Merge branch 'master' into erp5-cluster

Conflicts:
	software/neoppod/software.cfg
	stack/erp5/buildout.cfg
	stack/erp5/instance-erp5-cluster.cfg.in
	stack/erp5/instance-erp5-single.cfg.in
	stack/erp5/instance-zope.cfg.in

component/apache/buildout.cfg

@@ -17,15 +17,15 @@ extends =
 
 [apr]
 recipe = hexagonit.recipe.download
-version = 1.5.0
-md5sum = cc93bd2c12d0d037f68e21cc6385dc31
+version = 1.5.1
+md5sum = 5486180ec5a23efb5cae6d4292b300ab
 url = https://archive.apache.org/dist/apr/apr-${:version}.tar.bz2
 
 [apr-util]
 recipe = hexagonit.recipe.download
-version = 1.5.3
+version = 1.5.4
 url = https://archive.apache.org/dist/apr/apr-util-${:version}.tar.bz2
-md5sum = 6f3417691c7a27090f36e7cf4d94b36e
+md5sum = 2202b18f269ad606d70e1864857ed93c
 
 [apache]
 # inspired on http://old.aclark.net/team/aclark/blog/a-lamp-buildout-for-wordpress-and-other-php-apps/

component/bash/buildout.cfg

@@ -38,6 +38,8 @@ patches =
   http://ftp.gnu.org/pub/gnu/bash/bash-4.3-patches/bash43-026#922578e2be7ed03729454e92ee8d3f3a
   http://ftp.gnu.org/pub/gnu/bash/bash-4.3-patches/bash43-027#8ff6948b16f2db5c29b1b9ae1085bbe7
   http://ftp.gnu.org/pub/gnu/bash/bash-4.3-patches/bash43-028#dd51fa67913b5dca45a702b672b3323f
+  http://ftp.gnu.org/pub/gnu/bash/bash-4.3-patches/bash43-029#0729364c977ef4271e9f8dfafadacf67
+  http://ftp.gnu.org/pub/gnu/bash/bash-4.3-patches/bash43-030#efb709fdb1368945513de23ccbfae053
 configure-options =
   --with-curses
 environment =

component/ca-certificates/buildout.cfg

@@ -11,10 +11,10 @@ parts =
 
 [ca-certificates]
 recipe = slapos.recipe.cmmi
-version = 20140325
+version = 20141019
 url = ftp://ftp.free.fr/mirrors/ftp.debian.org/pool/main/c/ca-certificates/ca-certificates_${:version}.tar.xz
 patch-binary = ${patch:location}/bin/patch
-md5sum = 0436aba482091da310bd762e1deca8b4
+md5sum = f619282081c8bfc65ea64c37fa5285ed
 patches =
   ${:_profile_base_location_}/ca-certificates-sbin-dir.patch#0b4e7d82ce768823c01954ee41ef177b
 patch-options = -p0

component/curl/buildout.cfg

@@ -12,8 +12,8 @@ parts =
 
 [curl]
 recipe = slapos.recipe.cmmi
-url = http://curl.haxx.se/download/curl-7.38.0.tar.bz2
-md5sum = af6b3c299bd891f43cb5f76c4091b7b4
+url = http://curl.haxx.se/download/curl-7.39.0.tar.bz2
+md5sum = 1efecb5b0e43c17d968f0d228bbbbbbd
 configure-options =
   --disable-static
   --disable-ldap

component/file/buildout.cfg

@@ -8,8 +8,8 @@ extends =
 
 [file]
 recipe = slapos.recipe.cmmi
-url = ftp://ftp.astron.com/pub/file/file-5.19.tar.gz
-md5sum = e3526f59023f3f7d1ffa4d541335edab
+url = ftp://ftp.astron.com/pub/file/file-5.20.tar.gz
+md5sum = 5d5e13eb3e0e13839da869a31790faf2
 configure-options =
   --disable-static
 environment =

component/firefox/buildout.cfg

@@ -26,12 +26,12 @@ depends =
   ${liberation-fonts:location}
   ${ipaex-fonts:location}
 
-version = 32.0
+version = 33.0
 
 # MD5SUMs are available at :
 # https://ftp.mozilla.org/pub/mozilla.org/firefox/releases/${:version}/MD5SUMS
-x86 = http://download-installer.cdn.mozilla.net/pub/firefox/releases/${:version}/linux-i686/en-US/firefox-${:version}.tar.bz2 f086b7d0ac2421232d5cb31857980d2e
-x86-64 = http://download-installer.cdn.mozilla.net/pub/firefox/releases/${:version}/linux-x86_64/en-US/firefox-${:version}.tar.bz2 ec3b3976caa4ec5b3de137d059c74f8d
+x86 = http://download-installer.cdn.mozilla.net/pub/firefox/releases/${:version}/linux-i686/en-US/firefox-${:version}.tar.bz2 542b27d5f3afcb55406a7af72f68f3d5
+x86-64 = http://download-installer.cdn.mozilla.net/pub/firefox/releases/${:version}/linux-x86_64/en-US/firefox-${:version}.tar.bz2 ae4e8275750fd5f87963cbe263e8ba34
 
 script =
   if not self.options.get('url'): self.options['url'], self.options['md5sum'] = self.options[guessPlatform()].split(' ')
@@ -42,7 +42,7 @@ script =
   wrapper = open(wrapper_location, 'w')
   wrapper.write("""#!${dash:location}/bin/dash
   cd %(location)s
-  export LD_LIBRARY_PATH=%(location)s:${alsa:location}/lib:${atk:location}/lib:${bzip2:location}/lib:${cairo:location}/lib:${dbus:location}/lib:${dbus-glib:location}/lib:${fontconfig:location}/lib:${freetype:location}/lib:${gdk-pixbuf:location}/lib:${gettext:location}/lib:${glib:location}/lib:${gtk-2:location}/lib:${harfbuzz:location}/lib:${libICE:location}/lib:${libSM:location}/lib:${libX11:location}/lib:${libXau:location}/lib:${libXcursor:location}/lib:${libXext:location}/lib:${libXrender:location}/lib:${libXt:location}/lib:${libffi:location}/lib:${libpng:location}/lib:${libtool:location}/lib:${libuuid:location}/lib:${libxcb:location}/lib:${libxml2:location}/lib:${pango:location}/lib:${p11-kit:location}/lib:${pixman:location}/lib:${xfixes:location}/lib:${zlib:location}/lib
+  export LD_LIBRARY_PATH=%(location)s:${alsa:location}/lib:${atk:location}/lib:${bzip2:location}/lib:${cairo:location}/lib:${dbus:location}/lib:${dbus-glib:location}/lib:${fontconfig:location}/lib:${freetype:location}/lib:${gdk-pixbuf:location}/lib:${gettext:location}/lib:${glib:location}/lib:${gtk-2:location}/lib:${harfbuzz:location}/lib:${libICE:location}/lib:${libSM:location}/lib:${libX11:location}/lib:${libXau:location}/lib:${libXcomposite:location}/lib:${libXcursor:location}/lib:${libXext:location}/lib:${libXrender:location}/lib:${libXt:location}/lib:${libffi:location}/lib:${libpng:location}/lib:${libtool:location}/lib:${libuuid:location}/lib:${libxcb:location}/lib:${libxml2:location}/lib:${p11-kit:location}/lib:${pango:location}/lib:${pixman:location}/lib:${xdamage:location}/lib:${xfixes:location}/lib:${zlib:location}/lib
   export PATH=${fontconfig:location}/bin:$PATH
   exec %(location)s/firefox $*""")
   wrapper.close()

component/gcc/buildout.cfg

@@ -4,6 +4,7 @@
 extends =
   ../m4/buildout.cfg
   ../gmp/buildout.cfg
+  ../xz-utils/buildout.cfg
   ../zip/buildout.cfg
 
 parts =
@@ -11,59 +12,51 @@ parts =
 
 [mpfr]
 recipe = slapos.recipe.cmmi
-url = http://www.mpfr.org/mpfr-3.0.1/mpfr-3.0.1.tar.bz2
-md5sum = bfbecb2eacb6d48432ead5cfc3f7390a
+url = http://ftp.gnu.org/gnu/mpfr/mpfr-3.1.2.tar.xz
+md5sum = e3d203d188b8fe60bb6578dd3152e05c
 configure-options =
-  --with-gmp=${gmp-4:location}
+  --with-gmp=${gmp:location}
   --disable-static
 environment =
-  LDFLAGS=-Wl,-rpath=${gmp-4:location}/lib
+  PATH=${xz-utils:location}/bin:%(PATH)s
+  LDFLAGS=-Wl,-rpath=${gmp:location}/lib
 
 [mpc]
 recipe = slapos.recipe.cmmi
-url = http://www.multiprecision.org/mpc/download/mpc-0.9.tar.gz
-md5sum = 0d6acab8d214bd7d1fbbc593e83dd00d
+url = http://ftp.gnu.org/gnu/mpc/mpc-1.0.2.tar.gz
+md5sum = 68fadff3358fb3e7976c7a398a0af4c3
 configure-options =
-  --with-gmp=${gmp-4:location}
+  --with-gmp=${gmp:location}
   --with-mpfr=${mpfr:location}
   --disable-static
 environment =
-  LDFLAGS=-Wl,-rpath=${mpfr:location}/lib -Wl,-rpath=${gmp-4:location}/lib
+  LDFLAGS=-Wl,-rpath=${gmp:location}/lib -Wl,-rpath=${mpfr:location}/lib
 
-[ppl]
+[isl]
 recipe = slapos.recipe.cmmi
-# we should use version 0.11.x for gcc-4.7
-url = http://bugseng.com/external/ppl/download/ftp/releases/0.11/ppl-0.11.tar.bz2
-md5sum = c414abd2ecaf1f1f48949c78d383ee1a
-patches =
-  ${:_profile_base_location_}/ppl-0.11-gcc-4.9.patch#09e00eefba5f8e6e77a9c9c611fccc64
-
+url = ftp://gcc.gnu.org/pub/gcc/infrastructure/isl-0.12.2.tar.bz2
+md5sum = e039bfcfb6c2ab039b8ee69bf883e824
 configure-options =
-  --with-gmp=${gmp-4:location}
-  --with-gmp-prefix=${gmp-4:location}
+  --with-gmp-prefix=${gmp:location}
   --disable-static
 environment =
-  PATH=${m4:location}/bin:%(PATH)s
-  LDFLAGS=-Wl,-rpath=${gmp-4:location}/lib
-  CXXFLAGS=-fpermissive
-
+  LDFLAGS=-Wl,-rpath=${gmp:location}/lib
 
-[cloog-ppl]
+[cloog]
 recipe = slapos.recipe.cmmi
-url = ftp://gcc.gnu.org/pub/gcc/infrastructure/cloog-0.16.1.tar.gz
-md5sum = 947123350d1ff6dcb4b0774947ac015a
+url = ftp://gcc.gnu.org/pub/gcc/infrastructure/cloog-0.18.1.tar.gz
+md5sum = e34fca0540d840e5d0f6427e98c92252
 configure-options =
-  --with-gmp-prefix=${gmp-4:location}
-  --with-ppl=${ppl:location}
+  --with-gmp-prefix=${gmp:location}
+  --with-isl-prefix=${isl:location}
   --disable-static
 environment =
-  PATH=${m4:location}/bin:%(PATH)s
-  LDFLAGS=-Wl,-rpath=${gmp-4:location}/lib -Wl,-rpath=${ppl:location}/lib
+  LDFLAGS=-Wl,-rpath=${gmp:location}/lib -Wl,-rpath=${isl:location}/lib
 
 [gcc-common]
 recipe = slapos.recipe.cmmi
-url = http://ftp.gnu.org/gnu/gcc/gcc-4.7.4/gcc-4.7.4.tar.bz2
-md5sum = 4c696da46297de6ae77a82797d2abe28
+url = http://ftp.gnu.org/gnu/gcc/gcc-4.9.2/gcc-4.9.2.tar.bz2
+md5sum = 4df8ee253b7f3863ad0b86359cd39c43
 # make install does not work when several core are used
 make-targets = install -j1
 
@@ -72,41 +65,39 @@ make-targets = install -j1
 configure-options =
   --disable-bootstrap
   --disable-multilib
-  --with-gmp=${gmp-4:location}
+  --with-gmp=${gmp:location}
   --with-mpfr=${mpfr:location}
   --with-mpc=${mpc:location}
   --enable-languages="c,c++"
-  --with-ppl=${ppl:location}
-  --with-cloog=${cloog-ppl:location}
-  --enable-cloog-backend=isl
+  --with-isl=${isl:location}
+  --with-cloog=${cloog:location}
 environment =
-  LDFLAGS=-Wl,-rpath=${mpfr:location}/lib -Wl,-rpath=${gmp-4:location}/lib -Wl,-rpath=${mpc:location}/lib -Wl,-rpath=${ppl:location}/lib -Wl,-rpath=${cloog-ppl:location}/lib
+  LDFLAGS=-Wl,-rpath=${cloog:location}/lib -Wl,-rpath=${gmp:location}/lib -Wl,-rpath=${isl:location}/lib -Wl,-rpath=${mpc:location}/lib -Wl,-rpath=${mpfr:location}/lib
 
 [gcc-minimal]
 <= gcc-common
 configure-options =
   --disable-bootstrap
   --disable-multilib
-  --with-gmp=${gmp-4:location}
+  --with-gmp=${gmp:location}
   --with-mpfr=${mpfr:location}
   --with-mpc=${mpc:location}
   --enable-languages=c
-  --without-ppl
+  --without-isl
   --without-cloog
 environment =
-  LDFLAGS=-Wl,-rpath=${mpfr:location}/lib -Wl,-rpath=${gmp-4:location}/lib -Wl,-rpath=${mpc:location}/lib
+  LDFLAGS=-Wl,-rpath=${gmp:location}/lib -Wl,-rpath=${mpc:location}/lib -Wl,-rpath=${mpfr:location}/lib
 
 [gcc-fortran]
 <= gcc-common
 configure-options =
   --disable-bootstrap
   --disable-multilib
-  --with-gmp=${gmp-4:location}
+  --with-gmp=${gmp:location}
   --with-mpfr=${mpfr:location}
   --with-mpc=${mpc:location}
   --enable-languages="c,c++,fortran"
-  --with-ppl=${ppl:location}
-  --with-cloog=${cloog-ppl:location}
-  --enable-cloog-backend=isl
+  --with-isl=${isl:location}
+  --with-cloog=${cloog:location}
 environment =
-  LDFLAGS=-Wl,-rpath=${mpfr:location}/lib -Wl,-rpath=${gmp-4:location}/lib -Wl,-rpath=${mpc:location}/lib -Wl,-rpath=${ppl:location}/lib -Wl,-rpath=${cloog-ppl:location}/lib
+  LDFLAGS=-Wl,-rpath=${cloog:location}/lib -Wl,-rpath=${gmp:location}/lib -Wl,-rpath=${isl:location}/lib -Wl,-rpath=${mpc:location}/lib -Wl,-rpath=${mpfr:location}/lib

component/gcc/gcc-java-pre-4.4.patch

-Patch for http://gcc.gnu.org/bugzilla/show_bug.cgi?id=50888
-
---- a/src/libjava/libjava/prims.cc.orig	2012-01-20 11:30:18.586157610 +0100
-+++ b/src/libjava/libjava/prims.cc	2012-01-20 11:30:58.192770947 +0100
-@@ -38,6 +38,14 @@
- #endif
- 
- #ifndef DISABLE_GETENV_PROPERTIES
-+#ifdef __GLIBC__
-+/* glibc 2.15+ provides even for C++ inline optimized ::isspace etc.
-+   Unfortunately those inlines are throw (), and call a function pointer
-+   (which is throw () too, but with -fnon-call-exceptions this results
-+   in a __cxa_call_unexpected call.  This macro disables the optimized
-+   version.  */
-+#define __NO_CTYPE 1
-+#endif
- #include <ctype.h>
- #include <java-props.h>
- #define PROCESS_GCJ_PROPERTIES process_gcj_properties()
---- a/src/libjava/prims.cc.orig	2012-01-20 11:30:23.042818341 +0100
-+++ b/src/libjava/prims.cc	2012-01-20 11:31:01.389433254 +0100
-@@ -38,6 +38,14 @@
- #endif
- 
- #ifndef DISABLE_GETENV_PROPERTIES
-+#ifdef __GLIBC__
-+/* glibc 2.15+ provides even for C++ inline optimized ::isspace etc.
-+   Unfortunately those inlines are throw (), and call a function pointer
-+   (which is throw () too, but with -fnon-call-exceptions this results
-+   in a __cxa_call_unexpected call.  This macro disables the optimized
-+   version.  */
-+#define __NO_CTYPE 1
-+#endif
- #include <ctype.h>
- #include <java-props.h>
- #define PROCESS_GCJ_PROPERTIES process_gcj_properties()

component/gcc/ppl-0.11-gcc-4.9.patch

-# HG changeset patch
-# User Bernhard Walle <bernhard@bwalle.de>
-# Date 1399806036 -7200
-#      Sun May 11 13:00:36 2014 +0200
-# Node ID 5632a171e7c1dc4c0fb5c7a8352afb4188cd29d5
-# Parent  e35fa03cd2043496839aa5fa3a1a3d2d375b5535
-complibs/ppl: Fix build of ppl 0.11.2 with gcc 4.9
-
-  From: Roberto Bagnara <bagnara@cs.unipr.it>
-  Date: Tue, 29 Apr 2014 19:51:43 +0000 (+0200)
-  Subject: Added missing inclusions.  Use std::ptrdiff_t.
-  X-Git-Url: http://www.cs.unipr.it/git/gitweb.cgi?p=ppl%2Fppl.git;a=commitdiff_plain;h=61d4e14dfd9f1121e9b4521dead5728b2424dd7c
-
-  Added missing inclusions.  Use std::ptrdiff_t.
-  (Thanks to Paulo Cesar Pereira de Andrade.)
-
-Signed-off-by: Bernhard Walle <bernhard@bwalle.de>
-
-diff -r e35fa03cd204 -r 5632a171e7c1 patches/ppl/0.11.2/500-ptrdiff_t.patch
---- /dev/null	Thu Jan 01 00:00:00 1970 +0000
-+++ b/patches/ppl/0.11.2/500-ptrdiff_t.patch	Sun May 11 13:00:36 2014 +0200
-@@ -0,0 +1,95 @@
-+From: Bernhard Walle <bernhard@bwalle.de>
-+Subject: Fix compilation with gcc 4.9 (ptrdiff_t errors)
-+
-+This is a backport of following patch in ppl git
-+
-+  commit 61d4e14dfd9f1121e9b4521dead5728b2424dd7c
-+  Author: Roberto Bagnara <bagnara@cs.unipr.it>
-+  Date:   Tue Apr 29 21:51:43 2014 +0200
-+
-+      Added missing inclusions.  Use std::ptrdiff_t.
-+      (Thanks to Paulo Cesar Pereira de Andrade.)
-+
-+---
-+ src/Congruence_System.defs.hh     |    3 ++-
-+ src/Constraint_System.defs.hh     |    3 ++-
-+ src/Generator_System.defs.hh      |    3 ++-
-+ src/Grid_Generator_System.defs.hh |    3 ++-
-+ 4 files changed, 8 insertions(+), 4 deletions(-)
-+
-+--- a/src/Grid_Generator_System.defs.hh
-++++ b/src/Grid_Generator_System.defs.hh
-+@@ -30,6 +30,7 @@ site: http://www.cs.unipr.it/ppl/ . */
-+ #include "Variables_Set.types.hh"
-+ #include "Grid.types.hh"
-+ #include <iosfwd>
-++#include <cstddef>
-+ 
-+ namespace Parma_Polyhedra_Library {
-+ 
-+@@ -267,7 +268,7 @@ public:
-+   class const_iterator
-+     : public std::iterator<std::forward_iterator_tag,
-+ 			   Grid_Generator,
-+-			   ptrdiff_t,
-++			   std::ptrdiff_t,
-+ 			   const Grid_Generator*,
-+ 			   const Grid_Generator&>,
-+       private Generator_System::const_iterator {
-+--- a/src/Congruence_System.defs.hh
-++++ b/src/Congruence_System.defs.hh
-+@@ -33,6 +33,7 @@ site: http://www.cs.unipr.it/ppl/ . */
-+ #include "Grid.types.hh"
-+ #include "Grid_Certificate.types.hh"
-+ #include <iosfwd>
-++#include <cstddef>
-+ 
-+ namespace Parma_Polyhedra_Library {
-+ 
-+@@ -235,7 +236,7 @@ public:
-+   class const_iterator
-+     : public std::iterator<std::forward_iterator_tag,
-+ 			   Congruence,
-+-			   ptrdiff_t,
-++			   std::ptrdiff_t,
-+ 			   const Congruence*,
-+ 			   const Congruence&> {
-+   public:
-+--- a/src/Constraint_System.defs.hh
-++++ b/src/Constraint_System.defs.hh
-+@@ -33,6 +33,7 @@ site: http://www.cs.unipr.it/ppl/ . */
-+ #include "Congruence_System.types.hh"
-+ #include <iterator>
-+ #include <iosfwd>
-++#include <cstddef>
-+ 
-+ namespace Parma_Polyhedra_Library {
-+ 
-+@@ -204,7 +205,7 @@ public:
-+   class const_iterator
-+     : public std::iterator<std::forward_iterator_tag,
-+ 			   Constraint,
-+-			   ptrdiff_t,
-++			   std::ptrdiff_t,
-+ 			   const Constraint*,
-+ 			   const Constraint&> {
-+   public:
-+--- a/src/Generator_System.defs.hh
-++++ b/src/Generator_System.defs.hh
-+@@ -33,6 +33,7 @@ site: http://www.cs.unipr.it/ppl/ . */
-+ #include "Polyhedron.types.hh"
-+ #include "Poly_Con_Relation.defs.hh"
-+ #include <iosfwd>
-++#include <cstddef>
-+ 
-+ namespace Parma_Polyhedra_Library {
-+ 
-+@@ -250,7 +251,7 @@ public:
-+   class const_iterator
-+     : public std::iterator<std::forward_iterator_tag,
-+ 			   Generator,
-+-			   ptrdiff_t,
-++			   std::ptrdiff_t,
-+ 			   const Generator*,
-+ 			   const Generator&> {
-+   public:

component/gettext/buildout.cfg

@@ -10,8 +10,8 @@ extends =
 
 [gettext]
 recipe = slapos.recipe.cmmi
-url = http://ftp.gnu.org/pub/gnu/gettext/gettext-0.19.2.tar.xz
-md5sum = 1e6a827f5fbd98b3d40bd16b803acc44
+url = http://ftp.gnu.org/pub/gnu/gettext/gettext-0.19.3.tar.xz
+md5sum = 092c3f460553ceb4a638ff81d36434c4
 
 configure-options =
   --disable-static

component/ghostscript/buildout.cfg

@@ -33,8 +33,8 @@ environment =
 
 [ghostscript-9]
 <= ghostscript-common
-url = http://downloads.ghostscript.com/public/ghostscript-9.14.tar.bz2
-md5sum = 75f2e8ab8891d052ade9b64eb4eb5294
+url = http://downloads.ghostscript.com/public/ghostscript-9.15.tar.bz2
+md5sum = 0ee21d975c67a4883ba66bcc332418f5
 
 [ghostscript-fonts]
 recipe = hexagonit.recipe.download

component/gmp/buildout.cfg

@@ -6,21 +6,6 @@ extends =
 parts =
   gmp
 
-# we should use version 4.x for ppl-0.10
-[gmp-4]
-recipe = slapos.recipe.cmmi
-url = ftp://ftp.gmplib.org/pub/gmp-4.3.2/gmp-4.3.2.tar.bz2
-md5sum = dd60683d7057917e34630b4a787932e8
-# GMP does not correctly detect achitecture so it have to be given
-# as slapos.recipe.cmmi is using shell expansion in subproceses
-# backticks are working
-configure-options =
-  --build=`uname -m`-linux
-  --enable-cxx
-  --disable-static
-environment =
-  PATH=${m4:location}/bin:%(PATH)s
-
 [gmp]
 recipe = slapos.recipe.cmmi
 version = 6.0.0a

component/gnutls/buildout.cfg

@@ -11,8 +11,8 @@ parts = gnutls
 
 [gpg-error]
 recipe = slapos.recipe.cmmi
-url = ftp://ftp.gnupg.org/gcrypt/libgpg-error/libgpg-error-1.13.tar.bz2
-md5sum = fe0cfa7e15262ef8fdeee366109e9ff6
+url = ftp://ftp.gnupg.org/gcrypt/libgpg-error/libgpg-error-1.17.tar.bz2
+md5sum = b4f8d8b9ff14aed41f279aa844563539
 
 [gcrypt]
 recipe = slapos.recipe.cmmi
@@ -26,8 +26,8 @@ environment =
 
 [gnutls]
 recipe = slapos.recipe.cmmi
-url = ftp://ftp.gnutls.org/gcrypt/gnutls/v3.2/gnutls-3.2.16.tar.xz
-md5sum = d9a3fb49121b49bc332bfe9c3f49b005
+url = ftp://ftp.gnutls.org/gcrypt/gnutls/v3.2/gnutls-3.2.19.tar.xz
+md5sum = 8884492d0ad4339c831c6d507afbeea9
 configure-options =
   --disable-static
   --disable-libdane

component/groonga/buildout.cfg

@@ -11,9 +11,8 @@ extends =
 
 [groonga]
 recipe = slapos.recipe.cmmi
-version = 4.0.6
-url = http://packages.groonga.org/source/groonga/groonga-${:version}.tar.gz
-md5sum = d7eb0296b0b759d1856c31b6288670fa
+url = http://packages.groonga.org/source/groonga/groonga-4.0.7.tar.gz
+md5sum = 5d11546f921377d040f700bb74f1d7e4
 # temporary patch to respect more tokens in natural language mode.
 patches =
   ${:_profile_base_location_}/groonga.patch#9ed02fbe8400402d3eab47eee149978b

component/gzip/buildout.cfg

@@ -10,3 +10,12 @@ url = ftp://ftp.gnu.org/pub/gnu/gzip/gzip-1.6.tar.xz
 md5sum = da981f86677d58a106496e68de6f8995
 environment =
   PATH=${xz-utils:location}/bin:%(PATH)s
+patch-options = -p1
+# The --rsyncable patch is from debian/ubuntu,
+# specifically https://launchpad.net/ubuntu/+source/gzip/1.6-3ubuntu1
+# It is required to minimize the bandwidth used by rsync.
+# For an explanation, see http://beeznest.wordpress.com/2005/02/03/rsyncable-gzip/
+# Hunks for .texi files have been removed to avoid a dependency on makeinfo.
+patches =
+  ${:_profile_base_location_}/rsyncable.diff#0587af03a5580e2b7b4007469ee2b601
+

component/haproxy/buildout.cfg

@@ -11,8 +11,8 @@ parts = haproxy
 
 [haproxy]
 recipe = slapos.recipe.cmmi
-url = http://www.haproxy.org/download/1.5/src/haproxy-1.5.3.tar.gz
-md5sum = e999a547d57445d5a5ab7eb6a06df9a1
+url = http://www.haproxy.org/download/1.5/src/haproxy-1.5.8.tar.gz
+md5sum = 7bffa1afa069d90ce03b7cd9aa0557cd
 configure-command = true
 # If the system is running on Linux 2.6, we use "linux26" as the TARGET,
 # otherwise use "generic".

component/java/buildout.cfg

@@ -3,51 +3,31 @@
 parts =
   java
 
-[jdk-6u27-no-user-interaction-patch]
-recipe = hexagonit.recipe.download
-url = ${:_profile_base_location_}/${:filename}
-download-only = true
-filename = jdk-6u27-no-user-interaction.patch
-md5sum = 4c4303240647a114d07f3c411b2e6b5b
-
 [java]
 <= java-re
 
 [java-re]
-<= java-re-1.6.0
-
-[java-sdk]
-<= java-sdk-1.6.0
+<= java-re-7
 
-[java-re-1.6.0]
+[java-re-7]
 recipe = slapos.recipe.build
 slapos_promisee =
   directory:bin
   directory:lib
   directory:man
   directory:plugin
-  directory:javaws
   file:lib/rt.jar
   file:bin/java
-# http://java.com/en/download/manual_v6.jsp
-x86 = http://javadl.sun.com/webapps/download/AutoDL?BundleId=68284 7eda40e790de1a907d591b62949b6e72
-x86-64 = http://javadl.sun.com/webapps/download/AutoDL?BundleId=68286 839548714931443ba89719a995ece846
+# http://java.com/en/download/manual_java7.jsp
+x86 = http://javadl.sun.com/webapps/download/AutoDL?BundleId=97798 90a6b9e2a32d06c18a3f16b485f0d1ea
+x86-64 = http://javadl.sun.com/webapps/download/AutoDL?BundleId=97800 7605134662f6c87131eca5745895fe84
 script =
-  import glob
   if not self.options.get('url'): self.options['url'], self.options['md5sum'] = self.options[guessPlatform()].split(' ')
-  download_file = self.download(self.options['url'], self.options.get('md5sum'))
-  extract_dir = tempfile.mkdtemp(self.name)
-  os.chdir(extract_dir)
-  (download_dir, filename) = os.path.split(download_file)
-  auto_extract_bin = os.path.join(extract_dir, filename)
-  shutil.move(download_file, auto_extract_bin)
-  os.chmod(auto_extract_bin, 0755)
-  subprocess.call([auto_extract_bin])
-  self.cleanup_dir_list.append(extract_dir)
+  extract_dir = self.extract(self.download(self.options['url'], self.options.get('md5sum')))
   workdir = guessworkdir(extract_dir)
-  self.copyTree(glob.glob(os.path.join(workdir, "jre1.6.0_*"))[0], "%(location)s")
+  self.copyTree(workdir, "%(location)s")
 
-[java-re-1.7.0]
+[java-re-8]
 recipe = slapos.recipe.build
 slapos_promisee =
   directory:bin
@@ -56,9 +36,9 @@ slapos_promisee =
   directory:plugin
   file:lib/rt.jar
   file:bin/java
-# http://java.com/en/download/manual.jsp?locale=en
-x86 = http://javadl.sun.com/webapps/download/AutoDL?BundleId=68234 ea99bedd9db33e9e2970f4b70abd1e4b
-x86-64 = http://javadl.sun.com/webapps/download/AutoDL?BundleId=68236 5aa9bd26cdf1fa6afd2b15826b4ba139
+# http://java.com/en/download/manual_java7.jsp
+x86 = http://javadl.sun.com/webapps/download/AutoDL?BundleId=97358 22d970566c418499d331a2099d77c548
+x86-64 = http://javadl.sun.com/webapps/download/AutoDL?BundleId=97360 f4f7f7335eaf2e7b5ff455abece9d5ed
 script =
   if not self.options.get('url'): self.options['url'], self.options['md5sum'] = self.options[guessPlatform()].split(' ')
   extract_dir = self.extract(self.download(self.options['url'], self.options.get('md5sum')))

component/libatlas/buildout.cfg

@@ -30,7 +30,7 @@ configure-options =
   --prefix=${buildout:parts-directory}/${:_buildout_section_name_}
   --with-netlib-lapack-tarfile=${lapack-download:location}/${lapack-download:filename}
   --shared
-  -b "$(uname -p | grep -q 64 && echo 64 || echo 32)"
+  -b "$(uname -m | grep -q 64 && echo 64 || echo 32)"
   -Fa alg '-fPIC'
 make-options =
   -C build

component/libpng/buildout.cfg

@@ -23,10 +23,10 @@ md5sum = 4efba67fa5aa2b785c6fcec2cc3e90c9
 
 [libpng15]
 <= libpng-common
-url = http://download.sourceforge.net/libpng/libpng-1.5.18.tar.xz
-md5sum = 23fb6a0e9f4dbaab7b34b6fd317a4f2d
+url = http://download.sourceforge.net/libpng/libpng-1.5.19.tar.xz
+md5sum = 1e2afb372b9979b2133829ee8136f30f
 
 [libpng]
 <= libpng-common
-url = http://download.sourceforge.net/libpng/libpng-1.6.12.tar.xz
-md5sum = ee752da8aa75076500e4cf31318e636e
+url = http://download.sourceforge.net/libpng/libpng-1.6.14.tar.xz
+md5sum = aefd9fda2178acab908707737174a0cd

component/libreoffice-bin/buildout.cfg

@@ -9,14 +9,14 @@ parts =
 [libreoffice-bin]
 recipe = slapos.recipe.build
 # here, two %s are used, first one is for directory name (eg. x86_64), and second one is for filename (eg. x86-64).
-version = 4.2.6.3
+version = 4.3.3.2
 url = http://downloadarchive.documentfoundation.org/libreoffice/old/${:version}/rpm/%s/LibreOffice_${:version}_Linux_%s_rpm.tar.gz
 # supported architectures md5sums
-md5sum_x86 = 5a19002efc453a7af9443e1b01e7017a
-md5sum_x86-64 = 25d23958e8253182583e1af0ab9d1f5c
+md5sum_x86 = c24d4d7db03a5c983633efde754efae4
+md5sum_x86-64 = 94e0c1027d864f26a65e742874514b92
 
 # where office code can be found?
-officedir = libreoffice4.2
+officedir = libreoffice4.3
 
 # script to install
 script =

component/libtasn1/buildout.cfg

@@ -4,8 +4,8 @@ parts =
 
 [libtasn1]
 recipe = slapos.recipe.cmmi
-url = ftp://ftp.gnu.org/gnu/libtasn1/libtasn1-4.1.tar.gz
-md5sum = f9b37df3c2c9c6439d8bf427bfbfc521
+url = http://ftp.gnu.org/gnu/libtasn1/libtasn1-4.2.tar.gz
+md5sum = 414df906df421dee0a5cf7548788d153
 configure-options =
   --disable-static
   --disable-gtk-doc-html

component/mariadb/buildout.cfg

@@ -4,16 +4,17 @@
 [buildout]
 extends =
   ../cmake/buildout.cfg
-  ../zlib/buildout.cfg
-  ../jemalloc/buildout.cfg
   ../groonga/buildout.cfg
+  ../jemalloc/buildout.cfg
   ../libaio/buildout.cfg
   ../libevent/buildout.cfg
+  ../libxml2/buildout.cfg
   ../ncurses/buildout.cfg
   ../openssl/buildout.cfg
+  ../pcre/buildout.cfg
   ../pkgconfig/buildout.cfg
   ../readline/buildout.cfg
-  ../libxml2/buildout.cfg
+  ../zlib/buildout.cfg
 
 parts =
   mariadb
@@ -23,8 +24,6 @@ recipe = slapos.recipe.cmmi
 version = 10.0.14
 url = https://downloads.mariadb.org/f/mariadb-${:version}/source/mariadb-${:version}.tar.gz/from/http:/ftp.osuosl.org/pub/mariadb
 md5sum = 80fea71de54a9cfa7f5508df53d3f06d
-# compile directory is required to build mysql plugins.
-keep-compile-dir = true
 patch-options = -p0
 patches =
   ${:_profile_base_location_}/mariadb_10.0.8_create_system_tables__no_test.patch#a176d491cf45fccd53ee397c70393bc4
@@ -38,34 +37,40 @@ configure-options =
   -DWITH_ZLIB=system
   -DWITH_READLINE=0
   -DWITH_PIC=1
+  -DWITH_PCRE=system
   -DENABLE_DTRACE=0
   -DWITH_EXTRA_CHARSETS=complex
   -DWITH_EMBEDDED_SERVER=0
   -DWITHOUT_EXAMPLE_STORAGE_ENGINE=1
   -DWITHOUT_DAEMON_EXAMPLE=1
-  -DCMAKE_C_FLAGS="-I${jemalloc:location}/include -I${libaio:location}/include -I${libxml2:location}/include -I${ncurses:location}/include -I${openssl:location}/include -I${readline5:location}/include -I${zlib:location}/include"
-  -DCMAKE_CXX_FLAGS="-I${jemalloc:location}/include -I${libaio:location}/include -I${libxml2:location}/include -I${ncurses:location}/include -I${openssl:location}/include -I${readline5:location}/include -I${zlib:location}/include"
-  -DCMAKE_INSTALL_RPATH=${jemalloc:location}/lib:${libaio:location}/lib:${libxml2:location}/lib:${ncurses:location}/lib:${openssl:location}/lib:${readline5:location}/lib:${zlib:location}/lib
+  -DCMAKE_C_FLAGS="-I${jemalloc:location}/include -I${libaio:location}/include -I${libxml2:location}/include -I${ncurses:location}/include -I${openssl:location}/include -I${pcre:location}/include -I${readline5:location}/include -I${zlib:location}/include"
+  -DCMAKE_CXX_FLAGS="-I${jemalloc:location}/include -I${libaio:location}/include -I${libxml2:location}/include -I${ncurses:location}/include -I${openssl:location}/include -I${pcre:location}/include -I${readline5:location}/include -I${zlib:location}/include"
+  -DCMAKE_INSTALL_RPATH=${jemalloc:location}/lib:${libaio:location}/lib:${libxml2:location}/lib:${ncurses:location}/lib:${openssl:location}/lib:${pcre:location}/lib:${readline5:location}/lib:${zlib:location}/lib
   -DWITHOUT_TOKUDB=true
 environment =
   CMAKE_PROGRAM_PATH=${cmake:location}/bin
-  CMAKE_INCLUDE_PATH=${libaio:location}/include:${libaio:location}/include:${libxml2:location}/include:${ncurses:location}/include:${openssl:location}/include:${readline5:location}/include:${zlib:location}/include
-  CMAKE_LIBRARY_PATH=${libaio:location}/lib:{libaio:location}/lib:${libxml2:location}/lib:${ncurses:location}/lib:${openssl:location}/lib:${readline5:location}/lib:${zlib:location}/lib
-  LDFLAGS=-L${jemalloc:location}/lib -Wl,-rpath=${jemalloc:location}/lib -L${libaio:location}/lib -Wl,-rpath=${libaio:location}/lib -L${zlib:location}/lib
+  CMAKE_INCLUDE_PATH=${libaio:location}/include:${libaio:location}/include:${libxml2:location}/include:${ncurses:location}/include:${openssl:location}/include:${pcre:location}/include:${readline5:location}/include:${zlib:location}/include
+  CMAKE_LIBRARY_PATH=${libaio:location}/lib:{libaio:location}/lib:${libxml2:location}/lib:${ncurses:location}/lib:${openssl:location}/lib:${pcre:location}/lib:${readline5:location}/lib:${zlib:location}/lib
+  LDFLAGS=-L${jemalloc:location}/lib -Wl,-rpath=${jemalloc:location}/lib -L${libaio:location}/lib -Wl,-rpath=${libaio:location}/lib -L${pcre:location}/lib -L${zlib:location}/lib
 
 [mroonga-mariadb]
 # mroonga - a storage engine for MySQL. It provides fast fulltext search feature to all MySQL users.
 # http://mroonga.github.com/
 recipe = slapos.recipe.cmmi
-url = http://packages.groonga.org/source/mroonga/mroonga-4.06.tar.gz
-md5sum = 042bc1fbc3f25593be4484529d598144
+url = http://packages.groonga.org/source/mroonga/mroonga-4.07.tar.gz
+md5sum = fcfca7934b1f8df981fca579d3cd41bd
+configure-command = mkdir fake_mariadb_source && ln -s ${mariadb:location}/include/mysql/private fake_mariadb_source/sql && ./configure
 configure-options =
-  --with-mysql-source=${mariadb:location}__compile__/mariadb-${mariadb:version}
+  --prefix=${buildout:parts-directory}/${:_buildout_section_name_}
+  --with-mysql-source=fake_mariadb_source
   --with-mysql-config=${mariadb:location}/bin/mysql_config
   --disable-static
   --disable-document
+patch-options = -p1
+patches =
+  ${:_profile_base_location_}/mroonga_boolean.patch#36645770ae612515b74b90884ecc59fc
 environment =
   PATH=${groonga:location}/bin:${pkgconfig:location}/bin:%(PATH)s
-  CPPFLAGS=-I${groonga:location}/include/groonga
+  CPPFLAGS=-I${groonga:location}/include/groonga -I${pcre:location}/include
   LDFLAGS=-L${groonga:location}/lib
   PKG_CONFIG_PATH=${groonga:location}/lib/pkgconfig:${groonga-normalizer-mysql:location}/lib/pkgconfig

component/mariadb/mroonga_boolean.patch

+diff --git a/ha_mroonga.cpp b/ha_mroonga.cpp
+index dd9967d..c105a62 100644
+--- a/ha_mroonga.cpp
++++ b/ha_mroonga.cpp
+@@ -7678,7 +7678,7 @@ grn_rc ha_mroonga::generic_ft_init_ext_prepare_expression_in_boolean_mode(
+ 
+   const char *keyword, *keyword_original;
+   uint keyword_length, keyword_length_original;
+-  grn_operator default_operator = GRN_OP_OR;
++  grn_operator default_operator = GRN_OP_ADJUST;
+   grn_bool weight_specified = false;
+   keyword = keyword_original = key->ptr();
+   keyword_length = keyword_length_original = key->length();

component/mesa/buildout.cfg

 [buildout]
-extends = 
+extends =
+  ../autoconf/buildout.cfg
+  ../automake/buildout.cfg
+  ../bison/buildout.cfg
+  ../flex/buildout.cfg
+  ../libexpat/buildout.cfg
+  ../libtool/buildout.cfg
+  ../pkgconfig/buildout.cfg
   ../xorg/buildout.cfg
 
-parts = 
+parts =
   mesa
 
-[libdrm]
-#does not seem to compile with -jX
-recipe = slapos.recipe.cmmi
-url = http://dri.freedesktop.org/libdrm/libdrm-2.4.26.tar.bz2
-md5sum = 062569426773f69b11a47a7712bba770
-configure-options = 
-  --enable-nouveau-experimental-api
-environment =
-  PATH=${pkgconfig:location}/bin:%(PATH)s
-  PKG_CONFIG_PATH=${xorg-libpthread-stubs:location}/lib/pkgconfig:${pciaccess:location}/lib/pkgconfig
-
-
 [mesa]
 recipe = slapos.recipe.cmmi
-url = ftp://ftp.freedesktop.org//pub/mesa/7.11/MesaLib-7.11.tar.bz2
-md5sum = ff03aca82d0560009a076a87c888cf13
-configure-options = 
+url = ftp://ftp.freedesktop.org/pub/mesa/10.3.2/MesaLib-10.3.2.tar.bz2
+md5sum = c0a2a975899c4e7485124b388c16cd2f
+configure-options =
   --disable-static
-  --disable-driglx-direct
-  --disable-glu
-  --disable-glw
-  --disable-glut
+  --disable-dri
+  --disable-dri3
   --disable-egl
+  --disable-gbm
+  --disable-xvmc
+  --disable-vdpau
+  --enable-xlib-glx
+  --disable-gallium-gbm
+  --disable-shared-glapi
+  --disable-driglx-direct
   --disable-gallium-llvm
   --with-gallium-drivers=
   --with-dri-drivers=
-  --with-expat=${libexpat:location}
-  --with-driver=xlib
 environment =
-  PATH=${pkgconfig:location}/bin:${makedepend:location}/bin:${flex:location}/bin:${bison:location}/bin:%(PATH)s
-  PKG_CONFIG_PATH=${libdrm:location}/lib/pkgconfig:${dri2proto:location}/lib/pkgconfig:${xorg-libpthread-stubs:location}/lib/pkgconfig:${pciaccess:location}/lib/pkgconfig:${libX11:location}/lib/pkgconfig:${glproto:location}/lib/pkgconfig:${libX11:location}/lib/pkgconfig:${libXext:location}/lib/pkgconfig:${xfixes:location}/lib/pkgconfig:${xdamage:location}/lib/pkgconfig:${xproto:location}/lib/pkgconfig:${xextproto:location}/lib/pkgconfig
-  CPPFLAGS=-I${libexpat:location}/include -I${libdrm:location}/include/libdrm -I${libdrm:location}/include -I${xproto:location}/include -I${libX11:location}/include -I${libXext:location}/include/ -I${xextproto:location}/include
-  LDFLAGS=-L${libexpat:location}/lib
-  PYTHON=${buildout:executable}
+  PATH=${autoconf:location}/bin:${bison:location}/bin:${flex:location}/bin:${pkgconfig:location}/bin:%(PATH)s
+  PKG_CONFIG_PATH=${damageproto:location}/lib/pkgconfig:${fixesproto:location}/lib/pkgconfig:${glproto:location}/lib/pkgconfig:${kbproto:location}/lib/pkgconfig:${libX11:location}/lib/pkgconfig:${libXau:location}/lib/pkgconfig:${libXext:location}/lib/pkgconfig:${libexpat:location}/lib/pkgconfig:${libxcb:location}/lib/pkgconfig:${xdamage:location}/lib/pkgconfig:${xextproto:location}/lib/pkgconfig:${xfixes:location}/lib/pkgconfig:${xorg-libpthread-stubs:location}/lib/pkgconfig:${xproto:location}/lib/pkgconfig
+  PYTHON2=${buildout:executable}
+  ACLOCAL=${automake:location}/bin/aclocal -I${libtool:location}/share/aclocal -I${pkgconfig:location}/share/aclocal
+  AUTOCONF=${autoconf:location}/bin/autoconf
+  AUTOMAKE=${automake:location}/bin/automake
+
+[glu]
+recipe = slapos.recipe.cmmi
+# tar.bz2 cannot be well extracted why ?
+url = ftp://ftp.freedesktop.org/pub/mesa/glu/glu-9.0.0.tar.gz
+md5sum = bbc57d4fe3bd3fb095bdbef6fcb977c4
+configure-options =
+  --disable-static
+environment =
+  PATH=${pkgconfig:location}/bin:%(PATH)s
+  PKG_CONFIG_PATH=${damageproto:location}/lib/pkgconfig:${fixesproto:location}/lib/pkgconfig:${glproto:location}/lib/pkgconfig:${kbproto:location}/lib/pkgconfig:${libX11:location}/lib/pkgconfig:${libXau:location}/lib/pkgconfig:${libXext:location}/lib/pkgconfig:${libexpat:location}/lib/pkgconfig:${libxcb:location}/lib/pkgconfig:${mesa:location}/lib/pkgconfig:${xdamage:location}/lib/pkgconfig:${xextproto:location}/lib/pkgconfig:${xfixes:location}/lib/pkgconfig:${xorg-libpthread-stubs:location}/lib/pkgconfig:${xproto:location}/lib/pkgconfig

component/nginx/buildout.cfg

@@ -11,8 +11,8 @@ parts = nginx-output
 
 [nginx-common]
 recipe = slapos.recipe.cmmi
-url = http://nginx.org/download/nginx-1.7.5.tar.gz
-md5sum = e65aad627acc1cbe26527339a5814d57
+url = http://nginx.org/download/nginx-1.7.6.tar.gz
+md5sum = dd444e5333e0d324bec480e2ff67870a
 
 [nginx]
 <= nginx-common
@@ -38,6 +38,7 @@ configure-options =
   --with-http_ssl_module
   --with-mail
   --with-mail_ssl_module
+  --error-log-path=var/log/nginx.error.log
   --with-ld-opt=" -L ${libexpat:location}/lib -L ${openssl:location}/lib -L ${pcre:location}/lib -L ${zlib:location}/lib -Wl,-rpath=${libexpat:location}/lib -Wl,-rpath=${openssl:location}/lib -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${zlib:location}/lib"
   --with-cc-opt="-I ${libexpat:location}/include -I ${openssl:location}/include -I ${pcre:location}/include -I ${zlib:location}/include"
   --with-http_dav_module 

component/numpy/buildout.cfg

@@ -15,14 +15,8 @@ ATLAS = ${libatlas:location}/lib/libatlas.a
 [numpy]
 recipe = zc.recipe.egg:custom
 egg = numpy
-# patched version that supports rpath parameter in zc.recipe.egg:custom.
-find-links = http://www.nexedi.org/static/packages/source/numpy-1.8.2nxd001.tar.gz
 environment = numpy-env
 # no need to add libatlas location because they are statically linked.
 rpath =
   ${gcc-fortran:location}/lib
   ${gcc-fortran:location}/lib64
-
-[versions]
-# patched version that supports rpath parameter in zc.recipe.egg:custom.
-numpy = 1.8.2nxd001

component/openssl/buildout.cfg

@@ -16,8 +16,8 @@ parts =
 
 [openssl]
 recipe = slapos.recipe.cmmi
-url = https://www.openssl.org/source/openssl-1.0.1i.tar.gz
-md5sum = c8dc151a671b9b92ff3e4c118b174972
+url = https://www.openssl.org/source/openssl-1.0.1j.tar.gz
+md5sum = f7175c9cd3c39bb1907ac8bba9df8ed3
 patch-binary = ${patch:location}/bin/patch
 patches =
   ${:_profile_base_location_}/openssl-nodoc.patch#b4887a7b4e18402447bc6227d2493b92

component/pcre/buildout.cfg

@@ -4,8 +4,8 @@ parts =
 
 [pcre]
 recipe = slapos.recipe.cmmi
-url = ftp://ftp.csx.cam.ac.uk/pub/software/programming/pcre/pcre-8.35.tar.bz2
-md5sum = 6aacb23986adccd9b3bc626c00979958
+url = http://ftp.csx.cam.ac.uk/pub/software/programming/pcre/pcre-8.36.tar.bz2
+md5sum = b767bc9af0c20bc9c1fe403b0d41ad97
 configure-options =
   --disable-static
   --enable-unicode-properties

component/rdiff-backup/buildout.cfg

@@ -14,6 +14,8 @@ library-dirs =
   ${librsync:location}/lib/
 rpath =
   ${librsync:location}/lib/
+# If you need another release, override this link
+# and pin the egg in the [versions] section of the stack or SR.
 find-links = http://pkgs.fedoraproject.org/repo/pkgs/rdiff-backup/rdiff-backup-1.0.5.tar.gz/fa2a165fa07a94be52c52e3545bc7758/rdiff-backup-1.0.5.tar.gz
 
 [rdiff-backup]

component/sqlite3/buildout.cfg

@@ -5,8 +5,8 @@ parts =
 
 [sqlite3]
 recipe = slapos.recipe.cmmi
-url = http://www.sqlite.org/2014/sqlite-autoconf-3080600.tar.gz
-md5sum = f7e4a156b583abeba349629e2364224b
+url = http://www.sqlite.org/2014/sqlite-autoconf-3080701.tar.gz
+md5sum = 8ee4541ebb3e5739e7ef5e9046e30063
 configure-options =
   --disable-static
   --enable-readline

component/wget/buildout.cfg

@@ -10,8 +10,8 @@ parts =
 
 [wget]
 recipe = slapos.recipe.cmmi
-url = http://ftp.gnu.org/gnu/wget/wget-1.15.tar.xz
-md5sum = 7a279d5ac5594919124d5526e7143e28
+url = http://ftp.gnu.org/gnu/wget/wget-1.16.tar.xz
+md5sum = fe102975ab3a6c049777883f1bb9ad07
 configure-options =
   --enable-ipv6
   --enable-opie
@@ -24,7 +24,7 @@ patch-options =
   -p1
 
 patches =
-  ${:_profile_base_location_}/wget-doc.makefile.patch#2b89644c63a0dfe96290471d28a554f2
+  ${:_profile_base_location_}/wget-doc.makefile.patch#aabcc1695c7fb65ac44f295e04a2db78
 
 environment =
   PATH=${pkgconfig:location}/bin:${xz-utils:location}/bin:%(PATH)s

component/wget/wget-doc.makefile.patch

@@ -4,8 +4,8 @@
  ACLOCAL_AMFLAGS = -I m4
  
  # subdirectories in the distribution
--SUBDIRS = lib src doc po tests util
-+SUBDIRS = lib src po tests util
+-SUBDIRS = lib src doc po tests util testenv
++SUBDIRS = lib src po tests util testenv
  
  EXTRA_DIST = ChangeLog.README MAILING-LIST \
               msdos/ChangeLog msdos/config.h msdos/Makefile.DJ \
@@ -15,8 +15,8 @@
  ACLOCAL_AMFLAGS = -I m4
  
  # subdirectories in the distribution
--SUBDIRS = lib src doc po tests util
-+SUBDIRS = lib src po tests util
+-SUBDIRS = lib src doc po tests util testenv
++SUBDIRS = lib src po tests util testenv
  EXTRA_DIST = ChangeLog.README MAILING-LIST \
               msdos/ChangeLog msdos/config.h msdos/Makefile.DJ \
               msdos/Makefile.WC ABOUT-NLS \

component/xorg/buildout.cfg

@@ -465,6 +465,21 @@ environment =
   PATH=${pkgconfig:location}/bin:%(PATH)s
   PKG_CONFIG_PATH=${xextproto:location}/lib/pkgconfig:${libX11:location}/lib/pkgconfig:${libXext:location}/lib/pkgconfig:${libXt:location}/lib/pkgconfig:${xproto:location}/lib/pkgconfig:${kbproto:location}/lib/pkgconfig:${libxcb:location}/lib/pkgconfig:${xorg-libpthread-stubs:location}/lib/pkgconfig:${libXau:location}/lib/pkgconfig:${libICE:location}/lib/pkgconfig:${libSM:location}/lib/pkgconfig
 
+[compositeproto]
+recipe = slapos.recipe.cmmi
+url = http://www.x.org/releases/X11R7.7/src/everything/compositeproto-0.4.2.tar.bz2
+md5sum = 98482f65ba1e74a08bf5b056a4031ef0
+
+[libXcomposite]
+recipe = slapos.recipe.cmmi
+url = http://www.x.org/releases/X11R7.7/src/everything/libXcomposite-0.4.3.tar.bz2
+md5sum = a60e0b5c276d0aa9e2d3b982c98f61c8
+configure-options =
+  --disable-static
+environment =
+  PATH=${pkgconfig:location}/bin:%(PATH)s
+  PKG_CONFIG_PATH=${compositeproto:location}/lib/pkgconfig:${fixesproto:location}/lib/pkgconfig:${kbproto:location}/lib/pkgconfig:${libX11:location}/lib/pkgconfig:${libXau:location}/lib/pkgconfig:${libxcb:location}/lib/pkgconfig:${xextproto:location}/lib/pkgconfig:${xfixes:location}/lib/pkgconfig:${xorg-libpthread-stubs:location}/lib/pkgconfig:${xorg-util-macros:location}/share/pkgconfig:${xproto:location}/lib/pkgconfig
+
 [libXcursor]
 recipe = slapos.recipe.cmmi
 url = http://www.x.org/releases/X11R7.7/src/everything/libXcursor-1.1.13.tar.bz2

setup.py

@@ -28,7 +28,7 @@ from setuptools import setup, find_packages
 import glob
 import os
 
-version = '0.87'
+version = '0.91'
 name = 'slapos.cookbook'
 long_description = open("README.txt").read() + "\n" + \
     open("CHANGES.txt").read() + "\n"

slapos/recipe/addresiliency/__init__.py

@@ -40,44 +40,8 @@ class Recipe(GenericSlapRecipe):
     def _install(self):
         path_list = []
 
-        confpath = os.path.join(self.options['etc'], 'bully.conf')
-
-        ip_list = self.parameter_dict['ip-list']
-        print 'Creating bully configuration with ips : %s\n' % ip_list
-
-        conf = self.createFile(confpath,
-                               self.substituteTemplate(
-                               self.getTemplateFilename('bully.conf.in'),
-                               {
-                                   'self_id': int(self.parameter_dict['number']),
-                                   'ip_list': ip_list
-                                   }
-                               ))
-        path_list.append(conf)
-
         slap_connection = self.buildout['slap-connection']
 
-        if self.optionIsTrue('enable-bully-service', default=False):
-            bully_dir = self.options['services']
-        else:
-            bully_dir = self.options['bin']
-
-        bully_wrapper = self.createPythonScript(
-            name=os.path.join(bully_dir, self.options['wrapper-bully']),
-            absolute_function='slapos.recipe.addresiliency.bully.run',
-            arguments={
-                'confpath': confpath,
-                'server_url': slap_connection['server-url'],
-                'key_file': slap_connection.get('key-file'),
-                'cert_file': slap_connection.get('cert-file'),
-                'computer_id': slap_connection['computer-id'],
-                'partition_id': slap_connection['partition-id'],
-                'software': slap_connection['software-release-url'],
-                'namebase': self.parameter_dict['namebase'],
-            })
-
-        path_list.append(bully_wrapper)
-
         takeover_wrapper = self.createPythonScript(
             name=os.path.join(self.options['bin'], self.options['wrapper-takeover']),
             absolute_function='slapos.recipe.addresiliency.takeover.run',

slapos/recipe/addresiliency/bully.py

-# -*- coding: utf-8 -*-
-
-import logging
-import Queue
-import socket
-import thread
-import time
-
-import slapos.recipe.addresiliency.renamer
-import slapos
-
-log = logging.getLogger(__name__)
-logging.basicConfig(level=logging.DEBUG)
-
-
-BASE_PORT = 50000
-SLEEPING_MINS = 2           # XXX was 30, increase after testing
-
-MSG_PING = 'ping'
-MSG_HALT = 'halt'
-MSG_VICTORY = 'victory'
-
-MSG_OK = 'ok'
-
-STATE_NORMAL = 'normal'
-STATE_WAITINGCONFIRM = 'waitingConfirm'
-STATE_ELECTION = 'election'
-STATE_REORGANIZATION = 'reorganization'
-
-
-
-## Leader is always number 0
-
-class ResilientInstance(object):
-
-    def __init__(self, comm, renamer, confpath):
-        self.comm = comm
-        self.participant_id = 0
-        self.state = STATE_NORMAL
-        self.halter_id = 0
-        self.inElection = False
-        self.alive = True
-
-        self.mainCanal = self.comm.create_canal([MSG_PING, MSG_HALT, MSG_VICTORY])
-
-        self.renamer = renamer
-        self.okCanal = self.comm.create_canal([MSG_OK])
-        self.confpath = confpath
-        self.loadConnectionInfo()
-
-    def loadConnectionInfo(self):
-        params = open(self.confpath, 'r').readlines()
-        self.total_participants = len(params[0].split())
-        new_id = int(params[1])
-        if self.participant_id != new_id:
-            self.halter_id = new_id
-            self.participant_id = new_id 
-        log.debug('I am {} of {}'.format(self.participant_id, self.total_participants))
-
-    ## Needs to be changed to use the master
-    def aliveManagement(self):
-        while self.alive:
-            log.info('XXX sleeping for %d minutes' % SLEEPING_MINS)
-            time.sleep(SLEEPING_MINS*60)
-            if self.participant_id == 0:
-                continue
-            self.comm.send(MSG_PING, 0)
-            message, sender = self.okCanal.get()
-            if message:
-                continue
-            self.election()
-
-    def listen(self):
-        while self.alive:
-            self.comm.recv()
-
-    def main(self):
-        while self.alive:
-            message, sender = self.mainCanal.get()
-            if message == MSG_PING:
-                self.comm.send(MSG_OK, sender)
-
-            elif message == MSG_HALT:
-                self.state = STATE_WAITINGCONFIRM
-                self.halter_id = int(sender)
-                self.comm.send(MSG_OK, sender)
-
-            elif message == MSG_VICTORY:
-                if int(sender) == self.halter_id and self.state == STATE_WAITINGCONFIRM:
-                    log.info('{} thinks {} is the leader'.format(self.participant_id, sender))
-                    self.comm.send(MSG_OK, sender)
-                self.state = STATE_NORMAL
-
-    def election(self):
-        self.inElection = True
-        self.loadConnectionInfo()
-        # Check if I'm the highest instance alive
-        for higher in range(self.participant_id + 1, self.total_participants):
-            self.comm.send(MSG_PING, higher)
-            message, sender = self.okCanal.get()
-            if message:
-                log.info('{} is alive ({})'.format(higher, self.participant_id))
-                self.inElection = False
-                return False
-            continue
-
-        if not self.alive:
-            return False
-
-        # I should be the new coordinator, halt those below me
-        log.info('Should be ME : {}'.format(self.participant_id))
-        self.state = STATE_ELECTION
-        self.halter_id = self.participant_id
-        ups = []
-        for lower in range(self.participant_id):
-            self.comm.send(MSG_HALT, lower)
-            message, sender = self.okCanal.get()
-            if message:
-                ups.append(lower)
-
-        #Broadcast Victory
-        self.state = STATE_REORGANIZATION
-        for up in ups:
-            self.comm.send(MSG_VICTORY, up)
-            message, sender = self.okCanal.get()
-            if message:
-                continue
-            log.info('Something is wrong... let\'s start over')
-            return self.election()
-        self.state = STATE_NORMAL
-        self.active = True
-        log.info('{} Is THE LEADER'.format(self.participant_id))
-
-        self.renamer.failover()
-
-        self.inElection = False
-
-        return True
-
-
-
-class FilteredCanal(object):
-
-    def __init__(self, accept, timeout):
-        self.accept = accept
-        self.queue = Queue.Queue()
-        self.timeout = timeout
-
-    def append(self, message, sender):
-        if message in self.accept:
-            self.queue.put([message, sender])
-
-    def get(self):
-        try:
-            return self.queue.get(timeout=self.timeout)
-        except Queue.Empty:
-            return [None, None]
-
-
-
-class Wrapper(object):
-
-    def __init__(self, confpath, timeout=20):
-        self.canals = []
-        self.ips = []
-        self.participant_id = 0
-        self.timeout = timeout
-        self.confpath = confpath
-        self.getConnectionInfo()
-        self.socket = None
-
-    def getConnectionInfo(self):
-        params = open(self.confpath, 'r').readlines()
-        self.ips = params[0].split()
-        self.participant_id = int(params[1])
-        log.debug('I am {} of {}'.format(self.participant_id, self.ips))
-
-    def start(self):
-        self.getConnectionInfo()
-        self.socket = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
-        self.socket.bind((self.ips[self.participant_id], BASE_PORT + self.participant_id))
-        self.socket.listen(5)
-
-    def send(self, message, number):
-        self.getConnectionInfo()
-        try:
-            s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
-            s.connect((self.ips[number], BASE_PORT + number))
-            s.send(message + (' {}\n'.format(self.participant_id)))
-        except (socket.error, socket.herror, socket.gaierror, socket.timeout):
-            pass
-        finally:
-            s.close()
-
-    def create_canal(self, accept):
-        created = FilteredCanal(accept, self.timeout)
-        self.canals.append(created)
-        return created
-
-    def recv(self):
-        client, _ = self.socket.accept()
-        client_message = client.recv(1024)
-        if client_message:
-            message, sender = client_message.split()
-            for canal in self.canals:
-                canal.append(message, int(sender))
-
-
-
-
-def run(args):
-    confpath = args.pop('confpath')
-
-    renamer = slapos.recipe.addresiliency.renamer.Renamer(server_url = args.pop('server_url'),
-                                                          key_file = args.pop('key_file'),
-                                                          cert_file = args.pop('cert_file'),
-                                                          computer_guid = args.pop('computer_id'),
-                                                          partition_id = args.pop('partition_id'),
-                                                          software_release = args.pop('software'),
-                                                          namebase = args.pop('namebase'))
-
-    if args:
-        raise ValueError('Unknown arguments: %s' % ', '.join(args))
-
-    wrapper = Wrapper(confpath=confpath, timeout=20)
-
-    computer = ResilientInstance(wrapper, renamer=renamer, confpath=confpath)
-
-    # idle waiting for connection infos
-    while computer.total_participants < 2:
-        computer.loadConnectionInfo()
-        time.sleep(30)
-
-    log.info('Starting')
-
-    computer.comm.start()
-    thread.start_new_thread(computer.listen, ())
-    thread.start_new_thread(computer.aliveManagement, ())
-
-    computer.main()
-

slapos/recipe/addresiliency/template/bully.conf.in

-%(ip_list)s
-%(self_id)s

slapos/recipe/addresiliency/template/bully.py.in

-#!%(executable)s
-
-import select
-import socket
-import threading
-import time
-import sys
-
-sys.path[:] = %(syspath)s
-
-import slapos
-from slapos import slap as slapmodule
-
-port = 50000
-size = 1024
-wait = True
-
-
-def loadConnectionInfos():
-  connectionInfos = {}
-  file = open('%(confpath)s', 'r')
-  params = file.read().split('\n')
-  file.close()
-  ip_list = [x.strip("' ") for x in params[0].strip('[],').split(',')]
-  connectionInfos['self_id'] = int(params[1])
-
-  connectionInfos['server_list'] = \
-      [(i, ip_list[i]) for i in range(len(ip_list))]
-
-  connectionInfos['self_ip'] = ip_list[connectionInfos['self_id']]
-
-  return connectionInfos
-
-
-def rename_broken_and_stop():
-  try:
-    slap = slapmodule.slap()
-    slap.initializeConnection('%(server_url)s',
-                              '%(key_file)s',
-                              '%(cert_file)s')
-    computer_partition = slap.registerComputerPartition(computer_guid='%(computer_id)s',
-                                                        partition_id='%(partition_id)s')
-    broken = computer_partition.request(software_release='%(software)s',
-                                        software_type='frozen',
-                                        partition_reference='%(namebase)s0')
-
-    broken.rename('broken-%%s' %% (time.strftime("%%d-%%b_%%H:%%M:%%S", time.gmtime())))
-    broken.stopped()
-    computer_partition.rename('%(namebase)s0')
-    print 'renaming done\n'
-  except slapos.slap.slap.ServerError:
-    print 'Internal server error\n'
-
-def election():
-  global wait
-  connection = loadConnectionInfos()
-  message = "%%s, %%s" %% (connection['self_id'], "Election")
-  victory = True
-
-  for (remote_id, addr) in connection['server_list']:
-    if remote_id > connection['self_id']:
-      try:
-        s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
-        s.connect((addr, port + remote_id))
-        s.send(message)
-        reply = s.recv(size)
-        if reply == "%%s, %%s" %% (remote_id, "Alive"):
-          victory = False
-      except (socket.error, socket.herror, socket.gaierror, socket.timeout):
-        pass
-      finally:
-        s.close()
-
-  if victory:
-    wait = True
-    for (remote_id, addr) in connection['server_list']:
-      if remote_id < connection['self_id']:
-        try:
-          s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
-          s.connect((addr, port + remote_id))
-          s.send("%%s, %%s" %% (connection['self_id'], "Victory"))
-        except (socket.error, socket.herror, socket.gaierror, socket.timeout):
-          pass
-        finally:
-          s.close()
-    rename_broken_and_stop()
-
-
-def failure_detect():
-  global wait
-  connection = loadConnectionInfos()
-  while True:
-    time.sleep(30)
-    if wait:
-      print 'waiting 30 minutes\n'
-      time.sleep(30 * 60)
-      wait = False
-    if not connection['server_list'][0]:
-      continue
-    (remote_id, addr) = connection['server_list'][0]
-    try:
-      s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
-      s.connect((addr, port + remote_id))
-      s.close()
-    except (socket.error, socket.herror, socket.gaierror, socket.timeout):
-      s.close()
-      election()
-
-failure_detect_thread = threading.Thread(target=failure_detect)
-failure_detect_thread.start()
-
-connection = loadConnectionInfos()
-s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
-s.bind((connection['self_ip'], port + connection['self_id']))
-s.listen(5)
-#election()
-while True:
-  force_election = False
-  client, _ = s.accept()
-  client_message = client.recv(1024)
-  if client_message:
-    client_id, message = client_message.split(', ')
-    client_id = eval(client_id)
-    if message == "Victory":
-      wait = True
-      print "%%s wins" %% client_id
-    elif message == "Election":
-      print "%%s starts an election" %% client_id
-      if client_id < connection['self_id']:
-        client.send("%%s, %%s" %% (connection['self_id'], "Alive"))
-        force_election = True
-  client.close()
-  if force_election:
-    election()

slapos/recipe/addresiliency/template/bully_new.py.in

-#!%(executable)s
-
-import logging
-import os
-import socket
-import sys
-import thread
-import time
-
-sys.path[:] = %(syspath)s
-
-from slapos import slap as slapmodule
-import slapos
-
-BASE_PORT = 50000
-SLEEPING_MINS = 2
-
-log = logging.getLogger(__name__)
-logging.basicConfig(level=logging.DEBUG)
-
-
-
-class Renamer(object):
-    def __init__(self, server_url, key_file, cert_file, computer_guid,
-                 partition_id, software_release, namebase):
-        self.server_url = server_url
-        self.key_file = key_file
-        self.cert_file = cert_file
-        self.computer_guid = computer_guid
-        self.partition_id = partition_id
-        self.software_release = software_release
-        self.namebase = namebase
-
-    def _failover(self):
-        slap = slapmodule.slap()
-        slap.initializeConnection(self.server_url,
-                                  self.key_file,
-                                  self.cert_file)
-        computer_partition = slap.registerComputerPartition(computer_guid=self.computer_guid,
-                                                            partition_id=self.partition_id)
-        broken = computer_partition.request(software_release=self.software_release,
-                                            software_type='frozen',
-                                            partition_reference=self.namebase+'0')
-
-        broken.rename('broken-{}'.format(time.strftime("%%d-%%b_%%H:%%M:%%S", time.gmtime())))
-        broken.stopped()
-        computer_partition.rename(self.namebase+'0')
-
-    def failover(self):
-        try:
-            log.info('renaming done')
-        except slapos.slap.slap.ServerError:
-            log.info('Internal server error')
-
-
-
-## Leader is always number 0
-
-class ResilientInstance(object):
-
-    def __init__(self, comm, renamer, confpath):
-        self.comm = comm
-        self.id = 0
-        self.state = 'normal'
-        self.halter = 0
-        self.inElection = False
-        self.alive = True
-        self.lastPing = time.clock()
-
-        self.mainCanal = self.comm.canal(['ping', 'halt', 'victory'])
-
-        self.renamer = renamer
-        self.okCanal = self.comm.canal(['ok'])
-        self.confpath = confpath
-        self.loadConnectionInfos()
-
-    def loadConnectionInfos(self):
-        file = open(self.confpath, 'r')
-        params = file.read().split('\n')
-        file.close()
-        self.nbComp = len([x.strip("' ") for x in params[0].strip('[],').split(',')])
-        new_id = int(params[1])
-        if self.id != new_id:
-            self.halter = new_id
-            self.id = new_id 
-
-    ## Needs to be changed to use the master
-    def aliveManagement(self):
-        while self.alive:
-            log.info('XXX sleeping for %%d minutes' %% SLEEPING_MINS)
-            time.sleep(SLEEPING_MINS*60)
-            if self.id == 0:
-                continue
-            self.comm.send('ping', 0)
-            message, sender = self.okCanal.get()
-            if message:
-                continue
-            self.election()
-
-    def listen(self):
-        while self.alive:
-            self.comm.recv()
-
-    def main(self):
-        while self.alive:
-            message, sender = self.mainCanal.get()
-            if message == 'ping':
-                self.comm.send('ok', sender)
-
-            elif message == 'halt':
-                self.state = 'waitingConfirm'
-                self.halter = sender
-                self.comm.send('ok', sender)
-
-            elif message == 'victory':
-                if int(sender) == int(self.halter) and self.state == 'waitingConfirm':
-                    log.info('{} thinks {} is the leader'.format(self.id, sender))
-                    self.comm.send('ok', sender)
-                self.state = 'normal'
-
-    def election(self):
-        self.inElection = True
-        self.loadConnectionInfos()
-        #Check if I'm the highest instance alive
-        for higher in range(self.id + 1, self.nbComp):
-            self.comm.send('ping', higher)
-            message, sender = self.okCanal.get()
-            if message:
-                log.info('{} is alive ({})'.format(higher, self.id))
-                self.inElection = False
-                return False
-            continue
-
-        if not self.alive:
-            return False
-
-        #I should be the new coordinator, halt those below me
-        log.info('Should be ME : {}'.format(self.id))
-        self.state = 'election'
-        self.halter = self.id
-        ups = []
-        for lower in range(self.id):
-            self.comm.send('halt', lower)
-            message, sender = self.okCanal.get()
-            if message:
-                ups.append(lower)
-
-        #Broadcast Victory
-        self.state = 'reorganization'
-        for up in ups:
-            self.comm.send('victory', up)
-            message, sender = self.okCanal.get()
-            if message:
-                continue
-            log.info('Something is wrong... let\'s start over')
-            return self.election()
-        self.state = 'normal'
-        self.active = True
-        log.info('{} Is THE LEADER'.format(self.id))
-
-        self.renamer.failover()
-
-        self.inElection = False
-
-        return True
-
-
-class FilteredCanal(object):
-
-    def __init__(self, accept, timeout):
-        self.accept = accept
-        self.list = []
-        self.lock = thread.allocate_lock()
-        self.timeout = timeout
-
-    def append(self, message, sender):
-        if message in self.accept:
-            self.lock.acquire()
-            self.list.append([message, sender])
-            self.lock.release()
-
-    def get(self):
-        start = time.clock()
-        while (time.clock() - start < self.timeout):
-            self.lock.acquire()
-            if self.list:
-                self.lock.release()
-                return self.list.pop(0)
-            self.lock.release()
-        return [None, None]
-
-
-class Wrapper(object):
-
-    def __init__(self, confpath, timeout=20):
-        self.canals = []
-        self.ips = []
-        self.id = 0
-        self.timeout = timeout
-        self.confpath = confpath
-        self.getConnectionInfos()
-        self.socket = None
-
-    def getConnectionInfos(self):
-        file = open(self.confpath, 'r')
-        params = file.read().split('\n')
-        file.close()
-        self.ips = [x.strip("' ") for x in params[0].strip('[],').split(',')]
-        self.id = int(params[1])
-
-    def start(self):
-        self.getConnectionInfos()
-        self.socket = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
-        self.socket.bind((self.ips[self.id], BASE_PORT + self.id))
-        self.socket.listen(5)
-
-    def send(self, message, number):
-        self.getConnectionInfos()
-        try:
-            s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
-            s.connect((self.ips[number], BASE_PORT + number))
-            s.send(message + (' {}\n'.format(self.id)))
-        except (socket.error, socket.herror, socket.gaierror, socket.timeout):
-            pass
-        finally:
-            s.close()
-
-    def canal(self, accept):
-        created = FilteredCanal(accept, self.timeout)
-        self.canals.append(created)
-        return created
-
-    def recv(self):
-        client, _ = self.socket.accept()
-        client_message = client.recv(1024)
-        if client_message:
-            message, sender = client_message.split()
-            for canal in self.canals:
-                canal.append(message, int(sender))
-
-
-
-def main():
-
-    renamer = Renamer(server_url = '%(server_url)s',
-                      key_file = '%(key_file)s',
-                      cert_file = '%(cert_file)s',
-                      computer_guid = '%(computer_id)s',
-                      partition_id = '%(partition_id)s',
-                      software_release = '%(software)s',
-                      namebase = '%(namebase)s')
-
-    confpath = '%(confpath)s'
-
-    wrapper = Wrapper(confpath=confpath, timeout=20)
-
-    computer = ResilientInstance(wrapper, renamer=renamer, confpath=confpath)
-
-    #idle waiting for connection infos
-    while computer.nbComp < 2 :
-        computer.loadConnectionInfos()
-        time.sleep(30)
-
-    log.info('Starting')
-
-    computer.comm.start()
-    thread.start_new_thread(computer.listen, ())
-    thread.start_new_thread(computer.main, ())
-    thread.start_new_thread(computer.aliveManagement, ())
-
-    while True:
-        # XXX tight loop
-        continue
-
-
-if __name__ == '__main__':
-    main()
-

slapos/recipe/dropbear.py

@@ -112,6 +112,8 @@ class Recipe(GenericBaseRecipe):
     dropbear_cmd.extend(['-p', binding_address])
     # Single user mode
     dropbear_cmd.append('-n')
+    # Keep connection alive for 5 minutes
+    dropbear_cmd.extend(['-K', '300'])
 
     if 'dss-keyfile' in self.options:
       dropbear_cmd.extend(['-d', self.options['dss-keyfile']])

slapos/recipe/librecipe/generic.py

@@ -155,7 +155,7 @@ class GenericBaseRecipe(object):
           pidfile=%s
           if [ -e $pidfile ]; then
             pid=$(cat $pidfile)
-            if [ ! -z $(ps -p "$pid" | grep $(basename %s)) ]; then
+            if [ ! -z "$(ps -p $pid | grep $(basename %s))" ]; then
               echo "Already running with pid $pid."
               exit 1
             else

slapos/recipe/notifier.py

@@ -86,6 +86,7 @@ class Notify(GenericBaseRecipe):
                               command=notifier_binary,
                               parameters=parameters,
                               pidfile=pidfile,
+                              parameters_extra=True,
                               comments=[
                                   '',
                                   'Call an executable and send notification(s).',

slapos/recipe/request.py

@@ -25,6 +25,7 @@
 #
 ##############################################################################
 import logging
+from zc.buildout import UserError
 from slapos.recipe.librecipe import wrap, JSON_SERIALISED_MAGIC_KEY
 import json
 from slapos import slap as slapmodule
@@ -33,12 +34,6 @@ import traceback
 
 DEFAULT_SOFTWARE_TYPE = 'RootSoftwareInstance'
 
-def getListOption(option_dict, key, default=()):
-  result = option_dict.get(key, default)
-  if isinstance(result, basestring):
-    result = result.split()
-  return result
-
 class Recipe(object):
   """
   Request a partition to a slap master.
@@ -104,19 +99,20 @@ class Recipe(object):
     self.logger = logging.getLogger(name)
     software_url = options['software-url']
     name = options['name']
-    return_parameters = getListOption(options, 'return')
+    return_parameters = options.get('return', '').split()
     if not return_parameters:
       self.logger.debug("No parameter to return to main instance."
         "Be careful about that...")
     software_type = options.get('software-type', DEFAULT_SOFTWARE_TYPE)
-    filter_kw = dict(
-      (x, options['sla-' + x]) for x in getListOption(options, 'sla')
-      if options['sla-' + x]
-    )
-    partition_parameter_kw = self._filterForStorage(dict(
-      (x, options['config-' + x])
-      for x in getListOption(options, 'config')
-    ))
+    if 'config' in options or 'sla' in options:
+      raise UserError("'config' & 'sla' options are obsolete."
+                      " Clean up your software release.")
+    filter_kw = {k: v
+      for k, v in options.iteritems()
+      if k.startswith('sla-') and v}
+    partition_parameter_kw = self._filterForStorage({k: v
+      for k, v in options.iteritems()
+      if k.startswith('config-')})
     slave = options.get('slave', 'false').lower() in \
       librecipe.GenericBaseRecipe.TRUE_VALUES
 
@@ -267,9 +263,8 @@ class RequestEdge(Recipe):
       # Request will have its own copy of options dict
       local_options = original_options.copy()
       local_options['name'] = '%s-%s' % (country, name)
-      local_options['sla'] = "region"
       local_options['sla-region'] = country
-      
+
       self.request_dict[country] = Recipe(buildout, name, local_options)
       # "Bubble" all connection parameters
       for option, value in local_options.iteritems():

slapos/test/recipe/test_pbs.py

+
+import os
+import shutil
+import sys
+import tempfile
+import unittest
+
+from slapos.recipe import pbs
+
+
+class PBSTest(unittest.TestCase):
+
+    def new_recipe(self):
+        buildout = {
+                'buildout': {
+                    'bin-directory': '',
+                    'find-links': '',
+                    'allow-hosts': '',
+                    'develop-eggs-directory': '',
+                    'eggs-directory': '',
+                    'python': 'testpython',
+                    },
+                 'testpython': {
+                     'executable': sys.executable,
+                     },
+                 'slap-connection': {
+                     'computer-id': '',
+                     'partition-id': '',
+                     'server-url': '',
+                     'software-release-url': '',
+                     }
+                }
+
+        options = {
+                'rdiffbackup-binary': ''
+                }
+
+        return pbs.Recipe(buildout=buildout, name='pbs', options=options)
+
+    def test_push(self):
+        recipe = self.new_recipe()
+
+        with tempfile.NamedTemporaryFile() as rdiff_wrapper:
+            recipe.wrapper_push(remote_schema='TEST_REMOTE_SCHEMA',
+                                local_dir='TEST_LOCAL_DIR',
+                                remote_dir='TEST_REMOTE_DIR',
+                                rdiff_wrapper_path=rdiff_wrapper.name)
+            content = rdiff_wrapper.read()
+            self.assertIn('--remote-schema TEST_REMOTE_SCHEMA', content)
+            self.assertIn('TEST_LOCAL_DIR', content)
+            self.assertIn('TEST_REMOTE_DIR', content)
+
+    def test_pull(self):
+        recipe = self.new_recipe()
+
+        with tempfile.NamedTemporaryFile() as rdiff_wrapper:
+            recipe.wrapper_pull(remote_schema='TEST_REMOTE_SCHEMA',
+                                local_dir='TEST_LOCAL_DIR',
+                                remote_dir='TEST_REMOTE_DIR',
+                                rdiff_wrapper_path=rdiff_wrapper.name,
+                                remove_backup_older_than='TEST_OLDER')
+            content = rdiff_wrapper.read()
+            self.assertIn('--remote-schema TEST_REMOTE_SCHEMA', content)
+            self.assertIn('TEST_LOCAL_DIR', content)
+            self.assertIn('TEST_REMOTE_DIR', content)
+            self.assertIn('--remove-older-than TEST_OLDER', content)
+
+    def test_invalid_type(self):
+        recipe = self.new_recipe()
+
+        entry = {
+                'url': 'http://url.to.something/',
+                'type': 'invalid'
+                }
+
+        self.assertRaisesRegexp(ValueError,
+                                'type parameter must be either pull or push',
+                                recipe.add_slave, entry=entry, known_hosts_file={})
+
+    def test_install(self):
+        recipe = self.new_recipe()
+
+        promises_directory = tempfile.mkdtemp()
+        wrappers_directory = tempfile.mkdtemp()
+        directory = tempfile.mkdtemp()
+        feeds_directory = tempfile.mkdtemp()
+        run_directory = tempfile.mkdtemp()
+        cron_directory = tempfile.mkdtemp()
+
+        recipe.options.update({
+            'promises-directory': promises_directory,
+            'wrappers-directory': wrappers_directory,
+            'sshclient-binary': 'TEST_SSH_CLIENT',
+            'directory': directory,
+            'notifier-binary': 'TEST_NOTIFIER',
+            'feeds': feeds_directory,
+            'notifier-url': 'http://url.to.notifier/',
+            'run-directory': run_directory,
+            'cron-entries': cron_directory,
+            'known-hosts': 'TEST_KNOWN_HOSTS',
+            'slave-instance-list': '''[
+                {
+                 "url": "http://url.to.pull/",
+                 "type": "pull",
+                 "notification-id": "pulltest",
+                 "server-key": "TEST_SERVER_KEY",
+                 "name": "TEST_NAME",
+                 "notify": "http://url.to.notify/",
+                 "frequency": "TEST_FREQUENCY"
+                    }, {
+                 "url": "http://url.to.push/",
+                 "type": "push",
+                 "notification-id": "pushtest",
+                 "server-key": "TEST_SERVER_KEY",
+                 "name": "TEST_NAME",
+                 "notify": "http://url.to.notify/",
+                 "frequency": "TEST_FREQUENCY"
+                        }
+                ]
+                '''
+            })
+
+        recipe._install()
+
+        self.assertItemsEqual(os.listdir(promises_directory),
+                              ['pulltest', 'pushtest'])
+
+        self.assertItemsEqual(os.listdir(wrappers_directory),
+                              ['pulltest_raw', 'pulltest', 'pushtest_raw', 'pushtest'])
+
+        self.assertItemsEqual(os.listdir(directory),
+                              ['TEST_NAME'])
+
+        self.assertItemsEqual(os.listdir(feeds_directory),
+                              ['pulltest', 'pushtest'])
+
+        self.assertItemsEqual(os.listdir(run_directory),
+                              [])
+
+        self.assertItemsEqual(os.listdir(cron_directory),
+                              ['pulltest', 'pushtest'])
+
+        shutil.rmtree(promises_directory)
+        shutil.rmtree(wrappers_directory)
+        shutil.rmtree(directory)
+        shutil.rmtree(feeds_directory)
+        shutil.rmtree(run_directory)
+        shutil.rmtree(cron_directory)
+
+

software/ajaxplorer/instance-apachephp.cfg

@@ -177,7 +177,6 @@ name = MariaDB
 software-url = $${slap-connection:software-release-url}
 software-type = mariadb
 return = url
-sla = computer_guid
 sla-computer_guid = $${slap-connection:computer-id}
 
 [mariadb-urlparse]
@@ -260,7 +259,6 @@ name = Frontend-Website
 # XXX We have hardcoded SR URL here.
 software-url = $${slap-parameter:frontend-software-url}
 slave = true
-config = url custom_domain
 config-url = http://[$${apache-php:ip}]:$${apache-php:port}/
 return = site_url
 config-custom_domain = $${slap-parameter:domain}
@@ -272,7 +270,6 @@ name = Frontend-FileManager
 # XXX We have hardcoded SR URL here.
 software-url = $${slap-parameter:frontend-software-url}
 slave = true
-config = url custom_domain
 config-url = http://[$${httpd-conf:ip}]:$${httpd-conf:port2}/
 return = site_url
 config-custom_domain = $${slap-parameter:domain2}
@@ -284,7 +281,6 @@ name = Frontend-Website2
 # XXX We have hardcoded SR URL here.
 software-url = $${slap-parameter:frontend-software-url}
 slave = true
-config = url custom_domain
 config-url = http://[$${httpd-conf:ip}]:$${httpd-conf:port3}/
 return = site_url
 config-custom_domain = $${slap-parameter:domain3}

software/ajaxplorer/software.cfg

@@ -35,7 +35,7 @@ md5sum = 4c7936accb3658871b635158198b7905
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-apachephp.cfg
 output = ${buildout:directory}/template-apachephp.cfg
-md5sum = ef5c61127a21c016014021477d1791e3
+md5sum = 8be713bd2656e184651c8ec10579d668
 mode = 0644
 
 [instance-mariadb]
@@ -86,7 +86,6 @@ numpy = 1.6.2
 # websockify 0.4.1 doesn't install well
 websockify = 0.3.0
 plone.recipe.command = 1.1
-slapos.recipe.build = 0.11.6
 slapos.recipe.download = 1.0.dev-r4053
 slapos.recipe.template = 2.4.2
 z3c.recipe.mkdir = 0.5

software/apache-frontend/TODO.txt

 Apache:
 =======
-
+* (Minor Bug) Check if slave in slave_list before publishing information on it

software/apache-frontend/common.cfg

@@ -74,7 +74,7 @@ mode = 0644
 [template-apache-replicate]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/instance-apache-replicate.cfg.in
-md5sum = 140b3be4ce46aa18b33aea5d379d90e9
+md5sum = aafa98333cd51025938b33c2516f0b8d
 mode = 0644
 
 [template-slave-list]

software/apache-frontend/instance-apache-replicate.cfg.in

@@ -86,7 +86,6 @@ software-url = ${slap-connection:software-release-url}
 {% endif %}
 software-type = {{frontend_type}}
 return = private-ipv4 public-ipv4 slave-instance-information-list monitor_url
-config = _
 connection-monitor_url =
 
 {% for section, frontend_request in request_dict.iteritems() %}
@@ -102,7 +101,6 @@ state = {{ frontend_request.get('state') }}
 {%   do slave_configuration_dict.__setitem__("frontend-name", frontend_request.get('name')) %}
 config-_ = {{ json_module.dumps(slave_configuration_dict) }}
 {%   if frontend_request.get('sla') %}
-sla = {{ ' '.join(frontend_request.get('sla').keys()) }}
 {%     for parameter, value in frontend_request.get('sla').iteritems() -%}
 sla-{{ parameter }} = {{ value }}
 {%     endfor -%}

software/apache-frontend/instance-slave-apache-input-schema.json

@@ -13,7 +13,7 @@
       "title": "Custom Domain",
       "description": "Custom Domain to use for the website",
       "type": "string",
-      "pattern": "^([a-zA-Z0-9]([a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,6}$"
+      "pattern": "^([a-zA-Z0-9]([a-zA-Z0-9\\-]{0,61}[a-zA-Z0-9])?\\.)+[a-zA-Z]{2,6}$"
     },
     "type": {
       "title": "Backend Type",

software/apache-frontend/software.cfg

@@ -12,9 +12,8 @@ gitdb = 0.5.4
 plone.recipe.command = 1.1
 pycrypto = 2.6.1
 rdiff-backup = 1.0.5
-slapos.recipe.build = 0.12
 slapos.recipe.template = 2.5
-slapos.toolbox = 0.40.2
+slapos.toolbox = 0.40.4
 smmap = 0.8.2
 
 # Required by:

software/backupserver/software.cfg

@@ -120,6 +120,5 @@ mode = 0644
 rdiff-backup = 1.0.5
 gunicorn = 19.1.1
 plone.recipe.command = 1.1
-slapos.recipe.build = 0.11.6
 slapos.recipe.template = 2.4.2
 PyRSS2Gen = 1.1

software/cloudooo/instance-cloudoo.cfg.in

@@ -19,7 +19,7 @@ url = cloudooo://${haproxy:ip}:${haproxy:port}/
 recipe = slapos.cookbook:generic.cloudooo
 ip = {{ ipv4 }}
 environment =
-  LD_LIBRARY_PATH = {{ parameter_dict['cups'] }}/lib:{{ parameter_dict['cups'] }}/lib64:{{ parameter_dict['dbus'] }}/lib:{{ parameter_dict['dbus-glib'] }}/lib:{{ parameter_dict['file'] }}/lib:{{ parameter_dict['fontconfig'] }}/lib:{{ parameter_dict['freetype'] }}/lib:{{ parameter_dict['glib'] }}/lib:{{ parameter_dict['libffi'] }}/lib:{{ parameter_dict['libffi'] }}/lib64:{{ parameter_dict['libICE'] }}/lib:{{ parameter_dict['libpng12'] }}/lib:{{ parameter_dict['libSM'] }}/lib:{{ parameter_dict['libX11'] }}/lib:{{ parameter_dict['libXau'] }}/lib:{{ parameter_dict['libXdmcp'] }}/lib:{{ parameter_dict['libXext'] }}/lib:{{ parameter_dict['libxcb'] }}/lib:{{ parameter_dict['libXrender'] }}/lib:{{ parameter_dict['zlib'] }}/lib
+  LD_LIBRARY_PATH = {{ parameter_dict['cups'] }}/lib:{{ parameter_dict['cups'] }}/lib64:{{ parameter_dict['dbus'] }}/lib:{{ parameter_dict['dbus-glib'] }}/lib:{{ parameter_dict['file'] }}/lib:{{ parameter_dict['fontconfig'] }}/lib:{{ parameter_dict['freetype'] }}/lib:{{ parameter_dict['glib'] }}/lib:{{ parameter_dict['glu'] }}/lib:{{ parameter_dict['libICE'] }}/lib:{{ parameter_dict['libSM'] }}/lib:{{ parameter_dict['libX11'] }}/lib:{{ parameter_dict['libXau'] }}/lib:{{ parameter_dict['libXdmcp'] }}/lib:{{ parameter_dict['libXext'] }}/lib:{{ parameter_dict['libXrender'] }}/lib:{{ parameter_dict['libexpat'] }}/lib:{{ parameter_dict['libffi'] }}/lib:{{ parameter_dict['libffi'] }}/lib64:{{ parameter_dict['libpng12'] }}/lib:{{ parameter_dict['libxcb'] }}/lib:{{ parameter_dict['mesa'] }}/lib:{{ parameter_dict['xdamage'] }}/lib:{{ parameter_dict['xfixes'] }}/lib:{{ parameter_dict['zlib'] }}/lib
   FONTCONFIG_FILE = ${fontconfig-instance:conf-path}
   PATH = ${binary-link:target-directory}
 # Binary information

software/cloudooo/instance.cfg.in

@@ -25,31 +25,36 @@ context =
     ${:extra-context}
 
 [dynamic-template-cloudooo-parameters]
+buildout-bin-directory = {{ buildout_bin_directory }}
+coreutils = {{ coreutils_location }}
 cups = {{ cups_location }}
 dbus = {{ dbus_location }}
 dbus-glib = {{ dbus_glib_location }}
 file = {{ file_location }}
 fontconfig = {{ fontconfig_location }}
+fonts = {{ fonts_location }}
 freetype = {{ freetype_location }}
 glib = {{ glib_location }}
+glu = {{ glu_location }}
 haproxy = {{ haproxy_location }}
-libffi = {{ libffi_location }}
+imagemagick = {{ imagemagick_location }}
 libICE = {{ libICE_location }}
-libpng12 = {{ libpng12_location }}
 libSM = {{ libSM_location }}
 libX11 = {{ libX11_location }}
 libXau = {{ libXau_location }}
 libXdmcp = {{ libXdmcp_location }}
 libXext = {{ libXext_location }}
-libxcb = {{ libxcb_location }}
 libXrender = {{ libXrender_location }}
-zlib = {{ zlib_location }}
+libexpat = {{ libexpat_location }}
+libffi = {{ libffi_location }}
+libpng12 = {{ libpng12_location }}
 libreoffice-bin = {{ libreoffice_bin_location }}
-fonts = {{ fonts_location }}
-coreutils = {{ coreutils_location }}
-imagemagick = {{ imagemagick_location }}
+libxcb = {{ libxcb_location }}
+mesa = {{ mesa_location }}
 poppler = {{ poppler_location }}
-buildout-bin-directory = {{ buildout_bin_directory }}
+xdamage = {{ xdamage_location }}
+xfixes = {{ xfixes_location }}
+zlib = {{ zlib_location }}
 
 [dynamic-template-cloudooo]
 < = jinja2-template-base

software/cloudooo/software.cfg

@@ -37,12 +37,13 @@ context =
 # XXX: "template.cfg" is hardcoded in instanciation recipe
 filename = template.cfg
 template = ${:_profile_base_location_}/instance.cfg.in
-md5sum = 1c5dab821393845b118644686154710f
+md5sum = 2ad39c607cfa80bf87dd6258c29754c3
 extra-context =
     key buildout_bin_directory buildout:bin-directory
+    key coreutils_location coreutils:location
     key cups_location cups:location
-    key dbus_location dbus:location
     key dbus_glib_location dbus-glib:location
+    key dbus_location dbus:location
     key dcron_location dcron:location
     key file_location file:location
     key fontconfig_location fontconfig:location
@@ -50,6 +51,7 @@ extra-context =
     key freetype_location freetype:location
     key git_location git:location
     key glib_location glib:location
+    key glu_location glu:location
     key haproxy_location haproxy:location
     key imagemagick_location imagemagick:location
     key libICE_location libICE:location
@@ -59,18 +61,25 @@ extra-context =
     key libXdmcp_location libXdmcp:location
     key libXext_location libXext:location
     key libXrender_location libXrender:location
+    key libexpat_location libexpat:location
     key libffi_location libffi:location
     key libpng12_location libpng12:location
     key libreoffice_bin_location libreoffice-bin:location
     key libxcb_location libxcb:location
+    key mesa_location mesa:location
     key openssl_location openssl:location
     key poppler_location poppler:location
     key template_cloudooo template-cloudooo:target
+    key xdamage_location xdamage:location
+    key xfixes_location xfixes:location
     key zlib_location zlib:location
-    key coreutils_location coreutils:location
 
 [template-cloudooo]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/instance-cloudoo.cfg.in
-md5sum = 740a313b0d9d98800b099af8bf7cd0a7
+md5sum = 4bede3be20dbc2ecfdb5d49b3184742e
 mode = 640
+
+[versions]
+# use newest version of slapos.cookbook
+slapos.cookbook =

software/davstorage/common.cfg

@@ -44,6 +44,6 @@ mode = 0644
 [instance-davstorage]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-davstorage.cfg
-md5sum = 699ecf4678386667f58a3391bab7af0f
+md5sum = b83b0351a0f46aac35d52e681270ff03
 output = ${buildout:directory}/template-davstorage.cfg
 mode = 0644

software/davstorage/instance-davstorage.cfg

@@ -166,7 +166,6 @@ name = Frontend Ajax
 # XXX We have hardcoded SR URL here.
 software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
 slave = true
-config = url https-only
 config-https-only = true
 config-url = https://[$${davstorage:ip}]:$${davstorage:port_ajax}/
 return = domain
@@ -178,7 +177,6 @@ name = Frontend Webdav
 # XXX We have hardcoded SR URL here.
 software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
 slave = true
-config = url https-only
 config-https-only = true
 config-url = https://$${davstorage:user}:$${davstorage:password}@[$${davstorage:ip}]:$${davstorage:port_webdav}/
 return = domain

software/erp5testnode/instance-default.cfg

@@ -27,7 +27,7 @@ recipe = slapos.cookbook:erp5testnode
 slapos-directory = $${directory:slapos}
 working-directory = $${directory:testnode}
 test-suite-directory = $${directory:test-suite}
-proxy-host = $${slap-network-information:global-ipv6}
+proxy-host = $${slap-network-information:local-ipv4}
 proxy-port = 5000
 log-directory = $${directory:log}
 srv-directory = $${rootdirectory:srv}

software/erp5testnode/software.cfg

@@ -45,7 +45,12 @@ eggs =
   zc.buildout
   slapos.libnetworkcache
   slapos.core
+  jsonschema
+  hexagonit.recipe.download
+  netaddr
   inotifyx
+  lock_file
+  pytz
   erp5.util
   PyXML
 
@@ -70,11 +75,11 @@ recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-default.cfg
 output = ${buildout:directory}/template-default.cfg
 mode = 0644
-md5sum = b4556610d17ae769d759c49b0955b63e 
+md5sum = 22ffc8e212dcf2db8ad94cf0e5ac4772
 
 [versions]
 PyXML = 0.8.5
 erp5.util = 0.4.41
 plone.recipe.command = 1.1
-slapos.recipe.build = 0.12
 slapos.recipe.template = 2.5
+jsonschema = 2.4.0

software/etherpad-lite/common.cfg

@@ -58,7 +58,7 @@ mode = 0644
 [instance-etherpad-lite]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-etherpad-lite.cfg
-md5sum = fd7249be8988155110234c7bb877abb9
+md5sum = 28710a9a23320dd472387d81a698d12d
 output = ${buildout:directory}/template-etherpad-lite.cfg
 mode = 0644
 

software/etherpad-lite/instance-etherpad-lite.cfg

@@ -67,7 +67,6 @@ name = Frontend
 # XXX We have hardcoded SR URL here.
 software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
 slave = true
-config = url
 config-url = http://$${etherpad-conf-generation:ip}:$${etherpad-conf-generation:port}
 return = site_url
 

software/html5as/software.cfg

@@ -72,9 +72,8 @@ async = 0.6.1
 gitdb = 0.5.4
 pycrypto = 2.6
 rdiff-backup = 1.0.5
-slapos.recipe.build = 0.11.6
 slapos.recipe.template = 2.4.2
-slapos.toolbox = 0.40.2
+slapos.toolbox = 0.40.4
 smmap = 0.8.2
 z3c.recipe.scripts = 1.0.1
 plone.recipe.command = 1.1

software/kvm/common.cfg

@@ -11,9 +11,12 @@ extends =
   ../../component/noVNC/buildout.cfg
   ../../component/openssl/buildout.cfg
   ../../component/dcron/buildout.cfg
+  ../../stack/slapos.cfg
   ../../stack/nodejs.cfg
   ../../stack/resilient/buildout.cfg
-  ../../stack/slapos.cfg
+
+# stacks are listed from most generic to most specific,
+# to avoid versioning issues
 
 parts =
   template
@@ -44,6 +47,7 @@ eggs =
   slapos.cookbook
   slapos.toolbox
   erp5.util
+  cns.recipe.symlink
 
 [http-proxy]
 # https://github.com/nodejitsu/node-http-proxy
@@ -80,7 +84,7 @@ command =
 [template]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg.in
-md5sum = bc5a986c7208d02d3284a897ea90b39d
+md5sum = 4c8f07da2217e54163c265fe6fe3d41d
 output = ${buildout:directory}/template.cfg
 mode = 0644
 
@@ -88,7 +92,7 @@ mode = 0644
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/instance-kvm.cfg.jinja2
 mode = 644
-md5sum = e16c15f72fdeb92ce1854bc25daf5ad7
+md5sum = ac69266206830226185e576fb6e4935a
 download-only = true
 on-update = true
 
@@ -96,14 +100,14 @@ on-update = true
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/instance-kvm-resilient.cfg.jinja2
 mode = 644
-md5sum = a07c96b53fe9145278cd64a3b27a459a
+md5sum = 412d8540e054aa436bee723d85213a2b
 download-only = true
 on-update = true
 
 [template-kvm-resilient-test]
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/instance-kvm-resilient-test.cfg.jinja2
-md5sum = b4894680283d3912df4e9740f3e7848b
+md5sum = 080b5790b95a979e44c4a8e26dc02835
 mode = 0644
 download-only = true
 on-update = true
@@ -111,15 +115,15 @@ on-update = true
 [template-kvm-import]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-kvm-import.cfg.in
-md5sum = 7b36d6c61154b7ec3113a1bfaa25a904
+md5sum = 6835c9309ff4bf4a0efd1850e6c66b24
 output = ${buildout:directory}/template-kvm-import.cfg
 mode = 0644
 
 [template-kvm-import-script]
 recipe = hexagonit.recipe.download
-url = ${:_profile_base_location_}/template/kvm-import.sh.in
-filename = kvm-import.sh.in
-md5sum = e03ed049cddd8d157228b09e1ebc071a
+url = ${:_profile_base_location_}/template/kvm-import.sh.jinja2
+filename = kvm-import.sh.jinja2
+md5sum = 926a11421921c29f91fae8240bbcf585
 download-only = true
 mode = 0755
 
@@ -127,15 +131,15 @@ mode = 0755
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/instance-kvm-export.cfg.jinja2
 mode = 644
-md5sum = 900f416956903fa4858e67e93b5169a1
+md5sum = c9f13c1f481ed08c75089aef1d3c6981
 download-only = true
 on-update = true
 
 [template-kvm-export-script]
 recipe = hexagonit.recipe.download
-url = ${:_profile_base_location_}/template/kvm-export.sh.in
-filename = kvm-export.sh.in
-md5sum = 95fde96f35cbf90d677c44d18b60fafb
+url = ${:_profile_base_location_}/template/kvm-export.sh.jinja2
+filename = kvm-export.sh.jinja2
+md5sum = 22bd2e0c8fdb39a764a14c403a3bd752
 download-only = true
 mode = 0755
 

software/kvm/development.cfg

@@ -40,4 +40,3 @@ command =
   grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link &&
   grep parts ${buildout:develop-eggs-directory}/slapos.toolbox.egg-link &&
   grep parts ${buildout:develop-eggs-directory}/erp5.util.egg-link
-

software/kvm/instance-kvm-export.cfg.jinja2

@@ -17,13 +17,17 @@ parts +=
 
 # Create the exporter executable, which is a simple shell script
 [exporter]
-recipe = slapos.recipe.template
-url = {{ template_kvm_export }}
-output = ${directory:bin}/${slap-parameter:namebase}-exporter
-mode = 0755
-backup-disk-path = ${directory:backup}/virtual.qcow2
+recipe = slapos.recipe.template:jinja2
+template = {{ template_kvm_export }}
+rendered = ${directory:bin}/${slap-parameter:namebase}-exporter
 # Resilient stack wants a "wrapper" parameter
-wrapper = ${:output}
+wrapper = ${:rendered}
+mode = 0700
+context =
+    section directory directory
+    section buildout buildout
+    key socket_path kvm-instance:socket-path
+    raw gzip_binary {{ gzip_binary }}
 
 # Extends publish section with resilient parameters
 [publish-connection-information]

software/kvm/instance-kvm-import.cfg.in

@@ -25,13 +25,15 @@ crontabs = $${:etc}/crontabs
 cronstamps = $${:etc}/cronstamps
 
 [importer]
-recipe = slapos.recipe.template
-url = ${template-kvm-import-script:location}/${template-kvm-import-script:filename}
-output = $${directory:bin}/$${slap-parameter:namebase}-importer
-mode = 0755
-backup-disk-path = $${directory:backup}/virtual.qcow2
-disk-path = $${directory:srv}/virtual.qcow2
+recipe = slapos.recipe.template:jinja2
+template = ${template-kvm-import-script:location}/${template-kvm-import-script:filename}
+rendered = $${directory:bin}/$${slap-parameter:namebase}-importer
+mode = 0700
 # Resilient stack wants a "wrapper" parameter
-wrapper = $${:output}
+wrapper = $${:rendered}
+context =
+    section directory directory
+    raw zcat_binary ${gzip:location}/bin/zcat
+    raw gzip_binary ${gzip:location}/bin/gzip
 
 backup-disk-path = $${directory:backup}/virtual.qcow2

software/kvm/instance-kvm-resilient-input-schema.json

@@ -34,6 +34,12 @@
                 "description": "Remove all the backups in PBS that are older than specified value. It should be rdiff-backup-compatible.",
                 "type": "string",
                 "default": "3B"
+            },
+            "resilient-clone-number": {
+                "title": "Amount of backup(s) to create",
+                "description": "Amount of backup(s) to create. Each backup consists of a Pull Backup Server and a clone.",
+                "type": "integer",
+                "default": 2
             }
         }
     }

software/kvm/instance-kvm-resilient-test.cfg.jinja2

@@ -42,16 +42,13 @@ recipe = slapos.cookbook:request
 software-url = ${slap-connection:software-release-url}
 software-type = kvm-resilient
 name = Resilient KVM (Root Instance)
-{% set cluster_parameter_dict = slapparameter_dict.get('cluster', {}) -%}
-config = virtual-hard-drive-url virtual-hard-drive-md5sum resiliency-backup-periodicity {{ cluster_parameter_dict.keys() | join(' ') }}
-{% for key, value in cluster_parameter_dict.items() -%}
+{% for key, value in slapparameter_dict.get('cluster', {}).iteritems() -%}
 config-{{ key }} = {{ dumps(value) }}
 {% endfor -%}
 config-virtual-hard-drive-url = ${slap-parameter:virtual-hard-drive-url}
 config-virtual-hard-drive-md5sum = ${slap-parameter:virtual-hard-drive-md5sum}
 config-resiliency-backup-periodicity = */5 * * * *
 # XXX What to do?
-sla = computer_guid
 sla-computer_guid = ${slap-connection:computer-id}
 
 [slap-parameter]

software/kvm/instance-kvm-resilient.cfg.jinja2

@@ -3,6 +3,8 @@
 {% import 'parts' as parts %}
 {% import 'replicated' as replicated with context %}
 
+{% set backup_amount = slapparameter_dict.pop('resilient-clone-number', "1")|int + 1 -%}
+
 [buildout]
 eggs-directory = {{ eggs_directory }}
 develop-eggs-directory = {{ develop_eggs_directory }}
@@ -10,12 +12,12 @@ offline = true
 
 # += because we need to take up parts (like instance-custom, slapmonitor etc) from the profile we extended
 parts +=
-  {{ parts.replicate("kvm", "3") }}
+  {{ parts.replicate("kvm", backup_amount) }}
   publish-connection-informations
   kvm-frontend-url-promise
   kvm-backend-url-promise
 
-{{ replicated.replicate("kvm", "3", "kvm-export", "kvm-import", slapparameter_dict=slapparameter_dict) }}
+{{ replicated.replicate("kvm", backup_amount, "kvm-export", "kvm-import", slapparameter_dict=slapparameter_dict) }}
 
 [directory]
 recipe = slapos.cookbook:mkdirectory

software/kvm/instance-kvm.cfg.jinja2

@@ -196,11 +196,9 @@ partition-id = ${slap-connection:partition-id}
 name = VNC Frontend
 software-type = ${slap-parameter:frontend-software-type}
 slave = true
-config = host port
 config-host = ${novnc-instance:ip}
 config-port = ${novnc-instance:port}
 return = url resource port domainname
-sla = instance_guid
 sla-instance_guid = ${slap-parameter:frontend-instance-guid}
 
 [frontend-promise]

software/kvm/instance.cfg.in

@@ -82,6 +82,7 @@ context =
     raw kvm_template $${dynamic-template-kvm:rendered}
     raw template_kvm_export ${template-kvm-export-script:location}/${template-kvm-export-script:filename}
     raw pbsready_export_template ${pbsready-export:output}
+    raw gzip_binary ${gzip:location}/bin/gzip
 mode = 0644
 
 [dynamic-template-kvm-resilient-test]

software/kvm/software.cfg

@@ -5,35 +5,43 @@ extends = common.cfg
 PyRSS2Gen = 1.1
 apache-libcloud = 0.15.1
 async = 0.6.1
+cns.recipe.symlink = 0.2.3
 collective.recipe.template = 1.11
 ecdsa = 0.11
 erp5.util = 0.4.41
 gitdb = 0.5.4
 plone.recipe.command = 1.1
 pycrypto = 2.6.1
-rdiff-backup = 1.0.5
 slapos.recipe.download = 1.0.dev-r4053
-slapos.toolbox = 0.40.2
+slapos.toolbox = 0.40.4
 smmap = 0.8.2
 websockify = 0.6.0
 z3c.recipe.scripts = 1.0.1
 
 # Required by:
-# slapos.toolbox==0.40.2
+# websockify==0.6.0
+numpy = 1.8.2
+
+# Required by:
+# slapos.toolbox==0.40.4
 GitPython = 0.3.2.RC1
 
 # Required by:
-# slapos.toolbox==0.40.2
+# slapos.toolbox==0.40.4
 atomize = 0.2.0
 
 # Required by:
-# slapos.toolbox==0.40.2
+# slapos.toolbox==0.40.4
 feedparser = 5.1.3
 
+# Required by:
+# slapos.cookbook==0.87
+jsonschema = 2.4.0
+
 # Required by:
 # websockify==0.6.0
-numpy = 1.8.2
+numpy = 1.9.0
 
 # Required by:
-# slapos.toolbox==0.40.2
+# slapos.toolbox==0.40.4
 paramiko = 1.15.1

software/kvm/template/kvm-export.sh.in

-#!/bin/bash
-# Create a backup of the disk image of the virtual machine
-BACKUP_PATH=${:backup-disk-path}
-QMP_CLIENT=${buildout:directory}/software_release/bin/qemu-qmp-client
-
-if [ ! -f $DISK_PATH ]; then
-  echo "Nothing to backup, disk image doesn't exist yet."
-  exit 0;
-fi
-
-$QMP_CLIENT --socket ${kvm-instance:socket-path} --drive-backup $BACKUP_PATH
-

software/kvm/template/kvm-export.sh.jinja2

+#!/bin/bash
+#
+# Create a backup of the disk image of the virtual machine
+#
+LC_ALL=C
+export LC_ALL
+BACKUP_DIR={{ directory['backup'] }}
+BACKUP_FILE=virtual.qcow2
+
+QMP_CLIENT={{ buildout['directory'] }}/software_release/bin/qemu-qmp-client
+
+$QMP_CLIENT --socket {{ socket_path }} --drive-backup $BACKUP_DIR/$BACKUP_FILE
+
+# Due to the way qmp works, the VM file cannot be compressed on the fly.
+# Although the compression step is optional, the importer uses the .gz file
+# if present. So, remove it if you are disabling the compression.
+
+# The downside of compression, here, is the temporary usage of more disk space
+# in the exporter node. The goal is to minimize disk usage on the PBS server.
+
+# If you want to compress the file in-place:
+# truncate -s $(gzip -c $BACKUP_FILE | dd of=$BACKUP_FILE conv=notrunc 2>&1 | sed -n '$ s/ .*$// p') $BACKUP_FILE
+# but the importer script would have to be adapted.
+
+echo "Compressing backup..."
+{{ gzip_binary }} --force --rsyncable $BACKUP_DIR/$BACKUP_FILE
+
+cd $BACKUP_DIR && find -type f ! -name backup.signature -print0 | xargs -P4 -0 sha256sum | LC_ALL=C sort -k 66 > backup.signature
+

software/kvm/template/kvm-import.sh.in

-#!/bin/bash
-DISK_PATH=${:disk-path}
-BACKUP_PATH=${:backup-disk-path}
-
-# TODO: Use rdiff
-rm $DISK_PATH
-cp $BACKUP_PATH $DISK_PATH

software/kvm/template/kvm-import.sh.jinja2

+#!/bin/bash
+
+VM_DIR={{ directory['srv'] }}
+BACKUP_DIR={{ directory['backup'] }}
+VM_FILE=virtual.qcow2
+LC_ALL=C
+export LC_ALL
+umask 077
+
+write_backup_proof () {
+  cd {{ directory['backup'] }}
+  find -type f ! -name backup.signature ! -wholename "./rdiff-backup-data/*" -print0 | xargs -P4 -0 sha256sum  | LC_ALL=C sort -k 66 > {{ directory['srv'] }}/proof.signature
+  diff -ruw {{ directory['backup'] }} {{ directory['srv'] }}/proof.signature > {{ directory['srv'] }}/backup.diff
+}
+
+# For now we just make the diff before 
+write_backup_proof
+
+if [ -f "$BACKUP_DIR/${VM_FILE}.gz" ]; then
+    {{ gzip_binary }} -t "$BACKUP_DIR/${VM_FILE}.gz" || exit 10
+    {{ zcat_binary }} "$BACKUP_DIR/${VM_FILE}.gz" > $VM_DIR/$VM_FILE
+else
+    rm $VM_DIR/$VM_FILE
+    cp $BACKUP_DIR/$VM_FILE $VM_DIR/$VM_FILE
+fi
+

software/mariadb/software.cfg

@@ -67,5 +67,4 @@ output = ${buildout:directory}/template-recover.cfg
 mode = 0644
 
 [versions]
-slapos.recipe.build = 0.11.5
 slapos.recipe.template = 2.4.2

software/monitor/instance.cfg

@@ -71,7 +71,6 @@ name = Monitor Frontend
 # XXX We have hardcoded SR URL here.
 software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
 slave = true
-config = url domain
 config-url = $${monitor-parameters:url}
 config-domain = $${slap-parameter:frontend-domain}
 return = site_url domain

software/monitor/software.cfg

@@ -15,7 +15,7 @@ parts =
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg
 output = ${buildout:directory}/instance.cfg
-md5sum = 6ad84a0061df0c00a9f41d8b302adc45
+md5sum = b86685e4007296f4808c2f1f1b74a290
 mode = 0644
 
 # stupify index for now

software/neoppod/cluster.cfg.in

@@ -5,7 +5,6 @@
 [request-common]
 recipe = slapos.cookbook:request
 software-url = ${slap-connection:software-release-url}
-sla = computer_guid
 sla-computer_guid = ${slap-connection:computer-id}
 server-url = ${slap-connection:server-url}
 key-file = ${slap-connection:key-file}
@@ -13,22 +12,15 @@ cert-file = ${slap-connection:cert-file}
 computer-id = ${slap-connection:computer-id}
 partition-id = ${slap-connection:partition-id}
 config-cluster = {{ slapparameter_dict['cluster'] }}
-config-partitions = {{ slapparameter_dict.get('partitions', 12) }}
-config-replicas = {{ slapparameter_dict.get('replicas', 0) }}
-
-[request-config]
-base = cluster
-master = partitions replicas
-node = masters
 
 [node-base]
 < = request-common
-config = ${request-config:base} ${request-config:node}
 config-masters = ${all-masters:addresses}
 
 [mysql-storage-base]
 < = request-common
-config = ${request-config:base} ${request-config:master}
+config-partitions = {{ slapparameter_dict.get('partitions', 12) }}
+config-replicas = {{ slapparameter_dict.get('replicas', 0) }}
 software-type = {{ mysql_storage_software_type }}
 
 [mysql-storage-request-base]
@@ -57,7 +49,6 @@ name = {{ section_id }}
 < = mysql-storage-base
 	node-base
 name = {{ section_id }}
-config = ${request-config:base} ${request-config:master} ${request-config:node}
 
 {% endfor -%}
 {% for node_number in range(slapparameter_dict.get('admin-count', 1)) -%}

software/neoppod/software.cfg

@@ -37,7 +37,7 @@ scripts =
 [cluster]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/${:_buildout_section_name_}.cfg.in
-md5sum = 84cba584198a26289daacb3e6d199e2b
+md5sum = 3939e229108f96ac0d68224fc1630880
 
 [instance-neo-admin]
 < = cluster

software/nginx-push-stream/software.cfg

@@ -43,5 +43,4 @@ mode = 0644
 
 [versions]
 plone.recipe.command = 1.1
-slapos.recipe.build = 0.12
 slapos.recipe.template = 2.5

software/powerdns/instance-powerdns-input-schema.json

@@ -45,7 +45,7 @@
       "description": "Zone to be handled by the DNS cluster",
       "type": "string",
       "default": "domain.com",
-      "pattern": "^([a-zA-Z0-9]([a-zA-Z0-9\\-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,6}$"
+      "pattern": "^([a-zA-Z0-9]([a-zA-Z0-9\\-]{0,61}[a-zA-Z0-9])?\\.)+[a-zA-Z]{2,6}$"
     },
 
     "server-admin": {
@@ -59,7 +59,7 @@
       "title": "DNS domains template string",
       "description": "Template used to generate DNS domain name",
       "type": "string",
-      "default": "ns%s." + zone
+      "default": "ns%s. + zone"
     }
   }
 }

software/powerdns/instance-powerdns-replicate.cfg.jinja2

@@ -64,12 +64,9 @@ state = {{ slapparameter_dict.pop(state_key) }}
 {%   endif%}
 config-zone = {{ zone }}
 config-soa = {{ "%s,%s" % (dns_domain, server_admin) }}
-{%   if sla_parameters %}
-sla = {{ ' '.join(sla_parameters) }}
-{%     for parameter in sla_parameters -%}
+{%   for parameter in sla_parameters -%}
 sla-{{ parameter }} = {{ slapparameter_dict.pop( sla_key + parameter ) }}
-{%     endfor -%}
-{%   endif -%}
+{%   endfor -%}
 {% endfor -%}
 
 
@@ -86,7 +83,6 @@ software-url = ${slap-connection:software-release-url}
 {% endif %}
 software-type = {{dns_type}}
 return = private-ipv4 public-ipv4 slave-instance-information-list monitor_url
-config = {{ ' '.join(slapparameter_dict.keys()) + ' zone soa server-admin ns-record ' + slave_list_name }}
 config-server-admin = {{ server_admin }}
 config-ns-record = {{ ns_record }}
 {% for parameter, value in slapparameter_dict.iteritems() -%}

software/powerdns/software.cfg

@@ -38,7 +38,7 @@ mode = 640
 [template-dns-replicate]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/instance-powerdns-replicate.cfg.jinja2
-md5sum = 7cfc55bfb9821aad44272153f60450ef
+md5sum = 83cedc8fa923c59ca900b4600cc1c52f
 mode = 0644
 
 [iso-list]
@@ -64,5 +64,4 @@ PyRSS2Gen = 1.1
 cns.recipe.symlink = 0.2.3
 collective.recipe.template = 1.11
 plone.recipe.command = 1.1
-slapos.recipe.build = 0.12
 slapos.recipe.template = 2.5

software/seleniumrunner/software.cfg

@@ -47,5 +47,4 @@ mode = 0644
 
 [versions]
 plone.recipe.command = 1.1
-slapos.recipe.build = 0.12
 slapos.recipe.template = 2.5

software/slapos-master/instance-erp5.cfg.in

@@ -6,16 +6,11 @@
 [request-common]
 recipe = slapos.cookbook:request.serialised
 software-url = ${slap-connection:software-release-url}
-sla = computer_guid
 server-url = ${slap-connection:server-url}
 key-file = ${slap-connection:key-file}
 cert-file = ${slap-connection:cert-file}
 computer-id = ${slap-connection:computer-id}
 partition-id = ${slap-connection:partition-id}
-config =
-  use-ipv6
-  ${:extra-config}
-extra-config =
 config-use-ipv6 = {{ dumps(slapparameter_dict.get('use-ipv6', False)) }}
 
 {% macro request(name, software_type, config_key, config, ret={'url': True}) -%}
@@ -32,7 +27,6 @@ return = {{ ret.keys() | join(' ') }}
 {%   endif -%}
 {% endfor -%}
 sla-computer_guid = {{ dumps(slapparameter_dict.get(config_key + '-computer-guid', computer_id)) }}
-extra-config = {{ ' '.join(config) }}
 {% for option, value in config.items() -%}
 config-{{ option }} = {{ dumps(value) }}
 {% endfor -%}
@@ -43,8 +37,13 @@ config-{{ option }} = {{ dumps(value) }}
 {{ request('cloudooo', 'cloudooo', 'cloudooo', {'tcpv4-port': 2020}) }}
 {{ request('mariadb', 'mariadb', 'mariadb', {'tcpv4-port': 2099}, {'database-list': True, 'test-database-list': True}) }}
 {# Fail early if an unexpected value is provided -#}
-{% set possible_software_type_dict = {'zeo': 'zeo', 'neo': 'neo'} -%}
-{{ request('zodb', 'zodb-' ~ possible_software_type_dict[slapparameter_dict.get('zodb-software-type', 'zeo')], 'zodb', {'tcpv4-port': 2100, 'zodb-dict': {'root': {}}}, {'zodb-storage-type': False, 'zodb-dict': False, 'tidstorage-ip': False, 'tidstorage-port': False}) }}
+{% set zodb_type = slapparameter_dict.get('zodb-software-type') -%}
+{% set zodb_extern = slapparameter_dict.get('zodb-extern') -%}
+{% if {'zeo': 1, None: 0}[zodb_type] -%}
+{{   request('zodb', 'zodb-' ~ zodb_type, 'zodb', {'tcpv4-port': 2100, 'zodb-dict': {'root': {}}}, {'zodb-storage-type': False, 'zodb-dict': False, 'tidstorage-ip': False, 'tidstorage-port': False}) }}
+{% else -%}
+{% do zodb_extern[0] %}
+{% endif -%}
 
 [inituser-password]
 {% set inituser_password = slapparameter_dict.get('inituser-password') -%}
@@ -67,34 +66,6 @@ recipe = slapos.cookbook:generate.password
 return =
   zope-address-list
   hosts-dict
-extra-config =
-  bt5
-  bt5-repository-url
-  cloudooo-url
-  deadlock-debugger-password
-  developer-list
-  hosts-dict
-  inituser-login
-  inituser-password
-  instance-count
-  kumofs-url
-  longrequest-logger-interval
-  longrequest-logger-timeout
-  memcached-url
-  mysql-test-url-list
-  mysql-url-list
-  name
-  port-base
-  site-id
-  smtp-url
-  thread-amount
-  tidstorage-ip
-  tidstorage-port
-  timerserver-interval
-  timezone
-  webdav
-  zodb-dict
-  zodb-storage-type
 config-bt5 = {{ dumps(slapparameter_dict.get('bt5', 'erp5_full_text_myisam_catalog erp5_configurator_standard erp5_configurator_maxma_demo erp5_configurator_ung erp5_configurator_run_my_doc slapos_configurator')) }}
 config-bt5-repository-url = {{ dumps(slapparameter_dict.get('bt5-repository-url', local_bt5_repository)) }}
 config-cloudooo-url = ${request-cloudooo:connection-url}
@@ -109,11 +80,16 @@ config-mysql-test-url-list = ${request-mariadb:connection-test-database-list}
 config-mysql-url-list = ${request-mariadb:connection-database-list}
 config-site-id = {{ dumps(site_id) }}
 config-smtp-url = {{ dumps(slapparameter_dict.get('smtp-url', 'smtp://localhost:25/')) }}
+config-timezone = {{ dumps(slapparameter_dict.get('timezone', 'UTC')) }}
+{% if zodb_type -%}
 config-tidstorage-ip = ${request-zodb:connection-tidstorage-ip}
 config-tidstorage-port = ${request-zodb:connection-tidstorage-port}
-config-timezone = {{ dumps(slapparameter_dict.get('timezone', 'UTC')) }}
 config-zodb-dict = ${request-zodb:connection-zodb-dict}
 config-zodb-storage-type = ${request-zodb:connection-zodb-storage-type}
+{% endif -%}
+{% if zodb_extern -%}
+config-zodb-extern = {{ dumps(zodb_extern) }}
+{% endif -%}
 software-type = zope
 
 {% set zope_family_dict = {} -%}
@@ -168,19 +144,6 @@ config-url = ${request-balancer:{{ family_name }}-v6}
 name = balancer
 software-type = balancer
 sla-computer_guid = {{ dumps(slapparameter_dict.get('balancer-computer-guid', computer_id)) }}
-extra-config =
-  tcpv4-port
-  haproxy-server-check-path
-  backend-path-dict
-  ssl-authentication-dict
-  ssl
-  zope-family-dict
-  country-code
-  email
-  state
-  city
-  company
-  {{ zope_address_list_id_dict.values() | join(' ') }}
 return =
 {%- for family in zope_family_dict %}
   {{ family }}
@@ -207,7 +170,6 @@ config-company = {{ slapparameter_dict.get('company', 'Compagny') }}
 < = request-common
 software-url = {{ dumps(frontend_dict['software-url']) }}
 software-type = {{ dumps(frontend_dict.get('software-type', 'RootSoftwareInstance')) }}
-sla = instance_guid
 sla-instance_guid = {{ dumps(frontend_dict['instance-guid']) }}
 slave = true
 {% set config_dict = {
@@ -216,7 +178,6 @@ slave = true
 {%   if frontend_dict.get('domain') -%}
 {%     do config_dict.__setitem__('custom_domain', frontend_dict['domain']) -%}
 {%   endif -%}
-extra-config = url {{ config_dict.keys() | join(' ') }}
 {%   for name, value in config_dict.items() -%}
 config-{{ name }} = {{ value }}
 {%   endfor -%}

software/slapos-master/software.cfg

@@ -55,7 +55,7 @@ extra-paths +=
   ${vifib:location}/master
 
 [template-erp5]
-md5sum = 1b1ba4a3d57e6f91eb86e36b19f75463
+md5sum = 6ada1fd4af0a451516443bfb6d00b717
 
 [template-balancer]
 md5sum = 818ab59ae966114735866aecef7a8563

software/slapos-testing/instance.cfg

@@ -37,7 +37,7 @@ repository = ${slapos.core-repository:location}
 
 [slapos.core-setup]
 recipe = plone.recipe.command
-command = echo "Updating setup...";cd $${slapos.core:location}; export PATH="$${slapos-test-runner:prepend-path}:$PATH"; export CPPFLAGS="$${environment:CPPFLAGS}"; export LDFLAGS="$${environment:LDFLAGS}"; export PYTHONPATH="$${environment:PYTHONPATH}"; export LOCAL_IPV4="$${environment:LOCAL_IPV4}"; ${python2.7:location}/bin/python setup.py test -n; ${python2.7:location}/bin/python setup.py test -n; ${python2.7:location}/bin/python setup.py test -n; ${python2.7:location}/bin/python setup.py test -n 
+command = echo "Updating setup...";cd $${slapos.core:location}; export PATH="$${slapos-test-runner:prepend-path}:$PATH"; export CPPFLAGS="$${environment:CPPFLAGS}"; export LDFLAGS="$${environment:LDFLAGS}"; export PYTHONPATH="$${environment:PYTHONPATH}"; export LOCAL_IPV4="$${environment:LOCAL_IPV4}"; ${python2.7:location}/bin/python setup.py test -n; ${python2.7:location}/bin/python setup.py test -n; ${python2.7:location}/bin/python setup.py test -n; ${python2.7:location}/bin/python setup.py test -n
 update-command = $${:command}
 
 [slapos.package]
@@ -66,17 +66,9 @@ repository = ${erp5-util-repository:location}
 
 [erp5.util-setup]
 recipe = plone.recipe.command
-command = echo "Updating setup...";cd $${erp5-util:location}; export PATH="$${slapos-test-runner:prepend-path}:$PATH"; export CPPFLAGS="$${environment:CPPFLAGS}"; export LDFLAGS="$${environment:LDFLAGS}"; export PYTHONPATH="$${environment:PYTHONPATH}"; ${python2.7:location}/bin/python setup.py test -n; ${python2.7:location}/bin/python setup.py test -n; ${python2.7:location}/bin/python setup.py test -n; ${python2.7:location}/bin/python setup.py test -n; ${python2.7:location}/bin/python setup.py test -n 
+command = echo "Updating setup...";cd $${erp5-util:location}; export PATH="$${slapos-test-runner:prepend-path}:$PATH"; export CPPFLAGS="$${environment:CPPFLAGS}"; export LDFLAGS="$${environment:LDFLAGS}"; export PYTHONPATH="$${environment:PYTHONPATH}"; ${python2.7:location}/bin/python setup.py test -n; ${python2.7:location}/bin/python setup.py test -n; ${python2.7:location}/bin/python setup.py test -n; ${python2.7:location}/bin/python setup.py test -n; ${python2.7:location}/bin/python setup.py test -n
 update-command = $${:command}
 
-[officejs]
-<= download-source
-repository = ${officejs-repository:location}
-
-[jio]
-<= download-source
-repository = ${jio-repository:location}
-
 [slapos-test-runner]
 recipe = slapos.cookbook:egg_test
 run-test-suite = $${create-directory:bin}/runTestSuite
@@ -92,8 +84,6 @@ test-list =
   $${slapos.recipe.build:location}
   $${slapos.recipe.cmmi:location}
   $${erp5-util:location}
-  $${jio:location}
-  $${officejs:location}
 prepend-path = ${git:location}/bin:${libxslt:location}/bin:${python2.7:location}/bin
 environment = environment
 

software/slapos-testing/software.cfg

@@ -19,20 +19,20 @@ parts =
   slapos.recipe.build-repository
   slapos.recipe.cmmi-repository
   erp5-util-repository
-  officejs-repository
-  jio-repository
   eggs
   phantomjs
   template
 
 [eggs]
 recipe = zc.recipe.egg
-eggs = 
+eggs =
   ${lxml-python:egg}
   erp5.util
   slapos.cookbook
   collective.recipe.template
   plone.recipe.command
+  slapos.recipe.template
+  slapos.recipe.cmmi
 entry-points =
   runTestSuite=erp5.util.testsuite:runTestSuite
 scripts =
@@ -72,18 +72,10 @@ repository = http://git.erp5.org/repos/slapos.recipe.cmmi.git
 <= git-clone-repository
 repository = http://git.erp5.org/repos/erp5.git
 
-[officejs-repository]
-<= git-clone-repository
-repository = http://git.erp5.org/repos/officejs.git
-
-[jio-repository]
-<= git-clone-repository
-repository = http://git.erp5.org/repos/jio.git
-
 [template]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg
-md5sum = 94bfc33413cdfbb6e243002f6d9b6928
+md5sum = 934df9a6b59fd7cc43195931b3ba4520
 output =  ${buildout:directory}/template.cfg
 mode = 640
 
@@ -91,5 +83,4 @@ mode = 640
 Pygments = 1.6
 collective.recipe.template = 1.10
 plone.recipe.command = 1.1
-slapos.recipe.build = 0.12
 slapos.recipe.template = 2.5

software/slaprunner/common.cfg

@@ -21,6 +21,9 @@ extends =
   ../../stack/resilient/buildout.cfg
   ../../stack/monitor/buildout.cfg
 
+# stacks are listed from most generic to most specific,
+# to avoid versioning issues
+
 parts =
   template
   eggs
@@ -43,14 +46,14 @@ mode = 0644
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-runner.cfg
 output = ${buildout:directory}/template-runner.cfg.in
-md5sum = a4f3127bc82ff3a671a65a150f26c969
+md5sum = 8ebc0e59db33f0105968a813a3130236
 mode = 0644
 
 [template-runner-import-script]
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/template/runner-import.sh.jinja2
 download-only = true
-md5sum = d64529e263d42b0b56e5fe3a976d18c9
+md5sum = c0d05a26b06ce172efaad03c52ef92ca
 filename = runner-import.sh.jinja2
 mode = 0644
 
@@ -65,7 +68,7 @@ mode = 0644
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/template/runner-export.sh.jinja2
 download-only = true
-md5sum = e74a0cbda64182c7fdff42cc46f07a12
+md5sum = 0f290b46c0b89ff84aee5c10477e07ca
 filename = runner-export.sh.jinja2
 mode = 0644
 
@@ -87,7 +90,7 @@ mode = 0644
 recipe = hexagonit.recipe.download
 url = ${:_profile_base_location_}/instance-resilient-test.cfg.jinja2
 download-only = true
-md5sum = 01545742e97b4b95cd5e14d5d7cb1584
+md5sum = 105c7c427def7b2643b7480b2bada3a5
 filename = instance-resilient-test.cfg.jinja2
 mode = 0644
 
@@ -110,7 +113,7 @@ mode = 0644
 [template_launcher]
 recipe = slapos.recipe.download
 url = ${:_profile_base_location_}/launcher.in
-md5sum = c7f8b6e9ae84aa94686a9cbaaa3dd693
+md5sum = 525e37ea8b2acf6209869999b15071a6
 filename = launcher.in
 mode = 0644
 location = ${buildout:parts-directory}/${:_buildout_section_name_}

software/slaprunner/instance-resilient-test.cfg.jinja2

@@ -41,9 +41,7 @@ recipe = slapos.cookbook:request
 software-url = ${slap-connection:software-release-url}
 software-type = resilient
 name = Resilient Instance (Root Instance)
-{% set cluster_parameter_dict = slapparameter_dict.get('cluster', {}) -%}
-config = resiliency-backup-periodicity resilient-clone-number {{ cluster_parameter_dict.keys() | join(' ') }}
-{% for key, value in cluster_parameter_dict.items() -%}
+{% for key, value in slapparameter_dict.get('cluster', {}).iteritems() -%}
 config-{{ key }} = {{ dumps(value) }}
 {% endfor -%}
 config-resiliency-backup-periodicity = */6 * * * *
@@ -51,7 +49,6 @@ config-resilient-clone-number = 2
 # XXX hardcoded
 #config-frontend-domain = google.com
 # XXX Hack to deploy Root Instance on the same computer as the type-test Instance
-sla = computer_guid
 sla-computer_guid = ${slap-connection:computer-id}
 return = backend_url
 

software/slaprunner/instance-runner-input-schema.json

@@ -4,92 +4,92 @@
         "user-authorized-key": {
             "title": "User Authorized Key",
             "description": "SSH public key in order to connect to the SSH server of this runner instance.",
-            "type": "string",
+            "type": "string"
         },
         "instance-amount": {
             "title": "Partition Amount",
             "description": "Number of slappart to deploy inside the runner (default is 10). Needs instance to be restarted.",
             "type": "integer",
             "minimum": 1,
-            "maximum": 40,
+            "maximum": 40
         },
         "slapos-software": {
             "title": "Pre-selected Software Release",
             "description": "a relative path from the slapos git repo to a folder containing a software release, which will be automaticaly deployed while the runner instanciation, and only if the parameter auto-deploy is set to 'true'. For example: 'software/helloworld",
-            "type": "string",
+            "type": "string"
         },
         "auto-deploy": {
             "title": "Automatically Deploy Software",
             "description": "Authorizes the software declared with 'slapos-software' to be automatically deployed, or not. Needs instance to be restarted. (default is false)",
-            "type": "boolean",
+            "type": "boolean"
         },
         "slapos-repository": {
             "title": "SlapOS Git Repository URL",
             "description": "url of the default git repository that will be download by the runner while its instanciation. Will be cloned in a directory named 'slapos' (default is http://git.erp5.org/repos/slapos.git)",
             "type": "string",
             "format": "uri",
-            "pattern": "^(http|https|ftp)://",
+            "pattern": "^(http|https|ftp)://"
         },
         "slapos-reference": {
             "title": "SlapOS Git Branch Name",
             "description": "Branch or hash on which the default repository will checkout (default is master)",
-            "type": "string",
+            "type": "string"
         },
         "auto-deploy-instance": {
             "title": "Automatically Deploy Instances",
             "description": "Prevent the runner from deploying and starting instances. Needs instance to be restarted. It is set to false for instances of type 'import' in resiliency in any case (default is false)",
-            "type": "boolean",
+            "type": "boolean"
         },
         "autorun": {
             "title": "Automatically Run Sofware/Instance",
             "description": "Let automaticaly build and run a declared software with 'slapos-software'. Only works if 'slapos-software' is set, and 'auto-deploy' is true. Needs instance to be restarted. (default is false)",
-            "type": "boolean",
+            "type": "boolean"
         },
         "slapos-software-type": {
             "title": "Deployed Instance Software Type" ,
             "description": "Software type of your instance inside the runner",
-            "type": "string",
+            "type": "string"
         },
         "custom-frontend-backend-url": {
             "title": "Custom Frontend Backend URL",
             "description": "return an ipv4 frontend of the given ipv6(+optional port)",
             "type": "string",
-            "format": "uri",
+            "format": "uri"
         },
         "custom-frontend-backend-type": {
             "title": "Custom Frontend Backend Type",
             "description": "The type of the frontend slave instance to ask",
             "type": "string",
-            "enum": ["zope"],
+            "enum": ["zope"]
         },
         "custom-frontend-basic-auth": {
             "title": "Custom Frontend Basic Auth",
             "description": "if the ip given with 'custom-frontend-backend-url' is secure, set it to true for the promise do not fail",
-            "type": "boolean",
+            "type": "boolean"
         },
         "custom-frontend-instance-guid": {
             "title": "Custom Frontend Instance GUID",
             "description": "Instance guid of the frontend you whish to use",
-            "type": "string",
+            "type": "string"
         },
         "custom-frontend-software-type": {
             "title": "Custom Frontend Software Type",
             "description": "SoftwareType of the frontend you request (default is RootSoftwareInstance)",
             "type": "string",
-            "enum": ["RootSoftwareInstance", "default"],
+            "enum": ["RootSoftwareInstance", "default"]
         },
         "custom-frontend-software-url": {
             "title": "Custom Frontend Software URL",
             "description": "Software Url of the frontend you request (ie.: http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg)",
             "type": "string",
-            "format": "uri",
+            "format": "uri"
         },
         "monitor-port": {
             "title": "Monitor Port",
             "description": "Allow to manually change the port on wich the apache server running monitoring interface is listening. The default value for the webrunner is different from the default value of the standalone stack-monitor server (default 9684)",
             "type": "integer",
             "minimum": 9683,
-            "exclusiveMinimum": true,
+            "exclusiveMinimum": true
         },
         "cpu-usage-ratio": {
             "description": "Ratio of the CPU use for compilation, if value is set to n, compilation will use number-of-cpu/n of cpus (need instance restart)",
@@ -100,7 +100,7 @@
             "title": "Number of Resilient Clones",
             "description": "Number of clones to be deployed when a resilient runner, if missing a single clone is provided. Its value can be 1 or 2",
             "type": "integer",
-            "enum": [1, 2],
+            "enum": [1, 2]
         }
     }
 }

software/slaprunner/instance-runner-output-schema.json

@@ -15,7 +15,7 @@
       "type": "string"
     },
     "ssh_command": {
-      "description": "SSH command used to access your runner in ssh when you provided a ssh public key"
+      "description": "SSH command used to access your runner in ssh when you provided a ssh public key",
       "type": "string"
     },
     "monitor_url": {

software/slaprunner/instance-runner.cfg

@@ -78,16 +78,12 @@ computer-id = $${slap-connection:computer-id}
 partition-id = $${slap-connection:partition-id}
 
 {%- if slapparameter_dict.get('custom-frontend-instance-guid') -%}
-sla = instance_guid
 sla-instance_guid = $${slap-parameter:frontend-instance-guid}
 {% endif -%}
 
-{% set custom_frontend_backend_type = slapparameter_dict.get('custom-frontend-backend-type')%}
-{% if custom_frontend_backend_type %}
-config = url type
+{% set custom_frontend_backend_type = slapparameter_dict.get('custom-frontend-backend-type') -%}
+{% if custom_frontend_backend_type -%}
 config-type = {{ custom_frontend_backend_type }}
-{% else %}
-config = url
 {% endif -%}
 config-url = {{ slapparameter_dict.get('custom-frontend-backend-url') }}
 return = site_url domain
@@ -281,6 +277,7 @@ path_log = $${directory:log}/nginx.log
 path_access_log = $${directory:log}/nginx.access.log
 path_error_log = $${directory:log}/nginx.error.log
 path_tmp = $${directory:tmp}/
+nginx_prefix = $${buildout:directory}
 # Config files
 path_nginx_conf = $${directory:etc}/nginx.conf
 # Executables
@@ -421,7 +418,6 @@ name = SlapRunner Frontend
 # XXX We have hardcoded SR URL here.
 software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
 slave = true
-config = url domain
 config-url = $${slaprunner:access-url}
 config-domain = $${slap-parameter:frontend-domain}
 return = site_url domain
@@ -433,7 +429,6 @@ name = Monitor Frontend
 # XXX We have hardcoded SR URL here.
 software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
 slave = true
-config = url domain
 config-url = https://[$${monitor-httpd-configuration:listening-ip}]:$${monitor-parameters:port}
 config-domain = $${slap-parameter:frontend-domain}
 return = site_url domain

software/slaprunner/launcher.in

@@ -3,4 +3,6 @@
 # BEWARE: It will be overwritten automatically
 
 # Run nginx
-exec {{ param_nginx_frontend['bin_nginx'] }} -c {{ param_nginx_frontend['path_nginx_conf'] }}
+exec {{ param_nginx_frontend['bin_nginx'] }} \
+    -p {{ param_nginx_frontend['nginx_prefix'] }} \
+    -c {{ param_nginx_frontend['path_nginx_conf'] }}

software/slaprunner/software.cfg

@@ -20,25 +20,28 @@ gitdb = 0.5.4
 gunicorn = 19.1.1
 plone.recipe.command = 1.1
 pycrypto = 2.6.1
-rdiff-backup = 1.0.5
 slapos.recipe.build = 0.12
 slapos.recipe.download = 1.0.dev-r4053
-slapos.toolbox = 0.40.2
+slapos.toolbox = 0.40.4
 smmap = 0.8.2
 z3c.recipe.scripts = 1.0.1
 
 # Required by:
-# slapos.toolbox==0.40.2
+# slapos.toolbox==0.40.4
 GitPython = 0.3.2.RC1
 
 # Required by:
-# slapos.toolbox==0.40.2
+# slapos.toolbox==0.40.4
 atomize = 0.2.0
 
 # Required by:
-# slapos.toolbox==0.40.2
+# slapos.toolbox==0.40.4
 feedparser = 5.1.3
 
 # Required by:
-# slapos.toolbox==0.40.2
+# slapos.cookbook==0.87
+jsonschema = 2.4.0
+
+# Required by:
+# slapos.toolbox==0.40.4
 paramiko = 1.15.1

software/slaprunner/template/runner-export.sh.jinja2

 #!{{ shell_binary }}
-LANG=en_US
-export $LANG
+LC_ALL=C
+export LC_ALL
 umask 077
 sync_element () {
   path=$1
@@ -11,7 +11,7 @@ sync_element () {
   do
     cd $path;
     if [ -f $element ] || [ -d $element ]; then
-       {{ rsync_binary }} -av --safe-links --delete $element  $backup_path;
+       {{ rsync_binary }} -rlptgov --safe-links --delete $element  $backup_path;
     fi
   done
 }
@@ -23,3 +23,5 @@ sync_element {{ directory['etc'] }}  {{ directory['backup'] }}/etc/ config.json
 if [ -d {{ directory['backup'] }}/runner/software ]; then
   rm {{ directory['backup'] }}/runner/software/*
 fi
+
+cd {{ directory['backup'] }} && find -type f ! -name backup.signature -print0 | xargs -P4 -0 sha256sum | LC_ALL=C sort -k 66 > backup.signature