Merge branch 'master' of https://lab.nexedi.com/donkey-hotei/slapos

CHANGES.txt

 Changes
 =======
 
+1.0.31 (2016-05-30)
+-------------------
+
+  * Implement cross recipe cache for registerComputerPartition
+  * Fixup! workarround for long shebang (place script on bin)  
+
+1.0.30 (2016-05-23)
+-------------------
+
+  * Implement a workarround for long shebang
+  * Implement Validation for user inputs ssl certificates 
+
 1.0.25 (2016-04-15)
 -------------------
+
   * fixup slap configuration: provide instance and root instance title
 
 1.0.22 (2016-04-01)
 -------------------
+
   * slap configuration: provide instance and root instance title
 
 1.0.16 (2015-10.27)
 -------------------
+
   * kvm recipe: fix bugs dowload image and disk creation
 
 1.0.14 (2015-10.26)
+
 -------------------
   * kvm recipe: Allow to set keyboard layout language used by qemu and VNC
   * simplehttpserver-recipe: fix encoding error
 
 0.103 (2015-07-24)
 ------------------
+
   * kvm: fix issues with boolean parameters and add 'qed' in external disk format list.
   * simplehttpserver-recipe: Add support for POST method which only get content and save into specified file.
 
 0.102 (2015-05-22)
 ------------------
+
  * kvm-recipe: vm of kvm-cluster can get ipv4/hostname of all other vm in the same cluster
  * simplehttpserver-recipe: simple http server to serve files
 
 0.101 (2015-04-29)
 ------------------
+
  * kvm recipe: new parameters: external-disk-format, numa and cpu-options.
  * kvm recipe: allow guest VM to connect to host http service via a local predefined ipv4 address (guestfwd).
 
 0.100 (2015-04-20)
 ------------------
+
  * re6stnet recipe: re6st-registry log can now be reopened with SIGUSR1
  * re6stnet recipe: re6st certificate generation is improved.
 
 0.99 (2015-04-10)
 -----------------
+
  * re6stnet: new recipe to deploy re6st registry (re6st master) with slapos.
 
 0.98 (2015-04-09)
 -----------------
+
  * shellinabox: do not run in debug mode, it is much slower !
 
 0.97 (2015-03-26)
 -----------------
+
  * switch softwaretype recipe: the recipe is backward compatible with old slapos node packages.
  * kvm recipe: Avoid getting wrong storage path when creating kvm external disk
 
 0.96 (2015-03-20)
 -----------------
+
  * slap configuration: recipe can read from master network information releated to a tap interface
  * slap configuration: recipe will setup data folder in DATA directory of computer partition if disk is mounted
  * switch softwaretype recipe: also generate tap network information when they exist

component/cloudooo/buildout.eggversion.cfg

@@ -18,6 +18,7 @@ eggs =
   cloudooo.handler.pdf
   cloudooo.handler.ffmpeg
   cloudooo.handler.imagemagick
+  cloudooo.handler.wkhtmltopdf
   PasteScript
   python-magic
 entry-points =

component/curl/buildout.cfg

@@ -12,8 +12,8 @@ parts =
 
 [curl]
 recipe = slapos.recipe.cmmi
-url = http://curl.haxx.se/download/curl-7.47.0.tar.bz2
-md5sum = 85c58a00412476993050cb242a3f365d
+url = http://curl.haxx.se/download/curl-7.49.0.tar.bz2
+md5sum = 7416aaff4a9210b43edda7615ffa4169
 configure-options =
   --disable-static
   --disable-ldap

component/dbus/buildout.cfg

@@ -32,6 +32,6 @@ configure-options =
   --disable-gtk-doc-html
 environment =
   PATH=${pkgconfig:location}/bin:${glib:location}/bin:%(PATH)s
-  PKG_CONFIG_PATH=${dbus:location}/lib/pkgconfig:${glib:location}/lib/pkgconfig
+  PKG_CONFIG_PATH=${dbus:location}/lib/pkgconfig:${glib:location}/lib/pkgconfig:${pcre:location}/lib/pkgconfig
   CPPFLAGS=-I${libexpat:location}/include
   LDFLAGS=-L${libexpat:location}/lib -L${gettext:location}/lib -Wl,-rpath=${zlib:location}/lib

component/firewalld/buildout.cfg

@@ -32,7 +32,7 @@ pre-configure =
   ./autogen.sh
 
 environment =
-  PKG_CONFIG_PATH=${glib:location}/lib/pkgconfig
+  PKG_CONFIG_PATH=${glib:location}/lib/pkgconfig:${pcre:location}/lib/pkgconfig
   PATH=${autoconf:location}/bin:${automake:location}/bin:${pkgconfig:location}/bin:${libtool:location}/bin:${intltool:location}/bin:${gettext:location}/bin:${glib:location}/bin:${perl:location}/bin:%(PATH)s
   CPPFLAGS=-I${gettext:location}/include
   LDFLAGS=-L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib
@@ -49,7 +49,7 @@ url = http://dbus.freedesktop.org/releases/dbus-python/dbus-python-${:version}.t
 md5sum = b09cd2d1a057cc432ce944de3fc06bf7
 python-egg = ${buildout:parts-directory}/${:_buildout_section_name_}/lib/python2.7/site-packages
 environment =
-  PKG_CONFIG_PATH=${dbus:location}/lib/pkgconfig:${dbus-glib:location}/lib/pkgconfig:${glib:location}/lib/pkgconfig
+  PKG_CONFIG_PATH=${dbus:location}/lib/pkgconfig:${dbus-glib:location}/lib/pkgconfig:${glib:location}/lib/pkgconfig:${pcre:location}/lib/pkgconfig
   PATH=${pkgconfig:location}/bin:%(PATH)s
   DBUS_CFLAGS=-I${dbus:location}/include/dbus-1.0 -I${dbus:location}/lib/dbus-1.0/include
   DBUS_LIBS=-L${dbus:location}/lib -ldbus-1
@@ -73,7 +73,7 @@ configure-options =
 
 environment =
   PATH=${autoconf:location}/bin:${automake:location}/bin:${pkgconfig:location}/bin:${libtool:location}/bin:${intltool:location}/bin:${gettext:location}/bin:${glib:location}/bin:${flex:location}/bin:${bison:location}/bin:%(PATH)s
-  PKG_CONFIG_PATH=${glib:location}/lib/pkgconfig
+  PKG_CONFIG_PATH=${glib:location}/lib/pkgconfig:${pcre:location}/lib/pkgconfig
   CPPFLAGS=-I${glib:location}/include/glib-2.0 -I${glib:location}/lib/glib-2.0/include
   LDFLAGS=-L${glib:location}/lib -Wl,-rpath=${glib:location}/lib -L${libffi:location}/lib -Wl,-rpath=${libffi:location}/lib -lffi
   M4=${m4:location}/bin/m4
@@ -99,7 +99,7 @@ configure-options =
 
 environment =
   PATH=${pkgconfig:location}/bin:${libtool:location}/bin:${glib:location}/bin:${xz-utils:location}/bin:%(PATH)s
-  PKG_CONFIG_PATH=${glib:location}/lib/pkgconfig:${gobject-introspection:location}/lib/pkgconfig
+  PKG_CONFIG_PATH=${glib:location}/lib/pkgconfig:${gobject-introspection:location}/lib/pkgconfig:${pcre:location}/lib/pkgconfig
   PYTHON=${python2.7:location}/bin/python2.7
   FFI_CFLAGS=-I${libffi:location}/include
   FFI_LIBS=-L${libffi:location}/lib -Wl,-rpath=${libffi:location}/lib -lffi

component/glib/buildout.cfg

@@ -2,6 +2,7 @@
 extends =
   ../gettext/buildout.cfg
   ../libffi/buildout.cfg
+  ../pcre/buildout.cfg
   ../perl/buildout.cfg
   ../xz-utils/buildout.cfg
   ../zlib/buildout.cfg
@@ -10,8 +11,8 @@ parts =
 
 [glib]
 recipe = slapos.recipe.cmmi
-url = http://ftp.gnome.org/pub/gnome/core/3.16/3.16.2/sources/glib-2.44.1.tar.xz
-md5sum = 83efba4722a9674b97437d1d99af79db
+url = http://ftp.gnome.org/pub/gnome/core/3.20/3.20.2/sources/glib-2.48.1.tar.xz
+md5sum = 67bd3b75c9f6d5587b457dc01cdcd5bb
 configure-options =
   --with-python=${python2.7:location}/bin/python2.7
   --disable-static
@@ -25,6 +26,8 @@ environment =
   LDFLAGS=-L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
   LIBFFI_CFLAGS=-I${libffi:location}/include
   LIBFFI_LIBS= -L${libffi:location}/lib -Wl,-rpath=${libffi:location}/lib -lffi
+  PCRE_CFLAGS=-I${pcre:location}/include
+  PCRE_LIBS=-L${pcre:location}/lib -Wl,-rpath=${pcre:location}/lib -lpcre
 
 [x86-cygwin-glib]
 patches =

component/glibmm/buildout.cfg

@@ -11,9 +11,10 @@ parts =
 
 [glibmm]
 recipe = slapos.recipe.cmmi
+# we keep using glibmm-2.44 that is buildable with non-C++11 compiler
 url = http://ftp.gnome.org/pub/gnome/core/3.16/3.16.2/sources/glibmm-2.44.0.tar.xz
 md5sum = 32ee4150b436d097fe2506d0b0b13a75
-pkg_config_depends = ${glib:location}/lib/pkgconfig:${libsigc:location}/lib/pkgconfig
+pkg_config_depends = ${glib:location}/lib/pkgconfig:${libsigc:location}/lib/pkgconfig:${pcre:location}/lib/pkgconfig
 configure-options =
   --disable-documentation
 environment =

component/gtk-2/buildout.cfg

@@ -26,7 +26,7 @@ parts =
 recipe = slapos.recipe.cmmi
 url = http://cairographics.org/releases/cairo-1.14.2.tar.xz
 md5sum = e1cdfaf1c6c995c4d4c54e07215b0118
-pkg_config_depends = ${libXext:location}/lib/pkgconfig:${libXext:pkg_config_depends}:${libpng:location}/lib/pkgconfig:${fontconfig:location}/lib/pkgconfig:${fontconfig:pkg_config_depends}:${pixman:location}/lib/pkgconfig:${glib:location}/lib/pkgconfig
+pkg_config_depends = ${fontconfig:location}/lib/pkgconfig:${fontconfig:pkg_config_depends}:${glib:location}/lib/pkgconfig:${libXext:location}/lib/pkgconfig:${libXext:pkg_config_depends}:${libpng:location}/lib/pkgconfig:${pcre:location}/lib/pkgconfig:${pixman:location}/lib/pkgconfig
 configure-options =
   --disable-static
   --disable-gtk-doc-html
@@ -61,8 +61,8 @@ environment =
 
 [pango]
 recipe = slapos.recipe.cmmi
-url = http://ftp.gnome.org/pub/gnome/core/3.14/3.14.2/sources/pango-1.36.8.tar.xz
-md5sum = 217a9a753006275215fa9fa127760ece
+url = http://ftp.gnome.org/pub/gnome/core/3.20/3.20.2/sources/pango-1.40.1.tar.xz
+md5sum = 6fc88c6529890d6c8e03074d57a3eceb
 pkg_config_depends = ${harfbuzz:location}/lib/pkgconfig:${harfbuzz:pkg_config_depends}
 configure-options =
   --disable-static
@@ -75,9 +75,9 @@ environment =
 
 [gdk-pixbuf]
 recipe = slapos.recipe.cmmi
-url = http://ftp.gnome.org/pub/GNOME/sources/gdk-pixbuf/2.32/gdk-pixbuf-2.32.1.tar.xz
-md5sum = b1590189a2e89fab9f871959c168508c
-pkg_config_depends = ${glib:location}/lib/pkgconfig:${libX11:location}/lib/pkgconfig:${libX11:pkg_config_depends}
+url = http://ftp.gnome.org/pub/gnome/core/3.20/3.20.2/sources/gdk-pixbuf-2.35.1.tar.xz
+md5sum = 72e0c924d5dc96bfde58a3b65ed83744
+pkg_config_depends = ${glib:location}/lib/pkgconfig:${libX11:location}/lib/pkgconfig:${libX11:pkg_config_depends}:${pcre:location}/lib/pkgconfig
 configure-options =
   --disable-static
   --disable-gtk-doc-html
@@ -92,21 +92,21 @@ environment =
 
 [atk]
 recipe = slapos.recipe.cmmi
-url = http://ftp.gnome.org/pub/gnome/core/3.16/3.16.2/sources/atk-2.16.0.tar.xz
-md5sum = c7c5002bd6e58b4723a165f1bf312116
+url = http://ftp.gnome.org/pub/gnome/core/3.20/3.20.2/sources/atk-2.20.0.tar.xz
+md5sum = 5187b0972f4d3905f285540b31395e20
 configure-options =
   --with-python=${python2.7:location}/bin/python2.7
   --disable-gtk-doc-html
 environment =
   PATH=${glib:location}/bin:${pkgconfig:location}/bin:${xz-utils:location}/bin:%(PATH)s
-  PKG_CONFIG_PATH=${glib:location}/lib/pkgconfig
+  PKG_CONFIG_PATH=${glib:location}/lib/pkgconfig:${pcre:location}/lib/pkgconfig
   LDFLAGS=-L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib -L${glib:location}/lib -Wl,-rpath=${gettext:location}/lib
   LD_LIBRARY_PATH=${glib:location}/lib:${gettext:location}/lib
 
 [gtk-2]
 recipe = slapos.recipe.cmmi
-url = http://ftp.gnome.org/pub/gnome/core/3.16/3.16.2/sources/gtk+-2.24.28.tar.xz
-md5sum = bfacf87b2ea67e4e5c7866a9003e6526
+url = http://ftp.gnome.org/pub/gnome/core/3.20/3.20.2/sources/gtk+-2.24.30.tar.xz
+md5sum = 04568ba5c58b75e3c7543e45628ad789
 pkg_config_depends = ${pango:location}/lib/pkgconfig:${pango:pkg_config_depends}:${atk:location}/lib/pkgconfig:${gdk-pixbuf:location}/lib/pkgconfig
 configure-options =
   --disable-static

component/gtkmm/buildout.cfg

@@ -28,6 +28,7 @@ environment =
 
 [pangomm]
 recipe = slapos.recipe.cmmi
+# we keep using pangomm-2.36 for glibmm-2.44 that is buildable with non-C++11 compiler
 url = http://ftp.gnome.org/pub/gnome/core/3.16/3.16.2/sources/pangomm-2.36.0.tar.xz
 md5sum = 62910723211d86ab825b666b479871c9
 pkg_config_depends = ${pango:location}/lib/pkgconfig:${pango:pkg_config_depends}:${glibmm:location}/lib/pkgconfig:${glibmm:pkg_config_depends}:${cairomm:location}/lib/pkgconfig
@@ -41,9 +42,10 @@ environment =
 
 [atkmm]
 recipe = slapos.recipe.cmmi
+# we keep using atkmm-2.22 for glibmm-2.44 that is buildable with non-C++11 compiler
 url = http://ftp.gnome.org/pub/gnome/core/3.12/3.12.2/sources/atkmm-2.22.7.tar.xz
 md5sum = fec7db3fc47ba2e0c95d130ec865a236
-pkg_config_depends = ${glib:location}/lib/pkgconfig:${atk:location}/lib/pkgconfig:${libsigc:location}/lib/pkgconfig:${glibmm:location}/lib/pkgconfig:
+pkg_config_depends = ${atk:location}/lib/pkgconfig:${glib:location}/lib/pkgconfig:${glibmm:location}/lib/pkgconfig:${libsigc:location}/lib/pkgconfig:${pcre:location}/lib/pkgconfig
 configure-options =
   --disable-static
   --disable-documentation

component/haproxy/buildout.cfg

@@ -12,8 +12,8 @@ parts = haproxy
 
 [haproxy]
 recipe = slapos.recipe.cmmi
-url = http://www.haproxy.org/download/1.6/src/haproxy-1.6.4.tar.gz
-md5sum = ee107312ef58432859ee12bf048025ab
+url = http://www.haproxy.org/download/1.6/src/haproxy-1.6.5.tar.gz
+md5sum = 5290f278c04e682e42ab71fed26fc082
 configure-command = true
 # If the system is running on Linux 2.6, we use "linux26" as the TARGET,
 # otherwise use "generic".

component/inkscape/buildout.cfg

@@ -28,7 +28,7 @@ configure-options =
   --disable-static
 environment =
   PATH=${glib:location}/bin:${pkgconfig:location}/bin:%(PATH)s
-  PKG_CONFIG_PATH=${glib:location}/lib/pkgconfig:${atk:location}/lib/pkgconfig:${libsigc:location}/lib/pkgconfig:${glibmm:location}/lib/pkgconfig:
+  PKG_CONFIG_PATH=${atk:location}/lib/pkgconfig:${glib:location}/lib/pkgconfig:${glibmm:location}/lib/pkgconfig:${libsigc:location}/lib/pkgconfig:${pcre:location}/lib/pkgconfig
   LDFLAGS=-L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib
 
 [inkscape]

component/jasper/CVE-2016-1577.patch

+Description: CVE-2016-1577: Prevent double-free in jas_iccattrval_destroy()
+Origin: vendor, http://www.openwall.com/lists/oss-security/2016/03/03/12
+Bug-Ubuntu: https://launchpad.net/bugs/1547865
+Bug-Debian: https://bugs.debian.org/816625
+Forwarded: not-needed
+Author: Tyler Hicks <tyhicks@canonical.com>
+Reviewed-by: Salvatore Bonaccorso <carnil@debian.org>
+Last-Update: 2016-03-05
+
+--- a/src/libjasper/base/jas_icc.c
++++ b/src/libjasper/base/jas_icc.c
+@@ -300,6 +300,7 @@ jas_iccprof_t *jas_iccprof_load(jas_stre
+ 				if (jas_iccprof_setattr(prof, tagtabent->tag, attrval))
+ 					goto error;
+ 				jas_iccattrval_destroy(attrval);
++				attrval = 0;
+ 			} else {
+ #if 0
+ 				jas_eprintf("warning: skipping unknown tag type\n");

component/jasper/CVE-2016-2089.patch

+Description: CVE-2016-2089: matrix rows_ NULL pointer dereference in jas_matrix_clip()
+Origin: vendor
+Bug-RedHat: https://bugzilla.redhat.com/show_bug.cgi?id=1302636
+Bug-Debian: https://bugs.debian.org/812978
+Forwarded: not-needed
+Author: Tomas Hoger <thoger@redhat.com>
+Reviewed-by: Salvatore Bonaccorso <carnil@debian.org>
+Last-Update: 2016-03-05
+
+--- a/src/libjasper/base/jas_image.c
++++ b/src/libjasper/base/jas_image.c
+@@ -426,6 +426,10 @@ int jas_image_readcmpt(jas_image_t *imag
+ 		return -1;
+ 	}
+ 
++	if (!data->rows_) {
++		return -1;
++	}
++
+ 	if (jas_matrix_numrows(data) != height || jas_matrix_numcols(data) != width) {
+ 		if (jas_matrix_resize(data, height, width)) {
+ 			return -1;
+@@ -479,6 +483,10 @@ int jas_image_writecmpt(jas_image_t *ima
+ 		return -1;
+ 	}
+ 
++	if (!data->rows_) {
++		return -1;
++	}
++
+ 	if (jas_matrix_numrows(data) != height || jas_matrix_numcols(data) != width) {
+ 		return -1;
+ 	}
+--- a/src/libjasper/base/jas_seq.c
++++ b/src/libjasper/base/jas_seq.c
+@@ -262,6 +262,10 @@ void jas_matrix_divpow2(jas_matrix_t *ma
+ 	int rowstep;
+ 	jas_seqent_t *data;
+ 
++	if (!matrix->rows_) {
++		return;
++	}
++
+ 	rowstep = jas_matrix_rowstep(matrix);
+ 	for (i = matrix->numrows_, rowstart = matrix->rows_[0]; i > 0; --i,
+ 	  rowstart += rowstep) {
+@@ -282,6 +286,10 @@ void jas_matrix_clip(jas_matrix_t *matri
+ 	jas_seqent_t *data;
+ 	int rowstep;
+ 
++	if (!matrix->rows_) {
++		return;
++	}
++
+ 	rowstep = jas_matrix_rowstep(matrix);
+ 	for (i = matrix->numrows_, rowstart = matrix->rows_[0]; i > 0; --i,
+ 	  rowstart += rowstep) {
+@@ -306,6 +314,10 @@ void jas_matrix_asr(jas_matrix_t *matrix
+ 	int rowstep;
+ 	jas_seqent_t *data;
+ 
++	if (!matrix->rows_) {
++		return;
++	}
++
+ 	assert(n >= 0);
+ 	rowstep = jas_matrix_rowstep(matrix);
+ 	for (i = matrix->numrows_, rowstart = matrix->rows_[0]; i > 0; --i,
+@@ -325,6 +337,10 @@ void jas_matrix_asl(jas_matrix_t *matrix
+ 	int rowstep;
+ 	jas_seqent_t *data;
+ 
++	if (!matrix->rows_) {
++		return;
++	}
++
+ 	rowstep = jas_matrix_rowstep(matrix);
+ 	for (i = matrix->numrows_, rowstart = matrix->rows_[0]; i > 0; --i,
+ 	  rowstart += rowstep) {
+@@ -367,6 +383,10 @@ void jas_matrix_setall(jas_matrix_t *mat
+ 	int rowstep;
+ 	jas_seqent_t *data;
+ 
++	if (!matrix->rows_) {
++		return;
++	}
++
+ 	rowstep = jas_matrix_rowstep(matrix);
+ 	for (i = matrix->numrows_, rowstart = matrix->rows_[0]; i > 0; --i,
+ 	  rowstart += rowstep) {

component/jasper/CVE-2016-2116.patch

+Description: CVE-2016-2116: Prevent jas_stream_t memory leak in jas_iccprof_createfrombuf()
+Origin: vendor, http://www.openwall.com/lists/oss-security/2016/03/03/12
+Bug-Debian: https://bugs.debian.org/816626
+Forwarded: not-needed
+Author: Tyler Hicks <tyhicks@canoonical.com>
+Reviewed-by: Salvatore Bonaccorso <carnil@debian.org>
+Last-Update: 2016-03-05
+
+--- a/src/libjasper/base/jas_icc.c
++++ b/src/libjasper/base/jas_icc.c
+@@ -1693,6 +1693,8 @@ jas_iccprof_t *jas_iccprof_createfrombuf
+ 	jas_stream_close(in);
+ 	return prof;
+ error:
++	if (in)
++		jas_stream_close(in);
+ 	return 0;
+ }
+ 

component/jasper/buildout.cfg

@@ -19,6 +19,9 @@ patches =
   ${:_profile_base_location_}/CVE-2014-8138.patch#bfb9604fe84b6e686fea29bd760cf34d
   ${:_profile_base_location_}/CVE-2014-8157.patch#2fb5f62ba8a9f8afffc95a07d1194783
   ${:_profile_base_location_}/CVE-2014-8158.patch#9036077a1fab5de8819f210ea7b57a38
+  ${:_profile_base_location_}/CVE-2016-1577.patch#bc970cf3e8535559454781ec54db2d15
+  ${:_profile_base_location_}/CVE-2016-2089.patch#9b73eda015b04a6da493de89ce9b5685
+  ${:_profile_base_location_}/CVE-2016-2116.patch#387df217963281827e006ab4f14f869a
 configure-options =
   --disable-static
   --enable-shared

component/libexpat/CVE-2015-1283-refix.patch

+From 29a11774d8ebbafe8418b4a5ffb4cc1160b194a1 Mon Sep 17 00:00:00 2001
+From: Pascal Cuoq <cuoq@trust-in-soft.com>
+Date: Sun, 15 May 2016 09:05:46 +0200
+Subject: [PATCH] Avoid relying on undefined behavior in CVE-2015-1283 fix. It
+ does not really work: https://godbolt.org/g/Zl8gdF
+
+---
+ expat/lib/xmlparse.c | 6 ++++--
+ 1 file changed, 4 insertions(+), 2 deletions(-)
+
+diff --git a/lib/xmlparse.c b/lib/xmlparse.c
+index 13e080d..cdb12ef 100644
+--- a/lib/xmlparse.c
++++ b/lib/xmlparse.c
+@@ -1695,7 +1695,8 @@ XML_GetBuffer(XML_Parser parser, int len
+   }
+ 
+   if (len > bufferLim - bufferEnd) {
+-    int neededSize = len + (int)(bufferEnd - bufferPtr);
++    /* Do not invoke signed arithmetic overflow: */
++    int neededSize = (int) ((unsigned)len + (unsigned)(bufferEnd - bufferPtr));
+ /* BEGIN MOZILLA CHANGE (sanity check neededSize) */
+     if (neededSize < 0) {
+       errorCode = XML_ERROR_NO_MEMORY;
+@@ -1729,7 +1730,8 @@ XML_GetBuffer(XML_Parser parser, int len
+       if (bufferSize == 0)
+         bufferSize = INIT_BUFFER_SIZE;
+       do {
+-        bufferSize *= 2;
++        /* Do not invoke signed arithmetic overflow: */
++        bufferSize = (int) (2U * (unsigned) bufferSize);
+ /* BEGIN MOZILLA CHANGE (prevent infinite loop on overflow) */
+       } while (bufferSize < neededSize && bufferSize > 0);
+ /* END MOZILLA CHANGE */
+-- 
+2.8.2
+

component/libexpat/CVE-2015-1283.patch

+Description: fix multiple integer overflows in the XML_GetBuffer function
+ Multiple integer overflows in the XML_GetBuffer function in Expat through
+ 2.1.0, as used in Google Chrome before 44.0.2403.89 and other products,
+ allow remote attackers to cause a denial of service (heap-based buffer
+ overflow) or possibly have unspecified other impact via crafted XML data,
+ a related issue to CVE-2015-2716.
+Origin: Mozilla, https://hg.mozilla.org/releases/mozilla-esr31/rev/2f3e78643f5c
+Author: Eric Rahm <erahm@mozilla.com>
+Forwarded: not-needed
+Last-Update: 2015-07-24
+
+diff --git a/lib/xmlparse.c b/lib/xmlparse.c
+--- a/lib/xmlparse.c
++++ b/lib/xmlparse.c
+@@ -1673,29 +1673,40 @@ XML_ParseBuffer(XML_Parser parser, int l
+   XmlUpdatePosition(encoding, positionPtr, bufferPtr, &position);
+   positionPtr = bufferPtr;
+   return result;
+ }
+ 
+ void * XMLCALL
+ XML_GetBuffer(XML_Parser parser, int len)
+ {
++/* BEGIN MOZILLA CHANGE (sanity check len) */
++  if (len < 0) {
++    errorCode = XML_ERROR_NO_MEMORY;
++    return NULL;
++  }
++/* END MOZILLA CHANGE */
+   switch (ps_parsing) {
+   case XML_SUSPENDED:
+     errorCode = XML_ERROR_SUSPENDED;
+     return NULL;
+   case XML_FINISHED:
+     errorCode = XML_ERROR_FINISHED;
+     return NULL;
+   default: ;
+   }
+ 
+   if (len > bufferLim - bufferEnd) {
+-    /* FIXME avoid integer overflow */
+     int neededSize = len + (int)(bufferEnd - bufferPtr);
++/* BEGIN MOZILLA CHANGE (sanity check neededSize) */
++    if (neededSize < 0) {
++      errorCode = XML_ERROR_NO_MEMORY;
++      return NULL;
++    }
++/* END MOZILLA CHANGE */
+ #ifdef XML_CONTEXT_BYTES
+     int keep = (int)(bufferPtr - buffer);
+ 
+     if (keep > XML_CONTEXT_BYTES)
+       keep = XML_CONTEXT_BYTES;
+     neededSize += keep;
+ #endif  /* defined XML_CONTEXT_BYTES */
+     if (neededSize  <= bufferLim - buffer) {
+@@ -1714,17 +1725,25 @@ XML_GetBuffer(XML_Parser parser, int len
+     }
+     else {
+       char *newBuf;
+       int bufferSize = (int)(bufferLim - bufferPtr);
+       if (bufferSize == 0)
+         bufferSize = INIT_BUFFER_SIZE;
+       do {
+         bufferSize *= 2;
+-      } while (bufferSize < neededSize);
++/* BEGIN MOZILLA CHANGE (prevent infinite loop on overflow) */
++      } while (bufferSize < neededSize && bufferSize > 0);
++/* END MOZILLA CHANGE */
++/* BEGIN MOZILLA CHANGE (sanity check bufferSize) */
++      if (bufferSize <= 0) {
++        errorCode = XML_ERROR_NO_MEMORY;
++        return NULL;
++      }
++/* END MOZILLA CHANGE */
+       newBuf = (char *)MALLOC(bufferSize);
+       if (newBuf == 0) {
+         errorCode = XML_ERROR_NO_MEMORY;
+         return NULL;
+       }
+       bufferLim = newBuf + bufferSize;
+ #ifdef XML_CONTEXT_BYTES
+       if (bufferPtr) {
+
+
+
+

component/libexpat/buildout.cfg

@@ -2,9 +2,19 @@
 parts =
   libexpat
 
+extends =
+  ../patch/buildout.cfg
+
 [libexpat]
 recipe = slapos.recipe.cmmi
 url = http://downloads.sourceforge.net/project/expat/expat/2.1.0/expat-2.1.0.tar.gz
 md5sum = dd7dab7a5fea97d2a6a43f511449b7cd
 configure-options =
   --disable-static
+patch-options = -p1
+patches =
+  ${:_profile_base_location_}/CVE-2015-1283.patch#44b31d7377035591f37ad94a31a8042b
+  ${:_profile_base_location_}/CVE-2015-1283-refix.patch#47382fe30c9a49724c626cef793ef382
+  ${:_profile_base_location_}/CVE-2016-0718-v2-2-1.patch#4ee57e7a052ada99f2e11fa21a229727
+environment =
+  PATH=${patch:location}/bin:%(PATH)s

component/librsvg/buildout.cfg

@@ -10,19 +10,19 @@ extends =
 
 [libcroco]
 recipe = slapos.recipe.cmmi
-url = http://ftp.gnome.org/pub/gnome/core/3.12/3.12.2/sources/libcroco-0.6.8.tar.xz
-md5sum = 767e73c4174f75b99695d4530fd9bb80
+url = http://ftp.gnome.org/pub/gnome/core/3.20/3.20.2/sources/libcroco-0.6.11.tar.xz
+md5sum = dabc1911dfbfa85f8e6859ca47863168
 configure-options =
   --disable-static
 environment =
   PATH=${pkgconfig:location}/bin:${xz-utils:location}/bin:%(PATH)s
-  PKG_CONFIG_PATH=${pkgconfig:location}/lib/pkgconfig:${libxml2:location}/lib/pkgconfig:${glib:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig
+  PKG_CONFIG_PATH=${glib:location}/lib/pkgconfig:${libxml2:location}/lib/pkgconfig:${pcre:location}/lib/pkgconfig:${pkgconfig:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig
   LDFLAGS=-Wl,-rpath=${zlib:location}/lib
 
 [librsvg]
 recipe = slapos.recipe.cmmi
-url = http://ftp.gnome.org/pub/gnome/core/3.16/3.16.2/sources/librsvg-2.40.9.tar.xz
-md5sum = 31df15e3beaa8fbbf538ca3c52b400d2
+url = http://ftp.gnome.org/pub/gnome/core/3.20/3.20.2/sources/librsvg-2.40.15.tar.xz
+md5sum = 3a66ab5b4fe1fb43b471708e4ff39a0e
 pkg_config_depends = ${pango:location}/lib/pkgconfig:${pango:pkg_config_depends}:${zlib:location}/lib/pkgconfig:${gdk-pixbuf:location}/lib/pkgconfig:${libcroco:location}/lib/pkgconfig
 configure-options =
   --disable-static

component/libsigc/buildout.cfg

@@ -9,6 +9,7 @@ parts =
 
 [libsigc]
 recipe = slapos.recipe.cmmi
+# we keep using libsigc++-2.4 for glibmm-2.44 that is buildable with non-C++11 compiler
 url = http://ftp.gnome.org/pub/gnome/core/3.16/3.16.2/sources/libsigc++-2.4.1.tar.xz
 md5sum = 55945ba6e1652f89999e910f6b52047c
 configure-options =

component/libxml2/buildout.cfg

@@ -11,8 +11,8 @@ parts =
 
 [libxml2]
 recipe = slapos.recipe.cmmi
-url = http://xmlsoft.org/sources/libxml2-2.9.3.tar.gz
-md5sum = daece17e045f1c107610e137ab50c179
+url = http://xmlsoft.org/sources/libxml2-2.9.4.tar.gz
+md5sum = ae249165c173b1ff386ee8ad676815f5
 configure-options =
   --disable-static
   --without-python

component/libxslt/buildout.cfg

@@ -7,8 +7,8 @@ parts =
   libxslt
 
 [libxslt]
-url = ftp://xmlsoft.org/libxslt/libxslt-1.1.28.tar.gz
-md5sum = 9667bf6f9310b957254fdcf6596600b7
+url = ftp://xmlsoft.org/libxslt/libxslt-1.1.29.tar.gz
+md5sum = a129d3c44c022de3b9dcf6d6f288d72e
 recipe = slapos.recipe.cmmi
 configure-options =
   --disable-static

component/nginx/buildout.cfg

@@ -11,8 +11,8 @@ parts = nginx-output
 
 [nginx-common]
 recipe = slapos.recipe.cmmi
-url = http://nginx.org/download/nginx-1.9.14.tar.gz
-md5sum = a25818039f34b5d54b017d44c76321c4
+url = http://nginx.org/download/nginx-1.10.1.tar.gz
+md5sum = 088292d9caf6059ef328aa7dda332e44
 
 [nginx]
 <= nginx-common

component/pkgconfig/buildout.cfg

@@ -27,7 +27,7 @@ configure-options =
   --disable-host-tool
 environment =
   PATH=.:%(PATH)s
-  PKG_CONFIG_PATH=${glib:location}/lib/pkgconfig
+  PKG_CONFIG_PATH=${glib:location}/lib/pkgconfig:${pcre:location}/lib/pkgconfig
   CPPFLAGS=-I${glib:location}/include -I${popt:location}/include
   LDFLAGS=-L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib -L${glib:location}/lib -Wl,-rpath=${glib:location}/lib -L${popt:location}/lib -Wl,-rpath=${popt:location}/lib
   GLIB_CFLAGS=-I${glib:location}/include/glib-2.0 -I${glib:location}/lib/glib-2.0/include

component/qemu-kvm/buildout.cfg

@@ -37,7 +37,7 @@ configure-options =
   --disable-werror
 environment =
   PATH=${pkgconfig:location}/bin:%(PATH)s
-  PKG_CONFIG_PATH=${gnutls:location}/lib/pkgconfig:${glib:location}/lib/pkgconfig:${pixman:location}/lib/pkgconfig
+  PKG_CONFIG_PATH=${glib:location}/lib/pkgconfig:${gnutls:location}/lib/pkgconfig:${pcre:location}/lib/pkgconfig:${pixman:location}/lib/pkgconfig
   LDFLAGS=-L${pixman:location}/lib -Wl,-rpath=${pixman:location}/lib
 
 [debian-amd64-netinst.iso]

component/slapos/buildout.cfg

@@ -38,7 +38,7 @@ PATH=${bison:location}/bin:${bzip2:location}/bin:${gettext:location}/bin:${glib:
 CFLAGS=-I${bzip2:location}/include -I${gdbm:location}/include -I${gettext:location}/include -I${glib:location}/include -I${libxml2:location}/include -I${libxslt:location}/include -I${ncurses:location}/include -I${openssl:location}/include -I${popt:location}/include -I${readline:location}/include -I${sqlite3:location}/include -I${zlib:location}/include
 CPPFLAGS=${:CFLAGS}
 LDFLAGS=-L${bzip2:location}/lib -Wl,-rpath=${bzip2:location}/lib -L${gdbm:location}/lib -Wl,-rpath=${gdbm:location}/lib -L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib -L${glib:location}/lib -Wl,-rpath=${glib:location}/lib -L${libxml2:location}/lib -Wl,-rpath=${libxml2:location}/lib -L${libxslt:location}/lib -Wl,-rpath=${libxslt:location}/lib -L${ncurses:location}/lib -Wl,-rpath=${ncurses:location}/lib -L${openssl:location}/lib -Wl,-rpath=${openssl:location}/lib -L${popt:location}/lib -Wl,-rpath=${popt:location}/lib -L${readline:location}/lib -Wl,-rpath=${readline:location}/lib -L${sqlite3:location}/lib -Wl,-rpath=${sqlite3:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
-PKG_CONFIG_PATH=${glib:location}/lib/pkgconfig:${libxml2:location}/lib/pkgconfig:${libxslt:location}/lib/pkgconfig:${openssl:location}/lib/pkgconfig:${popt:location}/lib/pkgconfig:${python2.7:location}/lib/pkgconfig:${sqlite3:location}/lib/pkgconfig
+PKG_CONFIG_PATH=${glib:location}/lib/pkgconfig:${libxml2:location}/lib/pkgconfig:${libxslt:location}/lib/pkgconfig:${openssl:location}/lib/pkgconfig:${pcre:location}/lib/pkgconfig:${popt:location}/lib/pkgconfig:${python2.7:location}/lib/pkgconfig:${sqlite3:location}/lib/pkgconfig
 LD_LIBRARY_PATH=${bzip2:location}/lib:${gdbm:location}/lib:${gettext:location}/lib:${glib:location}/lib:${libxml2:location}/lib:${libxslt:location}/lib:${ncurses:location}/lib:${openssl:location}/lib:${popt:location}/lib:${readline:location}/lib:${sqlite3:location}/lib:${zlib:location}/lib
 
 [cfg-environment]

setup.py

@@ -28,7 +28,7 @@ from setuptools import setup, find_packages
 import glob
 import os
 
-version = '1.0.26.dev0'
+version = '1.0.31'
 name = 'slapos.cookbook'
 long_description = open("README.txt").read() + "\n" + \
     open("CHANGES.txt").read() + "\n"

slapos/recipe/certificate_authority/__init__.py

@@ -27,8 +27,10 @@
 import os
 import hashlib
 import ConfigParser
+import tempfile
 
 from slapos.recipe.librecipe import GenericBaseRecipe
+from certificate_authority import popenCommunicate
 
 class Recipe(GenericBaseRecipe):
 
@@ -93,6 +95,13 @@ class Recipe(GenericBaseRecipe):
 
 class Request(Recipe):
 
+  def setPath(self):
+    self.request_directory = self.options['requests-directory']
+    self.ca_private = self.options['ca-private']
+    self.ca_certs = self.options['ca-certs']
+    self.ca_key_ext = '.key'
+    self.ca_crt_ext = '.crt'
+
   def _options(self, options):
     if 'name' not in options:
       options['name'] = self.name
@@ -114,6 +123,7 @@ class Request(Recipe):
 
     # XXX Ugly hack to quickly provide custom certificate/key to everyone using the recipe
     if key_content and cert_content:
+      self._checkCertificateKeyConsistency(key_content, cert_content)
       open(key, 'w').write(key_content)
       open(certificate, 'w').write(cert_content)
       request_needed = False
@@ -145,3 +155,48 @@ class Request(Recipe):
       path_list.append(wrapper)
 
     return path_list
+
+  def _checkCertificateKeyConsistency(self, key, certificate, ca=""):
+
+    openssl_binary = self.options.get('openssl-binary', 'openssl')
+
+    tmpdir = tempfile.mkdtemp()
+    with open(tmpdir + "/ca", "w") as f:
+      f.write(ca)
+    
+    with open(tmpdir + "/key", "w") as f:
+      f.write(key)
+    
+    with open(tmpdir + "/cert", "w") as f:
+      f.write(certificate)
+
+    try:
+      # Simple test if the user/certificates are readable and don't raise
+      popenCommunicate([openssl_binary, 'x509', '-noout', '-text', '-in', tmpdir + "/cert"])
+      popenCommunicate([openssl_binary, 'rsa', '-noout', '-text', '-in', tmpdir + "/key"])
+      
+      # Get md5 to check if the key and certificate matches 
+      modulus_cert = popenCommunicate([openssl_binary, 'x509', '-noout', '-modulus', '-in', tmpdir + "/cert"])
+      modulus_key = popenCommunicate([openssl_binary, 'rsa', '-noout', '-modulus', '-in', tmpdir + "/key"])
+    
+      md5sum_cert = popenCommunicate([openssl_binary, 'md5'], modulus_cert)
+      md5sum_key = popenCommunicate([openssl_binary, 'md5'], modulus_key)
+    
+      if md5sum_cert != md5sum_key:
+        raise ValueError("The key and certificate provided don't patch each other. Please check your parameters") 
+    
+    except:
+      try:
+        file_list = [tmpdir + "/ca", tmpdir + "/key", tmpdir + "/cert"]
+        for f in file_list:
+          if os.path.exists(f):
+            os.unlink(f)
+
+        if os.path.exists(tmpdir):
+          os.rmdir(tmpdir)
+      except:
+        # do not raise during cleanup
+        pass
+      raise
+    else:
+      pass

slapos/recipe/librecipe/genericslap.py

@@ -29,6 +29,8 @@ import time
 
 from generic import GenericBaseRecipe
 
+CONNECTION_CACHE = {}
+
 class GenericSlapRecipe(GenericBaseRecipe):
   """Base class for all slap.recipe.* needing SLAP informations like instance
      parameters.
@@ -50,11 +52,18 @@ class GenericSlapRecipe(GenericBaseRecipe):
     self.cert_file = slap_connection.get('cert-file')
 
   def install(self):
+
+    cache_key = "%s_%s" % (self.computer_id, self.computer_partition_id)
+    self.computer_partition = CONNECTION_CACHE.get(cache_key, None)
+
     self.slap.initializeConnection(self.server_url, self.key_file,
         self.cert_file)
+    if self.computer_partition is None:
       self.computer_partition = self.slap.registerComputerPartition(
         self.computer_id,
         self.computer_partition_id)
+      CONNECTION_CACHE[cache_key] = self.computer_partition
+
     self.request = self.computer_partition.request
     self.setConnectionDict = self.computer_partition.setConnectionDict
     self.parameter_dict = self.computer_partition.getInstanceParameterDict()

slapos/recipe/wrapper.py

@@ -26,6 +26,7 @@
 ##############################################################################
 
 import shlex
+import os
 
 from slapos.recipe.librecipe import GenericBaseRecipe
 
@@ -57,8 +58,14 @@ class Recipe(GenericBaseRecipe):
 
         # We create a python script and a wrapper around the python
         # script because the python script might have a too long #! line
+        if os.path.exists(os.path.join(self.buildout['buildout']['directory'], "bin")): 
+          base_script_path = os.path.join(
+            self.buildout['buildout']['directory'], "bin/" + wrapper_path.split("/")[-1])
+        else:
+          base_script_path = os.path.join(
+            self.buildout['buildout']['directory'], wrapper_path.split("/")[-1])
         python_script = self.createPythonScript(
-            wrapper_path+'.py',
+            base_script_path +'.py',
             'slapos.recipe.librecipe.execute.generic_exec',
             (command_line, wait_files, environment,), )
         return [python_script, self.createWrapper(

software/agent/software.cfg

@@ -53,7 +53,7 @@ smmap = 0.9.0
 
 # Required by:
 # slapos.toolbox==0.55
-GitPython = 2.0.2
+GitPython = 2.0.5
 
 # Required by:
 # slapos.toolbox==0.55
@@ -65,7 +65,7 @@ feedparser = 5.2.1
 
 # Required by:
 # slapos.toolbox==0.55
-lockfile = 0.10.2
+lockfile = 0.12.2
 
 # Required by:
 # slapos.toolbox==0.55

software/anomaly-ids/software.cfg

+[buildout]
+versions = versions
+extends = 
+  https://lab.nexedi.com/nexedi/slapos/raw/master/software/wendelin/software.cfg
+parts +=
+  scikit-learn-repository
+  scikit-learn-egg
+
+[erp5_repository_list]
+repository_id_list = erp5 wendelin windelin
+
+[local-bt5-repository]
+list = ${erp5:location}/bt5 ${erp5:location}/product/ERP5/bootstrap ${wendelin:location}/bt5 ${windelin:location}/bt5
+
+# IsolationForest model with patch from fork of dev branch
+[scikit-learn-repository]
+recipe  = slapos.recipe.build:gitclone
+git-executable = ${git:location}/bin/git
+repository = https://github.com/donkey-hotei/scikit-learn.git
+location = ${buildout:parts-directory}/scikit-learn
+
+[scikit-learn-egg]
+recipe = zc.recipe.egg:develop
+egg    = scikit-learn
+setup  = ${scikit-learn-repository:location}/ 
+

software/apache-frontend/common.cfg

@@ -60,27 +60,27 @@ command =
 [template]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance.cfg
-md5sum = 51752d0e4eae8c761750117c44983594 
+md5sum = f686f765e55d1dce2e55a400f0714b3e 
 output = ${buildout:directory}/template.cfg
 mode = 0644
 
 [template-apache-frontend]
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/instance-apache-frontend.cfg
-md5sum = 07b6f864b46cb2f7b4b8aab71da95d13 
+md5sum = cc64151e4fe953f15f7ea8cf20718d84 
 output = ${buildout:directory}/template-apache-frontend.cfg
 mode = 0644
 
 [template-apache-replicate]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/instance-apache-replicate.cfg.in
-md5sum = a05bc0f925902ddf0b4d12146b10c9e1 
+md5sum = 9b17c835bcd927269cf510bf612f5985
 mode = 0644
 
 [template-slave-list]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/templates/apache-custom-slave-list.cfg.in
-md5sum = c01c0ebb749d16bda4649bf42d36c7ea 
+md5sum = 8b278b34a4fb063ba94e10186f725fcb
 mode = 640
 
 [template-slave-configuration]

software/apache-frontend/instance-apache-frontend.cfg

@@ -73,9 +73,8 @@ ca-dir = $${:srv}/ssl
 
 [switch-apache-softwaretype]
 recipe = slapos.cookbook:softwaretype
-single-default = $${dynamic-default-template-slave-list:rendered}
+single-default = $${dynamic-custom-personal-template-slave-list:rendered}
 single-custom-personal = $${dynamic-custom-personal-template-slave-list:rendered}
-single-custom-group = $${dynamic-custom-group-template-slave-list:rendered}
 
 [instance-parameter]
 # Fetches parameters defined in SlapOS Master for this instance.
@@ -164,53 +163,6 @@ extra-context =
     section apache_configuration apache-configuration
     key monitor_base_url monitor-instance-parameter:monitor-base-url 
 
-[dynamic-custom-group-template-slave-list]
-< = jinja2-template-base
-template = ${template-custom-slave-list:target}
-filename = custom-group-instance-slave-list.cfg
-extensions = jinja2.ext.do
-extra-context =
-    key apache_configuration_directory apache-directory:slave-configuration
-    key domain instance-parameter:configuration.domain
-    key http_port instance-parameter:configuration.plain_http_port
-    key https_port instance-parameter:configuration.port
-    key public_ipv4 instance-parameter:configuration.public-ipv4
-    key slave_instance_list instance-parameter:slave-instance-list
-    key extra_slave_instance_list instance-parameter:configuration.extra_slave_instance_list
-    key rewrite_cached_configuration apache-configuration:cached-rewrite-file
-    key custom_ssl_directory apache-directory:vh-ssl
-    key template_slave_configuration dynamic-virtualhost-template-slave:rendered
-    key apache_log_directory apache-directory:slave-log
-    key local_ipv4 instance-parameter:ipv4-random
-    key cache_port apache-configuration:cache-port
-    raw empty_template ${template-empty:target}
-    raw template_rewrite_cached ${template-rewrite-cached:target}
-    raw software_type single-custom-group
-
-[dynamic-default-template-slave-list]
-< = jinja2-template-base
-template = ${template-custom-slave-list:target}
-filename = default-instance-slave-list.cfg
-extensions = jinja2.ext.do
-extra-context =
-    key apache_configuration_directory apache-directory:slave-configuration
-    key domain instance-parameter:configuration.domain
-    key http_port instance-parameter:configuration.plain_http_port
-    key https_port instance-parameter:configuration.port
-    key public_ipv4 instance-parameter:configuration.public-ipv4
-    key slave_instance_list instance-parameter:slave-instance-list
-    key extra_slave_instance_list instance-parameter:configuration.extra_slave_instance_list
-    key rewrite_cached_configuration apache-configuration:cached-rewrite-file
-    key custom_ssl_directory apache-directory:vh-ssl
-    key apache_log_directory apache-directory:slave-log
-    key local_ipv4 instance-parameter:ipv4-random
-    key cache_port apache-configuration:cache-port
-    raw template_slave_configuration ${template-default-slave-virtualhost:target}
-    raw empty_template ${template-empty:target}
-    raw template_rewrite_cached ${template-rewrite-cached:target}
-    raw software_type single-default
-# XXXX Hack to allow two software types
-
 [dynamic-virtualhost-template-slave]
 <= jinja2-template-base
 template = ${template-slave-configuration:target}
@@ -608,12 +560,21 @@ apache-certificate =
 open-port = 80 443
 extra_slave_instance_list =
 frontend-name =
-
+monitor-cors-domains =
+monitor-httpd-port = 8072
+monitor-username = $${monitor-htpasswd:username}
+monitor-password = $${monitor-htpasswd:passwd}
 
 #######
 # Monitoring sections
 #
 
+[monitor-instance-parameter]
+monitor-httpd-port = $${slap-parameter:monitor-httpd-port}
+cors-domains = $${slap-parameter:monitor-cors-domains}
+username = $${slap-parameter:monitor-username}
+password = $${slap-parameter:monitor-password}
+
 [monitor-conf-parameters]
 private-path-list += 
   $${directory:logrotate-backup}
@@ -622,7 +583,7 @@ private-path-list +=
 [monitor-ats-cache-stats-wrapper]
 < = jinja2-template-base
 template = ${template-wrapper:output}
-rendered = $${monitor-directory:report}/ats-cache-stats
+rendered = $${monitor-directory:reports}/ats-cache-stats
 mode = 0700
 command = export TS_ROOT=$${buildout:directory} && echo "<pre>$(${trafficserver:location}/bin/traffic_shell $${monitor-ats-cache-stats-config:rendered})</pre>"
 extra-context =

software/apache-frontend/instance-apache-input-schema.json

@@ -5,8 +5,8 @@
 	"title": "Input Parameters",
 	"properties": {
 		"public-ipv4": {
-			"title": "Backend URL",
-			"description": "Url of the backend",
+			"title": "Public IPv4",
+			"description": "Public IPv4 that will be provided to Slaves to configure DNS.",
 			"type": "string"
 		},
 		"-frontend-authorized-slave-string": {
@@ -16,30 +16,32 @@
 		},
 		"apache-key": {
 			"title": "Apache Key",
-			"description": "Url of the Backend",
+			"description": "Apache Key",
+                        "textarea": true,
 			"type": "string"
 		},
 		"apache-certificate": {
 			"title": "Apache Certificate",
 			"description": "Apache Certificate",
+                        "textarea": true,
 			"type": "string"
 		},
 		"apache-ca-certificate": {
 			"title": "Apache CA Certificate",
 			"description": "Apache CA Certificate",
+                        "textarea": true,
 			"type": "string"
 		},
 		"domain": {
 			"title": "Domain",
-			"description": "Base Domain for create subdomains.",
+			"description": "Base Domain for create subdomains (ie.: example.com).",
 			"type": "string",
 			"pattern": "^([a-zA-Z0-9]([a-zA-Z0-9\\-]{0,61}[a-zA-Z0-9])?\\.)+[a-zA-Z]{2,6}$"
 		},
 		"-frontend-quantity": {
 			"title": "Frontend Replication Quantity",
 			"description": "Quantity of Frontends Replicate.",
-			"type": "integer",
-			"default": ""
+			"type": "integer"
 		}
 	}
 }

software/apache-frontend/instance-apache-replicate.cfg.in

@@ -124,6 +124,9 @@ monitor-base-url = ${monitor-conf-parameters:base-url}
 monitor-url = ${:monitor-base-url}/public/feeds
 monitor-user = ${monitor-instance-parameter:username}
 monitor-password = ${monitor-instance-parameter:password}
+{% set monitor_interface_url = slapparameter_dict.get('monitor-interface-url', 'https://monitor.app.officejs.com') -%}
+monitor-setup-url = {{ monitor_interface_url }}/#page=settings_configurator&url=${:monitor-url}
+
 
 #----------------------------
 #--

software/apache-frontend/instance-slave-apache-input-schema.json

@@ -7,7 +7,8 @@
     "url": {
       "title": "Backend URL",
       "description": "Url of the backend",
-      "type": "string"
+      "type": "string",
+      "pattern": "^(http|https|ftp)://"
     },
     "custom_domain": {
       "title": "Custom Domain",

software/apache-frontend/instance.cfg

@@ -33,10 +33,8 @@ recipe = slapos.cookbook:softwaretype
 default = $${dynamic-template-apache-replicate:rendered}
 RootSoftwareInstance = $${dynamic-template-apache-replicate:rendered}
 custom-personal = $${dynamic-template-apache-replicate:rendered}
-custom-group = $${dynamic-template-apache-replicate:rendered}
 single-default = ${template-apache-frontend:output}
 single-custom-personal = ${template-apache-frontend:output}
-single-custom-group = ${template-apache-frontend:output}
 replicate = $${dynamic-template-apache-replicate:rendered}
 
 [dynamic-template-apache-replicate]
@@ -47,6 +45,6 @@ extensions = jinja2.ext.do
 extra-context =
     raw template_publish_slave_information ${template-replicate-publish-slave-information:target}
 # Must match the key id in [switch-softwaretype] which uses this section.
-    raw software_type RootSoftwareInstance-default-custom-personal-custom-group-replicate
+    raw software_type RootSoftwareInstance-default-custom-personal-replicate
     raw template_monitor ${monitor2-template:rendered}
 

software/apache-frontend/software.cfg

@@ -16,7 +16,7 @@ smmap = 0.9.0
 
 # Required by:
 # slapos.toolbox==0.55
-GitPython = 2.0.2
+GitPython = 2.0.5
 
 # Required by:
 # slapos.toolbox==0.55
@@ -28,7 +28,7 @@ feedparser = 5.2.1
 
 # Required by:
 # slapos.toolbox==0.55
-lockfile = 0.10.2
+lockfile = 0.12.2
 
 # Required by:
 # slapos.toolbox==0.55

software/apache-frontend/templates/apache-custom-slave-list.cfg.in

@@ -98,7 +98,7 @@ command = {{frontend_configuration.get('apache-directory')}}/bin/htpasswd -cb ${
 #### Set Slave Certificates if needed
 
 # Set ssl certificates for each slave
-{%   for cert_name in ('ssl_key', 'ssl_crt', 'ssl_ca_crt', 'ssl_csr', 'ssl_proxy_ca_crt')-%}
+{%   for cert_name in ('ssl_ca_crt', 'ssl_csr', 'ssl_proxy_ca_crt')-%}
 {%     if cert_name in slave_instance -%}
 {%       set cert_title = '%s-%s' % (slave_reference, cert_name.replace('ssl_', '')) -%}
 {%       set cert_file = '/'.join([custom_ssl_directory, cert_title.replace('-','.')]) -%}
@@ -118,6 +118,43 @@ value = {{ dumps(slave_instance.get(cert_name)) }}
 {%     endif -%}
 {%   endfor -%}
 
+[cadirectory]
+recipe = slapos.cookbook:mkdirectory
+requests = {{ custom_ssl_directory }}/requests/
+private = {{ custom_ssl_directory }}/private/
+certs = {{ custom_ssl_directory }}/certs/
+newcerts = {{ custom_ssl_directory }}/newcerts/
+crl = {{ custom_ssl_directory }}/crl/
+
+{%   if 'ssl_key' in slave_instance and 'ssl_crt' in slave_instance -%}
+{%     set cert_title = '%s-crt' % (slave_reference) -%}
+{%     set key_title = '%s-key' % (slave_reference) -%}
+{%     set cert_file = '/'.join([custom_ssl_directory, cert_title.replace('-','.')]) -%}
+{%     set key_file = '/'.join([custom_ssl_directory, key_title.replace('-','.')]) -%}
+{%     do part_list.append(cert_title) -%}
+{%     do slave_parameter_dict.__setitem__("ssl_crt", cert_file) -%}
+{%     do slave_parameter_dict.__setitem__("ssl_key", key_file) -%}
+{%     do slave_instance.__setitem__('path_to_ssl_crt', cert_file) -%}
+{%     do slave_instance.__setitem__('path_to_ssl_key', key_file) -%}
+
+[{{cert_title}}]
+recipe = slapos.cookbook:certificate_authority.request
+#openssl-binary = ${openssl:location}/bin/openssl
+
+requests-directory = ${cadirectory:requests}
+ca-private = ${cadirectory:private}
+ca-certs = ${cadirectory:certs}
+ca-newcerts = ${cadirectory:newcerts}
+ca-crl = ${cadirectory:crl}
+
+key-file = {{ key_file }}
+cert-file = {{ cert_file }}
+key-content = {{ dumps(slave_instance.get('ssl_key')) }}
+cert-content = {{ dumps(slave_instance.get('ssl_crt')) }} 
+{%     endif -%}
+
+
+
 ############################
 #### Set Slave Configuration
 

software/apache-frontend/templates/apache-default-slave-list.cfg.in

-{% if slap_software_type in software_type  -%}
-
-{% set cached_server_dict = {} -%}
-{% set part_list = [] -%}
-{% set cache_access = "http://%s:%s" % (local_ipv4, cache_port) -%}
-{% set TRUE_VALUES = ['y', 'yes', '1', 'true'] -%}
-{% set generic_instance_parameter_dict = {'cache_access': cache_access,} -%}
-{% if extra_slave_instance_list -%}
-{%   set slave_instance_information_list = []-%}
-{%   set slave_instance_list = slave_instance_list + json_module.loads(extra_slave_instance_list) -%}
-{% endif -%}
-[jinja2-template-base]
-recipe = slapos.recipe.template:jinja2
-rendered = {{ apache_configuration_directory }}/${:filename}
-extra-context =
-context =
-    key eggs_directory buildout:eggs-directory
-    key develop_eggs_directory buildout:develop-eggs-directory
-    ${:extra-context}
-
-# Go throught slave list to set their configuration
-{% for slave_instance in slave_instance_list -%}
-{%   set slave_reference = slave_instance.get('slave_reference') -%}
-{%   set slave_section_title = 'dynamic-template-slave-instance-%s' % slave_reference -%}
-{%   set slave_parameter_dict = generic_instance_parameter_dict.copy() -%}
-# Set slave domain if none was defined
-{%   if slave_instance.get('domain', None) == None -%}
-# Backward compatibility
-{%     if slave_instance.get('custom_domain', None) != None -%}
-{%       do slave_instance.__setitem__('domain', slave_instance.get('custom_domain') )-%}
-{%     else -%}
-{%       do slave_instance.__setitem__('domain', "%s.%s" % (slave_instance.get('slave_reference').replace("-", "").lower(), domain)) -%}
-{%     endif -%}
-{%   endif -%}
-# Set personal log, two per slave
-{%   set access_log = '/'.join([apache_log_directory, '%s_access_log' % slave_reference]) -%}
-{%   set error_log = '/'.join([apache_log_directory, '%s_error_log' % slave_reference]) -%}
-# The slave use cache
-# Next line is forbidden and people who copy it will be hanged short
-{%   set enable_cache = ('' ~ slave_instance.get('enable_cache', '')).lower() in TRUE_VALUES -%}
-{%   if enable_cache -%}
-{%     do cached_server_dict.__setitem__(slave_instance.get('domain'), slave_instance.get('url')) -%}
-{%     do slave_instance.__setitem__('url', cache_access) -%}
-{%   endif -%}
-{%   do part_list.append(slave_section_title) -%}
-# Set up slave configuration file
-[{{ slave_section_title }}]
-< = jinja2-template-base
-template = {{ template_slave_configuration }}
-filename = {{ '%s.conf' % slave_reference }}
-extensions = jinja2.ext.do
-extra-context =
-    section slave_parameter {{ 'slave-instance-%s-configuration' % slave_reference }}
-    raw domain {{ domain }}
-    raw https_port {{ https_port }}
-    raw http_port {{ http_port }}
-    raw access_log {{ access_log }}
-    raw error_log {{ error_log }}
-{{ '\n' }}
-
-# Set ssl certificates for each slave
-{%   for cert_name in ('ssl_key', 'ssl_crt', 'ssl_ca_crt', 'ssl_csr')-%}
-{%     if cert_name in slave_instance -%}
-{%       set cert_title = '%s-%s' % (slave_reference, cert_name.replace('ssl_', '')) -%}
-{%       set cert_file = '/'.join([custom_ssl_directory, cert_title.replace('-','.')]) -%}
-{%       do part_list.append(cert_title) -%}
-{%       do slave_instance.__setitem__('path_to_' ~ cert_name, cert_file) -%}
-# Store certificates on fs
-[{{ cert_title }}]
-< = jinja2-template-base
-template = {{ empty_template }}
-rendered = {{ cert_file }}
-extra-context =
-    key content {{ cert_title + '-config:value' }}
-# Store certificate in config
-[{{ cert_title + '-config' }}]
-value = {{ dumps(slave_instance.get(cert_name)) }}
-{%     endif -%}
-{%   endfor -%}
-
-# Set apache configuration for slave
-[{{ ('slave-instance-%s-configuration' % slave_reference) }}]
-{%   for key, value in slave_instance.iteritems() -%}
-{{ key }} = {{ dumps(value) }}
-{%   endfor %}
-
-# Publish slave information
-{%   if not extra_slave_instance_list -%}
-{%     set publish_section_title = 'publish-%s-connection-information' % slave_instance.get('slave_reference') -%}
-{%     do part_list.append(publish_section_title) -%}
-[{{ publish_section_title }}]
-recipe = slapos.cookbook:publish
--slave-reference = {{ slave_instance.get('slave_reference') }}
-public-ipv4 = {{ public_ipv4 }}
-domain = {{ slave_instance.get('domain') }}
-url = http://{{ slave_instance.get('domain') }}
-# Backward compatibility
-site_url = ${:url}
-{%   else -%}
-{%     do slave_instance_information_list.append({'slave-reference':slave_instance.get('slave_reference'), 'public-ipv4':public_ipv4, 'domain':slave_instance.get('domain'), 'url':"http://%s" % slave_instance.get('domain'), 'site_url':"http://%s" % slave_instance.get('domain')}) -%}
-{%   endif -%}
-{% endfor -%}
-
-# Publish information for the instance
-{% set publish_section_title = 'publish-apache-information' -%}
-{% do part_list.append(publish_section_title) -%}
-[{{ publish_section_title }}]
-recipe = slapos.cookbook:publish
-public-ipv4 = {{ public_ipv4 }}
-private-ipv4 = {{ local_ipv4 }}
-domain = {{ domain }}
-{% if extra_slave_instance_list -%}
-slave-instance-information-list = {{ json_module.dumps(slave_instance_information_list) }}
-{% endif -%}
-
-
-{% do part_list.append('cached-rewrite-rules') -%}
-# Set rewrite rules for second apache
-[cached-rewrite-rules]
-< = jinja2-template-base
-template = {{ template_rewrite_cached }}
-rendered = {{ rewrite_cached_configuration }}
-extra-context =
-    import json_module json
-    key server_dict rewrite-rules:rules
-
-# Store Rewrite rules for second apache
-[rewrite-rules]
-rules = {{ dumps(cached_server_dict) }}
-
-# Add parts generated by template
-[buildout]
-parts +=
-{% for part in part_list -%}
-{{ '    %s' % part }}
-{% endfor -%}
-
-eggs-directory = {{ eggs_directory }}
-develop-eggs-directory = {{ develop_eggs_directory }}
-offline = true
-cache-access = {{ cache_access }}
-
-{% endif -%}

software/cloudooo/instance-cloudoo.cfg.in

@@ -98,6 +98,7 @@ link-binary =
   {{ parameter_dict['poppler'] }}/bin/pdfinfo
   {{ parameter_dict['poppler'] }}/bin/pdftotext
   {{ parameter_dict['poppler'] }}/bin/pdftohtml
+  {{ parameter_dict['wkhtmltopdf'] }}/wkhtmltopdf
 
 # rest of parts are candidates for some generic stuff
 [directory]

software/cloudooo/instance.cfg.in

@@ -54,6 +54,7 @@ libxcb = {{ libxcb_location }}
 mesa = {{ mesa_location }}
 poppler = {{ poppler_location }}
 pixman = {{ pixman_location }}
+wkhtmltopdf = {{ wkhtmltopdf_location }}
 xdamage = {{ xdamage_location }}
 xfixes = {{ xfixes_location }}
 zlib = {{ zlib_location }}

software/cloudooo/software.cfg

@@ -38,7 +38,7 @@ context =
 # XXX: "template.cfg" is hardcoded in instanciation recipe
 filename = template.cfg
 template = ${:_profile_base_location_}/instance.cfg.in
-md5sum = a26cb2a94b7bec07f62efcbe14fec608
+md5sum = 9871d0f315e49da0571b8a02f38e53c3
 extra-context =
     key buildout_bin_directory buildout:bin-directory
     key coreutils_location coreutils:location
@@ -73,6 +73,7 @@ extra-context =
     key pixman_location pixman:location
     key poppler_location poppler:location
     key template_cloudooo template-cloudooo:target
+    key wkhtmltopdf_location wkhtmltopdf:location
     key xdamage_location xdamage:location
     key xfixes_location xfixes:location
     key zlib_location zlib:location
@@ -80,5 +81,5 @@ extra-context =
 [template-cloudooo]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/instance-cloudoo.cfg.in
-md5sum = 439c6f397b851d2884803e88e5a2d8de
+md5sum = 3172833bcb1bf532a2b34cdc381b83d6
 mode = 640

software/erp5/software.cfg.json

+{
+  "name": "ERP5",
+  "description": "ERP5, Open-Source ERP",
+  "serialisation": "json-in-xml",
+  "software-type": {
+    "default": {
+      "title": "Default",
+      "description": "Default deployment of ERP5 with auto-create instance.",
+      "request": "instance-erp5-input-schema.json",
+      "response": "instance-erp5-output-schema.json",
+      "index": 0
+    },
+    "create-erp5-site": {
+      "title": "Create ERP5 Site", 
+      "description": "Default deployment of ERP5 with automatic creation of the instance.",
+      "request": "instance-erp5-input-schema.json",
+      "response": "instance-erp5-output-schema.json",
+      "index": 1
+    }
+  }
+}

software/html5as/software.cfg

@@ -79,7 +79,7 @@ plone.recipe.command = 1.1
 
 # Required by:
 # slapos.toolbox==0.40.2
-GitPython = 2.0.2
+GitPython = 2.0.5
 
 # Required by:
 # slapos.toolbox==0.40.2

software/kvm/development.cfg

@@ -55,5 +55,4 @@ slapos.cookbook =
 slapos.core =
 slapos.toolbox =
 erp5.util =
-# XXX Fix lockfile 0.11.0 requirement (pbr!=0.7,<1.0,>=0.6)
-lockfile = 0.10.2
+lockfile =
\ No newline at end of file

software/kvm/software.cfg

@@ -20,7 +20,7 @@ pycurl = 7.19.5.1
 
 # Required by:
 # slapos.toolbox==0.48
-GitPython = 2.0.2
+GitPython = 2.0.5
 
 # Required by:
 # slapos.toolbox==0.48
@@ -36,7 +36,7 @@ feedparser = 5.2.0.post1
 
 # Required by:
 # slapos.toolbox==0.48
-lockfile = 0.10.2
+lockfile = 0.12.2
 
 # Required by:
 # websockify==0.5.1

software/monitor/software.cfg

@@ -112,7 +112,7 @@ smmap = 0.9.0
 
 # Required by:
 # slapos.toolbox==0.55
-GitPython = 2.0.2
+GitPython = 2.0.5
 
 # Required by:
 # slapos.toolbox==0.55

software/neoppod/software-common.cfg

@@ -121,9 +121,9 @@ apache-libcloud = 0.20.1
 atomize = 0.2.0
 ecdsa = 0.13
 feedparser = 5.2.1
-GitPython = 2.0.2
+GitPython = 2.0.5
 gitdb = 0.6.4
-lockfile = 0.10.2
+lockfile = 0.12.2
 mysqlclient = 1.3.7
 paramiko = 2.0.0
 pycrypto = 2.6.1

software/neoppod/software-zodb4.cfg

@@ -12,6 +12,6 @@ ZODB-patches =
 ZODB-patch-options = -p1
 
 [versions]
-ZODB = 4.2.0+SlapOSPatched001
+ZODB = 4.3.1+SlapOSPatched001
 transaction =
 zdaemon =

software/re6stnet/software.cfg

@@ -117,7 +117,7 @@ smmap = 0.9.0
 
 # Required by:
 # slapos.toolbox==0.55
-GitPython = 2.0.2
+GitPython = 2.0.5
 
 # Required by:
 # slapos.toolbox==0.55
@@ -133,7 +133,7 @@ feedparser = 5.1.3
 
 # Required by:
 # slapos.toolbox==0.55
-lockfile = 0.10.2
+lockfile = 0.12.2
 
 # Required by:
 # re6stnet===0-413.gbec6b3c.dirty

software/slaprunner/common.cfg

@@ -92,7 +92,7 @@ mode = 0644
 [template-resilient]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/instance-resilient.cfg.jinja2
-md5sum = 6373b2cf3c0cee78bd858959976271d3
+md5sum = bf853ab13b564be991b42e74f631023a
 filename = instance-resilient.cfg.jinja2
 mode = 0644
 

software/slaprunner/development.cfg

@@ -11,13 +11,14 @@ parts =
   slapos.toolbox-dev
   slapos.cookbook-dev
   slapos.core-dev
+  erp5.util-dev
   slapos-cookbook
   ${:common-parts}
 
 [slapos.toolbox-repository]
 recipe = slapos.recipe.build:gitclone
 repository = https://lab.nexedi.com/nexedi/slapos.toolbox.git
-branch = slaprunner-paas
+branch = master
 git-executable = ${git:location}/bin/git
 develop = true
 
@@ -29,7 +30,7 @@ setup = ${slapos.toolbox-repository:location}
 [slapos.cookbook-repository]
 recipe = slapos.recipe.build:gitclone
 repository = https://lab.nexedi.com/nexedi/slapos.git
-branch = slaprunner-paas
+branch = master
 git-executable = ${git:location}/bin/git
 develop = true
 
@@ -67,5 +68,4 @@ setup = ${slapos.core-repository:location}
 slapos.cookbook =
 slapos.core =
 slapos.toolbox =
-# XXX Fix lockfile 0.11.0 requirement (pbr!=0.7,<1.0,>=0.6)
-lockfile = 0.10.2
+lockfile =

software/slaprunner/instance-resilient.cfg.jinja2

@@ -60,6 +60,9 @@ webdav_url = ${request-runner:connection-webdav_url}
 public_url = ${request-runner:connection-public_url}
 git_public_url = ${request-runner:connection-git_public_url}
 git_private_url = ${request-runner:connection-git_private_url}
+{% if slapparameter_dict.get('custom-frontend-backend-url') -%}
+custom-frontend-url = https://${request-custom-frontend:connection-domain}
+{% endif %}
 {% for key in monitor_return -%}
 {{ key }} = ${request-runner:connection-{{ key }}}
 {% endfor -%}

software/slaprunner/instance-runner-input-schema.json

@@ -4,6 +4,7 @@
         "user-authorized-key": {
             "title": "User Authorized Key",
             "description": "SSH public key in order to connect to the SSH server of this runner instance.",
+            "textarea": true,
             "type": "string"
         },
         "instance-amount": {

software/slaprunner/software.cfg

@@ -24,7 +24,7 @@ smmap = 0.9.0
 
 # Required by:
 # slapos.toolbox==0.55
-GitPython = 2.0.2
+GitPython = 2.0.5
 
 # Required by:
 # slapos.toolbox==0.55
@@ -36,7 +36,7 @@ feedparser = 5.2.1
 
 # Required by:
 # slapos.toolbox==0.55
-lockfile = 0.10.2
+lockfile = 0.12.2
 
 # Required by:
 # slapos.toolbox==0.55

stack/boinc/buildout.cfg

@@ -75,7 +75,7 @@ smmap = 0.8.2
 
 # Required by:
 # slapos.toolbox==0.40.2
-GitPython = 2.0.2
+GitPython = 2.0.5
 
 # Required by:
 # slapos.toolbox==0.40.2

stack/cloudooo.cfg

@@ -18,6 +18,7 @@ extends =
   ../component/glib/buildout.cfg
   ../component/haproxy/buildout.cfg
   ../component/imagemagick/buildout.cfg
+  ../component/wkhtmltopdf/buildout.cfg
   ../component/libffi/buildout.cfg
   ../component/libpng/buildout.cfg
   ../component/libreoffice-bin/buildout.cfg
@@ -61,6 +62,7 @@ parts =
   xdamage
   xfixes
   imagemagick
+  wkhtmltopdf
   file
   poppler
   ffmpeg

stack/erp5/apache.conf.in

@@ -58,7 +58,7 @@ SSLCARevocationFile {{ parameter_dict['crl'] }}
 
 ErrorLog "{{ parameter_dict['error-log'] }}"
 # Default apache log format with request time in microsecond at the end
-LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %D" combined
+LogFormat "%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %D" combined
 CustomLog "{{ parameter_dict['access-log'] }}" combined
 
 <Directory />

stack/erp5/buildout.cfg

@@ -370,12 +370,12 @@ md5sum = ec9321514674c084e509ca070763b4a1
 [template-apache-conf]
 <= download-base
 filename = apache.conf.in
-md5sum = cbe53c1879db9601a521e3ce1d546116
+md5sum = 30e5560c216042bc965959a78d43f7c4
  
 [template-haproxy-cfg]
 <= download-base
 filename = haproxy.cfg.in
-md5sum = 3defd473e2cea17ae36bba7752494858
+md5sum = 5f85c35ebf3d08d408df0ce9935c5424
 
 [bt5-repository]
 # Format:
@@ -676,10 +676,10 @@ Products.DCWorkflowGraph = 0.4.1
 Products.ExternalEditor = 1.1.1
 Products.GenericSetup = 1.8.3
 Products.LongRequestLogger = 2.0.0
-Products.MimetypesRegistry = 2.0.8
+Products.MimetypesRegistry = 2.0.9
 Products.PluginRegistry = 1.4
 Products.TIDStorage = 5.4.9
-PyPDF2 = 1.25.1
+PyPDF2 = 1.26.0
 PyStemmer = 1.3.0
 PyXML = 0.8.5
 Pympler = 0.4.3
@@ -704,8 +704,8 @@ interval = 1.0.0
 ipdb = 0.10.0
 ipykernel = 4.3.1
 ipython = 4.2.0
-ipywidgets = 5.1.3
-logilab-common = 1.2.0
+ipywidgets = 5.1.5
+logilab-common = 1.2.1
 matplotlib = 1.5.1
 mistune = 0.7.2
 notebook = 4.2.0
@@ -720,10 +720,10 @@ pycountry = 1.20
 pyflakes = 1.2.3
 # pylint 1.5.1 breaks testDynamicClassGeneration
 pylint = 1.4.4
-python-memcached = 1.57
+python-memcached = 1.58
 pytracemalloc = 1.2
 pyzmq = 15.2.0
-qrcode = 5.2.2
+qrcode = 5.3
 restkit = 4.2.2
 rtjp-eventlet = 0.3.2
 scikit-learn = 0.17.1
@@ -739,7 +739,7 @@ tornado = 4.3
 urlnorm = 1.1.2
 uuid = 1.30
 validictory = 1.0.1
-widgetsnbextension = 1.2.2
+widgetsnbextension = 1.2.3
 xfw = 0.10
 xupdate-processor = 0.4
 

stack/erp5/haproxy.cfg.in

@@ -35,6 +35,9 @@ defaults
   # is configured with maxconn to 1, without this option browsers are unable
   # to render a page
   option forceclose
+  # Use "option forwardfor" so that client's actual IP is used rather than the IP
+  # of the load balancer
+  option forwardfor
 
 {% for name, (port, backend_list) in sorted(parameter_dict['backend-dict'].iteritems()) -%}
 listen {{ name }}

stack/lamp.cfg

@@ -50,7 +50,7 @@ smmap = 0.8.2
 
 # Required by:
 # slapos.toolbox==0.40.2
-GitPython = 2.0.2
+GitPython = 2.0.5
 
 # Required by:
 # slapos.toolbox==0.40.2

stack/lamp/buildout.cfg

@@ -194,7 +194,7 @@ smmap = 0.8.2
 
 # Required by:
 # slapos.toolbox==0.40.2
-GitPython = 2.0.2
+GitPython = 2.0.5
 
 # Required by:
 # slapos.toolbox==0.40.2

stack/lapp/buildout.cfg

@@ -191,7 +191,7 @@ smmap = 0.8.2
 
 # Required by:
 # slapos.toolbox==0.40.2
-GitPython = 2.0.2
+GitPython = 2.0.5
 
 # Required by:
 # slapos.toolbox==0.40.2

stack/monitor/buildout.cfg

@@ -100,7 +100,7 @@ recipe = slapos.recipe.template:jinja2
 filename = template-monitor.cfg
 template = ${:_profile_base_location_}/instance-monitor.cfg.jinja2.in
 rendered = ${buildout:directory}/template-monitor.cfg
-md5sum = b6da7709da533c5c8280e5e103809c58  
+md5sum = 354da434447697b8b4f503a62599cf32  
 context =
     key apache_location apache:location
     key gzip_location gzip:location
@@ -131,13 +131,13 @@ depends =
 [monitor2-bin]
 <= monitor-template-script
 filename = monitor.py
-md5sum = 6db3d9fcfff32c556f2bedee804ffb47
+md5sum = 3166f5d110fda63aade1ca486dfb7996
 
 [run-promise-py]
 recipe = slapos.recipe.template:jinja2
 template = ${:_profile_base_location_}/scripts/run-promise.py
 rendered = ${buildout:parts-directory}/monitor-scripts/run-promise.py
-md5sum = 5694920c5789f4d953cb5d41adfa7f8b
+md5sum = 2570b11cd1f718f240a9c7a05ffa861d
 mode = 0755
 context =
   raw python ${buildout:directory}/bin/${extra-eggs:interpreter}
@@ -155,12 +155,12 @@ md5sum = f297779d0881f4bd48081506efb492a4
 [monitor-globalstate]
 <= monitor-template-script
 filename = globalstate.py
-md5sum = 384a1148cb3da9cf353a108fe70709c5
+md5sum = 1ebead4480d62d5e1d2612c2d9f4e1d8
 
 [monitor-collect]
 <= monitor-template-script
 filename = collect.py
-md5sum = cc65aebd4c35b3172a7ca83abde761bc
+md5sum = 78fbcb56761315bde354fe7914d3c54f
 
 [monitor-document-edit]
 <= monitor-template-script

stack/monitor/instance-monitor.cfg.jinja2.in

@@ -221,7 +221,8 @@ update-command = ${:command}
 recipe = plone.recipe.command
 stop-on-error = true
 htpasswd-path = ${monitor-directory:etc}/monitor-htpasswd
-command = {{ apache_location }}/bin/htpasswd -cb ${:htpasswd-path} ${:user} ${:password}
+command = if [ ! -f "${:htpasswd-path}" ]; then {{ apache_location }}/bin/htpasswd -cb ${:htpasswd-path} ${:user} ${:password}; fi
+update-command = ${:command}
 user = ${monitor-instance-parameter:username}
 password = ${monitor-instance-parameter:password}
 

stack/monitor/scripts/collect.py

@@ -262,51 +262,73 @@ def appendToJsonFile(file_path, content, stepback=2):
     jfile.seek(position)
     jfile.write('%s}' % ',"{}"]'.format(content))
 
+def initProcessDataFile(file_path):
+  with open(process_file, 'w') as fprocess:
+    data_dict = {
+      "date": time.time(),
+      "data": ["date, total process, CPU percent, CPU time, CPU threads"]
+    }
+    fprocess.write(json.dumps(data_dict))
+
+def initMemoryDataFile(file_path):
+  with open(mem_file, 'w') as fmem:
+    data_dict = {
+      "date": time.time(),
+      "data": ["date, memory used percent, memory used"]
+    }
+    fmem.write(json.dumps(data_dict))
+
+def initIODataFile(file_path):
+  with open(io_file, 'w') as fio:
+    data_dict = {
+      "date": time.time(),
+      "data": ["date, io rw counter, io cycles counter, disk used"]
+    }
+    fio.write(json.dumps(data_dict))
+
 if __name__ == "__main__":
   parser = parseArguments()
   if not os.path.exists(parser.output_folder) and os.path.isdir(parser.output_folder):
     raise Exception("Invalid ouput folder: %s" % parser.output_folder)
-  collector = RessourceCollect(parser.collector_db)
-
-  date_scope = datetime.now().strftime('%Y-%m-%d')
-  stat_info = os.stat(parser.output_folder)
-  partition_user = pwd.getpwuid(stat_info.st_uid)[0]
 
   # Consumption global status
   process_file = os.path.join(parser.output_folder, 'monitor_resource_process.data.json')
   mem_file = os.path.join(parser.output_folder, 'monitor_resource_memory.data.json')
   io_file = os.path.join(parser.output_folder, 'monitor_resource_io.data.json')
+  resource_file = os.path.join(parser.output_folder, 'monitor_process_resource.status.json')
+  status_file = os.path.join(parser.output_folder, 'monitor_resource.status.json')
+
+  if not os.path.exists(parser.collector_db):
+    print "Collector database not found..."
+    initProcessDataFile(process_file)
+    initMemoryDataFile(mem_file)
+    initIODataFile(io_file)
+    with open(status_file, "w") as status_file:
+      status_file.write('{"cpu_time": 0, "cpu_percent": 0, "memory_rss": 0, "memory_percent": 0, "io_rw_counter": 0, "date": "", "total_process": 0, "disk_used": 0, "io_cycles_counter": 0, "cpu_num_threads": 0}')
+    with open(resource_file, "w") as resource_file:
+      resource_file.write('[]')
+    exit(1)
+
+  collector = RessourceCollect(parser.collector_db)
+
+  date_scope = datetime.now().strftime('%Y-%m-%d')
+  stat_info = os.stat(parser.output_folder)
+  partition_user = pwd.getpwuid(stat_info.st_uid)[0]
 
   process_result, memory_result, io_result = collector.getPartitionComsumptionStatus(partition_user)
-  resource_file = os.path.join(parser.output_folder, 'monitor_process_resource.status.json')
 
   label_list = ['date', 'total_process', 'cpu_percent', 'cpu_time', 'cpu_num_threads',
                   'memory_percent', 'memory_rss', 'io_rw_counter', 'io_cycles_counter',
                   'disk_used']
   resource_status_dict = {}
   if not os.path.exists(process_file):
-    with open(process_file, 'w') as fprocess:
-      data_dict = {
-        "date": time.time(),
-        "data": ["date, total process, CPU percent, CPU time, CPU threads"]
-      }
-      fprocess.write(json.dumps(data_dict))
+    initProcessDataFile(process_file)
 
   if not os.path.exists(mem_file):
-    with open(mem_file, 'w') as fmem:
-      data_dict = {
-        "date": time.time(),
-        "data": ["date, memory used percent, memory used"]
-      }
-      fmem.write(json.dumps(data_dict))
+    initMemoryDataFile(mem_file)
 
   if not os.path.exists(io_file):
-    with open(io_file, 'w') as fio:
-      data_dict = {
-        "date": time.time(),
-        "data": ["date, io rw counter, io cycles counter, disk used"]
-      }
-      fio.write(json.dumps(data_dict))
+    initIODataFile(io_file)
 
   if process_result and process_result['total_process'] != 0.0:
     appendToJsonFile(process_file, ", ".join(
@@ -324,7 +346,7 @@ if __name__ == "__main__":
     )
     resource_status_dict.update(io_result)
 
-  with open(os.path.join(parser.output_folder, 'monitor_resource.status.json'), 'w') as fp:
+  with open(status_file, 'w') as fp:
     fp.write(json.dumps(resource_status_dict))
 
   # Consumption Ressource

stack/monitor/scripts/globalstate.py

@@ -104,6 +104,7 @@ def main(args_list):
         'success': success,
         'warning': warning,
       },
+      type='global',
       date=latest_date,
       _links={"rss_url": {"href": "%s/public/feed" % base_url},
               "public_url": {"href": "%s/share/jio_public/" % base_url},

stack/monitor/scripts/monitor.py

@@ -97,7 +97,6 @@ class Monitoring(object):
     self.report_script_folder = config.get("monitor", "report-folder")
     self.webdav_url = '%s/share' % config.get("monitor", "base-url")
     self.public_url = '%s/public' % config.get("monitor", "base-url")
-    self.status_history_folder = os.path.join(self.public_folder, 'history')
     self.python = config.get("monitor", "python") or "python"
     self.public_path_list = config.get("monitor", "public-path-list").split()
     self.private_path_list = config.get("monitor", "private-path-list").split()
@@ -107,7 +106,7 @@ class Monitoring(object):
     self.parameter_cfg_file = config.get("monitor", "parameter-file-path").strip()
 
     self.config_folder = os.path.join(self.private_folder, 'config')
-    self.report_folder = os.path.join(self.private_folder, 'report')
+    self.report_folder = self.private_folder
 
     self.promise_dict = {}
     for promise_folder in self.promise_folder_list:
@@ -237,14 +236,14 @@ class Monitoring(object):
       else:
         response = urllib2.urlopen(url)
     except urllib2.HTTPError:
-      return 'Unknow Instance'
+      return 'Unknown Instance'
     else:
       try:
         monitor_dict = json.loads(response.read())
-        return monitor_dict.get('title', 'Unknow Instance')
+        return monitor_dict.get('title', 'Unknown Instance')
       except ValueError, e:
         print "Bad Json file at %s" % url
-    return 'Unknow Instance'
+    return 'Unknown Instance'
 
   def configureFolders(self):
     # configure public and private folder
@@ -256,7 +255,6 @@ class Monitoring(object):
     jio_private = os.path.join(self.webdav_folder, 'jio_private')
     mkdirAll(jio_public)
     mkdirAll(jio_private)
-    mkdirAll(self.status_history_folder)
     try:
       os.symlink(self.public_folder, os.path.join(jio_public, '.jio_documents'))
     except OSError, e:
@@ -269,11 +267,9 @@ class Monitoring(object):
         raise
 
     self.data_folder = os.path.join(self.private_folder, 'data', '.jio_documents')
-    self.report_folder = os.path.join(self.report_folder, '.jio_documents')
     config_folder = os.path.join(self.config_folder, '.jio_documents')
     mkdirAll(self.data_folder)
     mkdirAll(config_folder)
-    mkdirAll(self.report_folder)
     try:
       os.symlink(os.path.join(self.private_folder, 'data'),
                   os.path.join(jio_private, 'data'))
@@ -285,11 +281,6 @@ class Monitoring(object):
     except OSError, e:
       if e.errno != os.errno.EEXIST:
         raise
-    try:
-      os.symlink(self.report_folder, os.path.join(jio_private, 'report'))
-    except OSError, e:
-      if e.errno != os.errno.EEXIST:
-        raise
 
   def makeConfigurationFiles(self):
     config_folder = os.path.join(self.config_folder, '.jio_documents')
@@ -397,7 +388,7 @@ class Monitoring(object):
   def generateReportCronEntries(self):
     cron_line_list = []
     # We should add the possibility to modify this parameter later from monitor interface
-    report_frequency = "*/30 * * * *"
+    report_frequency = "*/20 * * * *"
 
     report_name_list = [name.replace('.report.json', '')
       for name in os.listdir(self.report_folder) if name.endswith('.report.json')]
@@ -451,7 +442,7 @@ class Monitoring(object):
 
     for service_name, promise in self.promise_items:
       service_config = promise["configuration"]
-      service_status_path = "%s/%s.status.json" % (self.public_folder, service_name)  # hardcoded
+      service_status_path = "%s/%s.status.json" % (self.public_folder, service_name)
       mkdirAll(os.path.dirname(service_status_path))
 
       promise_cmd_line = [
@@ -463,7 +454,7 @@ class Monitoring(object):
         '--promise_script "%s"' % promise["path"],
         '--promise_name "%s"' % service_name,
         '--monitor_url "%s/jio_private/"' % self.webdav_url, # XXX hardcoded,
-        '--history_folder "%s"' % self.status_history_folder,
+        '--history_folder "%s"' % self.public_folder,
         '--instance_name "%s"' % self.title,
         '--hosting_name "%s"' % self.root_title]
 
@@ -539,13 +530,21 @@ class Monitoring(object):
 
     # Rotate monitor data files
     option_list = [
-      'daily', 'nocreate', 'noolddir', 'rotate 30',
+      'daily', 'nocreate', 'noolddir', 'rotate 5',
       'nocompress', 'extension .json', 'dateext',
       'dateformat -%Y-%m-%d', 'notifempty'
     ]
     file_list = ["%s/*.data.json" % self.data_folder]
     self.generateLogrotateEntry('monitor.data', file_list, option_list)
 
+    # Rotate public history status file, delete data of previous days
+    option_list = [
+      'daily', 'nocreate', 'rotate 0',
+      'nocompress', 'notifempty'
+    ]
+    file_list = ["%s/*.history.json" % self.public_folder]
+    self.generateLogrotateEntry('monitor.service.status', file_list, option_list)
+
     # Add cron entry for SlapOS Collect
     command = "%s %s --output_folder %s --collector_db %s" % (self.python,
       self.collect_script, self.data_folder, self.collector_db)

stack/monitor/scripts/run-promise.py

@@ -65,6 +65,7 @@ def main():
   status_json['title'] = parser.promise_name
   status_json['instance'] = parser.instance_name
   status_json['hosting_subscription'] = parser.hosting_name
+  status_json['type'] = parser.promise_type
 
   # Save the lastest status change date (needed for rss)
   status_json['change-time'] = ps_process.create_time()
@@ -85,9 +86,7 @@ def main():
   os.remove(parser.pid_path)
 
 def updateStatusHistoryFolder(name, status_file, history_folder, promise_type):
-  old_history_list = []
-  keep_item_amount = 25
-  history_path = os.path.join(history_folder, name, '.jio_documents')
+  history_path = os.path.join(history_folder)
   if not os.path.exists(status_file):
     return
   if not os.path.exists(history_folder):
@@ -101,14 +100,35 @@ def updateStatusHistoryFolder(name, status_file, history_folder, promise_type):
       else: raise
   with open(status_file, 'r') as sf:
     status_dict = json.loads(sf.read())
-    filename = '%s.%s.json' % (
-      status_dict['start-date'].replace(' ', '_').replace(':', ''),
-      promise_type)
+  
+  if promise_type == 'status':
+    filename = '%s.history.json' % name
+    history_file = os.path.join(history_path, filename)
+    # Remove links from history (not needed)
+    status_dict.pop('_links', None)
+    if not os.path.exists(history_file):
+      with open(history_file, 'w') as f_history:
+        data_dict = {
+          "date": time.time(),
+          "data": [status_dict]
+        }
+        f_history.write(json.dumps(data_dict))
+    else:
+      with open (history_file, mode="r+") as f_history:
+        f_history.seek(0,2)
+        position = f_history.tell() -2
+        f_history.seek(position)
+        #f_history.write(',%s]}' % str(status_dict))
+        f_history.write('%s}' % ',{}]'.format(json.dumps(status_dict)))
+  elif promise_type == 'report':
+    # keep_item_amount = 3
+    filename = '%s.history.json' % (
+      name)
 
     copyfile(status_file, os.path.join(history_path, filename))
-  # Don't let history foler grow too much, keep xx files
+    """# Don't let history foler grow too much, keep xx files
     file_list = filter(os.path.isfile,
-      glob.glob("%s/*.%s.json" % (history_path, promise_type))
+        glob.glob("%s/*.%s.history.json" % (history_path, promise_type))
       )
     file_count = len(file_list)
     if file_count > keep_item_amount:
@@ -119,7 +139,7 @@ def updateStatusHistoryFolder(name, status_file, history_folder, promise_type):
           os.unlink(to_delete)
           file_count -= 1
         except OSError:
-        raise
+          raise"""
 
 def generateStatusJsonFromProcess(process, start_date=None, title=None):
   stdout, stderr = process.communicate()

stack/slapos.cfg

@@ -112,10 +112,11 @@ PyYAML = 3.11
 Werkzeug = 0.11.9
 buildout-versions = 1.7
 cffi = 1.6.0
+click = 6.6
 cliff = 2.0.0
 cmd2 = 0.6.8
 collective.recipe.template = 1.13
-cryptography = 1.3.2
+cryptography = 1.3.4
 decorator = 4.0.9
 idna = 2.1
 inotifyx = 0.2.2
@@ -126,16 +127,16 @@ netaddr = 0.7.18
 pbr = 1.9.1
 plone.recipe.command = 1.1
 prettytable = 0.7.2
-psutil = 4.1.0
+psutil = 4.2.0
 pyOpenSSL = 16.0.0
 pyasn1 = 0.1.9
-pyparsing = 2.1.3
+pyparsing = 2.1.4
 pytz = 2016.4
 requests = 2.10.0
 setuptools = 19.6.2
 simplejson = 3.8.2
 six = 1.10.0
-slapos.cookbook = 1.0.25
+slapos.cookbook = 1.0.31
 slapos.core = 1.3.15
 slapos.extension.strip = 0.1
 slapos.libnetworkcache = 0.14.5
@@ -147,22 +148,22 @@ xml-marshaller = 0.9.7
 
 # Required by:
 # slapos.core==1.3.15
-Flask = 0.10.1
+Flask = 0.11
 
 # Required by:
 # Jinja2==2.8
 MarkupSafe = 0.23
 
 # Required by:
-# cryptography==1.3.2
-enum34 = 1.1.5
+# cryptography==1.3.4
+enum34 = 1.1.6
 
 # Required by:
 # jsonschema==2.5.1
 functools32 = 3.2.3.post2
 
 # Required by:
-# cryptography==1.3.2
+# cryptography==1.3.4
 ipaddress = 1.0.16
 
 # Required by:
@@ -183,7 +184,7 @@ pycparser = 2.14
 
 # Required by:
 # slapos.core==1.3.15
-supervisor = 3.2.3
+supervisor = 3.3.0
 
 # Required by:
 # slapos.core==1.3.15