Commit 1bea879b authored by Julien Muchembled's avatar Julien Muchembled

Merge branch 'master' into erp5-cluster

Conflicts:
	software/neoppod/software.cfg
	stack/erp5/buildout.cfg
	stack/erp5/instance-erp5-cluster.cfg.in
	stack/erp5/instance-erp5-single.cfg.in
	stack/erp5/instance-zope.cfg.in
parents a546487f 9d768e8f
...@@ -17,15 +17,15 @@ extends = ...@@ -17,15 +17,15 @@ extends =
[apr] [apr]
recipe = hexagonit.recipe.download recipe = hexagonit.recipe.download
version = 1.5.0 version = 1.5.1
md5sum = cc93bd2c12d0d037f68e21cc6385dc31 md5sum = 5486180ec5a23efb5cae6d4292b300ab
url = https://archive.apache.org/dist/apr/apr-${:version}.tar.bz2 url = https://archive.apache.org/dist/apr/apr-${:version}.tar.bz2
[apr-util] [apr-util]
recipe = hexagonit.recipe.download recipe = hexagonit.recipe.download
version = 1.5.3 version = 1.5.4
url = https://archive.apache.org/dist/apr/apr-util-${:version}.tar.bz2 url = https://archive.apache.org/dist/apr/apr-util-${:version}.tar.bz2
md5sum = 6f3417691c7a27090f36e7cf4d94b36e md5sum = 2202b18f269ad606d70e1864857ed93c
[apache] [apache]
# inspired on http://old.aclark.net/team/aclark/blog/a-lamp-buildout-for-wordpress-and-other-php-apps/ # inspired on http://old.aclark.net/team/aclark/blog/a-lamp-buildout-for-wordpress-and-other-php-apps/
......
...@@ -38,6 +38,8 @@ patches = ...@@ -38,6 +38,8 @@ patches =
http://ftp.gnu.org/pub/gnu/bash/bash-4.3-patches/bash43-026#922578e2be7ed03729454e92ee8d3f3a http://ftp.gnu.org/pub/gnu/bash/bash-4.3-patches/bash43-026#922578e2be7ed03729454e92ee8d3f3a
http://ftp.gnu.org/pub/gnu/bash/bash-4.3-patches/bash43-027#8ff6948b16f2db5c29b1b9ae1085bbe7 http://ftp.gnu.org/pub/gnu/bash/bash-4.3-patches/bash43-027#8ff6948b16f2db5c29b1b9ae1085bbe7
http://ftp.gnu.org/pub/gnu/bash/bash-4.3-patches/bash43-028#dd51fa67913b5dca45a702b672b3323f http://ftp.gnu.org/pub/gnu/bash/bash-4.3-patches/bash43-028#dd51fa67913b5dca45a702b672b3323f
http://ftp.gnu.org/pub/gnu/bash/bash-4.3-patches/bash43-029#0729364c977ef4271e9f8dfafadacf67
http://ftp.gnu.org/pub/gnu/bash/bash-4.3-patches/bash43-030#efb709fdb1368945513de23ccbfae053
configure-options = configure-options =
--with-curses --with-curses
environment = environment =
......
...@@ -11,10 +11,10 @@ parts = ...@@ -11,10 +11,10 @@ parts =
[ca-certificates] [ca-certificates]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
version = 20140325 version = 20141019
url = ftp://ftp.free.fr/mirrors/ftp.debian.org/pool/main/c/ca-certificates/ca-certificates_${:version}.tar.xz url = ftp://ftp.free.fr/mirrors/ftp.debian.org/pool/main/c/ca-certificates/ca-certificates_${:version}.tar.xz
patch-binary = ${patch:location}/bin/patch patch-binary = ${patch:location}/bin/patch
md5sum = 0436aba482091da310bd762e1deca8b4 md5sum = f619282081c8bfc65ea64c37fa5285ed
patches = patches =
${:_profile_base_location_}/ca-certificates-sbin-dir.patch#0b4e7d82ce768823c01954ee41ef177b ${:_profile_base_location_}/ca-certificates-sbin-dir.patch#0b4e7d82ce768823c01954ee41ef177b
patch-options = -p0 patch-options = -p0
......
...@@ -12,8 +12,8 @@ parts = ...@@ -12,8 +12,8 @@ parts =
[curl] [curl]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
url = http://curl.haxx.se/download/curl-7.38.0.tar.bz2 url = http://curl.haxx.se/download/curl-7.39.0.tar.bz2
md5sum = af6b3c299bd891f43cb5f76c4091b7b4 md5sum = 1efecb5b0e43c17d968f0d228bbbbbbd
configure-options = configure-options =
--disable-static --disable-static
--disable-ldap --disable-ldap
......
...@@ -8,8 +8,8 @@ extends = ...@@ -8,8 +8,8 @@ extends =
[file] [file]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
url = ftp://ftp.astron.com/pub/file/file-5.19.tar.gz url = ftp://ftp.astron.com/pub/file/file-5.20.tar.gz
md5sum = e3526f59023f3f7d1ffa4d541335edab md5sum = 5d5e13eb3e0e13839da869a31790faf2
configure-options = configure-options =
--disable-static --disable-static
environment = environment =
......
...@@ -26,12 +26,12 @@ depends = ...@@ -26,12 +26,12 @@ depends =
${liberation-fonts:location} ${liberation-fonts:location}
${ipaex-fonts:location} ${ipaex-fonts:location}
version = 32.0 version = 33.0
# MD5SUMs are available at : # MD5SUMs are available at :
# https://ftp.mozilla.org/pub/mozilla.org/firefox/releases/${:version}/MD5SUMS # https://ftp.mozilla.org/pub/mozilla.org/firefox/releases/${:version}/MD5SUMS
x86 = http://download-installer.cdn.mozilla.net/pub/firefox/releases/${:version}/linux-i686/en-US/firefox-${:version}.tar.bz2 f086b7d0ac2421232d5cb31857980d2e x86 = http://download-installer.cdn.mozilla.net/pub/firefox/releases/${:version}/linux-i686/en-US/firefox-${:version}.tar.bz2 542b27d5f3afcb55406a7af72f68f3d5
x86-64 = http://download-installer.cdn.mozilla.net/pub/firefox/releases/${:version}/linux-x86_64/en-US/firefox-${:version}.tar.bz2 ec3b3976caa4ec5b3de137d059c74f8d x86-64 = http://download-installer.cdn.mozilla.net/pub/firefox/releases/${:version}/linux-x86_64/en-US/firefox-${:version}.tar.bz2 ae4e8275750fd5f87963cbe263e8ba34
script = script =
if not self.options.get('url'): self.options['url'], self.options['md5sum'] = self.options[guessPlatform()].split(' ') if not self.options.get('url'): self.options['url'], self.options['md5sum'] = self.options[guessPlatform()].split(' ')
...@@ -42,7 +42,7 @@ script = ...@@ -42,7 +42,7 @@ script =
wrapper = open(wrapper_location, 'w') wrapper = open(wrapper_location, 'w')
wrapper.write("""#!${dash:location}/bin/dash wrapper.write("""#!${dash:location}/bin/dash
cd %(location)s cd %(location)s
export LD_LIBRARY_PATH=%(location)s:${alsa:location}/lib:${atk:location}/lib:${bzip2:location}/lib:${cairo:location}/lib:${dbus:location}/lib:${dbus-glib:location}/lib:${fontconfig:location}/lib:${freetype:location}/lib:${gdk-pixbuf:location}/lib:${gettext:location}/lib:${glib:location}/lib:${gtk-2:location}/lib:${harfbuzz:location}/lib:${libICE:location}/lib:${libSM:location}/lib:${libX11:location}/lib:${libXau:location}/lib:${libXcursor:location}/lib:${libXext:location}/lib:${libXrender:location}/lib:${libXt:location}/lib:${libffi:location}/lib:${libpng:location}/lib:${libtool:location}/lib:${libuuid:location}/lib:${libxcb:location}/lib:${libxml2:location}/lib:${pango:location}/lib:${p11-kit:location}/lib:${pixman:location}/lib:${xfixes:location}/lib:${zlib:location}/lib export LD_LIBRARY_PATH=%(location)s:${alsa:location}/lib:${atk:location}/lib:${bzip2:location}/lib:${cairo:location}/lib:${dbus:location}/lib:${dbus-glib:location}/lib:${fontconfig:location}/lib:${freetype:location}/lib:${gdk-pixbuf:location}/lib:${gettext:location}/lib:${glib:location}/lib:${gtk-2:location}/lib:${harfbuzz:location}/lib:${libICE:location}/lib:${libSM:location}/lib:${libX11:location}/lib:${libXau:location}/lib:${libXcomposite:location}/lib:${libXcursor:location}/lib:${libXext:location}/lib:${libXrender:location}/lib:${libXt:location}/lib:${libffi:location}/lib:${libpng:location}/lib:${libtool:location}/lib:${libuuid:location}/lib:${libxcb:location}/lib:${libxml2:location}/lib:${p11-kit:location}/lib:${pango:location}/lib:${pixman:location}/lib:${xdamage:location}/lib:${xfixes:location}/lib:${zlib:location}/lib
export PATH=${fontconfig:location}/bin:$PATH export PATH=${fontconfig:location}/bin:$PATH
exec %(location)s/firefox $*""") exec %(location)s/firefox $*""")
wrapper.close() wrapper.close()
......
...@@ -4,6 +4,7 @@ ...@@ -4,6 +4,7 @@
extends = extends =
../m4/buildout.cfg ../m4/buildout.cfg
../gmp/buildout.cfg ../gmp/buildout.cfg
../xz-utils/buildout.cfg
../zip/buildout.cfg ../zip/buildout.cfg
parts = parts =
...@@ -11,59 +12,51 @@ parts = ...@@ -11,59 +12,51 @@ parts =
[mpfr] [mpfr]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
url = http://www.mpfr.org/mpfr-3.0.1/mpfr-3.0.1.tar.bz2 url = http://ftp.gnu.org/gnu/mpfr/mpfr-3.1.2.tar.xz
md5sum = bfbecb2eacb6d48432ead5cfc3f7390a md5sum = e3d203d188b8fe60bb6578dd3152e05c
configure-options = configure-options =
--with-gmp=${gmp-4:location} --with-gmp=${gmp:location}
--disable-static --disable-static
environment = environment =
LDFLAGS=-Wl,-rpath=${gmp-4:location}/lib PATH=${xz-utils:location}/bin:%(PATH)s
LDFLAGS=-Wl,-rpath=${gmp:location}/lib
[mpc] [mpc]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
url = http://www.multiprecision.org/mpc/download/mpc-0.9.tar.gz url = http://ftp.gnu.org/gnu/mpc/mpc-1.0.2.tar.gz
md5sum = 0d6acab8d214bd7d1fbbc593e83dd00d md5sum = 68fadff3358fb3e7976c7a398a0af4c3
configure-options = configure-options =
--with-gmp=${gmp-4:location} --with-gmp=${gmp:location}
--with-mpfr=${mpfr:location} --with-mpfr=${mpfr:location}
--disable-static --disable-static
environment = environment =
LDFLAGS=-Wl,-rpath=${mpfr:location}/lib -Wl,-rpath=${gmp-4:location}/lib LDFLAGS=-Wl,-rpath=${gmp:location}/lib -Wl,-rpath=${mpfr:location}/lib
[ppl] [isl]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
# we should use version 0.11.x for gcc-4.7 url = ftp://gcc.gnu.org/pub/gcc/infrastructure/isl-0.12.2.tar.bz2
url = http://bugseng.com/external/ppl/download/ftp/releases/0.11/ppl-0.11.tar.bz2 md5sum = e039bfcfb6c2ab039b8ee69bf883e824
md5sum = c414abd2ecaf1f1f48949c78d383ee1a
patches =
${:_profile_base_location_}/ppl-0.11-gcc-4.9.patch#09e00eefba5f8e6e77a9c9c611fccc64
configure-options = configure-options =
--with-gmp=${gmp-4:location} --with-gmp-prefix=${gmp:location}
--with-gmp-prefix=${gmp-4:location}
--disable-static --disable-static
environment = environment =
PATH=${m4:location}/bin:%(PATH)s LDFLAGS=-Wl,-rpath=${gmp:location}/lib
LDFLAGS=-Wl,-rpath=${gmp-4:location}/lib
CXXFLAGS=-fpermissive
[cloog-ppl] [cloog]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
url = ftp://gcc.gnu.org/pub/gcc/infrastructure/cloog-0.16.1.tar.gz url = ftp://gcc.gnu.org/pub/gcc/infrastructure/cloog-0.18.1.tar.gz
md5sum = 947123350d1ff6dcb4b0774947ac015a md5sum = e34fca0540d840e5d0f6427e98c92252
configure-options = configure-options =
--with-gmp-prefix=${gmp-4:location} --with-gmp-prefix=${gmp:location}
--with-ppl=${ppl:location} --with-isl-prefix=${isl:location}
--disable-static --disable-static
environment = environment =
PATH=${m4:location}/bin:%(PATH)s LDFLAGS=-Wl,-rpath=${gmp:location}/lib -Wl,-rpath=${isl:location}/lib
LDFLAGS=-Wl,-rpath=${gmp-4:location}/lib -Wl,-rpath=${ppl:location}/lib
[gcc-common] [gcc-common]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
url = http://ftp.gnu.org/gnu/gcc/gcc-4.7.4/gcc-4.7.4.tar.bz2 url = http://ftp.gnu.org/gnu/gcc/gcc-4.9.2/gcc-4.9.2.tar.bz2
md5sum = 4c696da46297de6ae77a82797d2abe28 md5sum = 4df8ee253b7f3863ad0b86359cd39c43
# make install does not work when several core are used # make install does not work when several core are used
make-targets = install -j1 make-targets = install -j1
...@@ -72,41 +65,39 @@ make-targets = install -j1 ...@@ -72,41 +65,39 @@ make-targets = install -j1
configure-options = configure-options =
--disable-bootstrap --disable-bootstrap
--disable-multilib --disable-multilib
--with-gmp=${gmp-4:location} --with-gmp=${gmp:location}
--with-mpfr=${mpfr:location} --with-mpfr=${mpfr:location}
--with-mpc=${mpc:location} --with-mpc=${mpc:location}
--enable-languages="c,c++" --enable-languages="c,c++"
--with-ppl=${ppl:location} --with-isl=${isl:location}
--with-cloog=${cloog-ppl:location} --with-cloog=${cloog:location}
--enable-cloog-backend=isl
environment = environment =
LDFLAGS=-Wl,-rpath=${mpfr:location}/lib -Wl,-rpath=${gmp-4:location}/lib -Wl,-rpath=${mpc:location}/lib -Wl,-rpath=${ppl:location}/lib -Wl,-rpath=${cloog-ppl:location}/lib LDFLAGS=-Wl,-rpath=${cloog:location}/lib -Wl,-rpath=${gmp:location}/lib -Wl,-rpath=${isl:location}/lib -Wl,-rpath=${mpc:location}/lib -Wl,-rpath=${mpfr:location}/lib
[gcc-minimal] [gcc-minimal]
<= gcc-common <= gcc-common
configure-options = configure-options =
--disable-bootstrap --disable-bootstrap
--disable-multilib --disable-multilib
--with-gmp=${gmp-4:location} --with-gmp=${gmp:location}
--with-mpfr=${mpfr:location} --with-mpfr=${mpfr:location}
--with-mpc=${mpc:location} --with-mpc=${mpc:location}
--enable-languages=c --enable-languages=c
--without-ppl --without-isl
--without-cloog --without-cloog
environment = environment =
LDFLAGS=-Wl,-rpath=${mpfr:location}/lib -Wl,-rpath=${gmp-4:location}/lib -Wl,-rpath=${mpc:location}/lib LDFLAGS=-Wl,-rpath=${gmp:location}/lib -Wl,-rpath=${mpc:location}/lib -Wl,-rpath=${mpfr:location}/lib
[gcc-fortran] [gcc-fortran]
<= gcc-common <= gcc-common
configure-options = configure-options =
--disable-bootstrap --disable-bootstrap
--disable-multilib --disable-multilib
--with-gmp=${gmp-4:location} --with-gmp=${gmp:location}
--with-mpfr=${mpfr:location} --with-mpfr=${mpfr:location}
--with-mpc=${mpc:location} --with-mpc=${mpc:location}
--enable-languages="c,c++,fortran" --enable-languages="c,c++,fortran"
--with-ppl=${ppl:location} --with-isl=${isl:location}
--with-cloog=${cloog-ppl:location} --with-cloog=${cloog:location}
--enable-cloog-backend=isl
environment = environment =
LDFLAGS=-Wl,-rpath=${mpfr:location}/lib -Wl,-rpath=${gmp-4:location}/lib -Wl,-rpath=${mpc:location}/lib -Wl,-rpath=${ppl:location}/lib -Wl,-rpath=${cloog-ppl:location}/lib LDFLAGS=-Wl,-rpath=${cloog:location}/lib -Wl,-rpath=${gmp:location}/lib -Wl,-rpath=${isl:location}/lib -Wl,-rpath=${mpc:location}/lib -Wl,-rpath=${mpfr:location}/lib
Patch for http://gcc.gnu.org/bugzilla/show_bug.cgi?id=50888
--- a/src/libjava/libjava/prims.cc.orig 2012-01-20 11:30:18.586157610 +0100
+++ b/src/libjava/libjava/prims.cc 2012-01-20 11:30:58.192770947 +0100
@@ -38,6 +38,14 @@
#endif
#ifndef DISABLE_GETENV_PROPERTIES
+#ifdef __GLIBC__
+/* glibc 2.15+ provides even for C++ inline optimized ::isspace etc.
+ Unfortunately those inlines are throw (), and call a function pointer
+ (which is throw () too, but with -fnon-call-exceptions this results
+ in a __cxa_call_unexpected call. This macro disables the optimized
+ version. */
+#define __NO_CTYPE 1
+#endif
#include <ctype.h>
#include <java-props.h>
#define PROCESS_GCJ_PROPERTIES process_gcj_properties()
--- a/src/libjava/prims.cc.orig 2012-01-20 11:30:23.042818341 +0100
+++ b/src/libjava/prims.cc 2012-01-20 11:31:01.389433254 +0100
@@ -38,6 +38,14 @@
#endif
#ifndef DISABLE_GETENV_PROPERTIES
+#ifdef __GLIBC__
+/* glibc 2.15+ provides even for C++ inline optimized ::isspace etc.
+ Unfortunately those inlines are throw (), and call a function pointer
+ (which is throw () too, but with -fnon-call-exceptions this results
+ in a __cxa_call_unexpected call. This macro disables the optimized
+ version. */
+#define __NO_CTYPE 1
+#endif
#include <ctype.h>
#include <java-props.h>
#define PROCESS_GCJ_PROPERTIES process_gcj_properties()
# HG changeset patch
# User Bernhard Walle <bernhard@bwalle.de>
# Date 1399806036 -7200
# Sun May 11 13:00:36 2014 +0200
# Node ID 5632a171e7c1dc4c0fb5c7a8352afb4188cd29d5
# Parent e35fa03cd2043496839aa5fa3a1a3d2d375b5535
complibs/ppl: Fix build of ppl 0.11.2 with gcc 4.9
From: Roberto Bagnara <bagnara@cs.unipr.it>
Date: Tue, 29 Apr 2014 19:51:43 +0000 (+0200)
Subject: Added missing inclusions. Use std::ptrdiff_t.
X-Git-Url: http://www.cs.unipr.it/git/gitweb.cgi?p=ppl%2Fppl.git;a=commitdiff_plain;h=61d4e14dfd9f1121e9b4521dead5728b2424dd7c
Added missing inclusions. Use std::ptrdiff_t.
(Thanks to Paulo Cesar Pereira de Andrade.)
Signed-off-by: Bernhard Walle <bernhard@bwalle.de>
diff -r e35fa03cd204 -r 5632a171e7c1 patches/ppl/0.11.2/500-ptrdiff_t.patch
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/patches/ppl/0.11.2/500-ptrdiff_t.patch Sun May 11 13:00:36 2014 +0200
@@ -0,0 +1,95 @@
+From: Bernhard Walle <bernhard@bwalle.de>
+Subject: Fix compilation with gcc 4.9 (ptrdiff_t errors)
+
+This is a backport of following patch in ppl git
+
+ commit 61d4e14dfd9f1121e9b4521dead5728b2424dd7c
+ Author: Roberto Bagnara <bagnara@cs.unipr.it>
+ Date: Tue Apr 29 21:51:43 2014 +0200
+
+ Added missing inclusions. Use std::ptrdiff_t.
+ (Thanks to Paulo Cesar Pereira de Andrade.)
+
+---
+ src/Congruence_System.defs.hh | 3 ++-
+ src/Constraint_System.defs.hh | 3 ++-
+ src/Generator_System.defs.hh | 3 ++-
+ src/Grid_Generator_System.defs.hh | 3 ++-
+ 4 files changed, 8 insertions(+), 4 deletions(-)
+
+--- a/src/Grid_Generator_System.defs.hh
++++ b/src/Grid_Generator_System.defs.hh
+@@ -30,6 +30,7 @@ site: http://www.cs.unipr.it/ppl/ . */
+ #include "Variables_Set.types.hh"
+ #include "Grid.types.hh"
+ #include <iosfwd>
++#include <cstddef>
+
+ namespace Parma_Polyhedra_Library {
+
+@@ -267,7 +268,7 @@ public:
+ class const_iterator
+ : public std::iterator<std::forward_iterator_tag,
+ Grid_Generator,
+- ptrdiff_t,
++ std::ptrdiff_t,
+ const Grid_Generator*,
+ const Grid_Generator&>,
+ private Generator_System::const_iterator {
+--- a/src/Congruence_System.defs.hh
++++ b/src/Congruence_System.defs.hh
+@@ -33,6 +33,7 @@ site: http://www.cs.unipr.it/ppl/ . */
+ #include "Grid.types.hh"
+ #include "Grid_Certificate.types.hh"
+ #include <iosfwd>
++#include <cstddef>
+
+ namespace Parma_Polyhedra_Library {
+
+@@ -235,7 +236,7 @@ public:
+ class const_iterator
+ : public std::iterator<std::forward_iterator_tag,
+ Congruence,
+- ptrdiff_t,
++ std::ptrdiff_t,
+ const Congruence*,
+ const Congruence&> {
+ public:
+--- a/src/Constraint_System.defs.hh
++++ b/src/Constraint_System.defs.hh
+@@ -33,6 +33,7 @@ site: http://www.cs.unipr.it/ppl/ . */
+ #include "Congruence_System.types.hh"
+ #include <iterator>
+ #include <iosfwd>
++#include <cstddef>
+
+ namespace Parma_Polyhedra_Library {
+
+@@ -204,7 +205,7 @@ public:
+ class const_iterator
+ : public std::iterator<std::forward_iterator_tag,
+ Constraint,
+- ptrdiff_t,
++ std::ptrdiff_t,
+ const Constraint*,
+ const Constraint&> {
+ public:
+--- a/src/Generator_System.defs.hh
++++ b/src/Generator_System.defs.hh
+@@ -33,6 +33,7 @@ site: http://www.cs.unipr.it/ppl/ . */
+ #include "Polyhedron.types.hh"
+ #include "Poly_Con_Relation.defs.hh"
+ #include <iosfwd>
++#include <cstddef>
+
+ namespace Parma_Polyhedra_Library {
+
+@@ -250,7 +251,7 @@ public:
+ class const_iterator
+ : public std::iterator<std::forward_iterator_tag,
+ Generator,
+- ptrdiff_t,
++ std::ptrdiff_t,
+ const Generator*,
+ const Generator&> {
+ public:
...@@ -10,8 +10,8 @@ extends = ...@@ -10,8 +10,8 @@ extends =
[gettext] [gettext]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
url = http://ftp.gnu.org/pub/gnu/gettext/gettext-0.19.2.tar.xz url = http://ftp.gnu.org/pub/gnu/gettext/gettext-0.19.3.tar.xz
md5sum = 1e6a827f5fbd98b3d40bd16b803acc44 md5sum = 092c3f460553ceb4a638ff81d36434c4
configure-options = configure-options =
--disable-static --disable-static
......
...@@ -33,8 +33,8 @@ environment = ...@@ -33,8 +33,8 @@ environment =
[ghostscript-9] [ghostscript-9]
<= ghostscript-common <= ghostscript-common
url = http://downloads.ghostscript.com/public/ghostscript-9.14.tar.bz2 url = http://downloads.ghostscript.com/public/ghostscript-9.15.tar.bz2
md5sum = 75f2e8ab8891d052ade9b64eb4eb5294 md5sum = 0ee21d975c67a4883ba66bcc332418f5
[ghostscript-fonts] [ghostscript-fonts]
recipe = hexagonit.recipe.download recipe = hexagonit.recipe.download
......
...@@ -6,21 +6,6 @@ extends = ...@@ -6,21 +6,6 @@ extends =
parts = parts =
gmp gmp
# we should use version 4.x for ppl-0.10
[gmp-4]
recipe = slapos.recipe.cmmi
url = ftp://ftp.gmplib.org/pub/gmp-4.3.2/gmp-4.3.2.tar.bz2
md5sum = dd60683d7057917e34630b4a787932e8
# GMP does not correctly detect achitecture so it have to be given
# as slapos.recipe.cmmi is using shell expansion in subproceses
# backticks are working
configure-options =
--build=`uname -m`-linux
--enable-cxx
--disable-static
environment =
PATH=${m4:location}/bin:%(PATH)s
[gmp] [gmp]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
version = 6.0.0a version = 6.0.0a
......
...@@ -11,8 +11,8 @@ parts = gnutls ...@@ -11,8 +11,8 @@ parts = gnutls
[gpg-error] [gpg-error]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
url = ftp://ftp.gnupg.org/gcrypt/libgpg-error/libgpg-error-1.13.tar.bz2 url = ftp://ftp.gnupg.org/gcrypt/libgpg-error/libgpg-error-1.17.tar.bz2
md5sum = fe0cfa7e15262ef8fdeee366109e9ff6 md5sum = b4f8d8b9ff14aed41f279aa844563539
[gcrypt] [gcrypt]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
...@@ -26,8 +26,8 @@ environment = ...@@ -26,8 +26,8 @@ environment =
[gnutls] [gnutls]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
url = ftp://ftp.gnutls.org/gcrypt/gnutls/v3.2/gnutls-3.2.16.tar.xz url = ftp://ftp.gnutls.org/gcrypt/gnutls/v3.2/gnutls-3.2.19.tar.xz
md5sum = d9a3fb49121b49bc332bfe9c3f49b005 md5sum = 8884492d0ad4339c831c6d507afbeea9
configure-options = configure-options =
--disable-static --disable-static
--disable-libdane --disable-libdane
......
...@@ -11,9 +11,8 @@ extends = ...@@ -11,9 +11,8 @@ extends =
[groonga] [groonga]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
version = 4.0.6 url = http://packages.groonga.org/source/groonga/groonga-4.0.7.tar.gz
url = http://packages.groonga.org/source/groonga/groonga-${:version}.tar.gz md5sum = 5d11546f921377d040f700bb74f1d7e4
md5sum = d7eb0296b0b759d1856c31b6288670fa
# temporary patch to respect more tokens in natural language mode. # temporary patch to respect more tokens in natural language mode.
patches = patches =
${:_profile_base_location_}/groonga.patch#9ed02fbe8400402d3eab47eee149978b ${:_profile_base_location_}/groonga.patch#9ed02fbe8400402d3eab47eee149978b
......
...@@ -10,3 +10,12 @@ url = ftp://ftp.gnu.org/pub/gnu/gzip/gzip-1.6.tar.xz ...@@ -10,3 +10,12 @@ url = ftp://ftp.gnu.org/pub/gnu/gzip/gzip-1.6.tar.xz
md5sum = da981f86677d58a106496e68de6f8995 md5sum = da981f86677d58a106496e68de6f8995
environment = environment =
PATH=${xz-utils:location}/bin:%(PATH)s PATH=${xz-utils:location}/bin:%(PATH)s
patch-options = -p1
# The --rsyncable patch is from debian/ubuntu,
# specifically https://launchpad.net/ubuntu/+source/gzip/1.6-3ubuntu1
# It is required to minimize the bandwidth used by rsync.
# For an explanation, see http://beeznest.wordpress.com/2005/02/03/rsyncable-gzip/
# Hunks for .texi files have been removed to avoid a dependency on makeinfo.
patches =
${:_profile_base_location_}/rsyncable.diff#0587af03a5580e2b7b4007469ee2b601
--- a/deflate.c
+++ b/deflate.c
@@ -131,6 +131,14 @@
#endif
/* Matches of length 3 are discarded if their distance exceeds TOO_FAR */
+#ifndef RSYNC_WIN
+# define RSYNC_WIN 8192
+#endif
+/* Size of rsync window, must be < MAX_DIST */
+
+#define RSYNC_SUM_MATCH(sum) (((sum) & (RSYNC_WIN - 1)) == 0)
+/* Whether window sum matches magic value */
+
/* ===========================================================================
* Local data used by the "longest match" routines.
*/
@@ -212,6 +220,8 @@
unsigned good_match;
/* Use a faster search when the previous match is longer than this */
+local ulg rsync_sum; /* rolling sum of rsync window */
+local ulg rsync_chunk_end; /* next rsync sequence point */
/* Values for max_lazy_match, good_match and max_chain_length, depending on
* the desired pack level (0..9). The values given below have been tuned to
@@ -314,6 +324,10 @@
#endif
/* prev will be initialized on the fly */
+ /* rsync params */
+ rsync_chunk_end = 0xFFFFFFFFUL;
+ rsync_sum = 0;
+
/* Set the default configuration parameters:
*/
max_lazy_match = configuration_table[pack_level].max_lazy;
@@ -550,6 +564,8 @@
memcpy((char*)window, (char*)window+WSIZE, (unsigned)WSIZE);
match_start -= WSIZE;
strstart -= WSIZE; /* we now have strstart >= MAX_DIST: */
+ if (rsync_chunk_end != 0xFFFFFFFFUL)
+ rsync_chunk_end -= WSIZE;
block_start -= (long) WSIZE;
@@ -579,13 +595,44 @@
}
}
+local void rsync_roll(unsigned start, unsigned num)
+{
+ unsigned i;
+
+ if (start < RSYNC_WIN) {
+ /* before window fills. */
+ for (i = start; i < RSYNC_WIN; i++) {
+ if (i == start + num) return;
+ rsync_sum += (ulg)window[i];
+ }
+ num -= (RSYNC_WIN - start);
+ start = RSYNC_WIN;
+ }
+
+ /* buffer after window full */
+ for (i = start; i < start+num; i++) {
+ /* New character in */
+ rsync_sum += (ulg)window[i];
+ /* Old character out */
+ rsync_sum -= (ulg)window[i - RSYNC_WIN];
+ if (rsync_chunk_end == 0xFFFFFFFFUL && RSYNC_SUM_MATCH(rsync_sum))
+ rsync_chunk_end = i;
+ }
+}
+
+/* ===========================================================================
+ * Set rsync_chunk_end if window sum matches magic value.
+ */
+#define RSYNC_ROLL(s, n) \
+ do { if (rsync) rsync_roll((s), (n)); } while(0)
+
/* ===========================================================================
* Flush the current block, with given end-of-file flag.
* IN assertion: strstart is set to the end of the current match.
*/
#define FLUSH_BLOCK(eof) \
flush_block(block_start >= 0L ? (char*)&window[(unsigned)block_start] : \
- (char*)NULL, (long)strstart - block_start, (eof))
+ (char*)NULL, (long)strstart - block_start, flush-1, (eof))
/* ===========================================================================
* Processes a new input file and return its compressed length. This
@@ -596,7 +643,7 @@
local off_t deflate_fast()
{
IPos hash_head; /* head of the hash chain */
- int flush; /* set if current block must be flushed */
+ int flush = 0; /* set if current block must be flushed, 2=>and padded */
unsigned match_length = 0; /* length of best match */
prev_length = MIN_MATCH-1;
@@ -626,7 +673,8 @@
lookahead -= match_length;
- /* Insert new strings in the hash table only if the match length
+ RSYNC_ROLL(strstart, match_length);
+ /* Insert new strings in the hash table only if the match length
* is not too large. This saves time but degrades compression.
*/
if (match_length <= max_insert_length) {
@@ -654,9 +702,14 @@
/* No match, output a literal byte */
Tracevv((stderr,"%c",window[strstart]));
flush = ct_tally (0, window[strstart]);
+ RSYNC_ROLL(strstart, 1);
lookahead--;
strstart++;
}
+ if (rsync && strstart > rsync_chunk_end) {
+ flush = 2;
+ rsync_chunk_end = 0xFFFFFFFFUL;
+ }
if (flush) FLUSH_BLOCK(0), block_start = strstart;
/* Make sure that we always have enough lookahead, except
@@ -679,7 +732,7 @@
{
IPos hash_head; /* head of hash chain */
IPos prev_match; /* previous match */
- int flush; /* set if current block must be flushed */
+ int flush = 0; /* set if current block must be flushed */
int match_available = 0; /* set if previous match exists */
register unsigned match_length = MIN_MATCH-1; /* length of best match */
@@ -730,6 +783,7 @@
*/
lookahead -= prev_length-1;
prev_length -= 2;
+ RSYNC_ROLL(strstart, prev_length+1);
do {
strstart++;
INSERT_STRING(strstart, hash_head);
@@ -742,24 +796,38 @@
match_available = 0;
match_length = MIN_MATCH-1;
strstart++;
- if (flush) FLUSH_BLOCK(0), block_start = strstart;
+ if (rsync && strstart > rsync_chunk_end) {
+ rsync_chunk_end = 0xFFFFFFFFUL;
+ flush = 2;
+ }
+ if (flush) FLUSH_BLOCK(0), block_start = strstart;
} else if (match_available) {
/* If there was no match at the previous position, output a
* single literal. If there was a match but the current match
* is longer, truncate the previous match to a single literal.
*/
Tracevv((stderr,"%c",window[strstart-1]));
- if (ct_tally (0, window[strstart-1])) {
- FLUSH_BLOCK(0), block_start = strstart;
- }
+ flush = ct_tally (0, window[strstart-1]);
+ if (rsync && strstart > rsync_chunk_end) {
+ rsync_chunk_end = 0xFFFFFFFFUL;
+ flush = 2;
+ }
+ if (flush) FLUSH_BLOCK(0), block_start = strstart;
+ RSYNC_ROLL(strstart, 1);
strstart++;
lookahead--;
} else {
/* There is no previous match to compare with, wait for
* the next step to decide.
*/
+ if (rsync && strstart > rsync_chunk_end) {
+ rsync_chunk_end = 0xFFFFFFFFUL;
+ flush = 2;
+ FLUSH_BLOCK(0), block_start = strstart;
+ }
match_available = 1;
+ RSYNC_ROLL(strstart, 1);
strstart++;
lookahead--;
}
--- a/gzip.c
+++ b/gzip.c
@@ -215,6 +215,7 @@
unsigned insize; /* valid bytes in inbuf */
unsigned inptr; /* index of next byte to be processed in inbuf */
unsigned outcnt; /* bytes in output buffer */
+int rsync = 0; /* make ryncable chunks */
static int handled_sig[] =
{
@@ -275,7 +276,7 @@
{"best", 0, 0, '9'}, /* compress better */
{"lzw", 0, 0, 'Z'}, /* make output compatible with old compress */
{"bits", 1, 0, 'b'}, /* max number of bits per code (implies -Z) */
-
+ {"rsyncable", 0, 0, 'R'}, /* make rsync-friendly archive */
{ 0, 0, 0, 0 }
};
@@ -359,6 +360,7 @@
" -Z, --lzw produce output compatible with old compress",
" -b, --bits=BITS max number of bits per code (implies -Z)",
#endif
+ " --rsyncable Make rsync-friendly archive",
"",
"With no FILE, or when FILE is -, read standard input.",
"",
@@ -489,8 +491,11 @@
#else
recursive = 1;
#endif
- break;
- case 'S':
+ break;
+ case 'R':
+ rsync = 1; break;
+
+ case 'S':
#ifdef NO_MULTIPLE_DOTS
if (*optarg == '.') optarg++;
#endif
--- a/gzip.h
+++ b/gzip.h
@@ -140,6 +140,7 @@
extern unsigned insize; /* valid bytes in inbuf */
extern unsigned inptr; /* index of next byte to be processed in inbuf */
extern unsigned outcnt; /* bytes in output buffer */
+extern int rsync; /* deflate into rsyncable chunks */
extern off_t bytes_in; /* number of input bytes */
extern off_t bytes_out; /* number of output bytes */
@@ -287,7 +288,7 @@
/* in trees.c */
extern void ct_init (ush *attr, int *method);
extern int ct_tally (int dist, int lc);
-extern off_t flush_block (char *buf, ulg stored_len, int eof);
+extern off_t flush_block (char *buf, ulg stored_len, int pad, int eof);
/* in bits.c */
extern void bi_init (file_t zipfile);
--- a/gzip.1
+++ b/gzip.1
@@ -5,6 +5,7 @@
.ll +8
.B gzip
.RB [ " \-acdfhklLnNrtvV19 " ]
+.RB [ --rsyncable ]
.RB [ \-S\ suffix ]
[
.I "name \&..."
@@ -287,6 +288,16 @@
.I gunzip
).
.TP
+.B --rsyncable
+While compressing, synchronize the output occasionally based on the input.
+This increases size by less than 1 percent most cases, but means that the
+.BR rsync (1)
+program can take advantage of similarities in the uncompressed input
+when syncronizing two files compressed with this flag.
+.I gunzip
+cannot tell the difference between a compressed file created with this option,
+and one created without it.
+.TP
.B \-S .suf --suffix .suf
When compressing, use suffix .suf instead of .gz.
Any non-empty suffix can be given, but suffixes
--- a/trees.c
+++ b/trees.c
@@ -856,9 +856,10 @@
* trees or store, and output the encoded block to the zip file. This function
* returns the total compressed length for the file so far.
*/
-off_t flush_block(buf, stored_len, eof)
+off_t flush_block(buf, stored_len, pad, eof)
char *buf; /* input block, or NULL if too old */
ulg stored_len; /* length of input block */
+ int pad; /* pad output to byte boundary */
int eof; /* true if this is the last block for a file */
{
ulg opt_lenb, static_lenb; /* opt_len and static_len in bytes */
@@ -951,6 +952,10 @@
Assert (input_len == bytes_in, "bad input size");
bi_windup();
compressed_len += 7; /* align on byte boundary */
+ } else if (pad && (compressed_len % 8) != 0) {
+ send_bits((STORED_BLOCK<<1)+eof, 3); /* send block type */
+ compressed_len = (compressed_len + 3 + 7) & ~7L;
+ copy_block(buf, 0, 1); /* with header */
}
return compressed_len >> 3;
...@@ -11,8 +11,8 @@ parts = haproxy ...@@ -11,8 +11,8 @@ parts = haproxy
[haproxy] [haproxy]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
url = http://www.haproxy.org/download/1.5/src/haproxy-1.5.3.tar.gz url = http://www.haproxy.org/download/1.5/src/haproxy-1.5.8.tar.gz
md5sum = e999a547d57445d5a5ab7eb6a06df9a1 md5sum = 7bffa1afa069d90ce03b7cd9aa0557cd
configure-command = true configure-command = true
# If the system is running on Linux 2.6, we use "linux26" as the TARGET, # If the system is running on Linux 2.6, we use "linux26" as the TARGET,
# otherwise use "generic". # otherwise use "generic".
......
...@@ -3,51 +3,31 @@ ...@@ -3,51 +3,31 @@
parts = parts =
java java
[jdk-6u27-no-user-interaction-patch]
recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/${:filename}
download-only = true
filename = jdk-6u27-no-user-interaction.patch
md5sum = 4c4303240647a114d07f3c411b2e6b5b
[java] [java]
<= java-re <= java-re
[java-re] [java-re]
<= java-re-1.6.0 <= java-re-7
[java-sdk]
<= java-sdk-1.6.0
[java-re-1.6.0] [java-re-7]
recipe = slapos.recipe.build recipe = slapos.recipe.build
slapos_promisee = slapos_promisee =
directory:bin directory:bin
directory:lib directory:lib
directory:man directory:man
directory:plugin directory:plugin
directory:javaws
file:lib/rt.jar file:lib/rt.jar
file:bin/java file:bin/java
# http://java.com/en/download/manual_v6.jsp # http://java.com/en/download/manual_java7.jsp
x86 = http://javadl.sun.com/webapps/download/AutoDL?BundleId=68284 7eda40e790de1a907d591b62949b6e72 x86 = http://javadl.sun.com/webapps/download/AutoDL?BundleId=97798 90a6b9e2a32d06c18a3f16b485f0d1ea
x86-64 = http://javadl.sun.com/webapps/download/AutoDL?BundleId=68286 839548714931443ba89719a995ece846 x86-64 = http://javadl.sun.com/webapps/download/AutoDL?BundleId=97800 7605134662f6c87131eca5745895fe84
script = script =
import glob
if not self.options.get('url'): self.options['url'], self.options['md5sum'] = self.options[guessPlatform()].split(' ') if not self.options.get('url'): self.options['url'], self.options['md5sum'] = self.options[guessPlatform()].split(' ')
download_file = self.download(self.options['url'], self.options.get('md5sum')) extract_dir = self.extract(self.download(self.options['url'], self.options.get('md5sum')))
extract_dir = tempfile.mkdtemp(self.name)
os.chdir(extract_dir)
(download_dir, filename) = os.path.split(download_file)
auto_extract_bin = os.path.join(extract_dir, filename)
shutil.move(download_file, auto_extract_bin)
os.chmod(auto_extract_bin, 0755)
subprocess.call([auto_extract_bin])
self.cleanup_dir_list.append(extract_dir)
workdir = guessworkdir(extract_dir) workdir = guessworkdir(extract_dir)
self.copyTree(glob.glob(os.path.join(workdir, "jre1.6.0_*"))[0], "%(location)s") self.copyTree(workdir, "%(location)s")
[java-re-1.7.0] [java-re-8]
recipe = slapos.recipe.build recipe = slapos.recipe.build
slapos_promisee = slapos_promisee =
directory:bin directory:bin
...@@ -56,9 +36,9 @@ slapos_promisee = ...@@ -56,9 +36,9 @@ slapos_promisee =
directory:plugin directory:plugin
file:lib/rt.jar file:lib/rt.jar
file:bin/java file:bin/java
# http://java.com/en/download/manual.jsp?locale=en # http://java.com/en/download/manual_java7.jsp
x86 = http://javadl.sun.com/webapps/download/AutoDL?BundleId=68234 ea99bedd9db33e9e2970f4b70abd1e4b x86 = http://javadl.sun.com/webapps/download/AutoDL?BundleId=97358 22d970566c418499d331a2099d77c548
x86-64 = http://javadl.sun.com/webapps/download/AutoDL?BundleId=68236 5aa9bd26cdf1fa6afd2b15826b4ba139 x86-64 = http://javadl.sun.com/webapps/download/AutoDL?BundleId=97360 f4f7f7335eaf2e7b5ff455abece9d5ed
script = script =
if not self.options.get('url'): self.options['url'], self.options['md5sum'] = self.options[guessPlatform()].split(' ') if not self.options.get('url'): self.options['url'], self.options['md5sum'] = self.options[guessPlatform()].split(' ')
extract_dir = self.extract(self.download(self.options['url'], self.options.get('md5sum'))) extract_dir = self.extract(self.download(self.options['url'], self.options.get('md5sum')))
......
...@@ -30,7 +30,7 @@ configure-options = ...@@ -30,7 +30,7 @@ configure-options =
--prefix=${buildout:parts-directory}/${:_buildout_section_name_} --prefix=${buildout:parts-directory}/${:_buildout_section_name_}
--with-netlib-lapack-tarfile=${lapack-download:location}/${lapack-download:filename} --with-netlib-lapack-tarfile=${lapack-download:location}/${lapack-download:filename}
--shared --shared
-b "$(uname -p | grep -q 64 && echo 64 || echo 32)" -b "$(uname -m | grep -q 64 && echo 64 || echo 32)"
-Fa alg '-fPIC' -Fa alg '-fPIC'
make-options = make-options =
-C build -C build
......
...@@ -23,10 +23,10 @@ md5sum = 4efba67fa5aa2b785c6fcec2cc3e90c9 ...@@ -23,10 +23,10 @@ md5sum = 4efba67fa5aa2b785c6fcec2cc3e90c9
[libpng15] [libpng15]
<= libpng-common <= libpng-common
url = http://download.sourceforge.net/libpng/libpng-1.5.18.tar.xz url = http://download.sourceforge.net/libpng/libpng-1.5.19.tar.xz
md5sum = 23fb6a0e9f4dbaab7b34b6fd317a4f2d md5sum = 1e2afb372b9979b2133829ee8136f30f
[libpng] [libpng]
<= libpng-common <= libpng-common
url = http://download.sourceforge.net/libpng/libpng-1.6.12.tar.xz url = http://download.sourceforge.net/libpng/libpng-1.6.14.tar.xz
md5sum = ee752da8aa75076500e4cf31318e636e md5sum = aefd9fda2178acab908707737174a0cd
...@@ -9,14 +9,14 @@ parts = ...@@ -9,14 +9,14 @@ parts =
[libreoffice-bin] [libreoffice-bin]
recipe = slapos.recipe.build recipe = slapos.recipe.build
# here, two %s are used, first one is for directory name (eg. x86_64), and second one is for filename (eg. x86-64). # here, two %s are used, first one is for directory name (eg. x86_64), and second one is for filename (eg. x86-64).
version = 4.2.6.3 version = 4.3.3.2
url = http://downloadarchive.documentfoundation.org/libreoffice/old/${:version}/rpm/%s/LibreOffice_${:version}_Linux_%s_rpm.tar.gz url = http://downloadarchive.documentfoundation.org/libreoffice/old/${:version}/rpm/%s/LibreOffice_${:version}_Linux_%s_rpm.tar.gz
# supported architectures md5sums # supported architectures md5sums
md5sum_x86 = 5a19002efc453a7af9443e1b01e7017a md5sum_x86 = c24d4d7db03a5c983633efde754efae4
md5sum_x86-64 = 25d23958e8253182583e1af0ab9d1f5c md5sum_x86-64 = 94e0c1027d864f26a65e742874514b92
# where office code can be found? # where office code can be found?
officedir = libreoffice4.2 officedir = libreoffice4.3
# script to install # script to install
script = script =
......
...@@ -4,8 +4,8 @@ parts = ...@@ -4,8 +4,8 @@ parts =
[libtasn1] [libtasn1]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
url = ftp://ftp.gnu.org/gnu/libtasn1/libtasn1-4.1.tar.gz url = http://ftp.gnu.org/gnu/libtasn1/libtasn1-4.2.tar.gz
md5sum = f9b37df3c2c9c6439d8bf427bfbfc521 md5sum = 414df906df421dee0a5cf7548788d153
configure-options = configure-options =
--disable-static --disable-static
--disable-gtk-doc-html --disable-gtk-doc-html
...@@ -4,16 +4,17 @@ ...@@ -4,16 +4,17 @@
[buildout] [buildout]
extends = extends =
../cmake/buildout.cfg ../cmake/buildout.cfg
../zlib/buildout.cfg
../jemalloc/buildout.cfg
../groonga/buildout.cfg ../groonga/buildout.cfg
../jemalloc/buildout.cfg
../libaio/buildout.cfg ../libaio/buildout.cfg
../libevent/buildout.cfg ../libevent/buildout.cfg
../libxml2/buildout.cfg
../ncurses/buildout.cfg ../ncurses/buildout.cfg
../openssl/buildout.cfg ../openssl/buildout.cfg
../pcre/buildout.cfg
../pkgconfig/buildout.cfg ../pkgconfig/buildout.cfg
../readline/buildout.cfg ../readline/buildout.cfg
../libxml2/buildout.cfg ../zlib/buildout.cfg
parts = parts =
mariadb mariadb
...@@ -23,8 +24,6 @@ recipe = slapos.recipe.cmmi ...@@ -23,8 +24,6 @@ recipe = slapos.recipe.cmmi
version = 10.0.14 version = 10.0.14
url = https://downloads.mariadb.org/f/mariadb-${:version}/source/mariadb-${:version}.tar.gz/from/http:/ftp.osuosl.org/pub/mariadb url = https://downloads.mariadb.org/f/mariadb-${:version}/source/mariadb-${:version}.tar.gz/from/http:/ftp.osuosl.org/pub/mariadb
md5sum = 80fea71de54a9cfa7f5508df53d3f06d md5sum = 80fea71de54a9cfa7f5508df53d3f06d
# compile directory is required to build mysql plugins.
keep-compile-dir = true
patch-options = -p0 patch-options = -p0
patches = patches =
${:_profile_base_location_}/mariadb_10.0.8_create_system_tables__no_test.patch#a176d491cf45fccd53ee397c70393bc4 ${:_profile_base_location_}/mariadb_10.0.8_create_system_tables__no_test.patch#a176d491cf45fccd53ee397c70393bc4
...@@ -38,34 +37,40 @@ configure-options = ...@@ -38,34 +37,40 @@ configure-options =
-DWITH_ZLIB=system -DWITH_ZLIB=system
-DWITH_READLINE=0 -DWITH_READLINE=0
-DWITH_PIC=1 -DWITH_PIC=1
-DWITH_PCRE=system
-DENABLE_DTRACE=0 -DENABLE_DTRACE=0
-DWITH_EXTRA_CHARSETS=complex -DWITH_EXTRA_CHARSETS=complex
-DWITH_EMBEDDED_SERVER=0 -DWITH_EMBEDDED_SERVER=0
-DWITHOUT_EXAMPLE_STORAGE_ENGINE=1 -DWITHOUT_EXAMPLE_STORAGE_ENGINE=1
-DWITHOUT_DAEMON_EXAMPLE=1 -DWITHOUT_DAEMON_EXAMPLE=1
-DCMAKE_C_FLAGS="-I${jemalloc:location}/include -I${libaio:location}/include -I${libxml2:location}/include -I${ncurses:location}/include -I${openssl:location}/include -I${readline5:location}/include -I${zlib:location}/include" -DCMAKE_C_FLAGS="-I${jemalloc:location}/include -I${libaio:location}/include -I${libxml2:location}/include -I${ncurses:location}/include -I${openssl:location}/include -I${pcre:location}/include -I${readline5:location}/include -I${zlib:location}/include"
-DCMAKE_CXX_FLAGS="-I${jemalloc:location}/include -I${libaio:location}/include -I${libxml2:location}/include -I${ncurses:location}/include -I${openssl:location}/include -I${readline5:location}/include -I${zlib:location}/include" -DCMAKE_CXX_FLAGS="-I${jemalloc:location}/include -I${libaio:location}/include -I${libxml2:location}/include -I${ncurses:location}/include -I${openssl:location}/include -I${pcre:location}/include -I${readline5:location}/include -I${zlib:location}/include"
-DCMAKE_INSTALL_RPATH=${jemalloc:location}/lib:${libaio:location}/lib:${libxml2:location}/lib:${ncurses:location}/lib:${openssl:location}/lib:${readline5:location}/lib:${zlib:location}/lib -DCMAKE_INSTALL_RPATH=${jemalloc:location}/lib:${libaio:location}/lib:${libxml2:location}/lib:${ncurses:location}/lib:${openssl:location}/lib:${pcre:location}/lib:${readline5:location}/lib:${zlib:location}/lib
-DWITHOUT_TOKUDB=true -DWITHOUT_TOKUDB=true
environment = environment =
CMAKE_PROGRAM_PATH=${cmake:location}/bin CMAKE_PROGRAM_PATH=${cmake:location}/bin
CMAKE_INCLUDE_PATH=${libaio:location}/include:${libaio:location}/include:${libxml2:location}/include:${ncurses:location}/include:${openssl:location}/include:${readline5:location}/include:${zlib:location}/include CMAKE_INCLUDE_PATH=${libaio:location}/include:${libaio:location}/include:${libxml2:location}/include:${ncurses:location}/include:${openssl:location}/include:${pcre:location}/include:${readline5:location}/include:${zlib:location}/include
CMAKE_LIBRARY_PATH=${libaio:location}/lib:{libaio:location}/lib:${libxml2:location}/lib:${ncurses:location}/lib:${openssl:location}/lib:${readline5:location}/lib:${zlib:location}/lib CMAKE_LIBRARY_PATH=${libaio:location}/lib:{libaio:location}/lib:${libxml2:location}/lib:${ncurses:location}/lib:${openssl:location}/lib:${pcre:location}/lib:${readline5:location}/lib:${zlib:location}/lib
LDFLAGS=-L${jemalloc:location}/lib -Wl,-rpath=${jemalloc:location}/lib -L${libaio:location}/lib -Wl,-rpath=${libaio:location}/lib -L${zlib:location}/lib LDFLAGS=-L${jemalloc:location}/lib -Wl,-rpath=${jemalloc:location}/lib -L${libaio:location}/lib -Wl,-rpath=${libaio:location}/lib -L${pcre:location}/lib -L${zlib:location}/lib
[mroonga-mariadb] [mroonga-mariadb]
# mroonga - a storage engine for MySQL. It provides fast fulltext search feature to all MySQL users. # mroonga - a storage engine for MySQL. It provides fast fulltext search feature to all MySQL users.
# http://mroonga.github.com/ # http://mroonga.github.com/
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
url = http://packages.groonga.org/source/mroonga/mroonga-4.06.tar.gz url = http://packages.groonga.org/source/mroonga/mroonga-4.07.tar.gz
md5sum = 042bc1fbc3f25593be4484529d598144 md5sum = fcfca7934b1f8df981fca579d3cd41bd
configure-command = mkdir fake_mariadb_source && ln -s ${mariadb:location}/include/mysql/private fake_mariadb_source/sql && ./configure
configure-options = configure-options =
--with-mysql-source=${mariadb:location}__compile__/mariadb-${mariadb:version} --prefix=${buildout:parts-directory}/${:_buildout_section_name_}
--with-mysql-source=fake_mariadb_source
--with-mysql-config=${mariadb:location}/bin/mysql_config --with-mysql-config=${mariadb:location}/bin/mysql_config
--disable-static --disable-static
--disable-document --disable-document
patch-options = -p1
patches =
${:_profile_base_location_}/mroonga_boolean.patch#36645770ae612515b74b90884ecc59fc
environment = environment =
PATH=${groonga:location}/bin:${pkgconfig:location}/bin:%(PATH)s PATH=${groonga:location}/bin:${pkgconfig:location}/bin:%(PATH)s
CPPFLAGS=-I${groonga:location}/include/groonga CPPFLAGS=-I${groonga:location}/include/groonga -I${pcre:location}/include
LDFLAGS=-L${groonga:location}/lib LDFLAGS=-L${groonga:location}/lib
PKG_CONFIG_PATH=${groonga:location}/lib/pkgconfig:${groonga-normalizer-mysql:location}/lib/pkgconfig PKG_CONFIG_PATH=${groonga:location}/lib/pkgconfig:${groonga-normalizer-mysql:location}/lib/pkgconfig
diff --git a/ha_mroonga.cpp b/ha_mroonga.cpp
index dd9967d..c105a62 100644
--- a/ha_mroonga.cpp
+++ b/ha_mroonga.cpp
@@ -7678,7 +7678,7 @@ grn_rc ha_mroonga::generic_ft_init_ext_prepare_expression_in_boolean_mode(
const char *keyword, *keyword_original;
uint keyword_length, keyword_length_original;
- grn_operator default_operator = GRN_OP_OR;
+ grn_operator default_operator = GRN_OP_ADJUST;
grn_bool weight_specified = false;
keyword = keyword_original = key->ptr();
keyword_length = keyword_length_original = key->length();
[buildout] [buildout]
extends = extends =
../autoconf/buildout.cfg
../automake/buildout.cfg
../bison/buildout.cfg
../flex/buildout.cfg
../libexpat/buildout.cfg
../libtool/buildout.cfg
../pkgconfig/buildout.cfg
../xorg/buildout.cfg ../xorg/buildout.cfg
parts = parts =
mesa mesa
[libdrm]
#does not seem to compile with -jX
recipe = slapos.recipe.cmmi
url = http://dri.freedesktop.org/libdrm/libdrm-2.4.26.tar.bz2
md5sum = 062569426773f69b11a47a7712bba770
configure-options =
--enable-nouveau-experimental-api
environment =
PATH=${pkgconfig:location}/bin:%(PATH)s
PKG_CONFIG_PATH=${xorg-libpthread-stubs:location}/lib/pkgconfig:${pciaccess:location}/lib/pkgconfig
[mesa] [mesa]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
url = ftp://ftp.freedesktop.org//pub/mesa/7.11/MesaLib-7.11.tar.bz2 url = ftp://ftp.freedesktop.org/pub/mesa/10.3.2/MesaLib-10.3.2.tar.bz2
md5sum = ff03aca82d0560009a076a87c888cf13 md5sum = c0a2a975899c4e7485124b388c16cd2f
configure-options = configure-options =
--disable-static --disable-static
--disable-driglx-direct --disable-dri
--disable-glu --disable-dri3
--disable-glw
--disable-glut
--disable-egl --disable-egl
--disable-gbm
--disable-xvmc
--disable-vdpau
--enable-xlib-glx
--disable-gallium-gbm
--disable-shared-glapi
--disable-driglx-direct
--disable-gallium-llvm --disable-gallium-llvm
--with-gallium-drivers= --with-gallium-drivers=
--with-dri-drivers= --with-dri-drivers=
--with-expat=${libexpat:location}
--with-driver=xlib
environment = environment =
PATH=${pkgconfig:location}/bin:${makedepend:location}/bin:${flex:location}/bin:${bison:location}/bin:%(PATH)s PATH=${autoconf:location}/bin:${bison:location}/bin:${flex:location}/bin:${pkgconfig:location}/bin:%(PATH)s
PKG_CONFIG_PATH=${libdrm:location}/lib/pkgconfig:${dri2proto:location}/lib/pkgconfig:${xorg-libpthread-stubs:location}/lib/pkgconfig:${pciaccess:location}/lib/pkgconfig:${libX11:location}/lib/pkgconfig:${glproto:location}/lib/pkgconfig:${libX11:location}/lib/pkgconfig:${libXext:location}/lib/pkgconfig:${xfixes:location}/lib/pkgconfig:${xdamage:location}/lib/pkgconfig:${xproto:location}/lib/pkgconfig:${xextproto:location}/lib/pkgconfig PKG_CONFIG_PATH=${damageproto:location}/lib/pkgconfig:${fixesproto:location}/lib/pkgconfig:${glproto:location}/lib/pkgconfig:${kbproto:location}/lib/pkgconfig:${libX11:location}/lib/pkgconfig:${libXau:location}/lib/pkgconfig:${libXext:location}/lib/pkgconfig:${libexpat:location}/lib/pkgconfig:${libxcb:location}/lib/pkgconfig:${xdamage:location}/lib/pkgconfig:${xextproto:location}/lib/pkgconfig:${xfixes:location}/lib/pkgconfig:${xorg-libpthread-stubs:location}/lib/pkgconfig:${xproto:location}/lib/pkgconfig
CPPFLAGS=-I${libexpat:location}/include -I${libdrm:location}/include/libdrm -I${libdrm:location}/include -I${xproto:location}/include -I${libX11:location}/include -I${libXext:location}/include/ -I${xextproto:location}/include PYTHON2=${buildout:executable}
LDFLAGS=-L${libexpat:location}/lib ACLOCAL=${automake:location}/bin/aclocal -I${libtool:location}/share/aclocal -I${pkgconfig:location}/share/aclocal
PYTHON=${buildout:executable} AUTOCONF=${autoconf:location}/bin/autoconf
AUTOMAKE=${automake:location}/bin/automake
[glu]
recipe = slapos.recipe.cmmi
# tar.bz2 cannot be well extracted why ?
url = ftp://ftp.freedesktop.org/pub/mesa/glu/glu-9.0.0.tar.gz
md5sum = bbc57d4fe3bd3fb095bdbef6fcb977c4
configure-options =
--disable-static
environment =
PATH=${pkgconfig:location}/bin:%(PATH)s
PKG_CONFIG_PATH=${damageproto:location}/lib/pkgconfig:${fixesproto:location}/lib/pkgconfig:${glproto:location}/lib/pkgconfig:${kbproto:location}/lib/pkgconfig:${libX11:location}/lib/pkgconfig:${libXau:location}/lib/pkgconfig:${libXext:location}/lib/pkgconfig:${libexpat:location}/lib/pkgconfig:${libxcb:location}/lib/pkgconfig:${mesa:location}/lib/pkgconfig:${xdamage:location}/lib/pkgconfig:${xextproto:location}/lib/pkgconfig:${xfixes:location}/lib/pkgconfig:${xorg-libpthread-stubs:location}/lib/pkgconfig:${xproto:location}/lib/pkgconfig
...@@ -11,8 +11,8 @@ parts = nginx-output ...@@ -11,8 +11,8 @@ parts = nginx-output
[nginx-common] [nginx-common]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
url = http://nginx.org/download/nginx-1.7.5.tar.gz url = http://nginx.org/download/nginx-1.7.6.tar.gz
md5sum = e65aad627acc1cbe26527339a5814d57 md5sum = dd444e5333e0d324bec480e2ff67870a
[nginx] [nginx]
<= nginx-common <= nginx-common
...@@ -38,6 +38,7 @@ configure-options = ...@@ -38,6 +38,7 @@ configure-options =
--with-http_ssl_module --with-http_ssl_module
--with-mail --with-mail
--with-mail_ssl_module --with-mail_ssl_module
--error-log-path=var/log/nginx.error.log
--with-ld-opt=" -L ${libexpat:location}/lib -L ${openssl:location}/lib -L ${pcre:location}/lib -L ${zlib:location}/lib -Wl,-rpath=${libexpat:location}/lib -Wl,-rpath=${openssl:location}/lib -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${zlib:location}/lib" --with-ld-opt=" -L ${libexpat:location}/lib -L ${openssl:location}/lib -L ${pcre:location}/lib -L ${zlib:location}/lib -Wl,-rpath=${libexpat:location}/lib -Wl,-rpath=${openssl:location}/lib -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${zlib:location}/lib"
--with-cc-opt="-I ${libexpat:location}/include -I ${openssl:location}/include -I ${pcre:location}/include -I ${zlib:location}/include" --with-cc-opt="-I ${libexpat:location}/include -I ${openssl:location}/include -I ${pcre:location}/include -I ${zlib:location}/include"
--with-http_dav_module --with-http_dav_module
......
...@@ -15,14 +15,8 @@ ATLAS = ${libatlas:location}/lib/libatlas.a ...@@ -15,14 +15,8 @@ ATLAS = ${libatlas:location}/lib/libatlas.a
[numpy] [numpy]
recipe = zc.recipe.egg:custom recipe = zc.recipe.egg:custom
egg = numpy egg = numpy
# patched version that supports rpath parameter in zc.recipe.egg:custom.
find-links = http://www.nexedi.org/static/packages/source/numpy-1.8.2nxd001.tar.gz
environment = numpy-env environment = numpy-env
# no need to add libatlas location because they are statically linked. # no need to add libatlas location because they are statically linked.
rpath = rpath =
${gcc-fortran:location}/lib ${gcc-fortran:location}/lib
${gcc-fortran:location}/lib64 ${gcc-fortran:location}/lib64
[versions]
# patched version that supports rpath parameter in zc.recipe.egg:custom.
numpy = 1.8.2nxd001
...@@ -16,8 +16,8 @@ parts = ...@@ -16,8 +16,8 @@ parts =
[openssl] [openssl]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
url = https://www.openssl.org/source/openssl-1.0.1i.tar.gz url = https://www.openssl.org/source/openssl-1.0.1j.tar.gz
md5sum = c8dc151a671b9b92ff3e4c118b174972 md5sum = f7175c9cd3c39bb1907ac8bba9df8ed3
patch-binary = ${patch:location}/bin/patch patch-binary = ${patch:location}/bin/patch
patches = patches =
${:_profile_base_location_}/openssl-nodoc.patch#b4887a7b4e18402447bc6227d2493b92 ${:_profile_base_location_}/openssl-nodoc.patch#b4887a7b4e18402447bc6227d2493b92
......
...@@ -4,8 +4,8 @@ parts = ...@@ -4,8 +4,8 @@ parts =
[pcre] [pcre]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
url = ftp://ftp.csx.cam.ac.uk/pub/software/programming/pcre/pcre-8.35.tar.bz2 url = http://ftp.csx.cam.ac.uk/pub/software/programming/pcre/pcre-8.36.tar.bz2
md5sum = 6aacb23986adccd9b3bc626c00979958 md5sum = b767bc9af0c20bc9c1fe403b0d41ad97
configure-options = configure-options =
--disable-static --disable-static
--enable-unicode-properties --enable-unicode-properties
...@@ -14,6 +14,8 @@ library-dirs = ...@@ -14,6 +14,8 @@ library-dirs =
${librsync:location}/lib/ ${librsync:location}/lib/
rpath = rpath =
${librsync:location}/lib/ ${librsync:location}/lib/
# If you need another release, override this link
# and pin the egg in the [versions] section of the stack or SR.
find-links = http://pkgs.fedoraproject.org/repo/pkgs/rdiff-backup/rdiff-backup-1.0.5.tar.gz/fa2a165fa07a94be52c52e3545bc7758/rdiff-backup-1.0.5.tar.gz find-links = http://pkgs.fedoraproject.org/repo/pkgs/rdiff-backup/rdiff-backup-1.0.5.tar.gz/fa2a165fa07a94be52c52e3545bc7758/rdiff-backup-1.0.5.tar.gz
[rdiff-backup] [rdiff-backup]
......
...@@ -5,8 +5,8 @@ parts = ...@@ -5,8 +5,8 @@ parts =
[sqlite3] [sqlite3]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
url = http://www.sqlite.org/2014/sqlite-autoconf-3080600.tar.gz url = http://www.sqlite.org/2014/sqlite-autoconf-3080701.tar.gz
md5sum = f7e4a156b583abeba349629e2364224b md5sum = 8ee4541ebb3e5739e7ef5e9046e30063
configure-options = configure-options =
--disable-static --disable-static
--enable-readline --enable-readline
......
...@@ -10,8 +10,8 @@ parts = ...@@ -10,8 +10,8 @@ parts =
[wget] [wget]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
url = http://ftp.gnu.org/gnu/wget/wget-1.15.tar.xz url = http://ftp.gnu.org/gnu/wget/wget-1.16.tar.xz
md5sum = 7a279d5ac5594919124d5526e7143e28 md5sum = fe102975ab3a6c049777883f1bb9ad07
configure-options = configure-options =
--enable-ipv6 --enable-ipv6
--enable-opie --enable-opie
...@@ -24,7 +24,7 @@ patch-options = ...@@ -24,7 +24,7 @@ patch-options =
-p1 -p1
patches = patches =
${:_profile_base_location_}/wget-doc.makefile.patch#2b89644c63a0dfe96290471d28a554f2 ${:_profile_base_location_}/wget-doc.makefile.patch#aabcc1695c7fb65ac44f295e04a2db78
environment = environment =
PATH=${pkgconfig:location}/bin:${xz-utils:location}/bin:%(PATH)s PATH=${pkgconfig:location}/bin:${xz-utils:location}/bin:%(PATH)s
......
...@@ -4,8 +4,8 @@ ...@@ -4,8 +4,8 @@
ACLOCAL_AMFLAGS = -I m4 ACLOCAL_AMFLAGS = -I m4
# subdirectories in the distribution # subdirectories in the distribution
-SUBDIRS = lib src doc po tests util -SUBDIRS = lib src doc po tests util testenv
+SUBDIRS = lib src po tests util +SUBDIRS = lib src po tests util testenv
EXTRA_DIST = ChangeLog.README MAILING-LIST \ EXTRA_DIST = ChangeLog.README MAILING-LIST \
msdos/ChangeLog msdos/config.h msdos/Makefile.DJ \ msdos/ChangeLog msdos/config.h msdos/Makefile.DJ \
...@@ -15,8 +15,8 @@ ...@@ -15,8 +15,8 @@
ACLOCAL_AMFLAGS = -I m4 ACLOCAL_AMFLAGS = -I m4
# subdirectories in the distribution # subdirectories in the distribution
-SUBDIRS = lib src doc po tests util -SUBDIRS = lib src doc po tests util testenv
+SUBDIRS = lib src po tests util +SUBDIRS = lib src po tests util testenv
EXTRA_DIST = ChangeLog.README MAILING-LIST \ EXTRA_DIST = ChangeLog.README MAILING-LIST \
msdos/ChangeLog msdos/config.h msdos/Makefile.DJ \ msdos/ChangeLog msdos/config.h msdos/Makefile.DJ \
msdos/Makefile.WC ABOUT-NLS \ msdos/Makefile.WC ABOUT-NLS \
...@@ -465,6 +465,21 @@ environment = ...@@ -465,6 +465,21 @@ environment =
PATH=${pkgconfig:location}/bin:%(PATH)s PATH=${pkgconfig:location}/bin:%(PATH)s
PKG_CONFIG_PATH=${xextproto:location}/lib/pkgconfig:${libX11:location}/lib/pkgconfig:${libXext:location}/lib/pkgconfig:${libXt:location}/lib/pkgconfig:${xproto:location}/lib/pkgconfig:${kbproto:location}/lib/pkgconfig:${libxcb:location}/lib/pkgconfig:${xorg-libpthread-stubs:location}/lib/pkgconfig:${libXau:location}/lib/pkgconfig:${libICE:location}/lib/pkgconfig:${libSM:location}/lib/pkgconfig PKG_CONFIG_PATH=${xextproto:location}/lib/pkgconfig:${libX11:location}/lib/pkgconfig:${libXext:location}/lib/pkgconfig:${libXt:location}/lib/pkgconfig:${xproto:location}/lib/pkgconfig:${kbproto:location}/lib/pkgconfig:${libxcb:location}/lib/pkgconfig:${xorg-libpthread-stubs:location}/lib/pkgconfig:${libXau:location}/lib/pkgconfig:${libICE:location}/lib/pkgconfig:${libSM:location}/lib/pkgconfig
[compositeproto]
recipe = slapos.recipe.cmmi
url = http://www.x.org/releases/X11R7.7/src/everything/compositeproto-0.4.2.tar.bz2
md5sum = 98482f65ba1e74a08bf5b056a4031ef0
[libXcomposite]
recipe = slapos.recipe.cmmi
url = http://www.x.org/releases/X11R7.7/src/everything/libXcomposite-0.4.3.tar.bz2
md5sum = a60e0b5c276d0aa9e2d3b982c98f61c8
configure-options =
--disable-static
environment =
PATH=${pkgconfig:location}/bin:%(PATH)s
PKG_CONFIG_PATH=${compositeproto:location}/lib/pkgconfig:${fixesproto:location}/lib/pkgconfig:${kbproto:location}/lib/pkgconfig:${libX11:location}/lib/pkgconfig:${libXau:location}/lib/pkgconfig:${libxcb:location}/lib/pkgconfig:${xextproto:location}/lib/pkgconfig:${xfixes:location}/lib/pkgconfig:${xorg-libpthread-stubs:location}/lib/pkgconfig:${xorg-util-macros:location}/share/pkgconfig:${xproto:location}/lib/pkgconfig
[libXcursor] [libXcursor]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
url = http://www.x.org/releases/X11R7.7/src/everything/libXcursor-1.1.13.tar.bz2 url = http://www.x.org/releases/X11R7.7/src/everything/libXcursor-1.1.13.tar.bz2
......
...@@ -28,7 +28,7 @@ from setuptools import setup, find_packages ...@@ -28,7 +28,7 @@ from setuptools import setup, find_packages
import glob import glob
import os import os
version = '0.87' version = '0.91'
name = 'slapos.cookbook' name = 'slapos.cookbook'
long_description = open("README.txt").read() + "\n" + \ long_description = open("README.txt").read() + "\n" + \
open("CHANGES.txt").read() + "\n" open("CHANGES.txt").read() + "\n"
......
...@@ -40,44 +40,8 @@ class Recipe(GenericSlapRecipe): ...@@ -40,44 +40,8 @@ class Recipe(GenericSlapRecipe):
def _install(self): def _install(self):
path_list = [] path_list = []
confpath = os.path.join(self.options['etc'], 'bully.conf')
ip_list = self.parameter_dict['ip-list']
print 'Creating bully configuration with ips : %s\n' % ip_list
conf = self.createFile(confpath,
self.substituteTemplate(
self.getTemplateFilename('bully.conf.in'),
{
'self_id': int(self.parameter_dict['number']),
'ip_list': ip_list
}
))
path_list.append(conf)
slap_connection = self.buildout['slap-connection'] slap_connection = self.buildout['slap-connection']
if self.optionIsTrue('enable-bully-service', default=False):
bully_dir = self.options['services']
else:
bully_dir = self.options['bin']
bully_wrapper = self.createPythonScript(
name=os.path.join(bully_dir, self.options['wrapper-bully']),
absolute_function='slapos.recipe.addresiliency.bully.run',
arguments={
'confpath': confpath,
'server_url': slap_connection['server-url'],
'key_file': slap_connection.get('key-file'),
'cert_file': slap_connection.get('cert-file'),
'computer_id': slap_connection['computer-id'],
'partition_id': slap_connection['partition-id'],
'software': slap_connection['software-release-url'],
'namebase': self.parameter_dict['namebase'],
})
path_list.append(bully_wrapper)
takeover_wrapper = self.createPythonScript( takeover_wrapper = self.createPythonScript(
name=os.path.join(self.options['bin'], self.options['wrapper-takeover']), name=os.path.join(self.options['bin'], self.options['wrapper-takeover']),
absolute_function='slapos.recipe.addresiliency.takeover.run', absolute_function='slapos.recipe.addresiliency.takeover.run',
......
# -*- coding: utf-8 -*-
import logging
import Queue
import socket
import thread
import time
import slapos.recipe.addresiliency.renamer
import slapos
log = logging.getLogger(__name__)
logging.basicConfig(level=logging.DEBUG)
BASE_PORT = 50000
SLEEPING_MINS = 2 # XXX was 30, increase after testing
MSG_PING = 'ping'
MSG_HALT = 'halt'
MSG_VICTORY = 'victory'
MSG_OK = 'ok'
STATE_NORMAL = 'normal'
STATE_WAITINGCONFIRM = 'waitingConfirm'
STATE_ELECTION = 'election'
STATE_REORGANIZATION = 'reorganization'
## Leader is always number 0
class ResilientInstance(object):
def __init__(self, comm, renamer, confpath):
self.comm = comm
self.participant_id = 0
self.state = STATE_NORMAL
self.halter_id = 0
self.inElection = False
self.alive = True
self.mainCanal = self.comm.create_canal([MSG_PING, MSG_HALT, MSG_VICTORY])
self.renamer = renamer
self.okCanal = self.comm.create_canal([MSG_OK])
self.confpath = confpath
self.loadConnectionInfo()
def loadConnectionInfo(self):
params = open(self.confpath, 'r').readlines()
self.total_participants = len(params[0].split())
new_id = int(params[1])
if self.participant_id != new_id:
self.halter_id = new_id
self.participant_id = new_id
log.debug('I am {} of {}'.format(self.participant_id, self.total_participants))
## Needs to be changed to use the master
def aliveManagement(self):
while self.alive:
log.info('XXX sleeping for %d minutes' % SLEEPING_MINS)
time.sleep(SLEEPING_MINS*60)
if self.participant_id == 0:
continue
self.comm.send(MSG_PING, 0)
message, sender = self.okCanal.get()
if message:
continue
self.election()
def listen(self):
while self.alive:
self.comm.recv()
def main(self):
while self.alive:
message, sender = self.mainCanal.get()
if message == MSG_PING:
self.comm.send(MSG_OK, sender)
elif message == MSG_HALT:
self.state = STATE_WAITINGCONFIRM
self.halter_id = int(sender)
self.comm.send(MSG_OK, sender)
elif message == MSG_VICTORY:
if int(sender) == self.halter_id and self.state == STATE_WAITINGCONFIRM:
log.info('{} thinks {} is the leader'.format(self.participant_id, sender))
self.comm.send(MSG_OK, sender)
self.state = STATE_NORMAL
def election(self):
self.inElection = True
self.loadConnectionInfo()
# Check if I'm the highest instance alive
for higher in range(self.participant_id + 1, self.total_participants):
self.comm.send(MSG_PING, higher)
message, sender = self.okCanal.get()
if message:
log.info('{} is alive ({})'.format(higher, self.participant_id))
self.inElection = False
return False
continue
if not self.alive:
return False
# I should be the new coordinator, halt those below me
log.info('Should be ME : {}'.format(self.participant_id))
self.state = STATE_ELECTION
self.halter_id = self.participant_id
ups = []
for lower in range(self.participant_id):
self.comm.send(MSG_HALT, lower)
message, sender = self.okCanal.get()
if message:
ups.append(lower)
#Broadcast Victory
self.state = STATE_REORGANIZATION
for up in ups:
self.comm.send(MSG_VICTORY, up)
message, sender = self.okCanal.get()
if message:
continue
log.info('Something is wrong... let\'s start over')
return self.election()
self.state = STATE_NORMAL
self.active = True
log.info('{} Is THE LEADER'.format(self.participant_id))
self.renamer.failover()
self.inElection = False
return True
class FilteredCanal(object):
def __init__(self, accept, timeout):
self.accept = accept
self.queue = Queue.Queue()
self.timeout = timeout
def append(self, message, sender):
if message in self.accept:
self.queue.put([message, sender])
def get(self):
try:
return self.queue.get(timeout=self.timeout)
except Queue.Empty:
return [None, None]
class Wrapper(object):
def __init__(self, confpath, timeout=20):
self.canals = []
self.ips = []
self.participant_id = 0
self.timeout = timeout
self.confpath = confpath
self.getConnectionInfo()
self.socket = None
def getConnectionInfo(self):
params = open(self.confpath, 'r').readlines()
self.ips = params[0].split()
self.participant_id = int(params[1])
log.debug('I am {} of {}'.format(self.participant_id, self.ips))
def start(self):
self.getConnectionInfo()
self.socket = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
self.socket.bind((self.ips[self.participant_id], BASE_PORT + self.participant_id))
self.socket.listen(5)
def send(self, message, number):
self.getConnectionInfo()
try:
s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
s.connect((self.ips[number], BASE_PORT + number))
s.send(message + (' {}\n'.format(self.participant_id)))
except (socket.error, socket.herror, socket.gaierror, socket.timeout):
pass
finally:
s.close()
def create_canal(self, accept):
created = FilteredCanal(accept, self.timeout)
self.canals.append(created)
return created
def recv(self):
client, _ = self.socket.accept()
client_message = client.recv(1024)
if client_message:
message, sender = client_message.split()
for canal in self.canals:
canal.append(message, int(sender))
def run(args):
confpath = args.pop('confpath')
renamer = slapos.recipe.addresiliency.renamer.Renamer(server_url = args.pop('server_url'),
key_file = args.pop('key_file'),
cert_file = args.pop('cert_file'),
computer_guid = args.pop('computer_id'),
partition_id = args.pop('partition_id'),
software_release = args.pop('software'),
namebase = args.pop('namebase'))
if args:
raise ValueError('Unknown arguments: %s' % ', '.join(args))
wrapper = Wrapper(confpath=confpath, timeout=20)
computer = ResilientInstance(wrapper, renamer=renamer, confpath=confpath)
# idle waiting for connection infos
while computer.total_participants < 2:
computer.loadConnectionInfo()
time.sleep(30)
log.info('Starting')
computer.comm.start()
thread.start_new_thread(computer.listen, ())
thread.start_new_thread(computer.aliveManagement, ())
computer.main()
#!%(executable)s
import select
import socket
import threading
import time
import sys
sys.path[:] = %(syspath)s
import slapos
from slapos import slap as slapmodule
port = 50000
size = 1024
wait = True
def loadConnectionInfos():
connectionInfos = {}
file = open('%(confpath)s', 'r')
params = file.read().split('\n')
file.close()
ip_list = [x.strip("' ") for x in params[0].strip('[],').split(',')]
connectionInfos['self_id'] = int(params[1])
connectionInfos['server_list'] = \
[(i, ip_list[i]) for i in range(len(ip_list))]
connectionInfos['self_ip'] = ip_list[connectionInfos['self_id']]
return connectionInfos
def rename_broken_and_stop():
try:
slap = slapmodule.slap()
slap.initializeConnection('%(server_url)s',
'%(key_file)s',
'%(cert_file)s')
computer_partition = slap.registerComputerPartition(computer_guid='%(computer_id)s',
partition_id='%(partition_id)s')
broken = computer_partition.request(software_release='%(software)s',
software_type='frozen',
partition_reference='%(namebase)s0')
broken.rename('broken-%%s' %% (time.strftime("%%d-%%b_%%H:%%M:%%S", time.gmtime())))
broken.stopped()
computer_partition.rename('%(namebase)s0')
print 'renaming done\n'
except slapos.slap.slap.ServerError:
print 'Internal server error\n'
def election():
global wait
connection = loadConnectionInfos()
message = "%%s, %%s" %% (connection['self_id'], "Election")
victory = True
for (remote_id, addr) in connection['server_list']:
if remote_id > connection['self_id']:
try:
s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
s.connect((addr, port + remote_id))
s.send(message)
reply = s.recv(size)
if reply == "%%s, %%s" %% (remote_id, "Alive"):
victory = False
except (socket.error, socket.herror, socket.gaierror, socket.timeout):
pass
finally:
s.close()
if victory:
wait = True
for (remote_id, addr) in connection['server_list']:
if remote_id < connection['self_id']:
try:
s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
s.connect((addr, port + remote_id))
s.send("%%s, %%s" %% (connection['self_id'], "Victory"))
except (socket.error, socket.herror, socket.gaierror, socket.timeout):
pass
finally:
s.close()
rename_broken_and_stop()
def failure_detect():
global wait
connection = loadConnectionInfos()
while True:
time.sleep(30)
if wait:
print 'waiting 30 minutes\n'
time.sleep(30 * 60)
wait = False
if not connection['server_list'][0]:
continue
(remote_id, addr) = connection['server_list'][0]
try:
s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
s.connect((addr, port + remote_id))
s.close()
except (socket.error, socket.herror, socket.gaierror, socket.timeout):
s.close()
election()
failure_detect_thread = threading.Thread(target=failure_detect)
failure_detect_thread.start()
connection = loadConnectionInfos()
s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
s.bind((connection['self_ip'], port + connection['self_id']))
s.listen(5)
#election()
while True:
force_election = False
client, _ = s.accept()
client_message = client.recv(1024)
if client_message:
client_id, message = client_message.split(', ')
client_id = eval(client_id)
if message == "Victory":
wait = True
print "%%s wins" %% client_id
elif message == "Election":
print "%%s starts an election" %% client_id
if client_id < connection['self_id']:
client.send("%%s, %%s" %% (connection['self_id'], "Alive"))
force_election = True
client.close()
if force_election:
election()
#!%(executable)s
import logging
import os
import socket
import sys
import thread
import time
sys.path[:] = %(syspath)s
from slapos import slap as slapmodule
import slapos
BASE_PORT = 50000
SLEEPING_MINS = 2
log = logging.getLogger(__name__)
logging.basicConfig(level=logging.DEBUG)
class Renamer(object):
def __init__(self, server_url, key_file, cert_file, computer_guid,
partition_id, software_release, namebase):
self.server_url = server_url
self.key_file = key_file
self.cert_file = cert_file
self.computer_guid = computer_guid
self.partition_id = partition_id
self.software_release = software_release
self.namebase = namebase
def _failover(self):
slap = slapmodule.slap()
slap.initializeConnection(self.server_url,
self.key_file,
self.cert_file)
computer_partition = slap.registerComputerPartition(computer_guid=self.computer_guid,
partition_id=self.partition_id)
broken = computer_partition.request(software_release=self.software_release,
software_type='frozen',
partition_reference=self.namebase+'0')
broken.rename('broken-{}'.format(time.strftime("%%d-%%b_%%H:%%M:%%S", time.gmtime())))
broken.stopped()
computer_partition.rename(self.namebase+'0')
def failover(self):
try:
log.info('renaming done')
except slapos.slap.slap.ServerError:
log.info('Internal server error')
## Leader is always number 0
class ResilientInstance(object):
def __init__(self, comm, renamer, confpath):
self.comm = comm
self.id = 0
self.state = 'normal'
self.halter = 0
self.inElection = False
self.alive = True
self.lastPing = time.clock()
self.mainCanal = self.comm.canal(['ping', 'halt', 'victory'])
self.renamer = renamer
self.okCanal = self.comm.canal(['ok'])
self.confpath = confpath
self.loadConnectionInfos()
def loadConnectionInfos(self):
file = open(self.confpath, 'r')
params = file.read().split('\n')
file.close()
self.nbComp = len([x.strip("' ") for x in params[0].strip('[],').split(',')])
new_id = int(params[1])
if self.id != new_id:
self.halter = new_id
self.id = new_id
## Needs to be changed to use the master
def aliveManagement(self):
while self.alive:
log.info('XXX sleeping for %%d minutes' %% SLEEPING_MINS)
time.sleep(SLEEPING_MINS*60)
if self.id == 0:
continue
self.comm.send('ping', 0)
message, sender = self.okCanal.get()
if message:
continue
self.election()
def listen(self):
while self.alive:
self.comm.recv()
def main(self):
while self.alive:
message, sender = self.mainCanal.get()
if message == 'ping':
self.comm.send('ok', sender)
elif message == 'halt':
self.state = 'waitingConfirm'
self.halter = sender
self.comm.send('ok', sender)
elif message == 'victory':
if int(sender) == int(self.halter) and self.state == 'waitingConfirm':
log.info('{} thinks {} is the leader'.format(self.id, sender))
self.comm.send('ok', sender)
self.state = 'normal'
def election(self):
self.inElection = True
self.loadConnectionInfos()
#Check if I'm the highest instance alive
for higher in range(self.id + 1, self.nbComp):
self.comm.send('ping', higher)
message, sender = self.okCanal.get()
if message:
log.info('{} is alive ({})'.format(higher, self.id))
self.inElection = False
return False
continue
if not self.alive:
return False
#I should be the new coordinator, halt those below me
log.info('Should be ME : {}'.format(self.id))
self.state = 'election'
self.halter = self.id
ups = []
for lower in range(self.id):
self.comm.send('halt', lower)
message, sender = self.okCanal.get()
if message:
ups.append(lower)
#Broadcast Victory
self.state = 'reorganization'
for up in ups:
self.comm.send('victory', up)
message, sender = self.okCanal.get()
if message:
continue
log.info('Something is wrong... let\'s start over')
return self.election()
self.state = 'normal'
self.active = True
log.info('{} Is THE LEADER'.format(self.id))
self.renamer.failover()
self.inElection = False
return True
class FilteredCanal(object):
def __init__(self, accept, timeout):
self.accept = accept
self.list = []
self.lock = thread.allocate_lock()
self.timeout = timeout
def append(self, message, sender):
if message in self.accept:
self.lock.acquire()
self.list.append([message, sender])
self.lock.release()
def get(self):
start = time.clock()
while (time.clock() - start < self.timeout):
self.lock.acquire()
if self.list:
self.lock.release()
return self.list.pop(0)
self.lock.release()
return [None, None]
class Wrapper(object):
def __init__(self, confpath, timeout=20):
self.canals = []
self.ips = []
self.id = 0
self.timeout = timeout
self.confpath = confpath
self.getConnectionInfos()
self.socket = None
def getConnectionInfos(self):
file = open(self.confpath, 'r')
params = file.read().split('\n')
file.close()
self.ips = [x.strip("' ") for x in params[0].strip('[],').split(',')]
self.id = int(params[1])
def start(self):
self.getConnectionInfos()
self.socket = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
self.socket.bind((self.ips[self.id], BASE_PORT + self.id))
self.socket.listen(5)
def send(self, message, number):
self.getConnectionInfos()
try:
s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
s.connect((self.ips[number], BASE_PORT + number))
s.send(message + (' {}\n'.format(self.id)))
except (socket.error, socket.herror, socket.gaierror, socket.timeout):
pass
finally:
s.close()
def canal(self, accept):
created = FilteredCanal(accept, self.timeout)
self.canals.append(created)
return created
def recv(self):
client, _ = self.socket.accept()
client_message = client.recv(1024)
if client_message:
message, sender = client_message.split()
for canal in self.canals:
canal.append(message, int(sender))
def main():
renamer = Renamer(server_url = '%(server_url)s',
key_file = '%(key_file)s',
cert_file = '%(cert_file)s',
computer_guid = '%(computer_id)s',
partition_id = '%(partition_id)s',
software_release = '%(software)s',
namebase = '%(namebase)s')
confpath = '%(confpath)s'
wrapper = Wrapper(confpath=confpath, timeout=20)
computer = ResilientInstance(wrapper, renamer=renamer, confpath=confpath)
#idle waiting for connection infos
while computer.nbComp < 2 :
computer.loadConnectionInfos()
time.sleep(30)
log.info('Starting')
computer.comm.start()
thread.start_new_thread(computer.listen, ())
thread.start_new_thread(computer.main, ())
thread.start_new_thread(computer.aliveManagement, ())
while True:
# XXX tight loop
continue
if __name__ == '__main__':
main()
...@@ -112,6 +112,8 @@ class Recipe(GenericBaseRecipe): ...@@ -112,6 +112,8 @@ class Recipe(GenericBaseRecipe):
dropbear_cmd.extend(['-p', binding_address]) dropbear_cmd.extend(['-p', binding_address])
# Single user mode # Single user mode
dropbear_cmd.append('-n') dropbear_cmd.append('-n')
# Keep connection alive for 5 minutes
dropbear_cmd.extend(['-K', '300'])
if 'dss-keyfile' in self.options: if 'dss-keyfile' in self.options:
dropbear_cmd.extend(['-d', self.options['dss-keyfile']]) dropbear_cmd.extend(['-d', self.options['dss-keyfile']])
......
...@@ -155,7 +155,7 @@ class GenericBaseRecipe(object): ...@@ -155,7 +155,7 @@ class GenericBaseRecipe(object):
pidfile=%s pidfile=%s
if [ -e $pidfile ]; then if [ -e $pidfile ]; then
pid=$(cat $pidfile) pid=$(cat $pidfile)
if [ ! -z $(ps -p "$pid" | grep $(basename %s)) ]; then if [ ! -z "$(ps -p $pid | grep $(basename %s))" ]; then
echo "Already running with pid $pid." echo "Already running with pid $pid."
exit 1 exit 1
else else
......
...@@ -86,6 +86,7 @@ class Notify(GenericBaseRecipe): ...@@ -86,6 +86,7 @@ class Notify(GenericBaseRecipe):
command=notifier_binary, command=notifier_binary,
parameters=parameters, parameters=parameters,
pidfile=pidfile, pidfile=pidfile,
parameters_extra=True,
comments=[ comments=[
'', '',
'Call an executable and send notification(s).', 'Call an executable and send notification(s).',
......
...@@ -36,6 +36,7 @@ from slapos.recipe.librecipe import GenericSlapRecipe ...@@ -36,6 +36,7 @@ from slapos.recipe.librecipe import GenericSlapRecipe
from slapos.recipe.dropbear import KnownHostsFile from slapos.recipe.dropbear import KnownHostsFile
from slapos.recipe.notifier import Notify from slapos.recipe.notifier import Notify
from slapos.recipe.notifier import Callback from slapos.recipe.notifier import Callback
from slapos.recipe.librecipe import shlex
def promise(args): def promise(args):
...@@ -62,6 +63,161 @@ def promise(args): ...@@ -62,6 +63,161 @@ def promise(args):
class Recipe(GenericSlapRecipe, Notify, Callback): class Recipe(GenericSlapRecipe, Notify, Callback):
def wrapper_push(self, remote_schema, local_dir, remote_dir, rdiff_wrapper_path):
# Create a simple rdiff-backup wrapper that will push
template = textwrap.dedent("""\
#!/bin/sh
#
# Push data to a PBS *-import instance.
#
LC_ALL=C
export LC_ALL
RDIFF_BACKUP=%(rdiffbackup_binary)s
until $RDIFF_BACKUP \\
--remote-schema %(remote_schema)s \\
--restore-as-of now \\
--force \\
%(local_dir)s \\
%(remote_dir)s; do
echo "repeating rdiff-backup..."
sleep 10
done
""")
template_dict = {
'rdiffbackup_binary': shlex.quote(self.options['rdiffbackup-binary']),
'remote_schema': shlex.quote(remote_schema),
'remote_dir': shlex.quote(remote_dir),
'local_dir': shlex.quote(local_dir)
}
return self.createFile(
name=rdiff_wrapper_path,
content=template % template_dict,
mode=0o700
)
def wrapper_pull(self, remote_schema, local_dir, remote_dir, rdiff_wrapper_path, remove_backup_older_than):
# Wrap rdiff-backup call into a script that checks consistency of backup
# We need to manually escape the remote schema
template = textwrap.dedent("""\
#!/bin/sh
#
# Pull data from a PBS *-export instance.
#
sigint()
{
exit 1
}
trap sigint SIGINT # we can CTRL-C for ease of debugging
LC_ALL=C
export LC_ALL
is_first_backup=$(test -d %(rdiff_backup_data)s || echo yes)
RDIFF_BACKUP=%(rdiffbackup_binary)s
TMPDIR=%(tmpdir)s
BACKUP_DIR=%(local_dir)s
CORRUPTED_MSG="^Warning:\ Computed\ SHA1\ digest\ of\ "
CANTFIND_MSG="^Warning:\ Cannot\ find\ SHA1\ digest\ for\ file\ "
CORRUPTED_FILE=$TMPDIR/$$.rdiff_corrupted
CANTFIND_FILE=$TMPDIR/$$.rdiff_cantfind
SUCCEEDED=false
while ! $SUCCEEDED; do
# not using --fix-corrupted can lead to an infinite loop
# in case of manual changes to the backup repository.
CORRUPTED_ARGS=""
if [ "$1" = "--fix-corrupted" ]; then
VERIFY=$($RDIFF_BACKUP --verify $BACKUP_DIR 2>&1 >/dev/null)
echo "$VERIFY" | egrep "$CORRUPTED_MSG" | sed "s/$CORRUPTED_MSG//g" > $CORRUPTED_FILE
# Sometimes --verify reports this spurious warning:
echo "$VERIFY" | egrep "$CANTFIND_MSG" | sed "s/$CANTFIND_MSG\(.*\),/--always-snapshot\ '\\1'/g" > $CANTFIND_FILE
# There can be too many files, better not to provide them through separate command line parameters
CORRUPTED_ARGS="--always-snapshot-fromfile $CORRUPTED_FILE --always-snapshot-fromfile $CANTFIND_FILE"
if [ -s "$CORRUPTED_FILE" -o -s "$CANTFIND_FILE" ]; then
echo Retransmitting $(cat "$CORRUPTED_FILE" "$CANTFIND_FILE" | wc -l) corrupted/missing files
else
echo "No corrupted or missing files to retransmit"
fi
fi
$RDIFF_BACKUP \\
$CORRUPTED_ARGS \\
--remote-schema %(remote_schema)s \\
%(remote_dir)s \\
$BACKUP_DIR
[ "$CORRUPTED_ARGS" ] && rm -f "$CORRUPTED_FILE" "$CANTFIND_FILE"
if [ ! $? -eq 0 ]; then
# Check the backup, go to the last consistent backup, so that next
# run will be okay.
echo "Checking backup directory..."
$RDIFF_BACKUP --check-destination-dir $BACKUP_DIR
if [ ! $? -eq 0 ]; then
# Here, two possiblities:
if [ is_first_backup ]; then
continue
# The first backup failed, and check-destination as well.
# we may want to remove the backup.
else
continue
# The backup command has failed, while transferring an increment, and check-destination as well.
# XXX We may need to publish the failure and ask the the equeue, re-run this script again,
# instead do a push to the clone.
fi
fi
else
# Everything's okay, cleaning up...
$RDIFF_BACKUP --remove-older-than %(remove_backup_older_than)s --force $BACKUP_DIR
fi
SUCCEEDED=true
if [ -e %(backup_signature)s ]; then
cd $BACKUP_DIR
find -type f ! -name backup.signature ! -wholename "./rdiff-backup-data/*" -print0 | xargs -P4 -0 sha256sum | LC_ALL=C sort -k 66 > ../proof.signature
cmp backup.signature ../proof.signature || SUCCEEDED=false
diff -ruw backup.signature ../proof.signature > ../backup.diff
# XXX If there is a difference on the backup, we should publish the
# failure and ask the equeue, re-run this script again,
# instead do a push it to the clone.
fi
$SUCCEEDED || find $BACKUP_DIR -name rdiff-backup.tmp.* -exec rm -rf {} \;
done
""")
template_dict = {
'rdiffbackup_binary': shlex.quote(self.options['rdiffbackup-binary']),
'rdiff_backup_data': shlex.quote(os.path.join(local_dir, 'rdiff-backup-data')),
'backup_signature': shlex.quote(os.path.join(local_dir, 'backup.signature')),
'remote_schema': shlex.quote(remote_schema),
'remote_dir': shlex.quote(remote_dir),
'local_dir': shlex.quote(local_dir),
'tmpdir': '/tmp',
'remove_backup_older_than': shlex.quote(remove_backup_older_than)
}
return self.createFile(
name=rdiff_wrapper_path,
content=template % template_dict,
mode=0o700
)
def add_slave(self, entry, known_hosts_file): def add_slave(self, entry, known_hosts_file):
path_list = [] path_list = []
...@@ -88,86 +244,36 @@ class Recipe(GenericSlapRecipe, Notify, Callback): ...@@ -88,86 +244,36 @@ class Recipe(GenericSlapRecipe, Notify, Callback):
promise_dict) promise_dict)
path_list.append(promise) path_list.append(promise)
host = parsed_url.hostname known_hosts_file[parsed_url.hostname] = entry['server-key']
known_hosts_file[host] = entry['server-key']
notifier_wrapper_path = os.path.join(self.options['wrappers-directory'], slave_id) notifier_wrapper_path = os.path.join(self.options['wrappers-directory'], slave_id)
rdiff_wrapper_path = notifier_wrapper_path + '_raw' rdiff_wrapper_path = notifier_wrapper_path + '_raw'
# Create the rdiff-backup wrapper # Create the rdiff-backup wrapper
# It is useful to separate it from the notifier so that we can run it # It is useful to separate it from the notifier so that we can run it manually.
# Manually.
rdiffbackup_parameter_list = []
# XXX use -y because the host might not yet be in the # XXX use -y because the host might not yet be in the
# trusted hosts file until the next time slapgrid is run. # trusted hosts file until the next time slapgrid is run.
rdiffbackup_remote_schema = '%(ssh)s -y -p %%s %(user)s@%(host)s' % { remote_schema = '{ssh} -y -K 300 -p %s {username}@{hostname}'.format(
'ssh': self.options['sshclient-binary'], ssh=self.options['sshclient-binary'],
'user': parsed_url.username, username=parsed_url.username,
'host': parsed_url.hostname, hostname=parsed_url.hostname
} )
remote_directory = '%(port)s::%(path)s' % {'port': parsed_url.port, remote_dir = '{port}::{path}'.format(port=parsed_url.port, path=parsed_url.path)
'path': parsed_url.path} local_dir = self.createDirectory(self.options['directory'], entry['name'])
local_directory = self.createDirectory(self.options['directory'], entry['name'])
if slave_type == 'push': if slave_type == 'push':
# Create a simple rdiff-backup wrapper that will push rdiff_wrapper = self.wrapper_push(remote_schema,
rdiffbackup_parameter_list.extend(['--remote-schema', rdiffbackup_remote_schema]) local_dir,
rdiffbackup_parameter_list.extend(['--restore-as-of', 'now']) remote_dir,
rdiffbackup_parameter_list.append('--force') rdiff_wrapper_path)
rdiffbackup_parameter_list.append(local_directory)
rdiffbackup_parameter_list.append(remote_directory)
comments = ['', 'Push data to a PBS *-import instance.', '']
rdiff_wrapper = self.createWrapper(
name=rdiff_wrapper_path,
command=self.options['rdiffbackup-binary'],
parameters=rdiffbackup_parameter_list,
comments=comments,
pidfile=os.path.join(self.options['run-directory'], '%s_raw.pid' % slave_id),
)
elif slave_type == 'pull': elif slave_type == 'pull':
# Wrap rdiff-backup call into a script that checks consistency of backup # XXX: only 3 increments is not enough by default.
# We need to manually escape the remote schema rdiff_wrapper = self.wrapper_pull(remote_schema,
rdiffbackup_parameter_list.extend(['--remote-schema', '"%s"' % rdiffbackup_remote_schema]) local_dir,
rdiffbackup_parameter_list.append(remote_directory) remote_dir,
rdiffbackup_parameter_list.append(local_directory) rdiff_wrapper_path,
comments = ['', 'Pull data from a PBS *-export instance.', ''] entry.get('remove-backup-older-than', '3B'))
rdiff_wrapper_template = textwrap.dedent("""\
#!/bin/sh
# %(comment)s
RDIFF_BACKUP="%(rdiffbackup_binary)s"
$RDIFF_BACKUP %(rdiffbackup_parameter)s
if [ ! $? -eq 0 ]; then
# Check the backup, go to the last consistent backup, so that next
# run will be okay.
echo "Checking backup directory..."
$RDIFF_BACKUP --check-destination-dir %(local_directory)s
if [ ! $? -eq 0 ]; then
# Here, two possiblities:
# * The first backup failed. It is safe to remove it since there is nothing valuable there.
# * The backup has been complete, but is now in a really weird state. Not safe to remove it.
echo "Impossible to check backup: we move it to a safe place."
# XXX: bang
mv %(local_directory)s %(local_directory)s.$(date +%%s)
fi
else
# Everything's okay, cleaning up...
$RDIFF_BACKUP --remove-older-than %(remove_backup_older_than)s --force %(local_directory)s
fi
""")
rdiff_wrapper_content = rdiff_wrapper_template % {
'comment': comments,
'rdiffbackup_binary': self.options['rdiffbackup-binary'],
'local_directory': local_directory,
'rdiffbackup_parameter': ' \\\n '.join(rdiffbackup_parameter_list),
# XXX: only 10 increments is not enough by default.
'remove_backup_older_than': entry.get('remove-backup-older-than', '3B')
}
rdiff_wrapper = self.createFile(
name=rdiff_wrapper_path,
content=rdiff_wrapper_content,
mode=0700
)
path_list.append(rdiff_wrapper) path_list.append(rdiff_wrapper)
......
...@@ -25,6 +25,7 @@ ...@@ -25,6 +25,7 @@
# #
############################################################################## ##############################################################################
import logging import logging
from zc.buildout import UserError
from slapos.recipe.librecipe import wrap, JSON_SERIALISED_MAGIC_KEY from slapos.recipe.librecipe import wrap, JSON_SERIALISED_MAGIC_KEY
import json import json
from slapos import slap as slapmodule from slapos import slap as slapmodule
...@@ -33,12 +34,6 @@ import traceback ...@@ -33,12 +34,6 @@ import traceback
DEFAULT_SOFTWARE_TYPE = 'RootSoftwareInstance' DEFAULT_SOFTWARE_TYPE = 'RootSoftwareInstance'
def getListOption(option_dict, key, default=()):
result = option_dict.get(key, default)
if isinstance(result, basestring):
result = result.split()
return result
class Recipe(object): class Recipe(object):
""" """
Request a partition to a slap master. Request a partition to a slap master.
...@@ -104,19 +99,20 @@ class Recipe(object): ...@@ -104,19 +99,20 @@ class Recipe(object):
self.logger = logging.getLogger(name) self.logger = logging.getLogger(name)
software_url = options['software-url'] software_url = options['software-url']
name = options['name'] name = options['name']
return_parameters = getListOption(options, 'return') return_parameters = options.get('return', '').split()
if not return_parameters: if not return_parameters:
self.logger.debug("No parameter to return to main instance." self.logger.debug("No parameter to return to main instance."
"Be careful about that...") "Be careful about that...")
software_type = options.get('software-type', DEFAULT_SOFTWARE_TYPE) software_type = options.get('software-type', DEFAULT_SOFTWARE_TYPE)
filter_kw = dict( if 'config' in options or 'sla' in options:
(x, options['sla-' + x]) for x in getListOption(options, 'sla') raise UserError("'config' & 'sla' options are obsolete."
if options['sla-' + x] " Clean up your software release.")
) filter_kw = {k: v
partition_parameter_kw = self._filterForStorage(dict( for k, v in options.iteritems()
(x, options['config-' + x]) if k.startswith('sla-') and v}
for x in getListOption(options, 'config') partition_parameter_kw = self._filterForStorage({k: v
)) for k, v in options.iteritems()
if k.startswith('config-')})
slave = options.get('slave', 'false').lower() in \ slave = options.get('slave', 'false').lower() in \
librecipe.GenericBaseRecipe.TRUE_VALUES librecipe.GenericBaseRecipe.TRUE_VALUES
...@@ -267,7 +263,6 @@ class RequestEdge(Recipe): ...@@ -267,7 +263,6 @@ class RequestEdge(Recipe):
# Request will have its own copy of options dict # Request will have its own copy of options dict
local_options = original_options.copy() local_options = original_options.copy()
local_options['name'] = '%s-%s' % (country, name) local_options['name'] = '%s-%s' % (country, name)
local_options['sla'] = "region"
local_options['sla-region'] = country local_options['sla-region'] = country
self.request_dict[country] = Recipe(buildout, name, local_options) self.request_dict[country] = Recipe(buildout, name, local_options)
......
import os
import shutil
import sys
import tempfile
import unittest
from slapos.recipe import pbs
class PBSTest(unittest.TestCase):
def new_recipe(self):
buildout = {
'buildout': {
'bin-directory': '',
'find-links': '',
'allow-hosts': '',
'develop-eggs-directory': '',
'eggs-directory': '',
'python': 'testpython',
},
'testpython': {
'executable': sys.executable,
},
'slap-connection': {
'computer-id': '',
'partition-id': '',
'server-url': '',
'software-release-url': '',
}
}
options = {
'rdiffbackup-binary': ''
}
return pbs.Recipe(buildout=buildout, name='pbs', options=options)
def test_push(self):
recipe = self.new_recipe()
with tempfile.NamedTemporaryFile() as rdiff_wrapper:
recipe.wrapper_push(remote_schema='TEST_REMOTE_SCHEMA',
local_dir='TEST_LOCAL_DIR',
remote_dir='TEST_REMOTE_DIR',
rdiff_wrapper_path=rdiff_wrapper.name)
content = rdiff_wrapper.read()
self.assertIn('--remote-schema TEST_REMOTE_SCHEMA', content)
self.assertIn('TEST_LOCAL_DIR', content)
self.assertIn('TEST_REMOTE_DIR', content)
def test_pull(self):
recipe = self.new_recipe()
with tempfile.NamedTemporaryFile() as rdiff_wrapper:
recipe.wrapper_pull(remote_schema='TEST_REMOTE_SCHEMA',
local_dir='TEST_LOCAL_DIR',
remote_dir='TEST_REMOTE_DIR',
rdiff_wrapper_path=rdiff_wrapper.name,
remove_backup_older_than='TEST_OLDER')
content = rdiff_wrapper.read()
self.assertIn('--remote-schema TEST_REMOTE_SCHEMA', content)
self.assertIn('TEST_LOCAL_DIR', content)
self.assertIn('TEST_REMOTE_DIR', content)
self.assertIn('--remove-older-than TEST_OLDER', content)
def test_invalid_type(self):
recipe = self.new_recipe()
entry = {
'url': 'http://url.to.something/',
'type': 'invalid'
}
self.assertRaisesRegexp(ValueError,
'type parameter must be either pull or push',
recipe.add_slave, entry=entry, known_hosts_file={})
def test_install(self):
recipe = self.new_recipe()
promises_directory = tempfile.mkdtemp()
wrappers_directory = tempfile.mkdtemp()
directory = tempfile.mkdtemp()
feeds_directory = tempfile.mkdtemp()
run_directory = tempfile.mkdtemp()
cron_directory = tempfile.mkdtemp()
recipe.options.update({
'promises-directory': promises_directory,
'wrappers-directory': wrappers_directory,
'sshclient-binary': 'TEST_SSH_CLIENT',
'directory': directory,
'notifier-binary': 'TEST_NOTIFIER',
'feeds': feeds_directory,
'notifier-url': 'http://url.to.notifier/',
'run-directory': run_directory,
'cron-entries': cron_directory,
'known-hosts': 'TEST_KNOWN_HOSTS',
'slave-instance-list': '''[
{
"url": "http://url.to.pull/",
"type": "pull",
"notification-id": "pulltest",
"server-key": "TEST_SERVER_KEY",
"name": "TEST_NAME",
"notify": "http://url.to.notify/",
"frequency": "TEST_FREQUENCY"
}, {
"url": "http://url.to.push/",
"type": "push",
"notification-id": "pushtest",
"server-key": "TEST_SERVER_KEY",
"name": "TEST_NAME",
"notify": "http://url.to.notify/",
"frequency": "TEST_FREQUENCY"
}
]
'''
})
recipe._install()
self.assertItemsEqual(os.listdir(promises_directory),
['pulltest', 'pushtest'])
self.assertItemsEqual(os.listdir(wrappers_directory),
['pulltest_raw', 'pulltest', 'pushtest_raw', 'pushtest'])
self.assertItemsEqual(os.listdir(directory),
['TEST_NAME'])
self.assertItemsEqual(os.listdir(feeds_directory),
['pulltest', 'pushtest'])
self.assertItemsEqual(os.listdir(run_directory),
[])
self.assertItemsEqual(os.listdir(cron_directory),
['pulltest', 'pushtest'])
shutil.rmtree(promises_directory)
shutil.rmtree(wrappers_directory)
shutil.rmtree(directory)
shutil.rmtree(feeds_directory)
shutil.rmtree(run_directory)
shutil.rmtree(cron_directory)
...@@ -177,7 +177,6 @@ name = MariaDB ...@@ -177,7 +177,6 @@ name = MariaDB
software-url = $${slap-connection:software-release-url} software-url = $${slap-connection:software-release-url}
software-type = mariadb software-type = mariadb
return = url return = url
sla = computer_guid
sla-computer_guid = $${slap-connection:computer-id} sla-computer_guid = $${slap-connection:computer-id}
[mariadb-urlparse] [mariadb-urlparse]
...@@ -260,7 +259,6 @@ name = Frontend-Website ...@@ -260,7 +259,6 @@ name = Frontend-Website
# XXX We have hardcoded SR URL here. # XXX We have hardcoded SR URL here.
software-url = $${slap-parameter:frontend-software-url} software-url = $${slap-parameter:frontend-software-url}
slave = true slave = true
config = url custom_domain
config-url = http://[$${apache-php:ip}]:$${apache-php:port}/ config-url = http://[$${apache-php:ip}]:$${apache-php:port}/
return = site_url return = site_url
config-custom_domain = $${slap-parameter:domain} config-custom_domain = $${slap-parameter:domain}
...@@ -272,7 +270,6 @@ name = Frontend-FileManager ...@@ -272,7 +270,6 @@ name = Frontend-FileManager
# XXX We have hardcoded SR URL here. # XXX We have hardcoded SR URL here.
software-url = $${slap-parameter:frontend-software-url} software-url = $${slap-parameter:frontend-software-url}
slave = true slave = true
config = url custom_domain
config-url = http://[$${httpd-conf:ip}]:$${httpd-conf:port2}/ config-url = http://[$${httpd-conf:ip}]:$${httpd-conf:port2}/
return = site_url return = site_url
config-custom_domain = $${slap-parameter:domain2} config-custom_domain = $${slap-parameter:domain2}
...@@ -284,7 +281,6 @@ name = Frontend-Website2 ...@@ -284,7 +281,6 @@ name = Frontend-Website2
# XXX We have hardcoded SR URL here. # XXX We have hardcoded SR URL here.
software-url = $${slap-parameter:frontend-software-url} software-url = $${slap-parameter:frontend-software-url}
slave = true slave = true
config = url custom_domain
config-url = http://[$${httpd-conf:ip}]:$${httpd-conf:port3}/ config-url = http://[$${httpd-conf:ip}]:$${httpd-conf:port3}/
return = site_url return = site_url
config-custom_domain = $${slap-parameter:domain3} config-custom_domain = $${slap-parameter:domain3}
......
...@@ -35,7 +35,7 @@ md5sum = 4c7936accb3658871b635158198b7905 ...@@ -35,7 +35,7 @@ md5sum = 4c7936accb3658871b635158198b7905
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-apachephp.cfg url = ${:_profile_base_location_}/instance-apachephp.cfg
output = ${buildout:directory}/template-apachephp.cfg output = ${buildout:directory}/template-apachephp.cfg
md5sum = ef5c61127a21c016014021477d1791e3 md5sum = 8be713bd2656e184651c8ec10579d668
mode = 0644 mode = 0644
[instance-mariadb] [instance-mariadb]
...@@ -86,7 +86,6 @@ numpy = 1.6.2 ...@@ -86,7 +86,6 @@ numpy = 1.6.2
# websockify 0.4.1 doesn't install well # websockify 0.4.1 doesn't install well
websockify = 0.3.0 websockify = 0.3.0
plone.recipe.command = 1.1 plone.recipe.command = 1.1
slapos.recipe.build = 0.11.6
slapos.recipe.download = 1.0.dev-r4053 slapos.recipe.download = 1.0.dev-r4053
slapos.recipe.template = 2.4.2 slapos.recipe.template = 2.4.2
z3c.recipe.mkdir = 0.5 z3c.recipe.mkdir = 0.5
Apache: Apache:
======= =======
* (Minor Bug) Check if slave in slave_list before publishing information on it
...@@ -74,7 +74,7 @@ mode = 0644 ...@@ -74,7 +74,7 @@ mode = 0644
[template-apache-replicate] [template-apache-replicate]
recipe = slapos.recipe.build:download recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/instance-apache-replicate.cfg.in url = ${:_profile_base_location_}/instance-apache-replicate.cfg.in
md5sum = 140b3be4ce46aa18b33aea5d379d90e9 md5sum = aafa98333cd51025938b33c2516f0b8d
mode = 0644 mode = 0644
[template-slave-list] [template-slave-list]
......
...@@ -86,7 +86,6 @@ software-url = ${slap-connection:software-release-url} ...@@ -86,7 +86,6 @@ software-url = ${slap-connection:software-release-url}
{% endif %} {% endif %}
software-type = {{frontend_type}} software-type = {{frontend_type}}
return = private-ipv4 public-ipv4 slave-instance-information-list monitor_url return = private-ipv4 public-ipv4 slave-instance-information-list monitor_url
config = _
connection-monitor_url = connection-monitor_url =
{% for section, frontend_request in request_dict.iteritems() %} {% for section, frontend_request in request_dict.iteritems() %}
...@@ -102,7 +101,6 @@ state = {{ frontend_request.get('state') }} ...@@ -102,7 +101,6 @@ state = {{ frontend_request.get('state') }}
{% do slave_configuration_dict.__setitem__("frontend-name", frontend_request.get('name')) %} {% do slave_configuration_dict.__setitem__("frontend-name", frontend_request.get('name')) %}
config-_ = {{ json_module.dumps(slave_configuration_dict) }} config-_ = {{ json_module.dumps(slave_configuration_dict) }}
{% if frontend_request.get('sla') %} {% if frontend_request.get('sla') %}
sla = {{ ' '.join(frontend_request.get('sla').keys()) }}
{% for parameter, value in frontend_request.get('sla').iteritems() -%} {% for parameter, value in frontend_request.get('sla').iteritems() -%}
sla-{{ parameter }} = {{ value }} sla-{{ parameter }} = {{ value }}
{% endfor -%} {% endfor -%}
......
...@@ -13,7 +13,7 @@ ...@@ -13,7 +13,7 @@
"title": "Custom Domain", "title": "Custom Domain",
"description": "Custom Domain to use for the website", "description": "Custom Domain to use for the website",
"type": "string", "type": "string",
"pattern": "^([a-zA-Z0-9]([a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,6}$" "pattern": "^([a-zA-Z0-9]([a-zA-Z0-9\\-]{0,61}[a-zA-Z0-9])?\\.)+[a-zA-Z]{2,6}$"
}, },
"type": { "type": {
"title": "Backend Type", "title": "Backend Type",
......
...@@ -12,9 +12,8 @@ gitdb = 0.5.4 ...@@ -12,9 +12,8 @@ gitdb = 0.5.4
plone.recipe.command = 1.1 plone.recipe.command = 1.1
pycrypto = 2.6.1 pycrypto = 2.6.1
rdiff-backup = 1.0.5 rdiff-backup = 1.0.5
slapos.recipe.build = 0.12
slapos.recipe.template = 2.5 slapos.recipe.template = 2.5
slapos.toolbox = 0.40.2 slapos.toolbox = 0.40.4
smmap = 0.8.2 smmap = 0.8.2
# Required by: # Required by:
......
...@@ -120,6 +120,5 @@ mode = 0644 ...@@ -120,6 +120,5 @@ mode = 0644
rdiff-backup = 1.0.5 rdiff-backup = 1.0.5
gunicorn = 19.1.1 gunicorn = 19.1.1
plone.recipe.command = 1.1 plone.recipe.command = 1.1
slapos.recipe.build = 0.11.6
slapos.recipe.template = 2.4.2 slapos.recipe.template = 2.4.2
PyRSS2Gen = 1.1 PyRSS2Gen = 1.1
...@@ -19,7 +19,7 @@ url = cloudooo://${haproxy:ip}:${haproxy:port}/ ...@@ -19,7 +19,7 @@ url = cloudooo://${haproxy:ip}:${haproxy:port}/
recipe = slapos.cookbook:generic.cloudooo recipe = slapos.cookbook:generic.cloudooo
ip = {{ ipv4 }} ip = {{ ipv4 }}
environment = environment =
LD_LIBRARY_PATH = {{ parameter_dict['cups'] }}/lib:{{ parameter_dict['cups'] }}/lib64:{{ parameter_dict['dbus'] }}/lib:{{ parameter_dict['dbus-glib'] }}/lib:{{ parameter_dict['file'] }}/lib:{{ parameter_dict['fontconfig'] }}/lib:{{ parameter_dict['freetype'] }}/lib:{{ parameter_dict['glib'] }}/lib:{{ parameter_dict['libffi'] }}/lib:{{ parameter_dict['libffi'] }}/lib64:{{ parameter_dict['libICE'] }}/lib:{{ parameter_dict['libpng12'] }}/lib:{{ parameter_dict['libSM'] }}/lib:{{ parameter_dict['libX11'] }}/lib:{{ parameter_dict['libXau'] }}/lib:{{ parameter_dict['libXdmcp'] }}/lib:{{ parameter_dict['libXext'] }}/lib:{{ parameter_dict['libxcb'] }}/lib:{{ parameter_dict['libXrender'] }}/lib:{{ parameter_dict['zlib'] }}/lib LD_LIBRARY_PATH = {{ parameter_dict['cups'] }}/lib:{{ parameter_dict['cups'] }}/lib64:{{ parameter_dict['dbus'] }}/lib:{{ parameter_dict['dbus-glib'] }}/lib:{{ parameter_dict['file'] }}/lib:{{ parameter_dict['fontconfig'] }}/lib:{{ parameter_dict['freetype'] }}/lib:{{ parameter_dict['glib'] }}/lib:{{ parameter_dict['glu'] }}/lib:{{ parameter_dict['libICE'] }}/lib:{{ parameter_dict['libSM'] }}/lib:{{ parameter_dict['libX11'] }}/lib:{{ parameter_dict['libXau'] }}/lib:{{ parameter_dict['libXdmcp'] }}/lib:{{ parameter_dict['libXext'] }}/lib:{{ parameter_dict['libXrender'] }}/lib:{{ parameter_dict['libexpat'] }}/lib:{{ parameter_dict['libffi'] }}/lib:{{ parameter_dict['libffi'] }}/lib64:{{ parameter_dict['libpng12'] }}/lib:{{ parameter_dict['libxcb'] }}/lib:{{ parameter_dict['mesa'] }}/lib:{{ parameter_dict['xdamage'] }}/lib:{{ parameter_dict['xfixes'] }}/lib:{{ parameter_dict['zlib'] }}/lib
FONTCONFIG_FILE = ${fontconfig-instance:conf-path} FONTCONFIG_FILE = ${fontconfig-instance:conf-path}
PATH = ${binary-link:target-directory} PATH = ${binary-link:target-directory}
# Binary information # Binary information
......
...@@ -25,31 +25,36 @@ context = ...@@ -25,31 +25,36 @@ context =
${:extra-context} ${:extra-context}
[dynamic-template-cloudooo-parameters] [dynamic-template-cloudooo-parameters]
buildout-bin-directory = {{ buildout_bin_directory }}
coreutils = {{ coreutils_location }}
cups = {{ cups_location }} cups = {{ cups_location }}
dbus = {{ dbus_location }} dbus = {{ dbus_location }}
dbus-glib = {{ dbus_glib_location }} dbus-glib = {{ dbus_glib_location }}
file = {{ file_location }} file = {{ file_location }}
fontconfig = {{ fontconfig_location }} fontconfig = {{ fontconfig_location }}
fonts = {{ fonts_location }}
freetype = {{ freetype_location }} freetype = {{ freetype_location }}
glib = {{ glib_location }} glib = {{ glib_location }}
glu = {{ glu_location }}
haproxy = {{ haproxy_location }} haproxy = {{ haproxy_location }}
libffi = {{ libffi_location }} imagemagick = {{ imagemagick_location }}
libICE = {{ libICE_location }} libICE = {{ libICE_location }}
libpng12 = {{ libpng12_location }}
libSM = {{ libSM_location }} libSM = {{ libSM_location }}
libX11 = {{ libX11_location }} libX11 = {{ libX11_location }}
libXau = {{ libXau_location }} libXau = {{ libXau_location }}
libXdmcp = {{ libXdmcp_location }} libXdmcp = {{ libXdmcp_location }}
libXext = {{ libXext_location }} libXext = {{ libXext_location }}
libxcb = {{ libxcb_location }}
libXrender = {{ libXrender_location }} libXrender = {{ libXrender_location }}
zlib = {{ zlib_location }} libexpat = {{ libexpat_location }}
libffi = {{ libffi_location }}
libpng12 = {{ libpng12_location }}
libreoffice-bin = {{ libreoffice_bin_location }} libreoffice-bin = {{ libreoffice_bin_location }}
fonts = {{ fonts_location }} libxcb = {{ libxcb_location }}
coreutils = {{ coreutils_location }} mesa = {{ mesa_location }}
imagemagick = {{ imagemagick_location }}
poppler = {{ poppler_location }} poppler = {{ poppler_location }}
buildout-bin-directory = {{ buildout_bin_directory }} xdamage = {{ xdamage_location }}
xfixes = {{ xfixes_location }}
zlib = {{ zlib_location }}
[dynamic-template-cloudooo] [dynamic-template-cloudooo]
< = jinja2-template-base < = jinja2-template-base
......
...@@ -37,12 +37,13 @@ context = ...@@ -37,12 +37,13 @@ context =
# XXX: "template.cfg" is hardcoded in instanciation recipe # XXX: "template.cfg" is hardcoded in instanciation recipe
filename = template.cfg filename = template.cfg
template = ${:_profile_base_location_}/instance.cfg.in template = ${:_profile_base_location_}/instance.cfg.in
md5sum = 1c5dab821393845b118644686154710f md5sum = 2ad39c607cfa80bf87dd6258c29754c3
extra-context = extra-context =
key buildout_bin_directory buildout:bin-directory key buildout_bin_directory buildout:bin-directory
key coreutils_location coreutils:location
key cups_location cups:location key cups_location cups:location
key dbus_location dbus:location
key dbus_glib_location dbus-glib:location key dbus_glib_location dbus-glib:location
key dbus_location dbus:location
key dcron_location dcron:location key dcron_location dcron:location
key file_location file:location key file_location file:location
key fontconfig_location fontconfig:location key fontconfig_location fontconfig:location
...@@ -50,6 +51,7 @@ extra-context = ...@@ -50,6 +51,7 @@ extra-context =
key freetype_location freetype:location key freetype_location freetype:location
key git_location git:location key git_location git:location
key glib_location glib:location key glib_location glib:location
key glu_location glu:location
key haproxy_location haproxy:location key haproxy_location haproxy:location
key imagemagick_location imagemagick:location key imagemagick_location imagemagick:location
key libICE_location libICE:location key libICE_location libICE:location
...@@ -59,18 +61,25 @@ extra-context = ...@@ -59,18 +61,25 @@ extra-context =
key libXdmcp_location libXdmcp:location key libXdmcp_location libXdmcp:location
key libXext_location libXext:location key libXext_location libXext:location
key libXrender_location libXrender:location key libXrender_location libXrender:location
key libexpat_location libexpat:location
key libffi_location libffi:location key libffi_location libffi:location
key libpng12_location libpng12:location key libpng12_location libpng12:location
key libreoffice_bin_location libreoffice-bin:location key libreoffice_bin_location libreoffice-bin:location
key libxcb_location libxcb:location key libxcb_location libxcb:location
key mesa_location mesa:location
key openssl_location openssl:location key openssl_location openssl:location
key poppler_location poppler:location key poppler_location poppler:location
key template_cloudooo template-cloudooo:target key template_cloudooo template-cloudooo:target
key xdamage_location xdamage:location
key xfixes_location xfixes:location
key zlib_location zlib:location key zlib_location zlib:location
key coreutils_location coreutils:location
[template-cloudooo] [template-cloudooo]
recipe = slapos.recipe.build:download recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/instance-cloudoo.cfg.in url = ${:_profile_base_location_}/instance-cloudoo.cfg.in
md5sum = 740a313b0d9d98800b099af8bf7cd0a7 md5sum = 4bede3be20dbc2ecfdb5d49b3184742e
mode = 640 mode = 640
[versions]
# use newest version of slapos.cookbook
slapos.cookbook =
...@@ -44,6 +44,6 @@ mode = 0644 ...@@ -44,6 +44,6 @@ mode = 0644
[instance-davstorage] [instance-davstorage]
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-davstorage.cfg url = ${:_profile_base_location_}/instance-davstorage.cfg
md5sum = 699ecf4678386667f58a3391bab7af0f md5sum = b83b0351a0f46aac35d52e681270ff03
output = ${buildout:directory}/template-davstorage.cfg output = ${buildout:directory}/template-davstorage.cfg
mode = 0644 mode = 0644
...@@ -166,7 +166,6 @@ name = Frontend Ajax ...@@ -166,7 +166,6 @@ name = Frontend Ajax
# XXX We have hardcoded SR URL here. # XXX We have hardcoded SR URL here.
software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
slave = true slave = true
config = url https-only
config-https-only = true config-https-only = true
config-url = https://[$${davstorage:ip}]:$${davstorage:port_ajax}/ config-url = https://[$${davstorage:ip}]:$${davstorage:port_ajax}/
return = domain return = domain
...@@ -178,7 +177,6 @@ name = Frontend Webdav ...@@ -178,7 +177,6 @@ name = Frontend Webdav
# XXX We have hardcoded SR URL here. # XXX We have hardcoded SR URL here.
software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
slave = true slave = true
config = url https-only
config-https-only = true config-https-only = true
config-url = https://$${davstorage:user}:$${davstorage:password}@[$${davstorage:ip}]:$${davstorage:port_webdav}/ config-url = https://$${davstorage:user}:$${davstorage:password}@[$${davstorage:ip}]:$${davstorage:port_webdav}/
return = domain return = domain
......
...@@ -27,7 +27,7 @@ recipe = slapos.cookbook:erp5testnode ...@@ -27,7 +27,7 @@ recipe = slapos.cookbook:erp5testnode
slapos-directory = $${directory:slapos} slapos-directory = $${directory:slapos}
working-directory = $${directory:testnode} working-directory = $${directory:testnode}
test-suite-directory = $${directory:test-suite} test-suite-directory = $${directory:test-suite}
proxy-host = $${slap-network-information:global-ipv6} proxy-host = $${slap-network-information:local-ipv4}
proxy-port = 5000 proxy-port = 5000
log-directory = $${directory:log} log-directory = $${directory:log}
srv-directory = $${rootdirectory:srv} srv-directory = $${rootdirectory:srv}
......
...@@ -45,7 +45,12 @@ eggs = ...@@ -45,7 +45,12 @@ eggs =
zc.buildout zc.buildout
slapos.libnetworkcache slapos.libnetworkcache
slapos.core slapos.core
jsonschema
hexagonit.recipe.download
netaddr
inotifyx inotifyx
lock_file
pytz
erp5.util erp5.util
PyXML PyXML
...@@ -70,11 +75,11 @@ recipe = slapos.recipe.template ...@@ -70,11 +75,11 @@ recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-default.cfg url = ${:_profile_base_location_}/instance-default.cfg
output = ${buildout:directory}/template-default.cfg output = ${buildout:directory}/template-default.cfg
mode = 0644 mode = 0644
md5sum = b4556610d17ae769d759c49b0955b63e md5sum = 22ffc8e212dcf2db8ad94cf0e5ac4772
[versions] [versions]
PyXML = 0.8.5 PyXML = 0.8.5
erp5.util = 0.4.41 erp5.util = 0.4.41
plone.recipe.command = 1.1 plone.recipe.command = 1.1
slapos.recipe.build = 0.12
slapos.recipe.template = 2.5 slapos.recipe.template = 2.5
jsonschema = 2.4.0
...@@ -58,7 +58,7 @@ mode = 0644 ...@@ -58,7 +58,7 @@ mode = 0644
[instance-etherpad-lite] [instance-etherpad-lite]
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-etherpad-lite.cfg url = ${:_profile_base_location_}/instance-etherpad-lite.cfg
md5sum = fd7249be8988155110234c7bb877abb9 md5sum = 28710a9a23320dd472387d81a698d12d
output = ${buildout:directory}/template-etherpad-lite.cfg output = ${buildout:directory}/template-etherpad-lite.cfg
mode = 0644 mode = 0644
......
...@@ -67,7 +67,6 @@ name = Frontend ...@@ -67,7 +67,6 @@ name = Frontend
# XXX We have hardcoded SR URL here. # XXX We have hardcoded SR URL here.
software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
slave = true slave = true
config = url
config-url = http://$${etherpad-conf-generation:ip}:$${etherpad-conf-generation:port} config-url = http://$${etherpad-conf-generation:ip}:$${etherpad-conf-generation:port}
return = site_url return = site_url
......
...@@ -72,9 +72,8 @@ async = 0.6.1 ...@@ -72,9 +72,8 @@ async = 0.6.1
gitdb = 0.5.4 gitdb = 0.5.4
pycrypto = 2.6 pycrypto = 2.6
rdiff-backup = 1.0.5 rdiff-backup = 1.0.5
slapos.recipe.build = 0.11.6
slapos.recipe.template = 2.4.2 slapos.recipe.template = 2.4.2
slapos.toolbox = 0.40.2 slapos.toolbox = 0.40.4
smmap = 0.8.2 smmap = 0.8.2
z3c.recipe.scripts = 1.0.1 z3c.recipe.scripts = 1.0.1
plone.recipe.command = 1.1 plone.recipe.command = 1.1
......
...@@ -11,9 +11,12 @@ extends = ...@@ -11,9 +11,12 @@ extends =
../../component/noVNC/buildout.cfg ../../component/noVNC/buildout.cfg
../../component/openssl/buildout.cfg ../../component/openssl/buildout.cfg
../../component/dcron/buildout.cfg ../../component/dcron/buildout.cfg
../../stack/slapos.cfg
../../stack/nodejs.cfg ../../stack/nodejs.cfg
../../stack/resilient/buildout.cfg ../../stack/resilient/buildout.cfg
../../stack/slapos.cfg
# stacks are listed from most generic to most specific,
# to avoid versioning issues
parts = parts =
template template
...@@ -44,6 +47,7 @@ eggs = ...@@ -44,6 +47,7 @@ eggs =
slapos.cookbook slapos.cookbook
slapos.toolbox slapos.toolbox
erp5.util erp5.util
cns.recipe.symlink
[http-proxy] [http-proxy]
# https://github.com/nodejitsu/node-http-proxy # https://github.com/nodejitsu/node-http-proxy
...@@ -80,7 +84,7 @@ command = ...@@ -80,7 +84,7 @@ command =
[template] [template]
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg.in url = ${:_profile_base_location_}/instance.cfg.in
md5sum = bc5a986c7208d02d3284a897ea90b39d md5sum = 4c8f07da2217e54163c265fe6fe3d41d
output = ${buildout:directory}/template.cfg output = ${buildout:directory}/template.cfg
mode = 0644 mode = 0644
...@@ -88,7 +92,7 @@ mode = 0644 ...@@ -88,7 +92,7 @@ mode = 0644
recipe = hexagonit.recipe.download recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/instance-kvm.cfg.jinja2 url = ${:_profile_base_location_}/instance-kvm.cfg.jinja2
mode = 644 mode = 644
md5sum = e16c15f72fdeb92ce1854bc25daf5ad7 md5sum = ac69266206830226185e576fb6e4935a
download-only = true download-only = true
on-update = true on-update = true
...@@ -96,14 +100,14 @@ on-update = true ...@@ -96,14 +100,14 @@ on-update = true
recipe = hexagonit.recipe.download recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/instance-kvm-resilient.cfg.jinja2 url = ${:_profile_base_location_}/instance-kvm-resilient.cfg.jinja2
mode = 644 mode = 644
md5sum = a07c96b53fe9145278cd64a3b27a459a md5sum = 412d8540e054aa436bee723d85213a2b
download-only = true download-only = true
on-update = true on-update = true
[template-kvm-resilient-test] [template-kvm-resilient-test]
recipe = hexagonit.recipe.download recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/instance-kvm-resilient-test.cfg.jinja2 url = ${:_profile_base_location_}/instance-kvm-resilient-test.cfg.jinja2
md5sum = b4894680283d3912df4e9740f3e7848b md5sum = 080b5790b95a979e44c4a8e26dc02835
mode = 0644 mode = 0644
download-only = true download-only = true
on-update = true on-update = true
...@@ -111,15 +115,15 @@ on-update = true ...@@ -111,15 +115,15 @@ on-update = true
[template-kvm-import] [template-kvm-import]
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-kvm-import.cfg.in url = ${:_profile_base_location_}/instance-kvm-import.cfg.in
md5sum = 7b36d6c61154b7ec3113a1bfaa25a904 md5sum = 6835c9309ff4bf4a0efd1850e6c66b24
output = ${buildout:directory}/template-kvm-import.cfg output = ${buildout:directory}/template-kvm-import.cfg
mode = 0644 mode = 0644
[template-kvm-import-script] [template-kvm-import-script]
recipe = hexagonit.recipe.download recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/template/kvm-import.sh.in url = ${:_profile_base_location_}/template/kvm-import.sh.jinja2
filename = kvm-import.sh.in filename = kvm-import.sh.jinja2
md5sum = e03ed049cddd8d157228b09e1ebc071a md5sum = 926a11421921c29f91fae8240bbcf585
download-only = true download-only = true
mode = 0755 mode = 0755
...@@ -127,15 +131,15 @@ mode = 0755 ...@@ -127,15 +131,15 @@ mode = 0755
recipe = hexagonit.recipe.download recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/instance-kvm-export.cfg.jinja2 url = ${:_profile_base_location_}/instance-kvm-export.cfg.jinja2
mode = 644 mode = 644
md5sum = 900f416956903fa4858e67e93b5169a1 md5sum = c9f13c1f481ed08c75089aef1d3c6981
download-only = true download-only = true
on-update = true on-update = true
[template-kvm-export-script] [template-kvm-export-script]
recipe = hexagonit.recipe.download recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/template/kvm-export.sh.in url = ${:_profile_base_location_}/template/kvm-export.sh.jinja2
filename = kvm-export.sh.in filename = kvm-export.sh.jinja2
md5sum = 95fde96f35cbf90d677c44d18b60fafb md5sum = 22bd2e0c8fdb39a764a14c403a3bd752
download-only = true download-only = true
mode = 0755 mode = 0755
......
...@@ -40,4 +40,3 @@ command = ...@@ -40,4 +40,3 @@ command =
grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link && grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link &&
grep parts ${buildout:develop-eggs-directory}/slapos.toolbox.egg-link && grep parts ${buildout:develop-eggs-directory}/slapos.toolbox.egg-link &&
grep parts ${buildout:develop-eggs-directory}/erp5.util.egg-link grep parts ${buildout:develop-eggs-directory}/erp5.util.egg-link
...@@ -17,13 +17,17 @@ parts += ...@@ -17,13 +17,17 @@ parts +=
# Create the exporter executable, which is a simple shell script # Create the exporter executable, which is a simple shell script
[exporter] [exporter]
recipe = slapos.recipe.template recipe = slapos.recipe.template:jinja2
url = {{ template_kvm_export }} template = {{ template_kvm_export }}
output = ${directory:bin}/${slap-parameter:namebase}-exporter rendered = ${directory:bin}/${slap-parameter:namebase}-exporter
mode = 0755
backup-disk-path = ${directory:backup}/virtual.qcow2
# Resilient stack wants a "wrapper" parameter # Resilient stack wants a "wrapper" parameter
wrapper = ${:output} wrapper = ${:rendered}
mode = 0700
context =
section directory directory
section buildout buildout
key socket_path kvm-instance:socket-path
raw gzip_binary {{ gzip_binary }}
# Extends publish section with resilient parameters # Extends publish section with resilient parameters
[publish-connection-information] [publish-connection-information]
......
...@@ -25,13 +25,15 @@ crontabs = $${:etc}/crontabs ...@@ -25,13 +25,15 @@ crontabs = $${:etc}/crontabs
cronstamps = $${:etc}/cronstamps cronstamps = $${:etc}/cronstamps
[importer] [importer]
recipe = slapos.recipe.template recipe = slapos.recipe.template:jinja2
url = ${template-kvm-import-script:location}/${template-kvm-import-script:filename} template = ${template-kvm-import-script:location}/${template-kvm-import-script:filename}
output = $${directory:bin}/$${slap-parameter:namebase}-importer rendered = $${directory:bin}/$${slap-parameter:namebase}-importer
mode = 0755 mode = 0700
backup-disk-path = $${directory:backup}/virtual.qcow2
disk-path = $${directory:srv}/virtual.qcow2
# Resilient stack wants a "wrapper" parameter # Resilient stack wants a "wrapper" parameter
wrapper = $${:output} wrapper = $${:rendered}
context =
section directory directory
raw zcat_binary ${gzip:location}/bin/zcat
raw gzip_binary ${gzip:location}/bin/gzip
backup-disk-path = $${directory:backup}/virtual.qcow2 backup-disk-path = $${directory:backup}/virtual.qcow2
...@@ -34,6 +34,12 @@ ...@@ -34,6 +34,12 @@
"description": "Remove all the backups in PBS that are older than specified value. It should be rdiff-backup-compatible.", "description": "Remove all the backups in PBS that are older than specified value. It should be rdiff-backup-compatible.",
"type": "string", "type": "string",
"default": "3B" "default": "3B"
},
"resilient-clone-number": {
"title": "Amount of backup(s) to create",
"description": "Amount of backup(s) to create. Each backup consists of a Pull Backup Server and a clone.",
"type": "integer",
"default": 2
} }
} }
} }
......
...@@ -42,16 +42,13 @@ recipe = slapos.cookbook:request ...@@ -42,16 +42,13 @@ recipe = slapos.cookbook:request
software-url = ${slap-connection:software-release-url} software-url = ${slap-connection:software-release-url}
software-type = kvm-resilient software-type = kvm-resilient
name = Resilient KVM (Root Instance) name = Resilient KVM (Root Instance)
{% set cluster_parameter_dict = slapparameter_dict.get('cluster', {}) -%} {% for key, value in slapparameter_dict.get('cluster', {}).iteritems() -%}
config = virtual-hard-drive-url virtual-hard-drive-md5sum resiliency-backup-periodicity {{ cluster_parameter_dict.keys() | join(' ') }}
{% for key, value in cluster_parameter_dict.items() -%}
config-{{ key }} = {{ dumps(value) }} config-{{ key }} = {{ dumps(value) }}
{% endfor -%} {% endfor -%}
config-virtual-hard-drive-url = ${slap-parameter:virtual-hard-drive-url} config-virtual-hard-drive-url = ${slap-parameter:virtual-hard-drive-url}
config-virtual-hard-drive-md5sum = ${slap-parameter:virtual-hard-drive-md5sum} config-virtual-hard-drive-md5sum = ${slap-parameter:virtual-hard-drive-md5sum}
config-resiliency-backup-periodicity = */5 * * * * config-resiliency-backup-periodicity = */5 * * * *
# XXX What to do? # XXX What to do?
sla = computer_guid
sla-computer_guid = ${slap-connection:computer-id} sla-computer_guid = ${slap-connection:computer-id}
[slap-parameter] [slap-parameter]
......
...@@ -3,6 +3,8 @@ ...@@ -3,6 +3,8 @@
{% import 'parts' as parts %} {% import 'parts' as parts %}
{% import 'replicated' as replicated with context %} {% import 'replicated' as replicated with context %}
{% set backup_amount = slapparameter_dict.pop('resilient-clone-number', "1")|int + 1 -%}
[buildout] [buildout]
eggs-directory = {{ eggs_directory }} eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }} develop-eggs-directory = {{ develop_eggs_directory }}
...@@ -10,12 +12,12 @@ offline = true ...@@ -10,12 +12,12 @@ offline = true
# += because we need to take up parts (like instance-custom, slapmonitor etc) from the profile we extended # += because we need to take up parts (like instance-custom, slapmonitor etc) from the profile we extended
parts += parts +=
{{ parts.replicate("kvm", "3") }} {{ parts.replicate("kvm", backup_amount) }}
publish-connection-informations publish-connection-informations
kvm-frontend-url-promise kvm-frontend-url-promise
kvm-backend-url-promise kvm-backend-url-promise
{{ replicated.replicate("kvm", "3", "kvm-export", "kvm-import", slapparameter_dict=slapparameter_dict) }} {{ replicated.replicate("kvm", backup_amount, "kvm-export", "kvm-import", slapparameter_dict=slapparameter_dict) }}
[directory] [directory]
recipe = slapos.cookbook:mkdirectory recipe = slapos.cookbook:mkdirectory
......
...@@ -196,11 +196,9 @@ partition-id = ${slap-connection:partition-id} ...@@ -196,11 +196,9 @@ partition-id = ${slap-connection:partition-id}
name = VNC Frontend name = VNC Frontend
software-type = ${slap-parameter:frontend-software-type} software-type = ${slap-parameter:frontend-software-type}
slave = true slave = true
config = host port
config-host = ${novnc-instance:ip} config-host = ${novnc-instance:ip}
config-port = ${novnc-instance:port} config-port = ${novnc-instance:port}
return = url resource port domainname return = url resource port domainname
sla = instance_guid
sla-instance_guid = ${slap-parameter:frontend-instance-guid} sla-instance_guid = ${slap-parameter:frontend-instance-guid}
[frontend-promise] [frontend-promise]
......
...@@ -82,6 +82,7 @@ context = ...@@ -82,6 +82,7 @@ context =
raw kvm_template $${dynamic-template-kvm:rendered} raw kvm_template $${dynamic-template-kvm:rendered}
raw template_kvm_export ${template-kvm-export-script:location}/${template-kvm-export-script:filename} raw template_kvm_export ${template-kvm-export-script:location}/${template-kvm-export-script:filename}
raw pbsready_export_template ${pbsready-export:output} raw pbsready_export_template ${pbsready-export:output}
raw gzip_binary ${gzip:location}/bin/gzip
mode = 0644 mode = 0644
[dynamic-template-kvm-resilient-test] [dynamic-template-kvm-resilient-test]
......
...@@ -5,35 +5,43 @@ extends = common.cfg ...@@ -5,35 +5,43 @@ extends = common.cfg
PyRSS2Gen = 1.1 PyRSS2Gen = 1.1
apache-libcloud = 0.15.1 apache-libcloud = 0.15.1
async = 0.6.1 async = 0.6.1
cns.recipe.symlink = 0.2.3
collective.recipe.template = 1.11 collective.recipe.template = 1.11
ecdsa = 0.11 ecdsa = 0.11
erp5.util = 0.4.41 erp5.util = 0.4.41
gitdb = 0.5.4 gitdb = 0.5.4
plone.recipe.command = 1.1 plone.recipe.command = 1.1
pycrypto = 2.6.1 pycrypto = 2.6.1
rdiff-backup = 1.0.5
slapos.recipe.download = 1.0.dev-r4053 slapos.recipe.download = 1.0.dev-r4053
slapos.toolbox = 0.40.2 slapos.toolbox = 0.40.4
smmap = 0.8.2 smmap = 0.8.2
websockify = 0.6.0 websockify = 0.6.0
z3c.recipe.scripts = 1.0.1 z3c.recipe.scripts = 1.0.1
# Required by: # Required by:
# slapos.toolbox==0.40.2 # websockify==0.6.0
numpy = 1.8.2
# Required by:
# slapos.toolbox==0.40.4
GitPython = 0.3.2.RC1 GitPython = 0.3.2.RC1
# Required by: # Required by:
# slapos.toolbox==0.40.2 # slapos.toolbox==0.40.4
atomize = 0.2.0 atomize = 0.2.0
# Required by: # Required by:
# slapos.toolbox==0.40.2 # slapos.toolbox==0.40.4
feedparser = 5.1.3 feedparser = 5.1.3
# Required by:
# slapos.cookbook==0.87
jsonschema = 2.4.0
# Required by: # Required by:
# websockify==0.6.0 # websockify==0.6.0
numpy = 1.8.2 numpy = 1.9.0
# Required by: # Required by:
# slapos.toolbox==0.40.2 # slapos.toolbox==0.40.4
paramiko = 1.15.1 paramiko = 1.15.1
#!/bin/bash
# Create a backup of the disk image of the virtual machine
BACKUP_PATH=${:backup-disk-path}
QMP_CLIENT=${buildout:directory}/software_release/bin/qemu-qmp-client
if [ ! -f $DISK_PATH ]; then
echo "Nothing to backup, disk image doesn't exist yet."
exit 0;
fi
$QMP_CLIENT --socket ${kvm-instance:socket-path} --drive-backup $BACKUP_PATH
#!/bin/bash
#
# Create a backup of the disk image of the virtual machine
#
LC_ALL=C
export LC_ALL
BACKUP_DIR={{ directory['backup'] }}
BACKUP_FILE=virtual.qcow2
QMP_CLIENT={{ buildout['directory'] }}/software_release/bin/qemu-qmp-client
$QMP_CLIENT --socket {{ socket_path }} --drive-backup $BACKUP_DIR/$BACKUP_FILE
# Due to the way qmp works, the VM file cannot be compressed on the fly.
# Although the compression step is optional, the importer uses the .gz file
# if present. So, remove it if you are disabling the compression.
# The downside of compression, here, is the temporary usage of more disk space
# in the exporter node. The goal is to minimize disk usage on the PBS server.
# If you want to compress the file in-place:
# truncate -s $(gzip -c $BACKUP_FILE | dd of=$BACKUP_FILE conv=notrunc 2>&1 | sed -n '$ s/ .*$// p') $BACKUP_FILE
# but the importer script would have to be adapted.
echo "Compressing backup..."
{{ gzip_binary }} --force --rsyncable $BACKUP_DIR/$BACKUP_FILE
cd $BACKUP_DIR && find -type f ! -name backup.signature -print0 | xargs -P4 -0 sha256sum | LC_ALL=C sort -k 66 > backup.signature
#!/bin/bash
DISK_PATH=${:disk-path}
BACKUP_PATH=${:backup-disk-path}
# TODO: Use rdiff
rm $DISK_PATH
cp $BACKUP_PATH $DISK_PATH
#!/bin/bash
VM_DIR={{ directory['srv'] }}
BACKUP_DIR={{ directory['backup'] }}
VM_FILE=virtual.qcow2
LC_ALL=C
export LC_ALL
umask 077
write_backup_proof () {
cd {{ directory['backup'] }}
find -type f ! -name backup.signature ! -wholename "./rdiff-backup-data/*" -print0 | xargs -P4 -0 sha256sum | LC_ALL=C sort -k 66 > {{ directory['srv'] }}/proof.signature
diff -ruw {{ directory['backup'] }} {{ directory['srv'] }}/proof.signature > {{ directory['srv'] }}/backup.diff
}
# For now we just make the diff before
write_backup_proof
if [ -f "$BACKUP_DIR/${VM_FILE}.gz" ]; then
{{ gzip_binary }} -t "$BACKUP_DIR/${VM_FILE}.gz" || exit 10
{{ zcat_binary }} "$BACKUP_DIR/${VM_FILE}.gz" > $VM_DIR/$VM_FILE
else
rm $VM_DIR/$VM_FILE
cp $BACKUP_DIR/$VM_FILE $VM_DIR/$VM_FILE
fi
...@@ -67,5 +67,4 @@ output = ${buildout:directory}/template-recover.cfg ...@@ -67,5 +67,4 @@ output = ${buildout:directory}/template-recover.cfg
mode = 0644 mode = 0644
[versions] [versions]
slapos.recipe.build = 0.11.5
slapos.recipe.template = 2.4.2 slapos.recipe.template = 2.4.2
...@@ -71,7 +71,6 @@ name = Monitor Frontend ...@@ -71,7 +71,6 @@ name = Monitor Frontend
# XXX We have hardcoded SR URL here. # XXX We have hardcoded SR URL here.
software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
slave = true slave = true
config = url domain
config-url = $${monitor-parameters:url} config-url = $${monitor-parameters:url}
config-domain = $${slap-parameter:frontend-domain} config-domain = $${slap-parameter:frontend-domain}
return = site_url domain return = site_url domain
......
...@@ -15,7 +15,7 @@ parts = ...@@ -15,7 +15,7 @@ parts =
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg url = ${:_profile_base_location_}/instance.cfg
output = ${buildout:directory}/instance.cfg output = ${buildout:directory}/instance.cfg
md5sum = 6ad84a0061df0c00a9f41d8b302adc45 md5sum = b86685e4007296f4808c2f1f1b74a290
mode = 0644 mode = 0644
# stupify index for now # stupify index for now
......
...@@ -5,7 +5,6 @@ ...@@ -5,7 +5,6 @@
[request-common] [request-common]
recipe = slapos.cookbook:request recipe = slapos.cookbook:request
software-url = ${slap-connection:software-release-url} software-url = ${slap-connection:software-release-url}
sla = computer_guid
sla-computer_guid = ${slap-connection:computer-id} sla-computer_guid = ${slap-connection:computer-id}
server-url = ${slap-connection:server-url} server-url = ${slap-connection:server-url}
key-file = ${slap-connection:key-file} key-file = ${slap-connection:key-file}
...@@ -13,22 +12,15 @@ cert-file = ${slap-connection:cert-file} ...@@ -13,22 +12,15 @@ cert-file = ${slap-connection:cert-file}
computer-id = ${slap-connection:computer-id} computer-id = ${slap-connection:computer-id}
partition-id = ${slap-connection:partition-id} partition-id = ${slap-connection:partition-id}
config-cluster = {{ slapparameter_dict['cluster'] }} config-cluster = {{ slapparameter_dict['cluster'] }}
config-partitions = {{ slapparameter_dict.get('partitions', 12) }}
config-replicas = {{ slapparameter_dict.get('replicas', 0) }}
[request-config]
base = cluster
master = partitions replicas
node = masters
[node-base] [node-base]
< = request-common < = request-common
config = ${request-config:base} ${request-config:node}
config-masters = ${all-masters:addresses} config-masters = ${all-masters:addresses}
[mysql-storage-base] [mysql-storage-base]
< = request-common < = request-common
config = ${request-config:base} ${request-config:master} config-partitions = {{ slapparameter_dict.get('partitions', 12) }}
config-replicas = {{ slapparameter_dict.get('replicas', 0) }}
software-type = {{ mysql_storage_software_type }} software-type = {{ mysql_storage_software_type }}
[mysql-storage-request-base] [mysql-storage-request-base]
...@@ -57,7 +49,6 @@ name = {{ section_id }} ...@@ -57,7 +49,6 @@ name = {{ section_id }}
< = mysql-storage-base < = mysql-storage-base
node-base node-base
name = {{ section_id }} name = {{ section_id }}
config = ${request-config:base} ${request-config:master} ${request-config:node}
{% endfor -%} {% endfor -%}
{% for node_number in range(slapparameter_dict.get('admin-count', 1)) -%} {% for node_number in range(slapparameter_dict.get('admin-count', 1)) -%}
......
...@@ -37,7 +37,7 @@ scripts = ...@@ -37,7 +37,7 @@ scripts =
[cluster] [cluster]
recipe = slapos.recipe.build:download recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_buildout_section_name_}.cfg.in url = ${:_profile_base_location_}/${:_buildout_section_name_}.cfg.in
md5sum = 84cba584198a26289daacb3e6d199e2b md5sum = 3939e229108f96ac0d68224fc1630880
[instance-neo-admin] [instance-neo-admin]
< = cluster < = cluster
......
...@@ -43,5 +43,4 @@ mode = 0644 ...@@ -43,5 +43,4 @@ mode = 0644
[versions] [versions]
plone.recipe.command = 1.1 plone.recipe.command = 1.1
slapos.recipe.build = 0.12
slapos.recipe.template = 2.5 slapos.recipe.template = 2.5
...@@ -45,7 +45,7 @@ ...@@ -45,7 +45,7 @@
"description": "Zone to be handled by the DNS cluster", "description": "Zone to be handled by the DNS cluster",
"type": "string", "type": "string",
"default": "domain.com", "default": "domain.com",
"pattern": "^([a-zA-Z0-9]([a-zA-Z0-9\\-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,6}$" "pattern": "^([a-zA-Z0-9]([a-zA-Z0-9\\-]{0,61}[a-zA-Z0-9])?\\.)+[a-zA-Z]{2,6}$"
}, },
"server-admin": { "server-admin": {
...@@ -59,7 +59,7 @@ ...@@ -59,7 +59,7 @@
"title": "DNS domains template string", "title": "DNS domains template string",
"description": "Template used to generate DNS domain name", "description": "Template used to generate DNS domain name",
"type": "string", "type": "string",
"default": "ns%s." + zone "default": "ns%s. + zone"
} }
} }
} }
...@@ -64,12 +64,9 @@ state = {{ slapparameter_dict.pop(state_key) }} ...@@ -64,12 +64,9 @@ state = {{ slapparameter_dict.pop(state_key) }}
{% endif%} {% endif%}
config-zone = {{ zone }} config-zone = {{ zone }}
config-soa = {{ "%s,%s" % (dns_domain, server_admin) }} config-soa = {{ "%s,%s" % (dns_domain, server_admin) }}
{% if sla_parameters %}
sla = {{ ' '.join(sla_parameters) }}
{% for parameter in sla_parameters -%} {% for parameter in sla_parameters -%}
sla-{{ parameter }} = {{ slapparameter_dict.pop( sla_key + parameter ) }} sla-{{ parameter }} = {{ slapparameter_dict.pop( sla_key + parameter ) }}
{% endfor -%} {% endfor -%}
{% endif -%}
{% endfor -%} {% endfor -%}
...@@ -86,7 +83,6 @@ software-url = ${slap-connection:software-release-url} ...@@ -86,7 +83,6 @@ software-url = ${slap-connection:software-release-url}
{% endif %} {% endif %}
software-type = {{dns_type}} software-type = {{dns_type}}
return = private-ipv4 public-ipv4 slave-instance-information-list monitor_url return = private-ipv4 public-ipv4 slave-instance-information-list monitor_url
config = {{ ' '.join(slapparameter_dict.keys()) + ' zone soa server-admin ns-record ' + slave_list_name }}
config-server-admin = {{ server_admin }} config-server-admin = {{ server_admin }}
config-ns-record = {{ ns_record }} config-ns-record = {{ ns_record }}
{% for parameter, value in slapparameter_dict.iteritems() -%} {% for parameter, value in slapparameter_dict.iteritems() -%}
......
...@@ -38,7 +38,7 @@ mode = 640 ...@@ -38,7 +38,7 @@ mode = 640
[template-dns-replicate] [template-dns-replicate]
recipe = slapos.recipe.build:download recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/instance-powerdns-replicate.cfg.jinja2 url = ${:_profile_base_location_}/instance-powerdns-replicate.cfg.jinja2
md5sum = 7cfc55bfb9821aad44272153f60450ef md5sum = 83cedc8fa923c59ca900b4600cc1c52f
mode = 0644 mode = 0644
[iso-list] [iso-list]
...@@ -64,5 +64,4 @@ PyRSS2Gen = 1.1 ...@@ -64,5 +64,4 @@ PyRSS2Gen = 1.1
cns.recipe.symlink = 0.2.3 cns.recipe.symlink = 0.2.3
collective.recipe.template = 1.11 collective.recipe.template = 1.11
plone.recipe.command = 1.1 plone.recipe.command = 1.1
slapos.recipe.build = 0.12
slapos.recipe.template = 2.5 slapos.recipe.template = 2.5
...@@ -47,5 +47,4 @@ mode = 0644 ...@@ -47,5 +47,4 @@ mode = 0644
[versions] [versions]
plone.recipe.command = 1.1 plone.recipe.command = 1.1
slapos.recipe.build = 0.12
slapos.recipe.template = 2.5 slapos.recipe.template = 2.5
...@@ -6,16 +6,11 @@ ...@@ -6,16 +6,11 @@
[request-common] [request-common]
recipe = slapos.cookbook:request.serialised recipe = slapos.cookbook:request.serialised
software-url = ${slap-connection:software-release-url} software-url = ${slap-connection:software-release-url}
sla = computer_guid
server-url = ${slap-connection:server-url} server-url = ${slap-connection:server-url}
key-file = ${slap-connection:key-file} key-file = ${slap-connection:key-file}
cert-file = ${slap-connection:cert-file} cert-file = ${slap-connection:cert-file}
computer-id = ${slap-connection:computer-id} computer-id = ${slap-connection:computer-id}
partition-id = ${slap-connection:partition-id} partition-id = ${slap-connection:partition-id}
config =
use-ipv6
${:extra-config}
extra-config =
config-use-ipv6 = {{ dumps(slapparameter_dict.get('use-ipv6', False)) }} config-use-ipv6 = {{ dumps(slapparameter_dict.get('use-ipv6', False)) }}
{% macro request(name, software_type, config_key, config, ret={'url': True}) -%} {% macro request(name, software_type, config_key, config, ret={'url': True}) -%}
...@@ -32,7 +27,6 @@ return = {{ ret.keys() | join(' ') }} ...@@ -32,7 +27,6 @@ return = {{ ret.keys() | join(' ') }}
{% endif -%} {% endif -%}
{% endfor -%} {% endfor -%}
sla-computer_guid = {{ dumps(slapparameter_dict.get(config_key + '-computer-guid', computer_id)) }} sla-computer_guid = {{ dumps(slapparameter_dict.get(config_key + '-computer-guid', computer_id)) }}
extra-config = {{ ' '.join(config) }}
{% for option, value in config.items() -%} {% for option, value in config.items() -%}
config-{{ option }} = {{ dumps(value) }} config-{{ option }} = {{ dumps(value) }}
{% endfor -%} {% endfor -%}
...@@ -43,8 +37,13 @@ config-{{ option }} = {{ dumps(value) }} ...@@ -43,8 +37,13 @@ config-{{ option }} = {{ dumps(value) }}
{{ request('cloudooo', 'cloudooo', 'cloudooo', {'tcpv4-port': 2020}) }} {{ request('cloudooo', 'cloudooo', 'cloudooo', {'tcpv4-port': 2020}) }}
{{ request('mariadb', 'mariadb', 'mariadb', {'tcpv4-port': 2099}, {'database-list': True, 'test-database-list': True}) }} {{ request('mariadb', 'mariadb', 'mariadb', {'tcpv4-port': 2099}, {'database-list': True, 'test-database-list': True}) }}
{# Fail early if an unexpected value is provided -#} {# Fail early if an unexpected value is provided -#}
{% set possible_software_type_dict = {'zeo': 'zeo', 'neo': 'neo'} -%} {% set zodb_type = slapparameter_dict.get('zodb-software-type') -%}
{{ request('zodb', 'zodb-' ~ possible_software_type_dict[slapparameter_dict.get('zodb-software-type', 'zeo')], 'zodb', {'tcpv4-port': 2100, 'zodb-dict': {'root': {}}}, {'zodb-storage-type': False, 'zodb-dict': False, 'tidstorage-ip': False, 'tidstorage-port': False}) }} {% set zodb_extern = slapparameter_dict.get('zodb-extern') -%}
{% if {'zeo': 1, None: 0}[zodb_type] -%}
{{ request('zodb', 'zodb-' ~ zodb_type, 'zodb', {'tcpv4-port': 2100, 'zodb-dict': {'root': {}}}, {'zodb-storage-type': False, 'zodb-dict': False, 'tidstorage-ip': False, 'tidstorage-port': False}) }}
{% else -%}
{% do zodb_extern[0] %}
{% endif -%}
[inituser-password] [inituser-password]
{% set inituser_password = slapparameter_dict.get('inituser-password') -%} {% set inituser_password = slapparameter_dict.get('inituser-password') -%}
...@@ -67,34 +66,6 @@ recipe = slapos.cookbook:generate.password ...@@ -67,34 +66,6 @@ recipe = slapos.cookbook:generate.password
return = return =
zope-address-list zope-address-list
hosts-dict hosts-dict
extra-config =
bt5
bt5-repository-url
cloudooo-url
deadlock-debugger-password
developer-list
hosts-dict
inituser-login
inituser-password
instance-count
kumofs-url
longrequest-logger-interval
longrequest-logger-timeout
memcached-url
mysql-test-url-list
mysql-url-list
name
port-base
site-id
smtp-url
thread-amount
tidstorage-ip
tidstorage-port
timerserver-interval
timezone
webdav
zodb-dict
zodb-storage-type
config-bt5 = {{ dumps(slapparameter_dict.get('bt5', 'erp5_full_text_myisam_catalog erp5_configurator_standard erp5_configurator_maxma_demo erp5_configurator_ung erp5_configurator_run_my_doc slapos_configurator')) }} config-bt5 = {{ dumps(slapparameter_dict.get('bt5', 'erp5_full_text_myisam_catalog erp5_configurator_standard erp5_configurator_maxma_demo erp5_configurator_ung erp5_configurator_run_my_doc slapos_configurator')) }}
config-bt5-repository-url = {{ dumps(slapparameter_dict.get('bt5-repository-url', local_bt5_repository)) }} config-bt5-repository-url = {{ dumps(slapparameter_dict.get('bt5-repository-url', local_bt5_repository)) }}
config-cloudooo-url = ${request-cloudooo:connection-url} config-cloudooo-url = ${request-cloudooo:connection-url}
...@@ -109,11 +80,16 @@ config-mysql-test-url-list = ${request-mariadb:connection-test-database-list} ...@@ -109,11 +80,16 @@ config-mysql-test-url-list = ${request-mariadb:connection-test-database-list}
config-mysql-url-list = ${request-mariadb:connection-database-list} config-mysql-url-list = ${request-mariadb:connection-database-list}
config-site-id = {{ dumps(site_id) }} config-site-id = {{ dumps(site_id) }}
config-smtp-url = {{ dumps(slapparameter_dict.get('smtp-url', 'smtp://localhost:25/')) }} config-smtp-url = {{ dumps(slapparameter_dict.get('smtp-url', 'smtp://localhost:25/')) }}
config-timezone = {{ dumps(slapparameter_dict.get('timezone', 'UTC')) }}
{% if zodb_type -%}
config-tidstorage-ip = ${request-zodb:connection-tidstorage-ip} config-tidstorage-ip = ${request-zodb:connection-tidstorage-ip}
config-tidstorage-port = ${request-zodb:connection-tidstorage-port} config-tidstorage-port = ${request-zodb:connection-tidstorage-port}
config-timezone = {{ dumps(slapparameter_dict.get('timezone', 'UTC')) }}
config-zodb-dict = ${request-zodb:connection-zodb-dict} config-zodb-dict = ${request-zodb:connection-zodb-dict}
config-zodb-storage-type = ${request-zodb:connection-zodb-storage-type} config-zodb-storage-type = ${request-zodb:connection-zodb-storage-type}
{% endif -%}
{% if zodb_extern -%}
config-zodb-extern = {{ dumps(zodb_extern) }}
{% endif -%}
software-type = zope software-type = zope
{% set zope_family_dict = {} -%} {% set zope_family_dict = {} -%}
...@@ -168,19 +144,6 @@ config-url = ${request-balancer:{{ family_name }}-v6} ...@@ -168,19 +144,6 @@ config-url = ${request-balancer:{{ family_name }}-v6}
name = balancer name = balancer
software-type = balancer software-type = balancer
sla-computer_guid = {{ dumps(slapparameter_dict.get('balancer-computer-guid', computer_id)) }} sla-computer_guid = {{ dumps(slapparameter_dict.get('balancer-computer-guid', computer_id)) }}
extra-config =
tcpv4-port
haproxy-server-check-path
backend-path-dict
ssl-authentication-dict
ssl
zope-family-dict
country-code
email
state
city
company
{{ zope_address_list_id_dict.values() | join(' ') }}
return = return =
{%- for family in zope_family_dict %} {%- for family in zope_family_dict %}
{{ family }} {{ family }}
...@@ -207,7 +170,6 @@ config-company = {{ slapparameter_dict.get('company', 'Compagny') }} ...@@ -207,7 +170,6 @@ config-company = {{ slapparameter_dict.get('company', 'Compagny') }}
< = request-common < = request-common
software-url = {{ dumps(frontend_dict['software-url']) }} software-url = {{ dumps(frontend_dict['software-url']) }}
software-type = {{ dumps(frontend_dict.get('software-type', 'RootSoftwareInstance')) }} software-type = {{ dumps(frontend_dict.get('software-type', 'RootSoftwareInstance')) }}
sla = instance_guid
sla-instance_guid = {{ dumps(frontend_dict['instance-guid']) }} sla-instance_guid = {{ dumps(frontend_dict['instance-guid']) }}
slave = true slave = true
{% set config_dict = { {% set config_dict = {
...@@ -216,7 +178,6 @@ slave = true ...@@ -216,7 +178,6 @@ slave = true
{% if frontend_dict.get('domain') -%} {% if frontend_dict.get('domain') -%}
{% do config_dict.__setitem__('custom_domain', frontend_dict['domain']) -%} {% do config_dict.__setitem__('custom_domain', frontend_dict['domain']) -%}
{% endif -%} {% endif -%}
extra-config = url {{ config_dict.keys() | join(' ') }}
{% for name, value in config_dict.items() -%} {% for name, value in config_dict.items() -%}
config-{{ name }} = {{ value }} config-{{ name }} = {{ value }}
{% endfor -%} {% endfor -%}
......
...@@ -55,7 +55,7 @@ extra-paths += ...@@ -55,7 +55,7 @@ extra-paths +=
${vifib:location}/master ${vifib:location}/master
[template-erp5] [template-erp5]
md5sum = 1b1ba4a3d57e6f91eb86e36b19f75463 md5sum = 6ada1fd4af0a451516443bfb6d00b717
[template-balancer] [template-balancer]
md5sum = 818ab59ae966114735866aecef7a8563 md5sum = 818ab59ae966114735866aecef7a8563
......
...@@ -69,14 +69,6 @@ recipe = plone.recipe.command ...@@ -69,14 +69,6 @@ recipe = plone.recipe.command
command = echo "Updating setup...";cd $${erp5-util:location}; export PATH="$${slapos-test-runner:prepend-path}:$PATH"; export CPPFLAGS="$${environment:CPPFLAGS}"; export LDFLAGS="$${environment:LDFLAGS}"; export PYTHONPATH="$${environment:PYTHONPATH}"; ${python2.7:location}/bin/python setup.py test -n; ${python2.7:location}/bin/python setup.py test -n; ${python2.7:location}/bin/python setup.py test -n; ${python2.7:location}/bin/python setup.py test -n; ${python2.7:location}/bin/python setup.py test -n command = echo "Updating setup...";cd $${erp5-util:location}; export PATH="$${slapos-test-runner:prepend-path}:$PATH"; export CPPFLAGS="$${environment:CPPFLAGS}"; export LDFLAGS="$${environment:LDFLAGS}"; export PYTHONPATH="$${environment:PYTHONPATH}"; ${python2.7:location}/bin/python setup.py test -n; ${python2.7:location}/bin/python setup.py test -n; ${python2.7:location}/bin/python setup.py test -n; ${python2.7:location}/bin/python setup.py test -n; ${python2.7:location}/bin/python setup.py test -n
update-command = $${:command} update-command = $${:command}
[officejs]
<= download-source
repository = ${officejs-repository:location}
[jio]
<= download-source
repository = ${jio-repository:location}
[slapos-test-runner] [slapos-test-runner]
recipe = slapos.cookbook:egg_test recipe = slapos.cookbook:egg_test
run-test-suite = $${create-directory:bin}/runTestSuite run-test-suite = $${create-directory:bin}/runTestSuite
...@@ -92,8 +84,6 @@ test-list = ...@@ -92,8 +84,6 @@ test-list =
$${slapos.recipe.build:location} $${slapos.recipe.build:location}
$${slapos.recipe.cmmi:location} $${slapos.recipe.cmmi:location}
$${erp5-util:location} $${erp5-util:location}
$${jio:location}
$${officejs:location}
prepend-path = ${git:location}/bin:${libxslt:location}/bin:${python2.7:location}/bin prepend-path = ${git:location}/bin:${libxslt:location}/bin:${python2.7:location}/bin
environment = environment environment = environment
......
...@@ -19,8 +19,6 @@ parts = ...@@ -19,8 +19,6 @@ parts =
slapos.recipe.build-repository slapos.recipe.build-repository
slapos.recipe.cmmi-repository slapos.recipe.cmmi-repository
erp5-util-repository erp5-util-repository
officejs-repository
jio-repository
eggs eggs
phantomjs phantomjs
template template
...@@ -33,6 +31,8 @@ eggs = ...@@ -33,6 +31,8 @@ eggs =
slapos.cookbook slapos.cookbook
collective.recipe.template collective.recipe.template
plone.recipe.command plone.recipe.command
slapos.recipe.template
slapos.recipe.cmmi
entry-points = entry-points =
runTestSuite=erp5.util.testsuite:runTestSuite runTestSuite=erp5.util.testsuite:runTestSuite
scripts = scripts =
...@@ -72,18 +72,10 @@ repository = http://git.erp5.org/repos/slapos.recipe.cmmi.git ...@@ -72,18 +72,10 @@ repository = http://git.erp5.org/repos/slapos.recipe.cmmi.git
<= git-clone-repository <= git-clone-repository
repository = http://git.erp5.org/repos/erp5.git repository = http://git.erp5.org/repos/erp5.git
[officejs-repository]
<= git-clone-repository
repository = http://git.erp5.org/repos/officejs.git
[jio-repository]
<= git-clone-repository
repository = http://git.erp5.org/repos/jio.git
[template] [template]
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg url = ${:_profile_base_location_}/instance.cfg
md5sum = 94bfc33413cdfbb6e243002f6d9b6928 md5sum = 934df9a6b59fd7cc43195931b3ba4520
output = ${buildout:directory}/template.cfg output = ${buildout:directory}/template.cfg
mode = 640 mode = 640
...@@ -91,5 +83,4 @@ mode = 640 ...@@ -91,5 +83,4 @@ mode = 640
Pygments = 1.6 Pygments = 1.6
collective.recipe.template = 1.10 collective.recipe.template = 1.10
plone.recipe.command = 1.1 plone.recipe.command = 1.1
slapos.recipe.build = 0.12
slapos.recipe.template = 2.5 slapos.recipe.template = 2.5
...@@ -21,6 +21,9 @@ extends = ...@@ -21,6 +21,9 @@ extends =
../../stack/resilient/buildout.cfg ../../stack/resilient/buildout.cfg
../../stack/monitor/buildout.cfg ../../stack/monitor/buildout.cfg
# stacks are listed from most generic to most specific,
# to avoid versioning issues
parts = parts =
template template
eggs eggs
...@@ -43,14 +46,14 @@ mode = 0644 ...@@ -43,14 +46,14 @@ mode = 0644
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-runner.cfg url = ${:_profile_base_location_}/instance-runner.cfg
output = ${buildout:directory}/template-runner.cfg.in output = ${buildout:directory}/template-runner.cfg.in
md5sum = a4f3127bc82ff3a671a65a150f26c969 md5sum = 8ebc0e59db33f0105968a813a3130236
mode = 0644 mode = 0644
[template-runner-import-script] [template-runner-import-script]
recipe = hexagonit.recipe.download recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/template/runner-import.sh.jinja2 url = ${:_profile_base_location_}/template/runner-import.sh.jinja2
download-only = true download-only = true
md5sum = d64529e263d42b0b56e5fe3a976d18c9 md5sum = c0d05a26b06ce172efaad03c52ef92ca
filename = runner-import.sh.jinja2 filename = runner-import.sh.jinja2
mode = 0644 mode = 0644
...@@ -65,7 +68,7 @@ mode = 0644 ...@@ -65,7 +68,7 @@ mode = 0644
recipe = hexagonit.recipe.download recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/template/runner-export.sh.jinja2 url = ${:_profile_base_location_}/template/runner-export.sh.jinja2
download-only = true download-only = true
md5sum = e74a0cbda64182c7fdff42cc46f07a12 md5sum = 0f290b46c0b89ff84aee5c10477e07ca
filename = runner-export.sh.jinja2 filename = runner-export.sh.jinja2
mode = 0644 mode = 0644
...@@ -87,7 +90,7 @@ mode = 0644 ...@@ -87,7 +90,7 @@ mode = 0644
recipe = hexagonit.recipe.download recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/instance-resilient-test.cfg.jinja2 url = ${:_profile_base_location_}/instance-resilient-test.cfg.jinja2
download-only = true download-only = true
md5sum = 01545742e97b4b95cd5e14d5d7cb1584 md5sum = 105c7c427def7b2643b7480b2bada3a5
filename = instance-resilient-test.cfg.jinja2 filename = instance-resilient-test.cfg.jinja2
mode = 0644 mode = 0644
...@@ -110,7 +113,7 @@ mode = 0644 ...@@ -110,7 +113,7 @@ mode = 0644
[template_launcher] [template_launcher]
recipe = slapos.recipe.download recipe = slapos.recipe.download
url = ${:_profile_base_location_}/launcher.in url = ${:_profile_base_location_}/launcher.in
md5sum = c7f8b6e9ae84aa94686a9cbaaa3dd693 md5sum = 525e37ea8b2acf6209869999b15071a6
filename = launcher.in filename = launcher.in
mode = 0644 mode = 0644
location = ${buildout:parts-directory}/${:_buildout_section_name_} location = ${buildout:parts-directory}/${:_buildout_section_name_}
......
...@@ -41,9 +41,7 @@ recipe = slapos.cookbook:request ...@@ -41,9 +41,7 @@ recipe = slapos.cookbook:request
software-url = ${slap-connection:software-release-url} software-url = ${slap-connection:software-release-url}
software-type = resilient software-type = resilient
name = Resilient Instance (Root Instance) name = Resilient Instance (Root Instance)
{% set cluster_parameter_dict = slapparameter_dict.get('cluster', {}) -%} {% for key, value in slapparameter_dict.get('cluster', {}).iteritems() -%}
config = resiliency-backup-periodicity resilient-clone-number {{ cluster_parameter_dict.keys() | join(' ') }}
{% for key, value in cluster_parameter_dict.items() -%}
config-{{ key }} = {{ dumps(value) }} config-{{ key }} = {{ dumps(value) }}
{% endfor -%} {% endfor -%}
config-resiliency-backup-periodicity = */6 * * * * config-resiliency-backup-periodicity = */6 * * * *
...@@ -51,7 +49,6 @@ config-resilient-clone-number = 2 ...@@ -51,7 +49,6 @@ config-resilient-clone-number = 2
# XXX hardcoded # XXX hardcoded
#config-frontend-domain = google.com #config-frontend-domain = google.com
# XXX Hack to deploy Root Instance on the same computer as the type-test Instance # XXX Hack to deploy Root Instance on the same computer as the type-test Instance
sla = computer_guid
sla-computer_guid = ${slap-connection:computer-id} sla-computer_guid = ${slap-connection:computer-id}
return = backend_url return = backend_url
......
...@@ -4,92 +4,92 @@ ...@@ -4,92 +4,92 @@
"user-authorized-key": { "user-authorized-key": {
"title": "User Authorized Key", "title": "User Authorized Key",
"description": "SSH public key in order to connect to the SSH server of this runner instance.", "description": "SSH public key in order to connect to the SSH server of this runner instance.",
"type": "string", "type": "string"
}, },
"instance-amount": { "instance-amount": {
"title": "Partition Amount", "title": "Partition Amount",
"description": "Number of slappart to deploy inside the runner (default is 10). Needs instance to be restarted.", "description": "Number of slappart to deploy inside the runner (default is 10). Needs instance to be restarted.",
"type": "integer", "type": "integer",
"minimum": 1, "minimum": 1,
"maximum": 40, "maximum": 40
}, },
"slapos-software": { "slapos-software": {
"title": "Pre-selected Software Release", "title": "Pre-selected Software Release",
"description": "a relative path from the slapos git repo to a folder containing a software release, which will be automaticaly deployed while the runner instanciation, and only if the parameter auto-deploy is set to 'true'. For example: 'software/helloworld", "description": "a relative path from the slapos git repo to a folder containing a software release, which will be automaticaly deployed while the runner instanciation, and only if the parameter auto-deploy is set to 'true'. For example: 'software/helloworld",
"type": "string", "type": "string"
}, },
"auto-deploy": { "auto-deploy": {
"title": "Automatically Deploy Software", "title": "Automatically Deploy Software",
"description": "Authorizes the software declared with 'slapos-software' to be automatically deployed, or not. Needs instance to be restarted. (default is false)", "description": "Authorizes the software declared with 'slapos-software' to be automatically deployed, or not. Needs instance to be restarted. (default is false)",
"type": "boolean", "type": "boolean"
}, },
"slapos-repository": { "slapos-repository": {
"title": "SlapOS Git Repository URL", "title": "SlapOS Git Repository URL",
"description": "url of the default git repository that will be download by the runner while its instanciation. Will be cloned in a directory named 'slapos' (default is http://git.erp5.org/repos/slapos.git)", "description": "url of the default git repository that will be download by the runner while its instanciation. Will be cloned in a directory named 'slapos' (default is http://git.erp5.org/repos/slapos.git)",
"type": "string", "type": "string",
"format": "uri", "format": "uri",
"pattern": "^(http|https|ftp)://", "pattern": "^(http|https|ftp)://"
}, },
"slapos-reference": { "slapos-reference": {
"title": "SlapOS Git Branch Name", "title": "SlapOS Git Branch Name",
"description": "Branch or hash on which the default repository will checkout (default is master)", "description": "Branch or hash on which the default repository will checkout (default is master)",
"type": "string", "type": "string"
}, },
"auto-deploy-instance": { "auto-deploy-instance": {
"title": "Automatically Deploy Instances", "title": "Automatically Deploy Instances",
"description": "Prevent the runner from deploying and starting instances. Needs instance to be restarted. It is set to false for instances of type 'import' in resiliency in any case (default is false)", "description": "Prevent the runner from deploying and starting instances. Needs instance to be restarted. It is set to false for instances of type 'import' in resiliency in any case (default is false)",
"type": "boolean", "type": "boolean"
}, },
"autorun": { "autorun": {
"title": "Automatically Run Sofware/Instance", "title": "Automatically Run Sofware/Instance",
"description": "Let automaticaly build and run a declared software with 'slapos-software'. Only works if 'slapos-software' is set, and 'auto-deploy' is true. Needs instance to be restarted. (default is false)", "description": "Let automaticaly build and run a declared software with 'slapos-software'. Only works if 'slapos-software' is set, and 'auto-deploy' is true. Needs instance to be restarted. (default is false)",
"type": "boolean", "type": "boolean"
}, },
"slapos-software-type": { "slapos-software-type": {
"title": "Deployed Instance Software Type" , "title": "Deployed Instance Software Type" ,
"description": "Software type of your instance inside the runner", "description": "Software type of your instance inside the runner",
"type": "string", "type": "string"
}, },
"custom-frontend-backend-url": { "custom-frontend-backend-url": {
"title": "Custom Frontend Backend URL", "title": "Custom Frontend Backend URL",
"description": "return an ipv4 frontend of the given ipv6(+optional port)", "description": "return an ipv4 frontend of the given ipv6(+optional port)",
"type": "string", "type": "string",
"format": "uri", "format": "uri"
}, },
"custom-frontend-backend-type": { "custom-frontend-backend-type": {
"title": "Custom Frontend Backend Type", "title": "Custom Frontend Backend Type",
"description": "The type of the frontend slave instance to ask", "description": "The type of the frontend slave instance to ask",
"type": "string", "type": "string",
"enum": ["zope"], "enum": ["zope"]
}, },
"custom-frontend-basic-auth": { "custom-frontend-basic-auth": {
"title": "Custom Frontend Basic Auth", "title": "Custom Frontend Basic Auth",
"description": "if the ip given with 'custom-frontend-backend-url' is secure, set it to true for the promise do not fail", "description": "if the ip given with 'custom-frontend-backend-url' is secure, set it to true for the promise do not fail",
"type": "boolean", "type": "boolean"
}, },
"custom-frontend-instance-guid": { "custom-frontend-instance-guid": {
"title": "Custom Frontend Instance GUID", "title": "Custom Frontend Instance GUID",
"description": "Instance guid of the frontend you whish to use", "description": "Instance guid of the frontend you whish to use",
"type": "string", "type": "string"
}, },
"custom-frontend-software-type": { "custom-frontend-software-type": {
"title": "Custom Frontend Software Type", "title": "Custom Frontend Software Type",
"description": "SoftwareType of the frontend you request (default is RootSoftwareInstance)", "description": "SoftwareType of the frontend you request (default is RootSoftwareInstance)",
"type": "string", "type": "string",
"enum": ["RootSoftwareInstance", "default"], "enum": ["RootSoftwareInstance", "default"]
}, },
"custom-frontend-software-url": { "custom-frontend-software-url": {
"title": "Custom Frontend Software URL", "title": "Custom Frontend Software URL",
"description": "Software Url of the frontend you request (ie.: http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg)", "description": "Software Url of the frontend you request (ie.: http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg)",
"type": "string", "type": "string",
"format": "uri", "format": "uri"
}, },
"monitor-port": { "monitor-port": {
"title": "Monitor Port", "title": "Monitor Port",
"description": "Allow to manually change the port on wich the apache server running monitoring interface is listening. The default value for the webrunner is different from the default value of the standalone stack-monitor server (default 9684)", "description": "Allow to manually change the port on wich the apache server running monitoring interface is listening. The default value for the webrunner is different from the default value of the standalone stack-monitor server (default 9684)",
"type": "integer", "type": "integer",
"minimum": 9683, "minimum": 9683,
"exclusiveMinimum": true, "exclusiveMinimum": true
}, },
"cpu-usage-ratio": { "cpu-usage-ratio": {
"description": "Ratio of the CPU use for compilation, if value is set to n, compilation will use number-of-cpu/n of cpus (need instance restart)", "description": "Ratio of the CPU use for compilation, if value is set to n, compilation will use number-of-cpu/n of cpus (need instance restart)",
...@@ -100,7 +100,7 @@ ...@@ -100,7 +100,7 @@
"title": "Number of Resilient Clones", "title": "Number of Resilient Clones",
"description": "Number of clones to be deployed when a resilient runner, if missing a single clone is provided. Its value can be 1 or 2", "description": "Number of clones to be deployed when a resilient runner, if missing a single clone is provided. Its value can be 1 or 2",
"type": "integer", "type": "integer",
"enum": [1, 2], "enum": [1, 2]
} }
} }
} }
...@@ -15,7 +15,7 @@ ...@@ -15,7 +15,7 @@
"type": "string" "type": "string"
}, },
"ssh_command": { "ssh_command": {
"description": "SSH command used to access your runner in ssh when you provided a ssh public key" "description": "SSH command used to access your runner in ssh when you provided a ssh public key",
"type": "string" "type": "string"
}, },
"monitor_url": { "monitor_url": {
......
...@@ -78,16 +78,12 @@ computer-id = $${slap-connection:computer-id} ...@@ -78,16 +78,12 @@ computer-id = $${slap-connection:computer-id}
partition-id = $${slap-connection:partition-id} partition-id = $${slap-connection:partition-id}
{%- if slapparameter_dict.get('custom-frontend-instance-guid') -%} {%- if slapparameter_dict.get('custom-frontend-instance-guid') -%}
sla = instance_guid
sla-instance_guid = $${slap-parameter:frontend-instance-guid} sla-instance_guid = $${slap-parameter:frontend-instance-guid}
{% endif -%} {% endif -%}
{% set custom_frontend_backend_type = slapparameter_dict.get('custom-frontend-backend-type')%} {% set custom_frontend_backend_type = slapparameter_dict.get('custom-frontend-backend-type') -%}
{% if custom_frontend_backend_type %} {% if custom_frontend_backend_type -%}
config = url type
config-type = {{ custom_frontend_backend_type }} config-type = {{ custom_frontend_backend_type }}
{% else %}
config = url
{% endif -%} {% endif -%}
config-url = {{ slapparameter_dict.get('custom-frontend-backend-url') }} config-url = {{ slapparameter_dict.get('custom-frontend-backend-url') }}
return = site_url domain return = site_url domain
...@@ -281,6 +277,7 @@ path_log = $${directory:log}/nginx.log ...@@ -281,6 +277,7 @@ path_log = $${directory:log}/nginx.log
path_access_log = $${directory:log}/nginx.access.log path_access_log = $${directory:log}/nginx.access.log
path_error_log = $${directory:log}/nginx.error.log path_error_log = $${directory:log}/nginx.error.log
path_tmp = $${directory:tmp}/ path_tmp = $${directory:tmp}/
nginx_prefix = $${buildout:directory}
# Config files # Config files
path_nginx_conf = $${directory:etc}/nginx.conf path_nginx_conf = $${directory:etc}/nginx.conf
# Executables # Executables
...@@ -421,7 +418,6 @@ name = SlapRunner Frontend ...@@ -421,7 +418,6 @@ name = SlapRunner Frontend
# XXX We have hardcoded SR URL here. # XXX We have hardcoded SR URL here.
software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
slave = true slave = true
config = url domain
config-url = $${slaprunner:access-url} config-url = $${slaprunner:access-url}
config-domain = $${slap-parameter:frontend-domain} config-domain = $${slap-parameter:frontend-domain}
return = site_url domain return = site_url domain
...@@ -433,7 +429,6 @@ name = Monitor Frontend ...@@ -433,7 +429,6 @@ name = Monitor Frontend
# XXX We have hardcoded SR URL here. # XXX We have hardcoded SR URL here.
software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
slave = true slave = true
config = url domain
config-url = https://[$${monitor-httpd-configuration:listening-ip}]:$${monitor-parameters:port} config-url = https://[$${monitor-httpd-configuration:listening-ip}]:$${monitor-parameters:port}
config-domain = $${slap-parameter:frontend-domain} config-domain = $${slap-parameter:frontend-domain}
return = site_url domain return = site_url domain
......
...@@ -3,4 +3,6 @@ ...@@ -3,4 +3,6 @@
# BEWARE: It will be overwritten automatically # BEWARE: It will be overwritten automatically
# Run nginx # Run nginx
exec {{ param_nginx_frontend['bin_nginx'] }} -c {{ param_nginx_frontend['path_nginx_conf'] }} exec {{ param_nginx_frontend['bin_nginx'] }} \
-p {{ param_nginx_frontend['nginx_prefix'] }} \
-c {{ param_nginx_frontend['path_nginx_conf'] }}
...@@ -20,25 +20,28 @@ gitdb = 0.5.4 ...@@ -20,25 +20,28 @@ gitdb = 0.5.4
gunicorn = 19.1.1 gunicorn = 19.1.1
plone.recipe.command = 1.1 plone.recipe.command = 1.1
pycrypto = 2.6.1 pycrypto = 2.6.1
rdiff-backup = 1.0.5
slapos.recipe.build = 0.12 slapos.recipe.build = 0.12
slapos.recipe.download = 1.0.dev-r4053 slapos.recipe.download = 1.0.dev-r4053
slapos.toolbox = 0.40.2 slapos.toolbox = 0.40.4
smmap = 0.8.2 smmap = 0.8.2
z3c.recipe.scripts = 1.0.1 z3c.recipe.scripts = 1.0.1
# Required by: # Required by:
# slapos.toolbox==0.40.2 # slapos.toolbox==0.40.4
GitPython = 0.3.2.RC1 GitPython = 0.3.2.RC1
# Required by: # Required by:
# slapos.toolbox==0.40.2 # slapos.toolbox==0.40.4
atomize = 0.2.0 atomize = 0.2.0
# Required by: # Required by:
# slapos.toolbox==0.40.2 # slapos.toolbox==0.40.4
feedparser = 5.1.3 feedparser = 5.1.3
# Required by: # Required by:
# slapos.toolbox==0.40.2 # slapos.cookbook==0.87
jsonschema = 2.4.0
# Required by:
# slapos.toolbox==0.40.4
paramiko = 1.15.1 paramiko = 1.15.1
#!{{ shell_binary }} #!{{ shell_binary }}
LANG=en_US LC_ALL=C
export $LANG export LC_ALL
umask 077 umask 077
sync_element () { sync_element () {
path=$1 path=$1
...@@ -11,7 +11,7 @@ sync_element () { ...@@ -11,7 +11,7 @@ sync_element () {
do do
cd $path; cd $path;
if [ -f $element ] || [ -d $element ]; then if [ -f $element ] || [ -d $element ]; then
{{ rsync_binary }} -av --safe-links --delete $element $backup_path; {{ rsync_binary }} -rlptgov --safe-links --delete $element $backup_path;
fi fi
done done
} }
...@@ -23,3 +23,5 @@ sync_element {{ directory['etc'] }} {{ directory['backup'] }}/etc/ config.json ...@@ -23,3 +23,5 @@ sync_element {{ directory['etc'] }} {{ directory['backup'] }}/etc/ config.json
if [ -d {{ directory['backup'] }}/runner/software ]; then if [ -d {{ directory['backup'] }}/runner/software ]; then
rm {{ directory['backup'] }}/runner/software/* rm {{ directory['backup'] }}/runner/software/*
fi fi
cd {{ directory['backup'] }} && find -type f ! -name backup.signature -print0 | xargs -P4 -0 sha256sum | LC_ALL=C sort -k 66 > backup.signature
#!{{ shell_binary }} #!{{ shell_binary }}
LC_ALL=C
export LC_ALL
umask 077 umask 077
restore_element () { restore_element () {
backup_path=$1 backup_path=$1
...@@ -13,6 +15,16 @@ restore_element () { ...@@ -13,6 +15,16 @@ restore_element () {
fi fi
done done
} }
write_backup_proof () {
cd {{ directory['backup'] }}
find -type f ! -name backup.signature ! -wholename "./rdiff-backup-data/*" -print0 | xargs -P4 -0 sha256sum | LC_ALL=C sort -k 66 > {{ directory['srv'] }}/proof.signature
diff -ruw {{ directory['backup'] }} {{ directory['srv'] }}/proof.signature > {{ directory['srv'] }}/backup.diff
}
# For now we just make the diff before
write_backup_proof
restore_element {{ directory['backup'] }}/runner/ {{ directory['srv'] }}/runner instance project proxy.db restore_element {{ directory['backup'] }}/runner/ {{ directory['srv'] }}/runner instance project proxy.db
restore_element {{ directory['backup'] }}/etc/ {{ directory['etc'] }} config.json ssh restore_element {{ directory['backup'] }}/etc/ {{ directory['etc'] }} config.json ssh
cp -r {{ directory['backup'] }}/etc/.??* {{ directory['etc'] }}; cp -r {{ directory['backup'] }}/etc/.??* {{ directory['etc'] }};
......
...@@ -40,7 +40,6 @@ name = Test Root Instance - Slave ...@@ -40,7 +40,6 @@ name = Test Root Instance - Slave
software-url = $${slap-connection:software-release-url} software-url = $${slap-connection:software-release-url}
software-type = test software-type = test
slave = true slave = true
sla = instance_guid
sla-instance_guid = $${request-test-instance:instance-guid} sla-instance_guid = $${request-test-instance:instance-guid}
return = foo return = foo
......
...@@ -58,7 +58,7 @@ eggs = collective.recipe.template ...@@ -58,7 +58,7 @@ eggs = collective.recipe.template
[versions] [versions]
collective.recipe.template = 1.11 collective.recipe.template = 1.11
plone.recipe.command = 1.1 plone.recipe.command = 1.1
slapos.recipe.build = 0.12 slapos.recipe.build = 0.13
slapos.recipe.template = 2.5 slapos.recipe.template = 2.5
# Replicate slapos stack, but without shacache to not have to compile the entire world for a simple test. # Replicate slapos stack, but without shacache to not have to compile the entire world for a simple test.
......
[buildout]
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
parts = instance
[instance]
recipe = ${instance-recipe:egg}:${instance-recipe:module}
source = ${application:location}
httpd_binary = ${apache:location}/bin/httpd
mysql_binary = ${mariadb:location}/bin/mysql
mysql_install_binary = ${mariadb:location}/bin/mysql_install_db
mysql_upgrade_binary = ${mariadb:location}/bin/mysql_upgrade
mysqld_binary = ${mariadb:location}/libexec/mysqld
[buildout] [buildout]
versions = versions
parts =
template
apache-php
mariadb
eggs
instance-recipe-egg
extends = extends =
../../stack/lamp.cfg ../../../stack/lamp/buildout.cfg
[application] [application]
recipe = slapos.recipe.build:download-unpacked #url = https://forge.indepnet.net/attachments/download/1811/glpi-0.84.7.tar.gz
#md5sum = 885d7893460979197888cefafa5a75d0
url = https://forge.indepnet.net/attachments/download/1151/glpi-0.83.1.tar.gz url = https://forge.indepnet.net/attachments/download/1151/glpi-0.83.1.tar.gz
md5sum = 3f9cfb11c09c457a0f0a8fa1563b45a7 md5sum = 3f9cfb11c09c457a0f0a8fa1563b45a7
[instance-recipe]
egg = slapos.cookbook
module = lamp.simple
[template] [application-template]
# Default template for the instance. recipe = slapos.recipe.download
recipe = slapos.recipe.template url = ${:_profile_base_location_}/dummy-config.php.in
url = ${:_profile_base_location_}/instance.cfg md5sum = d41d8cd98f00b204e9800998ecf8427e
#${:_profile_base_location_}/instance.cfg download-only = True
#md5sum = Student shall put md5 of instance.cfg here filename = template.in
output = ${buildout:directory}/template.cfg
mode = 0644 mode = 0644
location = ${buildout:parts-directory}/${:_buildout_section_name_}
[instance-recipe-egg] [application-configuration]
recipe = zc.recipe.egg location = dummy-config.php
eggs =
${instance-recipe:egg}
#!/bin/bash
. environment.sh
DIR=var/gutenberg/raw-data
mkdir -p $DIR
wget -P $DIR -c http://www.gutenberg.org/cache/epub/103/pg103.txt
wget -P $DIR -c http://www.gutenberg.org/cache/epub/18857/pg18857.txt
wget -P $DIR -c http://www.gutenberg.org/cache/epub/2488/pg2488.txt
wget -P $DIR -c http://www.gutenberg.org/cache/epub/164/pg164.txt
wget -P $DIR -c http://www.gutenberg.org/cache/epub/1268/pg1268.txt
wget -P $DIR -c http://www.gutenberg.org/cache/epub/800/pg800.txt
wget -P $DIR -c http://www.gutenberg.org/cache/epub/4791/pg4791.txt
wget -P $DIR -c http://www.gutenberg.org/cache/epub/3526/pg3526.txt
wget -P $DIR -c http://www.gutenberg.org/cache/epub/2083/pg2083.txt
#!/usr/bin/env python
# http://www.michael-noll.com/tutorials/writing-an-hadoop-mapreduce-program-in-python/
import sys
# input comes from STDIN (standard input)
for line in sys.stdin:
# remove leading and trailing whitespace
line = line.strip()
# split the line into words
words = line.split()
# increase counters
for word in words:
# write the results to STDOUT (standard output);
# what we output here will be the input for the
# Reduce step, i.e. the input for reducer.py
#
# tab-delimited; the trivial word count is 1
print '%s\t%s' % (word, 1)
#!/bin/bash
# exit on error
# set -e
source environment.sh
hdfs dfs -mkdir var/gutenberg/input
RAW_DATA=var/gutenberg/raw-data
for file in `ls $RAW_DATA`; do
hdfs dfs -put $RAW_DATA/$file var/gutenberg/input
done
#!/usr/bin/env python
# http://www.michael-noll.com/tutorials/writing-an-hadoop-mapreduce-program-in-python/
from operator import itemgetter
import sys
current_word = None
current_count = 0
word = None
# input comes from STDIN
for line in sys.stdin:
# remove leading and trailing whitespace
line = line.strip()
# parse the input we got from mapper.py
word, count = line.split('\t', 1)
# convert count (currently a string) to int
try:
count = int(count)
except ValueError:
# count was not a number, so silently
# ignore/discard this line
continue
# this IF-switch only works because Hadoop sorts map output
# by key (here: word) before it is passed to the reducer
if current_word == word:
current_count += count
else:
if current_word:
# write result to STDOUT
print '%s\t%s' % (current_word, current_count)
current_count = count
current_word = word
# do not forget to output the last word if needed!
if current_word == word:
print '%s\t%s' % (current_word, current_count)
#!/bin/bash
# http://www.michael-noll.com/tutorials/writing-an-hadoop-mapreduce-program-in-python/
. environment.sh
hadoop jar software_release/parts/hadoop-streaming/*jar \
-mapper demo/gutenberg/mapper.py \
-reducer demo/gutenberg/reducer.py \
-input var/gutenberg/input/* \
-output var/gutenberg/output
[buildout]
extends =
${instance-stack:output}
parts =
sh-environment
start-daemons
deploy-tar
gutenberg-data-download
gutenberg-mapper
gutenberg-reducer
gutenberg-run
gutenberg-put-files
wikipedia-data-download
wikipedia-mapper
wikipedia-reducer
wikipedia-run
wikipedia-put-files
[directories]
demo = $${buildout:directory}/demo
gutenberg = $${:demo}/gutenberg
wikipedia = $${:demo}/wikipedia
[gutenberg-data-download]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/gutenberg/data-download.sh
#md5sum =
output = $${directories:gutenberg}/data-download.sh
mode = 0755
[gutenberg-mapper]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/gutenberg/mapper.py
#md5sum =
output = $${directories:gutenberg}/mapper.py
mode = 0755
[gutenberg-reducer]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/gutenberg/reducer.py
#md5sum =
output = $${directories:gutenberg}/reducer.py
mode = 0755
[gutenberg-run]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/gutenberg/run.sh.in
#md5sum =
output = $${directories:gutenberg}/run.sh
mode = 0755
[gutenberg-put-files]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/gutenberg/put-files.sh.in
#md5sum =
output = $${directories:gutenberg}/put-files.sh
mode = 0755
[wikipedia-data-download]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/wikipedia/data-download.sh
#md5sum =
output = $${directories:wikipedia}/data-download.sh
mode = 0755
[wikipedia-mapper]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/wikipedia/mapper.py
#md5sum =
output = $${directories:wikipedia}/mapper.py
mode = 0755
[wikipedia-reducer]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/wikipedia/reducer.py
#md5sum =
output = $${directories:wikipedia}/reducer.py
mode = 0755
[wikipedia-run]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/wikipedia/run.sh.in
#md5sum =
output = $${directories:wikipedia}/run.sh
mode = 0755
[wikipedia-put-files]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/wikipedia/put-files.sh.in
#md5sum =
output = $${directories:wikipedia}/put-files.sh
mode = 0755
[buildout]
extends =
../../../stack/hadoop/buildout.cfg
parts =
slapos-cookbook
libffi
python-cffi
eggs
java
hadoop
hadoop-streaming
instance-stack
instance
[instance]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg.in
output = ${buildout:directory}/instance.cfg
# md5sum =
mode = 0644
[versions]
collective.recipe.template = 1.11
cp.recipe.cmd = 0.5nxd001
plone.recipe.command = 1.1
slapos.recipe.template = 2.5
#!/bin/bash
. environment.sh
DIR=var/wikipedia/raw-data
mkdir -p $DIR
# http://dumps.wikimedia.org/enwiki/20140203/
# All pages, current versions only.
wget -P $DIR -c http://dumps.wikimedia.org/enwiki/20140203/enwiki-20140203-pages-meta-current1.xml-p000000010p000010000.bz2
wget -P $DIR -c http://dumps.wikimedia.org/enwiki/20140203/enwiki-20140203-pages-meta-current2.xml-p000010001p000025000.bz2
wget -P $DIR -c http://dumps.wikimedia.org/enwiki/20140203/enwiki-20140203-pages-meta-current3.xml-p000025001p000055000.bz2
wget -P $DIR -c http://dumps.wikimedia.org/enwiki/20140203/enwiki-20140203-pages-meta-current4.xml-p000055002p000104998.bz2
wget -P $DIR -c http://dumps.wikimedia.org/enwiki/20140203/enwiki-20140203-pages-meta-current5.xml-p000105001p000184999.bz2
wget -P $DIR -c http://dumps.wikimedia.org/enwiki/20140203/enwiki-20140203-pages-meta-current6.xml-p000185003p000305000.bz2
wget -P $DIR -c http://dumps.wikimedia.org/enwiki/20140203/enwiki-20140203-pages-meta-current7.xml-p000305002p000464997.bz2
wget -P $DIR -c http://dumps.wikimedia.org/enwiki/20140203/enwiki-20140203-pages-meta-current8.xml-p000465001p000665000.bz2
wget -P $DIR -c http://dumps.wikimedia.org/enwiki/20140203/enwiki-20140203-pages-meta-current9.xml-p000665001p000925000.bz2
# don't download the full dataset
# wget -P $DIR -c http://dumps.wikimedia.org/enwiki/20140203/enwiki-20140203-pages-meta-current10.xml-p000925001p001325000.bz2
# wget -P $DIR -c http://dumps.wikimedia.org/enwiki/20140203/enwiki-20140203-pages-meta-current11.xml-p001325001p001825000.bz2
# wget -P $DIR -c http://dumps.wikimedia.org/enwiki/20140203/enwiki-20140203-pages-meta-current12.xml-p001825001p002425000.bz2
# wget -P $DIR -c http://dumps.wikimedia.org/enwiki/20140203/enwiki-20140203-pages-meta-current13.xml-p002425001p003124998.bz2
# wget -P $DIR -c http://dumps.wikimedia.org/enwiki/20140203/enwiki-20140203-pages-meta-current14.xml-p003125001p003924999.bz2
# wget -P $DIR -c http://dumps.wikimedia.org/enwiki/20140203/enwiki-20140203-pages-meta-current15.xml-p003925001p004825000.bz2
# wget -P $DIR -c http://dumps.wikimedia.org/enwiki/20140203/enwiki-20140203-pages-meta-current16.xml-p004825002p006025000.bz2
# wget -P $DIR -c http://dumps.wikimedia.org/enwiki/20140203/enwiki-20140203-pages-meta-current17.xml-p006025001p007524997.bz2
# wget -P $DIR -c http://dumps.wikimedia.org/enwiki/20140203/enwiki-20140203-pages-meta-current18.xml-p007525002p009225000.bz2
# wget -P $DIR -c http://dumps.wikimedia.org/enwiki/20140203/enwiki-20140203-pages-meta-current19.xml-p009225001p011125000.bz2
# wget -P $DIR -c http://dumps.wikimedia.org/enwiki/20140203/enwiki-20140203-pages-meta-current20.xml-p011125001p013324998.bz2
# wget -P $DIR -c http://dumps.wikimedia.org/enwiki/20140203/enwiki-20140203-pages-meta-current21.xml-p013325001p015725000.bz2
# wget -P $DIR -c http://dumps.wikimedia.org/enwiki/20140203/enwiki-20140203-pages-meta-current22.xml-p015725003p018225000.bz2
# wget -P $DIR -c http://dumps.wikimedia.org/enwiki/20140203/enwiki-20140203-pages-meta-current23.xml-p018225001p020925000.bz2
# wget -P $DIR -c http://dumps.wikimedia.org/enwiki/20140203/enwiki-20140203-pages-meta-current24.xml-p020925002p023725000.bz2
# wget -P $DIR -c http://dumps.wikimedia.org/enwiki/20140203/enwiki-20140203-pages-meta-current25.xml-p023725001p026624999.bz2
# wget -P $DIR -c http://dumps.wikimedia.org/enwiki/20140203/enwiki-20140203-pages-meta-current26.xml-p026625002p029625000.bz2
# wget -P $DIR -c http://dumps.wikimedia.org/enwiki/20140203/enwiki-20140203-pages-meta-current27.xml-p029625001p041836446.bz2
#!/usr/bin/env python
import bz2
import os
import sys
import xml.sax
class WikipediaTitleHandler(xml.sax.ContentHandler):
def startElement(self, name, attrs):
self.chars = []
self.tag = name
def characters(self, content):
if self.tag == 'title':
self.chars.append(content)
def endElement(self, name):
if self.tag == 'title':
title = ''.join(self.chars)
if title.startswith('Talk:'):
return
if title.startswith('User talk:'):
return
if title.startswith('Wikipedia:'):
return
if title.startswith('Wikipedia talk:'):
return
if title.startswith('User:'):
return
print title.encode('utf8')
def process_xml(input):
parser = xml.sax.make_parser()
parser.setContentHandler(WikipediaTitleHandler())
parser.parse(input)
if __name__ == '__main__':
input = bz2.BZ2File('/dev/fd/0')
process_xml(input)
#!/bin/bash
# exit on error
# set -e
source environment.sh
hdfs dfs -mkdir var/wikipedia/input
RAW_DATA=var/wikipedia/raw-data
for file in `ls $RAW_DATA`; do
hdfs dfs -put $RAW_DATA/$file var/wikipedia/input
done
#!/usr/bin/env python
import bz2
import os
import sys
import xml.sax
class WikipediaTitleHandler(xml.sax.ContentHandler):
def startElement(self, name, attrs):
self.chars = []
self.tag = name
def characters(self, content):
if self.tag == 'title':
self.chars.append(content)
def endElement(self, name):
if self.tag == 'title':
title = ''.join(self.chars)
if title.startswith('Talk:'):
return
if title.startswith('User talk:'):
return
if title.startswith('Wikipedia:'):
return
if title.startswith('Wikipedia talk:'):
return
if title.startswith('User:'):
return
print title.encode('utf8')
def process_xml(input):
parser = xml.sax.make_parser()
parser.setContentHandler(WikipediaTitleHandler())
parser.parse(input)
if __name__ == '__main__':
input = bz2.BZ2File('/dev/fd/0')
process_xml(input)
# dirname = '/srv/slapgrid/slappart20/srv/runner/instance/slappart0/software_release/raw-data/'
# filenames = os.listdir(dirname)
# # ['enwiki-20140203-pages-meta-current1.xml-p000000010p000010000.bz2']
# for fname in filenames:
# process_xml(os.path.join(dirname, fname))
# input = bz2.BZ2File(process_xml(os.path.join(dirname, fname)))
#!/bin/bash
. environment.sh
hadoop jar software_release/parts/hadoop-streaming/*jar \
-mapper demo/wikipedia/mapper.py \
-reducer demo/wikipedia/reducer.py \
-input var/wikipedia/input/* \
-output var/wikipedia/output
...@@ -165,7 +165,6 @@ name = Frontend for $${slap-connection:computer-id} $${slap-connection:partition ...@@ -165,7 +165,6 @@ name = Frontend for $${slap-connection:computer-id} $${slap-connection:partition
# XXX We have hardcoded SR URL here. # XXX We have hardcoded SR URL here.
software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
slave = true slave = true
config = url
config-url = http://[$${instance-parameter:ipv6-random}]:5000 config-url = http://[$${instance-parameter:ipv6-random}]:5000
return = site_url return = site_url
......
...@@ -151,7 +151,6 @@ name = Frontend ...@@ -151,7 +151,6 @@ name = Frontend
# XXX We have hardcoded SR URL here. # XXX We have hardcoded SR URL here.
software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
slave = true slave = true
config = url custom_domain
config-url = http://[$${apache-php:ip}]:$${apache-php:port}/ config-url = http://[$${apache-php:ip}]:$${apache-php:port}/
return = site_url return = site_url
config-custom_domain = $${slap-parameter:domain} config-custom_domain = $${slap-parameter:domain}
......
...@@ -20,7 +20,6 @@ return = url backend-url ...@@ -20,7 +20,6 @@ return = url backend-url
country-list = $${slap-parameter:country-list} country-list = $${slap-parameter:country-list}
software-url = $${slap-connection:software-release-url} software-url = $${slap-connection:software-release-url}
software-type = default software-type = default
config = application-location
config-application-location = $${slap-parameter:application-location} config-application-location = $${slap-parameter:application-location}
return = backend-url return = backend-url
......
...@@ -13,7 +13,7 @@ md5sum = 26ac6629a83869140189a85e581e822f ...@@ -13,7 +13,7 @@ md5sum = 26ac6629a83869140189a85e581e822f
[instance-apache-php] [instance-apache-php]
url = ${:_profile_base_location_}/instance-apache-php.cfg.in url = ${:_profile_base_location_}/instance-apache-php.cfg.in
md5sum = fdc849c9f49b9d2fb043bf229d24076b md5sum = 062e77e6be6cda965cfdd0e6011df86e
[instance-edge] [instance-edge]
recipe = slapos.recipe.template recipe = slapos.recipe.template
......
[buildout] [buildout]
extends = extends =
../../component/git/buildout.cfg ../../../component/git/buildout.cfg
../../stack/lapp/buildout.cfg ../../../stack/lapp/buildout.cfg
# += since we need rdiff-backup and friends # += since we need rdiff-backup and friends
......
...@@ -75,7 +75,6 @@ name = Postgres ...@@ -75,7 +75,6 @@ name = Postgres
software-url = $${slap-connection:software-release-url} software-url = $${slap-connection:software-release-url}
software-type = postgres software-type = postgres
return = url return = url
sla = computer_guid
sla-computer_guid = $${slap-connection:computer-id} sla-computer_guid = $${slap-connection:computer-id}
[postgres-urlparse] [postgres-urlparse]
...@@ -139,7 +138,6 @@ name = Frontend ...@@ -139,7 +138,6 @@ name = Frontend
# XXX We have hardcoded SR URL here. # XXX We have hardcoded SR URL here.
software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
slave = true slave = true
config = url
config-url = $${mioga-url:direct_url} config-url = $${mioga-url:direct_url}
return = site_url return = site_url
......
...@@ -181,7 +181,6 @@ name = MariaDB ...@@ -181,7 +181,6 @@ name = MariaDB
software-url = $${slap-connection:software-release-url} software-url = $${slap-connection:software-release-url}
software-type = mariadb software-type = mariadb
return = url return = url
sla = computer_guid
sla-computer_guid = $${slap-connection:computer-id} sla-computer_guid = $${slap-connection:computer-id}
[mariadb-urlparse] [mariadb-urlparse]
...@@ -437,7 +436,6 @@ name = Frontend ...@@ -437,7 +436,6 @@ name = Frontend
# XXX We have hardcoded SR URL here. # XXX We have hardcoded SR URL here.
software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
slave = true slave = true
config = url custom_domain
config-url = http://[$${slap-network-information:global-ipv6}]:8080 config-url = http://[$${slap-network-information:global-ipv6}]:8080
return = site_url return = site_url
config-custom_domain = $${slap-parameter:domain} config-custom_domain = $${slap-parameter:domain}
......
...@@ -84,7 +84,7 @@ md5sum = bc6ed91a1862a10af661713aa0691848 ...@@ -84,7 +84,7 @@ md5sum = bc6ed91a1862a10af661713aa0691848
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-trac.cfg url = ${:_profile_base_location_}/instance-trac.cfg
output = ${buildout:directory}/template-trac.cfg output = ${buildout:directory}/template-trac.cfg
md5sum = 1546a60e086b4d083d58ad4fca6dcca8 md5sum = 2dc7ebfa9b7d5117274b74bafed4cfd4
mode = 0644 mode = 0644
[instance-mariadb] [instance-mariadb]
......
...@@ -196,11 +196,9 @@ partition-id = $${slap-connection:partition-id} ...@@ -196,11 +196,9 @@ partition-id = $${slap-connection:partition-id}
name = VNC Frontend name = VNC Frontend
software-type = $${slap-parameter:frontend-software-type} software-type = $${slap-parameter:frontend-software-type}
slave = true slave = true
config = host port
config-host = $${novnc-instance:ip} config-host = $${novnc-instance:ip}
config-port = $${novnc-instance:port} config-port = $${novnc-instance:port}
return = url resource port domainname return = url resource port domainname
sla = instance_guid
sla-instance_guid = $${slap-parameter:frontend-instance-guid} sla-instance_guid = $${slap-parameter:frontend-instance-guid}
[request-web-frontend] [request-web-frontend]
...@@ -210,7 +208,6 @@ name = Web Frontend ...@@ -210,7 +208,6 @@ name = Web Frontend
# XXX We have hardcoded SR URL here. # XXX We have hardcoded SR URL here.
software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
slave = true slave = true
config = url custom_domain
config-url = https://[$${tunnel-ipv6-kvm-https:ipv6}]:$${tunnel-ipv6-kvm-https:ipv6-port}/ config-url = https://[$${tunnel-ipv6-kvm-https:ipv6}]:$${tunnel-ipv6-kvm-https:ipv6-port}/
return = site_url return = site_url
config-custom_domain = $${slap-parameter:domain} config-custom_domain = $${slap-parameter:domain}
...@@ -223,7 +220,6 @@ name = SMTP Frontend ...@@ -223,7 +220,6 @@ name = SMTP Frontend
#software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/zimbra-kvm:/software/reverse-proxy-nginx/development.cfg #software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/zimbra-kvm:/software/reverse-proxy-nginx/development.cfg
software-url = /opt/slapdev/software/reverse-proxy-nginx/development.cfg software-url = /opt/slapdev/software/reverse-proxy-nginx/development.cfg
slave = true slave = true
config = smtp-upstream-host smtp-upstream-port
config-smtp-upstream-host = $${tunnel-ipv6-kvm-smtp:ipv6} config-smtp-upstream-host = $${tunnel-ipv6-kvm-smtp:ipv6}
config-smtp-upstream-port = $${tunnel-ipv6-kvm-smtp:port} config-smtp-upstream-port = $${tunnel-ipv6-kvm-smtp:port}
return = listening-ipv4 return = listening-ipv4
......
...@@ -101,4 +101,4 @@ scripts = ...@@ -101,4 +101,4 @@ scripts =
[versions] [versions]
erp5.util = 0.4.41 erp5.util = 0.4.41
slapos.toolbox = 0.40.2 slapos.toolbox = 0.40.4
...@@ -34,7 +34,7 @@ eggs = ...@@ -34,7 +34,7 @@ eggs =
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-boinc.cfg url = ${:_profile_base_location_}/instance-boinc.cfg
output = ${buildout:directory}/template-boinc.cfg output = ${buildout:directory}/template-boinc.cfg
md5sum = 6642942dde15ab5195d8d8533690ef29 md5sum = 5eb6057311fb1f73a54b72a40ead47c2
mode = 0644 mode = 0644
#Template for deploying MySQL Database Server #Template for deploying MySQL Database Server
...@@ -71,7 +71,7 @@ gitdb = 0.5.4 ...@@ -71,7 +71,7 @@ gitdb = 0.5.4
plone.recipe.command = 1.1 plone.recipe.command = 1.1
slapos.recipe.download = 1.0.dev-r4053 slapos.recipe.download = 1.0.dev-r4053
slapos.recipe.template = 2.4.2 slapos.recipe.template = 2.4.2
slapos.toolbox = 0.40.2 slapos.toolbox = 0.40.4
smmap = 0.8.2 smmap = 0.8.2
# Required by: # Required by:
......
...@@ -302,7 +302,6 @@ name = Frontend ...@@ -302,7 +302,6 @@ name = Frontend
# XXX We have hardcoded SR URL here. # XXX We have hardcoded SR URL here.
software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
slave = true slave = true
config = url custom_domain
config-url = http://[$${apache-php:ip}]:$${apache-php:port}/ config-url = http://[$${apache-php:ip}]:$${apache-php:port}/
return = site_url return = site_url
config-custom_domain = $${slap-parameter:domain} config-custom_domain = $${slap-parameter:domain}
......
...@@ -9,27 +9,28 @@ exec-sitecustomize = false ...@@ -9,27 +9,28 @@ exec-sitecustomize = false
extends = extends =
../stack/slapos.cfg ../stack/slapos.cfg
../component/cloudooo/buildout.cfg
../component/coreutils/buildout.cfg
../component/cups/buildout.cfg ../component/cups/buildout.cfg
../component/dbus/buildout.cfg ../component/dbus/buildout.cfg
../component/dcron/buildout.cfg
../component/ffmpeg/buildout.cfg
../component/file/buildout.cfg
../component/fonts/buildout.cfg
../component/git/buildout.cfg
../component/glib/buildout.cfg ../component/glib/buildout.cfg
../component/libreoffice-bin/buildout.cfg ../component/haproxy/buildout.cfg
../component/imagemagick/buildout.cfg
../component/libffi/buildout.cfg ../component/libffi/buildout.cfg
../component/libpng/buildout.cfg ../component/libpng/buildout.cfg
../component/libreoffice-bin/buildout.cfg
../component/lxml-python/buildout.cfg ../component/lxml-python/buildout.cfg
../component/python-2.7/buildout.cfg ../component/mesa/buildout.cfg
../component/xorg/buildout.cfg
../component/fonts/buildout.cfg
../component/git/buildout.cfg
../component/poppler/buildout.cfg ../component/poppler/buildout.cfg
../component/imagemagick/buildout.cfg ../component/python-2.7/buildout.cfg
../component/ffmpeg/buildout.cfg
../component/file/buildout.cfg
../component/stunnel/buildout.cfg
../component/rdiff-backup/buildout.cfg ../component/rdiff-backup/buildout.cfg
../component/dcron/buildout.cfg ../component/stunnel/buildout.cfg
../component/coreutils/buildout.cfg ../component/xorg/buildout.cfg
../component/cloudooo/buildout.cfg
../component/haproxy/buildout.cfg
versions = versions versions = versions
...@@ -57,8 +58,11 @@ parts = ...@@ -57,8 +58,11 @@ parts =
dbus dbus
dbus-glib dbus-glib
glib glib
libexpat
libffi libffi
libpng12 libpng12
xdamage
xfixes
imagemagick imagemagick
file file
poppler poppler
...@@ -77,7 +81,6 @@ WSGIUtils = 0.7 ...@@ -77,7 +81,6 @@ WSGIUtils = 0.7
plone.recipe.command = 1.1 plone.recipe.command = 1.1
python-magic = 0.4.6 python-magic = 0.4.6
rdiff-backup = 1.0.5 rdiff-backup = 1.0.5
slapos.recipe.build = 0.12
slapos.recipe.template = 2.5 slapos.recipe.template = 2.5
# Required by: # Required by:
......
...@@ -25,6 +25,8 @@ extends = ...@@ -25,6 +25,8 @@ extends =
../../component/libpng/buildout.cfg ../../component/libpng/buildout.cfg
../../component/libreoffice-bin/buildout.cfg ../../component/libreoffice-bin/buildout.cfg
../../component/lxml-python/buildout.cfg ../../component/lxml-python/buildout.cfg
../../component/mesa/buildout.cfg
../../component/numpy/buildout.cfg
../../component/percona-toolkit/buildout.cfg ../../component/percona-toolkit/buildout.cfg
../../component/mariadb/buildout.cfg ../../component/mariadb/buildout.cfg
../../component/mysql-python/buildout.cfg ../../component/mysql-python/buildout.cfg
...@@ -178,7 +180,7 @@ md5sum = 5ebf310655d5de27da039d71a63d209b ...@@ -178,7 +180,7 @@ md5sum = 5ebf310655d5de27da039d71a63d209b
[template-cloudooo] [template-cloudooo]
< = download-base < = download-base
filename = instance-cloudoo.cfg.in filename = instance-cloudoo.cfg.in
md5sum = 838e56f46ba749dfc53b8e69e21be008 md5sum = 48d5e8c3efc5bfd6fc1027b5ebe55e64
[template-zope-conf] [template-zope-conf]
< = download-base < = download-base
...@@ -215,7 +217,7 @@ md5sum = e40e2e39f4941c6372f4357e8589a5cf ...@@ -215,7 +217,7 @@ md5sum = e40e2e39f4941c6372f4357e8589a5cf
# XXX: "template.cfg" is hardcoded in instanciation recipe # XXX: "template.cfg" is hardcoded in instanciation recipe
filename = template.cfg filename = template.cfg
template = ${:_profile_base_location_}/instance.cfg.in template = ${:_profile_base_location_}/instance.cfg.in
md5sum = 99159fc1e32a0651b35f824709d561d0 md5sum = 7c622aaa7873e04ec340383b22ede3be
extra-context = extra-context =
key mariadb_link_binary template-mariadb:link-binary key mariadb_link_binary template-mariadb:link-binary
key zope_link_binary template-zope:link-binary key zope_link_binary template-zope:link-binary
...@@ -226,8 +228,8 @@ extra-context = ...@@ -226,8 +228,8 @@ extra-context =
key cups_location cups:location key cups_location cups:location
key curl_location curl:location key curl_location curl:location
key dash_location dash:location key dash_location dash:location
key dbus_location dbus:location
key dbus_glib_location dbus-glib:location key dbus_glib_location dbus-glib:location
key dbus_location dbus:location
key dcron_location dcron:location key dcron_location dcron:location
key erp5_location erp5:location key erp5_location erp5:location
key file_location file:location key file_location file:location
...@@ -236,6 +238,7 @@ extra-context = ...@@ -236,6 +238,7 @@ extra-context =
key fonts_location fonts:location key fonts_location fonts:location
key freetype_location freetype:location key freetype_location freetype:location
key glib_location glib:location key glib_location glib:location
key glu_location glu:location
key gzip_location gzip:location key gzip_location gzip:location
key haproxy_location haproxy:location key haproxy_location haproxy:location
key jsl_location jsl:location key jsl_location jsl:location
...@@ -247,6 +250,7 @@ extra-context = ...@@ -247,6 +250,7 @@ extra-context =
key libXdmcp_location libXdmcp:location key libXdmcp_location libXdmcp:location
key libXext_location libXext:location key libXext_location libXext:location
key libXrender_location libXrender:location key libXrender_location libXrender:location
key libexpat_location libexpat:location
key libffi_location libffi:location key libffi_location libffi:location
key libpng12_location libpng12:location key libpng12_location libpng12:location
key libreoffice_bin_location libreoffice-bin:location key libreoffice_bin_location libreoffice-bin:location
...@@ -255,6 +259,7 @@ extra-context = ...@@ -255,6 +259,7 @@ extra-context =
key local_bt5_repository local-bt5-repository:list key local_bt5_repository local-bt5-repository:list
key logrotate_location logrotate:location key logrotate_location logrotate:location
key mariadb_location mariadb:location key mariadb_location mariadb:location
key mesa_location mesa:location
key openssl_location openssl:location key openssl_location openssl:location
key sixtunnel_location 6tunnel:location key sixtunnel_location 6tunnel:location
key stunnel_location stunnel:location key stunnel_location stunnel:location
...@@ -276,12 +281,14 @@ extra-context = ...@@ -276,12 +281,14 @@ extra-context =
key template_zope_conf template-zope-conf:target key template_zope_conf template-zope-conf:target
key userhosts_location userhosts:location key userhosts_location userhosts:location
key wget_location wget:location key wget_location wget:location
key xdamage_location xdamage:location
key xfixes_location xfixes:location
key zlib_location zlib:location key zlib_location zlib:location
[template-erp5] [template-erp5]
< = download-base < = download-base
filename = instance-erp5.cfg.in filename = instance-erp5.cfg.in
md5sum = 2ac3bbeb031eb74fbd6f9d033c71b677 md5sum = 386f5f15d71636f855f22201d38367e5
[template-zeo] [template-zeo]
< = download-base < = download-base
...@@ -439,6 +446,7 @@ ZODB3-patch-binary = ${patch:location}/bin/patch ...@@ -439,6 +446,7 @@ ZODB3-patch-binary = ${patch:location}/bin/patch
[eggs] [eggs]
recipe = zc.recipe.egg recipe = zc.recipe.egg
eggs = eggs =
${numpy:egg}
${mysql-python:egg} ${mysql-python:egg}
${lxml-python:egg} ${lxml-python:egg}
${pillow-python:egg} ${pillow-python:egg}
...@@ -458,7 +466,6 @@ eggs = ...@@ -458,7 +466,6 @@ eggs =
ipdb ipdb
Jinja2 Jinja2
mechanize mechanize
numpy
paramiko paramiko
ply ply
pyflakes pyflakes
...@@ -563,7 +570,7 @@ setup = ${erp5:location} ...@@ -563,7 +570,7 @@ setup = ${erp5:location}
[cloudooo-repository] [cloudooo-repository]
branch = branch =
revision = f67af2797f74d4212f7221978645552353f341c8 revision = 8db3977b312e3cf8dbb64660c6f4f9e639b749c9
[slapos.cookbook-repository] [slapos.cookbook-repository]
branch = erp5-cluster branch = erp5-cluster
...@@ -595,9 +602,6 @@ PasteDeploy = 1.5.2 ...@@ -595,9 +602,6 @@ PasteDeploy = 1.5.2
Pygments = 1.6 Pygments = 1.6
coverage = 3.7.1 coverage = 3.7.1
# Known version with works
numpy = 1.6.2
# test_UserManagerInterfaces in testERP5Security fails with 1.10.0. # test_UserManagerInterfaces in testERP5Security fails with 1.10.0.
Products.PluggableAuthService = 1.9.0 Products.PluggableAuthService = 1.9.0
...@@ -632,10 +636,11 @@ zope.app.testing = 3.8.1 ...@@ -632,10 +636,11 @@ zope.app.testing = 3.8.1
# Pinned versions # Pinned versions
MySQL-python = 1.2.5 MySQL-python = 1.2.5
Pillow = 2.6.1
Products.CMFActionIcons = 2.1.3 Products.CMFActionIcons = 2.1.3
Products.DCWorkflowGraph = 0.4.1 Products.DCWorkflowGraph = 0.4.1
Products.ExternalEditor = 1.1.0 Products.ExternalEditor = 1.1.0
Products.GenericSetup = 1.7.4 Products.GenericSetup = 1.7.5
Products.LongRequestLogger = 1.1.0 Products.LongRequestLogger = 1.1.0
Products.MimetypesRegistry = 2.0.6 Products.MimetypesRegistry = 2.0.6
Products.PluginRegistry = 1.3 Products.PluginRegistry = 1.3
...@@ -648,7 +653,7 @@ WSGIUtils = 0.7 ...@@ -648,7 +653,7 @@ WSGIUtils = 0.7
apache-libcloud = 0.15.1 apache-libcloud = 0.15.1
astroid = 1.2.1 astroid = 1.2.1
async = 0.6.1 async = 0.6.1
chardet = 2.2.1 chardet = 2.3.0
csp-eventlet = 0.7.0 csp-eventlet = 0.7.0
ecdsa = 0.11 ecdsa = 0.11
elementtree = 1.2.7-20070827-preview elementtree = 1.2.7-20070827-preview
...@@ -657,17 +662,17 @@ eventlet = 0.15.2 ...@@ -657,17 +662,17 @@ eventlet = 0.15.2
five.formlib = 1.0.4 five.formlib = 1.0.4
five.localsitemanager = 2.0.5 five.localsitemanager = 2.0.5
gitdb = 0.5.4 gitdb = 0.5.4
greenlet = 0.4.4 greenlet = 0.4.5
http-parser = 0.8.3 http-parser = 0.8.3
httplib2 = 0.9 httplib2 = 0.9
huBarcode = 1.0.0 huBarcode = 1.0.0
interval = 1.0.0 interval = 1.0.0
ipdb = 0.8 ipdb = 0.8
ipython = 2.2.0 ipython = 2.3.0
logilab-common = 0.62.1 logilab-common = 0.63.0
minitage.paste = 1.4.6 minitage.paste = 1.4.6
minitage.recipe.egg = 1.107 minitage.recipe.egg = 1.107
pillow = 2.5.3 numpy = 1.9.1
plone.recipe.command = 1.1 plone.recipe.command = 1.1
ply = 3.4 ply = 3.4
polib = 1.0.5 polib = 1.0.5
...@@ -675,13 +680,12 @@ pprofile = 1.7.2 ...@@ -675,13 +680,12 @@ pprofile = 1.7.2
pycountry = 1.8 pycountry = 1.8
pyflakes = 0.8.1 pyflakes = 0.8.1
pylint = 1.3.1 pylint = 1.3.1
python-ldap = 2.4.15 python-ldap = 2.4.17
python-magic = 0.4.6 python-magic = 0.4.6
python-memcached = 1.53 python-memcached = 1.53
qrcode = 5.0.1 qrcode = 5.1
restkit = 4.2.2 restkit = 4.2.2
rtjp-eventlet = 0.3.2 rtjp-eventlet = 0.3.2
slapos.recipe.build = 0.12
slapos.recipe.template = 2.5 slapos.recipe.template = 2.5
slapos.toolbox = 0.43.0 slapos.toolbox = 0.43.0
smmap = 0.8.2 smmap = 0.8.2
......
...@@ -33,7 +33,7 @@ wrapper = ${directory:service}/cloudooo ...@@ -33,7 +33,7 @@ wrapper = ${directory:service}/cloudooo
data-directory = ${directory:cloudooo-data} data-directory = ${directory:cloudooo-data}
environment = environment =
LD_LIBRARY_PATH = {{ parameter_dict['cups'] }}/lib:{{ parameter_dict['cups'] }}/lib64:{{ parameter_dict['dbus'] }}/lib:{{ parameter_dict['dbus-glib'] }}/lib:{{ parameter_dict['file'] }}/lib:{{ parameter_dict['fontconfig'] }}/lib:{{ parameter_dict['freetype'] }}/lib:{{ parameter_dict['glib'] }}/lib:{{ parameter_dict['libffi'] }}/lib:{{ parameter_dict['libffi'] }}/lib64:{{ parameter_dict['libICE'] }}/lib:{{ parameter_dict['libpng12'] }}/lib:{{ parameter_dict['libSM'] }}/lib:{{ parameter_dict['libX11'] }}/lib:{{ parameter_dict['libXau'] }}/lib:{{ parameter_dict['libXdmcp'] }}/lib:{{ parameter_dict['libXext'] }}/lib:{{ parameter_dict['libxcb'] }}/lib:{{ parameter_dict['libXrender'] }}/lib:{{ parameter_dict['zlib'] }}/lib LD_LIBRARY_PATH = {{ parameter_dict['cups'] }}/lib:{{ parameter_dict['cups'] }}/lib64:{{ parameter_dict['dbus'] }}/lib:{{ parameter_dict['dbus-glib'] }}/lib:{{ parameter_dict['file'] }}/lib:{{ parameter_dict['fontconfig'] }}/lib:{{ parameter_dict['freetype'] }}/lib:{{ parameter_dict['glib'] }}/lib:{{ parameter_dict['glu'] }}/lib:{{ parameter_dict['libICE'] }}/lib:{{ parameter_dict['libSM'] }}/lib:{{ parameter_dict['libX11'] }}/lib:{{ parameter_dict['libXau'] }}/lib:{{ parameter_dict['libXdmcp'] }}/lib:{{ parameter_dict['libXext'] }}/lib:{{ parameter_dict['libXrender'] }}/lib:{{ parameter_dict['libexpat'] }}/lib:{{ parameter_dict['libffi'] }}/lib:{{ parameter_dict['libffi'] }}/lib64:{{ parameter_dict['libpng12'] }}/lib:{{ parameter_dict['libxcb'] }}/lib:{{ parameter_dict['mesa'] }}/lib:{{ parameter_dict['xdamage'] }}/lib:{{ parameter_dict['xfixes'] }}/lib:{{ parameter_dict['zlib'] }}/lib
FONTCONFIG_FILE = ${fontconfig-instance:conf-path} FONTCONFIG_FILE = ${fontconfig-instance:conf-path}
# Binary information # Binary information
......
...@@ -6,16 +6,11 @@ ...@@ -6,16 +6,11 @@
[request-common] [request-common]
recipe = slapos.cookbook:request.serialised recipe = slapos.cookbook:request.serialised
software-url = ${slap-connection:software-release-url} software-url = ${slap-connection:software-release-url}
sla = computer_guid
server-url = ${slap-connection:server-url} server-url = ${slap-connection:server-url}
key-file = ${slap-connection:key-file} key-file = ${slap-connection:key-file}
cert-file = ${slap-connection:cert-file} cert-file = ${slap-connection:cert-file}
computer-id = ${slap-connection:computer-id} computer-id = ${slap-connection:computer-id}
partition-id = ${slap-connection:partition-id} partition-id = ${slap-connection:partition-id}
config =
use-ipv6
${:extra-config}
extra-config =
config-use-ipv6 = {{ dumps(slapparameter_dict.get('use-ipv6', False)) }} config-use-ipv6 = {{ dumps(slapparameter_dict.get('use-ipv6', False)) }}
{% macro request(name, software_type, config_key, config, ret={'url': True}) -%} {% macro request(name, software_type, config_key, config, ret={'url': True}) -%}
...@@ -32,7 +27,6 @@ return = {{ ret.keys() | join(' ') }} ...@@ -32,7 +27,6 @@ return = {{ ret.keys() | join(' ') }}
{% endif -%} {% endif -%}
{% endfor -%} {% endfor -%}
sla-computer_guid = {{ dumps(slapparameter_dict.get(config_key + '-computer-guid', computer_id)) }} sla-computer_guid = {{ dumps(slapparameter_dict.get(config_key + '-computer-guid', computer_id)) }}
extra-config = {{ ' '.join(config) }}
{% for option, value in config.items() -%} {% for option, value in config.items() -%}
config-{{ option }} = {{ dumps(value) }} config-{{ option }} = {{ dumps(value) }}
{% endfor -%} {% endfor -%}
...@@ -72,30 +66,6 @@ recipe = slapos.cookbook:generate.password ...@@ -72,30 +66,6 @@ recipe = slapos.cookbook:generate.password
return = return =
zope-address-list zope-address-list
hosts-dict hosts-dict
{% set extra_config = '
bt5
bt5-repository-url
cloudooo-url
deadlock-debugger-password
developer-list
hosts-dict
inituser-login
inituser-password
instance-count
kumofs-url
longrequest-logger-interval
longrequest-logger-timeout
memcached-url
mysql-test-url-list
mysql-url-list
name
port-base
site-id
smtp-url
thread-amount
timerserver-interval
timezone
webdav' -%}
config-bt5 = {{ dumps(slapparameter_dict.get('bt5', 'erp5_full_text_myisam_catalog erp5_configurator_standard erp5_configurator_maxma_demo erp5_configurator_ung erp5_configurator_run_my_doc')) }} config-bt5 = {{ dumps(slapparameter_dict.get('bt5', 'erp5_full_text_myisam_catalog erp5_configurator_standard erp5_configurator_maxma_demo erp5_configurator_ung erp5_configurator_run_my_doc')) }}
config-bt5-repository-url = {{ dumps(slapparameter_dict.get('bt5-repository-url', local_bt5_repository)) }} config-bt5-repository-url = {{ dumps(slapparameter_dict.get('bt5-repository-url', local_bt5_repository)) }}
config-cloudooo-url = ${request-cloudooo:connection-url} config-cloudooo-url = ${request-cloudooo:connection-url}
...@@ -116,19 +86,11 @@ config-tidstorage-ip = ${request-zodb:connection-tidstorage-ip} ...@@ -116,19 +86,11 @@ config-tidstorage-ip = ${request-zodb:connection-tidstorage-ip}
config-tidstorage-port = ${request-zodb:connection-tidstorage-port} config-tidstorage-port = ${request-zodb:connection-tidstorage-port}
config-zodb-dict = ${request-zodb:connection-zodb-dict} config-zodb-dict = ${request-zodb:connection-zodb-dict}
config-zodb-storage-type = ${request-zodb:connection-zodb-storage-type} config-zodb-storage-type = ${request-zodb:connection-zodb-storage-type}
{% set extra_config = extra_config ~ '
tidstorage-ip
tidstorage-port
zodb-dict
zodb-storage-type' -%}
{% endif -%} {% endif -%}
{% if zodb_extern -%} {% if zodb_extern -%}
config-zodb-extern = {{ dumps(zodb_extern) }} config-zodb-extern = {{ dumps(zodb_extern) }}
{% set extra_config = extra_config ~ '
zodb-extern' -%}
{% endif -%} {% endif -%}
software-type = zope software-type = zope
extra-config = {{ extra_config }}
{% set zope_family_dict = {} -%} {% set zope_family_dict = {} -%}
{% for custom_name, zope_parameter_dict in slapparameter_dict.get('zope-partition-dict', {'1': {}}).items() -%} {% for custom_name, zope_parameter_dict in slapparameter_dict.get('zope-partition-dict', {'1': {}}).items() -%}
...@@ -177,13 +139,6 @@ config-url = ${request-balancer:{{ family_name }}-v6} ...@@ -177,13 +139,6 @@ config-url = ${request-balancer:{{ family_name }}-v6}
name = balancer name = balancer
software-type = balancer software-type = balancer
sla-computer_guid = {{ dumps(slapparameter_dict.get('balancer-computer-guid', computer_id)) }} sla-computer_guid = {{ dumps(slapparameter_dict.get('balancer-computer-guid', computer_id)) }}
extra-config =
tcpv4-port
haproxy-server-check-path
backend-path
ssl
zope-family-dict
{{ zope_address_list_id_dict.values() | join(' ') }}
return = return =
{%- for family in zope_family_dict %} {%- for family in zope_family_dict %}
{{ family }} {{ family }}
...@@ -204,7 +159,6 @@ config-ssl = {{ dumps(balancer_dict.get('ssl', {})) }} ...@@ -204,7 +159,6 @@ config-ssl = {{ dumps(balancer_dict.get('ssl', {})) }}
< = request-common < = request-common
software-url = {{ dumps(frontend_dict['software-url']) }} software-url = {{ dumps(frontend_dict['software-url']) }}
software-type = {{ dumps(frontend_dict.get('software-type', 'RootSoftwareInstance')) }} software-type = {{ dumps(frontend_dict.get('software-type', 'RootSoftwareInstance')) }}
sla = instance_guid
sla-instance_guid = {{ dumps(frontend_dict['instance-guid']) }} sla-instance_guid = {{ dumps(frontend_dict['instance-guid']) }}
slave = true slave = true
{% set config_dict = { {% set config_dict = {
...@@ -213,7 +167,6 @@ slave = true ...@@ -213,7 +167,6 @@ slave = true
{% if frontend_dict.get('domain') -%} {% if frontend_dict.get('domain') -%}
{% do config_dict.__setitem__('custom_domain', frontend_dict['domain']) -%} {% do config_dict.__setitem__('custom_domain', frontend_dict['domain']) -%}
{% endif -%} {% endif -%}
extra-config = url {{ config_dict.keys() | join(' ') }}
{% for name, value in config_dict.items() -%} {% for name, value in config_dict.items() -%}
config-{{ name }} = {{ value }} config-{{ name }} = {{ value }}
{% endfor -%} {% endfor -%}
......
...@@ -32,16 +32,21 @@ file = {{ file_location }} ...@@ -32,16 +32,21 @@ file = {{ file_location }}
fontconfig = {{ fontconfig_location }} fontconfig = {{ fontconfig_location }}
freetype = {{ freetype_location }} freetype = {{ freetype_location }}
glib = {{ glib_location }} glib = {{ glib_location }}
libffi = {{ libffi_location }} glu = {{ glu_location }}
libICE = {{ libICE_location }} libICE = {{ libICE_location }}
libpng12 = {{ libpng12_location }}
libSM = {{ libSM_location }} libSM = {{ libSM_location }}
libX11 = {{ libX11_location }} libX11 = {{ libX11_location }}
libXau = {{ libXau_location }} libXau = {{ libXau_location }}
libXdmcp = {{ libXdmcp_location }} libXdmcp = {{ libXdmcp_location }}
libXext = {{ libXext_location }} libXext = {{ libXext_location }}
libxcb = {{ libxcb_location }}
libXrender = {{ libXrender_location }} libXrender = {{ libXrender_location }}
libexpat = {{ libexpat_location }}
libffi = {{ libffi_location }}
libpng12 = {{ libpng12_location }}
libxcb = {{ libxcb_location }}
mesa = {{ mesa_location }}
xdamage = {{ xdamage_location }}
xfixes = {{ xfixes_location }}
zlib = {{ zlib_location }} zlib = {{ zlib_location }}
libreoffice-bin = {{ libreoffice_bin_location }} libreoffice-bin = {{ libreoffice_bin_location }}
fonts = {{ fonts_location }} fonts = {{ fonts_location }}
......
[buildout]
extends =
../../stack/slapos.cfg
../../component/libffi/buildout.cfg
../../component/pkgconfig/buildout.cfg
../../component/python-cffi/buildout.cfg
../../component/java/buildout.cfg
parts =
slapos-cookbook
libffi
python-cffi
eggs
java
hadoop
hadoop-streaming
instance-stack
[eggs]
recipe = zc.recipe.egg
eggs =
slapos.cookbook
collective.recipe.template
cp.recipe.cmd
plone.recipe.command
[hadoop]
recipe = hexagonit.recipe.download
version = 2.5.1
filename = hadoop-${:version}.tar.gz
url = http://apache.mirrors.spacedump.net/hadoop/common/hadoop-${:version}/${:filename}
md5sum = 015b12d790701deb4de9a647013f72a9
download-only = true
mode = 0644
strip-top-level-dir = true
[hadoop-streaming]
recipe = hexagonit.recipe.download
url = http://repo1.maven.org/maven2/org/apache/hadoop/hadoop-streaming/0.20.203.0/hadoop-streaming-0.20.203.0.jar
download-only = true
md5sum = 792c7df75b63e477fef12988b7cced6f
mode = 0644
[instance-stack]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-stack.cfg.in
output = ${buildout:directory}/instance-stack.cfg
md5sum = 3c846dd1269aeef43cd2d5448463563b
mode = 0644
[buildout]
parts =
sh-environment
start-daemons
deploy-tar
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
[sh-environment]
# environment needed for running/debugging the applications, exported to a shell script to be sourced.
recipe = collective.recipe.template
output = $${buildout:directory}/environment.sh
input = inline:
export JAVA_HOME="${java:location}"
export HADOOP_PREFIX="$${directories:hadoop-prefix}"
export PATH="$PATH:$HADOOP_PREFIX/bin"
export HADOOP_HOME="$HADOOP_PREFIX"
export HADOOP_COMMON_HOME="$HADOOP_PREFIX"
export HADOOP_CONF_DIR="$HADOOP_PREFIX/etc/"
export HADOOP_HDFS_HOME="$HADOOP_PREFIX"
export HADOOP_MAPRED_HOME="$HADOOP_PREFIX"
export HADOOP_YARN_HOME="$HADOOP_PREFIX"
[deploy-tar]
recipe = plone.recipe.command
update = true
command =
[ -d $${directories:hadoop-prefix}/bin} ] || tar xf ${hadoop:location}/${hadoop:filename} -C $${directories:hadoop-prefix} --strip-components=1
[directories]
recipe = slapos.cookbook:mkdirectory
bin = $${buildout:directory}/bin
etc = $${buildout:directory}/etc
var = $${buildout:directory}/var
hadoop-prefix = $${buildout:directory}/hadoop
services = $${directories:etc}/service
promises = $${directories:etc}/promise
[start-daemons]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/template/start-daemons.sh.in
output = $${directories:bin}/start-daemons.sh
md5sum = 35a51a7f826d29e47c564c57e954db80
mode = 0755
#!/bin/bash
## Start HDFS daemons
# Format the namenode directory (DO THIS ONLY ONCE, THE FIRST TIME)
#$HADOOP_PREFIX/bin/hdfs namenode -format
# Start the namenode daemon
$HADOOP_PREFIX/sbin/hadoop-daemon.sh start namenode
# Start the datanode daemon
$HADOOP_PREFIX/sbin/hadoop-daemon.sh start datanode
## Start YARN daemons
# Start the resourcemanager daemon
$HADOOP_PREFIX/sbin/yarn-daemon.sh start resourcemanager
# Start the nodemanager daemon
$HADOOP_PREFIX/sbin/yarn-daemon.sh start nodemanager
...@@ -44,10 +44,9 @@ apache-libcloud = 0.9.1 ...@@ -44,10 +44,9 @@ apache-libcloud = 0.9.1
async = 0.6.1 async = 0.6.1
gitdb = 0.5.4 gitdb = 0.5.4
plone.recipe.command = 1.1 plone.recipe.command = 1.1
slapos.recipe.build = 0.7
slapos.recipe.download = 1.0.dev-r4053 slapos.recipe.download = 1.0.dev-r4053
slapos.recipe.template = 2.3 slapos.recipe.template = 2.3
slapos.toolbox = 0.40.2 slapos.toolbox = 0.40.4
smmap = 0.8.2 smmap = 0.8.2
# Required by: # Required by:
......
...@@ -179,7 +179,6 @@ name = MariaDB ...@@ -179,7 +179,6 @@ name = MariaDB
software-url = $${slap-connection:software-release-url} software-url = $${slap-connection:software-release-url}
software-type = mariadb software-type = mariadb
return = url return = url
sla = computer_guid
sla-computer_guid = $${slap-connection:computer-id} sla-computer_guid = $${slap-connection:computer-id}
[mariadb-urlparse] [mariadb-urlparse]
...@@ -250,7 +249,6 @@ name = Frontend ...@@ -250,7 +249,6 @@ name = Frontend
# XXX We have hardcoded SR URL here. # XXX We have hardcoded SR URL here.
software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
slave = true slave = true
config = url custom_domain
config-url = http://[$${apache-network-configuration:listening-ip}]:$${apache-network-configuration:listening-port}/ config-url = http://[$${apache-network-configuration:listening-ip}]:$${apache-network-configuration:listening-port}/
return = site_url return = site_url
config-custom_domain = $${slap-parameter:domain} config-custom_domain = $${slap-parameter:domain}
......
...@@ -74,7 +74,7 @@ mode = 0644 ...@@ -74,7 +74,7 @@ mode = 0644
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/apache/instance-apache-php.cfg.in url = ${:_profile_base_location_}/apache/instance-apache-php.cfg.in
output = ${buildout:directory}/instance-apache-php.cfg output = ${buildout:directory}/instance-apache-php.cfg
md5sum = 3e2d71d3684aac3e52d2f55794df96bf md5sum = b2564a719fdde097f402ef80e0991903
mode = 0644 mode = 0644
[instance-apache-import] [instance-apache-import]
...@@ -88,6 +88,7 @@ mode = 0644 ...@@ -88,6 +88,7 @@ mode = 0644
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/apache/instance-apache-export.cfg.jinja2 template = ${:_profile_base_location_}/apache/instance-apache-export.cfg.jinja2
rendered = ${buildout:directory}/instance-apache-export.cfg rendered = ${buildout:directory}/instance-apache-export.cfg
extensions = jinja2.ext.do
context = key templateapache instance-apache-php:output context = key templateapache instance-apache-php:output
key templatepbsreadyexport pbsready-export:output key templatepbsreadyexport pbsready-export:output
...@@ -102,6 +103,7 @@ mode = 0644 ...@@ -102,6 +103,7 @@ mode = 0644
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/instance-resilient.cfg.jinja2 template = ${:_profile_base_location_}/instance-resilient.cfg.jinja2
rendered = ${buildout:directory}/instance-resilient.cfg rendered = ${buildout:directory}/instance-resilient.cfg
extensions = jinja2.ext.do
context = key buildout buildout:bin-directory context = key buildout buildout:bin-directory
key develop_eggs_directory buildout:develop-eggs-directory key develop_eggs_directory buildout:develop-eggs-directory
...@@ -110,7 +112,7 @@ context = key buildout buildout:bin-directory ...@@ -110,7 +112,7 @@ context = key buildout buildout:bin-directory
import-list = file parts template-parts:destination import-list = file parts template-parts:destination
file replicated template-replicated:destination file replicated template-replicated:destination
md5sum = f6dc9290194ad1fbe1ced553f5668922 md5sum = 5e2f917b032e43af6a736532966ae1f0
mode = 0644 mode = 0644
[instance-mariadb] [instance-mariadb]
...@@ -185,10 +187,9 @@ async = 0.6.1 ...@@ -185,10 +187,9 @@ async = 0.6.1
gitdb = 0.5.4 gitdb = 0.5.4
pycrypto = 2.6 pycrypto = 2.6
rdiff-backup = 1.0.5 rdiff-backup = 1.0.5
slapos.recipe.build = 0.11.6
slapos.recipe.download = 1.0.dev-r4053 slapos.recipe.download = 1.0.dev-r4053
slapos.recipe.template = 2.4.2 slapos.recipe.template = 2.4.2
slapos.toolbox = 0.40.2 slapos.toolbox = 0.40.4
smmap = 0.8.2 smmap = 0.8.2
# Required by: # Required by:
......
...@@ -19,7 +19,6 @@ parts += ...@@ -19,7 +19,6 @@ parts +=
[request-apache] [request-apache]
return = url ssh-public-key ssh-url notification-id ip url backend_url return = url ssh-public-key ssh-url notification-id ip url backend_url
# XXX: hardcoded values # XXX: hardcoded values
config = domain number authorized-key notify ip-list namebase mariadb1-computer-guid pbs-mariadb1-computer-guid mariadb2-computer-guid pbs-mariadb2-computer-guid mariadb3-computer-guid pbs-mariadb3-computer-guid
config-mariadb1-computer-guid = ${slap-parameter:mariadb1-computer-guid} config-mariadb1-computer-guid = ${slap-parameter:mariadb1-computer-guid}
config-pbs-mariadb1-computer-guid = ${slap-parameter:pbs-mariadb1-computer-guid} config-pbs-mariadb1-computer-guid = ${slap-parameter:pbs-mariadb1-computer-guid}
config-mariadb2-computer-guid = ${slap-parameter:mariadb2-computer-guid} config-mariadb2-computer-guid = ${slap-parameter:mariadb2-computer-guid}
......
...@@ -179,7 +179,6 @@ name = Postgres ...@@ -179,7 +179,6 @@ name = Postgres
software-url = $${slap-connection:software-release-url} software-url = $${slap-connection:software-release-url}
software-type = postgres software-type = postgres
return = url return = url
sla = computer_guid
sla-computer_guid = $${slap-connection:computer-id} sla-computer_guid = $${slap-connection:computer-id}
[postgres-urlparse] [postgres-urlparse]
...@@ -235,7 +234,6 @@ name = Frontend ...@@ -235,7 +234,6 @@ name = Frontend
# XXX We have hardcoded SR URL here. # XXX We have hardcoded SR URL here.
software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
slave = true slave = true
config = url custom_domain
config-url = http://[$${apache-php:ip}]:$${apache-php:port}/ config-url = http://[$${apache-php:ip}]:$${apache-php:port}/
return = site_url return = site_url
config-custom_domain = $${slap-parameter:domain} config-custom_domain = $${slap-parameter:domain}
......
...@@ -35,8 +35,8 @@ extends = ...@@ -35,8 +35,8 @@ extends =
../../component/rdiff-backup/buildout.cfg ../../component/rdiff-backup/buildout.cfg
../../component/stunnel/buildout.cfg ../../component/stunnel/buildout.cfg
../../component/dropbear/buildout.cfg ../../component/dropbear/buildout.cfg
../resilient/buildout.cfg
../slapos.cfg ../slapos.cfg
../resilient/buildout.cfg
#---------------- #----------------
...@@ -62,7 +62,7 @@ mode = 0644 ...@@ -62,7 +62,7 @@ mode = 0644
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/apache/instance-apache-php.cfg.in url = ${:_profile_base_location_}/apache/instance-apache-php.cfg.in
output = ${buildout:directory}/instance-apache-php.cfg output = ${buildout:directory}/instance-apache-php.cfg
md5sum = 823257dda6f3068a38c6b69c771cf307 md5sum = 7a0670c747f2f21a79c240bfbfbf9f6e
mode = 0644 mode = 0644
[instance-apache-import] [instance-apache-import]
...@@ -76,6 +76,7 @@ mode = 0644 ...@@ -76,6 +76,7 @@ mode = 0644
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/apache/instance-apache-export.cfg.jinja2 template = ${:_profile_base_location_}/apache/instance-apache-export.cfg.jinja2
rendered = ${buildout:directory}/instance-apache-export.cfg rendered = ${buildout:directory}/instance-apache-export.cfg
extensions = jinja2.ext.do
context = key templateapache instance-apache-php:output context = key templateapache instance-apache-php:output
key templatepbsreadyexport pbsready-export:output key templatepbsreadyexport pbsready-export:output
...@@ -90,6 +91,7 @@ mode = 0644 ...@@ -90,6 +91,7 @@ mode = 0644
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/instance-resilient.cfg.jinja2 template = ${:_profile_base_location_}/instance-resilient.cfg.jinja2
rendered = ${buildout:directory}/instance-resilient.cfg rendered = ${buildout:directory}/instance-resilient.cfg
extensions = jinja2.ext.do
context = key buildout buildout:bin-directory context = key buildout buildout:bin-directory
key develop_eggs_directory buildout:develop-eggs-directory key develop_eggs_directory buildout:develop-eggs-directory
...@@ -98,7 +100,7 @@ context = key buildout buildout:bin-directory ...@@ -98,7 +100,7 @@ context = key buildout buildout:bin-directory
import-list = file parts template-parts:destination import-list = file parts template-parts:destination
file replicated template-replicated:destination file replicated template-replicated:destination
md5sum = ef38aa9810ce20960382261f235abfcd md5sum = 35bbf8cb40fe7185664641d82b406334
mode = 0644 mode = 0644
[instance-postgres] [instance-postgres]
...@@ -182,11 +184,9 @@ cp.recipe.cmd = 0.4 ...@@ -182,11 +184,9 @@ cp.recipe.cmd = 0.4
gitdb = 0.5.4 gitdb = 0.5.4
plone.recipe.command = 1.1 plone.recipe.command = 1.1
pycrypto = 2.6 pycrypto = 2.6
rdiff-backup = 1.0.5
slapos.recipe.build = 0.11.6
slapos.recipe.download = 1.0.dev-r4053 slapos.recipe.download = 1.0.dev-r4053
slapos.recipe.template = 2.4.2 slapos.recipe.template = 2.4.2
slapos.toolbox = 0.40.2 slapos.toolbox = 0.40.4
smmap = 0.8.2 smmap = 0.8.2
# Required by: # Required by:
......
...@@ -19,7 +19,6 @@ parts += ...@@ -19,7 +19,6 @@ parts +=
[request-apache] [request-apache]
return = url ssh-public-key ssh-url notification-id ip url backend_url return = url ssh-public-key ssh-url notification-id ip url backend_url
# XXX: hardcoded values # XXX: hardcoded values
config = domain number authorized-key notify ip-list namebase postgres1-computer-guid pbs-postgres1-computer-guid postgres2-computer-guid pbs-postgres2-computer-guid postgres3-computer-guid pbs-postgres3-computer-guid maarch-sql-data-file
config-postgres1-computer-guid = ${slap-parameter:postgres1-computer-guid} config-postgres1-computer-guid = ${slap-parameter:postgres1-computer-guid}
config-pbs-postgres1-computer-guid = ${slap-parameter:pbs-postgres1-computer-guid} config-pbs-postgres1-computer-guid = ${slap-parameter:pbs-postgres1-computer-guid}
config-postgres2-computer-guid = ${slap-parameter:postgres2-computer-guid} config-postgres2-computer-guid = ${slap-parameter:postgres2-computer-guid}
......
...@@ -34,11 +34,11 @@ eggs = collective.recipe.template ...@@ -34,11 +34,11 @@ eggs = collective.recipe.template
[pbsready] [pbsready]
# Common parts for pbsready-import and pbsready-export. # Common parts for pbsready-import and pbsready-export.
# Provides rdiff-backup, notification queue, ssh authentication, # Provides rdiff-backup, notification queue, ssh authentication,
# dropbear server, and the bully script. # dropbear server, and the takeover script.
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/pbsready.cfg.in url = ${:_profile_base_location_}/pbsready.cfg.in
output = ${buildout:directory}/pbsready.cfg output = ${buildout:directory}/pbsready.cfg
md5sum = 28e9bc354cc146d1bec31c8cac263ee2 md5sum = e89d8378cc610704b518a89b095d3a19
mode = 0644 mode = 0644
[pbsready-import] [pbsready-import]
...@@ -47,7 +47,7 @@ mode = 0644 ...@@ -47,7 +47,7 @@ mode = 0644
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/pbsready-import.cfg.in url = ${:_profile_base_location_}/pbsready-import.cfg.in
output = ${buildout:directory}/pbsready-import.cfg output = ${buildout:directory}/pbsready-import.cfg
md5sum = 0f953067aac3e0132f72fc7e1ed38bd4 md5sum = a5570ecfeff7a9d1b5f8be08db4feefe
mode = 0644 mode = 0644
[pbsready-export] [pbsready-export]
...@@ -56,7 +56,7 @@ mode = 0644 ...@@ -56,7 +56,7 @@ mode = 0644
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/pbsready-export.cfg.in url = ${:_profile_base_location_}/pbsready-export.cfg.in
output = ${buildout:directory}/pbsready-export.cfg output = ${buildout:directory}/pbsready-export.cfg
md5sum = 135638b8c513c7723efb51e3d9182ae9 md5sum = 879fff114d1dbf1f58774ccbce9bdd22
mode = 0644 mode = 0644
[template-pull-backup] [template-pull-backup]
...@@ -69,14 +69,14 @@ mode = 0644 ...@@ -69,14 +69,14 @@ mode = 0644
[template-replicated] [template-replicated]
recipe = slapos.recipe.download recipe = slapos.recipe.download
url = ${:_profile_base_location_}/template-replicated.cfg.in url = ${:_profile_base_location_}/template-replicated.cfg.in
md5sum = 771e1ab7e7e77b35116c57bbae56ba62 md5sum = a21d751078dbf1fc80ecb93a6f499287
mode = 0644 mode = 0644
destination = ${buildout:directory}/template-replicated.cfg.in destination = ${buildout:directory}/template-replicated.cfg.in
[template-parts] [template-parts]
recipe = slapos.recipe.download recipe = slapos.recipe.download
url = ${:_profile_base_location_}/template-parts.cfg.in url = ${:_profile_base_location_}/template-parts.cfg.in
md5sum = a3f55a20881c3f1ec4416662146c06f7 md5sum = 071b1034ee8f5cc14f79b16fdeba2813
mode = 0644 mode = 0644
destination = ${buildout:directory}/template-parts.cfg.in destination = ${buildout:directory}/template-parts.cfg.in
...@@ -116,8 +116,16 @@ md5sum = 19ee9055de961acf402e2dfe5b9581d2 ...@@ -116,8 +116,16 @@ md5sum = 19ee9055de961acf402e2dfe5b9581d2
filename = monitor-check-resilient-feed.in filename = monitor-check-resilient-feed.in
mode = 0644 mode = 0644
[rdiff-backup-build]
# use our own version
find-links = http://www.nexedi.org/static/packages/source/rdiff-backup-1.3.4nxd2.tar.gz
[versions] [versions]
# Pin Jinja2 to 2.6, as 2.7 breaks current code # Pin Jinja2 to 2.6, as 2.7 breaks current code
Jinja2 = 2.6 Jinja2 = 2.6
# ... And newer s.r.template requires Jinja2 >= 2.7 # ... And newer s.r.template requires Jinja2 >= 2.7
slapos.recipe.template = 2.4.2 slapos.recipe.template = 2.4.2
rdiff-backup = 1.3.4nxd2
slapos.cookbook = 0.91
...@@ -54,7 +54,7 @@ pidfile = $${resilient-directory:pid}/$${:name}.pid ...@@ -54,7 +54,7 @@ pidfile = $${resilient-directory:pid}/$${:name}.pid
recipe = slapos.cookbook:cron.d recipe = slapos.cookbook:cron.d
name = backup name = backup
frequency = $${slap-parameter:resiliency-backup-periodicity} frequency = $${slap-parameter:resiliency-backup-periodicity}
command = $${notifier-exporter:wrapper} command = $${notifier-exporter:wrapper} --transaction-id `date +%s`
[slap-parameter] [slap-parameter]
# In cron.d format (i.e things like */15 * * * * are accepted). # In cron.d format (i.e things like */15 * * * * are accepted).
......
...@@ -36,6 +36,10 @@ takeover-password = $${resilient-web-takeover-password:passwd} ...@@ -36,6 +36,10 @@ takeover-password = $${resilient-web-takeover-password:passwd}
[dropbear-server] [dropbear-server]
port = 22220 port = 22220
# Define port of notifier (same reason)
[notifier]
port = 65533
[import-on-notification] [import-on-notification]
# notifier.callback runs a script when a notification (sent by a parent PBS) # notifier.callback runs a script when a notification (sent by a parent PBS)
# is received # is received
......
...@@ -128,14 +128,10 @@ rotate-num = 30 ...@@ -128,14 +128,10 @@ rotate-num = 30
#---------------- #----------------
#-- #--
#-- Resiliency script for the bully algorithm #-- Resiliency script for the node takeover
[resiliency] [resiliency]
# If enable-bully-service is true, the scripts will be run automatically.
# If false, they can be run with bin/bullly for all the PBSReady instances.
enable-bully-service = False
recipe = slapos.cookbook:addresiliency recipe = slapos.cookbook:addresiliency
wrapper-bully = bully
wrapper-takeover = takeover wrapper-takeover = takeover
services = $${basedirectory:services} services = $${basedirectory:services}
bin = $${rootdirectory:bin} bin = $${rootdirectory:bin}
......
...@@ -3,12 +3,10 @@ ...@@ -3,12 +3,10 @@
{% macro replicate(namebase, nbbackup) %} {% macro replicate(namebase, nbbackup) %}
request-{{namebase}} request-{{namebase}}
request-{{namebase}}-2
resilient-request-{{namebase}}-public-key-promise resilient-request-{{namebase}}-public-key-promise
{% for id in range(1,nbbackup|int) %} {% for id in range(1,nbbackup|int) %}
request-{{namebase}}-pseudo-replicating-{{id}} request-{{namebase}}-pseudo-replicating-{{id}}
request-{{namebase}}-pseudo-replicating-{{id}}-2
resilient-request-{{namebase}}-pseudo-replicating-{{id}}-public-key-promise resilient-request-{{namebase}}-pseudo-replicating-{{id}}-public-key-promise
{% endfor %} {% endfor %}
......
...@@ -24,10 +24,6 @@ recipe = slapos.cookbook:request ...@@ -24,10 +24,6 @@ recipe = slapos.cookbook:request
config-namebase = {{namebase}} config-namebase = {{namebase}}
software-url = ${slap-connection:software-release-url} software-url = ${slap-connection:software-release-url}
## Every request is double to provide the 3 IPs.
## First, we request with ip-list parameter hardcoded.
## Then, we request the same thing without this.
## XXX-Cedric: is it useful?
[request-{{namebase}}] [request-{{namebase}}]
<= resilient <= resilient
slap-connection slap-connection
...@@ -35,24 +31,15 @@ software-url = ${slap-connection:software-release-url} ...@@ -35,24 +31,15 @@ software-url = ${slap-connection:software-release-url}
software-type = {{typeexport}} software-type = {{typeexport}}
name = {{namebase}}0 name = {{namebase}}0
return = ssh-public-key ssh-url notification-id ip return = ssh-public-key ssh-url notification-id ip
config =
# Resilient related parameters
number authorized-key notify ip-list namebase
{% if slapparameter_dict is defined %}
# Software Instance related parameters
{% for parameter_name in slapparameter_dict.keys() %}{{parameter_name}} {% endfor %}
{% endif %}
config-number = 0 config-number = 0
config-authorized-key = {% for id in range(1,nbbackup|int) %} ${request-pbs-{{namebase}}-{{id}}:connection-ssh-key}{% endfor %} config-authorized-key = {% for id in range(1,nbbackup|int) %} ${request-pbs-{{namebase}}-{{id}}:connection-ssh-key}{% endfor %}
config-notify = {% for id in range(1,nbbackup|int) %} ${request-pbs-{{namebase}}-{{id}}:connection-notification-url}{% endfor %} config-notify = {% for id in range(1,nbbackup|int) %} ${request-pbs-{{namebase}}-{{id}}:connection-notification-url}{% endfor %}
config-ip-list =
# Bubble up all the instance parameters to the requested export instance. # Bubble up all the instance parameters to the requested export instance.
{% if slapparameter_dict is defined %} {% if slapparameter_dict is defined %}
{% for parameter_name, parameter_value in slapparameter_dict.items() %}config-{{parameter_name}} = {{parameter_value}} {% for parameter_name, parameter_value in slapparameter_dict.items() %}config-{{parameter_name}} = {{parameter_value}}
{% endfor %} {% endfor %}
{% endif %} {% endif %}
{% if sla_parameter_dict == {} -%} {% if sla_parameter_dict == {} -%}
sla = mode
sla-mode = unique_by_network sla-mode = unique_by_network
{% else %} {% else %}
{% set sla_key_main = "-sla-%s%s-" % (namebase, 0) -%} {% set sla_key_main = "-sla-%s%s-" % (namebase, 0) -%}
...@@ -67,13 +54,10 @@ sla-mode = unique_by_network ...@@ -67,13 +54,10 @@ sla-mode = unique_by_network
{% do sla_dict.__setitem__(key[sla_key_secondary_length:], sla_parameter_dict.get(key)) -%} {% do sla_dict.__setitem__(key[sla_key_secondary_length:], sla_parameter_dict.get(key)) -%}
{% endif -%} {% endif -%}
{% endfor -%} {% endfor -%}
{% if sla_dict %}
sla = {{ ' '.join(sla_dict.keys()) }}
{% for key, value in sla_dict.iteritems() -%} {% for key, value in sla_dict.iteritems() -%}
sla-{{ key }} = {{ value }} sla-{{ key }} = {{ value }}
{% endfor -%} {% endfor -%}
{% endif -%} {% endif -%}
{% endif -%}
{% for id in range(1,nbbackup|int) %} {% for id in range(1,nbbackup|int) %}
...@@ -90,13 +74,10 @@ return = ssh-public-key ssh-url notification-url ip ...@@ -90,13 +74,10 @@ return = ssh-public-key ssh-url notification-url ip
pbs-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-{{id}}-push pbs-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-{{id}}-push
config = number authorized-key on-notification ip-list namebase
config-number = {{id}} config-number = {{id}}
config-authorized-key = ${request-pbs-{{namebase}}-{{id}}:connection-ssh-key} config-authorized-key = ${request-pbs-{{namebase}}-{{id}}:connection-ssh-key}
config-on-notification = ${request-pbs-{{namebase}}-{{id}}:connection-feeds-url}${:pbs-notification-id} config-on-notification = ${request-pbs-{{namebase}}-{{id}}:connection-feeds-url}${:pbs-notification-id}
config-ip-list =
{% if sla_parameter_dict == {} -%} {% if sla_parameter_dict == {} -%}
sla = mode
sla-mode = unique_by_network sla-mode = unique_by_network
{% else %} {% else %}
{% set sla_key_main = "-sla-%s%s-" % (namebase, id) -%} {% set sla_key_main = "-sla-%s%s-" % (namebase, id) -%}
...@@ -111,23 +92,17 @@ sla-mode = unique_by_network ...@@ -111,23 +92,17 @@ sla-mode = unique_by_network
{% do sla_dict.__setitem__(key[sla_key_secondary_length:], sla_parameter_dict.get(key)) -%} {% do sla_dict.__setitem__(key[sla_key_secondary_length:], sla_parameter_dict.get(key)) -%}
{% endif -%} {% endif -%}
{% endfor -%} {% endfor -%}
{% if sla_dict %}
sla = {{ ' '.join(sla_dict.keys()) }}
{% for key, value in sla_dict.iteritems() -%} {% for key, value in sla_dict.iteritems() -%}
sla-{{ key }} = {{ value }} sla-{{ key }} = {{ value }}
{% endfor -%} {% endfor -%}
{% endif %} {% endif %}
{% endif %}
{% endfor -%} [publish-connection-informations]
feed-url-{{namebase}}-{{id}}-push = ${request-pbs-{{namebase}}-{{id}}:connection-feeds-url}${request-{{namebase}}-pseudo-replicating-{{id}}:pbs-notification-id}
{% endfor -%}
[iplist]
config-ip-list = ${request-{{namebase}}:connection-ip}{% for j in range(1,nbbackup|int) %} ${request-{{namebase}}-pseudo-replicating-{{j}}:connection-ip}{% endfor %}
[request-{{namebase}}-2]
<= request-{{namebase}}
iplist
[resilient-request-{{namebase}}-public-key-promise] [resilient-request-{{namebase}}-public-key-promise]
# Check that public-key-value parameter exists and is not empty # Check that public-key-value parameter exists and is not empty
...@@ -135,7 +110,7 @@ config-ip-list = ${request-{{namebase}}:connection-ip}{% for j in range(1,nbback ...@@ -135,7 +110,7 @@ config-ip-list = ${request-{{namebase}}:connection-ip}{% for j in range(1,nbback
recipe = collective.recipe.template recipe = collective.recipe.template
# XXX: don't use system executable # XXX: don't use system executable
input = inline:#!/bin/sh input = inline:#!/bin/sh
PUBLIC_KEY_CONTENT="${request-{{namebase}}-2:connection-ssh-public-key})" PUBLIC_KEY_CONTENT="${request-{{namebase}}:connection-ssh-public-key})"
if [[ ! -n "$PUBLIC_KEY_CONTENT" || "$PUBLIC_KEY_CONTENT" == *None* ]]; then if [[ ! -n "$PUBLIC_KEY_CONTENT" || "$PUBLIC_KEY_CONTENT" == *None* ]]; then
exit 1 exit 1
fi fi
...@@ -143,9 +118,6 @@ output = ${resilient-directory:promise}/resilient-request-{{namebase}}-public-ke ...@@ -143,9 +118,6 @@ output = ${resilient-directory:promise}/resilient-request-{{namebase}}-public-ke
mode = 700 mode = 700
{% for id in range(1,nbbackup|int) %} {% for id in range(1,nbbackup|int) %}
[request-{{namebase}}-pseudo-replicating-{{id}}-2]
<= request-{{namebase}}-pseudo-replicating-{{id}}
iplist
[resilient-request-{{namebase}}-pseudo-replicating-{{id}}-public-key-promise] [resilient-request-{{namebase}}-pseudo-replicating-{{id}}-public-key-promise]
# Check that public-key-value parameter exists and is not empty # Check that public-key-value parameter exists and is not empty
...@@ -153,7 +125,7 @@ mode = 700 ...@@ -153,7 +125,7 @@ mode = 700
recipe = collective.recipe.template recipe = collective.recipe.template
# XXX: don't use system executable # XXX: don't use system executable
input = inline:#!/bin/sh input = inline:#!/bin/sh
PUBLIC_KEY_CONTENT="${request-{{namebase}}-pseudo-replicating-{{id}}-2:connection-ssh-public-key})" PUBLIC_KEY_CONTENT="${request-{{namebase}}-pseudo-replicating-{{id}}:connection-ssh-public-key})"
if [[ ! -n "$PUBLIC_KEY_CONTENT" || "$PUBLIC_KEY_CONTENT" == *None* ]]; then if [[ ! -n "$PUBLIC_KEY_CONTENT" || "$PUBLIC_KEY_CONTENT" == *None* ]]; then
exit 1 exit 1
fi fi
...@@ -185,7 +157,6 @@ name = PBS ({{namebase}} / {{id}}) ...@@ -185,7 +157,6 @@ name = PBS ({{namebase}} / {{id}})
return = ssh-key notification-url feeds-url return = ssh-key notification-url feeds-url
slave = false slave = false
{% if sla_parameter_dict == {} -%} {% if sla_parameter_dict == {} -%}
sla = mode
sla-mode = unique_by_network sla-mode = unique_by_network
{% else %} {% else %}
{% set sla_key_main = "-sla-%s%s-" % ("pbs", id) -%} {% set sla_key_main = "-sla-%s%s-" % ("pbs", id) -%}
...@@ -200,13 +171,10 @@ sla-mode = unique_by_network ...@@ -200,13 +171,10 @@ sla-mode = unique_by_network
{% do sla_dict.__setitem__(key[sla_key_secondary_length:], sla_parameter_dict.get(key)) -%} {% do sla_dict.__setitem__(key[sla_key_secondary_length:], sla_parameter_dict.get(key)) -%}
{% endif -%} {% endif -%}
{% endfor -%} {% endfor -%}
{% if sla_dict %}
sla = {{ ' '.join(sla_dict.keys()) }}
{% for key, value in sla_dict.iteritems() -%} {% for key, value in sla_dict.iteritems() -%}
sla-{{ key }} = {{ value }} sla-{{ key }} = {{ value }}
{% endfor %} {% endfor %}
{% endif %} {% endif %}
{% endif %}
[resilient-request-pbs-{{namebase}}-{{id}}-public-key-promise] [resilient-request-pbs-{{namebase}}-{{id}}-public-key-promise]
# Check that public-key-value parameter exists and is not empty # Check that public-key-value parameter exists and is not empty
...@@ -225,7 +193,6 @@ mode = 700 ...@@ -225,7 +193,6 @@ mode = 700
[request-pull-backup-server-{{namebase}}-{{id}}] [request-pull-backup-server-{{namebase}}-{{id}}]
<= request-pbs-common <= request-pbs-common
name = PBS {{id}} pulling from ${request-{{namebase}}:name} name = PBS {{id}} pulling from ${request-{{namebase}}:name}
config = url name type server-key on-notification notify notification-id title remove-backup-older-than
config-url = ${request-{{namebase}}:connection-ssh-url} config-url = ${request-{{namebase}}:connection-ssh-url}
config-type = pull config-type = pull
config-server-key = ${request-{{namebase}}:connection-ssh-public-key} config-server-key = ${request-{{namebase}}:connection-ssh-public-key}
...@@ -236,13 +203,15 @@ config-name = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{n ...@@ -236,13 +203,15 @@ config-name = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{n
config-title = Pulling from {{namebase}} config-title = Pulling from {{namebase}}
config-remove-backup-older-than = {{ slapparameter_dict.get('remove-backup-older-than', '2W') }} config-remove-backup-older-than = {{ slapparameter_dict.get('remove-backup-older-than', '2W') }}
slave = true slave = true
sla = instance_guid
sla-instance_guid = ${request-pbs-{{namebase}}-{{id}}:instance_guid} sla-instance_guid = ${request-pbs-{{namebase}}-{{id}}:instance_guid}
[publish-connection-informations]
feed-url-{{namebase}}-{{id}}-pull = ${request-pbs-{{namebase}}-{{id}}:connection-feeds-url}${request-pull-backup-server-{{namebase}}-{{id}}:config-notification-id}
[request-pull-backup-server-{{namebase}}-backup-{{id}}] [request-pull-backup-server-{{namebase}}-backup-{{id}}]
<= request-pbs-common <= request-pbs-common
name = PBS pushing on ${request-{{namebase}}-pseudo-replicating-{{id}}:name} name = PBS pushing on ${request-{{namebase}}-pseudo-replicating-{{id}}:name}
config = url name type server-key on-notification notify notification-id title
config-url = ${request-{{namebase}}-pseudo-replicating-{{id}}:connection-ssh-url} config-url = ${request-{{namebase}}-pseudo-replicating-{{id}}:connection-ssh-url}
config-type = push config-type = push
config-server-key = ${request-{{namebase}}-pseudo-replicating-{{id}}:connection-ssh-public-key} config-server-key = ${request-{{namebase}}-pseudo-replicating-{{id}}:connection-ssh-public-key}
...@@ -252,7 +221,6 @@ config-notification-id = ${request-{{namebase}}-pseudo-replicating-{{id}}:pbs-no ...@@ -252,7 +221,6 @@ config-notification-id = ${request-{{namebase}}-pseudo-replicating-{{id}}:pbs-no
config-name = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-{{id}} config-name = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-{{id}}
config-title = Pushing to {{namebase}} backup {{id}} config-title = Pushing to {{namebase}} backup {{id}}
slave = true slave = true
sla = instance_guid
sla-instance_guid = ${request-pbs-{{namebase}}-{{id}}:instance_guid} sla-instance_guid = ${request-pbs-{{namebase}}-{{id}}:instance_guid}
{% endfor %} {% endfor %}
......
...@@ -16,6 +16,7 @@ download-cache = ...@@ -16,6 +16,7 @@ download-cache =
extensions += extensions +=
buildout-versions buildout-versions
mr.developer mr.developer
slapos.extension.strip
# Use shacache and lxml # Use shacache and lxml
extends = extends =
...@@ -111,34 +112,36 @@ Werkzeug = 0.9.6 ...@@ -111,34 +112,36 @@ Werkzeug = 0.9.6
buildout-versions = 1.7 buildout-versions = 1.7
cffi = 0.8.6 cffi = 0.8.6
cmd2 = 0.6.7 cmd2 = 0.6.7
cryptography = 0.5.4 cryptography = 0.6
itsdangerous = 0.24 itsdangerous = 0.24
lxml = 3.4.0 lxml = 3.4.0
meld3 = 1.0.0 meld3 = 1.0.0
mr.developer = 1.30 mr.developer = 1.31
prettytable = 0.7.2 prettytable = 0.7.2
pyOpenSSL = 0.14 pyOpenSSL = 0.14
pyparsing = 2.0.2 pyparsing = 2.0.3
setuptools = 5.8 setuptools = 7.0
six = 1.8.0 six = 1.8.0
simplejson = 3.6.5
slapos.cookbook = 0.87 slapos.cookbook = 0.87
slapos.core = 1.2.4 slapos.core = 1.3.3
slapos.libnetworkcache = 0.14.1 slapos.libnetworkcache = 0.14.2
slapos.recipe.build = 0.14
slapos.recipe.cmmi = 0.2 slapos.recipe.cmmi = 0.2
stevedore = 1.0.0 stevedore = 1.1.0
# Required by: # Required by:
# slapos.core==1.2.4 # slapos.core==1.3.3
Flask = 0.10.1 Flask = 0.10.1
# Required by: # Required by:
# cliff==1.7.0 # cliff==1.8.0
# stevedore==1.0.0 # stevedore==1.1.0
argparse = 1.2.1 argparse = 1.2.1
# Required by: # Required by:
# slapos.core==1.2.4 # slapos.core==1.3.3
cliff = 1.7.0 cliff = 1.8.0
# Required by: # Required by:
# slapos.cookbook==0.87 # slapos.cookbook==0.87
...@@ -153,12 +156,21 @@ lock-file = 2.0 ...@@ -153,12 +156,21 @@ lock-file = 2.0
netaddr = 0.7.12 netaddr = 0.7.12
# Required by: # Required by:
# slapos.core==1.2.4 # slapos.core==1.3.3
netifaces = 0.10.4 netifaces = 0.10.4
# Required by: # Required by:
# slapos.core==1.2.4 # cliff==1.8.0
psutil = 2.1.2 # stevedore==1.1.0
pbr = 0.10.0
# Required by:
# pbr==0.10.0
pip = 1.5.6
# Required by:
# slapos.core==1.3.3
psutil = 2.1.3
# Required by: # Required by:
# cffi==0.8.6 # cffi==0.8.6
...@@ -166,22 +178,26 @@ pycparser = 2.10 ...@@ -166,22 +178,26 @@ pycparser = 2.10
# Required by: # Required by:
# slapos.cookbook==0.87 # slapos.cookbook==0.87
pytz = 2014.7 pytz = 2014.9
# Required by:
# slapos.core==1.3.3
requests = 2.4.3
# Required by: # Required by:
# slapos.core==1.2.4 # slapos.core==1.3.3
requests = 2.4.1 supervisor = 3.1.3
# Required by: # Required by:
# slapos.core==1.2.4 # slapos.core==1.3.3
supervisor = 3.1.2 uritemplate = 0.6
# Required by: # Required by:
# slapos.cookbook==0.87 # slapos.cookbook==0.87
xml-marshaller = 0.9.7 xml-marshaller = 0.9.7
# Required by: # Required by:
# slapos.core==1.2.4 # slapos.core==1.3.3
zope.interface = 4.1.1 zope.interface = 4.1.1
[networkcache] [networkcache]
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment