Commit c63d05a6 authored by Cédric de Saint Martin's avatar Cédric de Saint Martin

Merge remote-tracking branch 'origin/master' into erp5-cluster

Conflicts:
	stack/erp5/buildout.cfg
	stack/erp5/instance-erp5-cluster.cfg.in
	stack/erp5/instance-mariadb.cfg.in
	stack/erp5/instance-tidstorage.cfg.in
	stack/erp5/instance.cfg.in
parents d181bc75 fc0013d7
Changes
=======
0.95 (2015-02-14)
-----------------
* resiliency stack: allow web takeover to work inside of webrunner/erp5testnode.
* resiliency takeover script: create lock file stating that takeover has been done.
0.94 (2015-02-06)
-----------------
* kvm: allow to configure tap and nat interface at the same time with use-nat and use-tap [d3d65916]
* kvm: use -netdev to configure network interface as -net is now obsolete [27baa9d4]
0.85 (2013-12-03)
-----------------
......
......@@ -30,9 +30,9 @@ md5sum = 2202b18f269ad606d70e1864857ed93c
[apache]
# inspired on http://old.aclark.net/team/aclark/blog/a-lamp-buildout-for-wordpress-and-other-php-apps/
recipe = slapos.recipe.cmmi
version = 2.4.10
version = 2.4.12
url = https://archive.apache.org/dist/httpd/httpd-${:version}.tar.bz2
md5sum = 44543dff14a4ebc1e9e2d86780507156
md5sum = b8dc8367a57a8d548a9b4ce16d264a13
configure-command = cp -ar ${apr:location}/apr-${apr:version} srclib/apr/; cp -ar ${apr-util:location}/apr-util-${apr-util:version} srclib/apr-util; ./configure
configure-options = --prefix=${buildout:parts-directory}/${:_buildout_section_name_}
--disable-static
......
......@@ -8,7 +8,7 @@ parts =
[automake]
recipe = slapos.recipe.cmmi
md5sum = 140e084223bf463a24c1a28427c6aec7
url = http://ftp.gnu.org/gnu/automake/automake-1.12.6.tar.xz
md5sum = 9a1ddb0e053474d9d1105cfe39b0c48d
url = http://ftp.gnu.org/gnu/automake/automake-1.15.tar.xz
environment =
PATH =${autoconf:location}/bin:${perl:location}/bin:${xz-utils:location}/bin:%(PATH)s
......@@ -10,8 +10,8 @@ parts =
[dbus]
recipe = slapos.recipe.cmmi
url = http://dbus.freedesktop.org/releases/dbus/dbus-1.8.14.tar.gz
md5sum = dfb810eda8780cf9debf7169b3ca5fe1
url = http://dbus.freedesktop.org/releases/dbus/dbus-1.8.16.tar.gz
md5sum = 020824a38850501e7d6ba8307a7c5ac3
location = ${buildout:parts-directory}/${:_buildout_section_name_}
configure-options =
--disable-static
......@@ -25,8 +25,8 @@ environment =
[dbus-glib]
recipe = slapos.recipe.cmmi
url = http://dbus.freedesktop.org/releases/dbus-glib/dbus-glib-0.102.tar.gz
md5sum = f76b8558fd575d0106c3a556eaa49184
url = http://dbus.freedesktop.org/releases/dbus-glib/dbus-glib-0.104.tar.gz
md5sum = 5497d2070709cf796f1878c75a72a039
configure-options =
--disable-static
--disable-gtk-doc-html
......
......@@ -26,12 +26,12 @@ md5sum = 5c781723a0d9ed6188960defba8e91cf
# http://ipafont.ipa.go.jp/
[ipaex-fonts]
<= fonts-base
url = http://ipafont.ipa.go.jp/ipaexfont/IPAexfont00201.php
url = http://dl.sourceforge.jp/ipafonts/57330/IPAexfont00201.zip
md5sum = 7bf84182a04a9632268dbcb03f100d05
[ipa-fonts]
<= fonts-base
url = http://ipafont.ipa.go.jp/ipafont/IPAfont00303.php
url = http://dl.sourceforge.jp/ipafonts/51868/IPAfont00303.zip
md5sum = 39a828acf27790adbe4944dfb4d94bb1
[ocrb-fonts]
......
......@@ -9,7 +9,7 @@ parts =
[gmp]
recipe = slapos.recipe.cmmi
version = 6.0.0a
url = ftp://ftp.gmplib.org/pub/gmp-6.0.0/gmp-${:version}.tar.xz
url = https://gmplib.org/download/gmp/gmp-${:version}.tar.xz
md5sum = 1e6da4e434553d2811437aa42c7f7c76
configure-options =
--enable-cxx
......
......@@ -6,14 +6,15 @@ parts =
groonga
extends =
../jemalloc/buildout.cfg
../libstemmer/buildout.cfg
../pkgconfig/buildout.cfg
../zlib/buildout.cfg
[groonga]
recipe = slapos.recipe.cmmi
url = http://packages.groonga.org/source/groonga/groonga-4.1.0.tar.gz
md5sum = 93c763da6c298595da4b6964d6b80769
url = http://packages.groonga.org/source/groonga/groonga-5.0.0.tar.gz
md5sum = a403fd685405fbdb424a3679082790d2
# temporary patch to respect more tokens in natural language mode.
patches =
${:_profile_base_location_}/groonga.patch#9ed02fbe8400402d3eab47eee149978b
......@@ -24,20 +25,21 @@ configure-options =
--disable-benchmark
--disable-document
--disable-groonga-httpd
--with-jemalloc
--with-libstemmer
--with-libstemmer-include=${libstemmer:location}/include
--with-libstemmer-lib=${libstemmer:location}/lib
--without-mecab
--without-kytea
environment =
CPPFLAGS=-I${libstemmer:location}/include -I${zlib:location}/include
LDFLAGS=-Wl,-rpath=${libstemmer:location}/lib -L${libstemmer:location}/lib -Wl,-rpath=${zlib:location}/lib -L${zlib:location}/lib
CPPFLAGS=-I${libstemmer:location}/include -I${libstemmer:location}/include -I${zlib:location}/include
LDFLAGS=-Wl,-rpath=${jemalloc:location}/lib -L${jemalloc:location}/lib -Wl,-rpath=${libstemmer:location}/lib -L${libstemmer:location}/lib -Wl,-rpath=${zlib:location}/lib -L${zlib:location}/lib
[groonga-normalizer-mysql]
recipe = slapos.recipe.cmmi
version = 1.0.6
version = 1.0.8
url = http://packages.groonga.org/source/groonga-normalizer-mysql/groonga-normalizer-mysql-${:version}.tar.gz
md5sum = 12740a835cfdf4b0dac66c3f2f152d84
md5sum = bcc01e8b715f0f9252effcf9d40338c1
location = ${groonga:location}
configure-options =
--disable-static
......
......@@ -5,8 +5,8 @@ parts = libtool
[libtool]
recipe = slapos.recipe.cmmi
md5sum = b5699a6d58f5594cdb0992c5e1f5e57e
url = http://ftp.gnu.org/gnu/libtool/libtool-2.4.3.tar.xz
md5sum = ee9c087775aeb98ce53a9c69da865a55
url = http://ftp.gnu.org/gnu/libtool/libtool-2.4.5.tar.xz
configure-options =
--disable-static
environment =
......
......@@ -7,7 +7,6 @@ extends =
../groonga/buildout.cfg
../jemalloc/buildout.cfg
../libaio/buildout.cfg
../libevent/buildout.cfg
../libxml2/buildout.cfg
../ncurses/buildout.cfg
../openssl/buildout.cfg
......@@ -21,9 +20,9 @@ parts =
[mariadb]
recipe = slapos.recipe.cmmi
version = 10.0.15
version = 10.0.16
url = https://downloads.mariadb.org/f/mariadb-${:version}/source/mariadb-${:version}.tar.gz/from/http:/ftp.osuosl.org/pub/mariadb
md5sum = 07e2ca1ad684c2dc5cb260717c4db41d
md5sum = 7b1a197ea5d4027833a3a2ed26fc4cae
patch-options = -p0
patches =
${:_profile_base_location_}/mariadb_10.0.8_create_system_tables__no_test.patch#a176d491cf45fccd53ee397c70393bc4
......@@ -41,6 +40,7 @@ configure-options =
-DENABLE_DTRACE=0
-DWITH_EXTRA_CHARSETS=complex
-DWITH_EMBEDDED_SERVER=0
-DWITH_JEMALLOC=yes
-DWITHOUT_EXAMPLE_STORAGE_ENGINE=1
-DWITHOUT_MROONGA_STORAGE_ENGINE=1
-DWITHOUT_DAEMON_EXAMPLE=1
......@@ -58,8 +58,8 @@ environment =
# mroonga - a storage engine for MySQL. It provides fast fulltext search feature to all MySQL users.
# http://mroonga.github.com/
recipe = slapos.recipe.cmmi
url = http://packages.groonga.org/source/mroonga/mroonga-4.09.tar.gz
md5sum = 426a572672d8fda3319bb63a7496cd3e
url = http://packages.groonga.org/source/mroonga/mroonga-5.00.tar.gz
md5sum = c0d49ef2ca60b82cd40eb9a842460f55
configure-command = mkdir fake_mariadb_source && ln -s ${mariadb:location}/include/mysql/private fake_mariadb_source/sql && ./configure
configure-options =
--prefix=${buildout:parts-directory}/${:_buildout_section_name_}
......
......@@ -4,7 +4,7 @@ parts =
[noVNC]
recipe = hexagonit.recipe.download
# Post-0.4 release from January 2013
url = http://github.com/kanaka/noVNC/zipball/3b2acc2258d36137a37edfbe0f03a3099189c49d
md5sum = a276be8fa193652bb5de8a271603f11f
# version-0.5.1 release from 29 Nov 2014
url = https://github.com/kanaka/noVNC/archive/v0.5.1.tar.gz
md5sum = ac55b2316b2164b6e09ae3bd89c37cb6
strip-top-level-dir = true
......@@ -9,7 +9,22 @@ parts =
nodejs
[nodejs]
<= nodejs-0.8
<= nodejs-0.10
[nodejs-0.10]
# Server-side Javascript.
recipe = slapos.recipe.cmmi
url = http://nodejs.org/dist/v0.10.36/node-v0.10.36.tar.gz
md5sum = 4b3527b830f2dacaba01aececd509c6f
configure-options =
--openssl-includes=${openssl:location}/include
--openssl-libpath=${openssl:location}/lib
environment =
HOME=${buildout:parts-directory}/${:_buildout_section_name_}
PATH=${pkgconfig:location}/bin:%(PATH)s
PKG_CONFIG_PATH=${openssl:location}/lib/pkgconfig/
CPPFLAGS=-I${zlib:location}/include
LDFLAGS=-Wl,-rpath=${openssl:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
[nodejs-0.8]
# Server-side Javascript.
......
......@@ -6,7 +6,7 @@ parts =
[patch]
recipe = slapos.recipe.cmmi
url = ftp://ftp.gnu.org/gnu/patch/patch-2.7.1.tar.xz
md5sum = e9ae5393426d3ad783a300a338c09b72
url = ftp://ftp.gnu.org/gnu/patch/patch-2.7.3.tar.xz
md5sum = 29b87be845e4662ab0ca0d48a805ecc6
environment =
PATH=${xz-utils:location}/bin:%(PATH)s
......@@ -13,12 +13,13 @@ extends =
[poppler]
recipe = slapos.recipe.cmmi
url = http://poppler.freedesktop.org/poppler-0.26.0.tar.xz
md5sum = 76c3b2f18002a0f1e2bbbc7cccd1b957
url = http://poppler.freedesktop.org/poppler-0.30.0.tar.xz
md5sum = fb0a9d82b5433c823421a123eb8e9789
configure-options =
--disable-cairo-output
--disable-cms
--disable-libcurl
--disable-libopenjpeg
--disable-gtk-doc-html
--disable-gtk-test
--disable-poppler-cpp
......
......@@ -5,8 +5,6 @@ extends =
parts =
pycrypto-python
versions = versions
[pycrypto-python]
recipe = zc.recipe.egg:custom
egg = pycrypto
......@@ -22,6 +20,3 @@ rpath =
ac_cv_func_malloc_0_nonnull = yes
LDFLAGS = -L${gmp:location}/lib
CPPFLAGS = -I${gmp:location}/include
[versions]
pycrypto = 2.6.1
......@@ -27,9 +27,9 @@ python = python2.7
[python2.7]
recipe = slapos.recipe.cmmi
package_version = 2.7.8
package_version = 2.7.9
package_version_suffix =
md5sum = d235bdfa75b8396942e360a70487ee00
md5sum = 38d530f7efc373d64a8fb1637e3baaa7
# This is actually the default setting for prefix, but we can't use it in
# other settings in this part if we don't set it explicitly here.
......@@ -39,8 +39,6 @@ executable = ${:prefix}/bin/python${:version}
patch-options = -p1
patches =
${:_profile_base_location_}/tls_sni.patch#c95af105e6e96aaa58a50137595872a0
${:_profile_base_location_}/tls_sni_httplib.patch#5c9d00d23b85169df792a936a056cbcc
${:_profile_base_location_}/fix_compiler_module_issue_20613.patch#94443a77f903e9de880a029967fa6aa7
url =
http://python.org/ftp/python/${:package_version}/Python-${:package_version}${:package_version_suffix}.tar.xz
......
Description: Support TLS SNI extension in ssl module
Author: markk
Bug-Python: http://bugs.python.org/issue5639
--- a/Lib/ssl.py
+++ b/Lib/ssl.py
@@ -202,6 +202,7 @@
def __init__(self, sock, keyfile=None, certfile=None,
server_side=False, cert_reqs=CERT_NONE,
ssl_version=PROTOCOL_SSLv23, ca_certs=None,
+ server_hostname=None,
do_handshake_on_connect=True,
suppress_ragged_eofs=True, ciphers=None):
# Can't use sock.type as other flags (such as SOCK_NONBLOCK) get
@@ -238,6 +239,7 @@
self._sslobj = _ssl.sslwrap(self._sock, server_side,
keyfile, certfile,
cert_reqs, ssl_version, ca_certs,
+ server_hostname,
ciphers)
if do_handshake_on_connect:
self.do_handshake()
@@ -246,6 +248,7 @@
self.cert_reqs = cert_reqs
self.ssl_version = ssl_version
self.ca_certs = ca_certs
+ self.server_hostname = server_hostname
self.ciphers = ciphers
self.do_handshake_on_connect = do_handshake_on_connect
self.suppress_ragged_eofs = suppress_ragged_eofs
@@ -411,7 +414,7 @@
raise ValueError("attempt to connect already-connected SSLSocket!")
self._sslobj = _ssl.sslwrap(self._sock, False, self.keyfile, self.certfile,
self.cert_reqs, self.ssl_version,
- self.ca_certs, self.ciphers)
+ self.ca_certs, self.server_hostname, self.ciphers)
try:
if return_errno:
rc = socket.connect_ex(self, addr)
@@ -452,6 +455,7 @@
cert_reqs=self.cert_reqs,
ssl_version=self.ssl_version,
ca_certs=self.ca_certs,
+ server_hostname=None,
ciphers=self.ciphers,
do_handshake_on_connect=self.do_handshake_on_connect,
suppress_ragged_eofs=self.suppress_ragged_eofs),
@@ -566,7 +570,7 @@
sock = sock._sock
ssl_sock = _ssl.sslwrap(sock, 0, keyfile, certfile, CERT_NONE,
- PROTOCOL_SSLv23, None)
+ PROTOCOL_SSLv23, None, None, None)
try:
sock.getpeername()
except socket_error:
--- a/Modules/_ssl.c
+++ b/Modules/_ssl.c
@@ -267,7 +267,7 @@
enum py_ssl_server_or_client socket_type,
enum py_ssl_cert_requirements certreq,
enum py_ssl_version proto_version,
- char *cacerts_file, char *ciphers)
+ char *cacerts_file, char *server_hostname, char *ciphers)
{
PySSLObject *self;
char *errstr = NULL;
@@ -389,6 +389,14 @@
PySSL_BEGIN_ALLOW_THREADS
self->ssl = SSL_new(self->ctx); /* New ssl struct */
+#if OPENSSL_VERSION_NUMBER >= 0x0090806fL && !defined(OPENSSL_NO_TLSEXT)
+ /* If SNI isn't supported, we just don't call it and fail silently,
+ * as there's not much else we can do.
+ */
+ if ((socket_type == PY_SSL_CLIENT) &&
+ (proto_version != PY_SSL_VERSION_SSL2) && server_hostname)
+ SSL_set_tlsext_host_name(self->ssl, server_hostname);
+#endif
PySSL_END_ALLOW_THREADS
SSL_set_fd(self->ssl, Sock->sock_fd); /* Set the socket for SSL */
#ifdef SSL_MODE_AUTO_RETRY
@@ -431,15 +439,16 @@
char *key_file = NULL;
char *cert_file = NULL;
char *cacerts_file = NULL;
+ char *server_hostname = NULL;
char *ciphers = NULL;
- if (!PyArg_ParseTuple(args, "O!i|zziizz:sslwrap",
+ if (!PyArg_ParseTuple(args, "O!i|zziizzz:sslwrap",
PySocketModule.Sock_Type,
&Sock,
&server_side,
&key_file, &cert_file,
&verification_mode, &protocol,
- &cacerts_file, &ciphers))
+ &cacerts_file, &server_hostname, &ciphers))
return NULL;
/*
@@ -452,13 +461,13 @@
return (PyObject *) newPySSLObject(Sock, key_file, cert_file,
server_side, verification_mode,
- protocol, cacerts_file,
+ protocol, cacerts_file, server_hostname,
ciphers);
}
PyDoc_STRVAR(ssl_doc,
"sslwrap(socket, server_side, [keyfile, certfile, certs_mode, protocol,\n"
-" cacertsfile, ciphers]) -> sslobject");
+" cacertsfile, ciphers, server_hostname]) -> sslobject");
/* SSL object methods */
Author: Arnaud Fontaine <arnaud.fontaine@nexedi.com>
Description: Enable TLS SNI support for httplib
--- a/Lib/httplib.py 2014-07-31 14:50:21.178088529 +0900
+++ b/Lib/httplib.py 2014-07-31 20:11:09.279081382 +0900
@@ -1195,7 +1195,12 @@
if self._tunnel_host:
self.sock = sock
self._tunnel()
- self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file)
+ server_hostname = self._tunnel_host
+ else:
+ server_hostname = self.host
+
+ self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file,
+ server_hostname=server_hostname)
__all__.append("HTTPSConnection")
--- a/Lib/ssl.py 2014-07-31 19:33:21.911968158 +0900
+++ b/Lib/ssl.py 2014-07-31 19:33:57.428391985 +0900
@@ -481,14 +481,15 @@
server_side=False, cert_reqs=CERT_NONE,
ssl_version=PROTOCOL_SSLv23, ca_certs=None,
do_handshake_on_connect=True,
- suppress_ragged_eofs=True, ciphers=None):
+ suppress_ragged_eofs=True, ciphers=None,
+ server_hostname=None):
return SSLSocket(sock, keyfile=keyfile, certfile=certfile,
server_side=server_side, cert_reqs=cert_reqs,
ssl_version=ssl_version, ca_certs=ca_certs,
do_handshake_on_connect=do_handshake_on_connect,
suppress_ragged_eofs=suppress_ragged_eofs,
- ciphers=ciphers)
+ ciphers=ciphers, server_hostname=server_hostname)
# some utility functions
......@@ -15,8 +15,8 @@ extends =
[kvm]
recipe = slapos.recipe.cmmi
# qemu-kvm and qemu are now the same since 1.3.
url = http://wiki.qemu-project.org/download/qemu-1.6.1.tar.bz2
md5sum = 3a897d722457c5a895cd6ac79a28fda0
url = http://wiki.qemu-project.org/download/qemu-2.2.0.tar.bz2
md5sum = f7a5e2da22d057eb838a91da7aff43c8
configure-options =
--target-list=x86_64-softmmu
--enable-system
......@@ -55,9 +55,9 @@ configure-options =
[debian-amd64-netinst.iso]
# Download the installer of Debian 7 (Wheezy)
recipe = hexagonit.recipe.download
url = http://cdimage.debian.org/debian-cd/7.2.0/amd64/iso-cd/debian-7.2.0-amd64-netinst.iso
url = http://cdimage.debian.org/debian-cd/7.8.0/amd64/iso-cd/debian-7.8.0-amd64-netinst.iso
filename = ${:_buildout_section_name_}
md5sum = b86774fe4de88be6378ba3d71b8029bd
md5sum = a91fba5001cf0fbccb44a7ae38c63b6e
download-only = true
mode = 0644
location = ${buildout:parts-directory}/${:_buildout_section_name_}
......
......@@ -5,8 +5,8 @@ parts =
[sqlite3]
recipe = slapos.recipe.cmmi
url = http://www.sqlite.org/2014/sqlite-autoconf-3080704.tar.gz
md5sum = 33bb8db0038317ce1b0480ca1185c7ba
url = http://sqlite.org/2015/sqlite-autoconf-3080801.tar.gz
md5sum = a6381941ffe8817ba19669ec0c0ede6f
configure-options =
--disable-static
--enable-readline
......
......@@ -6,8 +6,8 @@ parts =
[stunnel]
recipe = slapos.recipe.cmmi
url = https://www.stunnel.org/downloads/stunnel-5.02.tar.gz
md5sum = bb48b1c18cfc0a42708ef996b1a26926
url = https://www.stunnel.org/downloads/stunnel-5.10.tar.gz
md5sum = a0edda805eb7d6ea600a230fb0979ea1
configure-options =
--enable-ipv6
--disable-libwrap
......
[buildout]
extends =
../autoconf/buildout.cfg
../automake/buildout.cfg
../jbigkit/buildout.cfg
../leptonica/buildout.cfg
../libpng/buildout.cfg
../libtool/buildout.cfg
../leptonica/buildout.cfg
../jbigkit/buildout.cfg
../webp/buildout.cfg
parts =
......@@ -20,9 +21,8 @@ patches =
${:_profile_base_location_}/tesseract-3.00-gcc-4.7-build.patch#ca80db3ec489c547b03f3ee48879c1b1
${:_profile_base_location_}/tesseract-3.01-remove-bom.patch#2e691858cb492b7c17d23bf0912b3d24
configure-command =
aclocal -I ${libtool:location}/share/aclocal -I config
libtoolize -f -c
libtoolize --automake
aclocal -I ${libtool:location}/share/aclocal -I config
autoheader -f
automake -c -a -f
autoconf -Wno-portability
......
......@@ -28,7 +28,7 @@ from setuptools import setup, find_packages
import glob
import os
version = '0.93'
version = '0.96.dev0'
name = 'slapos.cookbook'
long_description = open("README.txt").read() + "\n" + \
open("CHANGES.txt").read() + "\n"
......
......@@ -43,7 +43,7 @@ class Recipe(GenericSlapRecipe):
slap_connection = self.buildout['slap-connection']
takeover_wrapper = self.createPythonScript(
name=os.path.join(self.options['bin'], self.options['wrapper-takeover']),
name=self.options['wrapper-takeover'],
absolute_function='slapos.recipe.addresiliency.takeover.run',
arguments={
'server_url': slap_connection['server-url'],
......@@ -53,6 +53,7 @@ class Recipe(GenericSlapRecipe):
'partition_id': slap_connection['partition-id'],
'software': slap_connection['software-release-url'],
'namebase': self.parameter_dict['namebase'],
'takeover_triggered_file_path': self.options['takeover-triggered-file-path'],
})
path_list.append(takeover_wrapper)
......
......@@ -11,7 +11,8 @@ logging.basicConfig(level=logging.DEBUG)
def takeover(server_url, key_file, cert_file, computer_guid,
partition_id, software_release, namebase,
winner_instance_suffix = None):
winner_instance_suffix = None,
takeover_triggered_file_path=None):
"""
This function does
......@@ -74,6 +75,10 @@ def takeover(server_url, key_file, cert_file, computer_guid,
# Note: Root instance will reconfigure itself the winning instance (software_type
# and parameters.)
# Create "lock" file preventing equeue to run import scripts
# XXX hardcoded
open(takeover_triggered_file_path, 'w').write('')
def run(args):
slapos.recipe.addresiliency.takeover.takeover(server_url = args.pop('server_url'),
key_file = args.pop('key_file'),
......@@ -81,5 +86,6 @@ def run(args):
computer_guid = args.pop('computer_id'),
partition_id = args.pop('partition_id'),
software_release = args.pop('software'),
namebase = args.pop('namebase'))
namebase = args.pop('namebase'),
takeover_triggered_file_path = args.pop('takeover_triggered_file_path'))
......@@ -36,6 +36,9 @@ class Recipe(GenericBaseRecipe):
'--lockfile', self.options['lockfile']
]
if 'takeover-triggered-file-path' in self.options:
parameters.extend(['--takeover-triggered-file-path', self.options['takeover-triggered-file-path']])
if 'loglevel' in self.options:
parameters.extend(['--loglevel', self.options['loglevel']])
......
......@@ -80,16 +80,19 @@ class Recipe(GenericBaseRecipe):
mysql_script_list = []
# user defined functions
udf_registration = ""
mroonga = self.options.get('mroonga', 'ha_mroonga.so')
if mroonga:
last_insert_grn_id = "CREATE FUNCTION last_insert_grn_id RETURNS " \
"INTEGER SONAME '" + mroonga + "';"
else:
last_insert_grn_id = ""
udf_registration += "CREATE FUNCTION last_insert_grn_id RETURNS " \
"INTEGER SONAME '" + mroonga + "';\n"
udf_registration += "CREATE FUNCTION mroonga_snippet RETURNS " \
"STRING SONAME '" + mroonga + "';\n"
udf_registration += "CREATE FUNCTION mroonga_command RETURNS " \
"STRING SONAME '" + mroonga + "';\n"
mysql_script_list.append(self.substituteTemplate(
self.getTemplateFilename('mysql-init-function.sql.in'),
{
'last_insert_grn_id': last_insert_grn_id,
'udf_registration': udf_registration,
}
))
# real database
......
USE mysql;
DROP FUNCTION IF EXISTS last_insert_grn_id;
%(last_insert_grn_id)s
DROP FUNCTION IF EXISTS mroonga_snippet;
DROP FUNCTION IF EXISTS mroonga_command;
DROP FUNCTION IF EXISTS sphinx_snippets;
#CREATE FUNCTION sphinx_snippets RETURNS STRING SONAME 'ha_sphinx.so';
%(udf_registration)s
......@@ -45,7 +45,7 @@ class Recipe(GenericBaseRecipe):
path_list = []
if not self.isTrueValue(self.options.get('use-tap')):
if self.isTrueValue(self.options.get('use-nat')):
# XXX This could be done using Jinja.
for port in self.options['nat-rules'].split():
tunnel_port = int(port) + 10000
......
......@@ -7,6 +7,8 @@ import os
import socket
import subprocess
import urllib
import gzip
import shutil
# XXX: give all of this through parameter, don't use this as template, but as module
qemu_img_path = '%(qemu-img-path)s'
......@@ -19,11 +21,14 @@ default_disk_image = '%(default-disk-image)s'
disk_path = '%(disk-path)s'
virtual_hard_drive_url = '%(virtual-hard-drive-url)s'.strip()
virtual_hard_drive_md5sum = '%(virtual-hard-drive-md5sum)s'.strip()
virtual_hard_drive_gzipped = '%(virtual-hard-drive-gzipped)s'.strip()
nat_rules = '%(nat-rules)s'.strip()
use_tap = '%(use-tap)s'
use_nat = '%(use-nat)s'
tap_interface = '%(tap-interface)s'
listen_ip = '%(ipv4)s'
mac_address = '%(mac-address)s'
tap_mac_address = '%(tap-mac-address)s'
smp_count = '%(smp-count)s'
ram_size = '%(ram-size)s'
pid_file_path = '%(pid-file-path)s'
......@@ -61,21 +66,29 @@ def getSocketStatus(host, port):
if not os.path.exists(disk_path) and virtual_hard_drive_url != '':
print('Downloading virtual hard drive...')
try:
urllib.urlretrieve(virtual_hard_drive_url, disk_path)
downloaded_disk = disk_path
if virtual_hard_drive_gzipped == 'True':
downloaded_disk = '%%s.gz' %% disk_path
urllib.urlretrieve(virtual_hard_drive_url, downloaded_disk)
except:
os.remove(disk_path)
os.remove(downloaded_disk)
raise
md5sum = virtual_hard_drive_md5sum.strip()
if md5sum:
print('Checking MD5 checksum...')
local_md5sum = md5Checksum(disk_path)
local_md5sum = md5Checksum(downloaded_disk)
if local_md5sum != md5sum:
os.remove(disk_path)
os.remove(downloaded_disk)
raise Exception('MD5 mismatch. MD5 of local file is %%s, Specified MD5 is %%s.' %% (
local_md5sum, md5sum))
print('MD5sum check passed.')
else:
print('Warning: not checksum specified.')
if downloaded_disk.endswith('.gz'):
with open(disk_path, 'w') as disk:
with gzip.open(downloaded_disk, 'rb') as disk_gz:
shutil.copyfileobj(disk_gz, disk)
os.remove(downloaded_disk)
# Create disk if doesn't exist
# XXX: move to Buildout profile
......@@ -87,22 +100,35 @@ if not os.path.exists(disk_path):
# Generate network parameters
# XXX: use_tap should be a boolean
tap_network_parameter = []
nat_network_parameter = []
number = -1
if use_nat == 'True':
number += 1
rules = 'user,id=lan%%s,' %% number + ','.join('hostfwd=tcp:%%s:%%s-:%%s' %% (listen_ip,
int(port) + 10000, port) for port in nat_rules.split())
nat_network_parameter = ['-netdev', rules,
'-device', 'e1000,netdev=lan%%s,mac=%%s' %% (number, mac_address)]
if use_tap == 'True':
qemu_network_parameter = 'tap,ifname=%%s,script=no,downscript=no' %% tap_interface
else:
qemu_network_parameter = 'user,' + ','.join('hostfwd=tcp:%%s:%%s-:%%s' %% (listen_ip, int(port) + 10000, port) for port in nat_rules.split())
number += 1
tap_network_parameter = ['-netdev',
'tap,id=lan%%s,ifname=%%s,script=no,downscript=no' %% (number,
tap_interface),
'-device', 'e1000,netdev=lan%%s,mac=%%s' %% (number, tap_mac_address)]
kvm_argument_list = [qemu_path,
'-enable-kvm', '-net', 'nic,macaddr=%%s' %% mac_address,
'-net', qemu_network_parameter,
'-smp', smp_count,
'-m', ram_size,
'-enable-kvm', '-smp', smp_count,
'-m', ram_size, '-vga', 'std',
'-drive', 'file=%%s,if=%%s' %% (disk_path, disk_type),
'-vnc', '%%s:1,ipv4,password' %% listen_ip,
'-boot', 'menu=on',
'-boot', 'order=cd,menu=on',
'-qmp', 'unix:%%s,server' %% socket_path,
'-pidfile', pid_file_path,
]
if tap_network_parameter == [] and nat_network_parameter == []:
print 'Warning : No network interface defined.'
else:
kvm_argument_list += nat_network_parameter + tap_network_parameter
# Try to connect to NBD server (and second nbd if defined).
# If not available, don't even specify it in qemu command line parameters.
......
......@@ -85,7 +85,7 @@ command =
[template]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg.in
md5sum = 4c8f07da2217e54163c265fe6fe3d41d
md5sum = 5fdeb07b7baaf0dfa9219f0d6ba1b140
output = ${buildout:directory}/template.cfg
mode = 0644
......@@ -93,7 +93,15 @@ mode = 0644
recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/instance-kvm.cfg.jinja2
mode = 644
md5sum = ac69266206830226185e576fb6e4935a
md5sum = 5ff1c3c0083ad7d331e9e2da7dc601cb
download-only = true
on-update = true
[template-kvm-cluster]
recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/instance-kvm-cluster.cfg.jinja2.in
mode = 644
md5sum = 9dc4b77be3d350b41baaf57d147e07e2
download-only = true
on-update = true
......
......@@ -19,7 +19,7 @@ develop =
[slapos.cookbook-repository]
recipe = slapos.recipe.build:gitclone
repository = http://git.erp5.org/repos/slapos.git
branch = master
branch = kvm-cluster
git-executable = ${git:location}/bin/git
[slapos.core-repository]
......
{
"type": "object",
"$schema": "http://json-schema.org/draft-04/schema",
"title": "Input Parameters",
"properties": {
"frontend": {
"frontend-instance-guid": {
"title": "Frontend Instance ID",
"description": "Unique identifier of the frontend instance, like \"SOFTINST-11031\".",
"type": "string",
"default": "SOFTINST-11031"
},
"frontend-software-type": {
"title": "Frontend Software Type",
"description": "Type of the frontend instance, like \"frontend\".",
"type": "string",
"default": "frontend"
},
"frontend-software-url": {
"title": "Frontend Software URL",
"description": "Software Release URL of the frontend instance, like \"http://example.com/path/to/software.cfg\".",
"type": "string",
"format": "uri",
"default": "http://git.erp5.org/gitweb/slapos.git/blob_plain/refs/tags/slapos-0.92:/software/kvm/software.cfg"
},
"type": "object"
},
"kvm-partition-dict": {
"description": "kvm instances definition",
"patternProperties": {
".*": {
"properties": {
"ram-size": {
"title": "RAM size",
"description": "RAM size, in MB.",
"type": "integer",
"default": 1024,
"minimum": 128,
"multipleOf": 128,
"maximum": 16384
},
"disk-size": {
"title": "Disk size",
"description": "Disk size, in GB.",
"type": "integer",
"default": 10,
"minimum": 1,
"maximum": 80
},
"disk-type": {
"title": "Disk type",
"description": "Type of QEMU disk drive.",
"type": "string",
"default": "virtio",
"enum": ["ide", "scsi", "sd", "mtd", "floppy", "pflash", "virtio"]
},
"cpu-count": {
"title": "CPU count",
"description": "Number of CPU cores.",
"type": "integer",
"minimum": 1,
"maximum": 8
},
"nbd-host": {
"title": "NBD hostname",
"description": "hostname (or IP) of the NBD server containing the boot image.",
"type": "string",
"format": ["host-name", "ip-address", "ipv6"],
"default": "debian.nbd.vifib.net"
},
"nbd-port": {
"title": "NBD port",
"description": "Port of the NBD server containing the boot image.",
"type": "integer",
"default": 1024,
"minimum": 1,
"maximum": 65535
},
"nbd2-host": {
"title": "Second NBD hostname",
"description": "hostname (or IP) of the second NBD server (containing drivers for example).",
"type": "string",
"format": ["host-name", "ip-address", "ipv6"]
},
"nbd2-port": {
"title": "Second NBD port",
"description": "Port of the second NBD server containing the boot image.",
"type": "integer",
"minimum": 1,
"maximum": 65535
},
"virtual-hard-drive-url": {
"title": "Existing disk image URL",
"description": "If specified, will download an existing disk image (qcow2, raw, ...), and will use it as main virtual hard drive. Can be used to download and use an already installed and customized virtual hard drive.",
"format": "uri",
"type": "string"
},
"virtual-hard-drive-md5sum": {
"title": "Checksum of virtual hard drive",
"description": "MD5 checksum of virtual hard drive, used if virtual-hard-drive-url is specified.",
"type": "string"
},
"virtual-hard-drive-gzipped": {
"title": "Define if virtual hard drive to download is gzipped",
"description": "Define if virtual hard drive to download is gzipped using gzip. This help to reduce size of file to download.",
"type": "boolean",
"default": false
},
"use-tap": {
"title": "Use QEMU TAP network interface",
"description": "Use QEMU TAP network interface, requires a bridge on SlapOS Node. If false, use user-mode network stack (NAT).",
"type": "boolean",
"default": false
},
"nat-rules": {
"title": "List of rules for NAT of QEMU user mode network stack.",
"description": "List of rules for NAT of QEMU user mode network stack, as comma-separated list of ports. For each port specified, it will redirect port x of the VM (example: 80) to the port x + 10000 of the public IPv6 (example: 10080). Defaults to \"22 80 443\". Ignored if \"use-tap\" parameter is enabled.",
"type": "string"
}
},
"type": "object"
}
},
"type": "object"
}
}
}
\ No newline at end of file
{% set publish_dict = {} -%}
{% set frontend_dict = slapparameter_dict.get('frontend', {}) -%}
[request-common]
recipe = slapos.cookbook:request
software-url = ${slap-connection:software-release-url}
server-url = ${slap-connection:server-url}
key-file = ${slap-connection:key-file}
cert-file = ${slap-connection:cert-file}
computer-id = ${slap-connection:computer-id}
partition-id = ${slap-connection:partition-id}
config-use-ipv6 = {{ dumps(slapparameter_dict.get('use-ipv6', False)) }}
# Request kvm instances
{% for instance_name, kvm_parameter_dict in slapparameter_dict.get('kvm-partition-dict', {'kvm-default': {}}).items() -%}
{% set section = 'request-' ~ instance_name -%}
[{{ section }}]
<= request-common
software-type = kvm
name = {{ instance_name }}
sla-computer_guid = {{ dumps(kvm_parameter_dict.get('computer-guid', computer_id)) }}
config-frontend-instance-name = {{ instance_name ~ ' VNC Frontend' }}
config-frontend-software-type = {{ dumps(frontend_dict.get('software-type', 'frontend')) }}
config-frontend-software-url = {{ dumps(frontend_dict.get('software-url', 'http://git.erp5.org/gitweb/slapos.git/blob_plain/refs/tags/slapos-0.92:/software/kvm/software.cfg')) }}
config-frontend-instance-guid = {{ dumps(frontend_dict.get('instance-guid', '')) }}
config-nbd-port = {{ dumps(kvm_parameter_dict.get('nbd-port', 1024)) }}
config-nbd-host = {{ dumps(kvm_parameter_dict.get('nbd-host', '')) }}
config-nbd2-port = {{ dumps(kvm_parameter_dict.get('nbd-port2', 1024)) }}
config-nbd2-host = {{ dumps(kvm_parameter_dict.get('nbd-host2', '')) }}
config-ram-size = {{ dumps(kvm_parameter_dict.get('ram-size', 1024)) }}
config-disk-size = {{ dumps(kvm_parameter_dict.get('disk-size', 10)) }}
config-disk-type = {{ dumps(kvm_parameter_dict.get('disk-type', 'virtio')) }}
config-cpu-count = {{ dumps(kvm_parameter_dict.get('cpu-count', 1)) }}
{% set nat_rules_list = kvm_parameter_dict.get('nat-rules', [22, 80, 443]) -%}
config-nat-rules = {{ nat_rules_list | join(' ') }}
config-use-nat = {{ dumps(kvm_parameter_dict.get('use-nat', True)) }}
config-use-tap = {{ dumps(kvm_parameter_dict.get('use-tap', False)) }}
config-virtual-hard-drive-url = {{ dumps(kvm_parameter_dict.get('virtual-hard-drive-url', '')) }}
config-virtual-hard-drive-md5sum = {{ dumps(kvm_parameter_dict.get('virtual-hard-drive-md5sum', '')) }}
config-virtual-hard-drive-gzipped = {{ dumps(kvm_parameter_dict.get('virtual-hard-drive-gzipped', False)) }}
return =
backend-url
url
{% do publish_dict.__setitem__(instance_name ~ '-backend-url', '${' ~ section ~ ':connection-backend-url}') -%}
{% do publish_dict.__setitem__(instance_name ~ '-url', '${' ~ section ~ ':connection-url}') -%}
{% endfor %}
[publish]
recipe = slapos.cookbook:publish
{% for name, value in publish_dict.items() -%}
{{ name }} = {{ value }}
{% endfor %}
[buildout]
parts = publish
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
\ No newline at end of file
......@@ -41,6 +41,10 @@ cronstamps = ${:etc}/cronstamps
recipe = slapos.cookbook:generate.mac
storage-path = ${directory:srv}/mac
[create-tap-mac]
recipe = slapos.cookbook:generate.mac
storage-path = ${directory:srv}/tap_mac
[gen-passwd]
recipe = slapos.cookbook:generate.password
storage-path = ${directory:srv}/passwd
......@@ -79,17 +83,20 @@ pid-file-path = ${directory:run}/pid_file
smp-count = ${slap-parameter:cpu-count}
ram-size = ${slap-parameter:ram-size}
mac-address = ${create-mac:mac-address}
tap-mac-address = ${create-tap-mac:mac-address}
# XXX-Cedric: should be named runner-wrapper-path and controller-wrapper-path
runner-path = ${directory:services}/kvm
controller-path = ${directory:scripts}/kvm_controller
use-tap = ${slap-parameter:use-tap}
use-nat = ${slap-parameter:use-nat}
nat-rules = ${slap-parameter:nat-rules}
6tunnel-wrapper-path = ${directory:services}/6tunnel
virtual-hard-drive-url = ${slap-parameter:virtual-hard-drive-url}
virtual-hard-drive-md5sum = ${slap-parameter:virtual-hard-drive-md5sum}
virtual-hard-drive-gzipped = ${slap-parameter:virtual-hard-drive-gzipped}
shell-path = {{ dash_executable_location }}
qemu-path = {{ qemu_executable_location }}
......@@ -107,7 +114,16 @@ port = ${kvm-instance:vnc-port}
# Check that disk image is not corrupted
recipe = collective.recipe.template
input = inline:#!/bin/sh
# Return code 0 is "OK"
# Return code 3 is "found leaks, but image is OK"
# http://git.qemu.org/?p=qemu.git;a=blob;f=qemu-img.c;h=4e9a7f5741c9cb863d978225829e68fefcae3947;hb=HEAD#l702
${kvm-instance:qemu-img-path} check ${kvm-instance:disk-path}
RETURN_CODE=$?
if [ $RETURN_CODE -eq 0 ] || [ $RETURN_CODE -eq 3 ]; then
exit 0
else
exit 1
fi
output = ${directory:promises}/kvm-disk-image-corruption
mode = 700
......@@ -193,7 +209,7 @@ key-file = ${slap-connection:key-file}
cert-file = ${slap-connection:cert-file}
computer-id = ${slap-connection:computer-id}
partition-id = ${slap-connection:partition-id}
name = VNC Frontend
name = ${slap-parameter:frontend-instance-name}
software-type = ${slap-parameter:frontend-software-type}
slave = true
config-host = ${novnc-instance:ip}
......@@ -227,6 +243,7 @@ url = ${request-slave-frontend:connection-url}/vnc_auto.html?host=${request-slav
frontend-software-type = frontend
frontend-software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/refs/tags/slapos-0.92:/software/kvm/software.cfg
frontend-instance-guid =
frontend-instance-name = VNC Frontend
nbd-port = 1024
nbd-host =
nbd2-port = 1024
......@@ -239,7 +256,9 @@ disk-type = virtio
cpu-count = 1
nat-rules = 22 80 443
use-nat = True
use-tap = False
virtual-hard-drive-url =
virtual-hard-drive-md5sum =
virtual-hard-drive-gzipped = False
......@@ -8,6 +8,7 @@ develop-eggs-directory = ${buildout:develop-eggs-directory}
[switch-softwaretype]
recipe = slapos.cookbook:softwaretype
default = $${:kvm}
kvm-cluster = $${dynamic-template-kvm-cluster:rendered}
kvm = $${dynamic-template-kvm:rendered}
nbd = ${template-nbd:output}
frontend = ${template-frontend:output}
......@@ -30,6 +31,30 @@ url = $${slap-connection:server-url}
key = $${slap-connection:key-file}
cert = $${slap-connection:cert-file}
[jinja2-template-base]
recipe = slapos.recipe.template:jinja2
rendered = $${buildout:directory}/$${:filename}
extensions = jinja2.ext.do
mode = 0644
extra-context =
context =
key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory
key ipv4 slap-configuration:ipv4
key ipv6 slap-configuration:ipv6
key slapparameter_dict slap-configuration:configuration
key computer_id slap-configuration:computer
$${:extra-context}
[dynamic-template-kvm-cluster-parameters]
[dynamic-template-kvm-cluster]
<= jinja2-template-base
template = ${template-kvm-cluster:location}/instance-kvm-cluster.cfg.jinja2.in
filename = template-kvm-cluster.cfg
extra-context =
section parameter_dict dynamic-template-kvm-cluster-parameters
[dynamic-template-kvm]
recipe = slapos.recipe.template:jinja2
template = ${template-kvm:location}/instance-kvm.cfg.jinja2
......
......@@ -3,41 +3,41 @@ extends = common.cfg
[versions]
PyRSS2Gen = 1.1
apache-libcloud = 0.15.1
async = 0.6.1
apache-libcloud = 0.16.0
cns.recipe.symlink = 0.2.3
collective.recipe.template = 1.11
ecdsa = 0.11
erp5.util = 0.4.41
gitdb = 0.5.4
erp5.util = 0.4.42
gitdb = 0.6.4
plone.recipe.command = 1.1
pycrypto = 2.6.1
slapos.recipe.download = 1.0.dev-r4053
slapos.toolbox = 0.40.4
smmap = 0.8.2
slapos.recipe.template = 2.6
slapos.toolbox = 0.46.1
smmap = 0.9.0
websockify = 0.6.0
z3c.recipe.scripts = 1.0.1
# Required by:
# websockify==0.6.0
numpy = 1.8.2
# Required by:
# slapos.toolbox==0.40.4
GitPython = 0.3.2.RC1
# slapos.toolbox==0.46.1
GitPython = 0.3.6
# Required by:
# slapos.toolbox==0.40.4
# slapos.toolbox==0.46.1
atomize = 0.2.0
# Required by:
# slapos.toolbox==0.40.4
# slapos.toolbox==0.46.1
feedparser = 5.1.3
# Required by:
# slapos.toolbox==0.46.1
lockfile = 0.10.2
# Required by:
# websockify==0.6.0
numpy = 1.9.0
numpy = 1.9.1
# Required by:
# slapos.toolbox==0.40.4
# slapos.toolbox==0.46.1
paramiko = 1.15.2
......@@ -38,21 +38,21 @@ parts =
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg
output = ${buildout:directory}/template.cfg
md5sum = 8a47421ac6158b4ee476acab212c67d9
md5sum = bb7e0bf9959c4437ff1e23e645315ccf
mode = 0644
[template-runner]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-runner.cfg
output = ${buildout:directory}/template-runner.cfg.in
md5sum = 3cac749e81d8b94dbbf1f1833bc031ce
md5sum = 67fd0af225c6c4f93f539f82dd23a2db
mode = 0644
[template-runner-import-script]
recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/template/runner-import.sh.jinja2
download-only = true
md5sum = 4134ea7e191d0c3b552d2efbae6b5894
md5sum = 42d083a2ccf5fc49c54ffd7cd67fddc6
filename = runner-import.sh.jinja2
mode = 0644
......@@ -60,14 +60,14 @@ mode = 0644
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-runner-import.cfg.in
output = ${buildout:directory}/instance-runner-import.cfg
md5sum = e9e3692ce4af4d603666c969ec1964d2
md5sum = 3900e8d448879a208ae0cc1ce92c1e6e
mode = 0644
[template-runner-export-script]
recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/template/runner-export.sh.jinja2
download-only = true
md5sum = 41c0213f5cc07ecbe5c2852ef1844ac9
md5sum = 9dc934fe5015ff53869830a833266192
filename = runner-export.sh.jinja2
mode = 0644
......@@ -89,7 +89,7 @@ mode = 0644
recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/instance-resilient-test.cfg.jinja2
download-only = true
md5sum = b95b230bcdd7d4eaf9452f73156b27e8
md5sum = fb8c45e5c35548331fb06c4633ec592a
filename = instance-resilient-test.cfg.jinja2
mode = 0644
......@@ -105,7 +105,7 @@ mode = 0644
recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/httpd_conf.in
download-only = true
md5sum = 94bc5d6f1afc4e075d530fed496f47c2
md5sum = b5d095f54f714d17dff12c0c5fe4afb7
filename = httpd_conf.in
mode = 0644
......@@ -120,7 +120,7 @@ location = ${buildout:parts-directory}/${:_buildout_section_name_}
[template-slapos-cfg]
recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/template/${:filename}
md5sum = ca57a8bf1d4da20344fab602a69b807f
md5sum = c490563b831fc959a09b4b233263a6c6
location = ${buildout:parts-directory}/${:_buildout_section_name_}
filename = slapos.cfg.in
download-only = true
......
......@@ -90,7 +90,7 @@ RewriteCond %{QUERY_STRING} service=git-receive-pack [OR]
RewriteCond %{REQUEST_URI} /git-receive-pack$
<LocationMatch "^/git/">
SetEnv GIT_PROJECT_ROOT:{{- parameters.project_private_folder -}}
SetEnv GIT_PROJECT_ROOT:{{ parameters.project_private_folder }}
Order Deny,Allow
Deny from env=AUTHREQUIRED
......@@ -101,7 +101,7 @@ RewriteCond %{REQUEST_URI} /git-receive-pack$
</LocationMatch>
<LocationMatch "^/git-public/">
SetEnv GIT_PROJECT_ROOT:{{- parameters.project_public_folder -}}
SetEnv GIT_PROJECT_ROOT:{{ parameters.project_public_folder }}
Order Deny,Allow
Deny from env=AUTHREQUIRED
......
......@@ -54,7 +54,7 @@ name = Resilient Instance (Root Instance)
{% for key, value in slapparameter_dict.get('cluster', {}).iteritems() -%}
config-{{ key }} = {{ dumps(value) }}
{% endfor -%}
config-resiliency-backup-periodicity = */6 * * * *
config-resiliency-backup-periodicity = */10 * * * *
config-resilient-clone-number = 1
config-ignore-known-hosts-file = true
# XXX hardcoded
......
......@@ -72,6 +72,7 @@ mode = 700
restore-exit-code-file=$${directory:srv}/importer-exit-code-file
context =
key backend_url slaprunner:access-url
key proxy_host slaprunner:ipv4
section directory directory
raw shell_binary ${dash:location}/bin/dash
raw rsync_binary ${rsync:location}/bin/rsync
......
......@@ -80,7 +80,7 @@ cert-file = $${slap-connection:cert-file}
computer-id = $${slap-connection:computer-id}
partition-id = $${slap-connection:partition-id}
{%- if slapparameter_dict.get('custom-frontend-instance-guid') -%}
{% if slapparameter_dict.get('custom-frontend-instance-guid') -%}
sla-instance_guid = $${slap-parameter:frontend-instance-guid}
{% endif -%}
......
......@@ -59,7 +59,7 @@ context =
mode = 0644
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
recipe = slapos.cookbook:slapconfiguration
computer = $${slap-connection:computer-id}
partition = $${slap-connection:partition-id}
url = $${slap-connection:server-url}
......
......@@ -21,26 +21,26 @@ plone.recipe.command = 1.1
pycrypto = 2.6.1
slapos.recipe.download = 1.0.dev-r4053
slapos.recipe.template = 2.6
slapos.toolbox = 0.46.0
slapos.toolbox = 0.47.3
smmap = 0.9.0
z3c.recipe.scripts = 1.0.1
# Required by:
# slapos.toolbox==0.46.0
# slapos.toolbox==0.47.3
GitPython = 0.3.5
# Required by:
# slapos.toolbox==0.46.0
# slapos.toolbox==0.47.3
atomize = 0.2.0
# Required by:
# slapos.toolbox==0.46.0
# slapos.toolbox==0.47.3
feedparser = 5.1.3
# Required by:
# slapos.toolbox==0.46.0
# slapos.toolbox==0.47.3
lockfile = 0.10.2
# Required by:
# slapos.toolbox==0.45.3
# slapos.toolbox==0.47.3
paramiko = 1.15.2
......@@ -13,6 +13,7 @@ sync_element () {
# Concatenate the exclude file of each partition of webrunner
# to create a global exclude file.
exclude_content="instance/supervisord.socket"
for partition in $srv_directory/runner/instance/slappart*
do
exclude_file="$partition/srv/exporter.exclude"
......
......@@ -13,7 +13,7 @@ restore_element () {
do
cd $backup_path;
if [ -f $element ] || [ -d $element ]; then
{{ rsync_binary }} -av --delete $backup_path/$element $restore_path;
{{ rsync_binary }} -av --delete --exclude *.sock --exclude *.pid --exclude .installed.cfg --exclude .installed-switch-softwaretype.cfg $backup_path/$element $restore_path;
fi
done
}
......@@ -46,6 +46,9 @@ $srv_directory/runner-import-restore || RESTORE_EXIT_CODE=$?
echo "Updating slapproxy database, software release and instances..."
HOME="{{ directory['home'] }}"
# XXX Hardcoded
export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
export MAKEFLAGS=-j4
SLAPOS="{{ directory['bin'] }}/slapos"
# XXX hardcoded
SQLITE3="$HOME/software_release/parts/sqlite3/bin/sqlite3"
......@@ -54,17 +57,34 @@ DATABASE="$HOME/srv/runner/proxy.db"
# XXX hardcoded
PARTITION=$(basename $HOME)
OLD_SOFTWARE_RELEASE=$($SQLITE3 $DATABASE "select software_release from partition11 where reference='slappart0';")
SOFTWARE_RELEASE=$(echo $OLD_SOFTWARE_RELEASE | sed -e 's/slappart[0-9][0-9]/'"$PARTITION"'/')
SOFTWARE_RELEASE=$(echo $OLD_SOFTWARE_RELEASE | sed -e 's/\(.*\)\(slappart\|test0-\)[0-9][0-9]\?/\1'"$PARTITION"'/')
$SQLITE3 $DATABASE "update partition11 set software_release='$SOFTWARE_RELEASE' where software_release NOT NULL;"
$SQLITE3 $DATABASE "update software11 set url='$SOFTWARE_RELEASE' where url='$OLD_SOFTWARE_RELEASE';" || $SQLITE3 $DATABASE "delete from software11 where url='$OLD_SOFTWARE_RELEASE';"
# Change slapproxy database to have all instances stopped
$SQLITE3 $DATABASE "update partition11 set slap_state='stopped';"
# Build newest software
$SLAPOS node software --cfg ~/etc/slapos.cfg --pidfile ~/var/run/slapos-node-software.pid --all >/dev/null 2>&1 || true
set -x
# Run slapproxy on different port (in case of running inside of erp5testnode
# with only one IP and port 50000 already taken by slapproxy of main instance)
HOST="{{ proxy_host }}"
PORT="50001"
URL="http://$HOST:$PORT"
$SLAPOS proxy start --cfg $HOME/etc/slapos.cfg --port $PORT >/dev/null 2>&1 &
SLAPPROXY_PID=$!
trap "kill $SLAPPROXY_PID" EXIT TERM INT
sleep 5
echo "Building newest software..."
$SLAPOS node software --cfg $HOME/etc/slapos.cfg --all --master-url=$URL --logfile $HOME/srv/runner/software.log --pidfile $HOME/var/run/slapos-node-software.pid >/dev/null 2>&1 ||
$SLAPOS node software --cfg $HOME/etc/slapos.cfg --all --master-url=$URL --logfile $HOME/srv/runner/software.log --pidfile $HOME/var/run/slapos-node-software.pid >/dev/null 2>&1 ||
$SLAPOS node software --cfg $HOME/etc/slapos.cfg --all --master-url=$URL --logfile $HOME/srv/runner/software.log --pidfile $HOME/var/run/slapos-node-software.pid >/dev/null 2>&1
# Remove defined scripts to force buildout to recreate them to have updated paths
rm $srv_directory/runner/instance/slappart*/srv/runner-import-restore
# Run slapos node instance
rm $srv_directory/runner/instance/slappart*/srv/runner-import-restore || true
echo "Running slapos node instance..."
# XXX hardcoded
$SLAPOS node instance --cfg ~/etc/slapos.cfg --pidfile ~/var/run/slapos-node-instance.pid >/dev/null 2>&1 || true
$SLAPOS node instance --cfg $HOME/etc/slapos.cfg --master-url=$URL --logfile $HOME/srv/runner/instance.log --pidfile $HOME/var/run/slapos-node-instance.pid >/dev/null 2>&1 || true
$SLAPOS node instance --cfg $HOME/etc/slapos.cfg --master-url=$URL --logfile $HOME/srv/runner/instance.log --pidfile $HOME/var/run/slapos-node-instance.pid >/dev/null 2>&1 || true
$SLAPOS node instance --cfg $HOME/etc/slapos.cfg --master-url=$URL --logfile $HOME/srv/runner/instance.log --pidfile $HOME/var/run/slapos-node-instance.pid >/dev/null 2>&1 || true
# Invoke defined scripts for each partition inside of slaprunner
for partition in $srv_directory/runner/instance/slappart*/
......
......@@ -5,6 +5,7 @@ master_url = http://{{ slaprunner['ipv4'] }}:{{ slaprunner['proxy_port'] }}
computer_id = slaprunner
maximal_delay = 0
root_check = {{ slaprunner['root_check'] }}
forbid_supervisord_automatic_launch = true
[slapformat]
partition_amount = {{ slaprunner['partition-amount'] }}
......
......@@ -78,7 +78,6 @@ parts =
Paste = 1.7.5.1
PasteScript = 1.7.5
WSGIUtils = 0.7
plone.recipe.command = 1.1
python-magic = 0.4.6
rdiff-backup = 1.0.5
slapos.recipe.template = 2.6
......@@ -90,4 +89,4 @@ PasteDeploy = 1.5.2
# Required by:
# cloudooo==1.2.5-dev
erp5.util = 0.4.41
erp5.util = 0.4.42
......@@ -651,7 +651,7 @@ Products.PluginRegistry = 1.3
Products.TIDStorage = 5.4.9
PyPDF2 = 1.24
PyXML = 0.8.5
Pympler = 0.3.1
Pympler = 0.4
StructuredText = 2.11.1
WSGIUtils = 0.7
apache-libcloud = 0.16.0
......@@ -659,7 +659,7 @@ astroid = 1.3.4
chardet = 2.3.0
collective.recipe.template = 1.11
csp-eventlet = 0.7.0
ecdsa = 0.11
ecdsa = 0.13
elementtree = 1.2.6.post20050316
erp5diff = 0.8.1.5
eventlet = 0.16.1
......@@ -673,15 +673,14 @@ huBarcode = 1.0.0
interval = 1.0.0
ipdb = 0.8
ipython = 2.3.1
jsonschema = 2.4.0
logilab-common = 0.63.2
neoppod = 1.3
numpy = 1.9.1
plone.recipe.command = 1.1
ply = 3.4
polib = 1.0.6
pprofile = 1.7.2
pycountry = 1.10
pycrypto = 2.6.1
pyflakes = 0.8.1
pylint = 1.4.1
python-ldap = 2.4.19
......@@ -691,7 +690,7 @@ qrcode = 5.1
restkit = 4.2.2
rtjp-eventlet = 0.3.2
slapos.recipe.template = 2.6
slapos.toolbox = 0.46.0
slapos.toolbox = 0.47.1
smmap = 0.9.0
socketpool = 0.5.3
spyne = 2.11.0
......@@ -702,22 +701,21 @@ urlnorm = 1.1.2
uuid = 1.30
validictory = 1.0.0
xupdate-processor = 0.4
xfw = 0.10
# Required by:
# slapos.toolbox==0.46.0
GitPython = 0.3.5
# slapos.toolbox==0.47.1
GitPython = 0.3.6
# Required by:
# Products.CMFCore==2.2.8
Products.ZSQLMethods = 2.13.4
# Required by:
# slapos.toolbox==0.46.0
# slapos.toolbox==0.47.1
atomize = 0.2.0
# Required by:
# slapos.toolbox==0.46.0
# slapos.toolbox==0.47.1
feedparser = 5.1.3
# Required by:
......@@ -725,11 +723,11 @@ feedparser = 5.1.3
fpconst = 0.7.2
# Required by:
# slapos.toolbox==0.46.0
# slapos.toolbox==0.47.1
lockfile = 0.10.2
# Required by:
# slapos.toolbox==0.46.0
# slapos.toolbox==0.47.1
paramiko = 1.15.2
# Required by:
......
#!/bin/sh
#!${:dash}
# DO NOT RUN THIS SCRIPT ON PRODUCTION INSTANCE
# OR MYSQL DATA WILL BE ERASED.
......@@ -10,6 +10,8 @@
# Depending on the output, it will create a file containing
# the status of the restoration (success or failure)
set -e
mysql_executable="${mariadb-instance:mysql-binary}"
mysqldump_executable="${binary-wrap-mysqldump:wrapper-path}"
mariadb_data_directory="${directory:mariadb-data}"
......@@ -28,7 +30,7 @@ if [ -e "$pid_file" ]; then
fi
echo "Deleting existing database..."
rm -r $mariadb_data_directory/*
rm -r $mariadb_data_directory/* >/dev/null 2>&1 || true
echo "Adapting binlog database to new paths..."
new_binlog_directory="$(dirname $binlog_path)"
......@@ -40,7 +42,22 @@ echo "Starting mariadb..."
# XXX hardcoded
$instance_directory/etc/run/mariadb &
mysqld_pid=$!
$instance_directory/etc/run/mariadb_update > /dev/null 2>&1
trap "kill $mysqld_pid" EXIT TERM INT
sleep 30
# If mysql has stopped, abort
if ! [ -d /proc/$mysql_pid ]; then
echo "mysqld exited, aborting."
exit 1
fi
$instance_directory/etc/run/mariadb_update &
mariadb_update_pid=$!
sleep 60
# If mariadb_update is still running, abort
if [ -d /proc/$mariadb_update_pid ]; then
echo "mariadb_update still running after timeout, aborting."
kill $mariadb_update_pid
exit 1
fi
echo "Importing data..."
# Use latest dump XXX can contain funny characters
......@@ -48,8 +65,6 @@ dump=$(ls -r $mariadb_backup_directory | head -1)
zcat "$mariadb_backup_directory/$dump" | $mysql_executable -u root --socket="$instance_directory/var/run/mariadb.sock"
RESTORE_EXIT_CODE=$?
kill "$mysqld_pid"
if [ $RESTORE_EXIT_CODE -eq 0 ]; then
echo 'Backup restoration successfully completed.'
else
......
......@@ -38,7 +38,7 @@ eggs = collective.recipe.template
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/pbsready.cfg.in
output = ${buildout:directory}/pbsready.cfg
md5sum = cbc5bdb360fb5c72418dba03135df526
md5sum = 7d7cb70cb76a41f853e74cc34a9ef428
mode = 0644
[pbsready-import]
......@@ -47,7 +47,7 @@ mode = 0644
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/pbsready-import.cfg.in
output = ${buildout:directory}/pbsready-import.cfg
md5sum = a5570ecfeff7a9d1b5f8be08db4feefe
md5sum = dd13497575d13b92c3abb0a633777e2c
mode = 0644
[pbsready-export]
......@@ -56,7 +56,7 @@ mode = 0644
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/pbsready-export.cfg.in
output = ${buildout:directory}/pbsready-export.cfg
md5sum = 879fff114d1dbf1f58774ccbce9bdd22
md5sum = bfd71e454140cf13179d408e10f95bf8
mode = 0644
[template-pull-backup]
......@@ -69,7 +69,7 @@ mode = 0644
[template-replicated]
recipe = slapos.recipe.download
url = ${:_profile_base_location_}/template-replicated.cfg.in
md5sum = c781ae17375c26b08b2a11bd9b800db8
md5sum = 0641a65142fe0f624bf7c112081c2527
mode = 0644
destination = ${buildout:directory}/template-replicated.cfg.in
......@@ -92,7 +92,7 @@ output = ${buildout:directory}/instance-frozen.cfg
[resilient-web-takeover-cgi-script-download]
recipe = slapos.recipe.download
url = ${:_profile_base_location_}/resilient-web-takeover-cgi-script.py.in
md5sum = 5c90dae56885eeb490cc5d7f82d2dc5b
md5sum = 3aa7624af1196062d7d01946d4de9f0e
mode = 0644
destination = ${buildout:directory}/resilient-web-takeover-cgi-script.py.in
......
......@@ -5,7 +5,6 @@ extends = ${pbsready:output}
# Explicitely define extended parts from pbsready
# then add local parts
parts =
resiliency
logrotate
logrotate-entry-cron
logrotate-entry-equeue
......
......@@ -5,7 +5,6 @@ extends = ${pbsready:output}
# Explicitely define extended parts from pbsready
# then add local parts
parts =
resiliency
logrotate
logrotate-entry-cron
logrotate-entry-equeue
......@@ -18,6 +17,7 @@ parts =
dropbear-server-pbs-authorized-key
notifier
resiliency-takeover-script
resilient-web-takeover-cgi-script
resilient-web-takeover-httpd-wrapper
resilient-web-takeover-httpd-promise
......@@ -48,6 +48,20 @@ recipe = slapos.cookbook:notifier.callback
on-notification-id = $${slap-parameter:on-notification}
callback = $${importer:wrapper}
###########
# Generate the takeover script
###########
[resiliency-takeover-script]
recipe = slapos.cookbook:addresiliency
wrapper-takeover = $${rootdirectory:bin}/takeover
takeover-triggered-file-path = $${rootdirectory:srv}/takeover_triggered
# Add path of file created by takeover script when takeover is triggered
# Takeover script will create this file
# equeue process will watch for file existence.
[equeue]
takeover-triggered-file-path = $${resiliency-takeover-script:takeover-triggered-file-path}
###########
# Deploy a webserver allowing to do takeover from a web browser.
###########
......
[buildout]
parts =
resiliency
logrotate
logrotate-entry-cron
logrotate-entry-equeue
......@@ -126,18 +125,6 @@ frequency = daily
rotate-num = 30
#----------------
#--
#-- Resiliency script for the node takeover
[resiliency]
recipe = slapos.cookbook:addresiliency
wrapper-takeover = takeover
services = $${basedirectory:services}
bin = $${rootdirectory:bin}
etc = $${rootdirectory:etc}
#----------------
#--
#-- Sets up an rdiff-backup server (with a dropbear server for ssh)
......
......@@ -2,6 +2,7 @@
equeue_database = '${equeue:database}'
equeue_lockfile = '${equeue:lockfile}'
takeover_script = '${resiliency-takeover-script:wrapper-takeover}'
import cgi
import cgitb
......@@ -68,6 +69,6 @@ if form['password'].value != '${:password}':
sys.exit(1)
# XXX hardcoded location
result = subprocess.check_output([os.path.expanduser("~/bin/takeover")], stderr=subprocess.STDOUT)
result = subprocess.check_output([takeover_script], stderr=subprocess.STDOUT)
print 'Success.'
print '<pre>%s</pre>' % result
......@@ -70,7 +70,7 @@ name = {{namebase}}{{id}}
software-url = ${slap-connection:software-release-url}
software-type = {{typeimport}}
return = ssh-public-key ssh-url notification-url ip
return = ssh-public-key ssh-url notification-url ip takeover-url takeover-password
pbs-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-{{id}}-push
......@@ -99,6 +99,8 @@ sla-{{ key }} = {{ value }}
[publish-connection-informations]
feed-url-{{namebase}}-{{id}}-push = ${request-pbs-{{namebase}}-{{id}}:connection-feeds-url}${request-{{namebase}}-pseudo-replicating-{{id}}:pbs-notification-id}
takeover-{{namebase}}-{{id}}-url = ${request-{{namebase}}-pseudo-replicating-{{id}}:connection-takeover-url}
takeover-{{namebase}}-{{id}}-password = ${request-{{namebase}}-pseudo-replicating-{{id}}:connection-takeover-password}
{% endfor -%}
......
......@@ -111,7 +111,7 @@ hexagonit.recipe.download = 1.7.post4
Jinja2 = 2.7.3
MarkupSafe = 0.23
Werkzeug = 0.9.6
Werkzeug = 0.10.1
buildout-versions = 1.7
cffi = 0.8.6
cmd2 = 0.6.8
......@@ -124,24 +124,24 @@ mr.developer = 1.31
netaddr = 0.7.13
pbr = 0.10.7
prettytable = 0.7.2
psutil = 2.2.0
psutil = 2.2.1
pyOpenSSL = 0.14
pyparsing = 2.0.3
pytz = 2014.10
requests = 2.5.1
setuptools = 11.3.1
setuptools = 12.0.4
simplejson = 3.6.5
six = 1.9.0
slapos.cookbook = 0.93
slapos.core = 1.3.6
slapos.cookbook = 0.95
slapos.core = 1.3.8
slapos.libnetworkcache = 0.14.2
slapos.recipe.build = 0.16
slapos.recipe.build = 0.18
slapos.recipe.cmmi = 0.2
stevedore = 1.2.0
xml-marshaller = 0.9.7
# Required by:
# slapos.core==1.3.5
# slapos.core==1.3.8
Flask = 0.10.1
# Required by:
......@@ -150,31 +150,31 @@ Flask = 0.10.1
argparse = 1.3.0
# Required by:
# slapos.core==1.3.5
# slapos.core==1.3.8
cliff = 1.9.0
# Required by:
# cryptography==0.7.1
# cryptography==0.7.2
enum34 = 1.0.4
# Required by:
# slapos.cookbook==0.93
# slapos.cookbook==0.95
jsonschema = 2.4.0
# Required by:
# slapos.cookbook==0.93
# slapos.cookbook==0.95
lock-file = 2.0
# Required by:
# slapos.core==1.3.5
# slapos.core==1.3.8
netifaces = 0.10.4
# Required by:
# pbr==0.10.7
pip = 6.0.6
pip = 6.0.8
# Required by:
# cryptography==0.7.1
# cryptography==0.7.2
pyasn1 = 0.1.7
# Required by:
......@@ -182,15 +182,15 @@ pyasn1 = 0.1.7
pycparser = 2.10
# Required by:
# slapos.core==1.3.5
# slapos.core==1.3.8
supervisor = 3.1.3
# Required by:
# slapos.core==1.3.5
# slapos.core==1.3.8
uritemplate = 0.6
# Required by:
# slapos.core==1.3.5
# slapos.core==1.3.8
zope.interface = 4.1.2
[networkcache]
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment