Commit d2cc7d2a authored by Cédric de Saint Martin's avatar Cédric de Saint Martin

Merge branch 'master' into lamp

parents 45aa7a28 a695668b
Changes Changes
======= =======
0.71.1 (2013-01-04)
-------------------
* Frontend: Sort instances by reference to avoid attacks. [Cedric de Saint
Martin]
* Frontend: Add public_ipv4 parameter support to ease deployment of slave
frontend. [Cedric de Saint Martin]
* Frontend: Move apache_frontend wrappers to watched directory (etc/service).
[Cedric de Saint Martin]
* Frontend: Add native path to varnish environment. [Cedric de Saint Martin]
0.71 (2012-12-20)
-----------------
* frontend: Add "path" parameter for Zope instances. [Cedric de Saint Martin]
0.70 (2012-11-05)
-----------------
* KVM: Add support for disk-type, second nbd and cpu-count. [Cedric de Saint
Martin]
0.69 (2012-10-30)
-----------------
* handle multiple notification_url values in notifier recipe [Marco Mariani]
* createWrapper() sh alternative to execute.execute() for simple cases
[Marco Mariani]
* fixed secret key generation in apachephp config [Marco Mariani]
0.68.1 (2012-10-03)
-------------------
* slaprunner: fix "logfile" parameter to "log_file"
0.68 (2012-10-02)
-----------------
* request.py: Remove useless calls to master, fix "update" method. [Cedric
de Saint Martin]
* Add webrunner test recipe. [Alain Takoudjou]
* Add logfile for slaprunner. [Cedric de Saint Martin]
* Fix check_url_available promise (syntax + checks + IPv6 support). [Cedric
de Saint Martin]
0.67 (2012-09-26)
-----------------
* Add check_page_content promise generator. [Cedric Le Ninivin]
* Fix check_url_available recipe. [Cedric de Saint Martin]
* Set up timezone database in mariab's mysql table so that we can use
timezone conversion function. [Kazuhiko Shiozaki]
* Add many resiliency-based recipes [Timothée Lacroix]
* Fix and unify request and requestoptional recipes [Cedric de Saint Martin]
* Fix Dropbear. [Antoine Catton]
0.66 (2012-09-10) 0.66 (2012-09-10)
----------------- -----------------
......
[buildout]
parts = apache-perl
extends =
../apache/buildout.cfg
../perl/buildout.cfg
../libuuid/buildout.cfg
[apache-perl]
# Note: Shall react on each build of apache and reinstall itself
recipe = hexagonit.recipe.cmmi
url = http://perl.apache.org/dist/mod_perl-2.0.5.tar.gz
md5sum = 03d01d135a122bd8cebd0cd5b185d674
configure-command =
${perl:location}/bin/perl Makefile.PL
configure-options =
MP_AP_PREFIX=${apache-2.2:location}
environment =
# CPPFLAGS=-I${libuuid:location}/include
MP_CCOPTS=-L${libuuid:location}/lib
...@@ -55,10 +55,50 @@ configure-options = ...@@ -55,10 +55,50 @@ configure-options =
--enable-bz2 --enable-bz2
--enable-ftp --enable-ftp
# Changing TMPDIR is required for PEAR installation.
# It will create a pear/temp directory under the SR instead of a shared /tmp/pear/temp.
# XXX we could mkdir tmp there
environment = environment =
PKG_CONFIG_PATH=${libxml2:location}/lib/pkgconfig:${openssl:location}/lib/pkgconfig PKG_CONFIG_PATH=${libxml2:location}/lib/pkgconfig:${openssl:location}/lib/pkgconfig
PATH=${pkgconfig:location}/bin:${bzip2:location}/bin:${libxml2:location}/bin:%(PATH)s PATH=${pkgconfig:location}/bin:${bzip2:location}/bin:${libxml2:location}/bin:%(PATH)s
LDFLAGS =-L${bzip2:location}/lib -Wl,-rpath -Wl,${bzip2:location}/lib -L${libtool:location}/lib -Wl,-rpath -Wl,${libtool:location}/lib -L${mariadb:location}/lib -Wl,-rpath -Wl,${mariadb:location}/lib -L${zlib:location}/lib -Wl,-rpath -Wl,${zlib:location}/lib -L${libmcrypt:location}/lib -Wl,-rpath -Wl,${libmcrypt:location}/libblkid LDFLAGS =-L${bzip2:location}/lib -Wl,-rpath -Wl,${bzip2:location}/lib -L${libtool:location}/lib -Wl,-rpath -Wl,${libtool:location}/lib -L${mariadb:location}/lib -Wl,-rpath -Wl,${mariadb:location}/lib -L${zlib:location}/lib -Wl,-rpath -Wl,${zlib:location}/lib -L${libmcrypt:location}/lib -Wl,-rpath -Wl,${libmcrypt:location}/libblkid
TMPDIR=${buildout:parts-directory}/${:_buildout_section_name_}
[apache-php-postgres]
<=apache-php
configure-options =
--with-apxs2=${apache:location}/bin/apxs
--with-libxml-dir=${libxml2:location}
--with-zlib-dir=${zlib:location}
--with-bz2-dir=${bzip2:location}
--with-mcrypt=${libmcrypt:location}
--with-gd
--with-jpeg-dir=${libjpeg:location}
--with-png-dir=${libpng:location}
--enable-gd-native-ttf
--with-ttf
--with-freetype-dir=${freetype:location}
--with-curl=${curl:location}
--with-zip-dir=${zip:location}
--with-imap=${cclient:location}
--with-iconv-dir=${libiconv:location}
--with-gettext=${gettext:location}
--with-ldap=${openldap:location}
--with-imap-ssl
--with-openssl=${openssl:location}
--enable-libxml
--enable-mbstring
--enable-session
--enable-exif
--enable-zip
--enable-bz2
--enable-ftp
--with-pgsql=${postgresql:location}
[libmcrypt] [libmcrypt]
......
...@@ -87,7 +87,7 @@ configure-options = --prefix=${buildout:parts-directory}/${:_buildout_section_na ...@@ -87,7 +87,7 @@ configure-options = --prefix=${buildout:parts-directory}/${:_buildout_section_na
environment = environment =
PATH=${pkgconfig:location}/bin:%(PATH)s PATH=${pkgconfig:location}/bin:%(PATH)s
PKG_CONFIG_PATH=${openssl:location}/lib/pkgconfig PKG_CONFIG_PATH=${openssl:location}/lib/pkgconfig
CPPFLAGS =-I${libuuid:location}/include CPPFLAGS =-I${libuuid:location}/include -I${openssl:location}/include
LDFLAGS =-Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${openssl:location}/lib -L${libuuid:location}/lib -Wl,-rpath=${libuuid:location}/lib -Wl,-rpath=${libexpat:location}/lib -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${sqlite3:location}/lib -Wl,-rpath=${gdbm:location}/lib LDFLAGS =-Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${openssl:location}/lib -L${libuuid:location}/lib -Wl,-rpath=${libuuid:location}/lib -Wl,-rpath=${libexpat:location}/lib -Wl,-rpath=${pcre:location}/lib -Wl,-rpath=${sqlite3:location}/lib -Wl,-rpath=${gdbm:location}/lib
[mod_antiloris-apache-2.4.patch] [mod_antiloris-apache-2.4.patch]
......
...@@ -5,7 +5,7 @@ parts = busybox ...@@ -5,7 +5,7 @@ parts = busybox
[busybox] [busybox]
recipe = slapos.recipe.build recipe = slapos.recipe.build
url = http://git.busybox.net/busybox/snapshot/busybox-1_20_1.tar.gz url = http://git.busybox.net/busybox/snapshot/busybox-1_20_1.tar.gz
md5sum = 15758fc37ae8051d6def1b8afb691821 md5sum = 2dcfee8add6b9c52d6a91e97ba705b66
script = script =
extract_dir = self.extract(self.download(%(url)r, %(md5sum)r)) extract_dir = self.extract(self.download(%(url)r, %(md5sum)r))
workdir = guessworkdir(extract_dir) workdir = guessworkdir(extract_dir)
......
...@@ -16,7 +16,7 @@ parts = ...@@ -16,7 +16,7 @@ parts =
[node-sm] [node-sm]
recipe = slapos.recipe.build:npm recipe = slapos.recipe.build:npm
packages = sm@0.2.7 packages = sm@0.2.11
node = nodejs node = nodejs
environment = environment =
PATH=${nodejs:location}/bin:%(PATH)s PATH=${nodejs:location}/bin:%(PATH)s
...@@ -25,10 +25,10 @@ environment = ...@@ -25,10 +25,10 @@ environment =
# Online IDE written in javascript/node.js # Online IDE written in javascript/node.js
# URL : c9.io # URL : c9.io
# You can use it using the following command : # You can use it using the following command :
# NODE_PATH=${:destination}/node_modules ${nodejs:node_location} ${:cloud9_js_location} # NODE_PATH=${:destination}/node_modules ${nodejs:node_location} ${:cloud9_js_location}
recipe = plone.recipe.command recipe = plone.recipe.command
stop-on-error = true stop-on-error = true
commit = 97db1467c517d265438684bd2a70b0b76ee282f6 commit = 5d18344936baf1d86b0fa5fc2c690051b4c77cb2
repository = https://github.com/ajaxorg/cloud9.git repository = https://github.com/ajaxorg/cloud9.git
location = ${buildout:parts-directory}/${:_buildout_section_name_} location = ${buildout:parts-directory}/${:_buildout_section_name_}
git-binary = ${git:location}/bin/git git-binary = ${git:location}/bin/git
...@@ -37,26 +37,33 @@ command = export GIT_SSL_NO_VERIFY=true; export HOME=${:location}; (${:git-binar ...@@ -37,26 +37,33 @@ command = export GIT_SSL_NO_VERIFY=true; export HOME=${:location}; (${:git-binar
update-command = update-command =
executable = ${:location}/bin/cloud9.js executable = ${:location}/bin/cloud9.js
[cloud9-session-directory.patch]
recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/${:filename}
filename = cloud9-session-directory.patch
download-only = true
md5sum = 5dc8cc28447ed3747b8a53c768d872aa
[cloud9-git] [cloud9-git]
# Online IDE written in javascript/node.js # Online IDE written in javascript/node.js
# URL : c9.io # URL : c9.io
# You can use it using the following command : # You can use it using the following command :
# NODE_PATH=${:destination}/node_modules ${nodejs:node_location} ${:cloud9_js_location} # NODE_PATH=${:destination}/node_modules ${nodejs:node_location} ${:cloud9_js_location}
recipe = plone.recipe.command recipe = plone.recipe.command
stop-on-error = true stop-on-error = true
commit = c66284221143c175fc889418d499da6f37492a7c commit = 6b0112363dd0f9523430808394a162d3cc699550
repository = https://github.com/ajaxorg/cloud9.git repository = https://github.com/ajaxorg/cloud9.git
location = ${buildout:parts-directory}/${:_buildout_section_name_} location = ${buildout:parts-directory}/${:_buildout_section_name_}
environment = export GIT_SSL_NO_VERIFY=true; export PATH=${git:location}/bin:${nodejs:location}/bin:${node-sm:location}/node_modules/sm/bin:$PATH; export CPPFLAGS="-I${libxml2:location}/include -I${nodejs:location}/include"; export LDFLAGS="-L${libxml2:location}/lib -Wl,-rpath=${libxml2:location}/lib"; export HOME=${:location}; environment = export GIT_SSL_NO_VERIFY=true; export PATH=${git:location}/bin:${nodejs:location}/bin:${node-sm:location}/node_modules/sm/bin:$PATH; export CPPFLAGS="-I${libxml2:location}/include -I${nodejs:location}/include"; export LDFLAGS="-L${libxml2:location}/lib -Wl,-rpath=${libxml2:location}/lib"; export HOME=${:location};
command = ${:environment} (git clone --quiet ${:repository} ${:location} && cd ${:location} && git reset --hard ${:commit} && ${node-sm:location}/node_modules/.bin/sm install) || (rm -fr ${:location}; exit 1) command = ${:environment} (git clone --quiet ${:repository} ${:location} && cd ${:location} && git reset --hard ${:commit} && ${node-sm:location}/node_modules/.bin/sm install && patch -p1 < ${cloud9-session-directory.patch:location}/${cloud9-session-directory.patch:filename}) || (rm -fr ${:location}; exit 1)
update-command = update-command =
executable = ${:location}/server.js executable = ${:location}/server.js
[cloud9-npm] [cloud9-npm]
# Online IDE written in javascript/node.js # Online IDE written in javascript/node.js
# URL : c9.io # URL : c9.io
# You can use it using the following command : # You can use it using the following command :
# NODE_PATH=${:destination}/node_modules ${nodejs:node_location} ${:cloud9_js_location} # NODE_PATH=${:destination}/node_modules ${nodejs:node_location} ${:cloud9_js_location}
recipe = slapos.recipe.npm recipe = slapos.recipe.npm
# Node part has to be specified, otherwise system node is used. # Node part has to be specified, otherwise system node is used.
node = nodejs node = nodejs
......
diff --git a/configs/default.js b/configs/default.js
index 6d1c85f..be35b37 100644
--- a/configs/default.js
+++ b/configs/default.js
@@ -22,6 +22,8 @@ var vfsUrl = "/vfs";
var port = argv.p || process.env.PORT || 3131;
var host = argv.l || process.env.IP || "localhost";
+var home = process.env['HOME']
+
var config = [
{
packagePath: "connect-architect/connect",
@@ -167,7 +169,7 @@ var config = [
},
{
packagePath: "connect-architect/connect.session.file",
- sessionsPath: __dirname + "/../.sessions"
+ sessionsPath: home + "/.sessions"
},
"./cloud9.permissions",
{
\ No newline at end of file
...@@ -3,6 +3,7 @@ parts = ...@@ -3,6 +3,7 @@ parts =
liberation-fonts liberation-fonts
ipaex-fonts ipaex-fonts
ipa-fonts ipa-fonts
ocrb-fonts
[fonts] [fonts]
location = ${buildout:parts-directory}/${:_buildout_section_name_} location = ${buildout:parts-directory}/${:_buildout_section_name_}
...@@ -32,3 +33,10 @@ strip-top-level-dir = true ...@@ -32,3 +33,10 @@ strip-top-level-dir = true
url = http://info.openlab.ipa.go.jp/ipafont/fontdata/IPAfont00303.zip url = http://info.openlab.ipa.go.jp/ipafont/fontdata/IPAfont00303.zip
md5sum = 39a828acf27790adbe4944dfb4d94bb1 md5sum = 39a828acf27790adbe4944dfb4d94bb1
destination = ${fonts:location}/${:_buildout_section_name_} destination = ${fonts:location}/${:_buildout_section_name_}
[ocrb-fonts]
recipe = hexagonit.recipe.download
strip-top-level-dir = true
url = http://sourceforge.jp/frs/redir.php?m=jaist&f=%2Ftsukurimashou%2F56948%2Focr-0.2.zip
md5sum = 9f2acd83291a31dbe053912f4115db75
destination = ${fonts:location}/${:_buildout_section_name_}
...@@ -170,4 +170,4 @@ make-targets = install -j1 ...@@ -170,4 +170,4 @@ make-targets = install -j1
# until gcc will be simplified by using more robust build recipe (like # until gcc will be simplified by using more robust build recipe (like
# slapos.recipe.build) each time any of parts which reuses this one gets updated # slapos.recipe.build) each time any of parts which reuses this one gets updated
# the hack-revision have to be increased # the hack-revision have to be increased
hack-revision = 1 hack-revision = 2
...@@ -29,5 +29,5 @@ environment = ...@@ -29,5 +29,5 @@ environment =
[ghostscript-9] [ghostscript-9]
<= ghostscript-common <= ghostscript-common
url = http://downloads.ghostscript.com/public/ghostscript-9.05.tar.bz2 url = http://downloads.ghostscript.com/public/ghostscript-9.06.tar.bz2
md5sum = 8bcef1f33ddf8a4d12b2cf8da385c191 md5sum = 46f9ebe40dc52755287b30704270db11
...@@ -12,8 +12,8 @@ parts = ...@@ -12,8 +12,8 @@ parts =
[git] [git]
recipe = hexagonit.recipe.cmmi recipe = hexagonit.recipe.cmmi
url = http://git-core.googlecode.com/files/git-1.7.10.4.tar.gz url = http://git-core.googlecode.com/files/git-1.7.12.tar.gz
md5sum = 68319d593d051ef76c26e945bbd2d7ac md5sum = ceb1a6b17a3e33bbc70eadf8fce5876c
configure-options = configure-options =
--with-curl=${curl:location} --with-curl=${curl:location}
--with-openssl=${openssl:location} --with-openssl=${openssl:location}
......
...@@ -7,12 +7,12 @@ parts = ...@@ -7,12 +7,12 @@ parts =
[groonga] [groonga]
recipe = hexagonit.recipe.cmmi recipe = hexagonit.recipe.cmmi
url = http://packages.groonga.org/source/groonga/groonga-2.0.5.tar.gz url = http://packages.groonga.org/source/groonga/groonga-2.0.6.tar.gz
md5sum = 6ea6634ceee97f701add89ee39f4c1da md5sum = 64467d1ab8d5e0df8348a636972a95a7
configure-options = configure-options =
--disable-static --disable-static
--disable-glibtest --disable-glibtest
--disable-benchmark --disable-benchmark
--disable-document --disable-document
--disable-nginx --disable-groonga-httpd
--without-mecab --without-mecab
...@@ -9,8 +9,8 @@ parts = haproxy ...@@ -9,8 +9,8 @@ parts = haproxy
[haproxy] [haproxy]
recipe = hexagonit.recipe.cmmi recipe = hexagonit.recipe.cmmi
url = http://haproxy.1wt.eu/download/1.4/src/haproxy-1.4.21.tar.gz url = http://haproxy.1wt.eu/download/1.4/src/haproxy-1.4.22.tar.gz
md5sum = f36d5d13fd4a44ab454eee5d06cb38a2 md5sum = a0b007c76f6a78524f3b3dd5e704979c
configure-command = true configure-command = true
# If the system is running on Linux 2.6, we use "linux26" as the TARGET, # If the system is running on Linux 2.6, we use "linux26" as the TARGET,
# otherwise use "generic". # otherwise use "generic".
......
...@@ -28,7 +28,7 @@ filename = imagemagick-6.6.6-1-no-gsx-gsc-probe.patch ...@@ -28,7 +28,7 @@ filename = imagemagick-6.6.6-1-no-gsx-gsc-probe.patch
[imagemagick] [imagemagick]
recipe = hexagonit.recipe.cmmi recipe = hexagonit.recipe.cmmi
url = ftp://ftp.imagemagick.org/pub/ImageMagick/ImageMagick-6.7.8-8.tar.bz2 url = http://ftp.vim.org/ImageMagick/ImageMagick-6.7.8-8.tar.bz2
md5sum = 4e5c8f102f3e7401587c924f5b4bca15 md5sum = 4e5c8f102f3e7401587c924f5b4bca15
depends = depends =
${libtiff:version} ${libtiff:version}
......
...@@ -29,9 +29,9 @@ slapos_promisee = ...@@ -29,9 +29,9 @@ slapos_promisee =
directory:javaws directory:javaws
file:lib/rt.jar file:lib/rt.jar
file:bin/java file:bin/java
# http://java.com/en/download/manual.jsp?locale=en # http://java.com/en/download/manual_v6.jsp
x86 = http://javadl.sun.com/webapps/download/AutoDL?BundleId=59621 7466b0b86bef21e3e31ae578b2b9f472 x86 = http://javadl.sun.com/webapps/download/AutoDL?BundleId=68284 7eda40e790de1a907d591b62949b6e72
x86-64 = http://javadl.sun.com/webapps/download/AutoDL?BundleId=59623 c5cfb8393d257b51e7a70e56b7784ac9 x86-64 = http://javadl.sun.com/webapps/download/AutoDL?BundleId=68286 839548714931443ba89719a995ece846
script = script =
import glob import glob
if not self.options.get('url'): self.options['url'], self.options['md5sum'] = self.options[guessPlatform()].split(' ') if not self.options.get('url'): self.options['url'], self.options['md5sum'] = self.options[guessPlatform()].split(' ')
...@@ -47,48 +47,20 @@ script = ...@@ -47,48 +47,20 @@ script =
workdir = guessworkdir(extract_dir) workdir = guessworkdir(extract_dir)
self.copyTree(glob.glob(os.path.join(workdir, "jre1.6.0_*"))[0], "%(location)s") self.copyTree(glob.glob(os.path.join(workdir, "jre1.6.0_*"))[0], "%(location)s")
[java-sdk-1.6.0] [java-re-1.7.0]
recipe = slapos.recipe.build recipe = slapos.recipe.build
slapos_promisee = slapos_promisee =
directory:bin directory:bin
directory:lib directory:lib
directory:man directory:man
directory:plugin directory:plugin
directory:javaws file:lib/rt.jar
file:jre/lib/rt.jar
file:bin/java
x86 = http://download.oracle.com/otn-pub/java/jdk/6u27-b07/jdk-6u27-linux-i586.bin bdb5f05bd20c6aa9a4729726191bf6fd
x86-64 = http://download.oracle.com/otn-pub/java/jdk/6u27-b07/jdk-6u27-linux-x64.bin 94f93a3ff03f824a238ecd79ad90433e
script =
if not self.options.get('url'): self.options['url'], self.options['md5sum'] = self.options[guessPlatform()].split(' ')
download_file = self.download(self.options['url'], self.options.get('md5sum'))
extract_dir = tempfile.mkdtemp(self.name)
os.chdir(extract_dir)
(download_dir, filename) = os.path.split(download_file)
auto_extract_bin = os.path.join(extract_dir, filename)
shutil.move(download_file, auto_extract_bin)
os.chmod(auto_extract_bin, 0777)
subprocess.call(["patch", auto_extract_bin, "-i", "${jdk-6u27-no-user-interaction-patch:location}/${jdk-6u27-no-user-interaction-patch:filename}"])
subprocess.call([auto_extract_bin])
self.cleanup_dir_list.append(extract_dir)
workdir = guessworkdir(extract_dir)
self.copyTree(os.path.join(workdir, "jdk1.6.0_27"), "%(location)s")
[java-sdk-1.7.0]
recipe = slapos.recipe.build
slapos_promisee =
directory:bin
directory:lib
directory:man
directory:jre
file:jre/lib/rt.jar
file:bin/java file:bin/java
file:bin/javac # http://java.com/en/download/manual.jsp?locale=en
x86 = http://download.oracle.com/otn-pub/java/jdk/7/jdk-7-linux-i586.tar.gz f97244a104f03731e5ff69f0dd5a9927 x86 = http://javadl.sun.com/webapps/download/AutoDL?BundleId=68234 ea99bedd9db33e9e2970f4b70abd1e4b
x86-64 = http://download.oracle.com/otn-pub/java/jdk/7/jdk-7-linux-x64.tar.gz b3c1ef5faea7b180469c129a49762b64 x86-64 = http://javadl.sun.com/webapps/download/AutoDL?BundleId=68236 5aa9bd26cdf1fa6afd2b15826b4ba139
script = script =
if not self.options.get('url'): self.options['url'], self.options['md5sum'] = self.options[guessPlatform()].split(' ') if not self.options.get('url'): self.options['url'], self.options['md5sum'] = self.options[guessPlatform()].split(' ')
extract_dir = self.extract(self.download(self.options['url'], self.options.get('md5sum'))) extract_dir = self.extract(self.download(self.options['url'], self.options.get('md5sum')))
workdir = guessworkdir(extract_dir) workdir = guessworkdir(extract_dir)
self.copyTree(workdir, "%(location)s") self.copyTree(workdir, "%(location)s")
...@@ -3,8 +3,10 @@ ...@@ -3,8 +3,10 @@
[buildout] [buildout]
extends = extends =
../bzip2/buildout.cfg
../imagemagick/buildout.cfg ../imagemagick/buildout.cfg
../jbigkit/buildout.cfg ../jbigkit/buildout.cfg
../zlib/buildout.cfg
parts = parts =
libdmtx libdmtx
dmtx-utils dmtx-utils
...@@ -26,4 +28,4 @@ environment = ...@@ -26,4 +28,4 @@ environment =
PATH=${pkgconfig:location}/bin:%(PATH)s PATH=${pkgconfig:location}/bin:%(PATH)s
PKG_CONFIG_PATH=${imagemagick:location}/lib/pkgconfig:${libdmtx:location}/lib/pkgconfig PKG_CONFIG_PATH=${imagemagick:location}/lib/pkgconfig:${libdmtx:location}/lib/pkgconfig
CPPFLAGS=-I${libdmtx:location}/include CPPFLAGS=-I${libdmtx:location}/include
LDFLAGS=-Wl,-rpath=${jbigkit:location}/lib LDFLAGS=-Wl,-rpath=${jbigkit:location}/lib -Wl,-rpath=${bzip2:location}/lib -Wl,-rpath=${zlib:location}/lib
...@@ -14,7 +14,8 @@ url = http://ftp.gnome.org/pub/gnome/sources/libcroco/0.6/libcroco-0.6.3.tar.bz2 ...@@ -14,7 +14,8 @@ url = http://ftp.gnome.org/pub/gnome/sources/libcroco/0.6/libcroco-0.6.3.tar.bz2
md5sum = e1e93eeff4367c896f3959af34ba20eb md5sum = e1e93eeff4367c896f3959af34ba20eb
environment = environment =
PATH=${pkgconfig:location}/bin:%(PATH)s PATH=${pkgconfig:location}/bin:%(PATH)s
PKG_CONFIG_PATH=${pkgconfig:location}/lib/pkgconfig:${libxml2:location}/lib/pkgconfig:${glib:location}/lib/pkgconfig PKG_CONFIG_PATH=${pkgconfig:location}/lib/pkgconfig:${libxml2:location}/lib/pkgconfig:${glib:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig
LDFLAGS=-Wl,-rpath=${zlib:location}/lib
[librsvg] [librsvg]
recipe = hexagonit.recipe.cmmi recipe = hexagonit.recipe.cmmi
...@@ -36,4 +37,4 @@ configure-options = ...@@ -36,4 +37,4 @@ configure-options =
environment = environment =
PATH=${gdk-pixbuf:location}/bin:${glib:location}/bin:${libxml2:location}/bin:${pkgconfig:location}/bin:${pango:location}/bin:%(PATH)s PATH=${gdk-pixbuf:location}/bin:${glib:location}/bin:${libxml2:location}/bin:${pkgconfig:location}/bin:${pango:location}/bin:%(PATH)s
PKG_CONFIG_PATH=${cairo:location}/lib/pkgconfig:${fontconfig:location}/lib/pkgconfig:${freetype:location}/lib/pkgconfig:${gdk-pixbuf:location}/lib/pkgconfig:${glib:location}/lib/pkgconfig:${kbproto:location}/lib/pkgconfig:${libX11:location}/lib/pkgconfig:${libpng:location}/lib/pkgconfig:${libXau:location}/lib/pkgconfig:${libXrender:location}/lib/pkgconfig:${libcroco:location}/lib/pkgconfig:${libxcb:location}/lib/pkgconfig:${libxml2:location}/lib/pkgconfig:${pango:location}/lib/pkgconfig:${pixman:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig:${xorg-libpthread-stubs:location}/lib/pkgconfig:${xproto:location}/lib/pkgconfig PKG_CONFIG_PATH=${cairo:location}/lib/pkgconfig:${fontconfig:location}/lib/pkgconfig:${freetype:location}/lib/pkgconfig:${gdk-pixbuf:location}/lib/pkgconfig:${glib:location}/lib/pkgconfig:${kbproto:location}/lib/pkgconfig:${libX11:location}/lib/pkgconfig:${libpng:location}/lib/pkgconfig:${libXau:location}/lib/pkgconfig:${libXrender:location}/lib/pkgconfig:${libcroco:location}/lib/pkgconfig:${libxcb:location}/lib/pkgconfig:${libxml2:location}/lib/pkgconfig:${pango:location}/lib/pkgconfig:${pixman:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig:${xorg-libpthread-stubs:location}/lib/pkgconfig:${xproto:location}/lib/pkgconfig
LDFLAGS=-L${bzip2:location}/lib -Wl,-rpath=${bzip2:location}/lib LDFLAGS=-L${bzip2:location}/lib -Wl,-rpath=${bzip2:location}/lib -Wl,-rpath=${zlib:location}/lib
...@@ -9,11 +9,9 @@ parts = ...@@ -9,11 +9,9 @@ parts =
[libtiff] [libtiff]
recipe = hexagonit.recipe.cmmi recipe = hexagonit.recipe.cmmi
version = 4.0.2 version = 4.0.3
#url = http://download.osgeo.org/libtiff/tiff-${:version}.tar.gz url = http://download.osgeo.org/libtiff/tiff-${:version}.tar.gz
# server is down - circumvent md5sum = 051c1068e6a0627f461948c365290410
url = http://www.imagemagick.org/download/delegates/tiff-${:version}.tar.gz
md5sum = 04a08fa1e07e696e820a0c3f32465a13
configure-options = configure-options =
--disable-static --disable-static
--without-x --without-x
......
...@@ -59,9 +59,11 @@ environment = ...@@ -59,9 +59,11 @@ environment =
LDFLAGS=-L${libaio:location}/lib LDFLAGS=-L${libaio:location}/lib
[mroonga-mariadb] [mroonga-mariadb]
# mroonga - a storage engine for MySQL. It provides fast fulltext search feature to all MySQL users.
# http://mroonga.github.com/
recipe = hexagonit.recipe.cmmi recipe = hexagonit.recipe.cmmi
url = https://github.com/downloads/mroonga/mroonga/mroonga-2.05.tar.gz url = https://github.com/downloads/mroonga/mroonga/mroonga-2.06.tar.gz
md5sum = 6aee0e6ff948bba7c02854326afc11df md5sum = 89ce640a57ed8f2161b8358ff4c29c66
configure-options = configure-options =
--with-mysql-source=${mariadb:location}__compile__/mariadb-${mariadb:version} --with-mysql-source=${mariadb:location}__compile__/mariadb-${mariadb:version}
--with-mysql-config=${mariadb:location}/bin/mysql_config --with-mysql-config=${mariadb:location}/bin/mysql_config
......
...@@ -3,6 +3,7 @@ ...@@ -3,6 +3,7 @@
extends = extends =
../cmake/buildout.cfg ../cmake/buildout.cfg
../glib/buildout.cfg ../glib/buildout.cfg
../pkgconfig/buildout.cfg
../openssl/buildout.cfg ../openssl/buildout.cfg
../pcre/buildout.cfg ../pcre/buildout.cfg
../mariadb/buildout.cfg ../mariadb/buildout.cfg
...@@ -22,6 +23,7 @@ buildout-bin-dir = ${buildout:bin-directory} ...@@ -22,6 +23,7 @@ buildout-bin-dir = ${buildout:bin-directory}
cmake-command = ${cmake:location}/bin/cmake cmake-command = ${cmake:location}/bin/cmake
mysql-config = ${mariadb:location}/bin/mysql_config mysql-config = ${mariadb:location}/bin/mysql_config
mysqllib = ${mariadb:location}/lib mysqllib = ${mariadb:location}/lib
path = ${pkgconfig:location}/bin
pkg-config-path = ${glib:location}/lib/pkgconfig/:${pcre:location}/lib/pkgconfig/:${openssl:location}/lib/pkgconfig/ pkg-config-path = ${glib:location}/lib/pkgconfig/:${pcre:location}/lib/pkgconfig/:${openssl:location}/lib/pkgconfig/
libraries = ${zlib:location}/lib/:${glib:location}/lib/:${pcre:location}/lib/:${mariadb:location}/lib/:${openssl:location}/lib/ libraries = ${zlib:location}/lib/:${glib:location}/lib/:${pcre:location}/lib/:${mariadb:location}/lib/:${openssl:location}/lib/
includes = ${zlib:location}/include/:${glib:location}/include/:${pcre:location}/include/:${mariadb:location}/include:${openssl:location}/include/ includes = ${zlib:location}/include/:${glib:location}/include/:${pcre:location}/include/:${mariadb:location}/include:${openssl:location}/include/
...@@ -39,7 +41,7 @@ script = ...@@ -39,7 +41,7 @@ script =
extract_dir = self.extract(url) extract_dir = self.extract(url)
workdir = guessworkdir(extract_dir) workdir = guessworkdir(extract_dir)
self.applyPatchList(self.options['mydumper-patches'], cwd=workdir) self.applyPatchList(self.options['mydumper-patches'], cwd=workdir)
env['PATH'] = self.options['buildout-bin-dir'] + ':' + env.get('PATH', '') env['PATH'] = self.options['path'] + ':' + self.options['buildout-bin-dir'] + ':' + env.get('PATH', '')
env['PKG_CONFIG_PATH'] = self.options['pkg-config-path'] + ':' + \ env['PKG_CONFIG_PATH'] = self.options['pkg-config-path'] + ':' + \
env.get('PKG_CONFIG_PATH', '') env.get('PKG_CONFIG_PATH', '')
env['CMAKE_INCLUDE_PATH'] = self.options['includes'] env['CMAKE_INCLUDE_PATH'] = self.options['includes']
......
...@@ -4,6 +4,6 @@ parts = ...@@ -4,6 +4,6 @@ parts =
[noVNC] [noVNC]
recipe = slapos.recipe.build:download-unpacked recipe = slapos.recipe.build:download-unpacked
url = http://cloud.github.com/downloads/kanaka/noVNC/novnc-0.3.tar.gz url = http://cloud.github.com/downloads/kanaka/noVNC/novnc-0.4.tar.gz
md5sum = 95d3c58921fa188c179491e8ef2acc12 md5sum = 5703d5d46022d8723796dcbbf821ee7f
strip-top-level-dir = true strip-top-level-dir = true
...@@ -14,8 +14,8 @@ parts = ...@@ -14,8 +14,8 @@ parts =
[nodejs-0.8] [nodejs-0.8]
# Server-side Javascript. # Server-side Javascript.
recipe = hexagonit.recipe.cmmi recipe = hexagonit.recipe.cmmi
url = http://nodejs.org/dist/v0.8.8/node-v0.8.8.tar.gz url = http://nodejs.org/dist/v0.8.14/node-v0.8.14.tar.gz
md5sum = f4dae84e96a94b768404c14633bccd49 md5sum = 284fd2c7578064c339d9cf6a3a475ac7
configure-options = configure-options =
--openssl-includes=${openssl:location}/include --openssl-includes=${openssl:location}/include
--openssl-libpath=${openssl:location}/lib --openssl-libpath=${openssl:location}/lib
......
...@@ -10,7 +10,7 @@ parts = ...@@ -10,7 +10,7 @@ parts =
recipe = hexagonit.recipe.cmmi recipe = hexagonit.recipe.cmmi
depends = depends =
${perl:version} ${perl:version}
url = http://www.percona.com/redir/downloads/percona-toolkit/2.0.4/percona-toolkit-2.0.4.tar.gz url = http://www.percona.com/redir/downloads/percona-toolkit/2.1.3/percona-toolkit-2.1.3.tar.gz
md5sum = df7dffcccb48d50f143849629228d4b4 md5sum = 6f8c4a0a0e43e467766cbdabaec9ef93
configure-command = configure-command =
${perl:location}/bin/perl Makefile.PL ${perl:location}/bin/perl Makefile.PL
...@@ -9,6 +9,9 @@ extends = ...@@ -9,6 +9,9 @@ extends =
../glib/buildout.cfg ../glib/buildout.cfg
../popt/buildout.cfg ../popt/buildout.cfg
[pkg-config]
<= pkgconfig
[pkgconfig] [pkgconfig]
recipe = hexagonit.recipe.cmmi recipe = hexagonit.recipe.cmmi
url = ftp://mirror.ovh.net/gentoo-distfiles/distfiles/pkg-config-0.26.tar.gz url = ftp://mirror.ovh.net/gentoo-distfiles/distfiles/pkg-config-0.26.tar.gz
......
...@@ -13,8 +13,8 @@ extends = ...@@ -13,8 +13,8 @@ extends =
[poppler] [poppler]
recipe = hexagonit.recipe.cmmi recipe = hexagonit.recipe.cmmi
md5sum = 45dd2c16c8c4d1a39e830e45745c4e25 md5sum = 28c40266f374e1960a7bcead17d39f96
url = http://poppler.freedesktop.org/poppler-0.20.2.tar.gz url = http://poppler.freedesktop.org/poppler-0.20.3.tar.gz
depends = depends =
${libtiff:version} ${libtiff:version}
configure-options = configure-options =
......
[buildout]
extends =
../openssl/buildout.cfg
../readline/buildout.cfg
../zlib/buildout.cfg
../ncurses/buildout.cfg
parts = postgresql
[postgresql]
recipe = hexagonit.recipe.cmmi
url = http://ftp.postgresql.org/pub/source/v9.1.6/postgresql-9.1.6.tar.bz2
md5sum = 000755f66c0de58bbd4cd2b89b45b8e2
configure-options = --with-openssl
environment =
CPPFLAGS=-I${zlib:location}/include -I${readline:location}/include -I${openssl:location}/include -I${ncurses:location}/lib
LDFLAGS=-L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${readline:location}/lib -Wl,-rpath=${readline:location}/lib -L${openssl:location}/lib -Wl,-rpath=${openssl:location}/lib -L${ncurses:location}/lib -Wl,-rpath=${ncurses:location}/lib
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
parts = python-setuptools parts = python-setuptools
[setuptools-download] [setuptools-download]
recipe = slapos.recipe.download recipe = slapos.recipe.build:download
filename = setuptools-0.6c11-py2.7.egg filename = setuptools-0.6c11-py2.7.egg
url = http://pypi.python.org/packages/2.7/s/setuptools/${:filename} url = http://pypi.python.org/packages/2.7/s/setuptools/${:filename}
md5sum = fe1f997bc722265116870bc7919059ea md5sum = fe1f997bc722265116870bc7919059ea
......
...@@ -4,5 +4,5 @@ parts = ...@@ -4,5 +4,5 @@ parts =
[rpm2cpio] [rpm2cpio]
recipe = slapos.recipe.build:download recipe = slapos.recipe.build:download
url = http://ruda.googlecode.com/hg-history/2989f0531de63872cc7327590798d1898ac91246/rpm/rpm2cpio.py url = https://raw.github.com/ruda/rpm2cpio/e196173f1f6b746463b7398e381b94a42edfa345/rpm2cpio.py
md5sum = c5bb6227d99e1ff5df880f997cbed2e3 md5sum = c5bb6227d99e1ff5df880f997cbed2e3
# This file is used to install SlapOS Node itself.
[buildout] [buildout]
extends = extends =
../../stack/shacache-client.cfg ../../stack/shacache-client.cfg
...@@ -25,6 +27,7 @@ parts = ...@@ -25,6 +27,7 @@ parts =
slapos slapos
cfg-environment cfg-environment
sh-environment sh-environment
py
find-links = find-links =
http://www.nexedi.org/static/packages/source/slapos.buildout/ http://www.nexedi.org/static/packages/source/slapos.buildout/
...@@ -46,13 +49,13 @@ exec-sitecustomize = false ...@@ -46,13 +49,13 @@ exec-sitecustomize = false
allowed-eggs-from-site-packages = allowed-eggs-from-site-packages =
[environment] [environment]
# Note: For now original PATH is appeneded to the end, as not all tools are # Note: For now original PATH is appended to the end, as not all tools are
# provided by SlapOS # provided by SlapOS
PATH=${bison:location}/bin:${bzip2:location}/bin:${gettext:location}/bin:${glib:location}/bin:${libxml2:location}/bin:${libxslt:location}/bin:${m4:location}/bin:${ncurses:location}/bin:${openssl:location}/bin:${pkgconfig:location}/bin:${python2.7:location}/bin:${readline:location}/bin:${sqlite3:location}/bin:${swig:location}/bin:${buildout:bin-directory}:${patch:location}/bin:$PATH PATH=${bison:location}/bin:${bzip2:location}/bin:${gettext:location}/bin:${glib:location}/bin:${libxml2:location}/bin:${libxslt:location}/bin:${m4:location}/bin:${ncurses:location}/bin:${openssl:location}/bin:${pkgconfig:location}/bin:${python2.7:location}/bin:${readline:location}/bin:${sqlite3:location}/bin:${swig:location}/bin:${buildout:bin-directory}:${patch:location}/bin:$PATH
CFLAGS=-I${bzip2:location}/include -I${gdbm:location}/include -I${gettext:location}/include -I${glib:location}/include -I${libxml2:location}/include -I${libxslt:location}/include -I${ncurses:location}/include -I${openssl:location}/include -I${popt:location}/include -I${readline:location}/include -I${sqlite3:location}/include -I${zlib:location}/include CFLAGS=-I${bzip2:location}/include -I${gdbm:location}/include -I${gettext:location}/include -I${glib:location}/include -I${libxml2:location}/include -I${libxslt:location}/include -I${ncurses:location}/include -I${openssl:location}/include -I${popt:location}/include -I${readline:location}/include -I${sqlite3:location}/include -I${zlib:location}/include
CPPFLAGS=${:CFLAGS} CPPFLAGS=${:CFLAGS}
LDFLAGS=-L${bzip2:location}/lib -Wl,-rpath=${bzip2:location}/lib -L${gdbm:location}/lib -Wl,-rpath=${gdbm:location}/lib -L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib -L${glib:location}/lib -Wl,-rpath=${glib:location}/lib -L${libxml2:location}/lib -Wl,-rpath=${libxml2:location}/lib -L${libxslt:location}/lib -Wl,-rpath=${libxslt:location}/lib -L${ncurses:location}/lib -Wl,-rpath=${ncurses:location}/lib -L${openssl:location}/lib -Wl,-rpath=${openssl:location}/lib -L${popt:location}/lib -Wl,-rpath=${popt:location}/lib -L${readline:location}/lib -Wl,-rpath=${readline:location}/lib -L${sqlite3:location}/lib -Wl,-rpath=${sqlite3:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib LDFLAGS=-L${bzip2:location}/lib -Wl,-rpath=${bzip2:location}/lib -L${gdbm:location}/lib -Wl,-rpath=${gdbm:location}/lib -L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib -L${glib:location}/lib -Wl,-rpath=${glib:location}/lib -L${libxml2:location}/lib -Wl,-rpath=${libxml2:location}/lib -L${libxslt:location}/lib -Wl,-rpath=${libxslt:location}/lib -L${ncurses:location}/lib -Wl,-rpath=${ncurses:location}/lib -L${openssl:location}/lib -Wl,-rpath=${openssl:location}/lib -L${popt:location}/lib -Wl,-rpath=${popt:location}/lib -L${readline:location}/lib -Wl,-rpath=${readline:location}/lib -L${sqlite3:location}/lib -Wl,-rpath=${sqlite3:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
PKG_CONFIG_PATH=${glib:location}/lib/pkgconfig:${libxml2:location}/lib/pkgconfig:${libxslt:location}/lib/pkgconfig:${openssl:location}/lib/pkgconfig:${popt:location}/lib/pkgconfig:${python2.7:location}/lib/pkconfig:${sqlite3:location}/lib/pkconfig PKG_CONFIG_PATH=${glib:location}/lib/pkgconfig:${libxml2:location}/lib/pkgconfig:${libxslt:location}/lib/pkgconfig:${openssl:location}/lib/pkgconfig:${popt:location}/lib/pkgconfig:${python2.7:location}/lib/pkgconfig:${sqlite3:location}/lib/pkgconfig
LD_LIBRARY_PATH=${bzip2:location}/lib:${gdbm:location}/lib:${gettext:location}/lib:${glib:location}/lib:${libxml2:location}/lib:${libxslt:location}/lib:${ncurses:location}/lib:${openssl:location}/lib:${popt:location}/lib:${readline:location}/lib:${sqlite3:location}/lib:${zlib:location}/lib LD_LIBRARY_PATH=${bzip2:location}/lib:${gdbm:location}/lib:${gettext:location}/lib:${glib:location}/lib:${libxml2:location}/lib:${libxslt:location}/lib:${ncurses:location}/lib:${openssl:location}/lib:${popt:location}/lib:${readline:location}/lib:${sqlite3:location}/lib:${zlib:location}/lib
[cfg-environment] [cfg-environment]
...@@ -112,26 +115,37 @@ scripts = ...@@ -112,26 +115,37 @@ scripts =
slapgrid-supervisord = slapos.grid.svcbackend:supervisord slapgrid-supervisord = slapos.grid.svcbackend:supervisord
slapproxy = slapos.proxy:main slapproxy = slapos.proxy:main
slapos = slapos.entry:main slapos = slapos.entry:main
slapos-watchdog = slapos.grid.watchdog:main
[py]
recipe = zc.recipe.egg
eggs =
${slapos:eggs}
python = python2.7
interpreter = py
scripts = py
[versions] [versions]
zc.buildout = 1.6.0-dev-SlapOS-006 # Use our own buildout version
zc.buildout = 1.6.0-dev-SlapOS-010
# Generated by buildout-versions
Jinja2 = 2.6 Jinja2 = 2.6
Werkzeug = 0.8.3 Werkzeug = 0.8.3
buildout-versions = 1.7 buildout-versions = 1.7
collective.recipe.template = 1.9 collective.recipe.template = 1.9
hexagonit.recipe.cmmi = 1.6 hexagonit.recipe.cmmi = 1.6
lxml = 3.0alpha2 lxml = 3.0.1
meld3 = 0.6.8 meld3 = 0.6.9
netaddr = 0.7.10 netaddr = 0.7.10
slapos.core = 0.28.6 slapos.core = 0.33.1
slapos.libnetworkcache = 0.13.2 slapos.libnetworkcache = 0.13.2
xml-marshaller = 0.9.7 xml-marshaller = 0.9.7
z3c.recipe.scripts = 1.0.1 z3c.recipe.scripts = 1.0.1
zc.recipe.egg = 1.3.2 zc.recipe.egg = 1.3.2
# Required by: # Required by:
# slapos.core==0.28.6 # slapos.core==0.33.1
Flask = 0.9 Flask = 0.9
# Required by: # Required by:
...@@ -139,21 +153,21 @@ Flask = 0.9 ...@@ -139,21 +153,21 @@ Flask = 0.9
hexagonit.recipe.download = 1.5.1 hexagonit.recipe.download = 1.5.1
# Required by: # Required by:
# slapos.core==0.28.6 # slapos.core==0.33.1
netifaces = 0.8 netifaces = 0.8
# Required by: # Required by:
# slapos.core==0.28.6 # slapos.core==0.33.1
# slapos.libnetworkcache==0.13.2 # slapos.libnetworkcache==0.13.2
# supervisor==3.0a12 # supervisor==3.0b1
# zc.buildout==1.6.0-dev-SlapOS-006 # zc.buildout==1.6.0-dev-SlapOS-010
# zope.interface==4.0.1 # zope.interface==4.0.1
setuptools = 0.6c12dev-r88846 setuptools = 0.6c12dev-r88846
# Required by: # Required by:
# slapos.core==0.28.6 # slapos.core==0.33.1
supervisor = 3.0a12 supervisor = 3.0b1
# Required by: # Required by:
# slapos.core==0.28.6 # slapos.core==0.33.1
zope.interface = 4.0.1 zope.interface = 4.0.1
This component is not used anymore - but might be revived within 4 months.
If not used within 6 months, feel free to delete this.
...@@ -20,8 +20,8 @@ filename = sphinx-1.10-beta-snowball.patch ...@@ -20,8 +20,8 @@ filename = sphinx-1.10-beta-snowball.patch
recipe = hexagonit.recipe.cmmi recipe = hexagonit.recipe.cmmi
depends = depends =
${mariadb:version} ${mariadb:version}
url = http://sphinxsearch.com/files/sphinx-2.0.2-beta.tar.gz url = http://sphinxsearch.com/files/sphinx-2.0.5-release.tar.gz
md5sum = fafe0f1a71d0ded32404c067eba7d0b3 md5sum = e71fdb5b0c2911247d48fb30550b9584
configure-options = configure-options =
--with-mysql --with-mysql
--with-mysql-includes=${mariadb:location}/include/mysql --with-mysql-includes=${mariadb:location}/include/mysql
......
...@@ -17,8 +17,8 @@ filename = stunnel-4-hooks.py ...@@ -17,8 +17,8 @@ filename = stunnel-4-hooks.py
[stunnel-4] [stunnel-4]
recipe = hexagonit.recipe.cmmi recipe = hexagonit.recipe.cmmi
url = http://mirror.bit.nl/stunnel/stunnel-4.53.tar.gz url = ftp://ftp.stunnel.org/stunnel/archive/4.x/stunnel-4.54.tar.gz
md5sum = ab3bfc915357d67da18c73f73610d593 md5sum = c2b1db99e3ed547214568959a8ed18ac
pre-configure-hook = ${stunnel-4-hook-download:location}/${stunnel-4-hook-download:filename}:pre_configure_hook pre-configure-hook = ${stunnel-4-hook-download:location}/${stunnel-4-hook-download:filename}:pre_configure_hook
configure-options = configure-options =
--enable-ipv6 --enable-ipv6
......
...@@ -45,6 +45,6 @@ patches = ...@@ -45,6 +45,6 @@ patches =
environment = environment =
PATH=${pkgconfig:location}/bin:%(PATH)s PATH=${pkgconfig:location}/bin:%(PATH)s
PKG_CONFIG_PATH=${openssl:location}/lib/pkgconfig:${garbage-collector:location}/lib/pkconfig PKG_CONFIG_PATH=${openssl:location}/lib/pkgconfig:${garbage-collector:location}/lib/pkgconfig
CPPFLAGS=-I${ncurses:location}/include/ -I${zlib:location}/include/ -I${garbage-collector:location}/include CPPFLAGS=-I${ncurses:location}/include/ -I${zlib:location}/include/ -I${garbage-collector:location}/include
LDFLAGS=-Wl,--as-needed -L${garbage-collector:location}/lib -Wl,-rpath=${garbage-collector:location}/lib -L${ncurses:location}/lib -Wl,-rpath=${ncurses:location}/lib -L${openssl:location}/lib -Wl,-rpath=${openssl:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib LDFLAGS=-Wl,--as-needed -L${garbage-collector:location}/lib -Wl,-rpath=${garbage-collector:location}/lib -L${ncurses:location}/lib -Wl,-rpath=${ncurses:location}/lib -L${openssl:location}/lib -Wl,-rpath=${openssl:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
##############################################################################
#
# Copyright (c) 2010-2013 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from setuptools import setup, find_packages from setuptools import setup, find_packages
import glob import glob
import os import os
version = '0.67-dev' version = '0.71.2-dev'
name = 'slapos.cookbook' name = 'slapos.cookbook'
long_description = open("README.txt").read() + "\n" + \ long_description = open("README.txt").read() + "\n" + \
open("CHANGES.txt").read() + "\n" open("CHANGES.txt").read() + "\n"
...@@ -26,7 +52,6 @@ setup(name=name, ...@@ -26,7 +52,6 @@ setup(name=name,
packages=find_packages(), packages=find_packages(),
include_package_data=True, include_package_data=True,
install_requires=[ install_requires=[
'PyXML', # for full blown python interpreter
'lxml', # for full blown python interpreter 'lxml', # for full blown python interpreter
'netaddr', # to manipulate on IP addresses 'netaddr', # to manipulate on IP addresses
'setuptools', # namespaces 'setuptools', # namespaces
...@@ -36,38 +61,41 @@ setup(name=name, ...@@ -36,38 +61,41 @@ setup(name=name,
'xml_marshaller', # need to communication with slapgrid 'xml_marshaller', # need to communication with slapgrid
'zc.buildout', # plays with buildout 'zc.buildout', # plays with buildout
'zc.recipe.egg', # for scripts generation 'zc.recipe.egg', # for scripts generation
'pytz', # for timezone database
], ],
zip_safe=True, zip_safe=True,
entry_points={ entry_points={
'zc.buildout': [ 'zc.buildout': [
'addresiliency = slapos.recipe.addresiliency:Recipe',
'agent = slapos.recipe.agent:Recipe', 'agent = slapos.recipe.agent:Recipe',
'apache.frontend = slapos.recipe.apache_frontend:Recipe', 'apache.frontend = slapos.recipe.apache_frontend:Recipe',
'apache.zope.backend = slapos.recipe.apache_zope_backend:Recipe',
'apachephp = slapos.recipe.apachephp:Recipe', 'apachephp = slapos.recipe.apachephp:Recipe',
'apacheproxy = slapos.recipe.apacheproxy:Recipe', 'apacheproxy = slapos.recipe.apacheproxy:Recipe',
'apache.zope.backend = slapos.recipe.apache_zope_backend:Recipe',
'certificate_authority.request = slapos.recipe.certificate_authority:Request',
'certificate_authority = slapos.recipe.certificate_authority:Recipe', 'certificate_authority = slapos.recipe.certificate_authority:Recipe',
'certificate_authority.request = slapos.recipe.certificate_authority:Request',
'check_page_content = slapos.recipe.check_page_content:Recipe',
'check_port_listening = slapos.recipe.check_port_listening:Recipe', 'check_port_listening = slapos.recipe.check_port_listening:Recipe',
'check_url_available = slapos.recipe.check_url_available:Recipe', 'check_url_available = slapos.recipe.check_url_available:Recipe',
'check_page_content = slapos.recipe.check_page_content:Recipe',
'cloud9 = slapos.recipe.cloud9:Recipe', 'cloud9 = slapos.recipe.cloud9:Recipe',
'cloudooo.test = slapos.recipe.erp5_test:CloudoooRecipe', 'cloudooo.test = slapos.recipe.erp5_test:CloudoooRecipe',
'cron.d = slapos.recipe.dcron:Part',
'cron = slapos.recipe.dcron:Recipe', 'cron = slapos.recipe.dcron:Recipe',
'cron.d = slapos.recipe.dcron:Part',
'davstorage = slapos.recipe.davstorage:Recipe', 'davstorage = slapos.recipe.davstorage:Recipe',
'downloader = slapos.recipe.downloader:Recipe', 'downloader = slapos.recipe.downloader:Recipe',
'dropbear = slapos.recipe.dropbear:Recipe',
'dropbear.add_authorized_key = slapos.recipe.dropbear:AddAuthorizedKey', 'dropbear.add_authorized_key = slapos.recipe.dropbear:AddAuthorizedKey',
'dropbear.client = slapos.recipe.dropbear:Client', 'dropbear.client = slapos.recipe.dropbear:Client',
'dropbear = slapos.recipe.dropbear:Recipe', 'dumpmdb = slapos.recipe.dumpmdb:Recipe',
'duplicity = slapos.recipe.duplicity:Recipe', 'duplicity = slapos.recipe.duplicity:Recipe',
'egg_test = slapos.recipe.erp5_test:EggTestRecipe', 'egg_test = slapos.recipe.erp5_test:EggTestRecipe',
'equeue = slapos.recipe.equeue:Recipe', 'equeue = slapos.recipe.equeue:Recipe',
'erp5.bootstrap = slapos.recipe.erp5_bootstrap:Recipe', 'erp5.bootstrap = slapos.recipe.erp5_bootstrap:Recipe',
'erp5.promise = slapos.recipe.erp5_promise:Recipe', 'erp5.promise = slapos.recipe.erp5_promise:Recipe',
'erp5scalabilitytestbed = slapos.recipe.erp5scalabilitytestbed:Recipe',
'erp5testnode = slapos.recipe.erp5testnode:Recipe',
'erp5.test = slapos.recipe.erp5_test:Recipe', 'erp5.test = slapos.recipe.erp5_test:Recipe',
'erp5.update = slapos.recipe.erp5_update:Recipe', 'erp5.update = slapos.recipe.erp5_update:Recipe',
'erp5scalabilitytestbed = slapos.recipe.erp5scalabilitytestbed:Recipe',
'erp5testnode = slapos.recipe.erp5testnode:Recipe',
'firefox = slapos.recipe.firefox:Recipe', 'firefox = slapos.recipe.firefox:Recipe',
'fontconfig = slapos.recipe.fontconfig:Recipe', 'fontconfig = slapos.recipe.fontconfig:Recipe',
'generate.mac = slapos.recipe.generatemac:Recipe', 'generate.mac = slapos.recipe.generatemac:Recipe',
...@@ -82,57 +110,62 @@ setup(name=name, ...@@ -82,57 +110,62 @@ setup(name=name,
'gitinit = slapos.recipe.gitinit:Recipe', 'gitinit = slapos.recipe.gitinit:Recipe',
'haproxy = slapos.recipe.haproxy:Recipe', 'haproxy = slapos.recipe.haproxy:Recipe',
'helloworld = slapos.recipe.helloworld:Recipe', 'helloworld = slapos.recipe.helloworld:Recipe',
'importmdb = slapos.recipe.importmdb:Recipe',
'java = slapos.recipe.java:Recipe', 'java = slapos.recipe.java:Recipe',
'kumofs = slapos.recipe.kumofs:Recipe', 'kumofs = slapos.recipe.kumofs:Recipe',
'kvm.frontend = slapos.recipe.kvm_frontend:Recipe',
'kvm = slapos.recipe.kvm:Recipe', 'kvm = slapos.recipe.kvm:Recipe',
'kvm.frontend = slapos.recipe.kvm_frontend:Recipe',
'lamp = slapos.recipe.lamp:Request',
'lamp.request = slapos.recipe.lamp:Request', 'lamp.request = slapos.recipe.lamp:Request',
'lamp.simple = slapos.recipe.lamp:Simple', 'lamp.simple = slapos.recipe.lamp:Simple',
'lamp = slapos.recipe.lamp:Request',
'lamp.static = slapos.recipe.lamp:Static', 'lamp.static = slapos.recipe.lamp:Static',
'libcloudrequest = slapos.recipe.libcloudrequest:Recipe',
'libcloud = slapos.recipe.libcloud:Recipe', 'libcloud = slapos.recipe.libcloud:Recipe',
'libcloudrequest = slapos.recipe.libcloudrequest:Recipe',
'lockfile = slapos.recipe.lockfile:Recipe', 'lockfile = slapos.recipe.lockfile:Recipe',
'logrotate.d = slapos.recipe.logrotate:Part',
'logrotate = slapos.recipe.logrotate:Recipe', 'logrotate = slapos.recipe.logrotate:Recipe',
'logrotate.d = slapos.recipe.logrotate:Part',
'memcached = slapos.recipe.memcached:Recipe', 'memcached = slapos.recipe.memcached:Recipe',
'mkdirectory = slapos.recipe.mkdirectory:Recipe', 'mkdirectory = slapos.recipe.mkdirectory:Recipe',
'mydumper = slapos.recipe.mydumper:Recipe', 'mydumper = slapos.recipe.mydumper:Recipe',
'mysql = slapos.recipe.mysql:Recipe', 'mysql = slapos.recipe.mysql:Recipe',
'nbdserver = slapos.recipe.nbdserver:Recipe', 'nbdserver = slapos.recipe.nbdserver:Recipe',
'nosqltestbed = slapos.recipe.nosqltestbed:NoSQLTestBed', 'nosqltestbed = slapos.recipe.nosqltestbed:NoSQLTestBed',
'notifier = slapos.recipe.notifier:Recipe',
'notifier.callback = slapos.recipe.notifier:Callback', 'notifier.callback = slapos.recipe.notifier:Callback',
'notifier.notify = slapos.recipe.notifier:Notify', 'notifier.notify = slapos.recipe.notifier:Notify',
'notifier = slapos.recipe.notifier:Recipe',
'novnc = slapos.recipe.novnc:Recipe', 'novnc = slapos.recipe.novnc:Recipe',
'onetimeupload = slapos.recipe.onetimeupload:Recipe', 'onetimeupload = slapos.recipe.onetimeupload:Recipe',
'pbs = slapos.recipe.pbs:Recipe', 'pbs = slapos.recipe.pbs:Recipe',
'postgres = slapos.recipe.postgres:Recipe',
'postgres.export = slapos.recipe.postgres:ExportRecipe',
'postgres.import = slapos.recipe.postgres:ImportRecipe',
'proactive = slapos.recipe.proactive:Recipe', 'proactive = slapos.recipe.proactive:Recipe',
'publish = slapos.recipe.publish:Recipe', 'publish = slapos.recipe.publish:Recipe',
'publishurl = slapos.recipe.publishurl:Recipe', 'publishurl = slapos.recipe.publishurl:Recipe',
'pwgen = slapos.recipe.pwgen:Recipe', 'pwgen = slapos.recipe.pwgen:Recipe',
'pwgen.stable = slapos.recipe.pwgen:StablePasswordGeneratorRecipe', 'pwgen.stable = slapos.recipe.pwgen:StablePasswordGeneratorRecipe',
'requestoptional = slapos.recipe.requestoptional:Recipe',
'request = slapos.recipe.request:Recipe', 'request = slapos.recipe.request:Recipe',
'requestoptional = slapos.recipe.request:RequestOptional',
'seleniumrunner = slapos.recipe.seleniumrunner:Recipe', 'seleniumrunner = slapos.recipe.seleniumrunner:Recipe',
'sheepdogtestbed = slapos.recipe.sheepdogtestbed:SheepDogTestBed', 'sheepdogtestbed = slapos.recipe.sheepdogtestbed:SheepDogTestBed',
'shellinabox = slapos.recipe.shellinabox:Recipe',
'shell = slapos.recipe.shell:Recipe', 'shell = slapos.recipe.shell:Recipe',
'shellinabox = slapos.recipe.shellinabox:Recipe',
'signalwrapper= slapos.recipe.signal_wrapper:Recipe', 'signalwrapper= slapos.recipe.signal_wrapper:Recipe',
'simplelogger = slapos.recipe.simplelogger:Recipe', 'simplelogger = slapos.recipe.simplelogger:Recipe',
'siptester = slapos.recipe.siptester:SipTesterRecipe', 'siptester = slapos.recipe.siptester:SipTesterRecipe',
'slapconfiguration = slapos.recipe.slapconfiguration:Recipe', 'slapconfiguration = slapos.recipe.slapconfiguration:Recipe',
'slapcontainer = slapos.recipe.container:Recipe', 'slapcontainer = slapos.recipe.container:Recipe',
'slapmonitor = slapos.recipe.slapmonitor:Recipe', 'slapmonitor = slapos.recipe.slapmonitor:MonitorRecipe',
'slapmonitor-xml = slapos.recipe.slapmonitor:MonitorXMLRecipe',
'slapreport = slapos.recipe.slapreport:Recipe', 'slapreport = slapos.recipe.slapreport:Recipe',
'slaprunner = slapos.recipe.slaprunner:Recipe', 'slaprunner = slapos.recipe.slaprunner:Recipe',
'slaprunner.test = slapos.recipe.slaprunner:Test',
'softwaretype = slapos.recipe.softwaretype:Recipe', 'softwaretype = slapos.recipe.softwaretype:Recipe',
'sphinx= slapos.recipe.sphinx:Recipe', 'sphinx= slapos.recipe.sphinx:Recipe',
'sshkeys_authority.request = slapos.recipe.sshkeys_authority:Request',
'sshkeys_authority = slapos.recipe.sshkeys_authority:Recipe', 'sshkeys_authority = slapos.recipe.sshkeys_authority:Recipe',
'sshkeys_authority.request = slapos.recipe.sshkeys_authority:Request',
'stunnel = slapos.recipe.stunnel:Recipe', 'stunnel = slapos.recipe.stunnel:Recipe',
'symbolic.link = slapos.recipe.symbolic_link:Recipe', 'symbolic.link = slapos.recipe.symbolic_link:Recipe',
'testnode = slapos.recipe.testnode:Recipe',
'tidstorage = slapos.recipe.tidstorage:Recipe', 'tidstorage = slapos.recipe.tidstorage:Recipe',
'urlparse = slapos.recipe._urlparse:Recipe', 'urlparse = slapos.recipe._urlparse:Recipe',
'uuid = slapos.recipe._uuid:Recipe', 'uuid = slapos.recipe._uuid:Recipe',
......
testnode
========
Generic testnode.
...@@ -24,23 +24,75 @@ ...@@ -24,23 +24,75 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# #
############################################################################## ##############################################################################
from slapos.recipe.librecipe import GenericSlapRecipe
import os import os
from slapos.recipe.librecipe import GenericBaseRecipe
class Recipe(GenericBaseRecipe): class Recipe(GenericSlapRecipe):
""" This class provides the installation of the resilience
scripts on the partition.
bin/takeover will perform a rename (must be run manually).
bin/bully will monitor, run elections and perform renames when needed.
"""
def _install(self):
path_list = []
confpath = os.path.join(self.options['etc'], 'bully.conf')
ip_list = self.parameter_dict['ip-list']
print 'Creating bully configuration with ips : %s\n' % ip_list
conf = self.createFile(confpath,
self.substituteTemplate(
self.getTemplateFilename('bully.conf.in'),
{
'self_id': int(self.parameter_dict['number']),
'ip_list': ip_list
}
))
path_list.append(conf)
slap_connection = self.buildout['slap-connection']
if self.optionIsTrue('enable-bully-service', default=False):
bully_dir = self.options['services']
else:
bully_dir = self.options['bin']
bully_wrapper = self.createPythonScript(
name=os.path.join(bully_dir, self.options['wrapper-bully']),
absolute_function='slapos.recipe.addresiliency.bully.run',
arguments={
'confpath': confpath,
'server_url': slap_connection['server-url'],
'key_file': slap_connection.get('key-file'),
'cert_file': slap_connection.get('cert-file'),
'computer_id': slap_connection['computer-id'],
'partition_id': slap_connection['partition-id'],
'software': slap_connection['software-release-url'],
'namebase': self.parameter_dict['namebase'],
})
path_list.append(bully_wrapper)
takeover_wrapper = self.createPythonScript(
name=os.path.join(self.options['bin'], self.options['wrapper-takeover']),
absolute_function='slapos.recipe.addresiliency.takeover.run',
arguments={
'server_url': slap_connection['server-url'],
'key_file': slap_connection.get('key-file'),
'cert_file': slap_connection.get('cert-file'),
'computer_id': slap_connection['computer-id'],
'partition_id': slap_connection['partition-id'],
'software': slap_connection['software-release-url'],
'namebase': self.parameter_dict['namebase'],
})
def install(self): path_list.append(takeover_wrapper)
mapping = self.options.copy()
for key in ('output', 'template', 'recipe', 'mode'):
if key in mapping:
del mapping[key]
with open(self.options['output'], 'w') as output, \ return path_list
open(self.options['template'], 'r') as template:
output.write(template.read() % mapping)
if 'mode' in self.options:
os.chmod(self.options['output'], int(self.options['mode'], 8))
return [self.options['output'], ]
# -*- coding: utf-8 -*-
import logging
import Queue
import socket
import thread
import time
import slapos.recipe.addresiliency.renamer
import slapos
log = logging.getLogger(__name__)
logging.basicConfig(level=logging.DEBUG)
BASE_PORT = 50000
SLEEPING_MINS = 2 # XXX was 30, increase after testing
MSG_PING = 'ping'
MSG_HALT = 'halt'
MSG_VICTORY = 'victory'
MSG_OK = 'ok'
STATE_NORMAL = 'normal'
STATE_WAITINGCONFIRM = 'waitingConfirm'
STATE_ELECTION = 'election'
STATE_REORGANIZATION = 'reorganization'
## Leader is always number 0
class ResilientInstance(object):
def __init__(self, comm, renamer, confpath):
self.comm = comm
self.participant_id = 0
self.state = STATE_NORMAL
self.halter_id = 0
self.inElection = False
self.alive = True
self.mainCanal = self.comm.create_canal([MSG_PING, MSG_HALT, MSG_VICTORY])
self.renamer = renamer
self.okCanal = self.comm.create_canal([MSG_OK])
self.confpath = confpath
self.loadConnectionInfo()
def loadConnectionInfo(self):
params = open(self.confpath, 'r').readlines()
self.total_participants = len(params[0].split())
new_id = int(params[1])
if self.participant_id != new_id:
self.halter_id = new_id
self.participant_id = new_id
log.debug('I am {} of {}'.format(self.participant_id, self.total_participants))
## Needs to be changed to use the master
def aliveManagement(self):
while self.alive:
log.info('XXX sleeping for %d minutes' % SLEEPING_MINS)
time.sleep(SLEEPING_MINS*60)
if self.participant_id == 0:
continue
self.comm.send(MSG_PING, 0)
message, sender = self.okCanal.get()
if message:
continue
self.election()
def listen(self):
while self.alive:
self.comm.recv()
def main(self):
while self.alive:
message, sender = self.mainCanal.get()
if message == MSG_PING:
self.comm.send(MSG_OK, sender)
elif message == MSG_HALT:
self.state = STATE_WAITINGCONFIRM
self.halter_id = int(sender)
self.comm.send(MSG_OK, sender)
elif message == MSG_VICTORY:
if int(sender) == self.halter_id and self.state == STATE_WAITINGCONFIRM:
log.info('{} thinks {} is the leader'.format(self.participant_id, sender))
self.comm.send(MSG_OK, sender)
self.state = STATE_NORMAL
def election(self):
self.inElection = True
self.loadConnectionInfo()
# Check if I'm the highest instance alive
for higher in range(self.participant_id + 1, self.total_participants):
self.comm.send(MSG_PING, higher)
message, sender = self.okCanal.get()
if message:
log.info('{} is alive ({})'.format(higher, self.participant_id))
self.inElection = False
return False
continue
if not self.alive:
return False
# I should be the new coordinator, halt those below me
log.info('Should be ME : {}'.format(self.participant_id))
self.state = STATE_ELECTION
self.halter_id = self.participant_id
ups = []
for lower in range(self.participant_id):
self.comm.send(MSG_HALT, lower)
message, sender = self.okCanal.get()
if message:
ups.append(lower)
#Broadcast Victory
self.state = STATE_REORGANIZATION
for up in ups:
self.comm.send(MSG_VICTORY, up)
message, sender = self.okCanal.get()
if message:
continue
log.info('Something is wrong... let\'s start over')
return self.election()
self.state = STATE_NORMAL
self.active = True
log.info('{} Is THE LEADER'.format(self.participant_id))
self.renamer.failover()
self.inElection = False
return True
class FilteredCanal(object):
def __init__(self, accept, timeout):
self.accept = accept
self.queue = Queue.Queue()
self.timeout = timeout
def append(self, message, sender):
if message in self.accept:
self.queue.put([message, sender])
def get(self):
try:
return self.queue.get(timeout=self.timeout)
except Queue.Empty:
return [None, None]
class Wrapper(object):
def __init__(self, confpath, timeout=20):
self.canals = []
self.ips = []
self.participant_id = 0
self.timeout = timeout
self.confpath = confpath
self.getConnectionInfo()
self.socket = None
def getConnectionInfo(self):
params = open(self.confpath, 'r').readlines()
self.ips = params[0].split()
self.participant_id = int(params[1])
log.debug('I am {} of {}'.format(self.participant_id, self.ips))
def start(self):
self.getConnectionInfo()
self.socket = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
self.socket.bind((self.ips[self.participant_id], BASE_PORT + self.participant_id))
self.socket.listen(5)
def send(self, message, number):
self.getConnectionInfo()
try:
s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
s.connect((self.ips[number], BASE_PORT + number))
s.send(message + (' {}\n'.format(self.participant_id)))
except (socket.error, socket.herror, socket.gaierror, socket.timeout):
pass
finally:
s.close()
def create_canal(self, accept):
created = FilteredCanal(accept, self.timeout)
self.canals.append(created)
return created
def recv(self):
client, _ = self.socket.accept()
client_message = client.recv(1024)
if client_message:
message, sender = client_message.split()
for canal in self.canals:
canal.append(message, int(sender))
def run(args):
confpath = args.pop('confpath')
renamer = slapos.recipe.addresiliency.renamer.Renamer(server_url = args.pop('server_url'),
key_file = args.pop('key_file'),
cert_file = args.pop('cert_file'),
computer_guid = args.pop('computer_id'),
partition_id = args.pop('partition_id'),
software_release = args.pop('software'),
namebase = args.pop('namebase'))
if args:
raise ValueError('Unknown arguments: %s' % ', '.join(args))
wrapper = Wrapper(confpath=confpath, timeout=20)
computer = ResilientInstance(wrapper, renamer=renamer, confpath=confpath)
# idle waiting for connection infos
while computer.total_participants < 2:
computer.loadConnectionInfo()
time.sleep(30)
log.info('Starting')
computer.comm.start()
thread.start_new_thread(computer.listen, ())
thread.start_new_thread(computer.aliveManagement, ())
computer.main()
# -*- coding: utf-8 -*-
import logging
import time
import slapos
log = logging.getLogger(__name__)
logging.basicConfig(level=logging.DEBUG)
class Renamer(object):
def __init__(self, server_url, key_file, cert_file, computer_guid,
partition_id, software_release, namebase):
self.server_url = server_url
self.key_file = key_file
self.cert_file = cert_file
self.computer_guid = computer_guid
self.partition_id = partition_id
self.software_release = software_release
self.namebase = namebase
def _failover(self):
"""\
This method does
- retrieve the broken computer partition
- change its reference to 'broken-...' and its software type to 'frozen'
- retrieve the winner computer partition (attached to this process)
- change its reference to replace the broken one.
later, slapgrid will change its software_type as well.
Then, after running slapgrid-cp a few times, the winner takes over and
a new cp is created to replace it as an importer.
"""
slap = slapos.slap.slap()
slap.initializeConnection(self.server_url, self.key_file, self.cert_file)
# partition that will take over.
cp_winner = slap.registerComputerPartition(computer_guid=self.computer_guid,
partition_id=self.partition_id)
# XXX although we can already rename cp_winner, to change its software type we need to
# get hold of the root cp as well
cp_exporter_ref = self.namebase + '0' # this is ok. the boss is always number zero.
# partition to be deactivated
cp_broken = cp_winner.request(software_release=self.software_release,
software_type='frozen',
state='stopped',
partition_reference=cp_exporter_ref)
broken_new_ref = 'broken-{}'.format(time.strftime("%d-%b_%H:%M:%S", time.gmtime()))
log.debug("Renaming {}: {}".format(cp_broken.getId(), broken_new_ref))
cp_broken.rename(new_name=broken_new_ref)
cp_broken.stopped()
log.debug("Renaming {}: {}".format(cp_winner.getId(), cp_exporter_ref))
# update name (and later, software type) for the partition that will take over
cp_winner.rename(new_name=cp_exporter_ref)
cp_winner.bang(message='partitions have been renamed!')
def failover(self):
try:
self._failover()
log.info('Renaming done')
except slapos.slap.ServerError:
log.info('Internal server error')
# -*- coding: utf-8 -*-
import slapos.recipe.addresiliency.renamer
def run(args):
renamer = slapos.recipe.addresiliency.renamer.Renamer(server_url = args.pop('server_url'),
key_file = args.pop('key_file'),
cert_file = args.pop('cert_file'),
computer_guid = args.pop('computer_id'),
partition_id = args.pop('partition_id'),
software_release = args.pop('software'),
namebase = args.pop('namebase'))
renamer.failover()
#!%(executable)s
import select
import socket
import threading
import time
import sys
sys.path[:] = %(syspath)s
import slapos
from slapos import slap as slapmodule
port = 50000
size = 1024
wait = True
def loadConnectionInfos():
connectionInfos = {}
file = open('%(confpath)s', 'r')
params = file.read().split('\n')
file.close()
ip_list = [x.strip("' ") for x in params[0].strip('[],').split(',')]
connectionInfos['self_id'] = int(params[1])
connectionInfos['server_list'] = \
[(i, ip_list[i]) for i in range(len(ip_list))]
connectionInfos['self_ip'] = ip_list[connectionInfos['self_id']]
return connectionInfos
def rename_broken_and_stop():
try:
slap = slapmodule.slap()
slap.initializeConnection('%(server_url)s',
'%(key_file)s',
'%(cert_file)s')
computer_partition = slap.registerComputerPartition(computer_guid='%(computer_id)s',
partition_id='%(partition_id)s')
broken = computer_partition.request(software_release='%(software)s',
software_type='frozen',
partition_reference='%(namebase)s0')
broken.rename('broken-%%s' %% (time.strftime("%%d-%%b_%%H:%%M:%%S", time.gmtime())))
broken.stopped()
computer_partition.rename('%(namebase)s0')
print 'renaming done\n'
except slapos.slap.slap.ServerError:
print 'Internal server error\n'
def election():
global wait
connection = loadConnectionInfos()
message = "%%s, %%s" %% (connection['self_id'], "Election")
victory = True
for (remote_id, addr) in connection['server_list']:
if remote_id > connection['self_id']:
try:
s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
s.connect((addr, port + remote_id))
s.send(message)
reply = s.recv(size)
if reply == "%%s, %%s" %% (remote_id, "Alive"):
victory = False
except (socket.error, socket.herror, socket.gaierror, socket.timeout):
pass
finally:
s.close()
if victory:
wait = True
for (remote_id, addr) in connection['server_list']:
if remote_id < connection['self_id']:
try:
s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
s.connect((addr, port + remote_id))
s.send("%%s, %%s" %% (connection['self_id'], "Victory"))
except (socket.error, socket.herror, socket.gaierror, socket.timeout):
pass
finally:
s.close()
rename_broken_and_stop()
def failure_detect():
global wait
connection = loadConnectionInfos()
while True:
time.sleep(30)
if wait:
print 'waiting 30 minutes\n'
time.sleep(30 * 60)
wait = False
if not connection['server_list'][0]:
continue
(remote_id, addr) = connection['server_list'][0]
try:
s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
s.connect((addr, port + remote_id))
s.close()
except (socket.error, socket.herror, socket.gaierror, socket.timeout):
s.close()
election()
failure_detect_thread = threading.Thread(target=failure_detect)
failure_detect_thread.start()
connection = loadConnectionInfos()
s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
s.bind((connection['self_ip'], port + connection['self_id']))
s.listen(5)
#election()
while True:
force_election = False
client, _ = s.accept()
client_message = client.recv(1024)
if client_message:
client_id, message = client_message.split(', ')
client_id = eval(client_id)
if message == "Victory":
wait = True
print "%%s wins" %% client_id
elif message == "Election":
print "%%s starts an election" %% client_id
if client_id < connection['self_id']:
client.send("%%s, %%s" %% (connection['self_id'], "Alive"))
force_election = True
client.close()
if force_election:
election()
#!%(executable)s
import logging
import os
import socket
import sys
import thread
import time
sys.path[:] = %(syspath)s
from slapos import slap as slapmodule
import slapos
BASE_PORT = 50000
SLEEPING_MINS = 2
log = logging.getLogger(__name__)
logging.basicConfig(level=logging.DEBUG)
class Renamer(object):
def __init__(self, server_url, key_file, cert_file, computer_guid,
partition_id, software_release, namebase):
self.server_url = server_url
self.key_file = key_file
self.cert_file = cert_file
self.computer_guid = computer_guid
self.partition_id = partition_id
self.software_release = software_release
self.namebase = namebase
def _failover(self):
slap = slapmodule.slap()
slap.initializeConnection(self.server_url,
self.key_file,
self.cert_file)
computer_partition = slap.registerComputerPartition(computer_guid=self.computer_guid,
partition_id=self.partition_id)
broken = computer_partition.request(software_release=self.software_release,
software_type='frozen',
partition_reference=self.namebase+'0')
broken.rename('broken-{}'.format(time.strftime("%%d-%%b_%%H:%%M:%%S", time.gmtime())))
broken.stopped()
computer_partition.rename(self.namebase+'0')
def failover(self):
try:
log.info('renaming done')
except slapos.slap.slap.ServerError:
log.info('Internal server error')
## Leader is always number 0
class ResilientInstance(object):
def __init__(self, comm, renamer, confpath):
self.comm = comm
self.id = 0
self.state = 'normal'
self.halter = 0
self.inElection = False
self.alive = True
self.lastPing = time.clock()
self.mainCanal = self.comm.canal(['ping', 'halt', 'victory'])
self.renamer = renamer
self.okCanal = self.comm.canal(['ok'])
self.confpath = confpath
self.loadConnectionInfos()
def loadConnectionInfos(self):
file = open(self.confpath, 'r')
params = file.read().split('\n')
file.close()
self.nbComp = len([x.strip("' ") for x in params[0].strip('[],').split(',')])
new_id = int(params[1])
if self.id != new_id:
self.halter = new_id
self.id = new_id
## Needs to be changed to use the master
def aliveManagement(self):
while self.alive:
log.info('XXX sleeping for %%d minutes' %% SLEEPING_MINS)
time.sleep(SLEEPING_MINS*60)
if self.id == 0:
continue
self.comm.send('ping', 0)
message, sender = self.okCanal.get()
if message:
continue
self.election()
def listen(self):
while self.alive:
self.comm.recv()
def main(self):
while self.alive:
message, sender = self.mainCanal.get()
if message == 'ping':
self.comm.send('ok', sender)
elif message == 'halt':
self.state = 'waitingConfirm'
self.halter = sender
self.comm.send('ok', sender)
elif message == 'victory':
if int(sender) == int(self.halter) and self.state == 'waitingConfirm':
log.info('{} thinks {} is the leader'.format(self.id, sender))
self.comm.send('ok', sender)
self.state = 'normal'
def election(self):
self.inElection = True
self.loadConnectionInfos()
#Check if I'm the highest instance alive
for higher in range(self.id + 1, self.nbComp):
self.comm.send('ping', higher)
message, sender = self.okCanal.get()
if message:
log.info('{} is alive ({})'.format(higher, self.id))
self.inElection = False
return False
continue
if not self.alive:
return False
#I should be the new coordinator, halt those below me
log.info('Should be ME : {}'.format(self.id))
self.state = 'election'
self.halter = self.id
ups = []
for lower in range(self.id):
self.comm.send('halt', lower)
message, sender = self.okCanal.get()
if message:
ups.append(lower)
#Broadcast Victory
self.state = 'reorganization'
for up in ups:
self.comm.send('victory', up)
message, sender = self.okCanal.get()
if message:
continue
log.info('Something is wrong... let\'s start over')
return self.election()
self.state = 'normal'
self.active = True
log.info('{} Is THE LEADER'.format(self.id))
self.renamer.failover()
self.inElection = False
return True
class FilteredCanal(object):
def __init__(self, accept, timeout):
self.accept = accept
self.list = []
self.lock = thread.allocate_lock()
self.timeout = timeout
def append(self, message, sender):
if message in self.accept:
self.lock.acquire()
self.list.append([message, sender])
self.lock.release()
def get(self):
start = time.clock()
while (time.clock() - start < self.timeout):
self.lock.acquire()
if self.list:
self.lock.release()
return self.list.pop(0)
self.lock.release()
return [None, None]
class Wrapper(object):
def __init__(self, confpath, timeout=20):
self.canals = []
self.ips = []
self.id = 0
self.timeout = timeout
self.confpath = confpath
self.getConnectionInfos()
self.socket = None
def getConnectionInfos(self):
file = open(self.confpath, 'r')
params = file.read().split('\n')
file.close()
self.ips = [x.strip("' ") for x in params[0].strip('[],').split(',')]
self.id = int(params[1])
def start(self):
self.getConnectionInfos()
self.socket = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
self.socket.bind((self.ips[self.id], BASE_PORT + self.id))
self.socket.listen(5)
def send(self, message, number):
self.getConnectionInfos()
try:
s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
s.connect((self.ips[number], BASE_PORT + number))
s.send(message + (' {}\n'.format(self.id)))
except (socket.error, socket.herror, socket.gaierror, socket.timeout):
pass
finally:
s.close()
def canal(self, accept):
created = FilteredCanal(accept, self.timeout)
self.canals.append(created)
return created
def recv(self):
client, _ = self.socket.accept()
client_message = client.recv(1024)
if client_message:
message, sender = client_message.split()
for canal in self.canals:
canal.append(message, int(sender))
def main():
renamer = Renamer(server_url = '%(server_url)s',
key_file = '%(key_file)s',
cert_file = '%(cert_file)s',
computer_guid = '%(computer_id)s',
partition_id = '%(partition_id)s',
software_release = '%(software)s',
namebase = '%(namebase)s')
confpath = '%(confpath)s'
wrapper = Wrapper(confpath=confpath, timeout=20)
computer = ResilientInstance(wrapper, renamer=renamer, confpath=confpath)
#idle waiting for connection infos
while computer.nbComp < 2 :
computer.loadConnectionInfos()
time.sleep(30)
log.info('Starting')
computer.comm.start()
thread.start_new_thread(computer.listen, ())
thread.start_new_thread(computer.main, ())
thread.start_new_thread(computer.aliveManagement, ())
while True:
# XXX tight loop
continue
if __name__ == '__main__':
main()
...@@ -25,65 +25,42 @@ ...@@ -25,65 +25,42 @@
# #
############################################################################# #############################################################################
import os
import sys
import zc.buildout import zc.buildout
import slapos.slap from slapos.recipe.librecipe import GenericBaseRecipe
from slapos.recipe.librecipe import BaseSlapRecipe import sys
from slapos.recipe.librecipe import GenericSlapRecipe
import json
import ConfigParser
# XXX: BaseSlapRecipe and GenericSlapRecipe are deprecated, use class Recipe(GenericBaseRecipe):
# GenericBaseRecipe and move partition parameter fetching to software release.
class Recipe(BaseSlapRecipe, GenericSlapRecipe):
def install(self): def install(self):
self.path_list = [] path_list = []
crond = self.installCrond()
configuration_path = self.options["config"]
header = """[DEFAULT]
master_url = %s
key = %s
slap = slapos.slap.slap() cert = %s
slap.initializeConnection(self.server_url, self.key_file, self.cert_file)
parameter_dict = slap.registerComputerPartition( max_install_duration = %s
self.computer_id, max_uninstall_duration = %s
self.computer_partition_id, max_request_duration = %s
).getInstanceParameterDict() max_destroy_duration = %s
""" % (self.options["master-url"],
"\n ".join(self.options["key"].split("\n")),
"\n ".join(self.options["cert"].split("\n")),
self.options["default_max_install_duration"],
self.options["default_max_uninstall_duration"],
self.options["default_max_request_duration"],
self.options["default_max_destroy_duration"])
# XXX: should probably expect one more (SR-originating) parameter instead
# of using self.work_directory .
configuration_path = os.path.join(self.work_directory, "agent.cfg")
with open(configuration_path, "w") as configuration: with open(configuration_path, "w") as configuration:
configuration.write(parameter_dict["configuration"]) configuration.write(header + self.options["configuration"])
agent_crond_path = os.path.join(crond, "agent")
with open(agent_crond_path, "w") as agent_crond: path_list.append(self.createPythonScript(
agent_crond.write("*/5 * * * * %s -S %s --pidfile=%s --log=%s " self.options['wrapper'],
"%s 2>&1 > /dev/null\n" % ( 'slapos.recipe.librecipe.execute.execute',
self.options["python_binary"], [self.options["agent_binary"], '--pidfile=%s' % self.options["pidfile"],
self.options["agent_binary"], "--log=%s" % self.options["log"], configuration_path]))
self.options["pidfile"],
self.options["log"],
configuration_path,
))
return self.path_list + [configuration_path, agent_crond_path] path_list.append(configuration_path)
def installCrond(self): return path_list
_, ws = self.egg.working_set()
timestamps = self.createDataDirectory('cronstamps')
cron_output = os.path.join(self.log_directory, 'cron-output')
self._createDirectory(cron_output)
catcher = zc.buildout.easy_install.scripts([('catchcron',
__name__ + '.catdatefile', 'catdatefile')], ws, sys.executable,
self.bin_directory, arguments=[cron_output])[0]
self.path_list.append(catcher)
cron_d = os.path.join(self.etc_directory, 'cron.d')
crontabs = os.path.join(self.etc_directory, 'crontabs')
self._createDirectory(cron_d)
self._createDirectory(crontabs)
wrapper = zc.buildout.easy_install.scripts([('crond',
'slapos.recipe.librecipe.execute', 'execute')], ws, sys.executable,
self.wrapper_directory, arguments=[
self.options['dcrond_binary'].strip(), '-s', cron_d, '-c', crontabs,
'-t', timestamps, '-f', '-l', '5', '-M', catcher]
)[0]
self.path_list.append(wrapper)
return cron_d
...@@ -28,6 +28,7 @@ from slapos.recipe.librecipe import BaseSlapRecipe ...@@ -28,6 +28,7 @@ from slapos.recipe.librecipe import BaseSlapRecipe
import os import os
import pkg_resources import pkg_resources
import hashlib import hashlib
import operator
import sys import sys
import zc.buildout import zc.buildout
import zc.recipe.egg import zc.recipe.egg
...@@ -43,6 +44,9 @@ class Recipe(BaseSlapRecipe): ...@@ -43,6 +44,9 @@ class Recipe(BaseSlapRecipe):
'template/%s' % template_name) 'template/%s' % template_name)
def _install(self): def _install(self):
# Define directory not defined in deprecated lib
self.service_directory = os.path.join(self.etc_directory, 'service')
# Check for mandatory arguments # Check for mandatory arguments
frontend_domain_name = self.parameter_dict.get("domain") frontend_domain_name = self.parameter_dict.get("domain")
if frontend_domain_name is None: if frontend_domain_name is None:
...@@ -69,9 +73,17 @@ class Recipe(BaseSlapRecipe): ...@@ -69,9 +73,17 @@ class Recipe(BaseSlapRecipe):
rewrite_rule_list = [] rewrite_rule_list = []
rewrite_rule_zope_list = [] rewrite_rule_zope_list = []
rewrite_rule_zope_path_list = []
slave_dict = {} slave_dict = {}
service_dict = {} service_dict = {}
# Sort slave instance by reference to avoid most security issues
slave_instance_list = sorted(slave_instance_list,
key=operator.itemgetter('slave_reference'))
# dict of used domains, only used to track duplicates
domain_dict = {}
for slave_instance in slave_instance_list: for slave_instance in slave_instance_list:
backend_url = slave_instance.get("url", None) backend_url = slave_instance.get("url", None)
reference = slave_instance.get("slave_reference") reference = slave_instance.get("slave_reference")
...@@ -97,6 +109,12 @@ class Recipe(BaseSlapRecipe): ...@@ -97,6 +109,12 @@ class Recipe(BaseSlapRecipe):
domain = "%s.%s" % (reference.replace("-", "").lower(), domain = "%s.%s" % (reference.replace("-", "").lower(),
frontend_domain_name) frontend_domain_name)
if domain_dict.get(domain):
# This domain already has been processed, skip this new one
continue
else:
domain_dict[domain] = True
# Define the URL where the instance will be available # Define the URL where the instance will be available
# WARNING: we use default ports (443, 80) here. # WARNING: we use default ports (443, 80) here.
slave_dict[reference] = "%s%s/" % (scheme, domain) slave_dict[reference] = "%s%s/" % (scheme, domain)
...@@ -118,6 +136,9 @@ class Recipe(BaseSlapRecipe): ...@@ -118,6 +136,9 @@ class Recipe(BaseSlapRecipe):
# RewriteMap for Zope Virtual Host Monster websites. # RewriteMap for Zope Virtual Host Monster websites.
if slave_instance.get("type", "").lower() in ['zope']: if slave_instance.get("type", "").lower() in ['zope']:
rewrite_rule_zope_list.append(rewrite_rule) rewrite_rule_zope_list.append(rewrite_rule)
# For Zope, we have another dict containing the path e.g '/erp5/...
rewrite_rule_path = "%s %s" % (domain, slave_instance.get('path', ''))
rewrite_rule_zope_path_list.append(rewrite_rule_path)
else: else:
rewrite_rule_list.append(rewrite_rule) rewrite_rule_list.append(rewrite_rule)
...@@ -152,6 +173,7 @@ class Recipe(BaseSlapRecipe): ...@@ -152,6 +173,7 @@ class Recipe(BaseSlapRecipe):
name=frontend_domain_name, name=frontend_domain_name,
rewrite_rule_list=rewrite_rule_list, rewrite_rule_list=rewrite_rule_list,
rewrite_rule_zope_list=rewrite_rule_zope_list, rewrite_rule_zope_list=rewrite_rule_zope_list,
rewrite_rule_zope_path_list=rewrite_rule_zope_path_list,
key=key, certificate=certificate) key=key, certificate=certificate)
# Send connection informations about each slave # Send connection informations about each slave
...@@ -160,9 +182,12 @@ class Recipe(BaseSlapRecipe): ...@@ -160,9 +182,12 @@ class Recipe(BaseSlapRecipe):
"instance: %s" % reference) "instance: %s" % reference)
try: try:
connection_dict = { connection_dict = {
'frontend_ipv6_address': self.getGlobalIPv6Address(), # Send the public IPs (if possible) so that user knows what IP
'frontend_ipv4_address': self.getLocalIPv4Address(), # to bind to its domain name
'site_url': url 'frontend_ipv6_address': self.getGlobalIPv6Address(),
'frontend_ipv4_address': self.parameter_dict.get("public-ipv4",
self.getLocalIPv4Address()),
'site_url': url,
} }
self.setConnectionDict(connection_dict, reference) self.setConnectionDict(connection_dict, reference)
except: except:
...@@ -289,7 +314,7 @@ class Recipe(BaseSlapRecipe): ...@@ -289,7 +314,7 @@ class Recipe(BaseSlapRecipe):
self._createDirectory(crontabs) self._createDirectory(crontabs)
wrapper = zc.buildout.easy_install.scripts([('crond', wrapper = zc.buildout.easy_install.scripts([('crond',
'slapos.recipe.librecipe.execute', 'execute')], self.ws, sys.executable, 'slapos.recipe.librecipe.execute', 'execute')], self.ws, sys.executable,
self.wrapper_directory, arguments=[ self.service_directory, arguments=[
self.options['dcrond_binary'].strip(), '-s', cron_d, '-c', crontabs, self.options['dcrond_binary'].strip(), '-s', cron_d, '-c', crontabs,
'-t', timestamps, '-f', '-l', '5', '-M', catcher] '-t', timestamps, '-f', '-l', '5', '-M', catcher]
)[0] )[0]
...@@ -346,10 +371,13 @@ class Recipe(BaseSlapRecipe): ...@@ -346,10 +371,13 @@ class Recipe(BaseSlapRecipe):
) )
self._writeFile(openssl_configuration, pkg_resources.resource_string( self._writeFile(openssl_configuration, pkg_resources.resource_string(
__name__, 'template/openssl.cnf.ca.in') % config) __name__, 'template/openssl.cnf.ca.in') % config)
# XXX-Cedric: Don't use this, but use slapos.recipe.certificate_authority
# from the instance profile.
self.path_list.extend(zc.buildout.easy_install.scripts([ self.path_list.extend(zc.buildout.easy_install.scripts([
('certificate_authority', __name__ + '.certificate_authority', ('certificate_authority', __name__ + '.certificate_authority',
'runCertificateAuthority')], 'runCertificateAuthority')],
self.ws, sys.executable, self.wrapper_directory, arguments=[dict( self.ws, sys.executable, self.service_directory, arguments=[dict(
openssl_configuration=openssl_configuration, openssl_configuration=openssl_configuration,
openssl_binary=self.options['openssl_binary'], openssl_binary=self.options['openssl_binary'],
certificate=os.path.join(self.ca_dir, 'cacert.pem'), certificate=os.path.join(self.ca_dir, 'cacert.pem'),
...@@ -382,6 +410,8 @@ class Recipe(BaseSlapRecipe): ...@@ -382,6 +410,8 @@ class Recipe(BaseSlapRecipe):
name + '.lock') name + '.lock')
apache_conf['document_root'] = os.path.join(self.data_root_directory, apache_conf['document_root'] = os.path.join(self.data_root_directory,
'htdocs') 'htdocs')
apache_conf['instance_home'] = os.path.join(self.work_directory)
apache_conf['httpd_home'] = self.options['httpd_home']
apache_conf['ip_list'] = ip_list apache_conf['ip_list'] = ip_list
apache_conf['port'] = port apache_conf['port'] = port
apache_conf['server_admin'] = 'admin@' apache_conf['server_admin'] = 'admin@'
...@@ -419,10 +449,11 @@ class Recipe(BaseSlapRecipe): ...@@ -419,10 +449,11 @@ class Recipe(BaseSlapRecipe):
"-f", config_file, "-f", config_file,
"-a", varnish_config["port"], "-T", varnish_config["control_port"], "-a", varnish_config["port"], "-T", varnish_config["control_port"],
"-s", varnish_config["storage"]] "-s", varnish_config["storage"]]
environment = dict(PATH=self.options["binutils_directory"]) environment = dict(PATH="%s:%s" % (self.options["binutils_directory"],
os.environ.get('PATH')))
wrapper = zc.buildout.easy_install.scripts([(name, wrapper = zc.buildout.easy_install.scripts([(name,
'slapos.recipe.librecipe.execute', 'executee')], self.ws, 'slapos.recipe.librecipe.execute', 'executee')], self.ws,
sys.executable, self.wrapper_directory, arguments=[varnish_argument_list, sys.executable, self.service_directory, arguments=[varnish_argument_list,
environment])[0] environment])[0]
self.path_list.append(wrapper) self.path_list.append(wrapper)
...@@ -461,7 +492,7 @@ class Recipe(BaseSlapRecipe): ...@@ -461,7 +492,7 @@ class Recipe(BaseSlapRecipe):
stunnel_conf)) stunnel_conf))
wrapper = zc.buildout.easy_install.scripts([('stunnel', wrapper = zc.buildout.easy_install.scripts([('stunnel',
'slapos.recipe.librecipe.execute', 'execute_wait')], self.ws, 'slapos.recipe.librecipe.execute', 'execute_wait')], self.ws,
sys.executable, self.wrapper_directory, arguments=[ sys.executable, self.service_directory, arguments=[
[self.options['stunnel_binary'].strip(), stunnel_conf_path], [self.options['stunnel_binary'].strip(), stunnel_conf_path],
[certificate, key]] [certificate, key]]
)[0] )[0]
...@@ -470,8 +501,17 @@ class Recipe(BaseSlapRecipe): ...@@ -470,8 +501,17 @@ class Recipe(BaseSlapRecipe):
def installFrontendApache(self, ip_list, key, certificate, name, def installFrontendApache(self, ip_list, key, certificate, name,
port=4443, plain_http_port=8080, port=4443, plain_http_port=8080,
rewrite_rule_list=[], rewrite_rule_zope_list=[], rewrite_rule_list=None,
rewrite_rule_zope_list=None,
rewrite_rule_zope_path_list=None,
access_control_string=None): access_control_string=None):
if rewrite_rule_list is None:
rewrite_rule_list = []
if rewrite_rule_zope_list is None:
rewrite_rule_zope_list = []
if rewrite_rule_zope_path_list is None:
rewrite_rule_zope_path_list = []
# Create htdocs, populate it with default 404 document # Create htdocs, populate it with default 404 document
htdocs_location = os.path.join(self.data_root_directory, 'htdocs') htdocs_location = os.path.join(self.data_root_directory, 'htdocs')
self._createDirectory(htdocs_location) self._createDirectory(htdocs_location)
...@@ -512,9 +552,14 @@ class Recipe(BaseSlapRecipe): ...@@ -512,9 +552,14 @@ class Recipe(BaseSlapRecipe):
# Create configuration file and rewritemaps # Create configuration file and rewritemaps
apachemap_name = "apachemap.txt" apachemap_name = "apachemap.txt"
apachemapzope_name = "apachemapzope.txt" apachemapzope_name = "apachemapzope.txt"
apachemapzopepath_name = "apachemapzopepath.txt"
self.createConfigurationFile(apachemap_name, "\n".join(rewrite_rule_list)) self.createConfigurationFile(apachemap_name, "\n".join(rewrite_rule_list))
self.createConfigurationFile(apachemapzope_name, self.createConfigurationFile(apachemapzope_name,
"\n".join(rewrite_rule_zope_list)) "\n".join(rewrite_rule_zope_list))
self.createConfigurationFile(apachemapzopepath_name,
"\n".join(rewrite_rule_zope_path_list))
apache_conf = self._getApacheConfigurationDict(name, ip_list, port) apache_conf = self._getApacheConfigurationDict(name, ip_list, port)
apache_conf['ssl_snippet'] = self.substituteTemplate( apache_conf['ssl_snippet'] = self.substituteTemplate(
self.getTemplateFilename('apache.ssl-snippet.conf.in'), self.getTemplateFilename('apache.ssl-snippet.conf.in'),
...@@ -532,12 +577,17 @@ class Recipe(BaseSlapRecipe): ...@@ -532,12 +577,17 @@ class Recipe(BaseSlapRecipe):
path = self.substituteTemplate( path = self.substituteTemplate(
self.getTemplateFilename('apache.conf.path-protected.in'), self.getTemplateFilename('apache.conf.path-protected.in'),
dict(path='/', access_control_string='none')) dict(path='/',
access_control_string='none',
document_root=apache_conf['document_root'],
)
)
apache_conf.update(**dict( apache_conf.update(**dict(
path_enable=path, path_enable=path,
apachemap_path=os.path.join(self.etc_directory, apachemap_name), apachemap_path=os.path.join(self.etc_directory, apachemap_name),
apachemapzope_path=os.path.join(self.etc_directory, apachemapzope_name), apachemapzope_path=os.path.join(self.etc_directory, apachemapzope_name),
apachemapzopepath_path=os.path.join(self.etc_directory, apachemapzopepath_name),
apache_domain=name, apache_domain=name,
https_port=port, https_port=port,
plain_http_port=plain_http_port, plain_http_port=plain_http_port,
...@@ -553,7 +603,7 @@ class Recipe(BaseSlapRecipe): ...@@ -553,7 +603,7 @@ class Recipe(BaseSlapRecipe):
self.path_list.extend(zc.buildout.easy_install.scripts([( self.path_list.extend(zc.buildout.easy_install.scripts([(
'frontend_apache', 'slapos.recipe.erp5.apache', 'runApache')], self.ws, 'frontend_apache', 'slapos.recipe.erp5.apache', 'runApache')], self.ws,
sys.executable, self.wrapper_directory, arguments=[ sys.executable, self.service_directory, arguments=[
dict( dict(
required_path_list=[key, certificate], required_path_list=[key, certificate],
binary=self.options['httpd_binary'], binary=self.options['httpd_binary'],
......
...@@ -5,12 +5,13 @@ ...@@ -5,12 +5,13 @@
PidFile "%(pid_file)s" PidFile "%(pid_file)s"
ServerName %(server_name)s ServerName %(server_name)s
DocumentRoot %(document_root)s DocumentRoot %(document_root)s
ServerRoot %(instance_home)s
%(listen)s %(listen)s
ServerAdmin %(server_admin)s ServerAdmin %(server_admin)s
DefaultType text/plain DefaultType text/plain
TypesConfig conf/mime.types TypesConfig %(httpd_home)s/conf/mime.types
AddType application/x-compress .Z AddType application/x-compress .Z
AddType application/x-gzip .gz .tgz AddType application/x-gzip .gz .tgz
...@@ -32,23 +33,23 @@ CustomLog "%(access_log)s" common ...@@ -32,23 +33,23 @@ CustomLog "%(access_log)s" common
#LoadModule unixd_module modules/mod_unixd.so #LoadModule unixd_module modules/mod_unixd.so
#LoadModule access_compat_module modules/mod_access_compat.so #LoadModule access_compat_module modules/mod_access_compat.so
#LoadModule authz_core_module modules/mod_authz_core.so #LoadModule authz_core_module modules/mod_authz_core.so
LoadModule authz_host_module modules/mod_authz_host.so LoadModule authz_host_module %(httpd_home)s/modules/mod_authz_host.so
LoadModule log_config_module modules/mod_log_config.so LoadModule log_config_module %(httpd_home)s/modules/mod_log_config.so
LoadModule deflate_module modules/mod_deflate.so LoadModule deflate_module %(httpd_home)s/modules/mod_deflate.so
LoadModule setenvif_module modules/mod_setenvif.so LoadModule setenvif_module %(httpd_home)s/modules/mod_setenvif.so
LoadModule version_module modules/mod_version.so LoadModule version_module %(httpd_home)s/modules/mod_version.so
LoadModule proxy_module modules/mod_proxy.so LoadModule proxy_module %(httpd_home)s/modules/mod_proxy.so
LoadModule proxy_http_module modules/mod_proxy_http.so LoadModule proxy_http_module %(httpd_home)s/modules/mod_proxy_http.so
LoadModule ssl_module modules/mod_ssl.so LoadModule ssl_module %(httpd_home)s/modules/mod_ssl.so
LoadModule mime_module modules/mod_mime.so LoadModule mime_module %(httpd_home)s/modules/mod_mime.so
LoadModule dav_module modules/mod_dav.so LoadModule dav_module %(httpd_home)s/modules/mod_dav.so
LoadModule dav_fs_module modules/mod_dav_fs.so LoadModule dav_fs_module %(httpd_home)s/modules/mod_dav_fs.so
LoadModule negotiation_module modules/mod_negotiation.so LoadModule negotiation_module %(httpd_home)s/modules/mod_negotiation.so
LoadModule rewrite_module modules/mod_rewrite.so LoadModule rewrite_module %(httpd_home)s/modules/mod_rewrite.so
LoadModule headers_module modules/mod_headers.so LoadModule headers_module %(httpd_home)s/modules/mod_headers.so
LoadModule cache_module modules/mod_cache.so LoadModule cache_module %(httpd_home)s/modules/mod_cache.so
LoadModule mem_cache_module modules/mod_mem_cache.so LoadModule mem_cache_module %(httpd_home)s/modules/mod_mem_cache.so
LoadModule antiloris_module modules/mod_antiloris.so LoadModule antiloris_module %(httpd_home)s/modules/mod_antiloris.so
# The following directives modify normal HTTP response behavior to # The following directives modify normal HTTP response behavior to
# handle known problems with browser implementations. # handle known problems with browser implementations.
...@@ -99,17 +100,24 @@ Header append Vary User-Agent ...@@ -99,17 +100,24 @@ Header append Vary User-Agent
ProxyTimeout 600 ProxyTimeout 600
RewriteEngine On RewriteEngine On
# Define the two rewritemaps : one for zope, one generic # Define the two RewriteMaps (key -> value store): one for Zope, one generic
# containing: rewritten URL -> original URL (a.k.a VirtualHostBase in Zope)
RewriteMap apachemapzope txt:%(apachemapzope_path)s RewriteMap apachemapzope txt:%(apachemapzope_path)s
RewriteMap apachemapgeneric txt:%(apachemap_path)s RewriteMap apachemapgeneric txt:%(apachemap_path)s
# Define another RewriteMap for Zope, containing:
# rewritten URL -> VirtualHostRoot
RewriteMap apachemapzopepath txt:%(apachemapzopepath_path)s
# First, we check if we have a zope backend server # First, we check if we have a zope backend server
# If so, let's use Virtual Host Daemon rewrite # If so, let's use Virtual Host Daemon rewrite
RewriteCond ${apachemapzope:%%{SERVER_NAME}} >"" RewriteCond ${apachemapzope:%%{SERVER_NAME}} >""
RewriteRule ^/(.*)$ ${apachemapzope:%%{SERVER_NAME}}/VirtualHostBase/https/%%{SERVER_NAME}:%%{SERVER_PORT}/VirtualHostRoot/$1 [L,P] # We suppose that Apache listens to 443 (even indirectly thanks to things like iptables)
RewriteRule ^/(.*)$ ${apachemapzope:%%{SERVER_NAME}}/VirtualHostBase/https/%%{SERVER_NAME}:443/${apachemapzopepath:%%{SERVER_NAME}}/VirtualHostRoot/$1 [L,P]
# If we have generic backend server, let's rewrite without virtual host daemon # If we have generic backend server, let's rewrite without virtual host daemon
RewriteCond ${apachemapgeneric:%%{SERVER_NAME}} >"" RewriteCond ${apachemapgeneric:%%{SERVER_NAME}} >""
# We suppose that Apache listens to 443 (even indirectly thanks to things like iptables)
RewriteRule ^/(.*)$ ${apachemapgeneric:%%{SERVER_NAME}}/$1 [L,P] RewriteRule ^/(.*)$ ${apachemapgeneric:%%{SERVER_NAME}}/$1 [L,P]
# If nothing exist : put a nice error # If nothing exist : put a nice error
......
# Path protected <Directory %(path)s>
<Location %(path)s>
Order Deny,Allow Order Deny,Allow
Allow from %(access_control_string)s Allow from %(access_control_string)s
</Location> </Directory>
<Directory %(document_root)s>
Order Allow,Deny
Allow from All
</Directory>
...@@ -78,7 +78,7 @@ class Recipe(GenericBaseRecipe): ...@@ -78,7 +78,7 @@ class Recipe(GenericBaseRecipe):
if not os.path.exists(secret_key_filename): if not os.path.exists(secret_key_filename):
secret_key = uuencode(os.urandom(45)).strip() secret_key = uuencode(os.urandom(45)).strip()
# Remove unsafe characters # Remove unsafe characters
secret_key = secret_key.translate(None, '"\'') secret_key = secret_key.translate(None, '"\'\\')
with open(secret_key_filename, 'w') as secret_key_file: with open(secret_key_filename, 'w') as secret_key_file:
secret_key_file.write(secret_key) secret_key_file.write(secret_key)
else: else:
......
...@@ -15,4 +15,4 @@ date.timezone = Europe/Paris ...@@ -15,4 +15,4 @@ date.timezone = Europe/Paris
file_uploads = On file_uploads = On
upload_max_filesize = 8M upload_max_filesize = 8M
post_max_size = 8M post_max_size = 8M
magic_quotes_gpc=0ff magic_quotes_gpc=Off
...@@ -3,7 +3,6 @@ ...@@ -3,7 +3,6 @@
# Basic server configuration # Basic server configuration
PidFile "%(pid_file)s" PidFile "%(pid_file)s"
LockFile "%(lock_file)s"
Listen %(ip)s:%(port)s Listen %(ip)s:%(port)s
ServerAdmin someone@email ServerAdmin someone@email
DefaultType text/plain DefaultType text/plain
...@@ -22,13 +21,15 @@ CustomLog "%(access_log)s" common ...@@ -22,13 +21,15 @@ CustomLog "%(access_log)s" common
<Directory /> <Directory />
Options FollowSymLinks Options FollowSymLinks
AllowOverride None AllowOverride None
Order deny,allow Require all denied
Deny from all
</Directory> </Directory>
ProxyPass / %(backend_url)s ProxyPass / %(backend_url)s
# List of modules # List of modules
LoadModule unixd_module modules/mod_unixd.so
LoadModule access_compat_module modules/mod_access_compat.so
LoadModule authz_core_module modules/mod_authz_core.so
LoadModule authz_host_module modules/mod_authz_host.so LoadModule authz_host_module modules/mod_authz_host.so
LoadModule log_config_module modules/mod_log_config.so LoadModule log_config_module modules/mod_log_config.so
LoadModule setenvif_module modules/mod_setenvif.so LoadModule setenvif_module modules/mod_setenvif.so
......
...@@ -9,11 +9,22 @@ if [ -z $URL ]; then ...@@ -9,11 +9,22 @@ if [ -z $URL ]; then
exit 3 exit 3
fi fi
CODE=$(%(curl_path)s -k -sL $URL -w %%{http_code} -o /dev/null) CODE=$(%(curl_path)s -g -k -sL $URL -w %%{http_code} -o /dev/null)
if [ $? -eq 3 ]; then
echo "URL malformed: $URL." >&2
exit 1
fi
if [ $? -eq 7 ]; then
echo "Failed to connect to host: $URL." >&2
exit 1
fi
if [ ! $CODE ]; then if [ ! $CODE ]; then
echo "$URL is not available (server not reachable)." >&2 echo "$URL is not available (server not reachable)." >&2
exit 1 exit 1
fi
if [ $CODE -eq 000 ]; then if [ $CODE -eq 000 ]; then
echo "$URL is not available (server not reachable)." >&2 echo "$URL is not available (server not reachable)." >&2
......
...@@ -33,26 +33,22 @@ class Recipe(GenericBaseRecipe): ...@@ -33,26 +33,22 @@ class Recipe(GenericBaseRecipe):
def install(self): def install(self):
self.logger.info("Installing dcron...") self.logger.info("Installing dcron...")
path_list = [] options = self.options
script = self.createWrapper(name=options['binary'],
command=options['dcrond-binary'].strip(),
parameters=[
'-s', options['cron-entries'],
'-c', options['crontabs'],
'-t', options['cronstamps'],
'-f', '-l', '5',
'-M', options['catcher']
])
cronstamps = self.options['cronstamps']
cron_d = self.options['cron-entries']
crontabs = self.options['crontabs']
catcher = self.options['catcher']
binary = self.options['binary']
script = self.createPythonScript(binary,
'slapos.recipe.librecipe.execute.execute',
[self.options['dcrond-binary'].strip(), '-s', cron_d, '-c', crontabs,
'-t', cronstamps, '-f', '-l', '5', '-M', catcher]
)
path_list.append(script)
self.logger.debug('Main cron executable created at : %r', script) self.logger.debug('Main cron executable created at : %r', script)
self.logger.info("dcron successfully installed.") self.logger.info("dcron successfully installed.")
return path_list return [script]
......
...@@ -158,6 +158,24 @@ class Client(GenericBaseRecipe): ...@@ -158,6 +158,24 @@ class Client(GenericBaseRecipe):
return [wrapper] return [wrapper]
def keysplit(s):
"""
Split a string like "ssh-rsa AKLFKJSL..... ssh-rsa AAAASAF...."
and return the individual key_type + key strings.
"""
si = iter(s.split(' '))
while True:
key_type = next(si)
try:
key_value = next(si)
except StopIteration:
# odd number of elements, should not happen, yield the last one by itself
yield key_type
break
yield '%s %s' % (key_type, key_value)
class AddAuthorizedKey(GenericBaseRecipe): class AddAuthorizedKey(GenericBaseRecipe):
def install(self): def install(self):
...@@ -167,6 +185,9 @@ class AddAuthorizedKey(GenericBaseRecipe): ...@@ -167,6 +185,9 @@ class AddAuthorizedKey(GenericBaseRecipe):
path_list.append(ssh) path_list.append(ssh)
authorized_keys = AuthorizedKeysFile(os.path.join(ssh, 'authorized_keys')) authorized_keys = AuthorizedKeysFile(os.path.join(ssh, 'authorized_keys'))
authorized_keys.append(self.options['key']) for key in keysplit(self.options['key']):
# XXX key might actually be the string 'None' or 'null'
authorized_keys.append(key)
return path_list return path_list
...@@ -30,17 +30,19 @@ class Recipe(GenericBaseRecipe): ...@@ -30,17 +30,19 @@ class Recipe(GenericBaseRecipe):
def install(self): def install(self):
commandline = [self.options['equeue-binary']] parameters = [
commandline.extend(['--database', self.options['database']]) '--database', self.options['database'],
commandline.extend(['-l', self.options['log']]) '-l', self.options['log'],
]
if 'loglevel' in self.options: if 'loglevel' in self.options:
commandline.extend(['--loglevel', self.options['loglevel']]) parameters.extend(['--loglevel', self.options['loglevel']])
commandline.append(self.options['socket']) parameters.append(self.options['socket'])
wrapper = self.createWrapper(name=self.options['wrapper'],
command=self.options['equeue-binary'],
parameters=parameters)
return [wrapper]
return [self.createPythonScript(
self.options['wrapper'],
'slapos.recipe.librecipe.execute.execute',
commandline,
)]
...@@ -154,26 +154,38 @@ class Recipe(GenericBaseRecipe): ...@@ -154,26 +154,38 @@ class Recipe(GenericBaseRecipe):
# TODO: move to a separate recipe (ack'ed by Cedric) # TODO: move to a separate recipe (ack'ed by Cedric)
# percona toolkit (formerly known as maatkit) installation # percona toolkit (formerly known as maatkit) installation
for pt_script_name in ( for pt_script_name in (
'pt-align',
'pt-archiver', 'pt-archiver',
'pt-collect',
'pt-config-diff', 'pt-config-diff',
'pt-deadlock-logger', 'pt-deadlock-logger',
'pt-diskstats',
'pt-duplicate-key-checker', 'pt-duplicate-key-checker',
'pt-fifo-split', 'pt-fifo-split',
'pt-find', 'pt-find',
'pt-fingerprint',
'pt-fk-error-logger', 'pt-fk-error-logger',
'pt-heartbeat', 'pt-heartbeat',
'pt-index-usage', 'pt-index-usage',
'pt-ioprofile',
'pt-kill', 'pt-kill',
'pt-log-player', 'pt-log-player',
'pt-mext',
'pt-mysql-summary',
'pt-online-schema-change', 'pt-online-schema-change',
'pt-pmp',
'pt-query-advisor', 'pt-query-advisor',
'pt-query-digest', 'pt-query-digest',
'pt-show-grants', 'pt-show-grants',
'pt-sift',
'pt-slave-delay', 'pt-slave-delay',
'pt-slave-find', 'pt-slave-find',
'pt-slave-restart', 'pt-slave-restart',
'pt-stalk',
'pt-summary',
'pt-table-checksum', 'pt-table-checksum',
'pt-table-sync', 'pt-table-sync',
'pt-table-usage',
'pt-tcp-model', 'pt-tcp-model',
'pt-trend', 'pt-trend',
'pt-upgrade', 'pt-upgrade',
......
...@@ -2,6 +2,7 @@ import os ...@@ -2,6 +2,7 @@ import os
import subprocess import subprocess
import time import time
import sys import sys
import pytz
def runMysql(args): def runMysql(args):
...@@ -41,32 +42,50 @@ def updateMysql(args): ...@@ -41,32 +42,50 @@ def updateMysql(args):
sleep = 30 sleep = 30
is_succeed = False is_succeed = False
while True: while True:
if not is_succeed: mysql_upgrade_list = [conf['mysql_upgrade_binary'], '--no-defaults', '--user=root', '--socket=%s' % conf['socket']]
mysql_upgrade_list = [conf['mysql_upgrade_binary'], '--no-defaults', '--user=root', '--socket=%s' % conf['socket']] mysql_upgrade = subprocess.Popen(mysql_upgrade_list, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
mysql_upgrade = subprocess.Popen(mysql_upgrade_list, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) result = mysql_upgrade.communicate()[0]
result = mysql_upgrade.communicate()[0] if mysql_upgrade.returncode is None:
if mysql_upgrade.returncode is None: mysql_upgrade.kill()
mysql_upgrade.kill() if mysql_upgrade.returncode != 0 and not 'is already upgraded' in result:
if mysql_upgrade.returncode != 0 and not 'is already upgraded' in result: print "Command %r failed with result:\n%s" % (mysql_upgrade_list, result)
print "Command %r failed with result:\n%s" % (mysql_upgrade_list, result) else:
print 'Sleeping for %ss and retrying' % sleep if mysql_upgrade.returncode == 0:
print "MySQL database upgraded with result:\n%s" % result
else: else:
if mysql_upgrade.returncode == 0: print "No need to upgrade MySQL database"
print "MySQL database upgraded with result:\n%s" % result mysql_list = [conf['mysql_binary'].strip(), '--no-defaults', '-B', '--user=root', '--socket=%s' % conf['socket']]
else: mysql = subprocess.Popen(mysql_list, stdin=subprocess.PIPE,
print "No need to upgrade MySQL database" stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
mysql_list = [conf['mysql_binary'].strip(), '--no-defaults', '-B', '--user=root', '--socket=%s' % conf['socket']] result = mysql.communicate(conf['mysql_script'])[0]
mysql = subprocess.Popen(mysql_list, stdin=subprocess.PIPE, if mysql.returncode is None:
stdout=subprocess.PIPE, stderr=subprocess.STDOUT) mysql.kill()
result = mysql.communicate(conf['mysql_script'])[0] if mysql.returncode != 0:
if mysql.returncode is None: print 'Command %r failed with:\n%s' % (mysql_list, result)
mysql.kill() else:
# import timezone database
mysql_tzinfo_to_sql_binary = os.path.join(
os.path.dirname(conf['mysql_binary'].strip()), 'mysql_tzinfo_to_sql')
zoneinfo_directory = '%s/zoneinfo' % os.path.dirname(pytz.__file__)
mysql_tzinfo_to_sql_list = [mysql_tzinfo_to_sql_binary, zoneinfo_directory]
mysql_tzinfo_to_sql = subprocess.Popen(mysql_tzinfo_to_sql_list, stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
timezone_sql = mysql_tzinfo_to_sql.communicate()[0]
if mysql.returncode != 0: if mysql.returncode != 0:
print 'Command %r failed with:\n%s' % (mysql_list, result) print 'Command %r failed with:\n%s' % (mysql_tzinfo_to_sql_list, result)
print 'Sleeping for %ss and retrying' % sleep
else: else:
mysql = subprocess.Popen(mysql_list + ['mysql',], stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
result = mysql.communicate(timezone_sql)[0]
if mysql.returncode is None:
mysql.kill()
if mysql.returncode != 0:
print 'Command %r failed with:\n%s' % (mysql_list, result)
is_succeed = True is_succeed = True
print 'SlapOS initialisation script succesfully applied on database.' if is_succeed:
print 'SlapOS initialisation script succesfully applied on database.'
break
print 'Sleeping for %ss and retrying' % sleep
sys.stdout.flush() sys.stdout.flush()
sys.stderr.flush() sys.stderr.flush()
time.sleep(sleep) time.sleep(sleep)
...@@ -24,7 +24,7 @@ long_query_time = 1 ...@@ -24,7 +24,7 @@ long_query_time = 1
max_allowed_packet = 128M max_allowed_packet = 128M
query_cache_size = 32M query_cache_size = 32M
plugin-load = ha_mroonga.so;ha_sphinx.so;handlersocket.so plugin-load = ha_mroonga.so;handlersocket.so
# By default only 100 connections are allowed, when using zeo # By default only 100 connections are allowed, when using zeo
# we may have much more connections # we may have much more connections
......
...@@ -73,7 +73,11 @@ class Recipe(GenericBaseRecipe): ...@@ -73,7 +73,11 @@ class Recipe(GenericBaseRecipe):
server_snippet = "" server_snippet = ""
i = 0 i = 0
name = self.options['name'] name = self.options['name']
for address in self.options['backend-list'].split(): backend_list = self.options['backend-list']
if isinstance(backend_list, str):
# BBB
backend_list = backend_list.split()
for address in backend_list:
i += 1 i += 1
server_snippet += self.substituteTemplate( server_snippet += self.substituteTemplate(
snippet_filename, dict( snippet_filename, dict(
......
...@@ -33,14 +33,25 @@ class Recipe(GenericBaseRecipe): ...@@ -33,14 +33,25 @@ class Recipe(GenericBaseRecipe):
kvm instance configuration. kvm instance configuration.
""" """
def install(self): def install(self):
# Sanitize drive type parameter
self.options.setdefault('disk-type', 'virtio')
if not self.options.get('disk-type') in ['ide', 'scsi', 'sd',
'mtd', 'floppy', 'pflash', 'virtio']:
print 'Warning: "disk-type" parameter is not in allowed values. Using ' \
'"virtio" value.'
self.options['disk-type'] = 'virtio'
config = dict( config = dict(
tap_interface=self.options['tap'], tap_interface=self.options['tap'],
vnc_ip=self.options['vnc-ip'], vnc_ip=self.options['vnc-ip'],
vnc_port=self.options['vnc-port'], vnc_port=self.options['vnc-port'],
nbd_ip=self.options['nbd-ip'], nbd_ip=self.options['nbd-host'],
nbd_port=self.options['nbd-port'], nbd_port=self.options['nbd-port'],
nbd2_ip=self.options.get('nbd2-host', ''),
nbd2_port=self.options.get('nbd2-port', 1024),
disk_path=self.options['disk-path'], disk_path=self.options['disk-path'],
disk_size=self.options['disk-size'], disk_size=self.options['disk-size'],
disk_type=self.options['disk-type'],
mac_address=self.options['mac-address'], mac_address=self.options['mac-address'],
smp_count=self.options['smp-count'], smp_count=self.options['smp-count'],
ram_size=self.options['ram-size'], ram_size=self.options['ram-size'],
...@@ -50,7 +61,6 @@ class Recipe(GenericBaseRecipe): ...@@ -50,7 +61,6 @@ class Recipe(GenericBaseRecipe):
shell_path=self.options['shell-path'], shell_path=self.options['shell-path'],
qemu_path=self.options['qemu-path'], qemu_path=self.options['qemu-path'],
qemu_img_path=self.options['qemu-img-path'], qemu_img_path=self.options['qemu-img-path'],
# XXX Weak password
vnc_passwd=self.options['passwd'] vnc_passwd=self.options['passwd']
) )
...@@ -67,4 +77,3 @@ class Recipe(GenericBaseRecipe): ...@@ -67,4 +77,3 @@ class Recipe(GenericBaseRecipe):
return [runner_path, controller_path] return [runner_path, controller_path]
...@@ -30,26 +30,32 @@ def getSocketStatus(host, port): ...@@ -30,26 +30,32 @@ def getSocketStatus(host, port):
disk_path = '%(disk_path)s' disk_path = '%(disk_path)s'
if not os.path.exists(disk_path): if not os.path.exists(disk_path):
subprocess.Popen(['%(qemu_img_path)s', 'create' ,'-f', 'qcow2', subprocess.Popen(['%(qemu_img_path)s', 'create' ,'-f', 'qcow2',
'%(disk_path)s', '%(disk_size)sG']) disk_path, '%(disk_size)sG'])
kvm_argument_list = ['%(qemu_path)s', '-enable-kvm', '-net', 'nic,macaddr=%(mac_address)s', kvm_argument_list = ['%(qemu_path)s',
'-enable-kvm', '-net', 'nic,macaddr=%(mac_address)s',
'-net', 'tap,ifname=%(tap_interface)s,script=no,downscript=no', '-net', 'tap,ifname=%(tap_interface)s,script=no,downscript=no',
'-smp', '%(smp_count)s', '-smp', '%(smp_count)s',
'-m', '%(ram_size)s', '-m', '%(ram_size)s',
'-drive', 'file=%(disk_path)s,if=virtio,boot=on', '-drive', 'file=%(disk_path)s,if=%(disk_type)s',
'-vnc', '%(vnc_ip)s:1,ipv4,password', '-vnc', '%(vnc_ip)s:1,ipv4,password',
'-boot', 'menu=on', '-boot', 'menu=on',
'-qmp', 'unix:%(socket_path)s,server', '-qmp', 'unix:%(socket_path)s,server',
'-pidfile', '%(pid_file_path)s', '-pidfile', '%(pid_file_path)s',
] ]
# Try to connect to NBD server # Try to connect to NBD server (and second nbd if defined)
s = getSocketStatus('%(nbd_ip)s', %(nbd_port)s) for nbd_ip, nbd_port in (
if s is None: ('%(nbd_ip)s', %(nbd_port)s), ('%(nbd2_ip)s', %(nbd2_port)s)):
# NBD is not available : launch kvm without it if nbd_ip and nbd_port:
print 'Warning : Nbd is not available.' s = getSocketStatus(nbd_ip, nbd_port)
os.execv('%(qemu_path)s', kvm_argument_list) if s is None:
else: # NBD is not available : launch kvm without it
# NBD is available print 'Warning : Nbd is not available.'
kvm_argument_list.extend(['-cdrom', 'nbd:[%(nbd_ip)s]:%(nbd_port)s']) else:
os.execv('%(qemu_path)s', kvm_argument_list) # NBD is available
kvm_argument_list.extend([
'-drive',
'file=nbd:[%%s]:%%s,media=cdrom' %% (nbd_ip, nbd_port)])
os.execv('%(qemu_path)s', kvm_argument_list)
...@@ -21,7 +21,7 @@ long_query_time = 5 ...@@ -21,7 +21,7 @@ long_query_time = 5
max_allowed_packet = 128M max_allowed_packet = 128M
query_cache_size = 32M query_cache_size = 32M
plugin-load = ha_mroonga.so;ha_sphinx.so plugin-load = ha_mroonga.so
# The following are important to configure and depend a lot on to the size of # The following are important to configure and depend a lot on to the size of
# your database and the available resources. # your database and the available resources.
......
...@@ -90,13 +90,22 @@ class BaseSlapRecipe: ...@@ -90,13 +90,22 @@ class BaseSlapRecipe:
] ]
# SLAP related information # SLAP related information
slap_connection = buildout['slap_connection'] try:
self.computer_id = slap_connection['computer_id'] slap_connection = buildout['slap_connection']
self.computer_partition_id = slap_connection['partition_id'] self.computer_id = slap_connection['computer_id']
self.server_url = slap_connection['server_url'] self.computer_partition_id = slap_connection['partition_id']
self.software_release_url = slap_connection['software_release_url'] self.server_url = slap_connection['server_url']
self.key_file = slap_connection.get('key_file') self.software_release_url = slap_connection['software_release_url']
self.cert_file = slap_connection.get('cert_file') self.key_file = slap_connection.get('key_file')
self.cert_file = slap_connection.get('cert_file')
except zc.buildout.buildout.MissingSection:
slap_connection = buildout['slap-connection']
self.computer_id = slap_connection['computer-id']
self.computer_partition_id = slap_connection['partition-id']
self.server_url = slap_connection['server-url']
self.software_release_url = slap_connection['software-release-url']
self.key_file = slap_connection.get('key-file')
self.cert_file = slap_connection.get('cert-file')
# setup egg to give possibility to generate scripts # setup egg to give possibility to generate scripts
self.egg = zc.recipe.egg.Egg(buildout, options['recipe'], options) self.egg = zc.recipe.egg.Egg(buildout, options['recipe'], options)
...@@ -145,13 +154,6 @@ class BaseSlapRecipe: ...@@ -145,13 +154,6 @@ class BaseSlapRecipe:
self._writeExecutable(wrapper_path, file_content) self._writeExecutable(wrapper_path, file_content)
return wrapper_path return wrapper_path
def createReportRunningWrapper(self, file_content):
"""Creates report runnig wrapper and returns its path"""
report_wrapper_path = os.path.join(self.wrapper_report_directory,
'slapreport')
self._writeExecutable(report_wrapper_path, file_content)
return report_wrapper_path
def substituteTemplate(self, template_location, mapping_dict): def substituteTemplate(self, template_location, mapping_dict):
"""Returns template content after substitution""" """Returns template content after substitution"""
return open(template_location, 'r').read() % mapping_dict return open(template_location, 'r').read() % mapping_dict
......
# -*- coding: utf-8 -*-
# vim: set et sts=2:
############################################################################## ##############################################################################
# #
# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved. # Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
...@@ -24,6 +26,7 @@ ...@@ -24,6 +26,7 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# #
############################################################################## ##############################################################################
import io
import logging import logging
import os import os
import sys import sys
...@@ -35,6 +38,8 @@ import urlparse ...@@ -35,6 +38,8 @@ import urlparse
import pkg_resources import pkg_resources
import zc.buildout import zc.buildout
from slapos.recipe.librecipe import shlex
class GenericBaseRecipe(object): class GenericBaseRecipe(object):
"""Boilerplate class for all Buildout recipes providing helpful methods like """Boilerplate class for all Buildout recipes providing helpful methods like
creating configuration file, creating wrappers, generating passwords, etc. creating configuration file, creating wrappers, generating passwords, etc.
...@@ -88,6 +93,21 @@ class GenericBaseRecipe(object): ...@@ -88,6 +93,21 @@ class GenericBaseRecipe(object):
def createExecutable(self, name, content, mode=0700): def createExecutable(self, name, content, mode=0700):
return self.createFile(name, content, mode) return self.createFile(name, content, mode)
def addLineToFile(self, filepath, line, encoding='utf8'):
"""Append a single line to a text file, if the line does not exist yet.
line must be unicode."""
try:
lines = io.open(filepath, 'r', encoding=encoding).readlines()
except IOError:
lines = []
if not line in lines:
lines.append(line)
with io.open(filepath, 'w+', encoding=encoding) as f:
f.write(u'\n'.join(lines))
def createPythonScript(self, name, absolute_function, arguments=''): def createPythonScript(self, name, absolute_function, arguments=''):
"""Create a python script using zc.buildout.easy_install.scripts """Create a python script using zc.buildout.easy_install.scripts
...@@ -107,6 +127,34 @@ class GenericBaseRecipe(object): ...@@ -107,6 +127,34 @@ class GenericBaseRecipe(object):
path, arguments=arguments)[0] path, arguments=arguments)[0]
return script return script
def createWrapper(self, name, command, parameters, comments=[], parameters_extra=False):
"""
Creates a very simple (one command) shell script for process replacement.
Takes care of quoting.
"""
lines = [ '#!/bin/sh' ]
for comment in comments:
lines.append('# %s' % comment)
lines.append('exec %s' % shlex.quote(command))
for param in parameters:
if len(lines[-1]) < 40:
lines[-1] += ' ' + shlex.quote(param)
else:
lines[-1] += ' \\'
lines.append('\t' + shlex.quote(param))
if parameters_extra:
# pass-through further parameters
lines[-1] += ' \\'
lines.append('\t$@')
content = '\n'.join(lines) + '\n'
return self.createFile(name, content, 0700)
def createDirectory(self, parent, name, mode=0700): def createDirectory(self, parent, name, mode=0700):
path = os.path.join(parent, name) path = os.path.join(parent, name)
if not os.path.exists(path): if not os.path.exists(path):
......
# -*- coding: utf-8 -*-
"""
backported part of shlex.py from Python 3.3
"""
import re
_find_unsafe = re.compile(r'[^\w@%+=:,./-]', 256).search
def quote(s):
"""Return a shell-escaped version of the string *s*."""
if not s:
return "''"
if _find_unsafe(s) is None:
return s
# use single quotes, and put single quotes into double quotes
# the string $'b is then quoted as '$'"'"'b'
return "'" + s.replace("'", "'\"'\"'") + "'"
...@@ -28,85 +28,76 @@ import subprocess ...@@ -28,85 +28,76 @@ import subprocess
from slapos.recipe.librecipe import GenericBaseRecipe from slapos.recipe.librecipe import GenericBaseRecipe
def dump(args):
mydumper_cmd = [args['mydumper']]
mydumper_cmd.extend(['-B', args['database']])
if args['socket'] is not None: def _mydumper_base_cmd(mydumper, database, user, password,
mydumper_cmd.extend(['-S', args['socket']]) socket=None, host=None, port=None, **kw):
else: cmd = [mydumper]
mydumper_cmd.extend(['-h', args['host']]) cmd.extend(['-B', database])
mydumper_cmd.etxned(['-P', args['port']])
mydumper_cmd.extend(['-u', args['user']])
if args['password'] is not None:
mydumper_cmd.extend(['-p', args['password']])
if args['compression']:
mydumper_cmd.append('--compress')
if args['rows'] is not None: if socket:
mydumper_cmd.extend(['-r', args['rows']]) cmd.extend(['-S', socket])
else:
mydumper_cmd.extend(['-o', args['directory']]) cmd.extend(['-h', host])
cmd.extend(['-P', port])
subprocess.check_call(mydumper_cmd) cmd.extend(['-u', user])
if password:
cmd.extend(['-p', password])
return cmd
def do_import(args): def do_export(args):
mydumper_cmd = [args['mydumper']] cmd = _mydumper_base_cmd(**args)
mydumper_cmd.extend(['-B', args['database']])
if args['socket'] is not None: if args['compression']:
mydumper_cmd.extend(['-S', args['socket']]) cmd.append('--compress')
else:
mydumper_cmd.extend(['-h', args['host']])
mydumper_cmd.etxned(['-P', args['port']])
mydumper_cmd.extend(['-u', args['user']]) if args['rows'] is not None:
if args['password'] is not None: cmd.extend(['-r', args['rows']])
mydumper_cmd.extend(['-p', args['password']])
mydumper_cmd.append('--overwrite-tables') cmd.extend(['-o', args['directory']])
mydumper_cmd.extend(['-d', args['directory']]) subprocess.check_call(cmd)
subprocess.check_call(mydumper_cmd)
def do_import(args):
cmd = _mydumper_base_cmd(**args)
cmd.append('--overwrite-tables')
cmd.extend(['-d', args['directory']])
subprocess.check_call(cmd)
class Recipe(GenericBaseRecipe): class Recipe(GenericBaseRecipe):
def install(self): def install(self):
# Host or socket should be defined config = {
try: 'database': self.options['database'],
self.options['host'] 'directory': self.options['backup-directory'],
except: 'user': self.options['user'],
self.options['socket'] 'password': self.options.get('password'),
}
config = dict(database=self.options['database'],
socket=self.options.get('socket'), if self.options.get('host'):
host=self.options.get('host'), config['host'] = self.options['host']
port=self.options.get('port', 3306), config['port'] = self.options.get('port', 3306)
directory=self.options['backup-directory'], elif self.options.get('socket'):
user=self.options['user'], config['socket'] = self.options['socket']
password=self.options.get('password'), else:
) raise ValueError("host or socket must be defined")
name = __name__
if self.optionIsTrue('import', False): if self.optionIsTrue('import', False):
config.update(mydumper=self.options['myloader-binary']) function = do_import
name += '.do_import' config['mydumper'] = self.options['myloader-binary']
else: else:
config.update(mydumper=self.options['mydumper-binary'], function = do_export
compression=self.optionIsTrue('compression', default=False), config['mydumper'] = self.options['mydumper-binary']
rows=self.options.get('rows'), config['compression'] = self.optionIsTrue('compression', default=False)
) config['rows'] = self.options.get('rows')
name += '.dump'
wrapper = self.createPythonScript(self.options['wrapper'], wrapper = self.createPythonScript(name=self.options['wrapper'],
name, absolute_function = '%s.%s' % (__name__, function.func_name),
config) arguments=config)
return [wrapper] return [wrapper]
...@@ -31,6 +31,7 @@ import subprocess ...@@ -31,6 +31,7 @@ import subprocess
from slapos.recipe.librecipe import GenericBaseRecipe from slapos.recipe.librecipe import GenericBaseRecipe
from slapos.recipe.librecipe import filehash from slapos.recipe.librecipe import filehash
class Recipe(GenericBaseRecipe): class Recipe(GenericBaseRecipe):
def _options(self, options): def _options(self, options):
......
...@@ -19,7 +19,7 @@ long_query_time = 5 ...@@ -19,7 +19,7 @@ long_query_time = 5
max_allowed_packet = 128M max_allowed_packet = 128M
query_cache_size = 32M query_cache_size = 32M
plugin-load = ha_mroonga.so;ha_sphinx.so plugin-load = ha_mroonga.so
# The following are important to configure and depend a lot on to the size of # The following are important to configure and depend a lot on to the size of
# your database and the available resources. # your database and the available resources.
......
...@@ -31,59 +31,76 @@ from slapos.recipe.librecipe import GenericBaseRecipe ...@@ -31,59 +31,76 @@ from slapos.recipe.librecipe import GenericBaseRecipe
class Recipe(GenericBaseRecipe): class Recipe(GenericBaseRecipe):
def install(self): def install(self):
commandline = [self.options['server-binary']] options = self.options
commandline.extend(['--callbacks', self.options['callbacks']]) script = self.createWrapper(name=options['wrapper'],
commandline.extend(['--feeds', self.options['feeds']]) command=options['server-binary'],
commandline.extend(['--equeue-socket', self.options['equeue-socket']]) parameters=[
commandline.append(self.options['host']) '--callbacks', options['callbacks'],
commandline.append(self.options['port']) '--feeds', options['feeds'],
'--equeue-socket', options['equeue-socket'],
options['host'], options['port']
],
comments=[
'',
'Upon receiving a notification, execute the callback(s).',
''])
return [script]
return [self.createPythonScript(self.options['wrapper'],
'slapos.recipe.librecipe.execute.execute',
commandline)]
class Callback(GenericBaseRecipe): class Callback(GenericBaseRecipe):
def createCallback(self, notification_id, callback): def createCallback(self, notification_id, callback):
callback_id = sha512(notification_id).hexdigest() callback_id = sha512(notification_id).hexdigest()
callback = self.createFile(os.path.join(self.options['callbacks'],
callback_id), filepath = os.path.join(self.options['callbacks'], callback_id)
callback) self.addLineToFile(filepath, callback)
return callback return filepath
def install(self): def install(self):
# XXX this path is returned multiple times, one for each callback that has been added.
return [self.createCallback(self.options['on-notification-id'], return [self.createCallback(self.options['on-notification-id'],
self.options['callback'])] self.options['callback'])]
class Notify(GenericBaseRecipe): class Notify(GenericBaseRecipe):
def createNotifier(self, notifier_binary, executable, wrapper, **kwargs): def createNotifier(self, notifier_binary, wrapper, executable,
if not os.path.exists(kwargs['log']): log, title, notification_url, feed_url):
if not os.path.exists(log):
# Just a touch # Just a touch
open(kwargs['log'], 'w').close() open(log, 'w').close()
commandline = [notifier_binary, parameters = [
'-l', kwargs['log'], '-l', log,
'--title', kwargs['title'], '--title', title,
'--feed', kwargs['feed_url'], '--feed', feed_url,
'--notification-url', kwargs['notification_url'], '--notification-url',
executable] ]
return self.createPythonScript(wrapper, parameters.extend(notification_url.split(' '))
'slapos.recipe.librecipe.execute.execute', parameters.extend(['--executable', executable])
[str(i) for i in commandline])
return self.createWrapper(name=wrapper,
command=notifier_binary,
parameters=parameters,
comments=[
'',
'Call an executable and send notification(s).',
''])
def install(self): def install(self):
feedurl = self.unparseUrl(scheme='http', host=self.options['host'], feed_url = self.unparseUrl(scheme='http', host=self.options['host'],
port=self.options['port'], port=self.options['port'],
path='/get/%s' % self.options['name']) path='/get/%s' % self.options['name'])
script = self.createNotifier( log = os.path.join(self.options['feeds'], self.options['name'])
self.options['notifier-binary'],
wrapper=self.options['wrapper'], options = self.options
executable=self.options['executable'], script = self.createNotifier(notifier_binary=options['notifier-binary'],
log=os.path.join(self.options['feeds'], self.options['name']), wrapper=options['wrapper'],
title=self.options['title'], executable=options['executable'],
notification_url=self.options['notify'], log=log,
feed_url=feedurl, title=options['title'],
) notification_url=options['notify'],
feed_url=feed_url)
return [script] return [script]
...@@ -24,14 +24,15 @@ ...@@ -24,14 +24,15 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# #
############################################################################## ##############################################################################
from json import loads as unjson
from hashlib import sha512 from hashlib import sha512
from urlparse import urlparse import inspect
import json
import os import os
import signal
import subprocess import subprocess
import sys import sys
import signal import urlparse
import inspect
from slapos.recipe.librecipe import GenericSlapRecipe from slapos.recipe.librecipe import GenericSlapRecipe
from slapos.recipe.dropbear import KnownHostsFile from slapos.recipe.dropbear import KnownHostsFile
...@@ -43,8 +44,7 @@ from slapos import slap as slapmodule ...@@ -43,8 +44,7 @@ from slapos import slap as slapmodule
def promise(args): def promise(args):
def failed_ssh(partition, ssh): def failed_ssh(partition, ssh):
# Bad python 2 syntax, looking forward python 3 to have print(file=) sys.stderr.write("SSH Connection failed\n")
print >> sys.stderr, "SSH Connection failed"
try: try:
ssh.terminate() ssh.terminate()
except: except:
...@@ -75,16 +75,20 @@ def promise(args): ...@@ -75,16 +75,20 @@ def promise(args):
slap = slapmodule.slap() slap = slapmodule.slap()
slap.initializeConnection(args['server_url'], slap.initializeConnection(args['server_url'],
key_file=args.get('key_file'), cert_file=args.get('cert_file')) key_file=args.get('key_file'),
cert_file=args.get('cert_file'))
partition = slap.registerComputerPartition(args['computer_id'], partition = slap.registerComputerPartition(args['computer_id'],
args['partition_id']) args['partition_id'])
ssh = subprocess.Popen([args['ssh_client'], '%(user)s@%(host)s/%(port)s' % args],
stdin=subprocess.PIPE,
stdout=open(os.devnull, 'w'),
stderr=open(os.devnull, 'w'))
# Rdiff Backup protocol quit command # Rdiff Backup protocol quit command
quitcommand = 'q' + chr(255) + chr(0) * 7 quitcommand = 'q' + chr(255) + chr(0) * 7
ssh_cmdline = [args['ssh_client'], '%(user)s@%(host)s/%(port)s' % args]
ssh = subprocess.Popen(ssh_cmdline, stdin=subprocess.PIPE,
stdout=open(os.devnull), stderr=open(os.devnull))
ssh.stdin.write(quitcommand) ssh.stdin.write(quitcommand)
ssh.stdin.flush() ssh.stdin.flush()
ssh.stdin.close() ssh.stdin.close()
...@@ -113,7 +117,7 @@ class Recipe(GenericSlapRecipe, Notify, Callback): ...@@ -113,7 +117,7 @@ class Recipe(GenericSlapRecipe, Notify, Callback):
promise_path = os.path.join(self.options['promises-directory'], promise_path = os.path.join(self.options['promises-directory'],
url_hash) url_hash)
parsed_url = urlparse(url) parsed_url = urlparse.urlparse(url)
promise_dict = self.promise_base_dict.copy() promise_dict = self.promise_base_dict.copy()
promise_dict.update(user=parsed_url.username, promise_dict.update(user=parsed_url.username,
host=parsed_url.hostname, host=parsed_url.hostname,
...@@ -127,15 +131,17 @@ class Recipe(GenericSlapRecipe, Notify, Callback): ...@@ -127,15 +131,17 @@ class Recipe(GenericSlapRecipe, Notify, Callback):
host = parsed_url.hostname host = parsed_url.hostname
known_hosts_file[host] = entry['server-key'] known_hosts_file[host] = entry['server-key']
remote_schema = '%(ssh)s -p %%s %(user)s@%(host)s' % \ # XXX use -y because the host might not yet be in the
# trusted hosts file until the next time slapgrid is run.
remote_schema = '%(ssh)s -y -p %%s %(user)s@%(host)s' % \
{ {
'ssh': self.options['sshclient-binary'], 'ssh': self.options['sshclient-binary'],
'user': parsed_url.username, 'user': parsed_url.username,
'host': parsed_url.hostname, 'host': parsed_url.hostname,
} }
command = [self.options['rdiffbackup-binary']] parameters = ['--remote-schema', remote_schema]
command.extend(['--remote-schema', remote_schema])
remote_directory = '%(port)s::%(path)s' % {'port': parsed_url.port, remote_directory = '%(port)s::%(path)s' % {'port': parsed_url.port,
'path': parsed_url.path} 'path': parsed_url.path}
...@@ -144,38 +150,39 @@ class Recipe(GenericSlapRecipe, Notify, Callback): ...@@ -144,38 +150,39 @@ class Recipe(GenericSlapRecipe, Notify, Callback):
name_hash) name_hash)
if entry['type'] == 'push': if entry['type'] == 'push':
command.extend(['--restore-as-of', 'now']) parameters.extend(['--restore-as-of', 'now'])
command.append('--force') parameters.append('--force')
command.extend([local_directory, remote_directory]) parameters.extend([local_directory, remote_directory])
comments = ['','Push data to a PBS *-import instance.','']
else: else:
command.extend([remote_directory, local_directory]) parameters.extend([remote_directory, local_directory])
comments = ['','Pull data from a PBS *-export instance.','']
wrapper_basepath = os.path.join(self.options['wrappers-directory'], wrapper_basepath = os.path.join(self.options['wrappers-directory'],
url_hash) url_hash)
wrapper_path = wrapper_basepath
if 'notify' in entry: if 'notify' in entry:
wrapper_path = '%s_raw' % wrapper_basepath wrapper_path = wrapper_basepath + '_raw'
else:
wrapper_path = wrapper_basepath
wrapper = self.createPythonScript( wrapper = self.createWrapper(name=wrapper_path,
wrapper_path, command=self.options['rdiffbackup-binary'],
'slapos.recipe.librecipe.execute.execute', parameters=parameters,
[str(i) for i in command] comments = comments)
)
path_list.append(wrapper) path_list.append(wrapper)
if 'notify' in entry: if 'notify' in entry:
feed_url = '%s/get/%s' % (self.options['notifier-url'], feed_url = '%s/get/%s' % (self.options['notifier-url'],
entry['notification-id']) entry['notification-id'])
wrapper = self.createNotifier( wrapper = self.createNotifier(notifier_binary=self.options['notifier-binary'],
self.options['notifier-binary'], wrapper=wrapper_basepath,
wrapper=wrapper_basepath, executable=wrapper_path,
executable=wrapper_path, log=os.path.join(self.options['feeds'], entry['notification-id']),
log=os.path.join(self.options['feeds'], entry['notification-id']), title=entry.get('title', 'Untitled'),
title=entry.get('title', 'Untitled'), notification_url=entry['notify'],
notification_url=entry['notify'], feed_url=feed_url,
feed_url=feed_url, )
)
path_list.append(wrapper) path_list.append(wrapper)
#self.setConnectionDict(dict(feed_url=feed_url), entry['slave_reference']) #self.setConnectionDict(dict(feed_url=feed_url), entry['slave_reference'])
...@@ -190,39 +197,39 @@ class Recipe(GenericSlapRecipe, Notify, Callback): ...@@ -190,39 +197,39 @@ class Recipe(GenericSlapRecipe, Notify, Callback):
return path_list return path_list
def _install(self): def _install(self):
path_list = [] path_list = []
if self.optionIsTrue('client', True): if self.optionIsTrue('client', True):
self.logger.info("Client mode") self.logger.info("Client mode")
slap_connection = self.buildout['slap-connection'] slap_connection = self.buildout['slap-connection']
self.promise_base_dict = dict( self.promise_base_dict = {
server_url=slap_connection['server-url'], 'server_url': slap_connection['server-url'],
computer_id=slap_connection['computer-id'], 'computer_id': slap_connection['computer-id'],
cert_file=slap_connection.get('cert-file'), 'cert_file': slap_connection.get('cert-file'),
key_file=slap_connection.get('key-file'), 'key_file': slap_connection.get('key-file'),
partition_id=slap_connection['partition-id'], 'partition_id': slap_connection['partition-id'],
ssh_client=self.options['sshclient-binary'], 'ssh_client': self.options['sshclient-binary'],
) }
slaves = unjson(self.options['slave-instance-list']) slaves = json.loads(self.options['slave-instance-list'])
known_hosts = KnownHostsFile(self.options['known-hosts']) known_hosts = KnownHostsFile(self.options['known-hosts'])
with known_hosts: with known_hosts:
# XXX this API could be cleaner
for slave in slaves: for slave in slaves:
path_list.extend(self.add_slave(slave, known_hosts)) path_list.extend(self.add_slave(slave, known_hosts))
else: else:
command = [self.options['rdiffbackup-binary']]
self.logger.info("Server mode") self.logger.info("Server mode")
command.extend(['--restrict', self.options['path']])
command.append('--server')
wrapper = self.createPythonScript( wrapper = self.createWrapper(name=self.options['wrapper'],
self.options['wrapper'], command=self.options['rdiffbackup-binary'],
'slapos.recipe.librecipe.execute.execute', parameters=[
command) '--restrict', self.options['path'],
'--server'
])
path_list.append(wrapper) path_list.append(wrapper)
return path_list return path_list
##############################################################################
#
# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import md5
import os
import subprocess
import textwrap
from zc.buildout import UserError
from slapos.recipe.librecipe import GenericBaseRecipe
class Recipe(GenericBaseRecipe):
"""\
This recipe creates:
- a Postgres cluster
- configuration to allow connections from IPV6 only (or unix socket)
- a superuser with provided name and generated password
- a database with provided name
- a foreground start script in the services directory
then adds the connection URL to the options.
The URL can be used as-is (ie. in sqlalchemy) or by the _urlparse.py recipe.
"""
def fetch_ipv6_host(self, options):
"""
Returns a string represtation of ipv6_host.
May receive a regular string, a set or a string serialized by buildout.
"""
ipv6_host = options['ipv6_host']
if isinstance(ipv6_host, set):
return ipv6_host.pop()
else:
return ipv6_host
def _options(self, options):
options['password'] = self.generatePassword()
options['url'] = 'postgresql://%(user)s:%(password)s@[%(ipv4_host)s]:%(port)s/%(dbname)s' % options
def install(self):
pgdata = self.options['pgdata-directory']
if not os.path.exists(pgdata):
self.createCluster()
self.createConfig()
self.createDatabase()
self.createSuperuser()
self.createRunScript()
return [
# XXX should we really return something here?
# os.path.join(pgdata, 'postgresql.conf')
]
def check_exists(self, path):
if not os.path.isfile(path):
raise IOError('File not found: %s' % path)
def createCluster(self):
initdb_binary = os.path.join(self.options['bin'], 'initdb')
self.check_exists(initdb_binary)
pgdata = self.options['pgdata-directory']
try:
subprocess.check_call([initdb_binary,
'-D', pgdata,
'-A', 'ident',
'-E', 'UTF8',
])
except subprocess.CalledProcessError:
raise UserError('Could not create cluster directory in %s' % pgdata)
def createConfig(self):
pgdata = self.options['pgdata-directory']
with open(os.path.join(pgdata, 'postgresql.conf'), 'wb') as cfg:
cfg.write(textwrap.dedent("""\
listen_addresses = '%s,%s'
logging_collector = on
log_rotation_size = 50MB
max_connections = 100
datestyle = 'iso, mdy'
lc_messages = 'en_US.UTF-8'
lc_monetary = 'en_US.UTF-8'
lc_numeric = 'en_US.UTF-8'
lc_time = 'en_US.UTF-8'
default_text_search_config = 'pg_catalog.english'
unix_socket_directory = '%s'
unix_socket_permissions = 0700
""" % (
self.options['ipv4_host'],
self.fetch_ipv6_host(self.options),
pgdata,
)))
with open(os.path.join(pgdata, 'pg_hba.conf'), 'wb') as cfg:
# see http://www.postgresql.org/docs/9.1/static/auth-pg-hba-conf.html
cfg.write(textwrap.dedent("""\
# TYPE DATABASE USER ADDRESS METHOD
# "local" is for Unix domain socket connections only (check unix_socket_permissions!)
local all all ident
host all all 127.0.0.1/32 md5
host all all %s/32 md5
host all all ::1/128 md5
host all all %s/128 md5
""" % (self.options['ipv4_host'], self.fetch_ipv6_host(self.options))))
def createDatabase(self):
self.runPostgresCommand(cmd='CREATE DATABASE "%s"' % self.options['dbname'])
def createSuperuser(self):
"""
Creates a Postgres superuser - other than "slapuser#" for use by the application.
"""
# http://postgresql.1045698.n5.nabble.com/Algorithm-for-generating-md5-encrypted-password-not-found-in-documentation-td4919082.html
user = self.options['user']
password = self.options['password']
# encrypt the password to avoid storing in the logs
enc_password = 'md5' + md5.md5(password+user).hexdigest()
self.runPostgresCommand(cmd="""CREATE USER "%s" ENCRYPTED PASSWORD '%s' SUPERUSER""" % (user, enc_password))
def runPostgresCommand(self, cmd):
"""
Executes a command in single-user mode, with no daemon running.
Multiple commands can be executed by providing newlines,
preceeded by backslash, between them.
See http://www.postgresql.org/docs/9.1/static/app-postgres.html
"""
pgdata = self.options['pgdata-directory']
postgres_binary = os.path.join(self.options['bin'], 'postgres')
try:
p = subprocess.Popen([postgres_binary,
'--single',
'-D', pgdata,
'postgres',
], stdin=subprocess.PIPE)
p.communicate(cmd+'\n')
except subprocess.CalledProcessError:
raise UserError('Could not create database %s' % pgdata)
def createRunScript(self):
"""
Creates a script that runs postgres in the foreground.
'exec' is used to allow easy control by supervisor.
"""
content = textwrap.dedent("""\
#!/bin/sh
exec %(bin)s/postgres \\
-D %(pgdata-directory)s
""" % self.options)
name = os.path.join(self.options['services'], 'postgres-start')
self.createExecutable(name, content=content)
class ExportRecipe(GenericBaseRecipe):
def install(self):
pgdata = self.options['pgdata-directory']
wrapper = self.options['wrapper']
self.createBackupScript(wrapper)
return [wrapper]
def createBackupScript(self, wrapper):
"""
Create a script to backup the database in 'custom' format.
"""
content = textwrap.dedent("""\
#!/bin/sh
umask 077
%(bin)s/pg_dump \\
--host=%(pgdata-directory)s \\
--format=custom \\
--file=%(backup-directory)s/database.dump \\
%(dbname)s
""" % self.options)
self.createExecutable(wrapper, content=content)
class ImportRecipe(GenericBaseRecipe):
def install(self):
pgdata = self.options['pgdata-directory']
wrapper = self.options['wrapper']
self.createRestoreScript(wrapper)
return [wrapper]
def createRestoreScript(self, wrapper):
"""
Create a script to restore the database from 'custom' format.
"""
content = textwrap.dedent("""\
#!/bin/sh
%(bin)s/pg_restore \\
--host=%(pgdata-directory)s \\
--dbname=%(dbname)s \\
--clean \\
--no-owner \\
--no-acl \\
%(backup-directory)s/database.dump
""" % self.options)
self.createExecutable(wrapper, content=content)
...@@ -25,8 +25,11 @@ ...@@ -25,8 +25,11 @@
# #
############################################################################## ##############################################################################
import logging import logging
from slapos import slap as slapmodule from slapos import slap as slapmodule
import slapos.recipe.librecipe.generic as librecipe
import traceback
DEFAULT_SOFTWARE_TYPE = 'RootSoftwareInstance'
class Recipe(object): class Recipe(object):
""" """
...@@ -97,17 +100,15 @@ class Recipe(object): ...@@ -97,17 +100,15 @@ class Recipe(object):
request = slap.registerComputerPartition( request = slap.registerComputerPartition(
options['computer-id'], options['partition-id']).request options['computer-id'], options['partition-id']).request
isSlave = options.get('slave', '').lower() in ['y', 'yes', 'true', '1']
return_parameters = [] return_parameters = []
if 'return' in options: if 'return' in options:
return_parameters = [str(parameter).strip() return_parameters = [str(parameter).strip()
for parameter in options['return'].split()] for parameter in options['return'].split()]
else: else:
self.logger.debug("No parameter to return to main instance." self.logger.debug("No parameter to return to main instance."
"Be careful about that...") "Be careful about that...")
software_type = options.get('software-type', 'RootInstanceSoftware') software_type = options.get('software-type', DEFAULT_SOFTWARE_TYPE)
filter_kw = {} filter_kw = {}
if 'sla' in options: if 'sla' in options:
...@@ -120,26 +121,46 @@ class Recipe(object): ...@@ -120,26 +121,46 @@ class Recipe(object):
partition_parameter_kw[config_parameter] = \ partition_parameter_kw[config_parameter] = \
options['config-%s' % config_parameter] options['config-%s' % config_parameter]
self.instance = instance = request(software_url, software_type, isSlave = options.get('slave', '').lower() in \
name, partition_parameter_kw=partition_parameter_kw, librecipe.GenericBaseRecipe.TRUE_VALUES
filter_kw=filter_kw, shared=isSlave)
self._raise_request_exception = None
self._raise_request_exception_formatted = None
self.instance = None
try:
self.instance = request(software_url, software_type,
name, partition_parameter_kw=partition_parameter_kw,
filter_kw=filter_kw, shared=isSlave)
# XXX what is the right way to get a global id?
options['instance_guid'] = self.instance.getId()
except (slapmodule.NotFoundError, slapmodule.ServerError, slapmodule.ResourceNotReady) as exc:
self._raise_request_exception = exc
self._raise_request_exception_formatted = traceback.format_exc()
for param in return_parameters: for param in return_parameters:
options['connection-%s' % param] = ''
if not self.instance:
continue
try: try:
options['connection-%s' % param] = str( options['connection-%s' % param] = str(
instance.getConnectionParameter(param)) self.instance.getConnectionParameter(param))
except slapmodule.NotFoundError: except (slapmodule.NotFoundError, slapmodule.ServerError, slapmodule.ResourceNotReady):
options['connection-%s' % param] = ''
if self.failed is None: if self.failed is None:
self.failed = param self.failed = param
def install(self): def install(self):
if self._raise_request_exception:
raise self._raise_request_exception
if self.failed is not None: if self.failed is not None:
# Check instance status to know if instance has been deployed # Check instance status to know if instance has been deployed
try: try:
status = self.instance.getState() if self.instance._computer_id is not None:
except slapmodule.NotFoundError: status = self.instance.getState()
status = 'not ready yet, please try again' else:
status = 'not ready yet'
except (slapmodule.NotFoundError, slapmodule.ServerError, slapmodule.ResourceNotReady):
status = 'not ready yet'
except AttributeError: except AttributeError:
status = 'unknown' status = 'unknown'
error_message = 'Connection parameter %s not found. '\ error_message = 'Connection parameter %s not found. '\
...@@ -150,3 +171,31 @@ class Recipe(object): ...@@ -150,3 +171,31 @@ class Recipe(object):
return [] return []
update = install update = install
class RequestOptional(Recipe):
"""
Request a SlapOS instance. Won't fail if request failed or is not ready.
Same as slapos.cookbook:request, but won't raise in case of problem.
"""
def install(self):
if self._raise_request_exception_formatted:
self.logger.warning('Optional request failed:')
self.logger.warning(self._raise_request_exception_formatted)
elif self.failed is not None:
# Check instance status to know if instance has been deployed
try:
if self.instance._computer_id is not None:
status = self.instance.getState()
else:
status = 'not ready yet'
except (slapmodule.NotFoundError, slapmodule.ServerError):
status = 'not ready yet'
except AttributeError:
status = 'unknown'
error_message = 'Connection parameter %s not found. '\
'Requested instance is currently %s. If this error persists, '\
'check status of this instance.' % (self.failed, status)
self.logger.warning(error_message)
return []
update = install
##############################################################################
#
# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import logging
from slapos import slap as slapmodule
class Recipe(object):
""" Request a SlapOS instance. Won't fail if instance is not ready.
"""
def __init__(self, buildout, name, options):
self.logger = logging.getLogger(name)
slap = slapmodule.slap()
self.software_release_url = options['software-url']
self.name = options['name']
slap.initializeConnection(options['server-url'],
options.get('key-file'),
options.get('cert-file'),
)
computer_partition = slap.registerComputerPartition(
options['computer-id'], options['partition-id'])
self.request = computer_partition.request
self.isSlave = False
if 'slave' in options:
self.isSlave = options['slave'].lower() in ['y', 'yes', 'true', '1']
self.return_parameters = []
if 'return' in options:
self.return_parameters = [str(parameter).strip()
for parameter in options['return'].split()]
else:
self.logger.debug("No parameter to return to main instance."
"Be careful about that...")
software_type = 'RootSoftwareInstance'
if 'software-type' in options:
software_type = options['software-type']
filter_kw = {}
if 'sla' in options:
for sla_parameter in options['sla'].split():
filter_kw[sla_parameter] = options['sla-%s' % sla_parameter]
partition_parameter_kw = {}
if 'config' in options:
for config_parameter in options['config'].split():
partition_parameter_kw[config_parameter] = \
options['config-%s' % config_parameter]
self.instance = self.request(options['software-url'], software_type,
self.name, partition_parameter_kw=partition_parameter_kw,
filter_kw=filter_kw, shared=self.isSlave)
self.failed = None
for param in self.return_parameters:
try:
options['connection-%s' % param] = str(
self.instance.getConnectionParameter(param))
except slapmodule.NotFoundError:
options['connection-%s' % param] = ''
if self.failed is None:
self.failed = param
def install(self):
if self.failed is not None:
# Check instance status to know if instance has been deployed
try:
status = self.instance.getState()
except slapmodule.NotFoundError:
status = 'not ready yet'
except AttributeError:
status = 'unknown'
error_message = 'Connection parameter %s not found. '\
'Requested instance is currently %s. If this error persists, '\
'check status of this instance.' % (self.failed, status)
self.logger.error(error_message)
return []
update = install
...@@ -24,31 +24,31 @@ ...@@ -24,31 +24,31 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# #
############################################################################## ##############################################################################
from slapos.recipe.librecipe import GenericBaseRecipe
class Recipe(GenericBaseRecipe): from slapos.recipe.librecipe import GenericBaseRecipe
"""
Slapmonitor instance configuration.
"""
def __init__(self, buildout, name, options): class MonitorRecipe(GenericBaseRecipe):
return GenericBaseRecipe.__init__(self, buildout, name, options)
def install(self): def install(self):
config = dict( options = self.options
pid_file_path=self.options['pid-file'], script = self.createWrapper(name=options['path'],
database_path=self.options['database-path'], command=options['slapmonitor-path'],
slapmonitor_path = self.options['slapmonitor-path'], parameters=[
shell_path=self.options['shell-path'], options['pid-file'],
) options['database-path'],
])
return [script]
# Runners
runner_path = self.createExecutable(
self.options['path'],
self.substituteTemplate(self.getTemplateFilename('slapmonitor_run.in'),
config))
return [runner_path] class MonitorXMLRecipe(GenericBaseRecipe):
def install(self):
options = self.options
script = self.createWrapper(name=options['path'],
command=options['slapmonitor-xml-path'],
parameters=[
options['database-path'],
],
parameters_extra=True)
return [script]
def update(self):
pass
#!%(shell_path)s
# BEWARE: This file is operated by slapgrid
# BEWARE: It will be overwritten automatically
#
exec %(slapmonitor_path)s %(pid_file_path)s %(database_path)s
...@@ -24,36 +24,23 @@ ...@@ -24,36 +24,23 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# #
############################################################################## ##############################################################################
from slapos.recipe.librecipe import GenericBaseRecipe from slapos.recipe.librecipe import GenericBaseRecipe
class Recipe(GenericBaseRecipe): class Recipe(GenericBaseRecipe):
"""
Slapmonitor instance configuration.
"""
def __init__(self, buildout, name, options):
return GenericBaseRecipe.__init__(self, buildout, name, options)
def install(self): def install(self):
config = dict( options = self.options
pid_file_path=self.options['pid-file'], script = self.createWrapper(name=options['path'],
consumption_log_path=self.options['consumption-log-path'], command=options['slapreport-path'],
database_path=self.options['database-path'], parameters=[
slapreport_path = self.options['slapreport-path'], options['pid-file'],
logbox_ip = self.options['logbox-ip'], options['consumption-log-path'],
logbox_port = self.options['logbox-port'], options['database-path'],
logbox_user = self.options['logbox-user'], options['logbox-ip'],
logbox_passwd = self.options['logbox-passwd'], options['logbox-port'],
shell_path=self.options['shell-path'], options['logbox-user'],
) options['logbox-passwd'],
])
# Runners return [script]
runner_path = self.createExecutable(
self.options['path'],
self.substituteTemplate(self.getTemplateFilename('slapreport_run.in'),
config))
return [runner_path]
def update(self):
pass
#!%(shell_path)s
# BEWARE: This file is operated by slapgrid
# BEWARE: It will be overwritten automatically
#
exec %(slapreport_path)s %(pid_file_path)s %(consumption_log_path)s %(database_path)s %(logbox_ip)s %(logbox_port)s %(logbox_user)s %(logbox_passwd)s
...@@ -39,6 +39,7 @@ class Recipe(GenericBaseRecipe): ...@@ -39,6 +39,7 @@ class Recipe(GenericBaseRecipe):
self.instance_directory = options['instance-directory'].strip() self.instance_directory = options['instance-directory'].strip()
self.partition_amount = options['partition-amount'].strip() self.partition_amount = options['partition-amount'].strip()
self.cloud9_url = options['cloud9-url'].strip() self.cloud9_url = options['cloud9-url'].strip()
self.log_file = os.path.join(options['log_dir'].strip(), 'slaprunner.log')
# Set slaprunner access URL # Set slaprunner access URL
options['access-url'] = 'http://[%s]:%s' % (self.ipv6, self.runner_port) options['access-url'] = 'http://[%s]:%s' % (self.ipv6, self.runner_port)
...@@ -59,6 +60,8 @@ class Recipe(GenericBaseRecipe): ...@@ -59,6 +60,8 @@ class Recipe(GenericBaseRecipe):
'supervisord.conf'), 'supervisord.conf'),
runner_workdir=self.workdir, runner_workdir=self.workdir,
etc_dir=self.options['etc_dir'], etc_dir=self.options['etc_dir'],
run_dir=self.options['run_dir'],
log_dir=self.options['log_dir'],
runner_host=self.ipv6, runner_host=self.ipv6,
runner_port=self.runner_port, runner_port=self.runner_port,
ipv4_address=self.ipv4, ipv4_address=self.ipv4,
...@@ -79,10 +82,12 @@ class Recipe(GenericBaseRecipe): ...@@ -79,10 +82,12 @@ class Recipe(GenericBaseRecipe):
path_list.append(config_file) path_list.append(config_file)
environment = dict( environment = dict(
PATH=os.path.dirname(self.options['git-binary']) + ':' + os.environ['PATH'], PATH=os.path.dirname(
self.options['git-binary']) + ':' + os.environ['PATH'],
GIT_SSH=self.options['ssh_client'] GIT_SSH=self.options['ssh_client']
) )
launch_args = [self.options['slaprunner'].strip(), config_file] launch_args = [self.options['slaprunner'].strip(), config_file,
'--log_file', self.log_file]
if self.optionIsTrue('debug', default=False): if self.optionIsTrue('debug', default=False):
launch_args.append('--debug') launch_args.append('--debug')
...@@ -93,3 +98,71 @@ class Recipe(GenericBaseRecipe): ...@@ -93,3 +98,71 @@ class Recipe(GenericBaseRecipe):
path_list.append(wrapper) path_list.append(wrapper)
return path_list return path_list
class Test(GenericBaseRecipe):
def _options(self, options):
self.ipv4 = options['ipv4'].strip()
self.ipv6 = options['ipv6'].strip()
self.proxy_port = options['proxy_port'].strip()
self.runner_port = options['runner_port'].strip()
self.workdir = options['working-directory'].strip()
self.software_directory = options['software-directory'].strip()
self.instance_directory = options['instance-directory'].strip()
self.partition_amount = options['partition-amount'].strip()
self.cloud9_url = options['cloud9-url'].strip()
# Set slaprunner access URL
options['access-url'] = 'http://[%s]:%s' % (self.ipv6, self.runner_port)
def install(self):
path_list = []
configuration = dict(
software_root=self.software_directory,
instance_root=self.instance_directory,
master_url='http://%s:%s/' % (self.ipv4, self.proxy_port),
computer_id='slaprunner',
partition_amount=self.partition_amount,
slapgrid_sr=self.options['slapgrid_sr'],
slapgrid_cp=self.options['slapgrid_cp'],
slapproxy=self.options['slapproxy'],
supervisor=self.options['supervisor'],
supervisord_config=os.path.join(self.instance_directory, 'etc',
'supervisord.conf'),
runner_workdir=self.workdir,
etc_dir=self.options['etc_dir'],
run_dir=self.options['etc_dir'],
log_dir=self.workdir,
runner_host=self.ipv6,
runner_port=self.runner_port,
ipv4_address=self.ipv4,
ipv6_address=self.ipv6,
proxy_host=self.ipv4,
proxy_port=self.proxy_port,
proxy_database=os.path.join(self.workdir, 'proxy.db'),
git=self.options['git-binary'],
ssh_client=self.options['ssh_client'],
public_key=self.options['public_key'],
private_key=self.options['private_key'],
cloud9_url=self.cloud9_url
)
config_file = self.createFile(self.options['slapos.cfg'],
self.substituteTemplate(self.getTemplateFilename('slapos.cfg.in'),
configuration))
path_list.append(config_file)
environment = dict(
PATH=os.path.dirname(
self.options['git-binary']) + ':' + os.environ['PATH'],
GIT_SSH=self.options['ssh_client'],
CONFIG_FILE_PATH=config_file
)
launch_args = [self.options['slaprunnertest'].strip()]
wrapper = self.createPythonScript(self.options['wrapper'],
'slapos.recipe.librecipe.execute.executee',
(launch_args, environment)
)
path_list.append(wrapper)
return path_list
import os
def execute(args):
"""Portable execution with process replacement"""
if args.get("path", None):
os.environ['PATH'] = args["path"]
os.execv(args["launch_args"][0], args["launch_args"])
...@@ -20,6 +20,8 @@ runner_port = %(runner_port)s ...@@ -20,6 +20,8 @@ runner_port = %(runner_port)s
ipv4_address = %(ipv4_address)s ipv4_address = %(ipv4_address)s
ipv6_address = %(ipv6_address)s ipv6_address = %(ipv6_address)s
etc_dir = %(etc_dir)s etc_dir = %(etc_dir)s
run_dir = %(run_dir)s
log_dir = %(log_dir)s
[slapproxy] [slapproxy]
host = %(proxy_host)s host = %(proxy_host)s
......
...@@ -87,13 +87,17 @@ class Recipe: ...@@ -87,13 +87,17 @@ class Recipe:
computer_partition_id) computer_partition_id)
self.parameter_dict = self.computer_partition.getInstanceParameterDict() self.parameter_dict = self.computer_partition.getInstanceParameterDict()
software_type = self.parameter_dict['slap_software_type'] software_type = self.parameter_dict['slap_software_type']
self.logger.info('Deploying instance with software type %s' % \
software_type)
# Raise if request software_type does not exist ...
if software_type not in self.options: if software_type not in self.options:
if 'default' in self.options: # ... Except for backward compatibility. Then use "default".
if software_type in ['RootSoftwareInstance']:
software_type = 'default' software_type = 'default'
else: else:
raise zc.buildout.UserError("This software type isn't mapped. And " raise zc.buildout.UserError("This software type (%s) isn't mapped." % \
"there's no default software type.") software_type)
instance_file_path = self.options[software_type] instance_file_path = self.options[software_type]
......
...@@ -27,6 +27,9 @@ ...@@ -27,6 +27,9 @@
import os import os
import logging import logging
import zc.buildout
class Recipe: class Recipe:
def __init__(self, buildout, name, options): def __init__(self, buildout, name, options):
self.buildout = buildout self.buildout = buildout
......
##############################################################################
#
# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from slapos.recipe.librecipe import BaseSlapRecipe
import os
import pkg_resources
import zc.buildout
import zc.recipe.egg
import sys
CONFIG = dict(
proxy_port='5000',
computer_id='COMPUTER',
partition_reference='test0',
)
class Recipe(BaseSlapRecipe):
def __init__(self, buildout, name, options):
self.egg = zc.recipe.egg.Egg(buildout, options['recipe'], options)
BaseSlapRecipe.__init__(self, buildout, name, options)
def installSlapOs(self):
CONFIG['slapos_directory'] = self.createDataDirectory('slapos')
CONFIG['working_directory'] = self.createDataDirectory('testnode')
CONFIG['software_root'] = os.path.join(CONFIG['slapos_directory'],
'software')
CONFIG['instance_root'] = os.path.join(CONFIG['slapos_directory'],
'instance')
CONFIG['proxy_database'] = os.path.join(CONFIG['slapos_directory'],
'proxy.db')
CONFIG['proxy_host'] = self.getLocalIPv4Address()
CONFIG['master_url'] = 'http://%s:%s' % (CONFIG['proxy_host'],
CONFIG['proxy_port'])
self._createDirectory(CONFIG['software_root'])
self._createDirectory(CONFIG['instance_root'])
CONFIG['slapos_config'] = self.createConfigurationFile('slapos.cfg',
self.substituteTemplate(pkg_resources.resource_filename(__name__,
'template/slapos.cfg.in'), CONFIG))
self.path_list.append(CONFIG['slapos_config'])
def setupRunningWrapper(self):
self.path_list.extend(zc.buildout.easy_install.scripts([(
'testnode',
__name__+'.testnode', 'run')], self.ws,
sys.executable, self.wrapper_directory, arguments=[
dict(
environment=self.getRuntimeEnvironment(),
computer_id=CONFIG['computer_id'],
instance_dict=eval(self.parameter_dict.get('instance_dict', '{}')),
instance_root=CONFIG['instance_root'],
ipv4_address=self.getLocalIPv4Address(),
ipv6_address=self.getGlobalIPv6Address(),
master_url=CONFIG['master_url'],
profile_url=self.parameter_dict['profile_url'],
proxy_database=CONFIG['proxy_database'],
slapgrid_partition_binary=self.options['slapgrid_partition_binary'],
slapgrid_software_binary=self.options['slapgrid_software_binary'],
slapos_config=CONFIG['slapos_config'],
slapproxy_binary=self.options['slapproxy_binary'],
software_root=CONFIG['software_root'],
buildbot_binary=self.options['buildbot_binary'],
working_directory=CONFIG['working_directory'],
buildbot_host=self.parameter_dict['buildbot_host'],
slave_name=self.parameter_dict['slave_name'],
slave_password=self.parameter_dict['slave_password'],
bin_directory=self.bin_directory,
# botenvironemnt is splittable string of key=value to substitute
# environment of running bot
bot_environment=self.parameter_dict.get('bot_environment', ''),
partition_reference=CONFIG['partition_reference'],
)
]))
def installLocalSvn(self):
svn_dict = dict(svn_binary = self.options['svn_binary'])
svn_dict.update(self.parameter_dict)
svn_path = os.path.join(self.bin_directory, 'svn')
self._writeExecutable(svn_path, """\
#!/bin/sh
%(svn_binary)s --username %(svn_username)s --password %(svn_password)s \
--non-interactive --trust-server-cert --no-auth-cache "$@" """% svn_dict)
self.path_list.append(svn_path)
svnversion = os.path.join(self.bin_directory, 'svnversion')
if os.path.lexists(svnversion):
os.unlink(svnversion)
os.symlink(self.options['svnversion_binary'], svnversion)
self.path_list.append(svnversion)
def installLocalGit(self):
git = os.path.join(self.bin_directory, 'git')
if os.path.lexists(git):
os.unlink(git)
os.symlink(self.options['git_binary'], git)
self.path_list.append(git)
def installLocalZip(self):
zip = os.path.join(self.bin_directory, 'zip')
if os.path.lexists(zip):
os.unlink(zip)
os.symlink(self.options['zip_binary'], zip)
self.path_list.append(zip)
def installLocalPython(self):
"""Installs local python fully featured with eggs"""
self.path_list.extend(zc.buildout.easy_install.scripts([], self.ws,
sys.executable, self.bin_directory, scripts=None,
interpreter='python'))
def installLocalRunUnitTest(self):
link = os.path.join(self.bin_directory, 'runUnitTest')
destination = os.path.join(CONFIG['instance_root'],
CONFIG['partition_reference'], 'bin', 'runUnitTest')
if os.path.lexists(link):
if not os.readlink(link) != destination:
os.unlink(link)
if not os.path.lexists(link):
os.symlink(destination, link)
self.path_list.append(link)
def _installBuildbot(self):
self.setupRunningWrapper()
self.installLocalPython()
self.installLocalGit()
self.installLocalSvn()
self.installLocalRunUnitTest()
return self.path_list
def getRuntimeEnvironment(self):
env = {}
env['PATH'] = ':'.join([self.bin_directory] +
os.environ['PATH'].split(':'))
return env
def _installProfileTesting(self):
self.path_list.extend(zc.buildout.easy_install.scripts([(
'testnode',
__name__+'.profile_testnode', 'run')], self.ws,
sys.executable, self.wrapper_directory, arguments=[
dict(
environment=self.getRuntimeEnvironment(),
slapgrid_environment=eval(self.parameter_dict.get(
'slapgrid_environment', '{}')),
profile_path=self.parameter_dict.get('profile_path',
'slapos/software.cfg'),
repository=self.parameter_dict['repository'],
# Optional URL of test aggreagation system
test_suite_master_url=self.parameter_dict['test_suite_master_url'],
suite_name=self.parameter_dict['suite_name'],
branch=self.parameter_dict.get('branch', 'master'),
# internal parameters
software_root=CONFIG['software_root'],
computer_id=CONFIG['computer_id'],
git_binary=self.options['git_binary'],
master_url=CONFIG['master_url'],
proxy_database=CONFIG['proxy_database'],
slapgrid_software_binary=self.options['slapgrid_software_binary'],
slapos_config=CONFIG['slapos_config'],
slapproxy_binary=self.options['slapproxy_binary'],
working_directory=CONFIG['working_directory'],
bin_directory=self.bin_directory,
partition_reference=CONFIG['partition_reference'],
)
]))
return self.path_list
def _install(self):
self.requirements, self.ws = self.egg.working_set()
self.path_list = []
self.installSlapOs()
self.installLocalZip()
flavour = self.parameter_dict.get('flavour', 'buildbot')
if flavour == 'buildbot':
return self._installBuildbot()
elif flavour == 'profile-testing':
return self._installProfileTesting()
raise NotImplementedError('Falvour %r is unknown'% flavour)
import urlparse
import urllib
import httplib
import mimetools
from random import randint
import tempfile
import os
import stat
import zipfile
import mimetypes
import datetime
TB_SEP = "============================================================="\
"========="
def get_content_type(f):
return mimetypes.guess_type(f.name)[0] or 'application/octet-stream'
class ConnectionHelper:
def __init__(self, url):
self.conn = urlparse.urlparse(url)
if self.conn.scheme == 'http':
connection_type = httplib.HTTPConnection
if self.conn.port is None:
self.port = 80
else:
connection_type = httplib.HTTPSConnection
if self.conn.port is None:
self.port = 443
self.connection_type = connection_type
def _connect(self):
self.connection = self.connection_type(self.conn.hostname + ':' +
str(self.conn.port or self.port))
def POST(self, path, parameter_dict, file_list=None):
self._connect()
parameter_dict.update(__ac_name=self.conn.username,
__ac_password=self.conn.password)
header_dict = {'Content-type': "application/x-www-form-urlencoded"}
if file_list is None:
body = urllib.urlencode(parameter_dict)
else:
boundary = mimetools.choose_boundary()
header_dict['Content-type'] = 'multipart/form-data; boundary=%s' % (
boundary,)
body = ''
for k, v in parameter_dict.iteritems():
body += '--%s\r\n' % boundary
body += 'Content-Disposition: form-data; name="%s"\r\n' % k
body += '\r\n'
body += '%s\r\n' % v
for name, filename in file_list:
f = open(filename, 'r')
body += '--%s\r\n' % boundary
body += 'Content-Disposition: form-data; name="%s"; filename="%s"\r\n'\
% (name, name)
body += 'Content-Type: %s\r\n' % get_content_type(f)
body += 'Content-Length: %d\r\n' % os.fstat(f.fileno())[stat.ST_SIZE]
body += '\r\n'
body += f.read()
f.close()
body += '\r\n'
self.connection.request("POST", self.conn.path + '/' + path,
body, header_dict)
self.response = self.connection.getresponse()
class ERP5TestReportHandler:
def __init__(self, url, suite_name):
# random test id
self.test_id = "%s-%X" % (
("%s" % datetime.date.today()).replace("-", ""),
randint(1, 10000000000000000),
)
self.connection_helper = ConnectionHelper(url)
self.suite_name = suite_name
def reportStart(self):
# report that test is running
print 'Starting test with id %s' % self.test_id
self.connection_helper.POST('TestResultModule_reportRunning', dict(
test_suite=self.suite_name,
test_report_id=self.test_id,
))
def reportFinished(self, out_file, revision, success, duration, text):
# make file parsable by erp5_test_results
tempcmd = tempfile.mkstemp()[1]
tempcmd2 = tempfile.mkstemp()[1]
tempout = tempfile.mkstemp()[1]
templog = tempfile.mkstemp()[1]
log_lines = open(out_file, 'r').readlines()
tl = open(templog, 'w')
tl.write(TB_SEP + '\n')
if len(log_lines) > 900:
tl.write('...[truncated]... \n\n')
for log_line in log_lines[-900:]:
starts = log_line.startswith
if starts('Ran') or starts('FAILED') or starts('OK') or starts(TB_SEP):
continue
if starts('ERROR: ') or starts('FAIL: '):
tl.write('internal-test: ' + log_line)
continue
tl.write(log_line)
tl.write("----------------------------------------------------------------------\n")
tl.write('Ran 1 test in %.2fs\n' % duration)
if success:
tl.write('OK\n')
else:
tl.write('FAILED (failures=1)\n')
tl.write(TB_SEP + '\n')
tl.close()
open(tempcmd, 'w').write("""svn info dummy""")
open(tempcmd2, 'w').write(self.suite_name)
open(tempout, 'w').write("Revision: %s\n%s" % (revision, text))
# create nice zip archive
tempzip = tempfile.mkstemp()[1]
zip = zipfile.ZipFile(tempzip, 'w')
zip.write(tempcmd, 'dummy/001/cmdline')
zip.write(tempout, 'dummy/001/stdout')
zip.write(templog, 'dummy/001/stderr')
zip.write(tempout, '%s/002/stdout' % self.suite_name)
zip.write(templog, '%s/002/stderr' % self.suite_name)
zip.write(tempcmd2, '%s/002/cmdline' % self.suite_name)
zip.close()
os.unlink(templog)
os.unlink(tempcmd)
os.unlink(tempout)
os.unlink(tempcmd2)
# post it to ERP5
self.connection_helper.POST('TestResultModule_reportCompleted', dict(
test_report_id=self.test_id),
file_list=[('filepath', tempzip)]
)
os.unlink(tempzip)
import os
import socket
import signal
import shutil
import slapos.slap
import subprocess
import time
import atexit
from erp5testreporthandler import ERP5TestReportHandler
process_group_pid_list = []
def clean():
for pgpid in process_group_pid_list:
try:
os.killpg(pgpid, signal.SIGTERM)
except:
pass
def sigterm_handler(signal, frame):
clean()
def sigint_handler(signal, frame):
clean()
raise KeyboardInterrupt
signal.signal(signal.SIGINT, sigint_handler)
signal.signal(signal.SIGTERM, sigterm_handler)
atexit.register(clean)
def getCurrentBranchName(config, p):
r = subprocess.Popen([config['git_binary'], 'branch'], stdout=subprocess.PIPE, cwd=p).communicate()[0]
for f in r.splitlines():
if f.startswith('*'):
return f.split()[1]
return ''
def getRevision(config, p):
return subprocess.Popen([config['git_binary'], 'rev-parse', 'HEAD'], stdout=subprocess.PIPE, cwd=p).communicate()[0].strip()
def getCurrentFetchRemote(config, p):
r = subprocess.Popen([config['git_binary'], 'remote', '-v'], stdout=subprocess.PIPE, cwd=p).communicate()[0]
remote = ''
for f in r.splitlines():
if f.startswith('origin') and f.endswith('(fetch)'):
if remote != '':
raise ValueError('Too many remotes: %s' % r)
remote = r.split()[1]
return remote
def getMachineIdString():
"""Returns machine identification string"""
kw = dict(stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
idstr = subprocess.Popen(["uname", "-m"], **kw).communicate()[0].strip()
# try to detect gcc version
try:
gcc_list = subprocess.Popen(["gcc", "-v"], **kw).communicate()[0].split(
'\n')
for gcc in gcc_list:
if gcc.startswith('gcc version'):
idstr += ' gcc:' + gcc.split()[2]
break
except IndexError:
pass
# try to detect libc version
try:
libdir = os.path.sep + 'lib'
for libso in os.listdir(libdir):
if libso.startswith('libc.') and os.path.islink(os.path.join(libdir,
libso)):
libc = os.readlink(os.path.join(libdir, libso))
if libc.endswith('.so'):
idstr += ' libc:' + libc.split('-')[1][:-3]
else:
idstr += ' ' + libc
break
except IndexError:
pass
return idstr
def run(args):
config = args[0]
for k,v in config['environment'].iteritems():
os.environ[k] = v
proxy = None
slapgrid = None
last_revision_file = os.path.join(config['working_directory'],
'revision.txt')
if os.path.exists(last_revision_file):
os.unlink(last_revision_file)
# fetch repository from git
repository_clone = os.path.join(config['working_directory'], 'repository')
profile_path = os.path.join(repository_clone, config['profile_path'])
if os.path.exists(config['proxy_database']):
os.unlink(config['proxy_database'])
proxy = subprocess.Popen([config['slapproxy_binary'],
config['slapos_config']], close_fds=True, preexec_fn=os.setsid)
process_group_pid_list.append(proxy.pid)
slap = slapos.slap.slap()
slap.initializeConnection(config['master_url'])
while True:
try:
slap.registerSupply().supply(profile_path,
computer_guid=config['computer_id'])
except socket.error:
time.sleep(1)
pass
else:
break
while True:
info_list = []
a = info_list.append
while True:
try:
if os.path.exists(repository_clone):
if getCurrentFetchRemote(config, repository_clone) != config['repository']:
shutil.rmtree(repository_clone)
if not os.path.exists(repository_clone):
subprocess.check_call([config['git_binary'], 'clone',
config['repository'], repository_clone])
# switch to branch
branch = getCurrentBranchName(config, repository_clone)
if branch != config['branch']:
subprocess.check_call([config['git_binary'], 'checkout', '--force',
'--track', '-b', config['branch'], 'origin/'+config['branch']],
cwd=repository_clone)
subprocess.check_call([config['git_binary'], 'reset', '--hard',
'@{upstream}'], cwd=repository_clone)
except Exception:
print 'Retrying git in 60s'
time.sleep(60)
else:
break
a('Tested repository: %s' % config['repository'])
a('Machine identification: %s' % getMachineIdString())
erp5_report = ERP5TestReportHandler(config['test_suite_master_url'],
'@'.join([config['suite_name'], branch]))
last_revision = ''
if os.path.exists(last_revision_file):
last_revision = open(last_revision_file).read().strip()
revision = getRevision(config, repository_clone)
open(last_revision_file, 'w').write(revision)
if revision != last_revision:
print 'Running for revision %r' % revision
while True:
try:
erp5_report.reportStart()
except Exception:
print 'Retrying in 5s'
time.sleep(5)
else:
break
if os.path.exists(config['software_root']):
shutil.rmtree(config['software_root'])
os.mkdir(config['software_root'])
out_file = os.path.join(config['working_directory'], 'slapgrid.out')
if os.path.exists(out_file):
os.unlink(out_file)
out = open(out_file, 'w')
begin = time.time()
slapgrid_environment = os.environ.copy()
for k, v in config['slapgrid_environment'].iteritems():
slapgrid_environment[k] = v
a('Slapgrid environment: %r'% config['slapgrid_environment'])
slapgrid = subprocess.Popen([config['slapgrid_software_binary'], '-vc',
config['slapos_config']], close_fds=True, preexec_fn=os.setsid,
stdout=out, stderr=subprocess.STDOUT, env=slapgrid_environment)
process_group_pid_list.append(slapgrid.pid)
slapgrid.communicate()
out.close()
while True:
try:
erp5_report.reportFinished(out_file,revision,
slapgrid.returncode == 0, time.time() - begin,
'\n'.join(info_list))
except Exception:
print 'Retrying in 5s'
time.sleep(5)
else:
break
print 'Sleeping for 600s'
time.sleep(600)
[slapos]
software_root = %(software_root)s
instance_root = %(instance_root)s
master_url = %(master_url)s
computer_id = %(computer_id)s
[slapproxy]
host = %(proxy_host)s
port = %(proxy_port)s
database_uri = %(proxy_database)s
from xml_marshaller import xml_marshaller
import os
import signal
import slapos.slap
import socket
import subprocess
import sys
import time
process_group_pid_list = []
process_pid_file_list = []
process_command_list = []
def sigterm_handler(signal, frame):
for pgpid in process_group_pid_list:
try:
os.killpg(pgpid, signal.SIGTERM)
except:
pass
for pid_file in process_pid_file_list:
try:
os.kill(int(open(pid_file).read().strip()), signal.SIGTERM)
except:
pass
for p in process_command_list:
try:
subprocess.call(p)
except:
pass
sys.exit(1)
signal.signal(signal.SIGTERM, sigterm_handler)
def run(args):
config = args[0]
for k,v in config['environment'].iteritems():
os.environ[k] = v
proxy = None
slapgrid = None
supervisord_pid_file = os.path.join(config['instance_root'], 'var', 'run',
'supervisord.pid')
if os.path.exists(config['proxy_database']):
os.unlink(config['proxy_database'])
try:
proxy = subprocess.Popen([config['slapproxy_binary'],
config['slapos_config']], close_fds=True, preexec_fn=os.setsid)
process_group_pid_list.append(proxy.pid)
slap = slapos.slap.slap()
slap.initializeConnection(config['master_url'])
while True:
try:
slap.registerSupply().supply(config['profile_url'],
computer_guid=config['computer_id'])
except socket.error:
time.sleep(1)
pass
else:
break
while True:
slapgrid = subprocess.Popen([config['slapgrid_software_binary'], '-vc',
config['slapos_config']], close_fds=True, preexec_fn=os.setsid)
process_group_pid_list.append(slapgrid.pid)
slapgrid.wait()
if slapgrid.returncode == 0:
print 'Software installed properly'
break
print 'Problem with software installation, trying again'
time.sleep(600)
computer = slap.registerComputer(config['computer_id'])
partition_reference = config['partition_reference']
partition_path = os.path.join(config['instance_root'], partition_reference)
if not os.path.exists(partition_path):
os.mkdir(partition_path)
os.chmod(partition_path, 0750)
computer.updateConfiguration(xml_marshaller.dumps({
'address': config['ipv4_address'],
'instance_root': config['instance_root'],
'netmask': '255.255.255.255',
'partition_list': [{'address_list': [{'addr': config['ipv4_address'],
'netmask': '255.255.255.255'},
{'addr': config['ipv6_address'],
'netmask': 'ffff:ffff:ffff::'},
],
'path': partition_path,
'reference': partition_reference,
'tap': {'name': partition_reference},
}
],
'reference': config['computer_id'],
'software_root': config['software_root']}))
slap.registerOpenOrder().request(config['profile_url'],
partition_reference='testing partition',
partition_parameter_kw=config['instance_dict'])
slapgrid = subprocess.Popen([config['slapgrid_partition_binary'], '-vc',
config['slapos_config']], close_fds=True, preexec_fn=os.setsid)
slapgrid.wait()
if slapgrid.returncode != 0:
raise ValueError('Slapgrid instance failed')
runUnitTest = os.path.join(partition_path, 'bin', 'runUnitTest')
if not os.path.exists(runUnitTest):
raise ValueError('No %r provided' % runUnitTest)
except:
try:
if os.path.exists(supervisord_pid_file):
os.kill(int(open(supervisord_pid_file).read().strip()), signal.SIGTERM)
except:
pass
raise
finally:
# Nice way to kill *everything* generated by run process -- process
# groups working only in POSIX compilant systems
# Exceptions are swallowed during cleanup phase
if proxy is not None:
os.killpg(proxy.pid, signal.SIGTERM)
if os.path.exists(config['proxy_database']):
os.unlink(config['proxy_database'])
if slapgrid is not None and slapgrid.returncode is None:
os.killpg(slapgrid.pid, signal.SIGTERM)
try:
bot_env = os.environ.copy()
bot_env['PATH'] = ':'.join([config['bin_directory']] +
bot_env['PATH'].split(':'))
for l in config['bot_environment'].split():
k, v = l.split('=')
bot_env[k] = v
if subprocess.call([config['buildbot_binary'], 'create-slave', '-f',
config['working_directory'], config['buildbot_host'],
config['slave_name'], config['slave_password']]) != 0:
raise ValueError('Buildbot call failed')
process_command_list.append([config['buildbot_binary'], 'stop',
config['working_directory']])
if os.path.exists(os.path.join(config['working_directory'],
'buildbot.tac.new')):
tac = os.path.join(config['working_directory'], 'buildbot.tac')
if os.path.exists(tac):
os.unlink(tac)
os.rename(os.path.join(config['working_directory'],
'buildbot.tac.new'), tac)
if subprocess.call([config['buildbot_binary'], 'start',
config['working_directory']], env=bot_env) != 0:
raise ValueError('Issue during starting buildbot')
while True:
time.sleep(3600)
finally:
try:
subprocess.call([config['buildbot_binary'], 'stop',
config['working_directory']])
except:
pass
try:
if os.path.exists(supervisord_pid_file):
os.kill(int(open(supervisord_pid_file).read().strip()), signal.SIGTERM)
except:
pass
...@@ -29,13 +29,21 @@ from slapos.recipe.librecipe import GenericBaseRecipe ...@@ -29,13 +29,21 @@ from slapos.recipe.librecipe import GenericBaseRecipe
class Recipe(GenericBaseRecipe): class Recipe(GenericBaseRecipe):
def install(self): def install(self):
path_list = [] configuration_file = self.createFile(
a = path_list.append self.options['configuration-path'],
configuration_file = self.createFile(self.options['configuration-path'], self.substituteTemplate(self.getTemplateFilename('tidstorage.py.in'), self.options)) self.substituteTemplate(
a(configuration_file) self.getTemplateFilename('tidstorage.py.in'), self.options))
tidstorage_wrapper = self.createPythonScript(self.options['tidstorage-wrapper'], 'slapos.recipe.librecipe.execute.execute', [self.options['tidstoraged-binary'], '--nofork', '--config', configuration_file])
a(tidstorage_wrapper)
repozo_wrapper = self.createPythonScript(self.options['repozo-wrapper'], 'slapos.recipe.librecipe.execute.execute', [self.options['tidstorage-repozo-binary'], '--config', configuration_file, '--repozo', self.options['repozo-binary'], '-z'])
a(repozo_wrapper)
return path_list tidstorage_wrapper = self.createPythonScript(
self.options['tidstorage-wrapper'],
'slapos.recipe.librecipe.execute.execute',
[self.options['tidstoraged-binary'], '--nofork', '--config',
configuration_file])
repozo_wrapper = self.createPythonScript(
self.options['repozo-wrapper'],
'slapos.recipe.librecipe.execute.execute',
[self.options['tidstorage-repozo-binary'], '--config',
configuration_file, '--repozo', self.options['repozo-binary'], '-z'])
return [configuration_file, tidstorage_wrapper, repozo_wrapper]
known_tid_storage_identifier_dict = %(known-tid-storage-identifier-dict)s known_tid_storage_identifier_dict = %(known-tid-storage-identifier-dict)s
base_url = '%(base-url)s'
address = '%(ip)s' address = '%(ip)s'
port = %(port)s port = %(port)s
......
...@@ -21,7 +21,7 @@ long_query_time = 5 ...@@ -21,7 +21,7 @@ long_query_time = 5
max_allowed_packet = 128M max_allowed_packet = 128M
query_cache_size = 32M query_cache_size = 32M
plugin-load = ha_mroonga.so;ha_sphinx.so plugin-load = ha_mroonga.so
# The following are important to configure and depend a lot on to the size of # The following are important to configure and depend a lot on to the size of
# your database and the available resources. # your database and the available resources.
......
[buildout]
parts =
instance
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
[instance]
recipe = ${instance-recipe:egg}:${instance-recipe:module}
agent_binary = ${buildout:directory}/bin/agent
pidfile = $${directory:srv}/agent.pid
log = $${directory:agentlog}/agent.log
wrapper = $${directory:run}/agent
config = $${directory:etc}/agent.cfg
master-url = $${slap-parameter:master-url}
key = $${slap-parameter:userkey}
cert = $${slap-parameter:usercertificate}
configuration = $${slap-parameter:configuration}
default_max_install_duration = $${slap-parameter:default_max_install_duration}
default_max_uninstall_duration = $${slap-parameter:default_max_uninstall_duration}
default_max_request_duration = $${slap-parameter:default_max_request_duration}
default_max_destroy_duration = $${slap-parameter:default_max_destroy_duration}
[directory]
recipe = slapos.cookbook:mkdirectory
etc = $${buildout:directory}/etc
run = $${:etc}/run
agentlog = $${buildout:directory}/var/log/agent
srv = $${buildout:directory}/srv
bin = $${buildout:directory}/bin
[buildout] [buildout]
parts = parts =
instance switch_softwaretype
eggs-directory = ${buildout:eggs-directory} eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory} develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
[instance] [switch_softwaretype]
recipe = ${instance-recipe:egg}:${instance-recipe:module} recipe = slapos.cookbook:softwaretype
agent_binary = ${buildout:directory}/bin/agent default = ${template-agent:output}
report_start = ${buildout:directory}/bin/report_start
report_stop = ${buildout:directory}/bin/report_stop
dcrond_binary = ${dcron:location}/sbin/crond
python_binary = ${python2.7:location}/bin/python
pidfile = $${rootdirectory:run}/agent.pid
log = $${rootdirectory:agentlog}/agent.log
[rootdirectory]
recipe = slapos.cookbook:mkdirectory
run = $${buildout:directory}/etc/run
agentlog = $${buildout:directory}/var/log/agent
srv = $${buildout:directory}/srv
bin = $${buildout:directory}/bin
[buildout] [buildout]
extends = extends =
../../component/dcron/buildout.cfg
../../component/python-2.7/buildout.cfg
../../component/lxml-python/buildout.cfg ../../component/lxml-python/buildout.cfg
../../component/git/buildout.cfg
../../stack/slapos.cfg ../../stack/slapos.cfg
develop =
${:parts-directory}/slapos.cookbook-repository
${:parts-directory}/slapos.toolbox-repository
parts = parts =
template template
eggs template-agent
slapos.cookbook-repository
check-recipe
slapos.toolbox-repository
check-recipe-toolbox
instance-recipe-egg instance-recipe-egg
dcron
python2.7
script script
# Local development
[slapos.cookbook-repository]
recipe = plone.recipe.command
stop-on-error = true
branch = agent
location = ${buildout:parts-directory}/${:_buildout_section_name_}
command = ${git:location}/bin/git clone --branch ${:branch} --quiet http://git.erp5.org/repos/slapos.git ${:location}
update-command = cd ${:location} && ${git:location}/bin/git pull --quiet
[check-recipe]
recipe = plone.recipe.command
stop-on-error = true
update-command = ${:command}
command = grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link
[slapos.toolbox-repository]
recipe = plone.recipe.command
stop-on-error = true
branch = agent
location = ${buildout:parts-directory}/${:_buildout_section_name_}
command = ${git:location}/bin/git clone --branch ${:branch} --quiet http://git.erp5.org/repos/slapos.toolbox.git ${:location}
update-command = cd ${:location} && ${git:location}/bin/git pull --quiet
[check-recipe-toolbox]
recipe = plone.recipe.command
stop-on-error = true
update-command = ${:command}
command = grep parts ${buildout:develop-eggs-directory}/slapos.toolbox.egg-link
[instance-recipe] [instance-recipe]
# XXX-Cedric: it can use newest slapos.cfg to not have duplication
egg = slapos.cookbook egg = slapos.cookbook
module = agent module = agent
[instance-recipe-egg] [instance-recipe-egg]
# XXX-Cedric: it can use newest slapos.cfg to not have duplication
recipe = zc.recipe.egg recipe = zc.recipe.egg
python = python2.7 eggs =
eggs = ${instance-recipe:egg} ${lxml-python:egg}
${instance-recipe:egg}
[template] [template]
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg url = ${:_profile_base_location_}/instance.cfg
output = ${buildout:directory}/template.cfg output = ${buildout:directory}/template.cfg
md5sum = c7cb98594f394d05baedabe424643f6f md5sum = bcd3b3cb8a305c83bb048d5ac1c583fe
mode = 0644 mode = 0644
[eggs] [template-agent]
python = python2.7 recipe = slapos.recipe.template
recipe = zc.recipe.egg url = ${:_profile_base_location_}/instance-agent.cfg
eggs = output = ${buildout:directory}/template-agent.cfg
${lxml-python:egg} md5sum = fd9670d8473be402b10990398b553b00
slapos.cookbook mode = 0644
slapos.toolbox
erp5.util
[lxml-python]
python = python2.7
[script] [script]
recipe = z3c.recipe.scripts recipe = z3c.recipe.scripts
python = python2.7
eggs = eggs =
zc.buildout zc.buildout
${lxml-python:egg}
slapos.core slapos.core
slapos.cookbook
slapos.toolbox[agent] slapos.toolbox[agent]
erp5.util
[networkcache] [networkcache]
# signature certificates of the following uploaders. # signature certificates of the following uploaders.
# Romain Courteaud # Romain Courteaud
# Cedric de Saint Martin
# Rafael Monnerat
signature-certificate-list = signature-certificate-list =
-----BEGIN CERTIFICATE----- -----BEGIN CERTIFICATE-----
MIIB4DCCAUkCADANBgkqhkiG9w0BAQsFADA5MQswCQYDVQQGEwJGUjEZMBcGA1UE MIIB4DCCAUkCADANBgkqhkiG9w0BAQsFADA5MQswCQYDVQQGEwJGUjEZMBcGA1UE
...@@ -68,3 +100,29 @@ signature-certificate-list = ...@@ -68,3 +100,29 @@ signature-certificate-list =
q7jdfWO18Zp/BG7tagz0jmmC4y/8akzHsVlruo2+2du2freE8dK746uoMlXlP93g q7jdfWO18Zp/BG7tagz0jmmC4y/8akzHsVlruo2+2du2freE8dK746uoMlXlP93g
QUUGLQ== QUUGLQ==
-----END CERTIFICATE----- -----END CERTIFICATE-----
-----BEGIN CERTIFICATE-----
MIIB9jCCAV+gAwIBAgIJAO4V/jiMoICoMA0GCSqGSIb3DQEBBQUAMBMxETAPBgNV
BAMMCENPTVAtMjMyMCAXDTEyMDIxNjExMTAyM1oYDzIxMTIwMTIzMTExMDIzWjAT
MREwDwYDVQQDDAhDT01QLTIzMjCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA
wi/3Z8W9pUiegUXIk/AiFDQ0UJ4JFAwjqr+HSRUirlUsHHT+8DzH/hfcTDX1I5BB
D1ADk+ydXjMm3OZrQcXjn29OUfM5C+g+oqeMnYQImN0DDQIOcUyr7AJc4xhvuXQ1
P2pJ5NOd3tbd0kexETa1LVhR6EgBC25LyRBRae76qosCAwEAAaNQME4wHQYDVR0O
BBYEFMDmW9aFy1sKTfCpcRkYnP6zUd1cMB8GA1UdIwQYMBaAFMDmW9aFy1sKTfCp
cRkYnP6zUd1cMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADgYEAskbFizHr
b6d3iIyN+wffxz/V9epbKIZVEGJd/6LrTdLiUfJPec7FaxVCWNyKBlCpINBM7cEV
Gn9t8mdVQflNqOlAMkOlUv1ZugCt9rXYQOV7rrEYJBWirn43BOMn9Flp2nibblby
If1a2ZoqHRxoNo2yTmm7TSYRORWVS+vvfjY=
-----END CERTIFICATE-----
-----BEGIN CERTIFICATE-----
MIIB9jCCAV+gAwIBAgIJAOcKrOH/2Da6MA0GCSqGSIb3DQEBBQUAMBMxETAPBgNV
BAMMCENPTVAtMjk3MCAXDTEyMDYyNjAzMDU1MVoYDzIxMTIwNjAyMDMwNTUxWjAT
MREwDwYDVQQDDAhDT01QLTI5NzCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA
xzbOGlcoin2q+Mtp52r26Njliz2aoxIXbnOBUyDc/OGtk9nWA5uBtTc2zwR17um6
KV0bGyvuBA78XcvU+AIV/5s0ohBAX7yjRKmEhAYcFvov3EyWSdjOrqqo4qFSzOrK
sVQBlxIDpjQBH4F3lf6dBv6/M+tCT3iSv3aOZbsG0E8CAwEAAaNQME4wHQYDVR0O
BBYEFLqtrfTu+BIVt+TFiRUkIoiWIYrxMB8GA1UdIwQYMBaAFLqtrfTu+BIVt+TF
iRUkIoiWIYrxMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADgYEAc8N5P5gW
Jrdk9gF/3Cpp6THDiy93+WcuAm7zFwXPFNttJtFKMNObP2YRZvsQkvjezfrZoRBF
j8LgKB3tZCbBj+HDj+AeD+q9V+cqMFLKc6LezvQYUuum6bZdfUNnPv1K1ULYSPjq
/jsRBbabCWSXqxR6gYEM6ooauj3udBMXhHE=
-----END CERTIFICATE-----
...@@ -9,15 +9,22 @@ It means that a single main instance of Apache will be used to act as frontend ...@@ -9,15 +9,22 @@ It means that a single main instance of Apache will be used to act as frontend
for many slaves. for many slaves.
How to use How to deploy a frontend server
========== ===============================
First, you will need to request a "master" instance of Apache Frontend with This is to deploy an entire frontend server with a public IPv4.
"domain" parameter, like:: If you want to use an already deployed frontend to make your service available
via ipv4, switch to the "Example" parts.
First, you will need to request a "master" instance of Apache Frontend with:
* A "domain" parameter where the frontend will be available
* A "public-ipv4" parameter to state which public IPv4 will be used
like::
<?xml version='1.0' encoding='utf-8'?> <?xml version='1.0' encoding='utf-8'?>
<instance> <instance>
<parameter id="domain">moulefrite.org</parameter> <parameter id="domain">moulefrite.org</parameter>
<parameter id="port">443</parameter> <parameter id="public-ipv4">xxx.xxx.xxx.xxx</parameter>
</instance> </instance>
Then, it is possible to request many slave instances Then, it is possible to request many slave instances
...@@ -69,11 +76,11 @@ url of backend to use. ...@@ -69,11 +76,11 @@ url of backend to use.
"url" is a mandatory parameter. "url" is a mandatory parameter.
Example: http://mybackend.com/myresource Example: http://mybackend.com/myresource
cache enable_cache
~~~~~ ~~~~~
Specify if slave instance should use a varnish / stunnel to connect to backend. Specify if slave instance should use a varnish / stunnel to connect to backend.
Possible values: "true", "false". Possible values: "true", "false".
"cache" is an optional parameter. Defaults to "false". "enable_cache" is an optional parameter. Defaults to "false".
Example: true Example: true
type type
...@@ -81,30 +88,83 @@ type ...@@ -81,30 +88,83 @@ type
Specify if slave instance will redirect to a zope backend. If specified, Apache Specify if slave instance will redirect to a zope backend. If specified, Apache
RewriteRule will use Zope's Virtual Host Daemon. RewriteRule will use Zope's Virtual Host Daemon.
Possible values: "zope", "default". Possible values: "zope", "default".
"type" is an optional parameter. Defaults to "default". "type" is an optional parameter. Defaults to "default".
Example: zope Example: zope
custom_domain custom_domain
~~~~~~~~~~~~~ ~~~~~~~~~~~~~
Domain name to use as frontend. The frontend will be accessible from this domain. Domain name to use as frontend. The frontend will be accessible from this domain.
"custom_domain" is an optional parameter. Defaults to "custom_domain" is an optional parameter. Defaults to
[instancereference].[masterdomain]. [instancereference].[masterdomain].
Example: www.mycustomdomain.com Example: www.mycustomdomain.com
path
~~~~
Only used if type is "zope".
Will append the specified path to the "VirtualHostRoot" of the zope's
VirtualHostMonster.
"path" is an optional parameter, ignored if not specified.
Example of value: "/erp5/web_site_module/hosting/"
Examples
========
Here are some example of how to make your SlapOS service available through
an already deployed frontend.
Simple Example
--------------
Request slave frontend instance so that https://[1:2:3:4:5:6:7:8]:1234 will be
redirected and accessible from the proxy::
instance = request(
software_release=apache_frontend,
software_type="RootSoftwareInstance",
partition_reference='my frontend',
shared=True,
partition_parameter_kw={
"url":"https://[1:2:3:4:5:6:7:8]:1234",
}
)
Zope Example
------------
Request slave frontend instance using a Zope backend so that
https://[1:2:3:4:5:6:7:8]:1234 will be redirected and accessible from the
proxy::
instance = request(
software_release=apache_frontend,
software_type="RootSoftwareInstance",
partition_reference='my frontend',
shared=True,
partition_parameter_kw={
"url":"https://[1:2:3:4:5:6:7:8]:1234",
"type":"zope",
}
)
Advanced example Advanced example
================ ----------------
Request slave frontend instance using a Zope backend, with Varnish activated, Request slave frontend instance using a Zope backend, with Varnish activated,
listening to a custom domain:: listening to a custom domain and redirecting to /erp5/ so that
https://[1:2:3:4:5:6:7:8]:1234/erp5/ will be redirected and accessible from
the proxy::
instance = request( instance = request(
software_release=apache_frontend, software_release=apache_frontend,
partition_reference='frontend2', software_type="RootSoftwareInstance",
partition_reference='my frontend',
shared=True, shared=True,
partition_parameter_kw={ partition_parameter_kw={
"url":"https://[1:2:3:4]:1234/someresource", "url":"https://[1:2:3:4:5:6:7:8]:1234",
"cache":"true", "enable_cache":"true",
"type":"zope", "type":"zope",
"path":"/erp5",
"custom_domain":"mycustomdomain.com", "custom_domain":"mycustomdomain.com",
} }
) )
...@@ -115,5 +175,9 @@ Notes ...@@ -115,5 +175,9 @@ Notes
It is not possible with slapos to listen to port <= 1024, because process are It is not possible with slapos to listen to port <= 1024, because process are
not run as root. It is a good idea then to go on the node where the instance is not run as root. It is a good idea then to go on the node where the instance is
and set some iptables rules like (if using default ports):: and set some iptables rules like (if using default ports)::
iptables -t nat -A PREROUTING -p tcp -d {public ip} --dport 443 -j DNAT --to-destination {listening ip}:4443
iptables -t nat -A PREROUTING -p tcp -d {public_ip} --dport 80 -j DNAT --to-destination {listening ip}:8080 iptables -t nat -A PREROUTING -p tcp -d {public_ipv4} --dport 443 -j DNAT --to-destination {listening_ipv4}:4443
iptables -t nat -A PREROUTING -p tcp -d {public_ipv4} --dport 80 -j DNAT --to-destination {listening_ipv4}:8080
Where {public ip} is the public IP of your server, or at least the LAN IP to where your NAT will forward to.
{listening ip} is the private ipv4 (like 10.0.34.123) that the instance is using and sending as connection parameter.
[buildout]
extends =
../../component/binutils/buildout.cfg
../../component/lxml-python/buildout.cfg
../../component/apache/buildout.cfg
../../component/stunnel/buildout.cfg
../../component/varnish/buildout.cfg
../../component/dcron/buildout.cfg
../../component/logrotate/buildout.cfg
../../component/rdiff-backup/buildout.cfg
../../stack/slapos.cfg
parts =
template
binutils
apache-2.2
apache-antiloris-apache-2.2
stunnel
varnish-2.1
dcron
logrotate
rdiff-backup
# Buildoutish
eggs
instance-recipe-egg
[instance-recipe]
# Note: In case if specific instantiation recipe is used this is the place to
# put its name
egg = slapos.cookbook
module = apache.frontend
[instance-recipe-egg]
recipe = zc.recipe.egg
eggs = ${instance-recipe:egg}
[eggs]
recipe = zc.recipe.egg
eggs =
${lxml-python:egg}
[template]
# Default template for apache instance.
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg
md5sum = fea902a2b9dbf8c80ff201bcf73f9396
output = ${buildout:directory}/template.cfg
mode = 0644
\ No newline at end of file
# Development profile of apache-frontend.
# Exactly the same as software.cfg, but fetch the slapos.cookbook
# from git repository instead of fetching stable version,
# allowing to play with bleeding edge environment.
# You'll need to run buildout twice for this profile.
[buildout]
extends =
# Extend in this order, otherwise "parts" will be taken from git profile
../../component/git/buildout.cfg
common.cfg
parts +=
slapos.cookbook-repository
develop =
${:parts-directory}/slapos.cookbook-repository
[slapos.cookbook-repository]
recipe = slapos.recipe.build:gitclone
repository = http://git.erp5.org/repos/slapos.git
branch = frontend
git-executable = ${git:location}/bin/git
[check-recipe]
recipe = plone.recipe.command
stop-on-error = true
update-command = ${:command}
command =
grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link &&
[buildout] [buildout]
parts = parts =
directory
instance instance
configtest
eggs-directory = ${buildout:eggs-directory} eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory} develop-eggs-directory = ${buildout:develop-eggs-directory}
# Create all needed directories
[directory]
recipe = slapos.cookbook:mkdirectory
etc = $${buildout:directory}/etc/
var = $${buildout:directory}/var/
srv = $${buildout:directory}/srv/
bin = $${buildout:directory}/bin/
service = $${:etc}/service
# Deploy Apache (old way, with monolithic recipe)
[instance] [instance]
recipe = ${instance-recipe:egg}:${instance-recipe:module} recipe = ${instance-recipe:egg}:${instance-recipe:module}
httpd_home = ${apache-2.2:location}
httpd_binary = ${apache-2.2:location}/bin/httpd httpd_binary = ${apache-2.2:location}/bin/httpd
logrotate_binary = ${logrotate:location}/usr/sbin/logrotate logrotate_binary = ${logrotate:location}/usr/sbin/logrotate
openssl_binary = ${openssl:location}/bin/openssl openssl_binary = ${openssl:location}/bin/openssl
...@@ -14,5 +28,12 @@ dcrond_binary = ${dcron:location}/sbin/crond ...@@ -14,5 +28,12 @@ dcrond_binary = ${dcron:location}/sbin/crond
varnishd_binary = ${varnish-2.1:location}/sbin/varnishd varnishd_binary = ${varnish-2.1:location}/sbin/varnishd
stunnel_binary = ${stunnel:location}/bin/stunnel stunnel_binary = ${stunnel:location}/bin/stunnel
rdiff_backup_binary = ${buildout:bin-directory}/rdiff-backup rdiff_backup_binary = ${buildout:bin-directory}/rdiff-backup
gcc_binary = ${gcc-java-minimal:location}/bin/gcc gcc_binary = gcc
binutils_directory = ${binutils:location}/bin/ binutils_directory = ${binutils:location}/bin/
# Create wrapper for "apachectl conftest" in bin
[configtest]
recipe = slapos.cookbook:wrapper
command-line = $${instance:httpd_binary} -f $${directory:etc}/apache_frontend.conf -t
output = $${directory:bin}/apache-configtest
This diff is collapsed.
...@@ -16,7 +16,7 @@ extends = ...@@ -16,7 +16,7 @@ extends =
[application] [application]
recipe = slapos.recipe.build:download-unpacked recipe = slapos.recipe.build:download-unpacked
url = http://download.dotclear.net/latest-2.0.tar.gz url = http://download.dotclear.net/latest-2.0.tar.gz
md5sum = 98263b4734f93b7c54d3ab436bf344e3 md5sum = 4ca12cbd12228ceee0b7c0f697e1ff09
[application-template] [application-template]
recipe = slapos.recipe.download recipe = slapos.recipe.download
......
...@@ -200,12 +200,23 @@ ...@@ -200,12 +200,23 @@
* ); * );
* @endcode * @endcode
*/ */
/*
+ * The apachephp recipe provided with both host+port in the same string.
+ * Split them again, php way.
+ * And it could be an ipv6 as well, so beware of colons.
+ */
$mysql_host_port = '%(mysql_host)s';
$mysql_port = substr(strrchr($mysql_host_port, ":"), 1);
$mysql_host = substr($mysql_host_port, 0, strlen($mysql_host_port) - strlen($mysql_port) - 1);
$databases['default']['default'] = array( $databases['default']['default'] = array(
'driver' => 'mysql', 'driver' => 'mysql',
'database' => '%(mysql_database)s', 'database' => '%(mysql_database)s',
'username' => '%(mysql_user)s', 'username' => '%(mysql_user)s',
'password' => '%(mysql_password)s', 'password' => '%(mysql_password)s',
'host' => '%(mysql_host)s', 'host' => $mysql_host,
'port' => $mysql_port,
'prefix' => '', 'prefix' => '',
); );
......
This diff is collapsed.
--- a/system.theme.css Mon Oct 29 11:20:30 2012 +0100
+++ b/system.theme.css Mon Oct 29 11:20:53 2012 +0100
@@ -1,3 +1,14 @@
+
+#install-settings-form .form-item-driver,
+#install-settings-form #edit-mysql * {
+ display: none;
+}
+
+#install-settings-form #edit-mysql:before {
+ content: "The database has been set up by SlapOS.";
+ font-weight: bold;
+ font-size: 120%;
+}
/**
* @file
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment