Skip to content

slapos
slapos
Commit 342b5393 authored by Aurélien Vermylen's avatar Aurélien Vermylen
Browse files
Options

Merge branch 'master' into clearroad

parents c44e2f89 503a278b
Hide whitespace changes
Inline Side-by-side
Showing with 1559 additions and 3421 deletions
component/dcron/buildout.cfg
View file @ 342b5393
......@@ -2,6 +2,7 @@
extends =
../coreutils/buildout.cfg
../patch/buildout.cfg
../randomsleep/buildout.cfg
parts = dcron-output
......@@ -18,6 +19,7 @@ make-options =
post-make-hook = ${:_profile_base_location_}/dcron-hooks.py#860e914dff4108b47565965fe5ebc7b5:post_make_hook
environment =
PATH=${patch:location}/bin:%(PATH)s
randomsleep_install = ${randomsleep:location}
[dcron-output]
# Shared binary location to ease migration
......
component/fontconfig/buildout.cfg
View file @ 342b5393
......@@ -7,14 +7,15 @@ extends =
../bzip2/buildout.cfg
../zlib/buildout.cfg
../bzip2/buildout.cfg
../gperf/buildout.cfg
parts =
fontconfig
[fontconfig]
recipe = slapos.recipe.cmmi
url = http://fontconfig.org/release/fontconfig-2.12.1.tar.bz2
md5sum = b5af5a423ee3b5cfc34846838963c058
url = http://fontconfig.org/release/fontconfig-2.12.6.tar.bz2
md5sum = 733f5e2371ca77b69707bd7b30cc2163
pkg_config_depends = ${freetype:pkg_config_depends}:${freetype:location}/lib/pkgconfig:${libxml2:location}/lib/pkgconfig
# XXX-Cedric : should we use --with-add-fonts={somefont:location}/share,{someotherfont:location}/share?
configure-options =
......@@ -23,7 +24,7 @@ configure-options =
--enable-libxml2
--with-default-fonts=${fonts:location}
environment =
PATH=${pkgconfig:location}/bin:%(PATH)s
PATH=${pkgconfig:location}/bin:${gperf:location}/bin:%(PATH)s
PKG_CONFIG_PATH=${:pkg_config_depends}
CPPFLAGS=-I${zlib:location}/include -I${bzip2:location}/include
LDFLAGS=-L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${bzip2:location}/lib -Wl,-rpath=${bzip2:location}/lib
component/gcc/buildout.cfg
View file @ 342b5393
......@@ -2,6 +2,7 @@
# Mostly required to support languages different than C or C++
[buildout]
extends =
../gettext/buildout.cfg
../gmp/buildout.cfg
../perl/buildout.cfg
../tar/buildout.cfg
......@@ -32,7 +33,7 @@ configure-options =
--with-as=${binutils:location}/bin/as
environment =
LDFLAGS=-Wl,-rpath=${gmp:location}/lib -Wl,-rpath=${isl:location}/lib -Wl,-rpath=${mpc:location}/lib -Wl,-rpath=${mpfr:location}/lib
PATH=${binutils:location}/bin:${perl:location}/bin:${tar:location}/bin:%(PATH)s
PATH=${binutils:location}/bin:${gettext:location}/bin:${perl:location}/bin:${tar:location}/bin:%(PATH)s
[gcc-minimal]
<= gcc-common
......
component/golang/buildout.cfg
View file @ 342b5393
......@@ -34,8 +34,8 @@ make-targets= cd src && ./make.bash && cp -alf .. ${:location}
[golang18]
<= golang-common
url = https://storage.googleapis.com/golang/go1.8.3.src.tar.gz
md5sum = 64e9380e07bba907e26a00cf5fcbe77e
url = https://dl.google.com/go/go1.8.7.src.tar.gz
md5sum = c61cfe9c85e7d42f903d3fe146d7cde6
# go1.8 needs go1.4 to bootstrap
environment-extra =
......@@ -43,8 +43,8 @@ environment-extra =
[golang19]
<= golang-common
url = https://storage.googleapis.com/golang/go1.9.2.src.tar.gz
md5sum = 44105c865a1a810464df79233a05a568
url = https://dl.google.com/go/go1.9.4.src.tar.gz
md5sum = 6816441fd6680c63865cdd5cb8bc1960
# go1.9 needs go1.4 to bootstrap
environment-extra =
......@@ -63,6 +63,11 @@ environment-extra =
# lab.nexedi.com/kirr/neo/go/... \
# github.com/pkg/profile \
# golang.org/x/perf/cmd/benchstat
#
# it is possible to specify Go build flags used for compilation e.g. this way:
#
# [gowork]
# buildflags = -race
[gowork]
directory = ${buildout:directory}/go.work
src = ${:directory}/src
......@@ -72,6 +77,9 @@ depends = ${gowork.goinstall:recipe}
# go version used for the workspace (possible to override in applications)
golang = ${golang19:location}
# no special build flags by default
buildflags =
# everything is done by dependent parts
recipe = plone.recipe.command
command = :
......@@ -100,7 +108,7 @@ stop-on-error = true
# clients should put package list to install to gowork:install ("..." requests installing everything)
[gowork.goinstall]
recipe = plone.recipe.command
command = bash -c ". ${gowork:env.sh} && go install -v ${gowork:install}"
command = bash -c ". ${gowork:env.sh} && go install ${gowork:buildflags} -v ${gowork:install}"
update-command = ${:command}
stop-on-error = true
......@@ -115,7 +123,7 @@ git-executable = ${git:location}/bin/git
# 3) provide repository (which is not the same as importpath in general case)
#
# the list of go packages for a go workspace state can be automatically
# generated with the help of go-pkg-snapshot tool.
# generated with the help of gowork-snapshot tool.
[go-git-package]
<= git-repository
location = ${gowork:src}/${:go.importpath}
component/golang/gowork-snapshot
View file @ 342b5393
......@@ -25,9 +25,16 @@ gogit_list() {
# git_upstream_url <repo> - show current branch upstream URL
git_upstream_url() {
repo=$1
head="`git -C $repo symbolic-ref --short HEAD`" # current branch - e.g. "t"
remote="`git -C $repo config --get branch.$head.remote`" # upstream name, e.g. "kirr"
url="`git -C $repo config --get remote.$remote.url`" # upstream URL
head="`git -C $repo symbolic-ref -q --short HEAD`" # current branch - e.g. "master"
if [ -z $head ] ; then
remote="origin" # If we are in detached HEAD, assume the upstream name is origin
else
remote="`git -C $repo config --get branch.$head.remote`" # upstream name, e.g. "origin"
fi
url="`git -C $repo config --get remote.$remote.url`" # upstream URL
if [ -z $url ] ; then
echo "ERROR: Failed to guess upstream url from $repo" 1>&2
fi
echo "$url"
}
......@@ -59,5 +66,5 @@ while read pkg url rev; do
echo "<= go-git-package"
echo "go.importpath = $pkg"
echo "repository = $url"
echo "revision = $rev"
echo "revision = $rev"
done
component/h5py/buildout.cfg
View file @ 342b5393
......@@ -2,6 +2,7 @@
extends =
../hdf5/buildout.cfg
../cython/buildout.cfg
../numpy/buildout.cfg
parts =
h5py
......@@ -12,7 +13,9 @@ CPPFLAGS = -I${hdf5:location}/include
[h5py]
recipe = zc.recipe.egg:custom
egg = h5py
setup-eggs = ${cython:egg}
setup-eggs =
${cython:egg}
${numpy:egg}
include-dirs =
${hdf5:location}/include
library-dirs =
......
component/libxml2/buildout.cfg
View file @ 342b5393
......@@ -11,8 +11,8 @@ parts =
[libxml2]
recipe = slapos.recipe.cmmi
url = http://xmlsoft.org/sources/libxml2-2.9.5.tar.gz
md5sum = 5ce0da9bdaa267b40c4ca36d35363b8b
url = http://xmlsoft.org/sources/libxml2-2.9.7.tar.gz
md5sum = 896608641a08b465098a40ddf51cefba
configure-options =
--disable-static
--without-python
......
component/libxslt/buildout.cfg
View file @ 342b5393
......@@ -7,8 +7,8 @@ parts =
libxslt
[libxslt]
url = ftp://xmlsoft.org/libxslt/libxslt-1.1.30.tar.gz
md5sum = 70becbbcb1dad55f14de0b84171b91d5
url = ftp://xmlsoft.org/libxslt/libxslt-1.1.32.tar.gz
md5sum = 1fc72f98e98bf4443f1651165f3aa146
recipe = slapos.recipe.cmmi
configure-options =
--disable-static
......@@ -20,5 +20,3 @@ configure-options =
environment =
CPPFLAGS=-I${zlib:location}/include
LDFLAGS=-L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
LIBXML_CFLAGS=-I${libxml2:location}/include
LIBXML_LIBS=-L${libxml2:location}/lib -Wl,-rpath=${libxml2:location}/lib
component/mariadb/buildout.cfg
View file @ 342b5393
......@@ -27,8 +27,8 @@ parts =
[mariadb]
recipe = slapos.recipe.cmmi
url = https://downloads.mariadb.org/f/mariadb-${:version}/source/mariadb-${:version}.tar.gz/from/http%3A//fr.mirror.babylon.network/mariadb/?serve
version = 10.1.29
md5sum = 6d2cc6b0f8c5c7525e08aaddad8dca96
version = 10.1.31
md5sum = 14ab0398c019eb531bc29f2c437ccb51
patch-options = -p0
patches =
${:_profile_base_location_}/mariadb_10.1.21_create_system_tables__no_test.patch#3c76aa9564a162f13aced7c0a3f783b3
......
component/mariadb/mariarocks.cfg
View file @ 342b5393
# Do not extend any file that touch buildout:parts.
[mariadb]
version = 10.2.11
md5sum = 954088299fe5f11b4fda3b540558adbd
version = 10.2.13
md5sum = 20c61bd4059ba287e54cfb2862bae81d
stable-patches =
configure-options +=
# force build of TokuDB due to a regression in 10.2.11
-DTOKUDB_OK=1
-DCMAKE_C_COMPILER=${gcc:location}/bin/gcc
-DCMAKE_CXX_COMPILER=${gcc:location}/bin/g++
extra_cflags = -I${zstd:location}/include
......
component/nginx/buildout.cfg
View file @ 342b5393
......@@ -18,6 +18,7 @@ patch-options = -p0
patch-binary = ${patch:location}/bin/patch
patches =
http://nginx.org/download/patch.2017.ranges.txt#40bf9f37c881cb3b10cfefd84ca92f6a
${:_profile_base_location_}/fix-gcc7-implicit-fallthrough-errors.patch
[nginx]
<= nginx-common
......
component/nginx/fix-gcc7-implicit-fallthrough-errors.patch 0 → 100644
View file @ 342b5393
commit 8449f750e62cd229026e9df3bd023ec7e073a7d4
Author: Maxim Dounin <mdounin@mdounin.ru>
Date: Thu Apr 27 16:57:18 2017 +0300
Added missing "fall through" comments (ticket #1259).
Found by gcc7 (-Wimplicit-fallthrough).
diff --git src/core/ngx_murmurhash.c src/core/ngx_murmurhash.c
index c31e0e03..5ade658d 100644
--- src/core/ngx_murmurhash.c
+++ src/core/ngx_murmurhash.c
@@ -35,8 +35,10 @@ ngx_murmur_hash2(u_char *data, size_t len)
switch (len) {
case 3:
h ^= data[2] << 16;
+ /* fall through */
case 2:
h ^= data[1] << 8;
+ /* fall through */
case 1:
h ^= data[0];
h *= 0x5bd1e995;
diff --git src/http/ngx_http_parse.c src/http/ngx_http_parse.c
index 36220fdc..e8e51563 100644
--- src/http/ngx_http_parse.c
+++ src/http/ngx_http_parse.c
@@ -1396,6 +1396,7 @@ ngx_http_parse_complex_uri(ngx_http_request_t *r, ngx_uint_t merge_slashes)
goto done;
case '+':
r->plus_in_uri = 1;
+ /* fall through */
default:
state = sw_usual;
*u++ = ch;
@@ -1437,6 +1438,7 @@ ngx_http_parse_complex_uri(ngx_http_request_t *r, ngx_uint_t merge_slashes)
goto done;
case '+':
r->plus_in_uri = 1;
+ /* fall through */
default:
state = sw_usual;
*u++ = ch;
@@ -1484,6 +1486,7 @@ ngx_http_parse_complex_uri(ngx_http_request_t *r, ngx_uint_t merge_slashes)
goto done;
case '+':
r->plus_in_uri = 1;
+ /* fall through */
default:
state = sw_usual;
*u++ = ch;
diff --git src/os/unix/ngx_process.c src/os/unix/ngx_process.c
index dd50b5ca..993c032a 100644
--- src/os/unix/ngx_process.c
+++ src/os/unix/ngx_process.c
@@ -413,6 +413,7 @@ ngx_signal_handler(int signo, siginfo_t *siginfo, void *ucontext)
break;
}
ngx_debug_quit = 1;
+ /* fall through */
case ngx_signal_value(NGX_SHUTDOWN_SIGNAL):
ngx_quit = 1;
action = ", shutting down";
component/ocropy/buildout.cfg
View file @ 342b5393
......@@ -32,8 +32,8 @@ setup-eggs =
${matplotlib:egg}
${pillow-python:egg}
patches =
${:_profile_base_location_}/ocropy.patch
patch-options = -p0
${:_profile_base_location_}/ocropy.patch#dd7a02e1e63ed9df68e3a539b3e919eb
patch-options = -p1
patch-binary = ${patch:location}/bin/patch
environment = ocropy-env
find-links = https://github.com/tmbdev/ocropy/tarball/4efbddca22bb2f0c639af0694e7a1386f2f097b5/ocropy-1.0.tar.gz
......
component/ocropy/ocropy.patch
View file @ 342b5393
diff --git ocrolib/__init__.py ocrolib/__init__.py
From 1bb1546b12b0c08b1b32b293207de2d58d43ff1c Mon Sep 17 00:00:00 2001
From: Francois Le Corre <francois.lecorre@nexedi.com>
Date: Thu, 6 Apr 2017 11:32:27 +0200
Subject: [PATCH] WIP
- New OCROPY_MODEL_PATH environment variable to specify the position of models
- Import for the first time a module that compiles C code from python,
allowing us to have the static library built before runtime and
preventing us from using gcc through zope.
---
ocrolib/__init__.py | 3 ++-
ocrolib/common.py | 7 ++++---
ocrolib/native.py | 1 +
setup.py | 15 ++++++++++++---
4 files changed, 19 insertions(+), 7 deletions(-)
diff --git a/ocrolib/__init__.py b/ocrolib/__init__.py
index 1e0d627..81e85fb 100644
--- ocrolib/__init__.py
+++ ocrolib/__init__.py
--- a/ocrolib/__init__.py
+++ b/ocrolib/__init__.py
@@ -1,7 +1,7 @@
__all__ = [
"binnednn","cairoextras","common","components","dbtables",
......@@ -11,18 +27,18 @@ index 1e0d627..81e85fb 100644
]
################################################################
@@ -9,5 +9,6 @@ __all__ = [
@@ -9,5 +9,6 @@
################################################################
import default
+from psegutils import *
from common import *
from default import traceback as trace
diff --git ocrolib/common.py ocrolib/common.py
diff --git a/ocrolib/common.py b/ocrolib/common.py
index 27c0f26..14f088f 100644
--- ocrolib/common.py
+++ ocrolib/common.py
@@ -14,6 +14,7 @@ import unicodedata
--- a/ocrolib/common.py
+++ b/ocrolib/common.py
@@ -14,6 +14,7 @@
import inspect
import glob
import cPickle
......@@ -38,7 +54,7 @@ index 27c0f26..14f088f 100644
def load_object(fname,zip=0,nofind=0,verbose=0):
"""Loads an object from disk. By default, this handles zipped files
and searches in the usual places for OCRopus. It also handles some
@@ -439,8 +441,7 @@ def load_object(fname,zip=0,nofind=0,verbose=0):
@@ -439,8 +441,7 @@ class names that have changed."""
if zip==0 and fname.endswith(".gz"):
zip = 1
if zip>0:
......@@ -57,10 +73,10 @@ index 27c0f26..14f088f 100644
possible_prefixes.append("/usr/local/share/ocropus")
diff --git ocrolib/native.py ocrolib/native.py
diff --git a/ocrolib/native.py b/ocrolib/native.py
index b7a207f..240450b 100644
--- ocrolib/native.py
+++ ocrolib/native.py
--- a/ocrolib/native.py
+++ b/ocrolib/native.py
@@ -44,6 +44,7 @@ class CompileError(Exception):
def compile_and_find(c_string,prefix=".pynative",opt="-g -O4",libs="-lm",
......@@ -69,22 +85,22 @@ index b7a207f..240450b 100644
if not os.path.exists(prefix):
os.mkdir(prefix)
m = hashlib.md5()
diff --git setup.py setup.py
index 2ec5832..6697b12 100644
--- setup.py
+++ setup.py
@@ -10,7 +10,9 @@ assert sys.version_info[0]==2 and sys.version_info[1]>=7,\
diff --git a/setup.py b/setup.py
index 2ec5832..0ad4d85 100644
--- a/setup.py
+++ b/setup.py
@@ -10,7 +10,9 @@
from distutils.core import setup #, Extension, Command
#from distutils.command.install_data import install_data
-if not os.path.exists("models/en-default.pyrnn.gz"):
+models = os.environ.get('OCROPY_MODEL_PATH', '').split(':') or \
+ [c for c in glob.glob("models/*pyrnn.gz")]
+models = os.environ.get('OCROPY_MODEL_PATH')
+models = models.split(':') if models else glob.glob("models/*pyrnn.gz")
+if not models:
print()
print("You should download the default model 'en-default.pyrnn.gz'")
print("and put it into ./models.")
@@ -18,16 +20,23 @@ if not os.path.exists("models/en-default.pyrnn.gz"):
@@ -18,16 +20,23 @@
print("Check https://github.com/tmbdev/ocropy for the location")
print("of model files.")
print()
......@@ -110,3 +126,6 @@ index 2ec5832..6697b12 100644
+ data_files= [('share/ocropus', models), ("", ["LICENSE"])],
scripts = scripts,
)
--
2.14.1
component/python-2.7/asyncore_poll_insteadof_select.patch 0 → 100644
View file @ 342b5393
From efa86aa5ad5b89ec329b3cde288cb119efb21c89 Mon Sep 17 00:00:00 2001
From: Kirill Smelkov <kirr@nexedi.com>
Date: Thu, 06 Jul 2017 21:15:31 +0200
Subject: [PATCH] asyncore: switch default to use poll instead of select
Recently we started to get the following errors on a Wendelin production
instance:
File ".../bin/runzope", line 211, in <module>
sys.exit(Zope2.Startup.run.run())
File ".../eggs/Zope2-2.13.24-py2.7.egg/Zope2/Startup/run.py", line 26, in run
starter.run()
File ".../eggs/Zope2-2.13.24-py2.7.egg/Zope2/Startup/__init__.py", line 111, in run
Lifetime.loop()
File ".../eggs/Zope2-2.13.24-py2.7.egg/Lifetime/__init__.py", line 43, in loop
lifetime_loop()
File ".../eggs/Zope2-2.13.24-py2.7.egg/Lifetime/__init__.py", line 53, in lifetime_loop
asyncore.poll(timeout, map)
File ".../parts/python2.7/lib/python2.7/asyncore.py", line 145, in poll
r, w, e = select.select(r, w, e, timeout)
ValueError: filedescriptor out of range in select()
Initially we thought the reason here is that number of file descriptors this
process uses goes beyond allowed limit (65K open files on that instance)
because wendelin.core uses 1 fd per an array view (opening
/dev/shm/ramh.XXXXX) but that turned out to be not strictly that case:
The reason here is that select() has limit for how many #fd it can
monitor at all. The limit is system-specific but on Linux it is usually
1024 - http://man7.org/linux/man-pages/man2/select.2.html#BUGS :
$ cat s.c
#include <sys/select.h>
#include <stdio.h>
int main() {
printf("%d\n", FD_SETSIZE);
return 0;
}
$ tcc -run s.c
1024
Also select() can monitor only file descriptors which are by itself
"< FD_SETSIZE", e.g. it cannot monitor fd=1025 even if there are only 10
opened file descriptors because 1025 is not < 1024.
As was said above in wendelin.core every array view uses 1 file
descriptor, so if we are using not so small amount of arrays, even
though #fd does not go beyond process-specific ulimit, because of
select() usage the total number of allowed opened file descriptors
becomes essentially 1024.
So let's switch from select() to poll() which does not have this 1024
#fd limit. Asyncore already support using poll() out of the box -
either via passing use_pull=True to asyncore.loop()
https://docs.python.org/2/library/asyncore.html#asyncore.loop
or by using asyncore.poll2() instead of asyncore.poll()
https://github.com/python/cpython/blob/2.7/Lib/asyncore.py#L170
https://github.com/python/cpython/blob/2.7/Lib/asyncore.py#L125
--------
One option would be to switch asyncore.poll() -> asyncore.poll2() for only 1
place in Zope - inside lifetime_loop(). There are however many such similar
places in Zope (e.g. in medusa) and in other software.
For this reason, for me, what makes sense to do is not to patch all such
places, but instead change via runtime-patching asyncore.poll to be
asyncore.poll2 - this way all software will automatically benefit from
poll() usage instead of select.
P.S.
What might also make sense to do in the future is to actually let
asyncore.poll to use epoll(), as both select() and poll() are doing all
fd registration on _every_ call, so when #fd grows this preparation
time grows too. For epoll() file descriptors are registered only once.
For this to work asyncore.socket_map has to be patched also, since there
are places in code which modify this fd registry directly (e.g. remove
fd from there etc)
Original patch & discussion: https://lab.nexedi.com/kirr/Zope/commit/c6addb05
---
Lib/asyncore.py | 9 +++++++++
1 file changed, 9 insertions(+)
diff --git a/Lib/asyncore.py b/Lib/asyncore.py
index 105982f..a3025a4 100644
--- a/Lib/asyncore.py
+++ b/Lib/asyncore.py
@@ -661,3 +661,12 @@ if os.name == 'posix':
self.socket = file_wrapper(fd)
self._fileno = self.socket.fileno()
self.add_channel()
+
+# monkeypatch to switch everyone's default to use poll instead of select
+# (select has 1024 fd limit)
+# TODO it is better to use epoll
+_poll_select = poll
+_poll_poll = poll2
+poll = _poll_poll
+
+#print >>sys.stderr, 'I(nxd): asyncore: using poll instead of select'
--
2.16.1.101.gde0f0111ea
component/python-2.7/buildout.cfg
View file @ 342b5393
......@@ -43,6 +43,7 @@ patches =
${:_profile_base_location_}/fix_compiler_module_issue_20613.patch#94443a77f903e9de880a029967fa6aa7
${:_profile_base_location_}/pytracemalloc_pep445.patch#46662cf0ccc7cb7cfb8289bbfd68b21a
${:_profile_base_location_}/disabled_module_list.patch#71ad30d32bcdbc50c19cf48675b1246e
${:_profile_base_location_}/asyncore_poll_insteadof_select.patch#ab6991c0ee6e25aeb8951e71f280a2f1
url =
http://www.python.org/ftp/python/${:package_version}/Python-${:package_version}${:package_version_suffix}.tar.xz
configure-options =
......
component/qemu-kvm/buildout.cfg
View file @ 342b5393
......@@ -19,12 +19,11 @@ extends =
[kvm]
recipe = slapos.recipe.cmmi
# qemu-kvm and qemu are now the same since 1.3.
url = http://wiki.qemu-project.org/download/qemu-2.8.0.tar.bz2
md5sum = 17940dce063b6ce450a12e719a6c9c43
url = http://wiki.qemu-project.org/download/qemu-2.11.0.tar.bz2
md5sum = 335994a755bc655e88a87aeb36bfc0b9
configure-options =
--target-list="$(uname -m 2>/dev/null|sed 's,^i[456]86$,i386,')-softmmu"
--enable-system
--with-system-pixman
--disable-sdl
--disable-xen
--disable-vnc-sasl
......@@ -58,25 +57,26 @@ location = ${buildout:parts-directory}/${:_buildout_section_name_}
<= debian-netinst-base
arch = amd64
[debian-amd64-squeeze-netinst.iso]
<= debian-amd64-netinst-base
version = 6.0.10
md5sum = 7f82d341561035f65933da43f94d5b52
[debian-amd64-wheezy-netinst.iso]
<= debian-amd64-netinst-base
version = 7.11.0
md5sum = 096c1c18b44c269808bd815d58c53c8f
[debian-amd64-jessie-netinst.iso]
<= debian-amd64-netinst-base
release = archive
version = 8.10.0
md5sum = 19dcfc381bd3e609c6056216d203f5bc
[debian-amd64-netinst.iso]
# Download the installer of Debian 8 (Jessie)
# Download the installer of Debian 9 (Stretch)
<= debian-amd64-netinst-base
release = release/current
version = 8.6.0
md5sum = e9f61bf327db6d8f7cee05a99f2353cc
version = 9.3.0
md5sum = db8ab7871bc2b7d456c4746e706fb5d3
[debian-amd64-testing-netinst.iso]
# Download the installer of Debian Stretch
# Download the installer of Debian Buster
<= debian-amd64-netinst-base
release = stretch_di_alpha7
version = stretch-DI-alpha7
......
component/randomsleep/buildout.cfg 0 → 100644
View file @ 342b5393
[buildout]
extends =
../coreutils/buildout.cfg
../bash/buildout.cfg
[randomsleep]
recipe = slapos.recipe.build
location = ${buildout:parts-directory}/${:_buildout_section_name_}
bin_dir = ${buildout:bin-directory}
bash_script_code =
if [ "$#" -ne 1 ]; then
echo "usage: randomsleep maxseconds"
exit
fi
${coreutils:location}/bin/sleep $((RANDOM % $1)).$((RANDOM % 100))
wrapper_script_code =
#!${bash:location}/bin/bash
${bash:location}/bin/bash ${:location}/randomsleep.bash "$@"
script =
os.makedirs(self.options['location'])
bash_script_path = os.path.join(self.options['location'], 'randomsleep.bash')
with open(bash_script_path, 'w') as f:
f.write(self.options['bash_script_code'])
wrapper_script_path = os.path.join(self.options['bin_dir'], 'randomsleep')
with open(wrapper_script_path, 'w') as f:
f.write(self.options['wrapper_script_code'])
os.chmod(wrapper_script_path, 0750)
component/rsync/buildout.cfg
View file @ 342b5393
......@@ -7,8 +7,8 @@ parts =
[rsync]
recipe = slapos.recipe.cmmi
url = http://rsync.samba.org/ftp/rsync/src/rsync-3.1.1.tar.gz
md5sum = 43bd6676f0b404326eee2d63be3cdcfe
url = https://download.samba.org/pub/rsync/rsync-3.1.2.tar.gz
md5sum = 0f758d7e000c0f7f7d3792610fad70cb
make-options =
PREFIX=${buildout:parts-directory}/${:_buildout_section_name_}
environment =
......
component/tensorboard/0.4.patch → component/tensorboard/0.4.0.patch
View file @ 342b5393
diff --git a/tensorboard/pip_package/build_pip_package.sh b/tensorboard/pip_package/build_pip_package.sh
index b386d59..f03b056 100755
index 88fb5a2..782dbc5 100755
--- a/tensorboard/pip_package/build_pip_package.sh
+++ b/tensorboard/pip_package/build_pip_package.sh
@@ -26,6 +26,7 @@ function main() {
......@@ -10,14 +10,14 @@ index b386d59..f03b056 100755
echo $(date) : "=== Using tmpdir: ${TMPDIR}"
@@ -45,8 +46,8 @@ function main() {
@@ -44,9 +45,7 @@ function main() {
pushd ${TMPDIR} >/dev/null
rm -f MANIFEST
echo $(date) : "=== Building wheel"
echo $(pwd)
- python setup.py bdist_wheel >/dev/null
- python3 setup.py bdist_wheel >/dev/null
echo $(date) : "=== Building python2 wheel in $PWD"
- python setup.py bdist_wheel --python-tag py2 >/dev/null
- echo $(date) : "=== Building python3 wheel in $PWD"
- python setup.py bdist_wheel --python-tag py3 >/dev/null
+ PYTHONPATH=${WORKDIR}/${RUNFILES} $PYTHON_BIN_PATH setup.py bdist_egg >/dev/null
+ #python3 setup.py bdist_wheel >/dev/null
mkdir -p ${DEST}
cp dist/* ${DEST}
popd
popd >/dev/null
component/tensorboard/buildout.cfg
View file @ 342b5393
......@@ -11,11 +11,11 @@ parts =
recipe = plone.recipe.command
stop-on-error = true
repository = https://github.com/tensorflow/tensorboard
tag = 0.4
tag = 0.4.0
git-binary = ${git:location}/bin/git
patch-binary = ${patch:location}/bin/patch
location = ${buildout:parts-directory}/${:_buildout_section_name_}
command = export HOME=${:location}; (${:git-binary} clone --quiet -b ${:tag} ${:repository} ${:location}; cd ${buildout:parts-directory} ; ${:patch-binary} -p1 -d ${:_buildout_section_name_} < ${:_profile_base_location_}/0.4.patch ) || (rm -fr ${:location}; exit 1)
command = export HOME=${:location}; (${:git-binary} clone --quiet -b ${:tag} ${:repository} ${:location}; cd ${buildout:parts-directory} ; ${:patch-binary} -p1 -d ${:_buildout_section_name_} < ${:_profile_base_location_}/0.4.0.patch ) || (rm -fr ${:location}; exit 1)
[tensorboard-build]
recipe = slapos.recipe.build
......@@ -61,7 +61,7 @@ numpy-python-command = ${buildout:bin-directory}/${numpy-egg:interpreter}
script =
os.makedirs(location)
workdir = self.options['tensorboard-repository-path']
egg_name = 'tensorflow_tensorboard-0.4.0rc3-py2.7.egg'
egg_name = 'tensorflow_tensorboard-0.4.0-py2.7.egg'
dist_dir = os.path.join(workdir, 'dist')
dest_dir = os.path.join(self.buildout['buildout']['eggs-directory'], egg_name)
env = {'PATH':':'.join([self.options['bazel-bin'],
......
component/util-linux/buildout.cfg
View file @ 342b5393
......@@ -5,8 +5,8 @@ extends =
[util-linux]
recipe = slapos.recipe.cmmi
url = http://www.kernel.org/pub/linux/utils/util-linux/v2.29/util-linux-2.29.tar.xz
md5sum = 07b6845f48a421ad5844aa9d58edb837
url = https://www.kernel.org/pub/linux/utils/util-linux/v2.31/util-linux-2.31.1.tar.xz
md5sum = 7733b583dcb51518944d42aa62ef19ea
configure-options =
--disable-static
--enable-libuuid
......@@ -29,7 +29,7 @@ configure-options =
--disable-rename
--disable-schedutils
--disable-setterm
--disable-libsmartcols
--enable-libsmartcols
--disable-switch_root
--disable-tinfo
--disable-tls
......
component/zbar/buildout.cfg 0 → 100644
View file @ 342b5393
[buildout]
parts=
zbar
[zbar]
recipe = slapos.recipe.cmmi
url = https://jaist.dl.sourceforge.net/project/zbar/zbar/0.10/zbar-0.10.tar.bz2
md5sum = 0fd61eb590ac1bab62a77913c8b086a5
configure-options =
--disable-video
--without-imagemagick
--without-gtk
--without-xv
--without-qt
--without-python
--without-x
--without-jpg
environment =
CFLAGS=
component/zbarlight/buildout.cfg 0 → 100644
View file @ 342b5393
[buildout]
extends =
../pillow/buildout.cfg
../zbar/buildout.cfg
parts =
zbarlight
[zbarlight]
recipe = zc.recipe.egg:custom
egg = zbarlight
include-dirs =
${zbar:location}/include
library-dirs =
${zbar:location}/lib
rpath =
${zbar:location}/lib
depend-eggs = ${pillow-python:egg}
component/zstd/buildout.cfg
View file @ 342b5393
......@@ -5,7 +5,7 @@ parts =
[zstd]
recipe = slapos.recipe.cmmi
location = ${buildout:parts-directory}/${:_buildout_section_name_}
url = https://github.com/facebook/zstd/archive/v1.3.1.tar.gz
md5sum = e849ceef2f090240f690c13fba6ca70b
url = https://github.com/facebook/zstd/archive/v1.3.3.tar.gz
md5sum = 187f8df17a75a74f78a23ea4806ac65f
configure-command = :
make-options = PREFIX=${:location}
setup.py
View file @ 342b5393
......@@ -72,7 +72,6 @@ setup(name=name,
'zc.buildout': [
'addresiliency = slapos.recipe.addresiliency:Recipe',
'accords = slapos.recipe.accords:Recipe',
'apache.zope.backend = slapos.recipe.apache_zope_backend:Recipe',
'apacheperl = slapos.recipe.apacheperl:Recipe',
'apachephp = slapos.recipe.apachephp:Recipe',
'apachephpconfigure = slapos.recipe.apachephpconfigure:Recipe',
......@@ -118,14 +117,11 @@ setup(name=name,
'generic.mysql.wrap_update_mysql = slapos.recipe.generic_mysql:WrapUpdateMySQL',
'generic.mysql.wrap_mysqld = slapos.recipe.generic_mysql:WrapMySQLd',
'generic.varnish = slapos.recipe.generic_varnish:Recipe',
'generic.zope = slapos.recipe.generic_zope:Recipe',
'generic.zope.zeo.client = slapos.recipe.generic_zope_zeo_client:Recipe',
'gitinit = slapos.recipe.gitinit:Recipe',
'haproxy = slapos.recipe.haproxy:Recipe',
'ipv4toipv6 = slapos.recipe.6tunnel:FourToSix',
'ipv6toipv4 = slapos.recipe.6tunnel:SixToFour',
'jsondump = slapos.recipe.jsondump:Recipe',
'kumofs = slapos.recipe.kumofs:Recipe',
'kvm.frontend = slapos.recipe.kvm_frontend:Recipe',
'lamp = slapos.recipe.lamp:Request',
'lamp.generic = slapos.recipe.lampgeneric:Recipe',
......@@ -136,7 +132,6 @@ setup(name=name,
'libcloudrequest = slapos.recipe.libcloudrequest:Recipe',
'logrotate = slapos.recipe.logrotate:Recipe',
'logrotate.d = slapos.recipe.logrotate:Part',
'memcached = slapos.recipe.memcached:Recipe',
'mkdirectory = slapos.recipe.mkdirectory:Recipe',
'mioga.instantiate = slapos.recipe.mioga.instantiate:Recipe',
'mydumper = slapos.recipe.mydumper:Recipe',
......@@ -196,7 +191,6 @@ setup(name=name,
'urlparse = slapos.recipe._urlparse:Recipe',
'uuid = slapos.recipe._uuid:Recipe',
'userinfo = slapos.recipe.userinfo:Recipe',
'waitfor = slapos.recipe.waitfor:Recipe',
'webchecker = slapos.recipe.web_checker:Recipe',
'wrapper = slapos.recipe.wrapper:Recipe',
'xvfb = slapos.recipe.xvfb:Recipe',
......
slapos/recipe/accords/__init__.py
View file @ 342b5393
......@@ -91,8 +91,8 @@ class Recipe(GenericSlapRecipe):
# Generate wrapper
wrapper_location = self.createPythonScript(self.options['accords-wrapper'],
'%s.accords.runAccords' % __name__,
parameter_dict)
__name__ + '.accords.runAccords',
(parameter_dict,))
path_list.append(wrapper_location)
# Generate helper for debug
......
slapos/recipe/addresiliency/__init__.py
View file @ 342b5393
......@@ -38,26 +38,18 @@ class Recipe(GenericSlapRecipe):
"""
def _install(self):
path_list = []
slap_connection = self.buildout['slap-connection']
takeover_wrapper = self.createPythonScript(
name=self.options['wrapper-takeover'],
absolute_function='slapos.recipe.addresiliency.takeover.run',
arguments={
return self.createPythonScript(
self.options['wrapper-takeover'],
__name__ + '.takeover.takeover',
kw={
'server_url': slap_connection['server-url'],
'key_file': slap_connection.get('key-file'),
'cert_file': slap_connection.get('cert-file'),
'computer_id': slap_connection['computer-id'],
'computer_guid': slap_connection['computer-id'],
'partition_id': slap_connection['partition-id'],
'software': slap_connection['software-release-url'],
'software_release': slap_connection['software-release-url'],
'namebase': self.parameter_dict['namebase'],
'takeover_triggered_file_path': self.options['takeover-triggered-file-path'],
})
path_list.append(takeover_wrapper)
return path_list
slapos/recipe/addresiliency/takeover.py
View file @ 342b5393
......@@ -78,14 +78,3 @@ def takeover(server_url, key_file, cert_file, computer_guid,
# Create "lock" file preventing equeue to run import scripts
# XXX hardcoded
open(takeover_triggered_file_path, 'w').write('')
def run(args):
slapos.recipe.addresiliency.takeover.takeover(server_url = args.pop('server_url'),
key_file = args.pop('key_file'),
cert_file = args.pop('cert_file'),
computer_guid = args.pop('computer_id'),
partition_id = args.pop('partition_id'),
software_release = args.pop('software'),
namebase = args.pop('namebase'),
takeover_triggered_file_path = args.pop('takeover_triggered_file_path'))
slapos/recipe/apache_zope_backend/__init__.py deleted 100644 → 0
View file @ c44e2f89
##############################################################################
#
# Copyright (c) 2011 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from slapos.recipe.librecipe import GenericBaseRecipe
import pkg_resources
class Recipe(GenericBaseRecipe):
def install(self):
try:
backend_list = self.options['backend-list']
except KeyError:
backend_list = [(self.options['port'], self.options['backend'])]
scheme = self.options['scheme']
if scheme == 'http':
required_path_list = []
ssl_enable = ssl_snippet = ''
elif scheme == 'https':
key = self.options['key-file']
certificate = self.options['cert-file']
required_path_list = [key, certificate]
ssl_snippet = self.substituteTemplate(self.getTemplateFilename('snippet.ssl.in'), {
'key': key,
'certificate': certificate,
'ssl_session_cache': self.options['ssl-session-cache'],
})
if 'ssl-authentication' in self.options and self.optionIsTrue(
'ssl-authentication'):
ssl_snippet += self.substituteTemplate(self.getTemplateFilename('snippet.ssl.ca.in'), {
'ca_certificate': self.options['ssl-authentication-certificate'],
'ca_crl': self.options['ssl-authentication-crl'],
})
ssl_enable = 'SSLEngine on'
else:
raise ValueError('Unsupported scheme %s' % scheme)
ip_list = self.options['ip']
if isinstance(ip_list, basestring):
ip_list = [ip_list]
backend_path = self.options.get('backend-path', '/')
vhost_template_name = self.getTemplateFilename('vhost.in')
apache_config_file = self.createFile(
self.options['configuration-file'],
self.substituteTemplate(
self.getTemplateFilename('apache.zope.conf.in'),
{
'path': '/',
'server_admin': 'admin@',
'pid_file': self.options['pid-file'],
'lock_file': self.options['lock-file'],
'error_log': self.options['error-log'],
'access_log': self.options['access-log'],
'access_control_string': self.options['access-control-string'],
'ssl_snippet': ssl_snippet,
'vhosts': ''.join(self.substituteTemplate(vhost_template_name, {
'ip': ip,
'port': port,
'backend': ('%s/%s' % (backend.rstrip('/'), backend_path.strip('/'))).rstrip('/'),
'ssl_enable': ssl_enable,
}) for (port, backend) in backend_list for ip in ip_list),
},
)
)
return [
apache_config_file,
self.createPythonScript(
self.options['wrapper'],
__name__ + '.apache.runApache',
[
{
'required_path_list': required_path_list,
'binary': self.options['apache-binary'],
'config': apache_config_file,
},
],
),
]
slapos/recipe/apache_zope_backend/apache.py deleted 100644 → 0
View file @ c44e2f89
import os
import sys
import time
def runApache(args):
sleep = 60
conf = args[0]
while True:
ready = True
for f in conf.get('required_path_list', []):
if not os.path.exists(f):
print 'File %r does not exists, sleeping for %s' % (f, sleep)
ready = False
if ready:
break
time.sleep(sleep)
apache_wrapper_list = [conf['binary'], '-f', conf['config'], '-DFOREGROUND']
apache_wrapper_list.extend(sys.argv[1:])
sys.stdout.flush()
sys.stderr.flush()
os.execl(apache_wrapper_list[0], *apache_wrapper_list)
slapos/recipe/apache_zope_backend/template/apache.zope.conf.in deleted 100644 → 0
View file @ c44e2f89
# Apache configuration file for Zope
# Automatically generated
# List of modules
LoadModule unixd_module modules/mod_unixd.so
LoadModule access_compat_module modules/mod_access_compat.so
LoadModule authz_core_module modules/mod_authz_core.so
LoadModule authz_host_module modules/mod_authz_host.so
LoadModule log_config_module modules/mod_log_config.so
LoadModule setenvif_module modules/mod_setenvif.so
LoadModule version_module modules/mod_version.so
LoadModule proxy_module modules/mod_proxy.so
LoadModule proxy_http_module modules/mod_proxy_http.so
LoadModule socache_shmcb_module modules/mod_socache_shmcb.so
LoadModule ssl_module modules/mod_ssl.so
LoadModule mime_module modules/mod_mime.so
LoadModule dav_module modules/mod_dav.so
LoadModule dav_fs_module modules/mod_dav_fs.so
LoadModule negotiation_module modules/mod_negotiation.so
LoadModule rewrite_module modules/mod_rewrite.so
LoadModule headers_module modules/mod_headers.so
# Basic server configuration
PidFile "%(pid_file)s"
ServerAdmin %(server_admin)s
TypesConfig conf/mime.types
AddType application/x-compress .Z
AddType application/x-gzip .gz .tgz
ServerTokens Prod
ServerSignature Off
TraceEnable Off
# Apache 2.4's default value (60 seconds) can be a bit too short
TimeOut 300
# As backend is trusting REMOTE_USER header unset it always
RequestHeader unset REMOTE_USER
%(ssl_snippet)s
# Log configuration
ErrorLog "%(error_log)s"
# Default apache log format with request time in microsecond at the end
LogFormat "%%h %%l %%u %%t \"%%r\" %%>s %%b \"%%{Referer}i\" \"%%{User-Agent}i\" %%D" combined
CustomLog "%(access_log)s" combined
# Directory protection
<Directory />
Options FollowSymLinks
AllowOverride None
Order deny,allow
Deny from all
</Directory>
# Path protected
<Location %(path)s>
Order Deny,Allow
Deny from all
Allow from %(access_control_string)s
</Location>
# Magic of Zope related rewrite
RewriteEngine On
%(vhosts)s
slapos/recipe/apache_zope_backend/template/snippet.ssl.ca.in deleted 100644 → 0
View file @ c44e2f89
SSLVerifyClient require
RequestHeader set REMOTE_USER %%{SSL_CLIENT_S_DN_CN}s
SSLCACertificateFile %(ca_certificate)s
SSLCARevocationPath %(ca_crl)s
slapos/recipe/apache_zope_backend/template/snippet.ssl.in deleted 100644 → 0
View file @ c44e2f89
SSLCertificateFile %(certificate)s
SSLCertificateKeyFile %(key)s
SSLRandomSeed startup builtin
SSLRandomSeed connect builtin
SSLProtocol all -SSLv2 -SSLv3
SSLCipherSuite ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:HIGH:!aNULL:!MD5
SSLHonorCipherOrder on
SSLSessionCache shmcb:%(ssl_session_cache)s(512000)
SSLProxyEngine On
slapos/recipe/apache_zope_backend/template/vhost.in deleted 100644 → 0
View file @ c44e2f89
Listen %(ip)s:%(port)s
<VirtualHost *:%(port)s>
%(ssl_enable)s
RewriteRule ^/(.*) %(backend)s/$1 [L,P]
</VirtualHost>
slapos/recipe/apacheperl/__init__.py
View file @ 342b5393
......@@ -57,10 +57,9 @@ class Recipe(GenericBaseRecipe):
)
path_list.append(httpd_conf)
wrapper = self.createPythonScript(self.options['wrapper'],
'slapos.recipe.librecipe.execute.execute',
[self.options['httpd-binary'], '-f', self.options['httpd-conf'],
'-DFOREGROUND']
wrapper = self.createWrapper(self.options['wrapper'],
(self.options['httpd-binary'], '-f', self.options['httpd-conf'],
'-DFOREGROUND'),
)
path_list.append(wrapper)
......
slapos/recipe/apachephp/__init__.py
View file @ 342b5393
......@@ -92,14 +92,13 @@ class Recipe(GenericBaseRecipe):
)
path_list.append(httpd_conf)
wrapper = self.createWrapper(name=self.options['wrapper'],
command=self.options['httpd-binary'],
parameters=[
wrapper = self.createWrapper(self.options['wrapper'],
(self.options['httpd-binary'],
'-f',
self.options['httpd-conf'],
'-DFOREGROUND'
],
environment=self.environ)
),
self.environ)
path_list.append(wrapper)
secret_key_filename = os.path.join(self.buildout['buildout']['directory'],
......
slapos/recipe/apachephpconfigure/__init__.py
View file @ 342b5393
......@@ -118,7 +118,7 @@ class Recipe(GenericBaseRecipe):
configureinstall_wrapper_path = self.createPythonScript(
self.options['configureinstall-location'],
__name__ + '.runner.executeRunner',
[argument, delete, rename, chmod, data]
(argument, delete, rename, chmod, data)
)
#TODO finish to port this and remove upper one
......
slapos/recipe/apachephpconfigure/runner.py
View file @ 342b5393
import subprocess
def executeRunner(args):
def executeRunner(arguments, delete, rename, chmod, data):
"""Start the instance configure. this may run a python script, move or/and rename
file or directory when dondition is filled. the condition may be when file exist or when an entry
exist into database.
"""
arguments, delete, rename, chmod, data = args
if delete != []:
if delete:
print "Calling lampconfigure with 'delete' arguments"
result = subprocess.Popen(arguments + delete)
result.wait()
if rename != []:
subprocess.call(arguments + delete)
if rename:
for parameters in rename:
print "Calling lampconfigure with 'rename' arguments"
result = subprocess.Popen(arguments + parameters)
result.wait()
if chmod != []:
subprocess.call(arguments + parameters)
if chmod:
print "Calling lampconfigure with 'chmod' arguments"
result = subprocess.Popen(arguments + chmod)
result.wait()
if data != []:
subprocess.call(arguments + chmod)
if data:
print "Calling lampconfigure with 'run' arguments"
print arguments + data
result = subprocess.Popen(arguments + data)
result.wait()
return
subprocess.call(arguments + data)
slapos/recipe/apacheproxy/__init__.py
View file @ 342b5393
......@@ -49,13 +49,12 @@ class Recipe(GenericBaseRecipe):
)
path_list.append(httpd_conf)
wrapper = self.createWrapper(name=self.options['wrapper'],
command=self.options['httpd-binary'],
parameters=[
wrapper = self.createWrapper(self.options['wrapper'],
(self.options['httpd-binary'],
'-f',
self.options['httpd-conf'],
'-DFOREGROUND',
])
))
path_list.append(wrapper)
......
slapos/recipe/boinc/__init__.py
View file @ 342b5393
......@@ -124,12 +124,15 @@ class Recipe(GenericBaseRecipe):
#Generate wrapper for php
wrapperphp = os.path.join(self.home, 'bin/php')
php_wrapper = self.createPythonScript(wrapperphp,
'slapos.recipe.librecipe.execute.executee',
([self.phpbin, '-c', self.phpini], os.environ)
php_wrapper = self.createWrapper(wrapperphp,
(self.phpbin, '-c', self.phpini),
)
path_list.append(php_wrapper)
mysql_dict = dict(db=self.database,
host=self.mysqlhost, port=self.mysqlport,
user=self.username, passwd=self.password)
#Generate python script for MySQL database test (starting)
file_status = os.path.join(self.home, '.boinc_config')
if os.path.exists(file_status):
......@@ -137,11 +140,7 @@ class Recipe(GenericBaseRecipe):
mysql_wrapper = self.createPythonScript(
os.path.join(self.wrapperdir, 'start_config'),
'%s.configure.checkMysql' % __name__,
dict(mysql_port=self.mysqlport, mysql_host=self.mysqlhost,
mysql_user=self.username, mysql_password=self.password,
database=self.database,
file_status=file_status, environment=environment
)
(environment, mysql_dict, file_status)
)
# Generate make project wrapper file
......@@ -164,8 +163,7 @@ class Recipe(GenericBaseRecipe):
install_wrapper = self.createPythonScript(
os.path.join(self.wrapperdir, 'make_project'),
'%s.configure.makeProject' % __name__,
dict(launch_args=launch_args, request_file=request_make_boinc,
make_sig=file_status, env=environment)
(file_status, launch_args, request_make_boinc, environment)
)
path_list.append(install_wrapper)
......@@ -197,7 +195,7 @@ class Recipe(GenericBaseRecipe):
)
start_service = self.createPythonScript(
os.path.join(self.wrapperdir, 'config_project'),
'%s.configure.services' % __name__, parameter
'%s.configure.services' % __name__, (parameter,)
)
path_list.append(start_service)
......@@ -208,14 +206,12 @@ class Recipe(GenericBaseRecipe):
os.unlink(start_boinc)
boinc_parameter = dict(service_status=service_status,
installroot=self.installroot, drop_install=drop_install,
mysql_port=self.mysqlport, mysql_host=self.mysqlhost,
mysql_user=self.username, mysql_password=self.password,
database=self.database, environment=environment,
mysql_dict=mysql_dict, environment=environment,
start_boinc=start_boinc)
start_wrapper = self.createPythonScript(os.path.join(self.wrapperdir,
'start_boinc'),
'%s.configure.restart_boinc' % __name__,
boinc_parameter
(boinc_parameter,)
)
path_list.append(start_wrapper)
......@@ -362,7 +358,7 @@ class App(GenericBaseRecipe):
)
deploy_app = self.createPythonScript(
os.path.join(wrapperdir, 'boinc_%s' % appname),
'%s.configure.deployApp' % __name__, parameter
'%s.configure.deployApp' % __name__, (parameter,)
)
path_list.append(deploy_app)
......@@ -404,17 +400,15 @@ class Client(GenericBaseRecipe):
cc_cmd = '--read_cc_config'
cmd = self.createPythonScript(cmd_wrapper,
'%s.configure.runCmd' % __name__,
dict(base_cmd=base_cmd, cc_cmd=cc_cmd, installdir=installdir,
project_url=url, key=key)
(base_cmd, cc_cmd, installdir, url, key)
)
path_list.append(cmd)
#Generate BOINC client wrapper
boinc = self.createPythonScript(boinc_wrapper,
'slapos.recipe.librecipe.execute.execute',
[boincbin, '--allow_multiple_clients', '--gui_rpc_port',
boinc = self.createWrapper(boinc_wrapper,
(boincbin, '--allow_multiple_clients', '--gui_rpc_port',
str(self.options['rpc-port']), '--allow_remote_gui_rpc',
'--dir', installdir, '--redirectio', '--check_all_logins']
'--dir', installdir, '--redirectio', '--check_all_logins'),
)
path_list.append(boinc)
......
slapos/recipe/boinc/configure.py
View file @ 342b5393
......@@ -35,27 +35,21 @@ import filecmp
from lock_file import LockFile
def checkMysql(args):
sys.path += args['environment']['PYTHONPATH'].split(':')
def checkMysql(environment, connect_kw, file_status=None):
sys.path += environment['PYTHONPATH'].split(':')
import MySQLdb
#Sleep until mysql server becomes available
while True:
try:
conn = MySQLdb.connect(host = args['mysql_host'],
user = args['mysql_user'],
port = int(args['mysql_port']),
passwd = args['mysql_password'],
db = args['database'])
conn.close()
print "Successfully connect to MySQL database... "
if args.has_key('file_status'):
writeFile(args['file_status'], "starting")
MySQLdb.connect(**connect_kw).close()
break
except Exception, ex:
print "The result is: \n" + ex.message
print "Could not connect to MySQL database... sleep for 2 secondes"
time.sleep(2)
print "Successfully connect to MySQL database... "
if file_status:
writeFile(file_status, "starting")
def checkFile(file, stime):
"""Loop until 'file' is created (exist)"""
......@@ -70,18 +64,16 @@ def checkFile(file, stime):
def restart_boinc(args):
"""Stop (if currently is running state) and start all Boinc service"""
environment = args['environment']
if args['drop_install']:
checkFile(args['service_status'], 3)
else:
checkMysql(args)
checkMysql(environment, args['mysql_dict'], args.get('file_status'))
print "Restart Boinc..."
env = os.environ
env['PATH'] = args['environment']['PATH']
env['PYTHONPATH'] = args['environment']['PYTHONPATH']
binstart = os.path.join(args['installroot'], 'bin/start')
binstop = os.path.join(args['installroot'], 'bin/stop')
os.system(binstop)
os.system(binstart)
env = os.environ.copy()
env.update(environment)
subprocess.call((os.path.join(args['installroot'], 'bin', 'stop'),), env=env)
subprocess.call((os.path.join(args['installroot'], 'bin', 'start'),), env=env)
writeFile(args['start_boinc'], "started")
print "Done."
......@@ -122,17 +114,16 @@ def startProcess(launch_args, env=None, cwd=None, stdout=subprocess.PIPE):
return False
return True
def makeProject(args):
def makeProject(make_sig, launch_args, request_file, extra_environ):
"""Run BOINC make_project script but once only"""
#Wait for DateBase initialization...
checkFile(args['make_sig'], 3)
checkFile(make_sig, 3)
print "Cheking if needed to run BOINC make_project..."
if os.path.exists(args['request_file']):
env = os.environ
env['PATH'] = args['env']['PATH']
env['PYTHONPATH'] = args['env']['PYTHONPATH']
if startProcess(args['launch_args'], env=env):
os.unlink(args['request_file'])
if os.path.exists(request_file):
env = os.environ.copy()
env.update(extra_environ)
if startProcess(launch_args, env=env):
os.unlink(request_file)
print "Finished running BOINC make_projet...Ending"
else:
print "No new request for make_project. Exiting..."
......@@ -155,9 +146,8 @@ def services(args):
return
print "execute script xadd..."
env = os.environ
env['PATH'] = args['environment']['PATH']
env['PYTHONPATH'] = args['environment']['PYTHONPATH']
env = os.environ.copy()
env.update(args['environment'])
if not startProcess([os.path.join(args['installroot'], 'bin/xadd')], env):
return
print "Update files and directories permissions..."
......@@ -212,9 +202,8 @@ def deployManagement(args):
newInstall = True
#Sleep until file .start_boinc exist (File indicate that BOINC has been started)
checkFile(args['start_boinc'], 3)
env = os.environ
env['PATH'] = args['environment']['PATH']
env['PYTHONPATH'] = args['environment']['PYTHONPATH']
env = os.environ.copy()
env.update(args['environment'])
print "setup directories..."
numversion = args['version'].replace('.', '')
......@@ -263,7 +252,7 @@ def deployManagement(args):
privateKeyFile = os.path.join(args['installroot'], 'keys/code_sign_private')
output = open(binary + '.sig', 'w')
p_sign = subprocess.Popen([sign, binary, privateKeyFile], stdout=output,
stderr=subprocess.STDOUT)
stderr=subprocess.STDOUT, env=env)
result = p_sign.communicate()[0]
if p_sign.returncode is None or p_sign.returncode != 0:
print "Failed to execute bin/sign_executable.\nThe error was: %s" % result
......@@ -290,10 +279,8 @@ def deployManagement(args):
create_wu(args, env)
print "Restart Boinc..."
binstart = os.path.join(args['installroot'], 'bin/start')
binstop = os.path.join(args['installroot'], 'bin/stop')
os.system(binstop)
os.system(binstart)
subprocess.call((os.path.join(args['installroot'], 'bin', 'stop'),), env=env)
subprocess.call((os.path.join(args['installroot'], 'bin', 'start'),), env=env)
print "Boinc Application deployment is done... writing end signal file..."
writeFile(token, str(args['wu_number']))
......@@ -315,22 +302,21 @@ def create_wu(args, env):
startProcess(launch_args, env, args['installroot'])
def runCmd(args):
def runCmd(base_cmd, cc_cmd, installdir, url, key):
"""Wait for Boinc Client started and run boinc cmd"""
client_config = os.path.join(args['installdir'], 'client_state.xml')
client_config = os.path.join(installdir, 'client_state.xml')
checkFile(client_config, 5)
time.sleep(10)
#Scan client state xml to find client ipv4 adress
host = re.search("<ip_addr>([\w\d\.:]+)</ip_addr>",
open(client_config, 'r').read()).group(1)
args['base_cmd'][2] = host + ':' + args['base_cmd'][2]
print "Run boinccmd with host at %s " % args['base_cmd'][2]
project_args = args['base_cmd'] + ['--project_attach', args['project_url'],
args['key']]
startProcess(project_args, cwd=args['installdir'])
if args['cc_cmd'] != '':
base_cmd[2] = host + ':' + base_cmd[2]
print "Run boinccmd with host at %s " % base_cmd[2]
project_args = base_cmd + ['--project_attach', url, key]
startProcess(project_args, cwd=installdir)
if cc_cmd:
#Load or reload cc_config file
startProcess(args['base_cmd'] + [args['cc_cmd']], cwd=args['installdir'])
startProcess(base_cmd + [cc_cmd], cwd=installdir)
def writeFile(file, content):
......
slapos/recipe/bonjourgrid/__init__.py
View file @ 342b5393
......@@ -62,8 +62,8 @@ class Recipe(GenericBaseRecipe):
condor_wrapper_list=condor_wrapper_list,
boinc_wrapper_list=boinc_wrapper_list)
bonjourGrid_wrapper = self.createPythonScript(grid_wrapper,
'%s.configure.launchScript' % __name__,
parameters
__name__ + '.configure.launchScript',
(parameters,)
)
path_list.append(bonjourGrid_wrapper)
......@@ -73,16 +73,15 @@ class Recipe(GenericBaseRecipe):
bg_wrapper = self.options['wrapper'].strip()
log = self.options['log_file'].strip()
pid_file = self.options['pid_file'].strip()
wrapper = self.createPythonScript(bg_wrapper,
'slapos.recipe.librecipe.execute.execute',
([python, bonjourgrid_master, '--log_file', log,
wrapper = self.createWrapper(bg_wrapper,
(python, bonjourgrid_master, '--log_file', log,
'--pid_file', pid_file,
'--master_wrapper', grid_wrapper,
'--directory', self.options['work_dir'].strip(),
'--server', self.options['redis-url'].strip(),
'--port', self.options['redis-port'].strip(),
'--num_workers', self.options['nworkers'].strip(),
])
),
)
path_list.append(wrapper)
......@@ -113,9 +112,8 @@ class Client(GenericBaseRecipe):
bg_wrapper = self.options['wrapper'].strip()
log = self.options['log_file'].strip()
pid_file = self.options['pid_file'].strip()
wrapper = self.createPythonScript(bg_wrapper,
'slapos.recipe.librecipe.execute.execute',
([python, bonjourgrid_client, '--log_file', log,
wrapper = self.createWrapper(bg_wrapper,
(python, bonjourgrid_client, '--log_file', log,
'--pid_file', pid_file,
'--boinc_wrapper', boinc_script,
'--condor_wrapper', condor_script,
......@@ -123,7 +121,7 @@ class Client(GenericBaseRecipe):
'--install_directory', self.options['install_dir'].strip(),
'--server', self.options['redis-url'].strip(),
'--port', self.options['redis-port'].strip(),
])
),
)
path_list.append(wrapper)
......
slapos/recipe/certificate_authority/__init__.py
View file @ 342b5393
......@@ -40,13 +40,10 @@ class Recipe(GenericBaseRecipe):
self.ca_private = self.options['ca-private']
self.ca_certs = self.options['ca-certs']
self.ca_newcerts = self.options['ca-newcerts']
self.ca_crl = self.options['ca-crl']
self.ca_key_ext = '.key'
self.ca_crt_ext = '.crt'
def install(self):
path_list = []
ca_country_code = self.options.get('country-code', 'XX')
ca_email = self.options.get('email', 'xx@example.com')
# XXX-BBB: State by mistake has been configured as string "('State',)"
......@@ -77,21 +74,15 @@ class Recipe(GenericBaseRecipe):
self.createFile(openssl_configuration, self.substituteTemplate(
self.getTemplateFilename('openssl.cnf.ca.in'), config))
ca_wrapper = self.createPythonScript(
return self.createPythonScript(
self.options['wrapper'],
'%s.certificate_authority.runCertificateAuthority' % __name__,
dict(
openssl_configuration=openssl_configuration,
openssl_binary=self.options['openssl-binary'],
certificate=os.path.join(self.ca_dir, 'cacert.pem'),
key=os.path.join(self.ca_private, 'cakey.pem'),
crl=self.ca_crl,
request_dir=self.request_directory
)
__name__ + '.certificate_authority.runCertificateAuthority',
(os.path.join(self.ca_private, 'cakey.pem'),
os.path.join(self.ca_dir, 'cacert.pem'),
self.options['openssl-binary'],
openssl_configuration,
self.request_directory)
)
path_list.append(ca_wrapper)
return path_list
class Request(Recipe):
......@@ -146,11 +137,10 @@ class Request(Recipe):
path_list = [key_file, cert_file]
if request_needed:
wrapper = self.createPythonScript(
wrapper = self.createWrapper(
self.options['wrapper'],
'slapos.recipe.librecipe.execute.execute_wait',
[ [self.options['executable']],
[certificate, key] ],
(self.options['executable'],),
wait_list=(certificate, key),
)
path_list.append(wrapper)
......
slapos/recipe/certificate_authority/certificate_authority.py
View file @ 342b5393
......@@ -102,10 +102,8 @@ class CertificateAuthority:
'certificate_file')):
print 'Created certificate %r' % parser.get('certificate', 'name')
def runCertificateAuthority(ca_conf):
ca = CertificateAuthority(ca_conf['key'], ca_conf['certificate'],
ca_conf['openssl_binary'], ca_conf['openssl_configuration'],
ca_conf['request_dir'])
def runCertificateAuthority(*args):
ca = CertificateAuthority(*args)
while True:
ca.checkAuthority()
ca.checkRequestDir()
......
slapos/recipe/cloud9/__init__.py
View file @ 342b5393
......@@ -42,8 +42,6 @@ class Recipe(GenericBaseRecipe):
options['access-url'] = 'http://[%s]:%s' % (self.ip, self.port)
def install(self):
path_list = []
environment = {
'PATH': os.path.dirname(self.git) + ':' + os.environ['PATH'],
}
......@@ -51,10 +49,4 @@ class Recipe(GenericBaseRecipe):
cloud9_args = [self.node_executable, self.cloud9, '-l', self.ip, '-p',
self.port, '-w', self.workdir]
wrapper = self.createPythonScript(self.wrapper,
'slapos.recipe.librecipe.execute.executee',
(cloud9_args, environment)
)
path_list.append(wrapper)
return path_list
return self.createWrapper(self.wrapper, cloud9_args, environment)
slapos/recipe/condor/__init__.py
View file @ 342b5393
......@@ -178,12 +178,11 @@ class Recipe(GenericBaseRecipe):
os.chmod(wrapper_location, 0744)
#generate script for start condor
start_condor = os.path.join(self.wrapperdir, 'start_condor')
start_bin = os.path.join(self.wrapper_sbin, 'condor_master')
condor_reconfig = os.path.join(self.wrapper_sbin, 'condor_reconfig')
wrapper = self.createPythonScript(start_condor,
'%s.configure.condorStart' % __name__,
dict(start_bin=start_bin, condor_reconfig=condor_reconfig)
wrapper = self.createPythonScript(
os.path.join(self.wrapperdir, 'start_condor'),
__name__ + '.configure.condorStart',
(os.path.join(self.wrapper_sbin, 'condor_reconfig'),
os.path.join(self.wrapper_sbin, 'condor_master'))
)
path_list.append(wrapper)
return path_list
......@@ -276,13 +275,11 @@ class AppSubmit(GenericBaseRecipe):
os.unlink(destination)
os.symlink(app_list[appname]['files'][file], destination)
#generate wrapper for submitting job
condor_submit = os.path.join(self.options['bin'].strip(), 'condor_submit')
parameter = dict(submit=condor_submit, sig_install=sig_install,
submit_file='submit',
appname=appname, appdir=appdir)
submit_job = self.createPythonScript(
os.path.join(self.options['wrapper-dir'].strip(), appname),
'%s.configure.submitJob' % __name__, parameter
__name__ + '.configure.submitJob',
(os.path.join(self.options['bin'].strip(), 'condor_submit'),
'submit', appdir, appname, sig_install)
)
path_list.append(submit_job)
return path_list
\ No newline at end of file
slapos/recipe/condor/configure.py
View file @ 342b5393
......@@ -29,27 +29,25 @@ import os
import subprocess
import time
def submitJob(args):
def submitJob(submit, submit_file, appdir, appname, sig_install):
"""Run condor_submit (if needed) for job deployment"""
time.sleep(10)
print "Check if needed to submit %s job's" % args['appname']
if not os.path.exists(args['sig_install']):
print "Check if needed to submit %s job's" % appname
if not os.path.exists(sig_install):
print "Nothing for install or update...Exited"
return
# '-a', "log = out.log", '-a', "error = error.log",
launch_args = [args['submit'], '-verbose', args['submit_file']]
launch_args = submit, '-verbose', submit_file
process = subprocess.Popen(launch_args, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, cwd=args['appdir'])
stderr=subprocess.STDOUT, cwd=appdir)
result = process.communicate()[0]
if process.returncode is None or process.returncode != 0:
print "Failed to execute condor_submit.\nThe error was: %s" % result
else:
os.unlink(args['sig_install'])
os.unlink(sig_install)
def condorStart(args):
def condorStart(condor_reconfig, start_bin):
"""Start Condor if deamons is currently stopped"""
result = os.system(args['condor_reconfig'])
if result != 0:
if subprocess.call(condor_reconfig):
#process failled to reconfig condor that mean that condor deamons is not curently started
os.system(args['start_bin'])
\ No newline at end of file
subprocess.call(start_bin)
slapos/recipe/davstorage/__init__.py
View file @ 342b5393
......@@ -98,26 +98,19 @@ class Recipe(GenericBaseRecipe):
)
path_list.append(config_file)
wrapper = self.createPythonScript(self.options['wrapper'],
'slapos.recipe.librecipe.execute.execute',
[self.options['apache-binary'], '-f', config_file, '-DFOREGROUND'])
wrapper = self.createWrapper(self.options['wrapper'],
(self.options['apache-binary'], '-f', config_file, '-DFOREGROUND'))
path_list.append(wrapper)
promise = self.createPythonScript(self.options['promise'],
__name__ + '.promise',
dict(host=self.options['ip'], port=int(self.options['port_webdav']),
user=self.options['user'], password=self.options['password'])
)
(self.options['ip'], int(self.options['port_webdav']),
self.options['user'], self.options['password']))
path_list.append(promise)
return path_list
def promise(args):
host = args['host']
port = args['port']
user = args['user']
password = args['password']
def promise(host, port, user, password):
connection = httplib.HTTPSConnection(host, port)
auth = base64.b64encode('%s:%s' % (user, password))
connection.request('OPTIONS', '/',
......
slapos/recipe/dcron.py
View file @ 342b5393
......@@ -35,15 +35,14 @@ class Recipe(GenericBaseRecipe):
self.logger.info("Installing dcron...")
options = self.options
script = self.createWrapper(name=options['binary'],
command=options['dcrond-binary'].strip(),
parameters=[
script = self.createWrapper(options['binary'],
(options['dcrond-binary'].strip(),
'-s', options['cron-entries'],
'-c', options['crontabs'],
'-t', options['cronstamps'],
'-f', '-l', '5',
'-M', options['catcher']
])
))
self.logger.debug('Main cron executable created at : %r', script)
......
slapos/recipe/dropbear.py
View file @ 342b5393
......@@ -57,8 +57,6 @@ class KnownHostsFile(dict):
class Recipe(GenericBaseRecipe):
def install(self):
path_list = []
dropbear_cmd = [self.options['dropbear-binary']]
# Don't fork into background
dropbear_cmd.append('-F')
......@@ -95,19 +93,12 @@ class Recipe(GenericBaseRecipe):
if 'shell' in self.options:
env['DROPBEAR_OVERRIDE_SHELL'] = self.options['shell']
wrapper = self.createPythonScript(
self.options['wrapper'],
'slapos.recipe.librecipe.execute.executee',
(dropbear_cmd, env, )
)
path_list.append(wrapper)
return path_list
return self.createWrapper(self.options['wrapper'], dropbear_cmd, env)
class Client(GenericBaseRecipe):
def install(self):
env = dict()
env = {}
if 'home' in self.options:
env['HOME'] = self.options['home']
......@@ -120,13 +111,7 @@ class Client(GenericBaseRecipe):
if 'identity-file' in self.options:
dropbear_cmd.extend(['-i', self.options['identity-file']])
wrapper = self.createPythonScript(
self.options['wrapper'],
'slapos.recipe.librecipe.execute.executee',
(dropbear_cmd, env, )
)
return [wrapper]
return self.createWrapper(self.options['wrapper'], dropbear_cmd, env)
class AddAuthorizedKey(GenericBaseRecipe):
......
slapos/recipe/duplicity.py
View file @ 342b5393
......@@ -46,7 +46,4 @@ class Recipe(GenericBaseRecipe):
cmd.extend(options)
cmd.extend([backup_directory, remote_url])
wrapper = self.createPythonScript(self.options['wrapper'],
'slapos.recipe.librecipe.execute.execute', cmd)
return [wrapper]
return self.createWrapper(self.options['wrapper'], cmd)
slapos/recipe/equeue.py
View file @ 342b5393
......@@ -30,23 +30,20 @@ class Recipe(GenericBaseRecipe):
def install(self):
parameters = [
args = [
self.options['equeue-binary'],
'--database', self.options['database'],
'--logfile', self.options['log'],
'--lockfile', self.options['lockfile']
]
if 'takeover-triggered-file-path' in self.options:
parameters.extend(['--takeover-triggered-file-path', self.options['takeover-triggered-file-path']])
args += ('--takeover-triggered-file-path',
self.options['takeover-triggered-file-path'])
if 'loglevel' in self.options:
parameters.extend(['--loglevel', self.options['loglevel']])
args += '--loglevel', self.options['loglevel']
parameters.append(self.options['socket'])
wrapper = self.createWrapper(name=self.options['wrapper'],
command=self.options['equeue-binary'],
parameters=parameters)
return [wrapper]
args.append(self.options['socket'])
return self.createWrapper(self.options['wrapper'], args)
slapos/recipe/erp5_test/__init__.py
View file @ 342b5393
......@@ -67,7 +67,7 @@ class Recipe(GenericBaseRecipe):
openssl_binary=self.options['openssl-binary'],
test_ca_path=self.options['certificate-authority-path'],
)
common_list = [
common_list = (
'--conversion_server_url=' + cloudooo_url,
# BBB: We still have test suites that only accept the following 2 options.
'--conversion_server_hostname=%s' % cloudooo_parsed.hostname,
......@@ -76,19 +76,19 @@ class Recipe(GenericBaseRecipe):
'--volatile_memcached_server_port=%s' % memcached_parsed.port,
'--persistent_memcached_server_hostname=%s' % kumofs_parsed.hostname,
'--persistent_memcached_server_port=%s' % kumofs_parsed.port,
]
)
path_list.append(self.createPythonScript(self.options['run-unit-test'],
__name__ + '.test.runUnitTest', [dict(
call_list=[self.options['run-unit-test-binary'],
__name__ + '.test.runUnitTest',
((self.options['run-unit-test-binary'],
'--erp5_sql_connection_string', mysql_connection_string,
'--extra_sql_connection_string_list', ','.join(
mysql_connection_string_list),
] + common_list, **common_dict)]))
) + common_list, common_dict)))
path_list.append(self.createPythonScript(self.options['run-test-suite'],
__name__ + '.test.runUnitTest', [dict(
call_list=[self.options['run-test-suite-binary'],
__name__ + '.test.runTestSuite',
((self.options['run-test-suite-binary'],
'--db_list', ','.join(mysql_connection_string_list),
] + common_list, **common_dict)]))
) + common_list, common_dict)))
return path_list
......@@ -98,20 +98,18 @@ class CloudoooRecipe(GenericBaseRecipe):
common_dict = dict(
prepend_path=self.options['prepend-path'],
)
common_list = [
common_list = (
"--paster_path", self.options['ooo-paster'],
self.options['configuration-file']
]
run_unit_test_path = self.createPythonScript(self.options['run-unit-test'],
__name__ + '.test.runUnitTest', [dict(
call_list=[self.options['run-unit-test-binary'],
] + common_list, **common_dict)])
path_list.append(run_unit_test_path)
)
path_list.append(self.createPythonScript(self.options['run-unit-test'],
__name__ + '.test.runUnitTest',
((self.options['run-unit-test-binary'],
) + common_list, common_dict)))
path_list.append(self.createPythonScript(self.options['run-test-suite'],
__name__ + '.test.runTestSuite', [dict(
call_list=[self.options['run-test-suite-binary'],
], **common_dict)]))
__name__ + '.test.runTestSuite',
((self.options['run-test-suite-binary'],
), common_dict)))
return path_list
......@@ -121,32 +119,20 @@ class EggTestRecipe(GenericBaseRecipe):
off a list of Python eggs.
"""
def install(self):
path_list = []
test_list = self.options['test-list'].strip().replace('\n', ',')
common_dict = {}
environment_dict = {}
common_dict = {}
if self.options.get('environment'):
environment_part = self.buildout.get(self.options['environment'])
if environment_part:
for key, value in environment_part.iteritems():
environment_dict[key] = value
common_dict['environment'] = dict(environment_part)
common_list = [ "--source_code_path_list", test_list]
argument_dict = dict(
call_list=[self.options['run-test-suite-binary'],] + common_list,
environment=environment_dict,
**common_dict
)
if 'prepend-path' in self.options:
argument_dict['prepend_path'] = self.options['prepend-path']
common_dict['prepend_path'] = self.options['prepend-path']
run_test_suite_script = self.createPythonScript(
return self.createPythonScript(
self.options['run-test-suite'], __name__ + '.test.runTestSuite',
[argument_dict]
((self.options['run-test-suite-binary'],
"--source_code_path_list", test_list),
common_dict)
)
path_list.append(run_test_suite_script)
return path_list
slapos/recipe/erp5_test/test.py
View file @ 342b5393
......@@ -26,15 +26,18 @@
##############################################################################
import os
import sys
def runTestSuite(args):
def runTestSuite(args, d):
env = os.environ.copy()
d = args[0]
if 'openssl_binary' in d:
env['OPENSSL_BINARY'] = d['openssl_binary']
if 'test_ca_path' in d:
env['TEST_CA_PATH'] = d['test_ca_path']
if 'prepend_path' in d:
env['PATH'] = ':'.join([d['prepend_path']] + os.environ.get('PATH', '').split(':'))
try:
env['PATH'] = d['prepend_path'] + ':' + env['PATH']
except KeyError:
env['PATH'] = d['prepend_path']
if 'instance_home' in d:
env['INSTANCE_HOME'] = d['instance_home']
env['REAL_INSTANCE_HOME'] = d['instance_home']
......@@ -44,42 +47,15 @@ def runTestSuite(args):
env.update(d['environment'])
# Deal with Shebang size limitation
executable_filepath = d['call_list'][0]
file_object = open(executable_filepath, 'r')
line = file_object.readline()
file_object.close()
argument_list = []
if line[:2] == '#!':
executable_filepath = line[2:].strip()
argument_list.append(executable_filepath)
argument_list.extend(d['call_list'])
argument_list.extend(sys.argv[1:])
argument_list.append(env)
os.execle(executable_filepath, *argument_list)
def runUnitTest(args):
env = os.environ.copy()
d = args[0]
if 'openssl_binary' in d:
env['OPENSSL_BINARY'] = d['openssl_binary']
if 'test_ca_path' in d:
env['TEST_CA_PATH'] = d['test_ca_path']
if 'prepend_path' in d:
env['PATH'] = ':'.join([d['prepend_path']] + os.environ.get('PATH', '').split(':'))
if 'instance_home' in d:
env['INSTANCE_HOME'] = d['instance_home']
env['REAL_INSTANCE_HOME'] = d['instance_home']
# Deal with Shebang size limitation
executable_filepath = d['call_list'][0]
file_object = open(executable_filepath, 'r')
line = file_object.readline()
file_object.close()
executable_filepath = args[0]
with open(executable_filepath, 'r') as f:
line = f.readline()
argument_list = []
if line[:2] == '#!':
executable_filepath = line[2:].strip()
argument_list.append(executable_filepath)
argument_list.extend(d['call_list'])
argument_list.extend(sys.argv[1:])
argument_list.append(env)
os.execle(executable_filepath, *argument_list)
argument_list += args
argument_list += sys.argv[1:]
os.execve(executable_filepath, argument_list, env)
runUnitTest = runTestSuite
slapos/recipe/erp5testnode/__init__.py
View file @ 342b5393
......@@ -68,17 +68,14 @@ class Recipe(GenericBaseRecipe):
)
self.path_list.append(configuration_file)
self.path_list.append(
self.createPythonScript(
self.options['wrapper'],
'slapos.recipe.librecipe.execute.executee',
[ # Executable
[ self.options['testnode'], '-l', self.options['log-file'],
configuration_file],
self.createWrapper(self.options['wrapper'],
# Command
( self.options['testnode'], '-l', self.options['log-file'],
configuration_file),
# Environment
{
'GIT_SSL_NO_VERIFY': '1',
}
],
)
)
self.installApache()
......@@ -106,9 +103,8 @@ class Recipe(GenericBaseRecipe):
apache_config)
)
self.path_list.append(config_file)
wrapper = self.createPythonScript(self.options['httpd-wrapper'],
'slapos.recipe.librecipe.execute.execute',
[self.options['apache-binary'], '-f', config_file, '-DFOREGROUND'])
wrapper = self.createWrapper(self.options['httpd-wrapper'],
(self.options['apache-binary'], '-f', config_file, '-DFOREGROUND'))
self.path_list.append(wrapper)
# create empty html page to not allow listing of /
page = open(os.path.join(self.options['log-directory'], "index.html"), "w")
......
slapos/recipe/generic_cloudooo/__init__.py
View file @ 342b5393
......@@ -118,5 +118,5 @@ class Recipe(GenericBaseRecipe):
path_list.append(config_file)
path_list.append(self.createPythonScript(self.options['wrapper'],
'slapos.recipe.librecipe.execute.execute_with_signal_translation',
[self.options['ooo-paster'].strip(), 'serve', config_file]))
((self.options['ooo-paster'].strip(), 'serve', config_file),)))
return path_list
slapos/recipe/generic_mysql/__init__.py
View file @ 342b5393
......@@ -154,7 +154,7 @@ class Recipe(GenericBaseRecipe):
)]
)
path_list.append(mysqld)
environment = dict(PATH='%s' % self.options['bin-directory'])
environment = {'PATH': self.options['bin-directory']}
# TODO: move to a separate recipe (ack'ed by Cedric)
if 'backup-script' in self.options:
# backup configuration
......@@ -165,9 +165,13 @@ class Recipe(GenericBaseRecipe):
'--defaults-file=%s' % mysql_conf_file,
'--socket=%s' % socket.strip(), '--user=root',
'--ibbackup=%s'% self.options['xtrabackup-binary']]
innobackupex_incremental = self.createPythonScript(self.options['innobackupex-incremental'], 'slapos.recipe.librecipe.execute.executee', [innobackupex_argument_list + ['--incremental'], environment])
innobackupex_incremental = self.createWrapper(
self.options['innobackupex-incremental'],
innobackupex_argument_list + ['--incremental'], environment)
path_list.append(innobackupex_incremental)
innobackupex_full = self.createPythonScript(self.options['innobackupex-full'], 'slapos.recipe.librecipe.execute.executee', [innobackupex_argument_list, environment])
innobackupex_full = self.createWrapper(
self.options['innobackupex-full'],
innobackupex_argument_list, environment)
path_list.append(innobackupex_full)
backup_controller = self.createPythonScript(self.options['backup-script'], __name__ + '.innobackupex.controller', [innobackupex_incremental, innobackupex_full, full_backup, incremental_backup])
path_list.append(backup_controller)
......@@ -215,7 +219,9 @@ class Recipe(GenericBaseRecipe):
'--defaults-file=%s' % mysql_conf_file,
'--socket=%s' % socket.strip(), '--user=root',
]
pt_exe = self.createPythonScript(os.path.join(self.options['bin-directory'], pt_script_name), 'slapos.recipe.librecipe.execute.executee', [pt_argument_list, environment])
pt_exe = self.createWrapper(
os.path.join(self.options['bin-directory'], pt_script_name),
pt_argument_list, environment)
path_list.append(pt_exe)
return path_list
......
slapos/recipe/generic_mysql/innobackupex.py
View file @ 342b5393
import os
import glob
def controller(args):
def controller(innobackupex_incremental, innobackupex_full,
full_backup, incremental_backup):
"""Creates full or incremental backup
If no full backup is done, it is created
......@@ -9,8 +10,6 @@ def controller(args):
base is the newest (according to date) full or incremental backup
"""
innobackupex_incremental, innobackupex_full, full_backup, incremental_backup \
= args
if len(os.listdir(full_backup)) == 0:
print 'Doing full backup in %r' % full_backup
os.execv(innobackupex_full, [innobackupex_full, full_backup])
......
slapos/recipe/generic_mysql/mysql.py
View file @ 342b5393
......@@ -5,9 +5,8 @@ import sys
import pytz
def runMysql(args):
def runMysql(conf):
sleep = 60
conf = args[0]
mysqld_wrapper_list = [conf['mysqld_binary'], '--defaults-file=%s' %
conf['configuration_file']]
# we trust mysql_install that if mysql directory is available mysql was
......@@ -54,8 +53,7 @@ def runMysql(args):
os.execl(mysqld_wrapper_list[0], *mysqld_wrapper_list)
def updateMysql(args):
conf = args[0]
def updateMysql(conf):
sleep = 30
is_succeed = False
try:
......
slapos/recipe/generic_zope/__init__.py deleted 100644 → 0
View file @ c44e2f89
##############################################################################
#
# Copyright (c) 2011 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from slapos.recipe.librecipe import GenericBaseRecipe
import binascii
import hashlib
import os
import re
import zc.buildout
_isurl = re.compile('([a-zA-Z0-9+.-]+)://').match
# based on Zope2.utilities.mkzopeinstance.write_inituser
def Zope2InitUser(path, username, password):
# Set password only once
# Currently, rely on existence of a simple file:
# Create it the first time, then next time, detect this file and do no-op.
inituser_done_path = '%s_done' % path
if os.path.exists(inituser_done_path):
return
if os.path.exists(path):
return
open(path, 'w').write('')
os.chmod(path, 0600)
open(path, 'w').write('%s:{SHA}%s\n' % (
username,binascii.b2a_base64(hashlib.sha1(password).digest())[:-1]))
open(inituser_done_path, 'w').write('"inituser" file already created once.')
class Recipe(GenericBaseRecipe):
def _options(self, options):
options['password'] = self.generatePassword()
options['deadlock-password'] = self.generatePassword()
def install(self):
"""
All zope have to share file created by portal_classes
(until everything is integrated into the ZODB).
So, do not request zope instance and create multiple in the same partition.
"""
path_list = []
Zope2InitUser(self.options['inituser'], self.options['user'],
self.options['password'])
# Symlink to BT5 repositories defined in instance config.
# Those paths will eventually end up in the ZODB, and having symlinks
# inside the XXX makes it possible to reuse such ZODB with another software
# release[ version].
# Note: this path cannot be used for development, it's really just a
# read-only repository.
repository_path = self.options['bt5-repository']
self.bt5_repository_list = []
append = self.bt5_repository_list.append
for repository in self.options.get('bt5-repository-list', '').split():
repository = repository.strip()
if not repository:
continue
if _isurl(repository) and not repository.startswith("file://"):
# XXX: assume it's a valid URL
append(repository)
continue
if repository.startswith('file://'):
repository = repository.replace('file://', '', '')
if os.path.isabs(repository):
repo_id = hashlib.sha1(repository).hexdigest()
link = os.path.join(repository_path, repo_id)
if os.path.lexists(link):
if not os.path.islink(link):
raise zc.buildout.UserError(
'Target link already %r exists but it is not link' % link)
os.unlink(link)
os.symlink(repository, link)
self.logger.debug('Created link %r -> %r' % (link, repository_path))
# Always provide a URL-Type
append("file://" + link)
# Create zope configuration file
zope_config = dict(
thread_amount=self.options['thread-amount'],
zodb_root_path=self.options['zodb-path'],
zodb_cache_size=int(self.options['zodb-cache-size']),
)
zope_environment = dict(
TMP=self.options['tmp-path'],
TMPDIR=self.options['tmp-path'],
HOME=self.options['tmp-path'],
PATH=self.options['bin-path']
)
# configure default Zope2 zcml
open(self.options['site-zcml'], 'w').write(open(self.getTemplateFilename(
'site.zcml')).read())
zope_config['instance'] = self.options['instance-path']
zope_config['event_log'] = self.options['event-log']
zope_config['z2_log'] = self.options['z2-log']
zope_config['pid-filename'] = self.options['pid-file']
zope_config['lock-filename'] = self.options['lock-file']
zope_config['products'] = 'products %s' % self.options['instance-products']
zope_config['address'] = '%s:%s' % (self.options['ip'], self.options['port'])
zope_config.update(dump_url=self.options['deadlock-path'],
secret=self.options['deadlock-password'])
zope_wrapper_template_location = self.getTemplateFilename('zope.conf.in')
zope_conf_content = self.substituteTemplate(zope_wrapper_template_location,
zope_config)
if ('promise-path' in self.options) and ('site-id' in self.options):
zope_conf_content += self.substituteTemplate(self.getTemplateFilename(
'zope.conf.promise.in'), {
'site-id': self.options['site-id'],
'promise-path': self.options['promise-path'],
})
zope_conf_path = self.createFile(self.options['configuration-file'], zope_conf_content)
path_list.append(zope_conf_path)
# Create init script
path_list.append(self.createPythonScript(self.options['wrapper'], 'slapos.recipe.librecipe.execute.executee', [[self.options['runzope-binary'].strip(), '-C', zope_conf_path], zope_environment]))
return path_list
slapos/recipe/generic_zope/template/site.zcml deleted 100644 → 0
View file @ c44e2f89
<configure
xmlns="http://namespaces.zope.org/zope"
xmlns:meta="http://namespaces.zope.org/meta"
xmlns:five="http://namespaces.zope.org/five">
<include package="Products.Five" />
<meta:redefinePermission from="zope2.Public" to="zope.Public" />
<!-- Load the meta -->
<include files="package-includes/*-meta.zcml" />
<five:loadProducts file="meta.zcml"/>
<!-- Load the configuration -->
<include files="package-includes/*-configure.zcml" />
<five:loadProducts />
<!-- Load the configuration overrides-->
<includeOverrides files="package-includes/*-overrides.zcml" />
<five:loadProductsOverrides />
<securityPolicy
component="Products.Five.security.FiveSecurityPolicy" />
</configure>
slapos/recipe/generic_zope/template/zope.conf.in deleted 100644 → 0
View file @ c44e2f89
## Zope 2 configuration file generated by SlapOS
# Some defines
%%define INSTANCE %(instance)s
instancehome $INSTANCE
# Used products
%(products)s
# Environment is setup in running wrapper script
# Reason: zope.conf is read too late for some componets
# No need to debug
debug-mode off
# One thread is safe enough
zserver-threads %(thread_amount)s
# File location
pid-filename %(pid-filename)s
lock-filename %(lock-filename)s
# Encoding
rest-input-encoding utf-8
rest-output-encoding utf-8
default-zpublisher-encoding utf-8
# Disable ownership checking to execute codes generated by alarm
skip-ownership-checking on
# Temporary storage database (for sessions)
<zodb_db temporary>
<temporarystorage>
name temporary storage for sessioning
</temporarystorage>
mount-point /temp_folder
container-class Products.TemporaryFolder.TemporaryContainer
</zodb_db>
# Logging configuration
<eventlog>
<logfile>
dateformat
path %(event_log)s
</logfile>
</eventlog>
<logger access>
<logfile>
dateformat
path %(z2_log)s
</logfile>
</logger>
# Serving configuration
<http-server>
address %(address)s
</http-server>
# ZODB configuration
<zodb_db root>
cache-size %(zodb_cache_size)d
<filestorage>
path %(zodb_root_path)s
</filestorage>
mount-point /
</zodb_db>
<zoperunner>
program $INSTANCE/bin/runzope
</zoperunner>
# DeadlockDebugger configuration
<product-config DeadlockDebugger>
dump_url %(dump_url)s
secret %(secret)s
</product-config>
# ERP5 Timer Service
%%import timerserver
<timer-server>
interval 5
</timer-server>
slapos/recipe/generic_zope/template/zope.conf.promise.in deleted 100644 → 0
View file @ c44e2f89
# ERP5 promise
<product-config /%(site-id)s>
promise_path %(promise-path)s
</product-config>
slapos/recipe/generic_zope_zeo_client/__init__.py deleted 100644 → 0
View file @ c44e2f89
##############################################################################
#
# Copyright (c) 2011 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from slapos.recipe.librecipe import GenericBaseRecipe
import binascii
import hashlib
import os
import re
import zc.buildout
_isurl = re.compile('([a-zA-Z0-9+.-]+)://').match
# based on Zope2.utilities.mkzopeinstance.write_inituser
def Zope2InitUser(path, username, password):
# Set password only once
# Currently, rely on existence of a simple file:
# Create it the first time, then next time, detect this file and do no-op.
inituser_done_path = '%s_done' % path
if os.path.exists(inituser_done_path):
return
if os.path.exists(path):
return
open(path, 'w').write('')
os.chmod(path, 0600)
open(path, 'w').write('%s:{SHA}%s\n' % (
username,binascii.b2a_base64(hashlib.sha1(password).digest())[:-1]))
open(inituser_done_path, 'w').write('"inituser" file already created once.')
class Recipe(GenericBaseRecipe):
def _options(self, options):
if 'password' not in options:
options['password'] = self.generatePassword()
def install(self):
"""
All zope have to share file created by portal_classes
(until everything is integrated into the ZODB).
So, do not request zope instance and create multiple in the same partition.
"""
path_list = []
Zope2InitUser(self.options['inituser'], self.options['user'],
self.options['password'])
# Symlink to BT5 repositories defined in instance config.
# Those paths will eventually end up in the ZODB, and having symlinks
# inside the XXX makes it possible to reuse such ZODB with another software
# release[ version].
# Note: this path cannot be used for development, it's really just a
# read-only repository.
repository_path = self.options['bt5-repository']
self.bt5_repository_list = []
append = self.bt5_repository_list.append
for repository in self.options.get('bt5-repository-list', '').split():
repository = repository.strip()
if not repository:
continue
if _isurl(repository) and not repository.startswith("file://"):
# XXX: assume it's a valid URL
append(repository)
continue
if repository.startswith('file://'):
repository = repository.replace('file://', '', '')
if os.path.isabs(repository):
repo_id = hashlib.sha1(repository).hexdigest()
link = os.path.join(repository_path, repo_id)
if os.path.lexists(link):
if not os.path.islink(link):
raise zc.buildout.UserError(
'Target link already %r exists but it is not link' % link)
os.unlink(link)
os.symlink(repository, link)
self.logger.debug('Created link %r -> %r' % (link, repository_path))
# Always provide a URL-Type
append("file://" + link)
zope_environment = {
'TMP': self.options['tmp-path'],
'TMPDIR': self.options['tmp-path'],
'HOME': self.options.get('home-path', self.options.get('tmp-path')),
'PATH': self.options['bin-path'],
'TZ': self.options['timezone'],
}
instance_home = self.options.get("instancehome-path", None)
if instance_home:
zope_environment["INSTANCE_HOME"] = instance_home
# configure default Zope2 zcml
open(self.options['site-zcml'], 'w').write(open(self.getTemplateFilename(
'site.zcml')).read())
# Create init script
path_list.append(self.createPythonScript(self.options['wrapper'], 'slapos.recipe.librecipe.execute.executee', [[self.options['runzope-binary'].strip(), '-C', self.options['configuration-file']], zope_environment]))
return path_list
slapos/recipe/generic_zope_zeo_client/template/site.zcml deleted 100644 → 0
View file @ c44e2f89
<configure
xmlns="http://namespaces.zope.org/zope"
xmlns:meta="http://namespaces.zope.org/meta"
xmlns:five="http://namespaces.zope.org/five">
<include package="Products.Five" />
<meta:redefinePermission from="zope2.Public" to="zope.Public" />
<!-- Load the meta -->
<include files="package-includes/*-meta.zcml" />
<five:loadProducts file="meta.zcml"/>
<!-- Load the configuration -->
<include files="package-includes/*-configure.zcml" />
<five:loadProducts />
<!-- Load the configuration overrides-->
<includeOverrides files="package-includes/*-overrides.zcml" />
<five:loadProductsOverrides />
<securityPolicy
component="Products.Five.security.FiveSecurityPolicy" />
</configure>
slapos/recipe/haproxy/__init__.py
View file @ 342b5393
......@@ -120,12 +120,11 @@ class Recipe(GenericBaseRecipe):
'server_text': server_snippet},
)
)
wrapper_path = self.createPythonScript(
wrapper_path = self.createWrapper(
self.options['wrapper-path'],
'slapos.recipe.librecipe.execute.execute',
arguments=[self.options['binary-path'].strip(), '-f', configuration_path],)
(self.options['binary-path'].strip(), '-f', configuration_path))
ctl_path = self.createPythonScript(
self.options['ctl-path'],
'%s.haproxy.haproxyctl' % __name__,
{'socket_path':self.options['socket-path']})
__name__ + '.haproxy.haproxyctl',
(self.options['socket-path'],))
return [configuration_path, wrapper_path, ctl_path]
slapos/recipe/haproxy/haproxy.py
View file @ 342b5393
......@@ -4,7 +4,7 @@ try:
except ImportError:
pass
def haproxyctl(conf):
def haproxyctl(socket_path):
while True:
try:
l = raw_input('> ')
......@@ -14,7 +14,7 @@ def haproxyctl(conf):
if l == 'quit':
break
s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
s.connect(conf['socket_path'])
s.connect(socket_path)
s.send('%s\n' % l)
while True:
r = s.recv(1024)
......
slapos/recipe/kumofs/__init__.py deleted 100644 → 0
View file @ c44e2f89
##############################################################################
#
# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from slapos.recipe.librecipe import BaseSlapRecipe
import hashlib
import os
import pkg_resources
import sys
import zc.buildout
import ConfigParser
class Recipe(BaseSlapRecipe):
def getTemplateFilename(self, template_name):
return pkg_resources.resource_filename(__name__,
'template/%s' % template_name)
def _install(self):
self.path_list = []
self.requirements, self.ws = self.egg.working_set()
# XXX-Cedric : add logrotate?
self.cron_d = self.installCrond()
kumo_conf = self.installKumo(self.getLocalIPv4Address())
ca_conf = self.installCertificateAuthority()
key, certificate = self.requestCertificate('Login Based Access')
stunnel_conf = self.installStunnel(self.getGlobalIPv6Address(),
self.getLocalIPv4Address(), 12345, kumo_conf['kumo_gateway_port'],
certificate, key, ca_conf['ca_crl'],
ca_conf['certificate_authority_path'])
self.linkBinary()
self.setConnectionDict(dict(
stunnel_ip = stunnel_conf['public_ip'],
stunnel_port = stunnel_conf['public_port'],
))
return self.path_list
def linkBinary(self):
"""Links binaries to instance's bin directory for easier exposal"""
for linkline in self.options.get('link_binary_list', '').splitlines():
if not linkline:
continue
target = linkline.split()
if len(target) == 1:
target = target[0]
path, linkname = os.path.split(target)
else:
linkname = target[1]
target = target[0]
link = os.path.join(self.bin_directory, linkname)
if os.path.lexists(link):
if not os.path.islink(link):
raise zc.buildout.UserError(
'Target link already %r exists but it is not link' % link)
os.unlink(link)
os.symlink(target, link)
self.logger.debug('Created link %r -> %r' % (link, target))
self.path_list.append(link)
def installCrond(self):
timestamps = self.createDataDirectory('cronstamps')
cron_output = os.path.join(self.log_directory, 'cron-output')
self._createDirectory(cron_output)
catcher = zc.buildout.easy_install.scripts([('catchcron',
__name__ + '.catdatefile', 'catdatefile')], self.ws, sys.executable,
self.bin_directory, arguments=[cron_output])[0]
self.path_list.append(catcher)
cron_d = os.path.join(self.etc_directory, 'cron.d')
crontabs = os.path.join(self.etc_directory, 'crontabs')
self._createDirectory(cron_d)
self._createDirectory(crontabs)
# Use execute from erp5.
wrapper = zc.buildout.easy_install.scripts([('crond',
'slapos.recipe.librecipe.execute', 'execute')], self.ws, sys.executable,
self.wrapper_directory, arguments=[
self.options['dcrond_binary'].strip(), '-s', cron_d, '-c', crontabs,
'-t', timestamps, '-f', '-l', '5', '-M', catcher]
)[0]
self.path_list.append(wrapper)
return cron_d
def installLogrotate(self):
"""Installs logortate main configuration file and registers its to cron"""
logrotate_d = os.path.abspath(os.path.join(self.etc_directory,
'logrotate.d'))
self._createDirectory(logrotate_d)
logrotate_backup = self.createBackupDirectory('logrotate')
logrotate_conf = self.createConfigurationFile("logrotate.conf",
"include %s" % logrotate_d)
logrotate_cron = os.path.join(self.cron_d, 'logrotate')
state_file = os.path.join(self.data_root_directory, 'logrotate.status')
open(logrotate_cron, 'w').write('0 0 * * * %s -s %s %s' %
(self.options['logrotate_binary'], state_file, logrotate_conf))
self.path_list.extend([logrotate_d, logrotate_conf, logrotate_cron])
return logrotate_d, logrotate_backup
def registerLogRotation(self, name, log_file_list, postrotate_script):
"""Register new log rotation requirement"""
open(os.path.join(self.logrotate_d, name), 'w').write(
self.substituteTemplate(self.getTemplateFilename(
'logrotate_entry.in'),
dict(file_list=' '.join(['"'+q+'"' for q in log_file_list]),
postrotate=postrotate_script, olddir=self.logrotate_backup)))
def installCertificateAuthority(self, ca_country_code='XX',
ca_email='xx@example.com', ca_state='State', ca_city='City',
ca_company='Company'):
backup_path = self.createBackupDirectory('ca')
self.ca_dir = os.path.join(self.data_root_directory, 'ca')
self._createDirectory(self.ca_dir)
self.ca_request_dir = os.path.join(self.ca_dir, 'requests')
self._createDirectory(self.ca_request_dir)
config = dict(ca_dir=self.ca_dir, request_dir=self.ca_request_dir)
self.ca_private = os.path.join(self.ca_dir, 'private')
self.ca_certs = os.path.join(self.ca_dir, 'certs')
self.ca_crl = os.path.join(self.ca_dir, 'crl')
self.ca_newcerts = os.path.join(self.ca_dir, 'newcerts')
self.ca_key_ext = '.key'
self.ca_crt_ext = '.crt'
for d in [self.ca_private, self.ca_crl, self.ca_newcerts, self.ca_certs]:
self._createDirectory(d)
for f in ['crlnumber', 'serial']:
if not os.path.exists(os.path.join(self.ca_dir, f)):
open(os.path.join(self.ca_dir, f), 'w').write('01')
if not os.path.exists(os.path.join(self.ca_dir, 'index.txt')):
open(os.path.join(self.ca_dir, 'index.txt'), 'w').write('')
openssl_configuration = os.path.join(self.ca_dir, 'openssl.cnf')
config.update(
working_directory=self.ca_dir,
country_code=ca_country_code,
state=ca_state,
city=ca_city,
company=ca_company,
email_address=ca_email,
)
self._writeFile(openssl_configuration, pkg_resources.resource_string(
__name__, 'template/openssl.cnf.ca.in') % config)
self.path_list.extend(zc.buildout.easy_install.scripts([
('certificate_authority',
__name__ + '.certificate_authority', 'runCertificateAuthority')],
self.ws, sys.executable, self.wrapper_directory, arguments=[dict(
openssl_configuration=openssl_configuration,
openssl_binary=self.options['openssl_binary'],
certificate=os.path.join(self.ca_dir, 'cacert.pem'),
key=os.path.join(self.ca_private, 'cakey.pem'),
crl=os.path.join(self.ca_crl),
request_dir=self.ca_request_dir
)]))
# configure backup
backup_cron = os.path.join(self.cron_d, 'ca_rdiff_backup')
open(backup_cron, 'w').write(
'''0 0 * * * %(rdiff_backup)s %(source)s %(destination)s'''%dict(
rdiff_backup=self.options['rdiff_backup_binary'],
source=self.ca_dir,
destination=backup_path))
self.path_list.append(backup_cron)
return dict(
ca_certificate=os.path.join(config['ca_dir'], 'cacert.pem'),
ca_crl=os.path.join(config['ca_dir'], 'crl'),
certificate_authority_path=config['ca_dir']
)
def requestCertificate(self, name):
hash = hashlib.sha512(name).hexdigest()
key = os.path.join(self.ca_private, hash + self.ca_key_ext)
certificate = os.path.join(self.ca_certs, hash + self.ca_crt_ext)
parser = ConfigParser.RawConfigParser()
parser.add_section('certificate')
parser.set('certificate', 'name', name)
parser.set('certificate', 'key_file', key)
parser.set('certificate', 'certificate_file', certificate)
parser.write(open(os.path.join(self.ca_request_dir, hash), 'w'))
return key, certificate
def installStunnel(self, public_ip, private_ip, public_port, private_port,
ca_certificate, key, ca_crl, ca_path):
"""Installs stunnel"""
template_filename = self.getTemplateFilename('stunnel.conf.in')
log = os.path.join(self.log_directory, 'stunnel.log')
pid_file = os.path.join(self.run_directory, 'stunnel.pid')
stunnel_conf = dict(
public_ip=public_ip,
private_ip=private_ip,
public_port=public_port,
pid_file=pid_file,
log=log,
cert = ca_certificate,
key = key,
ca_crl = ca_crl,
ca_path = ca_path,
private_port = private_port,
)
stunnel_conf_path = self.createConfigurationFile("stunnel.conf",
self.substituteTemplate(template_filename,
stunnel_conf))
wrapper = zc.buildout.easy_install.scripts([('stunnel',
'slapos.recipe.librecipe.execute', 'execute_wait')], self.ws,
sys.executable, self.wrapper_directory, arguments=[
[self.options['stunnel_binary'].strip(), stunnel_conf_path],
[ca_certificate, key]]
)[0]
self.path_list.append(wrapper)
return stunnel_conf
def installKumo(self, ip, kumo_manager_port=13101, kumo_server_port=13201,
kumo_server_listen_port=13202, kumo_gateway_port=13301):
# XXX: kumo is not storing pid in file, unless it is not running as daemon
# but running daemons is incompatible with SlapOS, so there is currently
# no way to have Kumo's pid files to rotate logs and send signals to them
config = dict(
kumo_gateway_binary=self.options['kumo_gateway_binary'],
kumo_gateway_ip=ip,
kumo_gateway_log=os.path.join(self.log_directory, "kumo-gateway.log"),
kumo_manager_binary=self.options['kumo_manager_binary'],
kumo_manager_ip=ip,
kumo_manager_log=os.path.join(self.log_directory, "kumo-manager.log"),
kumo_server_binary=self.options['kumo_server_binary'],
kumo_server_ip=ip,
kumo_server_log=os.path.join(self.log_directory, "kumo-server.log"),
kumo_server_storage=os.path.join(self.data_root_directory, "kumodb.tch"),
kumo_manager_port=kumo_manager_port,
kumo_server_port=kumo_server_port,
kumo_server_listen_port=kumo_server_listen_port,
kumo_gateway_port=kumo_gateway_port
)
self.path_list.append(self.createRunningWrapper('kumo_gateway',
self.substituteTemplate(self.getTemplateFilename('kumo_gateway.in'),
config)))
self.path_list.append(self.createRunningWrapper('kumo_manager',
self.substituteTemplate(self.getTemplateFilename('kumo_manager.in'),
config)))
self.path_list.append(self.createRunningWrapper('kumo_server',
self.substituteTemplate(self.getTemplateFilename('kumo_server.in'),
config)))
return dict(
kumo_address = '%s:%s' % (config['kumo_gateway_ip'],
config['kumo_gateway_port']),
kumo_gateway_ip=config['kumo_gateway_ip'],
kumo_gateway_port=config['kumo_gateway_port'],
)
slapos/recipe/kumofs/certificate_authority.py deleted 100644 → 0
View file @ c44e2f89
import os
import subprocess
import time
import ConfigParser
def popenCommunicate(command_list, input=None):
subprocess_kw = dict(stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
if input is not None:
subprocess_kw.update(stdin=subprocess.PIPE)
popen = subprocess.Popen(command_list, **subprocess_kw)
result = popen.communicate(input)[0]
if popen.returncode is None:
popen.kill()
if popen.returncode != 0:
raise ValueError('Issue during calling %r, result was:\n%s' % (
command_list, result))
return result
class CertificateAuthority:
def __init__(self, key, certificate, openssl_binary,
openssl_configuration, request_dir):
self.key = key
self.certificate = certificate
self.openssl_binary = openssl_binary
self.openssl_configuration = openssl_configuration
self.request_dir = request_dir
def checkAuthority(self):
file_list = [ self.key, self.certificate ]
ca_ready = True
for f in file_list:
if not os.path.exists(f):
ca_ready = False
break
if ca_ready:
return
for f in file_list:
if os.path.exists(f):
os.unlink(f)
try:
# no CA, let us create new one
popenCommunicate([self.openssl_binary, 'req', '-nodes', '-config',
self.openssl_configuration, '-new', '-x509', '-extensions',
'v3_ca', '-keyout', self.key, '-out', self.certificate,
'-days', '10950'], 'Automatic Certificate Authority\n')
except:
try:
for f in file_list:
if os.path.exists(f):
os.unlink(f)
except:
# do not raise during cleanup
pass
raise
def _checkCertificate(self, common_name, key, certificate):
file_list = [key, certificate]
ready = True
for f in file_list:
if not os.path.exists(f):
ready = False
break
if ready:
return False
for f in file_list:
if os.path.exists(f):
os.unlink(f)
csr = certificate + '.csr'
try:
popenCommunicate([self.openssl_binary, 'req', '-config',
self.openssl_configuration, '-nodes', '-new', '-keyout',
key, '-out', csr, '-days', '3650'],
common_name + '\n')
try:
popenCommunicate([self.openssl_binary, 'ca', '-batch', '-config',
self.openssl_configuration, '-out', certificate,
'-infiles', csr])
finally:
if os.path.exists(csr):
os.unlink(csr)
except:
try:
for f in file_list:
if os.path.exists(f):
os.unlink(f)
except:
# do not raise during cleanup
pass
raise
else:
return True
def checkRequestDir(self):
for request_file in os.listdir(self.request_dir):
parser = ConfigParser.RawConfigParser()
parser.readfp(open(os.path.join(self.request_dir, request_file), 'r'))
if self._checkCertificate(parser.get('certificate', 'name'),
parser.get('certificate', 'key_file'), parser.get('certificate',
'certificate_file')):
print 'Created certificate %r' % parser.get('certificate', 'name')
def runCertificateAuthority(args):
ca_conf = args[0]
ca = CertificateAuthority(ca_conf['key'], ca_conf['certificate'],
ca_conf['openssl_binary'], ca_conf['openssl_configuration'],
ca_conf['request_dir'])
while True:
ca.checkAuthority()
ca.checkRequestDir()
time.sleep(60)
slapos/recipe/kumofs/template/kumo_gateway.in deleted 100644 → 0
View file @ c44e2f89
#!/bin/sh
exec %(kumo_gateway_binary)s -F -E -m %(kumo_manager_ip)s:%(kumo_manager_port)s -t %(kumo_gateway_ip)s:%(kumo_gateway_port)s -o %(kumo_gateway_log)s
slapos/recipe/kumofs/template/kumo_manager.in deleted 100644 → 0
View file @ c44e2f89
#!/bin/sh
exec %(kumo_manager_binary)s -a -l %(kumo_manager_ip)s:%(kumo_manager_port)s -o %(kumo_manager_log)s
slapos/recipe/kumofs/template/kumo_server.in deleted 100644 → 0
View file @ c44e2f89
#!/bin/sh
exec %(kumo_server_binary)s -l %(kumo_server_ip)s:%(kumo_server_port)s -L %(kumo_server_listen_port)s -m %(kumo_manager_ip)s:%(kumo_manager_port)s -s %(kumo_server_storage)s -o %(kumo_server_log)s
slapos/recipe/kumofs/template/openssl.cnf.ca.in deleted 100644 → 0
View file @ c44e2f89
#
# OpenSSL example configuration file.
# This is mostly being used for generation of certificate requests.
#
# This definition stops the following lines choking if HOME isn't
# defined.
HOME = .
RANDFILE = $ENV::HOME/.rnd
# Extra OBJECT IDENTIFIER info:
#oid_file = $ENV::HOME/.oid
oid_section = new_oids
# To use this configuration file with the "-extfile" option of the
# "openssl x509" utility, name here the section containing the
# X.509v3 extensions to use:
# extensions =
# (Alternatively, use a configuration file that has only
# X.509v3 extensions in its main [= default] section.)
[ new_oids ]
# We can add new OIDs in here for use by 'ca', 'req' and 'ts'.
# Add a simple OID like this:
# testoid1=1.2.3.4
# Or use config file substitution like this:
# testoid2=${testoid1}.5.6
# Policies used by the TSA examples.
tsa_policy1 = 1.2.3.4.1
tsa_policy2 = 1.2.3.4.5.6
tsa_policy3 = 1.2.3.4.5.7
####################################################################
[ ca ]
default_ca = CA_default # The default ca section
####################################################################
[ CA_default ]
dir = %(working_directory)s # Where everything is kept
certs = $dir/certs # Where the issued certs are kept
crl_dir = $dir/crl # Where the issued crl are kept
database = $dir/index.txt # database index file.
#unique_subject = no # Set to 'no' to allow creation of
# several ctificates with same subject.
new_certs_dir = $dir/newcerts # default place for new certs.
certificate = $dir/cacert.pem # The CA certificate
serial = $dir/serial # The current serial number
crlnumber = $dir/crlnumber # the current crl number
# must be commented out to leave a V1 CRL
crl = $dir/crl.pem # The current CRL
private_key = $dir/private/cakey.pem # The private key
RANDFILE = $dir/private/.rand # private random number file
x509_extensions = usr_cert # The extentions to add to the cert
# Comment out the following two lines for the "traditional"
# (and highly broken) format.
name_opt = ca_default # Subject Name options
cert_opt = ca_default # Certificate field options
# Extension copying option: use with caution.
# copy_extensions = copy
# Extensions to add to a CRL. Note: Netscape communicator chokes on V2 CRLs
# so this is commented out by default to leave a V1 CRL.
# crlnumber must also be commented out to leave a V1 CRL.
# crl_extensions = crl_ext
default_days = 3650 # how long to certify for
default_crl_days= 30 # how long before next CRL
default_md = default # use public key default MD
preserve = no # keep passed DN ordering
# A few difference way of specifying how similar the request should look
# For type CA, the listed attributes must be the same, and the optional
# and supplied fields are just that :-)
policy = policy_match
# For the CA policy
[ policy_match ]
countryName = match
stateOrProvinceName = match
organizationName = match
organizationalUnitName = optional
commonName = supplied
emailAddress = optional
# For the 'anything' policy
# At this point in time, you must list all acceptable 'object'
# types.
[ policy_anything ]
countryName = optional
stateOrProvinceName = optional
localityName = optional
organizationName = optional
organizationalUnitName = optional
commonName = supplied
emailAddress = optional
####################################################################
[ req ]
default_bits = 2048
default_md = sha1
default_keyfile = privkey.pem
distinguished_name = req_distinguished_name
#attributes = req_attributes
x509_extensions = v3_ca # The extentions to add to the self signed cert
# Passwords for private keys if not present they will be prompted for
# input_password = secret
# output_password = secret
# This sets a mask for permitted string types. There are several options.
# default: PrintableString, T61String, BMPString.
# pkix : PrintableString, BMPString (PKIX recommendation before 2004)
# utf8only: only UTF8Strings (PKIX recommendation after 2004).
# nombstr : PrintableString, T61String (no BMPStrings or UTF8Strings).
# MASK:XXXX a literal mask value.
# WARNING: ancient versions of Netscape crash on BMPStrings or UTF8Strings.
string_mask = utf8only
# req_extensions = v3_req # The extensions to add to a certificate request
[ req_distinguished_name ]
countryName = Country Name (2 letter code)
countryName_value = %(country_code)s
countryName_min = 2
countryName_max = 2
stateOrProvinceName = State or Province Name (full name)
stateOrProvinceName_value = %(state)s
localityName = Locality Name (eg, city)
localityName_value = %(city)s
0.organizationName = Organization Name (eg, company)
0.organizationName_value = %(company)s
# we can do this but it is not needed normally :-)
#1.organizationName = Second Organization Name (eg, company)
#1.organizationName_default = World Wide Web Pty Ltd
commonName = Common Name (eg, your name or your server\'s hostname)
commonName_max = 64
emailAddress = Email Address
emailAddress_value = %(email_address)s
emailAddress_max = 64
# SET-ex3 = SET extension number 3
#[ req_attributes ]
#challengePassword = A challenge password
#challengePassword_min = 4
#challengePassword_max = 20
#
#unstructuredName = An optional company name
[ usr_cert ]
# These extensions are added when 'ca' signs a request.
# This goes against PKIX guidelines but some CAs do it and some software
# requires this to avoid interpreting an end user certificate as a CA.
basicConstraints=CA:FALSE
# Here are some examples of the usage of nsCertType. If it is omitted
# the certificate can be used for anything *except* object signing.
# This is OK for an SSL server.
# nsCertType = server
# For an object signing certificate this would be used.
# nsCertType = objsign
# For normal client use this is typical
# nsCertType = client, email
# and for everything including object signing:
# nsCertType = client, email, objsign
# This is typical in keyUsage for a client certificate.
# keyUsage = nonRepudiation, digitalSignature, keyEncipherment
# This will be displayed in Netscape's comment listbox.
nsComment = "OpenSSL Generated Certificate"
# PKIX recommendations harmless if included in all certificates.
subjectKeyIdentifier=hash
authorityKeyIdentifier=keyid,issuer
# This stuff is for subjectAltName and issuerAltname.
# Import the email address.
# subjectAltName=email:copy
# An alternative to produce certificates that aren't
# deprecated according to PKIX.
# subjectAltName=email:move
# Copy subject details
# issuerAltName=issuer:copy
#nsCaRevocationUrl = http://www.domain.dom/ca-crl.pem
#nsBaseUrl
#nsRevocationUrl
#nsRenewalUrl
#nsCaPolicyUrl
#nsSslServerName
# This is required for TSA certificates.
# extendedKeyUsage = critical,timeStamping
[ v3_req ]
# Extensions to add to a certificate request
basicConstraints = CA:FALSE
keyUsage = nonRepudiation, digitalSignature, keyEncipherment
[ v3_ca ]
# Extensions for a typical CA
# PKIX recommendation.
subjectKeyIdentifier=hash
authorityKeyIdentifier=keyid:always,issuer
# This is what PKIX recommends but some broken software chokes on critical
# extensions.
#basicConstraints = critical,CA:true
# So we do this instead.
basicConstraints = CA:true
# Key usage: this is typical for a CA certificate. However since it will
# prevent it being used as an test self-signed certificate it is best
# left out by default.
# keyUsage = cRLSign, keyCertSign
# Some might want this also
# nsCertType = sslCA, emailCA
# Include email address in subject alt name: another PKIX recommendation
# subjectAltName=email:copy
# Copy issuer details
# issuerAltName=issuer:copy
# DER hex encoding of an extension: beware experts only!
# obj=DER:02:03
# Where 'obj' is a standard or added object
# You can even override a supported extension:
# basicConstraints= critical, DER:30:03:01:01:FF
[ crl_ext ]
# CRL extensions.
# Only issuerAltName and authorityKeyIdentifier make any sense in a CRL.
# issuerAltName=issuer:copy
authorityKeyIdentifier=keyid:always
[ proxy_cert_ext ]
# These extensions should be added when creating a proxy certificate
# This goes against PKIX guidelines but some CAs do it and some software
# requires this to avoid interpreting an end user certificate as a CA.
basicConstraints=CA:FALSE
# Here are some examples of the usage of nsCertType. If it is omitted
# the certificate can be used for anything *except* object signing.
# This is OK for an SSL server.
# nsCertType = server
# For an object signing certificate this would be used.
# nsCertType = objsign
# For normal client use this is typical
# nsCertType = client, email
# and for everything including object signing:
# nsCertType = client, email, objsign
# This is typical in keyUsage for a client certificate.
# keyUsage = nonRepudiation, digitalSignature, keyEncipherment
# This will be displayed in Netscape's comment listbox.
nsComment = "OpenSSL Generated Certificate"
# PKIX recommendations harmless if included in all certificates.
subjectKeyIdentifier=hash
authorityKeyIdentifier=keyid,issuer
# This stuff is for subjectAltName and issuerAltname.
# Import the email address.
# subjectAltName=email:copy
# An alternative to produce certificates that aren't
# deprecated according to PKIX.
# subjectAltName=email:move
# Copy subject details
# issuerAltName=issuer:copy
#nsCaRevocationUrl = http://www.domain.dom/ca-crl.pem
#nsBaseUrl
#nsRevocationUrl
#nsRenewalUrl
#nsCaPolicyUrl
#nsSslServerName
# This really needs to be in place for it to be a proxy certificate.
proxyCertInfo=critical,language:id-ppl-anyLanguage,pathlen:3,policy:foo
####################################################################
[ tsa ]
default_tsa = tsa_config1 # the default TSA section
[ tsa_config1 ]
# These are used by the TSA reply generation only.
dir = /etc/pki/tls # TSA root directory
serial = $dir/tsaserial # The current serial number (mandatory)
crypto_device = builtin # OpenSSL engine to use for signing
signer_cert = $dir/tsacert.pem # The TSA signing certificate
# (optional)
certs = $dir/cacert.pem # Certificate chain to include in reply
# (optional)
signer_key = $dir/private/tsakey.pem # The TSA private key (optional)
default_policy = tsa_policy1 # Policy if request did not specify it
# (optional)
other_policies = tsa_policy2, tsa_policy3 # acceptable policies (optional)
digests = md5, sha1 # Acceptable message digests (mandatory)
accuracy = secs:1, millisecs:500, microsecs:100 # (optional)
clock_precision_digits = 0 # number of digits after dot. (optional)
ordering = yes # Is ordering defined for timestamps?
# (optional, default: no)
tsa_name = yes # Must the TSA name be included in the reply?
# (optional, default: no)
ess_cert_id_chain = no # Must the ESS cert id chain be included?
# (optional, default: no)
slapos/recipe/kumofs/template/stunnel.conf.in deleted 100644 → 0
View file @ c44e2f89
foreground = yes
output = %(log)s
pid = %(pid_file)s
syslog = no
CApath = %(ca_path)s
key = %(key)s
CRLpath = %(ca_crl)s
cert = %(cert)s
[service]
accept = %(public_ip)s:%(public_port)s
connect = %(private_ip)s:%(private_port)s
slapos/recipe/lamp/__init__.py
View file @ 342b5393
......@@ -298,9 +298,9 @@ class Request(BaseRecipe):
'local_host': local_host, 'local_port': local_port,
}))
wrapper = zc.buildout.easy_install.scripts([('stunnel',
'slapos.recipe.librecipe.execute', 'execute')], self.ws,
sys.executable, self.wrapper_directory, arguments=[
self.options['stunnel_binary'].strip(), stunnel_conf_path]
'slapos.recipe.librecipe.execute', 'generic_exec')], self.ws,
sys.executable, self.wrapper_directory, arguments='%r, %r'
% (self.options['stunnel_binary'].strip(), stunnel_conf_path)
)[0]
self.path_list.append(wrapper)
return (local_host, local_port,)
slapos/recipe/lampgeneric/__init__.py
View file @ 342b5393
......@@ -71,13 +71,12 @@ class Recipe(GenericBaseRecipe):
)
path_list.append(httpd_conf)
wrapper = self.createWrapper(name=self.options['wrapper'],
command=self.options['httpd-binary'],
parameters=[
wrapper = self.createWrapper(self.options['wrapper'],
(self.options['httpd-binary'],
'-f',
self.options['httpd-conf'],
'-DFOREGROUND'
])
))
path_list.append(wrapper)
......
slapos/recipe/librecipe/execute.py
View file @ 342b5393
......@@ -2,7 +2,6 @@ import sys
import os
import signal
import subprocess
import time
from collections import defaultdict
from inotify_simple import INotify, flags
......@@ -29,65 +28,96 @@ def _wait_files_creation(file_list):
if event.name in directory:
directory[event.name] = event.mask & (flags.CREATE | flags.MOVED_TO)
def execute(args):
"""Portable execution with process replacement"""
# XXX: Kept for backward compatibility
generic_exec([args, None, None])
def execute_wait(args):
"""Execution but after all files in args[1] exists"""
# XXX: Kept for backward compatibility
generic_exec([args[0], args[1], None])
def _libc():
from ctypes import CDLL, get_errno, c_char_p, c_int, c_ulong, util
libc = CDLL(util.find_library('c'), use_errno=True)
libc_mount = libc.mount
libc_mount.argtypes = c_char_p, c_char_p, c_char_p, c_ulong, c_char_p
def mount(source, target, filesystemtype, mountflags, data):
if libc_mount(source, target, filesystemtype, mountflags, data):
e = get_errno()
raise OSError(e, os.strerror(e))
libc_unshare = libc.unshare
libc_unshare.argtypes = c_int,
def unshare(flags):
if libc_unshare(flags):
e = get_errno()
raise OSError(e, os.strerror(e))
return mount, unshare
def generic_exec(args, extra_environ=None, wait_list=None,
pidfile=None, reserve_cpu=False, private_dev_shm=None,
#shebang_workaround=False, # XXX: still needed ?
):
args = list(args)
if pidfile:
import psutil
try:
with open(pidfile) as f:
pid = int(f.read())
running = psutil.Process(pid).cmdline()
except Exception:
pass
else:
# With chained shebangs, several paths may be inserted at the beginning.
n = len(args)
for i in xrange(1+len(running)-n):
if args == running[i:n+i]:
sys.exit("Already running with pid %s." % pid)
with open(pidfile, 'w') as f:
f.write(str(os.getpid()))
args += sys.argv[1:]
if reserve_cpu:
# If the CGROUPS cpuset is available (and prepared by slap format),
# request an exclusive CPU core for this process.
with open(os.path.expanduser('~/.slapos-cpu-exclusive'), 'a') as f:
f.write('%s\n' % os.getpid())
if wait_list:
_wait_files_creation(wait_list)
if private_dev_shm:
mount, unshare = _libc()
CLONE_NEWNS = 0x00020000
CLONE_NEWUSER = 0x10000000
uid = os.getuid()
gid = os.getgid()
unshare(CLONE_NEWUSER |CLONE_NEWNS)
with open('/proc/self/setgroups', 'wb') as f: f.write('deny')
with open('/proc/self/uid_map', 'wb') as f: f.write('%s %s 1' % (uid, uid))
with open('/proc/self/gid_map', 'wb') as f: f.write('%s %s 1' % (gid, gid))
mount('tmpfs', '/dev/shm', 'tmpfs', 0, 'size=' + private_dev_shm)
if extra_environ:
env = os.environ.copy()
env.update(extra_environ)
os.execve(args[0], args, env)
else:
os.execv(args[0], args)
child_pg = None
def executee(args):
"""Portable execution with process replacement and environment manipulation"""
# XXX: Kept for backward compatibility
generic_exec([args[0], None, args[1]])
def executee_wait(args):
"""Portable execution with process replacement and environment manipulation"""
# XXX: Kept for backward compatibility
generic_exec(args)
def generic_exec(args):
exec_list = list(args[0])
file_list = args[1]
environment_overriding = args[2]
exec_env = os.environ.copy()
if environment_overriding is not None:
exec_env.update(environment_overriding)
if file_list is not None:
_wait_files_creation(file_list)
os.execve(exec_list[0], exec_list + sys.argv[1:], exec_env)
def sig_handler(sig, frame):
print 'Received signal %r, killing children and exiting' % sig
if child_pg is not None:
os.killpg(child_pg, signal.SIGHUP)
os.killpg(child_pg, signal.SIGTERM)
sys.exit(0)
signal.signal(signal.SIGINT, sig_handler)
signal.signal(signal.SIGQUIT, sig_handler)
signal.signal(signal.SIGTERM, sig_handler)
sys.exit()
def execute_with_signal_translation(args):
"""Run process as children and translate from SIGTERM to another signal"""
global child_pg
signal.signal(signal.SIGINT, sig_handler)
signal.signal(signal.SIGQUIT, sig_handler)
signal.signal(signal.SIGTERM, sig_handler)
child = subprocess.Popen(args, close_fds=True, preexec_fn=os.setsid)
child_pg = child.pid
try:
print 'Process %r started' % args
while True:
time.sleep(10)
signal.pause()
finally:
os.killpg(child_pg, signal.SIGHUP)
os.killpg(child_pg, signal.SIGTERM)
slapos/recipe/librecipe/generic.py
View file @ 342b5393
......@@ -33,7 +33,6 @@ import sys
import inspect
import re
import shutil
from textwrap import dedent
import urllib
import urlparse
......@@ -61,7 +60,13 @@ class GenericBaseRecipe(object):
self._options(options) # Options Hook
self.options = options.copy() # Updated options dict
self._ws = self.getWorkingSet()
@property
def _ws(self):
# getWorkingSet() is slow and it is not always needed.
# So _ws should be a lazy attribute.
if getattr(self, '_ws_internal', None) is None:
self._ws_internal = self.getWorkingSet()
return self._ws_internal
def update(self):
"""By default update method does the same thing than install"""
......@@ -110,92 +115,60 @@ class GenericBaseRecipe(object):
with io.open(filepath, 'w+', encoding=encoding) as f:
f.write(u'\n'.join(lines))
def createPythonScript(self, name, absolute_function, arguments=''):
def createPythonScript(self, name, absolute_function, args=(), kw={}):
"""Create a python script using zc.buildout.easy_install.scripts
* function should look like 'module.function', or only 'function'
if it is a builtin function."""
absolute_function = tuple(absolute_function.rsplit('.', 1))
if len(absolute_function) == 1:
absolute_function = ('__builtin__',) + absolute_function
if len(absolute_function) != 2:
raise ValueError("A non valid function was given")
module, function = absolute_function
function = absolute_function.rsplit('.', 1)
if len(function) == 1:
module = '__builtin__'
function, = function
else:
module, function = function
path, filename = os.path.split(os.path.abspath(name))
script = zc.buildout.easy_install.scripts(
assert not isinstance(args, (basestring, dict)), args
args = map(repr, args)
args += map('%s=%r'.__mod__, kw.iteritems())
return zc.buildout.easy_install.scripts(
[(filename, module, function)], self._ws, sys.executable,
path, arguments=arguments)[0]
return script
path, arguments=', '.join(args))[0]
def createWrapper(self, name, command, parameters, comments=[],
parameters_extra=False, environment=None,
pidfile=None, reserve_cpu=False
):
"""
Creates a shell script for process replacement.
Takes care of quoting.
Takes care of #! line limitation when the wrapped command is a script.
if pidfile parameter is specified, then it will make the wrapper a singleton,
accepting to run only if no other instance is running.
def createWrapper(self, path, args, env=None, **kw):
"""Create a wrapper script for process replacement"""
assert args
if kw:
return self.createPythonScript(path,
'slapos.recipe.librecipe.execute.generic_exec',
(args, env) if env else (args,), kw)
:param reserve_cpu: bool, try to reserve one core for the `command`
"""
# Simple case: creates a basic shell script for process replacement.
# This must be kept minimal to avoid code duplication with generic_exec.
# In particular, do not implement workaround for shebang size limitation
# here (note that this can't be done correctly with a POSIX shell, because
# the process can't be given a name).
lines = ['#!/bin/sh']
if env:
for k, v in sorted(env.iteritems()):
lines.append('export %s=%s' % (k, shlex.quote(v)))
lines.append('exec')
lines = [ '#!/bin/sh' ]
if comments:
lines += '# ', '\n# '.join(comments), '\n'
lines.append('COMMAND=' + shlex.quote(command))
for key in environment or ():
lines.append('export %s=%s' % (key, environment[key]))
if pidfile:
lines.append(dedent("""
# Check for other instances
pidfile=%s
if [ -s $pidfile ]; then
if pid=`pgrep -F $pidfile -f "$COMMAND" 2>/dev/null`; then
echo "Already running with pid $pid."
exit 1
fi
fi
echo $$ > $pidfile""" % shlex.quote(pidfile)))
if reserve_cpu:
# if the CGROUPS cpuset is available (and prepared by slap format)
# request an exclusive CPU core for this process
lines.append(dedent("""
# put own PID into waiting list for exclusive CPU-core access
echo $$ >> ~/.slapos-cpu-exclusive
"""))
lines.append(dedent('''
# If the wrapped command uses a shebang, execute the referenced
# executable passing the script path as first argument.
# This is to workaround the limitation of 127 characters in #!
[ ! -f "$COMMAND" ] || {
[ "`head -c2`" != "#!" ] || read -r EXE ARG
} < "$COMMAND"
exec $EXE ${ARG:+"$ARG"} "$COMMAND"'''))
parameters = map(shlex.quote, parameters)
if parameters_extra:
# pass-through further parameters
parameters.append('"$@"')
for param in parameters:
args = map(shlex.quote, args)
args.append('"$@"')
for arg in args:
if len(lines[-1]) < 40:
lines[-1] += ' ' + param
lines[-1] += ' ' + arg
else:
lines[-1] += ' \\'
lines.append('\t' + param)
lines.append('\t' + arg)
lines.append('')
return self.createFile(name, '\n'.join(lines), 0700)
return self.createFile(path, '\n'.join(lines), 0700)
def createDirectory(self, parent, name, mode=0700):
path = os.path.join(parent, name)
......
slapos/recipe/logrotate.py
View file @ 342b5393
......@@ -46,10 +46,10 @@ class Recipe(GenericBaseRecipe):
state_file = self.options['state-file']
logrotate = self.createPythonScript(
logrotate = self.createWrapper(
self.options['wrapper'],
'slapos.recipe.librecipe.execute.execute',
[self.options['logrotate-binary'], '-s', state_file, logrotate_conf_file, ]
(self.options['logrotate-binary'],
'-s', state_file, logrotate_conf_file),
)
return [logrotate, logrotate_conf_file]
......
slapos/recipe/memcached/__init__.py deleted 100644 → 0
View file @ c44e2f89
##############################################################################
#
# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from slapos.recipe.librecipe import BaseSlapRecipe
import hashlib
import os
import pkg_resources
import sys
import zc.buildout
import ConfigParser
class Recipe(BaseSlapRecipe):
def getTemplateFilename(self, template_name):
return pkg_resources.resource_filename(__name__,
'template/%s' % template_name)
def _install(self):
self.path_list = []
self.requirements, self.ws = self.egg.working_set()
# XXX-Cedric : add logrotate?
self.cron_d = self.installCrond()
memcached_conf = self.installMemcached(ip=self.getLocalIPv4Address(),
port=11000)
ca_conf = self.installCertificateAuthority()
key, certificate = self.requestCertificate('Memcached')
stunnel_conf = self.installStunnel(self.getGlobalIPv6Address(),
self.getLocalIPv4Address(), 12345, memcached_conf['memcached_port'],
certificate, key, ca_conf['ca_crl'],
ca_conf['certificate_authority_path'])
self.linkBinary()
self.setConnectionDict(dict(
stunnel_ip = stunnel_conf['public_ip'],
stunnel_port = stunnel_conf['public_port'],
))
return self.path_list
def linkBinary(self):
"""Links binaries to instance's bin directory for easier exposal"""
for linkline in self.options.get('link_binary_list', '').splitlines():
if not linkline:
continue
target = linkline.split()
if len(target) == 1:
target = target[0]
path, linkname = os.path.split(target)
else:
linkname = target[1]
target = target[0]
link = os.path.join(self.bin_directory, linkname)
if os.path.lexists(link):
if not os.path.islink(link):
raise zc.buildout.UserError(
'Target link already %r exists but it is not link' % link)
os.unlink(link)
os.symlink(target, link)
self.logger.debug('Created link %r -> %r' % (link, target))
self.path_list.append(link)
def installCrond(self):
timestamps = self.createDataDirectory('cronstamps')
cron_output = os.path.join(self.log_directory, 'cron-output')
self._createDirectory(cron_output)
catcher = zc.buildout.easy_install.scripts([('catchcron',
__name__ + '.catdatefile', 'catdatefile')], self.ws, sys.executable,
self.bin_directory, arguments=[cron_output])[0]
self.path_list.append(catcher)
cron_d = os.path.join(self.etc_directory, 'cron.d')
crontabs = os.path.join(self.etc_directory, 'crontabs')
self._createDirectory(cron_d)
self._createDirectory(crontabs)
# Use execute from erp5.
wrapper = zc.buildout.easy_install.scripts([('crond',
'slapos.recipe.librecipe.execute', 'execute')], self.ws, sys.executable,
self.wrapper_directory, arguments=[
self.options['dcrond_binary'].strip(), '-s', cron_d, '-c', crontabs,
'-t', timestamps, '-f', '-l', '5', '-M', catcher]
)[0]
self.path_list.append(wrapper)
return cron_d
def installLogrotate(self):
"""Installs logortate main configuration file and registers its to cron"""
logrotate_d = os.path.abspath(os.path.join(self.etc_directory,
'logrotate.d'))
self._createDirectory(logrotate_d)
logrotate_backup = self.createBackupDirectory('logrotate')
logrotate_conf = self.createConfigurationFile("logrotate.conf",
"include %s" % logrotate_d)
logrotate_cron = os.path.join(self.cron_d, 'logrotate')
state_file = os.path.join(self.data_root_directory, 'logrotate.status')
open(logrotate_cron, 'w').write('0 0 * * * %s -s %s %s' %
(self.options['logrotate_binary'], state_file, logrotate_conf))
self.path_list.extend([logrotate_d, logrotate_conf, logrotate_cron])
return logrotate_d, logrotate_backup
def registerLogRotation(self, name, log_file_list, postrotate_script):
"""Register new log rotation requirement"""
open(os.path.join(self.logrotate_d, name), 'w').write(
self.substituteTemplate(self.getTemplateFilename(
'logrotate_entry.in'),
dict(file_list=' '.join(['"'+q+'"' for q in log_file_list]),
postrotate=postrotate_script, olddir=self.logrotate_backup)))
def installCertificateAuthority(self, ca_country_code='XX',
ca_email='xx@example.com', ca_state='State', ca_city='City',
ca_company='Company'):
backup_path = self.createBackupDirectory('ca')
self.ca_dir = os.path.join(self.data_root_directory, 'ca')
self._createDirectory(self.ca_dir)
self.ca_request_dir = os.path.join(self.ca_dir, 'requests')
self._createDirectory(self.ca_request_dir)
config = dict(ca_dir=self.ca_dir, request_dir=self.ca_request_dir)
self.ca_private = os.path.join(self.ca_dir, 'private')
self.ca_certs = os.path.join(self.ca_dir, 'certs')
self.ca_crl = os.path.join(self.ca_dir, 'crl')
self.ca_newcerts = os.path.join(self.ca_dir, 'newcerts')
self.ca_key_ext = '.key'
self.ca_crt_ext = '.crt'
for d in [self.ca_private, self.ca_crl, self.ca_newcerts, self.ca_certs]:
self._createDirectory(d)
for f in ['crlnumber', 'serial']:
if not os.path.exists(os.path.join(self.ca_dir, f)):
open(os.path.join(self.ca_dir, f), 'w').write('01')
if not os.path.exists(os.path.join(self.ca_dir, 'index.txt')):
open(os.path.join(self.ca_dir, 'index.txt'), 'w').write('')
openssl_configuration = os.path.join(self.ca_dir, 'openssl.cnf')
config.update(
working_directory=self.ca_dir,
country_code=ca_country_code,
state=ca_state,
city=ca_city,
company=ca_company,
email_address=ca_email,
)
self._writeFile(openssl_configuration, pkg_resources.resource_string(
__name__, 'template/openssl.cnf.ca.in') % config)
self.path_list.extend(zc.buildout.easy_install.scripts([
('certificate_authority',
__name__ + '.certificate_authority', 'runCertificateAuthority')],
self.ws, sys.executable, self.wrapper_directory, arguments=[dict(
openssl_configuration=openssl_configuration,
openssl_binary=self.options['openssl_binary'],
certificate=os.path.join(self.ca_dir, 'cacert.pem'),
key=os.path.join(self.ca_private, 'cakey.pem'),
crl=os.path.join(self.ca_crl),
request_dir=self.ca_request_dir
)]))
# configure backup
backup_cron = os.path.join(self.cron_d, 'ca_rdiff_backup')
open(backup_cron, 'w').write(
'''0 0 * * * %(rdiff_backup)s %(source)s %(destination)s'''%dict(
rdiff_backup=self.options['rdiff_backup_binary'],
source=self.ca_dir,
destination=backup_path))
self.path_list.append(backup_cron)
return dict(
ca_certificate=os.path.join(config['ca_dir'], 'cacert.pem'),
ca_crl=os.path.join(config['ca_dir'], 'crl'),
certificate_authority_path=config['ca_dir']
)
def requestCertificate(self, name):
hash = hashlib.sha512(name).hexdigest()
key = os.path.join(self.ca_private, hash + self.ca_key_ext)
certificate = os.path.join(self.ca_certs, hash + self.ca_crt_ext)
parser = ConfigParser.RawConfigParser()
parser.add_section('certificate')
parser.set('certificate', 'name', name)
parser.set('certificate', 'key_file', key)
parser.set('certificate', 'certificate_file', certificate)
parser.write(open(os.path.join(self.ca_request_dir, hash), 'w'))
return key, certificate
def installStunnel(self, public_ip, private_ip, public_port, private_port,
ca_certificate, key, ca_crl, ca_path):
"""Installs stunnel"""
template_filename = self.getTemplateFilename('stunnel.conf.in')
log = os.path.join(self.log_directory, 'stunnel.log')
pid_file = os.path.join(self.run_directory, 'stunnel.pid')
stunnel_conf = dict(
public_ip=public_ip,
private_ip=private_ip,
public_port=public_port,
pid_file=pid_file,
log=log,
cert = ca_certificate,
key = key,
ca_crl = ca_crl,
ca_path = ca_path,
private_port = private_port,
)
stunnel_conf_path = self.createConfigurationFile("stunnel.conf",
self.substituteTemplate(template_filename,
stunnel_conf))
wrapper = zc.buildout.easy_install.scripts([('stunnel',
'slapos.recipe.librecipe.execute', 'execute_wait')], self.ws,
sys.executable, self.wrapper_directory, arguments=[
[self.options['stunnel_binary'].strip(), stunnel_conf_path],
[ca_certificate, key]]
)[0]
self.path_list.append(wrapper)
return stunnel_conf
def installMemcached(self, ip, port):
config = dict(
memcached_binary=self.options['memcached_binary'],
memcached_ip=ip,
memcached_port=port,
)
self.path_list.append(self.createRunningWrapper('memcached',
self.substituteTemplate(self.getTemplateFilename('memcached.in'),
config)))
return dict(memcached_url='%s:%s' %
(config['memcached_ip'], config['memcached_port']),
memcached_ip=config['memcached_ip'],
memcached_port=config['memcached_port'])
slapos/recipe/memcached/certificate_authority.py deleted 100644 → 0
View file @ c44e2f89
import os
import subprocess
import time
import ConfigParser
def popenCommunicate(command_list, input=None):
subprocess_kw = dict(stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
if input is not None:
subprocess_kw.update(stdin=subprocess.PIPE)
popen = subprocess.Popen(command_list, **subprocess_kw)
result = popen.communicate(input)[0]
if popen.returncode is None:
popen.kill()
if popen.returncode != 0:
raise ValueError('Issue during calling %r, result was:\n%s' % (
command_list, result))
return result
class CertificateAuthority:
def __init__(self, key, certificate, openssl_binary,
openssl_configuration, request_dir):
self.key = key
self.certificate = certificate
self.openssl_binary = openssl_binary
self.openssl_configuration = openssl_configuration
self.request_dir = request_dir
def checkAuthority(self):
file_list = [ self.key, self.certificate ]
ca_ready = True
for f in file_list:
if not os.path.exists(f):
ca_ready = False
break
if ca_ready:
return
for f in file_list:
if os.path.exists(f):
os.unlink(f)
try:
# no CA, let us create new one
popenCommunicate([self.openssl_binary, 'req', '-nodes', '-config',
self.openssl_configuration, '-new', '-x509', '-extensions',
'v3_ca', '-keyout', self.key, '-out', self.certificate,
'-days', '10950'], 'Automatic Certificate Authority\n')
except:
try:
for f in file_list:
if os.path.exists(f):
os.unlink(f)
except:
# do not raise during cleanup
pass
raise
def _checkCertificate(self, common_name, key, certificate):
file_list = [key, certificate]
ready = True
for f in file_list:
if not os.path.exists(f):
ready = False
break
if ready:
return False
for f in file_list:
if os.path.exists(f):
os.unlink(f)
csr = certificate + '.csr'
try:
popenCommunicate([self.openssl_binary, 'req', '-config',
self.openssl_configuration, '-nodes', '-new', '-keyout',
key, '-out', csr, '-days', '3650'],
common_name + '\n')
try:
popenCommunicate([self.openssl_binary, 'ca', '-batch', '-config',
self.openssl_configuration, '-out', certificate,
'-infiles', csr])
finally:
if os.path.exists(csr):
os.unlink(csr)
except:
try:
for f in file_list:
if os.path.exists(f):
os.unlink(f)
except:
# do not raise during cleanup
pass
raise
else:
return True
def checkRequestDir(self):
for request_file in os.listdir(self.request_dir):
parser = ConfigParser.RawConfigParser()
parser.readfp(open(os.path.join(self.request_dir, request_file), 'r'))
if self._checkCertificate(parser.get('certificate', 'name'),
parser.get('certificate', 'key_file'), parser.get('certificate',
'certificate_file')):
print 'Created certificate %r' % parser.get('certificate', 'name')
def runCertificateAuthority(args):
ca_conf = args[0]
ca = CertificateAuthority(ca_conf['key'], ca_conf['certificate'],
ca_conf['openssl_binary'], ca_conf['openssl_configuration'],
ca_conf['request_dir'])
while True:
ca.checkAuthority()
ca.checkRequestDir()
time.sleep(60)
slapos/recipe/memcached/template/memcached.in deleted 100644 → 0
View file @ c44e2f89
#!/bin/sh
exec %(memcached_binary)s -p %(memcached_port)s -U %(memcached_port)s -l %(memcached_ip)s
slapos/recipe/memcached/template/openssl.cnf.ca.in deleted 100644 → 0
View file @ c44e2f89
#
# OpenSSL example configuration file.
# This is mostly being used for generation of certificate requests.
#
# This definition stops the following lines choking if HOME isn't
# defined.
HOME = .
RANDFILE = $ENV::HOME/.rnd
# Extra OBJECT IDENTIFIER info:
#oid_file = $ENV::HOME/.oid
oid_section = new_oids
# To use this configuration file with the "-extfile" option of the
# "openssl x509" utility, name here the section containing the
# X.509v3 extensions to use:
# extensions =
# (Alternatively, use a configuration file that has only
# X.509v3 extensions in its main [= default] section.)
[ new_oids ]
# We can add new OIDs in here for use by 'ca', 'req' and 'ts'.
# Add a simple OID like this:
# testoid1=1.2.3.4
# Or use config file substitution like this:
# testoid2=${testoid1}.5.6
# Policies used by the TSA examples.
tsa_policy1 = 1.2.3.4.1
tsa_policy2 = 1.2.3.4.5.6
tsa_policy3 = 1.2.3.4.5.7
####################################################################
[ ca ]
default_ca = CA_default # The default ca section
####################################################################
[ CA_default ]
dir = %(working_directory)s # Where everything is kept
certs = $dir/certs # Where the issued certs are kept
crl_dir = $dir/crl # Where the issued crl are kept
database = $dir/index.txt # database index file.
#unique_subject = no # Set to 'no' to allow creation of
# several ctificates with same subject.
new_certs_dir = $dir/newcerts # default place for new certs.
certificate = $dir/cacert.pem # The CA certificate
serial = $dir/serial # The current serial number
crlnumber = $dir/crlnumber # the current crl number
# must be commented out to leave a V1 CRL
crl = $dir/crl.pem # The current CRL
private_key = $dir/private/cakey.pem # The private key
RANDFILE = $dir/private/.rand # private random number file
x509_extensions = usr_cert # The extentions to add to the cert
# Comment out the following two lines for the "traditional"
# (and highly broken) format.
name_opt = ca_default # Subject Name options
cert_opt = ca_default # Certificate field options
# Extension copying option: use with caution.
# copy_extensions = copy
# Extensions to add to a CRL. Note: Netscape communicator chokes on V2 CRLs
# so this is commented out by default to leave a V1 CRL.
# crlnumber must also be commented out to leave a V1 CRL.
# crl_extensions = crl_ext
default_days = 3650 # how long to certify for
default_crl_days= 30 # how long before next CRL
default_md = default # use public key default MD
preserve = no # keep passed DN ordering
# A few difference way of specifying how similar the request should look
# For type CA, the listed attributes must be the same, and the optional
# and supplied fields are just that :-)
policy = policy_match
# For the CA policy
[ policy_match ]
countryName = match
stateOrProvinceName = match
organizationName = match
organizationalUnitName = optional
commonName = supplied
emailAddress = optional
# For the 'anything' policy
# At this point in time, you must list all acceptable 'object'
# types.
[ policy_anything ]
countryName = optional
stateOrProvinceName = optional
localityName = optional
organizationName = optional
organizationalUnitName = optional
commonName = supplied
emailAddress = optional
####################################################################
[ req ]
default_bits = 2048
default_md = sha1
default_keyfile = privkey.pem
distinguished_name = req_distinguished_name
#attributes = req_attributes
x509_extensions = v3_ca # The extentions to add to the self signed cert
# Passwords for private keys if not present they will be prompted for
# input_password = secret
# output_password = secret
# This sets a mask for permitted string types. There are several options.
# default: PrintableString, T61String, BMPString.
# pkix : PrintableString, BMPString (PKIX recommendation before 2004)
# utf8only: only UTF8Strings (PKIX recommendation after 2004).
# nombstr : PrintableString, T61String (no BMPStrings or UTF8Strings).
# MASK:XXXX a literal mask value.
# WARNING: ancient versions of Netscape crash on BMPStrings or UTF8Strings.
string_mask = utf8only
# req_extensions = v3_req # The extensions to add to a certificate request
[ req_distinguished_name ]
countryName = Country Name (2 letter code)
countryName_value = %(country_code)s
countryName_min = 2
countryName_max = 2
stateOrProvinceName = State or Province Name (full name)
stateOrProvinceName_value = %(state)s
localityName = Locality Name (eg, city)
localityName_value = %(city)s
0.organizationName = Organization Name (eg, company)
0.organizationName_value = %(company)s
# we can do this but it is not needed normally :-)
#1.organizationName = Second Organization Name (eg, company)
#1.organizationName_default = World Wide Web Pty Ltd
commonName = Common Name (eg, your name or your server\'s hostname)
commonName_max = 64
emailAddress = Email Address
emailAddress_value = %(email_address)s
emailAddress_max = 64
# SET-ex3 = SET extension number 3
#[ req_attributes ]
#challengePassword = A challenge password
#challengePassword_min = 4
#challengePassword_max = 20
#
#unstructuredName = An optional company name
[ usr_cert ]
# These extensions are added when 'ca' signs a request.
# This goes against PKIX guidelines but some CAs do it and some software
# requires this to avoid interpreting an end user certificate as a CA.
basicConstraints=CA:FALSE
# Here are some examples of the usage of nsCertType. If it is omitted
# the certificate can be used for anything *except* object signing.
# This is OK for an SSL server.
# nsCertType = server
# For an object signing certificate this would be used.
# nsCertType = objsign
# For normal client use this is typical
# nsCertType = client, email
# and for everything including object signing:
# nsCertType = client, email, objsign
# This is typical in keyUsage for a client certificate.
# keyUsage = nonRepudiation, digitalSignature, keyEncipherment
# This will be displayed in Netscape's comment listbox.
nsComment = "OpenSSL Generated Certificate"
# PKIX recommendations harmless if included in all certificates.
subjectKeyIdentifier=hash
authorityKeyIdentifier=keyid,issuer
# This stuff is for subjectAltName and issuerAltname.
# Import the email address.
# subjectAltName=email:copy
# An alternative to produce certificates that aren't
# deprecated according to PKIX.
# subjectAltName=email:move
# Copy subject details
# issuerAltName=issuer:copy
#nsCaRevocationUrl = http://www.domain.dom/ca-crl.pem
#nsBaseUrl
#nsRevocationUrl
#nsRenewalUrl
#nsCaPolicyUrl
#nsSslServerName
# This is required for TSA certificates.
# extendedKeyUsage = critical,timeStamping
[ v3_req ]
# Extensions to add to a certificate request
basicConstraints = CA:FALSE
keyUsage = nonRepudiation, digitalSignature, keyEncipherment
[ v3_ca ]
# Extensions for a typical CA
# PKIX recommendation.
subjectKeyIdentifier=hash
authorityKeyIdentifier=keyid:always,issuer
# This is what PKIX recommends but some broken software chokes on critical
# extensions.
#basicConstraints = critical,CA:true
# So we do this instead.
basicConstraints = CA:true
# Key usage: this is typical for a CA certificate. However since it will
# prevent it being used as an test self-signed certificate it is best
# left out by default.
# keyUsage = cRLSign, keyCertSign
# Some might want this also
# nsCertType = sslCA, emailCA
# Include email address in subject alt name: another PKIX recommendation
# subjectAltName=email:copy
# Copy issuer details
# issuerAltName=issuer:copy
# DER hex encoding of an extension: beware experts only!
# obj=DER:02:03
# Where 'obj' is a standard or added object
# You can even override a supported extension:
# basicConstraints= critical, DER:30:03:01:01:FF
[ crl_ext ]
# CRL extensions.
# Only issuerAltName and authorityKeyIdentifier make any sense in a CRL.
# issuerAltName=issuer:copy
authorityKeyIdentifier=keyid:always
[ proxy_cert_ext ]
# These extensions should be added when creating a proxy certificate
# This goes against PKIX guidelines but some CAs do it and some software
# requires this to avoid interpreting an end user certificate as a CA.
basicConstraints=CA:FALSE
# Here are some examples of the usage of nsCertType. If it is omitted
# the certificate can be used for anything *except* object signing.
# This is OK for an SSL server.
# nsCertType = server
# For an object signing certificate this would be used.
# nsCertType = objsign
# For normal client use this is typical
# nsCertType = client, email
# and for everything including object signing:
# nsCertType = client, email, objsign
# This is typical in keyUsage for a client certificate.
# keyUsage = nonRepudiation, digitalSignature, keyEncipherment
# This will be displayed in Netscape's comment listbox.
nsComment = "OpenSSL Generated Certificate"
# PKIX recommendations harmless if included in all certificates.
subjectKeyIdentifier=hash
authorityKeyIdentifier=keyid,issuer
# This stuff is for subjectAltName and issuerAltname.
# Import the email address.
# subjectAltName=email:copy
# An alternative to produce certificates that aren't
# deprecated according to PKIX.
# subjectAltName=email:move
# Copy subject details
# issuerAltName=issuer:copy
#nsCaRevocationUrl = http://www.domain.dom/ca-crl.pem
#nsBaseUrl
#nsRevocationUrl
#nsRenewalUrl
#nsCaPolicyUrl
#nsSslServerName
# This really needs to be in place for it to be a proxy certificate.
proxyCertInfo=critical,language:id-ppl-anyLanguage,pathlen:3,policy:foo
####################################################################
[ tsa ]
default_tsa = tsa_config1 # the default TSA section
[ tsa_config1 ]
# These are used by the TSA reply generation only.
dir = /etc/pki/tls # TSA root directory
serial = $dir/tsaserial # The current serial number (mandatory)
crypto_device = builtin # OpenSSL engine to use for signing
signer_cert = $dir/tsacert.pem # The TSA signing certificate
# (optional)
certs = $dir/cacert.pem # Certificate chain to include in reply
# (optional)
signer_key = $dir/private/tsakey.pem # The TSA private key (optional)
default_policy = tsa_policy1 # Policy if request did not specify it
# (optional)
other_policies = tsa_policy2, tsa_policy3 # acceptable policies (optional)
digests = md5, sha1 # Acceptable message digests (mandatory)
accuracy = secs:1, millisecs:500, microsecs:100 # (optional)
clock_precision_digits = 0 # number of digits after dot. (optional)
ordering = yes # Is ordering defined for timestamps?
# (optional, default: no)
tsa_name = yes # Must the TSA name be included in the reply?
# (optional, default: no)
ess_cert_id_chain = no # Must the ESS cert id chain be included?
# (optional, default: no)
slapos/recipe/memcached/template/stunnel.conf.in deleted 100644 → 0
View file @ c44e2f89
foreground = yes
output = %(log)s
pid = %(pid_file)s
syslog = no
CApath = %(ca_path)s
key = %(key)s
CRLpath = %(ca_crl)s
cert = %(cert)s
[service]
accept = %(public_ip)s:%(public_port)s
connect = %(private_ip)s:%(private_port)s
slapos/recipe/mioga/instantiate.py
View file @ 342b5393
......@@ -203,11 +203,10 @@ Include conf/extra/httpd-autoindex.conf
services_dir = self.options['services_dir']
httpd_wrapper = self.createPythonScript(
httpd_wrapper = self.createWrapper(
os.path.join(services_dir, 'httpd_wrapper'),
'slapos.recipe.librecipe.execute.execute',
[self.options['httpd_binary'], '-f', self.options['httpd_conf'],
'-DFOREGROUND']
(self.options['httpd_binary'],
'-f', self.options['httpd_conf'], '-DFOREGROUND'),
)
path_list.append(httpd_wrapper)
......@@ -220,19 +219,17 @@ Include conf/extra/httpd-autoindex.conf
site_perl_bin = os.path.join(self.options['site_perl'], 'bin')
mioga_conf_path = os.path.join(mioga_base, 'conf', 'Mioga.conf')
notifier_wrapper = self.createPythonScript(
notifier_wrapper = self.createWrapper(
os.path.join(services_dir, 'notifier'),
'slapos.recipe.librecipe.execute.execute',
[ os.path.join(site_perl_bin, 'notifier.pl'),
mioga_conf_path ]
(os.path.join(site_perl_bin, 'notifier.pl'),
mioga_conf_path),
)
path_list.append(notifier_wrapper)
searchengine_wrapper = self.createPythonScript(
searchengine_wrapper = self.createWrapper(
os.path.join(services_dir, 'searchengine'),
'slapos.recipe.librecipe.execute.execute',
[ os.path.join(site_perl_bin, 'searchengine.pl'),
mioga_conf_path ]
(os.path.join(site_perl_bin, 'searchengine.pl'),
mioga_conf_path),
)
path_list.append(searchengine_wrapper)
......
slapos/recipe/mydumper.py
View file @ 342b5393
......@@ -24,7 +24,7 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import subprocess
import os
from slapos.recipe.librecipe import GenericBaseRecipe
......@@ -58,14 +58,14 @@ def do_export(args):
cmd.extend(['-o', args['directory']])
subprocess.check_call(cmd)
os.execv(cmd[0], cmd)
def do_import(args):
cmd = _mydumper_base_cmd(**args)
cmd.append('--overwrite-tables')
cmd.extend(['-d', args['directory']])
subprocess.check_call(cmd)
os.execv(cmd[0], cmd)
class Recipe(GenericBaseRecipe):
......@@ -95,9 +95,5 @@ class Recipe(GenericBaseRecipe):
config['compression'] = self.optionIsTrue('compression', default=False)
config['rows'] = self.options.get('rows')
wrapper = self.createPythonScript(name=self.options['wrapper'],
absolute_function = '%s.%s' % (__name__, function.func_name),
arguments=config)
return [wrapper]
return self.createPythonScript(self.options['wrapper'],
'%s.%s' % (function.__module__, function.__name__), (config,))
slapos/recipe/mysql/__init__.py
View file @ 342b5393
......@@ -53,11 +53,10 @@ class Recipe(GenericBaseRecipe):
mysql_binary = self.options['mysql-binary']
socket = self.options['socket'],
post_rotate = self.createPythonScript(
post_rotate = self.createWrapper(
self.options['logrotate-post'],
'slapos.recipe.librecipe.execute.execute',
[mysql_binary, '--no-defaults', '-B', '-u', 'root', '--socket=%s' % socket, '-e',
'FLUSH LOGS']
(mysql_binary, '--no-defaults', '-B', '-u', 'root',
'--socket=%s' % socket, '-e', 'FLUSH LOGS'),
)
path_list.append(post_rotate)
......@@ -85,12 +84,12 @@ class Recipe(GenericBaseRecipe):
mysql_update = self.createPythonScript(
self.options['update-wrapper'],
'%s.mysql.updateMysql' % __name__,
dict(
(dict(
mysql_script=mysql_script,
mysql_binary=mysql_binary,
mysql_upgrade_binary=mysql_upgrade_binary,
socket=socket,
)
),)
)
path_list.append(mysql_update)
......@@ -98,7 +97,7 @@ class Recipe(GenericBaseRecipe):
mysqld = self.createPythonScript(
self.options['wrapper'],
'%s.mysql.runMysql' % __name__,
dict(
(dict(
mysql_install_binary=self.options['mysql-install-binary'],
mysqld_binary=mysqld_binary,
data_directory=mysql_conf['data_directory'],
......@@ -106,7 +105,7 @@ class Recipe(GenericBaseRecipe):
socket=socket,
configuration_file=mysql_conf_file,
cwd=self.options['mysql-base-directory'],
)
),)
)
path_list.append(mysqld)
......@@ -115,12 +114,12 @@ class Recipe(GenericBaseRecipe):
backup_script = self.createPythonScript(
self.options['backup-script'],
'%s.do_backup' % __name__,
dict(
(dict(
mydumper_binary=self.options['mydumper-binary'],
database=mysql_conf['mysql_database'],
socket=mysql_conf['socket'],
backup_directory=self.options['backup-directory']
),
),)
)
path_list.append(backup_script)
......@@ -129,7 +128,7 @@ class Recipe(GenericBaseRecipe):
recovering_script = self.createPythonScript(
self.options['recovering-wrapper'],
'%s.import_dump' % __name__,
{
({
'lock_file': os.path.join(self.work_directory,
'import_done'),
'database': mysql_conf['mysql_database'],
......@@ -140,7 +139,7 @@ class Recipe(GenericBaseRecipe):
'local_directory': self.mysql_backup_directory,
'dump_name': dump_filename,
'zcat_binary': self.options['zcat-binary'],
}
},)
)
path_list.append(recovering_script)
......
slapos/recipe/neoppod.py
View file @ 342b5393
......@@ -42,7 +42,8 @@ class NeoBaseRecipe(GenericBaseRecipe):
# Only then can this recipe start succeeding and actually doing anything
# useful, as per NEO deploying constraints.
raise UserError('"masters" parameter is mandatory')
option_list = [
args = [
options['binary'],
# Keep the -l option first, as expected by logrotate snippets.
'-l', options['logfile'],
'-m', options['masters'],
......@@ -53,17 +54,13 @@ class NeoBaseRecipe(GenericBaseRecipe):
]
if options['ssl']:
etc = os.path.join(self.buildout['buildout']['directory'], 'etc', '')
option_list += (
args += (
'--ca', etc + 'ca.crt',
'--cert', etc + 'neo.crt',
'--key', etc + 'neo.key',
)
option_list.extend(self._getOptionList())
return [self.createWrapper(
options['wrapper'],
options['binary'],
option_list
)]
args += self._getOptionList()
return self.createWrapper(options['wrapper'], args)
def _getBindingAddress(self):
options = self.options
......
slapos/recipe/notifier.py
View file @ 342b5393
......@@ -32,19 +32,15 @@ class Recipe(GenericBaseRecipe):
def install(self):
options = self.options
script = self.createWrapper(name=options['wrapper'],
command=options['server-binary'],
parameters=[
# Script that execute the callback(s) upon receiving a notification.
return self.createWrapper(options['wrapper'],
(options['server-binary'],
'--callbacks', options['callbacks'],
'--feeds', options['feeds'],
'--equeue-socket', options['equeue-socket'],
options['host'], options['port']
],
comments=[
'',
'Upon receiving a notification, execute the callback(s).',
''])
return [script]
),
)
class Callback(GenericBaseRecipe):
......@@ -80,35 +76,32 @@ class Notify(GenericBaseRecipe):
# Just a touch
open(log, 'w').close()
parameters = [
cmd = [notifier_binary,
'-l', log,
'--title', title,
'--feed', feed_url,
'--max-run', str(max_run),
'--notification-url',
]
parameters.extend(notification_url.split(' '))
parameters.extend(['--executable', executable])
cmd += notification_url.split(' ')
cmd += '--executable', executable
# For a more verbose mode, writing feed items for any action
instance_root_name = instance_root_name or self.options.get('instance-root-name', None)
log_url = log_url or self.options.get('log-url', None)
status_item_directory = status_item_directory or self.options.get('status-item-directory', None)
if instance_root_name and log_url and status_item_directory:
parameters.extend([
cmd += (
'--instance-root-name', instance_root_name,
'--log-url', log_url,
'--status-item-directory', status_item_directory,
])
return self.createWrapper(name=wrapper,
command=notifier_binary,
parameters=parameters,
pidfile=pidfile,
parameters_extra=True,
comments=[
'',
'Call an executable and send notification(s).',
''])
)
kw = {}
if pidfile:
kw['pidfile'] = pidfile
# Script that call an executable and send notification(s).
return self.createWrapper(wrapper, cmd, **kw)
def install(self):
......
slapos/recipe/novnc/__init__.py
View file @ 342b5393
......@@ -35,10 +35,9 @@ class Recipe(GenericBaseRecipe):
"""
def install(self):
runner_path = self.createPythonScript(
return self.createWrapper(
self.options['path'],
'slapos.recipe.librecipe.execute.execute_wait',
[[
(
self.options['websockify-path'],
'--web',
self.options['novnc-location'],
......@@ -47,8 +46,7 @@ class Recipe(GenericBaseRecipe):
'--ssl-only',
'%s:%s' % (self.options['ip'], self.options['port']),
'%s:%s' % (self.options['vnc-ip'], self.options['vnc-port']),
],
[self.options['ssl-key-path'], self.options['ssl-cert-path']]],
),
wait_list=(self.options['ssl-key-path'],
self.options['ssl-cert-path']),
)
return [runner_path]
slapos/recipe/pbs.py
View file @ 342b5393
......@@ -39,30 +39,18 @@ from slapos.recipe.notifier import Callback
from slapos.recipe.librecipe import shlex
def promise(args):
def promise(ssh_client, user, host, port):
# Redirect output to /dev/null
with open("/dev/null") as _dev_null:
with open(os.devnull) as _dev_null:
ssh = subprocess.Popen(
[args['ssh_client'], '%(user)s@%(host)s' % args, '-p', '%(port)s' % args],
stdin=subprocess.PIPE, stdout=_dev_null, stderr=None
)
# Rdiff Backup protocol quit command
quitcommand = 'q' + chr(255) + chr(0) * 7
ssh.stdin.write(quitcommand)
ssh.stdin.flush()
ssh.stdin.close()
ssh.wait()
if ssh.poll() is None:
return 1
if ssh.returncode != 0:
(ssh_client, '%s@%s' % (user, host), '-p', str(port)),
stdin=subprocess.PIPE, stdout=_dev_null)
ssh.communicate('q' + chr(255) + chr(0) * 7)
if ssh.returncode:
sys.stderr.write("SSH Connection failed\n")
return ssh.returncode
class Recipe(GenericSlapRecipe, Notify, Callback):
def _options(self, options):
options['rdiff-backup-data-folder'] = ""
......@@ -244,15 +232,11 @@ class Recipe(GenericSlapRecipe, Notify, Callback):
print 'Processing PBS slave %s with type %s' % (slave_id, slave_type)
promise_path = os.path.join(self.options['promises-directory'], "ssh-to-%s" % slave_id)
promise_dict = dict(ssh_client=self.options['sshclient-binary'],
user=parsed_url.username,
host=parsed_url.hostname,
port=parsed_url.port)
promise = self.createPythonScript(promise_path,
__name__ + '.promise',
promise_dict)
path_list.append(promise)
path_list.append(self.createPythonScript(
os.path.join(self.options['promises-directory'], "ssh-to-%s" % slave_id),
__name__ + '.promise',
(self.options['sshclient-binary'],
parsed_url.username, parsed_url.hostname, parsed_url.port)))
# Create known_hosts file by default.
# In some case, we don't want to create it (case where we share IP mong partitions)
......@@ -336,12 +320,11 @@ class Recipe(GenericSlapRecipe, Notify, Callback):
else:
self.logger.info("Server mode")
wrapper = self.createWrapper(name=self.options['wrapper'],
command=self.options['rdiffbackup-binary'],
parameters=[
wrapper = self.createWrapper(self.options['wrapper'],
(self.options['rdiffbackup-binary'],
'--restrict', self.options['path'],
'--server'
])
))
path_list.append(wrapper)
return path_list
slapos/recipe/re6stnet/__init__.py
View file @ 342b5393
......@@ -162,7 +162,6 @@ class Recipe(GenericBaseRecipe):
return hash_string
def install(self):
path_list = []
token_save_path = os.path.join(self.options['conf-dir'], 'token.json')
token_list_path = self.options['token-dir']
......@@ -190,20 +189,14 @@ class Recipe(GenericBaseRecipe):
self.createFile(token_save_path, json.dumps(token_dict))
service_dict = dict(token_base_path=token_list_path,
token_json=token_save_path,
partition_id=self.computer_partition_id,
computer_id=self.computer_id,
registry_url=registry_url,
server_url=self.server_url,
computer_dict = dict(partition_id=self.computer_partition_id,
computer_guid=self.computer_id,
master_url=self.server_url,
cert_file=self.cert_file,
key_file=self.key_file)
request_add = self.createPythonScript(
return self.createPythonScript(
self.options['manager-wrapper'].strip(),
'%s.re6stnet.manage' % __name__, service_dict
__name__ + '.re6stnet.manage',
(registry_url, token_list_path, token_save_path, computer_dict)
)
path_list.append(request_add)
return path_list
slapos/recipe/re6stnet/re6stnet.py
View file @ 342b5393
......@@ -17,10 +17,8 @@ logging.trace = logging.debug
def loadJsonFile(path):
if os.path.exists(path):
with open(path, 'r') as f:
content = f.read()
return json.loads(content)
else:
return {}
return json.load(f)
return {}
def writeFile(path, data):
with open(path, 'w') as f:
......@@ -39,29 +37,25 @@ def updateFile(file_path, value):
return True
return False
def getComputerPartition(server_url, key_file, cert_file, computer_guid, partition_id):
def getComputerPartition(master_url, key_file, cert_file,
computer_guid, partition_id):
slap = slapos.slap.slap()
# Redeploy instance to update published information
slap.initializeConnection(server_url,
key_file,
cert_file)
return slap.registerComputerPartition(computer_guid=computer_guid,
partition_id=partition_id)
slap.initializeConnection(master_url, key_file, cert_file)
return slap.registerComputerPartition(computer_guid, partition_id)
def requestAddToken(client, base_token_path):
def requestAddToken(client, token_base_path):
time.sleep(3)
path_list = [x for x in os.listdir(base_token_path) if x.endswith('.add')]
path_list = [x for x in os.listdir(token_base_path) if x.endswith('.add')]
log.info("Searching tokens to add at %s and found %s." % (base_token_path, path_list))
log.info("Searching tokens to add at %s and found %s." % (token_base_path, path_list))
if not path_list:
log.info("No new token to add. Exiting...")
return
for reference_key in path_list:
request_file = os.path.join(base_token_path, reference_key)
request_file = os.path.join(token_base_path, reference_key)
token = readFile(request_file)
log.info("Including token %s for %s" % (token, reference_key))
if token :
......@@ -79,21 +73,21 @@ def requestAddToken(client, base_token_path):
# update information
log.info("New token added for slave instance %s. Updating file status..." %
reference)
status_file = os.path.join(base_token_path, '%s.status' % reference)
status_file = os.path.join(token_base_path, '%s.status' % reference)
updateFile(status_file, 'TOKEN_ADDED')
os.unlink(request_file)
else:
log.debug('Bad token. Request add token fail for %s...' % request_file)
def requestRemoveToken(client, base_token_path):
path_list = [x for x in os.listdir(base_token_path) if x.endswith('.remove')]
def requestRemoveToken(client, token_base_path):
path_list = [x for x in os.listdir(token_base_path) if x.endswith('.remove')]
if not path_list:
log.info("No token to delete. Exiting...")
return
for reference_key in path_list:
request_file = os.path.join(base_token_path, reference_key)
request_file = os.path.join(token_base_path, reference_key)
token = readFile(request_file)
if token :
reference = reference_key.split('.')[0]
......@@ -108,7 +102,7 @@ def requestRemoveToken(client, base_token_path):
continue
else:
# certificate is invalidated, it will be revoked
writeFile(os.path.join(base_token_path, '%s.revoke' % reference), '')
writeFile(os.path.join(token_base_path, '%s.revoke' % reference), '')
if result in (True, 'True'):
# update information
......@@ -117,33 +111,17 @@ def requestRemoveToken(client, base_token_path):
if result in ['True', 'False']:
os.unlink(request_file)
status_file = os.path.join(base_token_path, '%s.status' % reference)
status_file = os.path.join(token_base_path, '%s.status' % reference)
if os.path.exists(status_file):
os.unlink(status_file)
ipv6_file = os.path.join(base_token_path, '%s.ipv6' % reference)
ipv6_file = os.path.join(token_base_path, '%s.ipv6' % reference)
if os.path.exists(ipv6_file):
os.unlink(ipv6_file)
else:
log.debug('Bad token. Request add token fail for %s...' % request_file)
def requestRevoqueCertificate(args):
base_token_path = args['token_base_path']
path_list = [x for x in os.listdir(base_token_path) if x.endswith('.revoke')]
for reference_key in path_list:
reference = reference_key.split('.')[0]
if revokeByMail(args['registry_url'],
'%s@slapos' % reference.lower(),
args['db']):
os.unlink(os.path.join(base_token_path, reference_key))
log.info("Certificate revoked for slave instance %s." % reference)
return
log.info("Failed to revoke email for %s" % reference)
def checkService(client, base_token_path, token_json, computer_partition):
def checkService(client, token_base_path, token_json, computer_partition):
token_dict = loadJsonFile(token_json)
updated = False
if not token_dict:
......@@ -152,7 +130,7 @@ def checkService(client, base_token_path, token_json, computer_partition):
# Check token status
for slave_reference, token in token_dict.iteritems():
log.info("%s %s" % (slave_reference, token))
status_file = os.path.join(base_token_path, '%s.status' % slave_reference)
status_file = os.path.join(token_base_path, '%s.status' % slave_reference)
if not os.path.exists(status_file):
# This token is not added yet!
log.info("Token %s dont exist yet." % status_file)
......@@ -206,31 +184,22 @@ def checkService(client, base_token_path, token_json, computer_partition):
slave_reference, traceback.format_exc())
def manage(args, can_bang=True):
computer_guid = args['computer_id']
partition_id = args['partition_id']
server_url = args['server_url']
key_file = args['key_file']
cert_file = args['cert_file']
def manage(registry_url, token_base_path, token_json,
computer_dict, can_bang=True):
client = registry.RegistryClient(args['registry_url'])
base_token_path = args['token_base_path']
token_json = args['token_json']
client = registry.RegistryClient(registry_url)
log.info("ADD TOKEN")
# Request Add new tokens
requestAddToken(client, base_token_path)
requestAddToken(client, token_base_path)
log.info("Remove TOKEN")
# Request delete removed token
requestRemoveToken(client, base_token_path)
requestRemoveToken(client, token_base_path)
computer_partition = getComputerPartition(server_url, key_file,
cert_file, computer_guid, partition_id)
computer_partition = getComputerPartition(**computer_dict)
log.info("Update Services")
# check status of all token
checkService(client, base_token_path,
token_json, computer_partition)
checkService(client, token_base_path, token_json, computer_partition)
slapos/recipe/redis/__init__.py
View file @ 342b5393
......@@ -25,7 +25,7 @@
#
##############################################################################
import os
import sys
from slapos.recipe.librecipe import GenericBaseRecipe
class Recipe(GenericBaseRecipe):
......@@ -56,10 +56,9 @@ class Recipe(GenericBaseRecipe):
configuration))
path_list.append(config)
redis = self.createPythonScript(
redis = self.createWrapper(
self.options['wrapper'],
'slapos.recipe.librecipe.execute.execute',
[self.options['server_bin'], config_file]
(self.options['server_bin'], config_file),
)
path_list.append(redis)
......@@ -67,11 +66,20 @@ class Recipe(GenericBaseRecipe):
if promise_script:
promise = self.createPythonScript(
promise_script,
'%s.promise.main' % __name__,
dict(host=self.options['ipv6'], port=self.options['port'],
unixsocket = self.options.get('unixsocket') )
__name__ + '.promise',
(self.options['ipv6'], int(self.options['port']),
self.options.get('unixsocket'))
)
path_list.append(promise)
return path_list
def promise(host, port, unixsocket):
from .MyRedis2410 import Redis
try:
r = Redis(host=host, port=port, unix_socket_path=unixsocket, db=0)
r.publish("Promise-Service","SlapOS Promise")
r.connection_pool.disconnect()
except Exception, e:
sys.exit(e)
slapos/recipe/redis/promise.py deleted 100644 → 0
View file @ c44e2f89
#! /usr/bin/env python
# -*- coding: utf-8 -*-
import slapos.recipe.redis.MyRedis2410 as redis
import sys
def main(args):
host = args['host']
port = int(args['port'])
unixsocket = args['unixsocket']
try:
r = redis.Redis(host=host, port=port, unix_socket_path=unixsocket, db=0)
r.publish("Promise-Service","SlapOS Promise")
r.connection_pool.disconnect()
sys.exit(0)
except Exception, e:
print str(e)
sys.exit(1)
\ No newline at end of file
slapos/recipe/reverse_proxy_nginx/__init__.py
View file @ 342b5393
......@@ -70,14 +70,12 @@ class Recipe(GenericSlapRecipe):
path_list.append(nginx_configuration_file)
# Generate Nginx wrapper
wrapper = self.createWrapper(
name=self.options['wrapper'],
command=self.options['nginx-executable'],
parameters=[
path_list.append(self.createWrapper(
self.options['wrapper'],
(self.options['nginx-executable'],
'-c', self.options['configuration-file'],
'-p', self.options['home-directory']
]
)
)))
# TODO: reload configuration or have feature like apache_map
......
slapos/recipe/seleniumrunner/__init__.py
View file @ 342b5393
......@@ -25,23 +25,15 @@
#
#############################################################################
import os
import sys
import zc.buildout
from slapos.recipe.librecipe import BaseSlapRecipe
from slapos.recipe.librecipe import GenericBaseRecipe
class Recipe(GenericBaseRecipe):
def install(self):
runner = self.createPythonScript(
def install(self):
return self.createPythonScript(
self.options['runner-path'],
__name__+'.testrunner.run',
arguments=[self.options['suite-url'],
self.options['report-url'],
self.options['report-project'],
self.options['browser'],
])
return [runner]
(self.options['suite-url'],
self.options['report-url'],
self.options['report-project'],
self.options['browser']))
slapos/recipe/seleniumrunner/testrunner.py
View file @ 342b5393
......@@ -36,12 +36,7 @@ import urlparse
from subprocess import Popen, PIPE
import signal
def run(args):
suite_url = args[0]
report_url = args[1]
project = args[2]
browser_binary = args[3]
def run(suite_url, report_url, project, browser_binary):
suite_parsed = urlparse.urlparse(suite_url)
config = {
......
slapos/recipe/shell.py
View file @ 342b5393
......@@ -31,27 +31,12 @@ from slapos.recipe.librecipe import GenericBaseRecipe
class Recipe(GenericBaseRecipe):
def install(self):
env = os.environ.copy()
path_list = self.options['path'].split('\n')
env.update(PATH=':'.join(path_list))
env.update(SHELL=self.options['shell'])
env.update(HOME=self.options['home'])
ps1 = self.options.get('ps1')
if ps1 is not None:
env.update(PS1=str(json.loads(ps1)))
else:
env.update(PS1=env.get('PS1', '> '))
wrapper = self.createPythonScript(
self.options['wrapper'],
'slapos.recipe.librecipe.execute.executee',
[ # Executable
[self.options['shell']],
# Environment
env
]
)
return [wrapper]
shell = self.options['shell']
env = {
'HOME': self.options['home'],
'PATH': ':'.join(self.options['path'].split('\n')),
'PS1': str(json.loads(ps1)) if ps1 else os.getenv('PS1', '> '),
'SHELL': shell,
}
return self.createWrapper(self.options['wrapper'], (shell,), env)
slapos/recipe/shellinabox.py
View file @ 342b5393
......@@ -33,25 +33,15 @@ import shlex
from slapos.recipe.librecipe import GenericBaseRecipe
def login_shell(args):
password_file = args['password-file']
def login_shell(password_file, shell):
if password_file:
with open(password_file, 'r') as password_file:
password = password_file.read()
if (password != ''):
entered_password = getpass()
else:
entered_password = ''
if not hmac.compare_digest(entered_password, password):
return 1
else:
commandline = shlex.split(args['shell'])
path = commandline[0]
os.execv(path, commandline)
else:
return 1
if not password or hmac.compare_digest(getpass(), password):
commandline = shlex.split(shell)
os.execv(commandline[0], commandline)
return 1
def shellinabox(args):
certificate_dir = args['certificate_dir']
......@@ -95,22 +85,16 @@ def shellinabox(args):
class Recipe(GenericBaseRecipe):
def install(self):
path_list = []
login_shell = self.createPythonScript(
login_shell_wrapper = self.createPythonScript(
self.options['login-shell'],
'%s.login_shell' % __name__,
{
'password-file': self.options['password-file'],
'shell': self.options['shell']
}
__name__ + '.login_shell',
(self.options['password-file'], self.options['shell'])
)
path_list.append(login_shell)
wrapper = self.createPythonScript(
shellinabox_wrapper = self.createPythonScript(
self.options['wrapper'],
'%s.shellinabox' % __name__,
dict(
__name__ + '.shellinabox',
(dict(
certificate_dir=self.options['certificate-directory'],
ssl_key=self.options['key-file'],
ssl_certificate=self.options['cert-file'],
......@@ -118,9 +102,8 @@ class Recipe(GenericBaseRecipe):
directory=self.options['directory'],
ipv6=self.options['ipv6'],
port=self.options['port'],
login_shell=login_shell,
)
login_shell=login_shell_wrapper,
),)
)
path_list.append(wrapper)
return [wrapper]
return login_shell_wrapper, shellinabox_wrapper
slapos/recipe/signal_wrapper.py
View file @ 342b5393
......@@ -32,6 +32,6 @@ class Recipe(GenericBaseRecipe):
self.createPythonScript(
self.options['wrapper-path'],
'slapos.recipe.librecipe.execute.execute_with_signal_translation',
[self.options['wrapped-path']]
((self.options['wrapped-path'],),)
)
]
slapos/recipe/simplehttpserver/__init__.py
View file @ 342b5393
......@@ -68,9 +68,8 @@ class Recipe(GenericBaseRecipe):
'root-dir': self.options['root-dir']
}
server = self.createPythonScript(
return self.createPythonScript(
self.options['wrapper'].strip(),
'%s.simplehttpserver.run' % __name__, parameters
__name__ + '.simplehttpserver.run',
(parameters,)
)
return [server]
slapos/recipe/simplelogger.py
View file @ 342b5393
......@@ -29,9 +29,9 @@ import time
from slapos.recipe.librecipe import GenericBaseRecipe
def log(args):
def log(filename):
prefix = time.strftime('%Y-%m-%d.%H:%M.%s:')
with open(args['filename'], 'aw') as logfile:
with open(filename, 'a') as logfile:
for line in sys.stdin:
print >> logfile, prefix, line,
print >> logfile, prefix, '------------------------'
......@@ -39,10 +39,7 @@ def log(args):
class Recipe(GenericBaseRecipe):
def install(self):
wrapper = self.options['wrapper']
log = self.options['log']
script = self.createPythonScript(wrapper,
__name__ + '.log',
arguments=dict(filename=log))
return [script]
return self.createPythonScript(
self.options['wrapper'],
__name__ + '.log',
(self.options['log'],))
slapos/recipe/sphinx/__init__.py
View file @ 342b5393
......@@ -48,11 +48,10 @@ class Recipe(GenericBaseRecipe):
)
# Create init script
wrapper = self.createPythonScript(
wrapper = self.createWrapper(
self.options['wrapper'],
'slapos.recipe.librecipe.execute.execute',
[self.options['sphinx-searchd-binary'].strip(), '-c',
sphinx_conf_path, '--nodetach'],
(self.options['sphinx-searchd-binary'].strip(), '-c',
sphinx_conf_path, '--nodetach'),
)
return [wrapper, sphinx_conf_path]
slapos/recipe/squid/__init__.py
View file @ 342b5393
......@@ -78,21 +78,15 @@ class Recipe(GenericBaseRecipe):
self.substituteTemplate(template_filename, config))
# Prepare directories
prepare_path = self.createPythonScript(
prepare_path = self.createWrapper(
self.options['prepare-path'],
'slapos.recipe.librecipe.execute.execute',
arguments=[self.options['binary-path'].strip(),
'-z',
'-f', configuration_path,
],)
(self.options['binary-path'].strip(),
'-z', '-f', configuration_path))
# Create running wrapper
wrapper_path = self.createPythonScript(
wrapper_path = self.createWrapper(
self.options['wrapper-path'],
'slapos.recipe.librecipe.execute.execute',
arguments=[self.options['binary-path'].strip(),
'-N',
'-f', configuration_path,
],)
(self.options['binary-path'].strip(),
'-N', '-f', configuration_path))
return [configuration_path, wrapper_path, prepare_path]
slapos/recipe/sshkeys_authority.py
View file @ 342b5393
......@@ -34,17 +34,14 @@ from slapos.recipe.librecipe import GenericBaseRecipe
from slapos.recipe.librecipe.inotify import subfiles
# This authority only works with dropbear or openssh sshkey generators
def sshkeys_authority(args):
requests_directory = args['requests']
keygen_binary = args['sshkeygen']
def sshkeys_authority(request_directory, keygen_binary):
if 'openssh' in keygen_binary:
authority_type = 'openssh'
else:
# Keep dropbear for compatibility
authority_type = 'dropbear'
for request_filename in subfiles(requests_directory):
for request_filename in subfiles(request_directory):
with open(request_filename) as request_file:
request = json.load(request_file)
......@@ -98,18 +95,13 @@ def sshkeys_authority(args):
public_key_file.write(public_key_value)
class Recipe(GenericBaseRecipe):
def install(self):
args = dict(
requests=self.options['request-directory'],
sshkeygen=self.options['keygen-binary'],
)
wrapper = self.createPythonScript(self.options['wrapper'],
__name__ + '.sshkeys_authority', args)
return [wrapper]
return self.createPythonScript(self.options['wrapper'],
__name__ + '.sshkeys_authority',
(self.options['request-directory'],
self.options['keygen-binary']))
class Request(GenericBaseRecipe):
......@@ -160,11 +152,9 @@ class Request(GenericBaseRecipe):
os.symlink(self.private_key, private_key_link)
# end-XXX
wrapper = self.createPythonScript(
wrapper = self.createWrapper(
self.options['wrapper'],
'slapos.recipe.librecipe.execute.execute_wait',
[ [self.options['executable']],
[self.private_key, self.public_key] ])
(self.options['executable'],),
wait_list=(self.private_key, self.public_key))
return [request_file, wrapper, public_key_link, private_key_link]
slapos/recipe/stunnel/__init__.py
View file @ 342b5393
......@@ -30,13 +30,16 @@ import errno
from slapos.recipe.librecipe import GenericBaseRecipe
def post_rotate(args):
pid_file = args['pid_file']
if os.path.exist(pid_file):
with open(pid_file, 'r') as file_:
pid = file_.read().strip()
os.kill(pid, signal.SIGUSR1)
def kill(pid_file, sig=signal.SIGUSR1):
if os.path.exists(pid_file):
with open(pid_file) as f:
pid = int(f.read().strip())
try:
os.kill(pid, sig)
except OSError, e:
if e.errno != errno.ESRCH: # No such process
raise e
os.unlink(pid_file)
class Recipe(GenericBaseRecipe):
......@@ -76,28 +79,18 @@ class Recipe(GenericBaseRecipe):
self.substituteTemplate(template, conf))
path_list.append(conf_file)
wrapper = self.createPythonScript(
wrapper = self.createWrapper(
self.options['wrapper'],
'slapos.recipe.librecipe.execute.execute',
[self.options['stunnel-binary'], conf_file]
(self.options['stunnel-binary'], conf_file),
)
path_list.append(wrapper)
if os.path.exists(pid_file):
with open(pid_file, 'r') as file_:
pid = file_.read().strip()
# Reload configuration
try:
os.kill(int(pid, 10), signal.SIGHUP)
except OSError, e:
if e.errno == errno.ESRCH: # No such process
os.unlink(pid_file)
else:
raise e
# Reload configuration
kill(pid_file, signal.SIGHUP)
if 'post-rotate-script' in self.options:
self.createPythonScript(self.options['post-rotate-script'],
__name__ + 'post_rotate',
dict(pid_file=pid_file))
path_list.append(self.createPythonScript(
self.options['post-rotate-script'],
__name__ + '.kill', (pid_file,)))
return path_list
slapos/recipe/tidstorage/__init__.py
View file @ 342b5393
......@@ -36,19 +36,17 @@ class Recipe(GenericBaseRecipe):
r = [configuration_file]
wrapper = self.options.get('tidstorage-wrapper')
wrapper and r.append(self.createPythonScript(wrapper,
'slapos.recipe.librecipe.execute.execute',
[self.options['tidstoraged-binary'], '--nofork', '--config',
configuration_file]))
wrapper and r.append(self.createWrapper(wrapper,
(self.options['tidstoraged-binary'],
'--nofork', '--config', configuration_file)))
r.append(self.createPythonScript(
r.append(self.createWrapper(
self.options['repozo-wrapper'],
'slapos.recipe.librecipe.execute.execute',
[self.options['tidstorage-repozo-binary'],
(self.options['tidstorage-repozo-binary'],
'--config', configuration_file,
'--repozo', self.options['repozo-binary'],
'--gzip',
'--quick',
]))
)))
return r
slapos/recipe/waitfor.py deleted 100644 → 0
View file @ c44e2f89
##############################################################################
#
# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import shlex
from slapos.recipe.librecipe import GenericBaseRecipe
class Recipe(GenericBaseRecipe):
def install(self):
files = [f for f in self.options['files'].split('\n') if f]
command_line = shlex.split(self.options['command-line'])
wrapper = self.createPythonScript(
self.options['wrapper'],
'slapos.recipe.librecipe.execute.execute_wait',
[ command_line,
files ],
)
return [wrapper]
slapos/recipe/wrapper.py
View file @ 342b5393
......@@ -26,7 +26,6 @@
##############################################################################
import shlex
import os
from slapos.recipe.librecipe import GenericBaseRecipe
......@@ -38,56 +37,31 @@ class Recipe(GenericBaseRecipe):
:param lines wait-for-files: list of files to wait for
:param str pidfile: path to pidfile ensure exclusivity for the process
:param bool parameters-extra: whether wrapper parameters are passed onto command
:param str private-dev-shm: size of private /dev/shm, using user namespaces
:param bool reserve-cpu: command will ask for an exclusive CPU core
"""
def install(self):
command_line = shlex.split(self.options['command-line'])
args = shlex.split(self.options['command-line'])
wrapper_path = self.options['wrapper-path']
wait_files = self.options.get('wait-for-files')
environment = self.options.get('environment')
parameters_extra = self.options.get('parameters-extra')
pidfile = self.options.get('pidfile')
reserve_cpu = self.options.get('reserve-cpu', False)
private_dev_shm = self.options.get('private-dev-shm')
if not wait_files and not environment:
# Create a simple wrapper as shell script
return [self.createWrapper(
name=wrapper_path,
command=command_line[0],
parameters=command_line[1:],
parameters_extra=parameters_extra,
pidfile=pidfile,
reserve_cpu=reserve_cpu
)]
environment = {}
for line in (self.options.get('environment') or '').splitlines():
line = line.strip()
if line:
k, v = line.split('=')
environment[k.rstrip()] = v.lstrip()
# More complex needs: create a Python script as wrapper
if wait_files is not None:
wait_files = [filename.strip() for filename in wait_files.split()
if filename.strip()]
if environment is not None:
environment = dict((k.strip(), v.strip()) for k, v in [
line.split('=') for line in environment.splitlines() if line.strip() ])
# We create a python script and a wrapper around the python
# script because the python script might have a too long #! line
if os.path.exists(os.path.join(self.buildout['buildout']['directory'], "bin")):
base_script_path = os.path.join(
self.buildout['buildout']['directory'], "bin/" + wrapper_path.split("/")[-1])
else:
base_script_path = os.path.join(
self.buildout['buildout']['directory'], wrapper_path.split("/")[-1])
python_script = self.createPythonScript(
base_script_path +'.py',
'slapos.recipe.librecipe.execute.generic_exec',
(command_line, wait_files, environment,), )
return [python_script, self.createWrapper(
name=wrapper_path,
command=python_script,
parameters=[],
parameters_extra=parameters_extra,
pidfile=pidfile,
reserve_cpu=reserve_cpu
)]
kw = {}
if wait_files:
kw['wait_list'] = wait_files.split()
if pidfile:
kw['pidfile'] = pidfile
if private_dev_shm:
kw['private_dev_shm'] = private_dev_shm
if self.isTrueValue(self.options.get('reserve-cpu')):
kw['reserve_cpu'] = True
return self.createWrapper(wrapper_path, args, environment, **kw)
slapos/recipe/zabbixagent/__init__.py
View file @ 342b5393
......@@ -71,10 +71,10 @@ class Recipe(BaseSlapRecipe):
self._createDirectory(cron_d)
self._createDirectory(crontabs)
wrapper = zc.buildout.easy_install.scripts([('crond',
'slapos.recipe.librecipe.execute', 'execute')], self.ws, sys.executable,
self.wrapper_directory, arguments=[
'slapos.recipe.librecipe.execute', 'generic_exec')], self.ws,
sys.executable, self.wrapper_directory, arguments=repr((
self.options['dcrond_binary'].strip(), '-s', cron_d, '-c', crontabs,
'-t', timestamps, '-f', '-l', '5', '-M', catcher]
'-t', timestamps, '-f', '-l', '5', '-M', catcher))[1:-1]
)[0]
self.path_list.append(wrapper)
return cron_d
......@@ -101,10 +101,10 @@ class Recipe(BaseSlapRecipe):
self.path_list.append(zabbix_agentd_path)
wrapper = zc.buildout.easy_install.scripts([('zabbixagentd',
'slapos.recipe.librecipe.execute', 'execute')], self.ws, sys.executable,
self.bin_directory, arguments=[
'slapos.recipe.librecipe.execute', 'generic_exec')], self.ws,
sys.executable, self.bin_directory, arguments=repr((
self.options['zabbix_agentd_binary'].strip(), '-c',
zabbix_agentd_path])[0]
zabbix_agentd_path))[1:-1])[0]
self.path_list.extend(zc.buildout.easy_install.scripts([
('zabbixagentd', __name__ + '.svcdaemon', 'svcdaemon')],
......
slapos/recipe/zeo/__init__.py
View file @ 342b5393
......@@ -89,10 +89,9 @@ class Recipe(GenericBaseRecipe):
self.substituteTemplate(template_filename, config))
# Create running wrapper
wrapper_path = self.createPythonScript(
wrapper_path = self.createWrapper(
self.options['wrapper-path'],
'slapos.recipe.librecipe.execute.execute',
arguments=[self.options['binary-path'].strip(), '-C',
self.options['conf-path']],)
(self.options['binary-path'].strip(),
'-C', self.options['conf-path']))
return [configuration_path, wrapper_path]
slapos/test/recipe/test_re6stnet.py
View file @ 342b5393
......@@ -79,15 +79,14 @@ class Re6stnetTest(unittest.TestCase):
token_file = os.path.join(self.options['conf-dir'], 'token.json')
with open(path, 'r') as f:
content = f.read()
self.assertIn("'token_json': '%s'" % token_file, content)
self.assertIn("('http://%s:%s/', %r, %r," % (
self.options['ipv4'], self.options['port'], self.token_dir, token_file),
content)
self.assertIn("'partition_id': 'slappart0'", content)
self.assertIn("'computer_id': 'comp-test'", content)
self.assertIn("'computer_guid': 'comp-test'", content)
self.assertIn("'key_file': '/path/to/key'", content)
self.assertIn("'cert_file': '/path/to/cert'", content)
self.assertIn("'server_url': 'http://server.com'", content)
self.assertIn("'token_base_path': '%s'" % self.token_dir, content)
self.assertIn("'registry_url': 'http://%s:%s/'" % (self.options['ipv4'],
self.options['port']), content)
self.assertIn("'master_url': 'http://server.com'", content)
def fake_generateCertificates(self):
return
......
software/agent/software.cfg
View file @ 342b5393
......@@ -50,8 +50,8 @@ gitdb = 0.6.4
pycrypto = 2.6.1
pycurl = 7.43.0
slapos.recipe.download = 1.0
slapos.recipe.template = 4.2
slapos.toolbox = 0.73
slapos.recipe.template = 4.3
slapos.toolbox = 0.74
smmap = 0.9.0
# Required by:
......
software/apache-frontend/software.cfg
View file @ 342b5393
......@@ -10,8 +10,8 @@ gitdb = 0.6.4
plone.recipe.command = 1.1
pycrypto = 2.6.1
rdiff-backup = 1.0.5+SlapOSPatched001
slapos.recipe.template = 4.2
slapos.toolbox = 0.73
slapos.recipe.template = 4.3
slapos.toolbox = 0.74
smmap = 0.9.0
numpy = 1.11.2
pyasn1 = 0.2.3
......
software/cdn-me/software.cfg
View file @ 342b5393
......@@ -40,4 +40,4 @@ cns.recipe.symlink = 0.2.3
collective.recipe.environment = 0.2.0
erp5.util = 0.4.49
plone.recipe.command = 1.1
slapos.recipe.template = 4.2
slapos.recipe.template = 4.3
software/cloudooo/buildout.hash.cfg 0 → 100644
View file @ 342b5393
# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
# The only allowed lines here are (regexes):
# - "^#" comments, copied verbatim
# - "^[" section beginings, copied verbatim
# - lines containing an "=" sign which must fit in the following categorie.
# - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
# But avoid directories, they are not portable.
# Copied verbatim.
# - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
# by the re-generation script.
# Re-generated.
# - other lines are copied verbatim
# Substitution (${...:...}), extension ([buildout] extends = ...) and
# section inheritance (< = ...) are NOT supported (but you should really
# not need these here).
[template-cloudooo]
filename = instance.cfg.in
md5sum = 6e4431cf4b0a0d034402604b1e2844c0
[template-cloudooo-instance]
filename = instance-cloudooo.cfg.in
md5sum = afbfed1d762e5cdf7c6fd1292e7b28e7
software/cloudooo/instance-cloudooo.cfg.in
View file @ 342b5393
......@@ -10,6 +10,7 @@
{% set next_port = slapparameter_dict.get(instance_parameter_dict['port-parameter-name'], next_port) | int -%}
{% endif -%}
{% do assert(next_port > 0) -%}
{% set next_port = itertools.count(next_port).next -%}
{% set backend_count = instance_parameter_dict['backend-count'] | int -%}
{% if instance_parameter_dict.get('backend-count-parameter-name') -%}
......@@ -33,9 +34,8 @@
{% set mimetype_entry_addition = mimetype_entry_addition ~ "\n" ~ slapparameter_dict.get(instance_parameter_dict['mimetype-entry-addition-parameter-name'], '') -%}
{% endif -%}
{% set apache_port = next_port -%}
{% set haproxy_port = next_port + 1 -%}
{% set next_port = next_port + 2 -%}
{% set apache_port = next_port() -%}
{% set haproxy_port = next_port() -%}
{% set apache_ip_list = [ipv4] -%}
{% if ipv6_set -%}
......@@ -184,13 +184,12 @@ ooo-uno-path = {{ parameter_dict['libreoffice-bin'] }}/basis-link/program
{% set name = 'cloudooo-' ~ index -%}
[{{ cloudooo(name) }}]
< = cloudooo-base
port = {{ next_port }}
openoffice-port = {{ next_port + 1 }}
port = {{ next_port() }}
openoffice-port = {{ next_port() }}
configuration-file = ${directory:etc}/{{ name }}.cfg
data-directory = ${directory:srv}/{{ name }}
wrapper = ${directory:services}/{{ name }}
{% set next_port = next_port + 2 -%}
{% endfor -%}
[haproxy]
......
software/cloudooo/instance.cfg.in
View file @ 342b5393
......@@ -39,6 +39,7 @@ extensions = jinja2.ext.do
extra-context =
section parameter_dict cloudooo-dynamic-template-parameter-dict
key ipv4_set slap-parameters:ipv4
import itertools itertools
[switch-softwaretype]
recipe = slapos.cookbook:softwaretype
......
software/cloudooo/software-common.cfg
View file @ 342b5393
[buildout]
extends =
buildout.hash.cfg
../../stack/cloudooo.cfg
../../stack/logrotate/buildout.cfg
parts =
......@@ -25,8 +26,7 @@ cert = ${slap-connection:cert-file}
recipe = slapos.recipe.template:jinja2
# XXX: "template.cfg" is hardcoded in instanciation recipe
rendered = ${buildout:directory}/template.cfg
template = ${:_profile_base_location_}/instance.cfg.in
md5sum = 4543e62c41726ef074454252c4951ac8
template = ${:_profile_base_location_}/${:filename}
mode = 640
[template-cloudooo-base]
......@@ -81,6 +81,5 @@ template-logrotate-base = ${template-logrotate-base:rendered}
[template-cloudooo-instance]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/instance-cloudooo.cfg.in
md5sum = 045830db85e1200f765788aa8b62289e
url = ${:_profile_base_location_}/${:filename}
mode = 640
software/dream/software.cfg
View file @ 342b5393
......@@ -53,7 +53,7 @@ mysqlclient = 1.3.12
# indirect dependancies
cp.recipe.cmd = 0.5
plone.recipe.command = 1.1
slapos.recipe.template = 4.2
slapos.recipe.template = 4.3
zope.exceptions = 4.0.7
zope.testing = 4.1.3
zc.recipe.testrunner = 2.0.0
......
software/erp5/README.rst
View file @ 342b5393
......@@ -3,12 +3,8 @@ Available ``software-type`` values
- ``default``
Recommended for production use.
- ``create-erp5-site``
Automated creation of ERP5Site instance, for easy deployment.
Usage in production discouraged due to the increased risk of data loss.
Recommended for developemnt and production use. Automatic creation of
erp5-site.
Notes
=====
......
software/erp5/instance-erp5-input-schema.json
View file @ 342b5393
......@@ -117,7 +117,11 @@
"timerserver-interval": {
"description": "Timerserver tick perdiod, in seconds, or 0 to disable",
"default": 5,
"type": "integer"
"type": "number"
},
"private-dev-shm": {
"description": "Size of private /dev/shm for wendelin.core. If sysctl kernel.unprivileged_userns_clone exists, it must be set to 1.",
"type": "string"
},
"ssl-authentication": {
"title": "Enable SSL Client authentication on this zope instance.",
......
software/erp5/software.cfg.json
View file @ 342b5393
......@@ -9,13 +9,6 @@
"request": "instance-erp5-input-schema.json",
"response": "instance-erp5-output-schema.json",
"index": 0
},
"create-erp5-site": {
"title": "Create ERP5 Site",
"description": "Automated ERP5Site creation on instanciation when ZODB is found empty.",
"request": "instance-erp5-input-schema.json",
"response": "instance-erp5-output-schema.json",
"index": 1
}
}
}
software/erp5testnode/software.cfg
View file @ 342b5393
......@@ -59,7 +59,7 @@ mode = 0644
[versions]
PyXML = 0.8.5
erp5.util = 0.4.50
slapos.recipe.template = 4.2
slapos.recipe.template = 4.3
ipython = 5.3.0
apache-libcloud = 2.1.0
gitdb2 = 2.0.2
......
software/erp5testnode/testsuite/deploy-test/runTestSuite.py
View file @ 342b5393
......@@ -106,6 +106,7 @@ def waitForSite(partition_path):
date=strftime("%Y/%m/%d %H:%M:%S", gmtime(end)),
duration=end - start,
)
print(try_info + 'status_dict %r' % (status_dict,))
return status_dict
......
software/erp5testnode/testsuite/dummy/software.cfg
View file @ 342b5393
......@@ -15,4 +15,4 @@ md5sum = efd3b712a2294207f265a9c45648d5cf
mode = 0644
[versions]
slapos.recipe.template = 4.2
slapos.recipe.template = 4.3
software/gitlab/software.cfg
View file @ 342b5393
......@@ -381,5 +381,5 @@ cns.recipe.symlink = 0.2.3
docutils = 0.12
plone.recipe.command = 1.1
rubygemsrecipe = 0.2.2+slapos001
slapos.recipe.template = 4.2
slapos.recipe.template = 4.3
z3c.recipe.scripts = 1.0.1
software/html5as/software.cfg
View file @ 342b5393
......@@ -72,7 +72,7 @@ async = 0.6.1
gitdb = 0.5.4
pycrypto = 2.6
rdiff-backup = 1.0.5+SlapOSPatched001
slapos.recipe.template = 4.2
slapos.recipe.template = 4.3
slapos.toolbox = 0.40.4
smmap = 0.8.2
plone.recipe.command = 1.1
......
software/html5ide/software.cfg
View file @ 342b5393
......@@ -48,4 +48,4 @@ md5sum = 8cde04bfd0c0e9bd56744b988275cfd8
PyRSS2Gen = 1.1
cns.recipe.symlink = 0.2.3
plone.recipe.command = 1.1
slapos.recipe.template = 4.2
slapos.recipe.template = 4.3
software/jstestnode/software.cfg
View file @ 342b5393
......@@ -113,5 +113,5 @@ mode = 0644
[versions]
erp5.util = 0.4.50
slapos.recipe.template = 4.2
slapos.recipe.template = 4.3
selenium = 3.8.0
software/jupyter/software.cfg
View file @ 342b5393
......@@ -85,7 +85,7 @@ pyzmq = 16.0.2
scikit-learn = 0.18.1
seaborn = 0.7.1
simplegeneric = 0.8.1
slapos.recipe.template = 4.2
slapos.recipe.template = 4.3
statsmodels = 0.8.0
terminado = 0.6
tornado = 4.4.2
......
software/kumofs/instance.cfg deleted 100644 → 0
View file @ c44e2f89
[buildout]
parts =
instance
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
[instance]
recipe = ${instance-recipe:egg}:${instance-recipe:module}
kumo_gateway_binary = ${kumo:location}/bin/kumo-gateway
kumo_manager_binary = ${kumo:location}/bin/kumo-manager
kumo_server_binary = ${kumo:location}/bin/kumo-server
dcrond_binary = ${dcron:location}/sbin/crond
openssl_binary = ${openssl:location}/bin/openssl
rdiff_backup_binary = ${buildout:bin-directory}/rdiff-backup
stunnel_binary = ${stunnel:location}/bin/stunnel
software/kumofs/software.cfg deleted 100644 → 0
View file @ c44e2f89
[buildout]
extends =
../../component/kumo/buildout.cfg
../../component/dcron/buildout.cfg
../../component/stunnel/buildout.cfg
../../component/rdiff-backup/buildout.cfg
../../component/lxml-python/buildout.cfg
../../stack/slapos.cfg
parts =
# Create instance template
template
rdiff-backup
dcron
kumo
stunnel
eggs
instance-recipe-egg
[instance-recipe]
egg = slapos.cookbook
module = kumofs
[instance-recipe-egg]
recipe = zc.recipe.egg
eggs = ${instance-recipe:egg}
[eggs]
recipe = zc.recipe.egg
eggs =
${lxml-python:egg}
[template]
# Default template for the instance.
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg
md5sum = 056a4af7128fd9e31da42c85cc039420
output = ${buildout:directory}/template.cfg
mode = 0644
[versions]
rdiff-backup = 1.0.5+SlapOSPatched001
slapos.recipe.template = 2.4.2
software/kvm/common.cfg
View file @ 342b5393
......@@ -41,6 +41,7 @@ parts = ${:common-parts}
[eggs]
recipe = zc.recipe.egg
interpreter = python.eggs
eggs =
${python-cffi:egg}
${python-cryptography:egg}
......@@ -89,7 +90,7 @@ command =
[template]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg.in
md5sum = bf5ef731c0d8da0267a4939882b4eeee
md5sum = 5a17fc127190bbc19361c5ffb10711b3
output = ${buildout:directory}/template.cfg
mode = 0644
......@@ -98,7 +99,7 @@ recipe = hexagonit.recipe.download
ignore-existing = true
url = ${:_profile_base_location_}/instance-kvm.cfg.jinja2
mode = 644
md5sum = e2b8f86bdc12c86e7d959b55c6d54f6d
md5sum = 68b66fb3e9020642e57f4a4ee266f2b3
download-only = true
on-update = true
......@@ -107,7 +108,7 @@ recipe = hexagonit.recipe.download
ignore-existing = true
url = ${:_profile_base_location_}/instance-kvm-cluster.cfg.jinja2.in
mode = 644
md5sum = 05b6004e8c7a94de14f247affcef4971
md5sum = ba3337b3678ed9d3578cc88749c5cd13
download-only = true
on-update = true
......@@ -185,7 +186,7 @@ ignore-existing = true
url = ${:_profile_base_location_}/template/template-kvm-run.in
mode = 644
filename = template-kvm-run.in
md5sum = 178a24cdad77cb6c2e519ac629dd0e74
md5sum = bd238397af6236b6b24b693012feeece
download-only = true
on-update = true
......@@ -195,7 +196,7 @@ ignore-existing = true
url = ${:_profile_base_location_}/template/kvm-controller-run.in
mode = 644
filename = kvm-controller-run.in
md5sum = 71afd2d13f6e56993ae413a168e012d7
md5sum = c86cd67bbdd26b7b14b7449a1bbd959b
download-only = true
on-update = true
......@@ -225,7 +226,7 @@ ignore-existing = true
url = ${:_profile_base_location_}/template/qemu-is-ready.in
mode = 644
filename = qemu-is-ready.in
md5sum = 0066fa0f5f3dd47bded5e5924df2550d
md5sum = b304eec8e2cb71f10ea83cac22f6db12
download-only = true
on-update = true
......
software/kvm/instance-kvm-cluster-input-schema.json
View file @ 342b5393
......@@ -189,13 +189,35 @@
"default": "started",
"enum": ["started", "stopped"]
},
"enable-device-hotplug": {
"title": "Enable device hotplug mode",
"description": "If yes, this will allow to Create devices like CPU and Memory in hotplug mode without restart the VM. Operatin System should be configured to Online new created devices.",
"type": "boolean",
"default": false
},
"ram-size": {
"title": "RAM size",
"description": "RAM size, in MB.",
"type": "integer",
"default": 1024,
"minimum": 128,
"multipleOf": 128
"minimum": 1024,
"multipleOf": 512
},
"ram-max-size": {
"title": "Maximum RAM size, in MB",
"description": "Define the maximum size of the memory. The size is in MB and should be a multiple of 512.",
"type": "integer",
"default": 51200,
"minimum": 1024,
"multipleOf": 512
},
"ram-hotplug-slot-size": {
"title": "Size of Hotpluggable RAM slot, in MB",
"description": "Define the RAM size to plug on one hotpluggable slot in MB, understand the size of one RAM bar. The RAM hotplugged on each slot will always have the same RAM size.",
"type": "integer",
"default": 512,
"minimum": 512,
"multipleOf": 512
},
"auto-ballooning": {
"title": "Enable qemu auto ballooning.",
......@@ -245,10 +267,13 @@
"type": "integer",
"minimum": 1
},
"cpu-options": {
"title": "CPU Additional options: cores, threads, sockets, maxcpus.",
"description": "Additional options to use with cpu-count. Options are separated by coma: [cores=cores][,threads=threads][,sockets=sockets][,maxcpus=maxcpus]. Set this option if you know what you're doing.",
"type": "string"
"cpu-max-count": {
"title": "Maximum CPU amount",
"description": "Specifies the maximum number of CPUs.",
"type": "integer",
"default": 24,
"minimum": 1,
"maximum": 64
},
"numa": {
"title": "Simulate a multi node NUMA system.",
......
software/kvm/instance-kvm-cluster.cfg.jinja2.in
View file @ 342b5393
......@@ -8,7 +8,7 @@
{% set slave_frontend_iguid = slave_frontend_dict.get('instance-guid', '') -%}
{% set kvm_instance_dict = {} -%}
{% set kvm_hostname_list = [] -%}
{% set monitor_url_list = [] -%}
{% set monitor_base_url_dict = {} -%}
{% macro setconfig(name, value) -%}
{# will set a config-name = value if value is not empty -#}
......@@ -51,12 +51,15 @@ config-authorized-key = {{ dumps(slapparameter_dict.get('authorized-keys') | joi
config-nbd-port = {{ dumps(kvm_parameter_dict.get('nbd-port', 1024)) }}
config-nbd2-port = {{ dumps(kvm_parameter_dict.get('nbd-port2', 1024)) }}
config-ram-size = {{ dumps(kvm_parameter_dict.get('ram-size', 1024)) }}
config-ram-max-size = {{ dumps(kvm_parameter_dict.get('ram-max-size', '51200')) }}
config-enable-device-hotplug = {{ dumps(kvm_parameter_dict.get('enable-device-hotplug', False)) }}
config-ram-hotplug-slot-size = {{ dumps(kvm_parameter_dict.get('ram-hotplug-slot-size', 512)) }}
config-disk-size = {{ dumps(kvm_parameter_dict.get('disk-size', 10)) }}
config-disk-type = {{ dumps(kvm_parameter_dict.get('disk-type', 'virtio')) }}
config-cpu-count = {{ dumps(kvm_parameter_dict.get('cpu-count', 1)) }}
config-cpu-max-count = {{ dumps(kvm_parameter_dict.get('cpu-max-count', 24)) }}
{{ setconfig('numa', kvm_parameter_dict.get('numa', '')) }}
{{ setconfig('machine-options', kvm_parameter_dict.get('machine-options', '')) }}
{{ setconfig('cpu-options', kvm_parameter_dict.get('cpu-options', '')) }}
{{ setconfig('nbd-host', kvm_parameter_dict.get('nbd-host', '')) }}
{{ setconfig('host2', kvm_parameter_dict.get('host2', '')) }}
......@@ -67,7 +70,7 @@ config-auto-ballooning = {{ dumps(kvm_parameter_dict.get('auto-ballooning', True
{{ setconfig('disk-cache', kvm_parameter_dict.get('disk-cache', '')) }}
{% set nat_rules_list = kvm_parameter_dict.get('nat-rules', []) -%}
{{ setconfig('nat-rules', ' '.join(nat_rules_list)) }}
{{ setconfig('nat-rules', nat_rules_list | join(' ')) }}
config-publish-nat-url = True
config-use-nat = {{ use_nat }}
config-use-tap = {{ dumps(kvm_parameter_dict.get('use-tap', True)) }}
......@@ -124,10 +127,10 @@ return =
{% if str(kvm_parameter_dict.get('use-tap', 'True')).lower() == 'true' -%}
{{ ' ' }}tap-ipv4
{% do monitor_url_list.append('${' ~ section ~ ':connection-monitor-base-url}') -%}
{% do publish_dict.__setitem__('lan-' ~ instance_name, '${' ~ section ~ ':connection-tap-ipv4}') -%}
{% do kvm_hostname_list.append(instance_name ~ ' ' ~ '${' ~ section ~ ':connection-tap-ipv4}') -%}
{% endif -%}
{% do monitor_base_url_dict.__setitem__(instance_name, '${' ~ section ~ ':connection-monitor-base-url}') -%}
{% do publish_dict.__setitem__(instance_name ~ '-backend-url', '${' ~ section ~ ':connection-backend-url}') -%}
{% do publish_dict.__setitem__(instance_name ~ '-url', '${' ~ section ~ ':connection-url}') -%}
{% do kvm_instance_dict.__setitem__(instance_name, (use_nat, nat_rules_list)) -%}
......@@ -241,11 +244,12 @@ cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.of
username = admin
password = ${monitor-htpasswd:passwd}
[monitor-conf-parameters]
monitor-url-list +=
{% for url in monitor_url_list -%}
{{ ' ' ~ url }}
[monitor-base-url-dict]
{% for key, value in monitor_base_url_dict.items() -%}
{{ key }} = {{ value }}
{% endfor %}
[monitor-conf-parameters]
private-path-list +=
${directory:webroot}/
......
software/kvm/instance-kvm-input-schema.json
View file @ 342b5393
......@@ -4,13 +4,35 @@
"title": "Input Parameters",
"properties": {
"enable-device-hotplug": {
"title": "Enable device hotplug mode",
"description": "If yes, this will allow to Create devices like CPU and Memory in hotplug mode without restart the VM. Operatin System should be configured to Online new created devices.",
"type": "boolean",
"default": false
},
"ram-size": {
"title": "RAM size",
"description": "RAM size, in MB.",
"type": "integer",
"default": 1024,
"minimum": 128,
"multipleOf": 128
"minimum": 1024,
"multipleOf": 512
},
"ram-max-size": {
"title": "Maximum RAM size, in MB",
"description": "Define the maximum size of the memory. The size is in MB and should be a multiple of 512.",
"type": "integer",
"default": 51200,
"minimum": 1024,
"multipleOf": 512
},
"ram-hotplug-slot-size": {
"title": "Size of Hotpluggable RAM slot, in MB",
"description": "Define the RAM size to plug on one hotpluggable slot in MB, understand the size of one RAM bar. The RAM hotplugged on each slot will always have the same RAM size.",
"type": "integer",
"default": 512,
"minimum": 512,
"multipleOf": 512
},
"auto-ballooning": {
"title": "Enable qemu auto ballooning.",
......@@ -54,10 +76,25 @@
"type": "integer",
"minimum": 1
},
"cpu-options": {
"title": "CPU Additional options: cores, threads, sockets, maxcpus.",
"description": "Additional options to use with cpu-count. Options are separated by coma: [cores=cores][,threads=threads][,sockets=sockets][,maxcpus=maxcpus]. Set this option if you know what you're doing.",
"type": "string"
"cpu-max-count": {
"title": "Maximum CPU amount",
"description": "Specifies the maximum number of CPUs.",
"type": "integer",
"default": 24,
"minimum": 1,
"maximum": 64
},
"max-cpu-hotplug-count": {
"title": "Maximum hotpluggable CPU amount",
"description": "Specifies the maximum number of hotpluggable CPUs.",
"type": "integer",
"default": 1
},
"cpu-hotplug-amount": {
"title": "CPU hotplug amount",
"description": "Specifies the number of CPUs to hotplug",
"type": "integer",
"default": 0
},
"numa": {
"title": "Simulate a multi node NUMA system.",
......
software/kvm/instance-kvm.cfg.jinja2
View file @ 342b5393
......@@ -5,6 +5,7 @@
{% set nat_restrict = slapparameter_dict.get('nat-restrict-mode', 'False').lower() -%}
{% set name = slapparameter_dict.get('name', 'localhost') -%}
{% set disable_ansible_promise = slapparameter_dict.get('disable-ansible-promise', 'True').lower() -%}
{% set enable_device_hotplug = slapparameter_dict.get('enable-device-hotplug', 'false').lower() -%}
{% set instance_type = slapparameter_dict.get('type', 'standalone') -%}
{% set nat_rule_list = slapparameter_dict.get('nat-rules', '22 80 443') -%}
{% set frontend_software_type = 'default' -%}
......@@ -62,10 +63,15 @@ storage-path = ${directory:srv}/passwd
bytes = 8
[kvm-controller-parameter-dict]
python-path = {{ python_executable }}
python-path = {{ python_eggs_executable }}
vnc-passwd = ${gen-passwd:passwd}
socket-path = ${directory:var}/qmp_socket
pid-file = ${directory:run}/pid_file
kvm-status-path = ${directory:var}/qemu-vm-is-ready
cpu-count = ${slap-parameter:cpu-count}
ram-hotplug-slot-size = ${slap-parameter:ram-hotplug-slot-size}
ram-size = ${slap-parameter:ram-size}
enable-device-hotplug = ${slap-parameter:enable-device-hotplug}
[kvm-parameter-dict]
python-path = {{ python_executable }}
......@@ -87,13 +93,16 @@ disk-size = ${slap-parameter:disk-size}
disk-type = ${slap-parameter:disk-type}
pid-file-path = ${directory:run}/pid_file
pid-file-path = ${kvm-controller-parameter-dict:pid-file}
socket-path = ${kvm-controller-parameter-dict:socket-path}
smp-count = ${slap-parameter:cpu-count}
smp-options = ${slap-parameter:cpu-options}
ram-size = ${slap-parameter:ram-size}
numa = ${slap-parameter:numa}
enable-device-hotplug = ${kvm-controller-parameter-dict:enable-device-hotplug}
smp-count = ${kvm-controller-parameter-dict:cpu-count}
smp-max-count = ${slap-parameter:cpu-max-count}
ram-size = ${kvm-controller-parameter-dict:ram-size}
ram-max-size = ${slap-parameter:ram-max-size}
init-ram-size = 1024
mac-address = ${create-mac:mac-address}
tap-mac-address = ${create-tap-mac:mac-address}
......@@ -143,7 +152,7 @@ disk-cache = ${slap-parameter:disk-cache}
disk-aio = ${slap-parameter:disk-aio}
auto-ballooning = ${slap-parameter:auto-ballooning}
machine-options = ${slap-parameter:machine-options}
cpu-options = ${slap-parameter:cpu-model}
cpu-hotplug-slot-size = ${slap-parameter:cpu-model}
log-file = ${directory:log}/qemu.log
......@@ -550,15 +559,15 @@ nbd-host =
nbd2-port = 1024
nbd2-host =
enable-device-hotplug = False
ram-size = 1024
ram-max-size = 51200
ram-hotplug-slot-size = 512
disk-size = 10
disk-type = virtio
cpu-count = 1
# cpu-option is a string: [cores=cores][,threads=threads][,sockets=sockets][,maxcpus=maxcpus]
cpu-options =
# list of numa options separate by space: node,nodeid=1,cpus=9-15 node,nodeid=2,cpus=1,3,7
numa =
cpu-max-count = 24
disk-cache = writeback
disk-aio = native
auto-ballooning = True
......
software/kvm/instance.cfg.in
View file @ 342b5393
......@@ -93,6 +93,7 @@ context =
raw netcat_bin ${netcat:location}/bin/netcat
raw openssl_executable_location ${openssl:location}/bin/openssl
raw python_executable ${buildout:executable}
raw python_eggs_executable ${buildout:bin-directory}/${eggs:interpreter}
raw qemu_executable_location ${kvm:location}/bin/qemu-system-x86_64
raw qemu_img_executable_location ${kvm:location}/bin/qemu-img
raw qemu_start_promise_tpl ${template-qemu-ready:location}/${template-qemu-ready:filename}
......
software/kvm/software.cfg
View file @ 342b5393
......@@ -5,13 +5,13 @@ extends = common.cfg
# XXX - use websockify = 0.5.1 for compatibility with kvm frontend
websockify = 0.5.1
slapos.toolbox = 0.73
slapos.toolbox = 0.74
erp5.util = 0.4.49
apache-libcloud = 1.1.0
collective.recipe.environment = 0.2.0
gitdb = 0.6.4
pycurl = 7.43.0
slapos.recipe.template = 4.2
slapos.recipe.template = 4.3
smmap = 0.9.0
# websockify = 0.8.0
......@@ -45,4 +45,4 @@ paramiko = 2.0.2
# Required by:
# slapos.toolbox==0.71
passlib = 1.6.5
\ No newline at end of file
passlib = 1.6.5
software/kvm/template/kvm-controller-run.in
View file @ 342b5393
......@@ -6,40 +6,62 @@
import socket
import time
import os
from slapos.qemuqmpclient import QemuQMPWrapper, getInitialQemuResourceDict
# XXX: to be factored with slapos.toolbox qemu qmp wrapper.
socket_path = '{{ parameter_dict.get("socket-path") }}'
pid_file = '{{ parameter_dict.get("pid-file") }}'
vnc_password = '{{ parameter_dict.get("vnc-passwd") }}'
status_path = '{{ parameter_dict.get("kvm-status-path") }}'
cpu_amount = {{ parameter_dict.get("cpu-count") }}
cpu_model = '{{ parameter_dict.get("cpu-model", "qemu64-x86_64-cpu") }}'
slot_hotplug_size = {{ parameter_dict.get("ram-hotplug-slot-size", 512) }}
ram_size = {{ parameter_dict.get("ram-size") }}
enable_device_hotplug = '{{ parameter_dict.get("enable-device-hotplug") }}'.lower()
if os.path.exists(status_path):
os.unlink(status_path)
def write(message):
with open(status_path, 'w') as status_file:
status_file.write(message)
def update():
# Connect to KVM qmp socket
so = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
connected = False
while not connected:
try:
so.connect(socket_path)
except socket.error:
time.sleep(1)
else:
connected = True
data = so.recv(1024)
# Enable qmp
so.send('{ "execute": "qmp_capabilities" }')
data = so.recv(1024)
# Set VNC password
so.send('{ "execute": "change", ' \
'"arguments": { "device": "vnc", "target": "password", ' \
' "arg": "' + vnc_password + '" } }')
data = so.recv(1024)
# Finish
so.close()
with open(status_path, 'w') as status_file:
status_file.write("OK")
init_dict = getInitialQemuResourceDict(pid_file)
if os.path.exists(status_path):
os.unlink(status_path)
if init_dict is None:
# qemu process is not OK
return
init_ram_size = int(init_dict['ram'].split('M')[0])
if cpu_amount < 1:
raise ValueError("CPU should be at least equal to 1.")
hotplug_ram = ram_size - init_ram_size
if hotplug_ram < 0:
raise ValueError("RAM size cannot be less than the initial value %s MB" % init_ram_size)
qemu_wrapper = QemuQMPWrapper(socket_path)
qemu_wrapper.setVNCPassword(vnc_password)
if enable_device_hotplug == 'true':
write("Qemu Controller is still running...")
qemu_wrapper.updateDevice({
'device': 'cpu',
'amount': cpu_amount,
'model': cpu_model
})
qemu_wrapper.updateDevice({
'device': 'memory',
'mem': hotplug_ram,
'slot': slot_hotplug_size,
'nslot': 128,
'canreboot': 1
})
except Exception, e:
write(str(e))
raise
write("")
if __name__ == "__main__":
update()
software/kvm/template/qemu-is-ready.in
View file @ 342b5393
#!{{ dash }}
if [ -f "{{ qemu_ready_path }}" ]; then
echo "VM correctly started."
FILE="{{ qemu_ready_path }}"
# don't start checks too fast
sleep 2
if [ -f "$FILE" ]; then
if [ "$(cat $FILE)" = "" ]; then
echo "VM correctly started."
else
>&2 echo "Qemu Controller failed"
>&2 cat $FILE
exit 1
fi
else
log_file="{{ qemu_service_log_file }}"
>&2 echo "Qemu process is not correctly started."
......
software/kvm/template/template-kvm-run.in
View file @ 342b5393
......@@ -39,13 +39,13 @@ tap_interface = '{{ parameter_dict.get("tap-interface") }}'
listen_ip = '{{ parameter_dict.get("ipv4") }}'
mac_address = '{{ parameter_dict.get("mac-address") }}'
tap_mac_address = '{{ parameter_dict.get("tap-mac-address") }}'
smp_count = '{{ parameter_dict.get("smp-count") }}'
smp_options = '{{ parameter_dict.get("smp-options") }}'.strip()
numa_list = '{{ parameter_dict.get("numa") }}'.split()
ram_size = '{{ parameter_dict.get("ram-size") }}'
numa_list = '{{ parameter_dict.get("numa", "") }}'.split()
ram_size = {{ parameter_dict.get("ram-size") }}
ram_max_size = '{{ parameter_dict.get("ram-max-size") }}'
init_ram_size = {{ parameter_dict.get("init-ram-size") }}
pid_file_path = '{{ parameter_dict.get("pid-file-path") }}'
external_disk_number = {{ parameter_dict.get("external-disk-number") }}
external_disk_size = '{{ parameter_dict.get("external-disk-size") }}'
external_disk_size = {{ parameter_dict.get("external-disk-size") }}
external_disk_format = '{{ parameter_dict.get("external-disk-format") }}'
disk_storage_dict = {}
disk_storage_list = """{{ parameter_dict.get("disk-storage-list") }}""".split('\n')
......@@ -69,9 +69,13 @@ disk_cache = disk_cache if disk_cache in ["none", "writeback", "unsafe",
disk_aio = '{{ parameter_dict.get("disk-aio") }}'.strip()
disk_aio = disk_aio if disk_aio in ["threads", "native"] and \
disk_cache == "directsync" else "threads"
smp_count = {{ parameter_dict.get("smp-count") }}
smp_max_count = {{ parameter_dict.get("smp-max-count") }}
machine_options = '{{ parameter_dict.get("machine-options", "") }}'.strip()
cpu_model = '{{ parameter_dict.get("cpu-model", "") }}'.strip()
enable_device_hotplug = '{{ parameter_dict.get("enable-device-hotplug") }}'.lower()
logfile = '{{ parameter_dict.get("log-file") }}'
if hasattr(ssl, '_create_unverified_context') and url_check_certificate == 'false':
......@@ -255,15 +259,14 @@ if use_tap == 'true':
tap_interface, vhost),
'-device', 'virtio-net-pci,netdev=lan%s,mac=%s' % (number, tap_mac_address)]
smp = smp_count
if smp_options:
for option in smp_options.split(','):
key, val = option.split('=')
if key in ('cores', 'threads', 'sockets', 'maxcpus') and val.isdigit():
smp += ',%s=%s' % (key, val)
if enable_device_hotplug != 'true':
smp = '%s,maxcpus=%s' % (smp_count, smp_max_count)
ram = '%sM,slots=128,maxmem=%sM' % (ram_size, ram_max_size)
else:
smp = '1,maxcpus=%s' % smp_max_count
ram = '%sM,slots=128,maxmem=%s' % (init_ram_size, ram_max_size)
kvm_argument_list = [qemu_path,
'-enable-kvm', '-smp', smp, '-name', vm_name,
'-m', ram_size, '-vga', 'std',
'-enable-kvm', '-smp', smp, '-name', vm_name, '-m', ram, '-vga', 'std',
'-drive', 'file=%s,if=%s,cache=%s,aio=%s' % (disk_path, disk_type, disk_cache, disk_aio),
'-vnc', '%s:1,ipv4,password' % listen_ip,
'-boot', 'order=cd,menu=on',
......
software/memcached/instance.cfg deleted 100644 → 0
View file @ c44e2f89
[buildout]
parts =
instance
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
[instance]
recipe = ${instance-recipe:egg}:${instance-recipe:module}
dcrond_binary = ${dcron:location}/sbin/crond
memcached_binary = ${memcached:location}/bin/memcached
openssl_binary = ${openssl:location}/bin/openssl
rdiff_backup_binary = ${buildout:bin-directory}/rdiff-backup
stunnel_binary = ${stunnel:location}/bin/stunnel
software/memcached/software.cfg deleted 100644 → 0
View file @ c44e2f89
[buildout]
extends =
../../component/memcached/buildout.cfg
../../component/dcron/buildout.cfg
../../component/stunnel/buildout.cfg
../../component/rdiff-backup/buildout.cfg
../../component/lxml-python/buildout.cfg
../../stack/slapos.cfg
parts =
# Create instance template
template
eggs
instance-recipe-egg
[instance-recipe]
egg = slapos.cookbook
module = memcached
[instance-recipe-egg]
recipe = zc.recipe.egg
eggs = ${instance-recipe:egg}
[eggs]
recipe = zc.recipe.egg
eggs =
${lxml-python:egg}
[template]
# Default template for the instance.
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg
md5sum = 837caf9897332a5f70c72438f1dc5bae
output = ${buildout:directory}/template.cfg
mode = 0644
[versions]
plone.recipe.command = 1.1
slapos.recipe.template = 2.3
software/nayuos/software.cfg
View file @ 342b5393
......@@ -17,7 +17,7 @@ parts +=
versions = versions
[versions]
slapos.recipe.template = 4.2
slapos.recipe.template = 4.3
[template-instance]
......
software/neoppod/software-common.cfg
View file @ 342b5393
......@@ -116,8 +116,8 @@ mysqlclient = 1.3.12
persistent = 4.2.3
pycrypto = 2.6.1
pycurl = 7.43.0
slapos.recipe.template = 4.2
slapos.toolbox = 0.73
slapos.recipe.template = 4.3
slapos.toolbox = 0.74
smmap2 = 2.0.1
transaction = 1.7.0
zodbpickle = 0.6.0
......
software/neotest/instance.cfg.in
View file @ 342b5393
# TODO instance which runs this test periodically automatically and ingests results to ERP5
# NEO test instance: run neotest under Nexedi testing infrastructure
[buildout]
parts = runTestSuite
# std stuff for slapos instance
eggs-directory = {{ buildout['eggs-directory'] }}
develop-eggs-directory = {{ buildout['develop-eggs-directory'] }}
offline = true
# software release we instantiate was supplied here
[software]
dir = {{ buildout['directory'] }}
bin = {{ buildout['bin-directory'] }}
# instance directories
[directory]
recipe = slapos.cookbook:mkdirectory
home = ${buildout:directory}
bin = ${:home}/bin
neotest = ${:home}/neotest
# script to run the testsuite from inside <instance>/neotest/
# located @ <instance>/bin/runTestSuite so testnode can see this as run tests entrypoint.
[runTestSuite]
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:bin}/${:_buildout_section_name_}
command-line = /bin/bash -c 'cd ${directory:neotest} && ${software:bin}/neotest-runTestSuite "$@"' runTestSuite
# vvv appends "$@" to argv ^^^ without shell-escaping
parameters-extra = true
software/neotest/neotest-env.sh.in
View file @ 342b5393
......@@ -2,6 +2,8 @@
. ${gowork:env.sh}
PATH="${coreutils:location}/bin:$PATH"
PATH="${util-linux:location}/bin:$PATH"
PATH="${ethtool:location}/sbin:$PATH"
PATH="${ioping:location}/bin:$PATH"
PATH="${lmbench:location}/bin:$PATH"
......
software/neotest/neotest-runTestSuite.in 0 → 100644
View file @ 342b5393
#!/bin/bash -e
# neotest's runTestSuite wraper so it could be run without any environment preset
. ${buildout:directory}/neotest-env.sh
exec ${gowork:src}/lab.nexedi.com/kirr/neo/go/neo/t/nxd/runTestSuite "$@"
software/neotest/software.cfg
View file @ 342b5393
......@@ -12,6 +12,8 @@ extends =
../../component/ethtool/buildout.cfg
../../component/ioping/buildout.cfg
../../component/lmbench/buildout.cfg
../../component/coreutils/buildout.cfg
../../component/util-linux/buildout.cfg
parts =
gowork
......@@ -19,6 +21,11 @@ parts =
ioping
ethtool
# neotest uses realpath & friends
coreutils
# ----//---- lsblk
util-linux
neoppod-develop
neoppod
wendelin.core-dev
......@@ -26,11 +33,12 @@ parts =
neotest-env.sh
neotest
neotest-runTestSuite
# for instance
slapos-deps-eggs
slapos-cookbook
# instance.cfg
instance.cfg
# go packages to install (+ automatically their dependencies)
......@@ -53,7 +61,7 @@ output = ${buildout:directory}/${:_buildout_section_name_}
[neotest-env.sh]
<= buildout-template
md5sum = 6e2203c7a86e8a3e8e56b8086115f5d4
md5sum = 595eebbbcb56b6a8464d48833cfae57b
[neotest]
<= buildout-template
......@@ -61,11 +69,25 @@ output = ${buildout:bin-directory}/${:_buildout_section_name_}
mode = 0755
md5sum = fb3b4109128c1db1739ef5bb6abd1d94
[neotest-runTestSuite]
<= buildout-template
output = ${buildout:bin-directory}/${:_buildout_section_name_}
mode = 0755
md5sum = 6a4281730b68cdba5c873817a6754428
# instance
[jinja2-template]
recipe = slapos.recipe.template:jinja2
template= ${:_profile_base_location_}/${:_buildout_section_name_}.in
rendered= ${buildout:directory}/${:_buildout_section_name_}
mode = 0644
context =
section buildout buildout
# instance (TODO)
[instance.cfg]
<= buildout-template
md5sum = 410e1b2d72829824b28cc0299adb472e
<= jinja2-template
md5sum = 7bd68ed8842cd25301bf04bcdcef88f6
# eggs:
......@@ -80,6 +102,8 @@ eggs =
wendelin.core
# for ZEO scripts (runzeo)
ZEO
# for nxd/runTestSuite
erp5.util
# wendelin.core: latest not yet released
......@@ -92,6 +116,7 @@ pyasn1 = 0.3.7
ZODB3 = 3.11.0
numpy = 1.13.3
zope.testing = 4.6.2
erp5.util = 0.4.50
# Required by:
# ZEO==4.3.1
......@@ -103,7 +128,3 @@ ZConfig = 3.2.0
# ZEO==4.3.1
# ZODB==4.4.5
zc.lockfile = 1.2.1
# Required by:
# slapos.toolbox==0.73
erp5.util = 0.4.49
software/nginx-push-stream/software.cfg
View file @ 342b5393
......@@ -45,5 +45,18 @@ output = ${buildout:directory}/instance-nginx.cfg.in
mode = 0644
[versions]
slapos.recipe.template = 4.2
slapos.toolbox = 0.73
slapos.recipe.template = 4.3
slapos.toolbox = 0.74
dnspython = 1.15.0
PyRSS2Gen = 1.1
erp5.util = 0.4.50
passlib = 1.7.1
GitPython = 2.1.8
lockfile = 0.12.2
apache-libcloud = 2.2.1
feedparser = 5.2.1
atomize = 0.2.0
inotifyx = 0.2.2
gitdb2 = 2.0.3
pyasn1 = 0.4.2
smmap2 = 2.0.3
software/powerdns/instance-powerdns-replicate.cfg.jinja2
View file @ 342b5393
......@@ -81,7 +81,7 @@ username = admin
[monitor-instance-parameter]
monitor-httpd-port = 8099
cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.officejs.com') }}
username = ${monitor-instance-parameter:username}
username = admin
password = ${monitor-htpasswd:passwd}
[monitor-conf-parameters]
......@@ -109,7 +109,7 @@ config-{{parameter}} = {{ value }}
{% endfor -%}
config-{{ slave_list_name }} = {{ json_module.dumps(slave_instance_list) }}
config-monitor-cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.node.vifib.com') }}
config-monitor-username = ${monitor-instance-parameter:username}
config-monitor-username = admin
config-monitor-password = ${monitor-htpasswd:passwd}
[publish-information]
......
software/powerdns/software.cfg
View file @ 342b5393
......@@ -38,7 +38,7 @@ mode = 640
[template-dns-replicate]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/instance-powerdns-replicate.cfg.jinja2
md5sum = 3699d0751632e0c11b466342789e27a3
md5sum = 46acd4ed071df8d7139dcd0434be42eb
mode = 0644
[iso-list]
......@@ -65,4 +65,4 @@ mode = 0644
PyRSS2Gen = 1.1
cns.recipe.symlink = 0.2.3
plone.recipe.command = 1.1
slapos.recipe.template = 4.2
slapos.recipe.template = 4.3
software/re6stnet/software.cfg
View file @ 342b5393
......@@ -110,8 +110,8 @@ gitdb = 0.6.4
plone.recipe.command = 1.1
pycrypto = 2.6.1
pycurl = 7.43.0
slapos.recipe.template = 4.2
slapos.toolbox = 0.73
slapos.recipe.template = 4.3
slapos.toolbox = 0.74
smmap = 0.9.0
# Required by:
......
software/seleniumrunner/software.cfg
View file @ 342b5393
......@@ -47,4 +47,4 @@ mode = 0644
[versions]
plone.recipe.command = 1.1
slapos.recipe.template = 4.2
slapos.recipe.template = 4.3
software/slapos-in-partition/software.cfg
View file @ 342b5393
......@@ -11,4 +11,4 @@ extends = common.cfg
Pygments = 1.6
collective.recipe.environment = 0.2.0
collective.recipe.template = 1.10
slapos.recipe.template = 4.2
slapos.recipe.template = 4.3
software/slapos-master/buildout.hash.cfg
View file @ 342b5393
......@@ -15,7 +15,7 @@
# not need these here).
[template-erp5]
filename = instance-erp5.cfg.in
md5sum = 4a77ee4a6367fee27552f8bfe9d87aab
md5sum = 7a1d2fe310764f5c06f636e8850c7d63
[template-balancer]
filename = instance-balancer.cfg.in
......
software/slapos-master/instance-erp5.cfg.in
View file @ 342b5393
......@@ -206,6 +206,7 @@ name = {{ partition_name }}
{{ root_common.sla(partition_name) }}
config-name = {{ dumps(custom_name) }}
config-instance-count = {{ dumps(zope_parameter_dict.get('instance-count', 1)) }}
config-private-dev-shm = {{ zope_parameter_dict.get('private-dev-shm', '') }}
config-thread-amount = {{ dumps(zope_parameter_dict.get('thread-amount', 4)) }}
config-timerserver-interval = {{ dumps(zope_parameter_dict.get('timerserver-interval', 5)) }}
config-longrequest-logger-interval = {{ dumps(zope_parameter_dict.get('longrequest-logger-interval', -1)) }}
......@@ -327,7 +328,7 @@ config-backend-path-dict = {{ dumps(zope_backend_path_dict) }}
config-ssl-authentication-dict = {{ dumps(ssl_authentication_dict) }}
config-apachedex-promise-threshold = {{ dumps(monitor_dict.get('apachedex-promise-threshold', 70)) }}
config-apachedex-configuration = {{ dumps(monitor_dict.get('apachedex-configuration',
'--erp5-base "/erp5(/|$|/\?)" --skip-user-agent Zabbix --error-detail --js-embed --quiet')) }}
'--erp5-base +erp5 .*/VirtualHostRoot/erp5(/|\\?|$) --base +other / --skip-user-agent Zabbix --error-detail --js-embed --quiet')) }}
[request-frontend-base]
{% if has_frontend -%}
......
software/slapos-testing/software.cfg
View file @ 342b5393
......@@ -97,4 +97,4 @@ mode = 640
Pygments = 2.1.3
collective.recipe.template = 1.10
plone.recipe.command = 1.1
slapos.recipe.template = 4.2
slapos.recipe.template = 4.3
software/slaprunner/buildout.hash.cfg
View file @ 342b5393
......@@ -19,7 +19,7 @@ md5sum = 713db528880282d568278f09458d2aab
[template-runner]
filename = instance-runner.cfg
md5sum = 7df86928aeef0fbada832b0283b9a454
md5sum = 8b1caca52ab1307343ebada59f5a6c66
[template-runner-import-script]
filename = template/runner-import.sh.jinja2
......@@ -35,7 +35,7 @@ md5sum = 98ce179badc6af5979a64a7c3d0a2ceb
[instance-runner-export]
filename = instance-runner-export.cfg.in
md5sum = 00ddcaa6e571505d1930612d12de8fde
md5sum = d1122988652127af5e252f1503eeacab
[template-resilient]
filename = instance-resilient.cfg.jinja2
......
software/slaprunner/instance-runner-export.cfg.in
View file @ 342b5393
......@@ -7,11 +7,13 @@ parts +=
nginx-launcher
certificate-authority
ca-nginx
logrotate-entry-nginx
gunicorn-launcher
gunicorn-graceful
publish-connection-information
slaprunner-promise
apache-httpd-promise
logrotate-entry-apache-httpd
slaprunner-supervisord-wrapper
runner-sshd-add-authorized-key
runner-sshd-graceful
......@@ -30,6 +32,7 @@ parts +=
bash-profile
supervisord-wrapper
supervisord-promise
logrotate-entry-supervisord
httpd-graceful-wrapper
## Monitoring part
## Monitor for runner
......
software/slaprunner/instance-runner.cfg
View file @ 342b5393
......@@ -4,10 +4,12 @@ parts =
nginx-launcher
certificate-authority
ca-nginx
logrotate-entry-nginx
gunicorn-launcher
gunicorn-graceful
publish-connection-information
slaprunner-promise
logrotate-entry-apache-httpd
apache-httpd-promise
slaprunner-supervisord-wrapper
runner-sshd-add-authorized-key
......@@ -27,6 +29,7 @@ parts =
bash-profile
supervisord-wrapper
supervisord-promise
logrotate-entry-supervisord
httpd-graceful-wrapper
{% if slapparameter_dict.get('no-ipv4-frontend', 'false') == 'false' %}
slaprunner-frontend-promise
......@@ -347,6 +350,12 @@ mode = 700
context =
section param_nginx_frontend nginx-frontend
[logrotate-entry-nginx]
<= logrotate-entry-base
name = nginx
log = $${directory:log}/nginx.access.log $${directory:log}/nginx.error.log
post = kill -SIGUSR1 $(cat $${buildout:directory}/var/run/nginx.pid)
[httpd-parameters]
path_pid = $${directory:run}/httpd.pid
path_error_log = $${directory:log}/httpd-error.log
......@@ -387,6 +396,12 @@ wait-for-files =
$${ca-nginx:cert-file}
$${ca-nginx:key-file}
[logrotate-entry-apache-httpd]
<= logrotate-entry-base
name = apache
log = $${directory:log}/httpd-access.log $${directory:log}/httpd-error.log
post = test ! -s $${buildout:directory}/var/run/httpd.pid || $${buildout:directory}/bin/slapos-kill --pidfile $${buildout:directory}/var/run/httpd.pid -s USR1
[httpd-graceful-wrapper]
recipe = collective.recipe.template
input = inline:
......@@ -777,6 +792,12 @@ recipe = slapos.cookbook:wrapper
command-line = $${buildout:directory}/bin/supervisord -c $${supervisord-conf:rendered} --nodaemon
wrapper-path = $${directory:services}/supervisord
[logrotate-entry-supervisord]
<= logrotate-entry-base
name = supervisord
log = $${directory:log}/slapproxy.log $${directory:log}/supervisord.log $${directory:log}/supervisord-errors.log
post = kill -SIGUSR2 $(cat $${buildout:directory}/var/run/supervisord.pid)
[supervisord-promise]
recipe = slapos.cookbook:check_port_listening
path = $${directory:promises}/supervisord
......
software/slaprunner/software.cfg
View file @ 342b5393
......@@ -15,8 +15,8 @@ gitdb = 0.6.4
gunicorn = 19.7.1
prettytable = 0.7.2
pycurl = 7.43.0
slapos.recipe.template = 4.2
slapos.toolbox = 0.73
slapos.recipe.template = 4.3
slapos.toolbox = 0.74
smmap = 0.9.0
# Required by:
......
software/test-slave-instance-deployment/software.cfg
View file @ 342b5393
......@@ -59,7 +59,7 @@ eggs = collective.recipe.template
collective.recipe.template = 1.11
plone.recipe.command = 1.1
slapos.recipe.build = 0.28
slapos.recipe.template = 4.2
slapos.recipe.template = 4.3
# Replicate slapos stack, but without shacache to not have to compile the entire world for a simple test.
[buildout]
......
software/varnish/software.cfg
View file @ 342b5393
......@@ -62,4 +62,4 @@ md5sum = 0ea12a4ad2d2e3d406476e35b8d3e3fb
mode = 640
[versions]
slapos.recipe.template = 4.2
slapos.recipe.template = 4.3
software/wendelin-scalability/test-neo.cfg
View file @ 342b5393
......@@ -10,12 +10,15 @@
# $x 8.787 0
# 0.100036621094
# 6556
#
# The 'start_process' command is similar but by growing a ZBigArray object.
# The random data has a compression ratio of 10%.
[buildout]
extends = test-common.cfg
parts += start_ingest
parts += start_ingest start_process
[start_ingest]
[start-script-common]
recipe = slapos.recipe.template:jinja2
rendered = ${buildout:bin-directory}/${:_buildout_section_name_}
mode = 0755
......@@ -27,20 +30,35 @@ template =
_('--site-id', default='erp5')
_('hostport', metavar='host[:port]', help='Zope address')
_('password', help="'zope' user password")
_('reference', help='Data Stream reference')
_('mu', type=float)
_('sigma', type=float)
_('chunks_per_transaction', nargs='?', type=int, help='default: 128 (8 MiB)')
options = []
def option(name, **kw):
_(name, **kw)
options.append(name)
${:options}
args = parser.parse_args()
qs = []
for k in 'reference', 'mu', 'sigma', 'chunks_per_transaction':
for k in options:
v = getattr(args, k)
if v is not None:
t = type(v)
qs.append('%s=%s' % (k if t is str else k + ':' + t.__name__, v))
c = httplib.HTTPConnection(args.hostport)
c.putrequest('GET', '/%s/ERP5Site_simulateFluentdIngestion?%s'
% (args.site_id, '&'.join(qs)))
c.putrequest('GET', '/%s/${:script}?%s' % (args.site_id, '&'.join(qs)))
c.putheader('Authorization',
'Basic ' + base64.b64encode('zope:'+args.password))
c.endheaders()
[start_ingest]
<= start-script-common
options =
option('id', help='Data Stream id')
option('mu', type=float)
option('sigma', type=float)
option('chunks_per_transaction', nargs='?', type=int, help='default: 128 (8 MiB)')
script = ERP5Site_simulateFluentdIngestion
[start_process]
<= start-script-common
options =
option('id', help='Data Array id')
script = ERP5Site_dummyZBigArrayProcessing
software/wendelin-scalability/test_scalability_fluentd/ExtensionTemplateItem/portal_components/extension.erp5.ScalabilityFluentd.py
View file @ 342b5393
......@@ -5,7 +5,7 @@ from random import lognormvariate
bigfile_chunk_size = 65536
def simulateFluentdIngestion(self, reference, mu, sigma,
def simulateFluentdIngestion(self, id, mu, sigma,
chunks_per_transaction=128):
from time import time
import transaction
......@@ -13,9 +13,9 @@ def simulateFluentdIngestion(self, reference, mu, sigma,
+ '/ingest')
module = self['data_stream_module']
try:
data_stream = module[reference]
data_stream = module[id]
except KeyError:
data_stream = module.newContent(reference, 'Data Stream')
data_stream = module.newContent(id, 'Data Stream')
transaction.commit()
pack = struct.Struct('!d').pack
......
software/wendelin-scalability/test_scalability_fluentd/ExtensionTemplateItem/portal_components/extension.erp5.ScalabilityZBigArray.py 0 → 100644
View file @ 342b5393
def dummyZBigArrayProcessing(self, id):
import numpy as np
from random import randrange, sample
import transaction
module = self['data_array_module']
try:
array = module[id]
except KeyError:
array = module.newContent(id, 'Data Array')
array.initArray(shape=(0, 64), dtype=np.int32)
transaction.commit()
note = array.getPath() + '/new_data'
array = array.getArray()
rows, cols = array.shape
y = xrange(cols)
n = 10 * (2<<20) // (cols*4)
z = np.ndarray(shape=(n, cols), dtype=array.dtype)
for row in z:
for i in sample(y, 8):
row[i] = randrange(0, 1000)
while 1:
txn = transaction.begin()
np.random.shuffle(z)
rows += n
array.resize((rows, cols))
array[-n:] = z
txn.note(note)
txn.commit()
software/wendelin-scalability/test_scalability_fluentd/ExtensionTemplateItem/portal_components/extension.erp5.ScalabilityZBigArray.xml 0 → 100644
View file @ 342b5393
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="Extension Component" module="erp5.portal_type"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_recorded_property_dict</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAI=</string> </persistent>
</value>
</item>
<item>
<key> <string>default_reference</string> </key>
<value> <string>ScalabilityZBigArray</string> </value>
</item>
<item>
<key> <string>description</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>extension.erp5.ScalabilityZBigArray</string> </value>
</item>
<item>
<key> <string>portal_type</string> </key>
<value> <string>Extension Component</string> </value>
</item>
<item>
<key> <string>sid</string> </key>
<value>
<none/>
</value>
</item>
<item>
<key> <string>text_content_error_message</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>text_content_warning_message</string> </key>
<value>
<tuple/>
</value>
</item>
<item>
<key> <string>version</string> </key>
<value> <string>erp5</string> </value>
</item>
<item>
<key> <string>workflow_history</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAM=</string> </persistent>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="2" aka="AAAAAAAAAAI=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary/>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="3" aka="AAAAAAAAAAM=">
<pickle>
<global name="PersistentMapping" module="Persistence.mapping"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>data</string> </key>
<value>
<dictionary>
<item>
<key> <string>component_validation_workflow</string> </key>
<value>
<persistent> <string encoding="base64">AAAAAAAAAAQ=</string> </persistent>
</value>
</item>
</dictionary>
</value>
</item>
</dictionary>
</pickle>
</record>
<record id="4" aka="AAAAAAAAAAQ=">
<pickle>
<global name="WorkflowHistoryList" module="Products.ERP5Type.patches.WorkflowTool"/>
</pickle>
<pickle>
<tuple>
<none/>
<list>
<dictionary>
<item>
<key> <string>action</string> </key>
<value> <string>validate</string> </value>
</item>
<item>
<key> <string>validation_state</string> </key>
<value> <string>validated</string> </value>
</item>
</dictionary>
</list>
</tuple>
</pickle>
</record>
</ZopeData>
software/wendelin-scalability/test_scalability_fluentd/PathTemplateItem/portal_skins/custom/ERP5Site_dummyZBigArrayProcessing.xml 0 → 100644
View file @ 342b5393
<?xml version="1.0"?>
<ZopeData>
<record id="1" aka="AAAAAAAAAAE=">
<pickle>
<global name="ExternalMethod" module="Products.ExternalMethod.ExternalMethod"/>
</pickle>
<pickle>
<dictionary>
<item>
<key> <string>_function</string> </key>
<value> <string>dummyZBigArrayProcessing</string> </value>
</item>
<item>
<key> <string>_module</string> </key>
<value> <string>ScalabilityZBigArray</string> </value>
</item>
<item>
<key> <string>id</string> </key>
<value> <string>ERP5Site_dummyZBigArrayProcessing</string> </value>
</item>
<item>
<key> <string>title</string> </key>
<value> <string></string> </value>
</item>
</dictionary>
</pickle>
</record>
</ZopeData>
software/wendelin-scalability/test_scalability_fluentd/PathTemplateItem/portal_skins/custom/DataStreamModule_getTotalSize.xml → software/wendelin-scalability/test_scalability_fluentd/PathTemplateItem/portal_skins/custom/Module_getTotalSize.xml
View file @ 342b5393
......@@ -54,7 +54,7 @@
</item>
<item>
<key> <string>id</string> </key>
<value> <string>DataStreamModule_getTotalSize</string> </value>
<value> <string>Module_getTotalSize</string> </value>
</item>
</dictionary>
</pickle>
......
software/wendelin-scalability/test_scalability_fluentd/bt/template_extension_id_list
View file @ 342b5393
extension.erp5.ScalabilityFluentd
\ No newline at end of file
extension.erp5.ScalabilityFluentd
extension.erp5.ScalabilityZBigArray
\ No newline at end of file
software/wendelin-scalability/test_scalability_fluentd/bt/template_path_list
View file @ 342b5393
portal_ingestion_policies/scalability_test_*
portal_skins/custom/DataStreamModule_getTotalSize
portal_skins/custom/ERP5Site_dummyZBigArrayProcessing
portal_skins/custom/ERP5Site_handleRawDataFluentdIngestion
portal_skins/custom/ERP5Site_simulateFluentdIngestion
\ No newline at end of file
portal_skins/custom/ERP5Site_simulateFluentdIngestion
portal_skins/custom/Module_getTotalSize
\ No newline at end of file
stack/caucase/buildout.cfg
View file @ 342b5393
......@@ -94,8 +94,8 @@ caucase = 0.1.4
futures = 3.1.1
gitdb2 = 2.0.2
gunicorn = 19.7.1
slapos.recipe.template = 4.2
slapos.toolbox = 0.73
slapos.recipe.template = 4.3
slapos.toolbox = 0.74
smmap2 = 2.0.3
# Required by:
......
stack/cloudooo.cfg
View file @ 342b5393
......@@ -89,7 +89,7 @@ PasteScript = 2.0.2
WSGIUtils = 0.7
python-magic = 0.4.6
rdiff-backup = 1.0.5+SlapOSPatched001
slapos.recipe.template = 4.2
slapos.recipe.template = 4.3
# Required by:
# PasteScript==2.0
......
stack/erp5/buildout.cfg
View file @ 342b5393
......@@ -63,6 +63,7 @@ extends =
../../component/findutils/buildout.cfg
../../component/userhosts/buildout.cfg
../../component/postfix/buildout.cfg
../../component/zbarlight/buildout.cfg
../monitor/buildout.cfg
../../software/caucase/software.cfg
../../software/jupyter/software.cfg
......@@ -465,6 +466,7 @@ eggs = ${neoppod:eggs}
${h5py:egg}
openpyxl
${statsmodels:egg}
${zbarlight:egg}
lock_file
astor
APacheDEX
......@@ -730,7 +732,7 @@ spyne = 2.12.14
suds = 0.4
facebook-sdk = 2.0.0
threadframe = 0.2
timerserver = 2.0.2
timerserver = 2.0.4
urlnorm = 1.1.4
uuid = 1.30
validictory = 1.1.0
......
stack/erp5/buildout.hash.cfg
View file @ 342b5393
......@@ -19,7 +19,7 @@ md5sum = c1f1083bf6c911a0e65dcb841fba327d
[mariadb-slow-query-report-script]
filename = mysql-querydigest.sh.in
md5sum = cfe6ab8ae54a521ecb269e9d9762cbeb
md5sum = 0c0d98a68230cd0ad36046bb25b35f4a
[mariadb-start-clone-from-backup]
filename = instance-mariadb-start-clone-from-backup.sh.in
......@@ -27,7 +27,7 @@ md5sum = 1af531c51f575a1d1362f2ca2d61620d
[template-mariadb]
filename = instance-mariadb.cfg.in
md5sum = 244e185ed26831fe0b5abf49c67753d6
md5sum = da7c36ecb490b67360d2afda94b41bff
[template-kumofs]
filename = instance-kumofs.cfg.in
......@@ -79,7 +79,7 @@ md5sum = d41d8cd98f00b204e9800998ecf8427e
[template-erp5]
filename = instance-erp5.cfg.in
md5sum = 2465af81147af322056cee9f6c7de14f
md5sum = 1d6735a803c9d28930bf2ad00706c06b
[template-zeo]
filename = instance-zeo.cfg.in
......@@ -87,11 +87,11 @@ md5sum = d1f33d406d528ae27d973e2dd0efb1ba
[template-zope]
filename = instance-zope.cfg.in
md5sum = fd7e8c507cef1950e6c0347ce2a01021
md5sum = e08c00d5973916d796bf08aa78dba34a
[template-balancer]
filename = instance-balancer.cfg.in
md5sum = f64c568f1365eb1164f12f48fede9a99
md5sum = a71ad387eab681b9020e271cba2c7a79
[template-haproxy-cfg]
filename = haproxy.cfg.in
......
stack/erp5/instance-balancer.cfg.in
View file @ 342b5393
......@@ -287,7 +287,7 @@ apachedex = ${monitor-directory:private}/apachedex
[{{ section('monitor-generate-apachedex-report') }}]
recipe = slapos.cookbook:wrapper
wrapper-path = ${monitor-directory:reports}/${:command}
command-line = "{{ parameter_dict['run-apachedex-location'] }}" "{{ parameter_dict['apachedex-location'] }}" "${directory:apachedex}" ${monitor-publish-parameters:monitor-base-url}/private/apachedex --apache-log-list "${apachedex-parameters:apache-log-list}" --config "${apachedex-parameters:configuration}"
command-line = "{{ parameter_dict['run-apachedex-location'] }}" "{{ parameter_dict['apachedex-location'] }}" "${directory:apachedex}" ${monitor-publish-parameters:monitor-base-url}/private/apachedex --apache-log-list "${apachedex-parameters:apache-log-list}" --configuration "${apachedex-parameters:configuration}"
command = apachedex_every_23_hour
[apachedex-parameters]
......
stack/erp5/instance-erp5.cfg.in
View file @ 342b5393
......@@ -205,6 +205,7 @@ name = {{ partition_name }}
{{ root_common.sla(partition_name) }}
config-name = {{ dumps(custom_name) }}
config-instance-count = {{ dumps(zope_parameter_dict.get('instance-count', 1)) }}
config-private-dev-shm = {{ zope_parameter_dict.get('private-dev-shm', '') }}
config-thread-amount = {{ dumps(zope_parameter_dict.get('thread-amount', 4)) }}
config-timerserver-interval = {{ dumps(zope_parameter_dict.get('timerserver-interval', 5)) }}
config-longrequest-logger-interval = {{ dumps(zope_parameter_dict.get('longrequest-logger-interval', -1)) }}
......@@ -284,7 +285,7 @@ config-backend-path-dict = {{ dumps(zope_backend_path_dict) }}
config-ssl-authentication-dict = {{ dumps(ssl_authentication_dict) }}
config-apachedex-promise-threshold = {{ dumps(monitor_dict.get('apachedex-promise-threshold', 70)) }}
config-apachedex-configuration = {{ dumps(monitor_dict.get('apachedex-configuration',
'--erp5-base "/erp5(/|$|/\?)" --skip-user-agent Zabbix --error-detail --js-embed --quiet')) }}
'--erp5-base +erp5 .*/VirtualHostRoot/erp5(/|\\?|$) --base +other / --skip-user-agent Zabbix --error-detail --js-embed --quiet')) }}
[request-frontend-base]
{% if has_frontend -%}
......
stack/erp5/instance-mariadb.cfg.in
View file @ 342b5393
......@@ -252,7 +252,7 @@ mariadb-ssl = ${:etc}/mariadb-ssl
var = ${buildout:directory}/var
log = ${:var}/log
run = ${:var}/run
slowquery = ${monitor-directory:private}/slowquerydex
slowquery = ${monitor-directory:private}/slowquery_digest
[{{ section('resiliency-exclude-file') }}]
# Generate rdiff exclude file in case of resiliency
......
stack/erp5/instance-zope.cfg.in
View file @ 342b5393
......@@ -206,6 +206,7 @@ environment =
{% endif %}
parameters-extra = true
command-line = '{{ parameter_dict['userhosts'] }}' '{{ bin_directory }}/runzope' -C '${:configuration-file}'
private-dev-shm = {{ slapparameter_dict['private-dev-shm'] }}
[{{ section('zcml') }}]
recipe = slapos.cookbook:copyfilelist
......
stack/erp5/mysql-querydigest.sh.in
View file @ 342b5393
......@@ -7,19 +7,20 @@ PT_QUERY_EXEC='{{pt_query_exec}}'
if [ ! -d "$OUTPUT_FOLDER" ]; then
echo "ERROR: output_folder don't exists"
exit 0
exit 1
fi
OUTPUT_FILE=${OUTPUT_FOLDER}/slowquery_digest.txt
TODAY=`date +%Y%m%d`
dashed_today=$(date +%Y-%m-%d)
today=$(date -d "$dashed_today" +%Y%m%d)
SLOW_LOG=$SLOW_QUERY_PATH-$TODAY
SLOW_LOG="$SLOW_QUERY_PATH-$today"
OUTPUT_FILE="$OUTPUT_FOLDER/slowquery_digest.txt-$dashed_today"
if [ ! -f "$SLOW_LOG" ]; then
echo "ERROR: cannot read mysql slow query log file $SLOW_LOG. Exiting."
exit 1
fi
$PT_QUERY_EXEC $SLOW_LOG > $OUTPUT_FILE
echo "ok"
"$PT_QUERY_EXEC" "$SLOW_LOG" > "$OUTPUT_FILE" && \
echo "Report generated successfully." || \
echo "Report failed with code $?"
stack/monitor/buildout.cfg
View file @ 342b5393
......@@ -72,7 +72,7 @@ md5sum = 1695c9a06a2b11ccfe893d7a224e489d
[monitor-conf]
<= monitor-template-base
filename = monitor.conf.in
md5sum = fc65084dd1c2fe2487b58a003b576f61
md5sum = 888e2845d09bfaa59c25f56f5bcf76b1
[monitor-instance-info]
<= monitor-template-base
......@@ -131,6 +131,6 @@ depends =
PyRSS2Gen = 1.1
cns.recipe.symlink = 0.2.3
pycurl = 7.43.0
slapos.toolbox = 0.73
slapos.toolbox = 0.74
pyasn1 = 0.3.7
stack/monitor/buildout.hash.cfg
View file @ 342b5393
......@@ -15,4 +15,4 @@
# not need these here).
[monitor2-template]
filename = instance-monitor.cfg.jinja2.in
md5sum = 5027f0b1abcc28ce3817cd70fb667a3b
md5sum = 03254b14a2ff242f7588a307d8c27f23
stack/monitor/instance-monitor.cfg.jinja2.in
View file @ 342b5393
......@@ -42,6 +42,7 @@ services = ${directory:services}
services-conf = ${directory:etc}/monitor.conf.d
log = ${directory:log}/monitor
monitor-var = ${directory:var}/monitor
monitor-log = ${directory:monitor}/private/monitor-log
[ca-directory]
recipe = slapos.cookbook:mkdirectory
......@@ -88,14 +89,11 @@ promise-folder = ${directory:promises}
monitor-promise-folder = ${monitor-directory:promises}
promises-timeout-file = ${monitor-promise-timeout-file:file}
pid-file = ${monitor-directory:pids}/monitor-bootstrap.pid
randomsleep = {{ bin_directory }}/randomsleep
public-path-list =
private-path-list =
${directory:log}
#
monitor-url-list =
${monitor-instance-parameter:monitor-url-list}
private-path-list = ${directory:log}
monitor-url-list = ${monitor-instance-parameter:monitor-url-list}
parameter-file-path = ${monitor-instance-parameter:configuration-file-path}
parameter-list =
......@@ -234,6 +232,12 @@ context =
raw dash_binary {{ dash_executable_location }}
command = kill -USR1 $(cat ${monitor-httpd-conf-parameter:pid-file})
[logrotate-entry-monitor-httpd]
<= logrotate-entry-base
name = monitor-apache
log = $${basedirectory:log}/monitor/monitor-httpd-error.log
post = test ! -s $${buildout:directory}/var/run/monitor-httpd.pid || $${buildout:directory}/bin/slapos-kill --pidfile $${buildout:directory}/var/run/monitor-httpd.pid -s USR1
[xnice-bin]
recipe = collective.recipe.template
input = inline:#!/bin/sh
......@@ -264,14 +268,14 @@ recipe = slapos.cookbook:cron.d
cron-entries = ${cron:cron-entries}
name = monitor-globalstate
frequency = * * * * *
command = ${monitor-globalstate-wrapper:wrapper-path}
command = {{ bin_directory }}/randomsleep 60 && ${monitor-globalstate-wrapper:wrapper-path}
[monitor-configurator-cron-entry]
recipe = slapos.cookbook:cron.d
cron-entries = ${cron:cron-entries}
name = monitor-configurator
frequency = * * * * *
command = ${monitor-configurator-wrapper:wrapper-path}
command = {{ bin_directory }}/randomsleep 60 && ${monitor-configurator-wrapper:wrapper-path}
[monitor-httpd-promise]
recipe = slapos.cookbook:check_url_available
......@@ -407,6 +411,7 @@ depends =
${monitor-bootstrap-promise:file}
${promise-check-slapgrid:output}
${promise-monitor-httpd-is-process-older-than-dependency-set:wrapper-path}
${logrotate-entry-monitor-httpd:name}
[monitor-publish]
monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
......
stack/monitor/templates/monitor.conf.in
View file @ 342b5393
[monitor]
{% set monitor_url_list = parameter_dict.pop("monitor-url-list", "").strip() -%}
{% for key, value in parameter_dict.items() -%}
{% if key == "monitor-url-list" and monitor_base_urls is defined -%}
{{ key }} =
{{ key }} = {{ value.strip().replace("\n", "\n ") }}
{% endfor -%}
monitor-url-list =
{% if monitor_url_list -%}
{{ ' ' ~ monitor_url_list.replace("\n", "\n ") }}
{% else -%}
{% for key, value in monitor_base_urls.items() -%}
{{ ' ' ~ value }}
{% endfor -%}
{% else -%}
{{ key }} = {{ value.strip().replace("\n", "\n ") }}
{% endif -%}
{% endfor -%}
stack/slapos.cfg
View file @ 342b5393
......@@ -93,7 +93,7 @@ eggs =
[versions]
setuptools = 33.1.1
# Use SlapOS patched zc.buildout
zc.buildout = 2.5.2+slapos011
zc.buildout = 2.5.2+slapos012
# Use SlapOS patched zc.recipe.egg (zc.recipe.egg 2.x is for Buildout 2)
zc.recipe.egg = 2.0.3+slapos003
# Use own version of h.r.download to be able to open .xz and .lz archives
......@@ -121,14 +121,14 @@ netaddr = 0.7.19
pbr = 2.0.0
plone.recipe.command = 1.1
prettytable = 0.7.2
psutil = 5.2.0
psutil = 5.4.3
pyOpenSSL = 17.2.0
pyparsing = 2.2.0
pytz = 2016.10
requests = 2.13.0
six = 1.10.0
slapos.cookbook = 1.0.53
slapos.core = 1.4.3
slapos.core = 1.4.4
slapos.extension.strip = 0.4
slapos.libnetworkcache = 0.15
slapos.rebootstrap = 4.1
......@@ -140,7 +140,7 @@ xml-marshaller = 0.9.7
paramiko = 2.1.3
# Required by:
# slapos.core==1.4.3
# slapos.core==1.4.4
Flask = 0.12
# Required by:
......@@ -160,7 +160,7 @@ ipaddress = 1.0.18
jsonschema = 2.6.0
# Required by:
# slapos.core==1.4.3
# slapos.core==1.4.4
# XXX 'slapos node format' raises an exception with netifaces 0.10.5.
netifaces = 0.10.4
......@@ -173,15 +173,15 @@ packaging = 16.8
pycparser = 2.17
# Required by:
# slapos.core==1.4.3
# slapos.core==1.4.4
supervisor = 3.3.3
# Required by:
# slapos.core==1.4.3
# slapos.core==1.4.4
uritemplate = 3.0.0
# Required by:
# slapos.core==1.4.3
# slapos.core==1.4.4
zope.interface = 4.3.3
[networkcache]
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7