Skip to content

slapos
slapos
Commit 56ad14fb authored by Rafael Monnerat's avatar Rafael Monnerat
Browse files
Options

Update Release candidate 

Conflicts:
	software/erp5/software.cfg
parents fb36acc7 578f65bd
Hide whitespace changes
Inline Side-by-side
Showing with 6892 additions and 5924 deletions
README.rst
View file @ 56ad14fb
......@@ -2,3 +2,14 @@ slapos.cookbook
===============
Cookbook of SlapOS recipes.
testing
=======
Unit tests for recipes can be found under ``slapos/test/recipe``. To run the
tests use provided unittest.defaultTestLoader inside ``slapos/test/test_recipe``
by invoking
python setup.py test --test-suite slapos.test.test_recipe.additional_tests
component/alsa/buildout.cfg
View file @ 56ad14fb
......@@ -5,15 +5,12 @@ parts =
[alsa]
# Contains libasound
recipe = slapos.recipe.cmmi
url = ftp://ftp.alsa-project.org/pub/lib/alsa-lib-1.0.24.1.tar.bz2
md5sum = 7cc05f25e1d5b65da8fb3fdcd540f226
url = ftp://ftp.alsa-project.org/pub/lib/alsa-lib-1.1.3.tar.bz2
md5sum = eefe5992567ba00d6110a540657aaf5c
configure-options =
--disable-static
--disable-aload
--disable-mixer
--disable-rawmidi
--disable-hwdep
--disable-seq
--disable-ucm
--disable-alisp
--disable-old-symbols
......
component/apache/apache-backend.conf.in
View file @ 56ad14fb
......@@ -108,12 +108,16 @@ SSLProxyEngine On
# As backend is trusting REMOTE_USER header unset it always
RequestHeader unset REMOTE_USER
RequestHeader unset SSL_CLIENT_SERIAL
{% if parameter_dict['ca-cert'] -%}
SSLVerifyClient require
SSLVerifyClient optional
RequestHeader set REMOTE_USER %{SSL_CLIENT_S_DN_CN}s
RequestHeader set SSL_CLIENT_SERIAL "%{SSL_CLIENT_M_SERIAL}s"
SSLCACertificateFile {{ parameter_dict['ca-cert'] }}
{% if parameter_dict['crl'] -%}
SSLCARevocationCheck chain
SSLCARevocationFile {{ parameter_dict['crl'] }}
{%- endif %}
{%- endif %}
ErrorLog "{{ parameter_dict['error-log'] }}"
......@@ -128,12 +132,24 @@ CustomLog "{{ parameter_dict['access-log'] }}" combined
</Directory>
RewriteEngine On
{% for port, _, backend in parameter_dict['backend-list'] -%}
{% for port, _, backend, enable_authentication in parameter_dict['backend-list'] -%}
{% for ip in parameter_dict['ip-list'] -%}
Listen {{ ip }}:{{ port }}
{% endfor -%}
<VirtualHost *:{{ port }}>
SSLEngine on
{% if enable_authentication and parameter_dict['ca-cert'] and parameter_dict['crl'] -%}
SSLVerifyClient require
SSLCACertificateFile {{ parameter_dict['ca-cert'] }}
SSLCARevocationCheck chain
SSLCARevocationFile {{ parameter_dict['crl'] }}
LogFormat "%h %l %{REMOTE_USER}i %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %D" combined
# We would like to separate the the authentificated logs.
ErrorLog "{{ parameter_dict['log-dir'] }}/apache-service-error.log"
CustomLog "{{ parameter_dict['log-dir'] }}/apache-service-access.log" combined
{% endif -%}
RewriteRule ^/(.*) {{ backend }}/$1 [L,P]
</VirtualHost>
{% endfor -%}
component/apache/buildout.cfg
View file @ 56ad14fb
......@@ -34,9 +34,9 @@ md5sum = 2202b18f269ad606d70e1864857ed93c
[apache]
# inspired on http://old.aclark.net/team/aclark/blog/a-lamp-buildout-for-wordpress-and-other-php-apps/
recipe = slapos.recipe.cmmi
version = 2.4.25
version = 2.4.26
url = https://archive.apache.org/dist/httpd/httpd-${:version}.tar.bz2
md5sum = 2826f49619112ad5813c0be5afcc7ddb
md5sum = d4d47749a44461cb2e6c9d78a22b522b
pre-configure =
cp -ar ${apr:location}/apr-${apr:version} srclib/apr/ &&
cp -ar ${apr-util:location}/apr-util-${apr-util:version} srclib/apr-util
......@@ -190,5 +190,5 @@ make-targets =
[template-apache-backend-conf]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/apache-backend.conf.in
md5sum = feef079241bda3407b7ceed5876cb61f
md5sum = 8f1f92ad308ab6bad973c790ee583428
mode = 640
component/automake/buildout.cfg
View file @ 56ad14fb
......@@ -9,8 +9,8 @@ parts =
[automake]
recipe = slapos.recipe.cmmi
md5sum = 9a1ddb0e053474d9d1105cfe39b0c48d
url = http://ftp.gnu.org/gnu/automake/automake-1.15.tar.xz
md5sum = 24cd3501b6ad8cd4d7e2546f07e8b4d4
url = http://ftp.gnu.org/gnu/automake/automake-1.15.1.tar.xz
patch-options = -p1
patches =
${:_profile_base_location_}/automake-1.15-shebang_workaround.patch#203f9199b0e629de3630b5959f8cf73e
......
component/bcrypt/buildout.cfg 0 → 100644
View file @ 56ad14fb
[buildout]
extends =
../libffi/buildout.cfg
parts =
bcrypt
[bcrypt-env]
CPPFLAGS=-I${libffi:location}/include
CFLAGS=-I${libffi:location}/include
[bcrypt]
recipe = zc.recipe.egg:custom
egg = bcrypt
rpath =
${libffi:location}/lib/
environment = bcrypt-env
component/ca-certificates/buildout.cfg
View file @ 56ad14fb
......@@ -11,9 +11,9 @@ parts =
[ca-certificates]
recipe = slapos.recipe.cmmi
url = http://http.debian.net/debian/pool/main/c/ca-certificates/ca-certificates_20161130.tar.xz
url = http://http.debian.net/debian/pool/main/c/ca-certificates/ca-certificates_20161130+nmu1.tar.xz
patch-binary = ${patch:location}/bin/patch
md5sum = 1a0a3a1b3390dc83affed4b0c2ae1c05
md5sum = a09e8b63126188fd0ed77f6fbaf5d35f
patches =
${:_profile_base_location_}/ca-certificates-any-python.patch#087b5e860c7a4b8ff6656c95c5835ee2
${:_profile_base_location_}/ca-certificates-sbin-dir.patch#0b4e7d82ce768823c01954ee41ef177b
......
component/consul/buildout.cfg 0 → 100644
View file @ 56ad14fb
[buildout]
parts =
consul
[consul]
recipe = slapos.recipe.build
# here, two %s are used, first one is for directory name (eg. x86_64), and second one is for filename (eg. x86-64).
url_x86-64 = https://releases.hashicorp.com/consul/0.8.3/consul_0.8.3_linux_amd64.zip
url_x86 = https://releases.hashicorp.com/consul/0.8.3/consul_0.8.3_linux_386.zip
# supported architectures md5sums
md5sum_x86 = dfdc0eedd79baab7e6bc56c1582fd02e
md5sum_x86-64 = d6bc0898ea37ae2198370a9e1978d1bb
# script to install.
script =
location = %(location)r
self.failIfPathExists(location)
import sys
ARCH_DIR_MAP = { 'x86': 'x86', 'x86-64': 'x86_64' }
WK_SUFIX_MAP = { 'x86': 'i386', 'x86-64': 'amd64' }
platform = guessPlatform()
url = self.options['url_' + platform]
md5sum = self.options['md5sum_' + platform]
extract_dir = self.extract(self.download(url, md5sum))
shutil.move(extract_dir, location)
component/curl/buildout.cfg
View file @ 56ad14fb
......@@ -12,8 +12,8 @@ parts =
[curl]
recipe = slapos.recipe.cmmi
url = http://curl.haxx.se/download/curl-7.54.0.tar.bz2
md5sum = 89bb7ba87384dfbf4f1a3f953da42458
url = http://curl.haxx.se/download/curl-7.54.1.tar.bz2
md5sum = 6b6eb722f512e7a24855ff084f54fe55
configure-options =
--disable-static
--disable-ldap
......
component/firewalld/buildout.cfg
View file @ 56ad14fb
......@@ -68,7 +68,7 @@ configure-options =
--disable-static
environment =
PATH=${pkgconfig:location}/bin:${gettext:location}/bin:${glib:location}/bin:${xz-utils:location}/bin:%(PATH)s
PATH=${pkgconfig:location}/bin:${gettext:location}/bin:${glib:location}/bin:${xz-utils:location}/bin:${flex:location}/bin:${bison:location}/bin:%(PATH)s
PKG_CONFIG_PATH=${glib:location}/lib/pkgconfig:${pcre:location}/lib/pkgconfig
CPPFLAGS=-I${glib:location}/include/glib-2.0 -I${glib:location}/lib/glib-2.0/include
LDFLAGS=-L${glib:location}/lib -Wl,-rpath=${glib:location}/lib -L${libffi:location}/lib -Wl,-rpath=${libffi:location}/lib -lffi
......
component/fonts/buildout.cfg
View file @ 56ad14fb
......@@ -20,10 +20,9 @@ strip-top-level-dir = true
# Liberation(tm) Fonts - a font family which aims at metric
# compatibility with Arial, Times New Roman, and Courier New.
# https://fedorahosted.org/liberation-fonts/
[liberation-fonts]
<= fonts-base
url = https://fedorahosted.org/releases/l/i/liberation-fonts/liberation-fonts-ttf-2.00.1.tar.gz
url = https://releases.pagure.org/liberation-fonts/liberation-fonts-ttf-2.00.1.tar.gz
md5sum = 5c781723a0d9ed6188960defba8e91cf
# IPAex and IPA Font - Japanese fonts provided by IPA
......
component/glib/buildout.cfg
View file @ 56ad14fb
......@@ -14,7 +14,7 @@ recipe = slapos.recipe.cmmi
url = http://ftp.gnome.org/pub/gnome/core/3.22/3.22.2/sources/glib-2.50.2.tar.xz
md5sum = 5eeb2bfaf78a07be59585e8b6e80b1d6
configure-options =
--with-python=${python2.7:location}/bin/python2.7
--with-python=${buildout:executable}
--disable-libmount
--disable-static
--disable-selinux
......
component/gnutls/buildout.cfg
View file @ 56ad14fb
[buildout]
extends =
../gettext/buildout.cfg
../gmp/buildout.cfg
../nettle/buildout.cfg
../p11-kit/buildout.cfg
../pkgconfig/buildout.cfg
../lunzip/buildout.cfg
../zlib/buildout.cfg
parts = gnutls
[gpg-error]
[gpg-common]
recipe = slapos.recipe.cmmi
url = ftp://ftp.gnupg.org/gcrypt/libgpg-error/libgpg-error-1.26.tar.bz2
md5sum = 97456709dbbcbb69354317ffe3e347cd
url = http://www.gnupg.org/ftp/gcrypt/${:_buildout_section_name_}/${:_buildout_section_name_}-${:version}.tar.bz2
# we'd like to combine <= and +=
configure-options = ${:configure-options-extra}
--enable-option-checking=fatal
--disable-dependency-tracking
environment = ${:environment-extra}
PATH=${gettext:location}/bin:%(PATH)s
environment-extra =
[gcrypt]
recipe = slapos.recipe.cmmi
url = ftp://ftp.gnupg.org/gcrypt/libgcrypt/libgcrypt-1.6.6.tar.bz2
md5sum = 944cf6595021d0c33478148a315b335b
configure-options =
--with-gpg-error-prefix=${gpg-error:location}
environment =
CPPFLAGS=-I${gpg-error:location}/include
LDFLAGS=-L${gpg-error:location}/lib -Wl,-rpath=${gpg-error:location}/lib
[libgpg-error]
<= gpg-common
version = 1.27
md5sum = 5217ef3e76a7275a2a3b569a12ddc989
configure-options-extra =
--disable-doc
--disable-tests
[libgcrypt]
<= gpg-common
version = 1.7.7
md5sum = d1769481b1b506a632fd66c5e5f62e41
configure-options-extra =
--with-gpg-error-prefix=${libgpg-error:location}
--disable-doc
environment-extra =
LDFLAGS=-Wl,-rpath=${libgpg-error:location}/lib
[gnutls]
recipe = slapos.recipe.cmmi
url = ftp://ftp.gnutls.org/gcrypt/gnutls/v3.3/gnutls-3.3.26.tar.xz
md5sum = 9840c06019bfa7db07ed8fd5e63106d5
configure-options =
<= gpg-common
url = http://www.gnupg.org/ftp/gcrypt/gnutls/v3.5/gnutls-3.5.13.tar.xz
md5sum = 4fd41ad86572933c2379b4cc321a0959
configure-options-extra =
--disable-doc
--disable-static
--disable-libdane
--disable-guile
--disable-crywrap
--without-tpm
--with-included-unistring
pkg-config-path = ${libtasn1:location}/lib/pkgconfig:${nettle:location}/lib/pkgconfig:${p11-kit:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig
environment =
PATH=${pkgconfig:location}/bin:${xz-utils:location}/bin:%(PATH)s
PKG_CONFIG_PATH=${nettle:location}/lib/pkgconfig:${p11-kit:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig
CPPFLAGS=-I${gmp:location}/include -I${zlib:location}/include
LDFLAGS=-L${gmp:location}/lib -Wl,-rpath=${gmp:location}/lib -Wl,-rpath=${nettle:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
PKG_CONFIG_PATH=${:pkg-config-path}
CPPFLAGS=-I${gmp:location}/include -I${libtasn1:location}/include
LDFLAGS=-L${gmp:location}/lib -Wl,-rpath=${gmp:location}/lib -Wl,-rpath=${nettle:location}/lib -Wl,-rpath=${zlib:location}/lib
component/groonga/buildout.cfg
View file @ 56ad14fb
......@@ -14,8 +14,8 @@ extends =
[groonga]
recipe = slapos.recipe.cmmi
url = http://packages.groonga.org/source/groonga/groonga-7.0.0.tar.gz
md5sum = 1b5383359ba1901e66b9e0910712569f
url = http://packages.groonga.org/source/groonga/groonga-7.0.3.tar.gz
md5sum = 4bddc049d43e8bbe40fae636ce03ac50
# temporary patch to respect more tokens in natural language mode.
patches =
${:_profile_base_location_}/groonga.patch#9ed02fbe8400402d3eab47eee149978b
......
component/haproxy/buildout.cfg
View file @ 56ad14fb
......@@ -12,8 +12,8 @@ parts = haproxy
[haproxy]
recipe = slapos.recipe.cmmi
url = http://www.haproxy.org/download/1.7/src/haproxy-1.7.2.tar.gz
md5sum = 7330b36f3764ebe409e9305803dc30e2
url = http://www.haproxy.org/download/1.7/src/haproxy-1.7.5.tar.gz
md5sum = ed84c80cb97852d2aa3161ed16c48a1c
configure-command = true
# If the system is running on Linux 2.6, we use "linux26" as the TARGET,
# otherwise use "generic".
......
component/headless-chromium/buildout.cfg 0 → 100644
View file @ 56ad14fb
[buildout]
extends =
../binutils/buildout.cfg
../bison/buildout.cfg
../pkgconfig/buildout.cfg
../gperf/buildout.cfg
../ninja/buildout.cfg
../freetype/buildout.cfg
../cmake/buildout.cfg
../git/buildout.cfg
../pkgconfig/buildout.cfg
../cups/buildout.cfg
../coreutils/buildout.cfg
../depot_tools/buildout.cfg
../findutils/buildout.cfg
../fontconfig/buildout.cfg
../gettext/buildout.cfg
../glib/buildout.cfg
../gtk-2/buildout.cfg
../libexpat/buildout.cfg
../libffi/buildout.cfg
../libpng/buildout.cfg
../libxml2/buildout.cfg
../mesa/buildout.cfg
../nspr/buildout.cfg
../nss/buildout.cfg
../pcre/buildout.cfg
../sqlite3/buildout.cfg
../xorg/buildout.cfg
../zlib/buildout.cfg
parts =
chromium
[gconf]
recipe = slapos.recipe.cmmi
url = http://ftp.gnome.org/pub/gnome/sources/GConf/3.2/GConf-3.2.6.tar.xz
md5sum = 2b16996d0e4b112856ee5c59130e822c
configure-options = --disable-orbit --disable-static
environment =
PATH=${pkgconfig:location}/bin:${intltool:location}/bin:${gettext:location}/bin:${glib:location}/bin:%(PATH)s
PKG_CONFIG_PATH=${glib:location}/lib/pkgconfig:${pcre:location}/lib/pkgconfig:${libxml2:location}/lib/pkgconfig:${dbus:location}/lib/pkgconfig:${dbus-glib:location}/lib/pkgconfig:$PKG_CONFIG_PATH
[chromedriver]
recipe = hexagonit.recipe.download
url = https://chromedriver.storage.googleapis.com/2.28/chromedriver_linux64.zip
md5sum = a72088c0a6b018ded2c0fff616da8f65
[chromium-download]
recipe = plone.recipe.command
# This revision is 56.0.2924.122. Because the chromedriver only support some certain
# version. Which version 56 is in the middle of 55-57.
revision = faf03429d9c3dbd483700dd42316b20776cbbd3c
path = ${buildout:parts-directory}/${:_buildout_section_name_}
command =
set -e
PATH=${depot_tools:location}:${git:location}/bin:$PATH
[ -d ${:path} ] && rm -r ${:path}
mkdir -p ${:path}
cd ${:path}
# Do never use `fetch` with the `--no-history` option unless you find an
# option to fetch directly at the wanted revision. `--no-history` could
# reduce the download size significantly but even if it retrieves enough
# commits at the time you test this section, development continues upstream
# and at some point the next command (gclient) would break.
# ...
# This command only could work in an empty dir.
fetch --nohooks chromium
gclient sync --revision ${:revision} --with_branch_heads
stop-on-error = true
[chromium]
recipe = slapos.recipe.cmmi
path = ${chromium-download:path}/src
location = ${buildout:parts-directory}/${:_buildout_section_name_}
configure-command =
gclient runhooks
mkdir -p ${:location}
echo 'use_udev = true
is_debug = false
enable_nacl = false' > ${:location}/args.gn
gn gen ${:location}
# Note you can run Chromium manually by: ${:location}/chrome --headless --no-sandbox --disable-gpu
make-binary = ninja -C ${:location} chrome
environment =
PKG_CONFIG_PATH=${freetype:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig:${libpng:location}/lib/pkgconfig:${randrproto:location}/lib/pkgconfig:$PKG_CONFIG_PATH
PATH=${chromedriver:location}:${dbus:location}/bin:${depot_tools:location}:${pkgconfig:location}/bin:${ninja:path}/:${bison:location}/bin:${gperf:location}/bin:${xserver:location}/bin:%(PATH)s
CPATH=${dbus:location}/include/dbus-1.0:${dbus:location}/lib/dbus-1.0/include/:${freetype:location}/include/freetype2:${libffi:location}/include:${mpfr:location}/include:${ncurses:location}/include:${openssl:location}/include:${readline:location}/include:${sqlite3:location}/include:${zlib:location}/include:${bzip2:location}/include:$CPATH
LD_LIBRARY_PATH=${alsa:location}/lib:${gconf:location}/lib:${libXScrnSaver:location}/lib:${glib:location}/lib:${atk:location}/lib:${cairo:location}/lib:${cups:location}/lib:${dbus:location}/lib:${dbus-glib:location}/lib:${fontconfig:location}/lib/:${gdk-pixbuf:location}/lib:${gettext:location}/lib:${glib:location}/lib:${gtk-2:location}/lib:${harfbuzz:location}/lib:${libX11:location}/lib:${libXau:location}/lib:${libXcomposite:location}/lib:${libXcursor:location}/lib:${libXext:location}/lib:${libXi:location}/lib:${libXrender:location}/lib/:${libXtst:location}/lib:${libexpat:location}/lib:${libffi:location}/lib:${libpng:location}/lib:${libpng12:location}/lib:${libxcb:location}/lib:${libxml2:location}/lib:${mesa:location}/lib:${nspr:location}/lib:${nss:location}/lib:${pango:location}/lib:${pcre:location}/lib:${pixman:location}/lib:${sqlite3:location}/lib:${xdamage:location}/lib:${xfixes:location}/lib:${zlib:location}/lib:$LD_LIBRARY_PATH
component/intltool/buildout.cfg
View file @ 56ad14fb
[buildout]
extends =
../gettext/buildout.cfg
../patch/buildout.cfg
../perl/buildout.cfg
../perl-XML-Parser/buildout.cfg
......@@ -9,10 +10,13 @@ parts =
[intltool]
recipe = slapos.recipe.cmmi
url = http://edge.launchpad.net/intltool/trunk/0.41.1/+download/intltool-0.41.1.tar.gz
md5sum = d6c91bf06681919ccfdf3624035b75dc
url = http://edge.launchpad.net/intltool/trunk/0.51.0/+download/intltool-0.51.0.tar.gz
md5sum = 12e517cac2b57a0121cda351570f1e63
patch-options = -p1
patches =
${:_profile_base_location_}/intltool-0.51.0.regexp.patch#3e7736d5ce7876e9a8d078c35e7f327b
depends =
${perl-XML-Parser:location}
environment =
PATH=${perl:location}/bin:${gettext:location}/bin:${pkgconfig:location}/bin:%(PATH)s
PATH=${patch:location}/bin:${perl:location}/bin:${gettext:location}/bin:${pkgconfig:location}/bin:%(PATH)s
PKG_CONFIG_PATH=${perl:location}/lib/pkgconfig
component/intltool/intltool-0.51.0.regexp.patch 0 → 100644
View file @ 56ad14fb
--- intltool-0.51.0.orig/intltool-update.in 2015-03-09 02:39:54.000000000 +0100
+++ intltool-0.51.0.orig/intltool-update.in 2015-06-19 01:52:07.171228154 +0200
@@ -1062,7 +1062,7 @@
}
}
- if ($str =~ /^(.*)\${?([A-Z_]+)}?(.*)$/)
+ if ($str =~ /^(.*)\$\{?([A-Z_]+)}?(.*)$/)
{
my $rest = $3;
my $untouched = $1;
@@ -1190,10 +1190,10 @@
$name =~ s/\(+$//g;
$version =~ s/\(+$//g;
- $varhash{"PACKAGE_NAME"} = $name if (not $name =~ /\${?AC_PACKAGE_NAME}?/);
- $varhash{"PACKAGE"} = $name if (not $name =~ /\${?PACKAGE}?/);
- $varhash{"PACKAGE_VERSION"} = $version if (not $name =~ /\${?AC_PACKAGE_VERSION}?/);
- $varhash{"VERSION"} = $version if (not $name =~ /\${?VERSION}?/);
+ $varhash{"PACKAGE_NAME"} = $name if (not $name =~ /\$\{?AC_PACKAGE_NAME}?/);
+ $varhash{"PACKAGE"} = $name if (not $name =~ /\$\{?PACKAGE}?/);
+ $varhash{"PACKAGE_VERSION"} = $version if (not $name =~ /\$\{?AC_PACKAGE_VERSION}?/);
+ $varhash{"VERSION"} = $version if (not $name =~ /\$\{?VERSION}?/);
}
if ($conf_source =~ /^AC_INIT\(([^,\)]+),([^,\)]+)[,]?([^,\)]+)?/m)
@@ -1219,11 +1219,11 @@
$version =~ s/\(+$//g;
$bugurl =~ s/\(+$//g if (defined $bugurl);
- $varhash{"PACKAGE_NAME"} = $name if (not $name =~ /\${?AC_PACKAGE_NAME}?/);
- $varhash{"PACKAGE"} = $name if (not $name =~ /\${?PACKAGE}?/);
- $varhash{"PACKAGE_VERSION"} = $version if (not $name =~ /\${?AC_PACKAGE_VERSION}?/);
- $varhash{"VERSION"} = $version if (not $name =~ /\${?VERSION}?/);
- $varhash{"PACKAGE_BUGREPORT"} = $bugurl if (defined $bugurl and not $bugurl =~ /\${?\w+}?/);
+ $varhash{"PACKAGE_NAME"} = $name if (not $name =~ /\$\{?AC_PACKAGE_NAME}?/);
+ $varhash{"PACKAGE"} = $name if (not $name =~ /\$\{?PACKAGE}?/);
+ $varhash{"PACKAGE_VERSION"} = $version if (not $name =~ /\$\{?AC_PACKAGE_VERSION}?/);
+ $varhash{"VERSION"} = $version if (not $name =~ /\$\{?VERSION}?/);
+ $varhash{"PACKAGE_BUGREPORT"} = $bugurl if (defined $bugurl and not $bugurl =~ /\$\{?\w+}?/);
}
# \s makes this not work, why?
component/libexpat/buildout.cfg
View file @ 56ad14fb
......@@ -4,7 +4,7 @@ parts =
[libexpat]
recipe = slapos.recipe.cmmi
url = http://downloads.sourceforge.net/project/expat/expat/2.2.0/expat-2.2.0.tar.bz2
md5sum = 2f47841c829facb346eb6e3fab5212e2
url = http://downloads.sourceforge.net/project/expat/expat/2.2.1/expat-2.2.1.tar.bz2
md5sum = d9c3baeab58774cefc2f04faf29f2cf8
configure-options =
--disable-static
component/libtasn1/buildout.cfg
View file @ 56ad14fb
......@@ -4,8 +4,8 @@ parts =
[libtasn1]
recipe = slapos.recipe.cmmi
url = http://ftp.gnu.org/gnu/libtasn1/libtasn1-4.8.tar.gz
md5sum = 9a6767705725544f2b86670dcfb34107
url = http://ftp.gnu.org/gnu/libtasn1/libtasn1-4.12.tar.gz
md5sum = 5c724bd1f73aaf4a311833e1cd297b21
configure-options =
--disable-static
--disable-gtk-doc-html
component/libtiff/buildout.cfg
View file @ 56ad14fb
......@@ -18,7 +18,7 @@ configure-options =
--disable-lzma
patch-options = -p1
patches =
${:_profile_base_location_}/tiff_4.0.7-5.debian.patch#f132ec0dc5cde03f9bbbe90b7a79fed4
${:_profile_base_location_}/tiff_4.0.7-6.debian.patch#b9249bb352e693b7975f9d343018994d
environment =
CPPFLAGS=-I${libjpeg:location}/include -I${jbigkit:location}/include -I${zlib:location}/include
LDFLAGS=-L${libjpeg:location}/lib -Wl,-rpath=${libjpeg:location}/lib -L${jbigkit:location}/lib -Wl,-rpath=${jbigkit:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
......
component/libtiff/tiff_4.0.7-5.debian.patch → component/libtiff/tiff_4.0.7-6.debian.patch
View file @ 56ad14fb
......@@ -1156,3 +1156,765 @@ index bdf754c..8bbcd52 100644
if (shortv != config && bitspersample != 8 && samplesperpixel > 1) {
fprintf(stderr,
"%s: Cannot handle different planar configuration w/ bits/sample != 8\n",
From 47f2fb61a3a64667bce1a8398a8fcb1b348ff122 Mon Sep 17 00:00:00 2001
From: erouault <erouault>
Date: Wed, 11 Jan 2017 12:15:01 +0000
Subject: [PATCH] * libtiff/tif_jpeg.c: avoid integer division by zero in
JPEGSetupEncode() when horizontal or vertical sampling is set to 0. Fixes
http://bugzilla.maptools.org/show_bug.cgi?id=2653
---
ChangeLog | 6 ++++++
libtiff/tif_jpeg.c | 7 +++++++
2 files changed, 13 insertions(+)
diff --git a/ChangeLog b/ChangeLog
index c82bc76..a7208f5 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,11 @@
2017-01-11 Even Rouault <even.rouault at spatialys.com>
+ * libtiff/tif_jpeg.c: avoid integer division by zero in
+ JPEGSetupEncode() when horizontal or vertical sampling is set to 0.
+ Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2653
+
+2017-01-11 Even Rouault <even.rouault at spatialys.com>
+
* tools/tiffcp.c: error out cleanly in cpContig2SeparateByRow and
cpSeparate2ContigByRow if BitsPerSample != 8 to avoid heap based overflow.
Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2656 and
diff --git a/libtiff/tif_jpeg.c b/libtiff/tif_jpeg.c
index 38595f9..6c17c38 100644
--- a/libtiff/tif_jpeg.c
+++ b/libtiff/tif_jpeg.c
@@ -1626,6 +1626,13 @@ JPEGSetupEncode(TIFF* tif)
case PHOTOMETRIC_YCBCR:
sp->h_sampling = td->td_ycbcrsubsampling[0];
sp->v_sampling = td->td_ycbcrsubsampling[1];
+ if( sp->h_sampling == 0 || sp->v_sampling == 0 )
+ {
+ TIFFErrorExt(tif->tif_clientdata, module,
+ "Invalig horizontal/vertical sampling value");
+ return (0);
+ }
+
/*
* A ReferenceBlackWhite field *must* be present since the
* default value is inappropriate for YCbCr. Fill in the
From 3cfd62d77c2a7e147a05bd678524c345fa9c2bb8 Mon Sep 17 00:00:00 2001
From: erouault <erouault>
Date: Wed, 11 Jan 2017 13:28:01 +0000
Subject: [PATCH] * libtiff/tif_dirread.c: avoid division by floating point 0
in TIFFReadDirEntryCheckedRational() and TIFFReadDirEntryCheckedSrational(),
and return 0 in that case (instead of infinity as before presumably)
Apparently some sanitizers do not like those divisions by zero. Fixes
http://bugzilla.maptools.org/show_bug.cgi?id=2644
---
ChangeLog | 8 ++++++++
libtiff/tif_dirread.c | 10 ++++++++--
2 files changed, 16 insertions(+), 2 deletions(-)
diff --git a/ChangeLog b/ChangeLog
index 6a752cd..722a405 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,13 @@
2017-01-11 Even Rouault <even.rouault at spatialys.com>
+ * libtiff/tif_dirread.c: avoid division by floating point 0 in
+ TIFFReadDirEntryCheckedRational() and TIFFReadDirEntryCheckedSrational(),
+ and return 0 in that case (instead of infinity as before presumably)
+ Apparently some sanitizers do not like those divisions by zero.
+ Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2644
+
+2017-01-11 Even Rouault <even.rouault at spatialys.com>
+
* libtiff/tif_jpeg.c: avoid integer division by zero in
JPEGSetupEncode() when horizontal or vertical sampling is set to 0.
Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2653
diff --git a/libtiff/tif_dirread.c b/libtiff/tif_dirread.c
index 570d0c3..8a1e42a 100644
--- a/libtiff/tif_dirread.c
+++ b/libtiff/tif_dirread.c
@@ -2872,7 +2872,10 @@ static enum TIFFReadDirEntryErr TIFFReadDirEntryCheckedRational(TIFF* tif, TIFFD
m.l = direntry->tdir_offset.toff_long8;
if (tif->tif_flags&TIFF_SWAB)
TIFFSwabArrayOfLong(m.i,2);
- if (m.i[0]==0)
+ /* Not completely sure what we should do when m.i[1]==0, but some */
+ /* sanitizers do not like division by 0.0: */
+ /* http://bugzilla.maptools.org/show_bug.cgi?id=2644 */
+ if (m.i[0]==0 || m.i[1]==0)
*value=0.0;
else
*value=(double)m.i[0]/(double)m.i[1];
@@ -2900,7 +2903,10 @@ static enum TIFFReadDirEntryErr TIFFReadDirEntryCheckedSrational(TIFF* tif, TIFF
m.l=direntry->tdir_offset.toff_long8;
if (tif->tif_flags&TIFF_SWAB)
TIFFSwabArrayOfLong(m.i,2);
- if ((int32)m.i[0]==0)
+ /* Not completely sure what we should do when m.i[1]==0, but some */
+ /* sanitizers do not like division by 0.0: */
+ /* http://bugzilla.maptools.org/show_bug.cgi?id=2644 */
+ if ((int32)m.i[0]==0 || m.i[1]==0)
*value=0.0;
else
*value=(double)((int32)m.i[0])/(double)m.i[1];
From 3144e57770c1e4d26520d8abee750f8ac8b75490 Mon Sep 17 00:00:00 2001
From: erouault <erouault>
Date: Wed, 11 Jan 2017 16:09:02 +0000
Subject: [PATCH] * libtiff/tif_dir.c, tif_dirread.c, tif_dirwrite.c: implement
various clampings of double to other data types to avoid undefined behaviour
if the output range isn't big enough to hold the input value. Fixes
http://bugzilla.maptools.org/show_bug.cgi?id=2643
http://bugzilla.maptools.org/show_bug.cgi?id=2642
http://bugzilla.maptools.org/show_bug.cgi?id=2646
http://bugzilla.maptools.org/show_bug.cgi?id=2647
---
ChangeLog | 10 ++++++
libtiff/tif_dir.c | 18 +++++++---
libtiff/tif_dirread.c | 10 +++++-
libtiff/tif_dirwrite.c | 90 ++++++++++++++++++++++++++++++++++++++++++++------
4 files changed, 113 insertions(+), 15 deletions(-)
diff --git a/ChangeLog b/ChangeLog
index 722a405..6517640 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,15 @@
2017-01-11 Even Rouault <even.rouault at spatialys.com>
+ * libtiff/tif_dir.c, tif_dirread.c, tif_dirwrite.c: implement various clampings
+ of double to other data types to avoid undefined behaviour if the output range
+ isn't big enough to hold the input value.
+ Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2643
+ http://bugzilla.maptools.org/show_bug.cgi?id=2642
+ http://bugzilla.maptools.org/show_bug.cgi?id=2646
+ http://bugzilla.maptools.org/show_bug.cgi?id=2647
+
+2017-01-11 Even Rouault <even.rouault at spatialys.com>
+
* libtiff/tif_dirread.c: avoid division by floating point 0 in
TIFFReadDirEntryCheckedRational() and TIFFReadDirEntryCheckedSrational(),
and return 0 in that case (instead of infinity as before presumably)
diff --git a/libtiff/tif_dir.c b/libtiff/tif_dir.c
index 68a55af..a04d28f 100644
--- a/libtiff/tif_dir.c
+++ b/libtiff/tif_dir.c
@@ -31,6 +31,7 @@
* (and also some miscellaneous stuff)
*/
#include "tiffiop.h"
+#include <float.h>
/*
* These are used in the backwards compatibility code...
@@ -154,6 +155,15 @@ checkInkNamesString(TIFF* tif, uint32 slen, const char* s)
return (0);
}
+static float TIFFClampDoubleToFloat( double val )
+{
+ if( val > FLT_MAX )
+ return FLT_MAX;
+ if( val < -FLT_MAX )
+ return -FLT_MAX;
+ return (float)val;
+}
+
static int
_TIFFVSetField(TIFF* tif, uint32 tag, va_list ap)
{
@@ -312,13 +322,13 @@ _TIFFVSetField(TIFF* tif, uint32 tag, va_list ap)
dblval = va_arg(ap, double);
if( dblval < 0 )
goto badvaluedouble;
- td->td_xresolution = (float) dblval;
+ td->td_xresolution = TIFFClampDoubleToFloat( dblval );
break;
case TIFFTAG_YRESOLUTION:
dblval = va_arg(ap, double);
if( dblval < 0 )
goto badvaluedouble;
- td->td_yresolution = (float) dblval;
+ td->td_yresolution = TIFFClampDoubleToFloat( dblval );
break;
case TIFFTAG_PLANARCONFIG:
v = (uint16) va_arg(ap, uint16_vap);
@@ -327,10 +337,10 @@ _TIFFVSetField(TIFF* tif, uint32 tag, va_list ap)
td->td_planarconfig = (uint16) v;
break;
case TIFFTAG_XPOSITION:
- td->td_xposition = (float) va_arg(ap, double);
+ td->td_xposition = TIFFClampDoubleToFloat( va_arg(ap, double) );
break;
case TIFFTAG_YPOSITION:
- td->td_yposition = (float) va_arg(ap, double);
+ td->td_yposition = TIFFClampDoubleToFloat( va_arg(ap, double) );
break;
case TIFFTAG_RESOLUTIONUNIT:
v = (uint16) va_arg(ap, uint16_vap);
diff --git a/libtiff/tif_dirread.c b/libtiff/tif_dirread.c
index 8a1e42a..77b0f37 100644
--- a/libtiff/tif_dirread.c
+++ b/libtiff/tif_dirread.c
@@ -40,6 +40,7 @@
*/
#include "tiffiop.h"
+#include <float.h>
#define IGNORE 0 /* tag placeholder used below */
#define FAILED_FII ((uint32) -1)
@@ -2406,7 +2407,14 @@ static enum TIFFReadDirEntryErr TIFFReadDirEntryFloatArray(TIFF* tif, TIFFDirEnt
ma=(double*)origdata;
mb=data;
for (n=0; n<count; n++)
- *mb++=(float)(*ma++);
+ {
+ double val = *ma++;
+ if( val > FLT_MAX )
+ val = FLT_MAX;
+ else if( val < -FLT_MAX )
+ val = -FLT_MAX;
+ *mb++=(float)val;
+ }
}
break;
}
diff --git a/libtiff/tif_dirwrite.c b/libtiff/tif_dirwrite.c
index c9e871b..2967da5 100644
--- a/libtiff/tif_dirwrite.c
+++ b/libtiff/tif_dirwrite.c
@@ -30,6 +30,7 @@
* Directory Write Support Routines.
*/
#include "tiffiop.h"
+#include <float.h>
#ifdef HAVE_IEEEFP
#define TIFFCvtNativeToIEEEFloat(tif, n, fp)
@@ -939,6 +940,69 @@ TIFFWriteDirectorySec(TIFF* tif, int isimage, int imagedone, uint64* pdiroff)
return(0);
}
+static float TIFFClampDoubleToFloat( double val )
+{
+ if( val > FLT_MAX )
+ return FLT_MAX;
+ if( val < -FLT_MAX )
+ return -FLT_MAX;
+ return (float)val;
+}
+
+static int8 TIFFClampDoubleToInt8( double val )
+{
+ if( val > 127 )
+ return 127;
+ if( val < -128 || val != val )
+ return -128;
+ return (int8)val;
+}
+
+static int16 TIFFClampDoubleToInt16( double val )
+{
+ if( val > 32767 )
+ return 32767;
+ if( val < -32768 || val != val )
+ return -32768;
+ return (int16)val;
+}
+
+static int32 TIFFClampDoubleToInt32( double val )
+{
+ if( val > 0x7FFFFFFF )
+ return 0x7FFFFFFF;
+ if( val < -0x7FFFFFFF-1 || val != val )
+ return -0x7FFFFFFF-1;
+ return (int32)val;
+}
+
+static uint8 TIFFClampDoubleToUInt8( double val )
+{
+ if( val < 0 )
+ return 0;
+ if( val > 255 || val != val )
+ return 255;
+ return (uint8)val;
+}
+
+static uint16 TIFFClampDoubleToUInt16( double val )
+{
+ if( val < 0 )
+ return 0;
+ if( val > 65535 || val != val )
+ return 65535;
+ return (uint16)val;
+}
+
+static uint32 TIFFClampDoubleToUInt32( double val )
+{
+ if( val < 0 )
+ return 0;
+ if( val > 0xFFFFFFFFU || val != val )
+ return 0xFFFFFFFFU;
+ return (uint32)val;
+}
+
static int
TIFFWriteDirectoryTagSampleformatArray(TIFF* tif, uint32* ndir, TIFFDirEntry* dir, uint16 tag, uint32 count, double* value)
{
@@ -959,7 +1023,7 @@ TIFFWriteDirectoryTagSampleformatArray(TIFF* tif, uint32* ndir, TIFFDirEntry* di
if (tif->tif_dir.td_bitspersample<=32)
{
for (i = 0; i < count; ++i)
- ((float*)conv)[i] = (float)value[i];
+ ((float*)conv)[i] = TIFFClampDoubleToFloat(value[i]);
ok = TIFFWriteDirectoryTagFloatArray(tif,ndir,dir,tag,count,(float*)conv);
}
else
@@ -971,19 +1035,19 @@ TIFFWriteDirectoryTagSampleformatArray(TIFF* tif, uint32* ndir, TIFFDirEntry* di
if (tif->tif_dir.td_bitspersample<=8)
{
for (i = 0; i < count; ++i)
- ((int8*)conv)[i] = (int8)value[i];
+ ((int8*)conv)[i] = TIFFClampDoubleToInt8(value[i]);
ok = TIFFWriteDirectoryTagSbyteArray(tif,ndir,dir,tag,count,(int8*)conv);
}
else if (tif->tif_dir.td_bitspersample<=16)
{
for (i = 0; i < count; ++i)
- ((int16*)conv)[i] = (int16)value[i];
+ ((int16*)conv)[i] = TIFFClampDoubleToInt16(value[i]);
ok = TIFFWriteDirectoryTagSshortArray(tif,ndir,dir,tag,count,(int16*)conv);
}
else
{
for (i = 0; i < count; ++i)
- ((int32*)conv)[i] = (int32)value[i];
+ ((int32*)conv)[i] = TIFFClampDoubleToInt32(value[i]);
ok = TIFFWriteDirectoryTagSlongArray(tif,ndir,dir,tag,count,(int32*)conv);
}
break;
@@ -991,19 +1055,19 @@ TIFFWriteDirectoryTagSampleformatArray(TIFF* tif, uint32* ndir, TIFFDirEntry* di
if (tif->tif_dir.td_bitspersample<=8)
{
for (i = 0; i < count; ++i)
- ((uint8*)conv)[i] = (uint8)value[i];
+ ((uint8*)conv)[i] = TIFFClampDoubleToUInt8(value[i]);
ok = TIFFWriteDirectoryTagByteArray(tif,ndir,dir,tag,count,(uint8*)conv);
}
else if (tif->tif_dir.td_bitspersample<=16)
{
for (i = 0; i < count; ++i)
- ((uint16*)conv)[i] = (uint16)value[i];
+ ((uint16*)conv)[i] = TIFFClampDoubleToUInt16(value[i]);
ok = TIFFWriteDirectoryTagShortArray(tif,ndir,dir,tag,count,(uint16*)conv);
}
else
{
for (i = 0; i < count; ++i)
- ((uint32*)conv)[i] = (uint32)value[i];
+ ((uint32*)conv)[i] = TIFFClampDoubleToUInt32(value[i]);
ok = TIFFWriteDirectoryTagLongArray(tif,ndir,dir,tag,count,(uint32*)conv);
}
break;
@@ -2094,6 +2094,7 @@ TIFFWriteDirectoryTagCheckedSlong8Array(
static int
TIFFWriteDirectoryTagCheckedRational(TIFF* tif, uint32* ndir, TIFFDirEntry* dir, uint16 tag, double value)
{
+ static const char module[] = "TIFFWriteDirectoryTagCheckedRational";
uint32 m[2];
assert(value>=0.0);
assert(sizeof(uint32)==4);
@@ -2102,7 +2102,12 @@ TIFFWriteDirectoryTagCheckedRational(TIF
m[0]=0;
m[1]=1;
}
- else if (value==(double)(uint32)value)
+ else if( value != value )
+ {
+ TIFFErrorExt(tif->tif_clientdata,module,"Not-a-number value is illegal");
+ return 0;
+ }
+ else if (value <= 0xFFFFFFFFU && value==(double)(uint32)value)
{
m[0]=(uint32)value;
m[1]=1;
@@ -2143,12 +2212,13 @@ TIFFWriteDirectoryTagCheckedRationalArray(TIFF* tif, uint32* ndir, TIFFDirEntry*
}
for (na=value, nb=m, nc=0; nc<count; na++, nb+=2, nc++)
{
- if (*na<=0.0)
+ if (*na<=0.0 || *na != *na)
{
nb[0]=0;
nb[1]=1;
}
- else if (*na==(float)(uint32)(*na))
+ else if (*na >= 0 && *na <= (float)0xFFFFFFFFU &&
+ *na==(float)(uint32)(*na))
{
nb[0]=(uint32)(*na);
nb[1]=1;
From 0a76a8c765c7b8327c59646284fa78c3c27e5490 Mon Sep 17 00:00:00 2001
From: erouault <erouault>
Date: Wed, 11 Jan 2017 16:13:50 +0000
Subject: [PATCH] * libtiff/tif_jpeg.c: validate BitsPerSample in
JPEGSetupEncode() to avoid undefined behaviour caused by invalid shift
exponent. Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2648
---
ChangeLog | 6 ++++++
libtiff/tif_jpeg.c | 7 +++++++
2 files changed, 13 insertions(+)
diff --git a/ChangeLog b/ChangeLog
index 6517640..8e202a2 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,11 @@
2017-01-11 Even Rouault <even.rouault at spatialys.com>
+ * libtiff/tif_jpeg.c: validate BitsPerSample in JPEGSetupEncode() to avoid
+ undefined behaviour caused by invalid shift exponent.
+ Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2648
+
+2017-01-11 Even Rouault <even.rouault at spatialys.com>
+
* libtiff/tif_dir.c, tif_dirread.c, tif_dirwrite.c: implement various clampings
of double to other data types to avoid undefined behaviour if the output range
isn't big enough to hold the input value.
diff --git a/libtiff/tif_jpeg.c b/libtiff/tif_jpeg.c
index 6c17c38..192989a 100644
--- a/libtiff/tif_jpeg.c
+++ b/libtiff/tif_jpeg.c
@@ -1632,6 +1632,13 @@ JPEGSetupEncode(TIFF* tif)
"Invalig horizontal/vertical sampling value");
return (0);
}
+ if( td->td_bitspersample > 16 )
+ {
+ TIFFErrorExt(tif->tif_clientdata, module,
+ "BitsPerSample %d not allowed for JPEG",
+ td->td_bitspersample);
+ return (0);
+ }
/*
* A ReferenceBlackWhite field *must* be present since the
From 66e7bd59520996740e4df5495a830b42fae48bc4 Mon Sep 17 00:00:00 2001
From: erouault <erouault>
Date: Wed, 11 Jan 2017 16:33:34 +0000
Subject: [PATCH] * libtiff/tif_read.c: avoid potential undefined behaviour on
signed integer addition in TIFFReadRawStrip1() in isMapped() case. Fixes
http://bugzilla.maptools.org/show_bug.cgi?id=2650
---
ChangeLog | 6 ++++++
libtiff/tif_read.c | 27 ++++++++++++++++++---------
2 files changed, 24 insertions(+), 9 deletions(-)
diff --git a/ChangeLog b/ChangeLog
index 8e202a2..3e31464 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,11 @@
2017-01-11 Even Rouault <even.rouault at spatialys.com>
+ * libtiff/tif_read.c: avoid potential undefined behaviour on signed integer
+ addition in TIFFReadRawStrip1() in isMapped() case.
+ Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2650
+
+2017-01-11 Even Rouault <even.rouault at spatialys.com>
+
* libtiff/tif_jpeg.c: validate BitsPerSample in JPEGSetupEncode() to avoid
undefined behaviour caused by invalid shift exponent.
Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2648
diff --git a/libtiff/tif_read.c b/libtiff/tif_read.c
index 52bbf50..b7aacbd 100644
--- a/libtiff/tif_read.c
+++ b/libtiff/tif_read.c
@@ -420,16 +420,25 @@ TIFFReadRawStrip1(TIFF* tif, uint32 strip, void* buf, tmsize_t size,
return ((tmsize_t)(-1));
}
} else {
- tmsize_t ma,mb;
+ tmsize_t ma;
tmsize_t n;
- ma=(tmsize_t)td->td_stripoffset[strip];
- mb=ma+size;
- if ((td->td_stripoffset[strip] > (uint64)TIFF_TMSIZE_T_MAX)||(ma>tif->tif_size))
- n=0;
- else if ((mb<ma)||(mb<size)||(mb>tif->tif_size))
- n=tif->tif_size-ma;
- else
- n=size;
+ if ((td->td_stripoffset[strip] > (uint64)TIFF_TMSIZE_T_MAX)||
+ ((ma=(tmsize_t)td->td_stripoffset[strip])>tif->tif_size))
+ {
+ n=0;
+ }
+ else if( ma > TIFF_TMSIZE_T_MAX - size )
+ {
+ n=0;
+ }
+ else
+ {
+ tmsize_t mb=ma+size;
+ if (mb>tif->tif_size)
+ n=tif->tif_size-ma;
+ else
+ n=size;
+ }
if (n!=size) {
#if defined(__WIN32__) && (defined(_MSC_VER) || defined(__MINGW32__))
TIFFErrorExt(tif->tif_clientdata, module,
From 48780b4fcc425cddc4ef8ffdf536f96a0d1b313b Mon Sep 17 00:00:00 2001
From: erouault <erouault>
Date: Wed, 11 Jan 2017 16:38:26 +0000
Subject: [PATCH] =?UTF-8?q?*=20libtiff/tif=5Fgetimage.c:=20add=20explicit?=
=?UTF-8?q?=20uint32=20cast=20in=20putagreytile=20to=20avoid=20UndefinedBe?=
=?UTF-8?q?haviorSanitizer=20warning.=20Patch=20by=20Nicol=C3=A1s=20Pe?=
=?UTF-8?q?=C3=B1a.=20Fixes=20http://bugzilla.maptools.org/show=5Fbug.cgi?=
=?UTF-8?q?=3Fid=3D2658?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
ChangeLog | 7 +++++++
libtiff/tif_getimage.c | 2 +-
2 files changed, 8 insertions(+), 1 deletion(-)
diff --git a/ChangeLog b/ChangeLog
index 3e31464..6a342e5 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,12 @@
2017-01-11 Even Rouault <even.rouault at spatialys.com>
+ * libtiff/tif_getimage.c: add explicit uint32 cast in putagreytile to
+ avoid UndefinedBehaviorSanitizer warning.
+ Patch by Nicolás Peña.
+ Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2658
+
+2017-01-11 Even Rouault <even.rouault at spatialys.com>
+
* libtiff/tif_read.c: avoid potential undefined behaviour on signed integer
addition in TIFFReadRawStrip1() in isMapped() case.
Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2650
diff --git a/libtiff/tif_getimage.c b/libtiff/tif_getimage.c
index fed31f1..2fa1775 100644
--- a/libtiff/tif_getimage.c
+++ b/libtiff/tif_getimage.c
@@ -1305,7 +1305,7 @@ DECLAREContigPutFunc(putagreytile)
while (h-- > 0) {
for (x = w; x-- > 0;)
{
- *cp++ = BWmap[*pp][0] & (*(pp+1) << 24 | ~A1);
+ *cp++ = BWmap[*pp][0] & ((uint32)*(pp+1) << 24 | ~A1);
pp += samplesperpixel;
}
cp += toskew;
From d60332057b9575ada4f264489582b13e30137be1 Mon Sep 17 00:00:00 2001
From: erouault <erouault>
Date: Wed, 11 Jan 2017 19:02:49 +0000
Subject: [PATCH] * libtiff/tiffiop.h, tif_unix.c, tif_win32.c, tif_vms.c: add
_TIFFcalloc()
* libtiff/tif_read.c: TIFFReadBufferSetup(): use _TIFFcalloc() to zero
initialize tif_rawdata.
Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2651
---
ChangeLog | 8 ++++++++
libtiff/tif_read.c | 4 +++-
libtiff/tif_unix.c | 8 ++++++++
libtiff/tif_vms.c | 8 ++++++++
libtiff/tif_win32.c | 8 ++++++++
libtiff/tiffio.h | 1 +
6 files changed, 36 insertions(+), 1 deletion(-)
diff --git a/ChangeLog b/ChangeLog
index 6a342e5..abd75d7 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,13 @@
2017-01-11 Even Rouault <even.rouault at spatialys.com>
+ * libtiff/tiffiop.h, tif_unix.c, tif_win32.c, tif_vms.c: add _TIFFcalloc()
+
+ * libtiff/tif_read.c: TIFFReadBufferSetup(): use _TIFFcalloc() to zero
+ initialize tif_rawdata.
+ Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2651
+
+2017-01-11 Even Rouault <even.rouault at spatialys.com>
+
* libtiff/tif_getimage.c: add explicit uint32 cast in putagreytile to
avoid UndefinedBehaviorSanitizer warning.
Patch by Nicolás Peña.
diff --git a/libtiff/tif_read.c b/libtiff/tif_read.c
index 277fdd6..4535ccb 100644
--- a/libtiff/tif_read.c
+++ b/libtiff/tif_read.c
@@ -985,7 +985,9 @@ TIFFReadBufferSetup(TIFF* tif, void* bp, tmsize_t size)
"Invalid buffer size");
return (0);
}
- tif->tif_rawdata = (uint8*) _TIFFmalloc(tif->tif_rawdatasize);
+ /* Initialize to zero to avoid uninitialized buffers in case of */
+ /* short reads (http://bugzilla.maptools.org/show_bug.cgi?id=2651) */
+ tif->tif_rawdata = (uint8*) _TIFFcalloc(1, tif->tif_rawdatasize);
tif->tif_flags |= TIFF_MYBUFFER;
}
if (tif->tif_rawdata == NULL) {
diff --git a/libtiff/tif_unix.c b/libtiff/tif_unix.c
index 7c7bc96..89dd32e 100644
--- a/libtiff/tif_unix.c
+++ b/libtiff/tif_unix.c
@@ -316,6 +316,14 @@ _TIFFmalloc(tmsize_t s)
return (malloc((size_t) s));
}
+void* _TIFFcalloc(tmsize_t nmemb, tmsize_t siz)
+{
+ if( nmemb == 0 || siz == 0 )
+ return ((void *) NULL);
+
+ return calloc((size_t) nmemb, (size_t)siz);
+}
+
void
_TIFFfree(void* p)
{
diff --git a/libtiff/tif_win32.c b/libtiff/tif_win32.c
index d730b3a..3e9001b 100644
--- a/libtiff/tif_win32.c
+++ b/libtiff/tif_win32.c
@@ -360,6 +360,14 @@ _TIFFmalloc(tmsize_t s)
return (malloc((size_t) s));
}
+void* _TIFFcalloc(tmsize_t nmemb, tmsize_t siz)
+{
+ if( nmemb == 0 || siz == 0 )
+ return ((void *) NULL);
+
+ return calloc((size_t) nmemb, (size_t)siz);
+}
+
void
_TIFFfree(void* p)
{
diff --git a/libtiff/tiffio.h b/libtiff/tiffio.h
index 732da17..fbd9171 100644
--- a/libtiff/tiffio.h
+++ b/libtiff/tiffio.h
@@ -293,6 +293,7 @@ extern TIFFCodec* TIFFGetConfiguredCODECs(void);
*/
extern void* _TIFFmalloc(tmsize_t s);
+extern void* _TIFFcalloc(tmsize_t nmemb, tmsize_t siz);
extern void* _TIFFrealloc(void* p, tmsize_t s);
extern void _TIFFmemset(void* p, int v, tmsize_t c);
extern void _TIFFmemcpy(void* d, const void* s, tmsize_t c);
From 8283e4d1b7e53340684d12932880cbcbaf23a8c1 Mon Sep 17 00:00:00 2001
From: erouault <erouault>
Date: Thu, 12 Jan 2017 17:43:25 +0000
Subject: [PATCH] =?UTF-8?q?*=20libtiff/tif=5Fojpeg.c:=20fix=20leak=20in=20?=
=?UTF-8?q?OJPEGReadHeaderInfoSecTablesAcTable=20when=20read=20fails.=20Pa?=
=?UTF-8?q?tch=20by=20Nicol=C3=A1s=20Pe=C3=B1a.=20Fixes=20http://bugzilla.?=
=?UTF-8?q?maptools.org/show=5Fbug.cgi=3Fid=3D2659?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
ChangeLog | 7 +++++++
libtiff/tif_ojpeg.c | 3 +++
2 files changed, 10 insertions(+)
diff --git a/ChangeLog b/ChangeLog
index 6e6f3b0..12e0370 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,10 @@
+2017-01-12 Even Rouault <even.rouault at spatialys.com>
+
+ * libtiff/tif_ojpeg.c: fix leak in OJPEGReadHeaderInfoSecTablesAcTable
+ when read fails.
+ Patch by Nicolás Peña.
+ Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2659
+
2017-01-11 Even Rouault <even.rouault at spatialys.com>
* libtiff/tiffiop.h, tif_unix.c, tif_win32.c, tif_vms.c: add _TIFFcalloc()
diff --git a/libtiff/tif_ojpeg.c b/libtiff/tif_ojpeg.c
index f19e8fd..b92f0eb 100644
--- a/libtiff/tif_ojpeg.c
+++ b/libtiff/tif_ojpeg.c
@@ -1918,7 +1918,10 @@ OJPEGReadHeaderInfoSecTablesAcTable(TIFF* tif)
rb[sizeof(uint32)+5+n]=o[n];
p=(uint32)TIFFReadFile(tif,&(rb[sizeof(uint32)+21]),q);
if (p!=q)
+ {
+ _TIFFfree(rb);
return(0);
+ }
sp->actable[m]=rb;
sp->sos_tda[m]=(sp->sos_tda[m]|m);
}
From 2ea32f7372b65c24b2816f11c04bf59b5090d05b Mon Sep 17 00:00:00 2001
From: erouault <erouault>
Date: Thu, 12 Jan 2017 19:23:20 +0000
Subject: [PATCH] * libtiff/tif_ojpeg.c: fix leak in
OJPEGReadHeaderInfoSecTablesQTable, OJPEGReadHeaderInfoSecTablesDcTable and
OJPEGReadHeaderInfoSecTablesAcTable
---
ChangeLog | 3 ++-
libtiff/tif_ojpeg.c | 6 ++++++
2 files changed, 8 insertions(+), 1 deletion(-)
diff --git a/ChangeLog b/ChangeLog
index 12e0370..cd2fa17 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
2017-01-12 Even Rouault <even.rouault at spatialys.com>
- * libtiff/tif_ojpeg.c: fix leak in OJPEGReadHeaderInfoSecTablesAcTable
+ * libtiff/tif_ojpeg.c: fix leak in OJPEGReadHeaderInfoSecTablesQTable,
+ OJPEGReadHeaderInfoSecTablesDcTable and OJPEGReadHeaderInfoSecTablesAcTable
when read fails.
Patch by Nicolás Peña.
Fixes http://bugzilla.maptools.org/show_bug.cgi?id=2659
diff --git a/libtiff/tif_ojpeg.c b/libtiff/tif_ojpeg.c
index b92f0eb..5f6c684 100644
--- a/libtiff/tif_ojpeg.c
+++ b/libtiff/tif_ojpeg.c
@@ -1790,7 +1790,10 @@ OJPEGReadHeaderInfoSecTablesQTable(TIFF* tif)
TIFFSeekFile(tif,sp->qtable_offset[m],SEEK_SET);
p=(uint32)TIFFReadFile(tif,&ob[sizeof(uint32)+5],64);
if (p!=64)
+ {
+ _TIFFfree(ob);
return(0);
+ }
sp->qtable[m]=ob;
sp->sof_tq[m]=m;
}
@@ -1854,7 +1857,10 @@ OJPEGReadHeaderInfoSecTablesDcTable(TIFF* tif)
rb[sizeof(uint32)+5+n]=o[n];
p=(uint32)TIFFReadFile(tif,&(rb[sizeof(uint32)+21]),q);
if (p!=q)
+ {
+ _TIFFfree(rb);
return(0);
+ }
sp->dctable[m]=rb;
sp->sos_tda[m]=(m<<4);
}
component/mariadb/buildout.cfg
View file @ 56ad14fb
......@@ -3,6 +3,7 @@
[buildout]
extends =
../bzip2/buildout.cfg
../cmake/buildout.cfg
../groonga/buildout.cfg
../jemalloc/buildout.cfg
......@@ -22,11 +23,13 @@ parts =
[mariadb]
recipe = slapos.recipe.cmmi
url = https://downloads.mariadb.com/MariaDB/mariadb-10.1.21/source/mariadb-10.1.21.tar.gz
md5sum = ff8b96edeeb22425117309a249055310
url = https://downloads.mariadb.com/MariaDB/mariadb-10.1.24/source/mariadb-10.1.24.tar.gz
md5sum = e1af4c62e3345b65975f2fa43a0040d2
patch-options = -p0
patches =
${:_profile_base_location_}/mariadb_10.1.21_create_system_tables__no_test.patch#3c76aa9564a162f13aced7c0a3f783b3
${:stable-patches}
stable-patches =
location = ${buildout:parts-directory}/${:_buildout_section_name_}
configure-command = ${cmake:location}/bin/cmake
configure-options =
......@@ -43,7 +46,7 @@ configure-options =
-DWITH_EXTRA_CHARSETS=complex
-DWITH_EMBEDDED_SERVER=0
-DWITH_JEMALLOC=yes
-DWITH_INNODB_BZIP2=OFF
-DWITH_INNODB_BZIP2=ON
-DWITH_INNODB_LZ4=OFF
-DWITH_INNODB_LZMA=ON
-DWITH_INNODB_LZO=OFF
......@@ -52,15 +55,15 @@ configure-options =
-DWITHOUT_EXAMPLE_STORAGE_ENGINE=1
-DWITHOUT_MROONGA_STORAGE_ENGINE=1
-DWITHOUT_DAEMON_EXAMPLE=1
-DCMAKE_C_FLAGS="-I${jemalloc:location}/include -I${libaio:location}/include -I${libxml2:location}/include -I${ncurses:location}/include -I${openssl:location}/include -I${pcre:location}/include -I${readline5:location}/include -I${xz-utils:location}/include -I${zlib:location}/include"
-DCMAKE_CXX_FLAGS="-I${jemalloc:location}/include -I${libaio:location}/include -I${libxml2:location}/include -I${ncurses:location}/include -I${openssl:location}/include -I${pcre:location}/include -I${readline5:location}/include -I${xz-utils:location}/include -I${zlib:location}/include"
-DCMAKE_C_FLAGS="-I${bzip2:location}/include -I${jemalloc:location}/include -I${libaio:location}/include -I${libxml2:location}/include -I${ncurses:location}/include -I${openssl:location}/include -I${pcre:location}/include -I${readline5:location}/include -I${xz-utils:location}/include -I${zlib:location}/include"
-DCMAKE_CXX_FLAGS="-I${bzip2:location}/include -I${jemalloc:location}/include -I${libaio:location}/include -I${libxml2:location}/include -I${ncurses:location}/include -I${openssl:location}/include -I${pcre:location}/include -I${readline5:location}/include -I${xz-utils:location}/include -I${zlib:location}/include"
-DCMAKE_INSTALL_RPATH=${:CMAKE_INSTALL_RPATH}
CMAKE_INSTALL_RPATH=${jemalloc:location}/lib:${libaio:location}/lib:${libxml2:location}/lib:${ncurses:location}/lib:${openssl:location}/lib:${pcre:location}/lib:${readline5:location}/lib:${xz-utils:location}/lib:${zlib:location}/lib
CMAKE_INSTALL_RPATH=${bzip2:location}/lib:${jemalloc:location}/lib:${libaio:location}/lib:${libxml2:location}/lib:${ncurses:location}/lib:${openssl:location}/lib:${pcre:location}/lib:${readline5:location}/lib:${xz-utils:location}/lib:${zlib:location}/lib
environment =
CMAKE_PROGRAM_PATH=${cmake:location}/bin
CMAKE_INCLUDE_PATH=${libaio:location}/include:${libaio:location}/include:${libxml2:location}/include:${ncurses:location}/include:${openssl:location}/include:${pcre:location}/include:${readline5:location}/include:${xz-utils:location}/include:${zlib:location}/include
CMAKE_LIBRARY_PATH=${libaio:location}/lib:{libaio:location}/lib:${libxml2:location}/lib:${ncurses:location}/lib:${openssl:location}/lib:${pcre:location}/lib:${readline5:location}/lib:${xz-utils:location}/lib:${zlib:location}/lib
LDFLAGS=-L${jemalloc:location}/lib -Wl,-rpath=${jemalloc:location}/lib -L${libaio:location}/lib -Wl,-rpath=${libaio:location}/lib -L${pcre:location}/lib -L${xz-utils:location}/lib -L${zlib:location}/lib
CMAKE_INCLUDE_PATH=${bzip2:location}/include:${libaio:location}/include:${libaio:location}/include:${libxml2:location}/include:${ncurses:location}/include:${openssl:location}/include:${pcre:location}/include:${readline5:location}/include:${xz-utils:location}/include:${zlib:location}/include
CMAKE_LIBRARY_PATH=${bzip2:location}/lib:${libaio:location}/lib:{libaio:location}/lib:${libxml2:location}/lib:${ncurses:location}/lib:${openssl:location}/lib:${pcre:location}/lib:${readline5:location}/lib:${xz-utils:location}/lib:${zlib:location}/lib
LDFLAGS=-L${bzip2:location}/lib -L${jemalloc:location}/lib -Wl,-rpath=${jemalloc:location}/lib -L${libaio:location}/lib -Wl,-rpath=${libaio:location}/lib -L${pcre:location}/lib -L${xz-utils:location}/lib -L${zlib:location}/lib
PATH=${patch:location}/bin:%(PATH)s
post-install =
mkdir -p ${:location}/include/wsrep &&
......@@ -70,11 +73,15 @@ post-install =
# mroonga - a storage engine for MySQL. It provides fast fulltext search feature to all MySQL users.
# http://mroonga.github.com/
recipe = slapos.recipe.cmmi
url = http://packages.groonga.org/source/mroonga/mroonga-7.00.tar.gz
md5sum = befcbb7de8f5dd27366ea513dc092f3f
pre-configure =
mkdir fake_mariadb_source &&
url = http://packages.groonga.org/source/mroonga/mroonga-7.03.tar.gz
md5sum = 045be7c932e80cedaa756ea94ae24883
pre-configure = set -e
rm -rf fake_mariadb_source
mkdir -p fake_mariadb_source
ln -s ${mariadb:location}/include/mysql/private fake_mariadb_source/sql
cp -ar ${mariadb:location}/include fake_mariadb_source/include
! test -f fake_mariadb_source/include/mysql/mariadb_stmt.h ||
sed -i -E -e '/^enum enum_(cursor|indicator)_type$/,/};/d' fake_mariadb_source/include/mysql/mariadb_stmt.h
configure-options =
--with-mysql-source=fake_mariadb_source
--with-mysql-config=${mariadb:location}/bin/mysql_config
......@@ -83,6 +90,8 @@ configure-options =
patch-options = -p1
patches =
${:_profile_base_location_}/mroonga_boolean.patch#29f85fc98e85d39b31e53b1e1683d288
pre-build =
sed -i -e "s,${mariadb:location}/include,$(pwd)/fake_mariadb_source/include,g" Makefile */Makefile
environment =
PATH=${groonga:location}/bin:${patch:location}/bin:${pkgconfig:location}/bin:%(PATH)s
CPPFLAGS=-I${groonga:location}/include/groonga -I${pcre:location}/include
......
component/mariadb/mariarocks.cfg
View file @ 56ad14fb
[mariadb]
url = https://downloads.mariadb.com/MariaDB/mariadb-10.2.5/source/mariadb-10.2.5.tar.gz
md5sum = 16fb01bb375c8d8a91a9ecf491397d51
pre-configure =
sed -i -e '/(rc == -1 || errno == EINTR)/s/||/\&\&/' libmariadb/plugins/pvio/pvio_socket.c
sed -i -e 's,set(SYSTEM_LIBS $${CMAKE_THREAD_LIBS_INIT}),set(SYSTEM_LIBS $${CMAKE_THREAD_LIBS_INIT} $${LIBRT}),' storage/rocksdb/build_rocksdb.cmake
url = https://downloads.mariadb.com/MariaDB/mariadb-10.2.6/source/mariadb-10.2.6.tar.gz
md5sum = 3d454cdadbd3208e1c9c23e5338a62b0
stable-patches =
configure-options +=
-DPLUGIN_DAEMON_EXAMPLE=NO
-DPLUGIN_EXAMPLE=NO
......
component/matplotlib/buildout.cfg
View file @ 56ad14fb
......@@ -29,7 +29,10 @@ CFLAGS = -I${libpng:location}/include -I${freetype:location}/include
recipe = zc.recipe.egg:custom
egg = matplotlib
environment = matplotlib-env
setup-eggs = ${numpy:egg}
setup-eggs =
cycler
python-dateutil
${numpy:egg}
rpath =
${libpng:location}/lib
${freetype:location}/lib
......
component/mysql-5.5/buildout.cfg deleted 100644 → 0
View file @ fb36acc7
[buildout]
extends =
../autoconf/buildout.cfg
../automake/buildout.cfg
../bison/buildout.cfg
../cmake/buildout.cfg
../groonga/buildout.cfg
../libaio/buildout.cfg
../libtool/buildout.cfg
../ncurses/buildout.cfg
../pkgconfig/buildout.cfg
../zlib/buildout.cfg
parts =
mysql-5.5
[mysql-5.5-sphinx-patch]
# this patch comes from sphinx-2.0.1-beta including changes for
# MySQL-5.5 in
# http://code.google.com/p/sphinxsearch/source/detail?r=2921
recipe = hexagonit.recipe.download
ignore-existing = true
url = ${:_profile_base_location_}/${:filename}
md5sum = 04549822ebfa91b5f84025ff49ef24c2
filename = mysql-5.5-sphinx-2.0.1-beta.diff
download-only = true
[mysql-5.5-no_test-patch]
recipe = hexagonit.recipe.download
ignore-existing = true
url = ${:_profile_base_location_}/${:filename}
md5sum = bb7ee34b9a98da1f913566c63ffbc6f5
filename = mysql_create_system_tables__no_test.patch
download-only = true
[mysql-5.5]
recipe = slapos.recipe.cmmi
version = 5.5.16
url = http://mysql.he.net/Downloads/MySQL-5.5/mysql-${:version}.tar.gz
md5sum = 462ab3752dd666ec06ca32f92673b59e
# compile directory is required to build mysql plugins.
keep-compile-dir = true
patch-options = -p0
patches =
${mysql-5.5-sphinx-patch:location}/${mysql-5.5-sphinx-patch:filename}
${mysql-5.5-no_test-patch:location}/${mysql-5.5-no_test-patch:filename}
configure-command = ${cmake:location}/bin/cmake
# we use embeded yassl instead of openssl to avoid compilation errors on sphinx search engine.
configure-options =
-DCMAKE_INSTALL_PREFIX=${buildout:parts-directory}/${:_buildout_section_name_}
-DBUILD_CONFIG=mysql_release
-DDEFAULT_CHARSET=utf8
-DDEFAULT_COLLATION=utf8_unicode_ci
-DWITH_SSL=bundled
-DWITH_ZLIB=system
-DWITHOUT_EXAMPLE_STORAGE_ENGINE=1
-DWITHOUT_DAEMON_EXAMPLE=1
-DWITH_SPHINX_STORAGE_ENGINE=1
-DCMAKE_C_FLAGS="-I${libaio:location}/include -I${ncurses:location}/include -I${zlib:location}/include"
-DCMAKE_INSTALL_RPATH=${libaio:location}/lib:${ncurses:location}/lib:${zlib:location}/lib
environment =
CMAKE_PROGRAM_PATH=${autoconf:location}/bin:${automake:location}/bin:${cmake:location}/bin:${libtool:location}/bin:${bison:location}/bin:${flex:location}/bin
CMAKE_INCLUDE_PATH=${libaio:location}/include:${ncurses:location}/include:${zlib:location}/include
CMAKE_LIBRARY_PATH=${libaio:location}/lib:${ncurses:location}/lib:${zlib:location}/lib
LDFLAGS=-L${libaio:location}/lib
[groonga-storage-engine-mysql-5.5]
recipe = slapos.recipe.cmmi
url = http://github.com/downloads/mroonga/mroonga/groonga-storage-engine-1.0.0.tar.gz
md5sum = 289b8b7919e790599ea79b6fe9270e04
configure-options =
--with-mysql-source=${mysql-5.5:location}__compile__/mysql-${mysql-5.5:version}
--with-mysql-config=${mysql-5.5:location}/bin/mysql_config
environment =
PATH=${groonga:location}/bin:${pkgconfig:location}/bin:%(PATH)s
CPPFLAGS=-I${groonga:location}/include/groonga
LDFLAGS=-L${groonga:location}/lib
PKG_CONFIG_PATH=${groonga:location}/lib/pkgconfig
component/mysql-5.5/mysql-5.5-sphinx-2.0.1-beta.diff deleted 100644 → 0
View file @ fb36acc7
This source diff could not be displayed because it is too large. You can view the blob instead.
component/mysql-5.5/mysql_create_system_tables__no_test.patch deleted 100644 → 0
View file @ fb36acc7
# 33_scripts__mysql_create_system_tables__no_test.dpatch by <ch@debian.org>
A user with no password prevents a normal user from login under certain
circumstances as it is checked first.
See http://bugs.debian.org/301741
and http://bugs.mysql.com/bug.php?id=6901
--- scripts/mysql_system_tables_data.sql 2008-12-04 22:59:44.000000000 +0100
+++ scripts/mysql_system_tables_data.sql 2008-12-04 23:00:07.000000000 +0100
@@ -26,8 +26,6 @@
-- Fill "db" table with default grants for anyone to
-- access database 'test' and 'test_%' if "db" table didn't exist
CREATE TEMPORARY TABLE tmp_db LIKE db;
-INSERT INTO tmp_db VALUES ('%','test','','Y','Y','Y','Y','Y','Y','N','Y','Y','Y','Y','Y','Y','Y','Y','N','N','Y','Y');
-INSERT INTO tmp_db VALUES ('%','test\_%','','Y','Y','Y','Y','Y','Y','N','Y','Y','Y','Y','Y','Y','Y','Y','N','N','Y','Y');
INSERT INTO db SELECT * FROM tmp_db WHERE @had_db_table=0;
DROP TABLE tmp_db;
@@ -40,8 +38,6 @@
REPLACE INTO tmp_user SELECT @current_hostname,'root','','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','','','','',0,0,0,0,'','' FROM dual WHERE LOWER( @current_hostname) != 'localhost';
REPLACE INTO tmp_user VALUES ('127.0.0.1','root','','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','','','','',0,0,0,0,'','');
REPLACE INTO tmp_user VALUES ('::1','root','','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','','','','',0,0,0,0,'','');
-INSERT INTO tmp_user (host,user) VALUES ('localhost','');
-INSERT INTO tmp_user (host,user) SELECT @current_hostname,'' FROM dual WHERE LOWER(@current_hostname ) != 'localhost';
INSERT INTO user SELECT * FROM tmp_user WHERE @had_user_table=0;
DROP TABLE tmp_user;
component/mysql-python/buildout.cfg deleted 100644 → 0
View file @ fb36acc7
[buildout]
extends =
../mariadb/buildout.cfg
../openssl/buildout.cfg
../zlib/buildout.cfg
parts =
mysql-python
[mysql-python-env]
PATH =${mariadb:location}/bin:%(PATH)s
[mysql-python]
recipe = zc.recipe.egg:custom
egg = MySQL-python
environment = mysql-python-env
library-dirs =
${zlib:location}/lib/
${openssl:location}/lib/
rpath =
${mariadb:location}/lib/
${zlib:location}/lib/
${openssl:location}/lib/
component/nano/buildout.cfg
View file @ 56ad14fb
[buildout]
parts =
nano
extends =
../file/buildout.cfg
../ncurses/buildout.cfg
[nano]
recipe = slapos.recipe.cmmi
version = 2.2.6
url = http://www.nano-editor.org/dist/v2.2/nano-2.2.6.tar.gz
md5sum = 03233ae480689a008eb98feb1b599807
url = http://www.nano-editor.org/dist/v2.8/nano-2.8.4.tar.xz
md5sum = 02ff28870194178595b287fc16fa611b
# The dummy PKG_CONFIG is in the case that both pkg-config and ncursesw
# are installed on the system.
environment=
CFLAGS=-I${ncurses:location}/include
LDFLAGS=-L${ncurses:location}/lib/ -Wl,-rpath=${ncurses:location}/lib/
\ No newline at end of file
PKG_CONFIG=false
NCURSESW_CONFIG=${ncurses:location}/bin/ncursesw6-config
CPPFLAGS=-I${file:location}/include
LDFLAGS=-L${file:location}/lib/ -Wl,-rpath=${file:location}/lib/
location = ${buildout:parts-directory}/${:_buildout_section_name_}
post-install =
cd ${:location} && mkdir etc &&
echo include "${:location}/share/nano/*.nanorc" > etc/nanorc
component/nettle/buildout.cfg
View file @ 56ad14fb
......@@ -6,10 +6,10 @@ extends =
[nettle]
recipe = slapos.recipe.cmmi
url = http://www.lysator.liu.se/~nisse/archive/nettle-2.7.1.tar.gz
md5sum = 003d5147911317931dd453520eb234a5
url = http://ftp.gnu.org/gnu/nettle/nettle-3.3.tar.gz
md5sum = 10f969f78a463704ae73529978148dbe
patches =
${:_profile_base_location_}/nettle-lib-location.patch#41dd0ce2a73487929bdc637b75dd62c9
${:_profile_base_location_}/nettle-lib-location.patch#3c5f5b285ffd5bc30436ee0f4c662084
configure-option =
--disable-static
--disable-assembler
......
component/nettle/nettle-lib-location.patch
View file @ 56ad14fb
--- configure.orig 2013-07-05 15:37:28.000000000 +0200
+++ configure 2013-07-05 15:47:48.000000000 +0200
@@ -4680,52 +4680,6 @@
--- configure.orig 2016-10-01 09:28:38.000000000 +0200
+++ configure 2017-06-22 11:22:13.865295091 +0200
@@ -6779,58 +6779,6 @@
if test "x$ABI" != xstandard ; then
{ $as_echo "$as_me:${as_lineno-$LINENO}: Compiler uses $ABI-bit ABI. To change, set CC." >&5
$as_echo "$as_me: Compiler uses $ABI-bit ABI. To change, set CC." >&6;}
......@@ -43,6 +43,12 @@
- *:freebsd*:64)
- libdir='${exec_prefix}/lib'
- ;;
- *:irix*:32)
- libdir='${exec_prefix}/lib32'
- ;;
- *:irix*:64)
- libdir='${exec_prefix}/lib64'
- ;;
- *)
- { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: Don't know where to install $ABI-bit libraries on this system." >&5
-$as_echo "$as_me: WARNING: Don't know where to install $ABI-bit libraries on this system." >&2;};
......@@ -52,4 +58,4 @@
- fi
fi
# Select assembler code
OPT_NETTLE_SOURCES=""
component/nghttp2/buildout.cfg
View file @ 56ad14fb
......@@ -3,6 +3,8 @@ extends =
../automake/buildout.cfg
../libtool/buildout.cfg
../pkgconfig/buildout.cfg
../python-2.7/buildout.cfg
../python-setuptools/buildout.cfg
parts =
nghttp2
......@@ -18,4 +20,4 @@ pre-configure =
automake
autoconf
environment =
PATH=${autoconf:location}/bin:${automake:location}/bin:${git:location}/bin:${libtool:location}/bin:${m4:location}/bin:%(PATH)s
PATH=${autoconf:location}/bin:${automake:location}/bin:${git:location}/bin:${libtool:location}/bin:${m4:location}/bin:${python2.7:location}/bin:%(PATH)s
component/ninja/buildout.cfg 0 → 100644
View file @ 56ad14fb
[buildout]
parts =
ninja
[ninja]
recipe = slapos.recipe.cmmi
version = v1.7.1
url = https://github.com/ninja-build/ninja/archive/${:version}.tar.gz
configure-command = ./configure.py
configure-options = --bootstrap
make-binary = true
post-install = ls && cp -r ./* ${buildout:parts-directory}/${:_buildout_section_name_}/
component/ocropy/buildout.cfg 0 → 100644
View file @ 56ad14fb
[buildout]
extends =
../git/buildout.cfg
../gzip/buildout.cfg
../scipy/buildout.cfg
../lxml-python/buildout.cfg
../matplotlib/buildout.cfg
../numpy/buildout.cfg
../patch/buildout.cfg
../pillow/buildout.cfg
../numpy/buildout.cfg
parts = ocropy
[ocropy-eng-traineddata]
recipe = hexagonit.recipe.download
filename = en-default.pyrnn.gz
md5sum = cedd140c7d7650e910f0550ad0f04727
download-only = true
url = http://www.tmbdev.net/en-default.pyrnn.gz
[ocropy-env]
OCROPY_MODEL_PATH = ${ocropy-eng-traineddata:location}/${ocropy-eng-traineddata:filename}
HOME = ${ocropy:egg}
[ocropy]
recipe = zc.recipe.egg:custom
egg = ocropy
setup-eggs =
${numpy:egg}
${scipy:egg}
${matplotlib:egg}
${pillow-python:egg}
patches =
${:_profile_base_location_}/ocropy.patch
patch-options = -p0
patch-binary = ${patch:location}/bin/patch
environment = ocropy-env
find-links = https://github.com/tmbdev/ocropy/tarball/4efbddca22bb2f0c639af0694e7a1386f2f097b5/ocropy-1.0.tar.gz
md5sum = 240b8866dd7248816e01af469a328c09
component/ocropy/ocropy.patch 0 → 100644
View file @ 56ad14fb
diff --git ocrolib/__init__.py ocrolib/__init__.py
index 1e0d627..81e85fb 100644
--- ocrolib/__init__.py
+++ ocrolib/__init__.py
@@ -1,7 +1,7 @@
__all__ = [
"binnednn","cairoextras","common","components","dbtables",
"fgen","gmmtree","gtkyield","hocr","lang","native",
- "mlp","multiclass","default","lineest"
+ "mlp","multiclass","default","lineest", "psegutils"
]
################################################################
@@ -9,5 +9,6 @@ __all__ = [
################################################################
import default
+from psegutils import *
from common import *
from default import traceback as trace
diff --git ocrolib/common.py ocrolib/common.py
index 27c0f26..14f088f 100644
--- ocrolib/common.py
+++ ocrolib/common.py
@@ -14,6 +14,7 @@ import unicodedata
import inspect
import glob
import cPickle
+import gzip
from ocrolib.exceptions import (BadClassLabel, BadInput, FileNotFound,
OcropusException)
@@ -428,6 +429,7 @@ def unpickle_find_global(mname,cname):
exec "import "+mname
return getattr(sys.modules[mname],cname)
+
def load_object(fname,zip=0,nofind=0,verbose=0):
"""Loads an object from disk. By default, this handles zipped files
and searches in the usual places for OCRopus. It also handles some
@@ -439,8 +441,7 @@ def load_object(fname,zip=0,nofind=0,verbose=0):
if zip==0 and fname.endswith(".gz"):
zip = 1
if zip>0:
- # with gzip.GzipFile(fname,"rb") as stream:
- with os.popen("gunzip < '%s'"%fname,"rb") as stream:
+ with gzip.GzipFile(fname,"rb") as stream:
unpickler = cPickle.Unpickler(stream)
unpickler.find_global = unpickle_find_global
return unpickler.load()
@@ -618,7 +619,7 @@ def ocropus_find_file(fname, gz=True):
possible_prefixes.append(os.path.normpath(os.path.join(
os.path.dirname(inspect.getfile(inspect.currentframe())),
- os.pardir, os.pardir, os.pardir, os.pardir, "share", "ocropus")))
+ os.pardir, "share", "ocropus")))
possible_prefixes.append("/usr/local/share/ocropus")
diff --git ocrolib/native.py ocrolib/native.py
index b7a207f..240450b 100644
--- ocrolib/native.py
+++ ocrolib/native.py
@@ -44,6 +44,7 @@ class CompileError(Exception):
def compile_and_find(c_string,prefix=".pynative",opt="-g -O4",libs="-lm",
options="-shared -fopenmp -std=c99 -fPIC",verbose=0):
+ prefix = os.path.join(os.path.dirname(__file__), prefix)
if not os.path.exists(prefix):
os.mkdir(prefix)
m = hashlib.md5()
diff --git setup.py setup.py
index 2ec5832..6697b12 100644
--- setup.py
+++ setup.py
@@ -10,7 +10,9 @@ assert sys.version_info[0]==2 and sys.version_info[1]>=7,\
from distutils.core import setup #, Extension, Command
#from distutils.command.install_data import install_data
-if not os.path.exists("models/en-default.pyrnn.gz"):
+models = os.environ.get('OCROPY_MODEL_PATH', '').split(':') or \
+ [c for c in glob.glob("models/*pyrnn.gz")]
+if not models:
print()
print("You should download the default model 'en-default.pyrnn.gz'")
print("and put it into ./models.")
@@ -18,16 +20,23 @@ if not os.path.exists("models/en-default.pyrnn.gz"):
print("Check https://github.com/tmbdev/ocropy for the location")
print("of model files.")
print()
+ sys.exit(1)
-models = [c for c in glob.glob("models/*pyrnn.gz")]
scripts = [c for c in glob.glob("ocropus-*") if "." not in c and "~" not in c]
+# compile pynative files now and include them in the build
+sys.path.insert(0, os.curdir)
+import ocrolib.nutils
+pynative_files = [os.path.join(*c.split(os.path.sep)[1:]) \
+ for c in glob.glob('ocrolib/.pynative/*')]
+
setup(
name = 'ocropy',
version = 'v1.0',
author = "Thomas Breuel",
description = "The OCRopy RNN-based Text Line Recognizer",
packages = ["ocrolib"],
- data_files= [('share/ocropus', models)],
+ package_data = {'ocrolib': pynative_files},
+ data_files= [('share/ocropus', models), ("", ["LICENSE"])],
scripts = scripts,
)
component/p11-kit/buildout.cfg
View file @ 56ad14fb
......@@ -9,8 +9,9 @@ extends =
[p11-kit]
recipe = slapos.recipe.cmmi
url = http://p11-glue.freedesktop.org/releases/p11-kit-0.23.1.tar.gz
md5sum = 96f073270c489c9a594e1c9413f42db8
url = https://github.com/p11-glue/p11-kit/releases/download/${:version}/p11-kit-${:version}.tar.gz
version = 0.23.7
md5sum = ebbefd123210594231adb4bde21b8560
configure-options =
--disable-static
--disable-doc-html
......
component/perl-CGI/buildout.cfg 0 → 100644
View file @ 56ad14fb
[buildout]
extends =
../perl/buildout.cfg
parts =
perl-CGI
[perl-CGI]
recipe = slapos.recipe.cmmi
url = http://www.cpan.org/modules/by-module/CGI/CGI-4.36.tar.gz
md5sum = 48566d81f430a7f755bb4d8f426ce35d
configure-command =
${perl:location}/bin/perl Makefile.PL
component/perl-DBD-MySQL/DBD-mysql-4.042.mariadb.patch 0 → 100644
View file @ 56ad14fb
diff -u DBD-mysql-4.042/dbdimp.c DBD-mysql-4.042/dbdimp.c
--- DBD-mysql-4.042/dbdimp.c 2017-02-28 14:36:40.000000000 +0100
+++ DBD-mysql-4.042/dbdimp.c 2017-06-08 12:43:39.704228470 +0200
@@ -2139,6 +2139,7 @@
if (result)
{
+ my_bool reconnect= 0;
#if MYSQL_VERSION_ID >=SERVER_PREPARE_VERSION
/* connection succeeded. */
/* imp_dbh == NULL when mysql_dr_connect() is called from mysql.xs
@@ -2155,7 +2156,7 @@
we turn off Mysql's auto reconnect and handle re-connecting ourselves
so that we can keep track of when this happens.
*/
- result->reconnect=0;
+ mysql_options(result, MYSQL_OPT_RECONNECT, &reconnect);
}
else {
/*
diff -u DBD-mysql-4.042/dbdimp.h DBD-mysql-4.042/dbdimp.h
--- DBD-mysql-4.042/dbdimp.h 2017-02-28 14:36:40.000000000 +0100
+++ DBD-mysql-4.042/dbdimp.h 2017-06-08 12:44:21.233948264 +0200
@@ -67,6 +67,12 @@
#define SSL_LAST_VERIFY_VERSION 50799
#define MYSQL_VERSION_5_0 50001
/* This is to avoid the ugly #ifdef mess in dbdimp.c */
+
+/* MariaDB Connector/C has MARIADB_VERSION_ID instead of MYSQL_VERSION_ID */
+#if !defined(MYSQL_VERSION_ID) && defined(MARIADB_VERSION_ID)
+#define MYSQL_VERSION_ID MARIADB_VERSION_ID
+#endif
+
#if MYSQL_VERSION_ID < SQL_STATE_VERSION
#define mysql_sqlstate(svsock) (NULL)
#endif
component/perl-DBD-MySQL/buildout.cfg
View file @ 56ad14fb
......@@ -3,6 +3,7 @@ extends =
../mysql-tritonn-5.0/buildout.cfg
../mariadb/buildout.cfg
../patch/buildout.cfg
../perl-Devel-CheckLib/buildout.cfg
parts =
perl-DBD-MySQL
......@@ -11,24 +12,24 @@ parts =
recipe = slapos.recipe.cmmi
depends =
${perl-DBI:location}
url = http://www.cpan.org/modules/by-module/DBD/DBD-mysql-4.027.tar.gz
md5sum = 11fa6449715ba9d3fa5f4ee5ae688d6f
${perl-Devel-CheckLib:location}
url = http://www.cpan.org/modules/by-module/DBD/DBD-mysql-4.042.tar.gz
md5sum = a144bd950b55af68835d44bc4ea6e5aa
patches =
${:_profile_base_location_}/DBD-mysql-4.027.rpathsupport.patch#a932982b7725e6621cfce3a3d7917e03
${:_profile_base_location_}/DBD-mysql-4.042.mariadb.patch#5864d36d19c4a05034b3a4873f7c659a
patch-options = -p1
configure-command =
${perl:location}/bin/perl Makefile.PL --libs="-L${zlib:location}/lib -L${openssl:location}/lib $(mysql_config --libs)"
[perl-DBD-MySQL]
<= perl-DBD-MySQL-common
configure-command =
${perl:location}/bin/perl Makefile.PL --mysql_config=${mysql-tritonn-5.0:location}/bin/mysql_config
environment =
OTHERLDFLAGS=-Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${mysql-tritonn-5.0:location}/lib/mysql -Wl,-rpath=${openssl:location}/lib
PATH=${patch:location}/bin:%(PATH)s
PATH=${mysql-tritonn-5.0:location}/bin:${patch:location}/bin:%(PATH)s
[perl-DBD-mariadb]
<= perl-DBD-MySQL-common
configure-command =
${perl:location}/bin/perl Makefile.PL --mysql_config=${mariadb:location}/bin/mysql_config
environment =
OTHERLDFLAGS=-L${zlib:location}/lib -L${openssl:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${mariadb:location}/lib -Wl,-rpath=${openssl:location}/lib
PATH=${patch:location}/bin:%(PATH)s
OTHERLDFLAGS=-Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${mariadb:location}/lib -Wl,-rpath=${openssl:location}/lib
PATH=${mariadb:location}/bin:${patch:location}/bin:%(PATH)s
component/perl-Devel-CheckLib/buildout.cfg 0 → 100644
View file @ 56ad14fb
[buildout]
extends =
../perl/buildout.cfg
[perl-Devel-CheckLib]
recipe = slapos.recipe.cmmi
url = http://www.cpan.org/modules/by-module/Devel/Devel-CheckLib-1.11.tar.gz
md5sum = c20873798591176283f51af4ad8088cd
configure-command =
${perl:location}/bin/perl Makefile.PL
component/perl-Encode-HanExtra/buildout.cfg
View file @ 56ad14fb
......@@ -10,7 +10,7 @@ recipe = slapos.recipe.cmmi
url = http://www.cpan.org/modules/by-module/Encode/Encode-HanExtra-0.23.tar.gz
md5sum = e1d3bc32c1c8ee304235a06fbcd5d5a4
configure-command =
${perl:location}/bin/perl Makefile.PL
${perl:location}/bin/perl -I. Makefile.PL
patch-options = -p1
patches =
${:_profile_base_location_}/Encode-HanExtra-0.23_use_shebang.patch#4ac804013a68a3e6b7c3bc4115d25a5e
......
component/perl-URI/buildout.cfg
View file @ 56ad14fb
......@@ -6,7 +6,7 @@ parts =
[perl-URI]
recipe = slapos.recipe.cmmi
url = http://www.cpan.org/modules/by-module/URI/URI-1.60.tar.gz
md5sum = 70f739be8ce28b8baba7c5920ffee4dc
url = http://www.cpan.org/modules/by-module/URI/URI-1.71.tar.gz
md5sum = 247c3da29a794f72730e01aa5a715daf
configure-command =
${perl:location}/bin/perl Makefile.PL
component/perl-XML-NamespaceSupport/buildout.cfg
View file @ 56ad14fb
......@@ -9,4 +9,4 @@ recipe = slapos.recipe.cmmi
url = http://www.cpan.org/modules/by-module/XML/XML-NamespaceSupport-1.11.tar.gz
md5sum = 222cca76161cd956d724286d36b607da
configure-command =
${perl:location}/bin/perl Makefile.PL
${perl:location}/bin/perl -I. Makefile.PL
component/perl/buildout.cfg
View file @ 56ad14fb
......@@ -7,9 +7,9 @@ parts =
[perl]
recipe = slapos.recipe.cmmi
version = 5.20.2
version = 5.26.0
url = http://www.cpan.org/src/5.0/perl-${:version}.tar.bz2
md5sum = 21062666f1c627aeb6dbff3c6952738b
md5sum = 8d34cb5a4eccd66e7a6a80e62b7b4aec
siteprefix = ${buildout:parts-directory}/site_${:_buildout_section_name_}
patch-options = -p1
patches =
......@@ -25,4 +25,4 @@ configure-command =
-Dusethreads
environment =
PATH=${patch:location}/bin:%(PATH)s
post-make-hook = ${:_profile_base_location_}/perl-create-libs-symlink.py#28d12de5c9c7985e65233d56817e3bec:post_make_hook
post-make-hook = ${:_profile_base_location_}/perl-create-libs-symlink.py#943453b7d3ff8d49ed12d44a7f7076ee:post_make_hook
component/perl/perl-create-libs-symlink.py
View file @ 56ad14fb
from zc.buildout import UserError
import fnmatch
import os
import pprint
def post_make_hook(options, buildout):
location = options['location']
matches = []
for root, dirnames, filenames in os.walk(location):
for filename in fnmatch.filter(filenames, 'libperl.a'):
matches.append(os.path.join(root, filename))
nr_matches = len(matches)
if nr_matches == 0:
print "ERROR - no libperl.* found!"
exit
elif nr_matches > 1:
print "WARNING - several libperl.a found, taking only the first one:", "\n".join(matches)
matches = [os.path.join(root, filename)
for root, dirnames, filenames in os.walk(location)
for filename in fnmatch.filter(filenames, 'libperl.a')]
if not matches:
raise UserError("ERROR - no libperl.* found!")
if len(matches) > 1:
print("WARNING - several libperl.a found, taking only the first one:",
"\n".join(matches))
# matches[0] is a prefix of "location"
# For the symlink, we want the relative path.
rel_link = os.path.relpath(os.path.dirname(matches[0]), location)
......@@ -24,4 +20,4 @@ def post_make_hook(options, buildout):
if os.path.islink(simlink_location):
os.unlink(simlink_location)
os.symlink(rel_link, simlink_location)
print "Created symlink \"libs-c\" to", rel_link
print("Created symlink \"libs-c\" to", rel_link)
component/powerdns/buildout.cfg
View file @ 56ad14fb
......@@ -23,12 +23,13 @@ url = http://downloads.powerdns.com/releases/pdns-3.3.1.tar.gz
md5sum = 074e2ff211fd12ecad25b5c1cc190dd4
configure-options =
--prefix=${buildout:parts-directory}/${:_buildout_section_name_}
--with-boost=${boost-lib:location}
--with-modules="geo"
--with-dynmodules=""
--without-lua
environment =
PATH=${make:location}/bin:${libtool:location}/bin:${pkgconfig:location}/bin:${bison:location}/bin:${flex:location}/bin:${git:location}/bin:${ragel:location}/bin:%(PATH)s
LDFLAGS = -L${boost-lib:location}/lib -Wl,-rpath=${boost-lib:location}/lib -L${zlib:location}/lib -Wl,-rpath -Wl,${zlib:location}/lib -lz
PATH=${gcc:location}/bin:${make:location}/bin:${libtool:location}/bin:${pkgconfig:location}/bin:${bison:location}/bin:${flex:location}/bin:${git:location}/bin:${ragel:location}/bin:%(PATH)s
LDFLAGS=-L${boost-lib:location}/lib -Wl,-rpath=${boost-lib:location}/lib -L${zlib:location}/lib -Wl,-rpath -Wl,${zlib:location}/lib -lz
CPPFLAGS=-I${boost-lib:location}/include
make-target =
install
component/python-2.4/buildout.cfg deleted 100644 → 0
View file @ fb36acc7
[buildout]
extends =
../bzip2/buildout.cfg
../gdbm/buildout.cfg
../gettext/buildout.cfg
../ncurses/buildout.cfg
../openssl/buildout.cfg
../readline/buildout.cfg
../sqlite3/buildout.cfg
../zlib/buildout.cfg
parts =
pythonbin2.4
[python2.4-dbm-patch]
recipe = hexagonit.recipe.download
ignore-existing = true
# original patch from http://bugs.gentoo.org/attachment.cgi?id=109117
url = ${:_profile_base_location_}/${:filename}
md5sum = 8611020af1463b42f253ac73a91b09a1
download-only = true
filename = python2.4-dbm.patch
[python2.4-no_system_inc_dirs-patch]
recipe = hexagonit.recipe.download
ignore-existing = true
url = ${:_profile_base_location_}/${:filename}
md5sum = 65c1193ac42c15109be0e6f9b7a671b8
download-only = true
filename = python2.4-no_system_inc_dirs.patch
[python2.4-CPPFLAGS-patch]
recipe = hexagonit.recipe.download
ignore-existing = true
url = ${:_profile_base_location_}/${:filename}
md5sum = 912b6d4b107e3f382995a4d3afcd3eca
download-only = true
filename = python2.4-backport-CPPFLAGS-setup-from-python2.6.patch
[python2.4]
recipe = slapos.recipe.cmmi
# This is actually the default setting for prefix, but we can't use it in
# other settings in this part if we don't set it explicitly here.
prefix = ${buildout:parts-directory}/${:_buildout_section_name_}
version = 2.4
package_version = ${:version}.6
executable = ${:prefix}/bin/python${:version}
python_version_major = 2.4
python_version_minor = 6
python_version_full = ${:python_version_major}.${:python_version_minor}
url =
http://python.org/ftp/python/${:python_version_full}/Python-${:python_version_full}.tgz
patches =
${python2.4-dbm-patch:location}/${python2.4-dbm-patch:filename}
${python2.4-no_system_inc_dirs-patch:location}/${python2.4-no_system_inc_dirs-patch:filename}
${python2.4-CPPFLAGS-patch:location}/${python2.4-CPPFLAGS-patch:filename}
configure-options =
--enable-unicode=ucs4
--with-threads
environment =
CPPFLAGS=-I${zlib:location}/include -I${readline:location}/include -I${ncurses:location}/include/ -I${ncurses:location}/include/ncursesw/ -I${bzip2:location}/include -I${gdbm:location}/include -I${openssl:location}/include -I${sqlite3:location}/include -I${gettext:location}/include
LDFLAGS=-L${zlib:location}/lib -L${readline:location}/lib -L${ncurses:location}/lib -L${bzip2:location}/lib -L${gdbm:location}/lib -L${openssl:location}/lib -L${sqlite3:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${readline:location}/lib -Wl,-rpath=${ncurses:location}/lib -Wl,-rpath=${bzip2:location}/lib -Wl,-rpath=${gdbm:location}/lib -Wl,-rpath=${openssl:location}/lib -Wl,-rpath=${sqlite3:location}/lib -L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib
[pythonbin2.4]
# XXX/Note: This is hackish way to have fully featured python interpreter
recipe = zc.recipe.egg:scripts
python = python2.4
eggs = ${eggs:eggs}
invokepython>=0.4
extra-paths =
${zope-2.8:location}/lib/python
scripts =
invokepython=python${python2.4:python_version_major}
ipython=ipython${python2.4:python_version_major}
[bootstrap2.4]
recipe = zc.recipe.egg
eggs = zc.buildout
suffix =
scripts =
buildout=bootstrap2.4
arguments = sys.argv[1:] + ["bootstrap"]
suffix = 2.4
python = python2.4
component/python-2.4/python2.4-backport-CPPFLAGS-setup-from-python2.6.patch deleted 100644 → 0
View file @ fb36acc7
From: Arnaud Fontaine <arnaud.fontaine@nexedi.com>
Date: Mon, 7 Mar 2011 13:02:05 +0900
Subject: [PATCH] Pass CPPFLAGS when building modules in Python 2.4
In contrary to Python 2.6, when building Python 2.4 modules, CPPFLAGS
specified to configure script is not passed at all. This patch backports
the fix from Python 2.6.
--- Makefile.pre.in 2006-10-09 02:41:25.000000000 +0900
+++ Makefile.pre.in 2011-03-07 14:58:34.368000777 +0900
@@ -56,7 +56,10 @@
OPT= @OPT@
BASECFLAGS= @BASECFLAGS@
CFLAGS= $(BASECFLAGS) $(OPT)
-CPPFLAGS= -I. -I$(srcdir)/Include
+# Both CPPFLAGS and LDFLAGS need to contain the shell's value for setup.py to
+# be able to build extension modules using the directories specified in the
+# environment variables
+CPPFLAGS= -I. -I$(srcdir)/Include @CPPFLAGS@
LDFLAGS= @LDFLAGS@
LDLAST= @LDLAST@
SGI_ABI= @SGI_ABI@
--- setup.py 2006-10-09 02:41:25.000000000 +0900
+++ setup.py 2011-03-07 14:53:36.208000779 +0900
@@ -3,7 +3,7 @@
__version__ = "$Revision: 52231 $"
-import sys, os, getopt, imp, re
+import sys, os, getopt, imp, re, optparse
from distutils import log
from distutils import sysconfig
@@ -243,6 +243,39 @@
add_dir_to_list(self.compiler.library_dirs, '/usr/local/lib')
add_dir_to_list(self.compiler.include_dirs, '/usr/local/include')
+ # Add paths specified in the environment variables LDFLAGS and
+ # CPPFLAGS for header and library files.
+ # We must get the values from the Makefile and not the environment
+ # directly since an inconsistently reproducible issue comes up where
+ # the environment variable is not set even though the value were passed
+ # into configure and stored in the Makefile (issue found on OS X 10.3).
+ for env_var, arg_name, dir_list in (
+ ('LDFLAGS', '-L', self.compiler.library_dirs),
+ ('CPPFLAGS', '-I', self.compiler.include_dirs)):
+ env_val = sysconfig.get_config_var(env_var)
+ if env_val:
+ # To prevent optparse from raising an exception about any
+ # options in env_val that is doesn't know about we strip out
+ # all double dashes and any dashes followed by a character
+ # that is not for the option we are dealing with.
+ #
+ # Please note that order of the regex is important! We must
+ # strip out double-dashes first so that we don't end up with
+ # substituting "--Long" to "-Long" and thus lead to "ong" being
+ # used for a library directory.
+ env_val = re.sub(r'(^|\s+)-(-|(?!%s))' % arg_name[1],
+ ' ', env_val)
+ parser = optparse.OptionParser()
+ # Make sure that allowing args interspersed with options is
+ # allowed
+ parser.allow_interspersed_args = True
+ parser.error = lambda msg: None
+ parser.add_option(arg_name, dest="dirs", action="append")
+ options = parser.parse_args(env_val.split())[0]
+ if options.dirs:
+ for directory in reversed(options.dirs):
+ add_dir_to_list(dir_list, directory)
+
# Add paths to popular package managers on OS X/darwin
if sys.platform == "darwin":
# Fink installs into /sw by default
--- Lib/optparse.py 2006-05-29 03:15:43.000000000 +0900
+++ Lib/optparse.py 2011-03-07 21:20:17.192000789 +0900
@@ -69,7 +69,13 @@
import sys, os
import types
import textwrap
-from gettext import gettext as _
+
+try:
+ from gettext import gettext
+except ImportError:
+ def gettext(message):
+ return message
+_ = gettext
def _repr(self):
return "<%s at 0x%x: %s>" % (self.__class__.__name__, id(self), self)
component/python-2.4/python2.4-no_system_inc_dirs.patch deleted 100644 → 0
View file @ fb36acc7
From: Arnaud Fontaine <arnaud.fontaine@nexedi.com>
Date: Mon, 7 Mar 2011 14:10:31 +0900
Subject: [PATCH] Do not use system include directory
Backport python-2.6.6-no_system_inc_dirs.patch to Python 2.4.
--- setup.py 2006-10-09 02:41:25.000000000 +0900
+++ setup.py 2011-03-07 15:15:00.724000778 +0900
@@ -270,7 +270,7 @@
'/lib64', '/usr/lib64',
'/lib', '/usr/lib',
]
- inc_dirs = self.compiler.include_dirs + ['/usr/include']
+ inc_dirs = self.compiler.include_dirs
exts = []
platform = self.get_platform()
@@ -528,31 +528,7 @@
# construct a list of paths to look for the header file in on
# top of the normal inc_dirs.
- db_inc_paths = [
- '/usr/include/db4',
- '/usr/local/include/db4',
- '/opt/sfw/include/db4',
- '/sw/include/db4',
- '/usr/include/db3',
- '/usr/local/include/db3',
- '/opt/sfw/include/db3',
- '/sw/include/db3',
- ]
- # 4.x minor number specific paths
- for x in (0,1,2,3,4):
- db_inc_paths.append('/usr/include/db4%d' % x)
- db_inc_paths.append('/usr/include/db4.%d' % x)
- db_inc_paths.append('/usr/local/BerkeleyDB.4.%d/include' % x)
- db_inc_paths.append('/usr/local/include/db4%d' % x)
- db_inc_paths.append('/pkg/db-4.%d/include' % x)
- db_inc_paths.append('/opt/db-4.%d/include' % x)
- # 3.x minor number specific paths
- for x in (2,3):
- db_inc_paths.append('/usr/include/db3%d' % x)
- db_inc_paths.append('/usr/local/BerkeleyDB.3.%d/include' % x)
- db_inc_paths.append('/usr/local/include/db3%d' % x)
- db_inc_paths.append('/pkg/db-3.%d/include' % x)
- db_inc_paths.append('/opt/db-3.%d/include' % x)
+ db_inc_paths = []
# Add some common subdirectories for Sleepycat DB to the list,
# based on the standard include directories. This way DB3/4 gets
component/python-2.6/python-2.6.6-no_system_inc_dirs.patch deleted 100644 → 0
View file @ fb36acc7
--- Python-2.6.6/setup.py~ 2010-12-11 12:05:13.000000000 +0100
+++ Python-2.6.6/setup.py 2011-03-05 13:33:04.556662701 +0100
@@ -411,7 +411,7 @@
'/lib64', '/usr/lib64',
'/lib', '/usr/lib',
]
- inc_dirs = self.compiler.include_dirs + ['/usr/include']
+ inc_dirs = self.compiler.include_dirs
exts = []
missing = []
@@ -818,15 +818,6 @@
# construct a list of paths to look for the header file in on
# top of the normal inc_dirs.
db_inc_paths = [
- '/usr/include/db4',
- '/usr/local/include/db4',
- '/opt/sfw/include/db4',
- '/usr/include/db3',
- '/usr/local/include/db3',
- '/opt/sfw/include/db3',
- # Fink defaults (http://fink.sourceforge.net/)
- '/sw/include/db4',
- '/sw/include/db3',
]
# 4.x minor number specific paths
for x in gen_db_minor_ver_nums(4):
component/python-2.7/buildout.cfg
View file @ 56ad14fb
......@@ -56,9 +56,10 @@ configure-options =
make-binary =
make-targets = make profile-opt && make install
extra-ldflags =
# the entry "-Wl,-rpath=${file:location}/lib" below is needed by python-magic,
# which would otherwise load the system libmagic.so with ctypes
environment =
PATH=${patch:location}/bin:${xz-utils:location}/bin:%(PATH)s
CPPFLAGS=-I${zlib:location}/include -I${readline:location}/include -I${libexpat:location}/include -I${libffi:location}/include -I${ncurses:location}/include -I${ncurses:location}/include/ncursesw -I${bzip2:location}/include -I${gdbm:location}/include -I${openssl:location}/include -I${sqlite3:location}/include -I${gettext:location}/include
LDFLAGS=-L${zlib:location}/lib -L${readline:location}/lib -L${libexpat:location}/lib -L${libffi:location}/lib -L${ncurses:location}/lib -L${bzip2:location}/lib -L${gdbm:location}/lib -L${openssl:location}/lib -L${sqlite3:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${readline:location}/lib -Wl,-rpath=${libexpat:location}/lib -Wl,-rpath=${libffi:location}/lib -Wl,-rpath=${ncurses:location}/lib -Wl,-rpath=${bzip2:location}/lib -Wl,-rpath=${gdbm:location}/lib -Wl,-rpath=${openssl:location}/lib -Wl,-rpath=${sqlite3:location}/lib -L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib -Wl,-rpath=${file:location}/lib
LDFLAGS=-L${zlib:location}/lib -L${readline:location}/lib -L${libexpat:location}/lib -L${libffi:location}/lib -L${ncurses:location}/lib -L${bzip2:location}/lib -L${gdbm:location}/lib -L${openssl:location}/lib -L${sqlite3:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${readline:location}/lib -Wl,-rpath=${libexpat:location}/lib -Wl,-rpath=${libffi:location}/lib -Wl,-rpath=${ncurses:location}/lib -Wl,-rpath=${bzip2:location}/lib -Wl,-rpath=${gdbm:location}/lib -Wl,-rpath=${openssl:location}/lib -Wl,-rpath=${sqlite3:location}/lib -L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib -Wl,-rpath=${file:location}/lib ${:extra-ldflags}
component/python-mysqlclient/buildout.cfg
View file @ 56ad14fb
......@@ -2,6 +2,7 @@
extends =
../mariadb/buildout.cfg
../openssl/buildout.cfg
../patch/buildout.cfg
../zlib/buildout.cfg
parts =
......@@ -13,6 +14,10 @@ PATH =${mariadb:location}/bin:%(PATH)s
[python-mysqlclient]
recipe = zc.recipe.egg:custom
egg = mysqlclient
patches =
https://github.com/PyMySQL/mysqlclient-python/commit/1693848c9f6ca863868d94d63499830f7f4f3a1f.diff#a493a91f9263243eb331fcab9901b8b0
patch-options = -p1
patch-binary = ${patch:location}/bin/patch
environment = python-mysqlclient-env
library-dirs =
${zlib:location}/lib/
......
component/python-slip/buildout.cfg
View file @ 56ad14fb
......@@ -5,7 +5,7 @@ parts =
[python-slip]
recipe = slapos.recipe.cmmi
url = https://fedorahosted.org/released/python-slip/python-slip-0.6.1.tar.bz2
url = https://github.com/nphilipp/python-slip/releases/download/python-slip-0.6.1/python-slip-0.6.1.tar.bz2
md5sum = a6d8ee96245fc21785d4c1c062c85f2f
configure-command = true
make-targets =
......
component/python-2.6/buildout.cfg → component/python3/buildout.cfg
View file @ 56ad14fb
[buildout]
# XXX: Extends shall not jump out of software
extends =
../bzip2/buildout.cfg
../file/buildout.cfg
../gdbm/buildout.cfg
../gettext/buildout.cfg
../libexpat/buildout.cfg
../libffi/buildout.cfg
../ncurses/buildout.cfg
../openssl/buildout.cfg
../patch/buildout.cfg
../readline/buildout.cfg
../sqlite3/buildout.cfg
../xz-utils/buildout.cfg
../zlib/buildout.cfg
../file/buildout.cfg
parts =
python2.6
python3.5
[python-2.6.6-no_system_inc_dirs.patch]
recipe = hexagonit.recipe.download
ignore-existing = true
url = ${:_profile_base_location_}/${:filename}
path = ${:filename}
md5sum = ed7f8e11a97e45e5e09649c8b242e917
download-only = true
filename = python-2.6.6-no_system_inc_dirs.patch
[python2.6]
[python3-common]
recipe = slapos.recipe.cmmi
package_version_suffix =
# This is actually the default setting for prefix, but we can't use it in
# other settings in this part if we don't set it explicitly here.
prefix = ${buildout:parts-directory}/${:_buildout_section_name_}
version = 2.6
package_version = ${:version}.8
package_version_suffix =
executable = ${:prefix}/bin/python${:version}
executable = ${:prefix}/bin/${:_buildout_section_name_}
url =
http://python.org/ftp/python/${:package_version}/Python-${:package_version}${:package_version_suffix}.tar.bz2
md5sum = c6e0420a21d8b23dee8b0195c9b9a125
patch-options = -p1
patches =
${python-2.6.6-no_system_inc_dirs.patch:location}/${python-2.6.6-no_system_inc_dirs.patch:filename}
${:_profile_base_location_}/default_encoding.patch#4ad9664e622d5556b4c32b1d9cb587ff
url =
http://www.python.org/ftp/python/${:package_version}/Python-${:package_version}${:package_version_suffix}.tar.xz
configure-options =
--enable-ipv6
--enable-unicode=ucs4
--without-ensurepip
--with-system-expat
--with-system-ffi
--with-threads
# Profiled build:
--enable-optimizations
pre-install = mkdir profile-opt
# the entry "-Wl,-rpath=${file:location}/lib" below is needed by python-magic,
# which would otherwise load the system libmagic.so with ctypes
environment =
CPPFLAGS=-I${zlib:location}/include -I${readline:location}/include -I${ncurses:location}/include/ -I${ncurses:location}/include/ncursesw/ -I${bzip2:location}/include -I${gdbm:location}/include -I${openssl:location}/include -I${sqlite3:location}/include -I${gettext:location}/include
LDFLAGS=-L${zlib:location}/lib -L${readline:location}/lib -L${ncurses:location}/lib -L${bzip2:location}/lib -L${gdbm:location}/lib -L${openssl:location}/lib -L${sqlite3:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${readline:location}/lib -Wl,-rpath=${ncurses:location}/lib -Wl,-rpath=${bzip2:location}/lib -Wl,-rpath=${gdbm:location}/lib -Wl,-rpath=${openssl:location}/lib -Wl,-rpath=${sqlite3:location}/lib -L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib -Wl,-rpath=${file:location}/lib
PATH=${patch:location}/bin:${xz-utils:location}/bin:%(PATH)s
CPPFLAGS=-I${zlib:location}/include -I${xz-utils:location}/include -I${readline:location}/include -I${libexpat:location}/include -I${libffi:location}/include -I${ncurses:location}/include -I${ncurses:location}/include/ncursesw -I${bzip2:location}/include -I${gdbm:location}/include -I${openssl:location}/include -I${sqlite3:location}/include -I${gettext:location}/include
LDFLAGS=-L${zlib:location}/lib -L${xz-utils:location}/lib -L${readline:location}/lib -L${libexpat:location}/lib -L${libffi:location}/lib -L${ncurses:location}/lib -L${bzip2:location}/lib -L${gdbm:location}/lib -L${openssl:location}/lib -L${sqlite3:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${xz-utils:location}/lib -Wl,-rpath=${readline:location}/lib -Wl,-rpath=${libexpat:location}/lib -Wl,-rpath=${libffi:location}/lib -Wl,-rpath=${ncurses:location}/lib -Wl,-rpath=${bzip2:location}/lib -Wl,-rpath=${gdbm:location}/lib -Wl,-rpath=${openssl:location}/lib -Wl,-rpath=${sqlite3:location}/lib -L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib -Wl,-rpath=${file:location}/lib
[bootstrap2.6]
recipe = zc.recipe.egg
eggs = zc.buildout
suffix =
scripts =
buildout=bootstrap2.6
arguments = sys.argv[1:] + ["bootstrap"]
python = python2.6
[python3.5]
<= python3-common
package_version = 3.5.3
md5sum = 57d1f8bfbabf4f2500273fb0706e6f21
component/python3/default_encoding.patch 0 → 100644
View file @ 56ad14fb
--- Python-3.5.3/Programs/python.c.orig 2017-01-17 08:57:53.000000000 +0100
+++ Python-3.5.3/Programs/python.c 2017-06-08 19:30:42.927077437 +0200
@@ -24,6 +24,9 @@
int i, res;
char *oldloc;
+ if (!getenv("LC_CTYPE") && !getenv("LC_ALL"))
+ setenv("LANG", "C.UTF-8", 0);
+
argv_copy = (wchar_t **)PyMem_RawMalloc(sizeof(wchar_t*) * (argc+1));
argv_copy2 = (wchar_t **)PyMem_RawMalloc(sizeof(wchar_t*) * (argc+1));
if (!argv_copy || !argv_copy2) {
component/qemu-kvm/buildout.cfg
View file @ 56ad14fb
......@@ -37,7 +37,7 @@ configure-options =
--enable-vnc-png
--disable-vnc-jpeg
--extra-cflags="-I${gnutls:location}/include -I${libuuid:location}/include -I${ncurses:location}/include -I${zlib:location}/include -I${libpng:location}/include -I${libaio:location}/include -I${attr:location}/include -I${libcap:location}/include"
--extra-ldflags="-Wl,-rpath -Wl,${glib:location}/lib -L${glib:location}/lib -Wl,-rpath -Wl,${gnutls:location}/lib -L${gnutls:location}/lib -Wl,-rpath -Wl,${gpg-error:location}/lib -L${gpg-error:location}/lib -L${gettext:location}/lib -Wl,-rpath -Wl,${gettext:location}/lib -Wl,-rpath -Wl,${libpng:location}/lib -L${libpng:location}/lib -L${libuuid:location}/lib -Wl,-rpath -Wl,${libuuid:location}/lib -L${libaio:location}/lib -Wl,-rpath=${libaio:location}/lib -L${ncurses:location}/lib -Wl,-rpath=${ncurses:location}/lib -L${zlib:location}/lib -Wl,-rpath -Wl,${zlib:location}/lib -lpng -lz -lgnutls -L${attr:location}/lib -Wl,-rpath=${attr:location}/lib -L${libcap:location}/lib -Wl,-rpath=${libcap:location}/lib"
--extra-ldflags="-Wl,-rpath -Wl,${glib:location}/lib -L${glib:location}/lib -Wl,-rpath -Wl,${gnutls:location}/lib -L${gnutls:location}/lib -Wl,-rpath -Wl,${libgpg-error:location}/lib -L${libgpg-error:location}/lib -L${gettext:location}/lib -Wl,-rpath -Wl,${gettext:location}/lib -Wl,-rpath -Wl,${libpng:location}/lib -L${libpng:location}/lib -L${libuuid:location}/lib -Wl,-rpath -Wl,${libuuid:location}/lib -L${libaio:location}/lib -Wl,-rpath=${libaio:location}/lib -L${ncurses:location}/lib -Wl,-rpath=${ncurses:location}/lib -L${zlib:location}/lib -Wl,-rpath -Wl,${zlib:location}/lib -lpng -lz -lgnutls -L${attr:location}/lib -Wl,-rpath=${attr:location}/lib -L${libcap:location}/lib -Wl,-rpath=${libcap:location}/lib"
--disable-werror
environment =
PATH=${pkgconfig:location}/bin:%(PATH)s
......
component/screen/buildout.cfg
View file @ 56ad14fb
[buildout]
extends =
../patch/buildout.cfg
parts =
screen
extends =
../ncurses/buildout.cfg
../texinfo/buildout.cfg
[screen]
recipe = slapos.recipe.cmmi
version = 4.0.3
url = http://ftp.gnu.org/gnu/screen/screen-4.0.3.tar.gz
md5sum = 8506fd205028a96c741e4037de6e3c42
# Compatibility with CentOS
patch-options = -p1 -b
patches =
${:_profile_base_location_}/screen-4.0.3-stropts.patch#5677917353067ffbcb08a23a59605aa4
url = http://ftp.gnu.org/gnu/screen/screen-${:version}.tar.gz
version = 4.5.1
md5sum = a8c5da2f42f8a18fa4dada2419d1549b
environment =
CFLAGS=-I${ncurses:location}/include
LDFLAGS=-L${ncurses:location}/lib -Wl,-rpath=${ncurses:location}/lib
PATH=${patch:location}/bin:%(PATH)s
PATH=${texinfo:location}/bin:%(PATH)s
component/screen/screen-4.0.3-stropts.patch deleted 100644 → 0
View file @ fb36acc7
diff -up screen-4.0.3/tty.sh.stropts screen-4.0.3/tty.sh
--- screen-4.0.3/tty.sh.stropts 2003-09-08 16:24:25.000000000 +0200
+++ screen-4.0.3/tty.sh 2008-04-07 11:28:34.000000000 +0200
@@ -76,7 +76,7 @@ exit 0
#endif
#include "config.h"
-#ifdef SVR4
+#if 0
#include <sys/stropts.h> /* for I_POP */
#endif
diff -up screen-4.0.3/screen.c.stropts screen-4.0.3/screen.c
--- screen-4.0.3/screen.c.stropts 2008-04-07 11:25:21.000000000 +0200
+++ screen-4.0.3/screen.c 2008-04-07 11:29:14.000000000 +0200
@@ -50,7 +50,7 @@
#include "config.h"
-#ifdef SVR4
+#if 0
# include <sys/stropts.h>
#endif
diff -up screen-4.0.3/process.c.stropts screen-4.0.3/process.c
--- screen-4.0.3/process.c.stropts 2003-09-18 14:53:54.000000000 +0200
+++ screen-4.0.3/process.c 2008-04-07 11:29:47.000000000 +0200
@@ -37,7 +37,7 @@
#include "config.h"
/* for solaris 2.1, Unixware (SVR4.2) and possibly others: */
-#ifdef SVR4
+#if 0
# include <sys/stropts.h>
#endif
diff -up screen-4.0.3/pty.c.stropts screen-4.0.3/pty.c
--- screen-4.0.3/pty.c.stropts 2003-09-08 16:26:18.000000000 +0200
+++ screen-4.0.3/pty.c 2008-04-07 11:30:07.000000000 +0200
@@ -34,7 +34,7 @@
#endif
/* for solaris 2.1, Unixware (SVR4.2) and possibly others */
-#ifdef HAVE_SVR4_PTYS
+#if 0
# include <sys/stropts.h>
#endif
component/slapos/buildout.cfg
View file @ 56ad14fb
......@@ -28,8 +28,6 @@ parts =
py
firewalld-patch
rebootstrap-directory = rebootstrap
[environment]
# Note: For now original PATH is appended to the end, as not all tools are
# provided by SlapOS
......
component/sqlite3/buildout.cfg
View file @ 56ad14fb
......@@ -5,8 +5,8 @@ parts =
[sqlite3]
recipe = slapos.recipe.cmmi
url = http://sqlite.org/2015/sqlite-autoconf-3081101.tar.gz
md5sum = 298c8d6af7ca314f68de92bc7a356cbe
url = http://sqlite.org/2017/sqlite-autoconf-3190000.tar.gz
md5sum = 2426883b4dd3a9fd6aeb28a16b81a72b
configure-options =
--disable-static
--enable-readline
......
component/w3-validator/buildout.cfg
View file @ 56ad14fb
[buildout]
extends =
../perl/buildout.cfg
../perl-CGI/buildout.cfg
../perl-Config-General/buildout.cfg
../perl-Encode-HanExtra/buildout.cfg
../perl-HTML-Encoding/buildout.cfg
......@@ -23,6 +24,7 @@ erp5.recipe.w3validator = 1.0.2
[w3-validator]
recipe = erp5.recipe.w3validator
depends =
${perl-CGI:location}
${perl-Config-General:location}
${perl-Encode-HanExtra:location}
${perl-HTML-Encoding:location}
......
component/xorg/buildout.cfg
View file @ 56ad14fb
......@@ -525,9 +525,9 @@ configure-options =
--with-sha1=libgcrypt
environment =
PKG_CONFIG_PATH=${pixman:location}/lib/pkgconfig:${openssl:location}/lib/pkgconfig:${xorg-util-macros:location}/share/pkgconfig:${fixesproto:location}/lib/pkgconfig:${damageproto:location}/lib/pkgconfig:${xcmiscproto:location}/lib/pkgconfig:${xtrans:location}/share/pkgconfig:${bigreqsproto:location}/lib/pkgconfig:${xproto:location}/lib/pkgconfig:${randrproto:location}/lib/pkgconfig:${renderproto:location}/lib/pkgconfig:${xextproto:location}/lib/pkgconfig:${inputproto:location}/lib/pkgconfig:${kbproto:location}/lib/pkgconfig:${fontsproto:location}/lib/pkgconfig:${videoproto:location}/lib/pkgconfig:${recordproto:location}/lib/pkgconfig:${resourceproto:location}/lib/pkgconfig:${xineramaproto:location}/lib/pkgconfig:${libXau:location}/lib/pkgconfig:${libxkbfile:location}/lib/pkgconfig:${libXfont:location}/lib/pkgconfig:${libX11:location}/lib/pkgconfig:${libxcb:location}/lib/pkgconfig:${xorg-libpthread-stubs:location}/lib/pkgconfig:${libfontenc:location}/lib/pkgconfig:${freetype:pkg_config_depends}:${freetype:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig
CPPFLAGS=-I${bzip2:location}/include -I${gcrypt:location}/include -I${gpg-error:location}/include -I${zlib:location}/include
CPPFLAGS=-I${bzip2:location}/include -I${libgcrypt:location}/include -I${libgpg-error:location}/include -I${zlib:location}/include
PATH=${patch:location}/bin:${pkgconfig:location}/bin:%(PATH)s
LDFLAGS=-L${bzip2:location}/lib -Wl,-rpath=${bzip2:location}/lib -L${gcrypt:location}/lib -Wl,-rpath=${gcrypt:location}/lib -L${gpg-error:location}/lib -Wl,-rpath=${gpg-error:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
LDFLAGS=-L${bzip2:location}/lib -Wl,-rpath=${bzip2:location}/lib -L${libgcrypt:location}/lib -Wl,-rpath=${libgcrypt:location}/lib -L${libgpg-error:location}/lib -Wl,-rpath=${libgpg-error:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
[libXi]
recipe = slapos.recipe.cmmi
......@@ -557,3 +557,20 @@ configure-options =
environment =
PATH=${pkgconfig:location}/bin:%(PATH)s
PKG_CONFIG_PATH=${libXext:location}/lib/pkgconfig:${libXext:pkg_config_depends}:${libXi:location}/lib/pkgconfig:${recordproto:location}/lib/pkgconfig
[scrnsaverproto]
recipe = slapos.recipe.cmmi
url = https://www.x.org/releases/X11R7.7/src/everything/scrnsaverproto-1.2.2.tar.bz2
md5sum = edd8a73775e8ece1d69515dd17767bfb
environment =
PATH=${pkgconfig:location}/bin:%(PATH)s
PKG_CONFIG_PATH=${xorg-util-macros:location}/share/pkgconfig
[libXScrnSaver]
recipe = slapos.recipe.cmmi
url = https://www.x.org/releases/X11R7.7/src/everything/libXScrnSaver-1.2.2.tar.bz2
md5sum = 7a773b16165e39e938650bcc9027c1d5
environment =
PATH=${pkgconfig:location}/bin:%(PATH)s
PKG_CONFIG_PATH=${libX11:location}/lib/pkgconfig:${xproto:location}/lib/pkgconfig:${kbproto:location}/lib/pkgconfig:${libxcb:location}/lib/pkgconfig:${xorg-libpthread-stubs:location}/lib/pkgconfig:${libXau:location}/lib/pkgconfig:${libXext:location}/lib/pkgconfig:${xextproto:location}/lib/pkgconfig:${scrnsaverproto:location}/lib/pkgconfig
component/zope-2.12/buildout.cfg deleted 100644 → 0
View file @ fb36acc7
[buildout]
# adapt https://svn.erp5.org/repos/public/experimental/erp5.buildout-zope-2.12/
component/zope-2.8/buildout.cfg deleted 100644 → 0
View file @ fb36acc7
# based on https://svn.erp5.org/repos/public/experimental/erp5.buildout
[buildout]
extends=
../python-2.4/buildout.cfg
[zope-2.8]
recipe = erp5.recipe.zope2install
url = http://www.zope.org/Products/Zope/2.8.11/Zope-2.8.11-final.tgz
md5sum = eff3e52d6ecde2d3669ea81e445b5a3a
python = python2.4
skip-fake-eggs =
ClientForm
mechanize
pytz
schema.json
View file @ 56ad14fb
......@@ -2,49 +2,56 @@
"$schema": "http://json-schema.org/draft-04/schema#",
"description": "Slapos Software Release instantiation descriptor",
"additionalProperties": false,
"required": [
"software-type"
],
"properties": {
"name": {
"description": "A short human-friendly name for the sofware release",
"default": "",
"type": "string"
},
"description": {
"description": "A short description of the sofware release",
"default": "",
"type": "string"
},
"serialisation": {
"description": "How the parameters and results are serialised",
"require": true,
"enum": ["xml", "json-in-xml"],
"type": "string"
},
"software-type": {
"description": "Existing software types",
"require": true,
"patternProperties": {
".*": {
"description": "Software type declaration",
"additionalProperties": false,
"required": [
"request",
"response"
],
"properties": {
"title": {
"description": "A human-friendly title of the software type",
"default": "",
"type": "string"
},
"description": {
"description": "A human-friendly description of the software type",
"default": "",
"type": "string"
},
"serialisation": {
"description": "How the parameters and results are serialised, if different from global setting",
"description": "How the parameters and results are serialised, if different from global setting, required if global setting is not provided",
"enum": ["xml", "json-in-xml"],
"type": "string"
},
"request": {
"require": true,
"description": "URL, relative to Software Release base path, of a json schema for values expected by instance of current software type",
"type": "string"
},
"response": {
"require": true,
"description": "URL, relative to Software Release base path, of a json schema for values published by instance of current software type",
"type": "string"
},
......@@ -54,10 +61,12 @@
},
"shared" : {
"description": "Define if the request will request a Slave or Software Instance.",
"default": "false",
"type": "boolean"
},
"index": {
"description": "Value to use instead of software type id to sort them (in order to display most relevant software types earlier in a list, for example)",
"default": 0,
"type": "number"
}
},
......
setup.py
View file @ 56ad14fb
......@@ -163,6 +163,7 @@ setup(name=name,
'publishsection = slapos.recipe.publish:PublishSection',
'publishurl = slapos.recipe.publishurl:Recipe',
'random.time = slapos.recipe.random:Time',
'random.integer = slapos.recipe.random:Integer',
'readline = slapos.recipe.readline:Recipe',
'redis.server = slapos.recipe.redis:Recipe',
'request = slapos.recipe.request:Recipe',
......
slapos/recipe/check_url_available/__init__.py
View file @ 56ad14fb
......@@ -42,6 +42,9 @@ class Recipe(GenericBaseRecipe):
'check_secure': self.options.get('check-secure', 0),
'http_code': self.options.get('http_code', '200'),
'time_out': self.options.get('timeout-file-path', timeout_file),
'ca-cert-file': self.options.get('ca-cert-file', ''),
'cert-file': self.options.get('cert-file', ''),
'key-file': self.options.get('key-file', ''),
}
# XXX-Cedric in this script, curl won't check certificate
......
slapos/recipe/check_url_available/template/check_url.in
View file @ 56ad14fb
......@@ -5,6 +5,9 @@
URL="%(url)s"
F_TIMEOUT="%(time_out)s"
TIMEOUT=20
CA_CERT="%(ca-cert-file)s"
CERT="%(cert-file)s"
KEY="%(key-file)s"
if [ -z "$URL" ]; then
echo "No URL specified." >&2
......@@ -20,7 +23,11 @@ if [ -s "$F_TIMEOUT" ]; then
fi
fi
CODE=$(%(curl_path)s -g -k -sL $URL -w %%{http_code} --max-time $TIMEOUT -o /dev/null)
if [ -z "$CA_CERT" ] || [ -z "$CERT" ] || [ -z "$KEY" ]; then
CODE=$(%(curl_path)s -g -k -sL $URL -w %%{http_code} --max-time $TIMEOUT -o /dev/null)
else
CODE=$(%(curl_path)s -g -k -sL $URL -w %%{http_code} --max-time $TIMEOUT -o /dev/null --cacert $CA_CERT --cert $CERT --key $KEY)
fi
if [ $? -eq 3 ]; then
echo "URL malformed: $URL." >&2
......
slapos/recipe/librecipe/execute.py
View file @ 56ad14fb
......@@ -22,8 +22,8 @@ def _wait_files_creation(file_list):
watchdescriptors = dict()
for dirname in directories.keys():
wd = inotifyx.add_watch(fd,
dirname,
inotifyx.IN_CREATE | inotifyx.IN_DELETE)
dirname,
inotifyx.IN_CREATE | inotifyx.IN_DELETE | inotifyx.IN_MOVE)
watchdescriptors[wd] = dirname
# Set to True the file wich exists
......
slapos/recipe/librecipe/generic.py
View file @ 56ad14fb
......@@ -131,7 +131,7 @@ class GenericBaseRecipe(object):
def createWrapper(self, name, command, parameters, comments=[],
parameters_extra=False, environment=None,
pidfile=None
pidfile=None, reserve_cpu=False
):
"""
Creates a shell script for process replacement.
......@@ -139,6 +139,8 @@ class GenericBaseRecipe(object):
Takes care of #! line limitation when the wrapped command is a script.
if pidfile parameter is specified, then it will make the wrapper a singleton,
accepting to run only if no other instance is running.
:param reserve_cpu: bool, try to reserve one core for the `command`
"""
lines = [ '#!/bin/sh' ]
......@@ -163,6 +165,14 @@ class GenericBaseRecipe(object):
fi
echo $$ > $pidfile""" % shlex.quote(pidfile)))
if reserve_cpu:
# if the CGROUPS cpuset is available (and prepared by slap format)
# request an exclusive CPU core for this process
lines.append(dedent("""
# put own PID into waiting list for exclusive CPU-core access
echo $$ >> ~/.slapos-cpu-exclusive
"""))
lines.append(dedent('''
# If the wrapped command uses a shebang, execute the referenced
# executable passing the script path as first argument.
......
slapos/recipe/neoppod.py
View file @ 56ad14fb
......@@ -43,6 +43,7 @@ class NeoBaseRecipe(GenericBaseRecipe):
# useful, as per NEO deploying constraints.
raise UserError('"masters" parameter is mandatory')
option_list = [
# Keep the -l option first, as expected by logrotate snippets.
'-l', options['logfile'],
'-m', options['masters'],
'-b', self._getBindingAddress(),
......@@ -91,6 +92,8 @@ class Storage(NeoBaseRecipe):
engine = self.options.get('engine')
if engine: # old versions of NEO don't support -e
r += '-e', engine
if self.options.get('disable-drop-partitions'):
r.append('--disable-drop-partitions')
return r
class Admin(NeoBaseRecipe):
......
slapos/recipe/random.py
View file @ 56ad14fb
......@@ -39,6 +39,28 @@ import string
from slapos.recipe.librecipe import GenericBaseRecipe
class Integer(object):
"""
Generate a random integer (see standard random.randint documentation).
Input options:
minimum (required)
integer(-ish) minmum bound, included.
maximum (required)
integer(-ish) maximum bound, included.
Output options:
value
Resulting integer.
"""
def __init__(self, buildout, name, options):
options['value'] = random.randint(int(options['minimum']), int(options['maximum']))
def install(self):
pass
update = install
class Time(object):
"""Generate a random time from a 24h time clock"""
......@@ -101,7 +123,6 @@ class Password(object):
"""
def __init__(self, buildout, name, options):
options_get = options.get
self.create_once = options.get('create-once', 'True').lower() \
in GenericBaseRecipe.TRUE_VALUES
try:
......@@ -118,13 +139,21 @@ class Password(object):
if e.errno != errno.ENOENT:
raise
if not passwd:
passwd = self.generatePassword(int(options_get('bytes', '8')))
passwd = self.generatePassword(int(options.get('bytes', '8')))
self.update = self.install
self.passwd = passwd.strip('\n')
# Password must not go into .installed file, for 2 reasons:
# security of course but also to prevent buildout to always reinstall.
options.get = lambda option, *args, **kw: passwd \
if option == 'passwd' else options_get(option, *args, **kw)
def get(option, *args, **kw):
return passwd if option == 'passwd' else options_get(option, *args, **kw)
try:
options_get = options._get
except AttributeError:
options_get = options.get
options.get = get
else:
options._get = get
generatePassword = staticmethod(generatePassword)
......
slapos/recipe/request.py
View file @ 56ad14fb
......@@ -170,6 +170,20 @@ class Recipe(object):
# Backward compatibility. Old SlapOS master and core don't know this.
self.logger.warning("Impossible to fetch instance GUID nor state.")
except (slapmodule.NotFoundError, slapmodule.ServerError, slapmodule.ResourceNotReady) as exc:
self.logger.warning(
'Request for %(request_name)r with software release '
'%(software_release)r and software type %(software_type)r failed '
'with partition_parameter_kw=%(partition_parameter_kw)r, '
'filter_kw=%(filter_kw)r, shared=%(shared)r, state=%(state)r.', dict(
software_release=software_url,
software_type=software_type,
request_name=name,
partition_parameter_kw=partition_parameter_kw,
filter_kw=filter_kw,
shared=slave,
state=requested_state
)
)
self._raise_request_exception = exc
self._raise_request_exception_formatted = traceback.format_exc()
return_parameter_dict = {}
......
slapos/recipe/slapconfiguration.py
View file @ 56ad14fb
......@@ -26,6 +26,7 @@
##############################################################################
import json
import logging
import os
import slapos.slap
......@@ -33,15 +34,21 @@ from slapos.recipe.librecipe import unwrap
from ConfigParser import RawConfigParser
from netaddr import valid_ipv4, valid_ipv6
from slapos.util import mkdir_p
from slapos import format as slapformat
logger = logging.getLogger("slapos")
class Recipe(object):
"""
Retrieves slap partition parameters, and makes them available to other
buildout section in various ways, and in various encodings.
Populates the buildout section it is used in with all slap partition
parameters.
Also provides access to partition properties: all IPv4, IPv6 and tap
interfaces it is allowed to use.
Retrieve slap partition parameters and make them available in buildout section.
There are two sources of parameters. First is configuration file slapos.cfg and
derived information.
Second is partition's resource_file which is made available in form of keys joined
with "-" and with all "_" replaced by "-".
For example {"tun": {"ipv4": <addr>}} would be available in buildout as ${instance:tun-ipv4}.
Input:
url
......@@ -67,7 +74,7 @@ class Recipe(object):
Example:
${storage-configuration:storage-home}
Output:
Output (keys derived from SlapOS configuration):
root-instance-title
Hosting subscription or root instance title
instance-title
......@@ -122,6 +129,12 @@ class Recipe(object):
options['configuration.' + key] = value
def fetch_parameter_dict(self, options, instance_root):
"""Gather parameters about current computer and partition.
Use two sources of truth
1. SlapOS Master - for external computer/partition information
2. format.Partition.resource_file - for partition specific details
"""
slap = slapos.slap.slap()
slap.initializeConnection(
options['url'],
......@@ -233,6 +246,27 @@ class Recipe(object):
options['storage-dict'] = storage_dict
options['tap'] = tap_set
# The external information transfered from Slap Master has been processed
# so we extend with information gathered from partition resource file
if hasattr(slapformat.Partition, "resource_file"):
resource_home = instance_root
while not os.path.exists(os.path.join(resource_home, slapformat.Partition.resource_file)):
resource_home = os.path.normpath(os.path.join(resource_home, '..'))
if resource_home == "/":
break
else:
# no break happened - let's add partition resources into options
logger.debug("Using partition resource file {}".format(
os.path.join(resource_home, slapformat.Partition.resource_file)))
with open(os.path.join(resource_home, slapformat.Partition.resource_file)) as fi:
partition_params = json.load(fi)
# be very careful with overriding master's information
for key, value in flatten_dict(partition_params).items():
if key not in options:
options[key] = value
# print out augmented options to see what we are passing
logger.debug(str(options))
return self._expandParameterDict(options, parameter_dict)
def _expandParameterDict(self, options, parameter_dict):
......@@ -261,3 +295,14 @@ class JsonDump(Recipe):
update = install
def flatten_dict(data, key_prefix=''):
"""Transform folded dict into one-level key-subkey-subsubkey dictionary."""
output = {}
for key, value in data.items():
prefixed_key = key_prefix + key.replace("_", "-") # to be consistent with `fetch_parameter_dict`
if isinstance(value, dict):
output.update(flatten_dict(value, key_prefix=prefixed_key + "-"))
continue
output[prefixed_key] = value
return output
slapos/recipe/wrapper.py
View file @ 56ad14fb
......@@ -31,6 +31,16 @@ import os
from slapos.recipe.librecipe import GenericBaseRecipe
class Recipe(GenericBaseRecipe):
"""Recipe to create a script from given command and options.
:param str command-line: shell command which launches the intended process
:param str wrapper-path: absolute path to file's destination
:param lines wait-for-files: list of files to wait for
:param str pidfile: path to pidfile ensure exclusivity for the process
:param bool parameters-extra: whether wrapper parameters are passed onto command
:param bool reserve-cpu: command will ask for an exclusive CPU core
"""
def install(self):
command_line = shlex.split(self.options['command-line'])
wrapper_path = self.options['wrapper-path']
......@@ -38,6 +48,7 @@ class Recipe(GenericBaseRecipe):
environment = self.options.get('environment')
parameters_extra = self.options.get('parameters-extra')
pidfile = self.options.get('pidfile')
reserve_cpu = self.options.get('reserve-cpu', False)
if not wait_files and not environment:
# Create a simple wrapper as shell script
......@@ -47,6 +58,7 @@ class Recipe(GenericBaseRecipe):
parameters=command_line[1:],
parameters_extra=parameters_extra,
pidfile=pidfile,
reserve_cpu=reserve_cpu
)]
# More complex needs: create a Python script as wrapper
......@@ -76,5 +88,6 @@ class Recipe(GenericBaseRecipe):
parameters=[],
parameters_extra=parameters_extra,
pidfile=pidfile,
reserve_cpu=reserve_cpu
)]
slapos/recipe/zero_knowledge.py
View file @ 56ad14fb
......@@ -36,32 +36,36 @@ class WriteRecipe(GenericBaseRecipe):
"""
"""
def __init__(self, buildout, name, options):
if not "filename" in options:
raise zc.buildout.UserError("You have to provide the parameter \"filename\"")
if not "filename" in options and not "file-path" in options:
raise zc.buildout.UserError("You have to provide the parameter either \"filename\" or \"file-path\"")
self.filename = options['filename'].strip()
self.path = os.path.join(buildout['buildout']['directory'], self.filename)
self._options = options.copy()
if options.get('filename'):
self.filename = options['filename'].strip()
self.path = os.path.join(buildout['buildout']['directory'], self.filename)
del self._options['filename']
else:
self.path = options['file-path'].strip()
del self._options['file-path']
del self._options['recipe']
self.name = name
self.options = options.copy()
del self.options['filename']
del self.options['recipe']
def install(self):
# Set up the parser, and write config file if needed
self.parser = ConfigParser.ConfigParser()
try:
self.parser.read(self.path)
#clean_options(options)
for key in self.options:
for key in self._options:
if key not in self.parser.options(self.name):
self.parser.set(self.name, key, self.options[key])
self.parser.set(self.name, key, self._options[key])
with open(self.path, 'w') as file:
self.parser.write(file)
# If the file or section do not exist
except (ConfigParser.NoSectionError, IOError) as e:
self.full_install()
install = update = lambda self: []
def full_install(self):
"""XXX-Nicolas : when some parameter's value is changed in
buildout profile, this will override custom user defined values"""
......@@ -69,21 +73,25 @@ class WriteRecipe(GenericBaseRecipe):
if self.parser.has_section(self.name):
self.parser.remove_section(self.name)
self.parser.add_section(self.name)
for key in self.options:
self.parser.set(self.name, key, self.options[key])
for key in self._options:
self.parser.set(self.name, key, self._options[key])
with open(self.path, 'w') as file:
self.parser.write(file)
update = install
class ReadRecipe(GenericBaseRecipe):
"""
"""
def __init__(self, buildout, name, options):
if not "filename" in options:
raise zc.buildout.UserError("You have to provide the parameter \"filename\"")
if not "filename" in options and not "file-path" in options:
raise zc.buildout.UserError("You have to provide the parameter either \"filename\" or file-path")
self.filename = options['filename'].strip()
self.path = os.path.join(buildout['buildout']['directory'], self.filename)
if options.get('filename'):
self.filename = options['filename'].strip()
self.path = os.path.join(buildout['buildout']['directory'], self.filename)
else:
self.path = options['file-path'].strip()
# Set up the parser, and write config file if needed
self.parser = ConfigParser.ConfigParser()
......
slapos/test/recipe/test_slaposconfiguration.py 0 → 100644
View file @ 56ad14fb
# coding: utf-8
import json
import mock
import os
import unittest
from collections import defaultdict
from slapos.recipe import slapconfiguration
from slapos import format as slapformat
class SlapConfigurationTest(unittest.TestCase):
def setUp(self):
"""Prepare files on filesystem."""
self.instance_root = "/tmp/instance_test_resourcefile"
os.mkdir(self.instance_root)
# create testing resource file
self.resource_file = os.path.join(self.instance_root, slapformat.Partition.resource_file)
self.resource = {
"tun": {
"ipv4": "192.168.0.1"
},
"address_list": [
10, 20
]
}
with open(self.resource_file, "wt") as fo:
json.dump(self.resource, fo)
# do your tests inside try block and clean up in finally
self.buildout = {
"buildout": {
"directory": self.instance_root
}
}
def tearDown(self):
os.unlink(self.resource_file)
os.rmdir(self.instance_root)
@mock.patch("slapos.slap.slap")
def test_correct_naming(self, MockClient):
"""Test correct naming of variables from resource file."""
MockClient.initializeConnection.return_value = None
MockClient.getInstanceParameterDict.return_value = dict()
options = defaultdict(str)
recipe = slapconfiguration.Recipe(self.buildout, "slapconfiguration", options)
self.assertEqual(options['tun-ipv4'], "192.168.0.1",
"Folded attrs should be separated by -")
self.assertEqual(options['address-list'], [10, 20],
"All underscores should be replaced with -")
software/agent/software.cfg
View file @ 56ad14fb
......@@ -45,35 +45,35 @@ eggs =
[versions]
apache-libcloud = 0.18.0
ecdsa = 0.13
erp5.util = 0.4.46
erp5.util = 0.4.49
gitdb = 0.6.4
pycrypto = 2.6.1
pycurl = 7.43.0
slapos.recipe.download = 1.0
slapos.recipe.template = 2.8
slapos.toolbox = 0.67
slapos.recipe.template = 3.0
slapos.toolbox = 0.69
smmap = 0.9.0
# Required by:
# slapos.toolbox = 0.67
# slapos.toolbox==0.69
GitPython = 2.0.8
# Required by:
# slapos.toolbox = 0.67
# slapos.toolbox==0.69
atomize = 0.2.0
# Required by:
# slapos.toolbox = 0.67
# slapos.toolbox==0.69
feedparser = 5.2.1
# Required by:
# slapos.toolbox = 0.67
# slapos.toolbox==0.69
lockfile = 0.12.2
# Required by:
# slapos.toolbox = 0.67
# slapos.toolbox==0.69
paramiko = 2.0.1
# Required by:
# slapos.toolbox = 0.67
# slapos.toolbox==0.69
rpdb = 0.1.5
software/apache-frontend/buildout.hash.cfg
View file @ 56ad14fb
......@@ -19,15 +19,15 @@ md5sum = f686f765e55d1dce2e55a400f0714b3e
[template-apache-frontend]
filename = instance-apache-frontend.cfg
md5sum = e2f60121bc1116319b4d20b4c4badc1c
md5sum = 979eecabeaad0114fb24f65bf52f7e6a
[template-apache-replicate]
filename = instance-apache-replicate.cfg.in
md5sum = 9b17c835bcd927269cf510bf612f5985
md5sum = c55506cfb92d87f915d50d32fb129a28
[template-slave-list]
filename = templates/apache-custom-slave-list.cfg.in
md5sum = 8a8db11499f0e3b00c0bd7d1e5afc99b
md5sum = 4010f7366262d00c33f9e7f53f63d42e
[template-slave-configuration]
filename = templates/custom-virtualhost.conf.in
......@@ -39,11 +39,11 @@ md5sum = 665e83d660c9b779249b2179d7ce4b4e
[template-apache-frontend-configuration]
filename = templates/apache.conf.in
md5sum = 51e0c8957e3cf30e09c7b836143c15c9
md5sum = 82cdb4ab02fec36285b9c1ce502f82f0
[template-custom-slave-list]
filename = templates/apache-custom-slave-list.cfg.in
md5sum = 8a8db11499f0e3b00c0bd7d1e5afc99b
md5sum = 4010f7366262d00c33f9e7f53f63d42e
[template-not-found-html]
filename = templates/notfound.html
......@@ -55,7 +55,7 @@ md5sum = d98a01182f38868612948c87d5231428
[template-default-slave-virtualhost]
filename = templates/default-virtualhost.conf.in
md5sum = 8fda2e88476ccb3652036302364f3506
md5sum = 5344bff68a3f7dead633a35771ad1a19
[template-cached-slave-virtualhost]
filename = templates/cached-virtualhost.conf.in
......@@ -92,3 +92,11 @@ md5sum = a5186f666acb2f040ede04c91e60408f
[template-nginx-notebook-slave-virtualhost]
filename = templates/nginx-notebook-slave.conf.in
md5sum = 82d74a7f2aceb2b4a7acc6259291b7f2
[template-apache-lazy-script-call]
filename = templates/apache-lazy-script-call.sh.in
md5sum = ebe5d3d19923eb812a40019cb11276d8
[template-apache-graceful-script]
filename = templates/apache-graceful-script.sh.in
md5sum = 41299cc64200e7b8217fb9dec20bb8b9
software/apache-frontend/common.cfg
View file @ 56ad14fb
......@@ -141,6 +141,14 @@ url = ${:_profile_base_location_}/templates/nginx.cfg.in
output = ${buildout:directory}/template-nginx.cfg.in
mode = 0644
[template-apache-lazy-script-call]
<=download-template
filename = apache-lazy-script-call.sh.in
[template-apache-graceful-script]
<=download-template
filename = apache-graceful-script.sh.in
[template-nginx-eventsource-slave-virtualhost]
<=download-template
filename = nginx-eventsource-slave.conf.in
......
software/apache-frontend/instance-apache-frontend.cfg
View file @ 56ad14fb
......@@ -23,6 +23,7 @@ parts =
promise-apache-frontend-cached
promise-apache-frontend-ssl-cached
promise-apache-is-process-older-than-dependency-set
promise-apache-mpm-watchdog
promise-nginx-frontend-v4-https
promise-nginx-frontend-v4-http
......@@ -44,7 +45,6 @@ parts =
## Nginx
nginx-frontend
## Monitor for apache
monitor-base
monitor-ats-cache-stats-wrapper
......@@ -172,8 +172,6 @@ extra-context =
key nginx_configuration_directory apache-directory:nginx-slave-configuration
key apache_cached_configuration_directory apache-directory:slave-with-cache-configuration
key slave_with_cache_configuration_directory apache-directory:slave-with-cache-configuration
key cached_port apache-configuration:cache-through-port
key ssl_cached_port apache-configuration:ssl-cache-through-port
key http_port instance-parameter:configuration.plain_http_port
key https_port instance-parameter:configuration.port
key nginx_http_port instance-parameter:configuration.plain_nginx_port
......@@ -185,7 +183,6 @@ extra-context =
key apache_log_directory apache-directory:slave-log
key local_ipv4 instance-parameter:ipv4-random
key global_ipv6 slap-network-information:global-ipv6
key cache_port apache-configuration:cache-port
key varnginx directory:varnginx
key empty_template software-release-path:template-empty
key template_custom_slave_configuration software-release-path:template-slave-configuration
......@@ -194,6 +191,7 @@ extra-context =
key template_eventsource_slave_configuration software-release-path:template-nginx-eventsource-slave-virtualhost
key template_notebook_slave_configuration software-release-path:template-nginx-notebook-slave-virtualhost
raw software_type single-custom-personal
key frontend_lazy_graceful_reload frontend-apache-lazy-graceful:rendered
section logrotate_dict logrotate
section frontend_configuration frontend-configuration
section apache_configuration apache-configuration
......@@ -372,7 +370,7 @@ name = apache
log = $${apache-configuration:error-log} $${apache-configuration:access-log}
frequency = daily
rotatep-num = 30
post = $${apache-configuration:frontend-graceful-command}
post = $${frontend-apache-lazy-graceful:rendered} &
sharedscripts = true
notifempty = true
create = true
......@@ -500,14 +498,35 @@ mode = 700
### End of ATS sections
### Apaches Graceful and promises
[frontend-apache-graceful]
[frontend-apache-graceful-bin]
< = jinja2-template-base
template = ${template-wrapper:output}
rendered = $${directory:etc-run}/frontend-apache-safe-graceful
rendered = $${directory:bin}/frontend-apache-safe-graceful
mode = 0700
extra-context =
key content apache-configuration:frontend-graceful-command
[frontend-apache-graceful]
< = jinja2-template-base
template = ${template-apache-graceful-script:target}
rendered = $${directory:etc-run}/frontend-apache-safe-graceful
mode = 0700
extra-context =
key directory_run directory:run
key directory_etc directory:etc
key apache_graceful_reload_command apache-configuration:frontend-graceful-command
[frontend-apache-lazy-graceful]
< = jinja2-template-base
template = ${template-apache-lazy-script-call:target}
rendered = $${directory:bin}/frontend-apache-lazy-graceful
mode = 0700
pid-file = $${directory:run}/lazy-graceful.pid
extra-context =
key pid_file :pid-file
raw wait_time 60
key lazy_command apache-configuration:frontend-graceful-command
# Promises checking configuration:
[promise-frontend-apache-configuration]
< = jinja2-template-base
......@@ -553,13 +572,18 @@ path = $${directory:promise}/apache_ssl_cached
hostname = $${instance-parameter:ipv4-random}
port = $${apache-configuration:ssl-cache-through-port}
[promise-apache-is-process-older-than-dependency-set]
recipe = slapos.cookbook:wrapper
command-line = ${buildout:bin-directory}/is-process-older-than-dependency-set $${apache-configuration:pid-file}
wrapper-path = $${directory:promise}/apache-frontend-is-running-actual-software-release
parameters-extra = true
[promise-apache-mpm-watchdog]
recipe = slapos.cookbook:wrapper
command-line = ${buildout:bin-directory}/apache-mpm-watchdog --url http://$${instance-parameter:ipv4-random}:$${instance-parameter:configuration.plain_http_port}/server-status -u admin -p $${monitor-htpasswd:passwd} -d $${directory:run}/apache-mpm-watchdog.json
wrapper-path = $${directory:promise}/apache-mpm-watchdog
parameters-extra = true
[slap_connection]
# Kept for backward compatiblity
......@@ -589,7 +613,7 @@ extra_slave_instance_list =
frontend-name =
monitor-cors-domains =
monitor-httpd-port = 8072
monitor-username = $${monitor-htpasswd:username}
monitor-username = $${monitor-instance-parameter:username}
monitor-password = $${monitor-htpasswd:passwd}
#######
......@@ -631,7 +655,7 @@ extra-context =
template = ${template-wrapper:output}
rendered = $${monitor-directory:reports}/monitor-apache-server-status-wrapper
mode = 0700
command = ${curl:location}/bin/curl -s http://$${instance-parameter:ipv4-random}:$${instance-parameter:configuration.plain_http_port}/server-status -u $${monitor-htpasswd:username}:$${monitor-htpasswd:passwd} 2>&1
command = ${curl:location}/bin/curl -s http://$${instance-parameter:ipv4-random}:$${instance-parameter:configuration.plain_http_port}/server-status -u $${monitor-instance-parameter:username}:$${monitor-htpasswd:passwd} 2>&1
extra-context =
key content monitor-apache-server-status-wrapper:command
......
software/apache-frontend/instance-apache-input-schema.json
View file @ 56ad14fb
......@@ -14,7 +14,44 @@
"description": "Value used to set IPReadLimit Parameter for antiloris.",
"type": "integer",
"default": 10
},
},
"mpm-server-limit": {
"title": "ServerLimit",
"description": "Value used to set ServerLimit on apache configuration.",
"type": "integer",
"default": 16
},
"mpm-max-clients": {
"title": "MaxClients",
"description": "Value used to set MaxClients on apache configuration.",
"type": "integer",
"default": 400
},
"mpm-start-servers": {
"title": "StartServers",
"description": "Value used to set StartServers on apache configuration.",
"type": "integer",
"default": 3
},
"mpm-thread-per-child": {
"title": "ThreadsPerChild",
"description": "Value used to set ThreadsPerChild on apache configuration.",
"type": "integer",
"default": 25
},
"mpm-graceful-shutdown-timeout": {
"title": "ThreadsPerChild",
"description": "Value used to set ThreadsPerChild on apache configuration.",
"type": "integer",
"default": 5
},
"enable-http2-by-default": {
"title": "Enable HTTP2 by Default",
"description": "Use HTTP2 as default Protocol",
"type": "string",
"default": "true",
"enum": ["true", "false"]
},
"re6st-verification-url": {
"title": "Test Verification URL",
"description": "Url to verify if the internet and/or re6stnet is working.",
......
software/apache-frontend/instance-apache-replicate.cfg.in
View file @ 56ad14fb
......@@ -81,7 +81,7 @@ context =
<= slap-connection
recipe = slapos.cookbook:requestoptional
config-monitor-cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.officejs.com') }}
config-monitor-username = ${monitor-htpasswd:username}
config-monitor-username = ${monitor-instance-parameter:username}
config-monitor-password = ${monitor-htpasswd:passwd}
{% set frontend_software_url_key = "-frontend-software-release-url" %}
......
software/apache-frontend/instance-slave-apache-input-schema.json
View file @ 56ad14fb
......@@ -140,7 +140,13 @@
"default": "false",
"enum": ["false", "true"]
},
"enable-http2": {
"title": "Enable HTTP2 Protocol",
"description": "Use HTTP2 Protocol for the site",
"type": "string",
"default": "true",
"enum": ["true", "false"]
},
"prefer-gzip-encoding-to-backend": {
"title": "Prefer gzip Encoding for Backend",
"description": "If set to true, frontend will rewrite Accept-Encoding request header to simply 'gzip' for all variants of Accept-Encoding containing 'gzip', in order to maximize cache hits for resources cached with Vary: Accept-Encoding when enable_cache is used",
......
software/apache-frontend/software.cfg
View file @ 56ad14fb
......@@ -10,8 +10,8 @@ gitdb = 0.6.4
plone.recipe.command = 1.1
pycrypto = 2.6.1
rdiff-backup = 1.0.5+SlapOSPatched001
slapos.recipe.template = 2.8
slapos.toolbox = 0.67
slapos.recipe.template = 3.0
slapos.toolbox = 0.69
smmap = 0.9.0
numpy = 1.11.2
pyasn1 = 0.2.3
......@@ -19,41 +19,41 @@ pyasn1 = 0.2.3
websockify = 0.8.0
# Required by:
# slapos.toolbox==0.61
# slapos.toolbox==0.69
dnspython = 1.15.0
# Required by:
# slapos.toolbox==0.61
erp5.util = 0.4.46
# slapos.toolbox==0.69
erp5.util = 0.4.49
# Required by:
# slapos.toolbox==0.61
# slapos.toolbox==0.69
passlib = 1.6.5
# Required by:
# slapos.toolbox = 0.67
# slapos.toolbox==0.69
GitPython = 2.0.8
# Required by:
# slapos.toolbox = 0.67
# slapos.toolbox==0.69
atomize = 0.2.0
# Required by:
# slapos.toolbox = 0.67
# slapos.toolbox==0.69
feedparser = 5.2.1
# Required by:
# slapos.toolbox = 0.67
# slapos.toolbox==0.69
lockfile = 0.12.2
# Required by:
# slapos.toolbox = 0.67
# slapos.toolbox==0.69
paramiko = 2.0.1
# Required by:
# slapos.toolbox = 0.67
# slapos.toolbox==0.69
pycurl = 7.43.0
# Required by:
# slapos.toolbox = 0.67
# slapos.toolbox==0.69
rpdb = 0.1.5
software/apache-frontend/templates/apache-custom-slave-list.cfg.in
View file @ 56ad14fb
......@@ -2,6 +2,9 @@
{% set cached_server_dict = {} -%}
{% set part_list = [] -%}
{% set cache_port = apache_configuration.get('cache-port') %}
{% set cached_port = apache_configuration.get('cache-through-port') %}
{% set ssl_cached_port = apache_configuration.get('ssl-cache-through-port') %}
{% set cache_access = "http://%s:%s" % (local_ipv4, cache_port) -%}
{% set ssl_cache_access = "http://%s:%s/HTTPS" % (local_ipv4, cache_port) -%}
{% set TRUE_VALUES = ['y', 'yes', '1', 'true'] -%}
......@@ -27,7 +30,7 @@ context =
{{ key }} = {{ value }}
{% endfor %}
post = {{ apache_configuration.get('frontend-graceful-command') }}
post = {{ frontend_lazy_graceful_reload }} &
frequency = daily
rotatep-num = 30
sharedscripts = true
......@@ -106,7 +109,7 @@ crl = {{ custom_ssl_directory }}/crl/
{% endif -%}
[slave-log-directories]
{{slave_reference}}-log-folder = {{ slave_log_folder }}
{{slave_reference}} = {{ slave_log_folder }}
{# Set slave logrotate entry #}
[{{slave_logrotate_section}}]
......
software/apache-frontend/templates/apache-graceful-script.sh.in 0 → 100644
View file @ 56ad14fb
#!/bin/sh
RUN_DIR={{ directory_run }}
ETC_DIR={{ directory_etc }}
APACHE_SIGNATURE_FILE=$RUN_DIR/apache_configuration.signature
NAPACHE_SIGNATURE_FILE=$RUN_DIR/napache_configuration.signature
touch $APACHE_SIGNATURE_FILE
sha256sum $ETC_DIR/apache*.conf $ETC_DIR/apache-*.d/*.conf $ETC_DIR/apache-*.d/ssl/*.*key $ETC_DIR/apache-*.d/ssl/*.*crt* | sort -k 66 > $NAPACHE_SIGNATURE_FILE
# If no diff, no restart for now
if diff "$APACHE_SIGNATURE_FILE" "$NAPACHE_SIGNATURE_FILE"; then
echo "Nothing Changed, so nothing to reload"
exit 0
fi
echo "Reloading apache.."
{{ apache_graceful_reload_command }}
mv "$NAPACHE_SIGNATURE_FILE" "$APACHE_SIGNATURE_FILE"
software/apache-frontend/templates/apache-lazy-script-call.sh.in 0 → 100644
View file @ 56ad14fb
#!/bin/bash
PID=$$
PIDFILE={{ pid_file }}
sleep $((RANDOM % 10))
if [ ! -f $PIDFILE ]; then
echo $PID > $PIDFILE
sleep {{ wait_time }}
{{ lazy_command }}
else
ps --pid `cat $PIDFILE` &>/dev/null
if [ $? -eq 0 ]; then
echo "Skipped"
else
echo $PID > $PIDFILE
sleep {{ wait_time }}
{{ lazy_command }}
fi
fi
software/apache-frontend/templates/apache.conf.in
View file @ 56ad14fb
......@@ -16,7 +16,6 @@ ServerRoot {{ instance_home }}
{% endfor -%}
ServerAdmin {{ server_admin }}
DefaultType text/plain
TypesConfig {{ httpd_home }}/conf/mime.types
AddType application/x-compress .Z
AddType application/x-gzip .gz .tgz
......@@ -31,7 +30,7 @@ TraceEnable off
# Log configuration
ErrorLog "{{ error_log }}"
LogLevel info
LogLevel notice
# LogFormat "%h %{REMOTE_USER}i %{Host}i %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined
# LogFormat "%h %{REMOTE_USER}i %{Host}i %l %u %t \"%r\" %>s %b" common
# CustomLog "{{ access_log }}" common
......@@ -83,6 +82,7 @@ LoadModule filter_module {{ httpd_home }}/modules/mod_filter.so
LoadModule http2_module {{ httpd_home }}/modules/mod_http2.so
LoadModule info_module {{ httpd_home }}/modules/mod_info.so
LoadModule status_module {{ httpd_home }}/modules/mod_status.so
LoadModule reqtimeout_module {{ httpd_home }}/modules/mod_reqtimeout.so
# The following directives modify normal HTTP response behavior to
# handle known problems with browser implementations.
......@@ -105,6 +105,11 @@ BrowserMatch "^gnome-vfs" redirect-carefully
BrowserMatch "^XML Spy" redirect-carefully
BrowserMatch "^Dreamweaver-WebDAV-SCM1" redirect-carefully
{% if slapparameter_dict.get('enable-http2-by-default', 'true') == 'true' %}
Protocols h2 http/1.1
{% endif %}
# Increase IPReadLimit to 10
<IfModule antiloris_module>
# IPReadLimit - Maximum simultaneous connections in READ state per IP address
......@@ -125,7 +130,12 @@ ExtendedStatus On
Require valid-user
</Location>
ServerLimit {{ slapparameter_dict.get('mpm-server-limit', '16') }}
MaxClients {{ slapparameter_dict.get('mpm-max-clients', '400') }}
StartServers {{ slapparameter_dict.get('mpm-start-servers', '3') }}
ThreadsPerChild {{ slapparameter_dict.get('mpm-thread-per-child', '25') }}
#MaxRequestsPerChild 0 // Default value is 0
GracefulShutdownTimeout {{ slapparameter_dict.get('mpm-graceful-shutdown-timeout', '5') }}
# Deflate
AddOutputFilterByType DEFLATE text/html text/plain text/xml text/css text/javascript application/x-javascript application/javascript
......@@ -156,13 +166,7 @@ SSLProxyCheckPeerCN off
SSLProxyCheckPeerExpire off
include {{frontend_configuration.get('log-access-configuration')}}
NameVirtualHost *:{{ http_port }}
NameVirtualHost *:{{ https_port }}
include {{ slave_configuration_directory }}/*.conf
NameVirtualHost *:{{ cached_port }}
NameVirtualHost *:{{ ssl_cached_port }}
include {{ slave_with_cache_configuration_directory }}/*.conf
ErrorDocument 404 /notfound.html
......
software/apache-frontend/templates/default-virtualhost.conf.in
View file @ 56ad14fb
......@@ -3,6 +3,7 @@
{%- set disable_via_header = ('' ~ slave_parameter.get('disable-via-header', '')).lower() in TRUE_VALUES -%}
{%- set prefer_gzip = ('' ~ slave_parameter.get('prefer-gzip-encoding-to-backend', '')).lower() in TRUE_VALUES -%}
{%- set server_alias_list = slave_parameter.get('server-alias', '').split() -%}
{%- set enable_h2 = ('' ~ slave_parameter.get('enable-http2', 'true')).lower() in TRUE_VALUES -%}
{%- set ssl_proxy_verify = ('' ~ slave_parameter.get('ssl-proxy-verify', '')).lower() in TRUE_VALUES -%}
{%- set disabled_cookie_list = slave_parameter.get('disabled-cookie-list', '').split() -%}
{%- set https_only = ('' ~ slave_parameter.get('https-only', '')).lower() in TRUE_VALUES -%}
......@@ -33,8 +34,10 @@
SSLProtocol all -SSLv2 -SSLv3
SSLCipherSuite ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:HIGH:!aNULL:!MD5
SSLHonorCipherOrder on
Protocols h2 http/1.1
{% if enable_h2 %}
Protocols h2 http/1.1
{% endif -%}
{% for key, value in ssl_configuration_list -%}
{% if value in slave_parameter -%}
......@@ -44,7 +47,7 @@
# One Slave two logs
ErrorLog "{{ slave_parameter.get('error_log') }}"
LogLevel info
LogLevel notice
LogFormat "%h %l %{REMOTE_USER}i %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %D" combined
CustomLog "{{ slave_parameter.get('access_log') }}" combined
......@@ -116,14 +119,16 @@
# One Slave two logs
ErrorLog "{{ slave_parameter.get('error_log') }}"
LogLevel info
LogLevel notice
LogFormat "%h %l %{REMOTE_USER}i %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %D" combined
CustomLog "{{ slave_parameter.get('access_log') }}" combined
# Remove "Secure" from cookies, as backend may be https
Header edit Set-Cookie "(?i)^(.+);secure$" "$1"
{% if enable_h2 %}
Protocols h2 http/1.1
{% endif -%}
{% if disable_no_cache_header %}
RequestHeader unset Cache-Control
......
software/caucase/instance-caucase-input-schema.json 0 → 100644
View file @ 56ad14fb
{
"type": "object",
"$schema": "http://json-schema.org/draft-04/schema",
"title": "Input Parameters",
"properties": {
"server-port": {
"title": "http port to use",
"description": "Caucase http port to use.",
"type": "integer",
"default": 8009,
"minimum": 1,
"maximum": 65535
},
"server-https-port": {
"title": "https port to use",
"description": "Caucase port to use for https connexion.",
"type": "integer",
"default": 8010,
"minimum": 1,
"maximum": 65535
},
"external-url": {
"title": "External http url",
"description": "External http url which point to caucase on http. This url will be added in signed certificate as CRL distribution point URI",
"type": "string",
"format": "uri"
},
"ca-subject": {
"title": "Subject of CA Certificate",
"description": "CA certificate subject as string. The format is: /C=XX/ST=State/L=City/OU=OUnit/O=Company/CN=CA Auth/emailAddress=xx@example.com. Only /CN is mandatory.",
"type": "string",
"default": "/C=FR/O=Company/CN=SlapOS Certificate Authority/emailAddress=xx@example.com"
},
"max-request-amount": {
"title": "Number of pending csr to accept",
"description": "Number of pending csr to accept. If this limit is reached, no more csr will be accepted by the CA.",
"type": "integer",
"default": 10
},
"crt-life-time": {
"title": "Signed Certificate life time",
"description": "The time in seconds before a generated certificate will expire. Default: 365*24*60*60 seconds (1 year)",
"type": "integer",
"default": 31536000,
"minimum": 86400
},
"crl-life-period": {
"title": "CRL life time period",
"description": "Number of individual certificate validity periods during which the CRL is valid. Default: 1/50.0",
"type": "float",
"default": 0.2
},
"ca-life-period": {
"title": "CA Certificate life period",
"description": "Number of individual certificate validity periods during which the CA certificate is valid. Default: 10",
"type": "float",
"default": 10
},
"crt-keep-time": {
"title": "Time before cleanup certificate content on CA",
"description": "The time in seconds before a generated certificate will be deleted on CA server. Set 0 to never delete. Default: 30*24*60*60 seconds (30 days)",
"default": 5184000,
"type": "integer"
}
}
}
\ No newline at end of file
software/caucase/instance-caucase-output-schema.json 0 → 100644
View file @ 56ad14fb
{
"$schema": "http://json-schema.org/draft-04/schema#",
"description": "Values returned by Caucase instantiation",
"properties": {
"http-url": {
"description": "Caucase URL on HTTP",
"type": "string"
},
"https-url": {
"description": "Caucase URL on HTTPS",
"type": "string"
}
},
"type": "object"
}
software/caucase/software.cfg 0 → 100644
View file @ 56ad14fb
[buildout]
extends =
../../stack/caucase/buildout.cfg
../../stack/slapos.cfg
parts =
slapos-cookbook
extra-eggs
instance-caucase
software/caucase/software.cfg.json 0 → 100644
View file @ 56ad14fb
{
"name": "Caucase",
"description": "Caucase certificate authority",
"serialisation": "json-in-xml",
"software-type": {
"default": {
"title": "Default",
"description": "Single caucase instance",
"request": "instance-caucase-input-schema.json",
"response": "instance-caucase-output-schema.json",
"index": 0
}
}
}
\ No newline at end of file
software/cdn-me/software.cfg
View file @ 56ad14fb
......@@ -38,6 +38,6 @@ eggs =
[versions]
cns.recipe.symlink = 0.2.3
collective.recipe.environment = 0.2.0
erp5.util = 0.4.46
erp5.util = 0.4.49
plone.recipe.command = 1.1
slapos.recipe.template = 2.8
slapos.recipe.template = 3.0
software/cloudooo/instance-cloudooo.cfg.in
View file @ 56ad14fb
......@@ -42,7 +42,7 @@
{% do apache_ip_list.append('[' ~ ipv6 ~ ']') -%}
{% endif -%}
{% set apache_dict = {} -%}
{% do apache_dict.__setitem__(publish_url_name, (apache_port, "https", 'http://' ~ ipv4 ~ ':' ~ haproxy_port)) -%}
{% do apache_dict.__setitem__(publish_url_name, (apache_port, "https", 'http://' ~ ipv4 ~ ':' ~ haproxy_port, False)) -%}
{% set bin_directory = parameter_dict['buildout-bin-directory'] -%}
{% set section_list = [] -%}
......
software/cloudooo/software-common.cfg
View file @ 56ad14fb
......@@ -83,5 +83,5 @@ template-logrotate-base = ${template-logrotate-base:rendered}
[template-cloudooo-instance]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/instance-cloudooo.cfg.in
md5sum = e1a4a73aeb8b978e953a3f4f39b98a4d
md5sum = 0a260754586e2a44010f3303c58a927a
mode = 640
software/dream/software.cfg
View file @ 56ad14fb
......@@ -48,12 +48,12 @@ scipy = 0.13.3
simpy = 3.0.5
zope.dottedname = 4.1.0
tablib = 0.10.0
mysqlclient = 1.3.10
mysqlclient = 1.3.10+SlapOSPatched001
# indirect dependancies
cp.recipe.cmd = 0.5
plone.recipe.command = 1.1
slapos.recipe.template = 2.8
slapos.recipe.template = 3.0
zope.exceptions = 4.0.7
zope.testing = 4.1.3
zc.recipe.testrunner = 2.0.0
......
software/erp5/instance-erp5-input-schema.json
View file @ 56ad14fb
......@@ -111,6 +111,12 @@
"default": 5,
"type": "integer"
},
"ssl-authentication": {
"title": "Enable SSL Client authentication on this zope instance.",
"description": "If set to true, will set SSL Client verification to required on apache VirtualHost which allow to access this zope instance.",
"type": "boolean",
"default": false
},
"webdav": {
"description": "Serve webdav queries, implies timerserver-interval=0 (disabled). Mixing webdav and non-webdav nodes in a single family will give unspecified results.",
"default": false,
......@@ -263,6 +269,28 @@
"description": "In wendelin.core there are 2 formats for storing data, so called ZBlk0 and ZBlk1. See https://lab.nexedi.com/nexedi/wendelin.core/blob/2e5e1d3d/bigfile/file_zodb.py#L19 for more details.",
"default": "",
"type": "string"
},
"caucase": {
"description": "Caucase certificate authority parameters",
"properties": {
"url": {
"title": "Caucase URL",
"description": "URL of existing caucase instance to use. If empty, a new caucase instance will be deployed. If not empty, other properties in this section will be ignored.",
"default": "",
"type": "string",
"format": "uri"
},
"crl-update-periodicity": {
"title": "Periodicity of CRL update",
"description": "Periodicity of CRL update, in cron format. The CRL will be downloaded from caucase URL and the new content will be saved if there was a change. Everytime a new CRL is writen, Apache reload will be called.",
"type": "string",
"default": "0 0 * * *"
}
},
"additionalProperties": {
"$ref": "../caucase/instance-caucase-input-schema.json#/properties"
},
"type": "object"
}
}
}
software/erp5/instance-erp5-output-schema.json
View file @ 56ad14fb
......@@ -72,6 +72,11 @@
"description": "Jupyter notebook web UI access information",
"pattern": "^https://",
"type": "string"
},
"caucase-http-url": {
"description": "Caucase url on HTTP. For HTTPS URL, uses https scheme, if port is explicitely specified in http URL, take that port and add 1 and use it as https port. If it is not specified.",
"pattern": "^http://",
"type": "string"
}
},
"patternProperties": {
......
software/erp5testnode/software.cfg
View file @ 56ad14fb
......@@ -60,6 +60,6 @@ md5sum = 05519f3887a309d3ec069e0aa9f52ebc
[versions]
PyXML = 0.8.5
erp5.util = 0.4.48
slapos.recipe.template = 2.7
erp5.util = 0.4.49
slapos.recipe.template = 3.0
ipython = 5.3.0
software/erp5testnode/testsuite/deploy-test/README.rst 0 → 100644
View file @ 56ad14fb
deply-test
==========
Introduction
------------
This is software release to run tests on VMs, but without direct access (like
ssh) to the VM.
It is supposed to be used as backend for ERP5TestNode.
Characteristics and limitations:
* partitions share the same user (as they access files directly)
* this SR will be installed and instantiated only from local file system
(like git clone)
software/erp5testnode/testsuite/deploy-test/deploy-script-controller 0 → 100644
View file @ 56ad14fb
#!/bin/bash
# Script for controlling deploy script
#
# Assumption: deployment script returning with code 0
# run correctly and everything is done
#
# This script is run inside of KVM, by passing its
# URL with bootstrap-script-url
# The script configuration happens by passing
# it in text form with data-to-vm
#
# Format of data-to-vm is shell script:
# URL=<url>\nWAITTIME=<seconds>\nTRIES=<amount>
#
# Expected values in configuration:
# * URL - the url of the script to download and test
# * WAITTIME - waiting time, before next try
# * TRIES - amount of tries
# Possible TODOs:
# * post results on each try
# * use function + trap to assure posting on exit
LOG_FILE=/var/log/test-script-deployment.log
wget -O /tmp/test-script.cfg.$$ -q http://10.0.2.100/data
source /tmp/test-script.cfg.$$
if [ -z "$LOG_FILE" ] ; then
echo "Output log file is missing"
exit 1
fi
if [ -z "$URL" ] ; then
echo "URL is missing" >> $LOG_FILE 2>&1
exit 1
fi
if [ -z "$WAITTIME" ] ; then
echo "WAITTIME missing" >> $LOG_FILE 2>&1
exit 1
fi
if [ -z "$TRIES" ] ; then
echo "TRIES missing" >> $LOG_FILE 2>&1
exit 1
fi
DEPLOYMENT_SCRIPT=/tmp/test-script-deployment.bash.$$
wget -O $DEPLOYMENT_SCRIPT -q $URL
if [[ ! -s "$DEPLOYMENT_SCRIPT" ]] ; then
echo "exit 1" > $DEPLOYMENT_SCRIPT
fi
function upload ()
{
try=$1
LOG_FILE=$2
t=`date '+%Y%m%d%H%S'`
mv $LOG_FILE ${LOG_FILE}.$t
# just to be sure flush all disk operations before uploading
flush
curl -q -X POST --data-urlencode "path=test-script-result/log-file.log.$t" --data-urlencode "content@${LOG_FILE}.$t" http://10.0.2.100/
}
try=1
while true; do
echo "$0: Try $try. Running '/bin/bash $DEPLOYMENT_SCRIPT'" >> $LOG_FILE 2>&1
export TEST_YML_PATH
/bin/bash $DEPLOYMENT_SCRIPT >> $LOG_FILE 2>&1
result=$?
if [ $result == 0 ] ; then
echo "$0: Try $try. Script executed successfully." >> $LOG_FILE 2>&1
upload $try $LOG_FILE
break
fi
if (( try > TRIES )) ; then
echo "$0: Try $try. Amount of tries $TRIES exceeded, giving up." >> $LOG_FILE 2>&1
upload $try $LOG_FILE
break
fi
# wait WAITTIME before checking the state
echo "$0: Try $try. Sleeping $WAITTIME before retry." >> $LOG_FILE 2>&1
upload $try $LOG_FILE
sleep $WAITTIME
((try++))
done
exit $result
software/erp5testnode/testsuite/deploy-test/instance-deploy-test.cfg.jinja2 0 → 100644
View file @ 56ad14fb
{# Get test type, default to script-from-url, as defined in instance-input-schema.json -#}
{% set test_type = slapparameter_dict.get('test-type', 'script-from-url') -%}
{# Choose parameters according to test type -#}
{% if test_type == 'script-from-url' -%}
{% set script_url = slapparameter_dict.get('script-to-test-url') -%}
{% set test_yml_path = '/not/required' -%}
{% elif test_type == 'cloned-playbook' -%}
{% set script_url = 'http://10.0.2.100/standalone-local-playbook' -%}
{% set test_yml_path = slapparameter_dict.get('yml-path-to-test') -%}
{% else -%}
The test_type = "{{ test_type }}" is unsupported.
{% endif -%}
[buildout]
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
parts =
request-kvm
[request-kvm]
<= slap-connection
recipe = slapos.cookbook:request
software-url = ${slap-connection:software-release-url}
software-type = kvm
name = test-kvm
sla-computer_guid = ${slap-connection:computer-id}
# Tested image
# Passed by request
config-virtual-hard-drive-url = {{ slapparameter_dict.get('image-to-test-url') }}
config-virtual-hard-drive-md5sum = {{ slapparameter_dict.get('image-to-test-md5sum') }}
# The test script
config-bootstrap-script-url = {{ in_vm_test_script }}#{{ in_vm_test_script_md5 }}
# Script configuration
config-data-to-vm =
URL={{ script_url }}
WAITTIME={{ waittime }}
TRIES={{ tries }}
TEST_YML_PATH={{ test_yml_path }}
# require HTTP server
config-enable-http-server = true
# VM options
config-ram-size = 2048
config-cpu-count = 2
# await for system to be ready
return =
url
backend-url
ipv6
nat-rule-port-443
nat-rule-port-80
[directory]
recipe = slapos.cookbook:mkdirectory
home = ${buildout:directory}
etc = ${:home}/etc/
var = ${:home}/var/
srv = ${:home}/srv/
bin = ${:home}/bin/
tmp = ${:home}/tmp/
log = ${:var}/log/
services = ${:etc}/service/
scripts = ${:etc}/run/
software/erp5testnode/testsuite/deploy-test/instance-input-schema.json 0 → 100644
View file @ 56ad14fb
{
"type": "object",
"$schema": "http://json-schema.org/draft-04/schema",
"title": "Input Parameters",
"properties": {
"image-to-test-url": {
"title": "Image To Test (URL)",
"description": "Absolute URL to QCOW2 vm-bootstrap compatible image.",
"type": "string"
},
"image-to-test-md5sum": {
"title": "MD5 checksum of Image To Test.",
"description": "MD5 checksum of QCOW2 vm-bootstrap compatible image.",
"type": "string"
},
"test-relative-directory": {
"title": "Relative directory where tests are found.",
"description": "The base of this directory is [slapos-package].",
"type": "string"
},
"test-type": {
"title": "Type of the test.",
"type": "string",
"default": "script-from-url",
"enum" : ["script-from-url", "cloned-playbook"]
},
"script-to-test-url": {
"title": "Optional URL of script to test, used for test-type=script-from-url.",
"description": "Optional URL of script to test, which will be injected into the VM.\nThis parameter will NOT use local git clone of [slapos-package] part, it will test fully provided script.",
"type": "string"
},
"yml-path-to-test": {
"title": "Optional YML path to test from the playbook, used for test-type=cloned-playbook.",
"description": "Optional YML path to test from the playbook, provided as relative path to playbook root.\nThis parameter will use local git clone of [slapos-package] part, from which playbook will be constructed.",
"type": "string"
}
},
"required": ["image-to-test-url", "image-to-test-md5sum", "test-type"]
}
software/erp5testnode/testsuite/deploy-test/instance.cfg.in 0 → 100644
View file @ 56ad14fb
[buildout]
extends = ${template:output}
parts +=
runTestSuite
copy-test-data
[copy-test-data]
# Note: This is a hack, as such while requesting KVM it is not possible
# to fill correctly http directory
# Note: This will only work inside of non-user related environment
# dest shell variable will evaluate to requested partition's
# srv/public directory
# The "if" statement makes this script be run only in partition
# *without* srv/public - so according to current knowledge, the
# one which requests KVM backend for testing
recipe = plone.recipe.command
stop-on-error = true
update-command = $${:command}
command =
if [ ! -d srv/public ] ; then
dest=`echo ../*/srv/public`
cp ${playbook:output} $dest/playbook.tar.gz &&
cp ${standalone-local-playbook:location} $dest/standalone-local-playbook
fi
[runTestSuite]
recipe = slapos.recipe.template:jinja2
rendered = $${buildout:directory}/bin/$${:_buildout_section_name_}
template = inline:
#!/bin/sh
export PATH=${python-with-eggs:location}:$PATH
exec ${buildout:bin-directory}/${runTestSuite_py:interpreter} ${:_profile_base_location_}/runTestSuite.py --partition_ipv4 {{ list(partition_ipv4)[0] }} --partition_path $${buildout:directory} --test_reference "{{ slapparameter_dict.get('image-to-test-url') }} {{ slapparameter_dict.get('script-to-test-url')}}" --test_location "${test-location:base}/{{ slapparameter_dict.get('test-relative-directory')}}" "$@"
mode = 0755
context =
key slapparameter_dict slap-configuration:configuration
key partition_ipv4 slap-configuration:ipv4
[switch_softwaretype]
default = $${:deploy-test}
deploy-test = $${dynamic-template-deploy-test:rendered}
[dynamic-template-deploy-test]
recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/instance-deploy-test.cfg.jinja2
rendered = $${buildout:directory}/template-deploy-test.cfg
context =
key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory
key slapparameter_dict slap-configuration:configuration
raw bin_directory ${buildout:bin-directory}
raw in_vm_test_script ${deploy-script-controller-script:location}
raw in_vm_test_script_md5 ${deploy-script-controller-script:md5sum}
raw waittime ${deploy-script-controller-script:waittime}
raw tries ${deploy-script-controller-script:tries}
mode = 0644
software/erp5testnode/testsuite/deploy-test/runTestSuite.py 0 → 100644
View file @ 56ad14fb
from __future__ import print_function
import argparse
import os
import glob
from time import gmtime, strftime, time, sleep
from erp5.util import taskdistribution, testsuite
import logging
import sys
import tempfile
import json
SLEEP_TIME = 10
TRY_AMOUNT = 3600
def waitForSite(partition_path):
status_dict = {'command': 'file not found'}
# find test result, wait 10h
try_num = 1
start = time()
result_found = False
while 1:
finished = False
try_info = 'Try %s/%s: ' % (try_num, TRY_AMOUNT)
test_result_glob = os.path.join(
partition_path,
'..',
'*',
'srv',
'public',
'test-script-result',
)
print(try_info + 'Waiting for data in %r.' % (test_result_glob,))
result_list = glob.glob(test_result_glob)
if len(result_list) > 0:
result_path = result_list[0]
print(try_info + 'Data directory %r found, looking for results.' % (
result_path,))
result_file_list = list((
os.path.join(dirname, filename)
for dirname, dirnames, filenames in os.walk(result_path)
for filename in filenames
))
if len(result_file_list):
print(try_info + 'No result posted, will check next try.')
for result_file in result_file_list:
print(try_info + 'Data found.')
result_found = True
result_file = os.path.abspath(result_file)
status_dict['command'] = result_file
result = open(result_file).read()
# remove result, as it is not required anymore
os.unlink(result_file)
print(try_info + 'Analysis of result %r:' % (result_file,))
print(try_info + result)
status_dict['stderr'] = 'Last result:\n%s' % (result,)
if 'FATAL: all hosts have already failed -- aborting' in result:
# failed
status_dict.update(
success=False
)
finished = False
status_dict['stdout'] = try_info + 'Build not yet successful.'
print(try_info + '%r: Found not yet finished run.' % (result_file,))
elif "\"msg\": \"[u'Build successful, connect to:', u'" in result:
# success
status_dict.update(
success=True
)
finished = True
print(try_info + '%r: Found finished successful run.' % (
result_file,))
status_dict['stdout'] = try_info + 'Build successful.'
break
else:
# unknown
status_dict.update(
success=False
)
status_dict['stdout'] = \
try_info + 'Cannot find success nor failure result in the output'
print(try_info + '%r: Found unknown run.' % (result_file,))
finished = False
if finished:
break
if try_num >= TRY_AMOUNT:
msg = try_info + 'Time exceeded, success not found.'
print(msg)
status_dict.setdefault('stdout', '')
status_dict['stdout'] = '\n'.join([status_dict['stdout'], msg])
break
try_num += 1
print(try_info + 'Sleeping for %ss.' % (SLEEP_TIME,))
sleep(SLEEP_TIME)
if not result_found:
status_dict['stdout'] = try_info + 'Test timed out and no result found.'
status_dict.update(
success=False
)
end = time()
status_dict.update(
date=strftime("%Y/%m/%d %H:%M:%S", gmtime(end)),
duration=end - start,
)
return status_dict
def main():
logger = logging.getLogger()
logger.addHandler(logging.StreamHandler(sys.stdout))
logger.setLevel(logging.DEBUG)
parser = argparse.ArgumentParser(description='Run a test suite.')
parser.add_argument('--test_suite', help='The test suite name')
parser.add_argument('--test_suite_title', help='The test suite title')
parser.add_argument('--test_node_title', help='The test node title')
parser.add_argument('--project_title', help='The project title')
parser.add_argument('--revision', help='The revision to test',
default='dummy_revision')
parser.add_argument('--node_quantity', type=int,
help='Number of CPUs to use for the VM')
parser.add_argument('--master_url',
help='The Url of Master controlling test suites')
# SlapOS and deploy test specific
parser.add_argument(
'--partition_path',
help="Path of a partition",
default=os.path.abspath(os.getcwd()))
parser.add_argument(
'--test_reference',
help="Reference of the test",
default="missing"
)
parser.add_argument(
'--partition_ipv4',
help="IPv4 of a partition"
)
parser.add_argument(
'--test_location',
help="Location of the tests"
)
args = parser.parse_args()
revision = args.revision
test_suite_title = args.test_suite_title or args.test_suite
os.environ['SOURCE_CODE_TO_TEST'] = args.test_location
suite = testsuite.EggTestSuite(
1, test_suite=args.test_suite, node_quantity=args.node_quantity,
revision=revision)
access_url_http = None
access_url_https = None
if args.partition_ipv4:
access_url_http = 'http://%s:10080' % (args.partition_ipv4,)
access_url_https = 'https://%s:10443' % (args.partition_ipv4,)
os.environ['TEST_ACCESS_URL_HTTP'] = access_url_http
os.environ['TEST_ACCESS_URL_HTTPS'] = access_url_https
tool = taskdistribution.TaskDistributionTool(
args.master_url,
logger=logger)
test_result = tool.createTestResult(
revision, suite.getTestList(), args.test_node_title,
suite.allow_restart, test_suite_title, args.project_title)
if test_result is None:
return
# Create the site
status_dict = waitForSite(args.partition_path)
status_file = tempfile.NamedTemporaryFile()
status_file.write(json.dumps(status_dict))
status_file.flush()
os.fsync(status_file.fileno())
os.environ['TEST_SITE_STATUS_JSON'] = status_file.name
assert revision == test_result.revision, (revision, test_result.revision)
while suite.acquire():
test = test_result.start(suite.running.keys())
if test is not None:
suite.start(test.name, lambda status_dict,
__test=test: __test.stop(**status_dict))
elif not suite.running:
break
return
if __name__ == "__main__":
main()
software/erp5testnode/testsuite/deploy-test/software.cfg 0 → 100644
View file @ 56ad14fb
[buildout]
extends = ../../../kvm/software.cfg
parts =
eggs
template-deploy-test
runTestSuite_py
playbook
[runTestSuite_py]
recipe = zc.recipe.egg
eggs =
erp5.util
interpreter = ${:_buildout_section_name_}
[python-with-eggs]
recipe = plone.recipe.command
location = ${buildout:parts-directory}/${:_buildout_section_name_}
stop-on-error = true
command =
rm -fr ${:location} &&
mkdir -p ${:location} &&
ln -s ${buildout:bin-directory}/pythonwitheggs ${:location}/python &&
ln -s ${buildout:bin-directory}/pythonwitheggs ${:location}/python2.7
update-command = ${:command}
[playbook]
recipe = plone.recipe.command
stop-on-error = true
environment = export PATH=${tar:location}/bin:${gzip:location}/bin:$PATH
location = ${buildout:parts-directory}/${:_buildout_section_name_}
output = ${:location}/playbook.tar.gz
command =
${:environment}
rm -fr ${:location}
mkdir -p ${:location}
cd ${slapos-package:location}/playbook
tar czf ${:output} .
update-command = ${:command}
[test-location]
base = ${slapos-package:location}
[slapos-package]
recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.com/nexedi/slapos.package.git
branch = cdn-test
git-executable = ${git:location}/bin/git
[template]
output = ${buildout:directory}/template-original.kvm.cfg
[deploy-script-controller-script]
filename = deploy-script-controller
location = ${:_profile_base_location_}/${:filename}
md5sum = d2b92f45257a52e5a7ff5c311d46d4ae
# configuration
waittime = 360
tries = 80
[standalone-local-playbook]
filename = standalone-local-playbook
location = ${:_profile_base_location_}/${:filename}
[template-deploy-test]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg.in
output = ${buildout:directory}/template.cfg
mode = 0644
[versions]
erp5.util = 0.4.49
software/erp5testnode/testsuite/deploy-test/software.cfg.json 0 → 100644
View file @ 56ad14fb
{
"name": "Deploy Test",
"description": "Deploy Testing software release",
"serialisation": "xml",
"software-type": {
"default": {
"title": "Default",
"description": "Standard ERP5TestNode hooked",
"request": "instance-input-schema.json",
"index": 0
}
}
}
software/erp5testnode/testsuite/deploy-test/standalone-local-playbook 0 → 100644
View file @ 56ad14fb
#!/bin/bash
set -e
#
# Paste content of function-common
# https://lab.nexedi.com/nexedi/slapos.package/blob/master/playbook/roles/install-script/files/function-common
#
#!/bin/bash
#
# functions-common - Common functions used by DevStack components
#
# The canonical copy of this file is maintained in the DevStack repo.
# All modifications should be made there and then sync'ed to other repos
# as required.
#
# This file is sorted alphabetically within the function groups.
#
# - Config Functions
# - Control Functions
# - Distro Functions
# - Git Functions
# - OpenStack Functions
# - Package Functions
# - Process Functions
# - Service Functions
# - System Functions
#
# The following variables are assumed to be defined by certain functions:
#
# - ``ENABLED_SERVICES``
# - ``ERROR_ON_CLONE``
# - ``FILES``
# - ``OFFLINE``
# - ``RECLONE``
# - ``REQUIREMENTS_DIR``
# - ``STACK_USER``
# - ``TRACK_DEPENDS``
# - ``UNDO_REQUIREMENTS``
# - ``http_proxy``, ``https_proxy``, ``no_proxy``
#
# Save trace setting
XTRACE=$(set +o | grep xtrace)
set +o xtrace
# Global Config Variables
declare -A GITREPO
declare -A GITBRANCH
declare -A GITDIR
TRACK_DEPENDS=${TRACK_DEPENDS:-False}
# Normalize config values to True or False
# Accepts as False: 0 no No NO false False FALSE
# Accepts as True: 1 yes Yes YES true True TRUE
# VAR=$(trueorfalse default-value test-value)
function trueorfalse {
local xtrace=$(set +o | grep xtrace)
set +o xtrace
local default=$1
local literal=$2
local testval=${!literal:-}
[[ -z "$testval" ]] && { echo "$default"; return; }
[[ "0 no No NO false False FALSE" =~ "$testval" ]] && { echo "False"; return; }
[[ "1 yes Yes YES true True TRUE" =~ "$testval" ]] && { echo "True"; return; }
echo "$default"
$xtrace
}
#function isset {
# [[ -v "$1" ]]
#}
#
# Control Functions
# =================
# Prints backtrace info
# filename:lineno:function
# backtrace level
function backtrace {
local level=$1
local deep=$((${#BASH_SOURCE[@]} - 1))
echo "[Call Trace]"
while [ $level -le $deep ]; do
echo "${BASH_SOURCE[$deep]}:${BASH_LINENO[$deep-1]}:${FUNCNAME[$deep-1]}"
deep=$((deep - 1))
done
}
# Prints line number and "message" then exits
# die $LINENO "message"
function die {
local exitcode=$?
set +o xtrace
local line=$1; shift
if [ $exitcode == 0 ]; then
exitcode=1
fi
backtrace 2
err $line "$*"
# Give buffers a second to flush
sleep 1
exit $exitcode
}
# Checks an environment variable is not set or has length 0 OR if the
# exit code is non-zero and prints "message" and exits
# NOTE: env-var is the variable name without a '$'
# die_if_not_set $LINENO env-var "message"
function die_if_not_set {
local exitcode=$?
local xtrace=$(set +o | grep xtrace)
set +o xtrace
local line=$1; shift
local evar=$1; shift
if ! is_set $evar || [ $exitcode != 0 ]; then
die $line "$*"
fi
$xtrace
}
# Prints line number and "message" in error format
# err $LINENO "message"
function err {
local exitcode=$?
local xtrace=$(set +o | grep xtrace)
set +o xtrace
local msg="[ERROR] ${BASH_SOURCE[2]}:$1 $2"
echo $msg 1>&2;
if [[ -n ${LOGDIR} ]]; then
echo $msg >> "${LOGDIR}/error.log"
fi
$xtrace
return $exitcode
}
# Checks an environment variable is not set or has length 0 OR if the
# exit code is non-zero and prints "message"
# NOTE: env-var is the variable name without a '$'
# err_if_not_set $LINENO env-var "message"
function err_if_not_set {
local exitcode=$?
local xtrace=$(set +o | grep xtrace)
set +o xtrace
local line=$1; shift
local evar=$1; shift
if ! is_set $evar || [ $exitcode != 0 ]; then
err $line "$*"
fi
$xtrace
return $exitcode
}
# Exit after outputting a message about the distribution not being supported.
# exit_distro_not_supported [optional-string-telling-what-is-missing]
function exit_distro_not_supported {
if [[ -z "$DISTRO" ]]; then
GetDistro
fi
if [ $# -gt 0 ]; then
die $LINENO "Support for $DISTRO is incomplete: no support for $@"
else
die $LINENO "Support for $DISTRO is incomplete."
fi
}
# Test if the named environment variable is set and not zero length
# is_set env-var
function is_set {
local var=\$"$1"
eval "[ -n \"$var\" ]" # For ex.: sh -c "[ -n \"$var\" ]" would be better, but several exercises depends on this
}
# Prints line number and "message" in warning format
# warn $LINENO "message"
function warn {
local exitcode=$?
local xtrace=$(set +o | grep xtrace)
set +o xtrace
local msg="[WARNING] ${BASH_SOURCE[2]}:$1 $2"
echo $msg 1>&2;
if [[ -n ${LOGDIR} ]]; then
echo $msg >> "${LOGDIR}/error.log"
fi
$xtrace
return $exitcode
}
# Distro Functions
# ================
# Determine OS Vendor, Release and Update
# Tested with OS/X, Ubuntu, RedHat, CentOS, Fedora
# Returns results in global variables:
# ``os_VENDOR`` - vendor name: ``Ubuntu``, ``Fedora``, etc
# ``os_RELEASE`` - major release: ``14.04`` (Ubuntu), ``20`` (Fedora)
# ``os_UPDATE`` - update: ex. the ``5`` in ``RHEL6.5``
# ``os_PACKAGE`` - package type: ``deb`` or ``rpm``
# ``os_CODENAME`` - vendor's codename for release: ``snow leopard``, ``trusty``
os_VENDOR=""
os_RELEASE=""
os_UPDATE=""
os_PACKAGE=""
os_CODENAME=""
# GetOSVersion
function GetOSVersion {
# Figure out which vendor we are
if [[ -x "`which sw_vers 2>/dev/null`" ]]; then
# OS/X
os_VENDOR=`sw_vers -productName`
os_RELEASE=`sw_vers -productVersion`
os_UPDATE=${os_RELEASE##*.}
os_RELEASE=${os_RELEASE%.*}
os_PACKAGE=""
if [[ "$os_RELEASE" =~ "10.7" ]]; then
os_CODENAME="lion"
elif [[ "$os_RELEASE" =~ "10.6" ]]; then
os_CODENAME="snow leopard"
elif [[ "$os_RELEASE" =~ "10.5" ]]; then
os_CODENAME="leopard"
elif [[ "$os_RELEASE" =~ "10.4" ]]; then
os_CODENAME="tiger"
elif [[ "$os_RELEASE" =~ "10.3" ]]; then
os_CODENAME="panther"
else
os_CODENAME=""
fi
elif [[ -x $(which lsb_release 2>/dev/null) ]]; then
os_VENDOR=$(lsb_release -i -s)
os_RELEASE=$(lsb_release -r -s)
os_UPDATE=""
os_PACKAGE="rpm"
if [[ "Debian,Ubuntu,LinuxMint" =~ $os_VENDOR ]]; then
os_PACKAGE="deb"
elif [[ "SUSE LINUX" =~ $os_VENDOR ]]; then
lsb_release -d -s | grep -q openSUSE
if [[ $? -eq 0 ]]; then
os_VENDOR="openSUSE"
fi
elif [[ $os_VENDOR == "openSUSE project" ]]; then
os_VENDOR="openSUSE"
elif [[ $os_VENDOR =~ Red.*Hat ]]; then
os_VENDOR="Red Hat"
fi
os_CODENAME=$(lsb_release -c -s)
elif [[ -r /etc/redhat-release ]]; then
# Red Hat Enterprise Linux Server release 5.5 (Tikanga)
# Red Hat Enterprise Linux Server release 7.0 Beta (Maipo)
# CentOS release 5.5 (Final)
# CentOS Linux release 6.0 (Final)
# Fedora release 16 (Verne)
# XenServer release 6.2.0-70446c (xenenterprise)
# Oracle Linux release 7
os_CODENAME=""
for r in "Red Hat" CentOS Fedora XenServer; do
os_VENDOR=$r
if [[ -n "`grep \"$r\" /etc/redhat-release`" ]]; then
ver=`sed -e 's/^.* \([0-9].*\) (\(.*\)).*$/\1\|\2/' /etc/redhat-release`
os_CODENAME=${ver#*|}
os_RELEASE=${ver%|*}
os_UPDATE=${os_RELEASE##*.}
os_RELEASE=${os_RELEASE%.*}
break
fi
os_VENDOR=""
done
if [ "$os_VENDOR" = "Red Hat" ] && [[ -r /etc/oracle-release ]]; then
os_VENDOR=OracleLinux
fi
os_PACKAGE="rpm"
elif [[ -r /etc/SuSE-release ]]; then
for r in openSUSE "SUSE Linux"; do
if [[ "$r" = "SUSE Linux" ]]; then
os_VENDOR="SUSE LINUX"
else
os_VENDOR=$r
fi
if [[ -n "`grep \"$r\" /etc/SuSE-release`" ]]; then
os_CODENAME=`grep "CODENAME = " /etc/SuSE-release | sed 's:.* = ::g'`
os_RELEASE=`grep "VERSION = " /etc/SuSE-release | sed 's:.* = ::g'`
os_UPDATE=`grep "PATCHLEVEL = " /etc/SuSE-release | sed 's:.* = ::g'`
break
fi
os_VENDOR=""
done
os_PACKAGE="rpm"
# If lsb_release is not installed, we should be able to detect Debian OS
elif [[ -f /etc/debian_version ]] && [[ $(cat /proc/version) =~ "Debian" ]]; then
os_VENDOR="Debian"
os_PACKAGE="deb"
os_CODENAME=$(awk '/VERSION=/' /etc/os-release | sed 's/VERSION=//' | sed -r 's/\"|\(|\)//g' | awk '{print $2}')
os_RELEASE=$(awk '/VERSION_ID=/' /etc/os-release | sed 's/VERSION_ID=//' | sed 's/\"//g')
fi
export os_VENDOR os_RELEASE os_UPDATE os_PACKAGE os_CODENAME
}
# Translate the OS version values into common nomenclature
# Sets global ``DISTRO`` from the ``os_*`` values
declare DISTRO
function GetDistro {
GetOSVersion
if [[ "$os_VENDOR" =~ (Ubuntu) || "$os_VENDOR" =~ (Debian) ]]; then
# 'Everyone' refers to Ubuntu / Debian releases by the code name adjective
DISTRO=$os_CODENAME
elif [[ "$os_VENDOR" =~ (Fedora) ]]; then
# For Fedora, just use 'f' and the release
DISTRO="f$os_RELEASE"
elif [[ "$os_VENDOR" =~ (openSUSE) ]]; then
DISTRO="opensuse-$os_RELEASE"
elif [[ "$os_VENDOR" =~ (SUSE LINUX) ]]; then
# For SLE, also use the service pack
if [[ -z "$os_UPDATE" ]]; then
DISTRO="sle${os_RELEASE}"
else
DISTRO="sle${os_RELEASE}sp${os_UPDATE}"
fi
elif [[ "$os_VENDOR" =~ (Red Hat) || \
"$os_VENDOR" =~ (CentOS) || \
"$os_VENDOR" =~ (OracleLinux) ]]; then
# Drop the . release as we assume it's compatible
DISTRO="rhel${os_RELEASE::1}"
elif [[ "$os_VENDOR" =~ (XenServer) ]]; then
DISTRO="xs$os_RELEASE"
else
# Catch-all for now is Vendor + Release + Update
DISTRO="$os_VENDOR-$os_RELEASE.$os_UPDATE"
fi
export DISTRO
}
# Utility function for checking machine architecture
# is_arch arch-type
function is_arch {
[[ "$(uname -m)" == "$1" ]]
}
# Determine if current distribution is an Oracle distribution
# is_oraclelinux
function is_oraclelinux {
if [[ -z "$os_VENDOR" ]]; then
GetOSVersion
fi
[ "$os_VENDOR" = "OracleLinux" ]
}
# Determine if current distribution is a Fedora-based distribution
# (Fedora, RHEL, CentOS, etc).
# is_fedora
function is_fedora {
if [[ -z "$os_VENDOR" ]]; then
GetOSVersion
fi
[ "$os_VENDOR" = "Fedora" ] || [ "$os_VENDOR" = "Red Hat" ] || \
[ "$os_VENDOR" = "CentOS" ] || [ "$os_VENDOR" = "OracleLinux" ]
}
# Determine if current distribution is a SUSE-based distribution
# (openSUSE, SLE).
# is_suse
function is_suse {
if [[ -z "$os_VENDOR" ]]; then
GetOSVersion
fi
[ "$os_VENDOR" = "openSUSE" ] || [ "$os_VENDOR" = "SUSE LINUX" ]
}
# Determine if current distribution is an Ubuntu-based distribution
# It will also detect non-Ubuntu but Debian-based distros
# is_ubuntu
function is_ubuntu {
if [[ -z "$os_PACKAGE" ]]; then
GetOSVersion
fi
[ "$os_PACKAGE" = "deb" ]
}
# Git Functions
# =============
# Returns openstack release name for a given branch name
# ``get_release_name_from_branch branch-name``
function get_release_name_from_branch {
local branch=$1
if [[ $branch =~ "stable/" || $branch =~ "proposed/" ]]; then
echo ${branch#*/}
else
echo "master"
fi
}
# git clone only if directory doesn't exist already. Since ``DEST`` might not
# be owned by the installation user, we create the directory and change the
# ownership to the proper user.
# Set global ``RECLONE=yes`` to simulate a clone when dest-dir exists
# Set global ``ERROR_ON_CLONE=True`` to abort execution with an error if the git repo
# does not exist (default is False, meaning the repo will be cloned).
# Uses globals ``ERROR_ON_CLONE``, ``OFFLINE``, ``RECLONE``
# git_clone remote dest-dir branch
function git_clone {
local git_remote=$1
local git_dest=$2
local git_ref=$3
local orig_dir=$(pwd)
local git_clone_flags=""
RECLONE=$(trueorfalse False RECLONE)
if [[ "${GIT_DEPTH}" -gt 0 ]]; then
git_clone_flags="$git_clone_flags --depth $GIT_DEPTH"
fi
if [[ "$OFFLINE" = "True" ]]; then
echo "Running in offline mode, clones already exist"
# print out the results so we know what change was used in the logs
cd $git_dest
git show --oneline | head -1
cd $orig_dir
return
fi
if echo $git_ref | egrep -q "^refs"; then
# If our branch name is a gerrit style refs/changes/...
if [[ ! -d $git_dest ]]; then
[[ "$ERROR_ON_CLONE" = "True" ]] && \
die $LINENO "Cloning not allowed in this configuration"
git_timed clone $git_clone_flags $git_remote $git_dest
fi
cd $git_dest
git_timed fetch $git_remote $git_ref && git checkout FETCH_HEAD
else
# do a full clone only if the directory doesn't exist
if [[ ! -d $git_dest ]]; then
[[ "$ERROR_ON_CLONE" = "True" ]] && \
die $LINENO "Cloning not allowed in this configuration"
git_timed clone $git_clone_flags $git_remote $git_dest
cd $git_dest
# This checkout syntax works for both branches and tags
git checkout $git_ref
elif [[ "$RECLONE" = "True" ]]; then
# if it does exist then simulate what clone does if asked to RECLONE
cd $git_dest
# set the url to pull from and fetch
git remote set-url origin $git_remote
git_timed fetch origin
# remove the existing ignored files (like pyc) as they cause breakage
# (due to the py files having older timestamps than our pyc, so python
# thinks the pyc files are correct using them)
find $git_dest -name '*.pyc' -delete
# handle git_ref accordingly to type (tag, branch)
if [[ -n "`git show-ref refs/tags/$git_ref`" ]]; then
git_update_tag $git_ref
elif [[ -n "`git show-ref refs/heads/$git_ref`" ]]; then
git_update_branch $git_ref
elif [[ -n "`git show-ref refs/remotes/origin/$git_ref`" ]]; then
git_update_remote_branch $git_ref
else
die $LINENO "$git_ref is neither branch nor tag"
fi
fi
fi
# print out the results so we know what change was used in the logs
cd $git_dest
git show --oneline | head -1
cd $orig_dir
}
# A variation on git clone that lets us specify a project by it's
# actual name, like oslo.config. This is exceptionally useful in the
# library installation case
function git_clone_by_name {
local name=$1
local repo=${GITREPO[$name]}
local dir=${GITDIR[$name]}
local branch=${GITBRANCH[$name]}
git_clone $repo $dir $branch
}
# git can sometimes get itself infinitely stuck with transient network
# errors or other issues with the remote end. This wraps git in a
# timeout/retry loop and is intended to watch over non-local git
# processes that might hang. GIT_TIMEOUT, if set, is passed directly
# to timeout(1); otherwise the default value of 0 maintains the status
# quo of waiting forever.
# usage: git_timed <git-command>
function git_timed {
local count=0
local timeout=0
if [[ -n "${GIT_TIMEOUT}" ]]; then
timeout=${GIT_TIMEOUT}
fi
until timeout -s SIGINT ${timeout} git "$@"; do
# 124 is timeout(1)'s special return code when it reached the
# timeout; otherwise assume fatal failure
if [[ $? -ne 124 ]]; then
die $LINENO "git call failed: [git $@]"
fi
count=$(($count + 1))
warn "timeout ${count} for git call: [git $@]"
if [ $count -eq 3 ]; then
die $LINENO "Maximum of 3 git retries reached"
fi
sleep 5
done
}
# git update using reference as a branch.
# git_update_branch ref
function git_update_branch {
local git_branch=$1
git checkout -f origin/$git_branch
# a local branch might not exist
git branch -D $git_branch || true
git checkout -b $git_branch
}
# git update using reference as a branch.
# git_update_remote_branch ref
function git_update_remote_branch {
local git_branch=$1
git checkout -b $git_branch -t origin/$git_branch
}
# git update using reference as a tag. Be careful editing source at that repo
# as working copy will be in a detached mode
# git_update_tag ref
function git_update_tag {
local git_tag=$1
git tag -d $git_tag
# fetching given tag only
git_timed fetch origin tag $git_tag
git checkout -f $git_tag
}
# OpenStack Functions
# ===================
# Get the default value for HOST_IP
# get_default_host_ip fixed_range floating_range host_ip_iface host_ip
function get_default_host_ip {
local fixed_range=$1
local floating_range=$2
local host_ip_iface=$3
local host_ip=$4
# Search for an IP unless an explicit is set by ``HOST_IP`` environment variable
if [ -z "$host_ip" -o "$host_ip" == "dhcp" ]; then
host_ip=""
# Find the interface used for the default route
host_ip_iface=${host_ip_iface:-$(ip route | awk '/default/ {print $5}' | head -1)}
local host_ips=$(LC_ALL=C ip -f inet addr show ${host_ip_iface} | awk '/inet/ {split($2,parts,"/"); print parts[1]}')
local ip
for ip in $host_ips; do
# Attempt to filter out IP addresses that are part of the fixed and
# floating range. Note that this method only works if the ``netaddr``
# python library is installed. If it is not installed, an error
# will be printed and the first IP from the interface will be used.
# If that is not correct set ``HOST_IP`` in ``localrc`` to the correct
# address.
if ! (address_in_net $ip $fixed_range || address_in_net $ip $floating_range); then
host_ip=$ip
break;
fi
done
fi
echo $host_ip
}
# Generates hex string from ``size`` byte of pseudo random data
# generate_hex_string size
function generate_hex_string {
local size=$1
hexdump -n "$size" -v -e '/1 "%02x"' /dev/urandom
}
# Grab a numbered field from python prettytable output
# Fields are numbered starting with 1
# Reverse syntax is supported: -1 is the last field, -2 is second to last, etc.
# get_field field-number
function get_field {
local data field
while read data; do
if [ "$1" -lt 0 ]; then
field="(\$(NF$1))"
else
field="\$$(($1 + 1))"
fi
echo "$data" | awk -F'[ \t]*\\|[ \t]*' "{print $field}"
done
}
# install default policy
# copy over a default policy.json and policy.d for projects
function install_default_policy {
local project=$1
local project_uc=$(echo $1|tr a-z A-Z)
local conf_dir="${project_uc}_CONF_DIR"
# eval conf dir to get the variable
conf_dir="${!conf_dir}"
local project_dir="${project_uc}_DIR"
# eval project dir to get the variable
project_dir="${!project_dir}"
local sample_conf_dir="${project_dir}/etc/${project}"
local sample_policy_dir="${project_dir}/etc/${project}/policy.d"
# first copy any policy.json
cp -p $sample_conf_dir/policy.json $conf_dir
# then optionally copy over policy.d
if [[ -d $sample_policy_dir ]]; then
cp -r $sample_policy_dir $conf_dir/policy.d
fi
}
# Add a policy to a policy.json file
# Do nothing if the policy already exists
# ``policy_add policy_file policy_name policy_permissions``
function policy_add {
local policy_file=$1
local policy_name=$2
local policy_perm=$3
if grep -q ${policy_name} ${policy_file}; then
echo "Policy ${policy_name} already exists in ${policy_file}"
return
fi
# Add a terminating comma to policy lines without one
# Remove the closing '}' and all lines following to the end-of-file
local tmpfile=$(mktemp)
uniq ${policy_file} | sed -e '
s/]$/],/
/^[}]/,$d
' > ${tmpfile}
# Append policy and closing brace
echo " \"${policy_name}\": ${policy_perm}" >>${tmpfile}
echo "}" >>${tmpfile}
mv ${tmpfile} ${policy_file}
}
# Gets or creates a domain
# Usage: get_or_create_domain <name> <description>
function get_or_create_domain {
local os_url="$KEYSTONE_SERVICE_URI_V3"
# Gets domain id
local domain_id=$(
# Gets domain id
openstack --os-token=$OS_TOKEN --os-url=$os_url \
--os-identity-api-version=3 domain show $1 \
-f value -c id 2>/dev/null ||
# Creates new domain
openstack --os-token=$OS_TOKEN --os-url=$os_url \
--os-identity-api-version=3 domain create $1 \
--description "$2" \
-f value -c id
)
echo $domain_id
}
# Gets or creates group
# Usage: get_or_create_group <groupname> [<domain> <description>]
function get_or_create_group {
local domain=${2:+--domain ${2}}
local desc="${3:-}"
local os_url="$KEYSTONE_SERVICE_URI_V3"
# Gets group id
local group_id=$(
# Creates new group with --or-show
openstack --os-token=$OS_TOKEN --os-url=$os_url \
--os-identity-api-version=3 group create $1 \
$domain --description "$desc" --or-show \
-f value -c id
)
echo $group_id
}
# Gets or creates user
# Usage: get_or_create_user <username> <password> [<email> [<domain>]]
function get_or_create_user {
if [[ ! -z "$3" ]]; then
local email="--email=$3"
else
local email=""
fi
local os_cmd="openstack"
local domain=""
if [[ ! -z "$4" ]]; then
domain="--domain=$4"
os_cmd="$os_cmd --os-url=$KEYSTONE_SERVICE_URI_V3 --os-identity-api-version=3"
fi
# Gets user id
local user_id=$(
# Creates new user with --or-show
$os_cmd user create \
$1 \
--password "$2" \
$email \
$domain \
--or-show \
-f value -c id
)
echo $user_id
}
# Gets or creates project
# Usage: get_or_create_project <name> [<domain>]
function get_or_create_project {
# Gets project id
local os_cmd="openstack"
local domain=""
if [[ ! -z "$2" ]]; then
domain="--domain=$2"
os_cmd="$os_cmd --os-url=$KEYSTONE_SERVICE_URI_V3 --os-identity-api-version=3"
fi
local project_id=$(
# Creates new project with --or-show
$os_cmd project create $1 $domain --or-show -f value -c id
)
echo $project_id
}
# Gets or creates role
# Usage: get_or_create_role <name>
function get_or_create_role {
local role_id=$(
# Creates role with --or-show
openstack role create $1 --or-show -f value -c id
)
echo $role_id
}
# Gets or adds user role to project
# Usage: get_or_add_user_project_role <role> <user> <project>
function get_or_add_user_project_role {
# Gets user role id
local user_role_id=$(openstack role list \
--user $2 \
--project $3 \
--column "ID" \
--column "Name" \
| grep " $1 " | get_field 1)
if [[ -z "$user_role_id" ]]; then
# Adds role to user
user_role_id=$(openstack role add \
$1 \
--user $2 \
--project $3 \
| grep " id " | get_field 2)
fi
echo $user_role_id
}
# Gets or adds group role to project
# Usage: get_or_add_group_project_role <role> <group> <project>
function get_or_add_group_project_role {
# Gets group role id
local group_role_id=$(openstack role list \
--group $2 \
--project $3 \
--column "ID" \
--column "Name" \
| grep " $1 " | get_field 1)
if [[ -z "$group_role_id" ]]; then
# Adds role to group
group_role_id=$(openstack role add \
$1 \
--group $2 \
--project $3 \
| grep " id " | get_field 2)
fi
echo $group_role_id
}
# Gets or creates service
# Usage: get_or_create_service <name> <type> <description>
function get_or_create_service {
# Gets service id
local service_id=$(
# Gets service id
openstack service show $1 -f value -c id 2>/dev/null ||
# Creates new service if not exists
openstack service create \
$2 \
--name $1 \
--description="$3" \
-f value -c id
)
echo $service_id
}
# Gets or creates endpoint
# Usage: get_or_create_endpoint <service> <region> <publicurl> <adminurl> <internalurl>
function get_or_create_endpoint {
# Gets endpoint id
local endpoint_id=$(openstack endpoint list \
--column "ID" \
--column "Region" \
--column "Service Name" \
| grep " $2 " \
| grep " $1 " | get_field 1)
if [[ -z "$endpoint_id" ]]; then
# Creates new endpoint
endpoint_id=$(openstack endpoint create \
$1 \
--region $2 \
--publicurl $3 \
--adminurl $4 \
--internalurl $5 \
| grep " id " | get_field 2)
fi
echo $endpoint_id
}
# Package Functions
# =================
# _get_package_dir
function _get_package_dir {
local base_dir=$1
local pkg_dir
if [[ -z "$base_dir" ]]; then
base_dir=$FILES
fi
if is_ubuntu; then
pkg_dir=$base_dir/debs
elif is_fedora; then
pkg_dir=$base_dir/rpms
elif is_suse; then
pkg_dir=$base_dir/rpms-suse
else
exit_distro_not_supported "list of packages"
fi
echo "$pkg_dir"
}
# Wrapper for ``apt-get`` to set cache and proxy environment variables
# Uses globals ``OFFLINE``, ``*_proxy``
# apt_get operation package [package ...]
function apt_get {
local xtrace=$(set +o | grep xtrace)
set +o xtrace
[[ "$OFFLINE" = "True" || -z "$@" ]] && return
local sudo="sudo"
[[ "$(id -u)" = "0" ]] && sudo="env"
$xtrace
$sudo DEBIAN_FRONTEND=noninteractive \
http_proxy=${http_proxy:-} https_proxy=${https_proxy:-} \
no_proxy=${no_proxy:-} \
apt-get --option "Dpkg::Options::=--force-confold" --assume-yes "$@"
}
function _parse_package_files {
local files_to_parse=$@
if [[ -z "$DISTRO" ]]; then
GetDistro
fi
for fname in ${files_to_parse}; do
local OIFS line package distros distro
[[ -e $fname ]] || continue
OIFS=$IFS
IFS=$'\n'
for line in $(<${fname}); do
if [[ $line =~ "NOPRIME" ]]; then
continue
fi
# Assume we want this package
package=${line%#*}
inst_pkg=1
# Look for # dist:xxx in comment
if [[ $line =~ (.*)#.*dist:([^ ]*) ]]; then
# We are using BASH regexp matching feature.
package=${BASH_REMATCH[1]}
distros=${BASH_REMATCH[2]}
# In bash ${VAR,,} will lowecase VAR
# Look for a match in the distro list
if [[ ! ${distros,,} =~ ${DISTRO,,} ]]; then
# If no match then skip this package
inst_pkg=0
fi
fi
if [[ $inst_pkg = 1 ]]; then
echo $package
fi
done
IFS=$OIFS
done
}
# get_packages() collects a list of package names of any type from the
# prerequisite files in ``files/{debs|rpms}``. The list is intended
# to be passed to a package installer such as apt or yum.
#
# Only packages required for the services in 1st argument will be
# included. Two bits of metadata are recognized in the prerequisite files:
#
# - ``# NOPRIME`` defers installation to be performed later in `stack.sh`
# - ``# dist:DISTRO`` or ``dist:DISTRO1,DISTRO2`` limits the selection
# of the package to the distros listed. The distro names are case insensitive.
function get_packages {
local xtrace=$(set +o | grep xtrace)
set +o xtrace
local services=$@
local package_dir=$(_get_package_dir)
local file_to_parse=""
local service=""
if [[ -z "$package_dir" ]]; then
echo "No package directory supplied"
return 1
fi
for service in ${services//,/ }; do
# Allow individual services to specify dependencies
if [[ -e ${package_dir}/${service} ]]; then
file_to_parse="${file_to_parse} ${package_dir}/${service}"
fi
# NOTE(sdague) n-api needs glance for now because that's where
# glance client is
if [[ $service == n-api ]]; then
if [[ ! $file_to_parse =~ $package_dir/nova ]]; then
file_to_parse="${file_to_parse} ${package_dir}/nova"
fi
if [[ ! $file_to_parse =~ $package_dir/glance ]]; then
file_to_parse="${file_to_parse} ${package_dir}/glance"
fi
elif [[ $service == c-* ]]; then
if [[ ! $file_to_parse =~ $package_dir/cinder ]]; then
file_to_parse="${file_to_parse} ${package_dir}/cinder"
fi
elif [[ $service == ceilometer-* ]]; then
if [[ ! $file_to_parse =~ $package_dir/ceilometer ]]; then
file_to_parse="${file_to_parse} ${package_dir}/ceilometer"
fi
elif [[ $service == s-* ]]; then
if [[ ! $file_to_parse =~ $package_dir/swift ]]; then
file_to_parse="${file_to_parse} ${package_dir}/swift"
fi
elif [[ $service == n-* ]]; then
if [[ ! $file_to_parse =~ $package_dir/nova ]]; then
file_to_parse="${file_to_parse} ${package_dir}/nova"
fi
elif [[ $service == g-* ]]; then
if [[ ! $file_to_parse =~ $package_dir/glance ]]; then
file_to_parse="${file_to_parse} ${package_dir}/glance"
fi
elif [[ $service == key* ]]; then
if [[ ! $file_to_parse =~ $package_dir/keystone ]]; then
file_to_parse="${file_to_parse} ${package_dir}/keystone"
fi
elif [[ $service == q-* ]]; then
if [[ ! $file_to_parse =~ $package_dir/neutron ]]; then
file_to_parse="${file_to_parse} ${package_dir}/neutron"
fi
elif [[ $service == ir-* ]]; then
if [[ ! $file_to_parse =~ $package_dir/ironic ]]; then
file_to_parse="${file_to_parse} ${package_dir}/ironic"
fi
fi
done
echo "$(_parse_package_files $file_to_parse)"
$xtrace
}
# get_plugin_packages() collects a list of package names of any type from a
# plugin's prerequisite files in ``$PLUGIN/devstack/files/{debs|rpms}``. The
# list is intended to be passed to a package installer such as apt or yum.
#
# Only packages required for enabled and collected plugins will included.
#
# The same metadata used in the main DevStack prerequisite files may be used
# in these prerequisite files, see get_packages() for more info.
function get_plugin_packages {
local xtrace=$(set +o | grep xtrace)
set +o xtrace
local files_to_parse=""
local package_dir=""
for plugin in ${DEVSTACK_PLUGINS//,/ }; do
local package_dir="$(_get_package_dir ${GITDIR[$plugin]}/devstack/files)"
files_to_parse+="$package_dir/$plugin"
done
echo "$(_parse_package_files $files_to_parse)"
$xtrace
}
# Distro-agnostic package installer
# Uses globals ``NO_UPDATE_REPOS``, ``REPOS_UPDATED``, ``RETRY_UPDATE``
# install_package package [package ...]
function update_package_repo {
NO_UPDATE_REPOS=${NO_UPDATE_REPOS:-False}
REPOS_UPDATED=${REPOS_UPDATED:-False}
RETRY_UPDATE=${RETRY_UPDATE:-False}
if [[ "$NO_UPDATE_REPOS" = "True" ]]; then
return 0
fi
if is_ubuntu; then
local xtrace=$(set +o | grep xtrace)
set +o xtrace
if [[ "$REPOS_UPDATED" != "True" || "$RETRY_UPDATE" = "True" ]]; then
# if there are transient errors pulling the updates, that's fine.
# It may be secondary repositories that we don't really care about.
apt_get update || /bin/true
REPOS_UPDATED=True
fi
$xtrace
fi
}
function real_install_package {
if is_ubuntu; then
apt_get install "$@"
elif is_fedora; then
yum_install "$@"
elif is_suse; then
zypper_install "$@"
else
exit_distro_not_supported "installing packages"
fi
}
# Distro-agnostic package installer
# install_package package [package ...]
function install_package {
update_package_repo
real_install_package $@ || RETRY_UPDATE=True update_package_repo && real_install_package $@
}
# Distro-agnostic function to tell if a package is installed
# is_package_installed package [package ...]
function is_package_installed {
if [[ -z "$@" ]]; then
return 1
fi
if [[ -z "$os_PACKAGE" ]]; then
GetOSVersion
fi
if [[ "$os_PACKAGE" = "deb" ]]; then
dpkg -s "$@" > /dev/null 2> /dev/null
elif [[ "$os_PACKAGE" = "rpm" ]]; then
rpm --quiet -q "$@"
else
exit_distro_not_supported "finding if a package is installed"
fi
}
# Distro-agnostic package uninstaller
# uninstall_package package [package ...]
function uninstall_package {
if is_ubuntu; then
apt_get purge "$@"
elif is_fedora; then
sudo ${YUM:-yum} remove -y "$@" ||:
elif is_suse; then
sudo zypper rm "$@"
else
exit_distro_not_supported "uninstalling packages"
fi
}
# Wrapper for ``yum`` to set proxy environment variables
# Uses globals ``OFFLINE``, ``*_proxy``, ``YUM``
# yum_install package [package ...]
function yum_install {
[[ "$OFFLINE" = "True" ]] && return
local sudo="sudo"
[[ "$(id -u)" = "0" ]] && sudo="env"
# The manual check for missing packages is because yum -y assumes
# missing packages are OK. See
# https://bugzilla.redhat.com/show_bug.cgi?id=965567
$sudo http_proxy="${http_proxy:-}" https_proxy="${https_proxy:-}" \
no_proxy="${no_proxy:-}" \
${YUM:-yum} install -y "$@" 2>&1 | \
awk '
BEGIN { fail=0 }
/No package/ { fail=1 }
{ print }
END { exit fail }' || \
die $LINENO "Missing packages detected"
# also ensure we catch a yum failure
if [[ ${PIPESTATUS[0]} != 0 ]]; then
die $LINENO "${YUM:-yum} install failure"
fi
}
# zypper wrapper to set arguments correctly
# Uses globals ``OFFLINE``, ``*_proxy``
# zypper_install package [package ...]
function zypper_install {
[[ "$OFFLINE" = "True" ]] && return
local sudo="sudo"
[[ "$(id -u)" = "0" ]] && sudo="env"
$sudo http_proxy="${http_proxy:-}" https_proxy="${https_proxy:-}" \
no_proxy="${no_proxy:-}" \
zypper --non-interactive install --auto-agree-with-licenses "$@"
}
# Process Functions
# =================
# _run_process() is designed to be backgrounded by run_process() to simulate a
# fork. It includes the dirty work of closing extra filehandles and preparing log
# files to produce the same logs as screen_it(). The log filename is derived
# from the service name.
# Uses globals ``CURRENT_LOG_TIME``, ``LOGDIR``, ``SCREEN_LOGDIR``, ``SCREEN_NAME``, ``SERVICE_DIR``
# If an optional group is provided sg will be used to set the group of
# the command.
# _run_process service "command-line" [group]
function _run_process {
local service=$1
local command="$2"
local group=$3
# Undo logging redirections and close the extra descriptors
exec 1>&3
exec 2>&3
exec 3>&-
exec 6>&-
local real_logfile="${LOGDIR}/${service}.log.${CURRENT_LOG_TIME}"
if [[ -n ${LOGDIR} ]]; then
exec 1>&"$real_logfile" 2>&1
ln -sf "$real_logfile" ${LOGDIR}/${service}.log
if [[ -n ${SCREEN_LOGDIR} ]]; then
# Drop the backward-compat symlink
ln -sf "$real_logfile" ${SCREEN_LOGDIR}/screen-${service}.log
fi
# TODO(dtroyer): Hack to get stdout from the Python interpreter for the logs.
export PYTHONUNBUFFERED=1
fi
# Run under ``setsid`` to force the process to become a session and group leader.
# The pid saved can be used with pkill -g to get the entire process group.
if [[ -n "$group" ]]; then
setsid sg $group "$command" & echo $! >$SERVICE_DIR/$SCREEN_NAME/$service.pid
else
setsid $command & echo $! >$SERVICE_DIR/$SCREEN_NAME/$service.pid
fi
# Just silently exit this process
exit 0
}
# Helper to remove the ``*.failure`` files under ``$SERVICE_DIR/$SCREEN_NAME``.
# This is used for ``service_check`` when all the ``screen_it`` are called finished
# Uses globals ``SCREEN_NAME``, ``SERVICE_DIR``
# init_service_check
function init_service_check {
SCREEN_NAME=${SCREEN_NAME:-stack}
SERVICE_DIR=${SERVICE_DIR:-${DEST}/status}
if [[ ! -d "$SERVICE_DIR/$SCREEN_NAME" ]]; then
mkdir -p "$SERVICE_DIR/$SCREEN_NAME"
fi
rm -f "$SERVICE_DIR/$SCREEN_NAME"/*.failure
}
# Find out if a process exists by partial name.
# is_running name
function is_running {
local name=$1
ps auxw | grep -v grep | grep ${name} > /dev/null
local exitcode=$?
# some times I really hate bash reverse binary logic
return $exitcode
}
# Run a single service under screen or directly
# If the command includes shell metachatacters (;<>*) it must be run using a shell
# If an optional group is provided sg will be used to run the
# command as that group.
# run_process service "command-line" [group]
function run_process {
local service=$1
local command="$2"
local group=$3
if is_service_enabled $service; then
if [[ "$USE_SCREEN" = "True" ]]; then
screen_process "$service" "$command" "$group"
else
# Spawn directly without screen
_run_process "$service" "$command" "$group" &
fi
fi
}
# Helper to launch a process in a named screen
# Uses globals ``CURRENT_LOG_TIME``, ```LOGDIR``, ``SCREEN_LOGDIR``, `SCREEN_NAME``,
# ``SERVICE_DIR``, ``USE_SCREEN``
# screen_process name "command-line" [group]
# Run a command in a shell in a screen window, if an optional group
# is provided, use sg to set the group of the command.
function screen_process {
local name=$1
local command="$2"
local group=$3
SCREEN_NAME=${SCREEN_NAME:-stack}
SERVICE_DIR=${SERVICE_DIR:-${DEST}/status}
USE_SCREEN=$(trueorfalse True USE_SCREEN)
screen -S $SCREEN_NAME -X screen -t $name
local real_logfile="${LOGDIR}/${name}.log.${CURRENT_LOG_TIME}"
echo "LOGDIR: $LOGDIR"
echo "SCREEN_LOGDIR: $SCREEN_LOGDIR"
echo "log: $real_logfile"
if [[ -n ${LOGDIR} ]]; then
screen -S $SCREEN_NAME -p $name -X logfile "$real_logfile"
screen -S $SCREEN_NAME -p $name -X log on
ln -sf "$real_logfile" ${LOGDIR}/${name}.log
if [[ -n ${SCREEN_LOGDIR} ]]; then
# Drop the backward-compat symlink
ln -sf "$real_logfile" ${SCREEN_LOGDIR}/screen-${1}.log
fi
fi
# sleep to allow bash to be ready to be send the command - we are
# creating a new window in screen and then sends characters, so if
# bash isn't running by the time we send the command, nothing
# happens. This sleep was added originally to handle gate runs
# where we needed this to be at least 3 seconds to pass
# consistently on slow clouds. Now this is configurable so that we
# can determine a reasonable value for the local case which should
# be much smaller.
sleep ${SCREEN_SLEEP:-3}
NL=`echo -ne '\015'`
# This fun command does the following:
# - the passed server command is backgrounded
# - the pid of the background process is saved in the usual place
# - the server process is brought back to the foreground
# - if the server process exits prematurely the fg command errors
# and a message is written to stdout and the process failure file
#
# The pid saved can be used in stop_process() as a process group
# id to kill off all child processes
if [[ -n "$group" ]]; then
command="sg $group '$command'"
fi
# Append the process to the screen rc file
screen_rc "$name" "$command"
screen -S $SCREEN_NAME -p $name -X stuff "$command & echo \$! >$SERVICE_DIR/$SCREEN_NAME/${name}.pid; fg || echo \"$name failed to start\" | tee \"$SERVICE_DIR/$SCREEN_NAME/${name}.failure\"$NL"
}
# Screen rc file builder
# Uses globals ``SCREEN_NAME``, ``SCREENRC``
# screen_rc service "command-line"
function screen_rc {
SCREEN_NAME=${SCREEN_NAME:-stack}
SCREENRC=$TOP_DIR/$SCREEN_NAME-screenrc
if [[ ! -e $SCREENRC ]]; then
# Name the screen session
echo "sessionname $SCREEN_NAME" > $SCREENRC
# Set a reasonable statusbar
echo "hardstatus alwayslastline '$SCREEN_HARDSTATUS'" >> $SCREENRC
# Some distributions override PROMPT_COMMAND for the screen terminal type - turn that off
echo "setenv PROMPT_COMMAND /bin/true" >> $SCREENRC
echo "screen -t shell bash" >> $SCREENRC
fi
# If this service doesn't already exist in the screenrc file
if ! grep $1 $SCREENRC 2>&1 > /dev/null; then
NL=`echo -ne '\015'`
echo "screen -t $1 bash" >> $SCREENRC
echo "stuff \"$2$NL\"" >> $SCREENRC
if [[ -n ${LOGDIR} ]]; then
echo "logfile ${LOGDIR}/${1}.log.${CURRENT_LOG_TIME}" >>$SCREENRC
echo "log on" >>$SCREENRC
fi
fi
}
# Stop a service in screen
# If a PID is available use it, kill the whole process group via TERM
# If screen is being used kill the screen window; this will catch processes
# that did not leave a PID behind
# Uses globals ``SCREEN_NAME``, ``SERVICE_DIR``, ``USE_SCREEN``
# screen_stop_service service
function screen_stop_service {
local service=$1
SCREEN_NAME=${SCREEN_NAME:-stack}
SERVICE_DIR=${SERVICE_DIR:-${DEST}/status}
USE_SCREEN=$(trueorfalse True USE_SCREEN)
if is_service_enabled $service; then
# Clean up the screen window
screen -S $SCREEN_NAME -p $service -X kill
fi
}
# Stop a service process
# If a PID is available use it, kill the whole process group via TERM
# If screen is being used kill the screen window; this will catch processes
# that did not leave a PID behind
# Uses globals ``SERVICE_DIR``, ``USE_SCREEN``
# stop_process service
function stop_process {
local service=$1
SERVICE_DIR=${SERVICE_DIR:-${DEST}/status}
USE_SCREEN=$(trueorfalse True USE_SCREEN)
if is_service_enabled $service; then
# Kill via pid if we have one available
if [[ -r $SERVICE_DIR/$SCREEN_NAME/$service.pid ]]; then
pkill -g $(cat $SERVICE_DIR/$SCREEN_NAME/$service.pid)
rm $SERVICE_DIR/$SCREEN_NAME/$service.pid
fi
if [[ "$USE_SCREEN" = "True" ]]; then
# Clean up the screen window
screen_stop_service $service
fi
fi
}
# Helper to get the status of each running service
# Uses globals ``SCREEN_NAME``, ``SERVICE_DIR``
# service_check
function service_check {
local service
local failures
SCREEN_NAME=${SCREEN_NAME:-stack}
SERVICE_DIR=${SERVICE_DIR:-${DEST}/status}
if [[ ! -d "$SERVICE_DIR/$SCREEN_NAME" ]]; then
echo "No service status directory found"
return
fi
# Check if there is any falure flag file under $SERVICE_DIR/$SCREEN_NAME
# make this -o errexit safe
failures=`ls "$SERVICE_DIR/$SCREEN_NAME"/*.failure 2>/dev/null || /bin/true`
for service in $failures; do
service=`basename $service`
service=${service%.failure}
echo "Error: Service $service is not running"
done
if [ -n "$failures" ]; then
die $LINENO "More details about the above errors can be found with screen, with ./rejoin-stack.sh"
fi
}
# Tail a log file in a screen if USE_SCREEN is true.
function tail_log {
local name=$1
local logfile=$2
USE_SCREEN=$(trueorfalse True USE_SCREEN)
if [[ "$USE_SCREEN" = "True" ]]; then
screen_process "$name" "sudo tail -f $logfile"
fi
}
# Deprecated Functions
# --------------------
# _old_run_process() is designed to be backgrounded by old_run_process() to simulate a
# fork. It includes the dirty work of closing extra filehandles and preparing log
# files to produce the same logs as screen_it(). The log filename is derived
# from the service name and global-and-now-misnamed ``SCREEN_LOGDIR``
# Uses globals ``CURRENT_LOG_TIME``, ``SCREEN_LOGDIR``, ``SCREEN_NAME``, ``SERVICE_DIR``
# _old_run_process service "command-line"
function _old_run_process {
local service=$1
local command="$2"
# Undo logging redirections and close the extra descriptors
exec 1>&3
exec 2>&3
exec 3>&-
exec 6>&-
if [[ -n ${SCREEN_LOGDIR} ]]; then
exec 1>&${SCREEN_LOGDIR}/screen-${1}.log.${CURRENT_LOG_TIME} 2>&1
ln -sf ${SCREEN_LOGDIR}/screen-${1}.log.${CURRENT_LOG_TIME} ${SCREEN_LOGDIR}/screen-${1}.log
# TODO(dtroyer): Hack to get stdout from the Python interpreter for the logs.
export PYTHONUNBUFFERED=1
fi
exec /bin/bash -c "$command"
die "$service exec failure: $command"
}
# old_run_process() launches a child process that closes all file descriptors and
# then exec's the passed in command. This is meant to duplicate the semantics
# of screen_it() without screen. PIDs are written to
# ``$SERVICE_DIR/$SCREEN_NAME/$service.pid`` by the spawned child process.
# old_run_process service "command-line"
function old_run_process {
local service=$1
local command="$2"
# Spawn the child process
_old_run_process "$service" "$command" &
echo $!
}
# Compatibility for existing start_XXXX() functions
# Uses global ``USE_SCREEN``
# screen_it service "command-line"
function screen_it {
if is_service_enabled $1; then
# Append the service to the screen rc file
screen_rc "$1" "$2"
if [[ "$USE_SCREEN" = "True" ]]; then
screen_process "$1" "$2"
else
# Spawn directly without screen
old_run_process "$1" "$2" >$SERVICE_DIR/$SCREEN_NAME/$1.pid
fi
fi
}
# Compatibility for existing stop_XXXX() functions
# Stop a service in screen
# If a PID is available use it, kill the whole process group via TERM
# If screen is being used kill the screen window; this will catch processes
# that did not leave a PID behind
# screen_stop service
function screen_stop {
# Clean up the screen window
stop_process $1
}
# Plugin Functions
# =================
DEVSTACK_PLUGINS=${DEVSTACK_PLUGINS:-""}
# enable_plugin <name> <url> [branch]
#
# ``name`` is an arbitrary name - (aka: glusterfs, nova-docker, zaqar)
# ``url`` is a git url
# ``branch`` is a gitref. If it's not set, defaults to master
function enable_plugin {
local name=$1
local url=$2
local branch=${3:-master}
DEVSTACK_PLUGINS+=",$name"
GITREPO[$name]=$url
GITDIR[$name]=$DEST/$name
GITBRANCH[$name]=$branch
}
# fetch_plugins
#
# clones all plugins
function fetch_plugins {
local plugins="${DEVSTACK_PLUGINS}"
local plugin
# short circuit if nothing to do
if [[ -z $plugins ]]; then
return
fi
echo "Fetching DevStack plugins"
for plugin in ${plugins//,/ }; do
git_clone_by_name $plugin
done
}
# load_plugin_settings
#
# Load settings from plugins in the order that they were registered
function load_plugin_settings {
local plugins="${DEVSTACK_PLUGINS}"
local plugin
# short circuit if nothing to do
if [[ -z $plugins ]]; then
return
fi
echo "Loading plugin settings"
for plugin in ${plugins//,/ }; do
local dir=${GITDIR[$plugin]}
# source any known settings
if [[ -f $dir/devstack/settings ]]; then
source $dir/devstack/settings
fi
done
}
# plugin_override_defaults
#
# Run an extremely early setting phase for plugins that allows default
# overriding of services.
function plugin_override_defaults {
local plugins="${DEVSTACK_PLUGINS}"
local plugin
# short circuit if nothing to do
if [[ -z $plugins ]]; then
return
fi
echo "Overriding Configuration Defaults"
for plugin in ${plugins//,/ }; do
local dir=${GITDIR[$plugin]}
# source any overrides
if [[ -f $dir/devstack/override-defaults ]]; then
# be really verbose that an override is happening, as it
# may not be obvious if things fail later.
echo "$plugin has overriden the following defaults"
cat $dir/devstack/override-defaults
source $dir/devstack/override-defaults
fi
done
}
# run_plugins
#
# Run the devstack/plugin.sh in all the plugin directories. These are
# run in registration order.
function run_plugins {
local mode=$1
local phase=$2
local plugins="${DEVSTACK_PLUGINS}"
local plugin
for plugin in ${plugins//,/ }; do
local dir=${GITDIR[$plugin]}
if [[ -f $dir/devstack/plugin.sh ]]; then
source $dir/devstack/plugin.sh $mode $phase
fi
done
}
function run_phase {
local mode=$1
local phase=$2
if [[ -d $TOP_DIR/extras.d ]]; then
for i in $TOP_DIR/extras.d/*.sh; do
[[ -r $i ]] && source $i $mode $phase
done
fi
# the source phase corresponds to settings loading in plugins
if [[ "$mode" == "source" ]]; then
load_plugin_settings
elif [[ "$mode" == "override_defaults" ]]; then
plugin_override_defaults
else
run_plugins $mode $phase
fi
}
# Service Functions
# =================
# remove extra commas from the input string (i.e. ``ENABLED_SERVICES``)
# _cleanup_service_list service-list
function _cleanup_service_list {
echo "$1" | sed -e '
s/,,/,/g;
s/^,//;
s/,$//
'
}
# disable_all_services() removes all current services
# from ``ENABLED_SERVICES`` to reset the configuration
# before a minimal installation
# Uses global ``ENABLED_SERVICES``
# disable_all_services
function disable_all_services {
ENABLED_SERVICES=""
}
# Remove all services starting with '-'. For example, to install all default
# services except rabbit (rabbit) set in ``localrc``:
# ENABLED_SERVICES+=",-rabbit"
# Uses global ``ENABLED_SERVICES``
# disable_negated_services
function disable_negated_services {
local tmpsvcs="${ENABLED_SERVICES}"
local service
for service in ${tmpsvcs//,/ }; do
if [[ ${service} == -* ]]; then
tmpsvcs=$(echo ${tmpsvcs}|sed -r "s/(,)?(-)?${service#-}(,)?/,/g")
fi
done
ENABLED_SERVICES=$(_cleanup_service_list "$tmpsvcs")
}
# disable_service() removes the services passed as argument to the
# ``ENABLED_SERVICES`` list, if they are present.
#
# For example:
# disable_service rabbit
#
# This function does not know about the special cases
# for nova, glance, and neutron built into is_service_enabled().
# Uses global ``ENABLED_SERVICES``
# disable_service service [service ...]
function disable_service {
local tmpsvcs=",${ENABLED_SERVICES},"
local service
for service in $@; do
if is_service_enabled $service; then
tmpsvcs=${tmpsvcs//,$service,/,}
fi
done
ENABLED_SERVICES=$(_cleanup_service_list "$tmpsvcs")
}
# enable_service() adds the services passed as argument to the
# ``ENABLED_SERVICES`` list, if they are not already present.
#
# For example:
# enable_service qpid
#
# This function does not know about the special cases
# for nova, glance, and neutron built into is_service_enabled().
# Uses global ``ENABLED_SERVICES``
# enable_service service [service ...]
function enable_service {
local tmpsvcs="${ENABLED_SERVICES}"
local service
for service in $@; do
if ! is_service_enabled $service; then
tmpsvcs+=",$service"
fi
done
ENABLED_SERVICES=$(_cleanup_service_list "$tmpsvcs")
disable_negated_services
}
# is_service_enabled() checks if the service(s) specified as arguments are
# enabled by the user in ``ENABLED_SERVICES``.
#
# Multiple services specified as arguments are ``OR``'ed together; the test
# is a short-circuit boolean, i.e it returns on the first match.
#
# There are special cases for some 'catch-all' services::
# **nova** returns true if any service enabled start with **n-**
# **cinder** returns true if any service enabled start with **c-**
# **ceilometer** returns true if any service enabled start with **ceilometer**
# **glance** returns true if any service enabled start with **g-**
# **neutron** returns true if any service enabled start with **q-**
# **swift** returns true if any service enabled start with **s-**
# **trove** returns true if any service enabled start with **tr-**
# For backward compatibility if we have **swift** in ENABLED_SERVICES all the
# **s-** services will be enabled. This will be deprecated in the future.
#
# Cells within nova is enabled if **n-cell** is in ``ENABLED_SERVICES``.
# We also need to make sure to treat **n-cell-region** and **n-cell-child**
# as enabled in this case.
#
# Uses global ``ENABLED_SERVICES``
# is_service_enabled service [service ...]
function is_service_enabled {
local xtrace=$(set +o | grep xtrace)
set +o xtrace
local enabled=1
local services=$@
local service
for service in ${services}; do
[[ ,${ENABLED_SERVICES}, =~ ,${service}, ]] && enabled=0
# Look for top-level 'enabled' function for this service
if type is_${service}_enabled >/dev/null 2>&1; then
# A function exists for this service, use it
is_${service}_enabled
enabled=$?
fi
# TODO(dtroyer): Remove these legacy special-cases after the is_XXX_enabled()
# are implemented
[[ ${service} == n-cell-* && ${ENABLED_SERVICES} =~ "n-cell" ]] && enabled=0
[[ ${service} == n-cpu-* && ${ENABLED_SERVICES} =~ "n-cpu" ]] && enabled=0
[[ ${service} == "nova" && ${ENABLED_SERVICES} =~ "n-" ]] && enabled=0
[[ ${service} == "cinder" && ${ENABLED_SERVICES} =~ "c-" ]] && enabled=0
[[ ${service} == "ceilometer" && ${ENABLED_SERVICES} =~ "ceilometer-" ]] && enabled=0
[[ ${service} == "glance" && ${ENABLED_SERVICES} =~ "g-" ]] && enabled=0
[[ ${service} == "ironic" && ${ENABLED_SERVICES} =~ "ir-" ]] && enabled=0
[[ ${service} == "neutron" && ${ENABLED_SERVICES} =~ "q-" ]] && enabled=0
[[ ${service} == "trove" && ${ENABLED_SERVICES} =~ "tr-" ]] && enabled=0
[[ ${service} == "swift" && ${ENABLED_SERVICES} =~ "s-" ]] && enabled=0
[[ ${service} == s-* && ${ENABLED_SERVICES} =~ "swift" ]] && enabled=0
done
$xtrace
return $enabled
}
# Toggle enable/disable_service for services that must run exclusive of each other
# $1 The name of a variable containing a space-separated list of services
# $2 The name of a variable in which to store the enabled service's name
# $3 The name of the service to enable
function use_exclusive_service {
local options=${!1}
local selection=$3
local out=$2
[ -z $selection ] || [[ ! "$options" =~ "$selection" ]] && return 1
local opt
for opt in $options;do
[[ "$opt" = "$selection" ]] && enable_service $opt || disable_service $opt
done
eval "$out=$selection"
return 0
}
# System Functions
# ================
# Only run the command if the target file (the last arg) is not on an
# NFS filesystem.
function _safe_permission_operation {
local xtrace=$(set +o | grep xtrace)
set +o xtrace
local args=( $@ )
local last
local sudo_cmd
local dir_to_check
let last="${#args[*]} - 1"
local dir_to_check=${args[$last]}
if [ ! -d "$dir_to_check" ]; then
dir_to_check=`dirname "$dir_to_check"`
fi
if is_nfs_directory "$dir_to_check" ; then
$xtrace
return 0
fi
if [[ $TRACK_DEPENDS = True ]]; then
sudo_cmd="env"
else
sudo_cmd="sudo"
fi
$xtrace
$sudo_cmd $@
}
# Exit 0 if address is in network or 1 if address is not in network
# ip-range is in CIDR notation: 1.2.3.4/20
# address_in_net ip-address ip-range
function address_in_net {
local ip=$1
local range=$2
local masklen=${range#*/}
local network=$(maskip ${range%/*} $(cidr2netmask $masklen))
local subnet=$(maskip $ip $(cidr2netmask $masklen))
[[ $network == $subnet ]]
}
# Add a user to a group.
# add_user_to_group user group
function add_user_to_group {
local user=$1
local group=$2
if [[ -z "$os_VENDOR" ]]; then
GetOSVersion
fi
# SLE11 and openSUSE 12.2 don't have the usual usermod
if ! is_suse || [[ "$os_VENDOR" = "openSUSE" && "$os_RELEASE" != "12.2" ]]; then
sudo usermod -a -G "$group" "$user"
else
sudo usermod -A "$group" "$user"
fi
}
# Convert CIDR notation to a IPv4 netmask
# cidr2netmask cidr-bits
function cidr2netmask {
local maskpat="255 255 255 255"
local maskdgt="254 252 248 240 224 192 128"
set -- ${maskpat:0:$(( ($1 / 8) * 4 ))}${maskdgt:$(( (7 - ($1 % 8)) * 4 )):3}
echo ${1-0}.${2-0}.${3-0}.${4-0}
}
# Gracefully cp only if source file/dir exists
# cp_it source destination
function cp_it {
if [ -e $1 ] || [ -d $1 ]; then
cp -pRL $1 $2
fi
}
# HTTP and HTTPS proxy servers are supported via the usual environment variables [1]
# ``http_proxy``, ``https_proxy`` and ``no_proxy``. They can be set in
# ``localrc`` or on the command line if necessary::
#
# [1] http://www.w3.org/Daemon/User/Proxies/ProxyClients.html
#
# http_proxy=http://proxy.example.com:3128/ no_proxy=repo.example.net ./stack.sh
# Unused function
#function export_proxy_variables {
# if isset http_proxy ; then
# export http_proxy=$http_proxy
# fi
# if isset https_proxy ; then
# export https_proxy=$https_proxy
# fi
# if isset no_proxy ; then
# export no_proxy=$no_proxy
# fi
#}
# Returns true if the directory is on a filesystem mounted via NFS.
function is_nfs_directory {
local mount_type=`stat -f -L -c %T $1`
test "$mount_type" == "nfs"
}
# Return the network portion of the given IP address using netmask
# netmask is in the traditional dotted-quad format
# maskip ip-address netmask
function maskip {
local ip=$1
local mask=$2
local l="${ip%.*}"; local r="${ip#*.}"; local n="${mask%.*}"; local m="${mask#*.}"
local subnet=$((${ip%%.*}&${mask%%.*})).$((${r%%.*}&${m%%.*})).$((${l##*.}&${n##*.})).$((${ip##*.}&${mask##*.}))
echo $subnet
}
# Service wrapper to restart services
# restart_service service-name
function restart_service {
if is_ubuntu; then
sudo /usr/sbin/service $1 restart
else
sudo /sbin/service $1 restart
fi
}
# Only change permissions of a file or directory if it is not on an
# NFS filesystem.
function safe_chmod {
_safe_permission_operation chmod $@
}
# Only change ownership of a file or directory if it is not on an NFS
# filesystem.
function safe_chown {
_safe_permission_operation chown $@
}
# Service wrapper to start services
# start_service service-name
function start_service {
if is_ubuntu; then
sudo /usr/sbin/service $1 start
else
sudo /sbin/service $1 start
fi
}
# Service wrapper to stop services
# stop_service service-name
function stop_service {
if is_ubuntu; then
sudo /usr/sbin/service $1 stop
else
sudo /sbin/service $1 stop
fi
}
# Restore xtrace
$XTRACE
# Local variables:
# mode: shell-script
# End:
if [[ $EUID -gt 0 ]]; then
echo "####################################################"
echo "# #"
echo "# ERROR: You must be root to run this script!!!! #"
echo "# #"
echo "####################################################"
exit 1
fi
export PATH=$PATH:/usr/local/bin:/usr/local/sbin
# XXX a bit brutal but otherwise it cannot work, without tty enabled.
if [ -f /etc/sudoers ]; then
sed -i "/requiretty/d" /etc/sudoers
fi
# Include Additional Functions
function download_playbook {
if [ ! -f /etc/opt/slapcache.cfg ]; then
slapcache-conf
fi
DFILE="/tmp/tmpplaybook$(basename $0).$$/"
TFILE="archive.tar.gz"
mkdir -p $DFILE
cd $DFILE
slapcache-download --destination=$TFILE
tar -xzvf $TFILE
rm $TFILE
}
# Determine what system we are running on. This provides ``os_VENDOR``,
# ``os_RELEASE``, ``os_UPDATE``, ``os_PACKAGE``, ``os_CODENAME``
# and ``DISTRO``
GetDistro
if [[ ${DISTRO} =~ (-.) ]] && [[ -f /etc/debian_version ]]; then
apt_get install lsb-release
GetDistro
fi
# Warn users who aren't on an explicitly supported distro, but allow them to
# override check and attempt installation with ``export FORCE=yes``
if [[ ! ${DISTRO} =~ (wheezy|jessie|trusty|vivid|xenial|rhel7|rhel6|LinuxMint-17.2) ]]; then
echo "WARNING: this script has not been tested on $DISTRO"
if [[ "$FORCE" != "yes" ]]; then
die $LINENO "If you wish to run this script anyway run with FORCE=yes"
fi
fi
# Make sure wheezy backports are available.
if [[ $DISTRO == "wheezy" ]]; then
echo "deb http://ftp.debian.org/debian wheezy-backports main contrib " > /etc/apt/sources.list.d/wheezy-backports.list
fi
if is_fedora && [[ $DISTRO == "rhel7" ]]; then
# RHEL requires EPEL for many Ansible dependencies
# NOTE: We always remove and install latest -- some environments
# use snapshot images, and if EPEL version updates they break
# unless we update them to latest version.
if sudo yum repolist enabled epel | grep -q 'epel'; then
uninstall_package epel-release || true
fi
# This trick installs the latest epel-release from a bootstrap
# repo, then removes itself (as epel-release installed the
# "real" repo).
#
# You would think that rather than this, you could use
# $releasever directly in .repo file we create below. However
# RHEL gives a $releasever of "6Server" which breaks the path;
# see https://bugzilla.redhat.com/show_bug.cgi?id=1150759
cat <<EOF | sudo tee /etc/yum.repos.d/epel-bootstrap.repo
[epel-bootstrap]
name=Bootstrap EPEL
mirrorlist=http://mirrors.fedoraproject.org/mirrorlist?repo=epel-7&arch=\$basearch
failovermethod=priority
enabled=0
gpgcheck=0
EOF
yum update -y
yum_install yum-utils
# Enable a bootstrap repo. It is removed after finishing
# the epel-release installation.
yum-config-manager --enable epel-bootstrap
yum_install epel-release || \
die $LINENO "Error installing EPEL repo, cannot continue"
# EPEL rpm has installed it's version
rm -f /etc/yum.repos.d/epel-bootstrap.repo
# ... and also optional to be enabled
is_package_installed yum-utils || install_package yum-utils
yum-config-manager --enable rhel-7-server-optional-rpms
fi
if is_fedora && [[ $DISTRO == "rhel6" ]]; then
sed -i "/requiretty/d" /etc/sudoers
if sudo yum repolist enabled epel | grep -q 'epel'; then
uninstall_package epel-release || true
fi
cat <<EOF | sudo tee /etc/yum.repos.d/epel-bootstrap.repo
[epel-bootstrap]
name=Bootstrap EPEL
mirrorlist=http://mirrors.fedoraproject.org/mirrorlist?repo=epel-6&arch=\$basearch
failovermethod=priority
enabled=0
gpgcheck=0
EOF
yum update -y
yum_install yum-utils
yum-config-manager --enable epel-bootstrap
yum_install epel-release || \
die $LINENO "Error installing EPEL repo, cannot continue"
# EPEL rpm has installed it's version
rm -f /etc/yum.repos.d/epel-bootstrap.repo
is_package_installed yum-utils || install_package yum-utils
yum-config-manager --enable rhel-6-server-optional-rpms
is_package_installed libselinux-python || install_package libselinux-python
fi
is_package_installed openssl || install_package openssl
# stick to a specific ansible package version
if is_ubuntu; then
# is_ubuntu also handle debian distro
is_package_installed ansible || install_package ansible=1.*
elif is_fedora; then
# is_fedora also handle CentOS distro
is_package_installed ansible || install_package ansible1.9.noarch
else
is_package_installed ansible || install_package ansible
fi
is_package_installed python-setuptools || install_package python-setuptools
if is_ubuntu && [[ $DISTRO == "trusty" ]]; then
is_package_installed python-apt || install_package python-apt
is_package_installed python-pycurl || install_package python-pycurl
fi
if is_ubuntu && [[ $DISTRO == "vivid" ]]; then
is_package_installed python-apt || install_package python-apt
is_package_installed python-pycurl || install_package python-pycurl
fi
if [[ $DISTRO == "LinuxMint-17.2" ]]; then
is_package_installed python-apt || install_package python-apt
is_package_installed python-pycurl || install_package python-pycurl
fi
ANSIBLE_PLUGIN_LOCATION="/usr/share/ansible_plugins/callback_plugins/"
mkdir -p $ANSIBLE_PLUGIN_LOCATION
/bin/cat << EOF > $ANSIBLE_PLUGIN_LOCATION/log_parse.py
import os
import time
import json
import ansible
baseModule = object
ANSIBLE_VERSION = 1
if hasattr(ansible, 'plugins') and hasattr(ansible.plugins, 'callback'):
baseModule = ansible.plugins.callback.CallbackBase
ANSIBLE_VERSION = 2
class CallbackModule(baseModule):
"""
logs playbook results, per host, in /var/log/ansible/hosts
"""
log_path = '/var/log/ansible/hosts'
fd_list = {}
def __init__(self):
if ANSIBLE_VERSION > 1:
super(CallbackModule, self).__init__()
if not os.path.exists(self.log_path):
os.makedirs(self.log_path)
else:
for filename in os.listdir(self.log_path):
filepath = os.path.join(self.log_path, filename)
if os.path.exists(filepath) and os.path.isfile(filepath):
os.unlink(filepath)
def writeLog(self, host, category, content):
if not self.fd_list.has_key(category):
self.fd_list[category] = open(
os.path.join(self.log_path, '%s_%s' % (host, category)), "a"
)
self.fd_list[category].write(content + '\n')
def log(self, host, category, data, ignore_errors=False):
if host == "localhost":
host = "127.0.0.1" # keep compatibility
if type(data) == dict:
if '_ansible_verbose_override' in data:
# avoid logging extraneous data
return
data = data.copy()
content = json.dumps(data)
if ignore_errors:
category = '%s_IGNORED' % category
self.writeLog(host, category, content)
def _stats(self, stats):
for key in self.fd_list:
self.fd_list[key].close()
def runner_on_failed(self, host, res, ignore_errors=False):
self.log(host, 'FAILED', res, ignore_errors)
def runner_on_ok(self, host, res):
self.log(host, 'OK', res)
def runner_on_skipped(self, host, item=None):
pass
def runner_on_unreachable(self, host, res):
self.log(host, 'UNREACHABLE', res)
def runner_on_async_failed(self, host, res, jid):
self.log(host, 'ASYNC_FAILED', res)
def playbook_on_import_for_host(self, host, imported_file):
self.log(host, 'IMPORTED', imported_file)
def playbook_on_not_import_for_host(self, host, missing_file):
self.log(host, 'NOTIMPORTED', missing_file)
def playbook_on_stats(self, stats):
self._stats(stats)
EOF
ansible localhost -m easy_install -a name=slapcache --connection=local
ansible localhost -m easy_install -a name=requests --connection=local
# Include Additional Functions
if [ ! -f /etc/opt/slapcache.cfg ]; then
slapcache-conf
fi
sed -i "s/key = slapos-global-key/key = slapos-global-official-vifib-key/g" /etc/opt/slapcache.cfg
DFILE="/tmp/tmpplaybook_unstable$(basename $0).$$/"
TFILE="archive.tar.gz"
mkdir -p $DFILE
cd $DFILE
wget -O $TFILE http://10.0.2.100/playbook.tar.gz
tar -xzvf $TFILE
rm $TFILE
clear
echo "Starting Ansible playbook:"
ansible-playbook $TEST_YML_PATH -i hosts --connection=local
software/erp5testnode/testsuite/dummy/software.cfg
View file @ 56ad14fb
......@@ -14,4 +14,4 @@ md5sum = efd3b712a2294207f265a9c45648d5cf
mode = 0644
[versions]
slapos.recipe.template = 2.9
slapos.recipe.template = 3.0
software/erp5testnode/testsuite/instance-resilient-test.cfg.jinja2
View file @ 56ad14fb
......@@ -50,6 +50,10 @@ config-resiliency-backup-periodicity = */10 * * * *
config-resilient-clone-number = 1
config-ignore-known-hosts-file = false
config-cpu-usage-ratio = 1
# Use one external folder (same of erp5testnode for build the software
# This is a way to preserve the latest software release build speeding up
# the tests.)
config-software-root = ${buildout:directory}/../../soft
# XXX hardcoded
#config-frontend-domain = google.com
# XXX Hack to deploy Root Instance on the same computer as the type-test Instance
......
software/erp5testnode/testsuite/slaprunner.cfg
View file @ 56ad14fb
......@@ -7,4 +7,8 @@ parts += template-erp5testnode
[template-resilient-test]
filename = instance-resilient-test.cfg.jinja2
md5sum = 268d07ed48199d34259aff1ba544d852
md5sum = dadf1e8599ba7d96c5af827ec389a9cf
[exporter-default-configuration]
# Define shorter interaction to speed up tests
backup_wait_time = 1
software/fluentd/fluentd-agent.conf.jinja2.in deleted 100644 → 0
View file @ fb36acc7
{% set conf_text = slapparameter_dict.get('conf_text', '') -%}
{{ conf_text }}
\ No newline at end of file
software/fluentd/instance.cfg.in
View file @ 56ad14fb
[buildout]
parts =
directory
service-fluentd
fluentd-agent-conf
eggs-directory = {{ buildout['eggs-directory'] }}
develop-eggs-directory = {{ buildout['develop-eggs-directory'] }}
......@@ -32,14 +30,16 @@ recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:service}/fluentd-service
command-line = {{ fluentd_location }}/bin/fluentd
-v
-c ${directory:etc}/fluentd-agent.conf
environment =
-c ${fluentd-agent-conf:rendered}
environment =
GEM_PATH={{ fluentd_location }}/lib/ruby/gems/1.8/
[fluentd-agent-conf]
recipe = slapos.recipe.template:jinja2
template = {{ fluentd_conf }}/fluentd-agent.conf.jinja2.in
rendered = ${buildout:directory}/etc/fluentd-agent.conf
template = inline:{% raw -%}
{{ slapparameter_dict.get('conf_text', '') }}
{%- endraw %}
rendered = ${directory:etc}/fluentd-agent.conf
mode = 0644
context =
context =
key slapparameter_dict instance-parameter:configuration
software/fluentd/software.cfg
View file @ 56ad14fb
[buildout]
extends =
../../component/fluentd/buildout.cfg
../../stack/slapos.cfg
parts =
fluentd
fluentd-agent-conf
parts =
instance-profile
slapos-cookbook
......@@ -18,17 +15,9 @@ mode = 0644
extensions = jinja2.ext.do
context =
section buildout buildout
raw fluentd_location ${fluentd:location}
raw fluentd_conf ${buildout:directory}
[fluentd-agent-conf]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/fluentd-agent.conf.jinja2.in
#md5sum = 4b81ddcbe2f16d23013caac37151d396
output = ${buildout:directory}/fluentd-agent.conf.jinja2.in
mode = 0644
key fluentd_location fluentd:location
[fluentd]
gems +=
gems +=
fluent-plugin-wendelin==0.1
fluent-plugin-bin
software/gitlab/software.cfg
View file @ 56ad14fb
......@@ -384,6 +384,6 @@ strip-top-level-dir = true
cns.recipe.symlink = 0.2.3
docutils = 0.12
plone.recipe.command = 1.1
rubygemsrecipe = 0.2.2
slapos.recipe.template = 2.9
rubygemsrecipe = 0.2.2+slapos001
slapos.recipe.template = 3.0
z3c.recipe.scripts = 1.0.1
software/html5ide/software.cfg
View file @ 56ad14fb
......@@ -48,4 +48,4 @@ md5sum = 8cde04bfd0c0e9bd56744b988275cfd8
PyRSS2Gen = 1.1
cns.recipe.symlink = 0.2.3
plone.recipe.command = 1.1
slapos.recipe.template = 2.8
slapos.recipe.template = 3.0
software/ipython_notebook/instance.cfg.in
View file @ 56ad14fb
......@@ -74,7 +74,13 @@ command-line =
wrapper-path = ${directory:service}/ipython_notebook
parameters-extra = true
# Explicitly define IPython Directory to be used while starting ipython notebook
environment = IPYTHONDIR=${directory:ipython_dir}
# Also explicitly force IPython to use en_US.utf8 encoding
environment =
IPYTHONDIR=${directory:ipython_dir}
JUPYTER_CONFIG_DIR=${directory:ipython_dir}
LC_ALL=C.UTF-8
LANG=C.UTF-8
LANGUAGE=C.UTF-8
[ipython-notebook-config]
<= dynamic-jinja2-template-base
......@@ -109,6 +115,7 @@ filename = knowledge0.cfg
[read-knowledge0]
recipe = slapos.cookbook:zero-knowledge.read
filename = knowledge0.cfg
password =
[monitor-instance-parameter]
monitor-base-url = ${monitor-frontend-promise:url}
......
software/ipython_notebook/software.cfg
View file @ 56ad14fb
......@@ -47,7 +47,7 @@ md5sum = d7d4a7e19d55bf14007819258bf42100
[erp5-kernel]
<= download-file-base
filename = ERP5kernel.py.jinja
md5sum = 5e7e0acef2f32e75f8f17bc851414aa1
md5sum = f7be6c3ef9cdfaaea3b8756895a22081
[kernel-json]
<= download-file-base
......@@ -57,14 +57,14 @@ md5sum = ab6e78ea20855e07d388b5b86d1770fe
[custom-js]
<= download-file-base
filename = custom.js.jinja
md5sum = a851a68667a4e676cfa86b34a86b4239
md5sum = 584aa839aa20263ba885537b74c83abd
[instance-jupyter]
recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/instance.cfg.in
rendered = ${buildout:directory}/template.cfg
mode = 0644
md5sum = b6555d25fd89bf92f47eda60a29c2b44
md5sum = c9d2c7a3658457b7f43cbbbc38ef247c
context =
key bin_directory buildout:bin-directory
key develop_eggs_directory buildout:develop-eggs-directory
......@@ -107,7 +107,7 @@ pyzmq = 16.0.2
scikit-learn = 0.18.1
seaborn = 0.7.1
simplegeneric = 0.8.1
slapos.recipe.template = 2.10
slapos.recipe.template = 3.0
statsmodels = 0.8.0
terminado = 0.6
tornado = 4.4.2
......
software/ipython_notebook/template/ERP5kernel.py.jinja
View file @ 56ad14fb
......@@ -106,8 +106,32 @@ class ERP5Kernel(Kernel):
# Get the magic value recived via code from frontend
magic_value = code.split()[1]
# Set magic_value to the required attribute
setattr(self, magic_info.variable_name , magic_value)
self.response = 'Your %s is %s. '%(magic_info.magic_name, magic_value)
if magic_info.magic_name == 'notebook_set_reference':
required_attributes = ['url', 'password', 'user']
missing_attributes = []
for attribute in required_attributes:
if not getattr(self, attribute):
missing_attributes.append(attribute)
if missing_attributes != []:
self.response = "You still haven't entered all required magics. \
Please do so before inputting your reference."
else:
if self.check_existing_reference(reference=magic_value):
self.response = 'WARNING: You already have a notebook with \
reference %s. It might be a good idea to use different references for new \
notebooks. \n' % magic_value
else:
self.response = ''
setattr(self, magic_info.variable_name , magic_value)
self.response = self.response + 'Your %s is %s. '%(magic_info.magic_name, magic_value)
elif magic_info.magic_name != 'erp5_password':
setattr(self, magic_info.variable_name , magic_value)
self.response = 'Your %s is %s. '%(magic_info.magic_name, magic_value)
else:
setattr(self, magic_info.variable_name , magic_value)
self.response = ""
# Catch exception while setting attribute and set message in response
except AttributeError:
......@@ -126,7 +150,8 @@ class ERP5Kernel(Kernel):
# Display the message/response from this fucntion before moving forward so
# as to keep track of the status
self.display_response(response=self.response)
if self.response != "":
self.display_response(response=(self.response + '\n'))
def check_required_attributes(self):
"""
......@@ -138,25 +163,32 @@ class ERP5Kernel(Kernel):
made to ask user to enter value.
"""
result_list = []
required_attributes = ['url', 'password', 'user', 'reference']
required_attributes = ['url', 'user', 'password', 'reference']
missing_attributes = []
# Set response to empty so as to flush the response set by some earlier fucntion call
self.response = ''
# Loop to check if the attributes are set
# Loop to check if the required attributes are set
for attribute in required_attributes:
if getattr(self, attribute):
result_list.append(True)
else:
# Set response/message for attributes which aren't set
self.response = '\nPlease enter %s in next cell. '%attribute
missing_attributes.append(attribute)
result_list.append(False)
# Compare result_list to get True for all True results and False for any False result
check_attributes = all(result_list)
if check_attributes:
self.response = 'You have entered all required magics. You may now use your notebook.'
else:
self.response = '''You have these required magics remaining: %s. \n''' % (
', '.join(map(str, missing_attributes)))
# Display response to frontend before moving forward
self.display_response(response=self.response)
self.display_response(response=(self.response + '\n'))
return check_attributes
......@@ -238,7 +270,7 @@ class ERP5Kernel(Kernel):
checked_attribute = self.check_required_attributes()
if checked_attribute and magic_info.send_request:
# Call the function to send request to erp5 with the arguments given
self.make_erp5_request(message='\nPlease proceed',
self.make_erp5_request(message='Please proceed\n',
request_reference=magic_info.request_reference,
display_message=magic_info.display_message)
......@@ -246,7 +278,8 @@ class ERP5Kernel(Kernel):
# Since this response would be either success message or failure
# error message, both of which are string type, so, we can simply
# display the stream response.
self.display_response(response=self.response)
if self.response != 'Please proceed\n':
self.display_response(response=self.response)
else:
# Set response if there is no magic or the magic name is not in MAGICS
......@@ -337,6 +370,31 @@ class ERP5Kernel(Kernel):
'user_expressions': {}}
return reply_content
# Checks the ERP5 site if there are existing notebooks with the same reference.
# Returns True if there are.
def check_existing_reference(self, reference):
if reference == None:
return False
modified_url = self.url[:self.url.rfind('/')] + '/Base_checkExistingReference'
result = True
try:
erp5_request = requests.post(
modified_url,
verify=False,
auth=(self.user, self.password),
data={
'reference': reference,
})
result = erp5_request.content
except requests.exceptions.RequestException as e:
self.response = str(e)
return result
if __name__ == '__main__':
IPKernelApp.launch_instance(kernel_class=ERP5Kernel)
software/ipython_notebook/template/custom.js.jinja
View file @ 56ad14fb
......@@ -85,15 +85,15 @@ $([IPython.events]).on('notebook_loaded.Notebook', function(){
var kernelname = IPython.notebook.kernel_selector.current_selection;
var display_text="<div class='output_subarea output_text output_result'>\
<pre>Follow these steps to customize your notebook with ERP5 kernel :-</br>\
1. Add <b>%erp5_url &lt;your_erp5_url&gt;/erp5/Base_executeJupyter</b>.</br>\
Make sure you have 'erp5_data_notebook' business template installed in your ERP5</br>\
2. <b>%notebook_set_reference &lt;your_notebook_refernce&gt;</b></br>\
1. Make sure you have 'erp5_data_notebook' business template installed in your ERP5</br>\
2. <b>%erp5_user &lt;your_erp5_username&gt;</b></br>\
3. <b>%erp5_password &lt;your_erp5_password&gt;</b></br>\
4. <b>%notebook_set_reference &lt;your_notebook_reference&gt;</b></br>\
It would be better to set the reference to match with erp5 reference pattern.</br>\
3. <b>%erp5_user &lt;your_erp5_username&gt;</b></br>\
4. <b>%erp5_password &lt;your_erp5_password&gt;</b></br>\
5. As soon as you see 'Please Proceed' message you can now access your erp5 using notebook.</br>\
<p><u>OTHER USEFUL MAGICS</u> -</br>\
<b>%my_notebooks</b> -This is used to display all the notebooks created by the specific user.</br>\
<b>%notebook_set_title</b> -This sets the title of the current notebook.</br>\
NOTE: Do not dynamically alter imported module objects as they are not being saved in DB, </br>\
so changes to them would be disregarded and would throw an error.</br>\
<p><u>About classes, functions and global state on modules:</u></p>\
......
software/jstestnode/software.cfg
View file @ 56ad14fb
......@@ -112,6 +112,6 @@ output = ${buildout:directory}/runTestSuite.in
mode = 0644
[versions]
erp5.util = 0.4.46
slapos.recipe.template = 2.9
erp5.util = 0.4.49
slapos.recipe.template = 3.0
selenium = 2.53.1
software/kvm/common.cfg
View file @ 56ad14fb
......@@ -89,7 +89,7 @@ command =
[template]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg.in
md5sum = 5c159864ce41394486867c27d65c2f50
md5sum = f40a938400e789361c95d5a9246bf0ef
output = ${buildout:directory}/template.cfg
mode = 0644
......@@ -98,7 +98,7 @@ recipe = hexagonit.recipe.download
ignore-existing = true
url = ${:_profile_base_location_}/instance-kvm.cfg.jinja2
mode = 644
md5sum = c66ec96a73470148707937bf96436fd6
md5sum = a849d4a6060fdb4e9e86917fb77ef153
download-only = true
on-update = true
......@@ -107,7 +107,7 @@ recipe = hexagonit.recipe.download
ignore-existing = true
url = ${:_profile_base_location_}/instance-kvm-cluster.cfg.jinja2.in
mode = 644
md5sum = 6ef7e38526f024356743e77973f0fb5f
md5sum = d9745bc9bd1d22a640a628c005f88ffb
download-only = true
on-update = true
......@@ -116,7 +116,7 @@ recipe = hexagonit.recipe.download
ignore-existing = true
url = ${:_profile_base_location_}/instance-kvm-resilient.cfg.jinja2
mode = 644
md5sum = 700676dff4a3835a9d6cde015d91922e
md5sum = 93e7143b46c6136b7cafe888fac90aba
download-only = true
on-update = true
......@@ -124,7 +124,7 @@ on-update = true
recipe = hexagonit.recipe.download
ignore-existing = true
url = ${:_profile_base_location_}/instance-kvm-import.cfg.jinja2.in
md5sum = 5e21ad759c148d26134ecefbb4d3d9f1
md5sum = dc3f3ad9ebd8b3b5c3ded57b91cee9c7
mode = 0644
download-only = true
on-update = true
......@@ -143,7 +143,7 @@ recipe = hexagonit.recipe.download
ignore-existing = true
url = ${:_profile_base_location_}/instance-kvm-export.cfg.jinja2
mode = 644
md5sum = c2ab8a5ff9d8cd351682382ea06b3328
md5sum = 13387d37bbf430f1d2b827c8f1acc804
download-only = true
on-update = true
......@@ -152,7 +152,7 @@ recipe = hexagonit.recipe.download
ignore-existing = true
url = ${:_profile_base_location_}/template/kvm-export.sh.jinja2
filename = kvm-export.sh.jinja2
md5sum = c18e958bf374d5fa35dbb952686595fb
md5sum = b617d64de73de1eed518185f310bbc82
download-only = true
mode = 0755
......
software/kvm/instance-kvm-cluster.cfg.jinja2.in
View file @ 56ad14fb
......@@ -80,7 +80,7 @@ config-data-to-vm = {{ dumps(kvm_parameter_dict.get('data-to-vm', '')) }}
config-disable-ansible-promise = {{ dumps(kvm_parameter_dict.get('disable-ansible-promise', False)) }}
config-enable-monitor = {{ enable_monitoring }}
config-monitor-cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.officejs.com') }}
config-monitor-username = ${monitor-htpasswd:username}
config-monitor-username = ${monitor-instance-parameter:username}
config-monitor-password = ${monitor-htpasswd:passwd}
# Enable simple http server on ipv6 so all VMs will access it
......@@ -226,16 +226,10 @@ mode = {{ mode }}
{{ writefile('cluster-data-content', '${directory:webroot}/${hash-code:passwd}/data', slapparameter_dict.get('cluster-data', ''), '700') }}
{% endif -%}
[monitor-htpasswd]
recipe = slapos.cookbook:generate.password
storage-path = ${directory:etc}/.monitor_user
bytes = 8
username = admin
[monitor-instance-parameter]
monitor-httpd-port = 8060
cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.officejs.com') }}
username = ${monitor-htpasswd:username}
username = admin
password = ${monitor-htpasswd:passwd}
[monitor-conf-parameters]
......@@ -254,7 +248,7 @@ recipe = slapos.cookbook:publish
{% if enable_monitoring -%}
{% set monitor_interface_url = slapparameter_dict.get('monitor-interface-url', 'https://monitor.app.officejs.com') -%}
{% do part_list.append('monitor-base') -%}
monitor-setup-url = {{ monitor_interface_url }}/#page=settings_configurator&url=${publish:monitor-url}&username=${publish:monitor-user}&password=${publish:monitor-password}
monitor-setup-url = {{ monitor_interface_url }}/#page=settings_configurator&url=${monitor-publish-parameters:monitor-url}&username=${monitor-publish-parameters:monitor-user}&password=${monitor-publish-parameters:monitor-password}
{% endif -%}
[buildout]
......
software/kvm/instance-kvm-export.cfg.jinja2
View file @ 56ad14fb
......@@ -35,3 +35,16 @@ context =
[publish-connection-information]
<= resilient-publish-connection-parameter
{% if str(slapparameter_dict.get('enable-monitor', True)).lower() == 'true' -%}
[monitor-instance-parameter]
monitor-httpd-port = 8026
monitor-title = {{ slapparameter_dict.get('name', 'KVM Standalone') }}
cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.officejs.com') }}
{% if slapparameter_dict.get('monitor-username', '') -%}
username = {{ slapparameter_dict['monitor-username'] }}
{% endif -%}
{% if slapparameter_dict.get('monitor-password', '') -%}
password = {{ slapparameter_dict['monitor-password'] }}
{% endif -%}
{% endif -%}
software/kvm/instance-kvm-import.cfg.jinja2.in
View file @ 56ad14fb
......@@ -4,16 +4,19 @@ eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
parts +=
kvm-disk-image-corruption-promise
# Here, we don't need KVM to run to import data, so we don't
# even extend the kvm instance profile.
extends =
{{ pbsready_import_template }}
[resilient-publish-connection-parameter]
monitor-base-url = ${publish:monitor-base-url}
monitor-url = ${publish:monitor-url}
monitor-user = ${publish:monitor-user}
monitor-password = ${publish:monitor-password}
monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
monitor-url = ${monitor-publish-parameters:monitor-url}
monitor-user = ${monitor-publish-parameters:monitor-user}
monitor-password = ${monitor-publish-parameters:monitor-password}
[monitor-instance-parameter]
monitor-httpd-port = 8276
......@@ -59,3 +62,21 @@ context =
raw gzip_binary {{ gzip_binary }}
backup-disk-path = ${directory:backup}/virtual.qcow2
[kvm-disk-image-corruption-promise]
# Check that disk image is not corrupted
recipe = collective.recipe.template
input = inline:#!/bin/sh
QEMU_IMAGE="${directory:srv}/virtual.qcow2"
if [ ! -s "$QEMU_IMAGE" ]; then
exit 0
fi
{{ qemu_location }}/bin/qemu-img check $QEMU_IMAGE
RETURN_CODE=$?
if [ $RETURN_CODE -eq 0 ] || [ $RETURN_CODE -eq 3 ]; then
exit 0
else
exit 1
fi
output = ${directory:promises}/kvm-disk-image-corruption
mode = 700
\ No newline at end of file
software/kvm/instance-kvm-resilient.cfg.jinja2
View file @ 56ad14fb
......@@ -68,8 +68,8 @@ backend-url = ${request-kvm:connection-backend-url}
url = ${request-kvm:connection-url}
ipv6 = ${request-kvm:connection-ip}
{% if monitor_dict -%}
monitor-base-url = ${publish:monitor-base-url}
monitor-setup-url = {{ monitor_interface_url }}/#page=settings_configurator&url=${publish:monitor-url}&username=${publish:monitor-user}&password=${publish:monitor-password}
monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
monitor-setup-url = {{ monitor_interface_url }}/#page=settings_configurator&url=${monitor-publish-parameters:monitor-url}&username=${monitor-publish-parameters:monitor-user}&password=${monitor-publish-parameters:monitor-password}
{% endif -%}
[kvm-frontend-url-promise]
......
software/kvm/instance-kvm.cfg.jinja2
View file @ 56ad14fb
......@@ -393,10 +393,10 @@ tap-ipv4 = ${slap-network-information:tap-ipv4}
{% endif %}
{% endif %}
{% if monitor -%}
monitor-base-url = ${publish:monitor-base-url}
monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
{% set monitor_interface_url = slapparameter_dict.get('monitor-interface-url', 'https://monitor.app.officejs.com') -%}
{% if monitor_interface_url -%}
monitor-setup-url = {{ monitor_interface_url }}/#page=settings_configurator&url=${publish:monitor-url}&username=${publish:monitor-user}&password=${publish:monitor-password}
monitor-setup-url = {{ monitor_interface_url }}/#page=settings_configurator&url=${monitor-publish-parameters:monitor-url}&username=${monitor-publish-parameters:monitor-user}&password=${monitor-publish-parameters:monitor-password}
{% endif -%}
{% endif -%}
......
software/kvm/instance.cfg.in
View file @ 56ad14fb
......@@ -149,6 +149,7 @@ extensions = jinja2.ext.do
context =
key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory
raw qemu_location ${kvm:location}
raw template_kvm_import ${template-kvm-import-script:location}/${template-kvm-import-script:filename}
raw pbsready_import_template ${pbsready-import:output}
key slapparameter_dict slap-configuration:configuration
......
software/kvm/software.cfg
View file @ 56ad14fb
......@@ -5,34 +5,34 @@ extends = common.cfg
# XXX - use websockify = 0.5.1 for compatibility with kvm frontend
websockify = 0.5.1
slapos.toolbox = 0.67
erp5.util = 0.4.46
slapos.toolbox = 0.69
erp5.util = 0.4.49
apache-libcloud = 1.1.0
collective.recipe.environment = 0.2.0
gitdb = 0.6.4
pycurl = 7.43.0
slapos.recipe.template = 2.9
slapos.recipe.template = 3.0
smmap = 0.9.0
# websockify = 0.8.0
# Required by:
# slapos.toolbox==0.61
# slapos.toolbox==0.69
GitPython = 2.0.8
# Required by:
# slapos.toolbox==0.61
# slapos.toolbox==0.69
atomize = 0.2.0
# Required by:
# slapos.toolbox==0.61
# slapos.toolbox==0.69
dnspython = 1.14.0
# Required by:
# slapos.toolbox==0.61
# slapos.toolbox==0.69
feedparser = 5.2.1
# Required by:
# slapos.toolbox==0.61
# slapos.toolbox==0.69
lockfile = 0.12.2
# Required by:
......@@ -40,9 +40,9 @@ lockfile = 0.12.2
numpy = 1.11.2rc1
# Required by:
# slapos.toolbox==0.61
# slapos.toolbox==0.69
paramiko = 2.0.2
# Required by:
# slapos.toolbox==0.61
# slapos.toolbox==0.69
passlib = 1.6.5
\ No newline at end of file
software/kvm/template/kvm-export.sh.jinja2
View file @ 56ad14fb
......@@ -2,6 +2,7 @@
#
# Create a backup of the disk image of the virtual machine
#
set -e
LC_ALL=C
export LC_ALL
BACKUP_DIR={{ directory['backup'] }}
......
software/monitor/instance-monitor.cfg.jinja2
View file @ 56ad14fb
......@@ -113,8 +113,8 @@ wrapper-path = ${monitor-directory:promises}/system-MEMORY-usage-check
[publish-connection-information]
recipe = slapos.cookbook:publish
monitor-setup-url = https://monitor.app.officejs.com/#page=settings_configurator&url=${publish:monitor-url}&username=${publish:monitor-user}&password=${publish:monitor-password}
server_log_url = ${publish:monitor-base-url}/${slap-configuration:private-hash}/
monitor-setup-url = https://monitor.app.officejs.com/#page=settings_configurator&url=${monitor-publish-parameters:monitor-url}&username=${monitor-publish-parameters:monitor-user}&password=${monitor-publish-parameters:monitor-password}
server_log_url = ${monitor-publish-parameters:monitor-base-url}/${slap-configuration:private-hash}/
[monitor-instance-parameter]
instance-configuration =
......
software/monitor/software.cfg
View file @ 56ad14fb
......@@ -32,7 +32,7 @@ mode = 0644
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/instance-monitor.cfg.jinja2
destination = ${buildout:directory}/template-base-monitor.cfg
md5sum = c513cf70b8671840b2ba049d2826a661
md5sum = c3fc83c851568bf0e56393676c489e18
mode = 0644
[template-monitor-distributor]
......@@ -102,6 +102,7 @@ scripts =
monitor.runpromise
monitor.genstatus
monitor.configwrite
is-process-older-than-dependency-set
[fluentd]
gems +=
......
software/nayuos/software.cfg
View file @ 56ad14fb
......@@ -11,7 +11,7 @@ parts +=
install-eggs-for-the-instance
[versions]
slapos.recipe.template = 2.8
slapos.recipe.template = 3.0
[verify-packages-exist-on-host]
recipe = plone.recipe.command
......
software/neoppod/instance-neo-admin.cfg.in
View file @ 56ad14fb
......@@ -25,12 +25,12 @@ path = ${directory:promises}/neo-admin-promise
< = logrotate-entry-base
name = neo-admin
log = ${neo-admin:logfile}
post = {{ bin_directory }}/slapos-kill -n neoadmin -s RTMIN+1 ${:log}
post = {{ bin_directory }}/slapos-kill -s RTMIN+1 -- ${neo-admin:binary} -l ${:log}
[neoctl]
recipe = slapos.cookbook:wrapper
command-line =
{{bin_directory}}/neoctl -a ${neo-admin:ip}:${neo-admin:port}
{{bin_directory}}/neoctl -a ${neo-admin:ip}:${neo-admin:port}
{%- if slapparameter_dict['ssl'] %}
--ca ${directory:etc}/ca.crt
--cert ${directory:etc}/neo.crt
......
software/neoppod/instance-neo-input-schema.json
View file @ 56ad14fb
......@@ -70,6 +70,11 @@
"default": 1,
"type": "integer"
},
"disable-drop-partitions": {
"description": "Set the --disable-drop-partitions option for storage nodes.",
"default": false,
"type": "boolean"
},
"mysql": {
"description": "Dictionary containing parameters for MySQL.",
"default": {},
......
software/neoppod/instance-neo-master.cfg.in
View file @ 56ad14fb
......@@ -33,4 +33,4 @@ path = ${directory:promises}/neo-master-promise
< = logrotate-entry-base
name = neo-master
log = ${neo-master:logfile}
post = {{ bin_directory }}/slapos-kill -n neomaster -s RTMIN+1 ${:log}
post = {{ bin_directory }}/slapos-kill -s RTMIN+1 -- ${neo-master:binary} -l ${:log}
software/neoppod/instance-neo-storage-mysql.cfg.in
View file @ 56ad14fb
......@@ -87,6 +87,7 @@ masters = ${publish:masters}
database-adapter = MySQL
wait-database = -1
engine = {{ slapparameter_dict.get('engine', '') }}
disable-drop-partitions = {{ dumps(bool(slapparameter_dict.get('disable-drop-partitions'))) }}
{% for i in range(slapparameter_dict.get('storage-count', 1)) -%}
{% set storage_id = 'neo-storage-' ~ i -%}
......@@ -101,7 +102,7 @@ database-parameters = root@neo{{ i }}${my-cnf-parameters:socket}
< = logrotate-entry-base
name = {{ storage_id }}
log = {{ '${' + storage_id + ':logfile}' }}
post = {{ bin_directory }}/slapos-kill -n neostorage -s RTMIN+1 ${:log}
post = {{ bin_directory }}/slapos-kill -s RTMIN+1 -- {{ bin_directory }}/neostorage -l ${:log}
{% endfor -%}
[init-script]
......
software/neoppod/software-common.cfg
View file @ 56ad14fb
......@@ -91,15 +91,15 @@ md5sum = f3259726bd5d824c569dc7db6b7d26a0
[instance-neo-admin]
<= download-base-neo
md5sum = d5eb14e3d83d7787950de913d7e29aad
md5sum = f671bb5bbb006d6bcecee8c7814bee57
[instance-neo-master]
<= download-base-neo
md5sum = c6e0db7344a26a55580102e3cc33c984
md5sum = 1fee10f02c2fa2a581e21878ca0fd704
[instance-neo-storage-mysql]
<= download-base-neo
md5sum = c1cdb4289ff458545de15aca65121b80
md5sum = 67d623d631c2f99e33bcabc79fc9cccf
[template-neo-my-cnf]
<= download-base-neo
......@@ -113,44 +113,44 @@ apache-libcloud = 1.5.0
ecdsa = 0.13
gitdb2 = 2.0.0
msgpack-python = 0.4.8
mysqlclient = 1.3.10
mysqlclient = 1.3.10+SlapOSPatched001
persistent = 4.2.3
pycrypto = 2.6.1
pycurl = 7.43.0
slapos.recipe.template = 2.10
slapos.toolbox = 0.67
slapos.recipe.template = 3.0
slapos.toolbox = 0.69
smmap2 = 2.0.1
transaction = 1.7.0
zodbpickle = 0.6.0
zodbtools = 0.0.0.dev3
# Required by:
# slapos.toolbox==0.67
# slapos.toolbox==0.69
GitPython = 2.1.3
# Required by:
# slapos.toolbox==0.67
# slapos.toolbox==0.69
PyRSS2Gen = 1.1
# Required by:
# slapos.toolbox==0.67
# slapos.toolbox==0.69
atomize = 0.2.0
# Required by:
# slapos.toolbox==0.67
# slapos.toolbox==0.69
dnspython = 1.14.0
# Required by:
# slapos.toolbox==0.67
# slapos.toolbox==0.69
feedparser = 5.2.1
# slapos.toolbox==0.67
# slapos.toolbox==0.69
lockfile = 0.12.2
# Required by:
# slapos.toolbox==0.67
# slapos.toolbox==0.69
paramiko = 2.1.2
# Required by:
# slapos.toolbox==0.67
# slapos.toolbox==0.69
passlib = 1.7.1
software/neoppod/software.cfg
View file @ 56ad14fb
......@@ -43,7 +43,7 @@ revision = c0adf81cfcfa2e4785d509e1c5d57559911e829a
[versions]
erp5.util = 0.4.46
erp5.util = 0.4.49
# To match ERP5
ZConfig = 2.9.3
zc.lockfile = 1.0.2
......
software/nginx-push-stream/software.cfg
View file @ 56ad14fb
......@@ -45,5 +45,5 @@ output = ${buildout:directory}/instance-nginx.cfg.in
mode = 0644
[versions]
slapos.recipe.template = 2.7
slapos.toolbox = 0.67
slapos.recipe.template = 3.0
slapos.toolbox = 0.69
software/powerdns/instance-powerdns-replicate.cfg.jinja2
View file @ 56ad14fb
......@@ -81,7 +81,7 @@ username = admin
[monitor-instance-parameter]
monitor-httpd-port = 8099
cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.officejs.com') }}
username = ${monitor-htpasswd:username}
username = ${monitor-instance-parameter:username}
password = ${monitor-htpasswd:passwd}
[monitor-conf-parameters]
......@@ -109,7 +109,7 @@ config-{{parameter}} = {{ value }}
{% endfor -%}
config-{{ slave_list_name }} = {{ json_module.dumps(slave_instance_list) }}
config-monitor-cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.node.vifib.com') }}
config-monitor-username = ${monitor-htpasswd:username}
config-monitor-username = ${monitor-instance-parameter:username}
config-monitor-password = ${monitor-htpasswd:passwd}
[publish-information]
......@@ -117,12 +117,12 @@ recipe = slapos.cookbook:publish
domain = {{ slapparameter_dict.get('domain') }}
slave-amount = {{ slave_instance_list | length }}
ns-record = {{ ns_record }}
monitor-url = ${publish:monitor-url}
monitor-user = ${publish:monitor-user}
monitor-password = ${publish:monitor-password}
monitor-url = ${monitor-publish-parameters:monitor-url}
monitor-user = ${monitor-publish-parameters:monitor-user}
monitor-password = ${monitor-publish-parameters:monitor-password}
{% set monitor_interface_url = slapparameter_dict.get('monitor-interface-url', 'https://monitor.app.officejs.com') -%}
{% if monitor_interface_url -%}
monitor-setup-url = {{ monitor_interface_url }}/#page=settings_configurator&url=${publish:monitor-url}
monitor-setup-url = {{ monitor_interface_url }}/#page=settings_configurator&url=${monitor-publish-parameters:monitor-url}
{% endif -%}
[buildout]
......@@ -154,4 +154,4 @@ cert_file = ${slap-connection:cert-file}
slave_instance_list =
-dns-quantity = 1
-dns-type = single-default
{%- endif %}
\ No newline at end of file
{%- endif %}
software/powerdns/instance-powerdns.cfg
View file @ 56ad14fb
......@@ -101,9 +101,9 @@ port = $${pdns:port}
[publish-connection-informations]
recipe = slapos.cookbook:publish
monitor-url = $${publish:monitor-url}
monitor-user = $${publish:monitor-user}
monitor-password = $${publish:monitor-password}
monitor-url = $${monitor-publish-parameters:monitor-url}
monitor-user = $${monitor-publish-parameters:monitor-user}
monitor-password = $${monitor-publish-parameters:monitor-password}
#####################
# Power DNS Slave configuration
......@@ -148,4 +148,4 @@ extends = ${monitor-template:output}
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
{% endif%}
\ No newline at end of file
{% endif%}
software/powerdns/software.cfg
View file @ 56ad14fb
......@@ -25,7 +25,7 @@ mode = 0644
[template-powerdns]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-powerdns.cfg
md5sum = 06ad5bc18936b04b0bd566353047d02f
md5sum = 7710007c0bd1817e3c1485ea9ece1e23
output = ${buildout:directory}/template-powerdns.cfg
mode = 0644
......@@ -38,7 +38,7 @@ mode = 640
[template-dns-replicate]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/instance-powerdns-replicate.cfg.jinja2
md5sum = ff2ac1f61fdd57fb349e209b9950f117
md5sum = 3699d0751632e0c11b466342789e27a3
mode = 0644
[iso-list]
......@@ -65,4 +65,4 @@ mode = 0644
PyRSS2Gen = 1.1
cns.recipe.symlink = 0.2.3
plone.recipe.command = 1.1
slapos.recipe.template = 2.8
slapos.recipe.template = 3.0
software/re6stnet/instance-re6stnet.cfg.in
View file @ 56ad14fb
......@@ -201,7 +201,7 @@ port = ${apache-conf:port}
{% do publish_dict.__setitem__('slave-amount', '${re6st-registry:slave-amount}') -%}
[publish]
recipe = slapos.cookbook:publish
monitor-setup-url = https://monitor.app.officejs.com/#page=settings_configurator&url=${publish:monitor-url}&username=${publish:monitor-user}&password=${publish:monitor-password}
monitor-setup-url = https://monitor.app.officejs.com/#page=settings_configurator&url=${monitor-publish-parameters:monitor-url}&username=${monitor-publish-parameters:monitor-user}&password=${monitor-publish-parameters:monitor-password}
{% for name, value in publish_dict.items() -%}
{{ name }} = {{ value }}
{% endfor -%}
......
software/re6stnet/software.cfg
View file @ 56ad14fb
......@@ -82,7 +82,7 @@ extra-context =
[template-re6stnet]
< = download-base
filename = instance-re6stnet.cfg.in
md5sum = 9e5e6cf010ec2a329e6bc8def7f95ba7
md5sum = 7d65efffeae3f9ae4e64699303a083b4
[template-apache-conf]
< = download-base
......@@ -111,16 +111,16 @@ gitdb = 0.6.4
plone.recipe.command = 1.1
pycrypto = 2.6.1
pycurl = 7.43.0
slapos.recipe.template = 2.7
slapos.toolbox = 0.67
slapos.recipe.template = 3.0
slapos.toolbox = 0.69
smmap = 0.9.0
# Required by:
# slapos.toolbox = 0.67
# slapos.toolbox==0.69
GitPython = 2.0.8
# Required by:
# slapos.toolbox = 0.67
# slapos.toolbox==0.69
atomize = 0.2.0
# Required by:
......@@ -128,11 +128,11 @@ atomize = 0.2.0
backports.ssl-match-hostname = 3.4.0.2
# Required by:
# slapos.toolbox = 0.67
# slapos.toolbox==0.69
feedparser = 5.1.3
# Required by:
# slapos.toolbox = 0.67
# slapos.toolbox==0.69
lockfile = 0.12.2
# Required by:
......@@ -140,10 +140,10 @@ lockfile = 0.12.2
miniupnpc = 1.9
# Required by:
# slapos.toolbox = 0.67
# slapos.toolbox==0.69
paramiko = 2.0.1
# Required by:
# slapos.toolbox = 0.67
# slapos.toolbox==0.69
rpdb = 0.1.5
software/seleniumrunner/software.cfg
View file @ 56ad14fb
......@@ -47,4 +47,4 @@ mode = 0644
[versions]
plone.recipe.command = 1.1
slapos.recipe.template = 2.7
slapos.recipe.template = 3.0
software/slapos-in-partition/software.cfg
View file @ 56ad14fb
......@@ -11,4 +11,4 @@ extends = common.cfg
Pygments = 1.6
collective.recipe.environment = 0.2.0
collective.recipe.template = 1.10
slapos.recipe.template = 2.7
slapos.recipe.template = 3.0
software/slapos-master/buildout.hash.cfg
View file @ 56ad14fb
......@@ -15,11 +15,11 @@
# not need these here).
[template-erp5]
filename = instance-erp5.cfg.in
md5sum = 1489091851f27c41243eeaa4967cc91c
md5sum = 50834c59ded302034cafcce7315a55c3
[template-balancer]
filename = instance-balancer.cfg.in
md5sum = a410f10fe9766c9321ffd7b18ace5a4f
md5sum = 5562e6738470b0fc96211f46bf8d41bc
[template-apache-backend-conf]
filename = apache-backend.conf.in
......
software/slapos-master/instance-balancer.cfg.in
View file @ 56ad14fb
......@@ -170,6 +170,7 @@ recipe = slapos.cookbook:publish.serialised
{{ family_name ~ '-v6' }} = {% if ipv6_set %}{{ scheme ~ '://[' ~ ipv6 ~ ']:' ~ apache_port }}{% endif %}
{{ family_name }} = {{ scheme ~ '://' ~ ipv4 ~ ':' ~ apache_port }}
{% endfor -%}
monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
[apache-ssl]
{% if ssl_parameter_dict.get('key') -%}
......@@ -232,11 +233,34 @@ private = ${:ca-dir}/private
certs = ${:ca-dir}/certs
newcerts = ${:ca-dir}/newcerts
crl = ${:ca-dir}/crl
apachedex = ${monitor-directory:private}/apachedex
[monitor-generate-apachedex-report]
recipe = slapos.cookbook:wrapper
wrapper-path = ${monitor-directory:reports}/${:command}
command-line = "{{ parameter_dict['run-apachedex-location'] }}" "{{ parameter_dict['apachedex-location'] }}" "${directory:apachedex}" --default "${apachedex-parameters:default}" --apache-log-list "${apachedex-parameters:apache-log-list}" --base-list "${apachedex-parameters:base-list}" --skip-base-list "${apachedex-parameters:skip-base-list}" --erp5-base-list "${apachedex-parameters:erp5-base-list}"
command = apachedex_every_3_hour
[apachedex-parameters]
default_parameter =
# XXX - Sample log file with curent date: apache_access.log-%(date)s.gz
# which will be equivalent to apache_access.log-20150112.gz if the date is 2015-01-12
apache-log-list = ${apache-conf-parameter-dict:access-log}
default = ${monitor-directory:etc}/apdex_default
base-list = ${monitor-directory:etc}/apdex_base_list
skip-base-list = ${monitor-directory:etc}/apdex_skip_base_list
erp5-base-list = ${monitor-directory:etc}/apdex_erp5_base_list
[monitor-instance-parameter]
monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }}
monitor-httpd-port = {{ next_port() }}
monitor-title = Balancer monitor
password = {{ slapparameter_dict['monitor-passwd'] }}
instance-configuration =
file apachedex-default ${apachedex-parameters:default}
file apachedex-base-list ${apachedex-parameters:base-list}
file apachedex-skip-base-list ${apachedex-parameters:skip-base-list}
file apachedex-erp5-base-list ${apachedex-parameters:erp5-base-list}
[buildout]
extends =
......@@ -245,4 +269,5 @@ extends =
parts +=
publish
logrotate-apache
monitor-generate-apachedex-report
{{ part_list | join('\n ') }}
software/slapos-master/instance-erp5.cfg.in
View file @ 56ad14fb
......@@ -8,6 +8,7 @@
{% set jupyter_dict = slapparameter_dict.get('jupyter', {}) -%}
{% set has_jupyter = jupyter_dict.get('enable', jupyter_enable_default.lower() in ('true', 'yes')) -%}
{% set jupyter_zope_family = jupyter_dict.get('zope-family', '') -%}
{% set monitor_base_url_dict = {} -%}
[request-common]
<= request-common-base
config-use-ipv6 = {{ dumps(slapparameter_dict.get('use-ipv6', False)) }}
......@@ -22,7 +23,10 @@ software-type = {{ software_type }}
return = {{ ' '.join(ret) }}
{% for ret, publish in ret.iteritems() -%}
{% if publish -%}
{% do publish_dict.__setitem__(name ~ '-' ~ ret, '${' ~ section ~ ':connection-' ~ ret ~ '}')%}
{% do publish_dict.__setitem__(name ~ '-' ~ ret, '${' ~ section ~ ':connection-' ~ ret ~ '}') %}
{% endif -%}
{% if ret == "monitor-base-url" -%}
{% do monitor_base_url_dict.__setitem__(section, '${' ~ section ~ ':connection-' ~ ret ~ '}') -%}
{% endif -%}
{% endfor -%}
{{ root_common.sla(name) }}
......@@ -34,10 +38,10 @@ config-{{ k }} = {{ '${' ~ v ~ '}' }}
{% endfor -%}
{% endmacro -%}
{{ request('memcached-persistent', 'kumofs', 'kumofs', {'tcpv4-port': 2000}) }}
{{ request('memcached-volatile', 'kumofs', 'memcached', {'tcpv4-port': 2010, 'ram-storage-size': 64}) }}
{{ request('cloudooo', 'cloudooo', 'cloudooo', {'tcpv4-port': 2020}) }}
{{ request('mariadb', 'mariadb', 'mariadb', {'tcpv4-port': 2099}, {'database-list': True, 'test-database-list': True}) }}
{{ request('memcached-persistent', 'kumofs', 'kumofs', {'tcpv4-port': 2000}, {'url': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
{{ request('memcached-volatile', 'kumofs', 'memcached', {'tcpv4-port': 2010, 'ram-storage-size': 64}, {'url': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
{{ request('cloudooo', 'cloudooo', 'cloudooo', {'tcpv4-port': 2020}, {'url': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
{{ request('mariadb', 'mariadb', 'mariadb', {'tcpv4-port': 2099}, {'database-list': True, 'test-database-list': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
{% if has_posftix -%}
{{ request('smtp', 'postfix', 'smtp', {'tcpv4-port': 2025, 'smtpd-sasl-user': 'erp5@nowhere'}, key_config={'smtpd-sasl-password': 'publish-early:smtpd-sasl-password'}) }}
{%- else %}
......@@ -74,7 +78,7 @@ connection-url = smtp://127.0.0.2:0/
{% else -%}
{{ assert(server_type == 'zeo', server_type) -}}
{# BBB: for compatibility, keep 'zodb' as partition_reference for ZEO -#}
{{ request('zodb', 'zodb-' ~ server_type, 'zodb-' ~ server_type, {'tcpv4-port': 2100, 'zodb-dict': server_dict}, dict.fromkeys(('storage-dict', 'tidstorage-ip', 'tidstorage-port'))) }}
{{ request('zodb', 'zodb-' ~ server_type, 'zodb-' ~ server_type, {'tcpv4-port': 2100, 'zodb-dict': server_dict}, dict.fromkeys(('storage-dict', 'tidstorage-ip', 'tidstorage-port', 'monitor-base-url')), key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
{% endif -%}
{% endfor -%}
......@@ -126,6 +130,7 @@ name = neo-${gen-neo-cluster-base:passwd}
return =
zope-address-list
hosts-dict
monitor-base-url
{% set bt5_default_list = 'erp5_full_text_myisam_catalog erp5_configurator_standard erp5_configurator_maxma_demo erp5_configurator_ung erp5_configurator_run_my_doc slapos_configurator' -%}
{% if has_jupyter -%}
{% set bt5_default_list = bt5_default_list + ' erp5_data_notebook' -%}
......@@ -141,6 +146,7 @@ config-inituser-login = {{ dumps(inituser_login) }}
config-inituser-password = ${publish-early:inituser-password}
config-kumofs-url = ${request-memcached-persistent:connection-url}
config-memcached-url = ${request-memcached-volatile:connection-url}
config-monitor-passwd = ${monitor-htpasswd:passwd}
config-mysql-test-url-list = ${request-mariadb:connection-test-database-list}
config-mysql-url-list = ${request-mariadb:connection-database-list}
config-site-id = {{ dumps(site_id) }}
......@@ -183,6 +189,7 @@ software-type = zope
[{{ section_name }}]
<= request-zope-base
name = {{ partition_name }}
{% do monitor_base_url_dict.__setitem__(section_name, '${' ~ section_name ~ ':connection-monitor-base-url}') -%}
{{ root_common.sla(partition_name) }}
config-name = {{ dumps(custom_name) }}
config-instance-count = {{ dumps(zope_parameter_dict.get('instance-count', 1)) }}
......@@ -284,10 +291,12 @@ name = balancer
software-type = balancer
{{ root_common.sla('balancer') }}
return =
monitor-base-url
{%- for family in zope_family_dict %}
{{ family }}
{{ family }}-v6
{% endfor -%}
{% do monitor_base_url_dict.__setitem__('request-balancer', '${' ~ 'request-balancer' ~ ':connection-monitor-base-url}') -%}
config-zope-family-dict = {{ dumps(zope_family_parameter_dict) }}
config-tcpv4-port = {{ dumps(balancer_dict.get('tcpv4-port', 2150)) }}
{% for zope_section_id, name in zope_address_list_id_dict.items() -%}
......@@ -300,6 +309,8 @@ config-ssl = {{ dumps(balancer_dict.get('ssl', {})) }}
config-backend-path-dict = {{ dumps(zope_backend_path_dict) }}
config-ssl-authentication-dict = {{ dumps(ssl_authentication_dict) }}
config-shared-certificate-authority-path = ${directory:ca-dir}
config-monitor-passwd = ${monitor-htpasswd:passwd}
[request-frontend-base]
{% if has_frontend -%}
......@@ -323,6 +334,7 @@ return = site_url
{% endif -%}{# if zope_partition_dict -#}
[publish]
<= monitor-publish
recipe = slapos.cookbook:publish.serialised
-extends = publish-early
{% if 'neo' in storage_dict -%}
......@@ -340,11 +352,25 @@ hosts-dict = {{ '${' ~ zope_address_list_id_dict.keys()[0] ~ ':connection-hosts-
{{ name }} = {{ value }}
{% endfor -%}
{{ root_common.common_section() }}
[monitor-instance-parameter]
monitor-httpd-port = 8386
{{ root_common.common_section() }}
[buildout]
extends = {{ template_monitor }}
parts +=
apache-certificate-authority
fix-ca-folder
publish
monitor-base
[monitor-conf-parameters]
monitor-title = ERP5 monitor
password = ${monitor-htpasswd:passwd}
[monitor-base-url-dict]
{% for key, value in monitor_base_url_dict.items() -%}
{{ key }} = {{ value }}
{% endfor %}
software/slapos-master/software.cfg
View file @ 56ad14fb
......@@ -33,7 +33,6 @@ update-command = ${:command}
eggs +=
suds
facebook-sdk
google-api-python-client
spyne
slapos.core
......@@ -77,7 +76,6 @@ revision = 2acee53ff365ef1f0db80800eb5f8f22a27546f2
[versions]
python-memcached = 1.47
facebook-sdk = 0.4.0
google-api-python-client = 1.2
facebook-sdk = 2.0.0
erp5diff = 0.8.1.6
software/slapos-testing/instance.cfg
View file @ 56ad14fb
......@@ -40,6 +40,10 @@ recipe = plone.recipe.command
command = echo "Updating setup...";cd $${slapos.core:location}; export PATH="$${slapos-test-runner:prepend-path}:$PATH"; export CPPFLAGS="$${environment:CPPFLAGS}"; export LDFLAGS="$${environment:LDFLAGS}"; export PYTHONPATH="$${environment:PYTHONPATH}"; export LOCAL_IPV4="$${environment:LOCAL_IPV4}"; ${python2.7:location}/bin/python setup.py test -n; ${python2.7:location}/bin/python setup.py test -n; ${python2.7:location}/bin/python setup.py test -n; ${python2.7:location}/bin/python setup.py test -n
update-command = $${:command}
[caucase]
<= download-source
repository = ${caucase-repository:location}
[slapos.cookbook]
<= download-source
repository = ${slapos.cookbook-repository:location}
......@@ -84,6 +88,7 @@ test-list =
$${slapos.recipe.cmmi:location}
$${slapos.toolbox:location}
$${erp5-util:location}
$${caucase:location}
prepend-path = ${curl:location}/bin:${openssl:location}/bin:${git:location}/bin:${libxslt:location}/bin:${python2.7:location}/bin
environment = environment
......
software/slapos-testing/software.cfg
View file @ 56ad14fb
......@@ -5,6 +5,7 @@ extends =
../../component/lxml-python/buildout.cfg
../../component/libxml2/buildout.cfg
../../component/libxslt/buildout.cfg
../../component/bcrypt/buildout.cfg
../../component/python-2.7/buildout.cfg
../../component/python-setuptools/buildout.cfg
../../component/zlib/buildout.cfg
......@@ -13,6 +14,7 @@ extends =
../../stack/slapos.cfg
parts =
caucase-repository
slapos.cookbook-repository
slapos.core-repository
slapos.recipe.template-repository
......@@ -30,8 +32,10 @@ eggs =
${lxml-python:egg}
${python-cryptography:egg}
${pycurl:egg}
${bcrypt:egg}
dnspython
Jinja2
caucase
erp5.util
slapos.cookbook
collective.recipe.template
......@@ -50,6 +54,10 @@ git-executable = ${git:location}/bin/git
forbid-download-cache = true
branch = master
[caucase-repository]
<= git-clone-repository
repository = https://lab.nexedi.com/nexedi/caucase.git
[slapos.cookbook-repository]
<= git-clone-repository
repository = https://lab.nexedi.com/nexedi/slapos.git
......@@ -81,7 +89,7 @@ repository = https://lab.nexedi.com/nexedi/erp5.git
[template]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg
md5sum = 51c82ee9139e26c7ab35be65263130c4
md5sum = 63253c24c5c09716a37f457f2a6b3b23
output = ${buildout:directory}/template.cfg
mode = 640
......@@ -89,4 +97,4 @@ mode = 640
Pygments = 2.1.3
collective.recipe.template = 1.10
plone.recipe.command = 1.1
slapos.recipe.template = 2.7
slapos.recipe.template = 3.0
software/slaprunner/buildout.hash.cfg
View file @ 56ad14fb
......@@ -15,27 +15,27 @@
# not need these here).
[template]
filename = instance.cfg
md5sum = 02755403c9f0b52d717160b0b2d0cfb7
md5sum = 713db528880282d568278f09458d2aab
[template-runner]
filename = instance-runner.cfg
md5sum = 07905ec872e84bc42c6ab089f5270ac2
md5sum = 8f49df215a1596efcec94e2cca009711
[template-runner-import-script]
filename = template/runner-import.sh.jinja2
md5sum = 5db39d7f56fd1f96ce6466e9055841a1
md5sum = 130193114cbbcd014af9704851410605
[instance-runner-import]
filename = instance-runner-import.cfg.in
md5sum = 9db9957f452bda370cb2d5cc2e833e85
md5sum = 5cfa49bcf20612844e1c50a85740d0b3
[template-runner-export-script]
filename = template/runner-export.sh.jinja2
md5sum = 86a429492dba25364f6ec2b318ba1f85
md5sum = ef87691a9675beb646ae1c99ef69e551
[instance-runner-export]
filename = instance-runner-export.cfg.in
md5sum = 852a2ed99af566d27e5e4403334a3376
md5sum = 00ddcaa6e571505d1930612d12de8fde
[template-resilient]
filename = instance-resilient.cfg.jinja2
......@@ -65,6 +65,10 @@ md5sum = f8446fcf254b4929eb828a9a1d7e5f62
filename = template/bash_profile.in
md5sum = 1c88cbca0c1e705eeb4b544ef4616097
[template-bashrc]
filename = template/bashrc.in
md5sum = d0b9060ec96d4d5b188e36c84240ea7d
[template-supervisord]
filename = template/supervisord.conf.in
md5sum = d294d0dafd265048399de6da8c96345f
......@@ -80,3 +84,7 @@ md5sum = acaac32cf1bd45714272468a89f4f119
[template-resilient-software-release-information]
filename = template/resilient_software_release_information.py.in
md5sum = 2451072826a9ad9425d62c9e9c7f6284
[template-slapuser-script]
filename = template/slapos-slapuser-script.in
md5sum = becafae59ab9973724e0edb238f4cb7a
software/slaprunner/common.cfg
View file @ 56ad14fb
......@@ -34,6 +34,7 @@ common-parts =
instance-runner-import
instance-runner-export
template-slapos-cfg
template-slapuser-script
# XXX: we have to manually add this for resilience
rdiff-backup
pbs-recipe-egg
......@@ -64,6 +65,7 @@ download-only = true
# Downloads from template directory into current directory
< = download-only-base
url = ${:_profile_base_location_}/template/${:filename}
location = ${buildout:parts-directory}/${:_buildout_section_name_}
[template]
< = template-base
......@@ -77,14 +79,14 @@ output = ${buildout:directory}/template-runner.cfg.in
< = template-download-base
filename = runner-import.sh.jinja2
[instance-runner-import]
< = template-base
output = ${buildout:directory}/instance-runner-import.cfg
[template-runner-export-script]
< = template-download-base
filename = runner-export.sh.jinja2
[instance-runner-import]
< = download-base
recipe = slapos.recipe.build:download
[instance-runner-export]
< = download-base
recipe = slapos.recipe.build:download
......@@ -102,41 +104,43 @@ recipe = slapos.recipe.build:download
[template_launcher]
< = download-base
recipe = slapos.recipe.build:download
location = ${buildout:parts-directory}/${:_buildout_section_name_}
[template-slapos-cfg]
< = template-download-base
location = ${buildout:parts-directory}/${:_buildout_section_name_}
filename = slapos.cfg.in
[template-parameters]
< = download-only-base
location = ${buildout:parts-directory}/${:_buildout_section_name_}
[template-bash-profile]
< = template-download-base
location = ${buildout:parts-directory}/${:_buildout_section_name_}
filename = bash_profile.in
[template-supervisord]
< = template-download-base
location = ${buildout:parts-directory}/${:_buildout_section_name_}
filename = supervisord.conf.in
[template-listener-slapgrid]
< = template-download-base
location = ${buildout:parts-directory}/${:_buildout_section_name_}
filename = listener_slapgrid.py.in
[monitor-check-webrunner-internal-instance]
< = template-download-base
destination = ${buildout:parts-directory}/${:filename}
destination = ${:location}/${:filename}
filename = monitor-check-webrunner-internal-instances.py
[template-resilient-software-release-information]
< = template-download-base
filename = resilient_software_release_information.py.in
[template-slapuser-script]
< = template-download-base
filename = slapos-slapuser-script.in
[template-bashrc]
< = template-download-base
filename = bashrc.in
[eggs]
recipe = zc.recipe.egg
eggs =
......
software/slaprunner/instance-runner-export.cfg.in
View file @ 56ad14fb
......@@ -70,7 +70,10 @@ rendered = ${directory:bin}/${slap-parameter:namebase}-exporter
wrapper = ${:rendered}
mode = 700
context =
import sys sys
import easy_install zc.buildout.easy_install
section directory directory
raw backup_wait_time {{ backup_wait_time }}
raw output_log_file ${directory:log}/resilient.log
raw shell_binary {{ bash_executable_location }}
raw rsync_binary {{ rsync_executable_location }}
......@@ -81,11 +84,25 @@ minimum = 8437
maximum = 8446
ip = ${slap-network-information:global-ipv6}
# XXX Redefine Monitor parameters for runner-export
[monitor-instance-parameter]
monitor-httpd-port = ${monitor-httpd-free-port:port}
# Pass some parameter to dispay in monitoring interface
{% if slapparameter_dict.get('name', '') -%}
monitor-title = {{ slapparameter_dict['name'] }}
{% endif -%}
cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.officejs.com') }}
{% if slapparameter_dict.get('monitor-username', '') -%}
username = {{ slapparameter_dict['monitor-username'] }}
{% endif -%}
{% if slapparameter_dict.get('monitor-password', '') -%}
password = {{ slapparameter_dict['monitor-password'] }}
{% endif -%}
{% if slapparameter_dict.get('monitor-url-list', '') -%}
monitor-url-list = {{ slapparameter_dict['monitor-url-list'] }}
{% endif -%}
instance-configuration =
httpdcors cors-domain ${slaprunner-httpd-cors:location} ${httpd-graceful-wrapper:output}
configuration-file-path = ${buildout:directory}/knowledge0.cfg
# Extends publish section with resilient parameters
[publish-connection-information]
......
software/slaprunner/instance-runner-import.cfg.in
View file @ 56ad14fb
[buildout]
extends = template-runner.cfg
${pbsready-import:output}
extends = {{ template_runner_path }}
{{ pbsready_import_template_path }}
parts +=
nginx_conf
......@@ -42,26 +42,26 @@ parts +=
recipe = slapos.cookbook:free_port
minimum = 49990
maximum = 49999
ip = $${slap-network-information:local-ipv4}
ip = ${slap-network-information:local-ipv4}
[runner-free-port]
recipe = slapos.cookbook:free_port
minimum = 50015
maximum = 50024
ip = $${slap-network-information:global-ipv6}
ip = ${slap-network-information:global-ipv6}
[slaprunner]
proxy_port = $${proxy-free-port:port}
runner_port = $${runner-free-port:port}
proxy_port = ${proxy-free-port:port}
runner_port = ${runner-free-port:port}
[supervisord-free-port]
recipe = slapos.cookbook:free_port
minimum = 39996
maximum = 40005
ip = $${slaprunner:ipv4}
ip = ${slaprunner:ipv4}
[supervisord]
port = $${supervisord-free-port:port}
port = ${supervisord-free-port:port}
[runner-sshd-port]
minimum = 22232
......@@ -74,17 +74,19 @@ recipe =
[importer]
recipe = slapos.recipe.template:jinja2
template = ${template-runner-import-script:location}/${template-runner-import-script:filename}
rendered = $${directory:bin}/$${slap-parameter:namebase}-importer
template = {{ importer_script_path }}
rendered = ${directory:bin}/${slap-parameter:namebase}-importer
# backward compatibility for resilient stack
wrapper = $${:rendered}
wrapper = ${:rendered}
mode = 700
restore-exit-code-file = $${directory:srv}/$${:restore-exit-code-file-basename}
restore-exit-code-file = ${directory:srv}/${:restore-exit-code-file-basename}
restore-exit-code-file-basename = importer-exit-code-file
restore-error-message-file = $${directory:srv}/$${:restore-error-message-file-basename}
restore-error-message-file = ${directory:srv}/${:restore-error-message-file-basename}
restore-error-message-file-basename = importer-error-message-file
resilient-log-basename = resilient.log
context =
import sys sys
import easy_install zc.buildout.easy_install
key backend_url slaprunner:access-url
key ipv4 slaprunner:ipv4
key ipv6 slaprunner:ipv6
......@@ -92,20 +94,20 @@ context =
key instance_folder slaprunner:instance_root
section directory directory
section supervisord supervisord
raw output_log_file $${directory:log}/$${:resilient-log-basename}
raw shell_binary ${bash:location}/bin/bash
raw rsync_binary ${rsync:location}/bin/rsync
raw restore_exit_code_file $${:restore-exit-code-file}
raw restore_error_message_file $${:restore-error-message-file}
raw output_log_file ${directory:log}/${:resilient-log-basename}
raw shell_binary {{ bash_executable_location }}
raw rsync_binary {{ rsync_executable_location }}
raw restore_exit_code_file ${:restore-exit-code-file}
raw restore_error_message_file ${:restore-error-message-file}
[importer-consistency-promise]
# Test that the importer script and "after-import" subscripts
# are not older than 2 days (1 day + some slack), and have succeeded
recipe = collective.recipe.template
input = inline: #!/bin/sh
EXIT_CODE_FILE="$${importer:restore-exit-code-file}"
RECENT_EXIT_CODE_FILE=$(find $${directory:srv} -maxdepth 1 -name "$${importer:restore-exit-code-file-basename}" -mtime -2)
RESILIENT_LOG_URL=$${publish:monitor-base-url}/log/$${importer:resilient-log-basename}
EXIT_CODE_FILE="${importer:restore-exit-code-file}"
RECENT_EXIT_CODE_FILE=$(find ${directory:srv} -maxdepth 1 -name "${importer:restore-exit-code-file-basename}" -mtime -2)
RESILIENT_LOG_URL=${monitor-publish-parameters:monitor-base-url}/log/${importer:resilient-log-basename}
if [ ! -f "$EXIT_CODE_FILE" ]; then
exit 0;
else
......@@ -113,23 +115,25 @@ input = inline: #!/bin/sh
echo "Consistency check is too old.";
exit 1;
else
cat $${importer:restore-error-message-file}
if [ -f "${importer:restore-error-message-file}" ]; then
cat ${importer:restore-error-message-file}
fi
echo "More information can be found here : $RESILIENT_LOG_URL";
exit $(cat $EXIT_CODE_FILE);
fi
fi
exit 1; # Something else went wrong
output = $${directory:promises}/importer-consistency-promise
output = ${directory:promises}/importer-consistency-promise
mode = 755
[software-release-deployment-promise]
recipe = collective.recipe.template
input = inline: #!/bin/sh
PROJECT_FILE=$(find "$${directory:etc}" -maxdepth 1 -name .project)
PROJECT_FILE=$(find "${directory:etc}" -maxdepth 1 -name .project)
if [ -z "$PROJECT_FILE" ]; then
exit 0;
else
INSTALLATION_FILE=$(find "$${runnerdirectory:software-root}" -mindepth 2 -maxdepth 2 -name .completed)
INSTALLATION_FILE=$(find "${runnerdirectory:software-root}" -mindepth 2 -maxdepth 2 -name .completed)
if [ -n "$INSTALLATION_FILE" ]; then
exit 0;
else
......@@ -137,41 +141,41 @@ input = inline: #!/bin/sh
fi
fi
exit 1
output = $${directory:promises}/software-release-deployment-promise
output = ${directory:promises}/software-release-deployment-promise
mode = 755
[resilient-software-release-information]
recipe = slapos.recipe.template
url = ${template-resilient-software-release-information:destination}/${template-resilient-software-release-information:filename}
output = $${directory:cgi-bin}/resilient_software_release_information.py
resilient-log-url = $${publish:monitor-base-url}/log/$${importer:resilient-log-basename}
url = {{ software_release_information_template }}
output = ${directory:cgi-bin}/resilient_software_release_information.py
resilient-log-url = ${monitor-publish-parameters:monitor-base-url}/log/${importer:resilient-log-basename}
mode = 0600
[slap-parameter]
auto-deploy-instance = false
auto-deploy = true
name = Webrunner import
monitor-cors-domains =
monitor-username = $${monitor-htpasswd:username}
monitor-password = $${monitor-htpasswd:passwd}
[resilient-publish-connection-parameter]
monitor-base-url = $${publish:monitor-base-url}
monitor-url = $${publish:monitor-url}
monitor-user = $${publish:monitor-user}
monitor-password = $${publish:monitor-password}
monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
monitor-url = ${monitor-publish-parameters:monitor-url}
monitor-user = ${monitor-publish-parameters:monitor-user}
monitor-password = ${monitor-publish-parameters:monitor-password}
[monitor-instance-parameter]
monitor-httpd-port = 8360
#monitor-title = $${slap-parameter:name}
#cors-domains = $${slap-parameter:monitor-cors-domains}
#username = $${slap-parameter:monitor-username}
#password = $${slap-parameter:monitor-password}
# Pass some parameter to dispay in monitoring interface
monitor-title = {{ slapparameter_dict.get('name', 'Webrunner import') }}
cors-domains = {{ slapparameter_dict.get('monitor-cors-domains', 'monitor.app.officejs.com') }}
{% if slapparameter_dict.get('monitor-username', '') -%}
username = {{ slapparameter_dict['monitor-username'] }}
{% endif -%}
{% if slapparameter_dict.get('monitor-password', '') -%}
password = {{ slapparameter_dict['monitor-password'] }}
{% endif -%}
instance-configuration =
raw takeover-url http://[$${resilient-web-takeover-httpd-configuration-file:listening-ip}]:$${resilient-web-takeover-httpd-configuration-file:listening-port}/
raw takeover-password $${resilient-web-takeover-password:passwd}
raw takeover-url http://[${resilient-web-takeover-httpd-configuration-file:listening-ip}]:${resilient-web-takeover-httpd-configuration-file:listening-port}/
raw takeover-password ${resilient-web-takeover-password:passwd}
configuration-file-path = ${buildout:directory}/knowledge0.cfg
[monitor-conf-parameters]
private-path-list +=
$${directory:logrotate-backup}
${directory:logrotate-backup}
software/slaprunner/instance-runner-input-schema.json
View file @ 56ad14fb
......@@ -57,11 +57,6 @@
"type": "integer",
"default" : 4
},
"instance-name": {
"title": "Displayed instance name",
"description": "Name of the instance, to show in the window title",
"type": "string"
},
"no-ipv4-frontend": {
"title": "No IPv4 frontend",
"description": "Prevent the slaprunner to order an IPv4 frontend for itself",
......@@ -129,12 +124,6 @@
"type": "string",
"default": "monitor.app.officejs.com"
},
"resilient-clone-number": {
"title": "Number of Resilient Clones",
"description": "Number of clones to be deployed when a resilient runner, if missing a single clone is provided. Its value can be 1 or 2",
"type": "integer",
"enum": [1, 2]
},
"instance-name": {
"title": "Instance Name",
"description": "Name of the instance, to show in the window title",
......
software/slaprunner/instance-runner-resilient-input-schema.json
View file @ 56ad14fb
......@@ -12,7 +12,8 @@
"description": "Amount of backup(s) to create. Each backup consists of a Pull Backup Server and a clone.",
"type": "integer",
"default": 1,
"enum": [1, 2],
"minimum": 0,
"maximum": 2,
"optional": true
},
"-sla-runner0-computer_guid": {
......
software/slaprunner/instance-runner.cfg
View file @ 56ad14fb
......@@ -38,6 +38,8 @@ parts =
## Monitoring part
monitor-base
monitor-check-webrunner-internal-instance
## Usability part
template-slapuser-script
extends = ${monitor2-template:rendered}
......@@ -114,7 +116,7 @@ home = $${directory:srv}/runner/
test = $${directory:srv}/test/
project = $${:home}/project
public = $${:home}/public
software-root = $${:home}/software
software-root = {{ slapparameter_dict.get('software-root', '$${:home}/software') }}
instance-root = $${:home}/instance
project-test = $${:test}/project
software-test = $${:test}/software
......@@ -544,7 +546,7 @@ init-password = $${runner-htpasswd:password}
ssh-command = ssh $${user-info:pw-name}@$${slap-network-information:global-ipv6} -p $${runner-sshd-port:port}
git-public-url = https://[$${httpd-parameters:global_ip}]:$${httpd-parameters:global_port}/git-public/
git-private-url = https://[$${httpd-parameters:global_ip}]:$${httpd-parameters:global_port}/git/
monitor-base-url = $${publish:monitor-base-url}
monitor-base-url = $${monitor-publish-parameters:monitor-base-url}
{% if slapparameter_dict.get('no-ipv4-frontend', 'false') == 'false' -%}
url = https://$${request-frontend:connection-domain}
webdav-url = $${request-httpd-frontend:connection-secure_access}/share/
......@@ -557,11 +559,11 @@ public-url = $${apache-httpd:access-url}/public/
{% if slapparameter_dict.get('instance-type', '') != 'resilient' -%}
{% set monitor_interface_url = slapparameter_dict.get('monitor-interface-url', 'https://monitor.app.officejs.com') -%}
monitor-setup-url = {{ monitor_interface_url }}/#page=settings_configurator&url=$${publish:monitor-url}&username=$${publish:monitor-user}&password=$${publish:monitor-password}
monitor-setup-url = {{ monitor_interface_url }}/#page=settings_configurator&url=$${monitor-publish-parameters:monitor-url}&username=$${monitor-publish-parameters:monitor-user}&password=$${monitor-publish-parameters:monitor-password}
{% else -%}
monitor-url = $${publish:monitor-url}
monitor-user = $${publish:monitor-user}
monitor-password = $${publish:monitor-password}
monitor-url = $${monitor-publish-parameters:monitor-url}
monitor-user = $${monitor-publish-parameters:monitor-user}
monitor-password = $${monitor-publish-parameters:monitor-password}
{% endif -%}
#---------------------------
#--
......@@ -712,7 +714,13 @@ rendered = $${buildout:directory}/.bash_profile
context =
raw path $PATH:${nano:location}/bin:${vim:location}/bin:${screen:location}/bin:${git:location}/bin:${curl:location}/bin:${python2.7:location}/bin:${tig:location}/bin:${zip:location}/bin
key workdir runnerdirectory:home
[bashrc]
recipe = slapos.recipe.template:jinja2
template = ${template-bashrc:location}/${template-bashrc:filename}
rendered = $${buildout:directory}/.bashrc
context =
raw bin_folder ${buildout:bin-directory}
#---------------------------
#--
......@@ -801,12 +809,8 @@ password = $${runner-htpasswd:password}
{% if slapparameter_dict.get('monitor-url-list', '') -%}
monitor-url-list = {{ slapparameter_dict['monitor-url-list'] }}
{% endif -%}
# check if not resilient runner
{% if not slapparameter_dict.get('authorized-key', '') -%}
# Pass some parameter to dispay in monitoring interface
instance-configuration =
httpdcors cors-domain $${slaprunner-httpd-cors:location} $${httpd-graceful-wrapper:output}
{% endif -%}
configuration-file-path = $${buildout:directory}/knowledge0.cfg
[monitor-check-webrunner-internal-instance]
......@@ -816,3 +820,12 @@ rendered = $${monitor-directory:reports}/$${:filename}
filename = monitor-check-webrunner-internal-instance
mode = 0744
## Slapuser slapos command script
[template-slapuser-script]
recipe = slapos.recipe.template:jinja2
template = ${template-slapuser-script:location}/${template-slapuser-script:filename}
rendered = $${buildout:bin-directory}/slapos
mode = 0744
context =
raw config_location $${slapos-cfg:rendered}
raw slapos_python_file_location ${buildout:bin-directory}/slapos
software/slaprunner/instance.cfg
View file @ 56ad14fb
......@@ -12,7 +12,7 @@ recipe = slapos.cookbook:softwaretype
default = $${instance-base-runner:rendered}
resilient = $${instance-resilient:rendered}
runner = $${instance-base-runner:rendered}
runner-import = ${instance-runner-import:output}
runner-import = $${template-runner-import:rendered}
runner-export = $${template-runner-export:rendered}
frozen = ${instance-frozen:output}
......@@ -55,12 +55,29 @@ mode = 640
context =
key pbsready_export_template_path template-pbsready-export:rendered
key template_runner_path instance-base-runner:rendered
key slapparameter_dict slap-configuration:configuration
raw backup_wait_time ${exporter-default-configuration:backup_wait_time}
raw exporter_script_path ${template-runner-export-script:location}/${template-runner-export-script:filename}
raw monitor_check_resilient_feed_template_path ${template-monitor-check-resilient-feed:location}/${template-monitor-check-resilient-feed:filename}
raw buildout_executable_location ${buildout:executable}
raw bash_executable_location ${bash:location}/bin/bash
raw rsync_executable_location ${rsync:location}/bin/rsync
[template-runner-import]
recipe = slapos.recipe.template:jinja2
template = ${instance-runner-import:target}
rendered = $${buildout:directory}/instance-runner-import.cfg
mode = 640
context =
key template_runner_path instance-base-runner:rendered
key slapparameter_dict slap-configuration:configuration
raw importer_script_path ${template-runner-import-script:location}/${template-runner-import-script:filename}
raw buildout_executable_location ${buildout:executable}
raw bash_executable_location ${bash:location}/bin/bash
raw rsync_executable_location ${rsync:location}/bin/rsync
raw pbsready_import_template_path ${pbsready-import:output}
raw software_release_information_template ${template-resilient-software-release-information:destination}/${template-resilient-software-release-information:filename}
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration
computer = $${slap-connection:computer-id}
......
software/slaprunner/software.cfg
View file @ 56ad14fb
......@@ -16,42 +16,42 @@ gitdb = 0.6.4
gunicorn = 19.4.5
prettytable = 0.7.2
pycurl = 7.43.0
slapos.recipe.template = 2.9
slapos.toolbox = 0.67
slapos.recipe.template = 3.0
slapos.toolbox = 0.69
smmap = 0.9.0
# Required by:
# slapos.toolbox==0.61
# slapos.toolbox==0.69
GitPython = 2.0.8
# Required by:
# slapos.toolbox==0.61
# slapos.toolbox==0.69
PyRSS2Gen = 1.1
# Required by:
# slapos.toolbox==0.61
# slapos.toolbox==0.69
atomize = 0.2.0
# Required by:
# slapos.toolbox==0.61
# slapos.toolbox==0.69
dnspython = 1.14.0
# Required by:
# slapos.toolbox==0.61
erp5.util = 0.4.46
# slapos.toolbox==0.69
erp5.util = 0.4.49
# Required by:
# slapos.toolbox==0.61
# slapos.toolbox==0.69
feedparser = 5.2.1
# Required by:
# slapos.toolbox==0.61
# slapos.toolbox==0.69
lockfile = 0.12.2
# Required by:
# slapos.toolbox==0.61
# slapos.toolbox==0.69
paramiko = 2.0.2
# Required by:
# slapos.toolbox==0.61
# slapos.toolbox==0.69
passlib = 1.6.5
software/slaprunner/template/bashrc.in 0 → 100644
View file @ 56ad14fb
export PATH={{ bin_folder }}:$PATH
software/slaprunner/template/runner-export.sh.jinja2
View file @ 56ad14fb
......@@ -4,71 +4,105 @@ export LC_ALL
umask 077
# Exit on any error, to prevent inconsistent backup
set -e
# Error on unset variable expansion
set -eu
# Redirect output to log
exec > >(tee -ai {{ output_log_file }})
exec > >(tee -ai '{{ output_log_file }}')
exec 2>&1
echo -e "\n\n$0 run at : $(date)"
srv_directory={{ directory['srv'] }}
sync_element () {
path=$1
backup_path=$2
shift 2
element_list=$*
# Concatenate the exclude file of each partition of webrunner
# to create a global exclude file.
exclude_content="instance/supervisord.socket"
for partition in $srv_directory/runner/instance/slappart*
do
exclude_file="$partition/srv/exporter.exclude"
if [ -e "$exclude_file" ]; then
partition_exclude_content_relative=$(cat "$exclude_file")
# For every line of the local exclude file, add the absolute path
for line in "$partition_exclude_content_relative"
do
if [ ! -z "$line" ]; then
exclude_content="$exclude_content\ninstance/$(basename $partition)/$line"
fi
done
fi
done
echo -e "$exclude_content" > $srv_directory/exporter.exclude
for element in $element_list
do
echo "Changing current directory to $path."
cd $path;
if [ -f $element ] || [ -d $element ]; then
command="{{ rsync_binary }} -rlptgov --stats --safe-links --exclude-from=$srv_directory/exporter.exclude --delete --delete-excluded $element $backup_path"
echo "Running : $command"
$command
fi
done
srv_directory='{{ directory["srv"] }}'
backup_directory='{{ directory["backup"] }}'
etc_directory='{{ directory["etc"] }}'
tmp_directory='{{ directory["tmp"] }}'
rsync () {
set -x
'{{ rsync_binary }}' -rlptgov --stats --safe-links --delete --delete-excluded "$@"
set +x
}
sync_element $srv_directory/runner {{ directory['backup'] }}/runner/ instance project proxy.db
(
# XXX: code duplication with runner-import.sh.jinja2
path=$srv_directory/runner
backup_path=$backup_directory/runner/
cd "$path"
if [ -d instance ]; then
# Concatenate the exclude file of each partition of webrunner
# to create a global exclude file.
# Also, ignore all buildout-managed files.
exclude=$({{ sys.executable }} - "$path" <<EOF
if 1:
import glob, errno, os, sys
sys.path[:0] = {{ repr(easy_install.buildout_and_setuptools_path) }}
from zc.buildout.configparser import parse
path = sys.argv[1]
def print_relative(path_list):
for p in path_list:
p = p.strip()
if p:
print(os.path.relpath(p, path))
print("*.sock")
print("*.socket")
print("*.pid")
print(".installed*.cfg")
for partition in glob.glob(path + "/instance/slappart*"):
os.chdir(partition)
try:
with open("srv/exporter.exclude") as f:
exclude = f.readlines()
except IOError as e:
if e.errno != errno.ENOENT:
raise
else:
print_relative(exclude)
for installed in glob.glob(".installed*.cfg"):
try:
with open(installed) as f:
installed = parse(f, installed)
except IOError as e:
if e.errno != errno.ENOENT:
raise
else:
for section in installed.itervalues():
print_relative(section.get(
'__buildout_installed__', '').splitlines())
EOF
)
echo "$exclude" |rsync --exclude-from=- instance "$backup_path"
fi
test -d project && rsync project "$backup_path"
test -d public && rsync public "$backup_path"
test -f proxy.db && rsync proxy.db "$backup_path"
)
# We sync .* appart
date +%s -u > {{ directory['etc'] }}/.resilient-timestamp
sync_element {{ directory['etc'] }} {{ directory['backup'] }}/etc/ config.json
# Hidden files are related to the webrunner's internals
cp -r {{ directory['etc'] }}/.??* {{ directory['backup'] }}/etc/
if [ -d {{ directory['backup'] }}/runner/software ]; then
rm {{ directory['backup'] }}/runner/software/*
(
cd "$etc_directory"
date +%s -u > .resilient-timestamp
rsync config.json "$backup_directory"/etc/
# Hidden files are related to the webrunner's internals
cp -r .??* "$backup_directory/etc/"
)
if [ -d "$backup_directory"/runner/software ]; then
rm "$backup_directory"/runner/software/*
fi
cd {{ directory['backup'] }} && find -type f ! -name backup.signature -print0 | xargs -0 sha256sum | LC_ALL=C sort -k 66 > backup.signature
(
cd "$backup_directory"
find -type f ! -name backup.signature -print0 | xargs -0 sha256sum | sort -k 66 > backup.signature
)
# Check that export didn't happen during backup of instances
tmp_backup_sum=$(mktemp -p {{ directory['tmp'] }})
tmp_filtered_signature=$(mktemp -p {{ directory['tmp'] }})
tmp_backup_sum=$(mktemp -p "$tmp_directory")
tmp_filtered_signature=$(mktemp -p "$tmp_directory")
remove_tmp_files () {
rm $tmp_backup_sum
rm $tmp_filtered_signature
rm "$tmp_backup_sum" "$tmp_filtered_signature"
}
trap remove_tmp_files EXIT
......@@ -83,15 +117,14 @@ if [ -z "$backup_directory_path" ]; then
fi
sleep 5
sha256sum $backup_directory_path | LC_ALL=C sort -k 66 > $tmp_backup_sum
cat {{ directory['backup'] }}/backup.signature | egrep "instance/slappart.*/srv/backup/" > $tmp_filtered_signature
sha256sum $backup_directory_path | sort -k 66 > "$tmp_backup_sum"
egrep "instance/slappart.*/srv/backup/" "$backup_directory/backup.signature" > "$tmp_filtered_signature"
# If the diff fails, then the notifier will restart this script
diff_status=0
diff $tmp_backup_sum $tmp_filtered_signature || diff_status=1
if [ $diff_status -ne 0 ]; then
echo "ERROR: Some backups are not consistents, exporter should be re-run."
echo "Let's sleep 10 minutes, to let the backup end..."
sleep 10m
exit 1
if diff "$tmp_backup_sum" "$tmp_filtered_signature"; then
exit 0
fi
echo "ERROR: Some backups are not consistent, exporter should be re-run."
echo "Let's sleep {{ backup_wait_time }} minutes, to let the backup end..."
sleep {{ backup_wait_time }}m
exit 1
software/slaprunner/template/runner-import.sh.jinja2
View file @ 56ad14fb
#!{{ shell_binary }}
set -e
LC_ALL=C
export LC_ALL
umask 077
# Exit on any error, to prevent inconsistent backup
# Error on unset variable expansion
set -eu
# Redirect output to log
exec > >(tee -ai {{ output_log_file }})
exec > >(tee -ai '{{ output_log_file }}')
exec 2>&1
RESTORE_EXIT_CODE_FILE="{{ restore_exit_code_file }}"
RESTORE_ERROR_MESSAGE_FILE="{{ restore_error_message_file }}"
echo -e "\n\n$0 run at : $(date)"
srv_directory='{{ directory["srv"] }}'
backup_directory='{{ directory["backup"] }}'
etc_directory='{{ directory["etc"] }}'
RESTORE_EXIT_CODE_FILE='{{ restore_exit_code_file }}'
RESTORE_ERROR_MESSAGE_FILE='{{ restore_error_message_file }}'
ERROR_MESSAGE=""
fail_with_exit_code () {
......@@ -17,7 +26,6 @@ fail_with_exit_code () {
echo -e "Failure during step : $ERROR_MESSAGE" > $RESTORE_ERROR_MESSAGE_FILE
exit 1
}
trap fail_with_exit_code ERR
log_message () {
......@@ -25,111 +33,160 @@ log_message () {
echo -e $1
}
# Delete the error message file, to not keep it even after a successful build
rm $RESTORE_ERROR_MESSAGE_FILE || true
srv_directory={{ directory['srv'] }}
restore_element () {
backup_path=$1
restore_path=$2
shift 2
element_list=$*
for element in $element_list
do
cd $backup_path;
if [ -f $element ] || [ -d $element ]; then
command="{{ rsync_binary }} --stats -av --delete --exclude *.sock --exclude *.socket --exclude *.pid --exclude .installed.cfg --exclude .installed-switch-softwaretype.cfg $backup_path/$element $restore_path"
echo "Running: \"$command\""
$command
fi
done
}
rm "$RESTORE_ERROR_MESSAGE_FILE" || true
echo -e "\n\nrunner-import run at : $(date)"
rsync () {
set -x
'{{ rsync_binary }}' -rlptgov --stats --safe-links --delete "$@"
set +x
}
log_message "Restoring WebRunner content..."
restore_element {{ directory['backup'] }}/runner/ $srv_directory/runner instance project proxy.db
(
# XXX: code duplication with runner-export.sh.jinja2
path=$srv_directory/runner
backup_path=$backup_directory/runner/
cd "$backup_path"
if [ -d instance ]; then
# Concatenate the exclude file of each partition of webrunner
# to create a global exclude file.
# Also, ignore all buildout-managed files.
exclude=$({{ sys.executable }} - "$path" <<EOF
if 1:
import glob, errno, os, sys
sys.path[:0] = {{ repr(easy_install.buildout_and_setuptools_path) }}
from zc.buildout.configparser import parse
path = sys.argv[1]
def print_relative(path_list):
for p in path_list:
p = p.strip()
if p:
print(os.path.relpath(p, path))
print("*.sock")
print("*.socket")
print("*.pid")
print(".installed*.cfg")
for partition in glob.glob(path + "/instance/slappart*"):
os.chdir(partition)
try:
with open("srv/exporter.exclude") as f:
exclude = f.readlines()
except IOError as e:
if e.errno != errno.ENOENT:
raise
else:
print_relative(exclude)
for installed in glob.glob(".installed*.cfg"):
try:
with open(installed) as f:
installed = parse(f, installed)
except IOError as e:
if e.errno != errno.ENOENT:
raise
else:
for section in installed.itervalues():
print_relative(section.get(
'__buildout_installed__', '').splitlines())
EOF
)
echo "$exclude" |rsync --exclude-from=- instance "$path"
fi
log_message "Restoring WebRunner config (etc directory)..."
restore_element {{ directory['backup'] }}/etc/ {{ directory['etc'] }} config.json
cp -r {{ directory['backup'] }}/etc/.??* {{ directory['etc'] }};
test -d project && rsync project "$path"
test -d public && rsync public "$path"
test -f proxy.db && rsync proxy.db "$path"
)
log_message "Restoring WebRunner config (etc directory)..."
(
cd "$backup_directory"/etc/
rsync config.json "$etc_directory"
# Hidden files are related to the webrunner's internals
cp -r .??* "$etc_directory"
)
# Invoke arbitrary script to perform specific restoration
# procedure.
runner_import_restore=$srv_directory/runner-import-restore
if [ ! -e "$runner_import_restore" ]; then
touch $runner_import_restore
chmod +x $runner_import_restore
if [ -x "$runner_import_restore" ]; then
log_message "Running $runner_import_restore..."
"$srv_directory/runner-import-restore"
fi
log_message "Running $runner_import_restore..."
$srv_directory/runner-import-restore
# If no "etc/.project" neither "srv/runner/proxy.db", we can safely assume
# that there is no instnace deployed on runner0
if [ ! -f "directory['etc']/.project" ] && [ ! -f "$srv_directory/runner/proxy.db" ]; then
# that there is no instance deployed on runner0
if [ ! -f "$etc_directory/.project" -a ! -f "$srv_directory/runner/proxy.db" ]; then
log_message "No Software Requested... Writing status file... End"
echo 0 > $RESTORE_EXIT_CODE_FILE
exit 0
fi
log_message "Updating slapproxy database..."
HOME="{{ directory['home'] }}"
HOME='{{ directory["home"] }}'
# XXX Hardcoded
export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
export MAKEFLAGS=-j4
SLAPOS="{{ directory['bin'] }}/slapos"
SLAPOS='{{ directory["bin"] }}'/slapos
# XXX hardcoded
SQLITE3="$HOME/software_release/parts/sqlite3/bin/sqlite3"
DATABASE="$HOME/srv/runner/proxy.db"
# Change slapproxy database to point instances to new software release
# XXX hardcoded
PARTITION=$(basename $HOME)
OLD_SOFTWARE_RELEASE=$($SQLITE3 $DATABASE "select software_release from partition11 where reference='slappart0';")
SOFTWARE_RELEASE=$(echo $OLD_SOFTWARE_RELEASE | sed -e 's/\(.*\)\(slappart\|test0-\)[0-9][0-9]\?/\1'"$PARTITION"'/')
$SQLITE3 $DATABASE "update partition11 set software_release='$SOFTWARE_RELEASE' where software_release NOT NULL;"
$SQLITE3 $DATABASE "update software11 set url='$SOFTWARE_RELEASE' where url='$OLD_SOFTWARE_RELEASE';" || $SQLITE3 $DATABASE "delete from software11 where url='$OLD_SOFTWARE_RELEASE';"
OLD_SOFTWARE_RELEASE=$("$SQLITE3" "$DATABASE" "select software_release from partition11 where reference='slappart0';")
SOFTWARE_RELEASE=$(echo $OLD_SOFTWARE_RELEASE | sed -e 's/\(.*\)\(slappart\|test0-\)[0-9][0-9][0-9]\?/\1'"$PARTITION"'/')
"$SQLITE3" "$DATABASE" "update partition11 set software_release='$SOFTWARE_RELEASE' where software_release NOT NULL;"
"$SQLITE3" "$DATABASE" "update software11 set url='$SOFTWARE_RELEASE' where url='$OLD_SOFTWARE_RELEASE';" || "$SQLITE3" "$DATABASE" "delete from software11 where url='$OLD_SOFTWARE_RELEASE';"
# Change slapproxy database to have all instances stopped
$SQLITE3 $DATABASE "update partition11 set requested_state='stopped';"
"$SQLITE3" "$DATABASE" "update partition11 set requested_state='stopped';"
# Change slapproxy database to get correct IPs
IPV4={{ ipv4 }}
IPV6={{ ipv6 }}
$SQLITE3 $DATABASE "update partition_network11 set address='$IPV4' where netmask='255.255.255.255';"
$SQLITE3 $DATABASE "update partition_network11 set address='$IPV6' where netmask='ffff:ffff:ffff::';"
IPV4='{{ ipv4 }}'
IPV6='{{ ipv6 }}'
"$SQLITE3" "$DATABASE" "update partition_network11 set address='$IPV4' where netmask='255.255.255.255';"
"$SQLITE3" "$DATABASE" "update partition_network11 set address='$IPV6' where netmask='ffff:ffff:ffff::';"
MASTERURL="http://{{ ipv4 }}:{{ proxy_port }}"
MASTERURL='http://{{ ipv4 }}:{{ proxy_port }}'
log_message "Removing old supervisord service description files..."
# XXX: Path hardcoded in slapos.core
rm {{ instance_folder }}/etc/supervisord.conf.d/* || true
rm '{{ instance_folder }}'/etc/supervisord.conf.d/* || true
SLAPOSCFG='{{ supervisord["slapos-cfg"] }}'
SLAPGRIDSRLOG='{{ supervisord["slapgrid-sr-log"] }}'
SLAPGRIDSRPID='{{ supervisord["slapgrid-sr-pid"] }}'
SLAPGRIDCPLOG='{{ supervisord["slapgrid-cp-log"] }}'
SLAPGRIDCPPID='{{ supervisord["slapgrid-cp-pid"] }}'
log_message "Building newest Software Release..."
$SLAPOS node software --cfg {{ supervisord['slapos-cfg'] }} --all --master-url=$MASTERURL --logfile {{ supervisord['slapgrid-sr-log'] }} --pidfile {{ supervisord['slapgrid-sr-pid'] }} >/dev/null 2>&1 ||
$SLAPOS node software --cfg {{ supervisord['slapos-cfg'] }} --all --master-url=$MASTERURL --logfile {{ supervisord['slapgrid-sr-log'] }} --pidfile {{ supervisord['slapgrid-sr-pid'] }} >/dev/null 2>&1 ||
$SLAPOS node software --cfg {{ supervisord['slapos-cfg'] }} --all --master-url=$MASTERURL --logfile {{ supervisord['slapgrid-sr-log'] }} --pidfile {{ supervisord['slapgrid-sr-pid'] }} >/dev/null 2>&1 ||
(tail -n 200 {{ supervisord['slapgrid-sr-log'] }} && false)
"$SLAPOS" node software --cfg "$SLAPOSCFG" --all --master-url="$MASTERURL" --logfile "$SLAPGRIDSRLOG" --pidfile "$SLAPGRIDSRPID" >/dev/null 2>&1 ||
"$SLAPOS" node software --cfg "$SLAPOSCFG" --all --master-url="$MASTERURL" --logfile "$SLAPGRIDSRLOG" --pidfile "$SLAPGRIDSRPID" >/dev/null 2>&1 ||
"$SLAPOS" node software --cfg "$SLAPOSCFG" --all --master-url="$MASTERURL" --logfile "$SLAPGRIDSRLOG" --pidfile "$SLAPGRIDSRPID" >/dev/null 2>&1 ||
(tail -n 200 "$SLAPGRIDSRLOG" && false)
# Remove defined scripts to force buildout to recreate them to have updated paths
rm $srv_directory/runner/instance/slappart*/srv/runner-import-restore || true
rm "$srv_directory"/runner/instance/slappart*/srv/runner-import-restore || true
log_message "Fixing Instances as needed after import..."
# XXX hardcoded
$SLAPOS node instance --cfg {{ supervisord['slapos-cfg'] }} --master-url=$MASTERURL --logfile {{ supervisord['slapgrid-cp-log'] }} --pidfile {{ supervisord['slapgrid-cp-pid'] }} >/dev/null 2>&1 ||
$SLAPOS node instance --cfg {{ supervisord['slapos-cfg'] }} --master-url=$MASTERURL --logfile {{ supervisord['slapgrid-cp-log'] }} --pidfile {{ supervisord['slapgrid-cp-pid'] }} >/dev/null 2>&1 ||
$SLAPOS node instance --cfg {{ supervisord['slapos-cfg'] }} --master-url=$MASTERURL --logfile {{ supervisord['slapgrid-cp-log'] }} --pidfile {{ supervisord['slapgrid-cp-pid'] }} >/dev/null 2>&1 ||
(tail -n 200 {{ supervisord['slapgrid-cp-log'] }} && false)
"$SLAPOS" node instance --cfg "$SLAPOSCFG" --master-url=$MASTERURL --logfile "$SLAPGRIDCPLOG" --pidfile "$SLAPGRIDCPPID" >/dev/null 2>&1 ||
"$SLAPOS" node instance --cfg "$SLAPOSCFG" --master-url=$MASTERURL --logfile "$SLAPGRIDCPLOG" --pidfile "$SLAPGRIDCPPID" >/dev/null 2>&1 ||
"$SLAPOS" node instance --cfg "$SLAPOSCFG" --master-url=$MASTERURL --logfile "$SLAPGRIDCPLOG" --pidfile "$SLAPGRIDCPPID" >/dev/null 2>&1 ||
(tail -n 200 "$SLAPGRIDCPLOG" && false)
# Invoke defined scripts for each partition inside of slaprunner
log_message "Invoke custom import scripts defined by each instances..."
for partition in $srv_directory/runner/instance/slappart*/
for partition in "$srv_directory"/runner/instance/slappart*/
do
script=$partition/srv/runner-import-restore
if [ -e "$script" ]; then
if [ -x "$script" ]; then
log_message "Running custom instance script : $script..."
$script
"$script"
fi
done
# Change back slapproxy database to have all instances started
log_message "Set instances as to start after takeover..."
$SQLITE3 $DATABASE "update partition11 set requested_state='started';"
"$SQLITE3" "$DATABASE" "update partition11 set requested_state='started';"
# Write exit code to an arbitrary file that will be checked by promise/monitor
log_message "Writing status file... End"
......
software/slaprunner/template/slapos-slapuser-script.in 0 → 100644
View file @ 56ad14fb
#!/bin/sh
# run slapos command inside slaprunner with appropriate config
SLAPOS_CONFIGURATION={{ config_location }} \
SLAPOS_CLIENT_CONFIGURATION=$SLAPOS_CONFIGURATION \
exec {{ slapos_python_file_location }} "$@"
software/test-slave-instance-deployment/software.cfg
View file @ 56ad14fb
......@@ -59,7 +59,7 @@ eggs = collective.recipe.template
collective.recipe.template = 1.11
plone.recipe.command = 1.1
slapos.recipe.build = 0.28
slapos.recipe.template = 2.7
slapos.recipe.template = 3.0
# Replicate slapos stack, but without shacache to not have to compile the entire world for a simple test.
[buildout]
......@@ -138,5 +138,5 @@ setuptools = 0.9.8
prettytable = 0.7.3-nxd001
[networkcache]
download-cache-url = http://www.shacache.org/shacache
download-dir-url = http://www.shacache.org/shadir
\ No newline at end of file
download-cache-url = http://download.shacache.org
download-dir-url = http://dir.shacache.org
software/unstable/lte/instance.jinja2.cfg 0 → 100644
View file @ 56ad14fb
#
# Deploy LTE instance
#
[buildout]
parts =
directory
lte
eggs-directory = {{ buildout['eggs-directory'] }}
develop-eggs-directory = {{ buildout['develop-eggs-directory'] }}
offline = true
[instance-parameter]
recipe = slapos.cookbook:slapconfiguration
computer = ${slap_connection:computer_id}
partition = ${slap_connection:partition_id}
url = ${slap_connection:server_url}
key = ${slap_connection:key_file}
cert = ${slap_connection:cert_file}
[directory]
recipe = slapos.cookbook:mkdirectory
home = ${buildout:directory}
etc = ${:home}/etc
var = ${:home}/var
script = ${:etc}/run
service = ${:etc}/service
promise = ${:etc}/promise
log = ${:var}/log
[lte]
recipe = slapos.recipe.build:download
url = {{ buildout['directory'] }}/lte.sh
destination = ${directory:service}/lte.sh
mode = 0775
md5sum = 9de4c400d411083d8ecd8d03d62b97f0
software/unstable/lte/lte.sh 0 → 100644
View file @ 56ad14fb
#!/bin/bash
function stopLTE {
sudo /bin/systemctl stop lte
echo "LTE service stopped"
exit 0
}
trap stopLTE TERM INT KILL
sudo /bin/systemctl start lte
while (( 1 )); do
sleep 1
done
software/unstable/lte/readme.rst 0 → 100644
View file @ 56ad14fb
LTE eNodeB software release
###########################
Simple software release which starts LTE service upon instantiation.
LTE is managed by systemd and we are taking advantage of this.
It is necessary to install SlapOS from "amarisoft" branch because
it has modified ``slapos node format`` to give group ``slapsoft``
the rights to operate ``systemctl <start/stop/enable/disable> lte``.
software/unstable/lte/software.cfg 0 → 100644
View file @ 56ad14fb
[buildout]
extends =
../../../stack/slapos.cfg
parts =
slapos-cookbook
instance-profile
lte.sh
[instance-profile]
recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/instance.jinja2.cfg
rendered = ${buildout:directory}/instance.cfg
md5sum = 3bbbb5297db5dd49f5dfc158e4291d4c
mode = 0644
extensions = jinja2.ext.do
context =
section buildout buildout
[copy-to-instance]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_buildout_section_name_}
destination = ${buildout:directory}/${:_buildout_section_name_}
[lte.sh]
<= copy-to-instance
md5sum = 9de4c400d411083d8ecd8d03d62b97f0
software/varnish/software.cfg
View file @ 56ad14fb
......@@ -62,4 +62,4 @@ md5sum = 0ea12a4ad2d2e3d406476e35b8d3e3fb
mode = 640
[versions]
slapos.recipe.template = 2.9
slapos.recipe.template = 3.0
stack/boinc/buildout.cfg
View file @ 56ad14fb
......@@ -69,7 +69,7 @@ pycrypto = 2.6
apache-libcloud = 0.12.4
async = 0.6.1
gitdb = 0.5.4
mysqlclient = 1.3.10
mysqlclient = 1.3.10+SlapOSPatched001
plone.recipe.command = 1.1
slapos.recipe.template = 2.4.2
slapos.toolbox = 0.40.4
......
stack/caucase/buildout.cfg 0 → 100644
View file @ 56ad14fb
[buildout]
extends =
buildout.hash.cfg
../../component/apache/buildout.cfg
../../component/nginx/buildout.cfg
../../component/curl/buildout.cfg
../../component/dash/buildout.cfg
../../component/openssl/buildout.cfg
../../component/bcrypt/buildout.cfg
../../stack/logrotate/buildout.cfg
parts =
instance-caucase
[extra-eggs]
recipe = zc.recipe.egg
interpreter = python_ca
eggs =
gunicorn # for WSGI HTTP Server
futures
caucase # certificate authority
${bcrypt:egg}
# are also required
plone.recipe.command
collective.recipe.template
slapos.toolbox
scripts =
slapos-kill
gunicorn
caucase
caucase-cli
caucase-cliweb
[template-ca-download-base]
recipe = hexagonit.recipe.download
ignore-existing = true
download-only = true
url = ${:_profile_base_location_}/${:filename}
mode = 0644
[template-httpd-auth-conf]
<= template-ca-download-base
[template-nginx-ca-conf]
<= template-ca-download-base
[template-caucase]
<= template-ca-download-base
url = ${:_profile_base_location_}/${:filename}
[template-authenticated-server]
recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/${:filename}
rendered = ${buildout:directory}/template-authenticated-server.cfg
context =
key apache_location apache:location
key template_logrotate_base template-logrotate-base:rendered
raw certificate_request_bin ${buildout:directory}/bin/caucase-cliweb
raw curl_executable_location ${curl:location}/bin/curl
raw dash_executable_location ${dash:location}/bin/dash
raw slapos_kill_bin ${buildout:directory}/bin/slapos-kill
raw template_httpd_auth_conf ${template-httpd-auth-conf:location}/${template-httpd-auth-conf:filename}
raw openssl_executable_location ${openssl:location}/bin/openssl
raw python_bin ${buildout:directory}/bin/${extra-eggs:interpreter}
[instance-caucase]
recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/${:filename}
rendered = ${buildout:directory}/template.cfg
context =
key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory
key nginx_location nginx:location
key template_logrotate_base template-logrotate-base:rendered
raw caucase_template ${template-caucase:location}/${template-caucase:filename}
raw curl_executable_location ${curl:location}/bin/curl
raw caucase_bin ${buildout:directory}/bin/caucase
raw certificate_request_bin ${buildout:directory}/bin/caucase-cliweb
raw template_nginx_ca_conf ${template-nginx-ca-conf:location}/${template-nginx-ca-conf:filename}
raw dash_executable_location ${dash:location}/bin/dash
raw slapos_kill_bin ${buildout:directory}/bin/slapos-kill
raw gunicorn_bin ${buildout:directory}/bin/gunicorn
raw openssl_executable_location ${openssl:location}/bin/openssl
raw python_bin ${buildout:directory}/bin/${extra-eggs:interpreter}
[versions]
Flask-User = 0.6.11
SQLAlchemy = 1.1.9
caucase = 0.1.3
futures = 3.1.1
gunicorn = 19.7.1
slapos.recipe.template = 3.0
APacheDEX = 1.6.2
bcrypt = 3.1.3
# Required by:
# caucase==0.1.3
Flask-AlchemyDumps = 0.0.10
# Required by:
# Flask-User==0.6.11
Flask-Login = 0.4.0
# Required by:
# Flask-User==0.6.11
Flask-Mail = 0.9.1
# Required by:
# Flask-AlchemyDumps==0.0.10
# Flask-User==0.6.11
Flask-SQLAlchemy = 2.2
# Required by:
# Flask-AlchemyDumps==0.0.10
Flask-Script = 2.0.5
# Required by:
# Flask-User==0.6.11
Flask-WTF = 0.14.2
# Required by:
# Flask-AlchemyDumps==0.0.10
Unipath = 1.1
# Required by:
# Flask-WTF==0.14.2
WTForms = 2.1
# Required by:
# Flask-Mail==0.9.1
blinker = 1.4
# Required by:
# caucase==0.1.3
pem = 16.1.0
# Required by:
# Flask-User==0.6.11
pycryptodome = 3.4.6
stack/caucase/buildout.hash.cfg 0 → 100644
View file @ 56ad14fb
# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
# The only allowed lines here are (regexes):
# - "^#" comments, copied verbatim
# - "^[" section beginings, copied verbatim
# - lines containing an "=" sign which must fit in the following categorie.
# - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
# But avoid directories, they are not portable.
# Copied verbatim.
# - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
# by the re-generation script.
# Re-generated.
# - other lines are copied verbatim
# Substitution (${...:...}), extension ([buildout] extends = ...) and
# section inheritance (< = ...) are NOT supported (but you should really
# not need these here).
[template-httpd-auth-conf]
filename = template-httpd-auth.conf.in
md5sum = ea445b0a9b143d12b5700a71ac06293c
[template-nginx-ca-conf]
filename = ca-nginx.conf.in
md5sum = d8bebf1629aacffd619541f363687b4a
[template-authenticated-server]
filename = instance-auth-server.cfg.jinja2.in
md5sum = a317d2f948cd3d16c860d05cc07ecf42
[template-caucase]
filename = instance-caucase.cfg.jinja2.in
md5sum = b801dfe4212ff97dc29191e4610df3a3
[instance-caucase]
filename = instance.cfg.in
md5sum = eb9d2ab646717d123b0472da5194d77f
stack/caucase/ca-nginx.conf.in 0 → 100644
View file @ 56ad14fb
worker_processes {{ parameter_dict['workers-processes'] }};
pid {{ parameter_dict['pid-file'] }};
error_log {{ parameter_dict['error-log'] }};
daemon off;
events {
worker_connections 1024;
accept_mutex off;
}
http {
# include mime.types;
default_type application/octet-stream;
access_log {{ parameter_dict['access-log'] }} combined;
client_max_body_size 10M;
map $http_upgrade $connection_upgrade {
default upgrade;
'' close;
}
sendfile on;
upstream app_server {
# for UNIX domain socket setups
server unix:{{ parameter_dict['socket'] }} fail_timeout=0;
}
{% if parameter_dict['cert-file'] and parameter_dict['key-file'] -%}
server {
listen [{{ parameter_dict['ip'] }}]:{{ parameter_dict['https-port'] }} ssl;
server_name _;
ssl_certificate {{ parameter_dict['cert-file'] }};
ssl_certificate_key {{ parameter_dict['key-file'] }};
ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
ssl_ciphers ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:HIGH:!aNULL:!MD5;
ssl_prefer_server_ciphers on;
keepalive_timeout 90s;
client_body_temp_path {{ parameter_dict['client-body-temp-path'] }};
proxy_temp_path {{ parameter_dict['proxy-temp-path'] }};
fastcgi_temp_path {{ parameter_dict['fastcgi-temp-path'] }};
uwsgi_temp_path {{ parameter_dict['uwsgi-temp-path'] }};
scgi_temp_path {{ parameter_dict['scgi-temp-path'] }};
location / {
proxy_redirect off;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Host $http_host;
proxy_set_header Host $http_host;
proxy_set_header Authorization $http_authorization;
proxy_pass_header Authorization;
proxy_connect_timeout 90;
proxy_send_timeout 90;
proxy_read_timeout 90;
send_timeout 90;
proxy_pass http://app_server;
}
}
{% endif -%}
server {
listen [{{ parameter_dict['ip'] }}]:{{ parameter_dict['port'] }};
server_name _;
keepalive_timeout 90s;
client_body_temp_path {{ parameter_dict['client-body-temp-path'] }};
proxy_temp_path {{ parameter_dict['proxy-temp-path'] }};
fastcgi_temp_path {{ parameter_dict['fastcgi-temp-path'] }};
uwsgi_temp_path {{ parameter_dict['uwsgi-temp-path'] }};
scgi_temp_path {{ parameter_dict['scgi-temp-path'] }};
location ~ ^(/admin|/user) {
# http is not used for /admin and /user
}
location / {
proxy_redirect off;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Host $http_host;
proxy_set_header Host $http_host;
proxy_set_header Authorization $http_authorization;
proxy_pass_header Authorization;
proxy_connect_timeout 90;
proxy_send_timeout 90;
proxy_read_timeout 90;
send_timeout 90;
proxy_pass http://app_server;
}
}
}
stack/caucase/instance-auth-server.cfg.jinja2.in 0 → 100644
View file @ 56ad14fb
[buildout]
extends =
{{ template_logrotate_base }}
parts =
authenticated-httpd-server
[authenticated-server-parameters]
ca-url =
common-name = instance@${slap-configuration:instance-title}
server-port = 8286
custom-httpd-file =
web-directory = ${directory:document-root}
[directory]
recipe = slapos.cookbook:mkdirectory
etc = ${buildout:directory}/etc
bin = ${buildout:directory}/bin
srv = ${buildout:directory}/srv
var = ${buildout:directory}/var
ssl = ${:etc}/ssl
run = ${:var}/run
log = ${:var}/log
scripts = ${:etc}/run
services = ${:etc}/service
promises = ${:etc}/promise
document-root = ${:srv}/private
[certificate-request-base]
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:bin}/request-instance-certificate
cert-file = ${directory:ssl}/instance.cert.pem
key-file = ${directory:ssl}/instance.key.pem
ca-cert = ${directory:ssl}/cacert.pem
parameters-extra = true
command-line = {{ certificate_request_bin }}
--crt-file ${:cert-file}
--key-file ${:key-file}
--ca-url ${authenticated-server-parameters:ca-url}
--ca-crt-file ${:ca-cert}
--no-check-certificate
# XXX - using --no-check-certificate which is insecure with https.
[server-certificate-request]
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:scripts}/request-instance-certificate
command-line =
${certificate-request-base:wrapper-path}
--cn ${authenticated-server-parameters:common-name}
--request
[authenticated-httpd-conf-parameter]
ip = ${slap-configuration:ipv6-random}
port = ${authenticated-server-parameters:server-port}
pid-file = ${directory:run}/httpd-auth.pid
dav-lock = ${directory:var}/DavLockdb
access-log = ${directory:log}/httpd-auth-access.log
error-log = ${directory:log}/httpd-auth-error.log
cert-file = ${certificate-request-base:cert-file}
key-file = ${certificate-request-base:key-file}
ca-cert = ${certificate-request-base:ca-cert}
url = https://[${:ip}]:${:port}
private = ${authenticated-server-parameters:web-directory}
httpd-include-file = ${authenticated-server-parameters:custom-httpd-file}
crl =
[authenticated-httpd-conf]
recipe = slapos.recipe.template:jinja2
template = {{ template_httpd_auth_conf }}
rendered = ${directory:etc}/httpd-auth.conf
mode = 0744
context =
section parameter_dict authenticated-httpd-conf-parameter
[authenticated-httpd-graceful]
recipe = collective.recipe.template
input = inline:
#!{{ dash_executable_location }}
kill -USR1 $(cat ${authenticated-httpd-conf-parameter:pid-file})
output = ${directory:scripts}/authenticated-httpd-graceful
mode = 700
[authenticated-httpd-server]
recipe = slapos.cookbook:wrapper
command-line = {{ apache_location }}/bin/httpd -f ${authenticated-httpd-conf:rendered} -DFOREGROUND
wrapper-path = ${directory:services}/authenticated-httpd-server
wait-for-files =
${certificate-request-base:cert-file}
${certificate-request-base:key-file}
${certificate-request-base:ca-cert}
url = ${authenticated-httpd-conf-parameter:url}
depends =
${authenticated-httpd-promise:filename}
${authenticated-httpd-graceful:output}
${server-certificate-request:wrapper-path}
${logrotate-authenticated-httpd:name}
${certificate-renew-cron-entry:name}
[certificate-renew]
recipe = collective.recipe.template
input = inline:
#!{{ dash_executable_location }}
d=$({{ openssl_executable_location }} x509 -enddate -noout -in ${certificate-request-base:cert-file} | cut -d'=' -f 2)
cert_time=$(date -d "$d" +"%s")
now=$(date +"%s")
thresold=2592000 # 30*24*60*60 equivalent to one month in seconds
remind=$(($cert_time - $now))
if [ $remind -lt $thresold ]; then
exec ${certificate-request-base:wrapper-path} --renew
exec ${authenticated-httpd-graceful:output}
fi
output = ${directory:bin}/certificate-renew
mode = 700
[certificate-renew-cron-entry]
recipe = slapos.cookbook:cron.d
cron-entries = ${cron:cron-entries}
name = certificate-auto-renew
frequency = 5 6 * * 6
command = ${certificate-renew:output}
[logrotate-authenticated-httpd]
< = logrotate-entry-base
name = authenticated-httpd-server
log = ${authenticated-httpd-conf-parameter:access-log} ${authenticated-httpd-conf-parameter:access-log}
post = {{ slapos_kill_bin }} --pidfile ${authenticated-httpd-conf-parameter:pid-file} -s USR1
[authenticated-httpd-promise]
recipe = slapos.cookbook:check_url_available
path = ${directory:promises}/${:filename}
filename = authenticated-httpd-is-available
url = ${authenticated-httpd-conf-parameter:url}
check-secure = 1
dash_path = {{ dash_executable_location }}
curl_path = {{ curl_executable_location }}
cert-file = ${certificate-request-base:cert-file}
key-file = ${certificate-request-base:key-file}
ca-cert-file = ${certificate-request-base:ca-cert}
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
computer = ${slap-connection:computer-id}
partition = ${slap-connection:partition-id}
url = ${slap-connection:server-url}
key = ${slap-connection:key-file}
cert = ${slap-connection:cert-file}
stack/caucase/instance-caucase.cfg.jinja2.in 0 → 100644
View file @ 56ad14fb
{% set part_list = [] -%}
{% set ipv6 = (ipv6 | list)[0] -%}
{% set default_subject = '/C=FR/O=Company/CN=SlapOS Certificate Authority/emailAddress=xx@example.com' -%}
{% if slapparameter_dict is not defined -%}
{% set slapparameter_dict = {} -%}
{% endif -%}
[ca-parameters]
server-port = {{ slapparameter_dict.get('server-port', 8009) }}
server-https-port = {{ slapparameter_dict.get('server-https-port', 8010) }}
ipv6 = {{ ipv6 }}
# Overrite this to set frontend or DNS URL (URL is used as CRL distribution point)
# Please set http not HTTPS scheme
crl-external-url = {{ slapparameter_dict.get('external-url', 'http://[${:ipv6}]:${:server-port}') }}
# /CN=XXX is required and should be unique
ca-subject = {{ slapparameter_dict.get('ca-subject', default_subject) }}
# Number of pending csr to accept
max-request-amount = {{ slapparameter_dict.get('max-request-amount', 10) }}
# one year (in seconds)
crt-life-time = {{ slapparameter_dict.get('crt-life-time', 31536000) }}
# crl-life-period correspond to about one week
crl-life-period = {{ slapparameter_dict.get('crl-life-period', 0.02) }}
# ca-life-period = ca-life-period * crt-life-time
ca-life-period = {{ slapparameter_dict.get('ca-life-period', 10) }}
# time before clean certificate on CA: 60*24*60*60
crt-keep-time = {{ slapparameter_dict.get('crt-keep-time', 5184000) }}
[directory]
recipe = slapos.cookbook:mkdirectory
etc = ${buildout:directory}/etc
bin = ${buildout:directory}/bin
srv = ${buildout:directory}/srv
var = ${buildout:directory}/var
run = ${:var}/run
log = ${:var}/log
scripts = ${:etc}/run
services = ${:etc}/service
promises = ${:etc}/promise
ssl=${:etc}/ssl
ca-dir = ${directory:srv}/ca
ca-temp = ${:ca-dir}/tmp
client-body-temp-path = ${:ca-temp}/client_body_temp_path
proxy-temp-path = ${:ca-temp}/proxy_temp_path
fastcgi-temp-path = ${:ca-temp}/fastcgi_temp_path
uwsgi-temp-path = ${:ca-temp}/uwsgi_temp_path
scgi-temp-path = ${:ca-temp}/scgi_temp_path
[nginx-certificate-request-base]
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:bin}/request-base-certificate
cert-file = ${ca-nginx-ssl-config:cert}
key-file = ${ca-nginx-ssl-config:key}
ca-cert = ${directory:ssl}/cacert.pem
parameters-extra = true
command-line = {{ certificate_request_bin }}
--crt-file ${:cert-file}
--key-file ${:key-file}
--ca-url http://[${ca-parameters:ipv6}]:${ca-parameters:server-port}
--ca-crt-file ${:ca-cert}
[nginx-certificate-request]
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:scripts}/request-server-certificate
command-line =
${nginx-certificate-request-base:wrapper-path}
--cn nginx@certificate.authority
--request
[ca-nginx-ssl-config]
# if ssl certificate is signed write to file so that zero-knowledge can read
recipe = plone.recipe.command
command =
if [ -s "${:key}" ] && [ -s "${:cert}" ]; then
cat << EOF > ${:output}
[ca-nginx-ssl]
key=${:key}
cert=${:cert}
EOF
fi
key = ${directory:ssl}/ca-cert.key
cert = ${directory:ssl}/ca-cert.crt
update-command = ${:command}
output = ${directory:etc}/ca-nginx-ssl.cfg
stop-on-error = true
[ca-nginx-ssl]
recipe = slapos.cookbook:zero-knowledge.read
file-path = ${ca-nginx-ssl-config:output}
# initials values are empty, the section https (ssl) in nginx config will be skipped
cert =
key =
[ca-nginx-conf-parameter]
ip = ${ca-parameters:ipv6}
port = ${ca-parameters:server-port}
https-port = ${ca-parameters:server-https-port}
pid-file = ${directory:run}/nginx-ca.pid
access-log = ${directory:log}/nginx-ca-access.log
error-log = ${directory:log}/nginx-ca-error.log
cert-file = ${ca-nginx-ssl:cert}
key-file = ${ca-nginx-ssl:key}
ca-conf = ${caucase-conf:output}
workers-processes = 1
client-body-temp-path = ${directory:client-body-temp-path}
proxy-temp-path = ${directory:proxy-temp-path}
fastcgi-temp-path = ${directory:fastcgi-temp-path}
uwsgi-temp-path = ${directory:uwsgi-temp-path}
scgi-temp-path = ${directory:scgi-temp-path}
socket = ${caucase-gunicorn:socket}
[ca-nginx-conf]
recipe = slapos.recipe.template:jinja2
template = {{ template_nginx_ca_conf }}
rendered = ${directory:etc}/nginx-ca.conf
mode = 0700
context =
section parameter_dict ca-nginx-conf-parameter
[caucase-conf]
recipe = collective.recipe.template
# Values here are intended to be changed in your instance. override this section
input = inline:
ca-dir ${directory:ca-dir}
# enable debug
# debug
# log-file ${directory:log}/ca-server.log
subject ${ca-parameters:ca-subject}
max-request-amount ${ca-parameters:max-request-amount}
external-url ${ca-parameters:crl-external-url}
# one year (in seconds)
crt-life-time ${ca-parameters:crt-life-time}
# crl-life-period correspond to about one week
crl-life-period ${ca-parameters:crl-life-period}
# ca-life-time = ca-life-period * crt-life-time
ca-life-period ${ca-parameters:ca-life-period}
# time before clean certificate on CA: 60*24*60*60
crt-keep-time ${ca-parameters:crt-keep-time}
output = ${directory:etc}/ca.conf
mode = 700
[ca-nginx-graceful]
recipe = collective.recipe.template
input = inline:#!{{ dash_executable_location }}
kill -HUP "$(cat '${ca-nginx-conf-parameter:pid-file}')"
output = ${directory:scripts}/ca-server-graceful
mode = 700
[caucase-gunicorn]
recipe = slapos.cookbook:wrapper
socket = ${directory:ca-dir}/ca.flaskserver.sock
command-line = {{ gunicorn_bin }} caucase.wsgi:app -b unix:${:socket} -e CA_CONFIGURATION_FILE=${caucase-conf:output} --error-logfile ${:log-file} --pid ${:pid-file} --capture-output --timeout 60 --threads 2 --log-level error --preload
log-file = ${directory:log}/ca-gunicorn-error.log
pid-file = ${directory:run}/ca-gunicorn.pid
wrapper-path = ${directory:services}/ca-gunicorn
[caucase-server]
recipe = slapos.cookbook:wrapper
command-line = {{ nginx_location }}/sbin/nginx -p ${directory:ca-dir} -c ${ca-nginx-conf:rendered}
wrapper-path = ${directory:services}/caucase-server
url = https://[${ca-parameters:ipv6}]:${ca-parameters:server-https-port}
http-url = ${ca-parameters:crl-external-url}
depends =
${nginx-certificate-request:wrapper-path}
${caucase-server-promise:filename}
${caucase-https-server-promise:filename}
${ca-nginx-graceful:output}
${ca-certificate-renew-cron-entry:name}
${logrotate-ca-nginx:name}
[ca-certificate-renew-cron-entry]
recipe = slapos.cookbook:cron.d
cron-entries = ${cron:cron-entries}
name = ca-server-certificate-auto-renew
# check renew every-week
time = weekly
# 2592000 = 30*24*60*60 equivalent to one month in seconds
command = ${nginx-certificate-request-base:wrapper-path} --renew --threshold 2592000 --on-renew="${ca-nginx-graceful:output}"
[logrotate-ca-nginx]
< = logrotate-entry-base
name = caucase-nginx-server
log = ${ca-nginx-conf-parameter:access-log} ${ca-nginx-conf-parameter:access-log}
post = {{ slapos_kill_bin }} --pidfile ${ca-nginx-conf-parameter:pid-file} -s USR1
[caucase-server-promise]
recipe = slapos.cookbook:check_url_available
path = ${directory:promises}/${:filename}
filename = caucase-server-listening-on-tcp
url = http://[${ca-parameters:ipv6}]:${ca-parameters:server-port}
dash_path = {{ dash_executable_location }}
curl_path = {{ curl_executable_location }}
[caucase-https-server-promise]
recipe = slapos.cookbook:check_url_available
path = ${directory:promises}/${:filename}
filename = caucase-server-https-on-${ca-parameters:server-https-port}
url = https://[${ca-parameters:ipv6}]:${ca-parameters:server-https-port}
check-secure = 1
dash_path = {{ dash_executable_location }}
curl_path = {{ curl_executable_location }}
{% if publish_parameter is defined and publish_parameter == 'yes' -%}
[publish-connection-parameter]
recipe = slapos.cookbook:publish.serialised
http-url = ${caucase-server:http-url}
https-url = ${caucase-server:url}
{% do part_list.append('publish-connection-parameter') -%}
{% endif -%}
[buildout]
extends =
{{ template_logrotate_base }}
parts =
caucase-server
# Complete parts with sections
{{ part_list | join('\n ') }}
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
\ No newline at end of file
stack/caucase/instance.cfg.in 0 → 100644
View file @ 56ad14fb
[buildout]
parts =
switch-softwaretype
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
computer = ${slap-connection:computer-id}
partition = ${slap-connection:partition-id}
url = ${slap-connection:server-url}
key = ${slap-connection:key-file}
cert = ${slap-connection:cert-file}
[dynamic-template-caucase]
recipe = slapos.recipe.template:jinja2
filename = ${:_buildout_section_name_}.cfg
rendered = ${buildout:parts-directory}/${:_buildout_section_name_}/${:filename}
template = {{ caucase_template }}
extensions = jinja2.ext.do
extra-context =
context =
key ipv4 slap-configuration:ipv4
key ipv6 slap-configuration:ipv6
key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory
key slapparameter_dict slap-configuration:configuration
raw gunicorn_bin {{ gunicorn_bin }}
raw template_logrotate_base {{ template_logrotate_base }}
raw certificate_request_bin {{ certificate_request_bin }}
raw template_nginx_ca_conf {{ template_nginx_ca_conf }}
raw nginx_location {{ nginx_location }}
raw slapos_kill_bin {{ slapos_kill_bin }}
raw dash_executable_location {{ dash_executable_location }}
raw curl_executable_location {{ curl_executable_location }}
raw publish_parameter yes
[switch-softwaretype]
recipe = slapos.cookbook:softwaretype
default = ${dynamic-template-caucase:rendered}
stack/caucase/template-httpd-auth.conf.in 0 → 100644
View file @ 56ad14fb
Listen [{{ parameter_dict['ip'] }}]:{{ parameter_dict['port'] }}
LoadModule unixd_module modules/mod_unixd.so
LoadModule access_compat_module modules/mod_access_compat.so
LoadModule authz_core_module modules/mod_authz_core.so
LoadModule authz_host_module modules/mod_authz_host.so
LoadModule mime_module modules/mod_mime.so
LoadModule dir_module modules/mod_dir.so
LoadModule ssl_module modules/mod_ssl.so
LoadModule alias_module modules/mod_alias.so
LoadModule autoindex_module modules/mod_autoindex.so
LoadModule proxy_module modules/mod_proxy.so
LoadModule proxy_http_module modules/mod_proxy_http.so
LoadModule rewrite_module modules/mod_rewrite.so
LoadModule headers_module modules/mod_headers.so
LoadModule env_module modules/mod_env.so
LoadModule setenvif_module modules/mod_setenvif.so
LoadModule log_config_module modules/mod_log_config.so
LoadModule dav_module modules/mod_dav.so
LoadModule dav_fs_module modules/mod_dav_fs.so
ServerAdmin admin@
TypesConfig conf/mime.types
AddType application/x-compress .Z
AddType application/x-gzip .gz .tgz
ServerTokens Prod
ServerSignature Off
TraceEnable Off
PidFile "{{ parameter_dict['pid-file'] }}"
ErrorLog "{{ parameter_dict['error-log'] }}"
# Default apache log format with request time in microsecond at the end
LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %D" combined
CustomLog "{{ parameter_dict['access-log'] }}" combined
# SSL Configuration
Define SSLConfigured
SSLCertificateFile {{ parameter_dict['cert-file'] }}
SSLCertificateKeyFile {{ parameter_dict['key-file'] }}
SSLRandomSeed startup builtin
SSLRandomSeed connect builtin
SSLRandomSeed startup /dev/urandom 256
SSLRandomSeed connect builtin
SSLProtocol all -SSLv2 -SSLv3
SSLCipherSuite ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:HIGH:!aNULL:!MD5
SSLHonorCipherOrder on
SSLEngine On
<Directory />
Options FollowSymLinks
AllowOverride None
Allow from all
</Directory>
<VirtualHost *:{{ parameter_dict['port'] }}>
DavLockDB {{ parameter_dict['dav-lock'] }}
SSLVerifyClient require
RequestHeader set REMOTE_USER %{SSL_CLIENT_S_DN_CN}s
SSLCACertificateFile {{ parameter_dict['ca-cert'] }}
{% if parameter_dict['crl'] -%}
SSLCARevocationCheck chain
SSLCARevocationFile {{ parameter_dict['crl'] }}
{%- endif %}
Alias / {{ parameter_dict['private'] }}/
<Directory {{ parameter_dict['private'] }}>
DirectoryIndex disabled
DAV On
Options Indexes FollowSymLinks
Order Allow,Deny
Allow from all
</Directory>
</VirtualHost>
{% if parameter_dict.get('httpd-include-file', '') -%}
# Custom apache configuration here
Include {{ parameter_dict['httpd-include-file'] }}
{% endif -%}
stack/cloudooo.cfg
View file @ 56ad14fb
......@@ -89,7 +89,7 @@ PasteScript = 2.0.2
WSGIUtils = 0.7
python-magic = 0.4.6
rdiff-backup = 1.0.5+SlapOSPatched001
slapos.recipe.template = 2.8
slapos.recipe.template = 3.0
# Required by:
# PasteScript==2.0
......@@ -98,4 +98,4 @@ PasteDeploy = 1.5.2
# Required by:
# cloudooo==1.2.5.dev0
erp5.util = 0.4.46
erp5.util = 0.4.49
stack/erp5/TODO
View file @ 56ad14fb
......@@ -4,8 +4,7 @@ General:
- ipv6 support (besides frontend-backend apache connection)
requires important changes at ERP5 level
- resilience
- make mariadb user accounts accept connections only from relevant IPs
or make x509 mandatory (needs ZMySQLD*A support)
- mariadb: make x509 mandatory (needs ZMySQLD*A support)
- make postfix log inside partition
- document postfix parameters (only once it actually works)
......
stack/erp5/buildout.cfg
View file @ 56ad14fb
......@@ -9,6 +9,7 @@ extends =
../../component/dbus/buildout.cfg
../../component/file/buildout.cfg
../../component/fonts/buildout.cfg
../../component/gcc/buildout.cfg
../../component/ghostscript/buildout.cfg
../../component/git/buildout.cfg
../../component/graphviz/buildout.cfg
......@@ -27,6 +28,7 @@ extends =
../../component/matplotlib/buildout.cfg
../../component/mesa/buildout.cfg
../../component/numpy/buildout.cfg
../../component/ocropy/buildout.cfg
../../component/onlyoffice-x2t/buildout.cfg
../../component/pandas/buildout.cfg
../../component/percona-toolkit/buildout.cfg
......@@ -58,7 +60,9 @@ extends =
../../component/findutils/buildout.cfg
../../component/userhosts/buildout.cfg
../../component/postfix/buildout.cfg
../monitor/buildout.cfg
../../software/ipython_notebook/software.cfg
../../software/caucase/software.cfg
../../software/neoppod/software-common.cfg
# keep neoppod extends last
......@@ -90,6 +94,8 @@ parts +=
libreoffice-bin
w3-validator
tesseract
scipy
ocropy
hookbox
percona-toolkit
zabbix-agent
......@@ -136,10 +142,17 @@ parts +=
ipython-notebook
instance-jupyter
# override python2.7 to add SlapOS libstdc++ in RPATH.
[python2.7]
extra-ldflags = -Wl,-rpath=${gcc:location}/lib -Wl,-rpath=${gcc:location}/lib64
# override instance-jupyter not to render into default template.cfg
[instance-jupyter]
rendered = ${buildout:directory}/template-jupyter.cfg
[instance-caucase]
rendered = ${buildout:directory}/instance-caucase.cfg
[download-base]
<= download-base-neo
url = ${:_profile_base_location_}/${:filename}
......@@ -154,6 +167,10 @@ command = grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link
<= download-base
mode = 755
[mariadb-slow-query-report-script]
<= download-base
mode = 755
[template-mariadb]
<= download-base
link-binary =
......@@ -218,6 +235,7 @@ context =
key bin_directory buildout:bin-directory
key buildout_bin_directory buildout:bin-directory
key cairo_location cairo:location
key caucase_template instance-caucase:rendered
key coreutils_location coreutils:location
key cups_location cups:location
key curl_location curl:location
......@@ -258,10 +276,13 @@ context =
key logrotate_location logrotate:location
key mariadb_location mariadb:location
key mariadb_resiliency_after_import_script mariadb-resiliency-after-import-script:target
key mariadb_slow_query_report_script mariadb-slow-query-report-script:target
key matplotlibrc_location matplotlibrc:location
key mesa_location mesa:location
key onlyoffice_x2t_location onlyoffice-x2t:location
key parts_directory buildout:parts-directory
key openssl_location openssl:location
key percona_toolkit_location percona-toolkit:location
key pixman_location pixman:location
key poppler_location poppler:location
key postfix_location postfix:location
......@@ -280,7 +301,7 @@ context =
key template_kumofs template-kumofs:target
key template_mariadb template-mariadb:target
key template_mariadb_initial_setup template-mariadb-initial-setup:target
key template_monitor monitor-template:rendered
key template_monitor monitor2-template:rendered
key template_my_cnf template-my-cnf:target
key template_postfix template-postfix:target
key template_postfix_aliases template-postfix-aliases:target
......@@ -298,13 +319,6 @@ context =
key zlib_location zlib:location
key extra_path_list eggs:extra-paths
[monitor-template-dummy]
<= download-base
# This is a placeholder, to be overriden by extending monitor SR
[monitor-template]
rendered = ${monitor-template-dummy:target}
[template-erp5]
<= download-base
......@@ -437,14 +451,17 @@ eggs = ${neoppod:eggs}
${numpy:egg}
${matplotlib:egg}
${lxml-python:egg}
${ocropy:egg}
${pandas:egg}
${pillow-python:egg}
${python-ldap-python:egg}
${pysvn-python:egg}
${pycrypto-python:egg}
${scipy:egg}
${scikit-learn:egg}
lock_file
astor
APacheDEX
PyStemmer
PyXML
Pympler
......@@ -542,6 +559,7 @@ eggs = ${neoppod:eggs}
# installation of python, which we don't want on an instance
interpreter = ${buildout:python}
scripts =
apachedex
repozo
runzope
runzeo
......@@ -590,6 +608,7 @@ eggs +=
scripts +=
is-local-tcp-port-opened
is-process-older-than-dependency-set
runApacheDex
zodbpack
[versions]
......@@ -599,6 +618,7 @@ scripts +=
# patched eggs
Acquisition = 2.13.9+SlapOSPatched001
Products.DCWorkflow = 2.2.4+SlapOSPatched001
ocropy = 1.0+SlapOSPatched001
pysvn = 1.7.10+SlapOSPatched002
python-ldap = 2.4.32+SlapOSPatched001
python-magic = 0.4.12+SlapOSPatched001
......@@ -641,6 +661,7 @@ zope.app.publication = 3.14.0
zope.app.testing = 3.8.1
# Pinned versions
APacheDEX = 1.6.2
Pillow = 4.0.0
Products.CMFActionIcons = 2.1.3
Products.DCWorkflowGraph = 0.4.1
......
stack/erp5/buildout.hash.cfg
View file @ 56ad14fb
......@@ -15,19 +15,23 @@
# not need these here).
[mariadb-resiliency-after-import-script]
filename = instance-mariadb-resiliency-after-import-script.sh.in
md5sum = 1e89de954d816b93f76d9b75820d192c
md5sum = 844d62cd6f9d6e3d1d78d52de2b72a49
[mariadb-slow-query-report-script]
filename = mysql-querydigest.sh.in
md5sum = dc974bd74cf967ae6250d81322629c44
[template-mariadb]
filename = instance-mariadb.cfg.in
md5sum = 11e7faca5233868c1643854e4c898588
md5sum = 3b6c19686ac52c2d452597dbacc88888
[template-kumofs]
filename = instance-kumofs.cfg.in
md5sum = 42d2a2c7cb5bf5122b6cfd8f53a5576f
md5sum = 253027f37c4e5bd908e6af551848f03a
[template-cloudooo]
filename = instance-cloudoo.cfg.in
md5sum = c553bbcdc5f80d893907a05acefc1356
md5sum = 43253cbfd5d361047ba6a99b116b2370
[template-zope-conf]
filename = zope.conf.in
......@@ -63,7 +67,7 @@ md5sum = 9ac81647368068a1a98a785d08074b43
[template-postfix-main-cf]
filename = postfix_main.cf.in
md5sum = 4fe9bb5225b9d7650d2916a0945821bd
md5sum = 886d1b004049c8dc1f89cbbfe4f95371
[template-postfix-aliases]
filename = postfix_aliases.in
......@@ -71,7 +75,7 @@ md5sum = 0969fbb25b05c02ef3c2d437b2f4e1a0
[template]
filename = instance.cfg.in
md5sum = 8ab417cf1ca98d2840c80a266f0e2be7
md5sum = e364ea67bfe786b6b6ebd6c4f0cd628a
[monitor-template-dummy]
filename = dummy.cfg
......@@ -79,19 +83,19 @@ md5sum = d41d8cd98f00b204e9800998ecf8427e
[template-erp5]
filename = instance-erp5.cfg.in
md5sum = 07f45a5d3a9fd0fe227af82c7ede7631
md5sum = 13638031b6b6c9ad9c0a9c4e6d9a202a
[template-zeo]
filename = instance-zeo.cfg.in
md5sum = 7610bafda245c008ccf0b6ea58ce21c2
md5sum = 2248d249cfc53c73e7c65665d4d53be0
[template-zope]
filename = instance-zope.cfg.in
md5sum = b7e92234825f9d72ccb9b6c4745b6ce7
md5sum = 6a64d1615c3ef9f6311c863d5aa0c58f
[template-balancer]
filename = instance-balancer.cfg.in
md5sum = d71c49f91b3455e6866f4b2db591009f
md5sum = f2fb0c537c124622fe8e89afe0188519
[template-haproxy-cfg]
filename = haproxy.cfg.in
......
stack/erp5/instance-balancer.cfg.in
View file @ 56ad14fb
{% set part_list = [] -%}
{% set ssl_parameter_dict = slapparameter_dict.get('ssl', {}) %}
{% set caucase_url = slapparameter_dict.get('caucase-url', '') -%}
{% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
{% set use_ipv6 = slapparameter_dict.get('use-ipv6', False) -%}
{#
......@@ -36,6 +37,56 @@ context = key content {{content_section_name}}:content
mode = {{ mode }}
{%- endmacro %}
[certificate-request-base]
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:bin}/request-instance-certificate
parameters-extra = true
command-line = {{ parameter_dict['bin-directory'] }}/caucase-cliweb
--crt-file ${apache-conf-ssl:cert}
--key-file ${apache-conf-ssl:key}
--crl-file ${apache-conf-ssl:crl}
--ca-url {{ caucase_url }}
--ca-crt-file ${apache-conf-ssl:ca-cert}
{% macro request_cert(name, common_name) -%}
{% set get_crl_periodicity = slapparameter_dict.get('crl-update-periodicity', 'daily') -%}
[{{ section(name ~ '-certificate-request') }}]
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:services}/request-{{ name }}-certificate
command-line =
${certificate-request-base:wrapper-path}
--cn {{ common_name }}
--request
[{{ section(name ~ '-renew-cron-entry') }}]
recipe = slapos.cookbook:cron.d
cron-entries = ${cron:cron-entries}
name = {{ name }}-certificate-auto-renew
time = weekly
# 2592000 = 30*24*60*60 equivalent to one month in seconds
command = ${certificate-request-base:wrapper-path} --renew --threshold 2592000 --on-renew="${apache-graceful:output}"
[{{ section(name ~ '-download-crl') }}]
# download the crl for the first time
recipe = plone.recipe.command
command =
if [ ! -s "${apache-conf-ssl:crl}" ]; then
${certificate-request-base:wrapper-path} --update-crl
fi
update-command = ${:command}
stop-on-error = true
[{{ section(name ~ '-update-crl-cron-entry') }}]
recipe = slapos.cookbook:cron.d
cron-entries = ${cron:cron-entries}
name = {{ name }}-update-crl
time = {{ get_crl_periodicity }}
# XXX - Update crl call apache graceful restart, it's not recommended to check crl too often, Apache
# has an issue with reload and can be frozen and stop responding. Default periodicity time = daily
command = ${certificate-request-base:wrapper-path} --update-crl --on-crl-update="${apache-graceful:output}"
{%- endmacro %}
{% if use_ipv6 -%}
[zope-tunnel-base]
recipe = slapos.cookbook:ipv4toipv6
......@@ -81,6 +132,7 @@ ipv6 = {{ zope_address.split(']:')[0][1:] }}
-#}
{% do zope_family_address_list[0][0] -%}
{% set haproxy_port = next_port() -%}
{% set backend_path = slapparameter_dict['backend-path-dict'][family_name] -%}
{% do haproxy_dict.__setitem__(family_name, (haproxy_port, zope_family_address_list)) -%}
{% if has_webdav -%}
{% set internal_scheme = 'http' -%}{# mod_rewrite does not recognise webdav scheme -#}
......@@ -89,7 +141,8 @@ ipv6 = {{ zope_address.split(']:')[0][1:] }}
{% set internal_scheme = 'http' -%}
{% set external_scheme = 'https' -%}
{% endif -%}
{% do apache_dict.__setitem__(family_name, (next_port(), external_scheme, internal_scheme ~ '://' ~ ipv4 ~ ':' ~ haproxy_port ~ slapparameter_dict['backend-path'])) -%}
{% set ssl_authentication = slapparameter_dict['ssl-authentication-dict'].get(family_name, False) -%}
{% do apache_dict.__setitem__(family_name, (next_port(), external_scheme, internal_scheme ~ '://' ~ ipv4 ~ ':' ~ haproxy_port ~ backend_path, ssl_authentication)) -%}
{% endfor -%}
[haproxy-cfg-parameter-dict]
......@@ -122,6 +175,7 @@ crl = ${directory:apache-conf}/crl.pem
backend-list = {{ dumps(apache_dict.values()) }}
ip-list = {{ dumps(apache_ip_list) }}
pid-file = ${directory:run}/apache.pid
log-dir = ${directory:log}
error-log = ${directory:log}/apache-error.log
access-log = ${directory:log}/apache-access.log
# Apache 2.4's default value (60 seconds) can be a bit too short
......@@ -145,6 +199,18 @@ context = section parameter_dict apache-conf-parameter-dict
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:services}/apache
command-line = "{{ parameter_dict['apache'] }}/bin/httpd" -f "${apache-conf:rendered}" -DFOREGROUND
wait-for-files =
${apache-conf-ssl:cert}
${apache-conf-ssl:key}
[apache-graceful]
recipe = collective.recipe.template
input = inline:
#!/bin/sh
kill -USR1 "$(cat '${apache-conf-parameter-dict:pid-file}')"
output = ${directory:bin}/apache-httpd-graceful
mode = 700
[{{ section('apache-promise') }}]
# Check any apache port in ipv4, expect other ports and ipv6 to behave consistently
......@@ -155,10 +221,11 @@ port = {{ apache_dict.values()[0][0] }}
[publish]
recipe = slapos.cookbook:publish.serialised
{% for family_name, (apache_port, scheme, _) in apache_dict.items() -%}
{% for family_name, (apache_port, scheme, _, _) in apache_dict.items() -%}
{{ family_name ~ '-v6' }} = {% if ipv6_set %}{{ scheme ~ '://[' ~ ipv6 ~ ']:' ~ apache_port }}{% endif %}
{{ family_name }} = {{ scheme ~ '://' ~ ipv4 ~ ':' ~ apache_port }}
{% endfor -%}
monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
[apache-ssl]
{% if ssl_parameter_dict.get('key') -%}
......@@ -166,6 +233,11 @@ key = ${apache-ssl-key:rendered}
cert = ${apache-ssl-cert:rendered}
{{ simplefile('apache-ssl-key', '${apache-conf-ssl:key}', ssl_parameter_dict['key']) }}
{{ simplefile('apache-ssl-cert', '${apache-conf-ssl:cert}', ssl_parameter_dict['cert']) }}
{% elif caucase_url -%}
key = ${apache-conf-ssl:key}
cert = ${apache-conf-ssl:cert}
{{ request_cert('erp5', 'instance.apache@erp5') }}
{% else %}
recipe = plone.recipe.command
command = "{{ parameter_dict['openssl'] }}/bin/openssl" req -newkey rsa -batch -new -x509 -days 3650 -nodes -keyout "${:key}" -out "${:cert}"
......@@ -179,6 +251,10 @@ cert = ${apache-ssl-ca:rendered}
crl = ${apache-ssl-crl:rendered}
{{ simplefile('apache-ssl-ca', '${apache-conf-ssl:ca-cert}', ssl_parameter_dict['ca-cert']) }}
{{ simplefile('apache-ssl-crl', '${apache-conf-ssl:crl}', ssl_parameter_dict['crl']) }}
{% elif caucase_url -%}
cert = ${apache-conf-ssl:ca-cert}
crl = ${apache-conf-ssl:crl}
{% else %}
cert =
crl =
......@@ -206,11 +282,34 @@ private = ${:ca-dir}/private
certs = ${:ca-dir}/certs
newcerts = ${:ca-dir}/newcerts
crl = ${:ca-dir}/crl
apachedex = ${monitor-directory:private}/apachedex
[monitor-generate-apachedex-report]
recipe = slapos.cookbook:wrapper
wrapper-path = ${monitor-directory:reports}/${:command}
command-line = "{{ parameter_dict['run-apachedex-location'] }}" "{{ parameter_dict['apachedex-location'] }}" "${directory:apachedex}" --default "${apachedex-parameters:default}" --apache-log-list "${apachedex-parameters:apache-log-list}" --base-list "${apachedex-parameters:base-list}" --skip-base-list "${apachedex-parameters:skip-base-list}" --erp5-base-list "${apachedex-parameters:erp5-base-list}"
command = apachedex_every_3_hour
[apachedex-parameters]
default_parameter =
# XXX - Sample log file with curent date: apache_access.log-%(date)s.gz
# which will be equivalent to apache_access.log-20150112.gz if the date is 2015-01-12
apache-log-list = ${apache-conf-parameter-dict:access-log}
default = ${monitor-directory:etc}/apdex_default
base-list = ${monitor-directory:etc}/apdex_base_list
skip-base-list = ${monitor-directory:etc}/apdex_skip_base_list
erp5-base-list = ${monitor-directory:etc}/apdex_erp5_base_list
[monitor-instance-parameter]
monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }}
monitor-httpd-port = {{ next_port() }}
monitor-title = Balancer monitor
password = {{ slapparameter_dict['monitor-passwd'] }}
instance-configuration =
file apachedex-default ${apachedex-parameters:default}
file apachedex-base-list ${apachedex-parameters:base-list}
file apachedex-skip-base-list ${apachedex-parameters:skip-base-list}
file apachedex-erp5-base-list ${apachedex-parameters:erp5-base-list}
[buildout]
extends =
......@@ -219,4 +318,5 @@ extends =
parts +=
publish
logrotate-apache
monitor-generate-apachedex-report
{{ part_list | join('\n ') }}
stack/erp5/instance-cloudoo.cfg.in
View file @ 56ad14fb
......@@ -20,6 +20,7 @@ url = http://[${ipv6toipv4:ipv6}]:${ipv6toipv4:ipv6-port}/
{% else -%}
url = http://${cloudooo-instance:ip}:${cloudooo-instance:port}/
{% endif -%}
monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
[cloudooo-instance]
recipe = slapos.cookbook:generic.cloudooo
......@@ -57,9 +58,10 @@ service-folder = ${directory:service}
[resiliency-exclude-file]
# Generate rdiff exclude file in case of resiliency
recipe = collective.recipe.template
input = inline: **
output = ${directory:srv}/exporter.exclude
recipe = slapos.recipe.template:jinja2
mode = 644
template = {{ 'inline:{{ "**\\n" }}' }}
rendered = ${directory:srv}/exporter.exclude
[promise]
recipe = slapos.cookbook:check_port_listening
......@@ -110,6 +112,7 @@ framebuffer = ${:srv}/framebuffer
monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }}
monitor-httpd-port = {{ tcpv4_port + 2 }}
monitor-title = Cloudooo monitor
password = {{ slapparameter_dict['monitor-passwd'] }}
[binary-link]
recipe = slapos.cookbook:symbolic.link
......
stack/erp5/instance-erp5.cfg.in
View file @ 56ad14fb
......@@ -8,6 +8,9 @@
{% set jupyter_dict = slapparameter_dict.get('jupyter', {}) -%}
{% set has_jupyter = jupyter_dict.get('enable', jupyter_enable_default.lower() in ('true', 'yes')) -%}
{% set jupyter_zope_family = jupyter_dict.get('zope-family', '') -%}
{% set monitor_base_url_dict = {} -%}
{% set caucase_url = slapparameter_dict.get('caucase', {}).pop('url', '') -%}
{% set crl_update_period = slapparameter_dict.get('caucase', {}).pop('crl-update-periodicity', 'daily') -%}
[request-common]
<= request-common-base
config-use-ipv6 = {{ dumps(slapparameter_dict.get('use-ipv6', False)) }}
......@@ -22,7 +25,10 @@ software-type = {{ software_type }}
return = {{ ' '.join(ret) }}
{% for ret, publish in ret.iteritems() -%}
{% if publish -%}
{% do publish_dict.__setitem__(name ~ '-' ~ ret, '${' ~ section ~ ':connection-' ~ ret ~ '}')%}
{% do publish_dict.__setitem__(name ~ '-' ~ ret, '${' ~ section ~ ':connection-' ~ ret ~ '}') %}
{% endif -%}
{% if ret == "monitor-base-url" -%}
{% do monitor_base_url_dict.__setitem__(section, '${' ~ section ~ ':connection-' ~ ret ~ '}') -%}
{% endif -%}
{% endfor -%}
{{ root_common.sla(name) }}
......@@ -34,10 +40,10 @@ config-{{ k }} = {{ '${' ~ v ~ '}' }}
{% endfor -%}
{% endmacro -%}
{{ request('memcached-persistent', 'kumofs', 'kumofs', {'tcpv4-port': 2000}) }}
{{ request('memcached-volatile', 'kumofs', 'memcached', {'tcpv4-port': 2010, 'ram-storage-size': 64}) }}
{{ request('cloudooo', 'cloudooo', 'cloudooo', {'tcpv4-port': 2020}) }}
{{ request('mariadb', 'mariadb', 'mariadb', {'tcpv4-port': 2099}, {'database-list': True, 'test-database-list': True}) }}
{{ request('memcached-persistent', 'kumofs', 'kumofs', {'tcpv4-port': 2000}, {'url': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
{{ request('memcached-volatile', 'kumofs', 'memcached', {'tcpv4-port': 2010, 'ram-storage-size': 64}, {'url': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
{{ request('cloudooo', 'cloudooo', 'cloudooo', {'tcpv4-port': 2020}, {'url': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
{{ request('mariadb', 'mariadb', 'mariadb', {'tcpv4-port': 2099}, {'database-list': True, 'test-database-list': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
{% if has_posftix -%}
{{ request('smtp', 'postfix', 'smtp', {'tcpv4-port': 2025, 'smtpd-sasl-user': 'erp5@nowhere'}, key_config={'smtpd-sasl-password': 'publish-early:smtpd-sasl-password'}) }}
{%- else %}
......@@ -46,6 +52,14 @@ config-{{ k }} = {{ '${' ~ v ~ '}' }}
connection-url = smtp://127.0.0.2:0/
{%- endif %}
{% if caucase_url -%}
{% do publish_dict.__setitem__('caucase-http-url', caucase_url) -%}
[request-caucase]
connection-http-url = {{ caucase_url }}
{%- else %}
{{ request('caucase', 'caucase', 'caucase', {'server-port': 8890, 'server-https-port': 8891}, {'http-url': True, 'https-url': False}) }}
{% endif -%}
{# ZODB -#}
{% set zodb_dict = {} -%}
{% set storage_dict = {} -%}
......@@ -74,7 +88,7 @@ connection-url = smtp://127.0.0.2:0/
{% else -%}
{{ assert(server_type == 'zeo', server_type) -}}
{# BBB: for compatibility, keep 'zodb' as partition_reference for ZEO -#}
{{ request('zodb', 'zodb-' ~ server_type, 'zodb-' ~ server_type, {'tcpv4-port': 2100, 'zodb-dict': server_dict}, dict.fromkeys(('storage-dict', 'tidstorage-ip', 'tidstorage-port'))) }}
{{ request('zodb', 'zodb-' ~ server_type, 'zodb-' ~ server_type, {'tcpv4-port': 2100, 'zodb-dict': server_dict}, dict.fromkeys(('storage-dict', 'tidstorage-ip', 'tidstorage-port', 'monitor-base-url')), key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
{% endif -%}
{% endfor -%}
......@@ -89,16 +103,16 @@ recipe = slapos.cookbook:publish-early
{%- if neo %}
neo-cluster gen-neo-cluster:name
{%- if neo[0] %}
neo-cluster = {{ neo[0] }}
neo-cluster = {{ dumps(neo[0]) }}
{%- endif %}
{%- endif %}
{%- set inituser_password = slapparameter_dict.get('inituser-password') %}
{%- if inituser_password %}
inituser-password = {{ inituser_password }}
inituser-password = {{ dumps(inituser_password) }}
{%- endif %}
{%- set deadlock_debugger_password = slapparameter_dict.get('deadlock-debugger-password') -%}
{%- if deadlock_debugger_password %}
deadlock-debugger-password = {{ deadlock_debugger_password }}
deadlock-debugger-password = {{ dumps(deadlock_debugger_password) }}
{%- endif %}
[gen-password]
......@@ -126,12 +140,14 @@ name = neo-${gen-neo-cluster-base:passwd}
return =
zope-address-list
hosts-dict
monitor-base-url
{% set bt5_default_list = 'erp5_full_text_myisam_catalog erp5_configurator_standard erp5_configurator_maxma_demo erp5_configurator_ung erp5_configurator_run_my_doc' -%}
{% if has_jupyter -%}
{% set bt5_default_list = bt5_default_list + ' erp5_data_notebook' -%}
{% endif -%}
config-bt5 = {{ dumps(slapparameter_dict.get('bt5', bt5_default_list)) }}
config-bt5-repository-url = {{ dumps(slapparameter_dict.get('bt5-repository-url', local_bt5_repository)) }}
config-caucase-url = ${request-caucase:connection-http-url}
config-cloudooo-url = ${request-cloudooo:connection-url}
config-deadlock-debugger-password = ${publish-early:deadlock-debugger-password}
config-developer-list = {{ dumps(slapparameter_dict.get('developer-list', [inituser_login])) }}
......@@ -141,6 +157,7 @@ config-inituser-login = {{ dumps(inituser_login) }}
config-inituser-password = ${publish-early:inituser-password}
config-kumofs-url = ${request-memcached-persistent:connection-url}
config-memcached-url = ${request-memcached-volatile:connection-url}
config-monitor-passwd = ${monitor-htpasswd:passwd}
config-mysql-test-url-list = ${request-mariadb:connection-test-database-list}
config-mysql-url-list = ${request-mariadb:connection-database-list}
config-site-id = {{ dumps(site_id) }}
......@@ -162,20 +179,26 @@ config-tidstorage-port = ${request-zodb:connection-tidstorage-port}
software-type = zope
{% set zope_family_dict = {} -%}
{% set zope_backend_path_dict = {} -%}
{% set ssl_authentication_dict = {} -%}
{% set jupyter_zope_family_default = [] -%}
{% for custom_name, zope_parameter_dict in zope_partition_dict.items() -%}
{% set partition_name = 'zope-' ~ custom_name -%}
{% set section_name = 'request-' ~ partition_name -%}
{% set zope_family = zope_parameter_dict.get('family', 'default') -%}
{% set backend_path = zope_parameter_dict.get('backend-path', '/') % {'site-id': site_id} %}
{# # default jupyter zope family is first zope family. -#}
{# # use list.append() to update it, because in jinja2 set changes only local scope. -#}
{% if not jupyter_zope_family_default -%}
{% do jupyter_zope_family_default.append(zope_family) -%}
{% endif -%}
{% do zope_family_dict.setdefault(zope_family, []).append(section_name) -%}
{% do zope_backend_path_dict.__setitem__(zope_family, backend_path) -%}
{% do ssl_authentication_dict.__setitem__(zope_family, zope_parameter_dict.get('ssl-authentication', False)) -%}
[{{ section_name }}]
<= request-zope-base
name = {{ partition_name }}
{% do monitor_base_url_dict.__setitem__(section_name, '${' ~ section_name ~ ':connection-monitor-base-url}') -%}
{{ root_common.sla(partition_name) }}
config-name = {{ dumps(custom_name) }}
config-instance-count = {{ dumps(zope_parameter_dict.get('instance-count', 1)) }}
......@@ -235,10 +258,13 @@ name = balancer
software-type = balancer
{{ root_common.sla('balancer') }}
return =
monitor-base-url
{%- for family in zope_family_dict %}
{{ family }}
{{ family }}-v6
{% endfor -%}
{% do monitor_base_url_dict.__setitem__('request-balancer', '${' ~ 'request-balancer' ~ ':connection-monitor-base-url}') -%}
config-zope-family-dict = {{ dumps(zope_family_parameter_dict) }}
config-tcpv4-port = {{ dumps(balancer_dict.get('tcpv4-port', 2150)) }}
{% for zope_section_id, name in zope_address_list_id_dict.items() -%}
......@@ -246,8 +272,12 @@ config-{{ name }} = {{ ' ${' ~ zope_section_id ~ ':connection-zope-address-list}
{% endfor -%}
# XXX: should those really be same for all families ?
config-haproxy-server-check-path = {{ dumps(balancer_dict.get('haproxy-server-check-path', '/') % {'site-id': site_id}) }}
config-backend-path = {{ dumps(balancer_dict.get('apache-backend-path', '/') % {'site-id': site_id}) }}
config-ssl = {{ dumps(balancer_dict.get('ssl', {})) }}
config-monitor-passwd = ${monitor-htpasswd:passwd}
config-caucase-url = ${request-caucase:connection-http-url}
config-crl-update-periodicity = {{ crl_update_period }}
config-backend-path-dict = {{ dumps(zope_backend_path_dict) }}
config-ssl-authentication-dict = {{ dumps(ssl_authentication_dict) }}
[request-frontend-base]
{% if has_frontend -%}
......@@ -271,6 +301,7 @@ return = site_url
{% endif -%}{# if zope_partition_dict -#}
[publish]
<= monitor-publish
recipe = slapos.cookbook:publish.serialised
-extends = publish-early
{% if 'neo' in storage_dict -%}
......@@ -288,4 +319,23 @@ hosts-dict = {{ '${' ~ zope_address_list_id_dict.keys()[0] ~ ':connection-hosts-
{{ name }} = {{ value }}
{% endfor -%}
[monitor-instance-parameter]
monitor-httpd-port = 8386
{{ root_common.common_section() }}
[buildout]
extends = {{ template_monitor }}
parts +=
monitor-base
[monitor-conf-parameters]
monitor-title = ERP5 monitor
password = ${monitor-htpasswd:passwd}
[monitor-base-url-dict]
{% for key, value in monitor_base_url_dict.items() -%}
{{ key }} = {{ value }}
{% endfor %}
stack/erp5/instance-kumofs.cfg.in
View file @ 56ad14fb
......@@ -20,6 +20,7 @@ url = memcached://[${kumofs-instance:ip}]:${kumofs-instance:gateway-port}/
{% else -%}
url = memcached://${kumofs-instance:ip}:${kumofs-instance:gateway-port}/
{% endif -%}
monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
[kumofs-instance]
recipe = slapos.cookbook:generic.kumofs
......@@ -77,9 +78,10 @@ kumofs-data = ${:srv}/kumofs
[resiliency-exclude-file]
# Generate rdiff exclude file in case of resiliency
recipe = collective.recipe.template
input = inline: **
output = ${directory:srv}/exporter.exclude
recipe = slapos.recipe.template:jinja2
mode = 644
template = {{ 'inline:{{ "**\\n" }}' }}
rendered = ${directory:srv}/exporter.exclude
# Deploy zope promises scripts
[promise-template]
......@@ -111,3 +113,4 @@ port = ${kumofs-instance:manager-port}
monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }}
monitor-httpd-port = {{ tcpv4_port + 4 }}
monitor-title = Kumofs monitor
password = {{ slapparameter_dict['monitor-passwd'] }}
stack/erp5/instance-mariadb-resiliency-after-import-script.sh.in
View file @ 56ad14fb
#!${:dash}
#!{{ dash }}
# DO NOT RUN THIS SCRIPT ON PRODUCTION INSTANCE
# OR MYSQL DATA WILL BE ERASED.
......@@ -12,14 +12,14 @@
set -e
mysql_executable='${binary-wrap-mysql:wrapper-path}'
mariadb_data_directory='${directory:mariadb-data}'
mariadb_backup_directory='${directory:mariadb-backup-full}'
instance_directory='${buildout:directory}'
pid_file='${my-cnf-parameters:pid-file}'
binlog_path='${my-cnf-parameters:binlog-path}'
server_executable='${mysqld:rendered}'
update_executable='${update-mysql:output}'
mysql_executable='{{ mysql_executable }}'
mariadb_data_directory='{{ mariadb_data_directory }}'
mariadb_backup_directory='{{ mariadb_backup_directory }}'
instance_directory='{{ instance_directory }}'
pid_file='{{ pid_file }}'
binlog_path='{{ binlog_path }}'
server_executable='{{ server_executable }}'
update_executable='{{ update_executable }}'
# Make sure mariadb is not already running
if [ -e "$pid_file" ]; then
......
stack/erp5/instance-mariadb.cfg.in
View file @ 56ad14fb
......@@ -16,9 +16,11 @@
{% else -%}
{% set ip = (ipv4_set | list)[0] -%}
{% endif -%}
{% set dash = parameter_dict['dash-location'] ~ '/bin/dash' %}
[publish]
[{{ section('publish') }}]
recipe = slapos.cookbook:publish.serialised
-extends = publish-early
{% macro render_database_list(database_list) -%}
{% set publish_database_list = [] -%}
{% for database in database_list -%}
......@@ -32,6 +34,21 @@ recipe = slapos.cookbook:publish.serialised
{% endmacro -%}
database-list = {{ render_database_list(database_list) }}
test-database-list = {{ render_database_list(test_database_list) }}
monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
[publish-early]
recipe = slapos.cookbook:publish-early
-init =
server-id gen-server-id:value
{%- set server_id = slapparameter_dict.get('server-id') %}
{%- if server_id %}
server-id = {{ dumps(server_id) }}
{%- endif %}
[gen-server-id]
recipe = slapos.cookbook:random.integer
minimum = {{ dumps(1) }}
maximum = {{ dumps(2**32 - 1) }}
[jinja2-template-base]
recipe = slapos.recipe.template:jinja2
......@@ -124,7 +141,7 @@ relaxed-writes = {{ dumps(slapparameter_dict.get('relaxed-writes', False)) }}
binlog-path = ${directory:mariadb-backup-incremental}/binlog
# XXX: binlog rotation happens along with other log's rotation
binlog-expire-days = {{ dumps(incremental_backup_retention_days) }}
server-id = {{ dumps(slapparameter_dict.get('server-id', 1)) }}
server-id = ${publish-early:server-id}
{% else %}
binlog-path =
{%- endif %}
......@@ -148,7 +165,7 @@ rendered = ${directory:etc}/mariadb_initial_setup.sql
template = {{ parameter_dict['template-mariadb-initial-setup'] }}
context = section parameter_dict init-script-parameters
[update-mysql]
[{{ section('update-mysql') }}]
recipe = slapos.cookbook:generic.mysql.wrap_update_mysql
output = ${directory:services}/mariadb_update
binary = ${binary-wrap-mysql_upgrade:wrapper-path}
......@@ -162,7 +179,7 @@ mysql_tzinfo_to_sql = ${binary-wrap-mysql_tzinfo_to_sql:wrapper-path}
# section is installed - so I only use jinja2 as a fancy way to write an
# executable file with partition-dependent but instance-parameters independent
# content.
template = inline:#!{{ parameter_dict['dash-location'] }}/bin/dash
template = inline:#!{{ dash }}
'{{ parameter_dict['mariadb-location'] }}/scripts/mysql_install_db' \
--defaults-file='${my-cnf:rendered}' \
--skip-name-resolve \
......@@ -173,13 +190,13 @@ template = inline:#!{{ parameter_dict['dash-location'] }}/bin/dash
"$@"
rendered = ${directory:services}/mariadb
[logrotate-entry-mariadb]
[{{ section('logrotate-entry-mariadb') }}]
< = logrotate-entry-base
name = mariadb
log = ${my-cnf-parameters:error-log} ${my-cnf-parameters:slow-query-log}
post = "${binary-wrap-mysql:wrapper-path}" -B -u root -e "FLUSH LOGS"
[binary-link]
[{{ section('binary-link') }}]
recipe = slapos.cookbook:symbolic.link
target-directory = ${directory:bin}
link-binary = {{ dumps(parameter_dict['link-binary']) }}
......@@ -213,6 +230,12 @@ command = mysql_upgrade
command-line = "{{ parameter_dict['mariadb-location'] }}/bin/${:command}"
command = mysql_tzinfo_to_sql
[binary-wrap-pt-digest]
<= binary-wrap-base
command-line = "{{ parameter_dict['percona-tools-location'] }}/bin/${:command}"
command = pt-query-digest
[directory]
recipe = slapos.cookbook:mkdirectory
bin = ${buildout:directory}/bin
......@@ -230,41 +253,60 @@ var = ${buildout:directory}/var
log = ${:var}/log
run = ${:var}/run
[resiliency-exclude-file]
[{{ section('resiliency-exclude-file') }}]
# Generate rdiff exclude file in case of resiliency
recipe = collective.recipe.template
input = inline: srv/mariadb/**
output = ${directory:srv}/exporter.exclude
< = jinja2-template-base
template = {{ 'inline:{{ "${directory:mariadb-data}/**\\n" }}' }}
rendered = ${directory:srv}/exporter.exclude
[dash]
dash = {{ dumps(dash) }}
[resiliency-after-import-script]
# Generate after import script used by importer instance of webrunner
recipe = collective.recipe.template
input = {{ parameter_dict['mariadb-resiliency-after-import-script'] }}
output = ${directory:srv}/runner-import-restore
< = jinja2-template-executable
template = {{ parameter_dict['mariadb-resiliency-after-import-script'] }}
rendered = ${directory:srv}/runner-import-restore
context =
key dash dash:dash
key mysql_executable binary-wrap-mysql:wrapper-path
key mariadb_data_directory directory:mariadb-data
key mariadb_backup_directory directory:mariadb-backup-full
key instance_directory buildout:directory
key pid_file my-cnf-parameters:pid-file
key binlog_path my-cnf-parameters:binlog-path
key server_executable mysqld:rendered
key update_executable update-mysql:output
[{{ section('monitor-generate-mariadb-slow-query-report') }}]
recipe = slapos.recipe.template:jinja2
template = {{ parameter_dict['mariadb-slow-query-report-script'] }}
rendered = ${monitor-directory:reports}/${:filename}
output-folder = ${directory:srv}/monitor/
filename = mariadb_slow_query_every_23_hour
mode = 755
dash = {{ parameter_dict['dash-location'] }}/bin/dash
context =
raw slow_query_path ${directory:srv}/backup/logrotate/mariadb_slowquery.log
raw pt_query_exec ${binary-wrap-pt-digest:wrapper-path}
raw dash {{ parameter_dict['dash-location'] }}/bin/dash
key output_folder :output-folder
[promise]
[{{ section('promise') }}]
recipe = slapos.cookbook:wrapper
command-line = "{{ parameter_dict['bin-directory'] }}/is-local-tcp-port-opened" "${my-cnf-parameters:ip}" "${my-cnf-parameters:port}"
wrapper-path = ${directory:promise}/mariadb
parameters-extra = true
{% set monitor_passwd = slapparameter_dict['monitor-passwd'] -%}
[monitor-instance-parameter]
monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }}
monitor-httpd-port = {{ port + 1 }}
monitor-title = Mariadb monitor
password = {{ slapparameter_dict['monitor-passwd'] }}
[buildout]
extends =
{{ logrotate_cfg }}
{{ parameter_dict['template-monitor'] }}
parts +=
publish
logrotate-entry-mariadb
binary-link
update-mysql
resiliency-exclude-file
resiliency-after-import-script
promise
{{ part_list | join('\n ') }}
stack/erp5/instance-zeo.cfg.in
View file @ 56ad14fb
......@@ -113,14 +113,16 @@ tidstorage-port = ${tidstorage:port}
tidstorage-ip =
tidstorage-port =
{% endif -%}
monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
# Used for ERP5 resiliency or (more probably)
# webrunner resiliency with erp5 inside.
[{{ section("resiliency-exclude-file") }}]
# Generate rdiff exclude file
recipe = collective.recipe.template
input = inline: srv/zodb/**
output = ${directory:srv}/exporter.exclude
recipe = slapos.recipe.template:jinja2
mode = 644
template = {{ 'inline:{{ "${directory:zodb}/**\\n" }}' }}
rendered = ${directory:srv}/exporter.exclude
[{{ section("resiliency-after-import-script") }}]
# Generate after import script used by importer instance of webrunner
......@@ -184,6 +186,7 @@ tidstorage = {{ tidstorage_repozo_path }}
monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }}
monitor-httpd-port = {{ next_port() }}
monitor-title = ZODB monitor
password = {{ slapparameter_dict['monitor-passwd'] }}
[buildout]
extends =
......
stack/erp5/instance-zope.cfg.in
View file @ 56ad14fb
......@@ -379,6 +379,7 @@ dict toward erp5 partition, violating the DRY principle and making the intent
hard to guess.
-#}
hosts-dict = {{ dumps(hosts_dict) }}
monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
[erp5-promise]
recipe = slapos.cookbook:erp5.promise
......@@ -394,6 +395,7 @@ bt5-repository-url = {{ dumps(slapparameter_dict['bt5-repository-url']) }}
monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }}
monitor-httpd-port = {{ next_port() }}
monitor-title = Zope monitor
password = {{ slapparameter_dict['monitor-passwd'] }}
[buildout]
extends =
......
stack/erp5/instance.cfg.in
View file @ 56ad14fb
[buildout]
extends = {{ instance_common_cfg }}
extends =
{{ instance_common_cfg }}
{{ caucase_template }}
[jinja2-template-base]
mode = 644
......@@ -72,6 +74,7 @@ extra-context =
[dynamic-template-erp5-parameters]
jupyter-enable-default = {{ jupyter_enable_default }}
local-bt5-repository = {{ local_bt5_repository }}
template-monitor = {{ dumps(template_monitor) }}
[dynamic-template-erp5]
<= jinja2-template-base
......@@ -80,6 +83,7 @@ filename = instance-erp5.cfg
extra-context =
key jupyter_enable_default dynamic-template-erp5-parameters:jupyter-enable-default
key local_bt5_repository dynamic-template-erp5-parameters:local-bt5-repository
key template_monitor dynamic-template-erp5-parameters:template-monitor
key openssl_location :openssl-location
import urlparse urlparse
import-list =
......@@ -92,7 +96,10 @@ apache = {{ apache_location }}
openssl = {{ openssl_location }}
haproxy = {{ haproxy_location }}
bin-directory = {{ bin_directory }}
apachedex-location = {{ bin_directory }}/apachedex
run-apachedex-location = {{ bin_directory }}/runApacheDex
6tunnel = {{ sixtunnel_location }}
curl-location = {{ curl_location }}
dash = {{ dash_location }}
template-haproxy-cfg = {{ template_haproxy_cfg }}
template-apache-conf = {{ template_apache_conf }}
......@@ -172,6 +179,8 @@ template-mariadb-initial-setup = {{ template_mariadb_initial_setup }}
link-binary = {{ dumps(mariadb_link_binary) }}
bin-directory = {{ bin_directory }}
mariadb-resiliency-after-import-script = {{ mariadb_resiliency_after_import_script }}
mariadb-slow-query-report-script = {{ mariadb_slow_query_report_script }}
percona-tools-location = {{ percona_toolkit_location }}
template-monitor = {{ template_monitor }}
[dynamic-template-mariadb]
......@@ -210,6 +219,7 @@ create-erp5-site = dynamic-template-create-erp5-site:rendered
RootSoftwareInstance = ${:default}
# Internal software types
kumofs = dynamic-template-kumofs:rendered
caucase = dynamic-template-caucase:rendered
cloudooo = dynamic-template-cloudooo:rendered
mariadb = dynamic-template-mariadb:rendered
balancer = dynamic-template-balancer:rendered
......
stack/erp5/mysql-querydigest.sh.in 0 → 100644
View file @ 56ad14fb
#!{{dash}}
# BEWARE: This file is operated by slapgrid
SLOW_QUERY_PATH='{{slow_query_path}}'
OUTPUT_FOLDER='{{output_folder}}'
PT_QUERY_EXEC='{{pt_query_exec}}'
if [ ! -d "$OUTPUT_FOLDER" ]; then
echo "ERROR: output_folder don't exists"
exit 0
fi
TODAY=`date +%Y%m%d`
SLOW_LOG=$SLOW_QUERY_PATH-$TODAY
if [ ! -f "$SLOW_LOG" ]; then
echo "ERROR: cannot read mysql slow query log file $SLOW_LOG. Exiting."
exit 1
fi
eval $PT_QUERY_EXEC $SLOW_LOG
stack/erp5/postfix_main.cf.in
View file @ 56ad14fb
......@@ -52,6 +52,10 @@ smtpd_relay_restrictions =
permit_sasl_authenticated
defer_unauth_destination
# We do not pass mail address in command lines, so accept those starting with
# a dash.
allow_min_user = yes
# Disable local delivery
local_transport = error
......
stack/lamp.cfg
View file @ 56ad14fb
......@@ -42,7 +42,7 @@ keep-compile-dir = false
apache-libcloud = 0.9.1
async = 0.6.1
gitdb = 0.5.4
mysqlclient = 1.3.10
mysqlclient = 1.3.10+SlapOSPatched001
plone.recipe.command = 1.1
slapos.recipe.template = 2.3
slapos.toolbox = 0.40.4
......
stack/lamp/buildout.cfg
View file @ 56ad14fb
......@@ -185,7 +185,7 @@ eggs =
apache-libcloud = 0.12.3
async = 0.6.1
gitdb = 0.5.4
mysqlclient = 1.3.10
mysqlclient = 1.3.10+SlapOSPatched001
pycrypto = 2.6
rdiff-backup = 1.0.5+SlapOSPatched001
slapos.recipe.template = 2.4.2
......
stack/monitor/README.md
View file @ 56ad14fb
......@@ -26,7 +26,7 @@ Extend monitor template and a monitor-base to parts:
...
monitor-base
Override monitor configuration section to define your custom parameters.
Override monitor configuration by adding monitor-instance-parameter section to define your custom parameters.
[monitor-instance-parameter]
monitor-title = ${slap-configuration:instance-title}
......@@ -38,7 +38,7 @@ Override monitor configuration section to define your custom parameters.
cors-domains = monitor.app.officejs.com
collector-db = ...
password = ${monitor-htpasswd:passwd}
username = ${monitor-htpasswd:username}
username = admin
instance-configuration = ...
configuration-file-path = ...
interface-url = ...
......@@ -53,11 +53,11 @@ You don't need to define all parameters, you can only set what is required to be
- root-instance-title: is the title of the hosting subscription.
- monitor-httpd-ipv6: is the ipv6 of the computer partition.
- monitor-httpd-port: the port to bind monitor httpd server on.
- monitor-base-url: this url that will be used/showed in monitor interface. This url is present in some monitor generated output files. There can be two value, the default: ${monitor-frontend-promise:url} which access monitor httpd server through the frontend and ${monitor-httpd-conf-parameter:url} which is the url with ipv6 (https://[IPv6]:port/).
- monitor-base-url: this url will be used/showed in monitor interface. This url is present in some monitor generated output files. There can be two value, the default: ${monitor-frontend-promise:url} which access monitor httpd server through the frontend and ${monitor-httpd-conf-parameter:url} which is the url with ipv6 (https://[IPv6]:port/).
- monitor-url-list: set list of Monitor Base URL of monitor sub-instances, if this is the root instance with at least one child.
- cors-domains: the domain used by the monitor web interface. The default is: monitor.app.officejs.com.
- username: monitor username, this should be the same in all sub-instances. Default is: admin.
- password: monitor password, this should be the same in all sub-instances. Default is generated (${monitor-htpasswd:username}).
- password: monitor password, this should be the same in all sub-instances. Default is generated (${monitor-instance-parameter:username}).
- instance-configuration: instance custom information or configuration to show in monitor web interface. There is many possibility:
raw CONFIG_KEY VALUE => non editable configuration, ie: raw monitor-password resqdsdsd34
file CONFIG_KEY PATH_TO_RESULT_FILE => editable configuration.
......@@ -65,6 +65,18 @@ You don't need to define all parameters, you can only set what is required to be
- configuration-file-path: path of knowledge0 cfg file where instance configuration will be written.
- interface-url: The URL of monitor web interface. This URL will be present in generated JSON files.
**Multiple Monitors**
If you have sub-instances, you should collect the base monitor url from all instances with monitor and send it to monitor-url-list or you can override "monitor-base-url-dict" section and add all the urls as key/value pairs in the root instance.
[monitor-base-url-list]
monitor1-url = https://[xxxx:xxx:xxxx:e:11::1fb1]:4200
monitor2-url = https://[xxxx:xxx:xxxx:e:22::2fb2]:4200
..
..
Also, All monitors of the sub instances need to have same password as the password of the root instance monitor.
NB: You should use double $ (ex: $${monitor-template:rendered}) instead of one $ in your instance template file if it's not a jinja template. See:
- Jinja template file exemple, use one $: https://lab.nexedi.com/nexedi/slapos/blob/master/software/slaprunner/instance-resilient-test.cfg.jinja2
- Non Jinja template file, use $$: https://lab.nexedi.com/nexedi/slapos/blob/master/software/slaprunner/instance.cfg
......@@ -187,13 +199,13 @@ MONITOR_BASE_URL/share is the webdav URL. public/ and private/ are linked to pub
Access to Monitor
-----------------
In monitor instance.cfg file, the section [publish] contain information about monitor access.
In monitor instance.cfg file, the section [monitor-publish-parameters] contain information about monitor access.
Usefull information are monitor-base-url, monitor-url, monitor-user and monitor-password.
- ${publish:monitor-base-url} is the url of monitor httpd server.
- ${publish:monitor-base-url}/public/feed is the RSS url of this monitor instance.
- ${publish:monitor-base-url}/public/feeds is the OPML URL of this monitor instance. To setup monitor instance in your monitoring interface, use OPML URL of the root instance. It should contain URL to others monitor instances.
- ${publish:monitor-base-url}/private is the monitor private directory. Username and password are reqired to connect.
- ${monitor-publish-parameters:monitor-base-url} is the url of monitor httpd server.
- ${monitor-publish-parameters:monitor-base-url}/public/feed is the RSS url of this monitor instance.
- ${monitor-publish-parameters:monitor-base-url}/public/feeds is the OPML URL of this monitor instance. To setup monitor instance in your monitoring interface, use OPML URL of the root instance. It should contain URL to others monitor instances.
- ${monitor-publish-parameters:monitor-base-url}/private is the monitor private directory. Username and password are reqired to connect.
The section [monitor-publish] contain parameters to publish with your instance connection information. It will publish "monitor-base-url" and
"monitor-setup-url" which is used to configure your instance to monitor interface in one click.
......@@ -218,7 +230,7 @@ Parameters can be editable (except raw parameter) directly from monitor interfa
[monitor-instance-parameter]
instance-configuration =
raw init-user ${publish-connection-information:init-user}
htpasswd monitor-password ${monitor-htpassword-file:password-file} ${monitor-instance-parameter:username} ${httpd-monitor-htpasswd:htpasswd-path}
htpasswd monitor-password ${httpd-monitor-htpasswd:password-file} ${monitor-instance-parameter:username} ${httpd-monitor-htpasswd:htpasswd-path}
file promise-timeout ${monitor-promise-timeout-file:file}
The user will see parameters:
......
stack/monitor/buildout.cfg
View file @ 56ad14fb
......@@ -72,7 +72,7 @@ md5sum = 1695c9a06a2b11ccfe893d7a224e489d
[monitor-conf]
<= monitor-template-base
filename = monitor.conf.in
md5sum = c8f024d741c6494d7c9ba01601d0b917
md5sum = fc65084dd1c2fe2487b58a003b576f61
[monitor-instance-info]
<= monitor-template-base
......@@ -131,5 +131,5 @@ depends =
PyRSS2Gen = 1.1
cns.recipe.symlink = 0.2.3
pycurl = 7.43.0
slapos.toolbox = 0.67
slapos.toolbox = 0.69
stack/monitor/buildout.hash.cfg
View file @ 56ad14fb
......@@ -15,4 +15,4 @@
# not need these here).
[monitor2-template]
filename = instance-monitor.cfg.jinja2.in
md5sum = c963c43189a9b50e48ba27f2dd4e7b02
md5sum = 5027f0b1abcc28ce3817cd70fb667a3b
stack/monitor/instance-monitor.cfg.jinja2.in
View file @ 56ad14fb
......@@ -100,7 +100,7 @@ parameter-file-path = ${monitor-instance-parameter:configuration-file-path}
parameter-list =
raw monitor-user ${monitor-instance-parameter:username}
htpasswd monitor-password ${monitor-htpassword-file:password-file} ${monitor-instance-parameter:username} ${httpd-monitor-htpasswd:htpasswd-path}
htpasswd monitor-password ${httpd-monitor-htpasswd:password-file} ${monitor-instance-parameter:username} ${httpd-monitor-htpasswd:htpasswd-path}
file promise-timeout ${monitor-promise-timeout-file:file}
file min-free-disk-MB ${promise-check-free-disk-space:config-file}
${monitor-instance-parameter:instance-configuration}
......@@ -122,12 +122,16 @@ command =
echo "20" > ${:file}
fi
[monitor-base-url-dict]
# place holder to be used to collect erp5 monitor urls
[monitor-conf]
recipe = slapos.recipe.template:jinja2
template = {{ monitor_conf_template }}
rendered = ${directory:etc}/${:filename}
filename = monitor.conf
context = section parameter_dict monitor-conf-parameters
section monitor_base_urls monitor-base-url-dict
[instance-info-parameters]
name = ${monitor-instance-parameter:monitor-title}
......@@ -156,7 +160,8 @@ update-command = ${:command}
[start-monitor]
recipe = slapos.cookbook:wrapper
command-line = {{ python_executable }} {{ monitor_bin }} --config_file ${monitor-conf:rendered}
wrapper-path = ${directory:scripts}/bootstrap-monitor
name = bootstrap-monitor
wrapper-path = ${directory:scripts}/${:name}
environment =
PATH=${python-symlink:target}:/usr/local/bin:/usr/bin:/bin
......@@ -164,25 +169,18 @@ environment =
recipe = slapos.cookbook:generate.password
storage-path = ${directory:etc}/.monitor_pwd
bytes = 8
username = admin
[monitor-htpassword-file]
recipe = plone.recipe.command
stop-on-error = true
password-file = ${directory:etc}/.monitor_pwd
command =
if [ ! -s "${:password-file}" ]; then echo "${monitor-instance-parameter:password}" > ${:password-file}; fi
update-command = ${:command}
[httpd-monitor-htpasswd]
recipe = plone.recipe.command
stop-on-error = true
password-file = ${directory:etc}/.monitor_pwd
htpasswd-path = ${monitor-directory:etc}/monitor-htpasswd
# Keep multiple lines as password can end with newline char.
command =
if [ ! -s "${:htpasswd-path}" ]; then
{{ apache_location }}/bin/htpasswd -cb ${:htpasswd-path} ${:user} ${:password}
fi
if [ ! -s "${:password-file}" ]; then echo "${monitor-instance-parameter:password}" > ${:password-file}; fi
update-command = ${:command}
user = ${monitor-instance-parameter:username}
password = ${monitor-instance-parameter:password}
......@@ -284,7 +282,7 @@ check-secure = 1
dash_path = {{ dash_executable_location }}
curl_path = {{ curl_executable_location }}
[publish]
[monitor-publish-parameters]
# XXX depends on monitor-base section
monitor-base-url = ${monitor-base:base-url}
monitor-url = ${:monitor-base-url}/public/feeds
......@@ -305,7 +303,7 @@ cors-domains = monitor.app.officejs.com
collector-db = /srv/slapgrid/var/data-log/collector.db
# Credentials
password = ${monitor-htpasswd:passwd}
username = ${monitor-htpasswd:username}
username = admin
# XXX: type key value
# ex raw monitor-password resqdsdsd34
instance-configuration =
......@@ -337,6 +335,7 @@ check-secure = 1
[monitor-bootstrap-promise]
recipe = collective.recipe.template
file = ${monitor-conf-parameters:promise-output-file}
error-log-file = ${buildout:directory}/.${slap-connection:partition-id}_${start-monitor:name}.log
input = inline:#!{{ dash_executable_location }}
pidfile=${monitor-conf-parameters:pid-file}
if [ -s $pidfile ]; then
......@@ -351,7 +350,18 @@ input = inline:#!{{ dash_executable_location }}
fi
done
fi
if [ ! -f "${:file}" ]; then echo "Monitor bootstrap exited with error." && exit 2; else echo "Bootstrap OK"; fi
if [ ! -f "${:file}" ]; then
echo "Monitor bootstrap exited with error."
log_file="${:error-log-file}"
if [ -s "$log_file" ]; then
echo " ---- Latest monitor-boostrap.log ----"
echo ""
tail -n 3 $log_file
fi
exit 2
else
echo "Bootstrap OK";
fi
output = ${directory:promises}/monitor-bootstrap-status
mode = 700
......@@ -399,8 +409,8 @@ depends =
${promise-monitor-httpd-is-process-older-than-dependency-set:wrapper-path}
[monitor-publish]
monitor-base-url = ${publish:monitor-base-url}
monitor-setup-url = ${monitor-instance-parameter:interface-url}/#page=settings_configurator&url=${publish:monitor-url}&username=${publish:monitor-user}&password=${publish:monitor-password}
monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
monitor-setup-url = ${monitor-instance-parameter:interface-url}/#page=settings_configurator&url=${monitor-publish-parameters:monitor-url}&username=${monitor-publish-parameters:monitor-user}&password=${monitor-publish-parameters:monitor-password}
[buildout]
......
stack/monitor/templates/monitor.conf.in
View file @ 56ad14fb
[monitor]
{% for key, value in parameter_dict.items() -%}
{% if key == "monitor-url-list" and monitor_base_urls is defined -%}
{{ key }} =
{% for key, value in monitor_base_urls.items() -%}
{{ ' ' ~ value }}
{% endfor -%}
{% else -%}
{{ key }} = {{ value.strip().replace("\n", "\n ") }}
{% endfor -%}
\ No newline at end of file
{% endif -%}
{% endfor -%}
stack/resilient/buildout.cfg
View file @ 56ad14fb
......@@ -121,6 +121,10 @@ download-only = true
filename = monitor-check-resilient-feed.in
mode = 0644
[exporter-default-configuration]
# Time in minutes to wait for the backup to finish
backup_wait_time = 10
[rdiff-backup-build]
# use our own version
find-links = http://www.nexedi.org/static/packages/source/rdiff-backup-1.3.4nxd2.tar.gz
......
stack/resilient/buildout.hash.cfg
View file @ 56ad14fb
......@@ -15,23 +15,23 @@
# not need these here).
[pbsready]
filename = pbsready.cfg.in
md5sum = 2adb49f45d27cbcb7430c98c2224d76a
md5sum = bceb082deb6b3ee9b015c8b04436b52c
[pbsready-import]
filename = pbsready-import.cfg.in
md5sum = a8f77b42782ec5e1fb49b5ae9891c757
md5sum = 9d36d08ac6ae351b598a67db41657cc6
[pbsready-export]
filename = pbsready-export.cfg.in
md5sum = bb463da1461d02d575717f1d1572bf97
md5sum = c6c11db5372150019debb1ce519b907d
[template-pull-backup]
filename = instance-pull-backup.cfg.in
md5sum = 7b4f8ac1a62680d624ac632f9601dab5
md5sum = fe4811540683bfcfac5ac12de084b623
[template-replicated]
filename = template-replicated.cfg.in
md5sum = 75686abd9cb8d6e1f4e15931e280fa56
md5sum = 55ef107f499e5e604b8d277098618fed
[template-parts]
filename = template-parts.cfg.in
......
stack/resilient/instance-pull-backup.cfg.in
View file @ 56ad14fb
......@@ -137,7 +137,7 @@ slave-instance-list = $${slap-parameter:slave_instance_list}
ignore-known-hosts-file = $${slap-parameter:ignore-known-hosts-file}
# To get a verbose feed about PBS state
instance-root-name = $${instance-info-parameters:root-name}
log-url = $${publish:monitor-base-url}/private/notifier/
log-url = $${monitor-publish-parameters:monitor-base-url}/private/notifier/
status-item-directory = $${directory:notifier-status-items}
[pbs-resilient-status-feed]
......@@ -260,11 +260,6 @@ parameters-extra = true
#----------------
#--
#-- Slave instance list (empty default).
[htpasswd]
recipe = slapos.cookbook:generate.password
storage-path = $${directory:etc}/.monitor_user
bytes = 8
username = admin
[slap-parameter]
slave_instance_list = []
......@@ -272,8 +267,8 @@ ignore-known-hosts-file = false
monitor-cors-domains =
monitor-httpd-port = 8070
monitor-title = PBS Instance
monitor-password = $${htpasswd:passwd}
monitor-username = $${htpasswd:username}
monitor-password = $${monitor-htpasswd:passwd}
monitor-username = admin
#----------------
#--
......@@ -341,10 +336,10 @@ recipe = slapos.cookbook:publish
ssh-key = $${sshkeys-openssh:public-key-value}
notification-url = http://[$${notifier:host}]:$${notifier:port}/notify
feeds-url = http://[$${notifier:host}]:$${notifier:port}/get/
monitor-base-url = $${publish:monitor-base-url}
monitor-url = $${publish:monitor-url}
monitor-user = $${publish:monitor-user}
monitor-password = $${publish:monitor-password}
monitor-base-url = $${monitor-publish-parameters:monitor-base-url}
monitor-url = $${monitor-publish-parameters:monitor-url}
monitor-user = $${monitor-publish-parameters:monitor-user}
monitor-password = $${monitor-publish-parameters:monitor-password}
#----------------
#--
......
stack/resilient/pbsready-export.cfg.in
View file @ 56ad14fb
......@@ -44,7 +44,7 @@ pidfile = ${resilient-directory:pid}/${:name}.pid
max-run = 3
[logrotate-entry-notifier]
output = ${rootdirectory:etc}/logrotate_notifier.conf
rendered = ${rootdirectory:etc}/logrotate_notifier.conf
[notifier-exporter-promise]
recipe = slapos.recipe.template:jinja2
......@@ -70,7 +70,7 @@ frequency = {{ resiliency_backup_periodicity }}
{% else %}
time = ${publish-early:resiliency-backup-periodicity}
{% endif %}
command = {{ logrotate_executable_location }} -s ${basedirectory:run}/logrotate.status ${logrotate-entry-notifier:output}; ${notifier-exporter:wrapper} --transaction-id `date +%s`
command = {{ logrotate_executable_location }} -s ${basedirectory:run}/logrotate.status ${logrotate-entry-notifier:rendered}; ${notifier-exporter:wrapper} --transaction-id `date +%s`
[gen-resiliency-backup-periodicity]
recipe = slapos.cookbook:random.time
......@@ -83,3 +83,4 @@ recipe = slapos.cookbook:publish-early
[slap-parameter]
# In cron.d format (i.e things like */15 * * * * are accepted).
resiliency-backup-periodicity =
notify =
\ No newline at end of file
stack/resilient/pbsready-import.cfg.in
View file @ 56ad14fb
......@@ -14,6 +14,10 @@ parts +=
backup-signature-link
[slap-parameter]
on-notification =
[resilient-publish-connection-parameter]
notification-url = http://[$${notifier:host}]:$${notifier:port}/notify
takeover-url = http://[$${resilient-web-takeover-httpd-configuration-file:listening-ip}]:$${resilient-web-takeover-httpd-configuration-file:listening-port}/
......@@ -107,7 +111,7 @@ input = ${resilient-web-takeover-cgi-script-download:target}
output = $${directory:cgi-bin}/web-takeover.cgi
password = $${resilient-web-takeover-password:passwd}
mode = 700
proof-signature-url = $${publish:monitor-base-url}/private/resilient/backup.signature
proof-signature-url = $${monitor-publish-parameters:monitor-base-url}/private/resilient/backup.signature
# XXX could it be something lighter?
# XXX Add SSL
......
stack/resilient/pbsready.cfg.in
View file @ 56ad14fb
......@@ -61,6 +61,10 @@ logrotate-backup = $${basedirectory:backup}/logrotate
cgi-bin = $${rootdirectory:srv}/cgi-bin
monitor-resilient = $${monitor-directory:private}/resilient
[slap-parameter]
authorized-key =
#----------------
#--
#-- Deploy cron.
......@@ -132,7 +136,7 @@ notifier-binary = ${buildout:bin-directory}/pubsubnotifier
host = $${slap-network-information:global-ipv6}
port = $${notifier-port:port}
instance-root-name = $${instance-info-parameters:root-name}
log-url = $${publish:monitor-base-url}/resilient/notifier-status-rss
log-url = $${monitor-publish-parameters:monitor-base-url}/resilient/notifier-status-rss
status-item-directory = $${directory:notifier-status-items}
context =
key content notifier:command
......
stack/resilient/template-replicated.cfg.in
View file @ 56ad14fb
......@@ -91,6 +91,10 @@ return = ssh-public-key ssh-url notification-url ip takeover-url takeover-passwo
pbs-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-{{id}}-push
{% if slapparameter_dict.get("software-root") %}
config-software-root = {{ slapparameter_dict.get("software-root") }}
{% endif %}
config-number = {{id}}
config-name = {{namebase}}{{id}}
config-authorized-key = ${request-pbs-{{namebase}}-{{id}}:connection-ssh-key}
......
stack/shacache-client.cfg
View file @ 56ad14fb
......@@ -3,5 +3,5 @@
networkcache-section = networkcache
[networkcache]
download-cache-url = http://www.shacache.org/shacache
download-dir-url = http://www.shacache.org/shadir
download-cache-url = http://download.shacache.org
download-dir-url = http://dir.shacache.org
stack/slapos.cfg
View file @ 56ad14fb
......@@ -100,7 +100,7 @@ eggs =
[versions]
# Use SlapOS patched zc.buildout
zc.buildout = 2.5.2+slapos007
zc.buildout = 2.5.2+slapos009
# Use SlapOS patched zc.recipe.egg (zc.recipe.egg 2.x is for Buildout 2)
zc.recipe.egg = 2.0.3+slapos003
# Use own version of h.r.download to be able to open .xz and .lz archives
......@@ -137,14 +137,15 @@ setuptools = 33.1.1
six = 1.10.0
slapos.cookbook = 1.0.50
slapos.core = 1.3.18
slapos.extension.strip = 0.2
slapos.libnetworkcache = 0.14.5
slapos.rebootstrap = 3.10
slapos.recipe.build = 0.33
slapos.recipe.cmmi = 0.5
slapos.extension.strip = 0.4
slapos.libnetworkcache = 0.15
slapos.rebootstrap = 4.1
slapos.recipe.build = 0.36
slapos.recipe.cmmi = 0.7
stevedore = 1.21.0
unicodecsv = 0.14.1
xml-marshaller = 0.9.7
paramiko = 2.1.3
# Required by:
# slapos.core==1.3.18
......@@ -192,8 +193,8 @@ uritemplate = 3.0.0
zope.interface = 4.3.3
[networkcache]
download-cache-url = http://www.shacache.org/shacache
download-dir-url = http://www.shacache.org/shadir
download-cache-url = http://download.shacache.org/
download-dir-url = http://dir.shacache.org/
# signature certificates of the following uploaders.
# Romain Courteaud
......@@ -204,6 +205,7 @@ download-dir-url = http://www.shacache.org/shadir
# Aurélien Calonne
# Rafael Monnerat
# Ivan Tyagov
# Julien Muchembled
signature-certificate-list =
-----BEGIN CERTIFICATE-----
MIIB4DCCAUkCADANBgkqhkiG9w0BAQsFADA5MQswCQYDVQQGEwJGUjEZMBcGA1UE
......@@ -309,3 +311,16 @@ signature-certificate-list =
8dEAOMIcdrPBzldf0JS+poCrLUw7qx9VZxa0ee3+F6WGEgsr2xhu2WpmTzpRWaFR
0I+saiiJCkOkLoyLxeQcgwe62em/DGn6PdcmRDYaVXBHygrRXus=
-----END CERTIFICATE-----
-----BEGIN CERTIFICATE-----
MIIB6jCCAVOgAwIBAgIJAMhx/tG6zHFJMA0GCSqGSIb3DQEBBQUAMA0xCzAJBgNV
BAMMAmptMCAXDTEzMDcwMTE3NDcyOVoYDzIxMTMwNjA3MTc0NzI5WjANMQswCQYD
VQQDDAJqbTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAyFJp6K3HAle6oUB4
/TxIvc06wju+ZzZuPX4fxq2GqOV9QX2jhPfYCyr/cJh4pOZEW1gwMw+9kzghmjNT
6RoBEWMS3E0F2K/waYb+doGb1Dffp2FIi3m547b20XnpnaIj3DS7FF30lRNo8jtR
1Dg+RDGpA23Pr1/3IAskC5qSydkCAwEAAaNQME4wHQYDVR0OBBYEFHHmUnqEVNEo
FpLiBjCmR5ZJsPndMB8GA1UdIwQYMBaAFHHmUnqEVNEoFpLiBjCmR5ZJsPndMAwG
A1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADgYEAPP5oOZWVrYjc5BB+pDVQm/rF
gVhgPPg+23oCiz0p9NzWgnYXhGmQWtzXVVVaL4aftkmX04mlT2Fgd4msEzQB6pAa
SOvU9E1802JN7Q1mrMbgJeNmk+18o4ze7ABKljqtvo+ZjUXthuWw3BNt+PtQjlX3
5wE+Yc4sypYGmAfU7Gc=
-----END CERTIFICATE-----
validate-schema 0 → 100755
View file @ 56ad14fb
#!/usr/bin/env python
import argparse
import json
import os.path
import urllib
from urlparse import urlparse, urlunparse, ParseResult
import jsonschema
# Adapted from slapos.core.git/slapos/slap/util.py
from lxml import etree
def xml2dict(infile):
result_dict = {}
for element in etree.parse(infile).iter(tag=etree.Element):
if element.tag == 'parameter':
key = element.get('id')
value = result_dict.get(key, None)
if value is not None:
value = value + ' ' + element.text
else:
value = element.text
result_dict[key] = value
return result_dict
def jsonInXML2dict(infile):
wrapped = xml2dict(infile)
if '_' in wrapped:
return json.loads(wrapped['_'])
return wrapped
class ValidatingRefResolver(jsonschema.RefResolver):
def resolve_remote(self, *args, **kw):
result = super(ValidatingRefResolver, self).resolve_remote(*args, **kw)
jsonschema.Draft4Validator.check_schema(result)
return result
DEFAULT_SLAPOS_SCHEMA_PATH = os.path.join(
os.path.dirname(__file__),
'schema.json',
)
def main():
parser = argparse.ArgumentParser(
description='Validates SlapOS Software Release descriptor schemas, '
'and optionally validates specific requests and/or responses with '
'these schemas.',
)
parser.add_argument('--software-type', type=str, help='Software type given request/response is intended for')
parser.add_argument('--request', type=argparse.FileType('r'), help='File containing request parameters')
parser.add_argument('--response', type=argparse.FileType('r'), help='File containing published values')
parser.add_argument('--slapos-schema', type=argparse.FileType('r'), help='SlapOS base schema. Default: %s' % (DEFAULT_SLAPOS_SCHEMA_PATH, ))
parser.add_argument('software_release', nargs=1, help='URL of the software release')
args = parser.parse_args()
if args.software_type is None and (
args.request is not None or
args.response is not None
):
parser.error(
'--software-type is required if --request or --response is provided',
)
slapos_schema_file = args.slapos_schema
if slapos_schema_file is None:
slapos_schema_file = open(DEFAULT_SLAPOS_SCHEMA_PATH)
slapos_schema = json.load(slapos_schema_file)
jsonschema.Draft4Validator.check_schema(slapos_schema)
software_release_url = urlparse(args.software_release[0])
if software_release_url.scheme == software_release_url.netloc == '':
# Assume software_release_url.path is a relative path for "file" scheme
software_release_url = ParseResult(
'file',
'',
os.path.abspath(software_release_url.path),
software_release_url.params,
software_release_url.query,
software_release_url.fragment,
)
software_release_base_path, software_release_file = software_release_url.path.rsplit('/', 1)
def loadurl(relative_url):
url = urlunparse((
software_release_url.scheme,
software_release_url.netloc,
software_release_base_path + '/' + relative_url,
software_release_url.params,
software_release_url.query,
software_release_url.fragment,
))
return url, json.loads(urllib.urlopen(url).read())
_, software_release_descriptor = loadurl(software_release_file + '.json')
jsonschema.Draft4Validator.check_schema(software_release_descriptor)
jsonschema.validate(software_release_descriptor, slapos_schema)
load = {
'xml': xml2dict,
'json-in-xml': jsonInXML2dict,
}[software_release_descriptor['serialisation']]
for software_type_name, software_type in software_release_descriptor['software-type'].iteritems():
for key in ('request', 'response'):
relative_url = software_type[key]
url, software_release_schema = loadurl(relative_url)
jsonschema.Draft4Validator.check_schema(software_release_schema)
validator = jsonschema.Draft4Validator(
schema=software_release_schema,
resolver=ValidatingRefResolver(
base_uri=url,
referrer=software_release_schema,
),
)
infile = getattr(args, key)
if infile is not None and software_type_name == args.software_type:
validator.validate(load(infile))
else:
validator.validate({})
if __name__ == '__main__':
main()
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7