Commit da630113 authored by Łukasz Nowak's avatar Łukasz Nowak

Update Release Candidate

parents e54dbf20 81c14ca3
Pipeline #11195 failed with stage
in 0 seconds
root = true
[*.{cfg,in,md,json}]
end_of_line = lf
charset = utf-8
insert_final_newline = true
[**.json]
indent_style = space
indent_size = 2
# business templates metadata
[**/bt/**]
insert_final_newline = false
......@@ -39,9 +39,9 @@ configure-options =
[apache]
recipe = slapos.recipe.cmmi
shared = true
version = 2.4.43
version = 2.4.46
url = https://archive.apache.org/dist/httpd/httpd-${:version}.tar.bz2
md5sum = 791c986b1e70fe61eb44060aacc89a64
md5sum = 7d661ea5e736dac5e2761d9f49fe8361
configure-options = --disable-static
--enable-authn-alias
--enable-bucketeer
......
......@@ -14,5 +14,4 @@
# not need these here).
[template-apache-backend-conf]
filename = apache-backend.conf.in
md5sum = ace264843228af84d025797f520ce70f
md5sum = 6f51eaa9930e7a7ef959de7c510c7c2f
......@@ -3,6 +3,7 @@ extends =
../autoconf/buildout.cfg
../automake/buildout.cfg
../m4/buildout.cfg
../patch/buildout.cfg
../perl/buildout.cfg
../xz-utils/buildout.cfg
parts =
......@@ -15,7 +16,7 @@ url = http://ftp.gnu.org/gnu/bison/bison-3.3.2.tar.xz
md5sum = c9b552dee234b2f6b66e56b27e5234c9
environment =
M4=${m4:location}/bin/m4
PATH=${autoconf:location}/bin:${automake:location}/bin:${perl:location}/bin:${xz-utils:location}/bin:%(PATH)s
PATH=${autoconf:location}/bin:${automake:location}/bin:${patch:location}/bin:${perl:location}/bin:${xz-utils:location}/bin:%(PATH)s
patch-options = -p1
patches =
${:_profile_base_location_}/01_inhibit_example_generation#ee2df22601a56d738939fa760f9dc19b
......@@ -71,4 +71,3 @@ install =
extract_dir = self.extract(self.download(*options[guessPlatform()].split()))
os.mkdir(location)
shutil.copy(os.path.join(extract_dir, 'chromedriver'), location)
......@@ -28,5 +28,3 @@ entry-points =
scripts =
paster=cloudooo_paster
runCloudoooUnitTest
[template-fonts-conf]
filename = fonts.conf.in
md5sum = 6967e553630d107fc0a59b14de8b0251
......@@ -32,4 +32,3 @@ eggs =
futures
PIL
gateone
......@@ -22,3 +22,4 @@ strip-top-level-dir = true
[versions]
geoip2 = 2.9.0
maxminddb = 1.5.4
......@@ -2,6 +2,7 @@
extends =
../gettext/buildout.cfg
../libffi/buildout.cfg
../patch/buildout.cfg
../pcre/buildout.cfg
../perl/buildout.cfg
../xz-utils/buildout.cfg
......@@ -25,13 +26,14 @@ configure-options =
--disable-xattr
--disable-man
environment =
PATH=${gettext:location}/bin:${perl:location}/bin:${xz-utils:location}/bin:%(PATH)s
PATH=${gettext:location}/bin:${patch:location}/bin:${perl:location}/bin:${xz-utils:location}/bin:%(PATH)s
CPPFLAGS=-I${gettext:location}/include -I${zlib:location}/include
LDFLAGS=-L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
LIBFFI_CFLAGS=-I${libffi:location}/include
LIBFFI_LIBS= -L${libffi:location}/lib -Wl,-rpath=${libffi:location}/lib -lffi
PCRE_CFLAGS=-I${pcre:location}/include
PCRE_LIBS=-L${pcre:location}/lib -Wl,-rpath=${pcre:location}/lib -lpcre
post-install = rm %(location)s/bin/gtester-report
[x86-cygwin-glib]
patches =
......
......@@ -15,8 +15,8 @@ extends =
[groonga]
recipe = slapos.recipe.cmmi
shared = true
url = https://packages.groonga.org/source/groonga/groonga-9.1.2.tar.gz
md5sum = 5266c49b758dde744854cb8cfe025812
url = https://packages.groonga.org/source/groonga/groonga-10.0.5.tar.gz
md5sum = 12f5c7b522afd72d0be2caa1e5f254da
groonga-plugin-dir = @@LOCATION@@/lib/groonga/plugins/
# temporary patch to respect more tokens in natural language mode.
patches =
......
......@@ -55,4 +55,3 @@ environment =
# not taken from pkg-config result...
CPPFLAGS=-I${cairo:location}/include -I${inputproto:location}/include -I${libX11:location}/include -I${libXi:location}/include -I${xproto:location}/include -I${kbproto:location}/include -I${libXrender:location}/include -I${renderext:location}/include -I${libXext:location}/include -I${libX11:location}/include -I${libXinerama:location}/include -I${gdk-pixbuf:location}/include -I${libXrandr:location}/include -I${xfixes:location}/include -I${libXcomposite:location}/include -I${xdamage:location}/include -I${fixesproto:location}/include
LDFLAGS=-L${libX11:location}/lib -L${libXi:location}/lib -L${libXext:location}/lib -L${libXrender:location}/lib -L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib -L${bzip2:location}/lib -Wl,-rpath=${bzip2:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${libX11:location}/lib -Wl,-rpath=${libX11:location}/lib -L${libXinerama:location}/lib -Wl,-rpath=${libXinerama:location}/lib -L${gdk-pixbuf:location}/lib -Wl,-rpath=${gdk-pixbuf:location}/lib -L${libXrandr:location}/lib -Wl,-rpath=${libXrandr:location}/lib -L${xfixes:location}/lib -Wl,-rpath=${xfixes:location}/lib -L${libXcomposite:location}/lib -Wl,-rpath=${libXcomposite:location}/lib -L${xdamage:location}/lib -Wl,-rpath=${xdamage:location}/lib
[buildout]
extends =
../cmake/buildout.cfg
../openssl/buildout.cfg
../zlib/buildout.cfg
../bison/buildout.cfg
../ruby/buildout.cfg
parts =
h2o
[h2o]
recipe = slapos.recipe.cmmi
shared = True
url = https://github.com/h2o/h2o/archive/v2.2.6.tar.gz
md5sum = 075283f92347fcb7b2f87dcc5251b922
configure-command = ${cmake:location}/bin/cmake .
configure-options =
-DCMAKE_INSTALL_PREFIX=@@LOCATION@@
-DWITH_MRUBY=on
-DCMAKE_C_FLAGS="${:CMAKE_CFLAGS}"
-DCMAKE_CXX_FLAGS="${:CMAKE_CFLAGS}"
-DCMAKE_INSTALL_RPATH=${:CMAKE_LIBRARY_PATH}
CMAKE_CFLAGS=-I${openssl:location}/include -I${zlib:location}/include
CMAKE_LIBRARY_PATH=${openssl:location}/lib:${zlib:location}/lib
environment =
PKG_CONFIG_PATH=${openssl:location}/lib/pkgconfig ${zlib:location}/lib/pkgconfig ${ruby:location}/lib/pkgconfig ${bison:location}/lib/pkgconfig
CMAKE_INCLUDE_PATH=${openssl:location}/include:${zlib:location}/include
CMAKE_LIBRARY_PATH=${:CMAKE_LIBRARY_PATH}
LDFLAGS=-L${openssl:location}/lib -Wl,-rpath -Wl,${openssl:location}/lib -L${zlib:location}/lib -Wl,-rpath -Wl,${zlib:location}/lib
PATH=${ruby:location}/bin:${bison:location}/bin:%(PATH)s
......@@ -13,8 +13,8 @@ parts = haproxy
[haproxy]
recipe = slapos.recipe.cmmi
shared = true
url = http://www.haproxy.org/download/2.0/src/haproxy-2.0.15.tar.gz
md5sum = 59f892991476f08e2d16ac460c502f61
url = http://www.haproxy.org/download/2.0/src/haproxy-2.0.17.tar.gz
md5sum = 786a967c73cc1455c938d42fbe333bfe
configure-command = true
# for Linux kernel 2.6.28 and above, we use "linux-glibc" as the TARGET,
# otherwise use "generic".
......
......@@ -102,5 +102,3 @@ input = inline:
#!/bin/sh
export BUNDLE_GEMFILE=${helloweb-ruby-bundle:path}/Gemfile
exec ${bundler:bundle} exec sh -c 'helloweb.rb "$@"' ${:_buildout_section_name_} "$@"
[buildout]
extends =
../attr/buildout.cfg
parts = libcap-ng
[libcap-ng]
recipe = slapos.recipe.cmmi
shared = true
url = https://people.redhat.com/sgrubb/libcap-ng/libcap-ng-0.7.10.tar.gz
md5sum = 57dc267e2949cdecb651a929f9206572
location = @@LOCATION@@
configure-options =
--with-python=no
--with-python3=no
# we set -lpthread to workaround https://github.com/stevegrubb/libcap-ng/issues/9
environment =
LDFLAGS=-lpthread
......@@ -6,4 +6,3 @@ parts =
recipe = slapos.recipe.cmmi
url = https://hyperrealm.github.io/libconfig/dist/libconfig-1.7.2.tar.gz
md5sum = 6bd98ee3a6e6b9126c82c916d7a9e690
......@@ -26,5 +26,3 @@ configure-options =
--without-libinnodb-prefix
--without-libsasl-prefix
--without-libsasl2-prefix
......@@ -6,4 +6,3 @@ parts =
recipe = slapos.recipe.cmmi
url = https://freefr.dl.sourceforge.net/project/lksctp/lksctp-tools/lksctp-tools-1.0.17.tar.gz
md5sum = 68e9b8fa4d4e331029b247b72d46d7a5
......@@ -20,4 +20,3 @@ make-options =
environment =
PATH=${bison:location}/bin:${flex:location}/bin:%(PATH)s
[buildout]
extends =
../xz-utils/buildout.cfg
../patch/buildout.cfg
parts =
m4
......@@ -10,7 +11,7 @@ shared = true
url = http://ftp.gnu.org/gnu/m4/m4-1.4.18.tar.xz
md5sum = 730bb15d96fffe47e148d1e09235af82
environment =
PATH=${xz-utils:location}/bin:%(PATH)s
PATH=${xz-utils:location}/bin:${patch:location}/bin:%(PATH)s
patch-options = -p1
patches =
${:_profile_base_location_}/01-fix-ftbfs-with-glibc-2.28.patch#058a786425e507f911649205b61ffcac
......@@ -30,14 +30,13 @@ parts =
recipe = slapos.recipe.cmmi
shared = true
url = https://downloads.mariadb.org/f/mariadb-${:version}/source/mariadb-${:version}.tar.gz/from/http%3A//fr.mirror.babylon.network/mariadb/?serve
version = 10.4.13
md5sum = 89e832d6d89dc9bd4c98657a954e0f2c
version = 10.4.14
md5sum = 9801120ae8acb33904ab4a3366a7714f
location = @@LOCATION@@
pre-configure =
set '\bSET(PLUGIN_AUTH_PAM YES)' cmake/build_configurations/mysql_release.cmake
set '\bSET(PLUGIN_AUTH_PAM YES CACHE BOOL "")' cmake/build_configurations/mysql_release.cmake
grep -q "$@"
sed -i "/$1/d" "$2"
mv cmake/Findzstd.cmake cmake/FindZSTD.cmake
configure-command = ${cmake:location}/bin/cmake
configure-options =
-DCMAKE_INSTALL_PREFIX=${:location}
......@@ -97,8 +96,8 @@ post-install =
# as plugin-dir ( https://mariadb.com/kb/en/server-system-variables/#plugin_dir )
recipe = slapos.recipe.cmmi
shared = true
url = https://packages.groonga.org/source/mroonga/mroonga-9.12.tar.gz
md5sum = d0af673f1bad3b9ccf33870bb2344a25
url = https://packages.groonga.org/source/mroonga/mroonga-10.05.tar.gz
md5sum = a690811be5cc0593acc12ecf4ac9c40f
pre-configure =
rm -rf fake_mariadb_source
mkdir -p fake_mariadb_source
......
......@@ -4,6 +4,7 @@ extends =
../pkgconfig/buildout.cfg
../libpng/buildout.cfg
../freetype/buildout.cfg
./buildout.hash.cfg
parts =
matplotlib
......@@ -11,10 +12,9 @@ parts =
[matplotlibrc]
recipe = slapos.recipe.template:jinja2
location = ${buildout:parts-directory}/${:_buildout_section_name_}
template = ${:_profile_base_location_}/matplotlibrc.in
template = ${:_profile_base_location_}/${:filename}
rendered = ${matplotlibrc:location}/matplotlibrc
mode = 0644
md5sum = 849f48f08bea21dd10d3825867cb1561
backend = agg
context =
key backend matplotlibrc:backend
......
[matplotlibrc]
filename = matplotlibrc.in
md5sum = 47a737b8f4db0b329e2b3fd70e5a325c
......@@ -136,4 +136,3 @@ component = vbucketmigrator
component-configure-options =
--without-sasl
--with-isasl
[buildout]
extends =
../perl/buildout.cfg
parts =
perl-Image-ExifTool
[perl-Image-ExifTool]
<= perl-CPAN-package
module = Image/Image-ExifTool
version = 12.00
md5sum = 8c3f99a9a8d110ba340ee3f7063c940e
......@@ -127,4 +127,3 @@ template = inline:
mode = 0755
rendered = ${buildout:bin-directory}/${:_buildout_section_name_}
perl-bin = ${:rendered}
......@@ -4,6 +4,7 @@ extends =
../gnutls/buildout.cfg
../libaio/buildout.cfg
../libcap/buildout.cfg
../libcap-ng/buildout.cfg
../libpng/buildout.cfg
../libuuid/buildout.cfg
../ncurses/buildout.cfg
......@@ -32,8 +33,8 @@ configure-options =
--enable-vnc
--enable-vnc-png
--disable-vnc-jpeg
--extra-cflags="-I${gnutls:location}/include -I${libuuid:location}/include -I${ncurses:location}/include -I${zlib:location}/include -I${libpng:location}/include -I${libaio:location}/include -I${attr:location}/include -I${libcap:location}/include"
--extra-ldflags="-Wl,-rpath -Wl,${glib:location}/lib -L${glib:location}/lib -Wl,-rpath -Wl,${gnutls:location}/lib -L${gnutls:location}/lib -Wl,-rpath -Wl,${libgpg-error:location}/lib -L${libgpg-error:location}/lib -L${gettext:location}/lib -Wl,-rpath -Wl,${gettext:location}/lib -Wl,-rpath -Wl,${libpng:location}/lib -L${libpng:location}/lib -L${libuuid:location}/lib -Wl,-rpath -Wl,${libuuid:location}/lib -L${libaio:location}/lib -Wl,-rpath=${libaio:location}/lib -L${ncurses:location}/lib -Wl,-rpath=${ncurses:location}/lib -L${zlib:location}/lib -Wl,-rpath -Wl,${zlib:location}/lib -lpng -lz -lgnutls -L${attr:location}/lib -Wl,-rpath=${attr:location}/lib -L${libcap:location}/lib -Wl,-rpath=${libcap:location}/lib"
--extra-cflags="-I${gnutls:location}/include -I${libuuid:location}/include -I${ncurses:location}/include -I${zlib:location}/include -I${libpng:location}/include -I${libaio:location}/include -I${attr:location}/include -I${libcap:location}/include -I${libcap-ng:location}/include "
--extra-ldflags="-Wl,-rpath -Wl,${glib:location}/lib -L${glib:location}/lib -Wl,-rpath -Wl,${gnutls:location}/lib -L${gnutls:location}/lib -Wl,-rpath -Wl,${libgpg-error:location}/lib -L${libgpg-error:location}/lib -L${gettext:location}/lib -Wl,-rpath -Wl,${gettext:location}/lib -Wl,-rpath -Wl,${libpng:location}/lib -L${libpng:location}/lib -L${libuuid:location}/lib -Wl,-rpath -Wl,${libuuid:location}/lib -L${libaio:location}/lib -Wl,-rpath=${libaio:location}/lib -L${ncurses:location}/lib -Wl,-rpath=${ncurses:location}/lib -L${zlib:location}/lib -Wl,-rpath -Wl,${zlib:location}/lib -lpng -lz -lgnutls -L${attr:location}/lib -Wl,-rpath=${attr:location}/lib -L${libcap:location}/lib -Wl,-rpath=${libcap:location}/lib -L${libcap-ng:location}/lib -Wl,-rpath=${libcap-ng:location}/lib"
--disable-werror
--disable-xfsctl
environment =
......@@ -63,16 +64,16 @@ md5sum = df0ce86d0b1d81e232ad08eef58754ed
[debian-amd64-stretch-netinst.iso]
<= debian-amd64-netinst-base
version = 9.12.0
md5sum = 50e7919f4276787f3a4631544edceae3
version = 9.13.0
md5sum = 6097fdb9cbab47c96471274b9044e983
[debian-amd64-netinst.iso]
# Download the installer of Debian 10 (Buster)
# XXX: This is not the latest version because
# Debian does not provide a stable URL for it.
<= debian-amd64-netinst-base
version = 10.3.0
md5sum = 5956434be4b81e6376151b64ef9b1596
version = 10.4.0
md5sum = e2ddc8268e4c164c32b4ba25be52c9af
[debian-amd64-testing-netinst.iso]
<= debian-amd64-netinst-base
......
......@@ -14,4 +14,3 @@ recipe = zc.recipe.egg
eggs =
${lxml-python:egg}
slapos.cookbook
......@@ -13,10 +13,6 @@ extensions =
extends-cache = extends-cache
download-cache = download-cache
[gcc]
# force usage of gcc from slapos
max_version = 0
# Uguu, upstream buildout.cfg must be patched as it works the other way
# around from a packager point of view at least, thus at the end static
# path, such as Python HOME directory, are wrong...
......
......@@ -8,8 +8,8 @@ parts =
[sqlite3]
recipe = slapos.recipe.cmmi
shared = true
url = https://sqlite.org/2019/sqlite-autoconf-3300100.tar.gz
md5sum = 51252dc6bc9094ba11ab151ba650ff3c
url = https://sqlite.org/2020/sqlite-autoconf-3320300.tar.gz
md5sum = 2e3911a3c15e85c2f2d040154bbe5ce3
configure-options =
--disable-static
--enable-readline
......
......@@ -42,5 +42,3 @@ environment =
CPPFLAGS=${:CMAKE_CFLAGS}
LDFLAGS=-L${bzip2:location}/lib -L${mbedtls:location}/lib -L${lksctp-tools:location}/lib -L${libconfig:location}/lib -L${boost-lib:location}/lib -L${fftw3:location}/lib
PATH=${cmake:location}/bin:${pkgconfig:location}/bin:%(PATH)s
......@@ -50,4 +50,3 @@ recipe = slapos.recipe.build:download
destination = ${tesseract-traineddata:location}/tessdata/osd.traineddata
url = https://github.com/tesseract-ocr/tessdata/raw/590567f20dc044f6948a8e2c61afc714c360ad0e/osd.traineddata
md5sum = 7611737524efd1ce2dde67eff629bbcf
[buildout]
extends =
../perl/buildout.cfg
../libcap-ng/buildout.cfg
../xz-utils/buildout.cfg
[util-linux]
recipe = slapos.recipe.cmmi
shared = true
url = https://www.kernel.org/pub/linux/utils/util-linux/v2.31/util-linux-2.31.1.tar.xz
md5sum = 7733b583dcb51518944d42aa62ef19ea
configure-options =
......@@ -31,7 +33,7 @@ configure-options =
--disable-setterm
--enable-libsmartcols
--disable-switch_root
--disable-tinfo
--without-tinfo
--disable-tls
--disable-ul
--disable-unshare
......@@ -43,5 +45,8 @@ configure-options =
--without-slang
--without-selinux
--without-audit
--with-cap-ng=${libcap-ng:location}
environment =
PATH=${perl:location}/bin:${xz-utils:location}/bin:%(PATH)s
LDFLAGS=-L${libcap-ng:location}/lib -Wl,-rpath=${libcap-ng:location}/lib
CFLAGS=-I${libcap-ng:location}/include
......@@ -8,7 +8,7 @@
"python2 ./format-json",
"git add"
],
"{components,software,stack}/**": [
"{component,software,stack}/**": [
"python -c 'import sys, os.path, subprocess; [subprocess.check_call((\"python2\", \"./update-hash\", buildout_hash)) for buildout_hash in { os.path.join(os.path.dirname(staged), \"buildout.hash.cfg\") for staged in sys.argv[1:]} if os.path.exists(buildout_hash)]'",
"python -c 'import sys, os.path, subprocess; [subprocess.check_call((\"git\", \"add\", buildout_hash)) for buildout_hash in { os.path.join(os.path.dirname(staged), \"buildout.hash.cfg\") for staged in sys.argv[1:]} if os.path.exists(buildout_hash)]'"
]
......
......@@ -15,4 +15,3 @@ if expected != "" and value != expected:
if not_expected != "" and value == not_expected:
print "FAIL: %%s == %%s" %% (value, not_expected)
sys.exit(127)
......@@ -2,4 +2,3 @@
path %(path)s
# pack-gc false
</filestorage>
......@@ -50,4 +50,3 @@ pycurl = 7.43.0
slapos.recipe.download = 1.0
slapos.recipe.template = 4.4
smmap = 0.9.0
......@@ -14,7 +14,7 @@
# not need these here).
[instance]
filename = instance.cfg
md5sum = 4c7936accb3658871b635158198b7905
md5sum = 1fbb72e4305ded9614ee80c0ee17b672
[instance-apache]
filename = instance-apachephp.cfg
......@@ -30,4 +30,4 @@ md5sum = c5695762361b801c284ee23a150cd1f1
[template-httpd-conf-hash]
filename = templates/apache.conf.in
md5sum = 6788381fadd8ca96f892d33df3163da0
md5sum = 4fc3f853abf702aeb6653942d4fb85d8
......@@ -14,7 +14,7 @@
# not need these here).
[template]
filename = instance.cfg
md5sum = f686f765e55d1dce2e55a400f0714b3e
md5sum = 4adae801babadd9aed6112c1ff209076
[template-apache-frontend]
filename = instance-apache-frontend.cfg
......@@ -34,7 +34,7 @@ md5sum = b3d5a3573fa381fec954ad5cbf1575a0
[template-replicate-publish-slave-information]
filename = templates/replicate-publish-slave-information.cfg.in
md5sum = 665e83d660c9b779249b2179d7ce4b4e
md5sum = 7c0025a35dc62eb3cbfd07a6b04d659e
[template-apache-frontend-configuration]
filename = templates/apache.conf.in
......@@ -50,7 +50,7 @@ md5sum = f20d6c3d2d94fb685f8d26dfca1e822b
[template-default-virtualhost]
filename = templates/000.conf.in
md5sum = d98a01182f38868612948c87d5231428
md5sum = 5fcc11ea7f3eae38b0135b94dfc42dd5
[template-default-slave-virtualhost]
filename = templates/default-virtualhost.conf.in
......@@ -58,7 +58,7 @@ md5sum = 7890469ecc5e7f46be5b2c3074a09b52
[template-cached-slave-virtualhost]
filename = templates/cached-virtualhost.conf.in
md5sum = 1a1a53d9ac4a1591c017d86850a94796
md5sum = f97dfcfae80527e242f285c9ebf11b27
[template-log-access]
filename = templates/template-log-access.conf.in
......@@ -66,11 +66,11 @@ md5sum = a20683faba37b5b9c035783e811dd88d
[template-empty]
filename = templates/empty.in
md5sum = c2314c3a9c3412a38d14b312d3df83c1
md5sum = 7155b18edfe128825b8d1f48071454a6
[template-wrapper]
filename = templates/wrapper.in
md5sum = 8cde04bfd0c0e9bd56744b988275cfd8
md5sum = 975177dedf677d24e14cede5d13187ce
[template-trafficserver-records-config]
filename = templates/trafficserver/records.config.jinja2
......@@ -94,7 +94,7 @@ md5sum = 82d74a7f2aceb2b4a7acc6259291b7f2
[template-apache-lazy-script-call]
filename = templates/apache-lazy-script-call.sh.in
md5sum = ebe5d3d19923eb812a40019cb11276d8
md5sum = 44e235d0451932f1232f9d9e0ecc59f9
[template-apache-graceful-script]
filename = templates/apache-graceful-script.sh.in
......
......@@ -47,4 +47,3 @@ extra-context =
# Must match the key id in [switch-softwaretype] which uses this section.
raw software_type RootSoftwareInstance-default-custom-personal-replicate
raw template_monitor ${monitor2-template:rendered}
......@@ -14,4 +14,3 @@ smmap = 0.9.0
numpy = 1.16.4
websockify = 0.8.0
......@@ -48,4 +48,3 @@
RewriteRule ^/(.*)$ {{ slave_parameter.get('https_backend_url', '') }}/$1 [L,P]
</VirtualHost>
......@@ -18,7 +18,7 @@ md5sum = 5c94d952305552dcbeaeaeb27dd28f3b
[template-nginx-configuration]
filename = template-nginx.cfg.in
md5sum = a7012aac9b47ddd4e3aa3197aee1ffac
md5sum = fc4f258e3ae56e0c9596484d736ccfed
[template-dcron-service]
filename = template-dcron-service.sh.in
......@@ -34,7 +34,7 @@ md5sum = 5cbd64f04da0601ba4286516a6161f5e
[template-crontab]
filename = template-crontab.in
md5sum = 072be0fd04896880c931d44d8eabde37
md5sum = f1f82101258de19068262b7213fc478b
[status2rss]
filename = status2rss.py
......
# min(0-59) hours(0-23) day(1-31) month(1-12) dow(0-7) command
MAILTO=admins@erp5.org
......@@ -3,6 +3,11 @@ Changes
Here are listed the most important changes, which might affect upgrades.
1.0.160 (2020-08-25)
--------------------
* haproxy updated from 2.0.15 to 2.0.17 in order to fix issue while accessing inaccessible backends
1.0.159 (2020-07-30)
--------------------
......
......@@ -129,9 +129,9 @@ Example sessions is::
curl -g -X GET --cacert "${frontend_name}.ca.crt" --crlfile "${frontend_name}.crl" master-key-generate-auth-url
> authtoken
cat certificate.pem key.pem ca-bundle.pem > master.pem
cat certificate.pem ca.pem key.pem > bundle.pem
curl -g -X PUT --cacert "${frontend_name}.ca.crt" --crlfile "${frontend_name}.crl" --data-binary @master.pem master-key-upload-url+authtoken
curl -g -X PUT --cacert "${frontend_name}.ca.crt" --crlfile "${frontend_name}.crl" --data-binary @bundle.pem master-key-upload-url+authtoken
This replaces old request parameters:
......@@ -157,9 +157,9 @@ Example sessions is::
curl -g -X GET --cacert "${frontend_name}.ca.crt" --crlfile "${frontend_name}.crl" key-generate-auth-url
> authtoken
cat certificate.pem key.pem ca-bundle.pem > master.pem
cat certificate.pem ca.pem key.pem > bundle.pem
curl -g -X PUT --cacert "${frontend_name}.ca.crt" --crlfile "${frontend_name}.crl" --data-binary @master.pem key-upload-url+authtoken
curl -g -X PUT --cacert "${frontend_name}.ca.crt" --crlfile "${frontend_name}.crl" --data-binary @bundle.pem key-upload-url+authtoken
This replaces old request parameters:
......
......@@ -18,7 +18,7 @@ md5sum = dae3bf6daf851b5610a1a6bd83057b29
[template-common]
filename = instance-common.cfg.in
md5sum = c801b7f9f11f0965677c22e6bbe9281b
md5sum = 5784bea3bd608913769ff9a8afcccb68
[template-apache-frontend]
filename = instance-apache-frontend.cfg.in
......@@ -62,11 +62,11 @@ md5sum = f8068179333ce19e95df561c70073857
[template-empty]
_update_hash_filename_ = templates/empty.in
md5sum = c2314c3a9c3412a38d14b312d3df83c1
md5sum = 7155b18edfe128825b8d1f48071454a6
[template-wrapper]
_update_hash_filename_ = templates/wrapper.in
md5sum = 8cde04bfd0c0e9bd56744b988275cfd8
md5sum = 975177dedf677d24e14cede5d13187ce
[template-trafficserver-records-config]
_update_hash_filename_ = templates/trafficserver/records.config.jinja2
......@@ -86,7 +86,7 @@ md5sum = 217a6c801b8330b0b825f7b8b4c77184
[template-caddy-lazy-script-call]
_update_hash_filename_ = templates/apache-lazy-script-call.sh.in
md5sum = b9f73f6323f9fceea054c46c854d2862
md5sum = 77d60840591de67b64ab3572e46273a0
[template-graceful-script]
_update_hash_filename_ = templates/graceful-script.sh.in
......
[template]
filename = instance.cfg
md5sum = bed788dee6daf05349c4577e7a7f1299
[instance-davstorage]
filename = instance-davstorage.cfg
md5sum = 081f97f9e575af7a4b7f9afc7c80dd4a
......@@ -9,6 +9,7 @@ extends =
../../component/logrotate/buildout.cfg
../../component/lxml-python/buildout.cfg
../../component/gzip/buildout.cfg
./buildout.hash.cfg
parts =
apache-php
......@@ -35,14 +36,12 @@ strip-top-level-dir = true
[template]
# Default template for the instance.
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg
md5sum = bed788dee6daf05349c4577e7a7f1299
url = ${:_profile_base_location_}/${:filename}
output = ${buildout:directory}/template.cfg
mode = 0644
[instance-davstorage]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-davstorage.cfg
md5sum = b83b0351a0f46aac35d52e681270ff03
url = ${:_profile_base_location_}/${:filename}
output = ${buildout:directory}/template-davstorage.cfg
mode = 0644
[instance]
filename = instance.cfg
md5sum = 94e1cd21bbdf7cabb5834223859407e0
\ No newline at end of file
md5sum = 646e50cfa93681e8bd85767621c7a39d
......@@ -75,4 +75,3 @@ log = $${:var}/log
[publish-connection-parameter]
recipe = slapos.cookbook:publishurl
url = $${dream_platform_parameter:url}
[buildout]
extends =
../../stack/erp5/buildout.cfg
......@@ -48,6 +48,8 @@ setup(name=name,
'psutil',
'requests',
'mysqlclient',
'cryptography',
'pyOpenSSL',
],
zip_safe=True,
test_suite='test',
......
......@@ -116,7 +116,7 @@ class TestFrontendXForwardedFor(ERP5InstanceTestCase):
pass
time.sleep(1)
else:
raise RuntimeError, 'caucased failed to start.'
raise RuntimeError('caucased failed to start.')
cau_args = [
caucase_path,
......@@ -145,7 +145,6 @@ class TestFrontendXForwardedFor(ERP5InstanceTestCase):
stderr=subprocess.STDOUT,
)
result = caucase_process.communicate()
print result
csr_id = result[0].split()[0]
subprocess.check_call(
......@@ -196,7 +195,7 @@ class TestFrontendXForwardedFor(ERP5InstanceTestCase):
else:
time.sleep(1)
else:
raise RuntimeError, 'getting service certificate failed.'
raise RuntimeError('getting service certificate failed.')
# start a caucased and server certificate.
cls.backend_caucase_dir = tempfile.mkdtemp()
......@@ -223,7 +222,7 @@ class TestFrontendXForwardedFor(ERP5InstanceTestCase):
pass
time.sleep(1)
else:
raise RuntimeError, 'caucased failed to start.'
raise RuntimeError('caucased failed to start.')
super(TestFrontendXForwardedFor, cls).setUpClass()
......
[template]
_update_hash_filename_ = instance.cfg
md5sum = 7ab2a242df988bf5c10bf8002acac3bd
[instance-etherpad-lite]
_update_hash_filename_ = instance-etherpad-lite.cfg
md5sum = 7ead62829a85f5922bb9ed0fc8dfb0df
[template-conf]
_update_hash_filename_ = templates/settings.json.in
md5sum = 19ab39e6b3256c82fd54ce074488b136
[template-run-script]
_update_hash_filename_ = templates/run.sh.in
md5sum = eac870b5f30e735e109a48913af2fae3
[template-deps-script]
_update_hash_filename_ = templates/installDeps.sh.in
md5sum = 53d0d53d419bd9ee592d3e1a1c84c758
......@@ -15,6 +15,7 @@ extends =
../../component/git/buildout.cfg
../../component/nodejs/buildout.cfg
../../component/postgresql/buildout.cfg
./buildout.hash.cfg
parts =
postgresql
......@@ -49,44 +50,36 @@ git-executable = ${git:location}/bin/git
[template]
# Default template for the instance.
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg
md5sum = 7ab2a242df988bf5c10bf8002acac3bd
url = ${:_profile_base_location_}/${:_update_hash_filename_}
output = ${buildout:directory}/template.cfg
mode = 0644
[instance-etherpad-lite]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-etherpad-lite.cfg
md5sum = 28710a9a23320dd472387d81a698d12d
url = ${:_profile_base_location_}/${:_update_hash_filename_}
output = ${buildout:directory}/template-etherpad-lite.cfg
mode = 0644
[template-conf]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/templates/${:filename}
url = ${:_profile_base_location_}/${:_update_hash_filename_}
mode = 0644
filename = settings.json.in
md5sum = 19ab39e6b3256c82fd54ce074488b136
location = ${buildout:parts-directory}/${:_buildout_section_name_}
[template-run-script]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/templates/${:filename}
url = ${:_profile_base_location_}/${:_update_hash_filename_}
mode = 0644
filename = run.sh.in
md5sum = eac870b5f30e735e109a48913af2fae3
location = ${buildout:parts-directory}/${:_buildout_section_name_}
[template-deps-script]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/templates/${:filename}
url = ${:_profile_base_location_}/${:_update_hash_filename_}
etherpad-location = ${etherpad-lite-repository:location}
nodejs-location = ${nodejs:location}
curl-location = ${curl:location}
postgre-location = ${postgresql:location}
mode = 0755
md5sum = 53d0d53d419bd9ee592d3e1a1c84c758
filename = installDeps.sh.in
output = ${etherpad-lite-repository:location}/bin/installDeps.sh
[install-deps]
......
[template]
_update_hash_filename_ = instance.cfg.in
md5sum = 428669a609aca3e0a7cae1387d332a75
[template-gateone]
filename = instance-gateone.cfg.in
md5sum = 7c4cb83d6ccf5a5bcdb7b972123ac3e4
[template-logrotate-base]
_update_hash_filename_ = instance-logrotate-base.cfg.in
md5sum = ebee34c726f30fff95a9e0ea167a635f
[template-nginx-conf]
_update_hash_filename_ = templates/nginx.conf.in
md5sum = fe055886e255d9dea9f9f29008f6e780
......@@ -154,4 +154,3 @@ develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
[slap-parameter]
......@@ -47,4 +47,3 @@ backup = ${:srv}/backup
etc = ${buildout:directory}/etc
services = ${:etc}/service
log = ${buildout:directory}/var/log
......@@ -13,6 +13,7 @@ extends =
../../component/dtach/buildout.cfg
../../component/nginx/buildout.cfg
../../stack/slapos.cfg
./buildout.hash.cfg
parts =
slapos-cookbook
......@@ -76,8 +77,7 @@ context =
[template]
< = template-jinja2-base
filename = template.cfg
template = ${:_profile_base_location_}/instance.cfg.in
md5sum = 428669a609aca3e0a7cae1387d332a75
template = ${:_profile_base_location_}/${:_update_hash_filename_}
extra-context =
key dash_location dash:location
key dtach_location dtach:location
......@@ -93,13 +93,10 @@ extra-context =
[template-gateone]
< = download-base
filename = instance-gateone.cfg.in
md5sum = e7096a17c36c3bd27a011de57b7abfc1
[template-logrotate-base]
< = template-jinja2-base
filename = instance-logrotate-base.cfg
md5sum = f28fbd310944f321ccb34b2a34c82005
extra-context =
key dcron_location dcron:location
key gzip_location gzip:location
......@@ -109,7 +106,6 @@ extra-context =
< = download-base
url = ${:_profile_base_location_}/templates/${:filename}.in
filename = nginx.conf
md5sum = 3d80d73a9cfffca6687813d86ddc25ba
[check-recipe]
recipe = plone.recipe.command
......
......@@ -30,7 +30,7 @@ md5sum = 61d1d04b9347b3168a1ad7676e4681ef
[gitconfig.in]
_update_hash_filename_ = template/gitconfig.in
md5sum = 7782f5c5d75663c2586e28d029c51e49
md5sum = eb1230fee50067924ba89f4dc6e82fa9
[gitlab-parameters.cfg]
_update_hash_filename_ = gitlab-parameters.cfg
......@@ -50,11 +50,11 @@ md5sum = f4cc0bc898b8d59010d61473e2adc53b
[gitaly-config.toml.in]
_update_hash_filename_ = template/gitaly-config.toml.in
md5sum = 056d7ed09e1bf20d022d3ef6b9363e00
md5sum = 0f1ec4077dab586cc003ae13f689eda2
[instance-gitlab.cfg.in]
_update_hash_filename_ = instance-gitlab.cfg.in
md5sum = 6effe68f7ef240780d1027dd834a562c
md5sum = dfeff229aa696bb6b6051a2aff4301fe
[instance-gitlab-export.cfg.in]
_update_hash_filename_ = instance-gitlab-export.cfg.in
......@@ -62,7 +62,7 @@ md5sum = 2af7dcf63f74e5edc53a3ff11fa4989b
[instance-gitlab-test.cfg.in]
_update_hash_filename_ = instance-gitlab-test.cfg.in
md5sum = 60714fb4e6c869c41bd5e9fada1b6e40
md5sum = 7ba08928e6a8998ec8ed1bb97851b726
[macrolib.cfg.in]
_update_hash_filename_ = macrolib.cfg.in
......@@ -70,7 +70,7 @@ md5sum = a56a44e96f65f5ed20211bb6a54279f4
[nginx-gitlab-http.conf.in]
_update_hash_filename_ = template/nginx-gitlab-http.conf.in
md5sum = 79d2b4e8a32abf7a74a3d4528844c593
md5sum = cd7471a8c5d6f6bc848c62ce62dca966
[nginx.conf.in]
_update_hash_filename_ = template/nginx.conf.in
......@@ -86,11 +86,11 @@ md5sum = 7c89a730889e3224548d9abe51a2d719
[smtp_settings.rb.in]
_update_hash_filename_ = template/smtp_settings.rb.in
md5sum = e2144b03f7247636143c65dc81550d75
md5sum = 4e1ced687a86e4cfff2dde91237e3942
[template-gitlab-resiliency-restore.sh.in]
_update_hash_filename_ = template/template-gitlab-resiliency-restore.sh.in
md5sum = 590fcadf26085fdd17487175bc0a469d
md5sum = 16b9f52f00d55feab7e31a88029ad351
[unicorn.rb.in]
_update_hash_filename_ = template/unicorn.rb.in
......
......@@ -57,4 +57,3 @@ command =
touch ${:backup-done}
fi
update-command = ${:command}
......@@ -14,8 +14,8 @@ extends =
../../component/nodejs/buildout.cfg
../../component/openssl/buildout.cfg
../../component/nginx/buildout.cfg
../../component/xz-utils/buildout.cfg
../../component/zlib/buildout.cfg
../../component/icu/buildout.cfg
gowork.cfg
# for instance
......@@ -182,14 +182,16 @@ bundle = ${bundler-4gitlab:bundle}
configure-command = cd ${:path} &&
${:bundle} config --local build.charlock_holmes --with-icu-dir=${icu:location} &&
${:bundle} config --local build.pg --with-pg-config=${postgresql10:location}/bin/pg_config &&
${:bundle} config --local build.re2 --with-re2-dir=${re2:location}
${:bundle} config --local build.re2 --with-re2-dir=${re2:location} &&
${:bundle} config --local build.nokogiri --with-zlib-dir=${zlib:location} --with-cflags=-I${xz-utils:location}/include --with-ldflags="-L${xz-utils:location}/lib -Wl,-rpath=${xz-utils:location}/lib"
make-binary =
make-targets= cd ${:path} &&
${:bundle} install --deployment --without development test mysql aws kerberos
environment =
PKG_CONFIG_PATH=${openssl-1.0:location}/lib/pkgconfig:${re2:location}/lib/pkgconfig
PKG_CONFIG_PATH=${openssl-1.0:location}/lib/pkgconfig:${re2:location}/lib/pkgconfig:${xz-utils:location}/lib/pkgconfig
PATH=${pkgconfig:location}/bin:%(PATH)s
CFLAGS=-I${xz-utils:location}/include
################## Google re2
[re2]
......
......@@ -49,4 +49,3 @@ dir = "{{ gitaly.location }}/ruby"
[gitlab-shell]
# The directory where gitlab-shell is installed
dir = "{{ gitlab_shell_work.location }}"
......@@ -213,4 +213,3 @@ server {
<%= @custom_gitlab_server_config %>
#}
}
......@@ -29,4 +29,3 @@ end
# SMTP disabled in instance configuration (see `smtp_enable` parameter).
# Mail sending, if enabled (see `email_enabled`), will be done via sendmail.
{% endif %}
......@@ -19,7 +19,7 @@ md5sum = 13f4f3806522b265dd1912ff169d146d
[influxdb-config-file]
filename = influxdb-config-file.cfg.in
md5sum = 7ce85159c0664b251e249eac4b37bea4
md5sum = a28972ced3e0f4aa776e43a9c44717c0
[telegraf-config-file]
filename = telegraf-config-file.cfg.in
......
......@@ -131,4 +131,3 @@ bind-address = "[{{ influxdb['local-host'] }}]:{{ influxdb['rpc-port'] }}"
enabled = true
query-stats-enabled = false
run-interval = "1s"
......@@ -74,4 +74,3 @@ rdiff-backup = 1.0.5+SlapOSPatched001
slapos.recipe.template = 4.4
smmap = 0.8.2
plone.recipe.command = 1.1
......@@ -22,7 +22,7 @@ md5sum = 09803fb71404edbccb32c44a0040dae4
[template-validator]
filename = instance-validator.cfg.in
md5sum = f2609d1f526ba6349b1374e3631a9f7c
md5sum = 2be286e367e37ce9e504170cd3a08007
[template]
filename = instance.cfg.in
......
......@@ -15,11 +15,11 @@
[instance]
filename = instance.cfg.in
md5sum = cd8648ca8a4a098e1a6faf7246f0d395
md5sum = 342fcf3fc18e33fa4f940518dab6af5c
[template-nginx-service]
filename = template-nginx-service.sh.in
md5sum = 529532e1240a66bdf39e3cbbef90ba87
md5sum = 5c94d952305552dcbeaeaeb27dd28f3b
[template-nginx-configuration]
filename = template-nginx.cfg.in
......
......@@ -15,7 +15,7 @@
[instance-jupyter-notebook]
filename = instance.cfg.in
md5sum = ba4e4c582b81deeb7339a13c82bbf34b
md5sum = 1d5fe6cc4e48672ae7be1c223794a932
[jupyter-notebook-config]
filename = jupyter_notebook_config.py.jinja
......
......@@ -149,4 +149,3 @@ context =
recipe = slapos.cookbook:symbolic.link
target-directory = ${directory:jupyter_custom_dir}
link-binary = {{ custom_js_location }}/custom.js
......@@ -15,15 +15,15 @@
[template]
filename = instance.cfg.in
md5sum = b36b6b3ccb15758d99f3b6258b141db9
md5sum = e6d5c7bb627b4f1d3e7c99721b7c58fe
[template-kvm]
filename = instance-kvm.cfg.jinja2
md5sum = fb4d76a83ff7c2d0f205d52fb314a9ee
md5sum = 50e78a2a34efe09afab161ae1c1efd46
[template-kvm-cluster]
filename = instance-kvm-cluster.cfg.jinja2.in
md5sum = 73b09e75d617888f6d84d363c0ada9c5
md5sum = a4788112008cd0b38a57cd28f7252fbd
[template-kvm-resilient]
filename = instance-kvm-resilient.cfg.jinja2
......@@ -51,11 +51,11 @@ md5sum = 6ea26f88252bf899c966d0f5675e7176
[template-ansible-promise]
filename = template/ansible-promise.in
md5sum = 2036bf145f472f62ef8dee5e729328fd
md5sum = b7e87479a289f472b634a046b44b5257
[template-kvm-run]
filename = template/template-kvm-run.in
md5sum = c319ca536b6bac5425245fae1684ca49
md5sum = dd1f581f34cf5a0b627576771347c710
[template-kvm-controller]
filename = template/kvm-controller-run.in
......@@ -63,15 +63,15 @@ md5sum = 3827b4f7624de190cf5f5d37e3b72e86
[template-apache-conf]
filename = template/apache.conf.in
md5sum = ac97f6a52e1c5a19a646242ef85abb8a
md5sum = e818aa16b3c736a0c2de3ec981663534
[template-content]
filename = template/template-content.in
md5sum = 822737e483864bf255ad1259237bef2a
md5sum = 752c91a4a6b72f5cf8226d6b940015f8
[template-qemu-ready]
filename = template/qemu-is-ready.in
md5sum = b304eec8e2cb71f10ea83cac22f6db12
md5sum = fb330a796fadb6cd5c85217f80a42af3
[file-download-script]
filename = template/download_file.in
......@@ -80,3 +80,11 @@ md5sum = 599dbbbd438fe7801e3f8642ae9e9a78
[template-httpd]
filename = instance-kvm-http.cfg.in
md5sum = d657884d02105deffddee0edae4b50a6
[image-download-controller]
_update_hash_filename_ = template/image-download-controller.py
md5sum = 7e4b54f8172c364bd12d28b07f8b1600
[image-download-config-creator]
_update_hash_filename_ = template/image-download-config-creator.py
md5sum = 7f6cd75096695922fddbba1c9292cef5
......@@ -42,6 +42,12 @@
"type": "string",
"format": "uri",
"default": "http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg"
},
"image-url-list": {
"title": "List of URLs images to download",
"description": "The list shall be list of direct URLs to images, followed by hash (#), then by image MD5SUM. Each image shall appear on newline, like: \"https://example.com/image.iso#06226c7fac5bacfa385872a19bb99684<newline>https://example.com/another-image.iso#31b40d58b18e038498ddb46caea1361c\". They will be provided in KVM image list according to the order on the list. After updating the list, the instance has to be restarted to refresh it. Amount of images is limited to 4, and one image can be maximum 5G. Image will be downloaded and checked against its MD5SUM 4 times, then it will be considered as impossible to download with given MD5SUM. Each image has to be downloaded in time shorter than 4 hours, so in case of very slow images to access, it can take up to 16 hours to download all of them. Note: The instance has to be restarted in order to update the list of available images in the VM.",
"type": "string",
"textarea": "true"
}
},
"type": "object"
......@@ -231,9 +237,8 @@
},
"ram-max-size": {
"title": "Maximum RAM size, in MB",
"description": "Define the maximum size of the memory. The size is in MB and should be a multiple of 512.",
"description": "Define the maximum size of the memory. The size is in MB and should be a multiple of 512. Defaults to ram-size + 512",
"type": "integer",
"default": 51200,
"minimum": 1024,
"multipleOf": 512
},
......@@ -318,9 +323,8 @@
},
"cpu-max-count": {
"title": "Maximum CPU amount",
"description": "Specifies the maximum number of CPUs.",
"description": "Specifies the maximum number of CPUs. Defaults to cpu-count + 1",
"type": "integer",
"default": 24,
"minimum": 1
},
"numa": {
......
......@@ -74,14 +74,14 @@ config-authorized-key = {{ dumps(slapparameter_dict.get('authorized-keys') | joi
config-nbd-port = {{ dumps(kvm_parameter_dict.get('nbd-port', 1024)) }}
config-nbd2-port = {{ dumps(kvm_parameter_dict.get('nbd-port2', 1024)) }}
config-ram-size = {{ dumps(kvm_parameter_dict.get('ram-size', 1024)) }}
config-ram-max-size = {{ dumps(kvm_parameter_dict.get('ram-max-size', '51200')) }}
config-ram-max-size = {{ dumps(kvm_parameter_dict.get('ram-max-size', int(kvm_parameter_dict.get('ram-size', 1024)) + 512)) }}
config-enable-device-hotplug = {{ dumps(kvm_parameter_dict.get('enable-device-hotplug', False)) }}
config-ram-hotplug-slot-size = {{ dumps(kvm_parameter_dict.get('ram-hotplug-slot-size', 512)) }}
config-disk-size = {{ dumps(kvm_parameter_dict.get('disk-size', 10)) }}
config-disk-type = {{ dumps(kvm_parameter_dict.get('disk-type', 'virtio')) }}
config-disk-format = {{ dumps(kvm_parameter_dict.get('disk-format', 'qcow2')) }}
config-cpu-count = {{ dumps(kvm_parameter_dict.get('cpu-count', 1)) }}
config-cpu-max-count = {{ dumps(kvm_parameter_dict.get('cpu-max-count', 24)) }}
config-cpu-max-count = {{ dumps(kvm_parameter_dict.get('cpu-max-count', int(kvm_parameter_dict.get('cpu-count', 1)) + 1)) }}
{{ setconfig('numa', kvm_parameter_dict.get('numa', '')) }}
{{ setconfig('machine-options', kvm_parameter_dict.get('machine-options', '')) }}
{{ setconfig('nbd-host', kvm_parameter_dict.get('nbd-host', '')) }}
......@@ -127,6 +127,10 @@ config-document-host = ${apache-conf:ip}
config-document-port = ${apache-conf:port}
config-document-path = ${hash-code:passwd}
config-keyboard-layout-language = {{ dumps(kvm_parameter_dict.get('keyboard-layout-language', 'fr')) }}
{%- if 'image-url-list' in kvm_parameter_dict %}
{#- play nice: if parameter was not constructed by the original request, do not send it at all #}
config-image-url-list = {{ kvm_parameter_dict['image-url-list'] }}
{%- endif %}
config-type = cluster
{% set bootstrap_script_url = slapparameter_dict.get('bootstrap-script-url', kvm_parameter_dict.get('bootstrap-script-url', '')) -%}
......
......@@ -19,9 +19,8 @@
},
"ram-max-size": {
"title": "Maximum RAM size, in MB",
"description": "Define the maximum size of the memory. The size is in MB and should be a multiple of 512.",
"description": "Define the maximum size of the memory. The size is in MB and should be a multiple of 512. Defaults to ram-size + 512",
"type": "integer",
"default": 51200,
"minimum": 1024,
"multipleOf": 512
},
......@@ -106,9 +105,8 @@
},
"cpu-max-count": {
"title": "Maximum CPU amount",
"description": "Specifies the maximum number of CPUs.",
"description": "Specifies the maximum number of CPUs. Defaults to cpu-count + 1",
"type": "integer",
"default": 24,
"minimum": 1
},
"numa": {
......@@ -367,6 +365,12 @@
"type": "string",
"format": "uri",
"default": "http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg"
},
"image-url-list": {
"title": "List of URLs images to download",
"description": "The list shall be list of direct URLs to images, followed by hash (#), then by image MD5SUM. Each image shall appear on newline, like: \"https://example.com/image.iso#06226c7fac5bacfa385872a19bb99684<newline>https://example.com/another-image.iso#31b40d58b18e038498ddb46caea1361c\". They will be provided in KVM image list according to the order on the list. After updating the list, the instance has to be restarted to refresh it. Amount of images is limited to 4, and one image can be maximum 5G. Image will be downloaded and checked against its MD5SUM 4 times, then it will be considered as impossible to download with given MD5SUM. Each image has to be downloaded in time shorter than 4 hours, so in case of very slow images to access, it can take up to 16 hours to download all of them. Note: The instance has to be restarted in order to update the list of available images in the VM.",
"type": "string",
"textarea": "true"
}
}
}
......@@ -9,6 +9,9 @@
{% set instance_type = slapparameter_dict.get('type', 'standalone') -%}
{% set nat_rule_list = slapparameter_dict.get('nat-rules', '22 80 443') -%}
{% set disk_device_path = slapparameter_dict.get('disk-device-path', None) -%}
{% set image_url_list_enabled = 'image-url-list' in slapparameter_dict %}
{% set cpu_max_count = dumps(slapparameter_dict.get('cpu-max-count', int(slapparameter_dict.get('cpu-count', 1)) + 1)) %}
{% set ram_max_size = dumps(slapparameter_dict.get('ram-max-size', int(slapparameter_dict.get('ram-size', 1024)) + 512)) %}
{% set extends_list = [] -%}
{% set part_list = [] -%}
......@@ -51,6 +54,11 @@ public = ${:srv}/public/
cron-entries = ${:etc}/cron.d
crontabs = ${:etc}/crontabs
cronstamps = ${:etc}/cronstamps
{%- if image_url_list_enabled %}
image-repository = ${:srv}/image-repository
image-url-list-var = ${:var}/image-url-list
image-url-list-expose = ${monitor-directory:private}/image-url-list
{%- endif %}
[create-mac]
recipe = slapos.cookbook:generate.mac
......@@ -65,6 +73,108 @@ recipe = slapos.cookbook:generate.password
storage-path = ${directory:srv}/passwd
bytes = 8
{% if image_url_list_enabled %}
## image-url-list support BEGIN
[empty-file-state-base-promise]
<= monitor-promise-base
module = check_file_state
name = ${:_buildout_section_name_}.py
config-state = empty
# It's very hard to put the username and password correctly, after schema://
# and before the host, as it's not the way how one can use monitor provided
# information, so just show the information in the URL
config-url = ${monitor-base:base-url}/private/image-url-list/${:filename} with username ${monitor-publish-parameters:monitor-user} and password ${monitor-publish-parameters:monitor-password}
[image-url-list-source-config]
# generates configuration of the image from the user parameter
# special "magic" is used, to properly support multiline image-url-list
# but in the same time correctly generate the configuration file
recipe = slapos.recipe.template:jinja2
template = inline:
{#- Do special trick to support image-url-list being None, if key is present with value "" #}
{%- raw %}
{{ slap_parameter.get('image-url-list') or '' }}
{% endraw -%}
context =
section slap_parameter slap-parameter
rendered = ${directory:etc}/image-url-list.conf
[image-url-list-processed-config]
# compares if the current configuration has been used by
# the image-url-list-download, if not, exposes it as not empty file with
# information
recipe = slapos.recipe.build
install =
import os
import hashlib
if not os.path.exists(location):
os.mkdir(location)
with open('${:state-file}', 'w') as state_handler:
try:
with open('${:config-file}', 'rb') as config_handler, open('${:processed-md5sum}') as processed_handler:
config_md5sum = hashlib.md5(config_handler.read()).hexdigest()
processed_md5sum = processed_handler.read()
if config_md5sum == processed_md5sum:
state_handler.write('')
else:
state_handler.write('config %s != processed %s' % (config_md5sum, processed_md5sum))
except Exception as e:
state_handler.write(str(e))
update = ${:install}
config-file = ${image-url-list-source-config:rendered}
state-filename = image-url-list-processed-config.state
state-file = ${directory:image-url-list-expose}/${:state-filename}
processed-md5sum = ${directory:image-url-list-var}/update-image-processed.md5sum
[image-url-list-processed-config-promise]
# promise to check if the configuration provided by the user has been already
# processed by the image-url-list-download script, which runs asynchronously
<= empty-file-state-base-promise
filename = ${image-url-list-processed-config:state-filename}
config-filename = ${image-url-list-processed-config:state-file}
[image-url-list-json-config]
# generates json configuration from user configuration
recipe = plone.recipe.command
command = {{ python_executable }} {{ image_download_config_creator }} ${image-url-list-source-config:rendered} ${:rendered} ${directory:image-repository} ${:error-state-file}
update-command = ${:command}
rendered = ${directory:image-url-list-var}/image-url-list.json
error-state-filename = image-url-list-json-config-error.txt
error-state-file = ${directory:image-url-list-expose}/${:error-state-filename}
[image-url-list-config-state-promise]
# promise to check if configuration has been parsed without errors
<= empty-file-state-base-promise
filename = ${image-url-list-json-config:error-state-filename}
config-filename = ${image-url-list-json-config:error-state-file}
[image-url-list-download-wrapper]
# wrapper to execute image-url-list-download on each run
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:scripts}/image-updater
command-line = {{ python_executable }} {{ image_download_controller }} ${image-url-list-json-config:rendered} {{ curl_executable_location }} ${:md5sum-state-file} ${:error-state-file} ${image-url-list-processed-config:processed-md5sum}
md5sum-state-filename = image-download-controller-md5sum-fail.json
md5sum-state-file = ${directory:image-url-list-expose}/${:md5sum-state-filename}
error-state-filename = image-download-controller-error.text
error-state-file = ${directory:image-url-list-expose}/${:error-state-filename}
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
[image-url-list-download-md5sum-promise]
# promise to report errors with problems with calculating md5sum of the
# downloaded images
<= empty-file-state-base-promise
filename = ${image-url-list-download-wrapper:md5sum-state-filename}
config-filename = ${image-url-list-download-wrapper:md5sum-state-file}
[image-url-list-download-state-promise]
# promise to report errors during download
<= empty-file-state-base-promise
filename = ${image-url-list-download-wrapper:error-state-filename}
config-filename = ${image-url-list-download-wrapper:error-state-file}
## image-url-list support END
{% endif %} {# if image_url_list_enabled #}
[kvm-controller-parameter-dict]
python-path = {{ python_eggs_executable }}
vnc-passwd = ${gen-passwd:passwd}
......@@ -85,6 +195,11 @@ vnc-ip = ${:ipv4}
vnc-port = 5901
default-cdrom-iso = {{ debian_amd64_netinst_location }}
{% if image_url_list_enabled %}
image-url-list-json-config = ${image-url-list-json-config:rendered}
{% else %}
image-url-list-json-config =
{% endif %}
nbd-host = ${slap-parameter:nbd-host}
nbd-port = ${slap-parameter:nbd-port}
nbd2-host = ${slap-parameter:nbd2-host}
......@@ -104,10 +219,10 @@ socket-path = ${kvm-controller-parameter-dict:socket-path}
enable-device-hotplug = ${kvm-controller-parameter-dict:enable-device-hotplug}
smp-count = ${kvm-controller-parameter-dict:cpu-count}
smp-max-count = ${slap-parameter:cpu-max-count}
smp-max-count = {{ cpu_max_count }}
ram-size = ${kvm-controller-parameter-dict:ram-size}
ram-max-size = ${slap-parameter:ram-max-size}
ram-max-size = {{ ram_max_size }}
init-ram-size = 1024
mac-address = ${create-mac:mac-address}
tap-mac-address = ${create-tap-mac:mac-address}
......@@ -708,10 +823,10 @@ nbd-port = 1024
nbd-host =
nbd2-port = 1024
nbd2-host =
image-url-list =
enable-device-hotplug = False
ram-size = 1024
ram-max-size = 51200
ram-hotplug-slot-size = 512
disk-size = 10
disk-type = virtio
......@@ -719,7 +834,6 @@ disk-format = qcow2
disk-device-path =
cpu-count = 1
cpu-max-count = 24
disk-cache = writeback
disk-aio = native
auto-ballooning = True
......@@ -759,6 +873,9 @@ keyboard-layout-language = fr
{% set key_list = v.split('\n') -%}
{{ k }} =
{{ key_list | join('\n ') }}
{% elif k == 'image-url-list' %}
{# needs to decorate possibly multiline or maybe unsafe value #}
{{ k }} = {{ dumps(v) }}
{% else -%}
{{ k }} = {{ v }}
{% endif -%}
......@@ -833,6 +950,13 @@ parts =
cron-service
cron-entry-logrotate
frontend-promise
{% if image_url_list_enabled %}
image-url-list-download-wrapper
image-url-list-config-state-promise
image-url-list-download-md5sum-promise
image-url-list-download-state-promise
image-url-list-processed-config-promise
{% endif %}
{% if additional_frontend %}
frontend-additional-promise
{% endif %}
......
......@@ -83,6 +83,8 @@ extra-context =
raw dcron_executable_location ${dcron:location}/sbin/crond
raw debian_amd64_netinst_location ${debian-amd64-netinst.iso:location}/${debian-amd64-netinst.iso:filename}
raw file_download_script ${file-download-script:location}/${file-download-script:filename}
raw image_download_controller ${image-download-controller:target}
raw image_download_config_creator ${image-download-config-creator:target}
raw logrotate_cfg ${template-logrotate-base:rendered}
raw novnc_location ${noVNC:location}
raw netcat_bin ${netcat:location}/bin/netcat
......
......@@ -190,6 +190,14 @@ context =
raw openssl_executable_location ${openssl:location}/bin/openssl
raw template_apache_conf ${template-apache-conf:location}/${template-apache-conf:filename}
[image-download-controller]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_update_hash_filename_}
mode = 640
[image-download-config-creator]
<= image-download-controller
[versions]
websockify = 0.9.0
......@@ -199,4 +207,3 @@ pycurl = 7.43.0
slapos.recipe.template = 4.4
smmap = 0.9.0
numpy = 1.16.4
#!/usr/bin/env python
import hashlib
import json
import re
import sys
if __name__ == "__main__":
source_configuration, destination_configuration, \
destination_directory, error_state_file = sys.argv[1:]
md5sum_re = re.compile(r"^([a-fA-F\d]{32})$")
image_prefix = 'image_'
maximum_image_amount = 4
# build currently wanted list
configuration_dict = {
'destination-directory': destination_directory,
}
image_list = []
error_list = []
print('INF: Storing errors in %s' % (error_state_file,))
with open(source_configuration, 'rb') as fh:
image_number = 0
data = fh.read()
configuration_dict['config-md5sum'] = hashlib.md5(data).hexdigest()
for entry in data.decode('utf-8').split():
split_entry = entry.split('#')
if len(split_entry) != 2:
error_list.append('ERR: entry %r is incorrect' % (entry,))
continue
url, md5sum = split_entry
if not md5sum_re.match(md5sum):
error_list.append('ERR: checksum in entry %r is malformed' % (entry, ))
continue
if md5sum not in [q['md5sum'] for q in image_list]:
image_number += 1
image_list.append({
'md5sum': md5sum,
'url': url,
'destination': md5sum,
'destination-tmp': md5sum + '_tmp',
'link': 'image_%03i' % (image_number,),
})
else:
print('INF: checksum %s repeated, used url %s' % (url, ))
image_amount = len(image_list)
if image_amount > maximum_image_amount:
error_list.append(
'ERR: Amount of images is %s, which is bigger than maximum (%s)' % (
image_amount, maximum_image_amount))
else:
configuration_dict['image-list'] = image_list
error_amount = len(error_list)
configuration_dict['error-amount'] = error_amount
with open(destination_configuration, 'w') as fh:
json.dump(configuration_dict, fh, indent=2)
with open(error_state_file, 'w') as fh:
if error_amount == 0:
print('INF: Configuration generated without errors')
fh.write('')
else:
print('ERR: Configuration generated with %s errors' % (error_amount,))
fh.write('\n'.join(error_list))
sys.exit(error_amount)
#!/usr/bin/env python
import hashlib
import json
import os
import re
import subprocess
import sys
# stolen from download_file.in
def md5Checksum(file_path):
with open(file_path, 'rb') as fh:
m = hashlib.md5()
while True:
data = fh.read(8192)
if not data:
break
m.update(data)
return m.hexdigest()
# Note: Assuring only one running instance is not done, as this script is only
# run from supervisord, which does it already
if __name__ == "__main__":
configuration, curl, md5sum_fail_file, error_state_file, \
processed_md5sum = sys.argv[1:]
error_list = []
md5sum_re = re.compile(r"^([a-fA-F\d]{32})$")
image_prefix = 'image_'
# build currently wanted list
with open(configuration) as fh:
try:
config = json.load(fh)
except Exception as e:
print('ERR: Problem loading configuration: %s' % (e,))
sys.exit(1)
if config['error-amount'] != 0:
print('ERR: There are problems with configuration')
print('INF: Storing errors in %s' % (error_state_file,))
# clean the destination directory
file_to_keep_list = []
for image in config['image-list']:
file_to_keep_list.append(image['destination'])
file_to_keep_list.append(image['link'])
for fname in os.listdir(config['destination-directory']):
if fname not in file_to_keep_list:
print('INF: Removing obsolete %s' % (fname,))
os.remove(os.path.join(config['destination-directory'], fname))
# prepare state dicts
# current and new are used to remove not existing configurations
# and also to allow re-add some configuration
try:
with open(md5sum_fail_file) as fh:
md5sum_state_dict = json.load(fh)
except Exception:
md5sum_state_dict = {}
new_md5sum_state_dict = {}
# fetch the wanted list
for image in config['image-list']:
destination = os.path.join(
config['destination-directory'], image['destination'])
if os.path.exists(destination):
if md5Checksum(destination) == image['md5sum']:
print('INF: %s : already downloaded' % (image['url'],))
continue
else:
print('INF: %s : Removed, as expected checksum does not match %s' % (
image['url'], image['md5sum']))
os.remove(destination)
# key is str, as the dict is dumped to JSON which does not accept tuples
md5sum_state_key = '%s#%s' % (image['url'], image['md5sum'])
md5sum_state_amount = md5sum_state_dict.get(md5sum_state_key, 0)
if md5sum_state_amount >= 4:
new_md5sum_state_dict[md5sum_state_key] = md5sum_state_amount
error_list.append(
'ERR: %s : Checksum is incorrect after %s tries, will not retry' % (
image['url'], md5sum_state_amount))
continue
print('INF: %s : Downloading' % (image['url'],))
download_success = True
destination_tmp = os.path.join(
config['destination-directory'], image['destination-tmp'])
try:
subprocess.check_output([
curl,
'--insecure', # allow any download
'--location', # follow redirects
'--no-progress-meter', # do not tell too much
'--max-time', '14400', # maximum time for download is 4 hours
'--max-filesize', '5368709120', # maximum 5G for an image
'--output', destination_tmp, image['url']],
stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
error_list.append('ERR: %s : Problem while downloading: %r' % (
image['url'], e.output.strip()))
continue
if not(os.path.exists(destination_tmp)):
error_list.append('ERR: %s : Image disappeared, will retry later')
continue
computed_md5sum = md5Checksum(destination_tmp)
if computed_md5sum != image['md5sum']:
try:
os.remove(destination_tmp)
except Exception:
pass
error_list.append(
'ERR: %s : MD5 mismatch expected is %s but got instead %s' % (
image['url'], image['md5sum'], computed_md5sum))
# Store yet another failure while computing md5sum for this
new_md5sum_state_dict[md5sum_state_key] = md5sum_state_amount + 1
else:
os.rename(destination_tmp, destination)
print('INF: %s : Stored with checksum %s' % (
image['url'], image['md5sum']))
for image in config['image-list']:
destination = os.path.join(
config['destination-directory'], image['destination'])
link = os.path.join(config['destination-directory'], image['link'])
if os.path.exists(destination):
if os.path.lexists(link):
if not os.path.islink(link):
os.remove(link)
if os.path.islink(link) and os.readlink(link) != destination:
os.remove(link)
if not os.path.lexists(link):
print('INF: %s : Symlinking %s -> %s' % (
image['url'], link, destination))
os.symlink(destination, link)
with open(md5sum_fail_file, 'w') as fh:
if new_md5sum_state_dict != {}:
json.dump(new_md5sum_state_dict, fh, indent=2)
else:
# if no problems reported, just empty the file
fh.write('')
with open(error_state_file, 'w') as fh:
fh.write('\n'.join(error_list))
with open(processed_md5sum, 'w') as fh:
fh.write(config['config-md5sum'])
sys.exit(len(error_list))
......@@ -15,6 +15,7 @@ import shutil
from random import shuffle
import glob
import re
import json
import ssl
......@@ -98,6 +99,8 @@ enable_device_hotplug = '{{ parameter_dict.get("enable-device-hotplug") }}'.lowe
logfile = '{{ parameter_dict.get("log-file") }}'
image_url_list_json_config = '{{ parameter_dict.get("image-url-list-json-config") }}'
if hasattr(ssl, '_create_unverified_context') and url_check_certificate == 'false':
opener = FancyURLopener(context=ssl._create_unverified_context())
else:
......@@ -366,11 +369,27 @@ for nbd_ip, nbd_port in nbd_list:
kvm_argument_list.extend([
'-drive',
'file=nbd:[%s]:%s,media=cdrom' % (nbd_ip, nbd_port)])
# If no NBD is specified/available: use internal disk image
else:
image_url_list_used = False
if image_url_list_json_config:
# Support image-url-list
with open(image_url_list_json_config) as fh:
image_config = json.load(fh)
if image_config['error-amount'] == 0:
for image in sorted(image_config['image-list'], key=lambda k: k['link']):
link = os.path.join(image_config['destination-directory'], image['link'])
if os.path.exists(link) and os.path.islink(link):
image_url_list_used = True
kvm_argument_list.extend([
'-drive',
'file=%s,media=cdrom' % (link,)
])
if not image_url_list_used:
# If no NBD is specified/available not downloadable image: use internal disk image
kvm_argument_list.extend([
'-drive', 'file=%s,media=cdrom' % default_cdrom_iso
])
print('Starting KVM: \n %s' % ' '.join(kvm_argument_list))
os.execv(qemu_path, kvm_argument_list)
......@@ -28,19 +28,24 @@
import six.moves.http_client as httplib
import json
import os
import hashlib
import psutil
import requests
import six
import slapos.util
import sqlite3
from six.moves.urllib.parse import parse_qs, urlparse
import unittest
import subprocess
from slapos.recipe.librecipe import generateHashFromFiles
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
from slapos.slap.standalone import SlapOSNodeCommandError
has_kvm = os.access('/dev/kvm', os.R_OK|os.W_OK)
has_kvm = os.access('/dev/kvm', os.R_OK | os.W_OK)
skipUnlessKvm = unittest.skipUnless(has_kvm, 'kvm not loaded or not allowed')
skipIfPython3 = unittest.skipIf(six.PY3, 'rdiff-backup is not compatible with Python 3 yet')
skipIfPython3 = unittest.skipIf(
six.PY3, 'rdiff-backup is not compatible with Python 3 yet')
if has_kvm:
setUpModule, InstanceTestCase = makeModuleSetUpAndTestCaseClass(
......@@ -57,7 +62,10 @@ else:
bootstrap_common_param_dict = {
# the bootstrap script is vm-bootstrap
"bootstrap-script-url": "http://shacache.org/shacache/05105cd25d1ad798b71fd46a206c9b73da2c285a078af33d0e739525a595886785725a68811578bc21f75d0a97700a66d5e75bce5b2721ca4556a0734cb13e65#c98825aa1b6c8087914d2bfcafec3058",
"bootstrap-script-url":
"http://shacache.org/shacache/05105cd25d1ad798b71fd46a206c9b73da2c285a078"
"af33d0e739525a595886785725a68811578bc21f75d0a97700a66d5e75bce5b2721ca455"
"6a0734cb13e65#c98825aa1b6c8087914d2bfcafec3058",
"slave-frontend": {
"slave-frontend-dict": {}
},
......@@ -69,7 +77,7 @@ bootstrap_common_param_dict = {
"fw-restricted-access": "off",
"fw-authorized-sources": [],
"fw-reject-sources": ["10.32.0.0/13"]
}
}
bootstrap_machine_param_dict = {
"computer-guid": "local",
......@@ -80,20 +88,23 @@ bootstrap_machine_param_dict = {
"cpu-count": 2,
"disk-size": 50,
# Debian 10 image
"virtual-hard-drive-url": "http://shacache.org/shacache/9d3e6d017754fdd08e5ecf78093dec27fd792fb183df6146006adf003b6f4b98c0388d5a11566627101f7855d77f60e3dd4ba7ce66850f4a8f030573b904d5ab",
"virtual-hard-drive-url":
"http://shacache.org/shacache/9d3e6d017754fdd08e5ecf78093dec27fd792fb183d"
"f6146006adf003b6f4b98c0388d5a11566627101f7855d77f60e3dd4ba7ce66850f4a8f0"
"30573b904d5ab",
"virtual-hard-drive-md5sum": "b7928d7b0a2b5e2888f5ddf68f5fe422",
"virtual-hard-drive-gzipped": False,
"hard-drive-url-check-certificate": False,
"use-tap": True,
"use-nat": True,
"nat-restrict-mode":True,
"nat-restrict-mode": True,
"enable-vhost": True,
"external-disk-number": 1,
"external-disk-size": 100,
"external-disk-format": "qcow2",
"enable-monitor": True,
"keyboard-layout-language": "fr"
}
}
@skipUnlessKvm
class ServicesTestCase(InstanceTestCase):
def test_hashes(self):
......@@ -237,14 +248,18 @@ class TestAccessDefaultAdditional(MonitorAccessMixin, InstanceTestCase):
)
self.assertIn('<title>noVNC</title>', result.text)
@skipUnlessKvm
class TestAccessDefaultBootstrap(MonitorAccessMixin, InstanceTestCase):
__partition_reference__ = 'adb'
expected_partition_with_monitor_base_url_count = 1
# as few gigabytes are being downloaded, wait a bit longer
instance_max_retry = 100
@classmethod
def getInstanceParameterDict(cls):
return {'_': json.dumps(dict(bootstrap_common_param_dict, **bootstrap_machine_param_dict))}
return {'_': json.dumps(dict(
bootstrap_common_param_dict, **bootstrap_machine_param_dict))}
def test(self):
connection_parameter_dict = self.computer_partition\
......@@ -257,6 +272,7 @@ class TestAccessDefaultBootstrap(MonitorAccessMixin, InstanceTestCase):
)
self.assertIn('<title>noVNC</title>', result.text)
@skipUnlessKvm
class TestAccessKvmCluster(MonitorAccessMixin, InstanceTestCase):
__partition_reference__ = 'akc'
......@@ -328,11 +344,14 @@ class TestAccessKvmClusterAdditional(MonitorAccessMixin, InstanceTestCase):
)
self.assertIn('<title>noVNC</title>', result.text)
@skipIfPython3
@skipUnlessKvm
class TestAccessKvmClusterBootstrap(MonitorAccessMixin, InstanceTestCase):
__partition_reference__ = 'akcb'
expected_partition_with_monitor_base_url_count = 3
# as few gigabytes are being downloaded, wait a bit longer
instance_max_retry = 100
@classmethod
def getInstanceSoftwareType(cls):
......@@ -345,7 +364,10 @@ class TestAccessKvmClusterBootstrap(MonitorAccessMixin, InstanceTestCase):
"test-machine1": bootstrap_machine_param_dict,
"test-machine2": dict(bootstrap_machine_param_dict, **{
# Debian 9 image
"virtual-hard-drive-url": "http://shacache.org/shacache/ce07873dbab7fa8501d1bf5565c2737b2eed6c8b9361b4997b21daf5f5d1590972db9ac00131cc5b27d9aa353f2f94071e073f9980cc61badd6d2427f592e6e8",
"virtual-hard-drive-url":
"http://shacache.org/shacache/ce07873dbab7fa8501d1bf5565c2737b2"
"eed6c8b9361b4997b21daf5f5d1590972db9ac00131cc5b27d9aa353f2f940"
"71e073f9980cc61badd6d2427f592e6e8",
"virtual-hard-drive-md5sum": "2b113e3cd8276b9740189622603d6f99"
})
}
......@@ -354,19 +376,22 @@ class TestAccessKvmClusterBootstrap(MonitorAccessMixin, InstanceTestCase):
def test(self):
connection_parameter_dict = self.computer_partition\
.getConnectionParameterDict()
result = requests.get(connection_parameter_dict['test-machine1-url'], verify=False)
result = requests.get(
connection_parameter_dict['test-machine1-url'], verify=False)
self.assertEqual(
httplib.OK,
result.status_code
)
self.assertIn('<title>noVNC</title>', result.text)
result = requests.get(connection_parameter_dict['test-machine2-url'], verify=False)
result = requests.get(
connection_parameter_dict['test-machine2-url'], verify=False)
self.assertEqual(
httplib.OK,
result.status_code
)
self.assertIn('<title>noVNC</title>', result.text)
@skipIfPython3
@skipUnlessKvm
class TestInstanceResilient(InstanceTestCase):
......@@ -395,6 +420,7 @@ class TestInstanceResilient(InstanceTestCase):
'takeover-kvm-1-url',
'url']))
@skipIfPython3
@skipUnlessKvm
class TestAccessResilientAdditional(InstanceTestCase):
......@@ -430,6 +456,7 @@ class TestAccessResilientAdditional(InstanceTestCase):
)
self.assertIn('<title>noVNC</title>', result.text)
class TestInstanceNbdServer(InstanceTestCase):
__partition_reference__ = 'ins'
instance_max_retry = 5
......@@ -448,10 +475,278 @@ class TestInstanceNbdServer(InstanceTestCase):
def test(self):
connection_parameter_dict = self.computer_partition\
.getConnectionParameterDict()
result = requests.get(connection_parameter_dict['upload_url'].strip(), verify=False)
result = requests.get(
connection_parameter_dict['upload_url'].strip(), verify=False)
self.assertEqual(
httplib.OK,
result.status_code
)
self.assertIn('<title>Upload new File</title>', result.text)
self.assertIn("WARNING", connection_parameter_dict['status_message'])
@skipUnlessKvm
class TestImageUrlList(InstanceTestCase):
__partition_reference__ = 'iul'
@classmethod
def getInstanceSoftwareType(cls):
return 'default'
@classmethod
def getInstanceParameterDict(cls):
# start with empty, but working configuration
return {}
def tearDown(self):
# clean up the instance for other tests
# 1st remove all images...
self.rerequestInstance({'image-url-list': ''})
self.slap.waitForInstance(max_retry=10)
# 2nd ...move instance to "default" state
self.rerequestInstance({})
self.slap.waitForInstance(max_retry=10)
def rerequestInstance(self, parameter_dict, state='started'):
software_url = self.getSoftwareURL()
software_type = self.getInstanceSoftwareType()
return self.slap.request(
software_release=software_url,
software_type=software_type,
partition_reference=self.default_partition_reference,
partition_parameter_kw=parameter_dict,
state=state)
fake_image, = (
"https://shacache.nxdcdn.com/shacache/05105cd25d1ad798b71fd46a206c9b73d"
"a2c285a078af33d0e739525a595886785725a68811578bc21f75d0a97700a66d5e75bc"
"e5b2721ca4556a0734cb13e65",)
fake_image_md5sum = "c98825aa1b6c8087914d2bfcafec3058"
fake_image2, = (
"https://shacache.nxdcdn.com/shacache/54f8a83a32bbf52602d9d211d592ee705"
"99f0c6b6aafe99e44aeadb0c8d3036a0e673aa994ffdb28d9fb0de155720123f74d814"
"2a74b7675a8d8ca20476dba6e",)
fake_image2_md5sum = "d4316a4d05f527d987b9d6e43e4c2bc6"
fake_image_wrong_md5sum = "c98825aa1b6c8087914d2bfcafec3057"
def raising_waitForInstance(self, max_retry):
with self.assertRaises(SlapOSNodeCommandError):
self.slap.waitForInstance(max_retry=max_retry)
def test(self):
partition_parameter_kw = {
'image-url-list': "%s#%s\n%s#%s" % (
self.fake_image, self.fake_image_md5sum, self.fake_image2,
self.fake_image2_md5sum)
}
self.rerequestInstance(partition_parameter_kw)
self.slap.waitForInstance(max_retry=10)
# check that image is correctly downloaded and linked
image_repository = os.path.join(
self.computer_partition_root_path, 'srv', 'image-repository')
image = os.path.join(image_repository, self.fake_image_md5sum)
image_link = os.path.join(image_repository, 'image_001')
self.assertTrue(os.path.exists(image))
with open(image, 'rb') as fh:
image_md5sum = hashlib.md5(fh.read()).hexdigest()
self.assertEqual(image_md5sum, self.fake_image_md5sum)
self.assertTrue(os.path.islink(image_link))
self.assertEqual(os.readlink(image_link), image)
image2 = os.path.join(image_repository, self.fake_image2_md5sum)
image2_link = os.path.join(image_repository, 'image_002')
self.assertTrue(os.path.exists(image2))
with open(image2, 'rb') as fh:
image2_md5sum = hashlib.md5(fh.read()).hexdigest()
self.assertEqual(image2_md5sum, self.fake_image2_md5sum)
self.assertTrue(os.path.islink(image2_link))
self.assertEqual(os.readlink(image2_link), image2)
# check that the image is NOT YET available in kvm
with self.slap.instance_supervisor_rpc as instance_supervisor:
kvm_pid = [q for q in instance_supervisor.getAllProcessInfo()
if 'kvm-' in q['name']][0]['pid']
kvm_process = psutil.Process(kvm_pid)
cmd_line = ''.join(kvm_process.cmdline())
self.assertNotIn(
'srv/image-repository/image_001,media=cdrom',
cmd_line
)
self.assertNotIn(
'srv/image-repository/image_002,media=cdrom',
cmd_line
)
# mimic the requirement: restart the instance by requesting it stopped and
# then started started, like user have to do it
self.rerequestInstance(partition_parameter_kw, state='stopped')
self.slap.waitForInstance(max_retry=1)
self.rerequestInstance(partition_parameter_kw, state='started')
self.slap.waitForInstance(max_retry=1)
# now the image is available in the kvm
with self.slap.instance_supervisor_rpc as instance_supervisor:
kvm_pid = [q for q in instance_supervisor.getAllProcessInfo()
if 'kvm-' in q['name']][0]['pid']
kvm_process = psutil.Process(kvm_pid)
cmd_line = ''.join(kvm_process.cmdline())
self.assertIn(
'srv/image-repository/image_001,media=cdrom',
cmd_line
)
self.assertIn(
'srv/image-repository/image_002,media=cdrom',
cmd_line
)
# cleanup of images works, also asserts that configuration changes are
# reflected
self.rerequestInstance({'image-url-list': ''})
self.slap.waitForInstance(max_retry=2)
self.assertEqual(
os.listdir(image_repository),
[]
)
def assertPromiseFails(self, promise):
monitor_run_promise = os.path.join(
self.computer_partition_root_path, 'software_release', 'bin',
'monitor.runpromise'
)
monitor_configuration = os.path.join(
self.computer_partition_root_path, 'etc', 'monitor.conf')
self.assertNotEqual(
0,
subprocess.call([
monitor_run_promise, '-c', monitor_configuration, '-a', '-f',
'--run-only', promise])
)
def test_bad_parameter(self):
self.rerequestInstance({
'image-url-list': "jsutbad"
})
self.raising_waitForInstance(3)
self.assertPromiseFails('image-url-list-config-state-promise.py')
def test_incorrect_md5sum(self):
self.rerequestInstance({
'image-url-list': "%s#" % (self.fake_image,)
})
self.raising_waitForInstance(3)
self.assertPromiseFails('image-url-list-config-state-promise.py')
self.rerequestInstance({
'image-url-list': "url#asdasd"
})
self.raising_waitForInstance(3)
self.assertPromiseFails('image-url-list-config-state-promise.py')
def test_not_matching_md5sum(self):
self.rerequestInstance({
'image-url-list': "%s#%s" % (
self.fake_image, self.fake_image_wrong_md5sum)
})
self.raising_waitForInstance(3)
self.assertPromiseFails('image-url-list-download-md5sum-promise.py')
self.assertPromiseFails('image-url-list-download-state-promise.py')
def test_unreachable_host(self):
self.rerequestInstance({
'image-url-list': "evennotahost#%s" % (
self.fake_image_md5sum,)
})
self.raising_waitForInstance(3)
self.assertPromiseFails('image-url-list-download-state-promise.py')
def test_too_many_images(self):
self.rerequestInstance({
'image-url-list': """
image1#11111111111111111111111111111111
image2#22222222222222222222222222222222
image3#33333333333333333333333333333333
image4#44444444444444444444444444444444
image5#55555555555555555555555555555555
image6#66666666666666666666666666666666
"""
})
self.raising_waitForInstance(3)
self.assertPromiseFails('image-url-list-config-state-promise.py')
@skipUnlessKvm
class TestImageUrlListKvmCluster(InstanceTestCase):
__partition_reference__ = 'iulkc'
@classmethod
def getInstanceSoftwareType(cls):
return 'kvm-cluster'
fake_image, = (
"https://shacache.nxdcdn.com/shacache/05105cd25d1ad798b71fd46a206c9b73d"
"a2c285a078af33d0e739525a595886785725a68811578bc21f75d0a97700a66d5e75bc"
"e5b2721ca4556a0734cb13e65",)
fake_image_md5sum = "c98825aa1b6c8087914d2bfcafec3058"
fake_image2, = (
"https://shacache.nxdcdn.com/shacache/54f8a83a32bbf52602d9d211d592ee705"
"99f0c6b6aafe99e44aeadb0c8d3036a0e673aa994ffdb28d9fb0de155720123f74d814"
"2a74b7675a8d8ca20476dba6e",)
fake_image2_md5sum = "d4316a4d05f527d987b9d6e43e4c2bc6"
@classmethod
def getInstanceParameterDict(cls):
return {'_': json.dumps({
"kvm-partition-dict": {
"KVM0": {
"disable-ansible-promise": True,
"image-url-list": "%s#%s" % (
cls.fake_image, cls.fake_image_md5sum)
},
"KVM1": {
"disable-ansible-promise": True,
"image-url-list": "%s#%s" % (
cls.fake_image2, cls.fake_image2_md5sum)
}
}
})}
def test(self):
# Note: As there is no way to introspect nicely where partition landed
# we assume ordering of the cluster requests
KVM0_config = os.path.join(
self.slap.instance_directory, self.__partition_reference__ + '1', 'etc',
'image-url-list.conf')
KVM1_config = os.path.join(
self.slap.instance_directory, self.__partition_reference__ + '2', 'etc',
'image-url-list.conf')
with open(KVM0_config, 'r') as fh:
self.assertEqual(
"%s#%s" % (self.fake_image, self.fake_image_md5sum),
fh.read()
)
with open(KVM1_config, 'r') as fh:
self.assertEqual(
"%s#%s" % (self.fake_image2, self.fake_image2_md5sum),
fh.read()
)
@skipUnlessKvm
class TestCpuMemMaxDynamic(InstanceTestCase):
__partition_reference__ = 'cmm'
@classmethod
def getInstanceParameterDict(cls):
return {
'cpu-count': 2,
'ram-size': 2048
}
def test(self):
with open(os.path.join(
self.computer_partition_root_path, 'bin', 'kvm_raw'), 'rb') as fh:
kvm_raw = fh.read()
self.assertTrue('smp_count = 2' in kvm_raw)
self.assertTrue('smp_max_count = 3' in kvm_raw)
self.assertTrue('ram_size = 2048' in kvm_raw)
self.assertTrue("ram_max_size = '2560'" in kvm_raw)
......@@ -34,7 +34,7 @@ md5sum = c535f4df6388cdc3f6c1df2a91d6dd53
[network-bench-cfg]
filename = network_bench.cfg.in
md5sum = cfcbf2002b8eff5153e2bf68ed24b720
md5sum = 4f4d1a0afc225656278154e453587aea
[monitor-collect-csv-dump]
filename = script/collect_csv_dump.py
......
......@@ -14,16 +14,16 @@
# not need these here).
[template-nextcloud-install.sh]
filename = nextcloud-install.sh.in
md5sum = a2281f86f6a26a8ff40a57a495505977
md5sum = 209abce3f82706fb80995f7efea70a9c
[template-apache-httpd]
filename = apache-httpd.conf.in
md5sum = 839258624e273aac71a96516bf34c7e6
md5sum = f7e8f6ea20f8685bb4e42cacaee4116f
[template-nextcloud-config.json]
filename = nextcloud-config.json.in
md5sum = fbc5eacda192cb02c8d9861cb628fe6f
md5sum = 6f42f0a8c5e5c0c657541a65c4d9ee57
[template-nextcloud-instance]
filename = nextcloud-instance.cfg.in
md5sum = 28f9a64176e86360a3398742d5ad8470
md5sum = b62f438d380363387fe225fa3d7e2291
......@@ -88,4 +88,3 @@ partition = $${slap-connection:partition-id}
url = $${slap-connection:server-url}
key = $${slap-connection:key-file}
cert = $${slap-connection:cert-file}
......@@ -18,7 +18,7 @@ md5sum = fddea033e1aa9d6147a1a47bd7cc4b62
[template-powerdns]
filename = instance-powerdns.cfg
md5sum = 2adb91323d60fc350f52910a3257d4a7
md5sum = 9cd4e436fa432f37b9f8f4de8350581b
[template-pdns-configuration]
_update_hash_filename_ = template/pdns.conf.jinja2
......
......@@ -58,7 +58,7 @@ configuration = $${pdns-directory:configuration}/pdns.conf
local-ipv4 = $${instance-parameter:ipv4-random}
ipv6 = $${instance-parameter:ipv6-random}
port = 5353
socket-directory = $${pdns-directory:socket}
socket-directory = $${pdns-directory:socket-directory}
webserver-port = 8088
[geo]
......@@ -68,7 +68,7 @@ database = ${geolite2-country:location}/GeoLite2-Country.mmdb
[pdns-directory]
recipe = slapos.cookbook:mkdirectory
configuration = $${directory:etc}/pdns
socket = $${directory:run}/pdns-socket
socket-directory = $${directory:run}/
[pdns-configuration-template]
< = jinja2-template-base
......
......@@ -16,5 +16,3 @@ http://www.proftpd.org/docs/
* make sure SFTPLog is useful (seems very verbose and does not contain more than stdout)
* make it easier to manage users ( using `mod_auth_web` against an ERP5 endpoint or accepting a list of user/password as instance parameter )
* allow configuring webhooks when new file is uploaded
......@@ -24,4 +24,3 @@ md5sum = 2a2c066d7d40dd8545f3008f434ee842
[proftpd-config-file]
filename = proftpd-config-file.cfg.in
md5sum = a7c0f4607c378b640379cc258a8aadfa
......@@ -5,4 +5,3 @@ https://www.pureftpd.org/project/pure-ftpd
# Features
* After each upload, call the script /opt/pureftpd/upload_script
......@@ -18,11 +18,11 @@ md5sum = 610fc6fd0444d3bab3fca4478572749a
[template-re6stnet]
filename = instance-re6stnet.cfg.in
md5sum = ee582dbab92c1c452874ddca3831aef6
md5sum = 002f7405f565c82219b0d4b92790ff8c
[template-apache-conf]
filename = apache.conf.in
md5sum = d64cafda1139b740a49a9f5e30a1b57b
md5sum = 2ed3c4e9b9d58d2e57cda227bdd454d2
[template-re6st-registry-conf]
filename = re6st-registry.conf.in
......@@ -30,7 +30,7 @@ md5sum = b0f0facfea82a4481f4fb5b0b263e09a
[template-wrapper]
filename = wrapper.in
md5sum = 69e63cb58267335e21da772bd867657e
md5sum = 7633bdfc0e638ff9979f963fbbca8f13
[template-registry-run]
filename = registry-run.in
......
......@@ -253,4 +253,3 @@ offline = true
[slap-parameter]
slave_instance_list = {}
......@@ -77,4 +77,3 @@ slapos.recipe.template = 4.4
# Required by:
# re6stnet==0.533
miniupnpc = 1.9
[template]
_update_hash_filename_ = instance.cfg
md5sum = 651973f038d8b0f8c62e11e299344506
[template-redis]
_update_hash_filename_ = instance-redis.cfg
md5sum = 1ed6238490b36eaae783d53d7a96cfea
......@@ -10,20 +10,19 @@ extends =
../../component/redis/buildout.cfg
../../component/git/buildout.cfg
../../stack/slapos.cfg
./buildout.hash.cfg
[template]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg
url = ${:_profile_base_location_}/${:_update_hash_filename_}
output = ${buildout:directory}/template.cfg
mode = 0644
md5sum = 5d2008c96a569c3ce498ddc0933afea3
[template-redis]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-redis.cfg
url = ${:_profile_base_location_}/${:_update_hash_filename_}
output = ${buildout:directory}/template-redis.cfg
mode = 0644
md5sum = f7b01ca7698c1b771f0653d64dc945a7
[versions]
plone.recipe.command = 1.1
......
......@@ -14,7 +14,7 @@
# not need these here).
[instance.cfg]
filename = instance.cfg.in
md5sum = b41f521b5f7980c64260ed0e5c494450
md5sum = 027cdcfe251b7bba9b779fee890a9162
[instance-repman.cfg]
_update_hash_filename_ = instance-repman.cfg.jinja2.in
......@@ -22,11 +22,11 @@ md5sum = 657ecdb1dfbbcf53e4e7932b3b5708c4
[config-toml.in]
_update_hash_filename_ = templates/config.toml.in
md5sum = 19fe38a342a5c9857e29f78eedb3c46e
md5sum = 2ee2c4bc5f985c11c2167b819d261256
[config-cluster-toml.in]
_update_hash_filename_ = templates/cluster-config.toml.in
md5sum = 079599a2841b5a0d5178bb12c4a30ae8
md5sum = c78c6be9537d3dffbb526b0e819bacb5
[nginx.conf.in]
_update_hash_filename_ = templates/nginx.conf.in
......@@ -42,19 +42,19 @@ md5sum = f3661b788099bb31d71ba6e7d36836d9
[template-mariadb-initial-setup]
_update_hash_filename_ = templates/mariadb_initial_setup.sql.in
md5sum = 9be53e2e92333b93e92556b8a01d9c42
md5sum = df44c2d6fb1971df582345daedad280c
[mariadb-init-root-sql]
_update_hash_filename_ = templates/mariadb_init_root.sql.in
md5sum = d927b5d36410bb02717d5ca125525785
md5sum = d8a0a7d72b02f911dafda655941b805d
[init-root-wrapper-in]
_update_hash_filename_ = templates/init_root_wrapper.in
md5sum = 83ef59b5afaf4454d368823c33aef9cb
md5sum = 7661e3b2f66982b9b17bd2ab73e2e8ef
[repman-manager-sh.in]
_update_hash_filename_ = templates/repman-manager.sh.in
md5sum = 50503bec392e31126328f51eadc11634
md5sum = 852dfab6d798aa1382eec4de2fd624f9
[dbjobs-in]
_update_hash_filename_ = templates/dbjobs.in
......
......@@ -101,5 +101,3 @@ template = {{ template_mariadb }}
filename = instance-mariadb.cfg
extra-context =
section parameter_dict template-mariadb-parameters
......@@ -66,5 +66,3 @@ switchover-slave-wait-catch = {{ setbool(parameter_dict['switchover-slave-wait-c
switchover-wait-kill = {{ parameter_dict['switchover-wait-kill'] }}
switchover-wait-trx = {{ parameter_dict['switchover-wait-trx'] }}
switchover-wait-write-query = {{ parameter_dict['switchover-wait-write-query'] }}
......@@ -67,4 +67,3 @@ sysbench-threads = 4
# Time to run benchmark (default 100)
sysbench-time = 100
sysbench-v1 = true
......@@ -15,4 +15,4 @@
[template]
filename = instance.cfg.in
md5sum = 1f31a063c79e19243dcc5508e08a6ae5
md5sum = 683fdb547b0c42460361786e1ab102ca
......@@ -6,4 +6,3 @@ parts =
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
......@@ -19,4 +19,4 @@ md5sum = c4ac5de141ae6a64848309af03e51d88
[template-selenium]
filename = instance-selenium.cfg.in
md5sum = 269ac5ce96695ede3ee917c6ce17ae2d
md5sum = 597991d7354970550d25324e3836adda
......@@ -47,4 +47,3 @@ LoadModule autoindex_module modules/mod_autoindex.so
order allow,deny
Allow from All
</Directory>
......@@ -20,4 +20,3 @@ input_definition_file = ${:computer-definition-file}
computer_xml = ${:computer-xml}
partition_amount = ${:partition-amount}
create_tap = false
......@@ -22,4 +22,4 @@ md5sum = d10a5ddfffa67b8ca01b3e38315bae2f
[template-apache-backend-conf]
filename = apache-backend.conf.in
md5sum = a169c1d6b0f2636f21f180e8a0b52137
md5sum = ba8f878083b79535c6a2d10de496d90f
......@@ -22,6 +22,11 @@ branch = erp5-vifib
repository = https://lab.nexedi.com/nexedi/slapos.core.git
branch = master
[slapos-bin]
<= erp5
repository = https://lab.nexedi.com/nexedi/slapos-bin.git
branch = master
[vifib-fix-products-paths]
recipe = plone.recipe.command
stop-on-error = true
......@@ -35,9 +40,11 @@ eggs +=
dummy +=
${vifib:location}
${slapos-bin:location}
extra-paths +=
${vifib:location}/master
${slapos-bin:location}
### Overwrite recipes to introduce customized changes
......
......@@ -16,40 +16,36 @@ slaprunner to develop a slapos profile, in the example `helloworld`, make
changes to the code, run tests and publish changes.
```bash
# install this software release
# install this software release and request an instance
SR=https://lab.nexedi.com/nexedi/slapos/raw/1.0/software/slapos-sr-testing/software.cfg
COMP=slaprunner
COMP=slaprunner # or "local" if using theia
INSTANCE_NAME=$COMP
slapos supply $SR $COMP
slapos node software
slapos request --node=node=$COMP $INSTANCE_NAME $SR
slapos request --node=computer_guid=$COMP $INSTANCE_NAME $SR
slapos node instance
# note the `environment-script` published value
slapos request --node=computer_guid=$COMP $INSTANCE_NAME $SR
# and load this script to set environment variables
source ( environment-script from step above )
# Clone a working copy somewhere
cd ~/srv/runner/project/
git clone https://lab.nexedi.com/nexedi/slapos.git slapos_work
# Create a directory to hold the slapos test environment
SLAPOS_TEST_WORKING_DIR=~/tmp/slapos/
mkdir -p $SLAPOS_TEST_WORKING_DIR
# change directory to the directory containing test for this software
cd ~/srv/runner/project/slapos_work/software/helloworld/test/
# run tests, using bundled python intepreter with pre-installed eggs dependencies.
SLAPOS_TEST_IPV6=::1 \
SLAPOS_TEST_IPV4=127.0.0.1 \
SLAPOS_TEST_WORKING_DIR=$SLAPOS_TEST_WORKING_DIR \
SLAPOS_TEST_SHARED_PART_LIST=/opt/slapos/shared/:~/srv/runner/shared \
SLAPOS_TEST_VERBOSE=1 \
SLAPOS_TEST_DEBUG=1 \
~/srv/runner/instance/slappart0/software_release/bin/python_for_test setup.py test
# run test (with debugging features activated)
SLAPOS_TEST_DEBUG= 1 python_for_test setup.py test
```
## Environment variables
The `environment-script` set all variabels except `SLAPOS_TEST_DEBUG` and `SLAPOS_TEST_VERBOSE` for you, but for reference, here is the list of variables which control the test runner:
| Variable | Description |
| --- | --- |
| `SLAPOS_TEST_IPV6` | ipv6 used by this instance. Usually you want to use a global address here to be able to connect to this instance. |
......@@ -83,3 +79,6 @@ To use a development version of `slapos.cookbook` on test nodes, you can try usi
### Test pass locally but fail on test nodes, what can I do ?
At the end of the test, a snapshot of the slapos instances is created. Sometimes examining the log files can help understanding what went wrong.
Most of the time, problems are because on test nodes paths are very long. One advanced technique to reproduce the problem in your development environment is to set `SLAPOS_TEST_WORKING_DIR` environment variable to a path with the same length as the ones on test nodes.
One way to make instances uses a slightly shorter path is to define `__partition_reference__` class attribute, so that the instances uses this as prefix instead of the class name.
......@@ -15,4 +15,4 @@
[template]
filename = instance.cfg
md5sum = 298bac4a631de3b30593b9a1dcf63e1c
md5sum = 25a4d7e438402d992edadf9339faf557
[buildout]
parts =
slapos-test-runner
publish
eggs-directory = {{ buildout['eggs-directory'] }}
develop-eggs-directory = {{ buildout['develop-eggs-directory'] }}
......@@ -26,6 +27,7 @@ repository = {{ slapos_location }}
recipe = slapos.cookbook:mkdirectory
bin = ${buildout:directory}/bin
working-dir = ${buildout:directory}/tmp
etc = ${buildout:directory}/etc
[slapos-test-runner]
recipe = slapos.cookbook:wrapper
......@@ -36,7 +38,36 @@ command-line =
--source_code_path_list={{ ','.join(tests.splitlines()) }}
environment =
PATH={{ buildout['bin-directory'] }}:{{ curl_location }}/bin/:/usr/bin/:/bin
SLAPOS_TEST_IPV4=${slap-configuration:ipv4-random}
SLAPOS_TEST_IPV6=${slap-configuration:ipv6-random}
SLAPOS_TEST_WORKING_DIR=${directory:working-dir}
PATH=${slapos-test-runner-environment:PATH}
SLAPOS_TEST_IPV4=${slapos-test-runner-environment:SLAPOS_TEST_IPV4}
SLAPOS_TEST_IPV6=${slapos-test-runner-environment:SLAPOS_TEST_IPV6}
SLAPOS_TEST_WORKING_DIR=${slapos-test-runner-environment:SLAPOS_TEST_WORKING_DIR}
[slapos-test-runner-environment]
PATH = {{ buildout['bin-directory'] }}:{{ curl_location }}/bin/:/usr/bin/:/bin
SLAPOS_TEST_IPV4 = ${slap-configuration:ipv4-random}
SLAPOS_TEST_IPV6 = ${slap-configuration:ipv6-random}
SLAPOS_TEST_WORKING_DIR = ${directory:working-dir}
[slapos-local-development-environment.sh]
recipe = slapos.recipe.template:jinja2
rendered = ${directory:etc}/${:_buildout_section_name_}
template = inline:
export PATH=${slapos-test-runner-environment:PATH}:$PATH
export SLAPOS_TEST_IPV4=${slapos-test-runner-environment:SLAPOS_TEST_IPV4}
export SLAPOS_TEST_IPV6=${slapos-test-runner-environment:SLAPOS_TEST_IPV6}
export SLAPOS_TEST_WORKING_DIR=${slapos-test-runner-environment:SLAPOS_TEST_WORKING_DIR}
{% set shared_part_list = [] %}
{% for shared_part in buildout['shared-part-list'].splitlines() -%}
{% do shared_part_list.append(shared_part) %}
{%- endfor %}
export SLAPOS_TEST_SHARED_PART_LIST={{ os.pathsep.join(shared_part_list) }}
echo "Environment loaded."
echo "To work on a test, execute:"
echo " SLAPOS_TEST_DEBUG=1 {{ interpreter }} setup.py test"
echo "from test folder"
echo
[publish]
recipe = slapos.cookbook:publish
environment-script = ${slapos-local-development-environment.sh:rendered}
......@@ -19,6 +19,8 @@ parts =
slapos-cookbook
template
shared-part-list =
[setup-develop-egg]
recipe = zc.recipe.egg:develop
......@@ -227,10 +229,12 @@ branch = master
[template]
recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/${:filename}
extensions = jinja2.ext.do
rendered = ${buildout:directory}/template.cfg
mode = 640
context =
section buildout buildout
import os os
key git_location git:location
key slapos_location slapos-repository:location
key interpreter eggs:interpreter
......
......@@ -16,4 +16,3 @@
[template]
filename = instance.cfg
md5sum = 47eb5f8e6f56729e2ce0e6213bccf4f7
......@@ -4,4 +4,3 @@ extends =
[python]
part = python3
......@@ -42,7 +42,7 @@ md5sum = 2b06f7eb9a1d45d250d4b92a944db925
[template_httpd_conf]
filename = httpd_conf.in
md5sum = 1d96c29b931383b2a87b9ae049d07d74
md5sum = 1d983937ad0909b3ef4d35d190645635
[template_launcher]
filename = launcher.in
......@@ -54,15 +54,15 @@ md5sum = fa90fc9a9010ce4bb1478ae37da9c56d
[template-slapformat-definition.cfg]
filename = template/slapformat-definition.cfg.in
md5sum = a94299dd2133956060486609e30b97ec
md5sum = bbb767dee9730f5a816ce4341403d89b
[template-parameters]
filename = parameters.xml.in
md5sum = f8446fcf254b4929eb828a9a1d7e5f62
md5sum = 0d7b2432aa809b859509ca1114accd8f
[template-bash-profile]
filename = template/bash_profile.in
md5sum = 7645048216fcf957f7773534cd0408dc
md5sum = d0ff3c86c76168d17d27b47b93a9b7aa
[template-supervisord]
filename = template/supervisord.conf.in
......@@ -78,7 +78,7 @@ md5sum = acaac32cf1bd45714272468a89f4f119
[template-resilient-software-release-information]
filename = template/resilient_software_release_information.py.in
md5sum = 2451072826a9ad9425d62c9e9c7f6284
md5sum = 869a3afbf3c2fff6c72602662c6a8730
[template-slapuser-script]
filename = template/slapos-slapuser-script.in
......@@ -86,4 +86,4 @@ md5sum = 75aab99c995ca841f93fc77fc9116c37
[template-buildout-shared-part-list]
filename = template/buildout-shared-part-list.in
md5sum = 3203c9ad0b30d3ee39a809a067efff8d
md5sum = f619c8c5897c4851442b7090c8509758
......@@ -95,4 +95,3 @@ Alias /share {{ parameters.runner_home }}
Require valid-user
</LimitExcept>
</Directory>
......@@ -15,4 +15,3 @@
{% endfor %}
{% endif %}
</instance>
......@@ -180,4 +180,3 @@ slapos.recipe.template = 4.4
collective.recipe.environment = 0.2.0
smmap = 0.9.0
lockfile = 0.12.2
......@@ -36,4 +36,3 @@ alias l='ls -CF'
echo "Welcome to SlapOS slaprunner shell"
echo
......@@ -10,4 +10,3 @@ pathname = slappart{{ partition_index }}
network_interface =
{% endfor %}
......@@ -52,4 +52,3 @@ md5sum = 869cca5a1c87a9766d0650f4ef374dc1
[config-user_db]
filename = config/user_db.csv
md5sum = e37cd54eb08de3202cc0250437a2f181
......@@ -19,4 +19,4 @@ md5sum = d027a2dccaf15ae6e7d3a28cc02d70c3
[template-turnserver]
filename = instance-turnserver.cfg.jinja2.in
md5sum = 62382da42ea56e9fd0124caeca42bcf3
md5sum = 539417d669c15b853ac8525f8d5cbc44
[application-template]
_update_hash_filename_ = wp-config.php.in
md5sum = d53056c1b3c59525a58f41aea53622aa
......@@ -50,4 +50,3 @@ command =
grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link &&
grep parts ${buildout:develop-eggs-directory}/slapos.core.egg-link &&
grep parts ${buildout:develop-eggs-directory}/slapos.toolbox.egg-link
[buildout]
extends =
../../stack/lamp/buildout.cfg
./buildout.hash.cfg
[application]
url = http://wordpress.org/wordpress-3.5.1.tar.gz
......@@ -8,8 +9,7 @@ md5sum = 409889c98b13cbdbb9fd121df859ae3e
[application-template]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/wp-config.php.in
md5sum = 0d62b28125ca3c780db0b547199953f2
url = ${:_profile_base_location_}/${:_update_hash_filename_}
download-only = True
filename = template.in
mode = 0644
......
......@@ -52,6 +52,7 @@ extends =
../../component/postfix/buildout.cfg
../../component/zbarlight/buildout.cfg
../../component/pylint/buildout.cfg
../../component/perl-Image-ExifTool/buildout.cfg
../../stack/caucase/buildout.cfg
../../software/jupyter/software.cfg
../../software/neoppod/software-common.cfg
......@@ -249,6 +250,7 @@ link-binary =
${mariadb:location}/bin/mysqldump
${openssl:location}/bin/openssl
${optipng:location}/bin/optipng
${perl-Image-ExifTool:location}/bin/exiftool
${poppler:location}/bin/pdfinfo
${poppler:location}/bin/pdftohtml
${poppler:location}/bin/pdftotext
......
......@@ -74,7 +74,7 @@ md5sum = 328ea2bb5f2bff18f8be8c541c01f260
[monitor-template-dummy]
filename = dummy.cfg
md5sum = d41d8cd98f00b204e9800998ecf8427e
md5sum = 68b329da9893e34099c7d8ad5cb9c940
[template-erp5]
filename = instance-erp5.cfg.in
......
......@@ -6,7 +6,7 @@ extends =
../../component/pkgconfig/buildout.cfg
../../component/python-cffi/buildout.cfg
../../component/java/buildout.cfg
./buildout.hash.cfg
parts =
slapos-cookbook
......@@ -51,9 +51,6 @@ mode = 0644
[instance-stack]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-stack.cfg.in
url = ${:_profile_base_location_}/${:filename}
output = ${buildout:directory}/instance-stack.cfg
md5sum = 3c846dd1269aeef43cd2d5448463563b
mode = 0644
[instance-stack]
filename = instance-stack.cfg.in
md5sum = 42da07374e980f1934e01589684f1811
......@@ -48,4 +48,3 @@ url = ${:_profile_base_location_}/template/start-daemons.sh.in
output = $${directories:bin}/start-daemons.sh
md5sum = 35a51a7f826d29e47c564c57e954db80
mode = 0755
......@@ -14,15 +14,15 @@
# not need these here).
[instance]
filename = instance.cfg.in
md5sum = 6efa60dd898d3cd568afb4a47b94d573
md5sum = be63b936ed521edaead8e0770ac64621
[instance-apache-php]
filename = instance-apache-php.cfg.in
md5sum = 5212aed6c82876509eae9eda0975ffa1
md5sum = a20bb27c0077b5a7252bda06ba7e069d
[instance-lamp]
filename = instance-lamp.cfg.jinja2.in
md5sum = 39e765fca5fcd92cb9fa13cd2b4d95fd
md5sum = 52b76d3c8aa23f467db33e32a2b50ed6
[template-apache.conf]
filename = apache.conf.in
......@@ -30,4 +30,4 @@ md5sum = 04080510698732d84122b464fdb08c6a
[template-php.ini]
filename = php.ini.in
md5sum = 7de35e3f8619324119ff296580d2880f
md5sum = 599358f7df4c1e0de86270f5992ef904
......@@ -104,4 +104,3 @@ extra-context =
key ipv6_set slap-configuration:ipv6
raw bin_directory {{ bin_directory }}
section parameter_dict dynamic-template-mariadb-parameters
......@@ -69,4 +69,3 @@ opcache.interned_strings_buffer=8
opcache.max_accelerated_files=10000
opcache.revalidate_freq=1
opcache.save_comments=1
......@@ -14,7 +14,7 @@
# not need these here).
[logrotate-conf-template]
filename = logrotate.conf.in
md5sum = 2fc0ea335969ffab7d8c4f98e3d04ae7
md5sum = 7c41026716d856bba7c1252b72adbf77
[logrotate-entry-template]
filename = logrotate_entry.in
......
......@@ -2,4 +2,3 @@ compresscmd {{ parameter_dict['compress-binary'] }}
compressoptions -9
uncompresscmd {{ parameter_dict['uncompress-binary'] }}
include {{ parameter_dict['logrotate-entries'] }}
......@@ -96,4 +96,3 @@ context =
[versions]
cns.recipe.symlink = 0.2.3
......@@ -22,11 +22,11 @@ md5sum = 0540fc5cc439a06079e9e724a5a55a70
[monitor-template-wrapper]
_update_hash_filename_ = templates/wrapper.in
md5sum = 1695c9a06a2b11ccfe893d7a224e489d
md5sum = e8566c00b28f6f86adde11b6b6371403
[monitor-conf]
_update_hash_filename_ = templates/monitor.conf.in
md5sum = 91c4c9bba1f7df788b9b7a059ed89ac2
md5sum = 76ec5b30a11e03bb2496776684259fdf
[monitor-httpd-cors]
_update_hash_filename_ = templates/httpd-cors.cfg.in
......
......@@ -30,7 +30,7 @@ md5sum = 382d9b57faac92da3c723760f2210c13
[template-replicated]
filename = template-replicated.cfg.in
md5sum = 1b25afe215e45a92e0988ea2ca9484f4
md5sum = 0917aaacb752526e6f114839a3e6e1de
[template-parts]
filename = template-parts.cfg.in
......@@ -38,11 +38,11 @@ md5sum = 071b1034ee8f5cc14f79b16fdeba2813
[template-resilient-templates]
filename = template-resilient-templates.cfg.in
md5sum = 3b426db6fddce431b363d095a22dd698
md5sum = 604b9d160ab85ec80718a2c54df2c4b7
[instance-frozen]
filename = instance-frozen.cfg.in
md5sum = d21472f0e58f928fb827f2cbf22c4d4a
md5sum = 2c9828b57a65ee1eda62fcc4e9a92714
[resilient-web-takeover-cgi-script-download]
filename = resilient-web-takeover-cgi-script.py.in
......@@ -50,7 +50,7 @@ md5sum = 4137b7ac6b20a6cd897ad58f51a763b1
[template-wrapper]
filename = templates/wrapper.in
md5sum = 8cde04bfd0c0e9bd56744b988275cfd8
md5sum = 975177dedf677d24e14cede5d13187ce
[notifier-feed-promise-template]
filename = templates/notifier-feed-promise.py.in
......@@ -59,4 +59,3 @@ md5sum = fa6521daaa02fef4dd2ce06d29ef90be
[template-monitor-check-resilient-feed]
filename = templates/monitor-check-resilient-feed.in
md5sum = af9787f8440fef19924b2e765372b20f
......@@ -3,4 +3,3 @@ eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
parts =
......@@ -301,4 +301,3 @@ pbs-{{namebase}}{{id}}-computer-guid =
ignore-known-hosts-file = false
{% endmacro %}
......@@ -43,4 +43,3 @@ context =
key slave_instance_list slap-configuration:slave-instance-list
key slapparameter_dict slap-configuration:configuration
key ipv6 slap-configuration:ipv6-random
......@@ -19,4 +19,3 @@ md5sum = 163c9f60e4ad3842162cbb11d771b7b8
[supervisord-conf]
_update_hash_filename_ = supervisord.conf.in
md5sum = d624f65151233493c6dbdafa83ae8cbd
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment