Skip to content

slapos
slapos
Commit 5081f0b0 authored by Thomas Gambier's avatar Thomas Gambier 🚴🏼
Browse files
Options

Update Release Candidate

parents 58ab5601 1a1efa41
Hide whitespace changes
Inline Side-by-side
Showing with 2824 additions and 468 deletions
component/chromedriver/buildout.cfg
View file @ 5081f0b0
# ChromeDriver - Webdriver for Chrome
# http://chromedriver.chromium.org/
# https://chromedriver.chromium.org/
# This is a binary download with wrapper scripts.
[buildout]
......@@ -13,6 +13,10 @@ parts =
chromedriver-wrapper
[chromedriver-wrapper-91]
<= chromedriver-wrapper
wrapper-name = chromedriver-91
part = ${chromedriver-91:location}
[chromedriver-wrapper-2.41]
<= chromedriver-wrapper
......@@ -46,7 +50,7 @@ install =
[chromedriver]
<= chromedriver-2.41
<= chromedriver-91
[chromedriver-2.41]
<= chromedriver-download
......@@ -54,6 +58,11 @@ version = 2.41
# Supports Chrome v67-69
md5sum-x86_64 = fbd8b9561575054e0e7e9cc53b680a70
[chromedriver-91]
<= chromedriver-download
version = 91.0.4472.101
# Supports Chrome v91
md5sum-x86_64 = cc43ba0babbfff7f22b48165ec8e8c81
[chromedriver-download]
# Installs chromedriver ${version}.
......
component/chromium/buildout.cfg
View file @ 5081f0b0
......@@ -66,16 +66,27 @@ install =
))
os.fchmod(f.fileno(), 0o755)
[chromium-wrapper-91]
<= chromium-wrapper
wrapper-name = chromium-91
part = ${chromium-91:location}
[chromium-wrapper-69]
<= chromium-wrapper
wrapper-name = chromium-69
part = ${chromium-69:location}
[chromium]
<= chromium-91
[chromium-91]
<= chromium-download
version = 91.0.4472.114
[chromium]
<= chromium-69
revision_x86-64 = 870763
md5sum-x86_64 = 74eab41580469c2b8117cf396db823cb
generation-x86_64 = 1617926496067901
[chromium-69]
......@@ -111,6 +122,7 @@ x86-64 = https://www.googleapis.com/download/storage/v1/b/chromium-browser-snaps
library =
${atk:location}/lib
${at-spi2-atk:location}/lib
${at-spi2-core:location}/lib
${alsa:location}/lib
${cairo:location}/lib
${cups:location}/lib
......@@ -133,11 +145,14 @@ library =
${libXtst:location}/lib
${libXScrnSaver:location}/lib
${libXrandr:location}/lib
${libdrm:location}/lib
${libexpat:location}/lib
${libffi:location}/lib
${libpng:location}/lib
${libpng12:location}/lib
${libxcb:location}/lib
${libxkbcommon:location}/lib
${libxshmfence:location}/lib
${libxml2:location}/lib
${mesa:location}/lib
${nspr:location}/lib
......
component/firefox/buildout.cfg
View file @ 5081f0b0
......@@ -51,6 +51,11 @@ install =
))
os.fchmod(f.fileno(), 0o755)
[firefox-wrapper-78]
<= firefox-wrapper
wrapper-name = firefox-78
part = ${firefox-78:location}
[firefox-wrapper-68]
<= firefox-wrapper
wrapper-name = firefox-68
......@@ -95,6 +100,12 @@ cache-dir =
# would not be created.
<= firefox-68
[firefox-78]
<= firefox-download
version = 78.1.0esr
i686-md5sum = 09595a1b9a99d17a618a51bc1f971e5e
x86_64-md5sum = 06f4d488721ce7229d9a86cb4c6786f3
[firefox-68]
<= firefox-download
version = 68.0.2esr
......
component/mariadb/mdev20693-mixin.cfg
View file @ 5081f0b0
# To be extended after mariadb's buildout.cfg
[mariadb]
[mariadb-10.4]
patches +=
${:_profile_base_location_}/mdev20693.patch#34ca907d6b36ba81d75bed118243f637
component/mesa/buildout.cfg
View file @ 5081f0b0
......@@ -9,6 +9,7 @@ extends =
../pkgconfig/buildout.cfg
../xorg/buildout.cfg
../xz-utils/buildout.cfg
../zlib/buildout.cfg
parts =
mesa
......@@ -16,32 +17,33 @@ parts =
[mesa]
recipe = slapos.recipe.cmmi
shared = true
url = ftp://ftp.freedesktop.org/pub/mesa/11.0.3/mesa-11.0.3.tar.xz
md5sum = bf9118bf0fbf360715cfe60baf7a1db5
url = https://archive.mesa3d.org/mesa-18.0.0.tar.xz
md5sum = c2a59fc5b56de3e197fa3a6023409e23
configure-options =
--enable-static
--disable-gles1
--disable-gles2
--disable-dri
--enable-dri
--disable-dri3
--enable-egl
--disable-gbm
--enable-gbm
--enable-sysfs
--disable-xvmc
--disable-vdpau
--disable-va
--enable-xlib-glx
--disable-shared-glapi
--disable-xlib-glx
--enable-shared-glapi
--disable-driglx-direct
--disable-gallium-llvm
--with-gallium-drivers=
--with-dri-drivers=
environment =
PATH=${autoconf:location}/bin:${bison:location}/bin:${flex:location}/bin:${pkgconfig:location}/bin:${xz-utils:location}/bin:%(PATH)s
PKG_CONFIG_PATH=${damageproto:location}/lib/pkgconfig:${fixesproto:location}/lib/pkgconfig:${glproto:location}/lib/pkgconfig:${kbproto:location}/lib/pkgconfig:${libX11:location}/lib/pkgconfig:${libXau:location}/lib/pkgconfig:${libXext:location}/lib/pkgconfig:${libexpat:location}/lib/pkgconfig:${libxcb:location}/lib/pkgconfig:${xdamage:location}/lib/pkgconfig:${xextproto:location}/lib/pkgconfig:${xfixes:location}/lib/pkgconfig:${xorg-libpthread-stubs:location}/lib/pkgconfig:${xproto:location}/lib/pkgconfig
PKG_CONFIG_PATH=${damageproto:location}/lib/pkgconfig:${fixesproto:location}/lib/pkgconfig:${glproto:location}/lib/pkgconfig:${kbproto:location}/lib/pkgconfig:${libX11:location}/lib/pkgconfig:${libXau:location}/lib/pkgconfig:${libXext:location}/lib/pkgconfig:${libexpat:location}/lib/pkgconfig:${libxcb:location}/lib/pkgconfig:${xdamage:location}/lib/pkgconfig:${xextproto:location}/lib/pkgconfig:${xfixes:location}/lib/pkgconfig:${xorg-libpthread-stubs:location}/lib/pkgconfig:${xproto:location}/lib/pkgconfig:${libdrm:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig
PYTHON2=${buildout:executable}
ACLOCAL=${automake:location}/bin/aclocal -I${libtool:location}/share/aclocal -I${pkgconfig:location}/share/aclocal
AUTOCONF=${autoconf:location}/bin/autoconf
AUTOMAKE=${automake:location}/bin/automake
LDFLAGS=-Wl,-rpath=${libdrm:location}/lib -Wl,-rpath=${zlib:location}/lib
make-options =
-j1
LIBTOOL=${libtool:location}/bin/libtool
......@@ -57,3 +59,4 @@ configure-options =
environment =
PATH=${pkgconfig:location}/bin:%(PATH)s
PKG_CONFIG_PATH=${damageproto:location}/lib/pkgconfig:${fixesproto:location}/lib/pkgconfig:${glproto:location}/lib/pkgconfig:${kbproto:location}/lib/pkgconfig:${libX11:location}/lib/pkgconfig:${libXau:location}/lib/pkgconfig:${libXext:location}/lib/pkgconfig:${libexpat:location}/lib/pkgconfig:${libxcb:location}/lib/pkgconfig:${mesa:location}/lib/pkgconfig:${xdamage:location}/lib/pkgconfig:${xextproto:location}/lib/pkgconfig:${xfixes:location}/lib/pkgconfig:${xorg-libpthread-stubs:location}/lib/pkgconfig:${xproto:location}/lib/pkgconfig
LDFLAGS=-Wl,-rpath=${libdrm:location}/lib -Wl,-rpath=${zlib:location}/lib
component/tempstorage/buildout.cfg 0 → 100644
View file @ 5081f0b0
# SlapOS component for tempstorage.
# https://github.com/zopefoundation/tempstorage
[buildout]
extends =
../ZODB/buildout.cfg
../git/buildout.cfg
# tempstorage provides tempstorage<X> depending on ZODB major version.
#
# - tempstorage >= 4 works only with ZODB5 because ZODB commit protocol was changed
# https://github.com/zopefoundation/tempstorage/commit/5cc223ea
# - tempstorage <= 5.2 is vulnerable to data corruption in loadBefore
# https://github.com/zopefoundation/tempstorage/issues/8
# https://github.com/zopefoundation/tempstorage/pull/16
# - tempstorage 3-nxd provides loadBefore backports to tempstorage 3.
[tempstorage]
recipe = slapos.recipe.build
depends = ${ZODB:egg}
init =
# link/depend to tempstorage<ZODB.major>
zodb = self.buildout['ZODB']
zmajor = zodb['major']
tempstorage_x = self.buildout['tempstorage'+zmajor]
options['depends'] += '$${%s:egg}' % tempstorage_x.name
options['egg'] = tempstorage_x['egg']
# update [versions] from what is needed by tempstorage<X>
self.buildout.parse('[_tempstorage-versions]\n' + tempstorage_x['egg-versions'])
versions = self.buildout['versions']
versions.update(self.buildout['_tempstorage-versions'])
# propagate updated [versions] -> easy_install
# (buildout does this in Buildout constructor)
import zc.buildout.easy_install
zc.buildout.easy_install.default_versions(versions)
# tempstorage5 is plain upstream egg
[tempstorage5]
recipe = zc.recipe.egg:eggs
egg = tempstorage
eggs = ${:egg}
egg-versions =
tempstorage = 5.2
# tempstorage4-wc2 is tempstorage 3 + backports for loadBefore fixes
[tempstorage4-wc2]
recipe = zc.recipe.egg:develop
setup = ${tempstorage4-wc2-repository:location}
egg = tempstorage
egg-versions =
tempstorage =
[tempstorage4-wc2-repository]
recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.com/nexedi/tempstorage.git
branch = 3-nxd
revision= 77b49295db78
location = ${buildout:parts-directory}/tempstorage
git-executable = ${git:location}/bin/git
# tempstorage4 is the same as tempstorage4-wc2 because of loadBefore fixes.
[tempstorage4]
<= tempstorage4-wc2
component/tesseract/buildout.hash.cfg
View file @ 5081f0b0
[download-tessdata.py]
filename = download-tessdata.py
md5sum = 2d283a6d8662d6bb8c9de7b26162b702
md5sum = 1f6cf2797f0daf9dec16c1d3a8c44907
component/tesseract/download-tessdata.py
View file @ 5081f0b0
......@@ -19,9 +19,13 @@ def post_make_hook(options, buildout, env):
for url in options['tessdata-urls'].splitlines():
url, _, md5sum = url.partition('#')
if url:
destination = os.path.join(
options['tessdata-location'],
os.path.basename(url),
)
download(
url,
md5sum=md5sum,
path=os.path.join(options['tessdata-location'],
os.path.basename(url)),
path=destination,
)
os.chmod(destination, 0o750)
component/wendelin.core/buildout.cfg
View file @ 5081f0b0
......@@ -38,6 +38,12 @@ GO = ${go:exe}
# wcfs needs this:
[gowork]
cpkgpath += ${zlib:location}/lib/pkgconfig
[wendelin.core-env]
# `pkg-config --libs zlib` emits only -L, but not -Wl,-rpath
# better set it via gowork:environment when "VAR += ..." support is there
# environment +=
# CGO_LDFLAGS += -Wl,-rpath=${zlib:location}/lib
CGO_LDFLAGS += -Wl,-rpath=${zlib:location}/lib
[wendelin.core-repository]
......
component/xorg/buildout.cfg
View file @ 5081f0b0
......@@ -2,20 +2,25 @@
extends =
../autoconf/buildout.cfg
../automake/buildout.cfg
../bison/buildout.cfg
../bzip2/buildout.cfg
../dash/buildout.cfg
../freetype/buildout.cfg
../gnutls/buildout.cfg
../icu/buildout.cfg
../intltool/buildout.cfg
../libtool/buildout.cfg
../libuuid/buildout.cfg
../libxml2/buildout.cfg
../libxslt/buildout.cfg
../meson/buildout.cfg
../ninja/buildout.cfg
../openssl/buildout.cfg
../patch/buildout.cfg
../perl/buildout.cfg
../perl-XML-Parser/buildout.cfg
../pkgconfig/buildout.cfg
../xz-utils/buildout.cfg
../zlib/buildout.cfg
./buildout.hash.cfg
......@@ -652,3 +657,43 @@ pkg_config_depends = ${libX11:location}/lib/pkgconfig:${xorgproto:pkg_config_dep
environment =
PKG_CONFIG_PATH=${:pkg_config_depends}
PATH=${pkgconfig:location}/bin:%(PATH)s
[libdrm]
recipe = slapos.recipe.cmmi
shared = true
configure-command = ${meson:location}/bin/meson builddir --libdir=lib -Dprefix=@@LOCATION@@
make-binary = ${ninja:location}/bin/ninja -C builddir
url = https://dri.freedesktop.org/libdrm/libdrm-2.4.106.tar.xz
md5sum = 4e316ae1966a1a63c31a3885313a8fb8
pkg_config_depends = ${pciaccess:location}/lib/pkgconfig
environment =
PKG_CONFIG_PATH=${:pkg_config_depends}
PATH=${xz-utils:location}/bin:${pkgconfig:location}/bin:${ninja:location}/bin:%(PATH)s
LDFLAGS=-Wl,-rpath=${pciaccess:location}/lib -Wl,-rpath=@@LOCATION@@/lib
[libxshmfence]
recipe = slapos.recipe.cmmi
shared = true
url = https://www.x.org/releases/individual/lib/libxshmfence-1.3.tar.gz
md5sum = ab3940af0bd3d3cc91eb35ecd33a779a
pkg_config_depends = ${xorgproto:pkg_config_depends}:${xorgproto:location}/share/pkgconfig
environment =
PKG_CONFIG_PATH=${:pkg_config_depends}
PATH=${pkgconfig:location}/bin:%(PATH)s
[libxkbcommon]
recipe = slapos.recipe.cmmi
shared = true
configure-command = ${meson:location}/bin/meson builddir --libdir=lib -Dprefix=@@LOCATION@@ -Denable-wayland=false -Denable-docs=false
make-binary = ${ninja:location}/bin/ninja -C builddir
url = https://xkbcommon.org/download/libxkbcommon-1.3.0.tar.xz
md5sum = 00b5275ec1309a1d427a645de5861605
pkg_config_depends = ${libxcb:pkg_config_depends}:${libxcb:location}/lib/pkgconfig:${libxml2:location}/lib/pkgconfig:${xz-utils:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig:${icu4c:location}/lib/pkgconfig
environment =
PKG_CONFIG_PATH=${:pkg_config_depends}
PATH=${xz-utils:location}/bin:${pkgconfig:location}/bin:${ninja:location}/bin:${bison:location}/bin:%(PATH)s
LDFLAGS=-Wl,-rpath=${libxcb:location}/lib -Wl,-rpath=${libxml2:location}/lib -Wl,-rpath=${xz-utils:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${icu4c:location}/lib -Wl,-rpath=@@LOCATION@@/lib
slapos/recipe/switch_softwaretype.py
View file @ 5081f0b0
......@@ -25,9 +25,8 @@
#
##############################################################################
from zc.buildout.buildout import Buildout
from zc.buildout.buildout import Buildout, MissingOption, MissingSection
from zc.buildout import UserError
class SubBuildout(Buildout):
"""Run buildout in buildout, partially copied from infrae.buildout
......@@ -71,8 +70,21 @@ class Recipe:
self.buildout = buildout
self.options = options
self.name = name
self.software_type = buildout["slap-configuration"]["slap-software-type"]
section, key = self.options[self.software_type].split(":")
try:
self.software_type = buildout["slap-configuration"]["slap-software-type"]
except (MissingSection, MissingOption):
raise UserError("The section to retrieve slap partition parameters "
"(with slapos.cookbook:slapconfiguration recipe or a derived one) "
"must be named [slap-configuration].")
try:
section, key = self.options[self.software_type].split(":")
except MissingOption:
raise MissingOption("This software type (%s) isn't mapped. RootSoftwareInstance "
"is the default software type." % self.software_type)
except ValueError:
raise UserError("The software types in the section [%s] must be separated "
"by a colon such as: 'section:key', where key is usually 'rendered'. "
"Don't use: ${section:key}" % self.name)
self.base = self.buildout[section][key]
def install(self):
......
software/backupserver/buildout.hash.cfg
View file @ 5081f0b0
......@@ -46,8 +46,8 @@ md5sum = ae4a0043414336a521b524d9c95f1c68
[template-pullrdiffbackup]
filename = instance-pullrdiffbackup.cfg.in
md5sum = c1f00c251298c2ab7fd095e7c4571d3b
md5sum = f2e6f30a0e8228cbfb93eaaae10fe884
[template]
filename = instance.cfg.in
md5sum = 42021b325159dff29e4bd4e33b8ff2f3
md5sum = 3df515def97f1e8a9f181514ae6ef03f
software/backupserver/instance-pullrdiffbackup.cfg.in
View file @ 5081f0b0
......@@ -194,9 +194,9 @@ virtual-depends =
[nginx-listen-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = nginx_listen.py
config-hostname = $${nginx-configuration:ip}
config-host = $${nginx-configuration:ip}
config-port = $${nginx-configuration:port}
[nginx-configuration]
......@@ -206,7 +206,7 @@ output = $${directory:etc}/nginx.cfg
mode = 0600
access_log = $${directory:log}/nginx-access.log
error_log = $${directory:log}/nginx-error.log
ip = $${slap-network-information:global-ipv6}
ip = {{ partition_ipv6 }}
port = 9443
ssl_key = $${directory:ssl}/nginx.key
ssl_csr = $${directory:ssl}/nginx.csr
......
software/backupserver/instance.cfg.in
View file @ 5081f0b0
......@@ -13,32 +13,21 @@ rendered = $${buildout:parts-directory}/$${:_buildout_section_name_}/$${:filenam
filename = instance-pullrdiffbackup.cfg
extensions = jinja2.ext.do
context =
key slave_instance_list instance-parameter:slave-instance-list
key slave_instance_list slap-configuration:slave-instance-list
# partition_ipv6 is the random ipv6 allocated to the local partition
key partition_ipv6 slap-configuration:ipv6-random
[switch-softwaretype]
recipe = slapos.cookbook:softwaretype
default = $${:pullrdiffbackup}
# pullrdiffbackup = ${template-pullrdiffbackup:output}
pullrdiffbackup = $${dynamic-template-pullrdiffbackup:rendered}
recipe = slapos.cookbook:switch-softwaretype
RootSoftwareInstance = $${:pullrdiffbackup}
pullrdiffbackup = dynamic-template-pullrdiffbackup:rendered
[slap-connection]
# part to migrate to new - separated words
computer-id = $${slap_connection:computer_id}
partition-id = $${slap_connection:partition_id}
server-url = $${slap_connection:server_url}
software-release-url = $${slap_connection:software_release_url}
key-file = $${slap_connection:key_file}
cert-file = $${slap_connection:cert_file}
# [slap-parameter]
# slave-instance-list = []
[instance-parameter]
[slap-configuration]
# Fetches parameters defined in SlapOS Master for this instance.
# Always the same.
recipe = slapos.cookbook:slapconfiguration.serialised
computer = $${slap_connection:computer_id}
partition = $${slap_connection:partition_id}
url = $${slap_connection:server_url}
key = $${slap_connection:key_file}
cert = $${slap_connection:cert_file}
computer = $${slap-connection:computer-id}
partition = $${slap-connection:partition-id}
url = $${slap-connection:server-url}
key = $${slap-connection:key-file}
cert = $${slap-connection:cert-file}
software/backupserver/software.cfg
View file @ 5081f0b0
......@@ -102,5 +102,3 @@ mode = 0644
[versions]
gunicorn = 19.1.1
plone.recipe.command = 1.1
PyRSS2Gen = 1.1
software/caddy-frontend/buildout.hash.cfg
View file @ 5081f0b0
......@@ -22,15 +22,15 @@ md5sum = 5784bea3bd608913769ff9a8afcccb68
[profile-caddy-frontend]
filename = instance-apache-frontend.cfg.in
md5sum = 28220d18308313d49a38d39c61a7e769
md5sum = 8507a2ace2f789b92c522cc62ca5aace
[profile-caddy-replicate]
filename = instance-apache-replicate.cfg.in
md5sum = ec421617d4932e8e1ae1e55a41fbfaab
md5sum = 8beb438d06bbb0f917d13e182fb12d17
[profile-slave-list]
_update_hash_filename_ = templates/apache-custom-slave-list.cfg.in
md5sum = a745fb8d61a7e2646e3aa55edf73e5a6
md5sum = 613f777a08373088cbaf7f51fd18ea70
[profile-replicate-publish-slave-information]
_update_hash_filename_ = templates/replicate-publish-slave-information.cfg.in
......@@ -106,7 +106,7 @@ md5sum = 38792c2dceae38ab411592ec36fff6a8
[profile-kedifa]
filename = instance-kedifa.cfg.in
md5sum = 225ce18b9218de4169327a206051a92e
md5sum = 16901e9eeb0d4f87e708ad91e7756f12
[template-backend-haproxy-rsyslogd-conf]
_update_hash_filename_ = templates/backend-haproxy-rsyslogd.conf.in
......
software/caddy-frontend/instance-apache-frontend.cfg.in
View file @ 5081f0b0
......@@ -535,9 +535,9 @@ context =
[trafficserver-promise-listen-port]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = trafficserver-port-listening.py
config-hostname = ${trafficserver-variable:local-ip}
config-host = ${trafficserver-variable:local-ip}
config-port = ${trafficserver-variable:input-port}
[trafficserver-ctl]
......@@ -655,44 +655,44 @@ config-verification-script = ${promise-helper-last-configuration-state:rendered}
[promise-caddy-frontend-v4-https]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = caddy_frontend_ipv4_https.py
config-hostname = {{ instance_parameter_dict['ipv4-random'] }}
config-host = {{ instance_parameter_dict['ipv4-random'] }}
config-port = ${configuration:port}
[promise-caddy-frontend-v4-http]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = caddy_frontend_ipv4_http.py
config-hostname = {{ instance_parameter_dict['ipv4-random'] }}
config-host = {{ instance_parameter_dict['ipv4-random'] }}
config-port = ${configuration:plain_http_port}
[promise-caddy-frontend-v6-https]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = caddy_frontend_ipv6_https.py
config-hostname = {{ instance_parameter_dict['ipv6-random'] }}
config-host = {{ instance_parameter_dict['ipv6-random'] }}
config-port = ${configuration:port}
[promise-caddy-frontend-v6-http]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = caddy_frontend_ipv6_http.py
config-hostname = {{ instance_parameter_dict['ipv6-random'] }}
config-host = {{ instance_parameter_dict['ipv6-random'] }}
config-port = ${configuration:plain_http_port}
[promise-backend-haproxy-http]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = backend_haproxy_http.py
config-hostname = {{ instance_parameter_dict['ipv4-random'] }}
config-host = {{ instance_parameter_dict['ipv4-random'] }}
config-port = ${backend-haproxy-configuration:http-port}
[promise-backend-haproxy-https]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = backend_haproxy_https.py
config-hostname = {{ instance_parameter_dict['ipv4-random'] }}
config-host = {{ instance_parameter_dict['ipv4-random'] }}
config-port = ${backend-haproxy-configuration:https-port}
[backend-haproxy-configuration]
......@@ -981,9 +981,9 @@ context =
[promise-slave-introspection-https]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = slave_introspection_https.py
config-hostname = {{ instance_parameter_dict['ipv6-random'] }}
config-host = {{ instance_parameter_dict['ipv6-random'] }}
config-port = ${frontend-configuration:slave-introspection-https-port}
[logrotate-entry-slave-introspection]
......
software/caddy-frontend/instance-apache-replicate.cfg.in
View file @ 5081f0b0
......@@ -865,14 +865,14 @@ rendered = ${directory:etc}/nginx-rejected-slave.conf
[promise-rejected-slave-publish-ip-port]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = rejected-slave-publish-ip-port-listening.py
config-hostname = ${rejected-slave-publish-configuration:ip}
config-host = ${rejected-slave-publish-configuration:ip}
config-port = ${rejected-slave-publish-configuration:port}
[rejected-slave-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
module = check_file_state
name = rejected-slave.py
config-filename = ${rejected-slave-json:rendered}
......
software/caddy-frontend/instance-kedifa.cfg.in
View file @ 5081f0b0
......@@ -193,9 +193,9 @@ template = inline:
[promise-expose-csr_id-ip-port]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = expose-csr_id-ip-port-listening.py
config-hostname = ${expose-csr_id-configuration:ip}
config-host = ${expose-csr_id-configuration:ip}
config-port = ${expose-csr_id-configuration:port}
[expose-csr_id]
......
software/caddy-frontend/templates/apache-custom-slave-list.cfg.in
View file @ 5081f0b0
......@@ -633,9 +633,9 @@ template = inline:
[promise-expose-csr_id-ip-port]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = expose-csr_id-ip-port-listening.py
config-hostname = ${expose-csr_id-configuration:ip}
config-host = ${expose-csr_id-configuration:ip}
config-port = ${expose-csr_id-configuration:port}
[expose-csr_id]
......
software/caddy-frontend/test/test.py
View file @ 5081f0b0
......@@ -79,6 +79,8 @@ else:
os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
# XXX Keep using slapos node instance --all, because of missing promises
SlapOSInstanceTestCase.slap._force_slapos_node_instance_all = True
# ports chosen to not collide with test systems
HTTP_PORT = '11080'
......
software/erp5/instance-erp5-input-schema.json
View file @ 5081f0b0
......@@ -78,7 +78,7 @@
"type": "string"
},
"developer-list": {
"description": "List of logins which should get the Developper role (required to modify portal_components' content), defaulting to inituser-login's value",
"description": "List of logins which should get the Developer role (required to modify portal_components' content), defaulting to inituser-login's value",
"items": {
"pattern": "^\\S+$",
"type": "string"
......@@ -215,7 +215,7 @@
"type": "integer"
},
"timerserver-interval": {
"description": "Timerserver tick perdiod, in seconds, or 0 to disable",
"description": "Timerserver tick period, in seconds, or 0 to disable",
"default": 5,
"type": "number"
},
......@@ -439,7 +439,7 @@
"type": "string"
},
{
"description": "Override value (parameter for maching nodes).",
"description": "Override value (parameter for matching nodes).",
"type": [
"integer",
"string"
......@@ -517,7 +517,7 @@
"type": "boolean"
},
"node-count": {
"description": "Number of tests this instance can execute in parrallel. This must be at least equal to the number of nodes configured on testnode running the test",
"description": "Number of tests this instance can execute in parallel. This must be at least equal to the number of nodes configured on testnode running the test",
"default": 3,
"type": "integer"
},
......@@ -525,6 +525,101 @@
"description": "Number of extra databases this instance tests will need.",
"default": 3,
"type": "integer"
},
"selenium": {
"default": {
"target": "firefox"
},
"examples": [
{
"target": "selenium-server",
"server-url": "https://selenium.example.com",
"desired-capabilities": {
"browserName": "firefox",
"version": "68.0.2esr",
"acceptInsecureCerts": true
}
},
{
"target": "selenium-server",
"server-url": "https://selenium.example.com",
"desired-capabilities": {
"browserName": "chrome",
"version": "91.0.4472.101"
}
}
],
"oneOf": [
{
"type": "object",
"title": "Selenium Server",
"description": "Configuration for Selenium server",
"additionalProperties": false,
"required": [
"desired-capabilities",
"server-url",
"target"
],
"properties": {
"target": {
"description": "Target system",
"type": "string",
"const": "selenium-server"
},
"server-url": {
"description": "URL of the selenium server",
"type": "string",
"format": "uri"
},
"verify-server-certificate": {
"description": "Verify the SSL/TLS certificate of the selenium server when using HTTPS",
"type": "boolean",
"default": true
},
"server-ca-certificate": {
"description": "PEM encoded bundle of CA certificates to verify the SSL/TLS certificate of the selenium server when using HTTPS",
"type": "string",
"default": "Root certificates from http://certifi.io/en/latest/"
},
"desired-capabilities": {
"description": "Desired browser capabilities",
"required": [
"browserName"
],
"type": "object",
"properties": {
"browserName": {
"description": "Name of the browser being used",
"type": "string",
"examples": [
"firefox",
"chrome",
"safari"
]
},
"version": {
"description": "The browser version",
"type": "string"
}
}
}
}
},
{
"type": "object",
"title": "Firefox",
"description": "Configuration for using firefox running as a sub-process",
"additionalProperties": false,
"properties": {
"target": {
"description": "Target system",
"const": "firefox",
"type": "string",
"default": "firefox"
}
}
}
]
}
},
"type": "object"
......
software/erp5/test/test/test_erp5.py
View file @ 5081f0b0
......@@ -218,6 +218,45 @@ class TestBalancerPorts(ERP5InstanceTestCase):
]))
class TestSeleniumTestRunner(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
"""Test ERP5 can be instantiated with selenium server for test runner.
"""
__partition_reference__ = 'sel'
@classmethod
def getInstanceParameterDict(cls):
return {
'_':
json.dumps({
'test-runner': {
'selenium': {
"target": "selenium-server",
"server-url": "https://example.com",
"verify-server-certificate": False,
"desired-capabilities": {
"browserName": "firefox",
"version": "68.0.2esr",
}
}
}
})
}
def test_test_runner_configuration_json_file(self):
runUnitTest_script, = glob.glob(
self.computer_partition_root_path + "/../*/bin/runUnitTest.real")
config_file = None
with open(runUnitTest_script) as f:
for line in f:
if 'ERP5_TEST_RUNNER_CONFIGURATION' in line:
_, config_file = line.split('=')
assert config_file
with open(config_file.strip()) as f:
self.assertEqual(
f.read(),
json.dumps(json.loads(self.getInstanceParameterDict()['_'])['test-runner']))
class TestDisableTestRunner(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
"""Test ERP5 can be instantiated without test runner.
"""
......
software/erp5testnode/buildout.hash.cfg
View file @ 5081f0b0
......@@ -18,4 +18,4 @@ md5sum = 307663d73ef3ef94b02567ecd322252e
[template-default]
filename = instance-default.cfg
md5sum = d10958c62d0be8d0d051300d695f4f44
md5sum = 24cc143b1886d443a4c29dcb8147a01c
software/erp5testnode/instance-default.cfg
View file @ 5081f0b0
......@@ -243,9 +243,9 @@ instance-promises =
[shellinabox-frontend-listen-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = $${:_buildout_section_name_}.py
config-hostname = $${shellinabox-frontend:hostname}
config-host = $${shellinabox-frontend:hostname}
config-port = $${shellinabox-frontend:port}
[testnode-log-frontend-promise]
......
software/galene/test/README.md 0 → 100644
View file @ 5081f0b0
Tests for Galene software release
software/galene/test/setup.py 0 → 100644
View file @ 5081f0b0
##############################################################################
#
# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from setuptools import setup, find_packages
version = '0.0.1.dev0'
name = 'slapos.test.galene'
long_description = open("README.md").read()
setup(
name=name,
version=version,
description="Test for SlapOS' Galene",
long_description=long_description,
long_description_content_type='text/markdown',
maintainer="Nexedi",
maintainer_email="info@nexedi.com",
url="https://lab.nexedi.com/nexedi/slapos",
packages=find_packages(),
install_requires=[
'slapos.core',
'slapos.libnetworkcache',
'requests',
],
zip_safe=True,
test_suite='test',
)
software/galene/test/test.py 0 → 100644
View file @ 5081f0b0
##############################################################################
#
# Copyright (c) 2019 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from __future__ import unicode_literals
import os
import requests
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
os.path.abspath(
os.path.join(os.path.dirname(__file__), '../software.cfg')))
class TestGalene(SlapOSInstanceTestCase):
__partition_reference__ = 'G'
def setUp(self):
self.connection_parameters = self.computer_partition.getConnectionParameterDict()
def test_url_get(self):
resp = requests.get(self.connection_parameters['url'], verify=False)
self.assertEqual(requests.codes.ok, resp.status_code)
software/gitlab/software.cfg
View file @ 5081f0b0
......@@ -134,7 +134,7 @@ git-executable = ${git:location}/bin/git
<= git-repository
repository = https://lab.nexedi.com/nexedi/gitlab-ce.git
# 9.5.10 + NXD patches:
revision = v9.5.10-8-gc290e22a08cb
revision = v9.5.10-9-g69b0ffae00bf
location = ${buildout:parts-directory}/gitlab
[gitlab-shell-repository]
......
software/helloworld/buildout.hash.cfg
View file @ 5081f0b0
......@@ -14,4 +14,4 @@
# not need these here).
[instance-profile]
filename = instance.cfg.in
md5sum = dcb9d2f540e0e397c9346c8b0c05f233
md5sum = c771dee1ef9aedad7c6ebf9418afe08e
software/helloworld/instance.cfg.in
View file @ 5081f0b0
......@@ -87,7 +87,7 @@ wrapper-path = ${directory:service}/helloweb-${:kind}
# promise, that checks that helloweb service is alive
[helloweb-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = helloweb-${:kind}.py
{# macro to instantiate service of `kind` to listen on `port` #}
......@@ -102,8 +102,8 @@ port = {{ port }}
[helloweb-{{ kind }}-promise]
<= helloweb-promise
kind = {{ kind }}
config-hostname= ${helloweb-{{ kind }}:ipv6}
config-port = {{ port }}
config-host = ${helloweb-{{ kind }}:ipv6}
config-port = {{ port }}
{% endmacro %}
# services instantiation
......
software/html5as-base/instance.cfg.in
View file @ 5081f0b0
......@@ -18,10 +18,19 @@ recipe = slapos.recipe.template:jinja2
template = {{ template_instance_html5as_target }}
rendered = ${buildout:directory}/${:filename}
filename = instance-html5as.cfg
context =
context =
section buildout buildout
section parameter_list profile-common
# partition_ipv6 is the random ipv6 allocated to the local partition
key partition_ipv6 slap-configuration:ipv6-random
[switch-softwaretype]
recipe = slapos.cookbook:softwaretype
default = ${instance-html5as:rendered}
recipe = slapos.cookbook:switch-softwaretype
RootSoftwareInstance = ${:default}
default = instance-html5as:rendered
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration
computer = ${slap-connection:computer-id}
partition = ${slap-connection:partition-id}
url = ${slap-connection:server-url}
software/html5as-base/instance_html5as.cfg.in
View file @ 5081f0b0
......@@ -70,7 +70,7 @@ scgi_temp_path = ${:tmp}/scgi_temp_path
nb_workers = 2
# Network
ip = ${slap-network-information:global-ipv6}
ip = {{ partition_ipv6 }}
port = 8081
access_url = http://[${:ip}]:${:port}
......
software/html5as-base/software.cfg
View file @ 5081f0b0
......@@ -27,7 +27,7 @@ recipe = slapos.recipe.template:jinja2
rendered = ${buildout:directory}/template.cfg
template = ${:_profile_base_location_}/${:filename}
filename = instance.cfg.in
md5sum = 5a6ebc126bcad3cdff1b51fb51f82a35
md5sum = 310aab063704794065ee3bc8f81fdc70
mode = 0644
context =
section buildout buildout
......@@ -43,7 +43,7 @@ context =
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_update_hash_filename_}
_update_hash_filename_ = instance_html5as.cfg.in
md5sum = 4a8c98cc5ca141f78f14fb9cec203cb8
md5sum = 9b7ed68551cac5967915979383238669
mode = 0644
[template_nginx_conf]
......@@ -71,8 +71,3 @@ mode = 0644
recipe = zc.recipe.egg
eggs =
plone.recipe.command
# Pin versions of eggs used that are not already pinned by stack/slapos.cfg
[versions]
slapos.recipe.template = 4.4
plone.recipe.command = 1.1
software/html5as/buildout.hash.cfg
View file @ 5081f0b0
......@@ -17,11 +17,11 @@
[template-cfg]
filename = instance.cfg.in
md5sum = 0a7aceffa5222e88125b72da42ddedd7
md5sum = 9e486efe4ab1aba8cb72b04f6c6da8ad
[instance_html5as]
_update_hash_filename_ = instance_html5as.cfg.in
md5sum = ec808dba866d85f7d37b85d75c13df2b
md5sum = 191ec2a8b967a3944971709365bdcd2d
[template_nginx_conf]
_update_hash_filename_ = templates/nginx_conf.in
......@@ -45,4 +45,4 @@ md5sum = 1c0ee16966e1fcdb3fd11c09f12ee2b2
[template_instance_replicate]
_update_hash_filename_ = instance_replicate.cfg.in
md5sum = 38d1d352307f79c9c99bf2a80a5c76b8
md5sum = 7ff7e11d05145115f53564ec1af205ef
software/html5as/instance.cfg.in
View file @ 5081f0b0
......@@ -9,11 +9,9 @@ offline = true
[profile-common]
nginx_location = {{ nginx_location }}
dash_location = {{ dash_location }}
tar_location = {{ tar_location }}
curl_location = {{ curl_location }}
template_nginx_conf = {{ template_nginx_conf_target }}
template_mime_types = {{ template_mime_types_target }}
template_launcher = {{ template_launcher_target }}
template_launcher = {{ template_launcher_target }}
template_index_html = {{ template_index_html_target }}
template_graceful = {{ template_graceful_target }}
template_monitor = {{ template_monitor }}
......@@ -26,24 +24,43 @@ filename = instance-html5as.cfg
context =
section buildout buildout
section parameter_list profile-common
# partition_ipv6 is the random ipv6 allocated to the local partition
key partition_ipv6 slap-configuration:ipv6-random
# slapparameter_dict: dictionary of all parameters
key slapparameter_dict slap-configuration:configuration
jsonkey default_parameter_dict :default-parameters
default-parameters =
{
"title": "",
"download_url": null,
"port": 8081,
"monitor-httpd-port": 8197
}
[instance-replicate]
recipe = slapos.recipe.template:jinja2
extensions = jinja2.ext.do
template = {{ template_instance_replicate }}
rendered = ${buildout:directory}/${:filename}
filename = instance-replicate-html5as.cfg
context =
section buildout buildout
section parameter_list profile-common
key slapparameter_dict slap-parameters:configuration
key slapparameter_dict slap-configuration:configuration
jsonkey default_parameter_dict :default-parameters
default-parameters =
{
"download_url": null,
"replicate-quantity": 1
}
[switch-softwaretype]
recipe = slapos.cookbook:softwaretype
default = ${instance-html5as:rendered}
replicate = ${instance-replicate:rendered}
recipe = slapos.cookbook:switch-softwaretype
RootSoftwareInstance = ${:default}
default = instance-html5as:rendered
replicate = instance-replicate:rendered
# Section needed to be added manually here to retrieve parameters
[slap-parameters]
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration
computer = ${slap-connection:computer-id}
partition = ${slap-connection:partition-id}
......
software/html5as/instance_html5as.cfg.in
View file @ 5081f0b0
......@@ -3,10 +3,13 @@
# Deploy html5as instance
#
#############################
# parameter_dict: a dictionary with the default parameters from instance.cfg.in
# replaces the values with the parameters of the instance request if there are any
{% set parameter_dict = dict(default_parameter_dict, **slapparameter_dict) %}
[buildout]
parts =
nginx_conf
downloader
mime_types
launcher
nginx-graceful
......@@ -58,7 +61,6 @@ service = ${directory:etc}/service
log = ${directory:var}/log
run = ${directory:var}/run
backup = ${directory:srv}/backup
data = ${directory:srv}/html5as
[tempdirectory]
recipe = slapos.cookbook:mkdirectory
......@@ -76,8 +78,8 @@ scgi_temp_path = ${:tmp}/scgi_temp_path
nb_workers = 2
# Network
ip = ${slap-network-information:global-ipv6}
port = ${slap-parameter:port}
ip = {{ partition_ipv6 }}
port = {{ parameter_dict['port'] }}
access_url = http://[${:ip}]:${:port}
# Paths
......@@ -88,15 +90,13 @@ path_access_log = ${basedirectory:log}/nginx.access.log
path_error_log = ${basedirectory:log}/nginx.error.log
path_tmp = ${tempdirectory:tmp}
# Docroot
docroot = ${basedirectory:data}
default_index = ${basedirectory:data}/index.html
docroot = ${downloader:location}
default_index = ${:docroot}/index.html
# Config files
path_nginx_conf = ${directory:etc}/nginx.conf
path_mime_types = ${directory:etc}/mime_types
# Binaries
path_shell = {{ parameter_list['dash_location'] }}/bin/dash
curl-binary = {{ parameter_list['curl_location'] }}/bin/curl
tar-binary = {{ parameter_list['tar_location'] }}/bin/tar
# Executables
bin_launcher = ${basedirectory:service}/launcher
......@@ -130,28 +130,48 @@ context =
# Command to put content in the docroot
[downloader]
recipe = plone.recipe.command
# This section will fail if the command fails.
stop-on-error = true
recipe = slapos.recipe.build
# Path where the recipe stores any produced file,
# it will be automatically removed at the beginning of "install".
location = ${directory:srv}/html5as
# All the keys in this section will be available as a dict called "self.options"
# We add: or '', otherwise jinja2 will render a 'None' string
url = {{ parameter_dict['download_url'] or '' }}
default_index_html = ${default_index_html:rendered}
# If a tarball is passed as a parameter in download url
# it's content will be served by the instance.
# If the parameter is not provided it fallback to the default template
command =
rm -rf ${html5as:docroot}/*;
URL="${slap-parameter:download_url}";
if [ -n "$URL" ];
then
${html5as:curl-binary} -Lks $URL | ${html5as:tar-binary} xzv -C ${html5as:docroot} --strip-components 1;
else
cp ${default_index_html:rendered} ${html5as:docroot}/;
fi
install =
import os, shutil
buildout_offline = self.buildout['buildout']['offline']
try:
# Allow to do self.download() which can only be used in "online" mode
self.buildout['buildout']['offline'] = 'false'
if self.options['url']:
# Use fonctions from the slapos.recipe.build repository
# Download a file from a URL to a temporary path
file = self.download(self.options['url'])
# Create a directory and extract the file that are compressed inside
extract_dir = self.extract(file)
# Return the right directory path
workdir = guessworkdir(extract_dir)
# Recursively copy directory
self.copyTree(workdir, location)
else:
# Create directory and copy the default template inside
os.makedirs(location)
shutil.copy(self.options['default_index_html'], location)
finally:
# reset the parameter
self.buildout['buildout']['offline'] = buildout_offline
[default_index_html]
recipe = slapos.recipe.template:jinja2
template = {{ parameter_list['template_index_html'] }}
rendered = ${directory:srv}/index.html
title = {{ parameter_dict['title'] }}
context =
key title slap-parameter:title
key title :title
### Nginx Graceful
[nginx-graceful]
......@@ -165,19 +185,19 @@ context =
# Port Listening checking promise
[port-listening-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = nginx-port-listening.py
config-hostname = ${html5as:ip}
config-host = ${html5as:ip}
config-port = ${html5as:port}
# Use a port different from the default one in order to be able to
# use it in a SlapOS webrunner or a Theia SlapOS Runner
[monitor-instance-parameter]
monitor-httpd-port = 8197
monitor-httpd-port = {{ parameter_dict['monitor-httpd-port'] }}
# Monitor Stack also provides logrotate stack. We only need to extend
# the logrotate-entry-base defined in instance-logrotate-base.cfg.in .
# More parameters can be added following the logrotate-entry-base section
# More parameters can be added following the logrotate-entry-base section
[logrotate-entry-nginx]
<= logrotate-entry-base
name = nginx
......@@ -191,7 +211,7 @@ recipe = slapos.cookbook:publish
# be deployed. The parameters needed for accessing monitoring will be published
<= monitor-publish
server_url = ${html5as:access_url}
title = Title ${slap-parameter:title}!
title = Title {{ parameter_dict['title'] }}!
# Add dependency to the promise so that frontend sections are processed
# and there is no need to declare the new part in buildout:parts
server-cdn-url = ${html5as-frontend-promise:url}
......@@ -202,7 +222,7 @@ server-cdn-url = ${html5as-frontend-promise:url}
<= slap-connection
# Recipe used to make requests
recipe = slapos.cookbook:requestoptional
name = HTM5AS frontend
name = HTML5AS frontend
# Specify the software url of the frontend software release
software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
# It is not a dedicated instance but an instance allocated on a shared instance
......@@ -220,8 +240,3 @@ name = html5as-http-frontend.py
url = ${html5as-frontend:connection-secure_access}
config-url = ${:url}
config-check-secure = 1
[slap-parameter]
title =
download_url =
port = 8081
software/html5as/instance_replicate.cfg.in
View file @ 5081f0b0
{% set replicate_quantity = slapparameter_dict.pop('replicate-quantity', '1') | int %}
{%- set parameter_dict = dict(default_parameter_dict, **slapparameter_dict) %}
{%- set replicate_quantity = int(parameter_dict['replicate-quantity']) %}
# Set default title and port for each replicate based on requested quantity
{%- for i in range(1, replicate_quantity + 1) %}
{%- do parameter_dict.setdefault("title-%d" % i, "") %}
{%- do parameter_dict.setdefault("port-%d" % i, 8081 + i) %}
{%- do parameter_dict.setdefault("monitor-httpd-port-%d" % i, 8197 + i) %}
{%- endfor %}
# Standard buildout section
[buildout]
parts =
publish-connection-information
......@@ -7,6 +16,11 @@ eggs-directory = {{ buildout['eggs-directory'] }}
develop-eggs-directory = {{ buildout['develop-eggs-directory'] }}
offline = true
################################
# Sections to Request instances
################################
# Macro section sharing request parameters
[instance-request-base]
<= slap-connection
recipe = slapos.cookbook:request
......@@ -17,29 +31,25 @@ software-type = default
# What parameter are neede to be retrieved
return = server_url server-cdn-url monitor-setup-url
# Provided parameters
config-title = ${slap-parameter:title}
config-download_url = ${slap-parameter:download_url}
# We add: or '', otherwise jinja2 will render a 'None' string
config-download_url = {{ parameter_dict['download_url'] or '' }}
# Create request section in a loop.
{% for i in range(1, replicate_quantity + 1) %}
# Request a normal html5as instance
[instance-{{ i }}]
<= instance-request-base
# Name of the instance
name = instance-html5as-{{ i }}
config-port = ${slap-parameter:port-{{ i }}}
config-title = ${slap-parameter:title-{{ i }}}
{% if "sla-%s-computer-guid" % i in slapparameter_dict -%}
sla-computer_guid = {{ slapparameter_dict["sla-%s-computer-guid" % i] }}
config-port = {{ parameter_dict["port-%s" % i] }}
config-title = {{ parameter_dict["title-%s" % i] }}
config-monitor-httpd-port = {{ parameter_dict["monitor-httpd-port-%s" % i] }}
{% if "sla-%s-computer-guid" % i in parameter_dict -%}
sla-computer_guid = {{ parameter_dict["sla-%s-computer-guid" % i] }}
{% endif -%}
{% endfor %}
[slap-parameter]
download_url =
{% for i in range(1, replicate_quantity + 1) %}
title-{{ i }} =
port-{{ i }} = 808{{ i }}
{% endfor %}
# Publish information to connect to the two instances
[publish-connection-information]
recipe = slapos.cookbook:publish
{% for i in range(1, replicate_quantity + 1) %}
......
software/html5as/software.cfg
View file @ 5081f0b0
......@@ -13,8 +13,6 @@ extends =
# In this example we extend needed components for html5as.
../../component/nginx/buildout.cfg
../../component/dash/buildout.cfg
../../component/tar/buildout.cfg
../../component/curl/buildout.cfg
parts =
# Call installation of slapos.cookbook egg defined in stack/slapos.cfg (needed
......@@ -36,8 +34,6 @@ context =
section buildout buildout
key nginx_location nginx:location
key dash_location dash:location
key curl_location curl:location
key tar_location tar:location
key template_nginx_conf_target template_nginx_conf:target
key template_mime_types_target template_mime_types:target
key template_launcher_target template_launcher:target
......@@ -79,7 +75,3 @@ mode = 0644
[template_instance_replicate]
<= download-base
# Pin versions of eggs used that are not already pinned by stack/slapos.cfg
[versions]
slapos.recipe.template = 4.4
software/htmlvalidatorserver/buildout.hash.cfg
View file @ 5081f0b0
......@@ -22,8 +22,8 @@ md5sum = 87781e6bcb523bb8434888d5f984f36c
[template-validator]
filename = instance-validator.cfg.in
md5sum = 2be286e367e37ce9e504170cd3a08007
md5sum = 9d12472bb2e337d3cc18f2cc6f235425
[template]
filename = instance.cfg.in
md5sum = 2b4d33e9ef1082dd4d6a53f55b391772
md5sum = 94fc13254c819cba33b03f30251bc469
software/htmlvalidatorserver/instance-validator.cfg.in
View file @ 5081f0b0
......@@ -49,15 +49,15 @@ recipe = slapos.recipe.template
url = ${template-tomcat-configuration:output}
output = $${basedirectory:catalina_conf}/server.xml
mode = 0600
ip = $${slap-network-information:global-ipv6}
ip = {{ partition_ipv6 }}
port = 8899
scheme = https
[tomcat-listen-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = tomcat_listen.py
config-hostname = $${tomcat-configuration:ip}
config-host = $${tomcat-configuration:ip}
config-port = $${tomcat-configuration:port}
#################################
......
software/htmlvalidatorserver/instance.cfg.in
View file @ 5081f0b0
......@@ -6,29 +6,26 @@ eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
[switch-softwaretype]
recipe = slapos.cookbook:softwaretype
default = $${:validator}
validator = ${template-validator:output}
[slap-connection]
# part to migrate to new - separated words
computer-id = $${slap_connection:computer_id}
partition-id = $${slap_connection:partition_id}
server-url = $${slap_connection:server_url}
software-release-url = $${slap_connection:software_release_url}
key-file = $${slap_connection:key_file}
cert-file = $${slap_connection:cert_file}
[dynamic-template-validator]
recipe = slapos.recipe.template:jinja2
template = ${template-validator:output}
rendered = $${buildout:parts-directory}/$${:_buildout_section_name_}/$${:filename}
filename = instance-validator.cfg
context =
# partition_ipv6 is the random ipv6 allocated to the local partition
key partition_ipv6 slap-configuration:ipv6-random
# [slap-parameter]
# slave-instance-list = []
[switch-softwaretype]
recipe = slapos.cookbook:switch-softwaretype
RootSoftwareInstance = $${:validator}
validator = dynamic-template-validator:rendered
[instance-parameter]
[slap-configuration]
# Fetches parameters defined in SlapOS Master for this instance.
# Always the same.
recipe = slapos.cookbook:slapconfiguration.serialised
computer = $${slap_connection:computer_id}
partition = $${slap_connection:partition_id}
url = $${slap_connection:server_url}
key = $${slap_connection:key_file}
cert = $${slap_connection:cert_file}
computer = $${slap-connection:computer-id}
partition = $${slap-connection:partition-id}
url = $${slap-connection:server-url}
key = $${slap-connection:key-file}
cert = $${slap-connection:cert-file}
software/htmlvalidatorserver/software.cfg
View file @ 5081f0b0
......@@ -49,5 +49,4 @@ mode = 0644
# 1.3.4nxd2 is invalid version string, thus pached version string is not '1.3.4nxd2+SlapOSPatched001'
# but '1.3.4nxd2-SlapOSPatched001'.
gunicorn = 19.1.1
plone.recipe.command = 1.1
inotifyx = 0.2.2
software/jscrawler/buildout.hash.cfg
View file @ 5081f0b0
......@@ -19,7 +19,7 @@ md5sum = 6c17361a49cfc47564063b867aab6e8c
[template-jscrawler]
filename = instance-jscrawler.cfg.jinja2.in
md5sum = bdf9e67077cd5c3c140974bcc56422ad
md5sum = fece076231740b414612da5ef3a1685a
[template-jscrawler-builder]
filename = template-jscrawler.builder.sh.in
......
software/jscrawler/instance-jscrawler.cfg.jinja2.in
View file @ 5081f0b0
......@@ -60,9 +60,9 @@ log = ${httpd-wrapper:log-file}
[httpd-listen-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = httpd-listen.py
config-hostname = ${httpd-wrapper:host}
config-host = ${httpd-wrapper:host}
config-port = ${httpd-wrapper:port}
[jscrawler-wrapper]
......
software/jstestnode/instance-jstestnode-input-schema.json
View file @ 5081f0b0
......@@ -23,6 +23,31 @@
"example": "https://softinst1234.host.vifib.net/"
},
"test-runner": {
"default": {
"target": "firefox"
},
"examples": [
{
"target": "selenium-server",
"server-url": "https://selenium.example.com",
"desired-capabilities": {
"browserName": "firefox",
"version": "68.0.2esr",
"acceptInsecureCerts": true
}
},
{
"target": "selenium-server",
"server-url": "https://selenium.example.com",
"desired-capabilities": {
"browserName": "chrome",
"version": "91.0.4472.101"
}
},
{
"target": "node"
}
],
"oneOf": [
{
"type": "object",
......@@ -46,14 +71,14 @@
"format": "uri"
},
"verify-server-certificate": {
"description": "Verify the SSL/TLS Certificats of the selenium server when using HTTPS",
"description": "Verify the SSL/TLS certificate of the selenium server when using HTTPS",
"type": "boolean",
"default": true
},
"server-ca-certificate": {
"description": "PEM encoded bundle of CA Certificates to verify the SSL/TLS Certificate of the selenium server when using HTTPS",
"description": "PEM encoded bundle of CA certificates to verify the SSL/TLS certificate of the selenium server when using HTTPS",
"type": "string",
"default": "root certificates from http://certifi.io/en/latest/"
"default": "Root certificates from http://certifi.io/en/latest/"
},
"desired-capabilities": {
"description": "Desired browser capabilities",
......@@ -63,8 +88,13 @@
"type": "object",
"properties": {
"browserName": {
"description": "Name of the browser being used, for example firefox, chrome",
"type": "string"
"description": "Name of the browser being used",
"type": "string",
"examples": [
"firefox",
"chrome",
"safari"
]
},
"version": {
"description": "The browser version",
......@@ -77,7 +107,7 @@
{
"type": "object",
"title": "Firefox",
"description": "Configuration for Firefox",
"description": "Configuration for using firefox running as a sub-process",
"additionalProperties": false,
"properties": {
"target": {
......
software/kvm/buildout.hash.cfg
View file @ 5081f0b0
......@@ -19,7 +19,7 @@ md5sum = 0d34ff81779115bf899f7bc752877b70
[template-kvm]
filename = instance-kvm.cfg.jinja2
md5sum = bf0c01ac7493693bb57ebef00bb20fa0
md5sum = d0f96be4e80b96e6ac33f6d474767b13
[template-kvm-cluster]
filename = instance-kvm-cluster.cfg.jinja2.in
......@@ -47,7 +47,7 @@ md5sum = b617d64de73de1eed518185f310bbc82
[template-nbd]
filename = instance-nbd.cfg.jinja2
md5sum = 6ea26f88252bf899c966d0f5675e7176
md5sum = 259e06f289f68297e0609e4ab1af8e86
[template-ansible-promise]
filename = template/ansible-promise.in
......@@ -55,7 +55,7 @@ md5sum = b7e87479a289f472b634a046b44b5257
[template-kvm-run]
filename = template/template-kvm-run.in
md5sum = be750fb62f7057c97dd6c6887b2149cc
md5sum = a502782244d1be536b732ebb40725f47
[template-kvm-controller]
filename = template/kvm-controller-run.in
......@@ -75,7 +75,7 @@ md5sum = fb330a796fadb6cd5c85217f80a42af3
[template-httpd]
filename = instance-kvm-http.cfg.in
md5sum = d657884d02105deffddee0edae4b50a6
md5sum = f4bcde62e008c2da9c65617ba7f73f08
[image-download-controller]
_update_hash_filename_ = template/image-download-controller.py
......
software/kvm/instance-kvm-cluster-input-schema.json
View file @ 5081f0b0
......@@ -217,7 +217,7 @@
},
"enable-device-hotplug": {
"title": "Enable device hotplug mode",
"description": "If yes, this will allow to Create devices like CPU and Memory in hotplug mode without restart the VM. Operatin System should be configured to Online new created devices.",
"description": "Allows to increase amount of RAM (ram-size) and CPU (cmp-count) without restart of the VM process, up to defined maximums (ram-max-size and cpu-max-count). Operation system have to support online addition of RAM and CPU.",
"type": "boolean",
"default": false
},
......
software/kvm/instance-kvm-http.cfg.in
View file @ 5081f0b0
......@@ -65,7 +65,7 @@ stop-on-error = true
[httpd-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = apache-httpd.py
config-hostname = ${apache-conf:ip}
config-host = ${apache-conf:ip}
config-port = ${apache-conf:port}
software/kvm/instance-kvm-input-schema.json
View file @ 5081f0b0
......@@ -5,7 +5,7 @@
"properties": {
"enable-device-hotplug": {
"title": "Enable device hotplug mode",
"description": "If yes, this will allow to Create devices like CPU and Memory in hotplug mode without restart the VM. Operatin System should be configured to Online new created devices.",
"description": "Allows to increase amount of RAM (ram-size) and CPU (cmp-count) without restart of the VM process, up to defined maximums (ram-max-size and cpu-max-count). Operation system have to support online addition of RAM and CPU.",
"type": "boolean",
"default": false
},
......
software/kvm/instance-kvm.cfg.jinja2
View file @ 5081f0b0
......@@ -437,13 +437,17 @@ disk-path = ${directory:srv}/virtual.${slap-parameter:disk-format}
pid-file-path = ${kvm-controller-parameter-dict:pid-file}
socket-path = ${kvm-controller-parameter-dict:socket-path}
enable-device-hotplug = ${kvm-controller-parameter-dict:enable-device-hotplug}
smp-count = ${kvm-controller-parameter-dict:cpu-count}
{%- set enable_device_hotplug = slapparameter_dict.get('enable-device-hotplug', 'false').lower() == 'true' %}
smp-max-count = {{ cpu_max_count }}
ram-size = ${kvm-controller-parameter-dict:ram-size}
ram-max-size = {{ ram_max_size }}
{%- if enable_device_hotplug %}
init-ram-size = 1024
init-smp-count = 1
{%- else %}
init-ram-size = ${kvm-controller-parameter-dict:ram-size}
init-smp-count = ${kvm-controller-parameter-dict:cpu-count}
{%- endif %}
mac-address = ${create-mac:mac-address}
tap-mac-address = ${create-tap-mac:mac-address}
......@@ -582,9 +586,9 @@ command-line = ${kvm-controller:rendered}
[kvm-vnc-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = vnc_promise.py
config-hostname = ${kvm-parameter-dict:vnc-ip}
config-host = ${kvm-parameter-dict:vnc-ip}
config-port = ${kvm-parameter-dict:vnc-port}
[kvm-disk-image-corruption-bin]
......@@ -704,9 +708,9 @@ wrapper = ${directory:bin}/websockify
[novnc-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = novnc_promise.py
config-hostname = ${novnc-instance:ip}
config-host = ${novnc-instance:ip}
config-port = ${novnc-instance:port}
......@@ -797,9 +801,9 @@ hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
[httpd-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = httpd.py
config-hostname = ${httpd:host}
config-host = ${httpd:host}
config-port = ${httpd:port}
{% endif %}
......
software/kvm/instance-nbd.cfg.jinja2
View file @ 5081f0b0
......@@ -65,9 +65,9 @@ key = ${gen-passwd:passwd}
[onetimeupload-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = onetimeupload_promise.py
config-hostname = ${onetimeupload-instance:ip}
config-host = ${onetimeupload-instance:ip}
config-port = ${onetimeupload-instance:port}
[publish-connection-information]
......
software/kvm/template/template-kvm-run.in
View file @ 5081f0b0
......@@ -37,9 +37,9 @@ mac_address = '{{ parameter_dict.get("mac-address") }}'
tap_mac_address = '{{ parameter_dict.get("tap-mac-address") }}'
tap_ipv6_addr = '{{ parameter_dict.get("tap-ipv6-addr") }}'
numa_list = '{{ parameter_dict.get("numa", "") }}'.split()
ram_size = {{ parameter_dict.get("ram-size") }}
ram_max_size = '{{ parameter_dict.get("ram-max-size") }}'
init_ram_size = {{ parameter_dict.get("init-ram-size") }}
init_smp_count = {{ parameter_dict.get("init-smp-count") }}
pid_file_path = '{{ parameter_dict.get("pid-file-path") }}'
external_disk_number = {{ parameter_dict.get("external-disk-number") }}
external_disk_size = {{ parameter_dict.get("external-disk-size") }}
......@@ -78,7 +78,6 @@ if not disk_info_list:
{%- endfor %}
})
smp_count = {{ parameter_dict.get("smp-count") }}
smp_max_count = {{ parameter_dict.get("smp-max-count") }}
machine_options = '{{ parameter_dict.get("machine-options", "") }}'.strip()
cpu_model = '{{ parameter_dict.get("cpu-model") }}'.strip()
......@@ -267,12 +266,8 @@ if use_tap == 'true':
tap_interface, vhost),
'-device', 'virtio-net-pci,netdev=lan%s,mac=%s' % (number, tap_mac_address)]
if enable_device_hotplug != 'true':
smp = '%s,maxcpus=%s' % (smp_count, smp_max_count)
ram = '%sM,slots=128,maxmem=%sM' % (ram_size, ram_max_size)
else:
smp = '1,maxcpus=%s' % smp_max_count
ram = '%sM,slots=128,maxmem=%sM' % (init_ram_size, ram_max_size)
smp = '%s,maxcpus=%s' % (init_smp_count, smp_max_count)
ram = '%sM,slots=128,maxmem=%sM' % (init_ram_size, ram_max_size)
kvm_argument_list = [qemu_path,
'-enable-kvm', '-smp', smp, '-name', vm_name, '-m', ram, '-vga', 'std',
......
software/kvm/test/setup.py
View file @ 5081f0b0
......@@ -43,6 +43,7 @@ setup(name=name,
install_requires=[
'slapos.core',
'slapos.cookbook',
'slapos.toolbox',
'slapos.libnetworkcache',
'erp5.util',
'supervisor',
......
software/kvm/test/test.py
View file @ 5081f0b0
......@@ -46,6 +46,7 @@ import time
import shutil
import sys
from slapos.qemuqmpclient import QemuQMPWrapper
from slapos.proxy.db_version import DB_VERSION
from slapos.recipe.librecipe import generateHashFromFiles
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
......@@ -60,6 +61,8 @@ if has_kvm:
os.path.abspath(
os.path.join(os.path.dirname(__file__), '..',
'software%s.cfg' % ("-py3" if six.PY3 else ""))))
# XXX Keep using slapos node instance --all, because of missing promises
InstanceTestCase.slap._force_slapos_node_instance_all = True
else:
setUpModule, InstanceTestCase = None, unittest.TestCase
......@@ -203,6 +206,108 @@ i0:whitelist-firewall-{hash} RUNNING""",
)
@skipUnlessKvm
class TestMemoryManagement(InstanceTestCase, KvmMixin):
__partition_reference__ = 'i'
def getKvmProcessInfo(self, switch_list):
return_list = []
with self.slap.instance_supervisor_rpc as instance_supervisor:
kvm_pid = [q for q in instance_supervisor.getAllProcessInfo()
if 'kvm-' in q['name']][0]['pid']
kvm_process = psutil.Process(kvm_pid)
get_next = False
for entry in kvm_process.cmdline():
if get_next:
return_list.append(entry)
get_next = False
elif entry in switch_list:
get_next = True
return kvm_pid, return_list
def test(self):
kvm_pid_1, info_list = self.getKvmProcessInfo(['-smp', '-m'])
self.assertEqual(
['1,maxcpus=2', '1024M,slots=128,maxmem=1536M'],
info_list
)
self.rerequestInstance({
'ram-size': '1536',
'cpu-count': '2',
})
self.slap.waitForInstance(max_retry=10)
kvm_pid_2, info_list = self.getKvmProcessInfo(['-smp', '-m'])
self.assertEqual(
['2,maxcpus=3', '1536M,slots=128,maxmem=2048M'],
info_list
)
# assert that process was restarted
self.assertNotEqual(kvm_pid_1, kvm_pid_2, "Unexpected: KVM not restarted")
def tearDown(self):
self.rerequestInstance({})
self.slap.waitForInstance(max_retry=10)
def test_enable_device_hotplug(self):
def getHotpluggedCpuRamValue():
qemu_wrapper = QemuQMPWrapper(os.path.join(
self.computer_partition_root_path, 'var', 'qmp_socket'))
ram_mb = sum(
[q['size']
for q in qemu_wrapper.getMemoryInfo()['hotplugged']]) / 1024 / 1024
cpu_count = len(
[q['CPU'] for q in qemu_wrapper.getCPUInfo()['hotplugged']])
return {'cpu_count': cpu_count, 'ram_mb': ram_mb}
kvm_pid_1, info_list = self.getKvmProcessInfo(['-smp', '-m'])
self.assertEqual(
['1,maxcpus=2', '1024M,slots=128,maxmem=1536M'],
info_list
)
self.assertEqual(
getHotpluggedCpuRamValue(),
{'cpu_count': 0, 'ram_mb': 0}
)
parameter_dict = {
'enable-device-hotplug': 'true',
# to avoid restarts the max RAM and CPU has to be static
'ram-max-size': '2048',
'cpu-max-count': '4',
}
self.rerequestInstance(parameter_dict)
self.slap.waitForInstance(max_retry=2)
kvm_pid_2, info_list = self.getKvmProcessInfo(['-smp', '-m'])
self.assertEqual(
['1,maxcpus=4', '1024M,slots=128,maxmem=2048M'],
info_list
)
self.assertEqual(
getHotpluggedCpuRamValue(),
{'cpu_count': 0, 'ram_mb': 0}
)
self.assertNotEqual(kvm_pid_1, kvm_pid_2, "Unexpected: KVM not restarted")
parameter_dict.update(**{
'ram-size': '1536',
'cpu-count': '2'
})
self.rerequestInstance(parameter_dict)
self.slap.waitForInstance(max_retry=10)
kvm_pid_3, info_list = self.getKvmProcessInfo(['-smp', '-m'])
self.assertEqual(
['1,maxcpus=4', '1024M,slots=128,maxmem=2048M'],
info_list
)
self.assertEqual(kvm_pid_2, kvm_pid_3, "Unexpected: KVM restarted")
self.assertEqual(
getHotpluggedCpuRamValue(),
{'cpu_count': 1, 'ram_mb': 512}
)
class MonitorAccessMixin(object):
def sqlite3_connect(self):
sqlitedb_file = os.path.join(
......@@ -223,7 +328,7 @@ class MonitorAccessMixin(object):
}
return db.execute(
"SELECT reference, xml, connection_xml, partition_reference,"
" software_release, requested_state, software_type"
" software_release, requested_state, software_type"
" FROM partition%s"
" WHERE slap_state='busy'" % DB_VERSION).fetchall()
finally:
......
software/monitor/test/test.py
View file @ 5081f0b0
......@@ -142,6 +142,23 @@ class EdgeSlaveMixin(MonitorTestMixin):
instance_max_retry = 20
expected_connection_parameter_dict = {}
@classmethod
def setUpClass(cls):
# XXX we run these tests with --all as a workaround for the fact that after
# requesting new shared instances we don't have promise to wait for the
# processing of these shared instances to be completed.
# The sequence is something like this:
# - `requestEdgetestSlaves` will request edgetest partition
# - first `waitForInstance` will process the edgetest partition, which will
# request a edgebot partition, but without promise to wait for the
# processing to be finished, so the first run of `slapos node instance`
# exits with success code and `waitForInstance` return.
# - second `waitForInstance` process the edgebot partition.
# Once we implement a promise (or something similar) here, we should not
# have to use --all
cls.slap._force_slapos_node_instance_all = True
return super(EdgeSlaveMixin, cls).setUpClass()
@classmethod
def getInstanceSoftwareType(cls):
return 'edgetest'
......
software/plantuml/buildout.hash.cfg
View file @ 5081f0b0
......@@ -15,7 +15,7 @@
[instance]
filename = instance.cfg.in
md5sum = d1dff3fc39eefc57b36dbaa195b6890e
md5sum = c962079a88a6ce97d8ce20fa4e8edfd1
[tomcat-server-xml]
filename = server.xml.in
......
software/plantuml/instance.cfg.in
View file @ 5081f0b0
......@@ -87,9 +87,9 @@ instance-promises =
[tomcat-listen-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = $${:_buildout_section_name_}.py
config-hostname= $${tomcat-instance:ip}
config-host = $${tomcat-instance:ip}
config-port = $${tomcat-instance:port}
[publish-connection-parameter]
......
software/powerdns/buildout.hash.cfg
View file @ 5081f0b0
......@@ -18,7 +18,7 @@ md5sum = fddea033e1aa9d6147a1a47bd7cc4b62
[template-powerdns]
filename = instance-powerdns.cfg
md5sum = 0920200cb05a68b1b4a161a927d9488f
md5sum = c04c3b490e7f9f35af3d204a9df51f35
[template-pdns-configuration]
_update_hash_filename_ = template/pdns.conf.jinja2
......@@ -26,7 +26,7 @@ md5sum = 20c37ea06a8fa405bc02470d5115fd11
[template-dns-replicate]
_update_hash_filename_ = instance-powerdns-replicate.cfg.jinja2
md5sum = 504d15f0bbf0e515d5ff16070f1ac802
md5sum = 4ff993a39da03d9d66d7c0f98efeb1e0
[iso-list]
_update_hash_filename_ = template/zz.countries.nexedi.dk.rbldnsd
......
software/powerdns/instance-powerdns-replicate.cfg.jinja2
View file @ 5081f0b0
......@@ -73,9 +73,9 @@ sla-{{ parameter }} = {{ slapparameter_dict.pop( sla_key + parameter ) }}
[{{promise_section_title}}]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = {{promise_section_title}}.py
config-hostname = {{ '${' ~ request_section_title ~ ':connection-powerdns-ipv6}' }}
config-host = {{ '${' ~ request_section_title ~ ':connection-powerdns-ipv6}' }}
config-port = {{ '${' ~ request_section_title ~ ':connection-powerdns-port}' }}
{% do monitor_url_list.append('${' ~ request_section_title ~ ':connection-monitor-base-url}') -%}
......
software/powerdns/instance-powerdns.cfg
View file @ 5081f0b0
......@@ -137,9 +137,9 @@ extra-context =
# Promises
[pdns-promise-listen-port]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = pdns-port-listening.py
config-hostname = $${pdns:ipv4}
config-host = $${pdns:ipv4}
config-port = $${pdns:port}
[publish-connection-informations]
......
software/proftpd/buildout.hash.cfg
View file @ 5081f0b0
......@@ -19,7 +19,7 @@ md5sum = efb4238229681447aa7fe73898dffad4
[instance-default]
filename = instance-default.cfg.in
md5sum = dae19ec06f8da9fa2980a6d2bdf3da54
md5sum = c6dce31a36e4e13de62687e9888aeb77
[proftpd-config-file]
filename = proftpd-config-file.cfg.in
......
software/proftpd/instance-default.cfg.in
View file @ 5081f0b0
......@@ -86,9 +86,9 @@ template = inline:{{ slapparameter_dict['ssh-key'] | indent }}
[proftpd-listen-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = ${:_buildout_section_name_}.py
config-hostname = ${proftpd:ipv6}
config-host = ${proftpd:ipv6}
config-port = ${proftpd:sftp-port}
......
software/pureftpd/buildout.hash.cfg
View file @ 5081f0b0
......@@ -15,4 +15,4 @@
[instance-profile]
filename = instance.cfg.in
md5sum = 134ed1cf1f6de63b14425031eb5c9043
md5sum = 500b773d1a63a6a895f9b8038a582b05
software/pureftpd/instance.cfg.in
View file @ 5081f0b0
......@@ -33,12 +33,12 @@ pureftpd-dir = ${:srv}/pureftpd/
[check-port-listening-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = check_nginx_port.py
[pureftpd-listen-promise]
<= check-port-listening-promise
config-hostname = ${pureftpd:ipv6}
config-host = ${pureftpd:ipv6}
config-port = ${pureftpd:ftp-port}
[pureftpd-userinfo]
......
software/re6stnet/buildout.hash.cfg
View file @ 5081f0b0
......@@ -18,7 +18,7 @@ md5sum = 71531ed9c9b79fa769ab367e7ea2d2a5
[template-re6stnet]
filename = instance-re6stnet.cfg.in
md5sum = 2309889cf3f5bc57ba63d389e662fc21
md5sum = 870c34cf58acaaee21c71182dd3cb0cf
[template-apache-conf]
filename = apache.conf.in
......
software/re6stnet/instance-re6stnet.cfg.in
View file @ 5081f0b0
......@@ -170,16 +170,16 @@ context =
[re6st-registry-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = re6st-registry.py
config-hostname = ${re6st-registry:ipv4}
config-host = ${re6st-registry:ipv4}
config-port = ${re6st-registry:port}
[apache-registry-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = apache-re6st-registry.py
config-hostname = ${apache-conf:ipv6}
config-host = ${apache-conf:ipv6}
config-port = ${apache-conf:port}
[publish]
......
software/repman/buildout.hash.cfg
View file @ 5081f0b0
......@@ -18,7 +18,7 @@ md5sum = 8a08be95a04f1a47098c4fdef80bdfed
[instance-repman.cfg]
_update_hash_filename_ = instance-repman.cfg.jinja2.in
md5sum = 1d6eba3984b3e2009682f6ce49b8ac4d
md5sum = 0c173313b48d0005c46d968db1c8ab5f
[config-toml.in]
_update_hash_filename_ = templates/config.toml.in
......
software/repman/instance-repman.cfg.jinja2.in
View file @ 5081f0b0
......@@ -216,23 +216,23 @@ depends =
[proxysql-{{ name }}-admin-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = proxysql-{{ name }}-admin-port-listening.py
config-hostname= {{ ipv4 }}
config-host = {{ ipv4 }}
config-port = {{ '${' ~ name ~ '-cluster-parameter:proxy-admin-port}' }}
[proxysql-{{ name }}-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = proxysql-{{ name }}-port-listening.py
config-hostname= {{ ipv4 }}
config-host = {{ ipv4 }}
config-port = {{ '${' ~ name ~ '-cluster-parameter:proxy-port}' }}
[proxysql-{{ name }}-ipv6-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = proxysql-{{ name }}-ipv6-port-listening.py
config-hostname= {{ ip }}
config-host = {{ ip }}
config-port = {{ '${' ~ name ~ '-cluster-parameter:proxy-port}' }}
{% set service_name = "proxysql-" ~ name -%}
......@@ -403,16 +403,16 @@ context =
[repman-listen-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = repman_service_listen.py
config-hostname = ${repman-parameter:ipv4}
config-host = ${repman-parameter:ipv4}
config-port = ${repman-parameter:port}
[repman-listen-ssl-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = repman_service_ssl_listen.py
config-hostname = ${repman-parameter:ipv4}
config-host = ${repman-parameter:ipv4}
config-port = ${repman-parameter:secure-port}
[nginx-conf]
......
software/restic-rest-server/buildout.hash.cfg 0 → 100644
View file @ 5081f0b0
# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
# The only allowed lines here are (regexes):
# - "^#" comments, copied verbatim
# - "^[" section beginings, copied verbatim
# - lines containing an "=" sign which must fit in the following categorie.
# - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
# Copied verbatim.
# - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
# by the re-generation script.
# Re-generated.
# - other lines are copied verbatim
# Substitution (${...:...}), extension ([buildout] extends = ...) and
# section inheritance (< = ...) are NOT supported (but you should really
# not need these here).
[instance.cfg.in]
filename = instance.cfg.in
md5sum = fee2097f3d12fd4bcfbc2698ecf49afc
software/restic-rest-server/instance.cfg.in 0 → 100644
View file @ 5081f0b0
{% import "caucase" as caucase with context %}
[buildout]
parts =
promises
publish-connection-parameter
extends =
{{ template_monitor }}
eggs-directory = {{ buildout['eggs-directory'] }}
develop-eggs-directory = {{ buildout['develop-eggs-directory'] }}
offline = true
[instance-parameter]
recipe = slapos.cookbook:slapconfiguration
computer = ${slap-connection:computer-id}
partition = ${slap-connection:partition-id}
url = ${slap-connection:server-url}
key = ${slap-connection:key-file}
cert = ${slap-connection:cert-file}
[slap-configuration]
# frontend reads from from a part named [slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
computer = ${slap-connection:computer-id}
partition = ${slap-connection:partition-id}
url = ${slap-connection:server-url}
key = ${slap-connection:key-file}
cert = ${slap-connection:cert-file}
[directory]
recipe = slapos.cookbook:mkdirectory
home = ${buildout:directory}
etc = ${:home}/etc
var = ${:home}/var
srv = ${:home}/srv
tmp = ${:home}/tmp
var-log = ${:var}/log
service = ${:etc}/service
promise = ${:etc}/promise
rest-server-data-dir = ${:srv}/restic
backup-caucased = ${:srv}/backup/caucased/
# Macros
[check-port-listening-promise]
recipe = slapos.cookbook:check_port_listening
path = ${directory:promise}/${:_buildout_section_name_}
[check-url-available-promise]
recipe = slapos.cookbook:check_url_available
path = ${directory:promise}/${:_buildout_section_name_}
dash_path = {{ dash_bin }}
curl_path = {{ curl_bin }}
# Caucase
[rest-server-certificate]
key-file = ${directory:etc}/${:_buildout_section_name_}.key
cert-file = ${directory:etc}/${:_buildout_section_name_}.crt
common-name = ${:_buildout_section_name_}
ca-file = ${directory:etc}/${:_buildout_section_name_}.ca.crt
crl-file = ${directory:etc}/${:_buildout_section_name_}.crl
{{
caucase.updater(
prefix='rest-server-certificate',
buildout_bin_directory=buildout['bin-directory'],
updater_path='${directory:service}/rest-server-certificate-updater',
url='${caucased:url}',
data_dir='${directory:srv}/caucase-updater',
crt_path='${rest-server-certificate:cert-file}',
ca_path='${rest-server-certificate:ca-file}',
crl_path='${rest-server-certificate:crl-file}',
key_path='${rest-server-certificate:key-file}',
template_csr='${rest-server-certificate-prepare-csr:csr}',
openssl=openssl_bin,
)}}
[rest-server-certificate-csr-config]
recipe = slapos.recipe.template:jinja2
mode = 644
template = inline:
[req]
prompt = no
req_extensions = req_ext
distinguished_name = dn
[ dn ]
CN = restic-rest-server
[ req_ext ]
subjectAltName = @alt_names
[ alt_names ]
IP.1 = ${instance-parameter:ipv4-random}
IP.2 = ${instance-parameter:ipv6-random}
rendered = ${buildout:parts-directory}/${:_buildout_section_name_}/${:_buildout_section_name_}.txt
[rest-server-certificate-prepare-csr]
recipe = plone.recipe.command
command =
if [ ! -f '${:csr}' ] ; then
{{ openssl_bin }} req \
-newkey rsa:2048 \
-batch \
-new \
-nodes \
-keyout /dev/null \
-config '${rest-server-certificate-csr-config:rendered}' \
-out '${:csr}'
fi
stop-on-error = true
csr = ${directory:srv}/${:_buildout_section_name_}.csr.pem
[caucased]
port = 18080
ip = ${instance-parameter:ipv6-random}
netloc = [${:ip}]:${:port}
url = http://${:netloc}/
{{
caucase.caucased(
prefix='caucased',
buildout_bin_directory=buildout['bin-directory'],
caucased_path='${directory:service}/caucased',
backup_dir='${directory:backup-caucased}',
data_dir='${directory:srv}/caucased',
netloc='${caucased:netloc}',
tmp='${directory:tmp}',
service_auto_approve_count=1,
user_auto_approve_count=0,
key_len=2048,
)}}
[rest-server-password]
recipe = slapos.cookbook:generate.password
user = backup
[rest-server-htpassword]
recipe = plone.recipe.command
command =
if [ ! -f '${:htpassword}' ] ; then
{{ htpasswd_bin }} \
-b \
-B \
-c ${:htpassword} \
${rest-server-password:user} \
${rest-server-password:passwd}
fi
htpassword = ${directory:rest-server-data-dir}/.htpasswd
stop-on-error = true
[rest-server]
recipe = slapos.cookbook:wrapper
command-line =
{{ gowork_bin }}/rest-server \
--listen [${instance-parameter:ipv6-random}]:${:port}
--log ${directory:var-log}/${:_buildout_section_name_}-access.log
--path ${directory:rest-server-data-dir}
--tls
--tls-cert ${rest-server-certificate:cert-file}
--tls-key ${rest-server-certificate:key-file}
--prometheus
wrapper-path = ${directory:service}/rest-server
port = 19080
ip = ${instance-parameter:ipv6-random}
url = https://[${:ip}]:${:port}
depends =
${rest-server-htpassword:recipe}
[rest-server-listen-promise]
<= check-port-listening-promise
hostname= ${rest-server:ip}
port = ${rest-server:port}
[frontend]
<= slap-connection
recipe = slapos.cookbook:requestoptional
name = Rest Server Frontend
# XXX We have hardcoded SR URL here.
software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
slave = true
config-url = ${rest-server:url}
return = domain secure_access
[frontend-available-promise]
<= check-url-available-promise
url = ${frontend:connection-secure_access}
check-secure = 1
[promises]
recipe =
instance-promises =
${caucased-promise:recipe}
${rest-server-certificate-promise:recipe}
${rest-server-listen-promise:path}
${frontend-available-promise:path}
[publish-connection-parameter]
recipe = slapos.cookbook:publish
rest-server-user = ${rest-server-password:user}
rest-server-password = ${rest-server-password:passwd}
url = ${frontend:connection-secure_access}
caucase-url = ${caucased:url}
software/restic-rest-server/software.cfg 0 → 100644
View file @ 5081f0b0
[buildout]
extends =
../../component/openssl/buildout.cfg
../../component/curl/buildout.cfg
../../component/dash/buildout.cfg
../../component/golang/buildout.cfg
../../component/restic/buildout.cfg
../../stack/caucase/buildout.cfg
../../stack/slapos.cfg
buildout.hash.cfg
parts =
slapos-cookbook
caucase-eggs
instance.cfg.in
restic
gowork
[python]
part = python3
[gowork]
install +=
${git.github.com_restic_rest-server:location}:./cmd/...
[git.github.com_restic_rest-server]
<= go-git-package
go.importpath = github.com/restic/rest-server
repository = https://github.com/restic/rest-server
revision = v0.10.0-0-g9313f19
[instance.cfg.in]
recipe = slapos.recipe.template:jinja2
rendered = ${buildout:directory}/instance.cfg
template = ${:_profile_base_location_}/${:filename}
mode = 0644
context =
section buildout buildout
key gowork_bin gowork:bin
raw openssl_bin ${openssl:location}/bin/openssl
raw htpasswd_bin ${apache:location}/bin/htpasswd
raw dash_bin ${dash:location}/bin/dash
raw curl_bin ${curl:location}/bin/curl
key template_monitor monitor2-template:rendered
import-list =
file caucase caucase-jinja2-library:target
software/restic-rest-server/test/README.md 0 → 100644
View file @ 5081f0b0
Tests for restic ReST Server software release
software/restic-rest-server/test/setup.py 0 → 100644
View file @ 5081f0b0
##############################################################################
#
# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from setuptools import setup, find_packages
version = '0.0.1.dev0'
name = 'slapos.test.restic_rest_server'
long_description = open("README.md").read()
setup(
name=name,
version=version,
description="Test for SlapOS' restic Rest Server",
long_description=long_description,
long_description_content_type='text/markdown',
maintainer="Nexedi",
maintainer_email="info@nexedi.com",
url="https://lab.nexedi.com/nexedi/slapos",
packages=find_packages(),
install_requires=[
'slapos.core',
'slapos.libnetworkcache',
'requests',
],
zip_safe=True,
test_suite='test',
)
software/restic-rest-server/test/test.py 0 → 100644
View file @ 5081f0b0
##############################################################################
#
# Copyright (c) 2021 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import glob
import os
import subprocess
import tempfile
import urllib.parse
import requests
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
class TestResticRestServer(SlapOSInstanceTestCase):
def setUp(self):
self.connection_parameters = \
self.computer_partition.getConnectionParameterDict()
parsed_url = urllib.parse.urlparse(self.connection_parameters['url'])
self.url_with_credentials = parsed_url._replace(
netloc='{}:{}@[{}]:{}'.format(
self.connection_parameters['rest-server-user'],
self.connection_parameters['rest-server-password'],
parsed_url.hostname,
parsed_url.port,
)).geturl()
self.ca_cert = self._getCaucaseServiceCACertificate()
def _getCaucaseServiceCACertificate(self):
ca_cert = tempfile.NamedTemporaryFile(
prefix="ca.crt.pem",
mode="w",
delete=False,
)
ca_cert.write(
requests.get(
urllib.parse.urljoin(
self.connection_parameters['caucase-url'],
'/cas/crt/ca.crt.pem',
)).text)
self.addCleanup(os.unlink, ca_cert.name)
return ca_cert.name
def test_http_get(self):
resp = requests.get(self.connection_parameters['url'], verify=self.ca_cert)
self.assertEqual(resp.status_code, requests.codes.unauthorized)
resp = requests.get(
urllib.parse.urljoin(
self.url_with_credentials,
'/metrics',
),
verify=self.ca_cert,
)
# a random metric
self.assertIn('process_cpu_seconds_total', resp.text)
resp.raise_for_status()
def test_backup_scenario(self):
restic_bin = os.path.join(
self.computer_partition_root_path,
'software_release',
'go.work',
'bin',
'restic',
)
def get_restic_output(*args, **kw):
return subprocess.check_output(
(
restic_bin,
'--cacert',
self.ca_cert,
'--password-file',
password_file.name,
'--repo',
'rest:' + self.url_with_credentials,
) + args,
universal_newlines=True,
**kw,
)
with tempfile.TemporaryDirectory() as work_directory,\
tempfile.NamedTemporaryFile(mode='w') as password_file:
password_file.write('secret')
password_file.flush()
with open(os.path.join(work_directory, 'data'), 'w') as f:
f.write('data to backup')
with self.assertRaises(subprocess.CalledProcessError) as exc_context:
get_restic_output('snapshots', stderr=subprocess.PIPE)
self.assertIn('Is there a repository at the following location?',
exc_context.exception.stderr)
out = get_restic_output('init')
self.assertIn('created restic repository', out)
out = get_restic_output('backup', work_directory)
self.assertIn('Added to the repo', out)
out = get_restic_output('snapshots')
self.assertEqual(out.splitlines()[-1], '1 snapshots')
snapshot_id = out.splitlines()[2].split()[0]
backup_path = out.splitlines()[2].split()[-1]
restore_directory = os.path.join(work_directory, 'restore')
out = get_restic_output(
'restore',
snapshot_id,
'--target',
restore_directory,
)
self.assertIn('restoring <Snapshot', out)
with open(os.path.join(restore_directory, backup_path, 'data')) as f:
self.assertEqual(f.read(), 'data to backup')
software/seleniumserver/buildout.hash.cfg
View file @ 5081f0b0
......@@ -15,8 +15,8 @@
[template]
filename = instance.cfg.in
md5sum = c4ac5de141ae6a64848309af03e51d88
md5sum = 0084214fae4ee1aad2c878aa393757af
[template-selenium]
filename = instance-selenium.cfg.in
md5sum = eea51d25c292c7ea305229184e380814
md5sum = 884196ea35de35fa9159517912441ce6
software/seleniumserver/instance-selenium.cfg.in
View file @ 5081f0b0
......@@ -61,7 +61,7 @@ command-line =
# newSessionWaitTimeout: let clients wait in the queue when no node are available
# maxSession: to accept enough clients
hostname = $${instance-parameter:ipv4-random}
hostname = $${slap-configuration:ipv4-random}
port = 4444
base-url = http://$${:hostname}:$${:port}
url = $${:base-url}/wd/hub
......@@ -94,7 +94,7 @@ environment =
XORG_LOCK_DIR=$${directory:tmp}
DISPLAY=$${xvfb-instance:display}
FONTCONFIG_FILE=$${fontconfig-conf:rendered}
hostname = $${instance-parameter:ipv4-random}
hostname = $${slap-configuration:ipv4-random}
[selenium-server-node-instance-firefox-52]
......@@ -115,11 +115,23 @@ capabilities = browserName=firefox,maxInstances=3,marionette=true,platform=LINUX
java-args = -Dwebdriver.gecko.driver=${geckodriver-0.24.0:location}
port = 7779
[selenium-server-node-instance-firefox-78]
<= selenium-server-node-instance
capabilities = browserName=firefox,maxInstances=3,marionette=true,platform=LINUX,version=${firefox-78:version},firefox_binary=${firefox-wrapper-78:location}
java-args = -Dwebdriver.gecko.driver=${geckodriver-0.24.0:location}
port = 7780
[selenium-server-node-instance-chromium-69]
<= selenium-server-node-instance
capabilities = browserName=chrome,maxInstances=3,platform=LINUX,version=${chromium-69:version},chrome_binary=${chromium-wrapper-69:location}
java-args = -Dwebdriver.chrome.driver=${chromedriver-wrapper-2.41:location}
port = 7780
port = 7781
[selenium-server-node-instance-chromium-91]
<= selenium-server-node-instance
capabilities = browserName=chrome,maxInstances=3,platform=LINUX,version=${chromium-91:version},chrome_binary=${chromium-wrapper-91:location}
java-args = -Dwebdriver.chrome.driver=${chromedriver-wrapper-91:location}
port = 7782
[selenium-server-admin-password]
......@@ -152,7 +164,7 @@ template = inline:
$${:path-hub}
}
}
ip = $${instance-parameter:ipv6-random}
ip = $${slap-configuration:ipv6-random}
hostname = [$${:ip}]
port = 9443
path-admin = /grid/console
......@@ -180,7 +192,7 @@ recipe = slapos.cookbook:userinfo
recipe = slapos.cookbook:free_port
minimum = 22222
maximum = 22231
ip = $${slap-network-information:global-ipv6}
ip = $${slap-configuration:ipv6-random}
hostname = $${:ip}
[ssh-keygen-base]
......@@ -251,7 +263,7 @@ recipe = plone.recipe.command
stop-on-error = true
location = $${buildout:directory}/.ssh
authorized-keys-file = $${:location}/authorized_keys
command = mkdir -p $${:location} && echo '$${instance-parameter:configuration.ssh-authorized-key}' > $${:authorized-keys-file}
command = mkdir -p $${:location} && echo '$${slap-configuration:configuration.ssh-authorized-key}' > $${:authorized-keys-file}
[promises]
......@@ -264,27 +276,29 @@ instance-promises =
$${selenium-server-node-firefox-52-listen-promise:name}
$${selenium-server-node-firefox-60-listen-promise:name}
$${selenium-server-node-firefox-68-listen-promise:name}
$${selenium-server-node-firefox-78-listen-promise:name}
$${selenium-server-node-instance-chromium-69-listen-promise:name}
$${selenium-server-node-instance-chromium-91-listen-promise:name}
[check-port-listening-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = $${:_buildout_section_name_}.py
[sshd-listen-promise]
<= check-port-listening-promise
config-hostname = $${sshd-address:hostname}
config-host = $${sshd-address:hostname}
config-port = $${sshd-address:port}
[selenium-server-frontend-listen-promise]
<= check-port-listening-promise
config-hostname = $${selenium-server-frontend-instance:ip}
config-host = $${selenium-server-frontend-instance:ip}
config-port = $${selenium-server-frontend-instance:port}
[selenium-server-hub-listen-promise]
<= check-port-listening-promise
config-hostname = $${selenium-server-hub-instance:hostname}
config-host = $${selenium-server-hub-instance:hostname}
config-port = $${selenium-server-hub-instance:port}
# Promise waiting for all nodes to be registered
......@@ -294,8 +308,8 @@ module = check_command_execute
name = $${:_buildout_section_name_}.py
config-command =
$${selenium-server-check-nodes-registered:rendered} $${selenium-server-hub-instance:api-url} $${:expected-node-count}
# We have 4 nodes with 3 slots each
expected-node-count = 12
# We have 6 nodes with 3 slots each
expected-node-count = 18
[selenium-server-check-nodes-registered]
recipe = slapos.recipe.template:jinja2
......@@ -313,24 +327,34 @@ template =
[selenium-server-node-firefox-52-listen-promise]
<= check-port-listening-promise
config-hostname = $${selenium-server-node-instance-firefox-52:hostname}
config-host = $${selenium-server-node-instance-firefox-52:hostname}
config-port = $${selenium-server-node-instance-firefox-52:port}
[selenium-server-node-firefox-60-listen-promise]
<= check-port-listening-promise
config-hostname = $${selenium-server-node-instance-firefox-60:hostname}
config-host = $${selenium-server-node-instance-firefox-60:hostname}
config-port = $${selenium-server-node-instance-firefox-60:port}
[selenium-server-node-firefox-68-listen-promise]
<= check-port-listening-promise
config-hostname = $${selenium-server-node-instance-firefox-68:hostname}
config-host = $${selenium-server-node-instance-firefox-68:hostname}
config-port = $${selenium-server-node-instance-firefox-68:port}
[selenium-server-node-firefox-78-listen-promise]
<= check-port-listening-promise
config-host = $${selenium-server-node-instance-firefox-78:hostname}
config-port = $${selenium-server-node-instance-firefox-78:port}
[selenium-server-node-instance-chromium-69-listen-promise]
<= check-port-listening-promise
config-hostname = $${selenium-server-node-instance-chromium-69:hostname}
config-host = $${selenium-server-node-instance-chromium-69:hostname}
config-port = $${selenium-server-node-instance-chromium-69:port}
[selenium-server-node-instance-chromium-91-listen-promise]
<= check-port-listening-promise
config-host = $${selenium-server-node-instance-chromium-91:hostname}
config-port = $${selenium-server-node-instance-chromium-91:port}
[publish-connection-parameter]
recipe = slapos.cookbook:publish
......@@ -347,7 +371,7 @@ ssh-fingerprint = $${ssh-key-fingerprint:fingerprint}
run-node-command = PORT=7999 bash -c 'trap '"'"'kill -TERM $SSHPID; wait $SSHPID '"'"' TERM INT; ssh -L 4444:$${selenium-server-hub-instance:hostname}:$${selenium-server-hub-instance:port} -R $PORT:127.0.0.1:$PORT -p $${sshd-service:port} $${sshd-service:username}@$${sshd-service:ip} & SSHPID=$!; java -jar selenium-server-standalone-3.14.0.jar -role node -host 127.0.0.1 -port $PORT ; wait "$SSHPID"'
[instance-parameter]
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration
computer = $${slap-connection:computer-id}
partition = $${slap-connection:partition-id}
......
software/seleniumserver/instance.cfg.in
View file @ 5081f0b0
......@@ -6,15 +6,21 @@ eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
[dynamic-template-selenium]
recipe = slapos.recipe.template:jinja2
template = ${template-selenium:output}
rendered = $${buildout:parts-directory}/$${:_buildout_section_name_}/$${:filename}
filename = instance-selenium.cfg
context =
key partition_ipv6 slap-configuration:ipv6-random
[switch-softwaretype]
recipe = slapos.cookbook:softwaretype
default = ${template-selenium:output}
recipe = slapos.cookbook:switch-softwaretype
RootSoftwareInstance = $${:default}
default = dynamic-template-selenium:rendered
[slap-connection]
# part to migrate to new - separated words
computer-id = $${slap_connection:computer_id}
partition-id = $${slap_connection:partition_id}
server-url = $${slap_connection:server_url}
software-release-url = $${slap_connection:software_release_url}
key-file = $${slap_connection:key_file}
cert-file = $${slap_connection:cert_file}
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration
computer = $${slap-connection:computer-id}
partition = $${slap-connection:partition-id}
url = $${slap-connection:server-url}
software/seleniumserver/software.cfg
View file @ 5081f0b0
......@@ -73,5 +73,6 @@ post-install =
post-install =
${symlink-extra-fonts-to-firefox-fonts-dir:install}
[versions]
plone.recipe.command = 1.1
[firefox-78]
post-install =
${symlink-extra-fonts-to-firefox-fonts-dir:install}
software/seleniumserver/test/test.py
View file @ 5081f0b0
......@@ -426,6 +426,14 @@ class TestFirefox68(
desired_capabilities = dict(DesiredCapabilities.FIREFOX, version='68.0.2esr')
user_agent = 'Gecko/20100101 Firefox/68.0'
class TestFirefox78(
BrowserCompatibilityMixin,
SeleniumServerTestCase,
ImageComparisonTestCase,
):
desired_capabilities = dict(DesiredCapabilities.FIREFOX, version='78.1.0esr')
user_agent = 'Gecko/20100101 Firefox/78.0'
class TestChrome69(
BrowserCompatibilityMixin,
......@@ -434,3 +442,12 @@ class TestChrome69(
):
desired_capabilities = dict(DesiredCapabilities.CHROME, version='69.0.3497.0')
user_agent = 'Chrome/69.0.3497.0'
class TestChrome91(
BrowserCompatibilityMixin,
SeleniumServerTestCase,
ImageComparisonTestCase,
):
desired_capabilities = dict(DesiredCapabilities.CHROME, version='91.0.4472.114')
user_agent = 'Chrome/91.0.4472.0'
software/slapos-master/buildout.hash.cfg
View file @ 5081f0b0
......@@ -14,11 +14,11 @@
# not need these here).
[template-erp5]
filename = instance-erp5.cfg.in
md5sum = c341609c8d0cfe240a3a2ae35bd713b6
md5sum = 84f099cc9852c4f53a075dccbb3880f0
[template-balancer]
filename = instance-balancer.cfg.in
md5sum = 9c67c77eab5195b2674e340fb44c48a2
md5sum = c7c0bb9abbd0f8cc6c7956d83a61c4b3
[template-apache-backend-conf]
filename = apache-backend.conf.in
......
software/slapos-master/instance-balancer.cfg.in
View file @ 5081f0b0
......@@ -2,6 +2,7 @@
{% set part_list = [] -%}
{% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
{% set ssl_parameter_dict = slapparameter_dict['ssl'] -%}
{% set frontend_caucase_url_list = ssl_parameter_dict.get('frontend-caucase-url-list', []) -%}
{% set shared_ca_path = slapparameter_dict.get('shared-certificate-authority-path') -%}
{#
XXX: This template only supports exactly one IPv4 and (if ipv6 is used) one IPv6
......@@ -38,7 +39,7 @@ mode = 644
{% set haproxy_dict = {} -%}
{% set apache_dict = {} -%}
{% set zope_virtualhost_monster_backend_dict = {} %}
{% set test_runner_url_dict = {} %} {# family_name => list of apache URLs #}
{% set test_runner_url_dict = {} %} {# family_name => list of URLs #}
{% set next_port = itertools.count(slapparameter_dict['tcpv4-port']).next -%}
{% for family_name, parameter_id_list in sorted(
slapparameter_dict['zope-family-dict'].iteritems()) -%}
......@@ -59,19 +60,19 @@ mode = 644
{% set test_runner_address_list = slapparameter_dict.get(parameter_id ~ '-test-runner-address-list', []) %}
{% if test_runner_address_list -%}
{% set test_runner_backend_mapping = {} %}
{% set test_runner_apache_url_list = [] %}
{% set test_runner_balancer_url_list = [] %}
{% set test_runner_external_port = next_port() %}
{% for i, (test_runner_internal_ip, test_runner_internal_port) in enumerate(test_runner_address_list) %}
{% do test_runner_backend_mapping.__setitem__(
'unit_test_' ~ i,
'http://' ~ test_runner_internal_ip ~ ':' ~ test_runner_internal_port ) %}
{% do test_runner_apache_url_list.append(
{% do test_runner_balancer_url_list.append(
'https://' ~ ipv4 ~ ':' ~ test_runner_external_port ~ '/unit_test_' ~ i ~ '/' ) %}
{% endfor %}
{% do zope_virtualhost_monster_backend_dict.__setitem__(
(ipv4, test_runner_external_port),
( ssl_authentication, test_runner_backend_mapping ) ) -%}
{% do test_runner_url_dict.__setitem__(family_name, test_runner_apache_url_list) -%}
{% do test_runner_url_dict.__setitem__(family_name, test_runner_balancer_url_list) -%}
{% endif -%}
{% endfor -%}
......@@ -123,6 +124,23 @@ caucase-key = ${directory:apache-conf}/apache-caucase.pem
ca-cert = ${directory:apache-conf}/ca.crt
crl = ${directory:apache-conf}/crl.pem
[simplefile]
< = jinja2-template-base
template = inline:{{ '{{ content }}' }}
{% macro simplefile(section_name, file_path, content, mode='') -%}
{% set content_section_name = section_name ~ '-content' -%}
[{{ content_section_name }}]
content = {{ dumps(content) }}
[{{ section(section_name) }}]
< = simplefile
rendered = {{ file_path }}
context = key content {{content_section_name}}:content
mode = {{ mode }}
{%- endmacro %}
[apache-ssl]
{% if ssl_parameter_dict.get('key') -%}
key = ${apache-ssl-key:rendered}
......@@ -185,9 +203,9 @@ input = inline:
[{{ section('apache-promise') }}]
# Check any apache port in ipv4, expect other ports and ipv6 to behave consistently
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = apache.py
config-hostname = {{ ipv4 }}
config-host = {{ ipv4 }}
config-port = {{ apache_dict.values()[0][0] }}
[{{ section('publish') }}]
......@@ -273,8 +291,6 @@ template = inline:
{% endfor %}
[apachedex-parameters]
# XXX - Sample log file with curent date: apache_access.log-%(date)s.gz
# which will be equivalent to apache_access.log-20150112.gz if the date is 2015-01-12
apache-log-list = ${apache-conf-parameter-dict:access-log}
configuration = ${monitor-apachedex-report-config:rendered}
promise-threshold = {{ slapparameter_dict['apachedex-promise-threshold'] }}
......
software/slapos-master/instance-erp5.cfg.in
View file @ 5081f0b0
......@@ -221,7 +221,10 @@ return =
{%- if test_runner_enabled %}
test-runner-address-list
{% endif %}
{% set bt5_default_list = 'erp5_full_text_mroonga_catalog slapos_configurator' -%}
{% set bt5_default_list = [
'erp5_full_text_mroonga_catalog',
'slapos_configurator',
] -%}
{% if has_jupyter -%}
{% do bt5_default_list.append('erp5_data_notebook') -%}
{% endif -%}
......@@ -254,6 +257,7 @@ config-wsgi = {{ dumps(slapparameter_dict.get('wsgi', True)) }}
config-test-runner-enabled = {{ dumps(test_runner_enabled) }}
config-test-runner-node-count = {{ dumps(test_runner_node_count) }}
config-wcfs_enable = {{ dumps(wcfs_enable) }}
config-test-runner-configuration = {{ dumps(slapparameter_dict.get('test-runner', {})) }}
software-type = zope
{% set global_publisher_timeout = slapparameter_dict.get('publisher-timeout') -%}
......
software/slapos-master/test/test/__init__.py
View file @ 5081f0b0
......@@ -30,11 +30,21 @@ import os
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
_setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', '..', 'software.cfg')))
setup_module_executed = False
def setUpModule():
# slapos.testing.testcase's only need to be executed once
global setup_module_executed
if not setup_module_executed:
_setUpModule()
setup_module_executed = True
class ERP5InstanceTestCase(SlapOSInstanceTestCase):
"""ERP5 base test case
"""
......
software/slapos-master/test/test/test_balancer.py 0 → 100644
View file @ 5081f0b0
import glob
import hashlib
import json
import logging
import os
import re
import shutil
import subprocess
import tempfile
import time
import urllib
import urlparse
from BaseHTTPServer import BaseHTTPRequestHandler
from typing import Dict
import mock
import OpenSSL.SSL
import pexpect
import psutil
import requests
from cryptography import x509
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives import hashes, serialization
from cryptography.hazmat.primitives.asymmetric import rsa
from cryptography.x509.oid import NameOID
from slapos.testing.testcase import ManagedResource
from slapos.testing.utils import (CrontabMixin, ManagedHTTPServer,
findFreeTCPPort)
from . import ERP5InstanceTestCase, setUpModule
setUpModule # pyflakes
class EchoHTTPServer(ManagedHTTPServer):
"""An HTTP Server responding with the request path and incoming headers,
encoded in json.
"""
class RequestHandler(BaseHTTPRequestHandler):
def do_GET(self):
# type: () -> None
self.send_response(200)
self.send_header("Content-Type", "application/json")
response = json.dumps(
{
'Path': self.path,
'Incoming Headers': self.headers.dict
},
indent=2,
)
self.end_headers()
self.wfile.write(response)
log_message = logging.getLogger(__name__ + '.EchoHTTPServer').info
class EchoHTTP11Server(ManagedHTTPServer):
"""An HTTP/1.1 Server responding with the request path and incoming headers,
encoded in json.
"""
class RequestHandler(BaseHTTPRequestHandler):
protocol_version = 'HTTP/1.1'
def do_GET(self):
# type: () -> None
self.send_response(200)
self.send_header("Content-Type", "application/json")
response = json.dumps(
{
'Path': self.path,
'Incoming Headers': self.headers.dict
},
indent=2,
)
self.send_header("Content-Length", len(response))
self.end_headers()
self.wfile.write(response)
log_message = logging.getLogger(__name__ + '.EchoHTTP11Server').info
class CaucaseService(ManagedResource):
"""A caucase service.
"""
url = None # type: str
directory = None # type: str
_caucased_process = None # type: subprocess.Popen
def open(self):
# type: () -> None
# start a caucased and server certificate.
software_release_root_path = os.path.join(
self._cls.slap._software_root,
hashlib.md5(self._cls.getSoftwareURL().encode()).hexdigest(),
)
caucased_path = os.path.join(software_release_root_path, 'bin', 'caucased')
self.directory = tempfile.mkdtemp()
caucased_dir = os.path.join(self.directory, 'caucased')
os.mkdir(caucased_dir)
os.mkdir(os.path.join(caucased_dir, 'user'))
os.mkdir(os.path.join(caucased_dir, 'service'))
backend_caucased_netloc = '%s:%s' % (self._cls._ipv4_address, findFreeTCPPort(self._cls._ipv4_address))
self.url = 'http://' + backend_caucased_netloc
self._caucased_process = subprocess.Popen(
[
caucased_path,
'--db', os.path.join(caucased_dir, 'caucase.sqlite'),
'--server-key', os.path.join(caucased_dir, 'server.key.pem'),
'--netloc', backend_caucased_netloc,
'--service-auto-approve-count', '1',
],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
for _ in range(30):
try:
if requests.get(self.url).status_code == 200:
break
except Exception:
pass
time.sleep(1)
else:
raise RuntimeError('caucased failed to start.')
def close(self):
# type: () -> None
self._caucased_process.terminate()
self._caucased_process.wait()
shutil.rmtree(self.directory)
class BalancerTestCase(ERP5InstanceTestCase):
@classmethod
def getInstanceSoftwareType(cls):
return 'balancer'
@classmethod
def _getInstanceParameterDict(cls):
# type: () -> Dict
return {
'shared-certificate-authority-path': os.path.join(
'~', 'srv', 'ssl'),
'tcpv4-port': 8000,
'computer-memory-percent-threshold': 100,
# XXX what is this ? should probably not be needed here
'name': cls.__name__,
'monitor-passwd': 'secret',
'apachedex-configuration': [
'--logformat', '%h %l %u %t "%r" %>s %O "%{Referer}i" "%{User-Agent}i" %{ms}T',
'--erp5-base', '+erp5', '.*/VirtualHostRoot/erp5(/|\\?|$)',
'--base', '+other', '/',
'--skip-user-agent', 'Zabbix',
'--error-detail',
'--js-embed',
'--quiet',
],
'apachedex-promise-threshold': 100,
'haproxy-server-check-path': '/',
'zope-family-dict': {
'default': ['dummy_http_server'],
},
'dummy_http_server': [[cls.getManagedResource("backend_web_server", EchoHTTPServer).netloc, 1, False]],
'backend-path-dict': {
'default': '',
},
'ssl-authentication-dict': {},
'ssl': {
'caucase-url': cls.getManagedResource("caucase", CaucaseService).url,
},
'family-path-routing-dict': {},
'path-routing-list': [],
}
@classmethod
def getInstanceParameterDict(cls):
# type: () -> Dict
return {'_': json.dumps(cls._getInstanceParameterDict())}
def setUp(self):
self.default_balancer_url = json.loads(
self.computer_partition.getConnectionParameterDict()['_'])['default']
class SlowHTTPServer(ManagedHTTPServer):
"""An HTTP Server which reply after 2 seconds.
"""
class RequestHandler(BaseHTTPRequestHandler):
def do_GET(self):
# type: () -> None
self.send_response(200)
self.send_header("Content-Type", "text/plain")
time.sleep(2)
self.end_headers()
self.wfile.write("OK\n")
log_message = logging.getLogger(__name__ + '.SlowHandler').info
class TestLog(BalancerTestCase, CrontabMixin):
"""Check logs emitted by balancer
"""
__partition_reference__ = 'l'
@classmethod
def _getInstanceParameterDict(cls):
# type: () -> Dict
parameter_dict = super(TestLog, cls)._getInstanceParameterDict()
# use a slow server instead
parameter_dict['dummy_http_server'] = [[cls.getManagedResource("slow_web_server", SlowHTTPServer).netloc, 1, False]]
return parameter_dict
def test_access_log_format(self):
# type: () -> None
requests.get(
urlparse.urljoin(self.default_balancer_url, '/url_path'),
verify=False,
)
time.sleep(.5) # wait a bit more until access is logged
with open(os.path.join(self.computer_partition_root_path, 'var', 'log', 'apache-access.log')) as access_log_file:
access_line = access_log_file.read().splitlines()[-1]
self.assertIn('/url_path', access_line)
# last \d is the request time in milli seconds, since this SlowHTTPServer
# sleeps for 2 seconds, it should take between 2 and 3 seconds to process
# the request - but our test machines can be slow sometimes, so we tolerate
# it can take up to 20 seconds.
match = re.match(
r'([(\d\.)]+) - - \[(.*?)\] "(.*?)" (\d+) (\d+) "(.*?)" "(.*?)" (\d+)',
access_line
)
self.assertTrue(match)
assert match
request_time = int(match.groups()[-1])
# XXX For slapos master, timing is in microsecond (not milisecond)
self.assertGreater(request_time, 2 * 1000000)
self.assertLess(request_time, 20 * 1000000)
def test_access_log_apachedex_report(self):
# type: () -> None
# make a request so that we have something in the logs
requests.get(self.default_balancer_url, verify=False)
# crontab for apachedex is executed
self._executeCrontabAtDate('generate-apachedex-report', '23:59')
# it creates a report for the day
apachedex_report, = glob.glob(
os.path.join(
self.computer_partition_root_path,
'srv',
'monitor',
'private',
'apachedex',
'ApacheDex-*.html',
))
with open(apachedex_report, 'r') as f:
report_text = f.read()
self.assertIn('APacheDEX', report_text)
# having this table means that apachedex could parse some lines.
self.assertIn('<h2>Hits per status code</h2>', report_text)
def test_access_log_rotation(self):
# type: () -> None
# run logrotate a first time so that it create state files
self._executeCrontabAtDate('logrotate', '2000-01-01')
# make a request so that we have something in the logs
requests.get(self.default_balancer_url, verify=False).raise_for_status()
# slow query crontab depends on crontab for log rotation
# to be executed first.
self._executeCrontabAtDate('logrotate', '2050-01-01')
# this logrotate leaves the log for the day as non compressed
rotated_log_file = os.path.join(
self.computer_partition_root_path,
'srv',
'backup',
'logrotate',
'apache-access.log-20500101',
)
self.assertTrue(os.path.exists(rotated_log_file))
requests.get(self.default_balancer_url, verify=False).raise_for_status()
# on next day execution of logrotate, log files are compressed
self._executeCrontabAtDate('logrotate', '2050-01-02')
self.assertTrue(os.path.exists(rotated_log_file + '.xz'))
self.assertFalse(os.path.exists(rotated_log_file))
def test_error_log(self):
# stop backend server
backend_server = self.getManagedResource("slow_web_server", SlowHTTPServer)
self.addCleanup(backend_server.open)
backend_server.close()
# after a while, balancer should detect and log this event in error log
time.sleep(5)
self.assertEqual(
requests.get(self.default_balancer_url, verify=False).status_code,
requests.codes.service_unavailable)
with open(os.path.join(self.computer_partition_root_path, 'var', 'log', 'apache-error.log')) as error_log_file:
error_line = error_log_file.read().splitlines()[-1]
self.assertIn('apache.conf -D FOREGROUND', error_line)
# this log also include a timestamp
# This regex is for haproxy mostly, so keep it commented for now, until we can
# Merge the slapos-master setup and erp5.
# self.assertRegexpMatches(error_line, r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}')
class BalancerCookieHTTPServer(ManagedHTTPServer):
"""An HTTP Server which can set balancer cookie.
This server set cookie when requested /set-cookie path.
The reply body is the name used when registering this resource
using getManagedResource. This way we can assert which
backend replied.
"""
@property
def RequestHandler(self):
server = self
class RequestHandler(BaseHTTPRequestHandler):
def do_GET(self):
# type: () -> None
self.send_response(200)
self.send_header("Content-Type", "text/plain")
if self.path == '/set_cookie':
# the balancer tells the backend what's the name of the balancer cookie with
# the X-Balancer-Current-Cookie header.
self.send_header('Set-Cookie', '%s=anything' % self.headers['X-Balancer-Current-Cookie'])
# The name of this cookie is SERVERID
assert self.headers['X-Balancer-Current-Cookie'] == 'SERVERID'
self.end_headers()
self.wfile.write(server._name)
log_message = logging.getLogger(__name__ + '.BalancerCookieHTTPServer').info
return RequestHandler
class TestBalancer(BalancerTestCase):
"""Check balancing capabilities
"""
__partition_reference__ = 'b'
@classmethod
def _getInstanceParameterDict(cls):
# type: () -> Dict
parameter_dict = super(TestBalancer, cls)._getInstanceParameterDict()
# use two backend servers
parameter_dict['dummy_http_server'] = [
[cls.getManagedResource("backend_web_server1", BalancerCookieHTTPServer).netloc, 1, False],
[cls.getManagedResource("backend_web_server2", BalancerCookieHTTPServer).netloc, 1, False],
]
return parameter_dict
def test_balancer_round_robin(self):
# requests are by default balanced to both servers
self.assertEqual(
{requests.get(self.default_balancer_url, verify=False).text for _ in range(10)},
{'backend_web_server1', 'backend_web_server2'}
)
def test_balancer_server_down(self):
# if one backend is down, it is excluded from balancer
self.getManagedResource("backend_web_server2", BalancerCookieHTTPServer).close()
self.addCleanup(self.getManagedResource("backend_web_server2", BalancerCookieHTTPServer).open)
self.assertEqual(
{requests.get(self.default_balancer_url, verify=False).text for _ in range(10)},
{'backend_web_server1',}
)
def test_balancer_set_cookie(self):
# if backend provides a "SERVERID" cookie, balancer will overwrite it with the
# backend selected by balancing algorithm
self.assertIn(
requests.get(urlparse.urljoin(self.default_balancer_url, '/set_cookie'), verify=False).cookies['SERVERID'],
('default-0', 'default-1'),
)
def test_balancer_respects_sticky_cookie(self):
# if request is made with the sticky cookie, the client stick on one balancer
cookies = dict(SERVERID='default-1')
self.assertEqual(
{requests.get(self.default_balancer_url, verify=False, cookies=cookies).text for _ in range(10)},
{'backend_web_server2',}
)
# if that backend becomes down, requests are balanced to another server
self.getManagedResource("backend_web_server2", BalancerCookieHTTPServer).close()
self.addCleanup(self.getManagedResource("backend_web_server2", BalancerCookieHTTPServer).open)
self.assertEqual(
requests.get(self.default_balancer_url, verify=False, cookies=cookies).text,
'backend_web_server1')
class TestTestRunnerEntryPoints(BalancerTestCase):
"""Check balancer has some entries for test runner.
"""
__partition_reference__ = 't'
@classmethod
def _getInstanceParameterDict(cls):
# type: () -> Dict
parameter_dict = super(
TestTestRunnerEntryPoints,
cls,
)._getInstanceParameterDict()
parameter_dict['dummy_http_server-test-runner-address-list'] = [
[
cls.getManagedResource("backend_0", EchoHTTPServer).hostname,
cls.getManagedResource("backend_0", EchoHTTPServer).port,
],
[
cls.getManagedResource("backend_1", EchoHTTPServer).hostname,
cls.getManagedResource("backend_1", EchoHTTPServer).port,
],
[
cls.getManagedResource("backend_2", EchoHTTPServer).hostname,
cls.getManagedResource("backend_2", EchoHTTPServer).port,
],
]
return parameter_dict
def test_use_proper_backend(self):
# requests are directed to proper backend based on URL path
test_runner_url_list = self.getRootPartitionConnectionParameterDict(
)['default-test-runner-url-list']
url_0, url_1, url_2 = test_runner_url_list
self.assertEqual(
urlparse.urlparse(url_0).netloc,
urlparse.urlparse(url_1).netloc)
self.assertEqual(
urlparse.urlparse(url_0).netloc,
urlparse.urlparse(url_2).netloc)
path_0 = '/VirtualHostBase/https/{netloc}/VirtualHostRoot/_vh_unit_test_0/something'.format(
netloc=urlparse.urlparse(url_0).netloc)
path_1 = '/VirtualHostBase/https/{netloc}/VirtualHostRoot/_vh_unit_test_1/something'.format(
netloc=urlparse.urlparse(url_0).netloc)
path_2 = '/VirtualHostBase/https/{netloc}/VirtualHostRoot/_vh_unit_test_2/something'.format(
netloc=urlparse.urlparse(url_0).netloc)
self.assertEqual(
{
requests.get(url_0 + 'something', verify=False).json()['Path']
for _ in range(10)
}, {path_0})
self.assertEqual(
{
requests.get(url_1 + 'something', verify=False).json()['Path']
for _ in range(10)
}, {path_1})
self.assertEqual(
{
requests.get(url_2 + 'something', verify=False).json()['Path']
for _ in range(10)
}, {path_2})
# If a test runner backend is down, others can be accessed.
self.getManagedResource("backend_0", EchoHTTPServer).close()
self.assertEqual(
{
requests.get(url_0 + 'something', verify=False).status_code
for _ in range(5)
}, {503})
self.assertEqual(
{
requests.get(url_1 + 'something', verify=False).json()['Path']
for _ in range(10)
}, {path_1})
class TestHTTP(BalancerTestCase):
"""Check HTTP protocol with a HTTP/1.1 backend
"""
@classmethod
def _getInstanceParameterDict(cls):
# type: () -> Dict
parameter_dict = super(TestHTTP, cls)._getInstanceParameterDict()
# use a HTTP/1.1 server instead
parameter_dict['dummy_http_server'] = [[cls.getManagedResource("HTTP/1.1 Server", EchoHTTP11Server).netloc, 1, False]]
return parameter_dict
__partition_reference__ = 'h'
def test_http_version(self):
# type: () -> None
self.assertEqual(
subprocess.check_output([
'curl',
'--silent',
'--show-error',
'--output',
'/dev/null',
'--insecure',
'--write-out',
'%{http_version}',
self.default_balancer_url,
]),
'1.1',
)
def test_keep_alive(self):
# type: () -> None
# when doing two requests, connection is established only once
session = requests.Session()
session.verify = False
# do a first request, which establish a first connection
session.get(self.default_balancer_url).raise_for_status()
# "break" new connection method and check we can make another request
with mock.patch(
"requests.packages.urllib3.connectionpool.HTTPSConnectionPool._new_conn",
) as new_conn:
session.get(self.default_balancer_url).raise_for_status()
new_conn.assert_not_called()
parsed_url = urlparse.urlparse(self.default_balancer_url)
# check that we have an open file for the ip connection
self.assertTrue([
c for c in psutil.Process(os.getpid()).connections()
if c.status == 'ESTABLISHED' and c.raddr.ip == parsed_url.hostname
and c.raddr.port == parsed_url.port
])
class ContentTypeHTTPServer(ManagedHTTPServer):
"""An HTTP/1.1 Server which reply with content type from path.
For example when requested http://host/text/plain it will reply
with Content-Type: text/plain header.
The body is always "OK"
"""
class RequestHandler(BaseHTTPRequestHandler):
protocol_version = 'HTTP/1.1'
def do_GET(self):
# type: () -> None
self.send_response(200)
if self.path == '/':
self.send_header("Content-Length", 0)
return self.end_headers()
content_type = self.path[1:]
body = "OK"
self.send_header("Content-Type", content_type)
self.send_header("Content-Length", len(body))
self.end_headers()
self.wfile.write(body)
log_message = logging.getLogger(__name__ + '.ContentTypeHTTPServer').info
class TestContentEncoding(BalancerTestCase):
"""Test how responses are gzip encoded or not depending on content type header.
"""
__partition_reference__ = 'ce'
@classmethod
def _getInstanceParameterDict(cls):
# type: () -> Dict
parameter_dict = super(TestContentEncoding, cls)._getInstanceParameterDict()
parameter_dict['dummy_http_server'] = [
[cls.getManagedResource("content_type_server", ContentTypeHTTPServer).netloc, 1, False],
]
return parameter_dict
# Disabled test until we can rework on it for apache, or drop
# apache on the backend.
def disabled_test_gzip_encoding(self):
# type: () -> None
for content_type in (
'text/cache-manifest',
'text/html',
'text/plain',
'text/css',
'application/hal+json',
'application/json',
'application/x-javascript',
'text/xml',
'application/xml',
'application/rss+xml',
'text/javascript',
'application/javascript',
'image/svg+xml',
'application/x-font-ttf',
'application/font-woff',
'application/font-woff2',
'application/x-font-opentype',
'application/wasm',):
resp = requests.get(
urlparse.urljoin(self.default_balancer_url, content_type),
verify=False,
headers={"Accept-Encoding": "gzip, deflate",})
self.assertEqual(resp.headers['Content-Type'], content_type)
self.assertEqual(
resp.headers.get('Content-Encoding'),
'gzip',
'%s uses wrong encoding: %s' % (content_type, resp.headers.get('Content-Encoding')))
self.assertEqual(resp.text, 'OK')
def test_no_gzip_encoding(self):
# type: () -> None
resp = requests.get(urlparse.urljoin(self.default_balancer_url, '/image/png'), verify=False)
self.assertNotIn('Content-Encoding', resp.headers)
self.assertEqual(resp.text, 'OK')
class CaucaseCertificate(ManagedResource):
"""A certificate signed by a caucase service.
"""
ca_crt_file = None # type: str
crl_file = None # type: str
csr_file = None # type: str
cert_file = None # type: str
key_file = None # type: str
def open(self):
# type: () -> None
self.tmpdir = tempfile.mkdtemp()
self.ca_crt_file = os.path.join(self.tmpdir, 'ca-crt.pem')
self.crl_file = os.path.join(self.tmpdir, 'ca-crl.pem')
self.csr_file = os.path.join(self.tmpdir, 'csr.pem')
self.cert_file = os.path.join(self.tmpdir, 'crt.pem')
self.key_file = os.path.join(self.tmpdir, 'key.pem')
def close(self):
# type: () -> None
shutil.rmtree(self.tmpdir)
@property
def _caucase_path(self):
# type: () -> str
"""path of caucase executable.
"""
software_release_root_path = os.path.join(
self._cls.slap._software_root,
hashlib.md5(self._cls.getSoftwareURL().encode()).hexdigest(),
)
return os.path.join(software_release_root_path, 'bin', 'caucase')
def request(self, common_name, caucase):
# type: (str, CaucaseService) -> None
"""Generate certificate and request signature to the caucase service.
This overwrite any previously requested certificate for this instance.
"""
cas_args = [
self._caucase_path,
'--ca-url', caucase.url,
'--ca-crt', self.ca_crt_file,
'--crl', self.crl_file,
]
key = rsa.generate_private_key(
public_exponent=65537,
key_size=2048,
backend=default_backend()
)
with open(self.key_file, 'wb') as f:
f.write(
key.private_bytes(
encoding=serialization.Encoding.PEM,
format=serialization.PrivateFormat.TraditionalOpenSSL,
encryption_algorithm=serialization.NoEncryption(),
))
csr = x509.CertificateSigningRequestBuilder().subject_name(
x509.Name([
x509.NameAttribute(
NameOID.COMMON_NAME,
common_name,
),
])).sign(
key,
hashes.SHA256(),
default_backend(),
)
with open(self.csr_file, 'wb') as f:
f.write(csr.public_bytes(serialization.Encoding.PEM))
csr_id = subprocess.check_output(
cas_args + [
'--send-csr', self.csr_file,
],
).split()[0]
assert csr_id
for _ in range(30):
if not subprocess.call(
cas_args + [
'--get-crt', csr_id, self.cert_file,
],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
) == 0:
break
else:
time.sleep(1)
else:
raise RuntimeError('getting service certificate failed.')
with open(self.cert_file) as f:
assert 'BEGIN CERTIFICATE' in f.read()
def revoke(self, caucase):
# type: (str, CaucaseService) -> None
"""Revoke the client certificate on this caucase instance.
"""
subprocess.check_call([
self._caucase_path,
'--ca-url', caucase.url,
'--ca-crt', self.ca_crt_file,
'--crl', self.crl_file,
'--revoke-crt', self.cert_file, self.key_file,
])
class TestServerTLSProvidedCertificate(BalancerTestCase):
"""Check that certificate and key can be provided as instance parameters.
"""
__partition_reference__ = 's'
@classmethod
def _getInstanceParameterDict(cls):
# type: () -> Dict
server_caucase = cls.getManagedResource('server_caucase', CaucaseService)
server_certificate = cls.getManagedResource('server_certificate', CaucaseCertificate)
server_certificate.request(cls._ipv4_address.decode(), server_caucase)
parameter_dict = super(TestServerTLSProvidedCertificate, cls)._getInstanceParameterDict()
with open(server_certificate.cert_file) as f:
parameter_dict['ssl']['cert'] = f.read()
with open(server_certificate.key_file) as f:
parameter_dict['ssl']['key'] = f.read()
return parameter_dict
def test_certificate_validates_with_provided_ca(self):
# type: () -> None
server_certificate = self.getManagedResource("server_certificate", CaucaseCertificate)
requests.get(self.default_balancer_url, verify=server_certificate.ca_crt_file)
software/slapos-master/test/test/test_erp5.py
View file @ 5081f0b0
......@@ -41,31 +41,50 @@ import requests
from . import ERP5InstanceTestCase
from . import setUpModule
setUpModule # pyflakes
setUpModule # pyflakes
class TestPublishedURLIsReachableMixin(object):
"""Mixin that checks that default page of ERP5 is reachable.
"""
def _checkERP5IsReachable(self, url):
def _checkERP5IsReachable(self, base_url, site_id, verify):
# We access ERP5 trough a "virtual host", which should make
# ERP5 produce URLs using https://virtual-host-name:1234/virtual_host_root
# as base.
virtual_host_url = urlparse.urljoin(
base_url,
'/VirtualHostBase/https/virtual-host-name:1234/{}/VirtualHostRoot/_vh_virtual_host_root/'
.format(site_id))
# What happens is that instantiation just create the services, but does not
# wait for ERP5 to be initialized. When this test run ERP5 instance is
# instantiated, but zope is still busy creating the site and haproxy
# replies with 503 Service Unavailable when zope is not started yet, with
# 404 when erp5 site is not created, with 500 when mysql is not yet
# reachable, so we retry in a loop until we get a succesful response.
for i in range(1, 60):
# XXX can we get CA from caucase already ?
r = requests.get(url, verify=False)
if r.status_code != requests.codes.ok:
delay = i * 2
self.logger.warn(
"ERP5 was not available, sleeping for %ds and retrying", delay)
time.sleep(delay)
continue
r.raise_for_status()
break
# instantiated, but zope is still busy creating the site and haproxy replies
# with 503 Service Unavailable when zope is not started yet, with 404 when
# erp5 site is not created, with 500 when mysql is not yet reachable, so we
# configure this requests session to retry.
# XXX we should probably add a promise instead
session = requests.Session()
session.mount(
base_url,
requests.adapters.HTTPAdapter(
max_retries=requests.packages.urllib3.util.retry.Retry(
total=60,
backoff_factor=.5,
status_forcelist=(404, 500, 503))))
r = session.get(virtual_host_url, verify=verify, allow_redirects=False)
self.assertEqual(r.status_code, requests.codes.found)
# access on / are redirected to login form, with virtual host preserved
self.assertEqual(r.headers.get('location'), 'https://virtual-host-name:1234/virtual_host_root/login_form')
# login page can be rendered and contain the text "ERP5"
r = session.get(
urlparse.urljoin(base_url, '{}/login_form'.format(site_id)),
verify=verify,
allow_redirects=False,
)
self.assertEqual(r.status_code, requests.codes.ok)
self.assertIn("ERP5", r.text)
def test_published_family_default_v6_is_reachable(self):
......@@ -73,15 +92,18 @@ class TestPublishedURLIsReachableMixin(object):
"""
param_dict = self.getRootPartitionConnectionParameterDict()
self._checkERP5IsReachable(
urlparse.urljoin(param_dict['family-default-v6'], param_dict['site-id']))
param_dict['family-default-v6'],
param_dict['site-id'],
verify=False)
def test_published_family_default_v4_is_reachable(self):
"""Tests the IPv4 URL published by the root partition is reachable.
"""
param_dict = self.getRootPartitionConnectionParameterDict()
self._checkERP5IsReachable(
urlparse.urljoin(param_dict['family-default'], param_dict['site-id']))
param_dict['family-default'],
param_dict['site-id'],
verify=False)
class TestDefaultParameters(
ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
......@@ -99,6 +121,30 @@ class TestMedusa(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
def getInstanceParameterDict(cls):
return {'_': json.dumps({'wsgi': False})}
class TestJupyter(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
"""Test ERP5 Jupyter notebook
"""
__partition_reference__ = 'jupyter'
@classmethod
def getInstanceParameterDict(cls):
return {'_': json.dumps({'jupyter': {'enable': True}})}
def test_jupyter_notebook_is_reachable(self):
param_dict = self.getRootPartitionConnectionParameterDict()
self.assertEqual(
'https://[%s]:8888/tree' % self._ipv6_address,
param_dict['jupyter-url']
)
result = requests.get(
param_dict['jupyter-url'], verify=False, allow_redirects=False)
self.assertEqual(
[requests.codes.found, True, '/login?next=%2Ftree'],
[result.status_code, result.is_redirect, result.headers['Location']]
)
class TestApacheBalancerPorts(ERP5InstanceTestCase):
"""Instantiate with two zope families, this should create for each family:
......@@ -132,19 +178,16 @@ class TestApacheBalancerPorts(ERP5InstanceTestCase):
self.assertTrue(parsed.port)
def test_published_family_parameters(self):
# when we request two families, we have two published family-{family_name}
# URLs
# when we request two families, we have two published family-{family_name} URLs
param_dict = self.getRootPartitionConnectionParameterDict()
for family_name in ('family1', 'family2'):
self.checkValidHTTPSURL(
param_dict['family-{family_name}'.format(family_name=family_name)])
self.checkValidHTTPSURL(
param_dict['family-{family_name}-v6'.format(
family_name=family_name)])
param_dict['family-{family_name}-v6'.format(family_name=family_name)])
def test_published_test_runner_url(self):
# each family's also a list of test test runner URLs, by default 3 per
# family
# each family's also a list of test test runner URLs, by default 3 per family
param_dict = self.getRootPartitionConnectionParameterDict()
for family_name in ('family1', 'family2'):
family_test_runner_url_list = param_dict[
......@@ -154,8 +197,7 @@ class TestApacheBalancerPorts(ERP5InstanceTestCase):
self.checkValidHTTPSURL(url)
def test_zope_listen(self):
# we requested 3 zope in family1 and 5 zopes in family2, we should have 8
# zope running.
# we requested 3 zope in family1 and 5 zopes in family2, we should have 8 zope running.
with self.slap.instance_supervisor_rpc as supervisor:
all_process_info = supervisor.getAllProcessInfo()
self.assertEqual(
......@@ -190,46 +232,7 @@ class TestApacheBalancerPorts(ERP5InstanceTestCase):
])
class TestDisableTestRunner(
ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
"""Test ERP5 can be instantiated without test runner.
"""
__partition_reference__ = 'distr'
@classmethod
def getInstanceParameterDict(cls):
return {'_': json.dumps({'test-runner': {'enabled': False}})}
def test_no_runUnitTestScript(self):
"""No runUnitTest script should be generated in any partition.
"""
# self.computer_partition_root_path is the path of root partition.
# we want to assert that no scripts exist in any partition.
bin_programs = map(
os.path.basename,
glob.glob(self.computer_partition_root_path + "/../*/bin/*"))
self.assertTrue(bin_programs) # just to check the glob was correct.
self.assertNotIn('runUnitTest', bin_programs)
self.assertNotIn('runTestSuite', bin_programs)
def test_no_apache_testrunner_port(self):
# Apache only listen on two ports, there is no apache ports allocated for
# test runner
with self.slap.instance_supervisor_rpc as supervisor:
all_process_info = supervisor.getAllProcessInfo()
process_info, = [p for p in all_process_info if p['name'] == 'apache']
apache_process = psutil.Process(process_info['pid'])
self.assertEqual(
sorted([socket.AF_INET, socket.AF_INET6]),
sorted(
c.family
for c in apache_process.connections()
if c.status == 'LISTEN'
))
class TestZopeNodeParameterOverride(
ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
class TestZopeNodeParameterOverride(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
"""Test override zope node parameters
"""
__partition_reference__ = 'override'
......@@ -244,7 +247,7 @@ class TestZopeNodeParameterOverride(
"server": {},
"cache-size-bytes": "20MB",
"cache-size-bytes!": [
("bb-0", 1 << 20),
("bb-0", 1<<20),
("bb-.*", "500MB"),
],
"pool-timeout": "10m",
......@@ -315,7 +318,7 @@ class TestZopeNodeParameterOverride(
partition = self.getComputerPartitionPath('zope-bb')
for zope in xrange(5):
checkConf({
"cache-size-bytes": "500MB" if zope else 1 << 20,
"cache-size-bytes": "500MB" if zope else 1<<20,
}, {
"cache-size": None,
})
......
software/slapos-master/test/test/test_mariadb.py 0 → 100644
View file @ 5081f0b0
##############################################################################
# coding: utf-8
#
# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import os
import json
import glob
import urlparse
import socket
import sys
import time
import contextlib
import datetime
import subprocess
import gzip
from backports import lzma
import MySQLdb
from slapos.testing.utils import CrontabMixin
from slapos.testing.utils import getPromisePluginParameterDict
from . import ERP5InstanceTestCase
from . import setUpModule
setUpModule # pyflakes
class MariaDBTestCase(ERP5InstanceTestCase):
"""Base test case for mariadb tests.
"""
__partition_reference__ = 'm'
@classmethod
def getInstanceSoftwareType(cls):
return "mariadb"
@classmethod
def _getInstanceParameterDict(cls):
return {
'tcpv4-port': 3306,
'max-connection-count': 5,
'long-query-time': 3,
'max-slowqueries-threshold': 1,
'slowest-query-threshold': 0.1,
# XXX what is this ? should probably not be needed here
'name': cls.__name__,
'monitor-passwd': 'secret',
# XXX should probably not be needed here
'computer-memory-percent-threshold': 100,
}
@classmethod
def getInstanceParameterDict(cls):
return {'_': json.dumps(cls._getInstanceParameterDict())}
def getDatabaseConnection(self):
connection_parameter_dict = json.loads(
self.computer_partition.getConnectionParameterDict()['_'])
db_url = urlparse.urlparse(connection_parameter_dict['database-list'][0])
self.assertEqual('mysql', db_url.scheme)
self.assertTrue(db_url.path.startswith('/'))
database_name = db_url.path[1:]
return MySQLdb.connect(
user=db_url.username,
passwd=db_url.password,
host=db_url.hostname,
port=db_url.port,
db=database_name,
)
class TestCrontabs(MariaDBTestCase, CrontabMixin):
def test_full_backup(self):
self._executeCrontabAtDate('mariadb-backup', '2050-01-01')
with gzip.open(
os.path.join(
self.computer_partition_root_path,
'srv',
'backup',
'mariadb-full',
'20500101000000.sql.gz',
),
'r') as dump:
self.assertIn('CREATE TABLE', dump.read())
def test_logrotate_and_slow_query_digest(self):
# slow query digest needs to run after logrotate, since it operates on the rotated
# file, so this tests both logrotate and slow query digest.
# run logrotate a first time so that it create state files
self._executeCrontabAtDate('logrotate', '2000-01-01')
# make two slow queries. We are using long-query-time=3, so the queries
# must take more than 3 seconds to be logged.
cnx = self.getDatabaseConnection()
with contextlib.closing(cnx):
cnx.query("SELECT SLEEP(3.1)")
cnx.store_result()
cnx.query("SELECT SLEEP(3.2)")
# slow query crontab depends on crontab for log rotation
# to be executed first.
self._executeCrontabAtDate('logrotate', '2050-01-01')
# this logrotate leaves the log for the day as non compressed
rotated_log_file = os.path.join(
self.computer_partition_root_path,
'srv',
'backup',
'logrotate',
'mariadb_slowquery.log-20500101',
)
self.assertTrue(os.path.exists(rotated_log_file))
# then crontab to generate slow query report is executed
self._executeCrontabAtDate('generate-mariadb-slow-query-report', '2050-01-01')
# and it creates a report for the day
slow_query_report = os.path.join(
self.computer_partition_root_path,
'srv',
'monitor',
'private',
'slowquery_digest',
'slowquery_digest.txt-2050-01-01.xz',
)
with lzma.open(slow_query_report, 'r') as f:
# this is the hash for our "select sleep(n)" slow query
self.assertIn("ID 0xF9A57DD5A41825CA", f.read())
# on next day execution of logrotate, log files are compressed
self._executeCrontabAtDate('logrotate', '2050-01-02')
self.assertTrue(os.path.exists(rotated_log_file + '.xz'))
self.assertFalse(os.path.exists(rotated_log_file))
# there's a promise checking that the threshold is not exceeded
# and it reports a problem since we set a threshold of 1 slow query
check_slow_query_promise_plugin = getPromisePluginParameterDict(
os.path.join(
self.computer_partition_root_path,
'etc',
'plugin',
'check-slow-query-pt-digest-result.py',
))
with self.assertRaises(subprocess.CalledProcessError) as error_context:
subprocess.check_output('faketime 2050-01-01 %s' % check_slow_query_promise_plugin['command'], shell=True)
self.assertEqual(
error_context.exception.output,
"""\
Threshold is lower than expected:
Expected total queries : 1.0 and current is: 2
Expected slowest query : 0.1 and current is: 3
""")
class TestMariaDB(MariaDBTestCase):
def test_utf8_collation(self):
cnx = self.getDatabaseConnection()
with contextlib.closing(cnx):
cnx.query(
"""
CREATE TABLE test_utf8_collation (
col1 CHAR(10)
)
""")
cnx.store_result()
cnx.query(
"""
insert into test_utf8_collation values ("à"), ("あ")
""")
cnx.store_result()
cnx.query(
"""
select * from test_utf8_collation where col1 = "a"
""")
self.assertEqual((('à',),), cnx.store_result().fetch_row(maxrows=2))
class TestMroonga(MariaDBTestCase):
def test_mroonga_plugin_loaded(self):
cnx = self.getDatabaseConnection()
with contextlib.closing(cnx):
cnx.query("show plugins")
plugins = cnx.store_result().fetch_row(maxrows=1000)
self.assertIn(
('Mroonga', 'ACTIVE', 'STORAGE ENGINE', 'ha_mroonga.so', 'GPL'),
plugins)
def test_mroonga_normalize_udf(self):
# example from https://mroonga.org/docs/reference/udf/mroonga_normalize.html#usage
cnx = self.getDatabaseConnection()
with contextlib.closing(cnx):
cnx.query(
"""
SELECT mroonga_normalize("ABCDあぃうぇ㍑")
""")
self.assertEqual((('abcdあぃうぇリットル',),),
cnx.store_result().fetch_row(maxrows=2))
if 0:
# this example fail with:
# OperationalError: (1123, "Can't initialize function 'mroonga_normalize'; mroonga_normalize(): nonexistent normalizer NormalizerMySQLUnicodeCIExceptKanaCI")
# same error on mroonga "official" docker images using mysql
# https://hub.docker.com/layers/groonga/mroonga/latest/images/sha256-e5a979801c95544ca3a1228d2c4d819820850e0162649553f2e94850e5e1c988?context=explore
# so it's probably OK to ignore
cnx.query(
"""
SELECT mroonga_normalize("aBｃＤあぃウェ㍑", "NormalizerMySQLUnicodeCIExceptKanaCIKanaWithVoicedSoundMark")
""")
self.assertEqual((('ABCDあぃうぇ㍑',),),
cnx.store_result().fetch_row(maxrows=2))
def test_mroonga_full_text_normalizer(self):
# example from https://mroonga.org//docs/tutorial/storage.html#how-to-specify-the-normalizer
cnx = self.getDatabaseConnection()
with contextlib.closing(cnx):
cnx.query("SET NAMES utf8")
cnx.store_result()
cnx.query(
"""
CREATE TABLE diaries (
day DATE PRIMARY KEY,
content VARCHAR(64) NOT NULL,
FULLTEXT INDEX (content) COMMENT 'normalizer "NormalizerAuto"'
) Engine=Mroonga DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
""")
cnx.store_result()
cnx.query(
"""INSERT INTO diaries VALUES ("2013-04-23", "ブラックコーヒーを飲んだ。")""")
cnx.store_result()
cnx.query(
"""
SELECT *
FROM diaries
WHERE MATCH (content) AGAINST ("+ふらつく" IN BOOLEAN MODE)
""")
self.assertEqual((), cnx.store_result().fetch_row(maxrows=2))
cnx.query(
"""
SELECT *
FROM diaries
WHERE MATCH (content) AGAINST ("+ﾌﾞﾗｯｸ" IN BOOLEAN MODE)
""")
self.assertEqual(
((datetime.date(2013, 4, 23), 'ブラックコーヒーを飲んだ。'),),
cnx.store_result().fetch_row(maxrows=2),
)
def test_mroonga_full_text_normalizer_TokenBigramSplitSymbolAlphaDigit(self):
# Similar to as ERP5's testI18NSearch with erp5_full_text_mroonga_catalog
cnx = self.getDatabaseConnection()
with contextlib.closing(cnx):
cnx.query(
"""
CREATE TABLE `full_text` (
`uid` BIGINT UNSIGNED NOT NULL,
`SearchableText` MEDIUMTEXT,
PRIMARY KEY (`uid`),
FULLTEXT `SearchableText` (`SearchableText`) COMMENT 'parser "TokenBigramSplitSymbolAlphaDigit"'
) ENGINE=mroonga
""")
cnx.store_result()
cnx.query(
"""
INSERT INTO full_text VALUES
(1, "Gabriel Fauré Quick brown fox jumps over the lazy dog"),
(2, "武者小路 実篤 Slow white fox jumps over the diligent dog."),
(3, "( - + )")""")
cnx.store_result()
cnx.query(
"""
SELECT uid
FROM full_text
WHERE MATCH (`full_text`.`SearchableText`) AGAINST ('*D+ Faure' IN BOOLEAN MODE)
""")
self.assertEqual(((1,),), cnx.store_result().fetch_row(maxrows=2))
cnx.query(
"""
SELECT uid
FROM full_text
WHERE MATCH (`full_text`.`SearchableText`) AGAINST ('*D+ 武者' IN BOOLEAN MODE)
""")
self.assertEqual(((2,),), cnx.store_result().fetch_row(maxrows=2))
cnx.query(
"""
SELECT uid
FROM full_text
WHERE MATCH (`full_text`.`SearchableText`) AGAINST ('*D+ +quick +fox +dog' IN BOOLEAN MODE)
""")
self.assertEqual(((1,),), cnx.store_result().fetch_row(maxrows=2))
def test_mroonga_full_text_stem(self):
# example from https://mroonga.org//docs/tutorial/storage.html#how-to-specify-the-token-filters
cnx = self.getDatabaseConnection()
with contextlib.closing(cnx):
cnx.query("SELECT mroonga_command('register token_filters/stem')")
self.assertEqual((('true',),), cnx.store_result().fetch_row(maxrows=2))
cnx.query(
"""
CREATE TABLE memos (
id INT NOT NULL PRIMARY KEY,
content TEXT NOT NULL,
FULLTEXT INDEX (content) COMMENT 'normalizer "NormalizerAuto", token_filters "TokenFilterStem"'
) Engine=Mroonga DEFAULT CHARSET=utf8
""")
cnx.store_result()
cnx.query(
"""INSERT INTO memos VALUES (1, "I develop Groonga"), (2, "I'm developing Groonga"), (3, "I developed Groonga")"""
)
cnx.store_result()
cnx.query(
"""
SELECT *
FROM memos
WHERE MATCH (content) AGAINST ("+develops" IN BOOLEAN MODE)
""")
self.assertEqual([
(1, "I develop Groonga"),
(2, "I'm developing Groonga"),
(3, "I developed Groonga"),
], list(sorted(cnx.store_result().fetch_row(maxrows=4))))
software/slapos-master/test/test/test_wcfs.py 0 → 100644
View file @ 5081f0b0
# Copyright (C) 2021 Nexedi SA and Contributors.
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
# the terms of any of the Free Software licenses or any of the Open Source
# Initiative approved licenses and Convey the resulting work. Corresponding
# source of such a combination shall include the source code for all other
# software used.
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options.
import json
import os.path
import unittest
from slapos.grid.utils import md5digest
from . import ERP5InstanceTestCase, setUpModule as _setUpModule
from .test_erp5 import TestPublishedURLIsReachableMixin
# skip tests when software release is built with wendelin.core 1.
def setUpModule():
_setUpModule()
cls = ERP5InstanceTestCase
if not os.path.exists(
os.path.join(
cls.slap.software_directory,
md5digest(cls.getSoftwareURL()),
'bin', 'wcfs')):
raise unittest.SkipTest("built with wendelin.core 1")
class TestWCFS(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
"""Test Wendelin Core File System
"""
__partition_reference__ = 'wcfs'
@classmethod
def getInstanceParameterDict(cls):
return {'_': json.dumps({'wcfs': {'enable': True}})}
def test_wcfs_accessible(self):
"""Verify that wcfs filesystem is basically accessible.
- we can read .wcfs/zurl
- its content is equal to published `serving-zurl`
"""
zurl = json.loads(
self.getComputerPartition('wcfs').getConnectionParameter('_')
)['serving-zurl']
mntpt = lookupMount(zurl)
zurl_ = readfile("%s/.wcfs/zurl" % mntpt)
self.assertEqual(zurl_, zurl)
# lookupMount returns /proc/mount entry for wcfs mounted to serve zurl.
def lookupMount(zurl):
for line in readfile('/proc/mounts').splitlines():
# <zurl> <mountpoint> fuse.wcfs ...
zurl_, mntpt, typ, _ = line.split(None, 3)
if typ != 'fuse.wcfs':
continue
if zurl_ == zurl:
return mntpt
raise KeyError("lookup mount %s: no /proc/mounts entry" % zurl)
# readfile returns content of file @path.
def readfile(path):
with open(path, 'r') as f:
return f.read()
software/slapos-sr-testing/software-py3.cfg
View file @ 5081f0b0
......@@ -13,6 +13,7 @@ extra-eggs +=
[template]
extra =
# The following list is for SR whose buildout runs only with Python 3.
galene ${slapos.test.galene-setup:setup}
helloworld ${slapos.test.helloworld-setup:setup}
jupyter ${slapos.test.jupyter-setup:setup}
monitor ${slapos.test.monitor-setup:setup}
......@@ -20,3 +21,4 @@ extra =
powerdns ${slapos.test.powerdns-setup:setup}
proftpd ${slapos.test.proftpd-setup:setup}
repman ${slapos.test.repman-setup:setup}
restic-rest-server ${slapos.test.restic_rest_server-setup:setup}
software/slapos-sr-testing/software.cfg
View file @ 5081f0b0
......@@ -5,6 +5,7 @@ extends =
../../component/curl/buildout.cfg
../../component/openssl/buildout.cfg
../../component/git/buildout.cfg
../../component/pycurl/buildout.cfg
../../component/faketime/buildout.cfg
../../component/pillow/buildout.cfg
../../component/python-cryptography/buildout.cfg
......@@ -98,6 +99,11 @@ setup = ${slapos-repository:location}/software/proftpd/test/
egg = slapos.test.re6stnet
setup = ${slapos-repository:location}/software/re6stnet/test/
[slapos.test.restic_rest_server-setup]
<= setup-develop-egg
egg = slapos.test.restic_rest_server
setup = ${slapos-repository:location}/software/restic-rest-server/test/
[slapos.test.seleniumserver-setup]
<= setup-develop-egg
egg = slapos.test.seleniumserver
......@@ -183,6 +189,11 @@ setup = ${slapos-repository:location}/software/repman/test/
egg = slapos.test.jscrawler
setup = ${slapos-repository:location}/software/jscrawler/test/
[slapos.test.galene-setup]
<= setup-develop-egg
egg = slapos.test.galene
setup = ${slapos-repository:location}/software/galene/test/
[slapos.core-repository]
<= git-clone-repository
repository = https://lab.nexedi.com/nexedi/slapos.core.git
......@@ -204,6 +215,7 @@ extra-eggs =
${lxml-python:egg}
${slapos.core-setup:egg}
${pillow-python:egg}
${pycurl:egg}
erp5.util
${python-pynacl:egg}
${python-cryptography:egg}
......@@ -225,6 +237,7 @@ extra-eggs =
${slapos.test.powerdns-setup:egg}
${slapos.test.proftpd-setup:egg}
${slapos.test.re6stnet-setup:egg}
${slapos.test.restic_rest_server-setup:egg}
${slapos.test.seleniumserver-setup:egg}
${slapos.test.slaprunner-setup:egg}
${slapos.test.jupyter-setup:egg}
......@@ -237,6 +250,7 @@ extra-eggs =
${slapos.test.repman-setup:egg}
${slapos.test.caucase-setup:egg}
${slapos.test.jscrawler-setup:egg}
${slapos.test.galene-setup:egg}
${slapos.test.html5as-setup:egg}
${slapos.test.html5as-base-setup:egg}
${slapos.test.fluentd-setup:egg}
......
software/slaprunner/buildout.hash.cfg
View file @ 5081f0b0
......@@ -18,7 +18,7 @@ md5sum = 8d6878ff1d2e75010c50a1a2b0c13b24
[template-runner]
filename = instance-runner.cfg
md5sum = 9f367deb7597957e7108bee719b78bcc
md5sum = ebdecea5c1653ed752e06fbc8be7e6b2
[template-runner-import-script]
filename = template/runner-import.sh.jinja2
......
software/slaprunner/instance-runner.cfg
View file @ 5081f0b0
......@@ -622,16 +622,16 @@ monitor-password = $${monitor-publish-parameters:monitor-password}
[slaprunner-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = slaprunner.py
config-hostname = $${slaprunner:ipv6}
config-host = $${slaprunner:ipv6}
config-port = $${slaprunner:runner_port}
[runner-sshd-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = runner-sshd.py
config-hostname = $${slap-network-information:global-ipv6}
config-host = $${slap-network-information:global-ipv6}
config-port = $${runner-sshd-port:port}
[slap-parameter]
......@@ -866,9 +866,9 @@ log = $${runnerdirectory:home}/instance/*/.slapgrid/log/instance.log $${runnerdi
[supervisord-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = supervisord.py
config-hostname = $${slaprunner:ipv4}
config-host = $${slaprunner:ipv4}
config-port = $${supervisord:port}
[slapos-supervisord-promise]
......@@ -879,9 +879,9 @@ config-command = ${buildout:bin-directory}/slapos node supervisorctl --cfg=$${sl
[slapos-proxy-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = slaproxy.py
config-hostname = $${slaprunner:ipv4}
config-host = $${slaprunner:ipv4}
config-port = $${slaprunner:proxy_port}
# XXX Monitor
......
software/theia/buildout.hash.cfg
View file @ 5081f0b0
......@@ -15,7 +15,7 @@
[instance-theia]
_update_hash_filename_ = instance-theia.cfg.jinja.in
md5sum = 11274189a4492204e046fef45fdd809c
md5sum = 557dec61b7bf58601051575234e7fd05
[instance]
_update_hash_filename_ = instance.cfg.in
......
software/theia/instance-theia.cfg.jinja.in
View file @ 5081f0b0
......@@ -77,16 +77,16 @@ instance-promises =
[theia-listen-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = $${:_buildout_section_name_}.py
config-hostname = $${theia-instance:ip}
config-host = $${theia-instance:ip}
config-port = $${theia-instance:port}
[frontend-listen-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = $${:_buildout_section_name_}.py
config-hostname = $${frontend-instance:ip}
config-host = $${frontend-instance:ip}
config-port = $${frontend-instance:port}
[remote-frontend-url-available-promise]
......@@ -107,10 +107,10 @@ config-check-secure = 1
[slapos-standalone-listen-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
# XXX promise plugins can not contain "slapos" in their names
name = standalone-listen-promise.py
config-hostname = $${slapos-standalone-instance:hostname}
config-host = $${slapos-standalone-instance:hostname}
config-port = $${slapos-standalone-instance:port}
[slapos-autorun-promise]
......
software/theia/test/dummy/software.cfg
View file @ 5081f0b0
......@@ -16,4 +16,4 @@ init =
json.dump(dict(os.environ), f)
[versions]
slapos.recipe.build = 0.46
slapos.recipe.build = 0.47
software/turnserver/buildout.hash.cfg
View file @ 5081f0b0
......@@ -15,12 +15,12 @@
[instance-cfg]
filename = instance.cfg.in
md5sum = 776c7de2054f78ba79382c22d85018be
md5sum = b43d5e8d1fc2d0eeb54f91cefe6a5bae
[template-turnserver]
filename = instance-turnserver.cfg.jinja2.in
md5sum = df8768f165036dbe0435bd1678b9deb3
md5sum = 7af3318d7249e9afe22436d9fe200159
[template-insecure-turnserver]
filename = instance-insecure-turnserver.cfg.jinja2.in
md5sum = e88678b9f322251a201860a13f4db6b8
md5sum = 3db65c3a16eb76ab438ac3817d1a5fea
software/turnserver/instance-insecure-turnserver.cfg.jinja2.in
View file @ 5081f0b0
{% set part_list = [] -%}
{% set server_name = slapparameter_dict.get('server-name', 'turn.example.com') -%}
{%- set parameter = dict(default_parameter_dict, **slapparameter_dict) %}
{%- set server_name = parameter['server-name'] %}
[directory]
recipe = slapos.cookbook:mkdirectory
......@@ -17,9 +18,11 @@ plugins = ${:etc}/plugin
recipe = slapos.cookbook:generate.password
bytes = 8
{% set turn_port = slapparameter_dict.get('port', 3478) -%}
{% set turn_tls_port = slapparameter_dict.get('tls-port', 5349) -%}
{% set listining_ip = slapparameter_dict.get('listening-ip', (ipv4 | list)[0]) -%}
{% set turn_port = parameter['port'] -%}
{% set turn_tls_port = parameter['tls-port'] -%}
# listening-ip parameter is mandatory
{% set listening_ip = slapparameter_dict['listening-ip'] -%}
[turnserver-config]
recipe = collective.recipe.template
user = nxdturn
......@@ -27,11 +30,11 @@ input = inline:
listening-port={{ turn_port }}
lt-cred-mech
realm={{ server_name }}
{% if slapparameter_dict.get('external-ip', '') %}
external-ip={{ slapparameter_dict['external-ip'] }}
{% if parameter['external-ip'] %}
external-ip={{ parameter['external-ip'] }}
{% endif %}
fingerprint
listening-ip={{ listining_ip }}
listening-ip={{ listening_ip }}
server-name={{ server_name }}
no-stdout-log
simple-log
......@@ -54,9 +57,9 @@ hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
[promise-check-turnserver-port]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = turnserver-port-listening.py
config-hostname = {{ listining_ip }}
config-host = {{ listening_ip }}
config-port = {{ turn_port }}
[publish-connection-information]
......
software/turnserver/instance-turnserver.cfg.jinja2.in
View file @ 5081f0b0
{% set part_list = [] -%}
{% set server_name = slapparameter_dict.get('server-name', 'turn.example.com') -%}
{%- set part_list = [] -%}
{%- set parameter = dict(default_parameter_dict, **slapparameter_dict) %}
{%- set server_name = parameter['server-name'] %}
[directory]
recipe = slapos.cookbook:mkdirectory
......@@ -31,9 +32,9 @@ mode = {{ mode }}
{% do part_list.append(section_name) -%}
{%- endmacro %}
{% if slapparameter_dict.get('ssl-key') and slapparameter_dict.get('ssl-crt') -%}
{{ simplefile('ssl-certificate', '${turnserver-ssl:certificate}', slapparameter_dict.get('ssl-crt')) }}
{{ simplefile('ssl-key', '${turnserver-ssl:key}', slapparameter_dict.get('ssl-key'), 600) }}
{% if parameter['ssl-key'] and parameter['ssl-crt'] -%}
{{ simplefile('ssl-certificate', '${turnserver-ssl:certificate}', parameter['ssl-crt']) }}
{{ simplefile('ssl-key', '${turnserver-ssl:key}', parameter['ssl-key'], 600) }}
{% else -%}
{% do part_list.append('gen-certificate') -%}
[gen-certificate]
......@@ -57,9 +58,9 @@ secret-file = ${directory:etc}/.turnsecret
command =
if [ ! -s "${:secret-file}" ]; then
cat <<EOF > ${:secret-file}
[turnserver]
secret = $("{{ parameter_dict['openssl'] }}/bin/openssl" rand -hex 32)
EOF
[turnserver]
secret = $("{{ parameter_dict['openssl'] }}/bin/openssl" rand -hex 32)
EOF
fi
chmod 600 ${:secret-file}
......@@ -68,9 +69,11 @@ recipe = slapos.cookbook:zero-knowledge.read
file-path = ${gen-secret:secret-file}
secret =
{% set turn_port = slapparameter_dict.get('port', 3478) -%}
{% set turn_tls_port = slapparameter_dict.get('tls-port', 5349) -%}
{% set listining_ip = slapparameter_dict.get('listening-ip', (ipv4 | list)[0]) -%}
{% set turn_port = parameter['port'] -%}
{% set turn_tls_port = parameter['tls-port'] -%}
# listening-ip parameter is mandatory
{% set listening_ip = slapparameter_dict['listening-ip'] -%}
[turnserver-config]
recipe = collective.recipe.template
input = inline:
......@@ -80,9 +83,9 @@ input = inline:
lt-cred-mech
use-auth-secret
static-auth-secret=${read-secret:secret}
listening-ip={{ listining_ip }}
{% if slapparameter_dict.get('external-ip', '') %}
external-ip={{ slapparameter_dict['external-ip'] }}
listening-ip={{ listening_ip }}
{% if parameter['external-ip'] %}
external-ip={{ parameter['external-ip'] }}
{% endif %}
server-name={{ server_name }}
realm={{ server_name }}
......@@ -120,16 +123,16 @@ hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
[promise-check-turnserver-port]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = turnserver-port-listening.py
config-hostname = {{ listining_ip }}
config-host = {{ listening_ip }}
config-port = {{ turn_port }}
[promise-check-turnserver-tls-port]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = turnserver-tls-port-listening.py
config-hostname = {{ listining_ip }}
config-host = {{ listening_ip }}
config-port = {{ turn_tls_port }}
[publish-connection-information]
......
software/turnserver/instance.cfg.in
View file @ 5081f0b0
......@@ -6,12 +6,11 @@ eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
[switch-softwaretype]
recipe = slapos.cookbook:softwaretype
default = $${dynamic-template-turnserver:rendered}
insecure = $${dynamic-template-insecure-turnserver:rendered}
recipe = slapos.cookbook:switch-softwaretype
RootSoftwareInstance = $${:default}
default = dynamic-template-turnserver:rendered
insecure = dynamic-template-insecure-turnserver:rendered
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
......@@ -38,13 +37,23 @@ context =
raw template_monitor ${monitor2-template:rendered}
raw logrotate_cfg ${template-logrotate-base:rendered}
$${:extra-context}
jsonkey default_parameter_dict :default-parameters
default-parameters =
{
"server-name" : "turn.example.com",
"ssl-key": "",
"ssl-crt": "",
"port": "3478",
"tls-port": "5349",
"external-ip": ""
# listening-ip parameter is mandatory
#"listening-ip": null,
}
[dynamic-template-turnserver-parameters]
openssl = ${openssl:location}
turnserver-location = ${coturn:location}
[dynamic-template-turnserver]
<= jinja2-template-base
template = ${template-turnserver:location}/${template-turnserver:filename}
......
software/turnserver/test/test.py
View file @ 5081f0b0
......@@ -60,6 +60,12 @@ class TurnServerTestCase(InstanceTestCase):
class TestServices(TurnServerTestCase):
@classmethod
def getInstanceParameterDict(cls):
return {
'listening-ip': cls._ipv4_address
}
def test_process_list(self):
hash_list = [
'software_release/buildout.cfg',
......@@ -138,7 +144,7 @@ class TestParameters(TurnServerTestCase):
'port': 3488,
'tls-port': 5369,
'external-ip': '127.0.0.1',
'listening-ip': '127.0.0.1'
'listening-ip': cls._ipv4_address
}
def test_turnserver_with_parameters(self):
......@@ -180,7 +186,7 @@ userdb=%(instance_path)s/srv/turndb
pidfile=%(instance_path)s/var/run/turnserver.pid
verbose""" % {'instance_path': self.partition_path,
'secret': secret,
'ipv4': '127.0.0.1',
'ipv4': self._ipv4_address,
'name': 'turn.site.com',
'external_ip': '127.0.0.1',
'port': 3488,
......@@ -193,6 +199,12 @@ verbose""" % {'instance_path': self.partition_path,
class TestInsecureServices(TurnServerTestCase):
@classmethod
def getInstanceParameterDict(cls):
return {
'listening-ip': cls._ipv4_address
}
@classmethod
def getInstanceSoftwareType(cls):
return 'insecure'
......
stack/erp5/buildout.cfg
View file @ 5081f0b0
......@@ -37,6 +37,7 @@ extends =
../../component/scikit-image/buildout.cfg
../../component/PyWavelets/buildout.cfg
../../component/subversion/buildout.cfg
../../component/tempstorage/buildout.cfg
../../component/tesseract/buildout.cfg
../../component/w3m/buildout.cfg
../../component/poppler/buildout.cfg
......@@ -497,6 +498,7 @@ eggs = ${neoppod:eggs}
# Zope
Zope2
${tempstorage:egg}
# Zope acquisition patch
Acquisition
# for runzeo
......@@ -708,7 +710,7 @@ uuid = 1.30
validictory = 1.1.0
xfw = 0.10
xupdate-processor = 0.4
selenium = 3.8.0
selenium = 3.14.1
scikit-image = 0.14.0
PyWavelets = 0.5.2
networkx = 2.1
......
stack/erp5/buildout.hash.cfg
View file @ 5081f0b0
......@@ -30,7 +30,7 @@ md5sum = 7d064777c1c4e7b275b255db4f4b1da9
[template-kumofs]
filename = instance-kumofs.cfg.in
md5sum = e91c0fbd0df441884f7422fa7976053c
md5sum = fed6dd2bdc389b4fc7e7b7ca32c5d4b6
[template-zope-conf]
filename = zope.conf.in
......@@ -50,7 +50,7 @@ md5sum = 1102c3e37a5a2e8aa2d8a2607ab633c8
[template-postfix]
filename = instance-postfix.cfg.in
md5sum = bbf84495576a3dbc522524895e9640ff
md5sum = 2a68a3e7c5c509cbd4cfa9e670ac91c7
[template-postfix-master-cf]
filename = postfix_master.cf.in
......@@ -70,15 +70,15 @@ md5sum = 7a14019abf48ca100eb94d9add20f5ae
[template]
filename = instance.cfg.in
md5sum = 8bd7f89b7c1e453ecc952659a01347e6
md5sum = bbef65b4edeb342f08309604ca3717d5
[template-erp5]
filename = instance-erp5.cfg.in
md5sum = f33e82cb68924decb77142b79dce4fab
md5sum = fcc8470824c448a56e2282c43b870cb5
[template-zeo]
filename = instance-zeo.cfg.in
md5sum = 0648e38bd5d3a15bb9f93264932740b9
md5sum = 79b6b422df512b5a075eba54a6895a01
[template-zodb-base]
filename = instance-zodb-base.cfg.in
......@@ -86,11 +86,11 @@ md5sum = bc821f9f9696953b10a03ad7b59a1936
[template-zope]
filename = instance-zope.cfg.in
md5sum = e495f7225a49f61c501ccc496a976d63
md5sum = 58ca95f6e0c067702a03fc3be66d50c1
[template-balancer]
filename = instance-balancer.cfg.in
md5sum = 8d3694226b6cbed961f6d608b6d6d294
md5sum = c6c1b3e4b2f3c6f256153dcfe9fbecad
[template-haproxy-cfg]
filename = haproxy.cfg.in
......
stack/erp5/instance-balancer.cfg.in
View file @ 5081f0b0
......@@ -303,9 +303,9 @@ certificate-and-key = ${directory:etc}/certificate-and-key-generated.pem
[{{ section('haproxy-promise') }}]
<= monitor-promise-base
# Check any haproxy port in ipv4, expect other ports and ipv6 to behave consistently
module = check_port_listening
module = check_socket_listening
name = haproxy.py
config-hostname = {{ ipv4 }}
config-host = {{ ipv4 }}
config-port = {{ haproxy_dict.values()[0][0] }}
[{{ section('publish') }}]
......
stack/erp5/instance-erp5.cfg.in
View file @ 5081f0b0
......@@ -71,7 +71,7 @@ recipe = slapos.cookbook:mkdirectory
bin = ${buildout:directory}/bin
service-on-watch = ${buildout:directory}/etc/service
srv = ${buildout:directory}/srv
tmp = {$buildout:directory}/tmp
tmp = ${buildout:directory}/tmp
backup-caucased = ${:srv}/backup/caucased
{% if not caucase_url -%}
......@@ -260,6 +260,7 @@ config-wsgi = {{ dumps(slapparameter_dict.get('wsgi', True)) }}
config-test-runner-enabled = {{ dumps(test_runner_enabled) }}
config-test-runner-node-count = {{ dumps(test_runner_node_count) }}
config-wcfs_enable = {{ dumps(wcfs_enable) }}
config-test-runner-configuration = {{ dumps(slapparameter_dict.get('test-runner', {})) }}
software-type = zope
{% set global_publisher_timeout = slapparameter_dict.get('publisher-timeout') -%}
......
stack/erp5/instance-kumofs.cfg.in
View file @ 5081f0b0
......@@ -86,8 +86,8 @@ rendered = ${directory:srv}/exporter.exclude
# Deploy zope promises scripts
[promise-template]
<= monitor-promise-base
module = check_port_listening
config-hostname = ${kumofs-instance:ip}
module = check_socket_listening
config-host = ${kumofs-instance:ip}
config-port = ${kumofs-instance:server-listen-port}
[promise-kumofs-server]
......
stack/erp5/instance-postfix.cfg.in
View file @ 5081f0b0
......@@ -80,9 +80,9 @@ wrapper-path = ${directory:run}/munnel
[{{ section('munnel-promise') }}]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = munnel.py
config-hostname = {{ ip }}
config-host = {{ ip }}
config-port = {{ milter_port }}
{% endif -%}
......@@ -262,9 +262,9 @@ wrapper-path = ${directory:run}/postfix-master
[{{ section('postfix-promise') }}]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = postfix.py
config-hostname = {{ ip }}
config-host = {{ ip }}
config-port = {{ tcpv4_port }}
[{{ section('promise-check-computer-memory') }}]
......
stack/erp5/instance-zeo.cfg.in
View file @ 5081f0b0
......@@ -55,9 +55,9 @@ post = test ! -s {{ "${" ~ zeo_section_name ~":pid-path}" }} || {{ bin_directory
[{{ section(zeo_section_name ~ "-promise") }}]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = zeo-{{ family }}.py
config-hostname = {{ "${" ~ zeo_section_name ~ ":ip}" }}
config-host = {{ "${" ~ zeo_section_name ~ ":ip}" }}
config-port = {{ "${" ~ zeo_section_name ~ ":port}" }}
{% endfor -%}
......@@ -89,9 +89,9 @@ tidstorage-wrapper = ${directory:services}/tidstoraged
[{{ section("promise-tidstorage") }}]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = tidstorage.py
config-hostname = ${tidstorage:ip}
config-host = ${tidstorage:ip}
config-port = ${tidstorage:port}
{% endif -%}
......
stack/erp5/instance-zope.cfg.in
View file @ 5081f0b0
......@@ -358,9 +358,9 @@ hash-existing-files =
[{{ section("promise-" ~ name) }}]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = {{ name }}.py
config-hostname = {{ ipv4 }}
config-host = {{ ipv4 }}
config-port = {{ port }}
{% if use_ipv6 -%}
......@@ -373,9 +373,9 @@ ipv4-port = {{ port }}
[{{ section("promise-tunnel-" ~ name) }}]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = {{ zope_tunnel_base_name }}.py
config-hostname = {{ '${' ~ zope_tunnel_section_name ~ ':ipv6}' }}
config-host = {{ '${' ~ zope_tunnel_section_name ~ ':ipv6}' }}
config-port = {{ '${' ~ zope_tunnel_section_name ~ ':ipv6-port}' }}
{% else -%}
{% do publish_list.append((ipv4 ~ ":" ~ port, thread_amount, webdav)) -%}
......@@ -470,6 +470,7 @@ environment-extra =
REAL_INSTANCE_HOME=${:instance-home}
SQLBENCH_PATH={{ parameter_dict['sqlbench_path'] }}
TEST_CA_PATH=${directory:ca-dir}
ERP5_TEST_RUNNER_CONFIGURATION=${test-runner-configuration:rendered}
{%- if wsgi %}
erp5_wsgi=1
{%- endif %}
......@@ -488,6 +489,12 @@ command-line =
--persistent_memcached_server_hostname=erp5-memcached-persistent
--persistent_memcached_server_port={{ port_dict['erp5-memcached-persistent'] }}
[test-runner-configuration]
recipe = slapos.recipe.template:jinja2
rendered = ${directory:etc}/${:_buildout_section_name_}.json
template = inline:
{{ json.dumps(slapparameter_dict['test-runner-configuration']) }}
[{{ section('runUnitTest') }}]
< = run-test-common
command-name = runUnitTest
......
stack/erp5/instance.cfg.in
View file @ 5081f0b0
......@@ -128,6 +128,7 @@ extra-context =
import urlparse urlparse
import hashlib hashlib
import itertools itertools
import json json
import-list =
file instance_zodb_base context:template-zodb-base
......
stack/erp5/zope-versions.cfg
View file @ 5081f0b0
......@@ -5,6 +5,7 @@ Acquisition = 2.13.12
DateTime = 2.12.8
DocumentTemplate = 2.13.6
ExtensionClass = 2.13.2
Jinja2 = 2.8.1
MarkupSafe = 1.1.1
Missing = 2.13.1
MultiMapping = 2.13.0
......
stack/erp5/ztk-versions.cfg
View file @ 5081f0b0
......@@ -74,6 +74,7 @@ zope.documenttemplate = 3.4.3
ClientForm = 0.2.10
distribute = 0.6.49
docutils = 0.7
Jinja2 = 2.5.5
# Newer versions of mechanize are not fully py24 compatible.
mechanize = 0.1.11
Paste = 1.7.5.1
......
stack/lamp/buildout.hash.cfg
View file @ 5081f0b0
......@@ -18,7 +18,7 @@ md5sum = 5c953c0f5d3376718eb9c4030288647a
[instance-apache-php]
filename = instance-apache-php.cfg.in
md5sum = a20bb27c0077b5a7252bda06ba7e069d
md5sum = 4afee4377fa9cbc1e4ff80647b2f279c
[instance-lamp]
filename = instance-lamp.cfg.jinja2.in
......
stack/lamp/instance-apache-php.cfg.in
View file @ 5081f0b0
......@@ -207,9 +207,9 @@ backend-url = ${apache-php-configuration:url}
[promise]
# Check any apache port in ipv4, expect other ports and ipv6 to behave consistently
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = apache-httpd-port-listening.py
config-hostname = ${apache-php-configuration:ip}
config-host = ${apache-php-configuration:ip}
config-port = ${apache-php-configuration:port}
[slap-parameter]
......
stack/monitor/README.md
View file @ 5081f0b0
......@@ -95,9 +95,9 @@ You will use slapos.cookbook:promise.plugin to generate your promise script into
[promise-check-site]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = check_site.py
config-hostname = ${publish:ipv6}
config-host = ${publish:ipv6}
config-port = 2020
config-foo = bar
......
stack/monitor/buildout.hash.cfg
View file @ 5081f0b0
......@@ -14,7 +14,7 @@
# not need these here).
[monitor2-template]
filename = instance-monitor.cfg.jinja2.in
md5sum = 84bc2cf29e34b48c51116d93e2be7636
md5sum = 200bb126dbfc33e5c5c165ae17bab64b
[monitor-httpd-conf]
_update_hash_filename_ = templates/monitor-httpd.conf.in
......
stack/monitor/instance-monitor.cfg.jinja2.in
View file @ 5081f0b0
......@@ -252,6 +252,11 @@ name = monitor-globalstate
frequency = */2 * * * *
command = {{ bin_directory }}/randomsleep 20 && ${monitor-globalstate-wrapper:wrapper-path}
[monitor-globalstate-first-run]
recipe = plone.recipe.command
command = ${monitor-globalstate-wrapper:wrapper-path}
stop-on-error = true
[monitor-configurator-cron-entry]
recipe = slapos.cookbook:cron.d
cron-entries = ${cron:cron-entries}
......@@ -388,6 +393,7 @@ update-command =
base-url = ${monitor-conf-parameters:base-url}
depends =
${monitor-globalstate-cron-entry:name}
${monitor-globalstate-first-run:recipe}
${monitor-configurator-cron-entry:name}
${monitor-collect-cron-entry:name}
${cron-entry-logrotate:name}
......
stack/resilient/buildout.hash.cfg
View file @ 5081f0b0
......@@ -14,7 +14,7 @@
# not need these here).
[pbsready]
filename = pbsready.cfg.in
md5sum = afb54d1aafd5854116028bc63501409b
md5sum = 9ceceeee21fa90837c887d2d6866859e
[pbsready-import]
filename = pbsready-import.cfg.in
......
stack/resilient/pbsready.cfg.in
View file @ 5081f0b0
......@@ -212,9 +212,9 @@ wrapper-path = $${basedirectory:scripts}/sshd-graceful
[sshd-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = sshd.py
config-hostname = $${slap-network-information:global-ipv6}
config-host = $${slap-network-information:global-ipv6}
config-port = $${sshd-port:port}
#----------------
......
stack/slapos.cfg
View file @ 5081f0b0
......@@ -195,10 +195,10 @@ slapos.extension.strip = 0.4
slapos.extension.shared = 1.0
slapos.libnetworkcache = 0.20
slapos.rebootstrap = 4.5
slapos.recipe.build = 0.46
slapos.recipe.build = 0.47
slapos.recipe.cmmi = 0.17
slapos.recipe.template = 4.6
slapos.toolbox = 0.119
slapos.toolbox = 0.122
stevedore = 1.21.0:whl
subprocess32 = 3.5.4
unicodecsv = 0.14.1
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7