Commit 5081f0b0 authored by Thomas Gambier's avatar Thomas Gambier 🚴🏼

Update Release Candidate

parents 58ab5601 1a1efa41
# ChromeDriver - Webdriver for Chrome
# http://chromedriver.chromium.org/
# https://chromedriver.chromium.org/
# This is a binary download with wrapper scripts.
[buildout]
......@@ -13,6 +13,10 @@ parts =
chromedriver-wrapper
[chromedriver-wrapper-91]
<= chromedriver-wrapper
wrapper-name = chromedriver-91
part = ${chromedriver-91:location}
[chromedriver-wrapper-2.41]
<= chromedriver-wrapper
......@@ -46,7 +50,7 @@ install =
[chromedriver]
<= chromedriver-2.41
<= chromedriver-91
[chromedriver-2.41]
<= chromedriver-download
......@@ -54,6 +58,11 @@ version = 2.41
# Supports Chrome v67-69
md5sum-x86_64 = fbd8b9561575054e0e7e9cc53b680a70
[chromedriver-91]
<= chromedriver-download
version = 91.0.4472.101
# Supports Chrome v91
md5sum-x86_64 = cc43ba0babbfff7f22b48165ec8e8c81
[chromedriver-download]
# Installs chromedriver ${version}.
......
......@@ -66,16 +66,27 @@ install =
))
os.fchmod(f.fileno(), 0o755)
[chromium-wrapper-91]
<= chromium-wrapper
wrapper-name = chromium-91
part = ${chromium-91:location}
[chromium-wrapper-69]
<= chromium-wrapper
wrapper-name = chromium-69
part = ${chromium-69:location}
[chromium]
<= chromium-91
[chromium-91]
<= chromium-download
version = 91.0.4472.114
[chromium]
<= chromium-69
revision_x86-64 = 870763
md5sum-x86_64 = 74eab41580469c2b8117cf396db823cb
generation-x86_64 = 1617926496067901
[chromium-69]
......@@ -111,6 +122,7 @@ x86-64 = https://www.googleapis.com/download/storage/v1/b/chromium-browser-snaps
library =
${atk:location}/lib
${at-spi2-atk:location}/lib
${at-spi2-core:location}/lib
${alsa:location}/lib
${cairo:location}/lib
${cups:location}/lib
......@@ -133,11 +145,14 @@ library =
${libXtst:location}/lib
${libXScrnSaver:location}/lib
${libXrandr:location}/lib
${libdrm:location}/lib
${libexpat:location}/lib
${libffi:location}/lib
${libpng:location}/lib
${libpng12:location}/lib
${libxcb:location}/lib
${libxkbcommon:location}/lib
${libxshmfence:location}/lib
${libxml2:location}/lib
${mesa:location}/lib
${nspr:location}/lib
......
......@@ -51,6 +51,11 @@ install =
))
os.fchmod(f.fileno(), 0o755)
[firefox-wrapper-78]
<= firefox-wrapper
wrapper-name = firefox-78
part = ${firefox-78:location}
[firefox-wrapper-68]
<= firefox-wrapper
wrapper-name = firefox-68
......@@ -95,6 +100,12 @@ cache-dir =
# would not be created.
<= firefox-68
[firefox-78]
<= firefox-download
version = 78.1.0esr
i686-md5sum = 09595a1b9a99d17a618a51bc1f971e5e
x86_64-md5sum = 06f4d488721ce7229d9a86cb4c6786f3
[firefox-68]
<= firefox-download
version = 68.0.2esr
......
# To be extended after mariadb's buildout.cfg
[mariadb]
[mariadb-10.4]
patches +=
${:_profile_base_location_}/mdev20693.patch#34ca907d6b36ba81d75bed118243f637
......@@ -9,6 +9,7 @@ extends =
../pkgconfig/buildout.cfg
../xorg/buildout.cfg
../xz-utils/buildout.cfg
../zlib/buildout.cfg
parts =
mesa
......@@ -16,32 +17,33 @@ parts =
[mesa]
recipe = slapos.recipe.cmmi
shared = true
url = ftp://ftp.freedesktop.org/pub/mesa/11.0.3/mesa-11.0.3.tar.xz
md5sum = bf9118bf0fbf360715cfe60baf7a1db5
url = https://archive.mesa3d.org/mesa-18.0.0.tar.xz
md5sum = c2a59fc5b56de3e197fa3a6023409e23
configure-options =
--enable-static
--disable-gles1
--disable-gles2
--disable-dri
--enable-dri
--disable-dri3
--enable-egl
--disable-gbm
--enable-gbm
--enable-sysfs
--disable-xvmc
--disable-vdpau
--disable-va
--enable-xlib-glx
--disable-shared-glapi
--disable-xlib-glx
--enable-shared-glapi
--disable-driglx-direct
--disable-gallium-llvm
--with-gallium-drivers=
--with-dri-drivers=
environment =
PATH=${autoconf:location}/bin:${bison:location}/bin:${flex:location}/bin:${pkgconfig:location}/bin:${xz-utils:location}/bin:%(PATH)s
PKG_CONFIG_PATH=${damageproto:location}/lib/pkgconfig:${fixesproto:location}/lib/pkgconfig:${glproto:location}/lib/pkgconfig:${kbproto:location}/lib/pkgconfig:${libX11:location}/lib/pkgconfig:${libXau:location}/lib/pkgconfig:${libXext:location}/lib/pkgconfig:${libexpat:location}/lib/pkgconfig:${libxcb:location}/lib/pkgconfig:${xdamage:location}/lib/pkgconfig:${xextproto:location}/lib/pkgconfig:${xfixes:location}/lib/pkgconfig:${xorg-libpthread-stubs:location}/lib/pkgconfig:${xproto:location}/lib/pkgconfig
PKG_CONFIG_PATH=${damageproto:location}/lib/pkgconfig:${fixesproto:location}/lib/pkgconfig:${glproto:location}/lib/pkgconfig:${kbproto:location}/lib/pkgconfig:${libX11:location}/lib/pkgconfig:${libXau:location}/lib/pkgconfig:${libXext:location}/lib/pkgconfig:${libexpat:location}/lib/pkgconfig:${libxcb:location}/lib/pkgconfig:${xdamage:location}/lib/pkgconfig:${xextproto:location}/lib/pkgconfig:${xfixes:location}/lib/pkgconfig:${xorg-libpthread-stubs:location}/lib/pkgconfig:${xproto:location}/lib/pkgconfig:${libdrm:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig
PYTHON2=${buildout:executable}
ACLOCAL=${automake:location}/bin/aclocal -I${libtool:location}/share/aclocal -I${pkgconfig:location}/share/aclocal
AUTOCONF=${autoconf:location}/bin/autoconf
AUTOMAKE=${automake:location}/bin/automake
LDFLAGS=-Wl,-rpath=${libdrm:location}/lib -Wl,-rpath=${zlib:location}/lib
make-options =
-j1
LIBTOOL=${libtool:location}/bin/libtool
......@@ -57,3 +59,4 @@ configure-options =
environment =
PATH=${pkgconfig:location}/bin:%(PATH)s
PKG_CONFIG_PATH=${damageproto:location}/lib/pkgconfig:${fixesproto:location}/lib/pkgconfig:${glproto:location}/lib/pkgconfig:${kbproto:location}/lib/pkgconfig:${libX11:location}/lib/pkgconfig:${libXau:location}/lib/pkgconfig:${libXext:location}/lib/pkgconfig:${libexpat:location}/lib/pkgconfig:${libxcb:location}/lib/pkgconfig:${mesa:location}/lib/pkgconfig:${xdamage:location}/lib/pkgconfig:${xextproto:location}/lib/pkgconfig:${xfixes:location}/lib/pkgconfig:${xorg-libpthread-stubs:location}/lib/pkgconfig:${xproto:location}/lib/pkgconfig
LDFLAGS=-Wl,-rpath=${libdrm:location}/lib -Wl,-rpath=${zlib:location}/lib
# SlapOS component for tempstorage.
# https://github.com/zopefoundation/tempstorage
[buildout]
extends =
../ZODB/buildout.cfg
../git/buildout.cfg
# tempstorage provides tempstorage<X> depending on ZODB major version.
#
# - tempstorage >= 4 works only with ZODB5 because ZODB commit protocol was changed
# https://github.com/zopefoundation/tempstorage/commit/5cc223ea
# - tempstorage <= 5.2 is vulnerable to data corruption in loadBefore
# https://github.com/zopefoundation/tempstorage/issues/8
# https://github.com/zopefoundation/tempstorage/pull/16
# - tempstorage 3-nxd provides loadBefore backports to tempstorage 3.
[tempstorage]
recipe = slapos.recipe.build
depends = ${ZODB:egg}
init =
# link/depend to tempstorage<ZODB.major>
zodb = self.buildout['ZODB']
zmajor = zodb['major']
tempstorage_x = self.buildout['tempstorage'+zmajor]
options['depends'] += '$${%s:egg}' % tempstorage_x.name
options['egg'] = tempstorage_x['egg']
# update [versions] from what is needed by tempstorage<X>
self.buildout.parse('[_tempstorage-versions]\n' + tempstorage_x['egg-versions'])
versions = self.buildout['versions']
versions.update(self.buildout['_tempstorage-versions'])
# propagate updated [versions] -> easy_install
# (buildout does this in Buildout constructor)
import zc.buildout.easy_install
zc.buildout.easy_install.default_versions(versions)
# tempstorage5 is plain upstream egg
[tempstorage5]
recipe = zc.recipe.egg:eggs
egg = tempstorage
eggs = ${:egg}
egg-versions =
tempstorage = 5.2
# tempstorage4-wc2 is tempstorage 3 + backports for loadBefore fixes
[tempstorage4-wc2]
recipe = zc.recipe.egg:develop
setup = ${tempstorage4-wc2-repository:location}
egg = tempstorage
egg-versions =
tempstorage =
[tempstorage4-wc2-repository]
recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.com/nexedi/tempstorage.git
branch = 3-nxd
revision= 77b49295db78
location = ${buildout:parts-directory}/tempstorage
git-executable = ${git:location}/bin/git
# tempstorage4 is the same as tempstorage4-wc2 because of loadBefore fixes.
[tempstorage4]
<= tempstorage4-wc2
[download-tessdata.py]
filename = download-tessdata.py
md5sum = 2d283a6d8662d6bb8c9de7b26162b702
md5sum = 1f6cf2797f0daf9dec16c1d3a8c44907
......@@ -19,9 +19,13 @@ def post_make_hook(options, buildout, env):
for url in options['tessdata-urls'].splitlines():
url, _, md5sum = url.partition('#')
if url:
destination = os.path.join(
options['tessdata-location'],
os.path.basename(url),
)
download(
url,
md5sum=md5sum,
path=os.path.join(options['tessdata-location'],
os.path.basename(url)),
path=destination,
)
os.chmod(destination, 0o750)
......@@ -38,6 +38,12 @@ GO = ${go:exe}
# wcfs needs this:
[gowork]
cpkgpath += ${zlib:location}/lib/pkgconfig
[wendelin.core-env]
# `pkg-config --libs zlib` emits only -L, but not -Wl,-rpath
# better set it via gowork:environment when "VAR += ..." support is there
# environment +=
# CGO_LDFLAGS += -Wl,-rpath=${zlib:location}/lib
CGO_LDFLAGS += -Wl,-rpath=${zlib:location}/lib
[wendelin.core-repository]
......
......@@ -2,20 +2,25 @@
extends =
../autoconf/buildout.cfg
../automake/buildout.cfg
../bison/buildout.cfg
../bzip2/buildout.cfg
../dash/buildout.cfg
../freetype/buildout.cfg
../gnutls/buildout.cfg
../icu/buildout.cfg
../intltool/buildout.cfg
../libtool/buildout.cfg
../libuuid/buildout.cfg
../libxml2/buildout.cfg
../libxslt/buildout.cfg
../meson/buildout.cfg
../ninja/buildout.cfg
../openssl/buildout.cfg
../patch/buildout.cfg
../perl/buildout.cfg
../perl-XML-Parser/buildout.cfg
../pkgconfig/buildout.cfg
../xz-utils/buildout.cfg
../zlib/buildout.cfg
./buildout.hash.cfg
......@@ -652,3 +657,43 @@ pkg_config_depends = ${libX11:location}/lib/pkgconfig:${xorgproto:pkg_config_dep
environment =
PKG_CONFIG_PATH=${:pkg_config_depends}
PATH=${pkgconfig:location}/bin:%(PATH)s
[libdrm]
recipe = slapos.recipe.cmmi
shared = true
configure-command = ${meson:location}/bin/meson builddir --libdir=lib -Dprefix=@@LOCATION@@
make-binary = ${ninja:location}/bin/ninja -C builddir
url = https://dri.freedesktop.org/libdrm/libdrm-2.4.106.tar.xz
md5sum = 4e316ae1966a1a63c31a3885313a8fb8
pkg_config_depends = ${pciaccess:location}/lib/pkgconfig
environment =
PKG_CONFIG_PATH=${:pkg_config_depends}
PATH=${xz-utils:location}/bin:${pkgconfig:location}/bin:${ninja:location}/bin:%(PATH)s
LDFLAGS=-Wl,-rpath=${pciaccess:location}/lib -Wl,-rpath=@@LOCATION@@/lib
[libxshmfence]
recipe = slapos.recipe.cmmi
shared = true
url = https://www.x.org/releases/individual/lib/libxshmfence-1.3.tar.gz
md5sum = ab3940af0bd3d3cc91eb35ecd33a779a
pkg_config_depends = ${xorgproto:pkg_config_depends}:${xorgproto:location}/share/pkgconfig
environment =
PKG_CONFIG_PATH=${:pkg_config_depends}
PATH=${pkgconfig:location}/bin:%(PATH)s
[libxkbcommon]
recipe = slapos.recipe.cmmi
shared = true
configure-command = ${meson:location}/bin/meson builddir --libdir=lib -Dprefix=@@LOCATION@@ -Denable-wayland=false -Denable-docs=false
make-binary = ${ninja:location}/bin/ninja -C builddir
url = https://xkbcommon.org/download/libxkbcommon-1.3.0.tar.xz
md5sum = 00b5275ec1309a1d427a645de5861605
pkg_config_depends = ${libxcb:pkg_config_depends}:${libxcb:location}/lib/pkgconfig:${libxml2:location}/lib/pkgconfig:${xz-utils:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig:${icu4c:location}/lib/pkgconfig
environment =
PKG_CONFIG_PATH=${:pkg_config_depends}
PATH=${xz-utils:location}/bin:${pkgconfig:location}/bin:${ninja:location}/bin:${bison:location}/bin:%(PATH)s
LDFLAGS=-Wl,-rpath=${libxcb:location}/lib -Wl,-rpath=${libxml2:location}/lib -Wl,-rpath=${xz-utils:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${icu4c:location}/lib -Wl,-rpath=@@LOCATION@@/lib
......@@ -25,9 +25,8 @@
#
##############################################################################
from zc.buildout.buildout import Buildout
from zc.buildout.buildout import Buildout, MissingOption, MissingSection
from zc.buildout import UserError
class SubBuildout(Buildout):
"""Run buildout in buildout, partially copied from infrae.buildout
......@@ -71,8 +70,21 @@ class Recipe:
self.buildout = buildout
self.options = options
self.name = name
self.software_type = buildout["slap-configuration"]["slap-software-type"]
section, key = self.options[self.software_type].split(":")
try:
self.software_type = buildout["slap-configuration"]["slap-software-type"]
except (MissingSection, MissingOption):
raise UserError("The section to retrieve slap partition parameters "
"(with slapos.cookbook:slapconfiguration recipe or a derived one) "
"must be named [slap-configuration].")
try:
section, key = self.options[self.software_type].split(":")
except MissingOption:
raise MissingOption("This software type (%s) isn't mapped. RootSoftwareInstance "
"is the default software type." % self.software_type)
except ValueError:
raise UserError("The software types in the section [%s] must be separated "
"by a colon such as: 'section:key', where key is usually 'rendered'. "
"Don't use: ${section:key}" % self.name)
self.base = self.buildout[section][key]
def install(self):
......
......@@ -46,8 +46,8 @@ md5sum = ae4a0043414336a521b524d9c95f1c68
[template-pullrdiffbackup]
filename = instance-pullrdiffbackup.cfg.in
md5sum = c1f00c251298c2ab7fd095e7c4571d3b
md5sum = f2e6f30a0e8228cbfb93eaaae10fe884
[template]
filename = instance.cfg.in
md5sum = 42021b325159dff29e4bd4e33b8ff2f3
md5sum = 3df515def97f1e8a9f181514ae6ef03f
......@@ -194,9 +194,9 @@ virtual-depends =
[nginx-listen-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = nginx_listen.py
config-hostname = $${nginx-configuration:ip}
config-host = $${nginx-configuration:ip}
config-port = $${nginx-configuration:port}
[nginx-configuration]
......@@ -206,7 +206,7 @@ output = $${directory:etc}/nginx.cfg
mode = 0600
access_log = $${directory:log}/nginx-access.log
error_log = $${directory:log}/nginx-error.log
ip = $${slap-network-information:global-ipv6}
ip = {{ partition_ipv6 }}
port = 9443
ssl_key = $${directory:ssl}/nginx.key
ssl_csr = $${directory:ssl}/nginx.csr
......
......@@ -13,32 +13,21 @@ rendered = $${buildout:parts-directory}/$${:_buildout_section_name_}/$${:filenam
filename = instance-pullrdiffbackup.cfg
extensions = jinja2.ext.do
context =
key slave_instance_list instance-parameter:slave-instance-list
key slave_instance_list slap-configuration:slave-instance-list
# partition_ipv6 is the random ipv6 allocated to the local partition
key partition_ipv6 slap-configuration:ipv6-random
[switch-softwaretype]
recipe = slapos.cookbook:softwaretype
default = $${:pullrdiffbackup}
# pullrdiffbackup = ${template-pullrdiffbackup:output}
pullrdiffbackup = $${dynamic-template-pullrdiffbackup:rendered}
recipe = slapos.cookbook:switch-softwaretype
RootSoftwareInstance = $${:pullrdiffbackup}
pullrdiffbackup = dynamic-template-pullrdiffbackup:rendered
[slap-connection]
# part to migrate to new - separated words
computer-id = $${slap_connection:computer_id}
partition-id = $${slap_connection:partition_id}
server-url = $${slap_connection:server_url}
software-release-url = $${slap_connection:software_release_url}
key-file = $${slap_connection:key_file}
cert-file = $${slap_connection:cert_file}
# [slap-parameter]
# slave-instance-list = []
[instance-parameter]
[slap-configuration]
# Fetches parameters defined in SlapOS Master for this instance.
# Always the same.
recipe = slapos.cookbook:slapconfiguration.serialised
computer = $${slap_connection:computer_id}
partition = $${slap_connection:partition_id}
url = $${slap_connection:server_url}
key = $${slap_connection:key_file}
cert = $${slap_connection:cert_file}
computer = $${slap-connection:computer-id}
partition = $${slap-connection:partition-id}
url = $${slap-connection:server-url}
key = $${slap-connection:key-file}
cert = $${slap-connection:cert-file}
......@@ -102,5 +102,3 @@ mode = 0644
[versions]
gunicorn = 19.1.1
plone.recipe.command = 1.1
PyRSS2Gen = 1.1
......@@ -22,15 +22,15 @@ md5sum = 5784bea3bd608913769ff9a8afcccb68
[profile-caddy-frontend]
filename = instance-apache-frontend.cfg.in
md5sum = 28220d18308313d49a38d39c61a7e769
md5sum = 8507a2ace2f789b92c522cc62ca5aace
[profile-caddy-replicate]
filename = instance-apache-replicate.cfg.in
md5sum = ec421617d4932e8e1ae1e55a41fbfaab
md5sum = 8beb438d06bbb0f917d13e182fb12d17
[profile-slave-list]
_update_hash_filename_ = templates/apache-custom-slave-list.cfg.in
md5sum = a745fb8d61a7e2646e3aa55edf73e5a6
md5sum = 613f777a08373088cbaf7f51fd18ea70
[profile-replicate-publish-slave-information]
_update_hash_filename_ = templates/replicate-publish-slave-information.cfg.in
......@@ -106,7 +106,7 @@ md5sum = 38792c2dceae38ab411592ec36fff6a8
[profile-kedifa]
filename = instance-kedifa.cfg.in
md5sum = 225ce18b9218de4169327a206051a92e
md5sum = 16901e9eeb0d4f87e708ad91e7756f12
[template-backend-haproxy-rsyslogd-conf]
_update_hash_filename_ = templates/backend-haproxy-rsyslogd.conf.in
......
......@@ -535,9 +535,9 @@ context =
[trafficserver-promise-listen-port]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = trafficserver-port-listening.py
config-hostname = ${trafficserver-variable:local-ip}
config-host = ${trafficserver-variable:local-ip}
config-port = ${trafficserver-variable:input-port}
[trafficserver-ctl]
......@@ -655,44 +655,44 @@ config-verification-script = ${promise-helper-last-configuration-state:rendered}
[promise-caddy-frontend-v4-https]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = caddy_frontend_ipv4_https.py
config-hostname = {{ instance_parameter_dict['ipv4-random'] }}
config-host = {{ instance_parameter_dict['ipv4-random'] }}
config-port = ${configuration:port}
[promise-caddy-frontend-v4-http]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = caddy_frontend_ipv4_http.py
config-hostname = {{ instance_parameter_dict['ipv4-random'] }}
config-host = {{ instance_parameter_dict['ipv4-random'] }}
config-port = ${configuration:plain_http_port}
[promise-caddy-frontend-v6-https]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = caddy_frontend_ipv6_https.py
config-hostname = {{ instance_parameter_dict['ipv6-random'] }}
config-host = {{ instance_parameter_dict['ipv6-random'] }}
config-port = ${configuration:port}
[promise-caddy-frontend-v6-http]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = caddy_frontend_ipv6_http.py
config-hostname = {{ instance_parameter_dict['ipv6-random'] }}
config-host = {{ instance_parameter_dict['ipv6-random'] }}
config-port = ${configuration:plain_http_port}
[promise-backend-haproxy-http]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = backend_haproxy_http.py
config-hostname = {{ instance_parameter_dict['ipv4-random'] }}
config-host = {{ instance_parameter_dict['ipv4-random'] }}
config-port = ${backend-haproxy-configuration:http-port}
[promise-backend-haproxy-https]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = backend_haproxy_https.py
config-hostname = {{ instance_parameter_dict['ipv4-random'] }}
config-host = {{ instance_parameter_dict['ipv4-random'] }}
config-port = ${backend-haproxy-configuration:https-port}
[backend-haproxy-configuration]
......@@ -981,9 +981,9 @@ context =
[promise-slave-introspection-https]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = slave_introspection_https.py
config-hostname = {{ instance_parameter_dict['ipv6-random'] }}
config-host = {{ instance_parameter_dict['ipv6-random'] }}
config-port = ${frontend-configuration:slave-introspection-https-port}
[logrotate-entry-slave-introspection]
......
......@@ -865,14 +865,14 @@ rendered = ${directory:etc}/nginx-rejected-slave.conf
[promise-rejected-slave-publish-ip-port]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = rejected-slave-publish-ip-port-listening.py
config-hostname = ${rejected-slave-publish-configuration:ip}
config-host = ${rejected-slave-publish-configuration:ip}
config-port = ${rejected-slave-publish-configuration:port}
[rejected-slave-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
module = check_file_state
name = rejected-slave.py
config-filename = ${rejected-slave-json:rendered}
......
......@@ -193,9 +193,9 @@ template = inline:
[promise-expose-csr_id-ip-port]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = expose-csr_id-ip-port-listening.py
config-hostname = ${expose-csr_id-configuration:ip}
config-host = ${expose-csr_id-configuration:ip}
config-port = ${expose-csr_id-configuration:port}
[expose-csr_id]
......
......@@ -633,9 +633,9 @@ template = inline:
[promise-expose-csr_id-ip-port]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = expose-csr_id-ip-port-listening.py
config-hostname = ${expose-csr_id-configuration:ip}
config-host = ${expose-csr_id-configuration:ip}
config-port = ${expose-csr_id-configuration:port}
[expose-csr_id]
......
......@@ -79,6 +79,8 @@ else:
os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
# XXX Keep using slapos node instance --all, because of missing promises
SlapOSInstanceTestCase.slap._force_slapos_node_instance_all = True
# ports chosen to not collide with test systems
HTTP_PORT = '11080'
......
......@@ -78,7 +78,7 @@
"type": "string"
},
"developer-list": {
"description": "List of logins which should get the Developper role (required to modify portal_components' content), defaulting to inituser-login's value",
"description": "List of logins which should get the Developer role (required to modify portal_components' content), defaulting to inituser-login's value",
"items": {
"pattern": "^\\S+$",
"type": "string"
......@@ -215,7 +215,7 @@
"type": "integer"
},
"timerserver-interval": {
"description": "Timerserver tick perdiod, in seconds, or 0 to disable",
"description": "Timerserver tick period, in seconds, or 0 to disable",
"default": 5,
"type": "number"
},
......@@ -439,7 +439,7 @@
"type": "string"
},
{
"description": "Override value (parameter for maching nodes).",
"description": "Override value (parameter for matching nodes).",
"type": [
"integer",
"string"
......@@ -517,7 +517,7 @@
"type": "boolean"
},
"node-count": {
"description": "Number of tests this instance can execute in parrallel. This must be at least equal to the number of nodes configured on testnode running the test",
"description": "Number of tests this instance can execute in parallel. This must be at least equal to the number of nodes configured on testnode running the test",
"default": 3,
"type": "integer"
},
......@@ -525,6 +525,101 @@
"description": "Number of extra databases this instance tests will need.",
"default": 3,
"type": "integer"
},
"selenium": {
"default": {
"target": "firefox"
},
"examples": [
{
"target": "selenium-server",
"server-url": "https://selenium.example.com",
"desired-capabilities": {
"browserName": "firefox",
"version": "68.0.2esr",
"acceptInsecureCerts": true
}
},
{
"target": "selenium-server",
"server-url": "https://selenium.example.com",
"desired-capabilities": {
"browserName": "chrome",
"version": "91.0.4472.101"
}
}
],
"oneOf": [
{
"type": "object",
"title": "Selenium Server",
"description": "Configuration for Selenium server",
"additionalProperties": false,
"required": [
"desired-capabilities",
"server-url",
"target"
],
"properties": {
"target": {
"description": "Target system",
"type": "string",
"const": "selenium-server"
},
"server-url": {
"description": "URL of the selenium server",
"type": "string",
"format": "uri"
},
"verify-server-certificate": {
"description": "Verify the SSL/TLS certificate of the selenium server when using HTTPS",
"type": "boolean",
"default": true
},
"server-ca-certificate": {
"description": "PEM encoded bundle of CA certificates to verify the SSL/TLS certificate of the selenium server when using HTTPS",
"type": "string",
"default": "Root certificates from http://certifi.io/en/latest/"
},
"desired-capabilities": {
"description": "Desired browser capabilities",
"required": [
"browserName"
],
"type": "object",
"properties": {
"browserName": {
"description": "Name of the browser being used",
"type": "string",
"examples": [
"firefox",
"chrome",
"safari"
]
},
"version": {
"description": "The browser version",
"type": "string"
}
}
}
}
},
{
"type": "object",
"title": "Firefox",
"description": "Configuration for using firefox running as a sub-process",
"additionalProperties": false,
"properties": {
"target": {
"description": "Target system",
"const": "firefox",
"type": "string",
"default": "firefox"
}
}
}
]
}
},
"type": "object"
......
......@@ -218,6 +218,45 @@ class TestBalancerPorts(ERP5InstanceTestCase):
]))
class TestSeleniumTestRunner(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
"""Test ERP5 can be instantiated with selenium server for test runner.
"""
__partition_reference__ = 'sel'
@classmethod
def getInstanceParameterDict(cls):
return {
'_':
json.dumps({
'test-runner': {
'selenium': {
"target": "selenium-server",
"server-url": "https://example.com",
"verify-server-certificate": False,
"desired-capabilities": {
"browserName": "firefox",
"version": "68.0.2esr",
}
}
}
})
}
def test_test_runner_configuration_json_file(self):
runUnitTest_script, = glob.glob(
self.computer_partition_root_path + "/../*/bin/runUnitTest.real")
config_file = None
with open(runUnitTest_script) as f:
for line in f:
if 'ERP5_TEST_RUNNER_CONFIGURATION' in line:
_, config_file = line.split('=')
assert config_file
with open(config_file.strip()) as f:
self.assertEqual(
f.read(),
json.dumps(json.loads(self.getInstanceParameterDict()['_'])['test-runner']))
class TestDisableTestRunner(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
"""Test ERP5 can be instantiated without test runner.
"""
......
......@@ -18,4 +18,4 @@ md5sum = 307663d73ef3ef94b02567ecd322252e
[template-default]
filename = instance-default.cfg
md5sum = d10958c62d0be8d0d051300d695f4f44
md5sum = 24cc143b1886d443a4c29dcb8147a01c
......@@ -243,9 +243,9 @@ instance-promises =
[shellinabox-frontend-listen-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = $${:_buildout_section_name_}.py
config-hostname = $${shellinabox-frontend:hostname}
config-host = $${shellinabox-frontend:hostname}
config-port = $${shellinabox-frontend:port}
[testnode-log-frontend-promise]
......
Tests for Galene software release
##############################################################################
#
# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from setuptools import setup, find_packages
version = '0.0.1.dev0'
name = 'slapos.test.galene'
long_description = open("README.md").read()
setup(
name=name,
version=version,
description="Test for SlapOS' Galene",
long_description=long_description,
long_description_content_type='text/markdown',
maintainer="Nexedi",
maintainer_email="info@nexedi.com",
url="https://lab.nexedi.com/nexedi/slapos",
packages=find_packages(),
install_requires=[
'slapos.core',
'slapos.libnetworkcache',
'requests',
],
zip_safe=True,
test_suite='test',
)
##############################################################################
#
# Copyright (c) 2019 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from __future__ import unicode_literals
import os
import requests
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
os.path.abspath(
os.path.join(os.path.dirname(__file__), '../software.cfg')))
class TestGalene(SlapOSInstanceTestCase):
__partition_reference__ = 'G'
def setUp(self):
self.connection_parameters = self.computer_partition.getConnectionParameterDict()
def test_url_get(self):
resp = requests.get(self.connection_parameters['url'], verify=False)
self.assertEqual(requests.codes.ok, resp.status_code)
......@@ -134,7 +134,7 @@ git-executable = ${git:location}/bin/git
<= git-repository
repository = https://lab.nexedi.com/nexedi/gitlab-ce.git
# 9.5.10 + NXD patches:
revision = v9.5.10-8-gc290e22a08cb
revision = v9.5.10-9-g69b0ffae00bf
location = ${buildout:parts-directory}/gitlab
[gitlab-shell-repository]
......
......@@ -14,4 +14,4 @@
# not need these here).
[instance-profile]
filename = instance.cfg.in
md5sum = dcb9d2f540e0e397c9346c8b0c05f233
md5sum = c771dee1ef9aedad7c6ebf9418afe08e
......@@ -87,7 +87,7 @@ wrapper-path = ${directory:service}/helloweb-${:kind}
# promise, that checks that helloweb service is alive
[helloweb-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = helloweb-${:kind}.py
{# macro to instantiate service of `kind` to listen on `port` #}
......@@ -102,8 +102,8 @@ port = {{ port }}
[helloweb-{{ kind }}-promise]
<= helloweb-promise
kind = {{ kind }}
config-hostname= ${helloweb-{{ kind }}:ipv6}
config-port = {{ port }}
config-host = ${helloweb-{{ kind }}:ipv6}
config-port = {{ port }}
{% endmacro %}
# services instantiation
......
......@@ -18,10 +18,19 @@ recipe = slapos.recipe.template:jinja2
template = {{ template_instance_html5as_target }}
rendered = ${buildout:directory}/${:filename}
filename = instance-html5as.cfg
context =
context =
section buildout buildout
section parameter_list profile-common
# partition_ipv6 is the random ipv6 allocated to the local partition
key partition_ipv6 slap-configuration:ipv6-random
[switch-softwaretype]
recipe = slapos.cookbook:softwaretype
default = ${instance-html5as:rendered}
recipe = slapos.cookbook:switch-softwaretype
RootSoftwareInstance = ${:default}
default = instance-html5as:rendered
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration
computer = ${slap-connection:computer-id}
partition = ${slap-connection:partition-id}
url = ${slap-connection:server-url}
......@@ -70,7 +70,7 @@ scgi_temp_path = ${:tmp}/scgi_temp_path
nb_workers = 2
# Network
ip = ${slap-network-information:global-ipv6}
ip = {{ partition_ipv6 }}
port = 8081
access_url = http://[${:ip}]:${:port}
......
......@@ -27,7 +27,7 @@ recipe = slapos.recipe.template:jinja2
rendered = ${buildout:directory}/template.cfg
template = ${:_profile_base_location_}/${:filename}
filename = instance.cfg.in
md5sum = 5a6ebc126bcad3cdff1b51fb51f82a35
md5sum = 310aab063704794065ee3bc8f81fdc70
mode = 0644
context =
section buildout buildout
......@@ -43,7 +43,7 @@ context =
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_update_hash_filename_}
_update_hash_filename_ = instance_html5as.cfg.in
md5sum = 4a8c98cc5ca141f78f14fb9cec203cb8
md5sum = 9b7ed68551cac5967915979383238669
mode = 0644
[template_nginx_conf]
......@@ -71,8 +71,3 @@ mode = 0644
recipe = zc.recipe.egg
eggs =
plone.recipe.command
# Pin versions of eggs used that are not already pinned by stack/slapos.cfg
[versions]
slapos.recipe.template = 4.4
plone.recipe.command = 1.1
......@@ -17,11 +17,11 @@
[template-cfg]
filename = instance.cfg.in
md5sum = 0a7aceffa5222e88125b72da42ddedd7
md5sum = 9e486efe4ab1aba8cb72b04f6c6da8ad
[instance_html5as]
_update_hash_filename_ = instance_html5as.cfg.in
md5sum = ec808dba866d85f7d37b85d75c13df2b
md5sum = 191ec2a8b967a3944971709365bdcd2d
[template_nginx_conf]
_update_hash_filename_ = templates/nginx_conf.in
......@@ -45,4 +45,4 @@ md5sum = 1c0ee16966e1fcdb3fd11c09f12ee2b2
[template_instance_replicate]
_update_hash_filename_ = instance_replicate.cfg.in
md5sum = 38d1d352307f79c9c99bf2a80a5c76b8
md5sum = 7ff7e11d05145115f53564ec1af205ef
......@@ -9,11 +9,9 @@ offline = true
[profile-common]
nginx_location = {{ nginx_location }}
dash_location = {{ dash_location }}
tar_location = {{ tar_location }}
curl_location = {{ curl_location }}
template_nginx_conf = {{ template_nginx_conf_target }}
template_mime_types = {{ template_mime_types_target }}
template_launcher = {{ template_launcher_target }}
template_launcher = {{ template_launcher_target }}
template_index_html = {{ template_index_html_target }}
template_graceful = {{ template_graceful_target }}
template_monitor = {{ template_monitor }}
......@@ -26,24 +24,43 @@ filename = instance-html5as.cfg
context =
section buildout buildout
section parameter_list profile-common
# partition_ipv6 is the random ipv6 allocated to the local partition
key partition_ipv6 slap-configuration:ipv6-random
# slapparameter_dict: dictionary of all parameters
key slapparameter_dict slap-configuration:configuration
jsonkey default_parameter_dict :default-parameters
default-parameters =
{
"title": "",
"download_url": null,
"port": 8081,
"monitor-httpd-port": 8197
}
[instance-replicate]
recipe = slapos.recipe.template:jinja2
extensions = jinja2.ext.do
template = {{ template_instance_replicate }}
rendered = ${buildout:directory}/${:filename}
filename = instance-replicate-html5as.cfg
context =
section buildout buildout
section parameter_list profile-common
key slapparameter_dict slap-parameters:configuration
key slapparameter_dict slap-configuration:configuration
jsonkey default_parameter_dict :default-parameters
default-parameters =
{
"download_url": null,
"replicate-quantity": 1
}
[switch-softwaretype]
recipe = slapos.cookbook:softwaretype
default = ${instance-html5as:rendered}
replicate = ${instance-replicate:rendered}
recipe = slapos.cookbook:switch-softwaretype
RootSoftwareInstance = ${:default}
default = instance-html5as:rendered
replicate = instance-replicate:rendered
# Section needed to be added manually here to retrieve parameters
[slap-parameters]
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration
computer = ${slap-connection:computer-id}
partition = ${slap-connection:partition-id}
......
......@@ -3,10 +3,13 @@
# Deploy html5as instance
#
#############################
# parameter_dict: a dictionary with the default parameters from instance.cfg.in
# replaces the values with the parameters of the instance request if there are any
{% set parameter_dict = dict(default_parameter_dict, **slapparameter_dict) %}
[buildout]
parts =
nginx_conf
downloader
mime_types
launcher
nginx-graceful
......@@ -58,7 +61,6 @@ service = ${directory:etc}/service
log = ${directory:var}/log
run = ${directory:var}/run
backup = ${directory:srv}/backup
data = ${directory:srv}/html5as
[tempdirectory]
recipe = slapos.cookbook:mkdirectory
......@@ -76,8 +78,8 @@ scgi_temp_path = ${:tmp}/scgi_temp_path
nb_workers = 2
# Network
ip = ${slap-network-information:global-ipv6}
port = ${slap-parameter:port}
ip = {{ partition_ipv6 }}
port = {{ parameter_dict['port'] }}
access_url = http://[${:ip}]:${:port}
# Paths
......@@ -88,15 +90,13 @@ path_access_log = ${basedirectory:log}/nginx.access.log
path_error_log = ${basedirectory:log}/nginx.error.log
path_tmp = ${tempdirectory:tmp}
# Docroot
docroot = ${basedirectory:data}
default_index = ${basedirectory:data}/index.html
docroot = ${downloader:location}
default_index = ${:docroot}/index.html
# Config files
path_nginx_conf = ${directory:etc}/nginx.conf
path_mime_types = ${directory:etc}/mime_types
# Binaries
path_shell = {{ parameter_list['dash_location'] }}/bin/dash
curl-binary = {{ parameter_list['curl_location'] }}/bin/curl
tar-binary = {{ parameter_list['tar_location'] }}/bin/tar
# Executables
bin_launcher = ${basedirectory:service}/launcher
......@@ -130,28 +130,48 @@ context =
# Command to put content in the docroot
[downloader]
recipe = plone.recipe.command
# This section will fail if the command fails.
stop-on-error = true
recipe = slapos.recipe.build
# Path where the recipe stores any produced file,
# it will be automatically removed at the beginning of "install".
location = ${directory:srv}/html5as
# All the keys in this section will be available as a dict called "self.options"
# We add: or '', otherwise jinja2 will render a 'None' string
url = {{ parameter_dict['download_url'] or '' }}
default_index_html = ${default_index_html:rendered}
# If a tarball is passed as a parameter in download url
# it's content will be served by the instance.
# If the parameter is not provided it fallback to the default template
command =
rm -rf ${html5as:docroot}/*;
URL="${slap-parameter:download_url}";
if [ -n "$URL" ];
then
${html5as:curl-binary} -Lks $URL | ${html5as:tar-binary} xzv -C ${html5as:docroot} --strip-components 1;
else
cp ${default_index_html:rendered} ${html5as:docroot}/;
fi
install =
import os, shutil
buildout_offline = self.buildout['buildout']['offline']
try:
# Allow to do self.download() which can only be used in "online" mode
self.buildout['buildout']['offline'] = 'false'
if self.options['url']:
# Use fonctions from the slapos.recipe.build repository
# Download a file from a URL to a temporary path
file = self.download(self.options['url'])
# Create a directory and extract the file that are compressed inside
extract_dir = self.extract(file)
# Return the right directory path
workdir = guessworkdir(extract_dir)
# Recursively copy directory
self.copyTree(workdir, location)
else:
# Create directory and copy the default template inside
os.makedirs(location)
shutil.copy(self.options['default_index_html'], location)
finally:
# reset the parameter
self.buildout['buildout']['offline'] = buildout_offline
[default_index_html]
recipe = slapos.recipe.template:jinja2
template = {{ parameter_list['template_index_html'] }}
rendered = ${directory:srv}/index.html
title = {{ parameter_dict['title'] }}
context =
key title slap-parameter:title
key title :title
### Nginx Graceful
[nginx-graceful]
......@@ -165,19 +185,19 @@ context =
# Port Listening checking promise
[port-listening-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = nginx-port-listening.py
config-hostname = ${html5as:ip}
config-host = ${html5as:ip}
config-port = ${html5as:port}
# Use a port different from the default one in order to be able to
# use it in a SlapOS webrunner or a Theia SlapOS Runner
[monitor-instance-parameter]
monitor-httpd-port = 8197
monitor-httpd-port = {{ parameter_dict['monitor-httpd-port'] }}
# Monitor Stack also provides logrotate stack. We only need to extend
# the logrotate-entry-base defined in instance-logrotate-base.cfg.in .
# More parameters can be added following the logrotate-entry-base section
# More parameters can be added following the logrotate-entry-base section
[logrotate-entry-nginx]
<= logrotate-entry-base
name = nginx
......@@ -191,7 +211,7 @@ recipe = slapos.cookbook:publish
# be deployed. The parameters needed for accessing monitoring will be published
<= monitor-publish
server_url = ${html5as:access_url}
title = Title ${slap-parameter:title}!
title = Title {{ parameter_dict['title'] }}!
# Add dependency to the promise so that frontend sections are processed
# and there is no need to declare the new part in buildout:parts
server-cdn-url = ${html5as-frontend-promise:url}
......@@ -202,7 +222,7 @@ server-cdn-url = ${html5as-frontend-promise:url}
<= slap-connection
# Recipe used to make requests
recipe = slapos.cookbook:requestoptional
name = HTM5AS frontend
name = HTML5AS frontend
# Specify the software url of the frontend software release
software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
# It is not a dedicated instance but an instance allocated on a shared instance
......@@ -220,8 +240,3 @@ name = html5as-http-frontend.py
url = ${html5as-frontend:connection-secure_access}
config-url = ${:url}
config-check-secure = 1
[slap-parameter]
title =
download_url =
port = 8081
{% set replicate_quantity = slapparameter_dict.pop('replicate-quantity', '1') | int %}
{%- set parameter_dict = dict(default_parameter_dict, **slapparameter_dict) %}
{%- set replicate_quantity = int(parameter_dict['replicate-quantity']) %}
# Set default title and port for each replicate based on requested quantity
{%- for i in range(1, replicate_quantity + 1) %}
{%- do parameter_dict.setdefault("title-%d" % i, "") %}
{%- do parameter_dict.setdefault("port-%d" % i, 8081 + i) %}
{%- do parameter_dict.setdefault("monitor-httpd-port-%d" % i, 8197 + i) %}
{%- endfor %}
# Standard buildout section
[buildout]
parts =
publish-connection-information
......@@ -7,6 +16,11 @@ eggs-directory = {{ buildout['eggs-directory'] }}
develop-eggs-directory = {{ buildout['develop-eggs-directory'] }}
offline = true
################################
# Sections to Request instances
################################
# Macro section sharing request parameters
[instance-request-base]
<= slap-connection
recipe = slapos.cookbook:request
......@@ -17,29 +31,25 @@ software-type = default
# What parameter are neede to be retrieved
return = server_url server-cdn-url monitor-setup-url
# Provided parameters
config-title = ${slap-parameter:title}
config-download_url = ${slap-parameter:download_url}
# We add: or '', otherwise jinja2 will render a 'None' string
config-download_url = {{ parameter_dict['download_url'] or '' }}
# Create request section in a loop.
{% for i in range(1, replicate_quantity + 1) %}
# Request a normal html5as instance
[instance-{{ i }}]
<= instance-request-base
# Name of the instance
name = instance-html5as-{{ i }}
config-port = ${slap-parameter:port-{{ i }}}
config-title = ${slap-parameter:title-{{ i }}}
{% if "sla-%s-computer-guid" % i in slapparameter_dict -%}
sla-computer_guid = {{ slapparameter_dict["sla-%s-computer-guid" % i] }}
config-port = {{ parameter_dict["port-%s" % i] }}
config-title = {{ parameter_dict["title-%s" % i] }}
config-monitor-httpd-port = {{ parameter_dict["monitor-httpd-port-%s" % i] }}
{% if "sla-%s-computer-guid" % i in parameter_dict -%}
sla-computer_guid = {{ parameter_dict["sla-%s-computer-guid" % i] }}
{% endif -%}
{% endfor %}
[slap-parameter]
download_url =
{% for i in range(1, replicate_quantity + 1) %}
title-{{ i }} =
port-{{ i }} = 808{{ i }}
{% endfor %}
# Publish information to connect to the two instances
[publish-connection-information]
recipe = slapos.cookbook:publish
{% for i in range(1, replicate_quantity + 1) %}
......
......@@ -13,8 +13,6 @@ extends =
# In this example we extend needed components for html5as.
../../component/nginx/buildout.cfg
../../component/dash/buildout.cfg
../../component/tar/buildout.cfg
../../component/curl/buildout.cfg
parts =
# Call installation of slapos.cookbook egg defined in stack/slapos.cfg (needed
......@@ -36,8 +34,6 @@ context =
section buildout buildout
key nginx_location nginx:location
key dash_location dash:location
key curl_location curl:location
key tar_location tar:location
key template_nginx_conf_target template_nginx_conf:target
key template_mime_types_target template_mime_types:target
key template_launcher_target template_launcher:target
......@@ -79,7 +75,3 @@ mode = 0644
[template_instance_replicate]
<= download-base
# Pin versions of eggs used that are not already pinned by stack/slapos.cfg
[versions]
slapos.recipe.template = 4.4
......@@ -22,8 +22,8 @@ md5sum = 87781e6bcb523bb8434888d5f984f36c
[template-validator]
filename = instance-validator.cfg.in
md5sum = 2be286e367e37ce9e504170cd3a08007
md5sum = 9d12472bb2e337d3cc18f2cc6f235425
[template]
filename = instance.cfg.in
md5sum = 2b4d33e9ef1082dd4d6a53f55b391772
md5sum = 94fc13254c819cba33b03f30251bc469
......@@ -49,15 +49,15 @@ recipe = slapos.recipe.template
url = ${template-tomcat-configuration:output}
output = $${basedirectory:catalina_conf}/server.xml
mode = 0600
ip = $${slap-network-information:global-ipv6}
ip = {{ partition_ipv6 }}
port = 8899
scheme = https
[tomcat-listen-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = tomcat_listen.py
config-hostname = $${tomcat-configuration:ip}
config-host = $${tomcat-configuration:ip}
config-port = $${tomcat-configuration:port}
#################################
......
......@@ -6,29 +6,26 @@ eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
[switch-softwaretype]
recipe = slapos.cookbook:softwaretype
default = $${:validator}
validator = ${template-validator:output}
[slap-connection]
# part to migrate to new - separated words
computer-id = $${slap_connection:computer_id}
partition-id = $${slap_connection:partition_id}
server-url = $${slap_connection:server_url}
software-release-url = $${slap_connection:software_release_url}
key-file = $${slap_connection:key_file}
cert-file = $${slap_connection:cert_file}
[dynamic-template-validator]
recipe = slapos.recipe.template:jinja2
template = ${template-validator:output}
rendered = $${buildout:parts-directory}/$${:_buildout_section_name_}/$${:filename}
filename = instance-validator.cfg
context =
# partition_ipv6 is the random ipv6 allocated to the local partition
key partition_ipv6 slap-configuration:ipv6-random
# [slap-parameter]
# slave-instance-list = []
[switch-softwaretype]
recipe = slapos.cookbook:switch-softwaretype
RootSoftwareInstance = $${:validator}
validator = dynamic-template-validator:rendered
[instance-parameter]
[slap-configuration]
# Fetches parameters defined in SlapOS Master for this instance.
# Always the same.
recipe = slapos.cookbook:slapconfiguration.serialised
computer = $${slap_connection:computer_id}
partition = $${slap_connection:partition_id}
url = $${slap_connection:server_url}
key = $${slap_connection:key_file}
cert = $${slap_connection:cert_file}
computer = $${slap-connection:computer-id}
partition = $${slap-connection:partition-id}
url = $${slap-connection:server-url}
key = $${slap-connection:key-file}
cert = $${slap-connection:cert-file}
......@@ -49,5 +49,4 @@ mode = 0644
# 1.3.4nxd2 is invalid version string, thus pached version string is not '1.3.4nxd2+SlapOSPatched001'
# but '1.3.4nxd2-SlapOSPatched001'.
gunicorn = 19.1.1
plone.recipe.command = 1.1
inotifyx = 0.2.2
......@@ -19,7 +19,7 @@ md5sum = 6c17361a49cfc47564063b867aab6e8c
[template-jscrawler]
filename = instance-jscrawler.cfg.jinja2.in
md5sum = bdf9e67077cd5c3c140974bcc56422ad
md5sum = fece076231740b414612da5ef3a1685a
[template-jscrawler-builder]
filename = template-jscrawler.builder.sh.in
......
......@@ -60,9 +60,9 @@ log = ${httpd-wrapper:log-file}
[httpd-listen-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = httpd-listen.py
config-hostname = ${httpd-wrapper:host}
config-host = ${httpd-wrapper:host}
config-port = ${httpd-wrapper:port}
[jscrawler-wrapper]
......
......@@ -23,6 +23,31 @@
"example": "https://softinst1234.host.vifib.net/"
},
"test-runner": {
"default": {
"target": "firefox"
},
"examples": [
{
"target": "selenium-server",
"server-url": "https://selenium.example.com",
"desired-capabilities": {
"browserName": "firefox",
"version": "68.0.2esr",
"acceptInsecureCerts": true
}
},
{
"target": "selenium-server",
"server-url": "https://selenium.example.com",
"desired-capabilities": {
"browserName": "chrome",
"version": "91.0.4472.101"
}
},
{
"target": "node"
}
],
"oneOf": [
{
"type": "object",
......@@ -46,14 +71,14 @@
"format": "uri"
},
"verify-server-certificate": {
"description": "Verify the SSL/TLS Certificats of the selenium server when using HTTPS",
"description": "Verify the SSL/TLS certificate of the selenium server when using HTTPS",
"type": "boolean",
"default": true
},
"server-ca-certificate": {
"description": "PEM encoded bundle of CA Certificates to verify the SSL/TLS Certificate of the selenium server when using HTTPS",
"description": "PEM encoded bundle of CA certificates to verify the SSL/TLS certificate of the selenium server when using HTTPS",
"type": "string",
"default": "root certificates from http://certifi.io/en/latest/"
"default": "Root certificates from http://certifi.io/en/latest/"
},
"desired-capabilities": {
"description": "Desired browser capabilities",
......@@ -63,8 +88,13 @@
"type": "object",
"properties": {
"browserName": {
"description": "Name of the browser being used, for example firefox, chrome",
"type": "string"
"description": "Name of the browser being used",
"type": "string",
"examples": [
"firefox",
"chrome",
"safari"
]
},
"version": {
"description": "The browser version",
......@@ -77,7 +107,7 @@
{
"type": "object",
"title": "Firefox",
"description": "Configuration for Firefox",
"description": "Configuration for using firefox running as a sub-process",
"additionalProperties": false,
"properties": {
"target": {
......
......@@ -19,7 +19,7 @@ md5sum = 0d34ff81779115bf899f7bc752877b70
[template-kvm]
filename = instance-kvm.cfg.jinja2
md5sum = bf0c01ac7493693bb57ebef00bb20fa0
md5sum = d0f96be4e80b96e6ac33f6d474767b13
[template-kvm-cluster]
filename = instance-kvm-cluster.cfg.jinja2.in
......@@ -47,7 +47,7 @@ md5sum = b617d64de73de1eed518185f310bbc82
[template-nbd]
filename = instance-nbd.cfg.jinja2
md5sum = 6ea26f88252bf899c966d0f5675e7176
md5sum = 259e06f289f68297e0609e4ab1af8e86
[template-ansible-promise]
filename = template/ansible-promise.in
......@@ -55,7 +55,7 @@ md5sum = b7e87479a289f472b634a046b44b5257
[template-kvm-run]
filename = template/template-kvm-run.in
md5sum = be750fb62f7057c97dd6c6887b2149cc
md5sum = a502782244d1be536b732ebb40725f47
[template-kvm-controller]
filename = template/kvm-controller-run.in
......@@ -75,7 +75,7 @@ md5sum = fb330a796fadb6cd5c85217f80a42af3
[template-httpd]
filename = instance-kvm-http.cfg.in
md5sum = d657884d02105deffddee0edae4b50a6
md5sum = f4bcde62e008c2da9c65617ba7f73f08
[image-download-controller]
_update_hash_filename_ = template/image-download-controller.py
......
......@@ -217,7 +217,7 @@
},
"enable-device-hotplug": {
"title": "Enable device hotplug mode",
"description": "If yes, this will allow to Create devices like CPU and Memory in hotplug mode without restart the VM. Operatin System should be configured to Online new created devices.",
"description": "Allows to increase amount of RAM (ram-size) and CPU (cmp-count) without restart of the VM process, up to defined maximums (ram-max-size and cpu-max-count). Operation system have to support online addition of RAM and CPU.",
"type": "boolean",
"default": false
},
......
......@@ -65,7 +65,7 @@ stop-on-error = true
[httpd-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = apache-httpd.py
config-hostname = ${apache-conf:ip}
config-host = ${apache-conf:ip}
config-port = ${apache-conf:port}
......@@ -5,7 +5,7 @@
"properties": {
"enable-device-hotplug": {
"title": "Enable device hotplug mode",
"description": "If yes, this will allow to Create devices like CPU and Memory in hotplug mode without restart the VM. Operatin System should be configured to Online new created devices.",
"description": "Allows to increase amount of RAM (ram-size) and CPU (cmp-count) without restart of the VM process, up to defined maximums (ram-max-size and cpu-max-count). Operation system have to support online addition of RAM and CPU.",
"type": "boolean",
"default": false
},
......
......@@ -437,13 +437,17 @@ disk-path = ${directory:srv}/virtual.${slap-parameter:disk-format}
pid-file-path = ${kvm-controller-parameter-dict:pid-file}
socket-path = ${kvm-controller-parameter-dict:socket-path}
enable-device-hotplug = ${kvm-controller-parameter-dict:enable-device-hotplug}
smp-count = ${kvm-controller-parameter-dict:cpu-count}
{%- set enable_device_hotplug = slapparameter_dict.get('enable-device-hotplug', 'false').lower() == 'true' %}
smp-max-count = {{ cpu_max_count }}
ram-size = ${kvm-controller-parameter-dict:ram-size}
ram-max-size = {{ ram_max_size }}
{%- if enable_device_hotplug %}
init-ram-size = 1024
init-smp-count = 1
{%- else %}
init-ram-size = ${kvm-controller-parameter-dict:ram-size}
init-smp-count = ${kvm-controller-parameter-dict:cpu-count}
{%- endif %}
mac-address = ${create-mac:mac-address}
tap-mac-address = ${create-tap-mac:mac-address}
......@@ -582,9 +586,9 @@ command-line = ${kvm-controller:rendered}
[kvm-vnc-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = vnc_promise.py
config-hostname = ${kvm-parameter-dict:vnc-ip}
config-host = ${kvm-parameter-dict:vnc-ip}
config-port = ${kvm-parameter-dict:vnc-port}
[kvm-disk-image-corruption-bin]
......@@ -704,9 +708,9 @@ wrapper = ${directory:bin}/websockify
[novnc-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = novnc_promise.py
config-hostname = ${novnc-instance:ip}
config-host = ${novnc-instance:ip}
config-port = ${novnc-instance:port}
......@@ -797,9 +801,9 @@ hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
[httpd-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = httpd.py
config-hostname = ${httpd:host}
config-host = ${httpd:host}
config-port = ${httpd:port}
{% endif %}
......
......@@ -65,9 +65,9 @@ key = ${gen-passwd:passwd}
[onetimeupload-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = onetimeupload_promise.py
config-hostname = ${onetimeupload-instance:ip}
config-host = ${onetimeupload-instance:ip}
config-port = ${onetimeupload-instance:port}
[publish-connection-information]
......
......@@ -37,9 +37,9 @@ mac_address = '{{ parameter_dict.get("mac-address") }}'
tap_mac_address = '{{ parameter_dict.get("tap-mac-address") }}'
tap_ipv6_addr = '{{ parameter_dict.get("tap-ipv6-addr") }}'
numa_list = '{{ parameter_dict.get("numa", "") }}'.split()
ram_size = {{ parameter_dict.get("ram-size") }}
ram_max_size = '{{ parameter_dict.get("ram-max-size") }}'
init_ram_size = {{ parameter_dict.get("init-ram-size") }}
init_smp_count = {{ parameter_dict.get("init-smp-count") }}
pid_file_path = '{{ parameter_dict.get("pid-file-path") }}'
external_disk_number = {{ parameter_dict.get("external-disk-number") }}
external_disk_size = {{ parameter_dict.get("external-disk-size") }}
......@@ -78,7 +78,6 @@ if not disk_info_list:
{%- endfor %}
})
smp_count = {{ parameter_dict.get("smp-count") }}
smp_max_count = {{ parameter_dict.get("smp-max-count") }}
machine_options = '{{ parameter_dict.get("machine-options", "") }}'.strip()
cpu_model = '{{ parameter_dict.get("cpu-model") }}'.strip()
......@@ -267,12 +266,8 @@ if use_tap == 'true':
tap_interface, vhost),
'-device', 'virtio-net-pci,netdev=lan%s,mac=%s' % (number, tap_mac_address)]
if enable_device_hotplug != 'true':
smp = '%s,maxcpus=%s' % (smp_count, smp_max_count)
ram = '%sM,slots=128,maxmem=%sM' % (ram_size, ram_max_size)
else:
smp = '1,maxcpus=%s' % smp_max_count
ram = '%sM,slots=128,maxmem=%sM' % (init_ram_size, ram_max_size)
smp = '%s,maxcpus=%s' % (init_smp_count, smp_max_count)
ram = '%sM,slots=128,maxmem=%sM' % (init_ram_size, ram_max_size)
kvm_argument_list = [qemu_path,
'-enable-kvm', '-smp', smp, '-name', vm_name, '-m', ram, '-vga', 'std',
......
......@@ -43,6 +43,7 @@ setup(name=name,
install_requires=[
'slapos.core',
'slapos.cookbook',
'slapos.toolbox',
'slapos.libnetworkcache',
'erp5.util',
'supervisor',
......
......@@ -46,6 +46,7 @@ import time
import shutil
import sys
from slapos.qemuqmpclient import QemuQMPWrapper
from slapos.proxy.db_version import DB_VERSION
from slapos.recipe.librecipe import generateHashFromFiles
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
......@@ -60,6 +61,8 @@ if has_kvm:
os.path.abspath(
os.path.join(os.path.dirname(__file__), '..',
'software%s.cfg' % ("-py3" if six.PY3 else ""))))
# XXX Keep using slapos node instance --all, because of missing promises
InstanceTestCase.slap._force_slapos_node_instance_all = True
else:
setUpModule, InstanceTestCase = None, unittest.TestCase
......@@ -203,6 +206,108 @@ i0:whitelist-firewall-{hash} RUNNING""",
)
@skipUnlessKvm
class TestMemoryManagement(InstanceTestCase, KvmMixin):
__partition_reference__ = 'i'
def getKvmProcessInfo(self, switch_list):
return_list = []
with self.slap.instance_supervisor_rpc as instance_supervisor:
kvm_pid = [q for q in instance_supervisor.getAllProcessInfo()
if 'kvm-' in q['name']][0]['pid']
kvm_process = psutil.Process(kvm_pid)
get_next = False
for entry in kvm_process.cmdline():
if get_next:
return_list.append(entry)
get_next = False
elif entry in switch_list:
get_next = True
return kvm_pid, return_list
def test(self):
kvm_pid_1, info_list = self.getKvmProcessInfo(['-smp', '-m'])
self.assertEqual(
['1,maxcpus=2', '1024M,slots=128,maxmem=1536M'],
info_list
)
self.rerequestInstance({
'ram-size': '1536',
'cpu-count': '2',
})
self.slap.waitForInstance(max_retry=10)
kvm_pid_2, info_list = self.getKvmProcessInfo(['-smp', '-m'])
self.assertEqual(
['2,maxcpus=3', '1536M,slots=128,maxmem=2048M'],
info_list
)
# assert that process was restarted
self.assertNotEqual(kvm_pid_1, kvm_pid_2, "Unexpected: KVM not restarted")
def tearDown(self):
self.rerequestInstance({})
self.slap.waitForInstance(max_retry=10)
def test_enable_device_hotplug(self):
def getHotpluggedCpuRamValue():
qemu_wrapper = QemuQMPWrapper(os.path.join(
self.computer_partition_root_path, 'var', 'qmp_socket'))
ram_mb = sum(
[q['size']
for q in qemu_wrapper.getMemoryInfo()['hotplugged']]) / 1024 / 1024
cpu_count = len(
[q['CPU'] for q in qemu_wrapper.getCPUInfo()['hotplugged']])
return {'cpu_count': cpu_count, 'ram_mb': ram_mb}
kvm_pid_1, info_list = self.getKvmProcessInfo(['-smp', '-m'])
self.assertEqual(
['1,maxcpus=2', '1024M,slots=128,maxmem=1536M'],
info_list
)
self.assertEqual(
getHotpluggedCpuRamValue(),
{'cpu_count': 0, 'ram_mb': 0}
)
parameter_dict = {
'enable-device-hotplug': 'true',
# to avoid restarts the max RAM and CPU has to be static
'ram-max-size': '2048',
'cpu-max-count': '4',
}
self.rerequestInstance(parameter_dict)
self.slap.waitForInstance(max_retry=2)
kvm_pid_2, info_list = self.getKvmProcessInfo(['-smp', '-m'])
self.assertEqual(
['1,maxcpus=4', '1024M,slots=128,maxmem=2048M'],
info_list
)
self.assertEqual(
getHotpluggedCpuRamValue(),
{'cpu_count': 0, 'ram_mb': 0}
)
self.assertNotEqual(kvm_pid_1, kvm_pid_2, "Unexpected: KVM not restarted")
parameter_dict.update(**{
'ram-size': '1536',
'cpu-count': '2'
})
self.rerequestInstance(parameter_dict)
self.slap.waitForInstance(max_retry=10)
kvm_pid_3, info_list = self.getKvmProcessInfo(['-smp', '-m'])
self.assertEqual(
['1,maxcpus=4', '1024M,slots=128,maxmem=2048M'],
info_list
)
self.assertEqual(kvm_pid_2, kvm_pid_3, "Unexpected: KVM restarted")
self.assertEqual(
getHotpluggedCpuRamValue(),
{'cpu_count': 1, 'ram_mb': 512}
)
class MonitorAccessMixin(object):
def sqlite3_connect(self):
sqlitedb_file = os.path.join(
......@@ -223,7 +328,7 @@ class MonitorAccessMixin(object):
}
return db.execute(
"SELECT reference, xml, connection_xml, partition_reference,"
" software_release, requested_state, software_type"
" software_release, requested_state, software_type"
" FROM partition%s"
" WHERE slap_state='busy'" % DB_VERSION).fetchall()
finally:
......
......@@ -142,6 +142,23 @@ class EdgeSlaveMixin(MonitorTestMixin):
instance_max_retry = 20
expected_connection_parameter_dict = {}
@classmethod
def setUpClass(cls):
# XXX we run these tests with --all as a workaround for the fact that after
# requesting new shared instances we don't have promise to wait for the
# processing of these shared instances to be completed.
# The sequence is something like this:
# - `requestEdgetestSlaves` will request edgetest partition
# - first `waitForInstance` will process the edgetest partition, which will
# request a edgebot partition, but without promise to wait for the
# processing to be finished, so the first run of `slapos node instance`
# exits with success code and `waitForInstance` return.
# - second `waitForInstance` process the edgebot partition.
# Once we implement a promise (or something similar) here, we should not
# have to use --all
cls.slap._force_slapos_node_instance_all = True
return super(EdgeSlaveMixin, cls).setUpClass()
@classmethod
def getInstanceSoftwareType(cls):
return 'edgetest'
......
......@@ -15,7 +15,7 @@
[instance]
filename = instance.cfg.in
md5sum = d1dff3fc39eefc57b36dbaa195b6890e
md5sum = c962079a88a6ce97d8ce20fa4e8edfd1
[tomcat-server-xml]
filename = server.xml.in
......
......@@ -87,9 +87,9 @@ instance-promises =
[tomcat-listen-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = $${:_buildout_section_name_}.py
config-hostname= $${tomcat-instance:ip}
config-host = $${tomcat-instance:ip}
config-port = $${tomcat-instance:port}
[publish-connection-parameter]
......
......@@ -18,7 +18,7 @@ md5sum = fddea033e1aa9d6147a1a47bd7cc4b62
[template-powerdns]
filename = instance-powerdns.cfg
md5sum = 0920200cb05a68b1b4a161a927d9488f
md5sum = c04c3b490e7f9f35af3d204a9df51f35
[template-pdns-configuration]
_update_hash_filename_ = template/pdns.conf.jinja2
......@@ -26,7 +26,7 @@ md5sum = 20c37ea06a8fa405bc02470d5115fd11
[template-dns-replicate]
_update_hash_filename_ = instance-powerdns-replicate.cfg.jinja2
md5sum = 504d15f0bbf0e515d5ff16070f1ac802
md5sum = 4ff993a39da03d9d66d7c0f98efeb1e0
[iso-list]
_update_hash_filename_ = template/zz.countries.nexedi.dk.rbldnsd
......
......@@ -73,9 +73,9 @@ sla-{{ parameter }} = {{ slapparameter_dict.pop( sla_key + parameter ) }}
[{{promise_section_title}}]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = {{promise_section_title}}.py
config-hostname = {{ '${' ~ request_section_title ~ ':connection-powerdns-ipv6}' }}
config-host = {{ '${' ~ request_section_title ~ ':connection-powerdns-ipv6}' }}
config-port = {{ '${' ~ request_section_title ~ ':connection-powerdns-port}' }}
{% do monitor_url_list.append('${' ~ request_section_title ~ ':connection-monitor-base-url}') -%}
......
......@@ -137,9 +137,9 @@ extra-context =
# Promises
[pdns-promise-listen-port]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = pdns-port-listening.py
config-hostname = $${pdns:ipv4}
config-host = $${pdns:ipv4}
config-port = $${pdns:port}
[publish-connection-informations]
......
......@@ -19,7 +19,7 @@ md5sum = efb4238229681447aa7fe73898dffad4
[instance-default]
filename = instance-default.cfg.in
md5sum = dae19ec06f8da9fa2980a6d2bdf3da54
md5sum = c6dce31a36e4e13de62687e9888aeb77
[proftpd-config-file]
filename = proftpd-config-file.cfg.in
......
......@@ -86,9 +86,9 @@ template = inline:{{ slapparameter_dict['ssh-key'] | indent }}
[proftpd-listen-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = ${:_buildout_section_name_}.py
config-hostname = ${proftpd:ipv6}
config-host = ${proftpd:ipv6}
config-port = ${proftpd:sftp-port}
......
......@@ -15,4 +15,4 @@
[instance-profile]
filename = instance.cfg.in
md5sum = 134ed1cf1f6de63b14425031eb5c9043
md5sum = 500b773d1a63a6a895f9b8038a582b05
......@@ -33,12 +33,12 @@ pureftpd-dir = ${:srv}/pureftpd/
[check-port-listening-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = check_nginx_port.py
[pureftpd-listen-promise]
<= check-port-listening-promise
config-hostname = ${pureftpd:ipv6}
config-host = ${pureftpd:ipv6}
config-port = ${pureftpd:ftp-port}
[pureftpd-userinfo]
......
......@@ -18,7 +18,7 @@ md5sum = 71531ed9c9b79fa769ab367e7ea2d2a5
[template-re6stnet]
filename = instance-re6stnet.cfg.in
md5sum = 2309889cf3f5bc57ba63d389e662fc21
md5sum = 870c34cf58acaaee21c71182dd3cb0cf
[template-apache-conf]
filename = apache.conf.in
......
......@@ -170,16 +170,16 @@ context =
[re6st-registry-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = re6st-registry.py
config-hostname = ${re6st-registry:ipv4}
config-host = ${re6st-registry:ipv4}
config-port = ${re6st-registry:port}
[apache-registry-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = apache-re6st-registry.py
config-hostname = ${apache-conf:ipv6}
config-host = ${apache-conf:ipv6}
config-port = ${apache-conf:port}
[publish]
......
......@@ -18,7 +18,7 @@ md5sum = 8a08be95a04f1a47098c4fdef80bdfed
[instance-repman.cfg]
_update_hash_filename_ = instance-repman.cfg.jinja2.in
md5sum = 1d6eba3984b3e2009682f6ce49b8ac4d
md5sum = 0c173313b48d0005c46d968db1c8ab5f
[config-toml.in]
_update_hash_filename_ = templates/config.toml.in
......
......@@ -216,23 +216,23 @@ depends =
[proxysql-{{ name }}-admin-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = proxysql-{{ name }}-admin-port-listening.py
config-hostname= {{ ipv4 }}
config-host = {{ ipv4 }}
config-port = {{ '${' ~ name ~ '-cluster-parameter:proxy-admin-port}' }}
[proxysql-{{ name }}-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = proxysql-{{ name }}-port-listening.py
config-hostname= {{ ipv4 }}
config-host = {{ ipv4 }}
config-port = {{ '${' ~ name ~ '-cluster-parameter:proxy-port}' }}
[proxysql-{{ name }}-ipv6-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = proxysql-{{ name }}-ipv6-port-listening.py
config-hostname= {{ ip }}
config-host = {{ ip }}
config-port = {{ '${' ~ name ~ '-cluster-parameter:proxy-port}' }}
{% set service_name = "proxysql-" ~ name -%}
......@@ -403,16 +403,16 @@ context =
[repman-listen-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = repman_service_listen.py
config-hostname = ${repman-parameter:ipv4}
config-host = ${repman-parameter:ipv4}
config-port = ${repman-parameter:port}
[repman-listen-ssl-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = repman_service_ssl_listen.py
config-hostname = ${repman-parameter:ipv4}
config-host = ${repman-parameter:ipv4}
config-port = ${repman-parameter:secure-port}
[nginx-conf]
......
# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
# The only allowed lines here are (regexes):
# - "^#" comments, copied verbatim
# - "^[" section beginings, copied verbatim
# - lines containing an "=" sign which must fit in the following categorie.
# - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
# Copied verbatim.
# - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
# by the re-generation script.
# Re-generated.
# - other lines are copied verbatim
# Substitution (${...:...}), extension ([buildout] extends = ...) and
# section inheritance (< = ...) are NOT supported (but you should really
# not need these here).
[instance.cfg.in]
filename = instance.cfg.in
md5sum = fee2097f3d12fd4bcfbc2698ecf49afc
{% import "caucase" as caucase with context %}
[buildout]
parts =
promises
publish-connection-parameter
extends =
{{ template_monitor }}
eggs-directory = {{ buildout['eggs-directory'] }}
develop-eggs-directory = {{ buildout['develop-eggs-directory'] }}
offline = true
[instance-parameter]
recipe = slapos.cookbook:slapconfiguration
computer = ${slap-connection:computer-id}
partition = ${slap-connection:partition-id}
url = ${slap-connection:server-url}
key = ${slap-connection:key-file}
cert = ${slap-connection:cert-file}
[slap-configuration]
# frontend reads from from a part named [slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
computer = ${slap-connection:computer-id}
partition = ${slap-connection:partition-id}
url = ${slap-connection:server-url}
key = ${slap-connection:key-file}
cert = ${slap-connection:cert-file}
[directory]
recipe = slapos.cookbook:mkdirectory
home = ${buildout:directory}
etc = ${:home}/etc
var = ${:home}/var
srv = ${:home}/srv
tmp = ${:home}/tmp
var-log = ${:var}/log
service = ${:etc}/service
promise = ${:etc}/promise
rest-server-data-dir = ${:srv}/restic
backup-caucased = ${:srv}/backup/caucased/
# Macros
[check-port-listening-promise]
recipe = slapos.cookbook:check_port_listening
path = ${directory:promise}/${:_buildout_section_name_}
[check-url-available-promise]
recipe = slapos.cookbook:check_url_available
path = ${directory:promise}/${:_buildout_section_name_}
dash_path = {{ dash_bin }}
curl_path = {{ curl_bin }}
# Caucase
[rest-server-certificate]
key-file = ${directory:etc}/${:_buildout_section_name_}.key
cert-file = ${directory:etc}/${:_buildout_section_name_}.crt
common-name = ${:_buildout_section_name_}
ca-file = ${directory:etc}/${:_buildout_section_name_}.ca.crt
crl-file = ${directory:etc}/${:_buildout_section_name_}.crl
{{
caucase.updater(
prefix='rest-server-certificate',
buildout_bin_directory=buildout['bin-directory'],
updater_path='${directory:service}/rest-server-certificate-updater',
url='${caucased:url}',
data_dir='${directory:srv}/caucase-updater',
crt_path='${rest-server-certificate:cert-file}',
ca_path='${rest-server-certificate:ca-file}',
crl_path='${rest-server-certificate:crl-file}',
key_path='${rest-server-certificate:key-file}',
template_csr='${rest-server-certificate-prepare-csr:csr}',
openssl=openssl_bin,
)}}
[rest-server-certificate-csr-config]
recipe = slapos.recipe.template:jinja2
mode = 644
template = inline:
[req]
prompt = no
req_extensions = req_ext
distinguished_name = dn
[ dn ]
CN = restic-rest-server
[ req_ext ]
subjectAltName = @alt_names
[ alt_names ]
IP.1 = ${instance-parameter:ipv4-random}
IP.2 = ${instance-parameter:ipv6-random}
rendered = ${buildout:parts-directory}/${:_buildout_section_name_}/${:_buildout_section_name_}.txt
[rest-server-certificate-prepare-csr]
recipe = plone.recipe.command
command =
if [ ! -f '${:csr}' ] ; then
{{ openssl_bin }} req \
-newkey rsa:2048 \
-batch \
-new \
-nodes \
-keyout /dev/null \
-config '${rest-server-certificate-csr-config:rendered}' \
-out '${:csr}'
fi
stop-on-error = true
csr = ${directory:srv}/${:_buildout_section_name_}.csr.pem
[caucased]
port = 18080
ip = ${instance-parameter:ipv6-random}
netloc = [${:ip}]:${:port}
url = http://${:netloc}/
{{
caucase.caucased(
prefix='caucased',
buildout_bin_directory=buildout['bin-directory'],
caucased_path='${directory:service}/caucased',
backup_dir='${directory:backup-caucased}',
data_dir='${directory:srv}/caucased',
netloc='${caucased:netloc}',
tmp='${directory:tmp}',
service_auto_approve_count=1,
user_auto_approve_count=0,
key_len=2048,
)}}
[rest-server-password]
recipe = slapos.cookbook:generate.password
user = backup
[rest-server-htpassword]
recipe = plone.recipe.command
command =
if [ ! -f '${:htpassword}' ] ; then
{{ htpasswd_bin }} \
-b \
-B \
-c ${:htpassword} \
${rest-server-password:user} \
${rest-server-password:passwd}
fi
htpassword = ${directory:rest-server-data-dir}/.htpasswd
stop-on-error = true
[rest-server]
recipe = slapos.cookbook:wrapper
command-line =
{{ gowork_bin }}/rest-server \
--listen [${instance-parameter:ipv6-random}]:${:port}
--log ${directory:var-log}/${:_buildout_section_name_}-access.log
--path ${directory:rest-server-data-dir}
--tls
--tls-cert ${rest-server-certificate:cert-file}
--tls-key ${rest-server-certificate:key-file}
--prometheus
wrapper-path = ${directory:service}/rest-server
port = 19080
ip = ${instance-parameter:ipv6-random}
url = https://[${:ip}]:${:port}
depends =
${rest-server-htpassword:recipe}
[rest-server-listen-promise]
<= check-port-listening-promise
hostname= ${rest-server:ip}
port = ${rest-server:port}
[frontend]
<= slap-connection
recipe = slapos.cookbook:requestoptional
name = Rest Server Frontend
# XXX We have hardcoded SR URL here.
software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
slave = true
config-url = ${rest-server:url}
return = domain secure_access
[frontend-available-promise]
<= check-url-available-promise
url = ${frontend:connection-secure_access}
check-secure = 1
[promises]
recipe =
instance-promises =
${caucased-promise:recipe}
${rest-server-certificate-promise:recipe}
${rest-server-listen-promise:path}
${frontend-available-promise:path}
[publish-connection-parameter]
recipe = slapos.cookbook:publish
rest-server-user = ${rest-server-password:user}
rest-server-password = ${rest-server-password:passwd}
url = ${frontend:connection-secure_access}
caucase-url = ${caucased:url}
[buildout]
extends =
../../component/openssl/buildout.cfg
../../component/curl/buildout.cfg
../../component/dash/buildout.cfg
../../component/golang/buildout.cfg
../../component/restic/buildout.cfg
../../stack/caucase/buildout.cfg
../../stack/slapos.cfg
buildout.hash.cfg
parts =
slapos-cookbook
caucase-eggs
instance.cfg.in
restic
gowork
[python]
part = python3
[gowork]
install +=
${git.github.com_restic_rest-server:location}:./cmd/...
[git.github.com_restic_rest-server]
<= go-git-package
go.importpath = github.com/restic/rest-server
repository = https://github.com/restic/rest-server
revision = v0.10.0-0-g9313f19
[instance.cfg.in]
recipe = slapos.recipe.template:jinja2
rendered = ${buildout:directory}/instance.cfg
template = ${:_profile_base_location_}/${:filename}
mode = 0644
context =
section buildout buildout
key gowork_bin gowork:bin
raw openssl_bin ${openssl:location}/bin/openssl
raw htpasswd_bin ${apache:location}/bin/htpasswd
raw dash_bin ${dash:location}/bin/dash
raw curl_bin ${curl:location}/bin/curl
key template_monitor monitor2-template:rendered
import-list =
file caucase caucase-jinja2-library:target
Tests for restic ReST Server software release
##############################################################################
#
# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from setuptools import setup, find_packages
version = '0.0.1.dev0'
name = 'slapos.test.restic_rest_server'
long_description = open("README.md").read()
setup(
name=name,
version=version,
description="Test for SlapOS' restic Rest Server",
long_description=long_description,
long_description_content_type='text/markdown',
maintainer="Nexedi",
maintainer_email="info@nexedi.com",
url="https://lab.nexedi.com/nexedi/slapos",
packages=find_packages(),
install_requires=[
'slapos.core',
'slapos.libnetworkcache',
'requests',
],
zip_safe=True,
test_suite='test',
)
##############################################################################
#
# Copyright (c) 2021 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import glob
import os
import subprocess
import tempfile
import urllib.parse
import requests
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
class TestResticRestServer(SlapOSInstanceTestCase):
def setUp(self):
self.connection_parameters = \
self.computer_partition.getConnectionParameterDict()
parsed_url = urllib.parse.urlparse(self.connection_parameters['url'])
self.url_with_credentials = parsed_url._replace(
netloc='{}:{}@[{}]:{}'.format(
self.connection_parameters['rest-server-user'],
self.connection_parameters['rest-server-password'],
parsed_url.hostname,
parsed_url.port,
)).geturl()
self.ca_cert = self._getCaucaseServiceCACertificate()
def _getCaucaseServiceCACertificate(self):
ca_cert = tempfile.NamedTemporaryFile(
prefix="ca.crt.pem",
mode="w",
delete=False,
)
ca_cert.write(
requests.get(
urllib.parse.urljoin(
self.connection_parameters['caucase-url'],
'/cas/crt/ca.crt.pem',
)).text)
self.addCleanup(os.unlink, ca_cert.name)
return ca_cert.name
def test_http_get(self):
resp = requests.get(self.connection_parameters['url'], verify=self.ca_cert)
self.assertEqual(resp.status_code, requests.codes.unauthorized)
resp = requests.get(
urllib.parse.urljoin(
self.url_with_credentials,
'/metrics',
),
verify=self.ca_cert,
)
# a random metric
self.assertIn('process_cpu_seconds_total', resp.text)
resp.raise_for_status()
def test_backup_scenario(self):
restic_bin = os.path.join(
self.computer_partition_root_path,
'software_release',
'go.work',
'bin',
'restic',
)
def get_restic_output(*args, **kw):
return subprocess.check_output(
(
restic_bin,
'--cacert',
self.ca_cert,
'--password-file',
password_file.name,
'--repo',
'rest:' + self.url_with_credentials,
) + args,
universal_newlines=True,
**kw,
)
with tempfile.TemporaryDirectory() as work_directory,\
tempfile.NamedTemporaryFile(mode='w') as password_file:
password_file.write('secret')
password_file.flush()
with open(os.path.join(work_directory, 'data'), 'w') as f:
f.write('data to backup')
with self.assertRaises(subprocess.CalledProcessError) as exc_context:
get_restic_output('snapshots', stderr=subprocess.PIPE)
self.assertIn('Is there a repository at the following location?',
exc_context.exception.stderr)
out = get_restic_output('init')
self.assertIn('created restic repository', out)
out = get_restic_output('backup', work_directory)
self.assertIn('Added to the repo', out)
out = get_restic_output('snapshots')
self.assertEqual(out.splitlines()[-1], '1 snapshots')
snapshot_id = out.splitlines()[2].split()[0]
backup_path = out.splitlines()[2].split()[-1]
restore_directory = os.path.join(work_directory, 'restore')
out = get_restic_output(
'restore',
snapshot_id,
'--target',
restore_directory,
)
self.assertIn('restoring <Snapshot', out)
with open(os.path.join(restore_directory, backup_path, 'data')) as f:
self.assertEqual(f.read(), 'data to backup')
......@@ -15,8 +15,8 @@
[template]
filename = instance.cfg.in
md5sum = c4ac5de141ae6a64848309af03e51d88
md5sum = 0084214fae4ee1aad2c878aa393757af
[template-selenium]
filename = instance-selenium.cfg.in
md5sum = eea51d25c292c7ea305229184e380814
md5sum = 884196ea35de35fa9159517912441ce6
......@@ -61,7 +61,7 @@ command-line =
# newSessionWaitTimeout: let clients wait in the queue when no node are available
# maxSession: to accept enough clients
hostname = $${instance-parameter:ipv4-random}
hostname = $${slap-configuration:ipv4-random}
port = 4444
base-url = http://$${:hostname}:$${:port}
url = $${:base-url}/wd/hub
......@@ -94,7 +94,7 @@ environment =
XORG_LOCK_DIR=$${directory:tmp}
DISPLAY=$${xvfb-instance:display}
FONTCONFIG_FILE=$${fontconfig-conf:rendered}
hostname = $${instance-parameter:ipv4-random}
hostname = $${slap-configuration:ipv4-random}
[selenium-server-node-instance-firefox-52]
......@@ -115,11 +115,23 @@ capabilities = browserName=firefox,maxInstances=3,marionette=true,platform=LINUX
java-args = -Dwebdriver.gecko.driver=${geckodriver-0.24.0:location}
port = 7779
[selenium-server-node-instance-firefox-78]
<= selenium-server-node-instance
capabilities = browserName=firefox,maxInstances=3,marionette=true,platform=LINUX,version=${firefox-78:version},firefox_binary=${firefox-wrapper-78:location}
java-args = -Dwebdriver.gecko.driver=${geckodriver-0.24.0:location}
port = 7780
[selenium-server-node-instance-chromium-69]
<= selenium-server-node-instance
capabilities = browserName=chrome,maxInstances=3,platform=LINUX,version=${chromium-69:version},chrome_binary=${chromium-wrapper-69:location}
java-args = -Dwebdriver.chrome.driver=${chromedriver-wrapper-2.41:location}
port = 7780
port = 7781
[selenium-server-node-instance-chromium-91]
<= selenium-server-node-instance
capabilities = browserName=chrome,maxInstances=3,platform=LINUX,version=${chromium-91:version},chrome_binary=${chromium-wrapper-91:location}
java-args = -Dwebdriver.chrome.driver=${chromedriver-wrapper-91:location}
port = 7782
[selenium-server-admin-password]
......@@ -152,7 +164,7 @@ template = inline:
$${:path-hub}
}
}
ip = $${instance-parameter:ipv6-random}
ip = $${slap-configuration:ipv6-random}
hostname = [$${:ip}]
port = 9443
path-admin = /grid/console
......@@ -180,7 +192,7 @@ recipe = slapos.cookbook:userinfo
recipe = slapos.cookbook:free_port
minimum = 22222
maximum = 22231
ip = $${slap-network-information:global-ipv6}
ip = $${slap-configuration:ipv6-random}
hostname = $${:ip}
[ssh-keygen-base]
......@@ -251,7 +263,7 @@ recipe = plone.recipe.command
stop-on-error = true
location = $${buildout:directory}/.ssh
authorized-keys-file = $${:location}/authorized_keys
command = mkdir -p $${:location} && echo '$${instance-parameter:configuration.ssh-authorized-key}' > $${:authorized-keys-file}
command = mkdir -p $${:location} && echo '$${slap-configuration:configuration.ssh-authorized-key}' > $${:authorized-keys-file}
[promises]
......@@ -264,27 +276,29 @@ instance-promises =
$${selenium-server-node-firefox-52-listen-promise:name}
$${selenium-server-node-firefox-60-listen-promise:name}
$${selenium-server-node-firefox-68-listen-promise:name}
$${selenium-server-node-firefox-78-listen-promise:name}
$${selenium-server-node-instance-chromium-69-listen-promise:name}
$${selenium-server-node-instance-chromium-91-listen-promise:name}
[check-port-listening-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = $${:_buildout_section_name_}.py
[sshd-listen-promise]
<= check-port-listening-promise
config-hostname = $${sshd-address:hostname}
config-host = $${sshd-address:hostname}
config-port = $${sshd-address:port}
[selenium-server-frontend-listen-promise]
<= check-port-listening-promise
config-hostname = $${selenium-server-frontend-instance:ip}
config-host = $${selenium-server-frontend-instance:ip}
config-port = $${selenium-server-frontend-instance:port}
[selenium-server-hub-listen-promise]
<= check-port-listening-promise
config-hostname = $${selenium-server-hub-instance:hostname}
config-host = $${selenium-server-hub-instance:hostname}
config-port = $${selenium-server-hub-instance:port}
# Promise waiting for all nodes to be registered
......@@ -294,8 +308,8 @@ module = check_command_execute
name = $${:_buildout_section_name_}.py
config-command =
$${selenium-server-check-nodes-registered:rendered} $${selenium-server-hub-instance:api-url} $${:expected-node-count}
# We have 4 nodes with 3 slots each
expected-node-count = 12
# We have 6 nodes with 3 slots each
expected-node-count = 18
[selenium-server-check-nodes-registered]
recipe = slapos.recipe.template:jinja2
......@@ -313,24 +327,34 @@ template =
[selenium-server-node-firefox-52-listen-promise]
<= check-port-listening-promise
config-hostname = $${selenium-server-node-instance-firefox-52:hostname}
config-host = $${selenium-server-node-instance-firefox-52:hostname}
config-port = $${selenium-server-node-instance-firefox-52:port}
[selenium-server-node-firefox-60-listen-promise]
<= check-port-listening-promise
config-hostname = $${selenium-server-node-instance-firefox-60:hostname}
config-host = $${selenium-server-node-instance-firefox-60:hostname}
config-port = $${selenium-server-node-instance-firefox-60:port}
[selenium-server-node-firefox-68-listen-promise]
<= check-port-listening-promise
config-hostname = $${selenium-server-node-instance-firefox-68:hostname}
config-host = $${selenium-server-node-instance-firefox-68:hostname}
config-port = $${selenium-server-node-instance-firefox-68:port}
[selenium-server-node-firefox-78-listen-promise]
<= check-port-listening-promise
config-host = $${selenium-server-node-instance-firefox-78:hostname}
config-port = $${selenium-server-node-instance-firefox-78:port}
[selenium-server-node-instance-chromium-69-listen-promise]
<= check-port-listening-promise
config-hostname = $${selenium-server-node-instance-chromium-69:hostname}
config-host = $${selenium-server-node-instance-chromium-69:hostname}
config-port = $${selenium-server-node-instance-chromium-69:port}
[selenium-server-node-instance-chromium-91-listen-promise]
<= check-port-listening-promise
config-host = $${selenium-server-node-instance-chromium-91:hostname}
config-port = $${selenium-server-node-instance-chromium-91:port}
[publish-connection-parameter]
recipe = slapos.cookbook:publish
......@@ -347,7 +371,7 @@ ssh-fingerprint = $${ssh-key-fingerprint:fingerprint}
run-node-command = PORT=7999 bash -c 'trap '"'"'kill -TERM $SSHPID; wait $SSHPID '"'"' TERM INT; ssh -L 4444:$${selenium-server-hub-instance:hostname}:$${selenium-server-hub-instance:port} -R $PORT:127.0.0.1:$PORT -p $${sshd-service:port} $${sshd-service:username}@$${sshd-service:ip} & SSHPID=$!; java -jar selenium-server-standalone-3.14.0.jar -role node -host 127.0.0.1 -port $PORT ; wait "$SSHPID"'
[instance-parameter]
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration
computer = $${slap-connection:computer-id}
partition = $${slap-connection:partition-id}
......
......@@ -6,15 +6,21 @@ eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
[dynamic-template-selenium]
recipe = slapos.recipe.template:jinja2
template = ${template-selenium:output}
rendered = $${buildout:parts-directory}/$${:_buildout_section_name_}/$${:filename}
filename = instance-selenium.cfg
context =
key partition_ipv6 slap-configuration:ipv6-random
[switch-softwaretype]
recipe = slapos.cookbook:softwaretype
default = ${template-selenium:output}
recipe = slapos.cookbook:switch-softwaretype
RootSoftwareInstance = $${:default}
default = dynamic-template-selenium:rendered
[slap-connection]
# part to migrate to new - separated words
computer-id = $${slap_connection:computer_id}
partition-id = $${slap_connection:partition_id}
server-url = $${slap_connection:server_url}
software-release-url = $${slap_connection:software_release_url}
key-file = $${slap_connection:key_file}
cert-file = $${slap_connection:cert_file}
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration
computer = $${slap-connection:computer-id}
partition = $${slap-connection:partition-id}
url = $${slap-connection:server-url}
......@@ -73,5 +73,6 @@ post-install =
post-install =
${symlink-extra-fonts-to-firefox-fonts-dir:install}
[versions]
plone.recipe.command = 1.1
[firefox-78]
post-install =
${symlink-extra-fonts-to-firefox-fonts-dir:install}
......@@ -426,6 +426,14 @@ class TestFirefox68(
desired_capabilities = dict(DesiredCapabilities.FIREFOX, version='68.0.2esr')
user_agent = 'Gecko/20100101 Firefox/68.0'
class TestFirefox78(
BrowserCompatibilityMixin,
SeleniumServerTestCase,
ImageComparisonTestCase,
):
desired_capabilities = dict(DesiredCapabilities.FIREFOX, version='78.1.0esr')
user_agent = 'Gecko/20100101 Firefox/78.0'
class TestChrome69(
BrowserCompatibilityMixin,
......@@ -434,3 +442,12 @@ class TestChrome69(
):
desired_capabilities = dict(DesiredCapabilities.CHROME, version='69.0.3497.0')
user_agent = 'Chrome/69.0.3497.0'
class TestChrome91(
BrowserCompatibilityMixin,
SeleniumServerTestCase,
ImageComparisonTestCase,
):
desired_capabilities = dict(DesiredCapabilities.CHROME, version='91.0.4472.114')
user_agent = 'Chrome/91.0.4472.0'
......@@ -14,11 +14,11 @@
# not need these here).
[template-erp5]
filename = instance-erp5.cfg.in
md5sum = c341609c8d0cfe240a3a2ae35bd713b6
md5sum = 84f099cc9852c4f53a075dccbb3880f0
[template-balancer]
filename = instance-balancer.cfg.in
md5sum = 9c67c77eab5195b2674e340fb44c48a2
md5sum = c7c0bb9abbd0f8cc6c7956d83a61c4b3
[template-apache-backend-conf]
filename = apache-backend.conf.in
......
......@@ -2,6 +2,7 @@
{% set part_list = [] -%}
{% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
{% set ssl_parameter_dict = slapparameter_dict['ssl'] -%}
{% set frontend_caucase_url_list = ssl_parameter_dict.get('frontend-caucase-url-list', []) -%}
{% set shared_ca_path = slapparameter_dict.get('shared-certificate-authority-path') -%}
{#
XXX: This template only supports exactly one IPv4 and (if ipv6 is used) one IPv6
......@@ -38,7 +39,7 @@ mode = 644
{% set haproxy_dict = {} -%}
{% set apache_dict = {} -%}
{% set zope_virtualhost_monster_backend_dict = {} %}
{% set test_runner_url_dict = {} %} {# family_name => list of apache URLs #}
{% set test_runner_url_dict = {} %} {# family_name => list of URLs #}
{% set next_port = itertools.count(slapparameter_dict['tcpv4-port']).next -%}
{% for family_name, parameter_id_list in sorted(
slapparameter_dict['zope-family-dict'].iteritems()) -%}
......@@ -59,19 +60,19 @@ mode = 644
{% set test_runner_address_list = slapparameter_dict.get(parameter_id ~ '-test-runner-address-list', []) %}
{% if test_runner_address_list -%}
{% set test_runner_backend_mapping = {} %}
{% set test_runner_apache_url_list = [] %}
{% set test_runner_balancer_url_list = [] %}
{% set test_runner_external_port = next_port() %}
{% for i, (test_runner_internal_ip, test_runner_internal_port) in enumerate(test_runner_address_list) %}
{% do test_runner_backend_mapping.__setitem__(
'unit_test_' ~ i,
'http://' ~ test_runner_internal_ip ~ ':' ~ test_runner_internal_port ) %}
{% do test_runner_apache_url_list.append(
{% do test_runner_balancer_url_list.append(
'https://' ~ ipv4 ~ ':' ~ test_runner_external_port ~ '/unit_test_' ~ i ~ '/' ) %}
{% endfor %}
{% do zope_virtualhost_monster_backend_dict.__setitem__(
(ipv4, test_runner_external_port),
( ssl_authentication, test_runner_backend_mapping ) ) -%}
{% do test_runner_url_dict.__setitem__(family_name, test_runner_apache_url_list) -%}
{% do test_runner_url_dict.__setitem__(family_name, test_runner_balancer_url_list) -%}
{% endif -%}
{% endfor -%}
......@@ -123,6 +124,23 @@ caucase-key = ${directory:apache-conf}/apache-caucase.pem
ca-cert = ${directory:apache-conf}/ca.crt
crl = ${directory:apache-conf}/crl.pem
[simplefile]
< = jinja2-template-base
template = inline:{{ '{{ content }}' }}
{% macro simplefile(section_name, file_path, content, mode='') -%}
{% set content_section_name = section_name ~ '-content' -%}
[{{ content_section_name }}]
content = {{ dumps(content) }}
[{{ section(section_name) }}]
< = simplefile
rendered = {{ file_path }}
context = key content {{content_section_name}}:content
mode = {{ mode }}
{%- endmacro %}
[apache-ssl]
{% if ssl_parameter_dict.get('key') -%}
key = ${apache-ssl-key:rendered}
......@@ -185,9 +203,9 @@ input = inline:
[{{ section('apache-promise') }}]
# Check any apache port in ipv4, expect other ports and ipv6 to behave consistently
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = apache.py
config-hostname = {{ ipv4 }}
config-host = {{ ipv4 }}
config-port = {{ apache_dict.values()[0][0] }}
[{{ section('publish') }}]
......@@ -273,8 +291,6 @@ template = inline:
{% endfor %}
[apachedex-parameters]
# XXX - Sample log file with curent date: apache_access.log-%(date)s.gz
# which will be equivalent to apache_access.log-20150112.gz if the date is 2015-01-12
apache-log-list = ${apache-conf-parameter-dict:access-log}
configuration = ${monitor-apachedex-report-config:rendered}
promise-threshold = {{ slapparameter_dict['apachedex-promise-threshold'] }}
......
......@@ -221,7 +221,10 @@ return =
{%- if test_runner_enabled %}
test-runner-address-list
{% endif %}
{% set bt5_default_list = 'erp5_full_text_mroonga_catalog slapos_configurator' -%}
{% set bt5_default_list = [
'erp5_full_text_mroonga_catalog',
'slapos_configurator',
] -%}
{% if has_jupyter -%}
{% do bt5_default_list.append('erp5_data_notebook') -%}
{% endif -%}
......@@ -254,6 +257,7 @@ config-wsgi = {{ dumps(slapparameter_dict.get('wsgi', True)) }}
config-test-runner-enabled = {{ dumps(test_runner_enabled) }}
config-test-runner-node-count = {{ dumps(test_runner_node_count) }}
config-wcfs_enable = {{ dumps(wcfs_enable) }}
config-test-runner-configuration = {{ dumps(slapparameter_dict.get('test-runner', {})) }}
software-type = zope
{% set global_publisher_timeout = slapparameter_dict.get('publisher-timeout') -%}
......
......@@ -30,11 +30,21 @@ import os
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
_setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', '..', 'software.cfg')))
setup_module_executed = False
def setUpModule():
# slapos.testing.testcase's only need to be executed once
global setup_module_executed
if not setup_module_executed:
_setUpModule()
setup_module_executed = True
class ERP5InstanceTestCase(SlapOSInstanceTestCase):
"""ERP5 base test case
"""
......
This diff is collapsed.
......@@ -41,31 +41,50 @@ import requests
from . import ERP5InstanceTestCase
from . import setUpModule
setUpModule # pyflakes
setUpModule # pyflakes
class TestPublishedURLIsReachableMixin(object):
"""Mixin that checks that default page of ERP5 is reachable.
"""
def _checkERP5IsReachable(self, url):
def _checkERP5IsReachable(self, base_url, site_id, verify):
# We access ERP5 trough a "virtual host", which should make
# ERP5 produce URLs using https://virtual-host-name:1234/virtual_host_root
# as base.
virtual_host_url = urlparse.urljoin(
base_url,
'/VirtualHostBase/https/virtual-host-name:1234/{}/VirtualHostRoot/_vh_virtual_host_root/'
.format(site_id))
# What happens is that instantiation just create the services, but does not
# wait for ERP5 to be initialized. When this test run ERP5 instance is
# instantiated, but zope is still busy creating the site and haproxy
# replies with 503 Service Unavailable when zope is not started yet, with
# 404 when erp5 site is not created, with 500 when mysql is not yet
# reachable, so we retry in a loop until we get a succesful response.
for i in range(1, 60):
# XXX can we get CA from caucase already ?
r = requests.get(url, verify=False)
if r.status_code != requests.codes.ok:
delay = i * 2
self.logger.warn(
"ERP5 was not available, sleeping for %ds and retrying", delay)
time.sleep(delay)
continue
r.raise_for_status()
break
# instantiated, but zope is still busy creating the site and haproxy replies
# with 503 Service Unavailable when zope is not started yet, with 404 when
# erp5 site is not created, with 500 when mysql is not yet reachable, so we
# configure this requests session to retry.
# XXX we should probably add a promise instead
session = requests.Session()
session.mount(
base_url,
requests.adapters.HTTPAdapter(
max_retries=requests.packages.urllib3.util.retry.Retry(
total=60,
backoff_factor=.5,
status_forcelist=(404, 500, 503))))
r = session.get(virtual_host_url, verify=verify, allow_redirects=False)
self.assertEqual(r.status_code, requests.codes.found)
# access on / are redirected to login form, with virtual host preserved
self.assertEqual(r.headers.get('location'), 'https://virtual-host-name:1234/virtual_host_root/login_form')
# login page can be rendered and contain the text "ERP5"
r = session.get(
urlparse.urljoin(base_url, '{}/login_form'.format(site_id)),
verify=verify,
allow_redirects=False,
)
self.assertEqual(r.status_code, requests.codes.ok)
self.assertIn("ERP5", r.text)
def test_published_family_default_v6_is_reachable(self):
......@@ -73,15 +92,18 @@ class TestPublishedURLIsReachableMixin(object):
"""
param_dict = self.getRootPartitionConnectionParameterDict()
self._checkERP5IsReachable(
urlparse.urljoin(param_dict['family-default-v6'], param_dict['site-id']))
param_dict['family-default-v6'],
param_dict['site-id'],
verify=False)
def test_published_family_default_v4_is_reachable(self):
"""Tests the IPv4 URL published by the root partition is reachable.
"""
param_dict = self.getRootPartitionConnectionParameterDict()
self._checkERP5IsReachable(
urlparse.urljoin(param_dict['family-default'], param_dict['site-id']))
param_dict['family-default'],
param_dict['site-id'],
verify=False)
class TestDefaultParameters(
ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
......@@ -99,6 +121,30 @@ class TestMedusa(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
def getInstanceParameterDict(cls):
return {'_': json.dumps({'wsgi': False})}
class TestJupyter(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
"""Test ERP5 Jupyter notebook
"""
__partition_reference__ = 'jupyter'
@classmethod
def getInstanceParameterDict(cls):
return {'_': json.dumps({'jupyter': {'enable': True}})}
def test_jupyter_notebook_is_reachable(self):
param_dict = self.getRootPartitionConnectionParameterDict()
self.assertEqual(
'https://[%s]:8888/tree' % self._ipv6_address,
param_dict['jupyter-url']
)
result = requests.get(
param_dict['jupyter-url'], verify=False, allow_redirects=False)
self.assertEqual(
[requests.codes.found, True, '/login?next=%2Ftree'],
[result.status_code, result.is_redirect, result.headers['Location']]
)
class TestApacheBalancerPorts(ERP5InstanceTestCase):
"""Instantiate with two zope families, this should create for each family:
......@@ -132,19 +178,16 @@ class TestApacheBalancerPorts(ERP5InstanceTestCase):
self.assertTrue(parsed.port)
def test_published_family_parameters(self):
# when we request two families, we have two published family-{family_name}
# URLs
# when we request two families, we have two published family-{family_name} URLs
param_dict = self.getRootPartitionConnectionParameterDict()
for family_name in ('family1', 'family2'):
self.checkValidHTTPSURL(
param_dict['family-{family_name}'.format(family_name=family_name)])
self.checkValidHTTPSURL(
param_dict['family-{family_name}-v6'.format(
family_name=family_name)])
param_dict['family-{family_name}-v6'.format(family_name=family_name)])
def test_published_test_runner_url(self):
# each family's also a list of test test runner URLs, by default 3 per
# family
# each family's also a list of test test runner URLs, by default 3 per family
param_dict = self.getRootPartitionConnectionParameterDict()
for family_name in ('family1', 'family2'):
family_test_runner_url_list = param_dict[
......@@ -154,8 +197,7 @@ class TestApacheBalancerPorts(ERP5InstanceTestCase):
self.checkValidHTTPSURL(url)
def test_zope_listen(self):
# we requested 3 zope in family1 and 5 zopes in family2, we should have 8
# zope running.
# we requested 3 zope in family1 and 5 zopes in family2, we should have 8 zope running.
with self.slap.instance_supervisor_rpc as supervisor:
all_process_info = supervisor.getAllProcessInfo()
self.assertEqual(
......@@ -190,46 +232,7 @@ class TestApacheBalancerPorts(ERP5InstanceTestCase):
])
class TestDisableTestRunner(
ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
"""Test ERP5 can be instantiated without test runner.
"""
__partition_reference__ = 'distr'
@classmethod
def getInstanceParameterDict(cls):
return {'_': json.dumps({'test-runner': {'enabled': False}})}
def test_no_runUnitTestScript(self):
"""No runUnitTest script should be generated in any partition.
"""
# self.computer_partition_root_path is the path of root partition.
# we want to assert that no scripts exist in any partition.
bin_programs = map(
os.path.basename,
glob.glob(self.computer_partition_root_path + "/../*/bin/*"))
self.assertTrue(bin_programs) # just to check the glob was correct.
self.assertNotIn('runUnitTest', bin_programs)
self.assertNotIn('runTestSuite', bin_programs)
def test_no_apache_testrunner_port(self):
# Apache only listen on two ports, there is no apache ports allocated for
# test runner
with self.slap.instance_supervisor_rpc as supervisor:
all_process_info = supervisor.getAllProcessInfo()
process_info, = [p for p in all_process_info if p['name'] == 'apache']
apache_process = psutil.Process(process_info['pid'])
self.assertEqual(
sorted([socket.AF_INET, socket.AF_INET6]),
sorted(
c.family
for c in apache_process.connections()
if c.status == 'LISTEN'
))
class TestZopeNodeParameterOverride(
ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
class TestZopeNodeParameterOverride(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
"""Test override zope node parameters
"""
__partition_reference__ = 'override'
......@@ -244,7 +247,7 @@ class TestZopeNodeParameterOverride(
"server": {},
"cache-size-bytes": "20MB",
"cache-size-bytes!": [
("bb-0", 1 << 20),
("bb-0", 1<<20),
("bb-.*", "500MB"),
],
"pool-timeout": "10m",
......@@ -315,7 +318,7 @@ class TestZopeNodeParameterOverride(
partition = self.getComputerPartitionPath('zope-bb')
for zope in xrange(5):
checkConf({
"cache-size-bytes": "500MB" if zope else 1 << 20,
"cache-size-bytes": "500MB" if zope else 1<<20,
}, {
"cache-size": None,
})
......
This diff is collapsed.
# Copyright (C) 2021 Nexedi SA and Contributors.
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
# the terms of any of the Free Software licenses or any of the Open Source
# Initiative approved licenses and Convey the resulting work. Corresponding
# source of such a combination shall include the source code for all other
# software used.
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options.
import json
import os.path
import unittest
from slapos.grid.utils import md5digest
from . import ERP5InstanceTestCase, setUpModule as _setUpModule
from .test_erp5 import TestPublishedURLIsReachableMixin
# skip tests when software release is built with wendelin.core 1.
def setUpModule():
_setUpModule()
cls = ERP5InstanceTestCase
if not os.path.exists(
os.path.join(
cls.slap.software_directory,
md5digest(cls.getSoftwareURL()),
'bin', 'wcfs')):
raise unittest.SkipTest("built with wendelin.core 1")
class TestWCFS(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
"""Test Wendelin Core File System
"""
__partition_reference__ = 'wcfs'
@classmethod
def getInstanceParameterDict(cls):
return {'_': json.dumps({'wcfs': {'enable': True}})}
def test_wcfs_accessible(self):
"""Verify that wcfs filesystem is basically accessible.
- we can read .wcfs/zurl
- its content is equal to published `serving-zurl`
"""
zurl = json.loads(
self.getComputerPartition('wcfs').getConnectionParameter('_')
)['serving-zurl']
mntpt = lookupMount(zurl)
zurl_ = readfile("%s/.wcfs/zurl" % mntpt)
self.assertEqual(zurl_, zurl)
# lookupMount returns /proc/mount entry for wcfs mounted to serve zurl.
def lookupMount(zurl):
for line in readfile('/proc/mounts').splitlines():
# <zurl> <mountpoint> fuse.wcfs ...
zurl_, mntpt, typ, _ = line.split(None, 3)
if typ != 'fuse.wcfs':
continue
if zurl_ == zurl:
return mntpt
raise KeyError("lookup mount %s: no /proc/mounts entry" % zurl)
# readfile returns content of file @path.
def readfile(path):
with open(path, 'r') as f:
return f.read()
......@@ -13,6 +13,7 @@ extra-eggs +=
[template]
extra =
# The following list is for SR whose buildout runs only with Python 3.
galene ${slapos.test.galene-setup:setup}
helloworld ${slapos.test.helloworld-setup:setup}
jupyter ${slapos.test.jupyter-setup:setup}
monitor ${slapos.test.monitor-setup:setup}
......@@ -20,3 +21,4 @@ extra =
powerdns ${slapos.test.powerdns-setup:setup}
proftpd ${slapos.test.proftpd-setup:setup}
repman ${slapos.test.repman-setup:setup}
restic-rest-server ${slapos.test.restic_rest_server-setup:setup}
......@@ -5,6 +5,7 @@ extends =
../../component/curl/buildout.cfg
../../component/openssl/buildout.cfg
../../component/git/buildout.cfg
../../component/pycurl/buildout.cfg
../../component/faketime/buildout.cfg
../../component/pillow/buildout.cfg
../../component/python-cryptography/buildout.cfg
......@@ -98,6 +99,11 @@ setup = ${slapos-repository:location}/software/proftpd/test/
egg = slapos.test.re6stnet
setup = ${slapos-repository:location}/software/re6stnet/test/
[slapos.test.restic_rest_server-setup]
<= setup-develop-egg
egg = slapos.test.restic_rest_server
setup = ${slapos-repository:location}/software/restic-rest-server/test/
[slapos.test.seleniumserver-setup]
<= setup-develop-egg
egg = slapos.test.seleniumserver
......@@ -183,6 +189,11 @@ setup = ${slapos-repository:location}/software/repman/test/
egg = slapos.test.jscrawler
setup = ${slapos-repository:location}/software/jscrawler/test/
[slapos.test.galene-setup]
<= setup-develop-egg
egg = slapos.test.galene
setup = ${slapos-repository:location}/software/galene/test/
[slapos.core-repository]
<= git-clone-repository
repository = https://lab.nexedi.com/nexedi/slapos.core.git
......@@ -204,6 +215,7 @@ extra-eggs =
${lxml-python:egg}
${slapos.core-setup:egg}
${pillow-python:egg}
${pycurl:egg}
erp5.util
${python-pynacl:egg}
${python-cryptography:egg}
......@@ -225,6 +237,7 @@ extra-eggs =
${slapos.test.powerdns-setup:egg}
${slapos.test.proftpd-setup:egg}
${slapos.test.re6stnet-setup:egg}
${slapos.test.restic_rest_server-setup:egg}
${slapos.test.seleniumserver-setup:egg}
${slapos.test.slaprunner-setup:egg}
${slapos.test.jupyter-setup:egg}
......@@ -237,6 +250,7 @@ extra-eggs =
${slapos.test.repman-setup:egg}
${slapos.test.caucase-setup:egg}
${slapos.test.jscrawler-setup:egg}
${slapos.test.galene-setup:egg}
${slapos.test.html5as-setup:egg}
${slapos.test.html5as-base-setup:egg}
${slapos.test.fluentd-setup:egg}
......
......@@ -18,7 +18,7 @@ md5sum = 8d6878ff1d2e75010c50a1a2b0c13b24
[template-runner]
filename = instance-runner.cfg
md5sum = 9f367deb7597957e7108bee719b78bcc
md5sum = ebdecea5c1653ed752e06fbc8be7e6b2
[template-runner-import-script]
filename = template/runner-import.sh.jinja2
......
......@@ -622,16 +622,16 @@ monitor-password = $${monitor-publish-parameters:monitor-password}
[slaprunner-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = slaprunner.py
config-hostname = $${slaprunner:ipv6}
config-host = $${slaprunner:ipv6}
config-port = $${slaprunner:runner_port}
[runner-sshd-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = runner-sshd.py
config-hostname = $${slap-network-information:global-ipv6}
config-host = $${slap-network-information:global-ipv6}
config-port = $${runner-sshd-port:port}
[slap-parameter]
......@@ -866,9 +866,9 @@ log = $${runnerdirectory:home}/instance/*/.slapgrid/log/instance.log $${runnerdi
[supervisord-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = supervisord.py
config-hostname = $${slaprunner:ipv4}
config-host = $${slaprunner:ipv4}
config-port = $${supervisord:port}
[slapos-supervisord-promise]
......@@ -879,9 +879,9 @@ config-command = ${buildout:bin-directory}/slapos node supervisorctl --cfg=$${sl
[slapos-proxy-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = slaproxy.py
config-hostname = $${slaprunner:ipv4}
config-host = $${slaprunner:ipv4}
config-port = $${slaprunner:proxy_port}
# XXX Monitor
......
......@@ -15,7 +15,7 @@
[instance-theia]
_update_hash_filename_ = instance-theia.cfg.jinja.in
md5sum = 11274189a4492204e046fef45fdd809c
md5sum = 557dec61b7bf58601051575234e7fd05
[instance]
_update_hash_filename_ = instance.cfg.in
......
......@@ -77,16 +77,16 @@ instance-promises =
[theia-listen-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = $${:_buildout_section_name_}.py
config-hostname = $${theia-instance:ip}
config-host = $${theia-instance:ip}
config-port = $${theia-instance:port}
[frontend-listen-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = $${:_buildout_section_name_}.py
config-hostname = $${frontend-instance:ip}
config-host = $${frontend-instance:ip}
config-port = $${frontend-instance:port}
[remote-frontend-url-available-promise]
......@@ -107,10 +107,10 @@ config-check-secure = 1
[slapos-standalone-listen-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
# XXX promise plugins can not contain "slapos" in their names
name = standalone-listen-promise.py
config-hostname = $${slapos-standalone-instance:hostname}
config-host = $${slapos-standalone-instance:hostname}
config-port = $${slapos-standalone-instance:port}
[slapos-autorun-promise]
......
......@@ -16,4 +16,4 @@ init =
json.dump(dict(os.environ), f)
[versions]
slapos.recipe.build = 0.46
slapos.recipe.build = 0.47
......@@ -15,12 +15,12 @@
[instance-cfg]
filename = instance.cfg.in
md5sum = 776c7de2054f78ba79382c22d85018be
md5sum = b43d5e8d1fc2d0eeb54f91cefe6a5bae
[template-turnserver]
filename = instance-turnserver.cfg.jinja2.in
md5sum = df8768f165036dbe0435bd1678b9deb3
md5sum = 7af3318d7249e9afe22436d9fe200159
[template-insecure-turnserver]
filename = instance-insecure-turnserver.cfg.jinja2.in
md5sum = e88678b9f322251a201860a13f4db6b8
md5sum = 3db65c3a16eb76ab438ac3817d1a5fea
{% set part_list = [] -%}
{% set server_name = slapparameter_dict.get('server-name', 'turn.example.com') -%}
{%- set parameter = dict(default_parameter_dict, **slapparameter_dict) %}
{%- set server_name = parameter['server-name'] %}
[directory]
recipe = slapos.cookbook:mkdirectory
......@@ -17,9 +18,11 @@ plugins = ${:etc}/plugin
recipe = slapos.cookbook:generate.password
bytes = 8
{% set turn_port = slapparameter_dict.get('port', 3478) -%}
{% set turn_tls_port = slapparameter_dict.get('tls-port', 5349) -%}
{% set listining_ip = slapparameter_dict.get('listening-ip', (ipv4 | list)[0]) -%}
{% set turn_port = parameter['port'] -%}
{% set turn_tls_port = parameter['tls-port'] -%}
# listening-ip parameter is mandatory
{% set listening_ip = slapparameter_dict['listening-ip'] -%}
[turnserver-config]
recipe = collective.recipe.template
user = nxdturn
......@@ -27,11 +30,11 @@ input = inline:
listening-port={{ turn_port }}
lt-cred-mech
realm={{ server_name }}
{% if slapparameter_dict.get('external-ip', '') %}
external-ip={{ slapparameter_dict['external-ip'] }}
{% if parameter['external-ip'] %}
external-ip={{ parameter['external-ip'] }}
{% endif %}
fingerprint
listening-ip={{ listining_ip }}
listening-ip={{ listening_ip }}
server-name={{ server_name }}
no-stdout-log
simple-log
......@@ -54,9 +57,9 @@ hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
[promise-check-turnserver-port]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = turnserver-port-listening.py
config-hostname = {{ listining_ip }}
config-host = {{ listening_ip }}
config-port = {{ turn_port }}
[publish-connection-information]
......
{% set part_list = [] -%}
{% set server_name = slapparameter_dict.get('server-name', 'turn.example.com') -%}
{%- set part_list = [] -%}
{%- set parameter = dict(default_parameter_dict, **slapparameter_dict) %}
{%- set server_name = parameter['server-name'] %}
[directory]
recipe = slapos.cookbook:mkdirectory
......@@ -31,9 +32,9 @@ mode = {{ mode }}
{% do part_list.append(section_name) -%}
{%- endmacro %}
{% if slapparameter_dict.get('ssl-key') and slapparameter_dict.get('ssl-crt') -%}
{{ simplefile('ssl-certificate', '${turnserver-ssl:certificate}', slapparameter_dict.get('ssl-crt')) }}
{{ simplefile('ssl-key', '${turnserver-ssl:key}', slapparameter_dict.get('ssl-key'), 600) }}
{% if parameter['ssl-key'] and parameter['ssl-crt'] -%}
{{ simplefile('ssl-certificate', '${turnserver-ssl:certificate}', parameter['ssl-crt']) }}
{{ simplefile('ssl-key', '${turnserver-ssl:key}', parameter['ssl-key'], 600) }}
{% else -%}
{% do part_list.append('gen-certificate') -%}
[gen-certificate]
......@@ -57,9 +58,9 @@ secret-file = ${directory:etc}/.turnsecret
command =
if [ ! -s "${:secret-file}" ]; then
cat <<EOF > ${:secret-file}
[turnserver]
secret = $("{{ parameter_dict['openssl'] }}/bin/openssl" rand -hex 32)
EOF
[turnserver]
secret = $("{{ parameter_dict['openssl'] }}/bin/openssl" rand -hex 32)
EOF
fi
chmod 600 ${:secret-file}
......@@ -68,9 +69,11 @@ recipe = slapos.cookbook:zero-knowledge.read
file-path = ${gen-secret:secret-file}
secret =
{% set turn_port = slapparameter_dict.get('port', 3478) -%}
{% set turn_tls_port = slapparameter_dict.get('tls-port', 5349) -%}
{% set listining_ip = slapparameter_dict.get('listening-ip', (ipv4 | list)[0]) -%}
{% set turn_port = parameter['port'] -%}
{% set turn_tls_port = parameter['tls-port'] -%}
# listening-ip parameter is mandatory
{% set listening_ip = slapparameter_dict['listening-ip'] -%}
[turnserver-config]
recipe = collective.recipe.template
input = inline:
......@@ -80,9 +83,9 @@ input = inline:
lt-cred-mech
use-auth-secret
static-auth-secret=${read-secret:secret}
listening-ip={{ listining_ip }}
{% if slapparameter_dict.get('external-ip', '') %}
external-ip={{ slapparameter_dict['external-ip'] }}
listening-ip={{ listening_ip }}
{% if parameter['external-ip'] %}
external-ip={{ parameter['external-ip'] }}
{% endif %}
server-name={{ server_name }}
realm={{ server_name }}
......@@ -120,16 +123,16 @@ hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
[promise-check-turnserver-port]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = turnserver-port-listening.py
config-hostname = {{ listining_ip }}
config-host = {{ listening_ip }}
config-port = {{ turn_port }}
[promise-check-turnserver-tls-port]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = turnserver-tls-port-listening.py
config-hostname = {{ listining_ip }}
config-host = {{ listening_ip }}
config-port = {{ turn_tls_port }}
[publish-connection-information]
......
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment