Skip to content

slapos
slapos
Commit 5081f0b0 authored by Thomas Gambier's avatar Thomas Gambier 🚴🏼
Browse files
Options

Update Release Candidate

parents 58ab5601 1a1efa41
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 2824 additions and 468 deletions
component/chromedriver/buildout.cfg
View file @ 5081f0b0
# ChromeDriver - Webdriver for Chrome
# http://chromedriver.chromium.org/
# https://chromedriver.chromium.org/
# This is a binary download with wrapper scripts.
[buildout]
......@@ -13,6 +13,10 @@ parts =
chromedriver-wrapper
[chromedriver-wrapper-91]
<= chromedriver-wrapper
wrapper-name = chromedriver-91
part = ${chromedriver-91:location}
[chromedriver-wrapper-2.41]
<= chromedriver-wrapper
......@@ -46,7 +50,7 @@ install =
[chromedriver]
<= chromedriver-2.41
<= chromedriver-91
[chromedriver-2.41]
<= chromedriver-download
......@@ -54,6 +58,11 @@ version = 2.41
# Supports Chrome v67-69
md5sum-x86_64 = fbd8b9561575054e0e7e9cc53b680a70
[chromedriver-91]
<= chromedriver-download
version = 91.0.4472.101
# Supports Chrome v91
md5sum-x86_64 = cc43ba0babbfff7f22b48165ec8e8c81
[chromedriver-download]
# Installs chromedriver ${version}.
......
component/chromium/buildout.cfg
View file @ 5081f0b0
......@@ -66,16 +66,27 @@ install =
))
os.fchmod(f.fileno(), 0o755)
[chromium-wrapper-91]
<= chromium-wrapper
wrapper-name = chromium-91
part = ${chromium-91:location}
[chromium-wrapper-69]
<= chromium-wrapper
wrapper-name = chromium-69
part = ${chromium-69:location}
[chromium]
<= chromium-91
[chromium-91]
<= chromium-download
version = 91.0.4472.114
[chromium]
<= chromium-69
revision_x86-64 = 870763
md5sum-x86_64 = 74eab41580469c2b8117cf396db823cb
generation-x86_64 = 1617926496067901
[chromium-69]
......@@ -111,6 +122,7 @@ x86-64 = https://www.googleapis.com/download/storage/v1/b/chromium-browser-snaps
library =
${atk:location}/lib
${at-spi2-atk:location}/lib
${at-spi2-core:location}/lib
${alsa:location}/lib
${cairo:location}/lib
${cups:location}/lib
......@@ -133,11 +145,14 @@ library =
${libXtst:location}/lib
${libXScrnSaver:location}/lib
${libXrandr:location}/lib
${libdrm:location}/lib
${libexpat:location}/lib
${libffi:location}/lib
${libpng:location}/lib
${libpng12:location}/lib
${libxcb:location}/lib
${libxkbcommon:location}/lib
${libxshmfence:location}/lib
${libxml2:location}/lib
${mesa:location}/lib
${nspr:location}/lib
......
component/firefox/buildout.cfg
View file @ 5081f0b0
......@@ -51,6 +51,11 @@ install =
))
os.fchmod(f.fileno(), 0o755)
[firefox-wrapper-78]
<= firefox-wrapper
wrapper-name = firefox-78
part = ${firefox-78:location}
[firefox-wrapper-68]
<= firefox-wrapper
wrapper-name = firefox-68
......@@ -95,6 +100,12 @@ cache-dir =
# would not be created.
<= firefox-68
[firefox-78]
<= firefox-download
version = 78.1.0esr
i686-md5sum = 09595a1b9a99d17a618a51bc1f971e5e
x86_64-md5sum = 06f4d488721ce7229d9a86cb4c6786f3
[firefox-68]
<= firefox-download
version = 68.0.2esr
......
component/mariadb/mdev20693-mixin.cfg
View file @ 5081f0b0
# To be extended after mariadb's buildout.cfg
[mariadb]
[mariadb-10.4]
patches +=
${:_profile_base_location_}/mdev20693.patch#34ca907d6b36ba81d75bed118243f637
component/mesa/buildout.cfg
View file @ 5081f0b0
......@@ -9,6 +9,7 @@ extends =
../pkgconfig/buildout.cfg
../xorg/buildout.cfg
../xz-utils/buildout.cfg
../zlib/buildout.cfg
parts =
mesa
......@@ -16,32 +17,33 @@ parts =
[mesa]
recipe = slapos.recipe.cmmi
shared = true
url = ftp://ftp.freedesktop.org/pub/mesa/11.0.3/mesa-11.0.3.tar.xz
md5sum = bf9118bf0fbf360715cfe60baf7a1db5
url = https://archive.mesa3d.org/mesa-18.0.0.tar.xz
md5sum = c2a59fc5b56de3e197fa3a6023409e23
configure-options =
--enable-static
--disable-gles1
--disable-gles2
--disable-dri
--enable-dri
--disable-dri3
--enable-egl
--disable-gbm
--enable-gbm
--enable-sysfs
--disable-xvmc
--disable-vdpau
--disable-va
--enable-xlib-glx
--disable-shared-glapi
--disable-xlib-glx
--enable-shared-glapi
--disable-driglx-direct
--disable-gallium-llvm
--with-gallium-drivers=
--with-dri-drivers=
environment =
PATH=${autoconf:location}/bin:${bison:location}/bin:${flex:location}/bin:${pkgconfig:location}/bin:${xz-utils:location}/bin:%(PATH)s
PKG_CONFIG_PATH=${damageproto:location}/lib/pkgconfig:${fixesproto:location}/lib/pkgconfig:${glproto:location}/lib/pkgconfig:${kbproto:location}/lib/pkgconfig:${libX11:location}/lib/pkgconfig:${libXau:location}/lib/pkgconfig:${libXext:location}/lib/pkgconfig:${libexpat:location}/lib/pkgconfig:${libxcb:location}/lib/pkgconfig:${xdamage:location}/lib/pkgconfig:${xextproto:location}/lib/pkgconfig:${xfixes:location}/lib/pkgconfig:${xorg-libpthread-stubs:location}/lib/pkgconfig:${xproto:location}/lib/pkgconfig
PKG_CONFIG_PATH=${damageproto:location}/lib/pkgconfig:${fixesproto:location}/lib/pkgconfig:${glproto:location}/lib/pkgconfig:${kbproto:location}/lib/pkgconfig:${libX11:location}/lib/pkgconfig:${libXau:location}/lib/pkgconfig:${libXext:location}/lib/pkgconfig:${libexpat:location}/lib/pkgconfig:${libxcb:location}/lib/pkgconfig:${xdamage:location}/lib/pkgconfig:${xextproto:location}/lib/pkgconfig:${xfixes:location}/lib/pkgconfig:${xorg-libpthread-stubs:location}/lib/pkgconfig:${xproto:location}/lib/pkgconfig:${libdrm:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig
PYTHON2=${buildout:executable}
ACLOCAL=${automake:location}/bin/aclocal -I${libtool:location}/share/aclocal -I${pkgconfig:location}/share/aclocal
AUTOCONF=${autoconf:location}/bin/autoconf
AUTOMAKE=${automake:location}/bin/automake
LDFLAGS=-Wl,-rpath=${libdrm:location}/lib -Wl,-rpath=${zlib:location}/lib
make-options =
-j1
LIBTOOL=${libtool:location}/bin/libtool
......@@ -57,3 +59,4 @@ configure-options =
environment =
PATH=${pkgconfig:location}/bin:%(PATH)s
PKG_CONFIG_PATH=${damageproto:location}/lib/pkgconfig:${fixesproto:location}/lib/pkgconfig:${glproto:location}/lib/pkgconfig:${kbproto:location}/lib/pkgconfig:${libX11:location}/lib/pkgconfig:${libXau:location}/lib/pkgconfig:${libXext:location}/lib/pkgconfig:${libexpat:location}/lib/pkgconfig:${libxcb:location}/lib/pkgconfig:${mesa:location}/lib/pkgconfig:${xdamage:location}/lib/pkgconfig:${xextproto:location}/lib/pkgconfig:${xfixes:location}/lib/pkgconfig:${xorg-libpthread-stubs:location}/lib/pkgconfig:${xproto:location}/lib/pkgconfig
LDFLAGS=-Wl,-rpath=${libdrm:location}/lib -Wl,-rpath=${zlib:location}/lib
component/tempstorage/buildout.cfg 0 → 100644
View file @ 5081f0b0
# SlapOS component for tempstorage.
# https://github.com/zopefoundation/tempstorage
[buildout]
extends =
../ZODB/buildout.cfg
../git/buildout.cfg
# tempstorage provides tempstorage<X> depending on ZODB major version.
#
# - tempstorage >= 4 works only with ZODB5 because ZODB commit protocol was changed
# https://github.com/zopefoundation/tempstorage/commit/5cc223ea
# - tempstorage <= 5.2 is vulnerable to data corruption in loadBefore
# https://github.com/zopefoundation/tempstorage/issues/8
# https://github.com/zopefoundation/tempstorage/pull/16
# - tempstorage 3-nxd provides loadBefore backports to tempstorage 3.
[tempstorage]
recipe = slapos.recipe.build
depends = ${ZODB:egg}
init =
# link/depend to tempstorage<ZODB.major>
zodb = self.buildout['ZODB']
zmajor = zodb['major']
tempstorage_x = self.buildout['tempstorage'+zmajor]
options['depends'] += '$${%s:egg}' % tempstorage_x.name
options['egg'] = tempstorage_x['egg']
# update [versions] from what is needed by tempstorage<X>
self.buildout.parse('[_tempstorage-versions]\n' + tempstorage_x['egg-versions'])
versions = self.buildout['versions']
versions.update(self.buildout['_tempstorage-versions'])
# propagate updated [versions] -> easy_install
# (buildout does this in Buildout constructor)
import zc.buildout.easy_install
zc.buildout.easy_install.default_versions(versions)
# tempstorage5 is plain upstream egg
[tempstorage5]
recipe = zc.recipe.egg:eggs
egg = tempstorage
eggs = ${:egg}
egg-versions =
tempstorage = 5.2
# tempstorage4-wc2 is tempstorage 3 + backports for loadBefore fixes
[tempstorage4-wc2]
recipe = zc.recipe.egg:develop
setup = ${tempstorage4-wc2-repository:location}
egg = tempstorage
egg-versions =
tempstorage =
[tempstorage4-wc2-repository]
recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.com/nexedi/tempstorage.git
branch = 3-nxd
revision= 77b49295db78
location = ${buildout:parts-directory}/tempstorage
git-executable = ${git:location}/bin/git
# tempstorage4 is the same as tempstorage4-wc2 because of loadBefore fixes.
[tempstorage4]
<= tempstorage4-wc2
component/tesseract/buildout.hash.cfg
View file @ 5081f0b0
[download-tessdata.py]
filename = download-tessdata.py
md5sum = 2d283a6d8662d6bb8c9de7b26162b702
md5sum = 1f6cf2797f0daf9dec16c1d3a8c44907
component/tesseract/download-tessdata.py
View file @ 5081f0b0
......@@ -19,9 +19,13 @@ def post_make_hook(options, buildout, env):
for url in options['tessdata-urls'].splitlines():
url, _, md5sum = url.partition('#')
if url:
destination = os.path.join(
options['tessdata-location'],
os.path.basename(url),
)
download(
url,
md5sum=md5sum,
path=os.path.join(options['tessdata-location'],
os.path.basename(url)),
path=destination,
)
os.chmod(destination, 0o750)
component/wendelin.core/buildout.cfg
View file @ 5081f0b0
......@@ -38,6 +38,12 @@ GO = ${go:exe}
# wcfs needs this:
[gowork]
cpkgpath += ${zlib:location}/lib/pkgconfig
[wendelin.core-env]
# `pkg-config --libs zlib` emits only -L, but not -Wl,-rpath
# better set it via gowork:environment when "VAR += ..." support is there
# environment +=
# CGO_LDFLAGS += -Wl,-rpath=${zlib:location}/lib
CGO_LDFLAGS += -Wl,-rpath=${zlib:location}/lib
[wendelin.core-repository]
......
component/xorg/buildout.cfg
View file @ 5081f0b0
......@@ -2,20 +2,25 @@
extends =
../autoconf/buildout.cfg
../automake/buildout.cfg
../bison/buildout.cfg
../bzip2/buildout.cfg
../dash/buildout.cfg
../freetype/buildout.cfg
../gnutls/buildout.cfg
../icu/buildout.cfg
../intltool/buildout.cfg
../libtool/buildout.cfg
../libuuid/buildout.cfg
../libxml2/buildout.cfg
../libxslt/buildout.cfg
../meson/buildout.cfg
../ninja/buildout.cfg
../openssl/buildout.cfg
../patch/buildout.cfg
../perl/buildout.cfg
../perl-XML-Parser/buildout.cfg
../pkgconfig/buildout.cfg
../xz-utils/buildout.cfg
../zlib/buildout.cfg
./buildout.hash.cfg
......@@ -652,3 +657,43 @@ pkg_config_depends = ${libX11:location}/lib/pkgconfig:${xorgproto:pkg_config_dep
environment =
PKG_CONFIG_PATH=${:pkg_config_depends}
PATH=${pkgconfig:location}/bin:%(PATH)s
[libdrm]
recipe = slapos.recipe.cmmi
shared = true
configure-command = ${meson:location}/bin/meson builddir --libdir=lib -Dprefix=@@LOCATION@@
make-binary = ${ninja:location}/bin/ninja -C builddir
url = https://dri.freedesktop.org/libdrm/libdrm-2.4.106.tar.xz
md5sum = 4e316ae1966a1a63c31a3885313a8fb8
pkg_config_depends = ${pciaccess:location}/lib/pkgconfig
environment =
PKG_CONFIG_PATH=${:pkg_config_depends}
PATH=${xz-utils:location}/bin:${pkgconfig:location}/bin:${ninja:location}/bin:%(PATH)s
LDFLAGS=-Wl,-rpath=${pciaccess:location}/lib -Wl,-rpath=@@LOCATION@@/lib
[libxshmfence]
recipe = slapos.recipe.cmmi
shared = true
url = https://www.x.org/releases/individual/lib/libxshmfence-1.3.tar.gz
md5sum = ab3940af0bd3d3cc91eb35ecd33a779a
pkg_config_depends = ${xorgproto:pkg_config_depends}:${xorgproto:location}/share/pkgconfig
environment =
PKG_CONFIG_PATH=${:pkg_config_depends}
PATH=${pkgconfig:location}/bin:%(PATH)s
[libxkbcommon]
recipe = slapos.recipe.cmmi
shared = true
configure-command = ${meson:location}/bin/meson builddir --libdir=lib -Dprefix=@@LOCATION@@ -Denable-wayland=false -Denable-docs=false
make-binary = ${ninja:location}/bin/ninja -C builddir
url = https://xkbcommon.org/download/libxkbcommon-1.3.0.tar.xz
md5sum = 00b5275ec1309a1d427a645de5861605
pkg_config_depends = ${libxcb:pkg_config_depends}:${libxcb:location}/lib/pkgconfig:${libxml2:location}/lib/pkgconfig:${xz-utils:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig:${icu4c:location}/lib/pkgconfig
environment =
PKG_CONFIG_PATH=${:pkg_config_depends}
PATH=${xz-utils:location}/bin:${pkgconfig:location}/bin:${ninja:location}/bin:${bison:location}/bin:%(PATH)s
LDFLAGS=-Wl,-rpath=${libxcb:location}/lib -Wl,-rpath=${libxml2:location}/lib -Wl,-rpath=${xz-utils:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${icu4c:location}/lib -Wl,-rpath=@@LOCATION@@/lib
slapos/recipe/switch_softwaretype.py
View file @ 5081f0b0
......@@ -25,9 +25,8 @@
#
##############################################################################
from zc.buildout.buildout import Buildout
from zc.buildout.buildout import Buildout, MissingOption, MissingSection
from zc.buildout import UserError
class SubBuildout(Buildout):
"""Run buildout in buildout, partially copied from infrae.buildout
......@@ -71,8 +70,21 @@ class Recipe:
self.buildout = buildout
self.options = options
self.name = name
self.software_type = buildout["slap-configuration"]["slap-software-type"]
section, key = self.options[self.software_type].split(":")
try:
self.software_type = buildout["slap-configuration"]["slap-software-type"]
except (MissingSection, MissingOption):
raise UserError("The section to retrieve slap partition parameters "
"(with slapos.cookbook:slapconfiguration recipe or a derived one) "
"must be named [slap-configuration].")
try:
section, key = self.options[self.software_type].split(":")
except MissingOption:
raise MissingOption("This software type (%s) isn't mapped. RootSoftwareInstance "
"is the default software type." % self.software_type)
except ValueError:
raise UserError("The software types in the section [%s] must be separated "
"by a colon such as: 'section:key', where key is usually 'rendered'. "
"Don't use: ${section:key}" % self.name)
self.base = self.buildout[section][key]
def install(self):
......
software/backupserver/buildout.hash.cfg
View file @ 5081f0b0
......@@ -46,8 +46,8 @@ md5sum = ae4a0043414336a521b524d9c95f1c68
[template-pullrdiffbackup]
filename = instance-pullrdiffbackup.cfg.in
md5sum = c1f00c251298c2ab7fd095e7c4571d3b
md5sum = f2e6f30a0e8228cbfb93eaaae10fe884
[template]
filename = instance.cfg.in
md5sum = 42021b325159dff29e4bd4e33b8ff2f3
md5sum = 3df515def97f1e8a9f181514ae6ef03f
software/backupserver/instance-pullrdiffbackup.cfg.in
View file @ 5081f0b0
......@@ -194,9 +194,9 @@ virtual-depends =
[nginx-listen-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = nginx_listen.py
config-hostname = $${nginx-configuration:ip}
config-host = $${nginx-configuration:ip}
config-port = $${nginx-configuration:port}
[nginx-configuration]
......@@ -206,7 +206,7 @@ output = $${directory:etc}/nginx.cfg
mode = 0600
access_log = $${directory:log}/nginx-access.log
error_log = $${directory:log}/nginx-error.log
ip = $${slap-network-information:global-ipv6}
ip = {{ partition_ipv6 }}
port = 9443
ssl_key = $${directory:ssl}/nginx.key
ssl_csr = $${directory:ssl}/nginx.csr
......
software/backupserver/instance.cfg.in
View file @ 5081f0b0
......@@ -13,32 +13,21 @@ rendered = $${buildout:parts-directory}/$${:_buildout_section_name_}/$${:filenam
filename = instance-pullrdiffbackup.cfg
extensions = jinja2.ext.do
context =
key slave_instance_list instance-parameter:slave-instance-list
key slave_instance_list slap-configuration:slave-instance-list
# partition_ipv6 is the random ipv6 allocated to the local partition
key partition_ipv6 slap-configuration:ipv6-random
[switch-softwaretype]
recipe = slapos.cookbook:softwaretype
default = $${:pullrdiffbackup}
# pullrdiffbackup = ${template-pullrdiffbackup:output}
pullrdiffbackup = $${dynamic-template-pullrdiffbackup:rendered}
recipe = slapos.cookbook:switch-softwaretype
RootSoftwareInstance = $${:pullrdiffbackup}
pullrdiffbackup = dynamic-template-pullrdiffbackup:rendered
[slap-connection]
# part to migrate to new - separated words
computer-id = $${slap_connection:computer_id}
partition-id = $${slap_connection:partition_id}
server-url = $${slap_connection:server_url}
software-release-url = $${slap_connection:software_release_url}
key-file = $${slap_connection:key_file}
cert-file = $${slap_connection:cert_file}
# [slap-parameter]
# slave-instance-list = []
[instance-parameter]
[slap-configuration]
# Fetches parameters defined in SlapOS Master for this instance.
# Always the same.
recipe = slapos.cookbook:slapconfiguration.serialised
computer = $${slap_connection:computer_id}
partition = $${slap_connection:partition_id}
url = $${slap_connection:server_url}
key = $${slap_connection:key_file}
cert = $${slap_connection:cert_file}
computer = $${slap-connection:computer-id}
partition = $${slap-connection:partition-id}
url = $${slap-connection:server-url}
key = $${slap-connection:key-file}
cert = $${slap-connection:cert-file}
software/backupserver/software.cfg
View file @ 5081f0b0
......@@ -102,5 +102,3 @@ mode = 0644
[versions]
gunicorn = 19.1.1
plone.recipe.command = 1.1
PyRSS2Gen = 1.1
software/caddy-frontend/buildout.hash.cfg
View file @ 5081f0b0
......@@ -22,15 +22,15 @@ md5sum = 5784bea3bd608913769ff9a8afcccb68
[profile-caddy-frontend]
filename = instance-apache-frontend.cfg.in
md5sum = 28220d18308313d49a38d39c61a7e769
md5sum = 8507a2ace2f789b92c522cc62ca5aace
[profile-caddy-replicate]
filename = instance-apache-replicate.cfg.in
md5sum = ec421617d4932e8e1ae1e55a41fbfaab
md5sum = 8beb438d06bbb0f917d13e182fb12d17
[profile-slave-list]
_update_hash_filename_ = templates/apache-custom-slave-list.cfg.in
md5sum = a745fb8d61a7e2646e3aa55edf73e5a6
md5sum = 613f777a08373088cbaf7f51fd18ea70
[profile-replicate-publish-slave-information]
_update_hash_filename_ = templates/replicate-publish-slave-information.cfg.in
......@@ -106,7 +106,7 @@ md5sum = 38792c2dceae38ab411592ec36fff6a8
[profile-kedifa]
filename = instance-kedifa.cfg.in
md5sum = 225ce18b9218de4169327a206051a92e
md5sum = 16901e9eeb0d4f87e708ad91e7756f12
[template-backend-haproxy-rsyslogd-conf]
_update_hash_filename_ = templates/backend-haproxy-rsyslogd.conf.in
......
software/caddy-frontend/instance-apache-frontend.cfg.in
View file @ 5081f0b0
......@@ -535,9 +535,9 @@ context =
[trafficserver-promise-listen-port]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = trafficserver-port-listening.py
config-hostname = ${trafficserver-variable:local-ip}
config-host = ${trafficserver-variable:local-ip}
config-port = ${trafficserver-variable:input-port}
[trafficserver-ctl]
......@@ -655,44 +655,44 @@ config-verification-script = ${promise-helper-last-configuration-state:rendered}
[promise-caddy-frontend-v4-https]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = caddy_frontend_ipv4_https.py
config-hostname = {{ instance_parameter_dict['ipv4-random'] }}
config-host = {{ instance_parameter_dict['ipv4-random'] }}
config-port = ${configuration:port}
[promise-caddy-frontend-v4-http]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = caddy_frontend_ipv4_http.py
config-hostname = {{ instance_parameter_dict['ipv4-random'] }}
config-host = {{ instance_parameter_dict['ipv4-random'] }}
config-port = ${configuration:plain_http_port}
[promise-caddy-frontend-v6-https]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = caddy_frontend_ipv6_https.py
config-hostname = {{ instance_parameter_dict['ipv6-random'] }}
config-host = {{ instance_parameter_dict['ipv6-random'] }}
config-port = ${configuration:port}
[promise-caddy-frontend-v6-http]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = caddy_frontend_ipv6_http.py
config-hostname = {{ instance_parameter_dict['ipv6-random'] }}
config-host = {{ instance_parameter_dict['ipv6-random'] }}
config-port = ${configuration:plain_http_port}
[promise-backend-haproxy-http]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = backend_haproxy_http.py
config-hostname = {{ instance_parameter_dict['ipv4-random'] }}
config-host = {{ instance_parameter_dict['ipv4-random'] }}
config-port = ${backend-haproxy-configuration:http-port}
[promise-backend-haproxy-https]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = backend_haproxy_https.py
config-hostname = {{ instance_parameter_dict['ipv4-random'] }}
config-host = {{ instance_parameter_dict['ipv4-random'] }}
config-port = ${backend-haproxy-configuration:https-port}
[backend-haproxy-configuration]
......@@ -981,9 +981,9 @@ context =
[promise-slave-introspection-https]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = slave_introspection_https.py
config-hostname = {{ instance_parameter_dict['ipv6-random'] }}
config-host = {{ instance_parameter_dict['ipv6-random'] }}
config-port = ${frontend-configuration:slave-introspection-https-port}
[logrotate-entry-slave-introspection]
......
software/caddy-frontend/instance-apache-replicate.cfg.in
View file @ 5081f0b0
......@@ -865,14 +865,14 @@ rendered = ${directory:etc}/nginx-rejected-slave.conf
[promise-rejected-slave-publish-ip-port]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = rejected-slave-publish-ip-port-listening.py
config-hostname = ${rejected-slave-publish-configuration:ip}
config-host = ${rejected-slave-publish-configuration:ip}
config-port = ${rejected-slave-publish-configuration:port}
[rejected-slave-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
module = check_file_state
name = rejected-slave.py
config-filename = ${rejected-slave-json:rendered}
......
software/caddy-frontend/instance-kedifa.cfg.in
View file @ 5081f0b0
......@@ -193,9 +193,9 @@ template = inline:
[promise-expose-csr_id-ip-port]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = expose-csr_id-ip-port-listening.py
config-hostname = ${expose-csr_id-configuration:ip}
config-host = ${expose-csr_id-configuration:ip}
config-port = ${expose-csr_id-configuration:port}
[expose-csr_id]
......
software/caddy-frontend/templates/apache-custom-slave-list.cfg.in
View file @ 5081f0b0
......@@ -633,9 +633,9 @@ template = inline:
[promise-expose-csr_id-ip-port]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = expose-csr_id-ip-port-listening.py
config-hostname = ${expose-csr_id-configuration:ip}
config-host = ${expose-csr_id-configuration:ip}
config-port = ${expose-csr_id-configuration:port}
[expose-csr_id]
......
software/caddy-frontend/test/test.py
View file @ 5081f0b0
......@@ -79,6 +79,8 @@ else:
os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
# XXX Keep using slapos node instance --all, because of missing promises
SlapOSInstanceTestCase.slap._force_slapos_node_instance_all = True
# ports chosen to not collide with test systems
HTTP_PORT = '11080'
......
software/erp5/instance-erp5-input-schema.json
View file @ 5081f0b0
......@@ -78,7 +78,7 @@
"type": "string"
},
"developer-list": {
"description": "List of logins which should get the Developper role (required to modify portal_components' content), defaulting to inituser-login's value",
"description": "List of logins which should get the Developer role (required to modify portal_components' content), defaulting to inituser-login's value",
"items": {
"pattern": "^\\S+$",
"type": "string"
......@@ -215,7 +215,7 @@
"type": "integer"
},
"timerserver-interval": {
"description": "Timerserver tick perdiod, in seconds, or 0 to disable",
"description": "Timerserver tick period, in seconds, or 0 to disable",
"default": 5,
"type": "number"
},
......@@ -439,7 +439,7 @@
"type": "string"
},
{
"description": "Override value (parameter for maching nodes).",
"description": "Override value (parameter for matching nodes).",
"type": [
"integer",
"string"
......@@ -517,7 +517,7 @@
"type": "boolean"
},
"node-count": {
"description": "Number of tests this instance can execute in parrallel. This must be at least equal to the number of nodes configured on testnode running the test",
"description": "Number of tests this instance can execute in parallel. This must be at least equal to the number of nodes configured on testnode running the test",
"default": 3,
"type": "integer"
},
......@@ -525,6 +525,101 @@
"description": "Number of extra databases this instance tests will need.",
"default": 3,
"type": "integer"
},
"selenium": {
"default": {
"target": "firefox"
},
"examples": [
{
"target": "selenium-server",
"server-url": "https://selenium.example.com",
"desired-capabilities": {
"browserName": "firefox",
"version": "68.0.2esr",
"acceptInsecureCerts": true
}
},
{
"target": "selenium-server",
"server-url": "https://selenium.example.com",
"desired-capabilities": {
"browserName": "chrome",
"version": "91.0.4472.101"
}
}
],
"oneOf": [
{
"type": "object",
"title": "Selenium Server",
"description": "Configuration for Selenium server",
"additionalProperties": false,
"required": [
"desired-capabilities",
"server-url",
"target"
],
"properties": {
"target": {
"description": "Target system",
"type": "string",
"const": "selenium-server"
},
"server-url": {
"description": "URL of the selenium server",
"type": "string",
"format": "uri"
},
"verify-server-certificate": {
"description": "Verify the SSL/TLS certificate of the selenium server when using HTTPS",
"type": "boolean",
"default": true
},
"server-ca-certificate": {
"description": "PEM encoded bundle of CA certificates to verify the SSL/TLS certificate of the selenium server when using HTTPS",
"type": "string",
"default": "Root certificates from http://certifi.io/en/latest/"
},
"desired-capabilities": {
"description": "Desired browser capabilities",
"required": [
"browserName"
],
"type": "object",
"properties": {
"browserName": {
"description": "Name of the browser being used",
"type": "string",
"examples": [
"firefox",
"chrome",
"safari"
]
},
"version": {
"description": "The browser version",
"type": "string"
}
}
}
}
},
{
"type": "object",
"title": "Firefox",
"description": "Configuration for using firefox running as a sub-process",
"additionalProperties": false,
"properties": {
"target": {
"description": "Target system",
"const": "firefox",
"type": "string",
"default": "firefox"
}
}
}
]
}
},
"type": "object"
......
software/erp5/test/test/test_erp5.py
View file @ 5081f0b0
......@@ -218,6 +218,45 @@ class TestBalancerPorts(ERP5InstanceTestCase):
]))
class TestSeleniumTestRunner(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
"""Test ERP5 can be instantiated with selenium server for test runner.
"""
__partition_reference__ = 'sel'
@classmethod
def getInstanceParameterDict(cls):
return {
'_':
json.dumps({
'test-runner': {
'selenium': {
"target": "selenium-server",
"server-url": "https://example.com",
"verify-server-certificate": False,
"desired-capabilities": {
"browserName": "firefox",
"version": "68.0.2esr",
}
}
}
})
}
def test_test_runner_configuration_json_file(self):
runUnitTest_script, = glob.glob(
self.computer_partition_root_path + "/../*/bin/runUnitTest.real")
config_file = None
with open(runUnitTest_script) as f:
for line in f:
if 'ERP5_TEST_RUNNER_CONFIGURATION' in line:
_, config_file = line.split('=')
assert config_file
with open(config_file.strip()) as f:
self.assertEqual(
f.read(),
json.dumps(json.loads(self.getInstanceParameterDict()['_'])['test-runner']))
class TestDisableTestRunner(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
"""Test ERP5 can be instantiated without test runner.
"""
......
software/erp5testnode/buildout.hash.cfg
View file @ 5081f0b0
......@@ -18,4 +18,4 @@ md5sum = 307663d73ef3ef94b02567ecd322252e
[template-default]
filename = instance-default.cfg
md5sum = d10958c62d0be8d0d051300d695f4f44
md5sum = 24cc143b1886d443a4c29dcb8147a01c
software/erp5testnode/instance-default.cfg
View file @ 5081f0b0
......@@ -243,9 +243,9 @@ instance-promises =
[shellinabox-frontend-listen-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = $${:_buildout_section_name_}.py
config-hostname = $${shellinabox-frontend:hostname}
config-host = $${shellinabox-frontend:hostname}
config-port = $${shellinabox-frontend:port}
[testnode-log-frontend-promise]
......
software/galene/test/README.md 0 → 100644
View file @ 5081f0b0
Tests for Galene software release
software/galene/test/setup.py 0 → 100644
View file @ 5081f0b0
##############################################################################
#
# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from setuptools import setup, find_packages
version = '0.0.1.dev0'
name = 'slapos.test.galene'
long_description = open("README.md").read()
setup(
name=name,
version=version,
description="Test for SlapOS' Galene",
long_description=long_description,
long_description_content_type='text/markdown',
maintainer="Nexedi",
maintainer_email="info@nexedi.com",
url="https://lab.nexedi.com/nexedi/slapos",
packages=find_packages(),
install_requires=[
'slapos.core',
'slapos.libnetworkcache',
'requests',
],
zip_safe=True,
test_suite='test',
)
software/galene/test/test.py 0 → 100644
View file @ 5081f0b0
##############################################################################
#
# Copyright (c) 2019 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from __future__ import unicode_literals
import os
import requests
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
os.path.abspath(
os.path.join(os.path.dirname(__file__), '../software.cfg')))
class TestGalene(SlapOSInstanceTestCase):
__partition_reference__ = 'G'
def setUp(self):
self.connection_parameters = self.computer_partition.getConnectionParameterDict()
def test_url_get(self):
resp = requests.get(self.connection_parameters['url'], verify=False)
self.assertEqual(requests.codes.ok, resp.status_code)
software/gitlab/software.cfg
View file @ 5081f0b0
......@@ -134,7 +134,7 @@ git-executable = ${git:location}/bin/git
<= git-repository
repository = https://lab.nexedi.com/nexedi/gitlab-ce.git
# 9.5.10 + NXD patches:
revision = v9.5.10-8-gc290e22a08cb
revision = v9.5.10-9-g69b0ffae00bf
location = ${buildout:parts-directory}/gitlab
[gitlab-shell-repository]
......
software/helloworld/buildout.hash.cfg
View file @ 5081f0b0
......@@ -14,4 +14,4 @@
# not need these here).
[instance-profile]
filename = instance.cfg.in
md5sum = dcb9d2f540e0e397c9346c8b0c05f233
md5sum = c771dee1ef9aedad7c6ebf9418afe08e
software/helloworld/instance.cfg.in
View file @ 5081f0b0
......@@ -87,7 +87,7 @@ wrapper-path = ${directory:service}/helloweb-${:kind}
# promise, that checks that helloweb service is alive
[helloweb-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = helloweb-${:kind}.py
{# macro to instantiate service of `kind` to listen on `port` #}
......@@ -102,8 +102,8 @@ port = {{ port }}
[helloweb-{{ kind }}-promise]
<= helloweb-promise
kind = {{ kind }}
config-hostname= ${helloweb-{{ kind }}:ipv6}
config-port = {{ port }}
config-host = ${helloweb-{{ kind }}:ipv6}
config-port = {{ port }}
{% endmacro %}
# services instantiation
......
software/html5as-base/instance.cfg.in
View file @ 5081f0b0
......@@ -18,10 +18,19 @@ recipe = slapos.recipe.template:jinja2
template = {{ template_instance_html5as_target }}
rendered = ${buildout:directory}/${:filename}
filename = instance-html5as.cfg
context =
context =
section buildout buildout
section parameter_list profile-common
# partition_ipv6 is the random ipv6 allocated to the local partition
key partition_ipv6 slap-configuration:ipv6-random
[switch-softwaretype]
recipe = slapos.cookbook:softwaretype
default = ${instance-html5as:rendered}
recipe = slapos.cookbook:switch-softwaretype
RootSoftwareInstance = ${:default}
default = instance-html5as:rendered
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration
computer = ${slap-connection:computer-id}
partition = ${slap-connection:partition-id}
url = ${slap-connection:server-url}
software/html5as-base/instance_html5as.cfg.in
View file @ 5081f0b0
......@@ -70,7 +70,7 @@ scgi_temp_path = ${:tmp}/scgi_temp_path
nb_workers = 2
# Network
ip = ${slap-network-information:global-ipv6}
ip = {{ partition_ipv6 }}
port = 8081
access_url = http://[${:ip}]:${:port}
......
software/html5as-base/software.cfg
View file @ 5081f0b0
......@@ -27,7 +27,7 @@ recipe = slapos.recipe.template:jinja2
rendered = ${buildout:directory}/template.cfg
template = ${:_profile_base_location_}/${:filename}
filename = instance.cfg.in
md5sum = 5a6ebc126bcad3cdff1b51fb51f82a35
md5sum = 310aab063704794065ee3bc8f81fdc70
mode = 0644
context =
section buildout buildout
......@@ -43,7 +43,7 @@ context =
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_update_hash_filename_}
_update_hash_filename_ = instance_html5as.cfg.in
md5sum = 4a8c98cc5ca141f78f14fb9cec203cb8
md5sum = 9b7ed68551cac5967915979383238669
mode = 0644
[template_nginx_conf]
......@@ -71,8 +71,3 @@ mode = 0644
recipe = zc.recipe.egg
eggs =
plone.recipe.command
# Pin versions of eggs used that are not already pinned by stack/slapos.cfg
[versions]
slapos.recipe.template = 4.4
plone.recipe.command = 1.1
software/html5as/buildout.hash.cfg
View file @ 5081f0b0
......@@ -17,11 +17,11 @@
[template-cfg]
filename = instance.cfg.in
md5sum = 0a7aceffa5222e88125b72da42ddedd7
md5sum = 9e486efe4ab1aba8cb72b04f6c6da8ad
[instance_html5as]
_update_hash_filename_ = instance_html5as.cfg.in
md5sum = ec808dba866d85f7d37b85d75c13df2b
md5sum = 191ec2a8b967a3944971709365bdcd2d
[template_nginx_conf]
_update_hash_filename_ = templates/nginx_conf.in
......@@ -45,4 +45,4 @@ md5sum = 1c0ee16966e1fcdb3fd11c09f12ee2b2
[template_instance_replicate]
_update_hash_filename_ = instance_replicate.cfg.in
md5sum = 38d1d352307f79c9c99bf2a80a5c76b8
md5sum = 7ff7e11d05145115f53564ec1af205ef
software/html5as/instance.cfg.in
View file @ 5081f0b0
......@@ -9,11 +9,9 @@ offline = true
[profile-common]
nginx_location = {{ nginx_location }}
dash_location = {{ dash_location }}
tar_location = {{ tar_location }}
curl_location = {{ curl_location }}
template_nginx_conf = {{ template_nginx_conf_target }}
template_mime_types = {{ template_mime_types_target }}
template_launcher = {{ template_launcher_target }}
template_launcher = {{ template_launcher_target }}
template_index_html = {{ template_index_html_target }}
template_graceful = {{ template_graceful_target }}
template_monitor = {{ template_monitor }}
......@@ -26,24 +24,43 @@ filename = instance-html5as.cfg
context =
section buildout buildout
section parameter_list profile-common
# partition_ipv6 is the random ipv6 allocated to the local partition
key partition_ipv6 slap-configuration:ipv6-random
# slapparameter_dict: dictionary of all parameters
key slapparameter_dict slap-configuration:configuration
jsonkey default_parameter_dict :default-parameters
default-parameters =
{
"title": "",
"download_url": null,
"port": 8081,
"monitor-httpd-port": 8197
}
[instance-replicate]
recipe = slapos.recipe.template:jinja2
extensions = jinja2.ext.do
template = {{ template_instance_replicate }}
rendered = ${buildout:directory}/${:filename}
filename = instance-replicate-html5as.cfg
context =
section buildout buildout
section parameter_list profile-common
key slapparameter_dict slap-parameters:configuration
key slapparameter_dict slap-configuration:configuration
jsonkey default_parameter_dict :default-parameters
default-parameters =
{
"download_url": null,
"replicate-quantity": 1
}
[switch-softwaretype]
recipe = slapos.cookbook:softwaretype
default = ${instance-html5as:rendered}
replicate = ${instance-replicate:rendered}
recipe = slapos.cookbook:switch-softwaretype
RootSoftwareInstance = ${:default}
default = instance-html5as:rendered
replicate = instance-replicate:rendered
# Section needed to be added manually here to retrieve parameters
[slap-parameters]
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration
computer = ${slap-connection:computer-id}
partition = ${slap-connection:partition-id}
......
software/html5as/instance_html5as.cfg.in
View file @ 5081f0b0
......@@ -3,10 +3,13 @@
# Deploy html5as instance
#
#############################
# parameter_dict: a dictionary with the default parameters from instance.cfg.in
# replaces the values with the parameters of the instance request if there are any
{% set parameter_dict = dict(default_parameter_dict, **slapparameter_dict) %}
[buildout]
parts =
nginx_conf
downloader
mime_types
launcher
nginx-graceful
......@@ -58,7 +61,6 @@ service = ${directory:etc}/service
log = ${directory:var}/log
run = ${directory:var}/run
backup = ${directory:srv}/backup
data = ${directory:srv}/html5as
[tempdirectory]
recipe = slapos.cookbook:mkdirectory
......@@ -76,8 +78,8 @@ scgi_temp_path = ${:tmp}/scgi_temp_path
nb_workers = 2
# Network
ip = ${slap-network-information:global-ipv6}
port = ${slap-parameter:port}
ip = {{ partition_ipv6 }}
port = {{ parameter_dict['port'] }}
access_url = http://[${:ip}]:${:port}
# Paths
......@@ -88,15 +90,13 @@ path_access_log = ${basedirectory:log}/nginx.access.log
path_error_log = ${basedirectory:log}/nginx.error.log
path_tmp = ${tempdirectory:tmp}
# Docroot
docroot = ${basedirectory:data}
default_index = ${basedirectory:data}/index.html
docroot = ${downloader:location}
default_index = ${:docroot}/index.html
# Config files
path_nginx_conf = ${directory:etc}/nginx.conf
path_mime_types = ${directory:etc}/mime_types
# Binaries
path_shell = {{ parameter_list['dash_location'] }}/bin/dash
curl-binary = {{ parameter_list['curl_location'] }}/bin/curl
tar-binary = {{ parameter_list['tar_location'] }}/bin/tar
# Executables
bin_launcher = ${basedirectory:service}/launcher
......@@ -130,28 +130,48 @@ context =
# Command to put content in the docroot
[downloader]
recipe = plone.recipe.command
# This section will fail if the command fails.
stop-on-error = true
recipe = slapos.recipe.build
# Path where the recipe stores any produced file,
# it will be automatically removed at the beginning of "install".
location = ${directory:srv}/html5as
# All the keys in this section will be available as a dict called "self.options"
# We add: or '', otherwise jinja2 will render a 'None' string
url = {{ parameter_dict['download_url'] or '' }}
default_index_html = ${default_index_html:rendered}
# If a tarball is passed as a parameter in download url
# it's content will be served by the instance.
# If the parameter is not provided it fallback to the default template
command =
rm -rf ${html5as:docroot}/*;
URL="${slap-parameter:download_url}";
if [ -n "$URL" ];
then
${html5as:curl-binary} -Lks $URL | ${html5as:tar-binary} xzv -C ${html5as:docroot} --strip-components 1;
else
cp ${default_index_html:rendered} ${html5as:docroot}/;
fi
install =
import os, shutil
buildout_offline = self.buildout['buildout']['offline']
try:
# Allow to do self.download() which can only be used in "online" mode
self.buildout['buildout']['offline'] = 'false'
if self.options['url']:
# Use fonctions from the slapos.recipe.build repository
# Download a file from a URL to a temporary path
file = self.download(self.options['url'])
# Create a directory and extract the file that are compressed inside
extract_dir = self.extract(file)
# Return the right directory path
workdir = guessworkdir(extract_dir)
# Recursively copy directory
self.copyTree(workdir, location)
else:
# Create directory and copy the default template inside
os.makedirs(location)
shutil.copy(self.options['default_index_html'], location)
finally:
# reset the parameter
self.buildout['buildout']['offline'] = buildout_offline
[default_index_html]
recipe = slapos.recipe.template:jinja2
template = {{ parameter_list['template_index_html'] }}
rendered = ${directory:srv}/index.html
title = {{ parameter_dict['title'] }}
context =
key title slap-parameter:title
key title :title
### Nginx Graceful
[nginx-graceful]
......@@ -165,19 +185,19 @@ context =
# Port Listening checking promise
[port-listening-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = nginx-port-listening.py
config-hostname = ${html5as:ip}
config-host = ${html5as:ip}
config-port = ${html5as:port}
# Use a port different from the default one in order to be able to
# use it in a SlapOS webrunner or a Theia SlapOS Runner
[monitor-instance-parameter]
monitor-httpd-port = 8197
monitor-httpd-port = {{ parameter_dict['monitor-httpd-port'] }}
# Monitor Stack also provides logrotate stack. We only need to extend
# the logrotate-entry-base defined in instance-logrotate-base.cfg.in .
# More parameters can be added following the logrotate-entry-base section
# More parameters can be added following the logrotate-entry-base section
[logrotate-entry-nginx]
<= logrotate-entry-base
name = nginx
......@@ -191,7 +211,7 @@ recipe = slapos.cookbook:publish
# be deployed. The parameters needed for accessing monitoring will be published
<= monitor-publish
server_url = ${html5as:access_url}
title = Title ${slap-parameter:title}!
title = Title {{ parameter_dict['title'] }}!
# Add dependency to the promise so that frontend sections are processed
# and there is no need to declare the new part in buildout:parts
server-cdn-url = ${html5as-frontend-promise:url}
......@@ -202,7 +222,7 @@ server-cdn-url = ${html5as-frontend-promise:url}
<= slap-connection
# Recipe used to make requests
recipe = slapos.cookbook:requestoptional
name = HTM5AS frontend
name = HTML5AS frontend
# Specify the software url of the frontend software release
software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
# It is not a dedicated instance but an instance allocated on a shared instance
......@@ -220,8 +240,3 @@ name = html5as-http-frontend.py
url = ${html5as-frontend:connection-secure_access}
config-url = ${:url}
config-check-secure = 1
[slap-parameter]
title =
download_url =
port = 8081
software/html5as/instance_replicate.cfg.in
View file @ 5081f0b0
{% set replicate_quantity = slapparameter_dict.pop('replicate-quantity', '1') | int %}
{%- set parameter_dict = dict(default_parameter_dict, **slapparameter_dict) %}
{%- set replicate_quantity = int(parameter_dict['replicate-quantity']) %}
# Set default title and port for each replicate based on requested quantity
{%- for i in range(1, replicate_quantity + 1) %}
{%- do parameter_dict.setdefault("title-%d" % i, "") %}
{%- do parameter_dict.setdefault("port-%d" % i, 8081 + i) %}
{%- do parameter_dict.setdefault("monitor-httpd-port-%d" % i, 8197 + i) %}
{%- endfor %}
# Standard buildout section
[buildout]
parts =
publish-connection-information
......@@ -7,6 +16,11 @@ eggs-directory = {{ buildout['eggs-directory'] }}
develop-eggs-directory = {{ buildout['develop-eggs-directory'] }}
offline = true
################################
# Sections to Request instances
################################
# Macro section sharing request parameters
[instance-request-base]
<= slap-connection
recipe = slapos.cookbook:request
......@@ -17,29 +31,25 @@ software-type = default
# What parameter are neede to be retrieved
return = server_url server-cdn-url monitor-setup-url
# Provided parameters
config-title = ${slap-parameter:title}
config-download_url = ${slap-parameter:download_url}
# We add: or '', otherwise jinja2 will render a 'None' string
config-download_url = {{ parameter_dict['download_url'] or '' }}
# Create request section in a loop.
{% for i in range(1, replicate_quantity + 1) %}
# Request a normal html5as instance
[instance-{{ i }}]
<= instance-request-base
# Name of the instance
name = instance-html5as-{{ i }}
config-port = ${slap-parameter:port-{{ i }}}
config-title = ${slap-parameter:title-{{ i }}}
{% if "sla-%s-computer-guid" % i in slapparameter_dict -%}
sla-computer_guid = {{ slapparameter_dict["sla-%s-computer-guid" % i] }}
config-port = {{ parameter_dict["port-%s" % i] }}
config-title = {{ parameter_dict["title-%s" % i] }}
config-monitor-httpd-port = {{ parameter_dict["monitor-httpd-port-%s" % i] }}
{% if "sla-%s-computer-guid" % i in parameter_dict -%}
sla-computer_guid = {{ parameter_dict["sla-%s-computer-guid" % i] }}
{% endif -%}
{% endfor %}
[slap-parameter]
download_url =
{% for i in range(1, replicate_quantity + 1) %}
title-{{ i }} =
port-{{ i }} = 808{{ i }}
{% endfor %}
# Publish information to connect to the two instances
[publish-connection-information]
recipe = slapos.cookbook:publish
{% for i in range(1, replicate_quantity + 1) %}
......
software/html5as/software.cfg
View file @ 5081f0b0
......@@ -13,8 +13,6 @@ extends =
# In this example we extend needed components for html5as.
../../component/nginx/buildout.cfg
../../component/dash/buildout.cfg
../../component/tar/buildout.cfg
../../component/curl/buildout.cfg
parts =
# Call installation of slapos.cookbook egg defined in stack/slapos.cfg (needed
......@@ -36,8 +34,6 @@ context =
section buildout buildout
key nginx_location nginx:location
key dash_location dash:location
key curl_location curl:location
key tar_location tar:location
key template_nginx_conf_target template_nginx_conf:target
key template_mime_types_target template_mime_types:target
key template_launcher_target template_launcher:target
......@@ -79,7 +75,3 @@ mode = 0644
[template_instance_replicate]
<= download-base
# Pin versions of eggs used that are not already pinned by stack/slapos.cfg
[versions]
slapos.recipe.template = 4.4
software/htmlvalidatorserver/buildout.hash.cfg
View file @ 5081f0b0
......@@ -22,8 +22,8 @@ md5sum = 87781e6bcb523bb8434888d5f984f36c
[template-validator]
filename = instance-validator.cfg.in
md5sum = 2be286e367e37ce9e504170cd3a08007
md5sum = 9d12472bb2e337d3cc18f2cc6f235425
[template]
filename = instance.cfg.in
md5sum = 2b4d33e9ef1082dd4d6a53f55b391772
md5sum = 94fc13254c819cba33b03f30251bc469
software/htmlvalidatorserver/instance-validator.cfg.in
View file @ 5081f0b0
......@@ -49,15 +49,15 @@ recipe = slapos.recipe.template
url = ${template-tomcat-configuration:output}
output = $${basedirectory:catalina_conf}/server.xml
mode = 0600
ip = $${slap-network-information:global-ipv6}
ip = {{ partition_ipv6 }}
port = 8899
scheme = https
[tomcat-listen-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = tomcat_listen.py
config-hostname = $${tomcat-configuration:ip}
config-host = $${tomcat-configuration:ip}
config-port = $${tomcat-configuration:port}
#################################
......
software/htmlvalidatorserver/instance.cfg.in
View file @ 5081f0b0
......@@ -6,29 +6,26 @@ eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
[switch-softwaretype]
recipe = slapos.cookbook:softwaretype
default = $${:validator}
validator = ${template-validator:output}
[slap-connection]
# part to migrate to new - separated words
computer-id = $${slap_connection:computer_id}
partition-id = $${slap_connection:partition_id}
server-url = $${slap_connection:server_url}
software-release-url = $${slap_connection:software_release_url}
key-file = $${slap_connection:key_file}
cert-file = $${slap_connection:cert_file}
[dynamic-template-validator]
recipe = slapos.recipe.template:jinja2
template = ${template-validator:output}
rendered = $${buildout:parts-directory}/$${:_buildout_section_name_}/$${:filename}
filename = instance-validator.cfg
context =
# partition_ipv6 is the random ipv6 allocated to the local partition
key partition_ipv6 slap-configuration:ipv6-random
# [slap-parameter]
# slave-instance-list = []
[switch-softwaretype]
recipe = slapos.cookbook:switch-softwaretype
RootSoftwareInstance = $${:validator}
validator = dynamic-template-validator:rendered
[instance-parameter]
[slap-configuration]
# Fetches parameters defined in SlapOS Master for this instance.
# Always the same.
recipe = slapos.cookbook:slapconfiguration.serialised
computer = $${slap_connection:computer_id}
partition = $${slap_connection:partition_id}
url = $${slap_connection:server_url}
key = $${slap_connection:key_file}
cert = $${slap_connection:cert_file}
computer = $${slap-connection:computer-id}
partition = $${slap-connection:partition-id}
url = $${slap-connection:server-url}
key = $${slap-connection:key-file}
cert = $${slap-connection:cert-file}
software/htmlvalidatorserver/software.cfg
View file @ 5081f0b0
......@@ -49,5 +49,4 @@ mode = 0644
# 1.3.4nxd2 is invalid version string, thus pached version string is not '1.3.4nxd2+SlapOSPatched001'
# but '1.3.4nxd2-SlapOSPatched001'.
gunicorn = 19.1.1
plone.recipe.command = 1.1
inotifyx = 0.2.2
software/jscrawler/buildout.hash.cfg
View file @ 5081f0b0
......@@ -19,7 +19,7 @@ md5sum = 6c17361a49cfc47564063b867aab6e8c
[template-jscrawler]
filename = instance-jscrawler.cfg.jinja2.in
md5sum = bdf9e67077cd5c3c140974bcc56422ad
md5sum = fece076231740b414612da5ef3a1685a
[template-jscrawler-builder]
filename = template-jscrawler.builder.sh.in
......
software/jscrawler/instance-jscrawler.cfg.jinja2.in
View file @ 5081f0b0
......@@ -60,9 +60,9 @@ log = ${httpd-wrapper:log-file}
[httpd-listen-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = httpd-listen.py
config-hostname = ${httpd-wrapper:host}
config-host = ${httpd-wrapper:host}
config-port = ${httpd-wrapper:port}
[jscrawler-wrapper]
......
software/jstestnode/instance-jstestnode-input-schema.json
View file @ 5081f0b0
......@@ -23,6 +23,31 @@
"example": "https://softinst1234.host.vifib.net/"
},
"test-runner": {
"default": {
"target": "firefox"
},
"examples": [
{
"target": "selenium-server",
"server-url": "https://selenium.example.com",
"desired-capabilities": {
"browserName": "firefox",
"version": "68.0.2esr",
"acceptInsecureCerts": true
}
},
{
"target": "selenium-server",
"server-url": "https://selenium.example.com",
"desired-capabilities": {
"browserName": "chrome",
"version": "91.0.4472.101"
}
},
{
"target": "node"
}
],
"oneOf": [
{
"type": "object",
......@@ -46,14 +71,14 @@
"format": "uri"
},
"verify-server-certificate": {
"description": "Verify the SSL/TLS Certificats of the selenium server when using HTTPS",
"description": "Verify the SSL/TLS certificate of the selenium server when using HTTPS",
"type": "boolean",
"default": true
},
"server-ca-certificate": {
"description": "PEM encoded bundle of CA Certificates to verify the SSL/TLS Certificate of the selenium server when using HTTPS",
"description": "PEM encoded bundle of CA certificates to verify the SSL/TLS certificate of the selenium server when using HTTPS",
"type": "string",
"default": "root certificates from http://certifi.io/en/latest/"
"default": "Root certificates from http://certifi.io/en/latest/"
},
"desired-capabilities": {
"description": "Desired browser capabilities",
......@@ -63,8 +88,13 @@
"type": "object",
"properties": {
"browserName": {
"description": "Name of the browser being used, for example firefox, chrome",
"type": "string"
"description": "Name of the browser being used",
"type": "string",
"examples": [
"firefox",
"chrome",
"safari"
]
},
"version": {
"description": "The browser version",
......@@ -77,7 +107,7 @@
{
"type": "object",
"title": "Firefox",
"description": "Configuration for Firefox",
"description": "Configuration for using firefox running as a sub-process",
"additionalProperties": false,
"properties": {
"target": {
......
software/kvm/buildout.hash.cfg
View file @ 5081f0b0
......@@ -19,7 +19,7 @@ md5sum = 0d34ff81779115bf899f7bc752877b70
[template-kvm]
filename = instance-kvm.cfg.jinja2
md5sum = bf0c01ac7493693bb57ebef00bb20fa0
md5sum = d0f96be4e80b96e6ac33f6d474767b13
[template-kvm-cluster]
filename = instance-kvm-cluster.cfg.jinja2.in
......@@ -47,7 +47,7 @@ md5sum = b617d64de73de1eed518185f310bbc82
[template-nbd]
filename = instance-nbd.cfg.jinja2
md5sum = 6ea26f88252bf899c966d0f5675e7176
md5sum = 259e06f289f68297e0609e4ab1af8e86
[template-ansible-promise]
filename = template/ansible-promise.in
......@@ -55,7 +55,7 @@ md5sum = b7e87479a289f472b634a046b44b5257
[template-kvm-run]
filename = template/template-kvm-run.in
md5sum = be750fb62f7057c97dd6c6887b2149cc
md5sum = a502782244d1be536b732ebb40725f47
[template-kvm-controller]
filename = template/kvm-controller-run.in
......@@ -75,7 +75,7 @@ md5sum = fb330a796fadb6cd5c85217f80a42af3
[template-httpd]
filename = instance-kvm-http.cfg.in
md5sum = d657884d02105deffddee0edae4b50a6
md5sum = f4bcde62e008c2da9c65617ba7f73f08
[image-download-controller]
_update_hash_filename_ = template/image-download-controller.py
......
software/kvm/instance-kvm-cluster-input-schema.json
View file @ 5081f0b0
......@@ -217,7 +217,7 @@
},
"enable-device-hotplug": {
"title": "Enable device hotplug mode",
"description": "If yes, this will allow to Create devices like CPU and Memory in hotplug mode without restart the VM. Operatin System should be configured to Online new created devices.",
"description": "Allows to increase amount of RAM (ram-size) and CPU (cmp-count) without restart of the VM process, up to defined maximums (ram-max-size and cpu-max-count). Operation system have to support online addition of RAM and CPU.",
"type": "boolean",
"default": false
},
......
software/kvm/instance-kvm-http.cfg.in
View file @ 5081f0b0
......@@ -65,7 +65,7 @@ stop-on-error = true
[httpd-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = apache-httpd.py
config-hostname = ${apache-conf:ip}
config-host = ${apache-conf:ip}
config-port = ${apache-conf:port}
software/kvm/instance-kvm-input-schema.json
View file @ 5081f0b0
......@@ -5,7 +5,7 @@
"properties": {
"enable-device-hotplug": {
"title": "Enable device hotplug mode",
"description": "If yes, this will allow to Create devices like CPU and Memory in hotplug mode without restart the VM. Operatin System should be configured to Online new created devices.",
"description": "Allows to increase amount of RAM (ram-size) and CPU (cmp-count) without restart of the VM process, up to defined maximums (ram-max-size and cpu-max-count). Operation system have to support online addition of RAM and CPU.",
"type": "boolean",
"default": false
},
......
software/kvm/instance-kvm.cfg.jinja2
View file @ 5081f0b0
......@@ -437,13 +437,17 @@ disk-path = ${directory:srv}/virtual.${slap-parameter:disk-format}
pid-file-path = ${kvm-controller-parameter-dict:pid-file}
socket-path = ${kvm-controller-parameter-dict:socket-path}
enable-device-hotplug = ${kvm-controller-parameter-dict:enable-device-hotplug}
smp-count = ${kvm-controller-parameter-dict:cpu-count}
{%- set enable_device_hotplug = slapparameter_dict.get('enable-device-hotplug', 'false').lower() == 'true' %}
smp-max-count = {{ cpu_max_count }}
ram-size = ${kvm-controller-parameter-dict:ram-size}
ram-max-size = {{ ram_max_size }}
{%- if enable_device_hotplug %}
init-ram-size = 1024
init-smp-count = 1
{%- else %}
init-ram-size = ${kvm-controller-parameter-dict:ram-size}
init-smp-count = ${kvm-controller-parameter-dict:cpu-count}
{%- endif %}
mac-address = ${create-mac:mac-address}
tap-mac-address = ${create-tap-mac:mac-address}
......@@ -582,9 +586,9 @@ command-line = ${kvm-controller:rendered}
[kvm-vnc-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = vnc_promise.py
config-hostname = ${kvm-parameter-dict:vnc-ip}
config-host = ${kvm-parameter-dict:vnc-ip}
config-port = ${kvm-parameter-dict:vnc-port}
[kvm-disk-image-corruption-bin]
......@@ -704,9 +708,9 @@ wrapper = ${directory:bin}/websockify
[novnc-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = novnc_promise.py
config-hostname = ${novnc-instance:ip}
config-host = ${novnc-instance:ip}
config-port = ${novnc-instance:port}
......@@ -797,9 +801,9 @@ hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
[httpd-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = httpd.py
config-hostname = ${httpd:host}
config-host = ${httpd:host}
config-port = ${httpd:port}
{% endif %}
......
software/kvm/instance-nbd.cfg.jinja2
View file @ 5081f0b0
......@@ -65,9 +65,9 @@ key = ${gen-passwd:passwd}
[onetimeupload-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = onetimeupload_promise.py
config-hostname = ${onetimeupload-instance:ip}
config-host = ${onetimeupload-instance:ip}
config-port = ${onetimeupload-instance:port}
[publish-connection-information]
......
software/kvm/template/template-kvm-run.in
View file @ 5081f0b0
......@@ -37,9 +37,9 @@ mac_address = '{{ parameter_dict.get("mac-address") }}'
tap_mac_address = '{{ parameter_dict.get("tap-mac-address") }}'
tap_ipv6_addr = '{{ parameter_dict.get("tap-ipv6-addr") }}'
numa_list = '{{ parameter_dict.get("numa", "") }}'.split()
ram_size = {{ parameter_dict.get("ram-size") }}
ram_max_size = '{{ parameter_dict.get("ram-max-size") }}'
init_ram_size = {{ parameter_dict.get("init-ram-size") }}
init_smp_count = {{ parameter_dict.get("init-smp-count") }}
pid_file_path = '{{ parameter_dict.get("pid-file-path") }}'
external_disk_number = {{ parameter_dict.get("external-disk-number") }}
external_disk_size = {{ parameter_dict.get("external-disk-size") }}
......@@ -78,7 +78,6 @@ if not disk_info_list:
{%- endfor %}
})
smp_count = {{ parameter_dict.get("smp-count") }}
smp_max_count = {{ parameter_dict.get("smp-max-count") }}
machine_options = '{{ parameter_dict.get("machine-options", "") }}'.strip()
cpu_model = '{{ parameter_dict.get("cpu-model") }}'.strip()
......@@ -267,12 +266,8 @@ if use_tap == 'true':
tap_interface, vhost),
'-device', 'virtio-net-pci,netdev=lan%s,mac=%s' % (number, tap_mac_address)]
if enable_device_hotplug != 'true':
smp = '%s,maxcpus=%s' % (smp_count, smp_max_count)
ram = '%sM,slots=128,maxmem=%sM' % (ram_size, ram_max_size)
else:
smp = '1,maxcpus=%s' % smp_max_count
ram = '%sM,slots=128,maxmem=%sM' % (init_ram_size, ram_max_size)
smp = '%s,maxcpus=%s' % (init_smp_count, smp_max_count)
ram = '%sM,slots=128,maxmem=%sM' % (init_ram_size, ram_max_size)
kvm_argument_list = [qemu_path,
'-enable-kvm', '-smp', smp, '-name', vm_name, '-m', ram, '-vga', 'std',
......
software/kvm/test/setup.py
View file @ 5081f0b0
......@@ -43,6 +43,7 @@ setup(name=name,
install_requires=[
'slapos.core',
'slapos.cookbook',
'slapos.toolbox',
'slapos.libnetworkcache',
'erp5.util',
'supervisor',
......
software/kvm/test/test.py
View file @ 5081f0b0
......@@ -46,6 +46,7 @@ import time
import shutil
import sys
from slapos.qemuqmpclient import QemuQMPWrapper
from slapos.proxy.db_version import DB_VERSION
from slapos.recipe.librecipe import generateHashFromFiles
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
......@@ -60,6 +61,8 @@ if has_kvm:
os.path.abspath(
os.path.join(os.path.dirname(__file__), '..',
'software%s.cfg' % ("-py3" if six.PY3 else ""))))
# XXX Keep using slapos node instance --all, because of missing promises
InstanceTestCase.slap._force_slapos_node_instance_all = True
else:
setUpModule, InstanceTestCase = None, unittest.TestCase
......@@ -203,6 +206,108 @@ i0:whitelist-firewall-{hash} RUNNING""",
)
@skipUnlessKvm
class TestMemoryManagement(InstanceTestCase, KvmMixin):
__partition_reference__ = 'i'
def getKvmProcessInfo(self, switch_list):
return_list = []
with self.slap.instance_supervisor_rpc as instance_supervisor:
kvm_pid = [q for q in instance_supervisor.getAllProcessInfo()
if 'kvm-' in q['name']][0]['pid']
kvm_process = psutil.Process(kvm_pid)
get_next = False
for entry in kvm_process.cmdline():
if get_next:
return_list.append(entry)
get_next = False
elif entry in switch_list:
get_next = True
return kvm_pid, return_list
def test(self):
kvm_pid_1, info_list = self.getKvmProcessInfo(['-smp', '-m'])
self.assertEqual(
['1,maxcpus=2', '1024M,slots=128,maxmem=1536M'],
info_list
)
self.rerequestInstance({
'ram-size': '1536',
'cpu-count': '2',
})
self.slap.waitForInstance(max_retry=10)
kvm_pid_2, info_list = self.getKvmProcessInfo(['-smp', '-m'])
self.assertEqual(
['2,maxcpus=3', '1536M,slots=128,maxmem=2048M'],
info_list
)
# assert that process was restarted
self.assertNotEqual(kvm_pid_1, kvm_pid_2, "Unexpected: KVM not restarted")
def tearDown(self):
self.rerequestInstance({})
self.slap.waitForInstance(max_retry=10)
def test_enable_device_hotplug(self):
def getHotpluggedCpuRamValue():
qemu_wrapper = QemuQMPWrapper(os.path.join(
self.computer_partition_root_path, 'var', 'qmp_socket'))
ram_mb = sum(
[q['size']
for q in qemu_wrapper.getMemoryInfo()['hotplugged']]) / 1024 / 1024
cpu_count = len(
[q['CPU'] for q in qemu_wrapper.getCPUInfo()['hotplugged']])
return {'cpu_count': cpu_count, 'ram_mb': ram_mb}
kvm_pid_1, info_list = self.getKvmProcessInfo(['-smp', '-m'])
self.assertEqual(
['1,maxcpus=2', '1024M,slots=128,maxmem=1536M'],
info_list
)
self.assertEqual(
getHotpluggedCpuRamValue(),
{'cpu_count': 0, 'ram_mb': 0}
)
parameter_dict = {
'enable-device-hotplug': 'true',
# to avoid restarts the max RAM and CPU has to be static
'ram-max-size': '2048',
'cpu-max-count': '4',
}
self.rerequestInstance(parameter_dict)
self.slap.waitForInstance(max_retry=2)
kvm_pid_2, info_list = self.getKvmProcessInfo(['-smp', '-m'])
self.assertEqual(
['1,maxcpus=4', '1024M,slots=128,maxmem=2048M'],
info_list
)
self.assertEqual(
getHotpluggedCpuRamValue(),
{'cpu_count': 0, 'ram_mb': 0}
)
self.assertNotEqual(kvm_pid_1, kvm_pid_2, "Unexpected: KVM not restarted")
parameter_dict.update(**{
'ram-size': '1536',
'cpu-count': '2'
})
self.rerequestInstance(parameter_dict)
self.slap.waitForInstance(max_retry=10)
kvm_pid_3, info_list = self.getKvmProcessInfo(['-smp', '-m'])
self.assertEqual(
['1,maxcpus=4', '1024M,slots=128,maxmem=2048M'],
info_list
)
self.assertEqual(kvm_pid_2, kvm_pid_3, "Unexpected: KVM restarted")
self.assertEqual(
getHotpluggedCpuRamValue(),
{'cpu_count': 1, 'ram_mb': 512}
)
class MonitorAccessMixin(object):
def sqlite3_connect(self):
sqlitedb_file = os.path.join(
......@@ -223,7 +328,7 @@ class MonitorAccessMixin(object):
}
return db.execute(
"SELECT reference, xml, connection_xml, partition_reference,"
" software_release, requested_state, software_type"
" software_release, requested_state, software_type"
" FROM partition%s"
" WHERE slap_state='busy'" % DB_VERSION).fetchall()
finally:
......
software/monitor/test/test.py
View file @ 5081f0b0
......@@ -142,6 +142,23 @@ class EdgeSlaveMixin(MonitorTestMixin):
instance_max_retry = 20
expected_connection_parameter_dict = {}
@classmethod
def setUpClass(cls):
# XXX we run these tests with --all as a workaround for the fact that after
# requesting new shared instances we don't have promise to wait for the
# processing of these shared instances to be completed.
# The sequence is something like this:
# - `requestEdgetestSlaves` will request edgetest partition
# - first `waitForInstance` will process the edgetest partition, which will
# request a edgebot partition, but without promise to wait for the
# processing to be finished, so the first run of `slapos node instance`
# exits with success code and `waitForInstance` return.
# - second `waitForInstance` process the edgebot partition.
# Once we implement a promise (or something similar) here, we should not
# have to use --all
cls.slap._force_slapos_node_instance_all = True
return super(EdgeSlaveMixin, cls).setUpClass()
@classmethod
def getInstanceSoftwareType(cls):
return 'edgetest'
......
software/plantuml/buildout.hash.cfg
View file @ 5081f0b0
......@@ -15,7 +15,7 @@
[instance]
filename = instance.cfg.in
md5sum = d1dff3fc39eefc57b36dbaa195b6890e
md5sum = c962079a88a6ce97d8ce20fa4e8edfd1
[tomcat-server-xml]
filename = server.xml.in
......
software/plantuml/instance.cfg.in
View file @ 5081f0b0
......@@ -87,9 +87,9 @@ instance-promises =
[tomcat-listen-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = $${:_buildout_section_name_}.py
config-hostname= $${tomcat-instance:ip}
config-host = $${tomcat-instance:ip}
config-port = $${tomcat-instance:port}
[publish-connection-parameter]
......
software/powerdns/buildout.hash.cfg
View file @ 5081f0b0
......@@ -18,7 +18,7 @@ md5sum = fddea033e1aa9d6147a1a47bd7cc4b62
[template-powerdns]
filename = instance-powerdns.cfg
md5sum = 0920200cb05a68b1b4a161a927d9488f
md5sum = c04c3b490e7f9f35af3d204a9df51f35
[template-pdns-configuration]
_update_hash_filename_ = template/pdns.conf.jinja2
......@@ -26,7 +26,7 @@ md5sum = 20c37ea06a8fa405bc02470d5115fd11
[template-dns-replicate]
_update_hash_filename_ = instance-powerdns-replicate.cfg.jinja2
md5sum = 504d15f0bbf0e515d5ff16070f1ac802
md5sum = 4ff993a39da03d9d66d7c0f98efeb1e0
[iso-list]
_update_hash_filename_ = template/zz.countries.nexedi.dk.rbldnsd
......
software/powerdns/instance-powerdns-replicate.cfg.jinja2
View file @ 5081f0b0
......@@ -73,9 +73,9 @@ sla-{{ parameter }} = {{ slapparameter_dict.pop( sla_key + parameter ) }}
[{{promise_section_title}}]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = {{promise_section_title}}.py
config-hostname = {{ '${' ~ request_section_title ~ ':connection-powerdns-ipv6}' }}
config-host = {{ '${' ~ request_section_title ~ ':connection-powerdns-ipv6}' }}
config-port = {{ '${' ~ request_section_title ~ ':connection-powerdns-port}' }}
{% do monitor_url_list.append('${' ~ request_section_title ~ ':connection-monitor-base-url}') -%}
......
software/powerdns/instance-powerdns.cfg
View file @ 5081f0b0
......@@ -137,9 +137,9 @@ extra-context =
# Promises
[pdns-promise-listen-port]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = pdns-port-listening.py
config-hostname = $${pdns:ipv4}
config-host = $${pdns:ipv4}
config-port = $${pdns:port}
[publish-connection-informations]
......
software/proftpd/buildout.hash.cfg
View file @ 5081f0b0
......@@ -19,7 +19,7 @@ md5sum = efb4238229681447aa7fe73898dffad4
[instance-default]
filename = instance-default.cfg.in
md5sum = dae19ec06f8da9fa2980a6d2bdf3da54
md5sum = c6dce31a36e4e13de62687e9888aeb77
[proftpd-config-file]
filename = proftpd-config-file.cfg.in
......
software/proftpd/instance-default.cfg.in
View file @ 5081f0b0
......@@ -86,9 +86,9 @@ template = inline:{{ slapparameter_dict['ssh-key'] | indent }}
[proftpd-listen-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = ${:_buildout_section_name_}.py
config-hostname = ${proftpd:ipv6}
config-host = ${proftpd:ipv6}
config-port = ${proftpd:sftp-port}
......
software/pureftpd/buildout.hash.cfg
View file @ 5081f0b0
......@@ -15,4 +15,4 @@
[instance-profile]
filename = instance.cfg.in
md5sum = 134ed1cf1f6de63b14425031eb5c9043
md5sum = 500b773d1a63a6a895f9b8038a582b05
software/pureftpd/instance.cfg.in
View file @ 5081f0b0
......@@ -33,12 +33,12 @@ pureftpd-dir = ${:srv}/pureftpd/
[check-port-listening-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = check_nginx_port.py
[pureftpd-listen-promise]
<= check-port-listening-promise
config-hostname = ${pureftpd:ipv6}
config-host = ${pureftpd:ipv6}
config-port = ${pureftpd:ftp-port}
[pureftpd-userinfo]
......
software/re6stnet/buildout.hash.cfg
View file @ 5081f0b0
......@@ -18,7 +18,7 @@ md5sum = 71531ed9c9b79fa769ab367e7ea2d2a5
[template-re6stnet]
filename = instance-re6stnet.cfg.in
md5sum = 2309889cf3f5bc57ba63d389e662fc21
md5sum = 870c34cf58acaaee21c71182dd3cb0cf
[template-apache-conf]
filename = apache.conf.in
......
software/re6stnet/instance-re6stnet.cfg.in
View file @ 5081f0b0
......@@ -170,16 +170,16 @@ context =
[re6st-registry-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = re6st-registry.py
config-hostname = ${re6st-registry:ipv4}
config-host = ${re6st-registry:ipv4}
config-port = ${re6st-registry:port}
[apache-registry-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = apache-re6st-registry.py
config-hostname = ${apache-conf:ipv6}
config-host = ${apache-conf:ipv6}
config-port = ${apache-conf:port}
[publish]
......
software/repman/buildout.hash.cfg
View file @ 5081f0b0
......@@ -18,7 +18,7 @@ md5sum = 8a08be95a04f1a47098c4fdef80bdfed
[instance-repman.cfg]
_update_hash_filename_ = instance-repman.cfg.jinja2.in
md5sum = 1d6eba3984b3e2009682f6ce49b8ac4d
md5sum = 0c173313b48d0005c46d968db1c8ab5f
[config-toml.in]
_update_hash_filename_ = templates/config.toml.in
......
software/repman/instance-repman.cfg.jinja2.in
View file @ 5081f0b0
......@@ -216,23 +216,23 @@ depends =
[proxysql-{{ name }}-admin-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = proxysql-{{ name }}-admin-port-listening.py
config-hostname= {{ ipv4 }}
config-host = {{ ipv4 }}
config-port = {{ '${' ~ name ~ '-cluster-parameter:proxy-admin-port}' }}
[proxysql-{{ name }}-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = proxysql-{{ name }}-port-listening.py
config-hostname= {{ ipv4 }}
config-host = {{ ipv4 }}
config-port = {{ '${' ~ name ~ '-cluster-parameter:proxy-port}' }}
[proxysql-{{ name }}-ipv6-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = proxysql-{{ name }}-ipv6-port-listening.py
config-hostname= {{ ip }}
config-host = {{ ip }}
config-port = {{ '${' ~ name ~ '-cluster-parameter:proxy-port}' }}
{% set service_name = "proxysql-" ~ name -%}
......@@ -403,16 +403,16 @@ context =
[repman-listen-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = repman_service_listen.py
config-hostname = ${repman-parameter:ipv4}
config-host = ${repman-parameter:ipv4}
config-port = ${repman-parameter:port}
[repman-listen-ssl-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = repman_service_ssl_listen.py
config-hostname = ${repman-parameter:ipv4}
config-host = ${repman-parameter:ipv4}
config-port = ${repman-parameter:secure-port}
[nginx-conf]
......
software/restic-rest-server/buildout.hash.cfg 0 → 100644
View file @ 5081f0b0
# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
# The only allowed lines here are (regexes):
# - "^#" comments, copied verbatim
# - "^[" section beginings, copied verbatim
# - lines containing an "=" sign which must fit in the following categorie.
# - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
# Copied verbatim.
# - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
# by the re-generation script.
# Re-generated.
# - other lines are copied verbatim
# Substitution (${...:...}), extension ([buildout] extends = ...) and
# section inheritance (< = ...) are NOT supported (but you should really
# not need these here).
[instance.cfg.in]
filename = instance.cfg.in
md5sum = fee2097f3d12fd4bcfbc2698ecf49afc
software/restic-rest-server/instance.cfg.in 0 → 100644
View file @ 5081f0b0
{% import "caucase" as caucase with context %}
[buildout]
parts =
promises
publish-connection-parameter
extends =
{{ template_monitor }}
eggs-directory = {{ buildout['eggs-directory'] }}
develop-eggs-directory = {{ buildout['develop-eggs-directory'] }}
offline = true
[instance-parameter]
recipe = slapos.cookbook:slapconfiguration
computer = ${slap-connection:computer-id}
partition = ${slap-connection:partition-id}
url = ${slap-connection:server-url}
key = ${slap-connection:key-file}
cert = ${slap-connection:cert-file}
[slap-configuration]
# frontend reads from from a part named [slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
computer = ${slap-connection:computer-id}
partition = ${slap-connection:partition-id}
url = ${slap-connection:server-url}
key = ${slap-connection:key-file}
cert = ${slap-connection:cert-file}
[directory]
recipe = slapos.cookbook:mkdirectory
home = ${buildout:directory}
etc = ${:home}/etc
var = ${:home}/var
srv = ${:home}/srv
tmp = ${:home}/tmp
var-log = ${:var}/log
service = ${:etc}/service
promise = ${:etc}/promise
rest-server-data-dir = ${:srv}/restic
backup-caucased = ${:srv}/backup/caucased/
# Macros
[check-port-listening-promise]
recipe = slapos.cookbook:check_port_listening
path = ${directory:promise}/${:_buildout_section_name_}
[check-url-available-promise]
recipe = slapos.cookbook:check_url_available
path = ${directory:promise}/${:_buildout_section_name_}
dash_path = {{ dash_bin }}
curl_path = {{ curl_bin }}
# Caucase
[rest-server-certificate]
key-file = ${directory:etc}/${:_buildout_section_name_}.key
cert-file = ${directory:etc}/${:_buildout_section_name_}.crt
common-name = ${:_buildout_section_name_}
ca-file = ${directory:etc}/${:_buildout_section_name_}.ca.crt
crl-file = ${directory:etc}/${:_buildout_section_name_}.crl
{{
caucase.updater(
prefix='rest-server-certificate',
buildout_bin_directory=buildout['bin-directory'],
updater_path='${directory:service}/rest-server-certificate-updater',
url='${caucased:url}',
data_dir='${directory:srv}/caucase-updater',
crt_path='${rest-server-certificate:cert-file}',
ca_path='${rest-server-certificate:ca-file}',
crl_path='${rest-server-certificate:crl-file}',
key_path='${rest-server-certificate:key-file}',
template_csr='${rest-server-certificate-prepare-csr:csr}',
openssl=openssl_bin,
)}}
[rest-server-certificate-csr-config]
recipe = slapos.recipe.template:jinja2
mode = 644
template = inline:
[req]
prompt = no
req_extensions = req_ext
distinguished_name = dn
[ dn ]
CN = restic-rest-server
[ req_ext ]
subjectAltName = @alt_names
[ alt_names ]
IP.1 = ${instance-parameter:ipv4-random}
IP.2 = ${instance-parameter:ipv6-random}
rendered = ${buildout:parts-directory}/${:_buildout_section_name_}/${:_buildout_section_name_}.txt
[rest-server-certificate-prepare-csr]
recipe = plone.recipe.command
command =
if [ ! -f '${:csr}' ] ; then
{{ openssl_bin }} req \
-newkey rsa:2048 \
-batch \
-new \
-nodes \
-keyout /dev/null \
-config '${rest-server-certificate-csr-config:rendered}' \
-out '${:csr}'
fi
stop-on-error = true
csr = ${directory:srv}/${:_buildout_section_name_}.csr.pem
[caucased]
port = 18080
ip = ${instance-parameter:ipv6-random}
netloc = [${:ip}]:${:port}
url = http://${:netloc}/
{{
caucase.caucased(
prefix='caucased',
buildout_bin_directory=buildout['bin-directory'],
caucased_path='${directory:service}/caucased',
backup_dir='${directory:backup-caucased}',
data_dir='${directory:srv}/caucased',
netloc='${caucased:netloc}',
tmp='${directory:tmp}',
service_auto_approve_count=1,
user_auto_approve_count=0,
key_len=2048,
)}}
[rest-server-password]
recipe = slapos.cookbook:generate.password
user = backup
[rest-server-htpassword]
recipe = plone.recipe.command
command =
if [ ! -f '${:htpassword}' ] ; then
{{ htpasswd_bin }} \
-b \
-B \
-c ${:htpassword} \
${rest-server-password:user} \
${rest-server-password:passwd}
fi
htpassword = ${directory:rest-server-data-dir}/.htpasswd
stop-on-error = true
[rest-server]
recipe = slapos.cookbook:wrapper
command-line =
{{ gowork_bin }}/rest-server \
--listen [${instance-parameter:ipv6-random}]:${:port}
--log ${directory:var-log}/${:_buildout_section_name_}-access.log
--path ${directory:rest-server-data-dir}
--tls
--tls-cert ${rest-server-certificate:cert-file}
--tls-key ${rest-server-certificate:key-file}
--prometheus
wrapper-path = ${directory:service}/rest-server
port = 19080
ip = ${instance-parameter:ipv6-random}
url = https://[${:ip}]:${:port}
depends =
${rest-server-htpassword:recipe}
[rest-server-listen-promise]
<= check-port-listening-promise
hostname= ${rest-server:ip}
port = ${rest-server:port}
[frontend]
<= slap-connection
recipe = slapos.cookbook:requestoptional
name = Rest Server Frontend
# XXX We have hardcoded SR URL here.
software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
slave = true
config-url = ${rest-server:url}
return = domain secure_access
[frontend-available-promise]
<= check-url-available-promise
url = ${frontend:connection-secure_access}
check-secure = 1
[promises]
recipe =
instance-promises =
${caucased-promise:recipe}
${rest-server-certificate-promise:recipe}
${rest-server-listen-promise:path}
${frontend-available-promise:path}
[publish-connection-parameter]
recipe = slapos.cookbook:publish
rest-server-user = ${rest-server-password:user}
rest-server-password = ${rest-server-password:passwd}
url = ${frontend:connection-secure_access}
caucase-url = ${caucased:url}
software/restic-rest-server/software.cfg 0 → 100644
View file @ 5081f0b0
[buildout]
extends =
../../component/openssl/buildout.cfg
../../component/curl/buildout.cfg
../../component/dash/buildout.cfg
../../component/golang/buildout.cfg
../../component/restic/buildout.cfg
../../stack/caucase/buildout.cfg
../../stack/slapos.cfg
buildout.hash.cfg
parts =
slapos-cookbook
caucase-eggs
instance.cfg.in
restic
gowork
[python]
part = python3
[gowork]
install +=
${git.github.com_restic_rest-server:location}:./cmd/...
[git.github.com_restic_rest-server]
<= go-git-package
go.importpath = github.com/restic/rest-server
repository = https://github.com/restic/rest-server
revision = v0.10.0-0-g9313f19
[instance.cfg.in]
recipe = slapos.recipe.template:jinja2
rendered = ${buildout:directory}/instance.cfg
template = ${:_profile_base_location_}/${:filename}
mode = 0644
context =
section buildout buildout
key gowork_bin gowork:bin
raw openssl_bin ${openssl:location}/bin/openssl
raw htpasswd_bin ${apache:location}/bin/htpasswd
raw dash_bin ${dash:location}/bin/dash
raw curl_bin ${curl:location}/bin/curl
key template_monitor monitor2-template:rendered
import-list =
file caucase caucase-jinja2-library:target
software/restic-rest-server/test/README.md 0 → 100644
View file @ 5081f0b0
Tests for restic ReST Server software release
software/restic-rest-server/test/setup.py 0 → 100644
View file @ 5081f0b0
##############################################################################
#
# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from setuptools import setup, find_packages
version = '0.0.1.dev0'
name = 'slapos.test.restic_rest_server'
long_description = open("README.md").read()
setup(
name=name,
version=version,
description="Test for SlapOS' restic Rest Server",
long_description=long_description,
long_description_content_type='text/markdown',
maintainer="Nexedi",
maintainer_email="info@nexedi.com",
url="https://lab.nexedi.com/nexedi/slapos",
packages=find_packages(),
install_requires=[
'slapos.core',
'slapos.libnetworkcache',
'requests',
],
zip_safe=True,
test_suite='test',
)
software/restic-rest-server/test/test.py 0 → 100644
View file @ 5081f0b0
##############################################################################
#
# Copyright (c) 2021 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import glob
import os
import subprocess
import tempfile
import urllib.parse
import requests
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
class TestResticRestServer(SlapOSInstanceTestCase):
def setUp(self):
self.connection_parameters = \
self.computer_partition.getConnectionParameterDict()
parsed_url = urllib.parse.urlparse(self.connection_parameters['url'])
self.url_with_credentials = parsed_url._replace(
netloc='{}:{}@[{}]:{}'.format(
self.connection_parameters['rest-server-user'],
self.connection_parameters['rest-server-password'],
parsed_url.hostname,
parsed_url.port,
)).geturl()
self.ca_cert = self._getCaucaseServiceCACertificate()
def _getCaucaseServiceCACertificate(self):
ca_cert = tempfile.NamedTemporaryFile(
prefix="ca.crt.pem",
mode="w",
delete=False,
)
ca_cert.write(
requests.get(
urllib.parse.urljoin(
self.connection_parameters['caucase-url'],
'/cas/crt/ca.crt.pem',
)).text)
self.addCleanup(os.unlink, ca_cert.name)
return ca_cert.name
def test_http_get(self):
resp = requests.get(self.connection_parameters['url'], verify=self.ca_cert)
self.assertEqual(resp.status_code, requests.codes.unauthorized)
resp = requests.get(
urllib.parse.urljoin(
self.url_with_credentials,
'/metrics',
),
verify=self.ca_cert,
)
# a random metric
self.assertIn('process_cpu_seconds_total', resp.text)
resp.raise_for_status()
def test_backup_scenario(self):
restic_bin = os.path.join(
self.computer_partition_root_path,
'software_release',
'go.work',
'bin',
'restic',
)
def get_restic_output(*args, **kw):
return subprocess.check_output(
(
restic_bin,
'--cacert',
self.ca_cert,
'--password-file',
password_file.name,
'--repo',
'rest:' + self.url_with_credentials,
) + args,
universal_newlines=True,
**kw,
)
with tempfile.TemporaryDirectory() as work_directory,\
tempfile.NamedTemporaryFile(mode='w') as password_file:
password_file.write('secret')
password_file.flush()
with open(os.path.join(work_directory, 'data'), 'w') as f:
f.write('data to backup')
with self.assertRaises(subprocess.CalledProcessError) as exc_context:
get_restic_output('snapshots', stderr=subprocess.PIPE)
self.assertIn('Is there a repository at the following location?',
exc_context.exception.stderr)
out = get_restic_output('init')
self.assertIn('created restic repository', out)
out = get_restic_output('backup', work_directory)
self.assertIn('Added to the repo', out)
out = get_restic_output('snapshots')
self.assertEqual(out.splitlines()[-1], '1 snapshots')
snapshot_id = out.splitlines()[2].split()[0]
backup_path = out.splitlines()[2].split()[-1]
restore_directory = os.path.join(work_directory, 'restore')
out = get_restic_output(
'restore',
snapshot_id,
'--target',
restore_directory,
)
self.assertIn('restoring <Snapshot', out)
with open(os.path.join(restore_directory, backup_path, 'data')) as f:
self.assertEqual(f.read(), 'data to backup')
software/seleniumserver/buildout.hash.cfg
View file @ 5081f0b0
......@@ -15,8 +15,8 @@
[template]
filename = instance.cfg.in
md5sum = c4ac5de141ae6a64848309af03e51d88
md5sum = 0084214fae4ee1aad2c878aa393757af
[template-selenium]
filename = instance-selenium.cfg.in
md5sum = eea51d25c292c7ea305229184e380814
md5sum = 884196ea35de35fa9159517912441ce6
software/seleniumserver/instance-selenium.cfg.in
View file @ 5081f0b0
......@@ -61,7 +61,7 @@ command-line =
# newSessionWaitTimeout: let clients wait in the queue when no node are available
# maxSession: to accept enough clients
hostname = $${instance-parameter:ipv4-random}
hostname = $${slap-configuration:ipv4-random}
port = 4444
base-url = http://$${:hostname}:$${:port}
url = $${:base-url}/wd/hub
......@@ -94,7 +94,7 @@ environment =
XORG_LOCK_DIR=$${directory:tmp}
DISPLAY=$${xvfb-instance:display}
FONTCONFIG_FILE=$${fontconfig-conf:rendered}
hostname = $${instance-parameter:ipv4-random}
hostname = $${slap-configuration:ipv4-random}
[selenium-server-node-instance-firefox-52]
......@@ -115,11 +115,23 @@ capabilities = browserName=firefox,maxInstances=3,marionette=true,platform=LINUX
java-args = -Dwebdriver.gecko.driver=${geckodriver-0.24.0:location}
port = 7779
[selenium-server-node-instance-firefox-78]
<= selenium-server-node-instance
capabilities = browserName=firefox,maxInstances=3,marionette=true,platform=LINUX,version=${firefox-78:version},firefox_binary=${firefox-wrapper-78:location}
java-args = -Dwebdriver.gecko.driver=${geckodriver-0.24.0:location}
port = 7780
[selenium-server-node-instance-chromium-69]
<= selenium-server-node-instance
capabilities = browserName=chrome,maxInstances=3,platform=LINUX,version=${chromium-69:version},chrome_binary=${chromium-wrapper-69:location}
java-args = -Dwebdriver.chrome.driver=${chromedriver-wrapper-2.41:location}
port = 7780
port = 7781
[selenium-server-node-instance-chromium-91]
<= selenium-server-node-instance
capabilities = browserName=chrome,maxInstances=3,platform=LINUX,version=${chromium-91:version},chrome_binary=${chromium-wrapper-91:location}
java-args = -Dwebdriver.chrome.driver=${chromedriver-wrapper-91:location}
port = 7782
[selenium-server-admin-password]
......@@ -152,7 +164,7 @@ template = inline:
$${:path-hub}
}
}
ip = $${instance-parameter:ipv6-random}
ip = $${slap-configuration:ipv6-random}
hostname = [$${:ip}]
port = 9443
path-admin = /grid/console
......@@ -180,7 +192,7 @@ recipe = slapos.cookbook:userinfo
recipe = slapos.cookbook:free_port
minimum = 22222
maximum = 22231
ip = $${slap-network-information:global-ipv6}
ip = $${slap-configuration:ipv6-random}
hostname = $${:ip}
[ssh-keygen-base]
......@@ -251,7 +263,7 @@ recipe = plone.recipe.command
stop-on-error = true
location = $${buildout:directory}/.ssh
authorized-keys-file = $${:location}/authorized_keys
command = mkdir -p $${:location} && echo '$${instance-parameter:configuration.ssh-authorized-key}' > $${:authorized-keys-file}
command = mkdir -p $${:location} && echo '$${slap-configuration:configuration.ssh-authorized-key}' > $${:authorized-keys-file}
[promises]
......@@ -264,27 +276,29 @@ instance-promises =
$${selenium-server-node-firefox-52-listen-promise:name}
$${selenium-server-node-firefox-60-listen-promise:name}
$${selenium-server-node-firefox-68-listen-promise:name}
$${selenium-server-node-firefox-78-listen-promise:name}
$${selenium-server-node-instance-chromium-69-listen-promise:name}
$${selenium-server-node-instance-chromium-91-listen-promise:name}
[check-port-listening-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = $${:_buildout_section_name_}.py
[sshd-listen-promise]
<= check-port-listening-promise
config-hostname = $${sshd-address:hostname}
config-host = $${sshd-address:hostname}
config-port = $${sshd-address:port}
[selenium-server-frontend-listen-promise]
<= check-port-listening-promise
config-hostname = $${selenium-server-frontend-instance:ip}
config-host = $${selenium-server-frontend-instance:ip}
config-port = $${selenium-server-frontend-instance:port}
[selenium-server-hub-listen-promise]
<= check-port-listening-promise
config-hostname = $${selenium-server-hub-instance:hostname}
config-host = $${selenium-server-hub-instance:hostname}
config-port = $${selenium-server-hub-instance:port}
# Promise waiting for all nodes to be registered
......@@ -294,8 +308,8 @@ module = check_command_execute
name = $${:_buildout_section_name_}.py
config-command =
$${selenium-server-check-nodes-registered:rendered} $${selenium-server-hub-instance:api-url} $${:expected-node-count}
# We have 4 nodes with 3 slots each
expected-node-count = 12
# We have 6 nodes with 3 slots each
expected-node-count = 18
[selenium-server-check-nodes-registered]
recipe = slapos.recipe.template:jinja2
......@@ -313,24 +327,34 @@ template =
[selenium-server-node-firefox-52-listen-promise]
<= check-port-listening-promise
config-hostname = $${selenium-server-node-instance-firefox-52:hostname}
config-host = $${selenium-server-node-instance-firefox-52:hostname}
config-port = $${selenium-server-node-instance-firefox-52:port}
[selenium-server-node-firefox-60-listen-promise]
<= check-port-listening-promise
config-hostname = $${selenium-server-node-instance-firefox-60:hostname}
config-host = $${selenium-server-node-instance-firefox-60:hostname}
config-port = $${selenium-server-node-instance-firefox-60:port}
[selenium-server-node-firefox-68-listen-promise]
<= check-port-listening-promise
config-hostname = $${selenium-server-node-instance-firefox-68:hostname}
config-host = $${selenium-server-node-instance-firefox-68:hostname}
config-port = $${selenium-server-node-instance-firefox-68:port}
[selenium-server-node-firefox-78-listen-promise]
<= check-port-listening-promise
config-host = $${selenium-server-node-instance-firefox-78:hostname}
config-port = $${selenium-server-node-instance-firefox-78:port}
[selenium-server-node-instance-chromium-69-listen-promise]
<= check-port-listening-promise
config-hostname = $${selenium-server-node-instance-chromium-69:hostname}
config-host = $${selenium-server-node-instance-chromium-69:hostname}
config-port = $${selenium-server-node-instance-chromium-69:port}
[selenium-server-node-instance-chromium-91-listen-promise]
<= check-port-listening-promise
config-host = $${selenium-server-node-instance-chromium-91:hostname}
config-port = $${selenium-server-node-instance-chromium-91:port}
[publish-connection-parameter]
recipe = slapos.cookbook:publish
......@@ -347,7 +371,7 @@ ssh-fingerprint = $${ssh-key-fingerprint:fingerprint}
run-node-command = PORT=7999 bash -c 'trap '"'"'kill -TERM $SSHPID; wait $SSHPID '"'"' TERM INT; ssh -L 4444:$${selenium-server-hub-instance:hostname}:$${selenium-server-hub-instance:port} -R $PORT:127.0.0.1:$PORT -p $${sshd-service:port} $${sshd-service:username}@$${sshd-service:ip} & SSHPID=$!; java -jar selenium-server-standalone-3.14.0.jar -role node -host 127.0.0.1 -port $PORT ; wait "$SSHPID"'
[instance-parameter]
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration
computer = $${slap-connection:computer-id}
partition = $${slap-connection:partition-id}
......
software/seleniumserver/instance.cfg.in
View file @ 5081f0b0
......@@ -6,15 +6,21 @@ eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
[dynamic-template-selenium]
recipe = slapos.recipe.template:jinja2
template = ${template-selenium:output}
rendered = $${buildout:parts-directory}/$${:_buildout_section_name_}/$${:filename}
filename = instance-selenium.cfg
context =
key partition_ipv6 slap-configuration:ipv6-random
[switch-softwaretype]
recipe = slapos.cookbook:softwaretype
default = ${template-selenium:output}
recipe = slapos.cookbook:switch-softwaretype
RootSoftwareInstance = $${:default}
default = dynamic-template-selenium:rendered
[slap-connection]
# part to migrate to new - separated words
computer-id = $${slap_connection:computer_id}
partition-id = $${slap_connection:partition_id}
server-url = $${slap_connection:server_url}
software-release-url = $${slap_connection:software_release_url}
key-file = $${slap_connection:key_file}
cert-file = $${slap_connection:cert_file}
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration
computer = $${slap-connection:computer-id}
partition = $${slap-connection:partition-id}
url = $${slap-connection:server-url}
software/seleniumserver/software.cfg
View file @ 5081f0b0
......@@ -73,5 +73,6 @@ post-install =
post-install =
${symlink-extra-fonts-to-firefox-fonts-dir:install}
[versions]
plone.recipe.command = 1.1
[firefox-78]
post-install =
${symlink-extra-fonts-to-firefox-fonts-dir:install}
software/seleniumserver/test/test.py
View file @ 5081f0b0
......@@ -426,6 +426,14 @@ class TestFirefox68(
desired_capabilities = dict(DesiredCapabilities.FIREFOX, version='68.0.2esr')
user_agent = 'Gecko/20100101 Firefox/68.0'
class TestFirefox78(
BrowserCompatibilityMixin,
SeleniumServerTestCase,
ImageComparisonTestCase,
):
desired_capabilities = dict(DesiredCapabilities.FIREFOX, version='78.1.0esr')
user_agent = 'Gecko/20100101 Firefox/78.0'
class TestChrome69(
BrowserCompatibilityMixin,
......@@ -434,3 +442,12 @@ class TestChrome69(
):
desired_capabilities = dict(DesiredCapabilities.CHROME, version='69.0.3497.0')
user_agent = 'Chrome/69.0.3497.0'
class TestChrome91(
BrowserCompatibilityMixin,
SeleniumServerTestCase,
ImageComparisonTestCase,
):
desired_capabilities = dict(DesiredCapabilities.CHROME, version='91.0.4472.114')
user_agent = 'Chrome/91.0.4472.0'
software/slapos-master/buildout.hash.cfg
View file @ 5081f0b0
......@@ -14,11 +14,11 @@
# not need these here).
[template-erp5]
filename = instance-erp5.cfg.in
md5sum = c341609c8d0cfe240a3a2ae35bd713b6
md5sum = 84f099cc9852c4f53a075dccbb3880f0
[template-balancer]
filename = instance-balancer.cfg.in
md5sum = 9c67c77eab5195b2674e340fb44c48a2
md5sum = c7c0bb9abbd0f8cc6c7956d83a61c4b3
[template-apache-backend-conf]
filename = apache-backend.conf.in
......
software/slapos-master/instance-balancer.cfg.in
View file @ 5081f0b0
......@@ -2,6 +2,7 @@
{% set part_list = [] -%}
{% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
{% set ssl_parameter_dict = slapparameter_dict['ssl'] -%}
{% set frontend_caucase_url_list = ssl_parameter_dict.get('frontend-caucase-url-list', []) -%}
{% set shared_ca_path = slapparameter_dict.get('shared-certificate-authority-path') -%}
{#
XXX: This template only supports exactly one IPv4 and (if ipv6 is used) one IPv6
......@@ -38,7 +39,7 @@ mode = 644
{% set haproxy_dict = {} -%}
{% set apache_dict = {} -%}
{% set zope_virtualhost_monster_backend_dict = {} %}
{% set test_runner_url_dict = {} %} {# family_name => list of apache URLs #}
{% set test_runner_url_dict = {} %} {# family_name => list of URLs #}
{% set next_port = itertools.count(slapparameter_dict['tcpv4-port']).next -%}
{% for family_name, parameter_id_list in sorted(
slapparameter_dict['zope-family-dict'].iteritems()) -%}
......@@ -59,19 +60,19 @@ mode = 644
{% set test_runner_address_list = slapparameter_dict.get(parameter_id ~ '-test-runner-address-list', []) %}
{% if test_runner_address_list -%}
{% set test_runner_backend_mapping = {} %}
{% set test_runner_apache_url_list = [] %}
{% set test_runner_balancer_url_list = [] %}
{% set test_runner_external_port = next_port() %}
{% for i, (test_runner_internal_ip, test_runner_internal_port) in enumerate(test_runner_address_list) %}
{% do test_runner_backend_mapping.__setitem__(
'unit_test_' ~ i,
'http://' ~ test_runner_internal_ip ~ ':' ~ test_runner_internal_port ) %}
{% do test_runner_apache_url_list.append(
{% do test_runner_balancer_url_list.append(
'https://' ~ ipv4 ~ ':' ~ test_runner_external_port ~ '/unit_test_' ~ i ~ '/' ) %}
{% endfor %}
{% do zope_virtualhost_monster_backend_dict.__setitem__(
(ipv4, test_runner_external_port),
( ssl_authentication, test_runner_backend_mapping ) ) -%}
{% do test_runner_url_dict.__setitem__(family_name, test_runner_apache_url_list) -%}
{% do test_runner_url_dict.__setitem__(family_name, test_runner_balancer_url_list) -%}
{% endif -%}
{% endfor -%}
......@@ -123,6 +124,23 @@ caucase-key = ${directory:apache-conf}/apache-caucase.pem
ca-cert = ${directory:apache-conf}/ca.crt
crl = ${directory:apache-conf}/crl.pem
[simplefile]
< = jinja2-template-base
template = inline:{{ '{{ content }}' }}
{% macro simplefile(section_name, file_path, content, mode='') -%}
{% set content_section_name = section_name ~ '-content' -%}
[{{ content_section_name }}]
content = {{ dumps(content) }}
[{{ section(section_name) }}]
< = simplefile
rendered = {{ file_path }}
context = key content {{content_section_name}}:content
mode = {{ mode }}
{%- endmacro %}
[apache-ssl]
{% if ssl_parameter_dict.get('key') -%}
key = ${apache-ssl-key:rendered}
......@@ -185,9 +203,9 @@ input = inline:
[{{ section('apache-promise') }}]
# Check any apache port in ipv4, expect other ports and ipv6 to behave consistently
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = apache.py
config-hostname = {{ ipv4 }}
config-host = {{ ipv4 }}
config-port = {{ apache_dict.values()[0][0] }}
[{{ section('publish') }}]
......@@ -273,8 +291,6 @@ template = inline:
{% endfor %}
[apachedex-parameters]
# XXX - Sample log file with curent date: apache_access.log-%(date)s.gz
# which will be equivalent to apache_access.log-20150112.gz if the date is 2015-01-12
apache-log-list = ${apache-conf-parameter-dict:access-log}
configuration = ${monitor-apachedex-report-config:rendered}
promise-threshold = {{ slapparameter_dict['apachedex-promise-threshold'] }}
......
software/slapos-master/instance-erp5.cfg.in
View file @ 5081f0b0
......@@ -221,7 +221,10 @@ return =
{%- if test_runner_enabled %}
test-runner-address-list
{% endif %}
{% set bt5_default_list = 'erp5_full_text_mroonga_catalog slapos_configurator' -%}
{% set bt5_default_list = [
'erp5_full_text_mroonga_catalog',
'slapos_configurator',
] -%}
{% if has_jupyter -%}
{% do bt5_default_list.append('erp5_data_notebook') -%}
{% endif -%}
......@@ -254,6 +257,7 @@ config-wsgi = {{ dumps(slapparameter_dict.get('wsgi', True)) }}
config-test-runner-enabled = {{ dumps(test_runner_enabled) }}
config-test-runner-node-count = {{ dumps(test_runner_node_count) }}
config-wcfs_enable = {{ dumps(wcfs_enable) }}
config-test-runner-configuration = {{ dumps(slapparameter_dict.get('test-runner', {})) }}
software-type = zope
{% set global_publisher_timeout = slapparameter_dict.get('publisher-timeout') -%}
......
software/slapos-master/test/test/__init__.py
View file @ 5081f0b0
......@@ -30,11 +30,21 @@ import os
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
_setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', '..', 'software.cfg')))
setup_module_executed = False
def setUpModule():
# slapos.testing.testcase's only need to be executed once
global setup_module_executed
if not setup_module_executed:
_setUpModule()
setup_module_executed = True
class ERP5InstanceTestCase(SlapOSInstanceTestCase):
"""ERP5 base test case
"""
......
software/slapos-master/test/test/test_balancer.py 0 → 100644
View file @ 5081f0b0
This diff is collapsed.
software/slapos-master/test/test/test_erp5.py
View file @ 5081f0b0
......@@ -41,31 +41,50 @@ import requests
from . import ERP5InstanceTestCase
from . import setUpModule
setUpModule # pyflakes
setUpModule # pyflakes
class TestPublishedURLIsReachableMixin(object):
"""Mixin that checks that default page of ERP5 is reachable.
"""
def _checkERP5IsReachable(self, url):
def _checkERP5IsReachable(self, base_url, site_id, verify):
# We access ERP5 trough a "virtual host", which should make
# ERP5 produce URLs using https://virtual-host-name:1234/virtual_host_root
# as base.
virtual_host_url = urlparse.urljoin(
base_url,
'/VirtualHostBase/https/virtual-host-name:1234/{}/VirtualHostRoot/_vh_virtual_host_root/'
.format(site_id))
# What happens is that instantiation just create the services, but does not
# wait for ERP5 to be initialized. When this test run ERP5 instance is
# instantiated, but zope is still busy creating the site and haproxy
# replies with 503 Service Unavailable when zope is not started yet, with
# 404 when erp5 site is not created, with 500 when mysql is not yet
# reachable, so we retry in a loop until we get a succesful response.
for i in range(1, 60):
# XXX can we get CA from caucase already ?
r = requests.get(url, verify=False)
if r.status_code != requests.codes.ok:
delay = i * 2
self.logger.warn(
"ERP5 was not available, sleeping for %ds and retrying", delay)
time.sleep(delay)
continue
r.raise_for_status()
break
# instantiated, but zope is still busy creating the site and haproxy replies
# with 503 Service Unavailable when zope is not started yet, with 404 when
# erp5 site is not created, with 500 when mysql is not yet reachable, so we
# configure this requests session to retry.
# XXX we should probably add a promise instead
session = requests.Session()
session.mount(
base_url,
requests.adapters.HTTPAdapter(
max_retries=requests.packages.urllib3.util.retry.Retry(
total=60,
backoff_factor=.5,
status_forcelist=(404, 500, 503))))
r = session.get(virtual_host_url, verify=verify, allow_redirects=False)
self.assertEqual(r.status_code, requests.codes.found)
# access on / are redirected to login form, with virtual host preserved
self.assertEqual(r.headers.get('location'), 'https://virtual-host-name:1234/virtual_host_root/login_form')
# login page can be rendered and contain the text "ERP5"
r = session.get(
urlparse.urljoin(base_url, '{}/login_form'.format(site_id)),
verify=verify,
allow_redirects=False,
)
self.assertEqual(r.status_code, requests.codes.ok)
self.assertIn("ERP5", r.text)
def test_published_family_default_v6_is_reachable(self):
......@@ -73,15 +92,18 @@ class TestPublishedURLIsReachableMixin(object):
"""
param_dict = self.getRootPartitionConnectionParameterDict()
self._checkERP5IsReachable(
urlparse.urljoin(param_dict['family-default-v6'], param_dict['site-id']))
param_dict['family-default-v6'],
param_dict['site-id'],
verify=False)
def test_published_family_default_v4_is_reachable(self):
"""Tests the IPv4 URL published by the root partition is reachable.
"""
param_dict = self.getRootPartitionConnectionParameterDict()
self._checkERP5IsReachable(
urlparse.urljoin(param_dict['family-default'], param_dict['site-id']))
param_dict['family-default'],
param_dict['site-id'],
verify=False)
class TestDefaultParameters(
ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
......@@ -99,6 +121,30 @@ class TestMedusa(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
def getInstanceParameterDict(cls):
return {'_': json.dumps({'wsgi': False})}
class TestJupyter(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
"""Test ERP5 Jupyter notebook
"""
__partition_reference__ = 'jupyter'
@classmethod
def getInstanceParameterDict(cls):
return {'_': json.dumps({'jupyter': {'enable': True}})}
def test_jupyter_notebook_is_reachable(self):
param_dict = self.getRootPartitionConnectionParameterDict()
self.assertEqual(
'https://[%s]:8888/tree' % self._ipv6_address,
param_dict['jupyter-url']
)
result = requests.get(
param_dict['jupyter-url'], verify=False, allow_redirects=False)
self.assertEqual(
[requests.codes.found, True, '/login?next=%2Ftree'],
[result.status_code, result.is_redirect, result.headers['Location']]
)
class TestApacheBalancerPorts(ERP5InstanceTestCase):
"""Instantiate with two zope families, this should create for each family:
......@@ -132,19 +178,16 @@ class TestApacheBalancerPorts(ERP5InstanceTestCase):
self.assertTrue(parsed.port)
def test_published_family_parameters(self):
# when we request two families, we have two published family-{family_name}
# URLs
# when we request two families, we have two published family-{family_name} URLs
param_dict = self.getRootPartitionConnectionParameterDict()
for family_name in ('family1', 'family2'):
self.checkValidHTTPSURL(
param_dict['family-{family_name}'.format(family_name=family_name)])
self.checkValidHTTPSURL(
param_dict['family-{family_name}-v6'.format(
family_name=family_name)])
param_dict['family-{family_name}-v6'.format(family_name=family_name)])
def test_published_test_runner_url(self):
# each family's also a list of test test runner URLs, by default 3 per
# family
# each family's also a list of test test runner URLs, by default 3 per family
param_dict = self.getRootPartitionConnectionParameterDict()
for family_name in ('family1', 'family2'):
family_test_runner_url_list = param_dict[
......@@ -154,8 +197,7 @@ class TestApacheBalancerPorts(ERP5InstanceTestCase):
self.checkValidHTTPSURL(url)
def test_zope_listen(self):
# we requested 3 zope in family1 and 5 zopes in family2, we should have 8
# zope running.
# we requested 3 zope in family1 and 5 zopes in family2, we should have 8 zope running.
with self.slap.instance_supervisor_rpc as supervisor:
all_process_info = supervisor.getAllProcessInfo()
self.assertEqual(
......@@ -190,46 +232,7 @@ class TestApacheBalancerPorts(ERP5InstanceTestCase):
])
class TestDisableTestRunner(
ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
"""Test ERP5 can be instantiated without test runner.
"""
__partition_reference__ = 'distr'
@classmethod
def getInstanceParameterDict(cls):
return {'_': json.dumps({'test-runner': {'enabled': False}})}
def test_no_runUnitTestScript(self):
"""No runUnitTest script should be generated in any partition.
"""
# self.computer_partition_root_path is the path of root partition.
# we want to assert that no scripts exist in any partition.
bin_programs = map(
os.path.basename,
glob.glob(self.computer_partition_root_path + "/../*/bin/*"))
self.assertTrue(bin_programs) # just to check the glob was correct.
self.assertNotIn('runUnitTest', bin_programs)
self.assertNotIn('runTestSuite', bin_programs)
def test_no_apache_testrunner_port(self):
# Apache only listen on two ports, there is no apache ports allocated for
# test runner
with self.slap.instance_supervisor_rpc as supervisor:
all_process_info = supervisor.getAllProcessInfo()
process_info, = [p for p in all_process_info if p['name'] == 'apache']
apache_process = psutil.Process(process_info['pid'])
self.assertEqual(
sorted([socket.AF_INET, socket.AF_INET6]),
sorted(
c.family
for c in apache_process.connections()
if c.status == 'LISTEN'
))
class TestZopeNodeParameterOverride(
ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
class TestZopeNodeParameterOverride(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
"""Test override zope node parameters
"""
__partition_reference__ = 'override'
......@@ -244,7 +247,7 @@ class TestZopeNodeParameterOverride(
"server": {},
"cache-size-bytes": "20MB",
"cache-size-bytes!": [
("bb-0", 1 << 20),
("bb-0", 1<<20),
("bb-.*", "500MB"),
],
"pool-timeout": "10m",
......@@ -315,7 +318,7 @@ class TestZopeNodeParameterOverride(
partition = self.getComputerPartitionPath('zope-bb')
for zope in xrange(5):
checkConf({
"cache-size-bytes": "500MB" if zope else 1 << 20,
"cache-size-bytes": "500MB" if zope else 1<<20,
}, {
"cache-size": None,
})
......
software/slapos-master/test/test/test_mariadb.py 0 → 100644
View file @ 5081f0b0
This diff is collapsed.
software/slapos-master/test/test/test_wcfs.py 0 → 100644
View file @ 5081f0b0
# Copyright (C) 2021 Nexedi SA and Contributors.
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
# the terms of any of the Free Software licenses or any of the Open Source
# Initiative approved licenses and Convey the resulting work. Corresponding
# source of such a combination shall include the source code for all other
# software used.
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options.
import json
import os.path
import unittest
from slapos.grid.utils import md5digest
from . import ERP5InstanceTestCase, setUpModule as _setUpModule
from .test_erp5 import TestPublishedURLIsReachableMixin
# skip tests when software release is built with wendelin.core 1.
def setUpModule():
_setUpModule()
cls = ERP5InstanceTestCase
if not os.path.exists(
os.path.join(
cls.slap.software_directory,
md5digest(cls.getSoftwareURL()),
'bin', 'wcfs')):
raise unittest.SkipTest("built with wendelin.core 1")
class TestWCFS(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
"""Test Wendelin Core File System
"""
__partition_reference__ = 'wcfs'
@classmethod
def getInstanceParameterDict(cls):
return {'_': json.dumps({'wcfs': {'enable': True}})}
def test_wcfs_accessible(self):
"""Verify that wcfs filesystem is basically accessible.
- we can read .wcfs/zurl
- its content is equal to published `serving-zurl`
"""
zurl = json.loads(
self.getComputerPartition('wcfs').getConnectionParameter('_')
)['serving-zurl']
mntpt = lookupMount(zurl)
zurl_ = readfile("%s/.wcfs/zurl" % mntpt)
self.assertEqual(zurl_, zurl)
# lookupMount returns /proc/mount entry for wcfs mounted to serve zurl.
def lookupMount(zurl):
for line in readfile('/proc/mounts').splitlines():
# <zurl> <mountpoint> fuse.wcfs ...
zurl_, mntpt, typ, _ = line.split(None, 3)
if typ != 'fuse.wcfs':
continue
if zurl_ == zurl:
return mntpt
raise KeyError("lookup mount %s: no /proc/mounts entry" % zurl)
# readfile returns content of file @path.
def readfile(path):
with open(path, 'r') as f:
return f.read()
software/slapos-sr-testing/software-py3.cfg
View file @ 5081f0b0
......@@ -13,6 +13,7 @@ extra-eggs +=
[template]
extra =
# The following list is for SR whose buildout runs only with Python 3.
galene ${slapos.test.galene-setup:setup}
helloworld ${slapos.test.helloworld-setup:setup}
jupyter ${slapos.test.jupyter-setup:setup}
monitor ${slapos.test.monitor-setup:setup}
......@@ -20,3 +21,4 @@ extra =
powerdns ${slapos.test.powerdns-setup:setup}
proftpd ${slapos.test.proftpd-setup:setup}
repman ${slapos.test.repman-setup:setup}
restic-rest-server ${slapos.test.restic_rest_server-setup:setup}
software/slapos-sr-testing/software.cfg
View file @ 5081f0b0
......@@ -5,6 +5,7 @@ extends =
../../component/curl/buildout.cfg
../../component/openssl/buildout.cfg
../../component/git/buildout.cfg
../../component/pycurl/buildout.cfg
../../component/faketime/buildout.cfg
../../component/pillow/buildout.cfg
../../component/python-cryptography/buildout.cfg
......@@ -98,6 +99,11 @@ setup = ${slapos-repository:location}/software/proftpd/test/
egg = slapos.test.re6stnet
setup = ${slapos-repository:location}/software/re6stnet/test/
[slapos.test.restic_rest_server-setup]
<= setup-develop-egg
egg = slapos.test.restic_rest_server
setup = ${slapos-repository:location}/software/restic-rest-server/test/
[slapos.test.seleniumserver-setup]
<= setup-develop-egg
egg = slapos.test.seleniumserver
......@@ -183,6 +189,11 @@ setup = ${slapos-repository:location}/software/repman/test/
egg = slapos.test.jscrawler
setup = ${slapos-repository:location}/software/jscrawler/test/
[slapos.test.galene-setup]
<= setup-develop-egg
egg = slapos.test.galene
setup = ${slapos-repository:location}/software/galene/test/
[slapos.core-repository]
<= git-clone-repository
repository = https://lab.nexedi.com/nexedi/slapos.core.git
......@@ -204,6 +215,7 @@ extra-eggs =
${lxml-python:egg}
${slapos.core-setup:egg}
${pillow-python:egg}
${pycurl:egg}
erp5.util
${python-pynacl:egg}
${python-cryptography:egg}
......@@ -225,6 +237,7 @@ extra-eggs =
${slapos.test.powerdns-setup:egg}
${slapos.test.proftpd-setup:egg}
${slapos.test.re6stnet-setup:egg}
${slapos.test.restic_rest_server-setup:egg}
${slapos.test.seleniumserver-setup:egg}
${slapos.test.slaprunner-setup:egg}
${slapos.test.jupyter-setup:egg}
......@@ -237,6 +250,7 @@ extra-eggs =
${slapos.test.repman-setup:egg}
${slapos.test.caucase-setup:egg}
${slapos.test.jscrawler-setup:egg}
${slapos.test.galene-setup:egg}
${slapos.test.html5as-setup:egg}
${slapos.test.html5as-base-setup:egg}
${slapos.test.fluentd-setup:egg}
......
software/slaprunner/buildout.hash.cfg
View file @ 5081f0b0
......@@ -18,7 +18,7 @@ md5sum = 8d6878ff1d2e75010c50a1a2b0c13b24
[template-runner]
filename = instance-runner.cfg
md5sum = 9f367deb7597957e7108bee719b78bcc
md5sum = ebdecea5c1653ed752e06fbc8be7e6b2
[template-runner-import-script]
filename = template/runner-import.sh.jinja2
......
software/slaprunner/instance-runner.cfg
View file @ 5081f0b0
......@@ -622,16 +622,16 @@ monitor-password = $${monitor-publish-parameters:monitor-password}
[slaprunner-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = slaprunner.py
config-hostname = $${slaprunner:ipv6}
config-host = $${slaprunner:ipv6}
config-port = $${slaprunner:runner_port}
[runner-sshd-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = runner-sshd.py
config-hostname = $${slap-network-information:global-ipv6}
config-host = $${slap-network-information:global-ipv6}
config-port = $${runner-sshd-port:port}
[slap-parameter]
......@@ -866,9 +866,9 @@ log = $${runnerdirectory:home}/instance/*/.slapgrid/log/instance.log $${runnerdi
[supervisord-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = supervisord.py
config-hostname = $${slaprunner:ipv4}
config-host = $${slaprunner:ipv4}
config-port = $${supervisord:port}
[slapos-supervisord-promise]
......@@ -879,9 +879,9 @@ config-command = ${buildout:bin-directory}/slapos node supervisorctl --cfg=$${sl
[slapos-proxy-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = slaproxy.py
config-hostname = $${slaprunner:ipv4}
config-host = $${slaprunner:ipv4}
config-port = $${slaprunner:proxy_port}
# XXX Monitor
......
software/theia/buildout.hash.cfg
View file @ 5081f0b0
......@@ -15,7 +15,7 @@
[instance-theia]
_update_hash_filename_ = instance-theia.cfg.jinja.in
md5sum = 11274189a4492204e046fef45fdd809c
md5sum = 557dec61b7bf58601051575234e7fd05
[instance]
_update_hash_filename_ = instance.cfg.in
......
software/theia/instance-theia.cfg.jinja.in
View file @ 5081f0b0
......@@ -77,16 +77,16 @@ instance-promises =
[theia-listen-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = $${:_buildout_section_name_}.py
config-hostname = $${theia-instance:ip}
config-host = $${theia-instance:ip}
config-port = $${theia-instance:port}
[frontend-listen-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = $${:_buildout_section_name_}.py
config-hostname = $${frontend-instance:ip}
config-host = $${frontend-instance:ip}
config-port = $${frontend-instance:port}
[remote-frontend-url-available-promise]
......@@ -107,10 +107,10 @@ config-check-secure = 1
[slapos-standalone-listen-promise]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
# XXX promise plugins can not contain "slapos" in their names
name = standalone-listen-promise.py
config-hostname = $${slapos-standalone-instance:hostname}
config-host = $${slapos-standalone-instance:hostname}
config-port = $${slapos-standalone-instance:port}
[slapos-autorun-promise]
......
software/theia/test/dummy/software.cfg
View file @ 5081f0b0
......@@ -16,4 +16,4 @@ init =
json.dump(dict(os.environ), f)
[versions]
slapos.recipe.build = 0.46
slapos.recipe.build = 0.47
software/turnserver/buildout.hash.cfg
View file @ 5081f0b0
......@@ -15,12 +15,12 @@
[instance-cfg]
filename = instance.cfg.in
md5sum = 776c7de2054f78ba79382c22d85018be
md5sum = b43d5e8d1fc2d0eeb54f91cefe6a5bae
[template-turnserver]
filename = instance-turnserver.cfg.jinja2.in
md5sum = df8768f165036dbe0435bd1678b9deb3
md5sum = 7af3318d7249e9afe22436d9fe200159
[template-insecure-turnserver]
filename = instance-insecure-turnserver.cfg.jinja2.in
md5sum = e88678b9f322251a201860a13f4db6b8
md5sum = 3db65c3a16eb76ab438ac3817d1a5fea
software/turnserver/instance-insecure-turnserver.cfg.jinja2.in
View file @ 5081f0b0
{% set part_list = [] -%}
{% set server_name = slapparameter_dict.get('server-name', 'turn.example.com') -%}
{%- set parameter = dict(default_parameter_dict, **slapparameter_dict) %}
{%- set server_name = parameter['server-name'] %}
[directory]
recipe = slapos.cookbook:mkdirectory
......@@ -17,9 +18,11 @@ plugins = ${:etc}/plugin
recipe = slapos.cookbook:generate.password
bytes = 8
{% set turn_port = slapparameter_dict.get('port', 3478) -%}
{% set turn_tls_port = slapparameter_dict.get('tls-port', 5349) -%}
{% set listining_ip = slapparameter_dict.get('listening-ip', (ipv4 | list)[0]) -%}
{% set turn_port = parameter['port'] -%}
{% set turn_tls_port = parameter['tls-port'] -%}
# listening-ip parameter is mandatory
{% set listening_ip = slapparameter_dict['listening-ip'] -%}
[turnserver-config]
recipe = collective.recipe.template
user = nxdturn
......@@ -27,11 +30,11 @@ input = inline:
listening-port={{ turn_port }}
lt-cred-mech
realm={{ server_name }}
{% if slapparameter_dict.get('external-ip', '') %}
external-ip={{ slapparameter_dict['external-ip'] }}
{% if parameter['external-ip'] %}
external-ip={{ parameter['external-ip'] }}
{% endif %}
fingerprint
listening-ip={{ listining_ip }}
listening-ip={{ listening_ip }}
server-name={{ server_name }}
no-stdout-log
simple-log
......@@ -54,9 +57,9 @@ hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
[promise-check-turnserver-port]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = turnserver-port-listening.py
config-hostname = {{ listining_ip }}
config-host = {{ listening_ip }}
config-port = {{ turn_port }}
[publish-connection-information]
......
software/turnserver/instance-turnserver.cfg.jinja2.in
View file @ 5081f0b0
{% set part_list = [] -%}
{% set server_name = slapparameter_dict.get('server-name', 'turn.example.com') -%}
{%- set part_list = [] -%}
{%- set parameter = dict(default_parameter_dict, **slapparameter_dict) %}
{%- set server_name = parameter['server-name'] %}
[directory]
recipe = slapos.cookbook:mkdirectory
......@@ -31,9 +32,9 @@ mode = {{ mode }}
{% do part_list.append(section_name) -%}
{%- endmacro %}
{% if slapparameter_dict.get('ssl-key') and slapparameter_dict.get('ssl-crt') -%}
{{ simplefile('ssl-certificate', '${turnserver-ssl:certificate}', slapparameter_dict.get('ssl-crt')) }}
{{ simplefile('ssl-key', '${turnserver-ssl:key}', slapparameter_dict.get('ssl-key'), 600) }}
{% if parameter['ssl-key'] and parameter['ssl-crt'] -%}
{{ simplefile('ssl-certificate', '${turnserver-ssl:certificate}', parameter['ssl-crt']) }}
{{ simplefile('ssl-key', '${turnserver-ssl:key}', parameter['ssl-key'], 600) }}
{% else -%}
{% do part_list.append('gen-certificate') -%}
[gen-certificate]
......@@ -57,9 +58,9 @@ secret-file = ${directory:etc}/.turnsecret
command =
if [ ! -s "${:secret-file}" ]; then
cat <<EOF > ${:secret-file}
[turnserver]
secret = $("{{ parameter_dict['openssl'] }}/bin/openssl" rand -hex 32)
EOF
[turnserver]
secret = $("{{ parameter_dict['openssl'] }}/bin/openssl" rand -hex 32)
EOF
fi
chmod 600 ${:secret-file}
......@@ -68,9 +69,11 @@ recipe = slapos.cookbook:zero-knowledge.read
file-path = ${gen-secret:secret-file}
secret =
{% set turn_port = slapparameter_dict.get('port', 3478) -%}
{% set turn_tls_port = slapparameter_dict.get('tls-port', 5349) -%}
{% set listining_ip = slapparameter_dict.get('listening-ip', (ipv4 | list)[0]) -%}
{% set turn_port = parameter['port'] -%}
{% set turn_tls_port = parameter['tls-port'] -%}
# listening-ip parameter is mandatory
{% set listening_ip = slapparameter_dict['listening-ip'] -%}
[turnserver-config]
recipe = collective.recipe.template
input = inline:
......@@ -80,9 +83,9 @@ input = inline:
lt-cred-mech
use-auth-secret
static-auth-secret=${read-secret:secret}
listening-ip={{ listining_ip }}
{% if slapparameter_dict.get('external-ip', '') %}
external-ip={{ slapparameter_dict['external-ip'] }}
listening-ip={{ listening_ip }}
{% if parameter['external-ip'] %}
external-ip={{ parameter['external-ip'] }}
{% endif %}
server-name={{ server_name }}
realm={{ server_name }}
......@@ -120,16 +123,16 @@ hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
[promise-check-turnserver-port]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = turnserver-port-listening.py
config-hostname = {{ listining_ip }}
config-host = {{ listening_ip }}
config-port = {{ turn_port }}
[promise-check-turnserver-tls-port]
<= monitor-promise-base
module = check_port_listening
module = check_socket_listening
name = turnserver-tls-port-listening.py
config-hostname = {{ listining_ip }}
config-host = {{ listening_ip }}
config-port = {{ turn_tls_port }}
[publish-connection-information]
......
software/turnserver/instance.cfg.in
View file @ 5081f0b0
This diff is collapsed.
software/turnserver/test/test.py
View file @ 5081f0b0
This diff is collapsed.
stack/erp5/buildout.cfg
View file @ 5081f0b0
This diff is collapsed.
stack/erp5/buildout.hash.cfg
View file @ 5081f0b0
This diff is collapsed.
stack/erp5/instance-balancer.cfg.in
View file @ 5081f0b0
This diff is collapsed.
stack/erp5/instance-erp5.cfg.in
View file @ 5081f0b0
This diff is collapsed.
stack/erp5/instance-kumofs.cfg.in
View file @ 5081f0b0
This diff is collapsed.
stack/erp5/instance-postfix.cfg.in
View file @ 5081f0b0
This diff is collapsed.
stack/erp5/instance-zeo.cfg.in
View file @ 5081f0b0
This diff is collapsed.
stack/erp5/instance-zope.cfg.in
View file @ 5081f0b0
This diff is collapsed.
stack/erp5/instance.cfg.in
View file @ 5081f0b0
This diff is collapsed.
stack/erp5/zope-versions.cfg
View file @ 5081f0b0
This diff is collapsed.
stack/erp5/ztk-versions.cfg
View file @ 5081f0b0
This diff is collapsed.
stack/lamp/buildout.hash.cfg
View file @ 5081f0b0
This diff is collapsed.
stack/lamp/instance-apache-php.cfg.in
View file @ 5081f0b0
This diff is collapsed.
stack/monitor/README.md
View file @ 5081f0b0
This diff is collapsed.
stack/monitor/buildout.hash.cfg
View file @ 5081f0b0
This diff is collapsed.
stack/resilient/buildout.hash.cfg
View file @ 5081f0b0
This diff is collapsed.
stack/resilient/pbsready.cfg.in
View file @ 5081f0b0
This diff is collapsed.
stack/slapos.cfg
View file @ 5081f0b0
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7