Commit f1d9dc31 authored by Łukasz Nowak's avatar Łukasz Nowak

Update Release Candidate

parents 747f918f 1cfb657f
[buildout] [buildout]
extends = extends =
../../component/golang/buildout.cfg ../../component/golang/buildout.cfg
gowork.cfg
parts = parts =
gowork gowork
caddy caddy
[gowork.goinstall] [caddy-get]
command = bash -c ". ${gowork:env.sh} && cd ${go_github.com_caddyserver_caddy:location} && GO111MODULE=on go install -v $(echo -n '${gowork:install}' |tr '\n' ' ')" <= go-git-package
go.importpath = github.com/caddyserver/caddy
repository = https://lab.nexedi.com/nexedi/caddy.git
revision = nxd-v1.0.3-1-g2c11cedc
[gowork] [gowork]
golang = ${golang1.14:location} golang = ${golang1.16:location}
install = install =
github.com/caddyserver/caddy/... ${caddy-get:location}:./...
[caddy] [caddy]
recipe = plone.recipe.command recipe = plone.recipe.command
......
# Code generated by gowork-snapshot; DO NOT EDIT.
# list of go git repositories to fetch
[gowork.goinstall]
depends_gitfetch =
${go_github.com_caddyserver_caddy:recipe}
[go_github.com_caddyserver_caddy]
<= go-git-package
go.importpath = github.com/caddyserver/caddy
repository = https://lab.nexedi.com/nexedi/caddy.git
revision = nxd-v1.0.3-1-g2c11cedc
...@@ -80,18 +80,16 @@ environment-extra = ...@@ -80,18 +80,16 @@ environment-extra =
# ---- infrastructure to build Go workspaces / projects ---- # ---- infrastructure to build Go workspaces / projects ----
# gowork is a top-level section representing workspace # gowork is the top-level section that defines Go workspace.
# #
# users should add `install` field to [gowork] to describe packages they want to # It specifies global settings that are used to build Go programs:
# be installed (+ automatically their dependencies are installed too). e.g. #
# - Go toolchain to use, e.g.
# #
# [gowork] # [gowork]
# install = # golang = ${golang1.15:location}
# lab.nexedi.com/kirr/neo/go/...
# github.com/pkg/profile
# golang.org/x/perf/cmd/benchstat
# #
# For Cgo support pkg-config is made pre-available by gowork, and users # - For Cgo support pkg-config is made pre-available by gowork, and users
# should list paths where to search for pkg-config files, e.g. this way: # should list paths where to search for pkg-config files, e.g. this way:
# #
# [gowork] # [gowork]
...@@ -99,11 +97,22 @@ environment-extra = ...@@ -99,11 +97,22 @@ environment-extra =
# ${sqlite3:location}/lib/pkgconfig # ${sqlite3:location}/lib/pkgconfig
# ${zlib:location}/lib/pkgconfig # ${zlib:location}/lib/pkgconfig
# #
# It is also possible to specify Go build flags used for compilation e.g. this way: # - It is also possible to specify Go build flags used for compilation e.g. this way:
# #
# [gowork] # [gowork]
# buildflags = -race # buildflags = -race
# #
#
# Users can also add `install` field to [gowork] to request Go programs to be
# automatically installed, for example:
#
# [gowork]
# install =
# lab.nexedi.com/kirr/neo/go/...@v0.0.0-20210103165133-f3effa6c535f
# golang.org/x/tools/gopls@v0.4.3
# ${helloweb:location}/go:./...
#
#
# ${go:exe} is standalone executable that runs go in activated gowork environment. # ${go:exe} is standalone executable that runs go in activated gowork environment.
[go] [go]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
...@@ -116,9 +125,9 @@ template= inline: ...@@ -116,9 +125,9 @@ template= inline:
exec go "$@" exec go "$@"
[gowork] [gowork]
directory = ${buildout:directory}/go.work directory = ${gowork.dir:directory}
src = ${:directory}/src src = ${gowork.dir:src}
bin = ${:directory}/bin bin = ${gowork.dir:bin}
depends = ${gowork.goinstall:recipe} depends = ${gowork.goinstall:recipe}
# go version used for the workspace (possible to override in applications) # go version used for the workspace (possible to override in applications)
...@@ -130,6 +139,9 @@ buildflags = ...@@ -130,6 +139,9 @@ buildflags =
# empty pkg-config path by default # empty pkg-config path by default
cpkgpath = cpkgpath =
# by default don't `go install` anything
install =
# everything is done by dependent parts # everything is done by dependent parts
recipe = plone.recipe.command recipe = plone.recipe.command
command = : command = :
...@@ -140,24 +152,52 @@ env.sh = ${gowork-env.sh:output} ...@@ -140,24 +152,52 @@ env.sh = ${gowork-env.sh:output}
[gowork-env.sh] [gowork-env.sh]
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/goenv.sh.in url = ${:_profile_base_location_}/goenv.sh.in
output = ${gowork:directory}/env.sh output = ${gowork.dir:directory}/env.sh
depends = ${gowork.mkdir:recipe} depends = ${gowork.dir:recipe}
md5sum = 6efdf5d63381c240c7d206d7939a63f7 md5sum = 6efdf5d63381c240c7d206d7939a63f7
[gowork.mkdir] [gowork.dir]
directory = ${buildout:directory}/go.work
src = ${:directory}/src
bin = ${:directory}/bin
# NOTE do not use slapos.cookbook:mkdirectory here - if anything in software (not instance) # NOTE do not use slapos.cookbook:mkdirectory here - if anything in software (not instance)
# uses slapos.cookbook:* in recipe - slapos.cookbook will get compiled against system # uses slapos.cookbook:* in recipe - slapos.cookbook will get compiled against system
# libxml/libxslt or fail to bootstrap at all if those are not present. # libxml/libxslt or fail to bootstrap at all if those are not present.
recipe = plone.recipe.command recipe = plone.recipe.command
command = mkdir -p ${gowork:directory} command = mkdir -p ${:directory}
update-command = ${:command} update-command = ${:command}
stop-on-error = true stop-on-error = true
# install go packages # install go programs
# clients should put package list to install to gowork:install ("..." requests installing everything) # clients can put program list to install to gowork:install
[gowork.goinstall] [gowork.goinstall]
recipe = plone.recipe.command recipe = plone.recipe.command
command = bash -c ". ${gowork:env.sh} && go install ${gowork:buildflags} -v $(echo -n '${gowork:install}' |tr '\n' ' ')" command = bash -c ". ${gowork:env.sh} &&
for x in $(echo -n '${gowork:install}' |tr '\n' ' '); do
case "\$x" in
# external module, e.g. golang.org/x/tools/gopls@v0.4.3
*@*)
echo GOMOD \$x
GO111MODULE=on go install ${gowork:buildflags} -v \$x
;;
# locally-cloned module source, e.g. <module-src-location>:./...
*:*)
echo GOMODSRC \$x
dir=\"\$${x%%:*}\"
arg=\"\$${x#*:}\"
(cd \$dir && GO111MODULE=on go install ${gowork:buildflags} -v \$arg)
;;
# non-module
*)
echo GOPKG \$x
GO111MODULE=off go install ${gowork:buildflags} -v \$x
;;
esac
done
"
update-command = ${:command} update-command = ${:command}
stop-on-error = true stop-on-error = true
...@@ -175,4 +215,4 @@ git-executable = ${git:location}/bin/git ...@@ -175,4 +215,4 @@ git-executable = ${git:location}/bin/git
# generated with the help of gowork-snapshot tool. # generated with the help of gowork-snapshot tool.
[go-git-package] [go-git-package]
<= git-repository <= git-repository
location = ${gowork:src}/${:go.importpath} location = ${gowork.dir:src}/${:go.importpath}
...@@ -4,7 +4,6 @@ extends = ...@@ -4,7 +4,6 @@ extends =
../git/buildout.cfg ../git/buildout.cfg
../ruby/buildout.cfg ../ruby/buildout.cfg
../golang/buildout.cfg ../golang/buildout.cfg
gowork.cfg
parts = parts =
helloweb-python helloweb-python
...@@ -21,9 +20,15 @@ parts = ...@@ -21,9 +20,15 @@ parts =
# cloning+building manually. However to be able to use third-party Go packages # cloning+building manually. However to be able to use third-party Go packages
# we need gowork support, and other languages can use helloweb repository from # we need gowork support, and other languages can use helloweb repository from
# under gowork as well. # under gowork as well.
[helloweb]
<= go-git-package
go.importpath = lab.nexedi.com/nexedi/helloweb
repository = https://lab.nexedi.com/nexedi/helloweb.git
revision = a7c788ae71
[gowork] [gowork]
install = install =
lab.nexedi.com/nexedi/helloweb/go/... ${helloweb:location}/go:./...
golang = ${golang1.16:location} golang = ${golang1.16:location}
...@@ -44,7 +49,7 @@ input = inline: ...@@ -44,7 +49,7 @@ input = inline:
[helloweb-egg] [helloweb-egg]
recipe = zc.recipe.egg:develop recipe = zc.recipe.egg:develop
egg = helloweb egg = helloweb
setup = ${gowork:src}/lab.nexedi.com/nexedi/helloweb/python/ setup = ${helloweb:location}/python/
[helloweb-python] [helloweb-python]
recipe = zc.recipe.egg:scripts recipe = zc.recipe.egg:scripts
...@@ -86,7 +91,7 @@ environment = ...@@ -86,7 +91,7 @@ environment =
[helloweb-ruby-bundle] [helloweb-ruby-bundle]
recipe = slapos.recipe.cmmi recipe = slapos.recipe.cmmi
path = ${gowork:src}/lab.nexedi.com/nexedi/helloweb/ruby/ path = ${helloweb:location}/ruby/
configure-command = : configure-command = :
make-binary = make-binary =
......
# Code generated by gowork-snapshot; DO NOT EDIT.
# list of go git repositories to fetch
[gowork.goinstall]
depends_gitfetch =
${go_lab.nexedi.com_nexedi_helloweb:recipe}
[go_lab.nexedi.com_nexedi_helloweb]
<= go-git-package
go.importpath = lab.nexedi.com/nexedi/helloweb
repository = https://lab.nexedi.com/nexedi/helloweb.git
revision = 8bfedac656
...@@ -14,6 +14,12 @@ parts = ...@@ -14,6 +14,12 @@ parts =
[nodejs] [nodejs]
<= nodejs-8.9.4 <= nodejs-8.9.4
[nodejs-14.16.0]
<= nodejs-base
openssl_location = ${openssl:location}
version = v14.16.0
md5sum = 7dc3666f407bf4e12a01ce1be2883d31
[nodejs-12.18.3] [nodejs-12.18.3]
<= nodejs-base <= nodejs-base
openssl_location = ${openssl:location} openssl_location = ${openssl:location}
......
...@@ -6,7 +6,6 @@ parts = ...@@ -6,7 +6,6 @@ parts =
gowork gowork
[gowork] [gowork]
golang = ${golang1.13:location}
install = install =
buildflags = -v --tags server --ldflags "-extldflags 'static' -w -s -X main.GoOS=linux -X main.GoArch=amd64 -X main.Version=2.1 -X main.FullVersion=$FULLVERSION -X main.Build=$(date +%FT%T%z) -X main.WithProvisioning=ON -X main.WithOpenSVC=OFF -X main.WithHaproxy=ON -X main.WithMaxscale=ON -X main.WithMariadbshardproxy=ON -X main.WithProxysql=ON -X main.WithSphinx=ON -X main.WithArbitration=OFF -X main.WithArbitrationClient=ON -X main.WithMonitoring=ON -X main.WithHttp=ON -X main.WithBackup=ON -X main.WithMail=ON -X main.WithEnforce=ON -X main.WithDeprecate=ON" buildflags = -v --tags server --ldflags "-extldflags 'static' -w -s -X main.GoOS=linux -X main.GoArch=amd64 -X main.Version=2.1 -X main.FullVersion=$FULLVERSION -X main.Build=$(date +%FT%T%z) -X main.WithProvisioning=ON -X main.WithOpenSVC=OFF -X main.WithHaproxy=ON -X main.WithMaxscale=ON -X main.WithMariadbshardproxy=ON -X main.WithProxysql=ON -X main.WithSphinx=ON -X main.WithArbitration=OFF -X main.WithArbitrationClient=ON -X main.WithMonitoring=ON -X main.WithHttp=ON -X main.WithBackup=ON -X main.WithMail=ON -X main.WithEnforce=ON -X main.WithDeprecate=ON"
......
...@@ -5,9 +5,6 @@ extends = ...@@ -5,9 +5,6 @@ extends =
parts = parts =
restic restic
[gowork]
golang = ${golang1.13:location}
[restic] [restic]
recipe = plone.recipe.command recipe = plone.recipe.command
......
...@@ -3,7 +3,10 @@ ...@@ -3,7 +3,10 @@
[buildout] [buildout]
extends = extends =
../pygolang/buildout.cfg ../pygolang/buildout.cfg
../zodbtools/buildout.cfg
../numpy/buildout.cfg ../numpy/buildout.cfg
../golang/buildout.cfg
../zlib/buildout.cfg
../git/buildout.cfg ../git/buildout.cfg
parts = parts =
...@@ -21,12 +24,18 @@ environment = wendelin.core-env ...@@ -21,12 +24,18 @@ environment = wendelin.core-env
# dependent eggs that must come through in-tree recipes # dependent eggs that must come through in-tree recipes
depends = depends =
${zodbtools:egg}
${numpy:egg} ${numpy:egg}
[wendelin.core-env] [wendelin.core-env]
# wendelin.core needs git(dev) to build # wendelin.core needs git(dev) and go to build
PATH = ${git:location}/bin:%(PATH)s PATH = ${git:location}/bin:%(PATH)s
GO = ${go:exe}
# wcfs needs this:
[gowork]
cpkgpath += ${zlib:location}/lib/pkgconfig
[wendelin.core-repository] [wendelin.core-repository]
......
...@@ -32,11 +32,12 @@ eggs = ...@@ -32,11 +32,12 @@ eggs =
neoppod[tests] neoppod[tests]
ZEO[test] ZEO[test]
# env.sh for that python to be on $PATH # env.sh for that python + go to be on $PATH
[wendelin.core-env.sh] [wendelin.core-env.sh]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
rendered = ${buildout:directory}/${:_buildout_section_name_} rendered = ${buildout:directory}/${:_buildout_section_name_}
template = inline: template = inline:
. ${gowork:env.sh}
export PS1="(wendelin.core-env) $PS1" export PS1="(wendelin.core-env) $PS1"
export PATH=${buildout:bin-directory}:${gdb:location}/bin:$PATH export PATH=${buildout:bin-directory}:${gdb:location}/bin:$PATH
......
# https://classic.yarnpkg.com/ package manager for nodejs
[buildout]
extends =
../nodejs/buildout.cfg
[yarn]
<= yarn-1.22.10
[yarn-1.22.10]
<= yarn-wrapper
yarn-download = ${yarn-download-1.22.10:location}
[yarn-1.17.3]
<= yarn-wrapper
yarn-download = ${yarn-download-1.17.3:location}
[yarn-1.16.0]
<= yarn-wrapper
yarn-download = ${yarn-download-1.16.0:location}
[yarn-1.3.2]
<= yarn-wrapper
yarn-download = ${yarn-download-1.3.2:location}
[yarn-wrapper]
recipe = slapos.recipe.template:jinja2
rendered = ${:location}/bin/yarn
template = inline:
#!/bin/sh
PATH=${nodejs:location}/bin/:$PATH
exec ${:yarn-download}/bin/yarn $@
location = ${buildout:parts-directory}/${:_buildout_section_name_}
bin-yarn = ${:rendered}
[yarn-download]
recipe = slapos.recipe.build:download-unpacked
url = https://github.com/yarnpkg/yarn/releases/download/v${:version}/yarn-v${:version}.tar.gz
[yarn-download-1.22.10]
<= yarn-download
version = 1.22.10
md5sum = 52e8dbe9d0cb90683dd3ee2ebf2becb8
[yarn-download-1.17.3]
<= yarn-download
version = 1.17.3
md5sum = 4a02e1687a150113ad6b0215f9afdb3e
[yarn-download-1.3.2]
<= yarn-download
version = 1.3.2
md5sum = db82fa09c996e9318f2f1d2ab99228f9
[yarn-download-1.16.0]
<= yarn-download
version = 1.16.0
md5sum = 46790033c23803387890f545e4040690
...@@ -21,6 +21,7 @@ Here are listed the most important changes, which might affect upgrades. ...@@ -21,6 +21,7 @@ Here are listed the most important changes, which might affect upgrades.
* fix: use kedifa with with for file with multiple CAs * fix: use kedifa with with for file with multiple CAs
* feature: support query string (the characters after ? in the url) in url and https-url * feature: support query string (the characters after ? in the url) in url and https-url
* fix: by having unique acl names fix rare bug of directing traffic to https-url instead of url or otherwise * fix: by having unique acl names fix rare bug of directing traffic to https-url instead of url or otherwise
* feature: failover backend
1.0.164 (2020-09-24) 1.0.164 (2020-09-24)
-------------------- --------------------
......
...@@ -236,6 +236,8 @@ This set of parameters is used to control the way how the backend checks will be ...@@ -236,6 +236,8 @@ This set of parameters is used to control the way how the backend checks will be
Please be aware that the `health-check-timeout` is really short by default, so in case if `/` of the backend is slow to reply configure proper path with `health-check-http-path` to not mark such backend down too fast, before increasing the check timeout. Please be aware that the `health-check-timeout` is really short by default, so in case if `/` of the backend is slow to reply configure proper path with `health-check-http-path` to not mark such backend down too fast, before increasing the check timeout.
Thanks to using health-check it's possible to configure failover system. By providing `health-check-failover-url` or `health-check-failover-https-url` some special backend can be used to reply in case if original backend replies with error (codes like `5xx`). As a note one can setup this failover URL like `https://failover.example.com/?p=` so that the path from the incoming request will be passed as parameter. Additionally authentication to failover URL is supported with `health-check-authenticate-to-failover-backend` and SSL Proxy verification with `health-check-failover-ssl-proxy-verify` and `health-check-failover-ssl-proxy-ca-crt`.
Examples Examples
======== ========
......
...@@ -26,11 +26,11 @@ md5sum = a6a626fd1579fd1d4b80ea67433ca16a ...@@ -26,11 +26,11 @@ md5sum = a6a626fd1579fd1d4b80ea67433ca16a
[profile-caddy-replicate] [profile-caddy-replicate]
filename = instance-apache-replicate.cfg.in filename = instance-apache-replicate.cfg.in
md5sum = 1ab3fc07bb186601b54c584a3ccaf1c3 md5sum = 1248911409cbeea980a838b04ee451d2
[profile-slave-list] [profile-slave-list]
_update_hash_filename_ = templates/apache-custom-slave-list.cfg.in _update_hash_filename_ = templates/apache-custom-slave-list.cfg.in
md5sum = 9eb14b83ee6fc8a5afa8267d9bcf4772 md5sum = 8ce1d5bf09662d941f940be7e6493918
[profile-replicate-publish-slave-information] [profile-replicate-publish-slave-information]
_update_hash_filename_ = templates/replicate-publish-slave-information.cfg.in _update_hash_filename_ = templates/replicate-publish-slave-information.cfg.in
...@@ -50,7 +50,7 @@ md5sum = a0ae858a3db8825c22d33d323392f588 ...@@ -50,7 +50,7 @@ md5sum = a0ae858a3db8825c22d33d323392f588
[template-backend-haproxy-configuration] [template-backend-haproxy-configuration]
_update_hash_filename_ = templates/backend-haproxy.cfg.in _update_hash_filename_ = templates/backend-haproxy.cfg.in
md5sum = 8c4e2548a12c8fd7dba74f940201745a md5sum = 17f9582671327d8e4321a7fd1cdcb0fe
[template-empty] [template-empty]
_update_hash_filename_ = templates/empty.in _update_hash_filename_ = templates/empty.in
......
...@@ -200,7 +200,7 @@ context = ...@@ -200,7 +200,7 @@ context =
{% endfor %} {% endfor %}
{% do slave.__setitem__('server-alias', ' '.join(slave_server_alias_unclashed)) %} {% do slave.__setitem__('server-alias', ' '.join(slave_server_alias_unclashed)) %}
{% endif %} {% endif %}
{% for url_key in ['url', 'https-url'] %} {% for url_key in ['url', 'https-url', 'health-check-failover-url', 'health-check-failover-https-url'] %}
{% if url_key in slave %} {% if url_key in slave %}
{% set url = (slave[url_key] or '').strip() %} {% set url = (slave[url_key] or '').strip() %}
{% if not validators.url(url) %} {% if not validators.url(url) %}
...@@ -210,14 +210,16 @@ context = ...@@ -210,14 +210,16 @@ context =
{% endif %} {% endif %}
{% endif %} {% endif %}
{% endfor %} {% endfor %}
{% if 'ssl_proxy_ca_crt' in slave %} {% for k in ['ssl_proxy_ca_crt', 'health-check-failover-ssl-proxy-ca-crt'] %}
{% set ssl_proxy_ca_crt = slave.get('ssl_proxy_ca_crt', '') %} {% if k in slave %}
{% set crt = slave.get(k, '') %}
{% set check_popen = popen([software_parameter_dict['openssl'], 'x509', '-noout']) %} {% set check_popen = popen([software_parameter_dict['openssl'], 'x509', '-noout']) %}
{% do check_popen.communicate(ssl_proxy_ca_crt) %} {% do check_popen.communicate(crt) %}
{% if check_popen.returncode != 0 %} {% if check_popen.returncode != 0 %}
{% do slave_error_list.append('ssl_proxy_ca_crt is invalid') %} {% do slave_error_list.append('%s is invalid' % (k,)) %}
{% endif %} {% endif %}
{% endif %} {% endif %}
{% endfor %}
{# BBB: SlapOS Master non-zero knowledge BEGIN #} {# BBB: SlapOS Master non-zero knowledge BEGIN #}
{% for key in ['ssl_key', 'ssl_crt', 'ssl_ca_crt'] %} {% for key in ['ssl_key', 'ssl_crt', 'ssl_ca_crt'] %}
{% if key in slave %} {% if key in slave %}
......
...@@ -286,6 +286,44 @@ ...@@ -286,6 +286,44 @@
"default": "1", "default": "1",
"type": "integer" "type": "integer"
}, },
"health-check-failover-url": {
"description": "URL of the failover backend",
"pattern": "^(http|https|ftp)://",
"title": "Failover backend URL",
"type": "string"
},
"health-check-failover-https-url": {
"description": "HTTPS URL of the failover backend if it is different from health-check-failover-url parameter. Note: It requires https-url to be configured, as otherwise the differentiation does not make sense..",
"pattern": "^(http|https|ftp)://",
"title": "Failover HTTPS Backend URL",
"type": "string"
},
"health-check-authenticate-to-failover-backend": {
"description": "If set to true the frontend certificate will be used as authentication certificate to the failover backend. Note: failover backend might have to know the frontend CA, available with 'backend-client-caucase-url'.",
"enum": [
"false",
"true"
],
"title": "Authenticate to failover backend",
"type": "string"
},
"health-check-failover-ssl-proxy-verify": {
"default": "false",
"description": "If set to true, failover backend SSL Certificates will be checked and frontend will refuse to proxy if certificate is invalid",
"enum": [
"false",
"true"
],
"title": "Verify failover backend certificates",
"type": "string"
},
"health-check-failover-ssl-proxy-ca-crt": {
"default": "",
"description": "Content of the SSL Certificate Authority file of the failover backend (to be used with health-check-failover-ssl-proxy-verify)",
"textarea": true,
"title": "SSL failover backend Authority's Certificate",
"type": "string"
},
"strict-transport-security": { "strict-transport-security": {
"title": "Strict Transport Security", "title": "Strict Transport Security",
"description": "Enables Strict Transport Security (HSTS) on the slave, the default 0 results with option disabled. Setting the value enables HSTS and sets the value of max-age. More information: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Strict-Transport-Security", "description": "Enables Strict Transport Security (HSTS) on the slave, the default 0 results with option disabled. Setting the value enables HSTS and sets the value of max-age. More information: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Strict-Transport-Security",
......
...@@ -29,7 +29,7 @@ parts += ...@@ -29,7 +29,7 @@ parts +=
recipe = slapos.recipe.build:gitclone recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.com/nexedi/kedifa.git repository = https://lab.nexedi.com/nexedi/kedifa.git
git-executable = ${git:location}/bin/git git-executable = ${git:location}/bin/git
revision = 1e38bcb3bf56f544262780be7e8858f3e6ceedc5 revision = 3fccc2ec945c59b644a12fa40225795abd61e0b0
[kedifa-develop] [kedifa-develop]
recipe = zc.recipe.egg:develop recipe = zc.recipe.egg:develop
......
...@@ -58,6 +58,18 @@ context = ...@@ -58,6 +58,18 @@ context =
{%- do slave_instance.__setitem__(prefix, info_dict) %} {%- do slave_instance.__setitem__(prefix, info_dict) %}
{%- endfor %} {%- endfor %}
{%- do slave_instance.__setitem__('ssl_proxy_verify', ('' ~ slave_instance.get('ssl-proxy-verify', '')).lower() in TRUE_VALUES) %} {%- do slave_instance.__setitem__('ssl_proxy_verify', ('' ~ slave_instance.get('ssl-proxy-verify', '')).lower() in TRUE_VALUES) %}
{%- for key, prefix in [('health-check-failover-url', 'http_backend'), ('health-check-failover-https-url', 'https_backend')] %}
{%- set parsed = urlparse_module.urlparse(slave_instance.get(key, '').strip()) %}
{%- set info_dict = slave_instance[prefix] %}
{%- do info_dict.__setitem__('health-check-failover-scheme', parsed.scheme) %}
{%- do info_dict.__setitem__('health-check-failover-hostname', parsed.hostname) %}
{%- do info_dict.__setitem__('health-check-failover-port', parsed.port or DEFAULT_PORT[parsed.scheme]) %}
{%- do info_dict.__setitem__('health-check-failover-path', parsed.path) %}
{%- do info_dict.__setitem__('health-check-failover-query', parsed.query) %}
{%- do info_dict.__setitem__('health-check-failover-fragment', parsed.fragment) %}
{%- do slave_instance.__setitem__(prefix, info_dict) %}
{%- endfor %}
{%- do slave_instance.__setitem__('health-check-failover-ssl-proxy-verify', ('' ~ slave_instance.get('health-check-failover-ssl-proxy-verify', '')).lower() in TRUE_VALUES) %}
{%- do slave_instance.__setitem__('enable-http2', ('' ~ slave_instance.get('enable-http2', configuration['enable-http2-by-default'])).lower() in TRUE_VALUES) %} {%- do slave_instance.__setitem__('enable-http2', ('' ~ slave_instance.get('enable-http2', configuration['enable-http2-by-default'])).lower() in TRUE_VALUES) %}
{%- for key in ['https-only', 'websocket-transparent'] %} {%- for key in ['https-only', 'websocket-transparent'] %}
{%- do slave_instance.__setitem__(key, ('' ~ slave_instance.get(key, 'true')).lower() in TRUE_VALUES) %} {%- do slave_instance.__setitem__(key, ('' ~ slave_instance.get(key, 'true')).lower() in TRUE_VALUES) %}
...@@ -135,6 +147,7 @@ context = ...@@ -135,6 +147,7 @@ context =
{%- endfor %} {%- endfor %}
{%- do slave_instance.__setitem__('strict-transport-security', int(slave_instance['strict-transport-security'])) %} {%- do slave_instance.__setitem__('strict-transport-security', int(slave_instance['strict-transport-security'])) %}
{%- do slave_instance.__setitem__('authenticate-to-backend', ('' ~ slave_instance.get('authenticate-to-backend', '')).lower() in TRUE_VALUES) %} {%- do slave_instance.__setitem__('authenticate-to-backend', ('' ~ slave_instance.get('authenticate-to-backend', '')).lower() in TRUE_VALUES) %}
{%- do slave_instance.__setitem__('health-check-authenticate-to-failover-backend', ('' ~ slave_instance.get('health-check-authenticate-to-failover-backend', '')).lower() in TRUE_VALUES) %}
{#- Setup active check #} {#- Setup active check #}
{%- do slave_instance.__setitem__('health-check', ('' ~ slave_instance.get('health-check', '')).lower() in TRUE_VALUES) %} {%- do slave_instance.__setitem__('health-check', ('' ~ slave_instance.get('health-check', '')).lower() in TRUE_VALUES) %}
{%- if slave_instance['health-check'] %} {%- if slave_instance['health-check'] %}
...@@ -266,7 +279,7 @@ command = {{ software_parameter_dict['htpasswd'] }} -cb ${:file} {{ slave_refere ...@@ -266,7 +279,7 @@ command = {{ software_parameter_dict['htpasswd'] }} -cb ${:file} {{ slave_refere
{%- set certificate = '%s/%s' % (autocert, cert_name) %} {%- set certificate = '%s/%s' % (autocert, cert_name) %}
{%- do slave_parameter_dict.__setitem__('certificate', certificate )%} {%- do slave_parameter_dict.__setitem__('certificate', certificate )%}
{#- Set ssl certificates for each slave #} {#- Set ssl certificates for each slave #}
{%- for cert_name in ('ssl_csr', 'ssl_proxy_ca_crt')%} {%- for cert_name in ('ssl_csr', 'ssl_proxy_ca_crt', 'health-check-failover-ssl-proxy-ca-crt')%}
{%- set cert_file_key = 'path_to_' + cert_name %} {%- set cert_file_key = 'path_to_' + cert_name %}
{%- if cert_name in slave_instance %} {%- if cert_name in slave_instance %}
{%- set cert_title = '%s-%s' % (slave_reference, cert_name.replace('ssl_', '')) %} {%- set cert_title = '%s-%s' % (slave_reference, cert_name.replace('ssl_', '')) %}
......
...@@ -32,9 +32,15 @@ defaults ...@@ -32,9 +32,15 @@ defaults
{%- endif %} {%- endif %}
{%- endfor %} {%- endfor %}
{%- if matched['count'] > 0 %} {%- if matched['count'] > 0 %}
{%- if slave_instance[SCHEME_PREFIX_MAPPING[scheme]]['health-check-failover-hostname'] %}
acl is_failover_{{ slave_instance['slave_reference'] }}_{{ scheme }} nbsrv({{ slave_instance['slave_reference'] }}-{{ scheme }}) eq 0
use_backend {{ slave_instance['slave_reference'] }}-{{ scheme }} if is_{{ slave_instance['slave_reference'] }}_{{ scheme }} ! is_failover_{{ slave_instance['slave_reference'] }}_{{ scheme }}
use_backend {{ slave_instance['slave_reference'] }}-{{ scheme }}-failover if is_{{ slave_instance['slave_reference'] }}_{{ scheme }} is_failover_{{ slave_instance['slave_reference'] }}_{{ scheme }}
{%- else %}
use_backend {{ slave_instance['slave_reference'] }}-{{ scheme }} if is_{{ slave_instance['slave_reference'] }}_{{ scheme }} use_backend {{ slave_instance['slave_reference'] }}-{{ scheme }} if is_{{ slave_instance['slave_reference'] }}_{{ scheme }}
{%- endif %} {%- endif %}
{%- endif %} {%- endif %}
{%- endif %}
{%- endmacro %} {%- endmacro %}
# statistic # statistic
...@@ -123,5 +129,44 @@ backend {{ slave_instance['slave_reference'] }}-{{ scheme }} ...@@ -123,5 +129,44 @@ backend {{ slave_instance['slave_reference'] }}-{{ scheme }}
{%- endif %} {%- endif %}
{%- endif %} {%- endif %}
{%- endif %} {%- endif %}
{%- if info_dict['health-check-failover-hostname'] and info_dict['health-check-failover-port'] %}
{%- set ssl_list = [] %}
{%- if info_dict['health-check-failover-scheme'] == 'https' %}
{%- if slave_instance['health-check-authenticate-to-failover-backend'] %}
{%- do ssl_list.append('crt %s' % (configuration['certificate'],)) %}
{%- endif %}
{%- do ssl_list.append('ssl verify') %}
{%- if slave_instance['health-check-failover-ssl-proxy-verify'] %}
{%- if slave_instance['path_to_health-check-failover-ssl-proxy-ca-crt'] %}
{%- do ssl_list.append('required ca-file %s' % (slave_instance['path_to_health-check-failover-ssl-proxy-ca-crt'],)) %}
{%- else %}
{#- Backend SSL shall be verified, but not CA provided, disallow connection #}
{#- Simply dropping hostname from the dict will result with ignoring it... #}
{%- do info_dict.__setitem__('health-check-failover-hostname', '') %}
{%- endif %}
{%- else %}
{%- do ssl_list.append('none') %}
{%- endif %}
{%- endif %}
backend {{ slave_instance['slave_reference'] }}-{{ scheme }}-failover
{%- set hostname = info_dict['health-check-failover-hostname'] %}
{%- set port = info_dict['health-check-failover-port'] %}
{%- set path_list = [info_dict['health-check-failover-path'].rstrip('/')] %}
{%- set query = info_dict['health-check-failover-query'] %}
{%- if query %}
{%- do path_list.append(query) %}
{%- endif %}
{%- set path = '?'.join(path_list) %}
{%- if hostname and port %}
timeout server {{ slave_instance['request-timeout'] }}s
timeout connect {{ slave_instance['backend-connect-timeout'] }}s
retries {{ slave_instance['backend-connect-retries'] }}
server {{ slave_instance['slave_reference'] }}-backend {{ hostname }}:{{ port }} {{ ' '.join(ssl_list) }}
{%- if path %}
http-request set-path {{ path }}%[path]
{%- endif %}
{%- endif %}
{%- endif %}
{%- endfor %} {%- endfor %}
{%- endfor %} {%- endfor %}
...@@ -6146,6 +6146,16 @@ class TestSlaveRejectReportUnsafeDamaged(SlaveHttpFrontendTestCase): ...@@ -6146,6 +6146,16 @@ class TestSlaveRejectReportUnsafeDamaged(SlaveHttpFrontendTestCase):
'ssl-proxy-verify': True, 'ssl-proxy-verify': True,
'ssl_proxy_ca_crt': '', 'ssl_proxy_ca_crt': '',
}, },
'health-check-failover-SSL-PROXY-VERIFY_SSL_PROXY_CA_CRT_DAMAGED': {
'url': cls.backend_https_url,
'health-check-failover-ssl-proxy-verify': True,
'health-check-failover-ssl-proxy-ca-crt': 'damaged',
},
'health-check-failover-SSL-PROXY-VERIFY_SSL_PROXY_CA_CRT_EMPTY': {
'url': cls.backend_https_url,
'health-check-failover-ssl-proxy-verify': True,
'health-check-failover-ssl-proxy-ca-crt': '',
},
'BAD-BACKEND': { 'BAD-BACKEND': {
'url': 'http://1:2:3:4', 'url': 'http://1:2:3:4',
'https-url': 'http://host.domain:badport', 'https-url': 'http://host.domain:badport',
...@@ -6263,8 +6273,8 @@ class TestSlaveRejectReportUnsafeDamaged(SlaveHttpFrontendTestCase): ...@@ -6263,8 +6273,8 @@ class TestSlaveRejectReportUnsafeDamaged(SlaveHttpFrontendTestCase):
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address, 'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
'domain': 'example.com', 'domain': 'example.com',
'accepted-slave-amount': '5', 'accepted-slave-amount': '5',
'rejected-slave-amount': '26', 'rejected-slave-amount': '28',
'slave-amount': '31', 'slave-amount': '33',
'rejected-slave-dict': { 'rejected-slave-dict': {
'_HTTPS-URL': ['slave https-url "https://[fd46::c2ae]:!py!u\'123123\'"' '_HTTPS-URL': ['slave https-url "https://[fd46::c2ae]:!py!u\'123123\'"'
' invalid'], ' invalid'],
...@@ -6303,6 +6313,12 @@ class TestSlaveRejectReportUnsafeDamaged(SlaveHttpFrontendTestCase): ...@@ -6303,6 +6313,12 @@ class TestSlaveRejectReportUnsafeDamaged(SlaveHttpFrontendTestCase):
'_EMPTY-BACKEND': [ '_EMPTY-BACKEND': [
"slave https-url '' invalid", "slave https-url '' invalid",
"slave url '' invalid"], "slave url '' invalid"],
'_health-check-failover-SSL-PROXY-VERIFY_SSL_PROXY_CA_CRT_DAMAGED': [
'health-check-failover-ssl-proxy-ca-crt is invalid'
],
'_health-check-failover-SSL-PROXY-VERIFY_SSL_PROXY_CA_CRT_EMPTY': [
'health-check-failover-ssl-proxy-ca-crt is invalid'
],
'_health-check-fall': [ '_health-check-fall': [
'Wrong health-check-fall WRONG'], 'Wrong health-check-fall WRONG'],
'_health-check-fall-negative': [ '_health-check-fall-negative': [
...@@ -6373,6 +6389,28 @@ class TestSlaveRejectReportUnsafeDamaged(SlaveHttpFrontendTestCase): ...@@ -6373,6 +6389,28 @@ class TestSlaveRejectReportUnsafeDamaged(SlaveHttpFrontendTestCase):
parameter_dict parameter_dict
) )
def test_health_check_failover_ssl_proxy_ca_crt_damaged(self):
parameter_dict = self.parseSlaveParameterDict(
'health-check-failover-SSL-PROXY-VERIFY_SSL_PROXY_CA_CRT_DAMAGED')
self.assertEqual(
{
'request-error-list': [
"health-check-failover-ssl-proxy-ca-crt is invalid"]
},
parameter_dict
)
def test_health_check_failover_ssl_proxy_ca_crt_empty(self):
parameter_dict = self.parseSlaveParameterDict(
'health-check-failover-SSL-PROXY-VERIFY_SSL_PROXY_CA_CRT_EMPTY')
self.assertEqual(
{
'request-error-list': [
"health-check-failover-ssl-proxy-ca-crt is invalid"]
},
parameter_dict
)
def test_server_alias_same(self): def test_server_alias_same(self):
parameter_dict = self.parseSlaveParameterDict('SERVER-ALIAS-SAME') parameter_dict = self.parseSlaveParameterDict('SERVER-ALIAS-SAME')
self.assertLogAccessUrlWithPop(parameter_dict) self.assertLogAccessUrlWithPop(parameter_dict)
...@@ -7059,6 +7097,67 @@ class TestSlaveHealthCheck(SlaveHttpFrontendTestCase, TestDataMixin): ...@@ -7059,6 +7097,67 @@ class TestSlaveHealthCheck(SlaveHttpFrontendTestCase, TestDataMixin):
'health-check-rise': '3', 'health-check-rise': '3',
'health-check-fall': '7', 'health-check-fall': '7',
}, },
'health-check-failover-url': {
'https-only': False, # http and https access to check
'health-check-timeout': 1, # fail fast for test
'health-check-interval': 1, # fail fast for test
'url': cls.backend_url + 'url',
'https-url': cls.backend_url + 'https-url',
'health-check': True,
'health-check-http-path': '/health-check-failover-url',
'health-check-failover-url': cls.backend_url + 'failover-url?a=b&c=',
'health-check-failover-https-url':
cls.backend_url + 'failover-https-url?a=b&c=',
},
'health-check-failover-url-auth-to-backend': {
'https-only': False, # http and https access to check
'health-check-timeout': 1, # fail fast for test
'health-check-interval': 1, # fail fast for test
'url': cls.backend_url + 'url',
'https-url': cls.backend_url + 'https-url',
'health-check': True,
'health-check-http-path': '/health-check-failover-url-auth-to-backend',
'health-check-authenticate-to-failover-backend': True,
'health-check-failover-url': 'https://%s:%s/failover-url?a=b&c=' % (
cls._ipv4_address, cls._server_https_auth_port),
'health-check-failover-https-url':
'https://%s:%s/failover-https-url?a=b&c=' % (
cls._ipv4_address, cls._server_https_auth_port),
},
'health-check-failover-url-ssl-proxy-verified': {
'url': cls.backend_url,
'health-check-timeout': 1, # fail fast for test
'health-check-interval': 1, # fail fast for test
'health-check': True,
'health-check-http-path': '/health-check-failover-url-ssl-proxy'
'-verified',
'health-check-failover-url': cls.backend_https_url,
'health-check-failover-ssl-proxy-verify': True,
'health-check-failover-ssl-proxy-ca-crt':
cls.test_server_ca.certificate_pem,
},
'health-check-failover-url-ssl-proxy-verify-unverified': {
'url': cls.backend_url,
'health-check-timeout': 1, # fail fast for test
'health-check-interval': 1, # fail fast for test
'health-check': True,
'health-check-http-path': '/health-check-failover-url-ssl-proxy-verify'
'-unverified',
'health-check-failover-url': cls.backend_https_url,
'health-check-failover-ssl-proxy-verify': True,
'health-check-failover-ssl-proxy-ca-crt':
cls.another_server_ca.certificate_pem,
},
'health-check-failover-url-ssl-proxy-verify-missing': {
'url': cls.backend_url,
'health-check-timeout': 1, # fail fast for test
'health-check-interval': 1, # fail fast for test
'health-check': True,
'health-check-http-path': '/health-check-failover-url-ssl-proxy-verify'
'-missing',
'health-check-failover-url': cls.backend_https_url,
'health-check-failover-ssl-proxy-verify': True,
},
} }
@classmethod @classmethod
...@@ -7137,6 +7236,173 @@ backend _health-check-default-http ...@@ -7137,6 +7236,173 @@ backend _health-check-default-http
def test_health_check_custom(self): def test_health_check_custom(self):
self._test('health-check-custom') self._test('health-check-custom')
def test_health_check_failover_url(self):
parameter_dict = self.assertSlaveBase('health-check-failover-url')
slave_parameter_dict = self.getSlaveParameterDictDict()[
'health-check-failover-url']
# check normal access
result = fakeHTTPResult(parameter_dict['domain'], '/path')
self.assertEqualResultJson(result, 'Path', '/url/path')
result = fakeHTTPSResult(parameter_dict['domain'], '/path')
self.assertEqual(self.certificate_pem, der2pem(result.peercert))
self.assertEqualResultJson(result, 'Path', '/https-url/path')
# start replying with bad status code
result = requests.put(
self.backend_url + slave_parameter_dict[
'health-check-http-path'].strip('/'),
headers={'X-Reply-Status-Code': '502'})
self.assertEqual(result.status_code, httplib.CREATED)
time.sleep(3) # > health-check-timeout + health-check-interval
result = fakeHTTPSResult(parameter_dict['domain'], '/failoverpath')
self.assertEqual(self.certificate_pem, der2pem(result.peercert))
self.assertEqualResultJson(
result, 'Path', '/failover-https-url?a=b&c=/failoverpath')
result = fakeHTTPResult(parameter_dict['domain'], '/failoverpath')
self.assertEqualResultJson(
result, 'Path', '/failover-url?a=b&c=/failoverpath')
def test_health_check_failover_url_auth_to_backend(self):
parameter_dict = self.assertSlaveBase(
'health-check-failover-url-auth-to-backend')
slave_parameter_dict = self.getSlaveParameterDictDict()[
'health-check-failover-url-auth-to-backend']
self.startAuthenticatedServerProcess()
self.addCleanup(self.stopAuthenticatedServerProcess)
# assert that you can't fetch nothing without key
try:
requests.get(self.backend_https_auth_url, verify=False)
except Exception:
pass
else:
self.fail(
'Access to %r shall be not possible without certificate' % (
self.backend_https_auth_url,))
# check normal access
result = fakeHTTPResult(parameter_dict['domain'], '/path')
self.assertEqualResultJson(result, 'Path', '/url/path')
self.assertNotIn('X-Backend-Identification', result.headers)
result = fakeHTTPSResult(parameter_dict['domain'], '/path')
self.assertEqual(self.certificate_pem, der2pem(result.peercert))
self.assertEqualResultJson(result, 'Path', '/https-url/path')
self.assertNotIn('X-Backend-Identification', result.headers)
# start replying with bad status code
result = requests.put(
self.backend_url + slave_parameter_dict[
'health-check-http-path'].strip('/'),
headers={'X-Reply-Status-Code': '502'})
self.assertEqual(result.status_code, httplib.CREATED)
time.sleep(3) # > health-check-timeout + health-check-interval
result = fakeHTTPSResult(parameter_dict['domain'], '/failoverpath')
self.assertEqual(self.certificate_pem, der2pem(result.peercert))
self.assertEqualResultJson(
result, 'Path', '/failover-https-url?a=b&c=/failoverpath')
self.assertEqual(
'Auth Backend', result.headers['X-Backend-Identification'])
result = fakeHTTPResult(parameter_dict['domain'], '/failoverpath')
self.assertEqualResultJson(
result, 'Path', '/failover-url?a=b&c=/failoverpath')
self.assertEqual(
'Auth Backend', result.headers['X-Backend-Identification'])
def test_health_check_failover_url_ssl_proxy_verified(self):
parameter_dict = self.assertSlaveBase(
'health-check-failover-url-ssl-proxy-verified')
slave_parameter_dict = self.getSlaveParameterDictDict()[
'health-check-failover-url-ssl-proxy-verified']
# check normal access
result = fakeHTTPSResult(parameter_dict['domain'], '/path')
self.assertEqual(self.certificate_pem, der2pem(result.peercert))
self.assertEqualResultJson(result, 'Path', '/path')
# start replying with bad status code
result = requests.put(
self.backend_url + slave_parameter_dict[
'health-check-http-path'].strip('/'),
headers={'X-Reply-Status-Code': '502'})
self.assertEqual(result.status_code, httplib.CREATED)
time.sleep(3) # > health-check-timeout + health-check-interval
result = fakeHTTPSResult(
parameter_dict['domain'], '/test-path')
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
self.assertEqualResultJson(result, 'Path', '/test-path')
def test_health_check_failover_url_ssl_proxy_unverified(self):
parameter_dict = self.assertSlaveBase(
'health-check-failover-url-ssl-proxy-verify-unverified')
slave_parameter_dict = self.getSlaveParameterDictDict()[
'health-check-failover-url-ssl-proxy-verify-unverified']
# check normal access
result = fakeHTTPSResult(parameter_dict['domain'], '/path')
self.assertEqual(self.certificate_pem, der2pem(result.peercert))
self.assertEqualResultJson(result, 'Path', '/path')
# start replying with bad status code
result = requests.put(
self.backend_url + slave_parameter_dict[
'health-check-http-path'].strip('/'),
headers={'X-Reply-Status-Code': '502'})
self.assertEqual(result.status_code, httplib.CREATED)
time.sleep(3) # > health-check-timeout + health-check-interval
result = fakeHTTPSResult(
parameter_dict['domain'], '/test-path')
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
# as ssl proxy verification failed, service is unavailable
self.assertEqual(result.status_code, httplib.SERVICE_UNAVAILABLE)
def test_health_check_failover_url_ssl_proxy_missing(self):
parameter_dict = self.assertSlaveBase(
'health-check-failover-url-ssl-proxy-verify-missing')
slave_parameter_dict = self.getSlaveParameterDictDict()[
'health-check-failover-url-ssl-proxy-verify-missing']
# check normal access
result = fakeHTTPSResult(parameter_dict['domain'], '/path')
self.assertEqual(self.certificate_pem, der2pem(result.peercert))
self.assertEqualResultJson(result, 'Path', '/path')
# start replying with bad status code
result = requests.put(
self.backend_url + slave_parameter_dict[
'health-check-http-path'].strip('/'),
headers={'X-Reply-Status-Code': '502'})
self.assertEqual(result.status_code, httplib.CREATED)
time.sleep(3) # > health-check-timeout + health-check-interval
result = fakeHTTPSResult(
parameter_dict['domain'], '/test-path')
self.assertEqual(
self.certificate_pem,
der2pem(result.peercert))
# as ssl proxy verification failed, service is unavailable
self.assertEqual(result.status_code, httplib.SERVICE_UNAVAILABLE)
if __name__ == '__main__': if __name__ == '__main__':
class HTTP6Server(ThreadedHTTPServer): class HTTP6Server(ThreadedHTTPServer):
......
...@@ -21,6 +21,21 @@ T-2/var/log/httpd/_health-check-default_error_log ...@@ -21,6 +21,21 @@ T-2/var/log/httpd/_health-check-default_error_log
T-2/var/log/httpd/_health-check-disabled_access_log T-2/var/log/httpd/_health-check-disabled_access_log
T-2/var/log/httpd/_health-check-disabled_backend_log T-2/var/log/httpd/_health-check-disabled_backend_log
T-2/var/log/httpd/_health-check-disabled_error_log T-2/var/log/httpd/_health-check-disabled_error_log
T-2/var/log/httpd/_health-check-failover-url-auth-to-backend_access_log
T-2/var/log/httpd/_health-check-failover-url-auth-to-backend_backend_log
T-2/var/log/httpd/_health-check-failover-url-auth-to-backend_error_log
T-2/var/log/httpd/_health-check-failover-url-ssl-proxy-verified_access_log
T-2/var/log/httpd/_health-check-failover-url-ssl-proxy-verified_backend_log
T-2/var/log/httpd/_health-check-failover-url-ssl-proxy-verified_error_log
T-2/var/log/httpd/_health-check-failover-url-ssl-proxy-verify-missing_access_log
T-2/var/log/httpd/_health-check-failover-url-ssl-proxy-verify-missing_backend_log
T-2/var/log/httpd/_health-check-failover-url-ssl-proxy-verify-missing_error_log
T-2/var/log/httpd/_health-check-failover-url-ssl-proxy-verify-unverified_access_log
T-2/var/log/httpd/_health-check-failover-url-ssl-proxy-verify-unverified_backend_log
T-2/var/log/httpd/_health-check-failover-url-ssl-proxy-verify-unverified_error_log
T-2/var/log/httpd/_health-check-failover-url_access_log
T-2/var/log/httpd/_health-check-failover-url_backend_log
T-2/var/log/httpd/_health-check-failover-url_error_log
T-2/var/log/monitor-httpd-access.log T-2/var/log/monitor-httpd-access.log
T-2/var/log/monitor-httpd-error.log T-2/var/log/monitor-httpd-error.log
T-2/var/log/slave-introspection-access.log T-2/var/log/slave-introspection-access.log
......
...@@ -3,6 +3,28 @@ ...@@ -3,6 +3,28 @@
"description": "Parameters to instantiate ERP5", "description": "Parameters to instantiate ERP5",
"additionalProperties": false, "additionalProperties": false,
"definitions": { "definitions": {
"routing-rule-list": {
"description": "Maps the path received in requests to given zope path. Rules are applied in the order they are given. This requires the path received from the outside world (typically: frontend) to have its root correspond to Zope's root (for frontend: 'path' parameter must be empty), with the customary VirtualHostMonster construct (for frontend: 'type' must be 'zope').",
"type": "array",
"default": [["/", "/"]],
"items": {
"type": "array",
"minItems": 2,
"maxItems": 2,
"items": [
{
"title": "External path",
"description": "Path as received from the outside world, based on VirtualHostRoot element.",
"type": "string"
},
{
"title": "Internal path",
"description": "Zope path, based on Zope root object, the external path should correspond to. '%(site-id)s' is replaced by the site-id value, and '%%' replaced by '%'.",
"type": "string"
}
]
}
},
"tcpv4port": { "tcpv4port": {
"$ref": "./schemas-definitions.json#/tcpv4port" "$ref": "./schemas-definitions.json#/tcpv4port"
} }
...@@ -59,6 +81,42 @@ ...@@ -59,6 +81,42 @@
"uniqueItems": true, "uniqueItems": true,
"type": "array" "type": "array"
}, },
"activity-timeout": {
"description": "How long a CMFActivity-initiated transaction may last, in seconds",
"default": null,
"type": ["number", "null"]
},
"publisher-timeout": {
"description": "How long a publisher-initiated transaction may last, in seconds",
"default": null,
"type": ["number", "null"]
},
"family-override": {
"description": "Family-wide options, possibly overriding global options",
"default": {},
"patternProperties": {
".*": {
"default": {},
"properties": {
"webdav": {
"description": "Serve webdav queries, implies timerserver-interval=0 (disabled)",
"default": false,
"type": "boolean"
},
"activity-timeout": {
"description": "Override global activity timeout",
"type": ["number", "null"]
},
"publisher-timeout": {
"description": "Override global publisher timeout",
"type": ["number", "null"]
}
},
"type": "object"
}
},
"type": "object"
},
"hostalias-dict": { "hostalias-dict": {
"description": "Hostname-to-domain-name mapping", "description": "Hostname-to-domain-name mapping",
"default": {}, "default": {},
...@@ -154,11 +212,6 @@ ...@@ -154,11 +212,6 @@
"type": "boolean", "type": "boolean",
"default": false "default": false
}, },
"webdav": {
"description": "Serve webdav queries, implies timerserver-interval=0 (disabled). Mixing webdav and non-webdav nodes in a single family will give unspecified results.",
"default": false,
"type": "boolean"
},
"longrequest-logger-interval": { "longrequest-logger-interval": {
"description": "Period, in seconds, with which LongRequestLogger polls worker thread stack traces, or -1 to disable", "description": "Period, in seconds, with which LongRequestLogger polls worker thread stack traces, or -1 to disable",
"default": -1, "default": -1,
...@@ -452,6 +505,20 @@ ...@@ -452,6 +505,20 @@
"balancer": { "balancer": {
"description": "HTTP(S) load balancer proxy parameters", "description": "HTTP(S) load balancer proxy parameters",
"properties": { "properties": {
"path-routing-list": {
"$ref": "#/definitions/routing-rule-list",
"title": "Global path routing rules"
},
"family-path-routing-dict": {
"type": "object",
"title": "Family-specific path routing rules",
"description": "Applied, only for the eponymous family, before global path routing rules.",
"patternProperties": {
".+": {
"$ref": "#/definitions/routing-rule-list"
}
}
},
"ssl": { "ssl": {
"description": "HTTPS certificate generation parameters", "description": "HTTPS certificate generation parameters",
"properties": { "properties": {
......
...@@ -8,6 +8,7 @@ import shutil ...@@ -8,6 +8,7 @@ import shutil
import subprocess import subprocess
import tempfile import tempfile
import time import time
import urllib
import urlparse import urlparse
from BaseHTTPServer import BaseHTTPRequestHandler from BaseHTTPServer import BaseHTTPRequestHandler
from typing import Dict from typing import Dict
...@@ -166,7 +167,9 @@ class BalancerTestCase(ERP5InstanceTestCase): ...@@ -166,7 +167,9 @@ class BalancerTestCase(ERP5InstanceTestCase):
'ssl-authentication-dict': {}, 'ssl-authentication-dict': {},
'ssl': { 'ssl': {
'caucase-url': cls.getManagedResource("caucase", CaucaseService).url, 'caucase-url': cls.getManagedResource("caucase", CaucaseService).url,
} },
'family-path-routing-dict': {},
'path-routing-list': [],
} }
@classmethod @classmethod
...@@ -877,3 +880,93 @@ class TestClientTLS(BalancerTestCase): ...@@ -877,3 +880,93 @@ class TestClientTLS(BalancerTestCase):
with self.assertRaisesRegexp(Exception, 'certificate revoked'): with self.assertRaisesRegexp(Exception, 'certificate revoked'):
_make_request() _make_request()
class TestPathBasedRouting(BalancerTestCase):
"""Check path-based routing rewrites URLs as expected.
"""
__partition_reference__ = 'pbr'
@classmethod
def _getInstanceParameterDict(cls):
# type: () -> Dict
parameter_dict = super(
TestPathBasedRouting,
cls,
)._getInstanceParameterDict()
parameter_dict['zope-family-dict'][
'second'
] = parameter_dict['zope-family-dict'][
'default'
]
# Routing rules outermost slashes mean nothing. They are internally
# stripped and rebuilt in order to correctly represent the request's URL.
parameter_dict['family-path-routing-dict'] = {
'default': [
['foo/bar', 'erp5/boo/far/faz'], # no outermost slashes
['/foo', '/erp5/somewhere'],
['/foo/shadowed', '/foo_shadowed'], # unreachable
['/next', '/erp5/web_site_module/another_next_website'],
],
}
parameter_dict['path-routing-list'] = [
['/next', '/erp5/web_site_module/the_next_website'],
['/next2', '/erp5/web_site_module/the_next2_website'],
['//', '//erp5/web_site_module/123//'], # extraneous slashes
]
return parameter_dict
def test_routing(self):
# type: () -> None
published_dict = json.loads(self.computer_partition.getConnectionParameterDict()['_'])
scheme = 'scheme'
netloc = 'example.com:8080'
prefix = '/VirtualHostBase/' + scheme + '//' + urllib.quote(
netloc,
safe='',
)
# For easier reading of test data, visualy separating the virtual host
# base from the virtual host root
vhr = '/VirtualHostRoot'
def assertRoutingEqual(family, path, expected_path):
# sanity check: unlike the rules, this test is sensitive to outermost
# slashes, and paths must be absolute-ish for code simplicity.
assert path.startswith('/')
# Frontend is expected to provide URLs with the following path structure:
# /VirtualHostBase/<scheme>//<netloc>/VirtualHostRoot<path>
# where:
# - scheme is the user-input scheme
# - netloc is the user-input netloc
# - path is the user-input path
# Someday, frontends will instead propagate scheme and netloc via other
# means (likely: HTTP headers), in which case this test and the SR will
# need to be amended to reconstruct Virtual Host urls itself, and this
# test will need to be updated accordingly.
self.assertEqual(
requests.get(
urlparse.urljoin(published_dict[family], prefix + vhr + path),
verify=False,
).json()['Path'],
expected_path,
)
# Trailing slash presence is preserved.
assertRoutingEqual('default', '/foo/bar', prefix + '/erp5/boo/far/faz' + vhr + '/_vh_foo/bar')
assertRoutingEqual('default', '/foo/bar/', prefix + '/erp5/boo/far/faz' + vhr + '/_vh_foo/bar/')
# Subpaths are preserved.
assertRoutingEqual('default', '/foo/bar/hey', prefix + '/erp5/boo/far/faz' + vhr + '/_vh_foo/bar/hey')
# Rule precedence: later less-specific rules are applied.
assertRoutingEqual('default', '/foo', prefix + '/erp5/somewhere' + vhr + '/_vh_foo')
assertRoutingEqual('default', '/foo/', prefix + '/erp5/somewhere' + vhr + '/_vh_foo/')
assertRoutingEqual('default', '/foo/baz', prefix + '/erp5/somewhere' + vhr + '/_vh_foo/baz')
# Rule precedence: later more-specific rules are meaningless.
assertRoutingEqual('default', '/foo/shadowed', prefix + '/erp5/somewhere' + vhr + '/_vh_foo/shadowed')
# Rule precedence: family rules applied before general rules.
assertRoutingEqual('default', '/next', prefix + '/erp5/web_site_module/another_next_website' + vhr + '/_vh_next')
# Fallback on general rules when no family-specific rule matches
# Note: the root is special in that there is aways a trailing slash in the
# produced URL.
assertRoutingEqual('default', '/', prefix + '/erp5/web_site_module/123' + vhr + '/')
# Rule-less family reach general rules.
assertRoutingEqual('second', '/foo/bar', prefix + '/erp5/web_site_module/123' + vhr + '/foo/bar') # Rules match whole-elements, so the rule order does not matter to
# elements which share a common prefix.
assertRoutingEqual('second', '/next', prefix + '/erp5/web_site_module/the_next_website' + vhr + '/_vh_next')
assertRoutingEqual('second', '/next2', prefix + '/erp5/web_site_module/the_next2_website' + vhr + '/_vh_next2')
...@@ -3,6 +3,7 @@ ...@@ -3,6 +3,7 @@
extends = extends =
buildout.hash.cfg buildout.hash.cfg
../../stack/slapos.cfg ../../stack/slapos.cfg
../../stack/nodejs.cfg
../../stack/monitor/buildout.cfg ../../stack/monitor/buildout.cfg
../../component/ruby/buildout.cfg ../../component/ruby/buildout.cfg
../../component/golang/buildout.cfg ../../component/golang/buildout.cfg
...@@ -11,7 +12,6 @@ extends = ...@@ -11,7 +12,6 @@ extends =
../../component/cmake/buildout.cfg ../../component/cmake/buildout.cfg
../../component/icu/buildout.cfg ../../component/icu/buildout.cfg
../../component/pkgconfig/buildout.cfg ../../component/pkgconfig/buildout.cfg
../../component/nodejs/buildout.cfg
../../component/openssl/buildout.cfg ../../component/openssl/buildout.cfg
../../component/nginx/buildout.cfg ../../component/nginx/buildout.cfg
../../component/xz-utils/buildout.cfg ../../component/xz-utils/buildout.cfg
...@@ -67,12 +67,11 @@ parts = ...@@ -67,12 +67,11 @@ parts =
[slapos.cookbook-repository] [slapos.cookbook-repository]
revision = 571d6514f7290e8faa9439c4b86aa2f6c87df261 revision = 571d6514f7290e8faa9439c4b86aa2f6c87df261
[nodejs]
<= nodejs-8.12.0
[yarn] [yarn]
# need this version of Yarn <= yarn-1.3.2
recipe = slapos.recipe.build:download-unpacked
url = https://github.com/yarnpkg/yarn/releases/download/v1.3.2/yarn-v1.3.2.tar.gz
md5sum = db82fa09c996e9318f2f1d2ab99228f9
############################ ############################
# Software compilation # # Software compilation #
...@@ -122,7 +121,7 @@ bundle1.17.3 = ${buildout:parts-directory}/${:_buildout_section_name_}/lib/ruby/ ...@@ -122,7 +121,7 @@ bundle1.17.3 = ${buildout:parts-directory}/${:_buildout_section_name_}/lib/ruby/
# (python-4gitlab puts interpreter into ${buildout:bin-directory}) # (python-4gitlab puts interpreter into ${buildout:bin-directory})
environment = environment =
PATH = ${yarn:location}/bin:${:ruby-location}/bin:${cmake:location}/bin:${pkgconfig:location}/bin:${nodejs-8.12.0:location}/bin:${postgresql10:location}/bin:${redis28:location}/bin:${git:location}/bin:${buildout:bin-directory}:%(PATH)s PATH = ${yarn:location}/bin:${:ruby-location}/bin:${cmake:location}/bin:${pkgconfig:location}/bin:${nodejs:location}/bin:${postgresql10:location}/bin:${redis28:location}/bin:${git:location}/bin:${buildout:bin-directory}:%(PATH)s
# gitlab, gitlab-shell & gitlab-workhorse checked out as git repositories # gitlab, gitlab-shell & gitlab-workhorse checked out as git repositories
...@@ -210,7 +209,7 @@ make-binary = ...@@ -210,7 +209,7 @@ make-binary =
make-targets= cd ${:path} && npm install make-targets= cd ${:path} && npm install
environment = environment =
PATH=${nodejs-8.12.0:location}/bin/:%(PATH)s PATH=${nodejs:location}/bin/:%(PATH)s
#our go infrastructure not currently supporting submodules, IIRC #our go infrastructure not currently supporting submodules, IIRC
# https://lab.nexedi.com/nexedi/slapos/merge_requests/337 # https://lab.nexedi.com/nexedi/slapos/merge_requests/337
...@@ -269,7 +268,9 @@ configure-command = cd ${:path}/ruby && ...@@ -269,7 +268,9 @@ configure-command = cd ${:path}/ruby &&
${:bundle} config --local build.charlock_holmes --with-icu-dir=${icu:location} ${:bundle} config --local build.charlock_holmes --with-icu-dir=${icu:location}
make-binary = make-binary =
make-targets = make-targets =
. ${gowork:env.sh} && make . ${gowork:env.sh} &&
unset GOBIN &&
make
environment = environment =
PKG_CONFIG_PATH=${openssl-1.0:location}/lib/pkgconfig:${icu:location}/lib/pkgconfig PKG_CONFIG_PATH=${openssl-1.0:location}/lib/pkgconfig:${icu:location}/lib/pkgconfig
PATH=${pkgconfig:location}/bin:${ruby2.3:location}/bin:%(PATH)s PATH=${pkgconfig:location}/bin:${ruby2.3:location}/bin:%(PATH)s
......
...@@ -8,10 +8,10 @@ This is an experimental integration, mainly to evaluate these solutions. ...@@ -8,10 +8,10 @@ This is an experimental integration, mainly to evaluate these solutions.
See https://github.com/influxdata/telegraf to learn about plugins. See https://github.com/influxdata/telegraf to learn about plugins.
Useful plugins in this context are probably Useful plugins in this context are probably
[exec](https://github.com/influxdata/telegraf/tree/1.11.1/plugins/inputs/exec), [exec](https://github.com/influxdata/telegraf/tree/v1.17.3/plugins/inputs/exec),
[logparser](https://github.com/influxdata/telegraf/tree/1.11.1/plugins/inputs/logparser) [logparser](https://github.com/influxdata/telegraf/tree/v1.17.3/plugins/inputs/logparser)
or or
[http](https://github.com/influxdata/telegraf/tree/1.11.1/plugins/inputs/http). [http](https://github.com/influxdata/telegraf/tree/v1.17.3/plugins/inputs/http).
Telegraf will save in the `telegraf` database from the embedded influxdb server. Telegraf will save in the `telegraf` database from the embedded influxdb server.
......
...@@ -15,7 +15,7 @@ ...@@ -15,7 +15,7 @@
[instance-profile] [instance-profile]
filename = instance.cfg.in filename = instance.cfg.in
md5sum = 13f4f3806522b265dd1912ff169d146d md5sum = 3ccdd2299e759488545b62368c7a0b91
[influxdb-config-file] [influxdb-config-file]
filename = influxdb-config-file.cfg.in filename = influxdb-config-file.cfg.in
...@@ -35,8 +35,8 @@ md5sum = 3aa0f1ed752b2a59ea2b5e7c1733daf3 ...@@ -35,8 +35,8 @@ md5sum = 3aa0f1ed752b2a59ea2b5e7c1733daf3
[loki-config-file] [loki-config-file]
filename = loki-config-file.cfg.in filename = loki-config-file.cfg.in
md5sum = 02ba5acf23fcf88f5594919f46838533 md5sum = ad2baf4599a937d7352034a41fa24814
[promtail-config-file] [promtail-config-file]
filename = promtail-config-file.cfg.in filename = promtail-config-file.cfg.in
md5sum = c77788d0a3cc654ad9393eb4b1f31e94 md5sum = c8c9d815dd7b427788c066f041f04573
...@@ -20,22 +20,22 @@ revision = 1f7c19e5f5 ...@@ -20,22 +20,22 @@ revision = 1f7c19e5f5
<= go-git-package <= go-git-package
go.importpath = github.com/grafana/grafana go.importpath = github.com/grafana/grafana
repository = https://github.com/grafana/grafana repository = https://github.com/grafana/grafana
revision = v6.3.0-0-g830da0fda0 revision = v7.4.3-0-g010f20c1c8
[go_github.com_grafana_loki] [go_github.com_grafana_loki]
<= go-git-package <= go-git-package
go.importpath = github.com/grafana/loki go.importpath = github.com/grafana/loki
repository = https://github.com/grafana/loki repository = https://github.com/grafana/loki
revision = v0.3.0-0-gc673380bb3 revision = v2.1.0-188-g877f524c36
[go_github.com_influxdata_influxdb] [go_github.com_influxdata_influxdb]
<= go-git-package <= go-git-package
go.importpath = github.com/influxdata/influxdb go.importpath = github.com/influxdata/influxdb
repository = https://github.com/influxdata/influxdb repository = https://github.com/influxdata/influxdb
revision = v1.7.8rc1-0-ga7afa2ecf1 revision = v1.8.4-0-gbc8ec4384e
[go_github.com_influxdata_telegraf] [go_github.com_influxdata_telegraf]
<= go-git-package <= go-git-package
go.importpath = github.com/influxdata/telegraf go.importpath = github.com/influxdata/telegraf
repository = https://github.com/influxdata/telegraf repository = https://github.com/influxdata/telegraf
revision = 1.11.1-0-gf91de60d37 revision = v1.17.3-0-g24a552b90b
...@@ -132,7 +132,7 @@ command-line = ...@@ -132,7 +132,7 @@ command-line =
bash -c "{{ influx_bin }} \ bash -c "{{ influx_bin }} \
-username ${influxdb:auth-username} \ -username ${influxdb:auth-username} \
-password ${influxdb:auth-password} \ -password ${influxdb:auth-password} \
-host ${influxdb:host} \ -host [${influxdb:host}] \
-port ${influxdb:http-port} \ -port ${influxdb:http-port} \
-unsafeSsl \ -unsafeSsl \
-ssl \ -ssl \
...@@ -219,6 +219,7 @@ storage-boltdb-dir = ${directory:loki-storage-boltdb-dir} ...@@ -219,6 +219,7 @@ storage-boltdb-dir = ${directory:loki-storage-boltdb-dir}
storage-filesystem-dir = ${directory:loki-storage-filesystem-dir} storage-filesystem-dir = ${directory:loki-storage-filesystem-dir}
ip = ${instance-parameter:ipv4-random} ip = ${instance-parameter:ipv4-random}
port = 3100 port = 3100
grpc-port = 9095
url = http://${:ip}:${:port} url = http://${:ip}:${:port}
...@@ -238,9 +239,10 @@ command-line = ...@@ -238,9 +239,10 @@ command-line =
wrapper-path = ${directory:service}/promtail wrapper-path = ${directory:service}/promtail
dir = ${directory:promtail-dir} dir = ${directory:promtail-dir}
http_port = 19080 http-port = 19080
grpc-port = 19095
ip = ${instance-parameter:ipv4-random} ip = ${instance-parameter:ipv4-random}
url = http://${:ip}:${:http_port} url = http://${:ip}:${:http-port}
[promtail-config-file] [promtail-config-file]
<= config-file <= config-file
...@@ -252,7 +254,7 @@ context = ...@@ -252,7 +254,7 @@ context =
[promtail-listen-promise] [promtail-listen-promise]
<= check-port-listening-promise <= check-port-listening-promise
hostname= ${promtail:ip} hostname= ${promtail:ip}
port = ${promtail:http_port} port = ${promtail:http-port}
......
auth_enabled: false auth_enabled: false
server: server:
http_listen_address: {{ loki['ip'] }}
http_listen_port: {{ loki['port'] }} http_listen_port: {{ loki['port'] }}
grpc_listen_address: {{ loki['ip'] }}
grpc_listen_port: {{ loki['grpc-port'] }}
ingester: ingester:
lifecycler: lifecycler:
......
# https://github.com/grafana/loki/blob/master/docs/logentry/processing-log-lines.md
server: server:
http_listen_port: {{ promtail['http_port'] }} http_listen_address: {{ promtail['ip'] }}
# XXX this external_url does not work ... promtail still listen on all IPs http_listen_port: {{ promtail['http-port'] }}
grpc_listen_address: {{ promtail['ip'] }}
grpc_listen_port: {{ promtail['grpc-port'] }}
external_url: {{ promtail['url'] }} external_url: {{ promtail['url'] }}
grpc_listen_port: 0
positions: positions:
filename: {{ promtail['dir'] }}/positions.yaml filename: {{ promtail['dir'] }}/positions.yaml
......
...@@ -22,17 +22,14 @@ parts = ...@@ -22,17 +22,14 @@ parts =
loki-config-file loki-config-file
promtail-config-file promtail-config-file
[python]
part = python3
[nodejs] [nodejs]
<= nodejs-10.6.0 <= nodejs-14.16.0
[yarn] [yarn]
# this could become a component, but it needs to be invoked from nodejs explicitly, <= yarn-1.22.10
# otherwise it uses system's nodejs
recipe = slapos.recipe.build:download-unpacked
url = https://github.com/yarnpkg/yarn/releases/download/v1.16.0/yarn-v1.16.0.tar.gz
md5sum = 46790033c23803387890f545e4040690
[gowork] [gowork]
# All the softwares installed in the go workspace have "non standard" installation # All the softwares installed in the go workspace have "non standard" installation
...@@ -49,7 +46,7 @@ loki-bin = ${:bin}/loki ...@@ -49,7 +46,7 @@ loki-bin = ${:bin}/loki
promtail-bin = ${:bin}/promtail promtail-bin = ${:bin}/promtail
# use recent go # use recent go
golang = ${golang1.12:location} golang = ${golang1.16:location}
[gowork.goinstall] [gowork.goinstall]
command = : command = :
...@@ -65,8 +62,7 @@ depends = ...@@ -65,8 +62,7 @@ depends =
command = bash -c ". ${gowork:env.sh} && \ command = bash -c ". ${gowork:env.sh} && \
go install -v github.com/golang/dep/cmd/dep && \ go install -v github.com/golang/dep/cmd/dep && \
cd ${gowork:directory}/src/github.com/influxdata/influxdb && \ cd ${gowork:directory}/src/github.com/influxdata/influxdb && \
dep ensure && \ go install ./..."
go install ./cmd/..."
update-command = update-command =
[telegraf-install] [telegraf-install]
...@@ -80,14 +76,17 @@ update-command = ...@@ -80,14 +76,17 @@ update-command =
[grafana-install] [grafana-install]
<= gowork.goinstall <= gowork.goinstall
# yarn and go run build.go needs our nodejs in $PATH command = bash -c ". ${gowork:env.sh} && \
command = bash -c "export PATH=${nodejs:location}/bin/:$PATH && \
. ${gowork:env.sh} && \
cd ${gowork:directory}/src/github.com/grafana/grafana && \ cd ${gowork:directory}/src/github.com/grafana/grafana && \
go run build.go setup && \ go run build.go setup && \
go run build.go build && \ go run build.go build && \
${yarn:location}/bin/yarn install --pure-lockfile && \ ${yarn:location}/bin/yarn install --pure-lockfile && \
${nodejs:location}/bin/npm run build" ${yarn:location}/bin/yarn run build && \
${yarn:location}/bin/yarn run plugins:build-bundled && \
# Cleanup yarn and Cypress caches
rm -rf ${buildout:directory}/.cache/Cypress/ && \
rm -rf ${buildout:directory}/.cache/yarn/
"
update-command = update-command =
[loki-install] [loki-install]
......
...@@ -25,12 +25,13 @@ ...@@ -25,12 +25,13 @@
# #
############################################################################## ##############################################################################
import os
import textwrap
import logging import logging
import os
import tempfile import tempfile
import textwrap
import time import time
import psutil
import requests import requests
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
...@@ -44,11 +45,11 @@ setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass( ...@@ -44,11 +45,11 @@ setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
class GrafanaTestCase(SlapOSInstanceTestCase): class GrafanaTestCase(SlapOSInstanceTestCase):
"""Base test case for grafana. """Base test case for grafana.
Since the instances takes timte to start and stop, Since the instances takes time to start and stop,
we increate as lot the number of retries. we increase the number of retries.
""" """
instance_max_retry = 50
report_max_retry = 30 report_max_retry = 30
instance_max_retry = 30
class TestGrafana(GrafanaTestCase): class TestGrafana(GrafanaTestCase):
...@@ -144,8 +145,14 @@ class TestLoki(GrafanaTestCase): ...@@ -144,8 +145,14 @@ class TestLoki(GrafanaTestCase):
r''' r'''
- job_name: {cls.__name__} - job_name: {cls.__name__}
pipeline_stages: pipeline_stages:
- match:
selector: '{{job="{cls.__name__}"}}'
stages:
- multiline:
firstline: '^\d{{4}}-\d{{2}}-\d{{2}}\s\d{{1,2}}\:\d{{2}}\:\d{{2}}\,\d{{3}}'
max_wait_time: 3s
- regex: - regex:
expression: "^(?P<timestamp>.*) - (?P<name>\\S+) - (?P<level>\\S+) - (?P<message>.*)" expression: '^(?P<timestamp>.*) - (?P<name>\S+) - (?P<level>\S+) - (?P<message>.*)'
- timestamp: - timestamp:
format: 2006-01-02T15:04:05Z00:00 format: 2006-01-02T15:04:05Z00:00
source: timestamp source: timestamp
...@@ -180,6 +187,7 @@ class TestLoki(GrafanaTestCase): ...@@ -180,6 +187,7 @@ class TestLoki(GrafanaTestCase):
# create a logger logging to the file that we have # create a logger logging to the file that we have
# configured in instance parameter. # configured in instance parameter.
test_logger = logging.getLogger(self.id()) test_logger = logging.getLogger(self.id())
test_logger.propagate = False
test_logger.setLevel(logging.INFO) test_logger.setLevel(logging.INFO)
test_handler = logging.FileHandler(filename=self._logfile.name) test_handler = logging.FileHandler(filename=self._logfile.name)
test_handler.setFormatter( test_handler.setFormatter(
...@@ -189,16 +197,25 @@ class TestLoki(GrafanaTestCase): ...@@ -189,16 +197,25 @@ class TestLoki(GrafanaTestCase):
test_logger.info("testing message") test_logger.info("testing message")
test_logger.info("testing another message") test_logger.info("testing another message")
test_logger.warning("testing warn") test_logger.warning("testing warn")
# log an exception, which will be multi line in log file.
def nested1():
def nested2():
raise ValueError('boom')
nested2()
try:
nested1()
except ValueError:
test_logger.exception("testing exception")
# Check our messages have been ingested # Check our messages have been ingested
# we retry a few times, because there's a short delay until messages are # we retry a few times, because there's a short delay until messages are
# ingested and returned. # ingested and returned.
for i in range(10): for i in range(60):
resp = requests.get( resp = requests.get(
'{self.loki_url}/api/prom/query?query={{job="TestLoki"}}'.format( '{self.loki_url}/api/prom/query?query={{job="TestLoki"}}'.format(
**locals()), **locals()),
verify=False).json() verify=False).json()
if not resp: if len(resp.get('streams', [])) < 3:
time.sleep(0.5 * i) time.sleep(0.5 * i)
continue continue
...@@ -220,9 +237,83 @@ class TestLoki(GrafanaTestCase): ...@@ -220,9 +237,83 @@ class TestLoki(GrafanaTestCase):
line for line in info_stream['entries'] line for line in info_stream['entries']
if "testing another message" in line['line'] if "testing another message" in line['line']
]) ])
error_stream_list = [stream for stream in resp['streams'] if 'level="ERROR"' in stream['labels']]
self.assertEqual(1, len(error_stream_list), resp['streams'])
error_stream, = error_stream_list
line, = [line['line'] for line in error_stream['entries']]
# this entry is multi-line
self.assertIn('testing exception\nTraceback (most recent call last):\n', line)
self.assertIn('ValueError: boom', line)
# The labels we have configued are also available # The labels we have configued are also available
resp = requests.get( resp = requests.get(
'{self.loki_url}/api/prom/label'.format(**locals()), '{self.loki_url}/api/prom/label'.format(**locals()),
verify=False).json() verify=False).json()
self.assertIn('level', resp['values']) self.assertIn('level', resp['values'])
self.assertIn('name', resp['values']) self.assertIn('name', resp['values'])
class TestListenInPartition(GrafanaTestCase):
def setUp(self):
with self.slap.instance_supervisor_rpc as supervisor:
all_process_info = supervisor.getAllProcessInfo()
self.process_dict = {
p['name'].replace('-on-watch', ''): psutil.Process(p['pid'])
for p in all_process_info if p['name'] != 'watchdog'
}
def test_grafana_listen(self):
self.assertEqual(
[
c.laddr for c in self.process_dict['grafana'].connections()
if c.status == 'LISTEN'
],
[(self._ipv6_address, 8180)],
)
def test_influxdb_listen(self):
self.assertEqual(
sorted([
c.laddr for c in self.process_dict['influxdb'].connections()
if c.status == 'LISTEN'
]),
[
(self._ipv4_address, 8088),
(self._ipv6_address, 8086),
],
)
def test_telegraph_listen(self):
self.assertEqual(
[
c.laddr for c in self.process_dict['telegraf'].connections()
if c.status == 'LISTEN'
],
[],
)
def test_loki_listen(self):
self.assertEqual(
sorted([
c.laddr for c in self.process_dict['loki'].connections()
if c.status == 'LISTEN'
]),
[
(self._ipv4_address, 3100),
(self._ipv4_address, 9095),
],
)
def test_promtail_listen(self):
self.assertEqual(
sorted([
c.laddr for c in self.process_dict['promtail'].connections()
if c.status == 'LISTEN'
]),
[
(self._ipv4_address, 19080),
(self._ipv4_address, 19095),
],
)
...@@ -19,7 +19,7 @@ md5sum = e6d5c7bb627b4f1d3e7c99721b7c58fe ...@@ -19,7 +19,7 @@ md5sum = e6d5c7bb627b4f1d3e7c99721b7c58fe
[template-kvm] [template-kvm]
filename = instance-kvm.cfg.jinja2 filename = instance-kvm.cfg.jinja2
md5sum = 23493c541efef97ac5fe435114910b8e md5sum = 0c00ca332d364a2c620cd260d51d4649
[template-kvm-cluster] [template-kvm-cluster]
filename = instance-kvm-cluster.cfg.jinja2.in filename = instance-kvm-cluster.cfg.jinja2.in
...@@ -55,7 +55,7 @@ md5sum = b7e87479a289f472b634a046b44b5257 ...@@ -55,7 +55,7 @@ md5sum = b7e87479a289f472b634a046b44b5257
[template-kvm-run] [template-kvm-run]
filename = template/template-kvm-run.in filename = template/template-kvm-run.in
md5sum = cc1b748c82ca15022744558dab05995c md5sum = d1748b18fc76544cc71be5c40ed9cbb9
[template-kvm-controller] [template-kvm-controller]
filename = template/kvm-controller-run.in filename = template/kvm-controller-run.in
......
...@@ -1036,10 +1036,16 @@ keyboard-layout-language = fr ...@@ -1036,10 +1036,16 @@ keyboard-layout-language = fr
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
template = inline: template = inline:
{%- raw %} {%- raw %}
[{"disk": "{{disk_device_path}}"}] {%- set disk_list = [] %}
{%- for disk in disk_device_path.split() %}
{%- do disk_list.append({'disk': disk}) %}
{%- endfor -%}
{{ json_module.dumps(disk_list) }}
{% endraw -%} {% endraw -%}
rendered = ${buildout:directory}/.slapos-disk-permission rendered = ${buildout:directory}/.slapos-disk-permission
extensions = jinja2.ext.do
context = context =
import json_module json
raw disk_device_path {{disk_device_path}} raw disk_device_path {{disk_device_path}}
{% do part_list.append('wipe-disk-device-wrapper') -%} {% do part_list.append('wipe-disk-device-wrapper') -%}
......
...@@ -83,12 +83,23 @@ disk_aio = disk_aio if disk_aio in ["threads", "native"] else "threads" ...@@ -83,12 +83,23 @@ disk_aio = disk_aio if disk_aio in ["threads", "native"] else "threads"
# If a device (ie.: /dev/sdb) is provided, use it instead # If a device (ie.: /dev/sdb) is provided, use it instead
# the disk_path with disk_format # the disk_path with disk_format
disk_device_path = '{{ parameter_dict.get("disk-device-path", "") }}' disk_info_list = []
if disk_device_path.startswith("/dev/"): for disk_device_path in '{{ parameter_dict.get("disk-device-path", "") }}'.split():
disk_path = disk_device_path if disk_device_path.startswith("/dev/"):
disk_format = "raw" disk_info_list.append({
disk_aio = "native" 'path': disk_device_path,
disk_cache = "none" 'format': "raw",
'io': "native",
'cache': "none"
})
if len(disk_info_list) == 0:
disk_info_list.append({
'path': disk_path,
'format': disk_format,
'io': disk_aio,
'cache': disk_cache,
})
smp_count = {{ parameter_dict.get("smp-count") }} smp_count = {{ parameter_dict.get("smp-count") }}
smp_max_count = {{ parameter_dict.get("smp-max-count") }} smp_max_count = {{ parameter_dict.get("smp-max-count") }}
...@@ -167,12 +178,12 @@ def getMapStorageList(disk_storage_dict, external_disk_number): ...@@ -167,12 +178,12 @@ def getMapStorageList(disk_storage_dict, external_disk_number):
return id_list, external_disk_number return id_list, external_disk_number
# Download existing hard drive if needed at first boot # Download existing hard drive if needed at first boot
if not os.path.exists(disk_path) and virtual_hard_drive_url != '': if len(disk_info_list) == 1 and not os.path.exists(disk_info_list[0]['path']) and virtual_hard_drive_url != '':
print('Downloading virtual hard drive...') print('Downloading virtual hard drive...')
try: try:
downloaded_disk = disk_path downloaded_disk = disk_info_list[0]['path']
if virtual_hard_drive_gzipped == 'true': if virtual_hard_drive_gzipped == 'true':
downloaded_disk = '%s.gz' % disk_path downloaded_disk = '%s.gz' % disk_info_list[0]['path']
opener.retrieve(virtual_hard_drive_url, downloaded_disk) opener.retrieve(virtual_hard_drive_url, downloaded_disk)
except: except:
if os.path.exists(downloaded_disk): if os.path.exists(downloaded_disk):
...@@ -191,21 +202,21 @@ if not os.path.exists(disk_path) and virtual_hard_drive_url != '': ...@@ -191,21 +202,21 @@ if not os.path.exists(disk_path) and virtual_hard_drive_url != '':
print('Warning: not checksum specified.') print('Warning: not checksum specified.')
if downloaded_disk.endswith('.gz'): if downloaded_disk.endswith('.gz'):
try: try:
with open(disk_path, 'w') as disk: with open(disk_info_list[0]['path'], 'w') as disk:
with gzip.open(downloaded_disk, 'rb') as disk_gz: with gzip.open(downloaded_disk, 'rb') as disk_gz:
shutil.copyfileobj(disk_gz, disk) shutil.copyfileobj(disk_gz, disk)
except Exception: except Exception:
if os.path.exists(disk_path): if os.path.exists(disk_info_list[0]['path']):
os.remove(disk_path) os.remove(disk_info_list[0]['path'])
raise raise
os.remove(downloaded_disk) os.remove(downloaded_disk)
# Create disk if doesn't exist # Create disk if doesn't exist
# XXX: move to Buildout profile # XXX: move to Buildout profile
if not disk_device_path.startswith("/dev/") and not os.path.exists(disk_path): if len(disk_info_list) == 1 and not disk_info_list[0]['path'].startswith("/dev/") and not os.path.exists(disk_info_list[0]['path']):
print('Creating virtual hard drive...') print('Creating virtual hard drive...')
subprocess.check_call([qemu_img_path, 'create' ,'-f', disk_format, subprocess.check_call([qemu_img_path, 'create' ,'-f', disk_info_list[0]['format'],
disk_path, '%sG' % disk_size]) disk_info_list[0]['path'], '%sG' % disk_size])
print('Done.') print('Done.')
# Check and create external disk # Check and create external disk
...@@ -240,13 +251,6 @@ if disk_storage_dict: ...@@ -240,13 +251,6 @@ if disk_storage_dict:
print('Data folder %s was not used to create external disk %r' % (index +1)) print('Data folder %s was not used to create external disk %r' % (index +1))
index += 1 index += 1
additional_disk_options = ''
if disk_aio == 'native':
additional_disk_options += ',cache.direct=on'
if disk_format == "raw":
additional_disk_options += ',discard=on'
# Generate network parameters # Generate network parameters
# XXX: use_tap should be a boolean # XXX: use_tap should be a boolean
tap_network_parameter = [] tap_network_parameter = []
...@@ -303,8 +307,6 @@ else: ...@@ -303,8 +307,6 @@ else:
kvm_argument_list = [qemu_path, kvm_argument_list = [qemu_path,
'-enable-kvm', '-smp', smp, '-name', vm_name, '-m', ram, '-vga', 'std', '-enable-kvm', '-smp', smp, '-name', vm_name, '-m', ram, '-vga', 'std',
'-drive', 'file=%s,if=%s,cache=%s,aio=%s%s' % (disk_path, disk_type, disk_cache,
disk_aio, additional_disk_options),
'-vnc', '%s:1,ipv4,password' % listen_ip, '-vnc', '%s:1,ipv4,password' % listen_ip,
'-boot', 'order=cd,menu=on', '-boot', 'order=cd,menu=on',
'-qmp', 'unix:%s,server,nowait' % socket_path, '-qmp', 'unix:%s,server,nowait' % socket_path,
...@@ -312,6 +314,18 @@ kvm_argument_list = [qemu_path, ...@@ -312,6 +314,18 @@ kvm_argument_list = [qemu_path,
'-D', logfile, '-D', logfile,
'-nodefaults', '-nodefaults',
] ]
for disk_info in disk_info_list:
additional_disk_options = ''
if disk_info['io'] == 'native':
additional_disk_options += ',cache.direct=on'
if disk_info['format'] == "raw":
additional_disk_options += ',discard=on'
kvm_argument_list.extend([
'-drive',
'file=%s,if=%s,cache=%s,aio=%s%s' % ( disk_info['path'], disk_type, disk_info['cache'], disk_info['io'], additional_disk_options)
])
rgx = re.compile('^[\w*\,][\=\d+\-\,\w]*$') rgx = re.compile('^[\w*\,][\=\d+\-\,\w]*$')
for numa in numa_list: for numa in numa_list:
......
...@@ -14,7 +14,7 @@ ...@@ -14,7 +14,7 @@
# not need these here). # not need these here).
[instance-common] [instance-common]
filename = instance-common.cfg.in filename = instance-common.cfg.in
md5sum = 6da513940e5bf7d06b3fb0aeb39c8ad5 md5sum = 2bd1779425b7561682c0de5496d808ed
[root-common] [root-common]
filename = root-common.cfg.in filename = root-common.cfg.in
......
...@@ -23,6 +23,7 @@ context = ...@@ -23,6 +23,7 @@ context =
key ipv4_set slap-configuration:ipv4 key ipv4_set slap-configuration:ipv4
key ipv6_set slap-configuration:ipv6 key ipv6_set slap-configuration:ipv6
key slapparameter_dict slap-configuration:configuration key slapparameter_dict slap-configuration:configuration
key computer slap-configuration:computer
raw logrotate_cfg {{ template_logrotate_base }} raw logrotate_cfg {{ template_logrotate_base }}
raw template_monitor {{ template_monitor }} raw template_monitor {{ template_monitor }}
raw bin_directory {{ bin_directory }} raw bin_directory {{ bin_directory }}
......
[buildout]
extends = software.cfg
[neoppod]
eggs = neoppod[admin, ctl, master, storage-importer, storage-mysqldb, tests]
${python-mysqlclient:egg}
ZODB3
patch-binary = ${patch:location}/bin/patch
ZODB3-patch-options = -p1
ZODB3-patches =
${neoppod-repository:location}/ZODB3.patch
[versions]
ZODB3 = 3.10.7+SlapOSPatched001
transaction = 1.1.1
zdaemon = 2.0.7
...@@ -12,9 +12,6 @@ ZODB = 5.6.0 ...@@ -12,9 +12,6 @@ ZODB = 5.6.0
ZEO = 5.2.2 ZEO = 5.2.2
transaction = 2.4.0 transaction = 2.4.0
# BBB: ZEO
msgpack = 0.5.6
# Required by: # Required by:
# ZEO==5.2.0 # ZEO==5.2.0
# trollius==2.2.post1 # trollius==2.2.post1
......
# Code generated by gowork-snapshot; DO NOT EDIT.
# list of go git repositories to fetch
[gowork.goinstall]
depends_gitfetch =
${go_crawshaw.io_sqlite:recipe}
${go_github.com_DataDog_czlib:recipe}
${go_github.com_cznic_strutil:recipe}
${go_github.com_davecgh_go-spew:recipe}
${go_github.com_fsnotify_fsnotify:recipe}
${go_github.com_golang_glog:recipe}
${go_github.com_gwenn_gosqlite:recipe}
${go_github.com_gwenn_yacr:recipe}
${go_github.com_kisielk_og-rek:recipe}
${go_github.com_kylelemons_godebug:recipe}
${go_github.com_philhofer_fwd:recipe}
${go_github.com_pkg_errors:recipe}
${go_github.com_pkg_profile:recipe}
${go_github.com_pmezard_go-difflib:recipe}
${go_github.com_shamaton_msgpack:recipe}
${go_github.com_soheilhy_cmux:recipe}
${go_github.com_someonegg_gocontainer:recipe}
${go_github.com_someonegg_gox:recipe}
${go_github.com_stretchr_testify:recipe}
${go_github.com_tinylib_msgp:recipe}
${go_github.com_ttacon_chalk:recipe}
${go_golang.org_x_crypto:recipe}
${go_golang.org_x_mod:recipe}
${go_golang.org_x_net:recipe}
${go_golang.org_x_perf:recipe}
${go_golang.org_x_sync:recipe}
${go_golang.org_x_sys:recipe}
${go_golang.org_x_text:recipe}
${go_golang.org_x_tools:recipe}
${go_golang.org_x_xerrors:recipe}
${go_gopkg.in_yaml.v3:recipe}
${go_lab.nexedi.com_kirr_go123:recipe}
${go_lab.nexedi.com_kirr_neo:recipe}
[go_crawshaw.io_sqlite]
<= go-git-package
go.importpath = crawshaw.io/sqlite
repository = https://github.com/crawshaw/sqlite
revision = v0.3.2-7-gc582b9de4f
[go_github.com_DataDog_czlib]
<= go-git-package
go.importpath = github.com/DataDog/czlib
repository = https://github.com/DataDog/czlib
revision = 4bc9a24e37
[go_github.com_cznic_strutil]
<= go-git-package
go.importpath = github.com/cznic/strutil
repository = https://github.com/cznic/strutil
revision = 529a34b1c1
[go_github.com_davecgh_go-spew]
<= go-git-package
go.importpath = github.com/davecgh/go-spew
repository = https://github.com/davecgh/go-spew
revision = v1.1.1-1-gd8f796af33
[go_github.com_fsnotify_fsnotify]
<= go-git-package
go.importpath = github.com/fsnotify/fsnotify
repository = https://github.com/fsnotify/fsnotify
revision = 7f4cf4dd2b
[go_github.com_golang_glog]
<= go-git-package
go.importpath = github.com/golang/glog
repository = https://github.com/golang/glog
revision = 23def4e6c1
[go_github.com_gwenn_gosqlite]
<= go-git-package
go.importpath = github.com/gwenn/gosqlite
repository = https://github.com/gwenn/gosqlite
revision = 24878be1a2
[go_github.com_gwenn_yacr]
<= go-git-package
go.importpath = github.com/gwenn/yacr
repository = https://github.com/gwenn/yacr
revision = bbe82c1f4d
[go_github.com_kisielk_og-rek]
<= go-git-package
go.importpath = github.com/kisielk/og-rek
repository = https://github.com/kisielk/og-rek
revision = 24bb08c22e
[go_github.com_kylelemons_godebug]
<= go-git-package
go.importpath = github.com/kylelemons/godebug
repository = https://github.com/kylelemons/godebug
revision = fa7b53cdfc
[go_github.com_philhofer_fwd]
<= go-git-package
go.importpath = github.com/philhofer/fwd
repository = https://github.com/philhofer/fwd
revision = 414ae1bb9e
[go_github.com_pkg_errors]
<= go-git-package
go.importpath = github.com/pkg/errors
repository = https://github.com/pkg/errors
revision = v0.8.1-31-g614d223910
[go_github.com_pkg_profile]
<= go-git-package
go.importpath = github.com/pkg/profile
repository = https://github.com/pkg/profile
revision = v1.5.0-1-g3704c8d233
[go_github.com_pmezard_go-difflib]
<= go-git-package
go.importpath = github.com/pmezard/go-difflib
repository = https://github.com/pmezard/go-difflib
revision = v1.0.0-4-g5d4384ee4f
[go_github.com_shamaton_msgpack]
<= go-git-package
go.importpath = github.com/shamaton/msgpack
repository = https://github.com/shamaton/msgpack
revision = 29a4ba0bb9
[go_github.com_soheilhy_cmux]
<= go-git-package
go.importpath = github.com/soheilhy/cmux
repository = https://github.com/soheilhy/cmux
revision = 8a8ea3c539
[go_github.com_someonegg_gocontainer]
<= go-git-package
go.importpath = github.com/someonegg/gocontainer
repository = https://github.com/someonegg/gocontainer
revision = 24fb283ab9
[go_github.com_someonegg_gox]
<= go-git-package
go.importpath = github.com/someonegg/gox
repository = https://github.com/someonegg/gox
revision = c9bfdf8eb4
[go_github.com_stretchr_testify]
<= go-git-package
go.importpath = github.com/stretchr/testify
repository = https://github.com/stretchr/testify
revision = v1.2.2-210-g54d05a4e18
[go_github.com_tinylib_msgp]
<= go-git-package
go.importpath = github.com/tinylib/msgp
repository = https://github.com/tinylib/msgp
revision = 87c1ec45d5
[go_github.com_ttacon_chalk]
<= go-git-package
go.importpath = github.com/ttacon/chalk
repository = https://github.com/ttacon/chalk
revision = v0.1-3-g22c06c80ed
[go_golang.org_x_crypto]
<= go-git-package
go.importpath = golang.org/x/crypto
repository = https://go.googlesource.com/crypto
revision = 5c72a88397
[go_golang.org_x_mod]
<= go-git-package
go.importpath = golang.org/x/mod
repository = https://go.googlesource.com/mod
revision = ce943fd024
[go_golang.org_x_net]
<= go-git-package
go.importpath = golang.org/x/net
repository = https://go.googlesource.com/net
revision = 62affa334b
[go_golang.org_x_perf]
<= go-git-package
go.importpath = golang.org/x/perf
repository = https://go.googlesource.com/perf
revision = d949658356
[go_golang.org_x_sync]
<= go-git-package
go.importpath = golang.org/x/sync
repository = https://go.googlesource.com/sync
revision = 6e8e738ad2
[go_golang.org_x_sys]
<= go-git-package
go.importpath = golang.org/x/sys
repository = https://go.googlesource.com/sys
revision = aee5d888a8
[go_golang.org_x_text]
<= go-git-package
go.importpath = golang.org/x/text
repository = https://go.googlesource.com/text
revision = v0.3.3-2-ga8b4671254
[go_golang.org_x_tools]
<= go-git-package
go.importpath = golang.org/x/tools
repository = https://go.googlesource.com/tools
revision = ba800b16d8
[go_golang.org_x_xerrors]
<= go-git-package
go.importpath = golang.org/x/xerrors
repository = https://go.googlesource.com/xerrors
revision = 5ec99f83af
[go_gopkg.in_yaml.v3]
<= go-git-package
go.importpath = gopkg.in/yaml.v3
repository = https://gopkg.in/yaml.v3
revision = v2.1.1-97-geeeca48fe7
[go_lab.nexedi.com_kirr_go123]
<= go-git-package
go.importpath = lab.nexedi.com/kirr/go123
repository = https://lab.nexedi.com/kirr/go123.git
revision = c8d9907ef7
[go_lab.nexedi.com_kirr_neo]
<= go-git-package
go.importpath = lab.nexedi.com/kirr/neo
repository = https://lab.nexedi.com/kirr/neo.git
revision = v1.12-2791-gf3effa6c53
...@@ -2,12 +2,7 @@ ...@@ -2,12 +2,7 @@
[buildout] [buildout]
extends = software.cfg extends = software.cfg
[neo]
[go_lab.nexedi.com_kirr_go123]
branch = master
revision=
[go_lab.nexedi.com_kirr_neo]
branch = t branch = t
revision= revision=
......
...@@ -8,7 +8,6 @@ extends = ...@@ -8,7 +8,6 @@ extends =
../../stack/slapos.cfg ../../stack/slapos.cfg
../neoppod/software-common.cfg ../neoppod/software-common.cfg
../../component/golang/buildout.cfg ../../component/golang/buildout.cfg
gowork.cfg
../../component/sqlite3/buildout.cfg ../../component/sqlite3/buildout.cfg
../../component/zlib/buildout.cfg ../../component/zlib/buildout.cfg
../../component/wendelin.core/buildout.cfg ../../component/wendelin.core/buildout.cfg
...@@ -49,12 +48,23 @@ parts = ...@@ -49,12 +48,23 @@ parts =
instance.cfg instance.cfg
# go packages to install (+ automatically their dependencies) [neo]
<= go-git-package
go.importpath = lab.nexedi.com/kirr/neo
repository = https://lab.nexedi.com/kirr/neo.git
revision = v1.12-3107-gf278d611
[goperf]
<= go-git-package
go.importpath = golang.org/x/perf
repository = https://lab.nexedi.com/kirr/go.perf.git
revision = e8bebf9314da
# go programs to install
[gowork] [gowork]
install = install =
lab.nexedi.com/kirr/neo/go/... ${neo:location}/go:./...
github.com/pkg/profile ${goperf:location}:./cmd/benchstat
golang.org/x/perf/cmd/benchstat
cpkgpath = cpkgpath =
${sqlite3:location}/lib/pkgconfig ${sqlite3:location}/lib/pkgconfig
...@@ -62,7 +72,7 @@ cpkgpath = ...@@ -62,7 +72,7 @@ cpkgpath =
# dev-install neo from go checkout # dev-install neo from go checkout
[neoppod-develop] [neoppod-develop]
setup = ${go_lab.nexedi.com_kirr_neo:location} setup = ${neo:location}
# neotest-env.sh + neotest wrapper # neotest-env.sh + neotest wrapper
...@@ -99,7 +109,7 @@ template = inline: ...@@ -99,7 +109,7 @@ template = inline:
[tnxdtest] [tnxdtest]
recipe = plone.recipe.command recipe = plone.recipe.command
stop-on-error = yes stop-on-error = yes
command = ln -s -t $${directory:t} ${gowork:src}/lab.nexedi.com/kirr/neo/go/neo/t/.nxdtest command = ln -s -t $${directory:t} ${neo:location}/go/neo/t/.nxdtest
...@@ -119,10 +129,6 @@ eggs = ...@@ -119,10 +129,6 @@ eggs =
${wendelin.core:egg} ${wendelin.core:egg}
# wendelin.core: latest not yet released
[wendelin.core-repository]
revision= v0.11-15-gf785ac079b
# ping eggs versions # ping eggs versions
[versions] [versions]
ZODB3 = 3.11.0 ZODB3 = 3.11.0
......
...@@ -43,9 +43,6 @@ setup( ...@@ -43,9 +43,6 @@ setup(
install_requires=[ install_requires=[
'slapos.core', 'slapos.core',
'slapos.libnetworkcache', 'slapos.libnetworkcache',
'erp5.util',
'supervisor',
'pexpect',
'requests', 'requests',
], ],
zip_safe=True, zip_safe=True,
......
...@@ -24,16 +24,10 @@ ...@@ -24,16 +24,10 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# #
############################################################################## ##############################################################################
from __future__ import unicode_literals
import os import os
import textwrap from six.moves.urllib.parse import urljoin
import logging
import tempfile
import time
from six.moves.urllib.parse import urlparse, urljoin
import pexpect
import requests import requests
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
...@@ -44,17 +38,46 @@ setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass( ...@@ -44,17 +38,46 @@ setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
class TestRepman(SlapOSInstanceTestCase): class TestRepman(SlapOSInstanceTestCase):
__partition_reference__ = 'R' # solve path too long for postgresql and unicorn __partition_reference__ = 'R'
@classmethod
def getInstanceSoftwareType(cls):
return 'default'
def setUp(self): def setUp(self):
self.backend_url = self.computer_partition.getConnectionParameterDict( self.url = self.computer_partition.getConnectionParameterDict()['url']
)['backend-url']
def test_http_get(self): def test_http_get(self):
resp = requests.get(self.backend_url, verify=False) connection_parameter_dict = \
self.assertTrue( self.computer_partition.getConnectionParameterDict()
resp.status_code in [requests.codes.ok, requests.codes.found]) resp = requests.get(self.url, verify=False)
self.assertEqual(resp.status_code, requests.codes.ok)
resp = requests.post(
urljoin(self.url, '/api/login'),
json={
'username': connection_parameter_dict['username'],
'password': connection_parameter_dict['repman-password'],
},
verify=False,
)
self.assertEqual(resp.status_code, requests.codes.ok)
token = resp.json()['token']
headers = {"authorization": "Bearer " + token}
resp = requests.get(
urljoin(self.url, '/api/monitor'),
headers=headers,
verify=False,
)
self.assertEqual(resp.status_code, requests.codes.ok)
resp = requests.get(
urljoin(self.url, '/api/clusters'),
params={
'query': '{"method":"GET","isArray":false}',
},
headers=headers,
verify=False,
)
self.assertEqual(resp.status_code, requests.codes.ok)
cluster, = resp.json()
self.assertTrue(cluster['isProvision'])
self.assertTrue(cluster['isFailable'])
self.assertFalse(cluster['isDown'])
...@@ -14,7 +14,7 @@ ...@@ -14,7 +14,7 @@
# not need these here). # not need these here).
[template-erp5] [template-erp5]
filename = instance-erp5.cfg.in filename = instance-erp5.cfg.in
md5sum = 4d0839e359c98ba3cd516903b72f798b md5sum = 3d2683f8b0031f4f5690c6b9388c0566
[template-balancer] [template-balancer]
filename = instance-balancer.cfg.in filename = instance-balancer.cfg.in
......
...@@ -177,9 +177,9 @@ return = ...@@ -177,9 +177,9 @@ return =
{% endif %} {% endif %}
{% set bt5_default_list = 'erp5_full_text_mroonga_catalog slapos_configurator' -%} {% set bt5_default_list = 'erp5_full_text_mroonga_catalog slapos_configurator' -%}
{% if has_jupyter -%} {% if has_jupyter -%}
{% set bt5_default_list = bt5_default_list + ' erp5_data_notebook' -%} {% do bt5_default_list.append('erp5_data_notebook') -%}
{% endif -%} {% endif -%}
config-bt5 = {{ dumps(slapparameter_dict.get('bt5', bt5_default_list)) }} config-bt5 = {{ dumps(slapparameter_dict.get('bt5', ' '.join(bt5_default_list))) }}
config-bt5-repository-url = {{ dumps(slapparameter_dict.get('bt5-repository-url', local_bt5_repository)) }} config-bt5-repository-url = {{ dumps(slapparameter_dict.get('bt5-repository-url', local_bt5_repository)) }}
config-cloudooo-url = {{ dumps(slapparameter_dict.get('cloudooo-url', default_cloudooo_url)) }} config-cloudooo-url = {{ dumps(slapparameter_dict.get('cloudooo-url', default_cloudooo_url)) }}
config-caucase-url = {{ dumps(caucase_url) }} config-caucase-url = {{ dumps(caucase_url) }}
...@@ -187,6 +187,7 @@ config-deadlock-debugger-password = ${publish-early:deadlock-debugger-password} ...@@ -187,6 +187,7 @@ config-deadlock-debugger-password = ${publish-early:deadlock-debugger-password}
config-developer-list = {{ dumps(slapparameter_dict.get('developer-list', [inituser_login])) }} config-developer-list = {{ dumps(slapparameter_dict.get('developer-list', [inituser_login])) }}
config-saucelabs-dict = {{ dumps(slapparameter_dict.get('saucelabs-dict', {})) }} config-saucelabs-dict = {{ dumps(slapparameter_dict.get('saucelabs-dict', {})) }}
config-hosts-dict = {{ dumps(slapparameter_dict.get('hosts-dict', {})) }} config-hosts-dict = {{ dumps(slapparameter_dict.get('hosts-dict', {})) }}
config-computer-hosts-dict = {{ dumps(slapparameter_dict.get('computer-hosts-dict', {})) }}
config-hostalias-dict = {{ dumps(slapparameter_dict.get('hostalias-dict', {})) }} config-hostalias-dict = {{ dumps(slapparameter_dict.get('hostalias-dict', {})) }}
config-id-store-interval = {{ dumps(slapparameter_dict.get('id-store-interval')) }} config-id-store-interval = {{ dumps(slapparameter_dict.get('id-store-interval')) }}
config-zope-longrequest-logger-error-threshold = {{ dumps(monitor_dict.get('zope-longrequest-logger-error-threshold', 20)) }} config-zope-longrequest-logger-error-threshold = {{ dumps(monitor_dict.get('zope-longrequest-logger-error-threshold', 20)) }}
......
...@@ -15,6 +15,7 @@ extends = ...@@ -15,6 +15,7 @@ extends =
../../stack/slapos.cfg ../../stack/slapos.cfg
../../stack/nxdtest.cfg ../../stack/nxdtest.cfg
../../stack/caucase/buildout.cfg
./buildout.hash.cfg ./buildout.hash.cfg
...@@ -314,11 +315,6 @@ selenium = 3.141.0 ...@@ -314,11 +315,6 @@ selenium = 3.141.0
# Patched eggs # Patched eggs
PyPDF2 = 1.26.0+SlapOSPatched001 PyPDF2 = 1.26.0+SlapOSPatched001
# caucase and its dependencies
caucase = 0.9.4
pem = 18.2.0
PyJWT = 1.6.4
# Django 1.11 is python 2 compatible # Django 1.11 is python 2 compatible
Django = 1.11 Django = 1.11
......
[buildout] [buildout]
extends = extends =
../../component/nodejs/buildout.cfg
../../component/caddy/buildout.cfg ../../component/caddy/buildout.cfg
../../component/git/buildout.cfg ../../component/git/buildout.cfg
../../component/bash/buildout.cfg ../../component/bash/buildout.cfg
...@@ -13,6 +12,7 @@ extends = ...@@ -13,6 +12,7 @@ extends =
../../component/coreutils/buildout.cfg ../../component/coreutils/buildout.cfg
../../component/java-jdk/buildout.cfg ../../component/java-jdk/buildout.cfg
../../component/fonts/buildout.cfg ../../component/fonts/buildout.cfg
../../stack/nodejs.cfg
../../stack/slapos.cfg ../../stack/slapos.cfg
../../stack/monitor/buildout.cfg ../../stack/monitor/buildout.cfg
../../component/defaults.cfg ../../component/defaults.cfg
...@@ -37,13 +37,7 @@ max_version = 0 ...@@ -37,13 +37,7 @@ max_version = 0
<= nodejs-12.18.3 <= nodejs-12.18.3
[yarn] [yarn]
# this could become a component, but it needs to be invoked from nodejs explicitly, <= yarn-1.17.3
# otherwise it uses system's nodejs
# XXX why don't we build a wrapper ?
version = 1.17.3
recipe = slapos.recipe.build:download-unpacked
url = https://github.com/yarnpkg/yarn/releases/download/v${:version}/yarn-v${:version}.tar.gz
md5sum = 4a02e1687a150113ad6b0215f9afdb3e
[slapos-standalone] [slapos-standalone]
recipe = zc.recipe.egg recipe = zc.recipe.egg
......
...@@ -36,6 +36,6 @@ mode = 0644 ...@@ -36,6 +36,6 @@ mode = 0644
depends = ${caucase-jinja2-library-eggs:eggs} depends = ${caucase-jinja2-library-eggs:eggs}
[versions] [versions]
caucase = 0.9.8 caucase = 0.9.9
pem = 20.1.0 pem = 21.1.0
PyJWT = 1.7.1 PyJWT = 1.7.1
...@@ -34,7 +34,7 @@ md5sum = e91c0fbd0df441884f7422fa7976053c ...@@ -34,7 +34,7 @@ md5sum = e91c0fbd0df441884f7422fa7976053c
[template-zope-conf] [template-zope-conf]
filename = zope.conf.in filename = zope.conf.in
md5sum = c43da8f7b4db22e40a4864e6cfcaef44 md5sum = 5ce9d5e99880d5db9ed339bbefe28d34
[site-zcml] [site-zcml]
filename = site.zcml filename = site.zcml
...@@ -74,7 +74,7 @@ md5sum = b5ac16fdeed8863e465e955ba6d1e12a ...@@ -74,7 +74,7 @@ md5sum = b5ac16fdeed8863e465e955ba6d1e12a
[template-erp5] [template-erp5]
filename = instance-erp5.cfg.in filename = instance-erp5.cfg.in
md5sum = 548d99118afa736e5a7c428b0c8ed560 md5sum = ac155f28a096747fc267f32a1cec46e1
[template-zeo] [template-zeo]
filename = instance-zeo.cfg.in filename = instance-zeo.cfg.in
...@@ -82,15 +82,15 @@ md5sum = 0648e38bd5d3a15bb9f93264932740b9 ...@@ -82,15 +82,15 @@ md5sum = 0648e38bd5d3a15bb9f93264932740b9
[template-zope] [template-zope]
filename = instance-zope.cfg.in filename = instance-zope.cfg.in
md5sum = c03f93f95333e6a61b857dcfab7f9c0e md5sum = 9fa66b93fbf6a40aa8136c651ad9f539
[template-balancer] [template-balancer]
filename = instance-balancer.cfg.in filename = instance-balancer.cfg.in
md5sum = 8ad9137310ae0403d433bb3c0d93be9f md5sum = 8d3694226b6cbed961f6d608b6d6d294
[template-haproxy-cfg] [template-haproxy-cfg]
filename = haproxy.cfg.in filename = haproxy.cfg.in
md5sum = 8de18a61607bd66341a44b95640d293f md5sum = 452c502fabd5a6066c9dee533dfb1c77
[template-rsyslogd-cfg] [template-rsyslogd-cfg]
filename = rsyslogd.cfg.in filename = rsyslogd.cfg.in
......
...@@ -147,6 +147,8 @@ defaults ...@@ -147,6 +147,8 @@ defaults
log {{ parameter_dict['log-socket'] }} local0 info log {{ parameter_dict['log-socket'] }} local0 info
{% set bind_ssl_crt = 'ssl crt ' ~ parameter_dict['cert'] ~ ' alpn h2,http/1.1' %} {% set bind_ssl_crt = 'ssl crt ' ~ parameter_dict['cert'] ~ ' alpn h2,http/1.1' %}
{% set family_path_routing_dict = parameter_dict['family-path-routing-dict'] %}
{% set path_routing_list = parameter_dict['path-routing-list'] %}
{% for name, (port, _, certificate_authentication, backend_list) in sorted(parameter_dict['backend-dict'].iteritems()) -%} {% for name, (port, _, certificate_authentication, backend_list) in sorted(parameter_dict['backend-dict'].iteritems()) -%}
listen family_{{ name }} listen family_{{ name }}
...@@ -172,6 +174,11 @@ listen family_{{ name }} ...@@ -172,6 +174,11 @@ listen family_{{ name }}
capture request header User-Agent len 512 capture request header User-Agent len 512
log-format "%{+Q}o %{-Q}ci - - [%trg] %r %ST %B %{+Q}[capture.req.hdr(0)] %{+Q}[capture.req.hdr(1)] %Tt" log-format "%{+Q}o %{-Q}ci - - [%trg] %r %ST %B %{+Q}[capture.req.hdr(0)] %{+Q}[capture.req.hdr(1)] %Tt"
{% for outer_prefix, inner_prefix in family_path_routing_dict.get(name, []) + path_routing_list %}
{% set outer_prefix = outer_prefix.strip('/') -%}
http-request replace-path ^(/+VirtualHostBase/+[^/]+/+[^/]+)/+VirtualHostRoot/+({% if outer_prefix %}{{ outer_prefix }}($|/.*){% else %}.*{% endif %}) \1/{{ inner_prefix.strip('/') }}/VirtualHostRoot/{% if outer_prefix %}_vh_{% endif %}\2
{% endfor %}
{% set has_webdav = [] -%} {% set has_webdav = [] -%}
{% for address, connection_count, webdav in backend_list -%} {% for address, connection_count, webdav in backend_list -%}
{% if webdav %}{% do has_webdav.append(None) %}{% endif -%} {% if webdav %}{% do has_webdav.append(None) %}{% endif -%}
......
...@@ -190,6 +190,8 @@ ca-cert = ${haproxy-conf-ssl:ca-cert} ...@@ -190,6 +190,8 @@ ca-cert = ${haproxy-conf-ssl:ca-cert}
crl = ${haproxy-conf-ssl:crl} crl = ${haproxy-conf-ssl:crl}
{% endif %} {% endif %}
stats-socket = ${directory:run}/haproxy.sock stats-socket = ${directory:run}/haproxy.sock
path-routing-list = {{ dumps(slapparameter_dict['path-routing-list']) }}
family-path-routing-dict = {{ dumps(slapparameter_dict['family-path-routing-dict']) }}
pidfile = ${directory:run}/haproxy.pid pidfile = ${directory:run}/haproxy.pid
log-socket = ${rsyslogd-cfg-parameter-dict:log-socket} log-socket = ${rsyslogd-cfg-parameter-dict:log-socket}
server-check-path = {{ dumps(slapparameter_dict['haproxy-server-check-path']) }} server-check-path = {{ dumps(slapparameter_dict['haproxy-server-check-path']) }}
......
...@@ -24,6 +24,7 @@ ...@@ -24,6 +24,7 @@
{% set use_ipv6 = slapparameter_dict.get('use-ipv6', False) -%} {% set use_ipv6 = slapparameter_dict.get('use-ipv6', False) -%}
{% set partition_thread_count_list = [] -%} {% set partition_thread_count_list = [] -%}
{% set zope_partition_dict = slapparameter_dict.get('zope-partition-dict', {'1': {}}) -%} {% set zope_partition_dict = slapparameter_dict.get('zope-partition-dict', {'1': {}}) -%}
{% set zope_family_override_dict = slapparameter_dict.get('family-override', {}) -%}
{% for zope_parameter_dict in zope_partition_dict.values() -%} {% for zope_parameter_dict in zope_partition_dict.values() -%}
{# Apply some zope_parameter_dict default values, to avoid duplication. -#} {# Apply some zope_parameter_dict default values, to avoid duplication. -#}
{% do zope_parameter_dict.setdefault('thread-amount', 4) -%} {% do zope_parameter_dict.setdefault('thread-amount', 4) -%}
...@@ -34,7 +35,8 @@ ...@@ -34,7 +35,8 @@
<= request-common-base <= request-common-base
config-use-ipv6 = {{ dumps(slapparameter_dict.get('use-ipv6', False)) }} config-use-ipv6 = {{ dumps(slapparameter_dict.get('use-ipv6', False)) }}
config-computer-memory-percent-threshold = {{ dumps(monitor_dict.get('computer-memory-percent-threshold', 80)) }} config-computer-memory-percent-threshold = {{ dumps(monitor_dict.get('computer-memory-percent-threshold', 80)) }}
{% set caucase_url = slapparameter_dict.get('caucase', {}).pop('url', '') -%} {% set caucase_dict = slapparameter_dict.get('caucase', {}) -%}
{% set caucase_url = caucase_dict.get('url', '') -%}
{% macro request(name, software_type, config_key, config, ret={'url': True}, key_config={}) -%} {% macro request(name, software_type, config_key, config, ret={'url': True}, key_config={}) -%}
{% do config.update(slapparameter_dict.get(config_key, {})) -%} {% do config.update(slapparameter_dict.get(config_key, {})) -%}
...@@ -69,8 +71,6 @@ service-on-watch = ${buildout:directory}/etc/service ...@@ -69,8 +71,6 @@ service-on-watch = ${buildout:directory}/etc/service
srv = ${buildout:directory}/srv srv = ${buildout:directory}/srv
backup-caucased = ${:srv}/backup/caucased backup-caucased = ${:srv}/backup/caucased
{% set caucase_dict = slapparameter_dict.get('caucase', {}) -%}
{% set caucase_url = caucase_dict.get('url') -%}
{% if not caucase_url -%} {% if not caucase_url -%}
{% if use_ipv6 -%} {% if use_ipv6 -%}
{% set caucase_host = '[' ~ (ipv6_set | list)[0] ~ ']' %} {% set caucase_host = '[' ~ (ipv6_set | list)[0] ~ ']' %}
...@@ -95,8 +95,35 @@ backup-caucased = ${:srv}/backup/caucased ...@@ -95,8 +95,35 @@ backup-caucased = ${:srv}/backup/caucased
{% do root_common.section('caucased-promise') -%} {% do root_common.section('caucased-promise') -%}
{% endif -%} {% endif -%}
{% do publish_dict.__setitem__('caucase-http-url', caucase_url) -%} {% do publish_dict.__setitem__('caucase-http-url', caucase_url) -%}
{% set balancer_dict = slapparameter_dict.get('balancer', {}) -%} {% set balancer_dict = slapparameter_dict.setdefault('balancer', {}) -%}
{% do balancer_dict.setdefault('ssl', {}).setdefault('caucase-url', caucase_url) -%} {% do balancer_dict.setdefault('ssl', {}).setdefault('caucase-url', caucase_url) -%}
{% do balancer_dict.setdefault('tcpv4-port', 2150) -%}
{% do balancer_dict.__setitem__('haproxy-server-check-path', balancer_dict.get('haproxy-server-check-path', '/') % {'site-id': site_id}) -%}
{% set routing_path_template_field_dict = {"site-id": site_id} -%}
{% macro expandRoutingPath(output, input) -%}
{% for outer_prefix, inner_prefix in input -%}
{% do output.append((outer_prefix, inner_prefix % routing_path_template_field_dict)) -%}
{% endfor -%}
{% endmacro -%}
{% set path_routing_list = [] -%}
{% do expandRoutingPath(
path_routing_list,
balancer_dict.get(
'path-routing-list',
(('/', '/'), ),
),
) -%}
{% do balancer_dict.__setitem__('path-routing-list', path_routing_list) -%}
{% set family_path_routing_dict = {} -%}
{% for name, family_path_routing_list in balancer_dict.get(
'family-path-routing-dict',
{},
).items() -%}
{% set path_routing_list = [] -%}
{% do expandRoutingPath(path_routing_list, family_path_routing_list) -%}
{% do family_path_routing_dict.__setitem__(name, path_routing_list) -%}
{% endfor -%}
{% do balancer_dict.__setitem__('family-path-routing-dict', family_path_routing_dict) -%}
{{ request('memcached-persistent', 'kumofs', 'kumofs', {'tcpv4-port': 2000}, {'url': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }} {{ request('memcached-persistent', 'kumofs', 'kumofs', {'tcpv4-port': 2000}, {'url': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
{{ request('memcached-volatile', 'kumofs', 'memcached', {'tcpv4-port': 2010, 'ram-storage-size': 64}, {'url': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }} {{ request('memcached-volatile', 'kumofs', 'memcached', {'tcpv4-port': 2010, 'ram-storage-size': 64}, {'url': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
...@@ -193,6 +220,7 @@ config-deadlock-debugger-password = ${publish-early:deadlock-debugger-password} ...@@ -193,6 +220,7 @@ config-deadlock-debugger-password = ${publish-early:deadlock-debugger-password}
config-developer-list = {{ dumps(slapparameter_dict.get('developer-list', [inituser_login])) }} config-developer-list = {{ dumps(slapparameter_dict.get('developer-list', [inituser_login])) }}
config-selenium-server-configuration-dict = {{ dumps(slapparameter_dict.get('selenium-server-configuration-dict', {})) }} config-selenium-server-configuration-dict = {{ dumps(slapparameter_dict.get('selenium-server-configuration-dict', {})) }}
config-hosts-dict = {{ dumps(slapparameter_dict.get('hosts-dict', {})) }} config-hosts-dict = {{ dumps(slapparameter_dict.get('hosts-dict', {})) }}
config-computer-hosts-dict = {{ dumps(slapparameter_dict.get('computer-hosts-dict', {})) }}
config-hostalias-dict = {{ dumps(slapparameter_dict.get('hostalias-dict', {})) }} config-hostalias-dict = {{ dumps(slapparameter_dict.get('hostalias-dict', {})) }}
config-id-store-interval = {{ dumps(slapparameter_dict.get('id-store-interval')) }} config-id-store-interval = {{ dumps(slapparameter_dict.get('id-store-interval')) }}
config-zope-longrequest-logger-error-threshold = {{ dumps(monitor_dict.get('zope-longrequest-logger-error-threshold', 20)) }} config-zope-longrequest-logger-error-threshold = {{ dumps(monitor_dict.get('zope-longrequest-logger-error-threshold', 20)) }}
...@@ -226,6 +254,8 @@ config-tidstorage-port = ${request-zodb:connection-tidstorage-port} ...@@ -226,6 +254,8 @@ config-tidstorage-port = ${request-zodb:connection-tidstorage-port}
{% endfor -%} {% endfor -%}
software-type = zope software-type = zope
{% set global_publisher_timeout = slapparameter_dict.get('publisher-timeout') -%}
{% set global_activity_timeout = slapparameter_dict.get('activity-timeout') -%}
{% set zope_family_dict = {} -%} {% set zope_family_dict = {} -%}
{% set zope_family_name_list = [] -%} {% set zope_family_name_list = [] -%}
{% set zope_backend_path_dict = {} -%} {% set zope_backend_path_dict = {} -%}
...@@ -246,6 +276,7 @@ software-type = zope ...@@ -246,6 +276,7 @@ software-type = zope
{% do zope_family_dict.setdefault(zope_family, []).append(section_name) -%} {% do zope_family_dict.setdefault(zope_family, []).append(section_name) -%}
{% do zope_backend_path_dict.__setitem__(zope_family, backend_path) -%} {% do zope_backend_path_dict.__setitem__(zope_family, backend_path) -%}
{% do ssl_authentication_dict.__setitem__(zope_family, zope_parameter_dict.get('ssl-authentication', False)) -%} {% do ssl_authentication_dict.__setitem__(zope_family, zope_parameter_dict.get('ssl-authentication', False)) -%}
{% set current_zope_family_override_dict = zope_family_override_dict.get(zope_family, {}) -%}
[{{ section_name }}] [{{ section_name }}]
<= request-zope-base <= request-zope-base
name = {{ partition_name }} name = {{ partition_name }}
...@@ -260,7 +291,10 @@ config-longrequest-logger-interval = {{ dumps(zope_parameter_dict.get('longreque ...@@ -260,7 +291,10 @@ config-longrequest-logger-interval = {{ dumps(zope_parameter_dict.get('longreque
config-longrequest-logger-timeout = {{ dumps(zope_parameter_dict.get('longrequest-logger-timeout', 1)) }} config-longrequest-logger-timeout = {{ dumps(zope_parameter_dict.get('longrequest-logger-timeout', 1)) }}
config-large-file-threshold = {{ dumps(zope_parameter_dict.get('large-file-threshold', "10MB")) }} config-large-file-threshold = {{ dumps(zope_parameter_dict.get('large-file-threshold', "10MB")) }}
config-port-base = {{ dumps(zope_parameter_dict.get('port-base', 2200)) }} config-port-base = {{ dumps(zope_parameter_dict.get('port-base', 2200)) }}
config-webdav = {{ dumps(zope_parameter_dict.get('webdav', False)) }} {# BBB: zope_parameter_dict used to contain 'webdav', so fallback to it -#}
config-webdav = {{ dumps(current_zope_family_override_dict.get('webdav', zope_parameter_dict.get('webdav', False))) }}
config-publisher-timeout = {{ dumps(current_zope_family_override_dict.get('publisher-timeout', global_publisher_timeout)) }}
config-activity-timeout = {{ dumps(current_zope_family_override_dict.get('activity-timeout', global_activity_timeout)) }}
{% if test_runner_enabled -%} {% if test_runner_enabled -%}
config-test-runner-apache-url-list = ${publish-early:{{ zope_family }}-test-runner-url-list} config-test-runner-apache-url-list = ${publish-early:{{ zope_family }}-test-runner-url-list}
...@@ -319,40 +353,39 @@ config-url = ${request-jupyter:connection-url} ...@@ -319,40 +353,39 @@ config-url = ${request-jupyter:connection-url}
{% endif -%} {% endif -%}
{%- endif %} {%- endif %}
[request-balancer] {% set balancer_ret_dict = {'monitor-base-url': False} -%}
<= request-common {% for family in zope_family_dict -%}
name = balancer {% do balancer_ret_dict.__setitem__(family, False) -%}
software-type = balancer {% do balancer_ret_dict.__setitem__(family + '-v6', False) -%}
{{ root_common.sla('balancer') }} {% if test_runner_enabled -%}
return = {% do balancer_ret_dict.__setitem__(family + '-test-runner-url-list', False) -%}
monitor-base-url {% endif -%}
{%- for family in zope_family_dict %}
{{ family }}
{{ family }}-v6
{% if test_runner_enabled %}
{{ family }}-test-runner-url-list
{% endif %}
{% endfor -%} {% endfor -%}
{% do monitor_base_url_dict.__setitem__('request-balancer', '${' ~ 'request-balancer' ~ ':connection-monitor-base-url}') -%} {% set balancer_key_config_dict = {
config-zope-family-dict = {{ dumps(zope_family_parameter_dict) }} 'monitor-passwd': 'monitor-htpasswd:passwd',
config-tcpv4-port = {{ dumps(balancer_dict.get('tcpv4-port', 2150)) }} } -%}
{% for zope_section_id, name in zope_address_list_id_dict.items() -%} {% for zope_section_id, name in zope_address_list_id_dict.items() -%}
config-{{ name }} = {{ ' ${' ~ zope_section_id ~ ':connection-zope-address-list}' }} {% do balancer_key_config_dict.__setitem__(
name,
zope_section_id + ':connection-zope-address-list',
) -%}
{% if test_runner_enabled -%} {% if test_runner_enabled -%}
config-{{ name }}-test-runner-address-list = {{ ' ${' ~ zope_section_id ~ ':connection-test-runner-address-list}' }} {% do balancer_key_config_dict.__setitem__(
name + '-test-runner-address-list',
zope_section_id + ':connection-test-runner-address-list',
) -%}
{% endif -%} {% endif -%}
{% endfor -%} {% endfor -%}
# XXX: should those really be same for all families ? {{ request(
config-haproxy-server-check-path = {{ dumps(balancer_dict.get('haproxy-server-check-path', '/') % {'site-id': site_id}) }} name='balancer',
config-monitor-passwd = ${monitor-htpasswd:passwd} software_type='balancer',
config-ssl = {{ dumps(balancer_dict['ssl']) }} config_key='balancer',
config-name = ${:name} config={
config-backend-path-dict = {{ dumps(zope_backend_path_dict) }} 'zope-family-dict': zope_family_parameter_dict,
config-ssl-authentication-dict = {{ dumps(ssl_authentication_dict) }} 'backend-path-dict': zope_backend_path_dict,
config-apachedex-promise-threshold = {{ dumps(monitor_dict.get('apachedex-promise-threshold', 70)) }} 'ssl-authentication-dict': ssl_authentication_dict,
config-apachedex-configuration = {{ 'apachedex-promise-threshold': monitor_dict.get('apachedex-promise-threshold', 70),
dumps( 'apachedex-configuration': monitor_dict.get(
monitor_dict.get(
'apachedex-configuration', 'apachedex-configuration',
[ [
'--logformat', '%h %l %u %t "%r" %>s %O "%{Referer}i" "%{User-Agent}i" %{ms}T', '--logformat', '%h %l %u %t "%r" %>s %O "%{Referer}i" "%{User-Agent}i" %{ms}T',
...@@ -361,7 +394,13 @@ config-apachedex-configuration = {{ ...@@ -361,7 +394,13 @@ config-apachedex-configuration = {{
'--skip-user-agent', 'Zabbix', '--skip-user-agent', 'Zabbix',
'--error-detail', '--error-detail',
'--js-embed', '--js-embed',
'--quiet'])) }} '--quiet',
],
),
},
ret=balancer_ret_dict,
key_config=balancer_key_config_dict,
) }}
[request-frontend-base] [request-frontend-base]
{% if has_frontend -%} {% if has_frontend -%}
......
...@@ -22,7 +22,7 @@ partition. No more (undefined result), no less (IndexError). ...@@ -22,7 +22,7 @@ partition. No more (undefined result), no less (IndexError).
-#} -#}
{% set ipv4 = (ipv4_set | list)[0] -%} {% set ipv4 = (ipv4_set | list)[0] -%}
{% set hosts_dict = {} -%} {% set publishable_hosts_dict = {} -%}
{% set port_dict = {} -%} {% set port_dict = {} -%}
{% for alias, url in ( {% for alias, url in (
('erp5-memcached-volatile', slapparameter_dict['memcached-url']), ('erp5-memcached-volatile', slapparameter_dict['memcached-url']),
...@@ -32,15 +32,17 @@ partition. No more (undefined result), no less (IndexError). ...@@ -32,15 +32,17 @@ partition. No more (undefined result), no less (IndexError).
) -%} ) -%}
{% set parsed_url = urlparse.urlparse(url) -%} {% set parsed_url = urlparse.urlparse(url) -%}
{% do port_dict.__setitem__(alias, parsed_url.port) -%} {% do port_dict.__setitem__(alias, parsed_url.port) -%}
{% do hosts_dict.__setitem__(alias, parsed_url.hostname) -%} {% do publishable_hosts_dict.__setitem__(alias, parsed_url.hostname) -%}
{%- endfor %} {%- endfor %}
{% for i, url in enumerate(slapparameter_dict['mysql-url-list']) -%} {% for i, url in enumerate(slapparameter_dict['mysql-url-list']) -%}
{% do hosts_dict.__setitem__( {% do publishable_hosts_dict.__setitem__(
'erp5-catalog-' ~ i, 'erp5-catalog-' ~ i,
urlparse.urlparse(url).hostname, urlparse.urlparse(url).hostname,
) -%} ) -%}
{%- endfor %} {%- endfor %}
{% do hosts_dict.update(slapparameter_dict['hosts-dict']) -%} {% do publishable_hosts_dict.update(slapparameter_dict['hosts-dict']) -%}
{% set hosts_dict = publishable_hosts_dict.copy() -%}
{% do hosts_dict.update(slapparameter_dict['computer-hosts-dict'].get(computer, {})) -%}
[jinja2-template-base] [jinja2-template-base]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
...@@ -274,6 +276,8 @@ cloudooo-url = {{ (cloudooo if cloudooo.port == None else ...@@ -274,6 +276,8 @@ cloudooo-url = {{ (cloudooo if cloudooo.port == None else
{% endif -%} {% endif -%}
{% endfor -%} {% endfor -%}
developer-list = {{ dumps(slapparameter_dict['developer-list']) }} developer-list = {{ dumps(slapparameter_dict['developer-list']) }}
publisher-timeout = {{ dumps(slapparameter_dict['publisher-timeout']) }}
activity-timeout = {{ dumps(slapparameter_dict['activity-timeout']) }}
instance = ${directory:instance} instance = ${directory:instance}
instance-products = ${directory:instance-products} instance-products = ${directory:instance-products}
deadlock-path = /manage_debug_threads deadlock-path = /manage_debug_threads
...@@ -553,7 +557,7 @@ This has the downside of making each zope partition publish the (hopefuly) same ...@@ -553,7 +557,7 @@ This has the downside of making each zope partition publish the (hopefuly) same
dict toward erp5 partition, violating the DRY principle and making the intent dict toward erp5 partition, violating the DRY principle and making the intent
hard to guess. hard to guess.
-#} -#}
hosts-dict = {{ dumps(hosts_dict) }} hosts-dict = {{ dumps(publishable_hosts_dict) }}
monitor-base-url = ${monitor-publish-parameters:monitor-base-url} monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
test-runner-address-list = {{ dumps(test_runner_address_list) }} test-runner-address-list = {{ dumps(test_runner_address_list) }}
software-release-url = ${slap-connection:software-release-url} software-release-url = ${slap-connection:software-release-url}
......
...@@ -140,10 +140,14 @@ large-file-threshold {{ parameter_dict['large-file-threshold'] }} ...@@ -140,10 +140,14 @@ large-file-threshold {{ parameter_dict['large-file-threshold'] }}
</zodb_db> </zodb_db>
{% set developer_list = parameter_dict['developer-list'] -%} {% set developer_list = parameter_dict['developer-list'] -%}
{% if developer_list -%} {% set publisher_timeout = parameter_dict['publisher-timeout'] -%}
{% set activity_timeout = parameter_dict['activity-timeout'] -%}
{% if developer_list or publisher_timeout or activity_timeout -%}
%import Products.ERP5Type %import Products.ERP5Type
<ERP5Type erp5> <ERP5Type erp5>
developers {{ developer_list | join(' ') }} {% if developer_list %}developers {{ developer_list | join(' ') }}{% endif %}
{% if publisher_timeout %}publisher-timeout {{ publisher_timeout }}{% endif %}
{% if activity_timeout %}activity-timeout {{ activity_timeout }}{% endif %}
</ERP5Type> </ERP5Type>
{% endif -%} {% endif -%}
......
[buildout] [buildout]
extends = extends =
../component/nodejs/buildout.cfg ../component/nodejs/buildout.cfg
../component/yarn/buildout.cfg
../stack/slapos.cfg ../stack/slapos.cfg
parts = parts =
......
...@@ -200,7 +200,7 @@ setuptools-dso = 1.7 ...@@ -200,7 +200,7 @@ setuptools-dso = 1.7
rubygemsrecipe = 0.3.0 rubygemsrecipe = 0.3.0
six = 1.12.0 six = 1.12.0
slapos.cookbook = 1.0.171 slapos.cookbook = 1.0.171
slapos.core = 1.6.5 slapos.core = 1.6.6
slapos.extension.strip = 0.4 slapos.extension.strip = 0.4
slapos.extension.shared = 1.0 slapos.extension.shared = 1.0
slapos.libnetworkcache = 0.20 slapos.libnetworkcache = 0.20
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment