Commit f1d9dc31 authored by Łukasz Nowak's avatar Łukasz Nowak

Update Release Candidate

parents 747f918f 1cfb657f
Pipeline #14100 failed with stage
in 0 seconds
[buildout]
extends =
../../component/golang/buildout.cfg
gowork.cfg
parts =
gowork
caddy
[gowork.goinstall]
command = bash -c ". ${gowork:env.sh} && cd ${go_github.com_caddyserver_caddy:location} && GO111MODULE=on go install -v $(echo -n '${gowork:install}' |tr '\n' ' ')"
[caddy-get]
<= go-git-package
go.importpath = github.com/caddyserver/caddy
repository = https://lab.nexedi.com/nexedi/caddy.git
revision = nxd-v1.0.3-1-g2c11cedc
[gowork]
golang = ${golang1.14:location}
golang = ${golang1.16:location}
install =
github.com/caddyserver/caddy/...
${caddy-get:location}:./...
[caddy]
recipe = plone.recipe.command
......
# Code generated by gowork-snapshot; DO NOT EDIT.
# list of go git repositories to fetch
[gowork.goinstall]
depends_gitfetch =
${go_github.com_caddyserver_caddy:recipe}
[go_github.com_caddyserver_caddy]
<= go-git-package
go.importpath = github.com/caddyserver/caddy
repository = https://lab.nexedi.com/nexedi/caddy.git
revision = nxd-v1.0.3-1-g2c11cedc
......@@ -80,30 +80,39 @@ environment-extra =
# ---- infrastructure to build Go workspaces / projects ----
# gowork is a top-level section representing workspace
# gowork is the top-level section that defines Go workspace.
#
# users should add `install` field to [gowork] to describe packages they want to
# be installed (+ automatically their dependencies are installed too). e.g.
# It specifies global settings that are used to build Go programs:
#
# - Go toolchain to use, e.g.
#
# [gowork]
# install =
# lab.nexedi.com/kirr/neo/go/...
# github.com/pkg/profile
# golang.org/x/perf/cmd/benchstat
# golang = ${golang1.15:location}
#
# For Cgo support pkg-config is made pre-available by gowork, and users
# should list paths where to search for pkg-config files, e.g. this way:
# - For Cgo support pkg-config is made pre-available by gowork, and users
# should list paths where to search for pkg-config files, e.g. this way:
#
# [gowork]
# cpkgpath =
# ${sqlite3:location}/lib/pkgconfig
# ${zlib:location}/lib/pkgconfig
#
# It is also possible to specify Go build flags used for compilation e.g. this way:
# - It is also possible to specify Go build flags used for compilation e.g. this way:
#
# [gowork]
# buildflags = -race
#
#
# Users can also add `install` field to [gowork] to request Go programs to be
# automatically installed, for example:
#
# [gowork]
# install =
# lab.nexedi.com/kirr/neo/go/...@v0.0.0-20210103165133-f3effa6c535f
# golang.org/x/tools/gopls@v0.4.3
# ${helloweb:location}/go:./...
#
#
# ${go:exe} is standalone executable that runs go in activated gowork environment.
[go]
recipe = slapos.recipe.template:jinja2
......@@ -116,9 +125,9 @@ template= inline:
exec go "$@"
[gowork]
directory = ${buildout:directory}/go.work
src = ${:directory}/src
bin = ${:directory}/bin
directory = ${gowork.dir:directory}
src = ${gowork.dir:src}
bin = ${gowork.dir:bin}
depends = ${gowork.goinstall:recipe}
# go version used for the workspace (possible to override in applications)
......@@ -130,6 +139,9 @@ buildflags =
# empty pkg-config path by default
cpkgpath =
# by default don't `go install` anything
install =
# everything is done by dependent parts
recipe = plone.recipe.command
command = :
......@@ -140,24 +152,52 @@ env.sh = ${gowork-env.sh:output}
[gowork-env.sh]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/goenv.sh.in
output = ${gowork:directory}/env.sh
depends = ${gowork.mkdir:recipe}
output = ${gowork.dir:directory}/env.sh
depends = ${gowork.dir:recipe}
md5sum = 6efdf5d63381c240c7d206d7939a63f7
[gowork.mkdir]
[gowork.dir]
directory = ${buildout:directory}/go.work
src = ${:directory}/src
bin = ${:directory}/bin
# NOTE do not use slapos.cookbook:mkdirectory here - if anything in software (not instance)
# uses slapos.cookbook:* in recipe - slapos.cookbook will get compiled against system
# libxml/libxslt or fail to bootstrap at all if those are not present.
recipe = plone.recipe.command
command = mkdir -p ${gowork:directory}
command = mkdir -p ${:directory}
update-command = ${:command}
stop-on-error = true
# install go packages
# clients should put package list to install to gowork:install ("..." requests installing everything)
# install go programs
# clients can put program list to install to gowork:install
[gowork.goinstall]
recipe = plone.recipe.command
command = bash -c ". ${gowork:env.sh} && go install ${gowork:buildflags} -v $(echo -n '${gowork:install}' |tr '\n' ' ')"
command = bash -c ". ${gowork:env.sh} &&
for x in $(echo -n '${gowork:install}' |tr '\n' ' '); do
case "\$x" in
# external module, e.g. golang.org/x/tools/gopls@v0.4.3
*@*)
echo GOMOD \$x
GO111MODULE=on go install ${gowork:buildflags} -v \$x
;;
# locally-cloned module source, e.g. <module-src-location>:./...
*:*)
echo GOMODSRC \$x
dir=\"\$${x%%:*}\"
arg=\"\$${x#*:}\"
(cd \$dir && GO111MODULE=on go install ${gowork:buildflags} -v \$arg)
;;
# non-module
*)
echo GOPKG \$x
GO111MODULE=off go install ${gowork:buildflags} -v \$x
;;
esac
done
"
update-command = ${:command}
stop-on-error = true
......@@ -175,4 +215,4 @@ git-executable = ${git:location}/bin/git
# generated with the help of gowork-snapshot tool.
[go-git-package]
<= git-repository
location = ${gowork:src}/${:go.importpath}
location = ${gowork.dir:src}/${:go.importpath}
......@@ -4,7 +4,6 @@ extends =
../git/buildout.cfg
../ruby/buildout.cfg
../golang/buildout.cfg
gowork.cfg
parts =
helloweb-python
......@@ -21,9 +20,15 @@ parts =
# cloning+building manually. However to be able to use third-party Go packages
# we need gowork support, and other languages can use helloweb repository from
# under gowork as well.
[helloweb]
<= go-git-package
go.importpath = lab.nexedi.com/nexedi/helloweb
repository = https://lab.nexedi.com/nexedi/helloweb.git
revision = a7c788ae71
[gowork]
install =
lab.nexedi.com/nexedi/helloweb/go/...
${helloweb:location}/go:./...
golang = ${golang1.16:location}
......@@ -44,7 +49,7 @@ input = inline:
[helloweb-egg]
recipe = zc.recipe.egg:develop
egg = helloweb
setup = ${gowork:src}/lab.nexedi.com/nexedi/helloweb/python/
setup = ${helloweb:location}/python/
[helloweb-python]
recipe = zc.recipe.egg:scripts
......@@ -86,7 +91,7 @@ environment =
[helloweb-ruby-bundle]
recipe = slapos.recipe.cmmi
path = ${gowork:src}/lab.nexedi.com/nexedi/helloweb/ruby/
path = ${helloweb:location}/ruby/
configure-command = :
make-binary =
......
# Code generated by gowork-snapshot; DO NOT EDIT.
# list of go git repositories to fetch
[gowork.goinstall]
depends_gitfetch =
${go_lab.nexedi.com_nexedi_helloweb:recipe}
[go_lab.nexedi.com_nexedi_helloweb]
<= go-git-package
go.importpath = lab.nexedi.com/nexedi/helloweb
repository = https://lab.nexedi.com/nexedi/helloweb.git
revision = 8bfedac656
......@@ -14,6 +14,12 @@ parts =
[nodejs]
<= nodejs-8.9.4
[nodejs-14.16.0]
<= nodejs-base
openssl_location = ${openssl:location}
version = v14.16.0
md5sum = 7dc3666f407bf4e12a01ce1be2883d31
[nodejs-12.18.3]
<= nodejs-base
openssl_location = ${openssl:location}
......
......@@ -6,7 +6,6 @@ parts =
gowork
[gowork]
golang = ${golang1.13:location}
install =
buildflags = -v --tags server --ldflags "-extldflags 'static' -w -s -X main.GoOS=linux -X main.GoArch=amd64 -X main.Version=2.1 -X main.FullVersion=$FULLVERSION -X main.Build=$(date +%FT%T%z) -X main.WithProvisioning=ON -X main.WithOpenSVC=OFF -X main.WithHaproxy=ON -X main.WithMaxscale=ON -X main.WithMariadbshardproxy=ON -X main.WithProxysql=ON -X main.WithSphinx=ON -X main.WithArbitration=OFF -X main.WithArbitrationClient=ON -X main.WithMonitoring=ON -X main.WithHttp=ON -X main.WithBackup=ON -X main.WithMail=ON -X main.WithEnforce=ON -X main.WithDeprecate=ON"
......
......@@ -5,9 +5,6 @@ extends =
parts =
restic
[gowork]
golang = ${golang1.13:location}
[restic]
recipe = plone.recipe.command
......
......@@ -3,7 +3,10 @@
[buildout]
extends =
../pygolang/buildout.cfg
../zodbtools/buildout.cfg
../numpy/buildout.cfg
../golang/buildout.cfg
../zlib/buildout.cfg
../git/buildout.cfg
parts =
......@@ -21,12 +24,18 @@ environment = wendelin.core-env
# dependent eggs that must come through in-tree recipes
depends =
${zodbtools:egg}
${numpy:egg}
[wendelin.core-env]
# wendelin.core needs git(dev) to build
# wendelin.core needs git(dev) and go to build
PATH = ${git:location}/bin:%(PATH)s
GO = ${go:exe}
# wcfs needs this:
[gowork]
cpkgpath += ${zlib:location}/lib/pkgconfig
[wendelin.core-repository]
......
......@@ -32,11 +32,12 @@ eggs =
neoppod[tests]
ZEO[test]
# env.sh for that python to be on $PATH
# env.sh for that python + go to be on $PATH
[wendelin.core-env.sh]
recipe = slapos.recipe.template:jinja2
rendered = ${buildout:directory}/${:_buildout_section_name_}
template = inline:
. ${gowork:env.sh}
export PS1="(wendelin.core-env) $PS1"
export PATH=${buildout:bin-directory}:${gdb:location}/bin:$PATH
......
# https://classic.yarnpkg.com/ package manager for nodejs
[buildout]
extends =
../nodejs/buildout.cfg
[yarn]
<= yarn-1.22.10
[yarn-1.22.10]
<= yarn-wrapper
yarn-download = ${yarn-download-1.22.10:location}
[yarn-1.17.3]
<= yarn-wrapper
yarn-download = ${yarn-download-1.17.3:location}
[yarn-1.16.0]
<= yarn-wrapper
yarn-download = ${yarn-download-1.16.0:location}
[yarn-1.3.2]
<= yarn-wrapper
yarn-download = ${yarn-download-1.3.2:location}
[yarn-wrapper]
recipe = slapos.recipe.template:jinja2
rendered = ${:location}/bin/yarn
template = inline:
#!/bin/sh
PATH=${nodejs:location}/bin/:$PATH
exec ${:yarn-download}/bin/yarn $@
location = ${buildout:parts-directory}/${:_buildout_section_name_}
bin-yarn = ${:rendered}
[yarn-download]
recipe = slapos.recipe.build:download-unpacked
url = https://github.com/yarnpkg/yarn/releases/download/v${:version}/yarn-v${:version}.tar.gz
[yarn-download-1.22.10]
<= yarn-download
version = 1.22.10
md5sum = 52e8dbe9d0cb90683dd3ee2ebf2becb8
[yarn-download-1.17.3]
<= yarn-download
version = 1.17.3
md5sum = 4a02e1687a150113ad6b0215f9afdb3e
[yarn-download-1.3.2]
<= yarn-download
version = 1.3.2
md5sum = db82fa09c996e9318f2f1d2ab99228f9
[yarn-download-1.16.0]
<= yarn-download
version = 1.16.0
md5sum = 46790033c23803387890f545e4040690
......@@ -21,6 +21,7 @@ Here are listed the most important changes, which might affect upgrades.
* fix: use kedifa with with for file with multiple CAs
* feature: support query string (the characters after ? in the url) in url and https-url
* fix: by having unique acl names fix rare bug of directing traffic to https-url instead of url or otherwise
* feature: failover backend
1.0.164 (2020-09-24)
--------------------
......
......@@ -236,6 +236,8 @@ This set of parameters is used to control the way how the backend checks will be
Please be aware that the `health-check-timeout` is really short by default, so in case if `/` of the backend is slow to reply configure proper path with `health-check-http-path` to not mark such backend down too fast, before increasing the check timeout.
Thanks to using health-check it's possible to configure failover system. By providing `health-check-failover-url` or `health-check-failover-https-url` some special backend can be used to reply in case if original backend replies with error (codes like `5xx`). As a note one can setup this failover URL like `https://failover.example.com/?p=` so that the path from the incoming request will be passed as parameter. Additionally authentication to failover URL is supported with `health-check-authenticate-to-failover-backend` and SSL Proxy verification with `health-check-failover-ssl-proxy-verify` and `health-check-failover-ssl-proxy-ca-crt`.
Examples
========
......
......@@ -26,11 +26,11 @@ md5sum = a6a626fd1579fd1d4b80ea67433ca16a
[profile-caddy-replicate]
filename = instance-apache-replicate.cfg.in
md5sum = 1ab3fc07bb186601b54c584a3ccaf1c3
md5sum = 1248911409cbeea980a838b04ee451d2
[profile-slave-list]
_update_hash_filename_ = templates/apache-custom-slave-list.cfg.in
md5sum = 9eb14b83ee6fc8a5afa8267d9bcf4772
md5sum = 8ce1d5bf09662d941f940be7e6493918
[profile-replicate-publish-slave-information]
_update_hash_filename_ = templates/replicate-publish-slave-information.cfg.in
......@@ -50,7 +50,7 @@ md5sum = a0ae858a3db8825c22d33d323392f588
[template-backend-haproxy-configuration]
_update_hash_filename_ = templates/backend-haproxy.cfg.in
md5sum = 8c4e2548a12c8fd7dba74f940201745a
md5sum = 17f9582671327d8e4321a7fd1cdcb0fe
[template-empty]
_update_hash_filename_ = templates/empty.in
......
......@@ -200,7 +200,7 @@ context =
{% endfor %}
{% do slave.__setitem__('server-alias', ' '.join(slave_server_alias_unclashed)) %}
{% endif %}
{% for url_key in ['url', 'https-url'] %}
{% for url_key in ['url', 'https-url', 'health-check-failover-url', 'health-check-failover-https-url'] %}
{% if url_key in slave %}
{% set url = (slave[url_key] or '').strip() %}
{% if not validators.url(url) %}
......@@ -210,14 +210,16 @@ context =
{% endif %}
{% endif %}
{% endfor %}
{% if 'ssl_proxy_ca_crt' in slave %}
{% set ssl_proxy_ca_crt = slave.get('ssl_proxy_ca_crt', '') %}
{% set check_popen = popen([software_parameter_dict['openssl'], 'x509', '-noout']) %}
{% do check_popen.communicate(ssl_proxy_ca_crt) %}
{% if check_popen.returncode != 0 %}
{% do slave_error_list.append('ssl_proxy_ca_crt is invalid') %}
{% for k in ['ssl_proxy_ca_crt', 'health-check-failover-ssl-proxy-ca-crt'] %}
{% if k in slave %}
{% set crt = slave.get(k, '') %}
{% set check_popen = popen([software_parameter_dict['openssl'], 'x509', '-noout']) %}
{% do check_popen.communicate(crt) %}
{% if check_popen.returncode != 0 %}
{% do slave_error_list.append('%s is invalid' % (k,)) %}
{% endif %}
{% endif %}
{% endif %}
{% endfor %}
{# BBB: SlapOS Master non-zero knowledge BEGIN #}
{% for key in ['ssl_key', 'ssl_crt', 'ssl_ca_crt'] %}
{% if key in slave %}
......
......@@ -286,6 +286,44 @@
"default": "1",
"type": "integer"
},
"health-check-failover-url": {
"description": "URL of the failover backend",
"pattern": "^(http|https|ftp)://",
"title": "Failover backend URL",
"type": "string"
},
"health-check-failover-https-url": {
"description": "HTTPS URL of the failover backend if it is different from health-check-failover-url parameter. Note: It requires https-url to be configured, as otherwise the differentiation does not make sense..",
"pattern": "^(http|https|ftp)://",
"title": "Failover HTTPS Backend URL",
"type": "string"
},
"health-check-authenticate-to-failover-backend": {
"description": "If set to true the frontend certificate will be used as authentication certificate to the failover backend. Note: failover backend might have to know the frontend CA, available with 'backend-client-caucase-url'.",
"enum": [
"false",
"true"
],
"title": "Authenticate to failover backend",
"type": "string"
},
"health-check-failover-ssl-proxy-verify": {
"default": "false",
"description": "If set to true, failover backend SSL Certificates will be checked and frontend will refuse to proxy if certificate is invalid",
"enum": [
"false",
"true"
],
"title": "Verify failover backend certificates",
"type": "string"
},
"health-check-failover-ssl-proxy-ca-crt": {
"default": "",
"description": "Content of the SSL Certificate Authority file of the failover backend (to be used with health-check-failover-ssl-proxy-verify)",
"textarea": true,
"title": "SSL failover backend Authority's Certificate",
"type": "string"
},
"strict-transport-security": {
"title": "Strict Transport Security",
"description": "Enables Strict Transport Security (HSTS) on the slave, the default 0 results with option disabled. Setting the value enables HSTS and sets the value of max-age. More information: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Strict-Transport-Security",
......
......@@ -29,7 +29,7 @@ parts +=
recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.com/nexedi/kedifa.git
git-executable = ${git:location}/bin/git
revision = 1e38bcb3bf56f544262780be7e8858f3e6ceedc5
revision = 3fccc2ec945c59b644a12fa40225795abd61e0b0
[kedifa-develop]
recipe = zc.recipe.egg:develop
......
......@@ -58,6 +58,18 @@ context =
{%- do slave_instance.__setitem__(prefix, info_dict) %}
{%- endfor %}
{%- do slave_instance.__setitem__('ssl_proxy_verify', ('' ~ slave_instance.get('ssl-proxy-verify', '')).lower() in TRUE_VALUES) %}
{%- for key, prefix in [('health-check-failover-url', 'http_backend'), ('health-check-failover-https-url', 'https_backend')] %}
{%- set parsed = urlparse_module.urlparse(slave_instance.get(key, '').strip()) %}
{%- set info_dict = slave_instance[prefix] %}
{%- do info_dict.__setitem__('health-check-failover-scheme', parsed.scheme) %}
{%- do info_dict.__setitem__('health-check-failover-hostname', parsed.hostname) %}
{%- do info_dict.__setitem__('health-check-failover-port', parsed.port or DEFAULT_PORT[parsed.scheme]) %}
{%- do info_dict.__setitem__('health-check-failover-path', parsed.path) %}
{%- do info_dict.__setitem__('health-check-failover-query', parsed.query) %}
{%- do info_dict.__setitem__('health-check-failover-fragment', parsed.fragment) %}
{%- do slave_instance.__setitem__(prefix, info_dict) %}
{%- endfor %}
{%- do slave_instance.__setitem__('health-check-failover-ssl-proxy-verify', ('' ~ slave_instance.get('health-check-failover-ssl-proxy-verify', '')).lower() in TRUE_VALUES) %}
{%- do slave_instance.__setitem__('enable-http2', ('' ~ slave_instance.get('enable-http2', configuration['enable-http2-by-default'])).lower() in TRUE_VALUES) %}
{%- for key in ['https-only', 'websocket-transparent'] %}
{%- do slave_instance.__setitem__(key, ('' ~ slave_instance.get(key, 'true')).lower() in TRUE_VALUES) %}
......@@ -135,6 +147,7 @@ context =
{%- endfor %}
{%- do slave_instance.__setitem__('strict-transport-security', int(slave_instance['strict-transport-security'])) %}
{%- do slave_instance.__setitem__('authenticate-to-backend', ('' ~ slave_instance.get('authenticate-to-backend', '')).lower() in TRUE_VALUES) %}
{%- do slave_instance.__setitem__('health-check-authenticate-to-failover-backend', ('' ~ slave_instance.get('health-check-authenticate-to-failover-backend', '')).lower() in TRUE_VALUES) %}
{#- Setup active check #}
{%- do slave_instance.__setitem__('health-check', ('' ~ slave_instance.get('health-check', '')).lower() in TRUE_VALUES) %}
{%- if slave_instance['health-check'] %}
......@@ -266,7 +279,7 @@ command = {{ software_parameter_dict['htpasswd'] }} -cb ${:file} {{ slave_refere
{%- set certificate = '%s/%s' % (autocert, cert_name) %}
{%- do slave_parameter_dict.__setitem__('certificate', certificate )%}
{#- Set ssl certificates for each slave #}
{%- for cert_name in ('ssl_csr', 'ssl_proxy_ca_crt')%}
{%- for cert_name in ('ssl_csr', 'ssl_proxy_ca_crt', 'health-check-failover-ssl-proxy-ca-crt')%}
{%- set cert_file_key = 'path_to_' + cert_name %}
{%- if cert_name in slave_instance %}
{%- set cert_title = '%s-%s' % (slave_reference, cert_name.replace('ssl_', '')) %}
......
......@@ -32,7 +32,13 @@ defaults
{%- endif %}
{%- endfor %}
{%- if matched['count'] > 0 %}
{%- if slave_instance[SCHEME_PREFIX_MAPPING[scheme]]['health-check-failover-hostname'] %}
acl is_failover_{{ slave_instance['slave_reference'] }}_{{ scheme }} nbsrv({{ slave_instance['slave_reference'] }}-{{ scheme }}) eq 0
use_backend {{ slave_instance['slave_reference'] }}-{{ scheme }} if is_{{ slave_instance['slave_reference'] }}_{{ scheme }} ! is_failover_{{ slave_instance['slave_reference'] }}_{{ scheme }}
use_backend {{ slave_instance['slave_reference'] }}-{{ scheme }}-failover if is_{{ slave_instance['slave_reference'] }}_{{ scheme }} is_failover_{{ slave_instance['slave_reference'] }}_{{ scheme }}
{%- else %}
use_backend {{ slave_instance['slave_reference'] }}-{{ scheme }} if is_{{ slave_instance['slave_reference'] }}_{{ scheme }}
{%- endif %}
{%- endif %}
{%- endif %}
{%- endmacro %}
......@@ -123,5 +129,44 @@ backend {{ slave_instance['slave_reference'] }}-{{ scheme }}
{%- endif %}
{%- endif %}
{%- endif %}
{%- if info_dict['health-check-failover-hostname'] and info_dict['health-check-failover-port'] %}
{%- set ssl_list = [] %}
{%- if info_dict['health-check-failover-scheme'] == 'https' %}
{%- if slave_instance['health-check-authenticate-to-failover-backend'] %}
{%- do ssl_list.append('crt %s' % (configuration['certificate'],)) %}
{%- endif %}
{%- do ssl_list.append('ssl verify') %}
{%- if slave_instance['health-check-failover-ssl-proxy-verify'] %}
{%- if slave_instance['path_to_health-check-failover-ssl-proxy-ca-crt'] %}
{%- do ssl_list.append('required ca-file %s' % (slave_instance['path_to_health-check-failover-ssl-proxy-ca-crt'],)) %}
{%- else %}
{#- Backend SSL shall be verified, but not CA provided, disallow connection #}
{#- Simply dropping hostname from the dict will result with ignoring it... #}
{%- do info_dict.__setitem__('health-check-failover-hostname', '') %}
{%- endif %}
{%- else %}
{%- do ssl_list.append('none') %}
{%- endif %}
{%- endif %}
backend {{ slave_instance['slave_reference'] }}-{{ scheme }}-failover
{%- set hostname = info_dict['health-check-failover-hostname'] %}
{%- set port = info_dict['health-check-failover-port'] %}
{%- set path_list = [info_dict['health-check-failover-path'].rstrip('/')] %}
{%- set query = info_dict['health-check-failover-query'] %}
{%- if query %}
{%- do path_list.append(query) %}
{%- endif %}
{%- set path = '?'.join(path_list) %}
{%- if hostname and port %}
timeout server {{ slave_instance['request-timeout'] }}s
timeout connect {{ slave_instance['backend-connect-timeout'] }}s
retries {{ slave_instance['backend-connect-retries'] }}
server {{ slave_instance['slave_reference'] }}-backend {{ hostname }}:{{ port }} {{ ' '.join(ssl_list) }}
{%- if path %}
http-request set-path {{ path }}%[path]
{%- endif %}
{%- endif %}
{%- endif %}
{%- endfor %}
{%- endfor %}
This diff is collapsed.
......@@ -21,6 +21,21 @@ T-2/var/log/httpd/_health-check-default_error_log
T-2/var/log/httpd/_health-check-disabled_access_log
T-2/var/log/httpd/_health-check-disabled_backend_log
T-2/var/log/httpd/_health-check-disabled_error_log
T-2/var/log/httpd/_health-check-failover-url-auth-to-backend_access_log
T-2/var/log/httpd/_health-check-failover-url-auth-to-backend_backend_log
T-2/var/log/httpd/_health-check-failover-url-auth-to-backend_error_log
T-2/var/log/httpd/_health-check-failover-url-ssl-proxy-verified_access_log
T-2/var/log/httpd/_health-check-failover-url-ssl-proxy-verified_backend_log
T-2/var/log/httpd/_health-check-failover-url-ssl-proxy-verified_error_log
T-2/var/log/httpd/_health-check-failover-url-ssl-proxy-verify-missing_access_log
T-2/var/log/httpd/_health-check-failover-url-ssl-proxy-verify-missing_backend_log
T-2/var/log/httpd/_health-check-failover-url-ssl-proxy-verify-missing_error_log
T-2/var/log/httpd/_health-check-failover-url-ssl-proxy-verify-unverified_access_log
T-2/var/log/httpd/_health-check-failover-url-ssl-proxy-verify-unverified_backend_log
T-2/var/log/httpd/_health-check-failover-url-ssl-proxy-verify-unverified_error_log
T-2/var/log/httpd/_health-check-failover-url_access_log
T-2/var/log/httpd/_health-check-failover-url_backend_log
T-2/var/log/httpd/_health-check-failover-url_error_log
T-2/var/log/monitor-httpd-access.log
T-2/var/log/monitor-httpd-error.log
T-2/var/log/slave-introspection-access.log
......
......@@ -3,6 +3,28 @@
"description": "Parameters to instantiate ERP5",
"additionalProperties": false,
"definitions": {
"routing-rule-list": {
"description": "Maps the path received in requests to given zope path. Rules are applied in the order they are given. This requires the path received from the outside world (typically: frontend) to have its root correspond to Zope's root (for frontend: 'path' parameter must be empty), with the customary VirtualHostMonster construct (for frontend: 'type' must be 'zope').",
"type": "array",
"default": [["/", "/"]],
"items": {
"type": "array",
"minItems": 2,
"maxItems": 2,
"items": [
{
"title": "External path",
"description": "Path as received from the outside world, based on VirtualHostRoot element.",
"type": "string"
},
{
"title": "Internal path",
"description": "Zope path, based on Zope root object, the external path should correspond to. '%(site-id)s' is replaced by the site-id value, and '%%' replaced by '%'.",
"type": "string"
}
]
}
},
"tcpv4port": {
"$ref": "./schemas-definitions.json#/tcpv4port"
}
......@@ -59,6 +81,42 @@
"uniqueItems": true,
"type": "array"
},
"activity-timeout": {
"description": "How long a CMFActivity-initiated transaction may last, in seconds",
"default": null,
"type": ["number", "null"]
},
"publisher-timeout": {
"description": "How long a publisher-initiated transaction may last, in seconds",
"default": null,
"type": ["number", "null"]
},
"family-override": {
"description": "Family-wide options, possibly overriding global options",
"default": {},
"patternProperties": {
".*": {
"default": {},
"properties": {
"webdav": {
"description": "Serve webdav queries, implies timerserver-interval=0 (disabled)",
"default": false,
"type": "boolean"
},
"activity-timeout": {
"description": "Override global activity timeout",
"type": ["number", "null"]
},
"publisher-timeout": {
"description": "Override global publisher timeout",
"type": ["number", "null"]
}
},
"type": "object"
}
},
"type": "object"
},
"hostalias-dict": {
"description": "Hostname-to-domain-name mapping",
"default": {},
......@@ -154,11 +212,6 @@
"type": "boolean",
"default": false
},
"webdav": {
"description": "Serve webdav queries, implies timerserver-interval=0 (disabled). Mixing webdav and non-webdav nodes in a single family will give unspecified results.",
"default": false,
"type": "boolean"
},
"longrequest-logger-interval": {
"description": "Period, in seconds, with which LongRequestLogger polls worker thread stack traces, or -1 to disable",
"default": -1,
......@@ -452,6 +505,20 @@
"balancer": {
"description": "HTTP(S) load balancer proxy parameters",
"properties": {
"path-routing-list": {
"$ref": "#/definitions/routing-rule-list",
"title": "Global path routing rules"
},
"family-path-routing-dict": {
"type": "object",
"title": "Family-specific path routing rules",
"description": "Applied, only for the eponymous family, before global path routing rules.",
"patternProperties": {
".+": {
"$ref": "#/definitions/routing-rule-list"
}
}
},
"ssl": {
"description": "HTTPS certificate generation parameters",
"properties": {
......
......@@ -8,6 +8,7 @@ import shutil
import subprocess
import tempfile
import time
import urllib
import urlparse
from BaseHTTPServer import BaseHTTPRequestHandler
from typing import Dict
......@@ -166,7 +167,9 @@ class BalancerTestCase(ERP5InstanceTestCase):
'ssl-authentication-dict': {},
'ssl': {
'caucase-url': cls.getManagedResource("caucase", CaucaseService).url,
}
},
'family-path-routing-dict': {},
'path-routing-list': [],
}
@classmethod
......@@ -877,3 +880,93 @@ class TestClientTLS(BalancerTestCase):
with self.assertRaisesRegexp(Exception, 'certificate revoked'):
_make_request()
class TestPathBasedRouting(BalancerTestCase):
"""Check path-based routing rewrites URLs as expected.
"""
__partition_reference__ = 'pbr'
@classmethod
def _getInstanceParameterDict(cls):
# type: () -> Dict
parameter_dict = super(
TestPathBasedRouting,
cls,
)._getInstanceParameterDict()
parameter_dict['zope-family-dict'][
'second'
] = parameter_dict['zope-family-dict'][
'default'
]
# Routing rules outermost slashes mean nothing. They are internally
# stripped and rebuilt in order to correctly represent the request's URL.
parameter_dict['family-path-routing-dict'] = {
'default': [
['foo/bar', 'erp5/boo/far/faz'], # no outermost slashes
['/foo', '/erp5/somewhere'],
['/foo/shadowed', '/foo_shadowed'], # unreachable
['/next', '/erp5/web_site_module/another_next_website'],
],
}
parameter_dict['path-routing-list'] = [
['/next', '/erp5/web_site_module/the_next_website'],
['/next2', '/erp5/web_site_module/the_next2_website'],
['//', '//erp5/web_site_module/123//'], # extraneous slashes
]
return parameter_dict
def test_routing(self):
# type: () -> None
published_dict = json.loads(self.computer_partition.getConnectionParameterDict()['_'])
scheme = 'scheme'
netloc = 'example.com:8080'
prefix = '/VirtualHostBase/' + scheme + '//' + urllib.quote(
netloc,
safe='',
)
# For easier reading of test data, visualy separating the virtual host
# base from the virtual host root
vhr = '/VirtualHostRoot'
def assertRoutingEqual(family, path, expected_path):
# sanity check: unlike the rules, this test is sensitive to outermost
# slashes, and paths must be absolute-ish for code simplicity.
assert path.startswith('/')
# Frontend is expected to provide URLs with the following path structure:
# /VirtualHostBase/<scheme>//<netloc>/VirtualHostRoot<path>
# where:
# - scheme is the user-input scheme
# - netloc is the user-input netloc
# - path is the user-input path
# Someday, frontends will instead propagate scheme and netloc via other
# means (likely: HTTP headers), in which case this test and the SR will
# need to be amended to reconstruct Virtual Host urls itself, and this
# test will need to be updated accordingly.
self.assertEqual(
requests.get(
urlparse.urljoin(published_dict[family], prefix + vhr + path),
verify=False,
).json()['Path'],
expected_path,
)
# Trailing slash presence is preserved.
assertRoutingEqual('default', '/foo/bar', prefix + '/erp5/boo/far/faz' + vhr + '/_vh_foo/bar')
assertRoutingEqual('default', '/foo/bar/', prefix + '/erp5/boo/far/faz' + vhr + '/_vh_foo/bar/')
# Subpaths are preserved.
assertRoutingEqual('default', '/foo/bar/hey', prefix + '/erp5/boo/far/faz' + vhr + '/_vh_foo/bar/hey')
# Rule precedence: later less-specific rules are applied.
assertRoutingEqual('default', '/foo', prefix + '/erp5/somewhere' + vhr + '/_vh_foo')
assertRoutingEqual('default', '/foo/', prefix + '/erp5/somewhere' + vhr + '/_vh_foo/')
assertRoutingEqual('default', '/foo/baz', prefix + '/erp5/somewhere' + vhr + '/_vh_foo/baz')
# Rule precedence: later more-specific rules are meaningless.
assertRoutingEqual('default', '/foo/shadowed', prefix + '/erp5/somewhere' + vhr + '/_vh_foo/shadowed')
# Rule precedence: family rules applied before general rules.
assertRoutingEqual('default', '/next', prefix + '/erp5/web_site_module/another_next_website' + vhr + '/_vh_next')
# Fallback on general rules when no family-specific rule matches
# Note: the root is special in that there is aways a trailing slash in the
# produced URL.
assertRoutingEqual('default', '/', prefix + '/erp5/web_site_module/123' + vhr + '/')
# Rule-less family reach general rules.
assertRoutingEqual('second', '/foo/bar', prefix + '/erp5/web_site_module/123' + vhr + '/foo/bar') # Rules match whole-elements, so the rule order does not matter to
# elements which share a common prefix.
assertRoutingEqual('second', '/next', prefix + '/erp5/web_site_module/the_next_website' + vhr + '/_vh_next')
assertRoutingEqual('second', '/next2', prefix + '/erp5/web_site_module/the_next2_website' + vhr + '/_vh_next2')
......@@ -3,6 +3,7 @@
extends =
buildout.hash.cfg
../../stack/slapos.cfg
../../stack/nodejs.cfg
../../stack/monitor/buildout.cfg
../../component/ruby/buildout.cfg
../../component/golang/buildout.cfg
......@@ -11,7 +12,6 @@ extends =
../../component/cmake/buildout.cfg
../../component/icu/buildout.cfg
../../component/pkgconfig/buildout.cfg
../../component/nodejs/buildout.cfg
../../component/openssl/buildout.cfg
../../component/nginx/buildout.cfg
../../component/xz-utils/buildout.cfg
......@@ -67,12 +67,11 @@ parts =
[slapos.cookbook-repository]
revision = 571d6514f7290e8faa9439c4b86aa2f6c87df261
[nodejs]
<= nodejs-8.12.0
[yarn]
# need this version of Yarn
recipe = slapos.recipe.build:download-unpacked
url = https://github.com/yarnpkg/yarn/releases/download/v1.3.2/yarn-v1.3.2.tar.gz
md5sum = db82fa09c996e9318f2f1d2ab99228f9
<= yarn-1.3.2
############################
# Software compilation #
......@@ -122,7 +121,7 @@ bundle1.17.3 = ${buildout:parts-directory}/${:_buildout_section_name_}/lib/ruby/
# (python-4gitlab puts interpreter into ${buildout:bin-directory})
environment =
PATH = ${yarn:location}/bin:${:ruby-location}/bin:${cmake:location}/bin:${pkgconfig:location}/bin:${nodejs-8.12.0:location}/bin:${postgresql10:location}/bin:${redis28:location}/bin:${git:location}/bin:${buildout:bin-directory}:%(PATH)s
PATH = ${yarn:location}/bin:${:ruby-location}/bin:${cmake:location}/bin:${pkgconfig:location}/bin:${nodejs:location}/bin:${postgresql10:location}/bin:${redis28:location}/bin:${git:location}/bin:${buildout:bin-directory}:%(PATH)s
# gitlab, gitlab-shell & gitlab-workhorse checked out as git repositories
......@@ -210,7 +209,7 @@ make-binary =
make-targets= cd ${:path} && npm install
environment =
PATH=${nodejs-8.12.0:location}/bin/:%(PATH)s
PATH=${nodejs:location}/bin/:%(PATH)s
#our go infrastructure not currently supporting submodules, IIRC
# https://lab.nexedi.com/nexedi/slapos/merge_requests/337
......@@ -269,7 +268,9 @@ configure-command = cd ${:path}/ruby &&
${:bundle} config --local build.charlock_holmes --with-icu-dir=${icu:location}
make-binary =
make-targets =
. ${gowork:env.sh} && make
. ${gowork:env.sh} &&
unset GOBIN &&
make
environment =
PKG_CONFIG_PATH=${openssl-1.0:location}/lib/pkgconfig:${icu:location}/lib/pkgconfig
PATH=${pkgconfig:location}/bin:${ruby2.3:location}/bin:%(PATH)s
......
......@@ -8,10 +8,10 @@ This is an experimental integration, mainly to evaluate these solutions.
See https://github.com/influxdata/telegraf to learn about plugins.
Useful plugins in this context are probably
[exec](https://github.com/influxdata/telegraf/tree/1.11.1/plugins/inputs/exec),
[logparser](https://github.com/influxdata/telegraf/tree/1.11.1/plugins/inputs/logparser)
[exec](https://github.com/influxdata/telegraf/tree/v1.17.3/plugins/inputs/exec),
[logparser](https://github.com/influxdata/telegraf/tree/v1.17.3/plugins/inputs/logparser)
or
[http](https://github.com/influxdata/telegraf/tree/1.11.1/plugins/inputs/http).
[http](https://github.com/influxdata/telegraf/tree/v1.17.3/plugins/inputs/http).
Telegraf will save in the `telegraf` database from the embedded influxdb server.
......
......@@ -15,7 +15,7 @@
[instance-profile]
filename = instance.cfg.in
md5sum = 13f4f3806522b265dd1912ff169d146d
md5sum = 3ccdd2299e759488545b62368c7a0b91
[influxdb-config-file]
filename = influxdb-config-file.cfg.in
......@@ -35,8 +35,8 @@ md5sum = 3aa0f1ed752b2a59ea2b5e7c1733daf3
[loki-config-file]
filename = loki-config-file.cfg.in
md5sum = 02ba5acf23fcf88f5594919f46838533
md5sum = ad2baf4599a937d7352034a41fa24814
[promtail-config-file]
filename = promtail-config-file.cfg.in
md5sum = c77788d0a3cc654ad9393eb4b1f31e94
md5sum = c8c9d815dd7b427788c066f041f04573
......@@ -20,22 +20,22 @@ revision = 1f7c19e5f5
<= go-git-package
go.importpath = github.com/grafana/grafana
repository = https://github.com/grafana/grafana
revision = v6.3.0-0-g830da0fda0
revision = v7.4.3-0-g010f20c1c8
[go_github.com_grafana_loki]
<= go-git-package
go.importpath = github.com/grafana/loki
repository = https://github.com/grafana/loki
revision = v0.3.0-0-gc673380bb3
revision = v2.1.0-188-g877f524c36
[go_github.com_influxdata_influxdb]
<= go-git-package
go.importpath = github.com/influxdata/influxdb
repository = https://github.com/influxdata/influxdb
revision = v1.7.8rc1-0-ga7afa2ecf1
revision = v1.8.4-0-gbc8ec4384e
[go_github.com_influxdata_telegraf]
<= go-git-package
go.importpath = github.com/influxdata/telegraf
repository = https://github.com/influxdata/telegraf
revision = 1.11.1-0-gf91de60d37
revision = v1.17.3-0-g24a552b90b
......@@ -132,7 +132,7 @@ command-line =
bash -c "{{ influx_bin }} \
-username ${influxdb:auth-username} \
-password ${influxdb:auth-password} \
-host ${influxdb:host} \
-host [${influxdb:host}] \
-port ${influxdb:http-port} \
-unsafeSsl \
-ssl \
......@@ -219,6 +219,7 @@ storage-boltdb-dir = ${directory:loki-storage-boltdb-dir}
storage-filesystem-dir = ${directory:loki-storage-filesystem-dir}
ip = ${instance-parameter:ipv4-random}
port = 3100
grpc-port = 9095
url = http://${:ip}:${:port}
......@@ -238,9 +239,10 @@ command-line =
wrapper-path = ${directory:service}/promtail
dir = ${directory:promtail-dir}
http_port = 19080
http-port = 19080
grpc-port = 19095
ip = ${instance-parameter:ipv4-random}
url = http://${:ip}:${:http_port}
url = http://${:ip}:${:http-port}
[promtail-config-file]
<= config-file
......@@ -252,7 +254,7 @@ context =
[promtail-listen-promise]
<= check-port-listening-promise
hostname= ${promtail:ip}
port = ${promtail:http_port}
port = ${promtail:http-port}
......
auth_enabled: false
server:
http_listen_address: {{ loki['ip'] }}
http_listen_port: {{ loki['port'] }}
grpc_listen_address: {{ loki['ip'] }}
grpc_listen_port: {{ loki['grpc-port'] }}
ingester:
lifecycler:
......
# https://github.com/grafana/loki/blob/master/docs/logentry/processing-log-lines.md
server:
http_listen_port: {{ promtail['http_port'] }}
# XXX this external_url does not work ... promtail still listen on all IPs
http_listen_address: {{ promtail['ip'] }}
http_listen_port: {{ promtail['http-port'] }}
grpc_listen_address: {{ promtail['ip'] }}
grpc_listen_port: {{ promtail['grpc-port'] }}
external_url: {{ promtail['url'] }}
grpc_listen_port: 0
positions:
filename: {{ promtail['dir'] }}/positions.yaml
......
......@@ -22,17 +22,14 @@ parts =
loki-config-file
promtail-config-file
[python]
part = python3
[nodejs]
<= nodejs-10.6.0
<= nodejs-14.16.0
[yarn]
# this could become a component, but it needs to be invoked from nodejs explicitly,
# otherwise it uses system's nodejs
recipe = slapos.recipe.build:download-unpacked
url = https://github.com/yarnpkg/yarn/releases/download/v1.16.0/yarn-v1.16.0.tar.gz
md5sum = 46790033c23803387890f545e4040690
<= yarn-1.22.10
[gowork]
# All the softwares installed in the go workspace have "non standard" installation
......@@ -49,7 +46,7 @@ loki-bin = ${:bin}/loki
promtail-bin = ${:bin}/promtail
# use recent go
golang = ${golang1.12:location}
golang = ${golang1.16:location}
[gowork.goinstall]
command = :
......@@ -65,8 +62,7 @@ depends =
command = bash -c ". ${gowork:env.sh} && \
go install -v github.com/golang/dep/cmd/dep && \
cd ${gowork:directory}/src/github.com/influxdata/influxdb && \
dep ensure && \
go install ./cmd/..."
go install ./..."
update-command =
[telegraf-install]
......@@ -80,14 +76,17 @@ update-command =
[grafana-install]
<= gowork.goinstall
# yarn and go run build.go needs our nodejs in $PATH
command = bash -c "export PATH=${nodejs:location}/bin/:$PATH && \
. ${gowork:env.sh} && \
command = bash -c ". ${gowork:env.sh} && \
cd ${gowork:directory}/src/github.com/grafana/grafana && \
go run build.go setup && \
go run build.go build && \
${yarn:location}/bin/yarn install --pure-lockfile && \
${nodejs:location}/bin/npm run build"
${yarn:location}/bin/yarn run build && \
${yarn:location}/bin/yarn run plugins:build-bundled && \
# Cleanup yarn and Cypress caches
rm -rf ${buildout:directory}/.cache/Cypress/ && \
rm -rf ${buildout:directory}/.cache/yarn/
"
update-command =
[loki-install]
......
......@@ -25,12 +25,13 @@
#
##############################################################################
import os
import textwrap
import logging
import os
import tempfile
import textwrap
import time
import psutil
import requests
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
......@@ -44,11 +45,11 @@ setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
class GrafanaTestCase(SlapOSInstanceTestCase):
"""Base test case for grafana.
Since the instances takes timte to start and stop,
we increate as lot the number of retries.
Since the instances takes time to start and stop,
we increase the number of retries.
"""
instance_max_retry = 50
report_max_retry = 30
instance_max_retry = 30
class TestGrafana(GrafanaTestCase):
......@@ -144,21 +145,27 @@ class TestLoki(GrafanaTestCase):
r'''
- job_name: {cls.__name__}
pipeline_stages:
- regex:
expression: "^(?P<timestamp>.*) - (?P<name>\\S+) - (?P<level>\\S+) - (?P<message>.*)"
- timestamp:
format: 2006-01-02T15:04:05Z00:00
source: timestamp
- labels:
level:
name:
- match:
selector: '{{job="{cls.__name__}"}}'
stages:
- multiline:
firstline: '^\d{{4}}-\d{{2}}-\d{{2}}\s\d{{1,2}}\:\d{{2}}\:\d{{2}}\,\d{{3}}'
max_wait_time: 3s
- regex:
expression: '^(?P<timestamp>.*) - (?P<name>\S+) - (?P<level>\S+) - (?P<message>.*)'
- timestamp:
format: 2006-01-02T15:04:05Z00:00
source: timestamp
- labels:
level:
name:
static_configs:
- targets:
- localhost
labels:
job: {cls.__name__}
__path__: {cls._logfile.name}
''').format(**locals())
- targets:
- localhost
labels:
job: {cls.__name__}
__path__: {cls._logfile.name}
''').format(**locals())
}
@classmethod
......@@ -180,6 +187,7 @@ class TestLoki(GrafanaTestCase):
# create a logger logging to the file that we have
# configured in instance parameter.
test_logger = logging.getLogger(self.id())
test_logger.propagate = False
test_logger.setLevel(logging.INFO)
test_handler = logging.FileHandler(filename=self._logfile.name)
test_handler.setFormatter(
......@@ -189,16 +197,25 @@ class TestLoki(GrafanaTestCase):
test_logger.info("testing message")
test_logger.info("testing another message")
test_logger.warning("testing warn")
# log an exception, which will be multi line in log file.
def nested1():
def nested2():
raise ValueError('boom')
nested2()
try:
nested1()
except ValueError:
test_logger.exception("testing exception")
# Check our messages have been ingested
# we retry a few times, because there's a short delay until messages are
# ingested and returned.
for i in range(10):
for i in range(60):
resp = requests.get(
'{self.loki_url}/api/prom/query?query={{job="TestLoki"}}'.format(
**locals()),
verify=False).json()
if not resp:
if len(resp.get('streams', [])) < 3:
time.sleep(0.5 * i)
continue
......@@ -220,9 +237,83 @@ class TestLoki(GrafanaTestCase):
line for line in info_stream['entries']
if "testing another message" in line['line']
])
error_stream_list = [stream for stream in resp['streams'] if 'level="ERROR"' in stream['labels']]
self.assertEqual(1, len(error_stream_list), resp['streams'])
error_stream, = error_stream_list
line, = [line['line'] for line in error_stream['entries']]
# this entry is multi-line
self.assertIn('testing exception\nTraceback (most recent call last):\n', line)
self.assertIn('ValueError: boom', line)
# The labels we have configued are also available
resp = requests.get(
'{self.loki_url}/api/prom/label'.format(**locals()),
verify=False).json()
self.assertIn('level', resp['values'])
self.assertIn('name', resp['values'])
class TestListenInPartition(GrafanaTestCase):
def setUp(self):
with self.slap.instance_supervisor_rpc as supervisor:
all_process_info = supervisor.getAllProcessInfo()
self.process_dict = {
p['name'].replace('-on-watch', ''): psutil.Process(p['pid'])
for p in all_process_info if p['name'] != 'watchdog'
}
def test_grafana_listen(self):
self.assertEqual(
[
c.laddr for c in self.process_dict['grafana'].connections()
if c.status == 'LISTEN'
],
[(self._ipv6_address, 8180)],
)
def test_influxdb_listen(self):
self.assertEqual(
sorted([
c.laddr for c in self.process_dict['influxdb'].connections()
if c.status == 'LISTEN'
]),
[
(self._ipv4_address, 8088),
(self._ipv6_address, 8086),
],
)
def test_telegraph_listen(self):
self.assertEqual(
[
c.laddr for c in self.process_dict['telegraf'].connections()
if c.status == 'LISTEN'
],
[],
)
def test_loki_listen(self):
self.assertEqual(
sorted([
c.laddr for c in self.process_dict['loki'].connections()
if c.status == 'LISTEN'
]),
[
(self._ipv4_address, 3100),
(self._ipv4_address, 9095),
],
)
def test_promtail_listen(self):
self.assertEqual(
sorted([
c.laddr for c in self.process_dict['promtail'].connections()
if c.status == 'LISTEN'
]),
[
(self._ipv4_address, 19080),
(self._ipv4_address, 19095),
],
)
......@@ -19,7 +19,7 @@ md5sum = e6d5c7bb627b4f1d3e7c99721b7c58fe
[template-kvm]
filename = instance-kvm.cfg.jinja2
md5sum = 23493c541efef97ac5fe435114910b8e
md5sum = 0c00ca332d364a2c620cd260d51d4649
[template-kvm-cluster]
filename = instance-kvm-cluster.cfg.jinja2.in
......@@ -55,7 +55,7 @@ md5sum = b7e87479a289f472b634a046b44b5257
[template-kvm-run]
filename = template/template-kvm-run.in
md5sum = cc1b748c82ca15022744558dab05995c
md5sum = d1748b18fc76544cc71be5c40ed9cbb9
[template-kvm-controller]
filename = template/kvm-controller-run.in
......
......@@ -1036,10 +1036,16 @@ keyboard-layout-language = fr
recipe = slapos.recipe.template:jinja2
template = inline:
{%- raw %}
[{"disk": "{{disk_device_path}}"}]
{%- set disk_list = [] %}
{%- for disk in disk_device_path.split() %}
{%- do disk_list.append({'disk': disk}) %}
{%- endfor -%}
{{ json_module.dumps(disk_list) }}
{% endraw -%}
rendered = ${buildout:directory}/.slapos-disk-permission
extensions = jinja2.ext.do
context =
import json_module json
raw disk_device_path {{disk_device_path}}
{% do part_list.append('wipe-disk-device-wrapper') -%}
......
......@@ -83,12 +83,23 @@ disk_aio = disk_aio if disk_aio in ["threads", "native"] else "threads"
# If a device (ie.: /dev/sdb) is provided, use it instead
# the disk_path with disk_format
disk_device_path = '{{ parameter_dict.get("disk-device-path", "") }}'
if disk_device_path.startswith("/dev/"):
disk_path = disk_device_path
disk_format = "raw"
disk_aio = "native"
disk_cache = "none"
disk_info_list = []
for disk_device_path in '{{ parameter_dict.get("disk-device-path", "") }}'.split():
if disk_device_path.startswith("/dev/"):
disk_info_list.append({
'path': disk_device_path,
'format': "raw",
'io': "native",
'cache': "none"
})
if len(disk_info_list) == 0:
disk_info_list.append({
'path': disk_path,
'format': disk_format,
'io': disk_aio,
'cache': disk_cache,
})
smp_count = {{ parameter_dict.get("smp-count") }}
smp_max_count = {{ parameter_dict.get("smp-max-count") }}
......@@ -167,12 +178,12 @@ def getMapStorageList(disk_storage_dict, external_disk_number):
return id_list, external_disk_number
# Download existing hard drive if needed at first boot
if not os.path.exists(disk_path) and virtual_hard_drive_url != '':
if len(disk_info_list) == 1 and not os.path.exists(disk_info_list[0]['path']) and virtual_hard_drive_url != '':
print('Downloading virtual hard drive...')
try:
downloaded_disk = disk_path
downloaded_disk = disk_info_list[0]['path']
if virtual_hard_drive_gzipped == 'true':
downloaded_disk = '%s.gz' % disk_path
downloaded_disk = '%s.gz' % disk_info_list[0]['path']
opener.retrieve(virtual_hard_drive_url, downloaded_disk)
except:
if os.path.exists(downloaded_disk):
......@@ -191,21 +202,21 @@ if not os.path.exists(disk_path) and virtual_hard_drive_url != '':
print('Warning: not checksum specified.')
if downloaded_disk.endswith('.gz'):
try:
with open(disk_path, 'w') as disk:
with open(disk_info_list[0]['path'], 'w') as disk:
with gzip.open(downloaded_disk, 'rb') as disk_gz:
shutil.copyfileobj(disk_gz, disk)
except Exception:
if os.path.exists(disk_path):
os.remove(disk_path)
if os.path.exists(disk_info_list[0]['path']):
os.remove(disk_info_list[0]['path'])
raise
os.remove(downloaded_disk)
# Create disk if doesn't exist
# XXX: move to Buildout profile
if not disk_device_path.startswith("/dev/") and not os.path.exists(disk_path):
if len(disk_info_list) == 1 and not disk_info_list[0]['path'].startswith("/dev/") and not os.path.exists(disk_info_list[0]['path']):
print('Creating virtual hard drive...')
subprocess.check_call([qemu_img_path, 'create' ,'-f', disk_format,
disk_path, '%sG' % disk_size])
subprocess.check_call([qemu_img_path, 'create' ,'-f', disk_info_list[0]['format'],
disk_info_list[0]['path'], '%sG' % disk_size])
print('Done.')
# Check and create external disk
......@@ -240,13 +251,6 @@ if disk_storage_dict:
print('Data folder %s was not used to create external disk %r' % (index +1))
index += 1
additional_disk_options = ''
if disk_aio == 'native':
additional_disk_options += ',cache.direct=on'
if disk_format == "raw":
additional_disk_options += ',discard=on'
# Generate network parameters
# XXX: use_tap should be a boolean
tap_network_parameter = []
......@@ -303,8 +307,6 @@ else:
kvm_argument_list = [qemu_path,
'-enable-kvm', '-smp', smp, '-name', vm_name, '-m', ram, '-vga', 'std',
'-drive', 'file=%s,if=%s,cache=%s,aio=%s%s' % (disk_path, disk_type, disk_cache,
disk_aio, additional_disk_options),
'-vnc', '%s:1,ipv4,password' % listen_ip,
'-boot', 'order=cd,menu=on',
'-qmp', 'unix:%s,server,nowait' % socket_path,
......@@ -312,6 +314,18 @@ kvm_argument_list = [qemu_path,
'-D', logfile,
'-nodefaults',
]
for disk_info in disk_info_list:
additional_disk_options = ''
if disk_info['io'] == 'native':
additional_disk_options += ',cache.direct=on'
if disk_info['format'] == "raw":
additional_disk_options += ',discard=on'
kvm_argument_list.extend([
'-drive',
'file=%s,if=%s,cache=%s,aio=%s%s' % ( disk_info['path'], disk_type, disk_info['cache'], disk_info['io'], additional_disk_options)
])
rgx = re.compile('^[\w*\,][\=\d+\-\,\w]*$')
for numa in numa_list:
......
......@@ -14,7 +14,7 @@
# not need these here).
[instance-common]
filename = instance-common.cfg.in
md5sum = 6da513940e5bf7d06b3fb0aeb39c8ad5
md5sum = 2bd1779425b7561682c0de5496d808ed
[root-common]
filename = root-common.cfg.in
......
......@@ -23,6 +23,7 @@ context =
key ipv4_set slap-configuration:ipv4
key ipv6_set slap-configuration:ipv6
key slapparameter_dict slap-configuration:configuration
key computer slap-configuration:computer
raw logrotate_cfg {{ template_logrotate_base }}
raw template_monitor {{ template_monitor }}
raw bin_directory {{ bin_directory }}
......
[buildout]
extends = software.cfg
[neoppod]
eggs = neoppod[admin, ctl, master, storage-importer, storage-mysqldb, tests]
${python-mysqlclient:egg}
ZODB3
patch-binary = ${patch:location}/bin/patch
ZODB3-patch-options = -p1
ZODB3-patches =
${neoppod-repository:location}/ZODB3.patch
[versions]
ZODB3 = 3.10.7+SlapOSPatched001
transaction = 1.1.1
zdaemon = 2.0.7
......@@ -12,9 +12,6 @@ ZODB = 5.6.0
ZEO = 5.2.2
transaction = 2.4.0
# BBB: ZEO
msgpack = 0.5.6
# Required by:
# ZEO==5.2.0
# trollius==2.2.post1
......
# Code generated by gowork-snapshot; DO NOT EDIT.
# list of go git repositories to fetch
[gowork.goinstall]
depends_gitfetch =
${go_crawshaw.io_sqlite:recipe}
${go_github.com_DataDog_czlib:recipe}
${go_github.com_cznic_strutil:recipe}
${go_github.com_davecgh_go-spew:recipe}
${go_github.com_fsnotify_fsnotify:recipe}
${go_github.com_golang_glog:recipe}
${go_github.com_gwenn_gosqlite:recipe}
${go_github.com_gwenn_yacr:recipe}
${go_github.com_kisielk_og-rek:recipe}
${go_github.com_kylelemons_godebug:recipe}
${go_github.com_philhofer_fwd:recipe}
${go_github.com_pkg_errors:recipe}
${go_github.com_pkg_profile:recipe}
${go_github.com_pmezard_go-difflib:recipe}
${go_github.com_shamaton_msgpack:recipe}
${go_github.com_soheilhy_cmux:recipe}
${go_github.com_someonegg_gocontainer:recipe}
${go_github.com_someonegg_gox:recipe}
${go_github.com_stretchr_testify:recipe}
${go_github.com_tinylib_msgp:recipe}
${go_github.com_ttacon_chalk:recipe}
${go_golang.org_x_crypto:recipe}
${go_golang.org_x_mod:recipe}
${go_golang.org_x_net:recipe}
${go_golang.org_x_perf:recipe}
${go_golang.org_x_sync:recipe}
${go_golang.org_x_sys:recipe}
${go_golang.org_x_text:recipe}
${go_golang.org_x_tools:recipe}
${go_golang.org_x_xerrors:recipe}
${go_gopkg.in_yaml.v3:recipe}
${go_lab.nexedi.com_kirr_go123:recipe}
${go_lab.nexedi.com_kirr_neo:recipe}
[go_crawshaw.io_sqlite]
<= go-git-package
go.importpath = crawshaw.io/sqlite
repository = https://github.com/crawshaw/sqlite
revision = v0.3.2-7-gc582b9de4f
[go_github.com_DataDog_czlib]
<= go-git-package
go.importpath = github.com/DataDog/czlib
repository = https://github.com/DataDog/czlib
revision = 4bc9a24e37
[go_github.com_cznic_strutil]
<= go-git-package
go.importpath = github.com/cznic/strutil
repository = https://github.com/cznic/strutil
revision = 529a34b1c1
[go_github.com_davecgh_go-spew]
<= go-git-package
go.importpath = github.com/davecgh/go-spew
repository = https://github.com/davecgh/go-spew
revision = v1.1.1-1-gd8f796af33
[go_github.com_fsnotify_fsnotify]
<= go-git-package
go.importpath = github.com/fsnotify/fsnotify
repository = https://github.com/fsnotify/fsnotify
revision = 7f4cf4dd2b
[go_github.com_golang_glog]
<= go-git-package
go.importpath = github.com/golang/glog
repository = https://github.com/golang/glog
revision = 23def4e6c1
[go_github.com_gwenn_gosqlite]
<= go-git-package
go.importpath = github.com/gwenn/gosqlite
repository = https://github.com/gwenn/gosqlite
revision = 24878be1a2
[go_github.com_gwenn_yacr]
<= go-git-package
go.importpath = github.com/gwenn/yacr
repository = https://github.com/gwenn/yacr
revision = bbe82c1f4d
[go_github.com_kisielk_og-rek]
<= go-git-package
go.importpath = github.com/kisielk/og-rek
repository = https://github.com/kisielk/og-rek
revision = 24bb08c22e
[go_github.com_kylelemons_godebug]
<= go-git-package
go.importpath = github.com/kylelemons/godebug
repository = https://github.com/kylelemons/godebug
revision = fa7b53cdfc
[go_github.com_philhofer_fwd]
<= go-git-package
go.importpath = github.com/philhofer/fwd
repository = https://github.com/philhofer/fwd
revision = 414ae1bb9e
[go_github.com_pkg_errors]
<= go-git-package
go.importpath = github.com/pkg/errors
repository = https://github.com/pkg/errors
revision = v0.8.1-31-g614d223910
[go_github.com_pkg_profile]
<= go-git-package
go.importpath = github.com/pkg/profile
repository = https://github.com/pkg/profile
revision = v1.5.0-1-g3704c8d233
[go_github.com_pmezard_go-difflib]
<= go-git-package
go.importpath = github.com/pmezard/go-difflib
repository = https://github.com/pmezard/go-difflib
revision = v1.0.0-4-g5d4384ee4f
[go_github.com_shamaton_msgpack]
<= go-git-package
go.importpath = github.com/shamaton/msgpack
repository = https://github.com/shamaton/msgpack
revision = 29a4ba0bb9
[go_github.com_soheilhy_cmux]
<= go-git-package
go.importpath = github.com/soheilhy/cmux
repository = https://github.com/soheilhy/cmux
revision = 8a8ea3c539
[go_github.com_someonegg_gocontainer]
<= go-git-package
go.importpath = github.com/someonegg/gocontainer
repository = https://github.com/someonegg/gocontainer
revision = 24fb283ab9
[go_github.com_someonegg_gox]
<= go-git-package
go.importpath = github.com/someonegg/gox
repository = https://github.com/someonegg/gox
revision = c9bfdf8eb4
[go_github.com_stretchr_testify]
<= go-git-package
go.importpath = github.com/stretchr/testify
repository = https://github.com/stretchr/testify
revision = v1.2.2-210-g54d05a4e18
[go_github.com_tinylib_msgp]
<= go-git-package
go.importpath = github.com/tinylib/msgp
repository = https://github.com/tinylib/msgp
revision = 87c1ec45d5
[go_github.com_ttacon_chalk]
<= go-git-package
go.importpath = github.com/ttacon/chalk
repository = https://github.com/ttacon/chalk
revision = v0.1-3-g22c06c80ed
[go_golang.org_x_crypto]
<= go-git-package
go.importpath = golang.org/x/crypto
repository = https://go.googlesource.com/crypto
revision = 5c72a88397
[go_golang.org_x_mod]
<= go-git-package
go.importpath = golang.org/x/mod
repository = https://go.googlesource.com/mod
revision = ce943fd024
[go_golang.org_x_net]
<= go-git-package
go.importpath = golang.org/x/net
repository = https://go.googlesource.com/net
revision = 62affa334b
[go_golang.org_x_perf]
<= go-git-package
go.importpath = golang.org/x/perf
repository = https://go.googlesource.com/perf
revision = d949658356
[go_golang.org_x_sync]
<= go-git-package
go.importpath = golang.org/x/sync
repository = https://go.googlesource.com/sync
revision = 6e8e738ad2
[go_golang.org_x_sys]
<= go-git-package
go.importpath = golang.org/x/sys
repository = https://go.googlesource.com/sys
revision = aee5d888a8
[go_golang.org_x_text]
<= go-git-package
go.importpath = golang.org/x/text
repository = https://go.googlesource.com/text
revision = v0.3.3-2-ga8b4671254
[go_golang.org_x_tools]
<= go-git-package
go.importpath = golang.org/x/tools
repository = https://go.googlesource.com/tools
revision = ba800b16d8
[go_golang.org_x_xerrors]
<= go-git-package
go.importpath = golang.org/x/xerrors
repository = https://go.googlesource.com/xerrors
revision = 5ec99f83af
[go_gopkg.in_yaml.v3]
<= go-git-package
go.importpath = gopkg.in/yaml.v3
repository = https://gopkg.in/yaml.v3
revision = v2.1.1-97-geeeca48fe7
[go_lab.nexedi.com_kirr_go123]
<= go-git-package
go.importpath = lab.nexedi.com/kirr/go123
repository = https://lab.nexedi.com/kirr/go123.git
revision = c8d9907ef7
[go_lab.nexedi.com_kirr_neo]
<= go-git-package
go.importpath = lab.nexedi.com/kirr/neo
repository = https://lab.nexedi.com/kirr/neo.git
revision = v1.12-2791-gf3effa6c53
......@@ -2,12 +2,7 @@
[buildout]
extends = software.cfg
[go_lab.nexedi.com_kirr_go123]
branch = master
revision=
[go_lab.nexedi.com_kirr_neo]
[neo]
branch = t
revision=
......
......@@ -8,7 +8,6 @@ extends =
../../stack/slapos.cfg
../neoppod/software-common.cfg
../../component/golang/buildout.cfg
gowork.cfg
../../component/sqlite3/buildout.cfg
../../component/zlib/buildout.cfg
../../component/wendelin.core/buildout.cfg
......@@ -49,12 +48,23 @@ parts =
instance.cfg
# go packages to install (+ automatically their dependencies)
[neo]
<= go-git-package
go.importpath = lab.nexedi.com/kirr/neo
repository = https://lab.nexedi.com/kirr/neo.git
revision = v1.12-3107-gf278d611
[goperf]
<= go-git-package
go.importpath = golang.org/x/perf
repository = https://lab.nexedi.com/kirr/go.perf.git
revision = e8bebf9314da
# go programs to install
[gowork]
install =
lab.nexedi.com/kirr/neo/go/...
github.com/pkg/profile
golang.org/x/perf/cmd/benchstat
${neo:location}/go:./...
${goperf:location}:./cmd/benchstat
cpkgpath =
${sqlite3:location}/lib/pkgconfig
......@@ -62,7 +72,7 @@ cpkgpath =
# dev-install neo from go checkout
[neoppod-develop]
setup = ${go_lab.nexedi.com_kirr_neo:location}
setup = ${neo:location}
# neotest-env.sh + neotest wrapper
......@@ -99,7 +109,7 @@ template = inline:
[tnxdtest]
recipe = plone.recipe.command
stop-on-error = yes
command = ln -s -t $${directory:t} ${gowork:src}/lab.nexedi.com/kirr/neo/go/neo/t/.nxdtest
command = ln -s -t $${directory:t} ${neo:location}/go/neo/t/.nxdtest
......@@ -119,10 +129,6 @@ eggs =
${wendelin.core:egg}
# wendelin.core: latest not yet released
[wendelin.core-repository]
revision= v0.11-15-gf785ac079b
# ping eggs versions
[versions]
ZODB3 = 3.11.0
......
......@@ -43,9 +43,6 @@ setup(
install_requires=[
'slapos.core',
'slapos.libnetworkcache',
'erp5.util',
'supervisor',
'pexpect',
'requests',
],
zip_safe=True,
......
......@@ -24,16 +24,10 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from __future__ import unicode_literals
import os
import textwrap
import logging
import tempfile
import time
from six.moves.urllib.parse import urlparse, urljoin
from six.moves.urllib.parse import urljoin
import pexpect
import requests
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
......@@ -44,17 +38,46 @@ setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
class TestRepman(SlapOSInstanceTestCase):
__partition_reference__ = 'R' # solve path too long for postgresql and unicorn
@classmethod
def getInstanceSoftwareType(cls):
return 'default'
__partition_reference__ = 'R'
def setUp(self):
self.backend_url = self.computer_partition.getConnectionParameterDict(
)['backend-url']
self.url = self.computer_partition.getConnectionParameterDict()['url']
def test_http_get(self):
resp = requests.get(self.backend_url, verify=False)
self.assertTrue(
resp.status_code in [requests.codes.ok, requests.codes.found])
connection_parameter_dict = \
self.computer_partition.getConnectionParameterDict()
resp = requests.get(self.url, verify=False)
self.assertEqual(resp.status_code, requests.codes.ok)
resp = requests.post(
urljoin(self.url, '/api/login'),
json={
'username': connection_parameter_dict['username'],
'password': connection_parameter_dict['repman-password'],
},
verify=False,
)
self.assertEqual(resp.status_code, requests.codes.ok)
token = resp.json()['token']
headers = {"authorization": "Bearer " + token}
resp = requests.get(
urljoin(self.url, '/api/monitor'),
headers=headers,
verify=False,
)
self.assertEqual(resp.status_code, requests.codes.ok)
resp = requests.get(
urljoin(self.url, '/api/clusters'),
params={
'query': '{"method":"GET","isArray":false}',
},
headers=headers,
verify=False,
)
self.assertEqual(resp.status_code, requests.codes.ok)
cluster, = resp.json()
self.assertTrue(cluster['isProvision'])
self.assertTrue(cluster['isFailable'])
self.assertFalse(cluster['isDown'])
......@@ -14,7 +14,7 @@
# not need these here).
[template-erp5]
filename = instance-erp5.cfg.in
md5sum = 4d0839e359c98ba3cd516903b72f798b
md5sum = 3d2683f8b0031f4f5690c6b9388c0566
[template-balancer]
filename = instance-balancer.cfg.in
......
......@@ -177,9 +177,9 @@ return =
{% endif %}
{% set bt5_default_list = 'erp5_full_text_mroonga_catalog slapos_configurator' -%}
{% if has_jupyter -%}
{% set bt5_default_list = bt5_default_list + ' erp5_data_notebook' -%}
{% do bt5_default_list.append('erp5_data_notebook') -%}
{% endif -%}
config-bt5 = {{ dumps(slapparameter_dict.get('bt5', bt5_default_list)) }}
config-bt5 = {{ dumps(slapparameter_dict.get('bt5', ' '.join(bt5_default_list))) }}
config-bt5-repository-url = {{ dumps(slapparameter_dict.get('bt5-repository-url', local_bt5_repository)) }}
config-cloudooo-url = {{ dumps(slapparameter_dict.get('cloudooo-url', default_cloudooo_url)) }}
config-caucase-url = {{ dumps(caucase_url) }}
......@@ -187,6 +187,7 @@ config-deadlock-debugger-password = ${publish-early:deadlock-debugger-password}
config-developer-list = {{ dumps(slapparameter_dict.get('developer-list', [inituser_login])) }}
config-saucelabs-dict = {{ dumps(slapparameter_dict.get('saucelabs-dict', {})) }}
config-hosts-dict = {{ dumps(slapparameter_dict.get('hosts-dict', {})) }}
config-computer-hosts-dict = {{ dumps(slapparameter_dict.get('computer-hosts-dict', {})) }}
config-hostalias-dict = {{ dumps(slapparameter_dict.get('hostalias-dict', {})) }}
config-id-store-interval = {{ dumps(slapparameter_dict.get('id-store-interval')) }}
config-zope-longrequest-logger-error-threshold = {{ dumps(monitor_dict.get('zope-longrequest-logger-error-threshold', 20)) }}
......@@ -430,7 +431,7 @@ hosts-dict = {{ '${' ~ zope_address_list_id_dict.keys()[0] ~ ':connection-hosts-
{% endfor -%}
{% if test_runner_enabled -%}
{% for zope_family_name in zope_family_name_list -%}
{{ zope_family_name }}-test-runner-url-list = ${request-balancer:connection-{{ zope_family_name }}-test-runner-url-list}
{{ zope_family_name }}-test-runner-url-list = ${request-balancer:connection-{{ zope_family_name }}-test-runner-url-list}
{% endfor -%}
{% endif -%}
......
......@@ -15,6 +15,7 @@ extends =
../../stack/slapos.cfg
../../stack/nxdtest.cfg
../../stack/caucase/buildout.cfg
./buildout.hash.cfg
......@@ -314,11 +315,6 @@ selenium = 3.141.0
# Patched eggs
PyPDF2 = 1.26.0+SlapOSPatched001
# caucase and its dependencies
caucase = 0.9.4
pem = 18.2.0
PyJWT = 1.6.4
# Django 1.11 is python 2 compatible
Django = 1.11
......
[buildout]
extends =
../../component/nodejs/buildout.cfg
../../component/caddy/buildout.cfg
../../component/git/buildout.cfg
../../component/bash/buildout.cfg
......@@ -13,6 +12,7 @@ extends =
../../component/coreutils/buildout.cfg
../../component/java-jdk/buildout.cfg
../../component/fonts/buildout.cfg
../../stack/nodejs.cfg
../../stack/slapos.cfg
../../stack/monitor/buildout.cfg
../../component/defaults.cfg
......@@ -37,13 +37,7 @@ max_version = 0
<= nodejs-12.18.3
[yarn]
# this could become a component, but it needs to be invoked from nodejs explicitly,
# otherwise it uses system's nodejs
# XXX why don't we build a wrapper ?
version = 1.17.3
recipe = slapos.recipe.build:download-unpacked
url = https://github.com/yarnpkg/yarn/releases/download/v${:version}/yarn-v${:version}.tar.gz
md5sum = 4a02e1687a150113ad6b0215f9afdb3e
<= yarn-1.17.3
[slapos-standalone]
recipe = zc.recipe.egg
......
......@@ -36,6 +36,6 @@ mode = 0644
depends = ${caucase-jinja2-library-eggs:eggs}
[versions]
caucase = 0.9.8
pem = 20.1.0
caucase = 0.9.9
pem = 21.1.0
PyJWT = 1.7.1
......@@ -34,7 +34,7 @@ md5sum = e91c0fbd0df441884f7422fa7976053c
[template-zope-conf]
filename = zope.conf.in
md5sum = c43da8f7b4db22e40a4864e6cfcaef44
md5sum = 5ce9d5e99880d5db9ed339bbefe28d34
[site-zcml]
filename = site.zcml
......@@ -74,7 +74,7 @@ md5sum = b5ac16fdeed8863e465e955ba6d1e12a
[template-erp5]
filename = instance-erp5.cfg.in
md5sum = 548d99118afa736e5a7c428b0c8ed560
md5sum = ac155f28a096747fc267f32a1cec46e1
[template-zeo]
filename = instance-zeo.cfg.in
......@@ -82,15 +82,15 @@ md5sum = 0648e38bd5d3a15bb9f93264932740b9
[template-zope]
filename = instance-zope.cfg.in
md5sum = c03f93f95333e6a61b857dcfab7f9c0e
md5sum = 9fa66b93fbf6a40aa8136c651ad9f539
[template-balancer]
filename = instance-balancer.cfg.in
md5sum = 8ad9137310ae0403d433bb3c0d93be9f
md5sum = 8d3694226b6cbed961f6d608b6d6d294
[template-haproxy-cfg]
filename = haproxy.cfg.in
md5sum = 8de18a61607bd66341a44b95640d293f
md5sum = 452c502fabd5a6066c9dee533dfb1c77
[template-rsyslogd-cfg]
filename = rsyslogd.cfg.in
......
......@@ -147,6 +147,8 @@ defaults
log {{ parameter_dict['log-socket'] }} local0 info
{% set bind_ssl_crt = 'ssl crt ' ~ parameter_dict['cert'] ~ ' alpn h2,http/1.1' %}
{% set family_path_routing_dict = parameter_dict['family-path-routing-dict'] %}
{% set path_routing_list = parameter_dict['path-routing-list'] %}
{% for name, (port, _, certificate_authentication, backend_list) in sorted(parameter_dict['backend-dict'].iteritems()) -%}
listen family_{{ name }}
......@@ -172,6 +174,11 @@ listen family_{{ name }}
capture request header User-Agent len 512
log-format "%{+Q}o %{-Q}ci - - [%trg] %r %ST %B %{+Q}[capture.req.hdr(0)] %{+Q}[capture.req.hdr(1)] %Tt"
{% for outer_prefix, inner_prefix in family_path_routing_dict.get(name, []) + path_routing_list %}
{% set outer_prefix = outer_prefix.strip('/') -%}
http-request replace-path ^(/+VirtualHostBase/+[^/]+/+[^/]+)/+VirtualHostRoot/+({% if outer_prefix %}{{ outer_prefix }}($|/.*){% else %}.*{% endif %}) \1/{{ inner_prefix.strip('/') }}/VirtualHostRoot/{% if outer_prefix %}_vh_{% endif %}\2
{% endfor %}
{% set has_webdav = [] -%}
{% for address, connection_count, webdav in backend_list -%}
{% if webdav %}{% do has_webdav.append(None) %}{% endif -%}
......
......@@ -190,6 +190,8 @@ ca-cert = ${haproxy-conf-ssl:ca-cert}
crl = ${haproxy-conf-ssl:crl}
{% endif %}
stats-socket = ${directory:run}/haproxy.sock
path-routing-list = {{ dumps(slapparameter_dict['path-routing-list']) }}
family-path-routing-dict = {{ dumps(slapparameter_dict['family-path-routing-dict']) }}
pidfile = ${directory:run}/haproxy.pid
log-socket = ${rsyslogd-cfg-parameter-dict:log-socket}
server-check-path = {{ dumps(slapparameter_dict['haproxy-server-check-path']) }}
......
......@@ -24,6 +24,7 @@
{% set use_ipv6 = slapparameter_dict.get('use-ipv6', False) -%}
{% set partition_thread_count_list = [] -%}
{% set zope_partition_dict = slapparameter_dict.get('zope-partition-dict', {'1': {}}) -%}
{% set zope_family_override_dict = slapparameter_dict.get('family-override', {}) -%}
{% for zope_parameter_dict in zope_partition_dict.values() -%}
{# Apply some zope_parameter_dict default values, to avoid duplication. -#}
{% do zope_parameter_dict.setdefault('thread-amount', 4) -%}
......@@ -34,7 +35,8 @@
<= request-common-base
config-use-ipv6 = {{ dumps(slapparameter_dict.get('use-ipv6', False)) }}
config-computer-memory-percent-threshold = {{ dumps(monitor_dict.get('computer-memory-percent-threshold', 80)) }}
{% set caucase_url = slapparameter_dict.get('caucase', {}).pop('url', '') -%}
{% set caucase_dict = slapparameter_dict.get('caucase', {}) -%}
{% set caucase_url = caucase_dict.get('url', '') -%}
{% macro request(name, software_type, config_key, config, ret={'url': True}, key_config={}) -%}
{% do config.update(slapparameter_dict.get(config_key, {})) -%}
......@@ -69,8 +71,6 @@ service-on-watch = ${buildout:directory}/etc/service
srv = ${buildout:directory}/srv
backup-caucased = ${:srv}/backup/caucased
{% set caucase_dict = slapparameter_dict.get('caucase', {}) -%}
{% set caucase_url = caucase_dict.get('url') -%}
{% if not caucase_url -%}
{% if use_ipv6 -%}
{% set caucase_host = '[' ~ (ipv6_set | list)[0] ~ ']' %}
......@@ -95,8 +95,35 @@ backup-caucased = ${:srv}/backup/caucased
{% do root_common.section('caucased-promise') -%}
{% endif -%}
{% do publish_dict.__setitem__('caucase-http-url', caucase_url) -%}
{% set balancer_dict = slapparameter_dict.get('balancer', {}) -%}
{% set balancer_dict = slapparameter_dict.setdefault('balancer', {}) -%}
{% do balancer_dict.setdefault('ssl', {}).setdefault('caucase-url', caucase_url) -%}
{% do balancer_dict.setdefault('tcpv4-port', 2150) -%}
{% do balancer_dict.__setitem__('haproxy-server-check-path', balancer_dict.get('haproxy-server-check-path', '/') % {'site-id': site_id}) -%}
{% set routing_path_template_field_dict = {"site-id": site_id} -%}
{% macro expandRoutingPath(output, input) -%}
{% for outer_prefix, inner_prefix in input -%}
{% do output.append((outer_prefix, inner_prefix % routing_path_template_field_dict)) -%}
{% endfor -%}
{% endmacro -%}
{% set path_routing_list = [] -%}
{% do expandRoutingPath(
path_routing_list,
balancer_dict.get(
'path-routing-list',
(('/', '/'), ),
),
) -%}
{% do balancer_dict.__setitem__('path-routing-list', path_routing_list) -%}
{% set family_path_routing_dict = {} -%}
{% for name, family_path_routing_list in balancer_dict.get(
'family-path-routing-dict',
{},
).items() -%}
{% set path_routing_list = [] -%}
{% do expandRoutingPath(path_routing_list, family_path_routing_list) -%}
{% do family_path_routing_dict.__setitem__(name, path_routing_list) -%}
{% endfor -%}
{% do balancer_dict.__setitem__('family-path-routing-dict', family_path_routing_dict) -%}
{{ request('memcached-persistent', 'kumofs', 'kumofs', {'tcpv4-port': 2000}, {'url': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
{{ request('memcached-volatile', 'kumofs', 'memcached', {'tcpv4-port': 2010, 'ram-storage-size': 64}, {'url': True, 'monitor-base-url': False}, key_config={'monitor-passwd': 'monitor-htpasswd:passwd'}) }}
......@@ -193,6 +220,7 @@ config-deadlock-debugger-password = ${publish-early:deadlock-debugger-password}
config-developer-list = {{ dumps(slapparameter_dict.get('developer-list', [inituser_login])) }}
config-selenium-server-configuration-dict = {{ dumps(slapparameter_dict.get('selenium-server-configuration-dict', {})) }}
config-hosts-dict = {{ dumps(slapparameter_dict.get('hosts-dict', {})) }}
config-computer-hosts-dict = {{ dumps(slapparameter_dict.get('computer-hosts-dict', {})) }}
config-hostalias-dict = {{ dumps(slapparameter_dict.get('hostalias-dict', {})) }}
config-id-store-interval = {{ dumps(slapparameter_dict.get('id-store-interval')) }}
config-zope-longrequest-logger-error-threshold = {{ dumps(monitor_dict.get('zope-longrequest-logger-error-threshold', 20)) }}
......@@ -226,6 +254,8 @@ config-tidstorage-port = ${request-zodb:connection-tidstorage-port}
{% endfor -%}
software-type = zope
{% set global_publisher_timeout = slapparameter_dict.get('publisher-timeout') -%}
{% set global_activity_timeout = slapparameter_dict.get('activity-timeout') -%}
{% set zope_family_dict = {} -%}
{% set zope_family_name_list = [] -%}
{% set zope_backend_path_dict = {} -%}
......@@ -246,6 +276,7 @@ software-type = zope
{% do zope_family_dict.setdefault(zope_family, []).append(section_name) -%}
{% do zope_backend_path_dict.__setitem__(zope_family, backend_path) -%}
{% do ssl_authentication_dict.__setitem__(zope_family, zope_parameter_dict.get('ssl-authentication', False)) -%}
{% set current_zope_family_override_dict = zope_family_override_dict.get(zope_family, {}) -%}
[{{ section_name }}]
<= request-zope-base
name = {{ partition_name }}
......@@ -260,7 +291,10 @@ config-longrequest-logger-interval = {{ dumps(zope_parameter_dict.get('longreque
config-longrequest-logger-timeout = {{ dumps(zope_parameter_dict.get('longrequest-logger-timeout', 1)) }}
config-large-file-threshold = {{ dumps(zope_parameter_dict.get('large-file-threshold', "10MB")) }}
config-port-base = {{ dumps(zope_parameter_dict.get('port-base', 2200)) }}
config-webdav = {{ dumps(zope_parameter_dict.get('webdav', False)) }}
{# BBB: zope_parameter_dict used to contain 'webdav', so fallback to it -#}
config-webdav = {{ dumps(current_zope_family_override_dict.get('webdav', zope_parameter_dict.get('webdav', False))) }}
config-publisher-timeout = {{ dumps(current_zope_family_override_dict.get('publisher-timeout', global_publisher_timeout)) }}
config-activity-timeout = {{ dumps(current_zope_family_override_dict.get('activity-timeout', global_activity_timeout)) }}
{% if test_runner_enabled -%}
config-test-runner-apache-url-list = ${publish-early:{{ zope_family }}-test-runner-url-list}
......@@ -319,40 +353,39 @@ config-url = ${request-jupyter:connection-url}
{% endif -%}
{%- endif %}
[request-balancer]
<= request-common
name = balancer
software-type = balancer
{{ root_common.sla('balancer') }}
return =
monitor-base-url
{%- for family in zope_family_dict %}
{{ family }}
{{ family }}-v6
{% if test_runner_enabled %}
{{ family }}-test-runner-url-list
{% endif %}
{% set balancer_ret_dict = {'monitor-base-url': False} -%}
{% for family in zope_family_dict -%}
{% do balancer_ret_dict.__setitem__(family, False) -%}
{% do balancer_ret_dict.__setitem__(family + '-v6', False) -%}
{% if test_runner_enabled -%}
{% do balancer_ret_dict.__setitem__(family + '-test-runner-url-list', False) -%}
{% endif -%}
{% endfor -%}
{% do monitor_base_url_dict.__setitem__('request-balancer', '${' ~ 'request-balancer' ~ ':connection-monitor-base-url}') -%}
config-zope-family-dict = {{ dumps(zope_family_parameter_dict) }}
config-tcpv4-port = {{ dumps(balancer_dict.get('tcpv4-port', 2150)) }}
{% set balancer_key_config_dict = {
'monitor-passwd': 'monitor-htpasswd:passwd',
} -%}
{% for zope_section_id, name in zope_address_list_id_dict.items() -%}
config-{{ name }} = {{ ' ${' ~ zope_section_id ~ ':connection-zope-address-list}' }}
{% do balancer_key_config_dict.__setitem__(
name,
zope_section_id + ':connection-zope-address-list',
) -%}
{% if test_runner_enabled -%}
config-{{ name }}-test-runner-address-list = {{ ' ${' ~ zope_section_id ~ ':connection-test-runner-address-list}' }}
{% do balancer_key_config_dict.__setitem__(
name + '-test-runner-address-list',
zope_section_id + ':connection-test-runner-address-list',
) -%}
{% endif -%}
{% endfor -%}
# XXX: should those really be same for all families ?
config-haproxy-server-check-path = {{ dumps(balancer_dict.get('haproxy-server-check-path', '/') % {'site-id': site_id}) }}
config-monitor-passwd = ${monitor-htpasswd:passwd}
config-ssl = {{ dumps(balancer_dict['ssl']) }}
config-name = ${:name}
config-backend-path-dict = {{ dumps(zope_backend_path_dict) }}
config-ssl-authentication-dict = {{ dumps(ssl_authentication_dict) }}
config-apachedex-promise-threshold = {{ dumps(monitor_dict.get('apachedex-promise-threshold', 70)) }}
config-apachedex-configuration = {{
dumps(
monitor_dict.get(
{{ request(
name='balancer',
software_type='balancer',
config_key='balancer',
config={
'zope-family-dict': zope_family_parameter_dict,
'backend-path-dict': zope_backend_path_dict,
'ssl-authentication-dict': ssl_authentication_dict,
'apachedex-promise-threshold': monitor_dict.get('apachedex-promise-threshold', 70),
'apachedex-configuration': monitor_dict.get(
'apachedex-configuration',
[
'--logformat', '%h %l %u %t "%r" %>s %O "%{Referer}i" "%{User-Agent}i" %{ms}T',
......@@ -361,7 +394,13 @@ config-apachedex-configuration = {{
'--skip-user-agent', 'Zabbix',
'--error-detail',
'--js-embed',
'--quiet'])) }}
'--quiet',
],
),
},
ret=balancer_ret_dict,
key_config=balancer_key_config_dict,
) }}
[request-frontend-base]
{% if has_frontend -%}
......@@ -407,7 +446,7 @@ hosts-dict = {{ '${' ~ zope_address_list_id_dict.keys()[0] ~ ':connection-hosts-
{% endfor -%}
{% if test_runner_enabled -%}
{% for zope_family_name in zope_family_name_list -%}
{{ zope_family_name }}-test-runner-url-list = ${request-balancer:connection-{{ zope_family_name }}-test-runner-url-list}
{{ zope_family_name }}-test-runner-url-list = ${request-balancer:connection-{{ zope_family_name }}-test-runner-url-list}
{% endfor -%}
{% endif -%}
......
......@@ -22,7 +22,7 @@ partition. No more (undefined result), no less (IndexError).
-#}
{% set ipv4 = (ipv4_set | list)[0] -%}
{% set hosts_dict = {} -%}
{% set publishable_hosts_dict = {} -%}
{% set port_dict = {} -%}
{% for alias, url in (
('erp5-memcached-volatile', slapparameter_dict['memcached-url']),
......@@ -32,15 +32,17 @@ partition. No more (undefined result), no less (IndexError).
) -%}
{% set parsed_url = urlparse.urlparse(url) -%}
{% do port_dict.__setitem__(alias, parsed_url.port) -%}
{% do hosts_dict.__setitem__(alias, parsed_url.hostname) -%}
{% do publishable_hosts_dict.__setitem__(alias, parsed_url.hostname) -%}
{%- endfor %}
{% for i, url in enumerate(slapparameter_dict['mysql-url-list']) -%}
{% do hosts_dict.__setitem__(
{% do publishable_hosts_dict.__setitem__(
'erp5-catalog-' ~ i,
urlparse.urlparse(url).hostname,
) -%}
{%- endfor %}
{% do hosts_dict.update(slapparameter_dict['hosts-dict']) -%}
{% do publishable_hosts_dict.update(slapparameter_dict['hosts-dict']) -%}
{% set hosts_dict = publishable_hosts_dict.copy() -%}
{% do hosts_dict.update(slapparameter_dict['computer-hosts-dict'].get(computer, {})) -%}
[jinja2-template-base]
recipe = slapos.recipe.template:jinja2
......@@ -274,6 +276,8 @@ cloudooo-url = {{ (cloudooo if cloudooo.port == None else
{% endif -%}
{% endfor -%}
developer-list = {{ dumps(slapparameter_dict['developer-list']) }}
publisher-timeout = {{ dumps(slapparameter_dict['publisher-timeout']) }}
activity-timeout = {{ dumps(slapparameter_dict['activity-timeout']) }}
instance = ${directory:instance}
instance-products = ${directory:instance-products}
deadlock-path = /manage_debug_threads
......@@ -553,7 +557,7 @@ This has the downside of making each zope partition publish the (hopefuly) same
dict toward erp5 partition, violating the DRY principle and making the intent
hard to guess.
-#}
hosts-dict = {{ dumps(hosts_dict) }}
hosts-dict = {{ dumps(publishable_hosts_dict) }}
monitor-base-url = ${monitor-publish-parameters:monitor-base-url}
test-runner-address-list = {{ dumps(test_runner_address_list) }}
software-release-url = ${slap-connection:software-release-url}
......
......@@ -140,10 +140,14 @@ large-file-threshold {{ parameter_dict['large-file-threshold'] }}
</zodb_db>
{% set developer_list = parameter_dict['developer-list'] -%}
{% if developer_list -%}
{% set publisher_timeout = parameter_dict['publisher-timeout'] -%}
{% set activity_timeout = parameter_dict['activity-timeout'] -%}
{% if developer_list or publisher_timeout or activity_timeout -%}
%import Products.ERP5Type
<ERP5Type erp5>
developers {{ developer_list | join(' ') }}
{% if developer_list %}developers {{ developer_list | join(' ') }}{% endif %}
{% if publisher_timeout %}publisher-timeout {{ publisher_timeout }}{% endif %}
{% if activity_timeout %}activity-timeout {{ activity_timeout }}{% endif %}
</ERP5Type>
{% endif -%}
......
[buildout]
extends =
../component/nodejs/buildout.cfg
../component/yarn/buildout.cfg
../stack/slapos.cfg
parts =
......
......@@ -200,7 +200,7 @@ setuptools-dso = 1.7
rubygemsrecipe = 0.3.0
six = 1.12.0
slapos.cookbook = 1.0.171
slapos.core = 1.6.5
slapos.core = 1.6.6
slapos.extension.strip = 0.4
slapos.extension.shared = 1.0
slapos.libnetworkcache = 0.20
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment