Commit 7ff43824 authored by Jérome Perrin's avatar Jérome Perrin

ERP5: Move frontend virtualhost logic on backend

 - use caucase for balancer certificate
 - move virtual host logic on the backend
 - change "frontend" parameter to request "" type (and no longer "zope")

See merge request nexedi/slapos!1504
parents cb78214e 6e735808
From 8e7c9a6a86104e306aee2224ff5e517ee201b28f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=A9rome=20Perrin?= <jerome@nexedi.com>
Date: Tue, 9 Jan 2024 17:15:11 +0900
Subject: [PATCH] Fix redirections to URLS with host given as IP-litteral
(#1192)
When redirecting to an URL with an IPv6 host with surrounding brackets,
we should not escape the surrounding brackets.
The patch updates referenced RFC from 2396 to 3986, which obsoletes it
and change the safe characters for the netloc part to allow [ and ].
The RFC specifies that [ and ] are only allowed when they are the first
and last characters, but we don't need to be more specific here, because
using [ or ] in other places of the host is rejected by urlparse above.
Fixes #1191
---
src/ZPublisher/HTTPResponse.py | 14 +++++++-------
src/ZPublisher/tests/testHTTPResponse.py | 8 ++++++--
2 files changed, 13 insertions(+), 9 deletions(-)
diff --git a/src/ZPublisher/HTTPResponse.py b/src/ZPublisher/HTTPResponse.py
index b0b4ca2b1..b1a824151 100644
--- a/src/ZPublisher/HTTPResponse.py
+++ b/src/ZPublisher/HTTPResponse.py
@@ -230,24 +230,24 @@ class HTTPBaseResponse(BaseResponse):
# To be entirely correct, we must make sure that all non-ASCII
# characters are quoted correctly.
parsed = list(urlparse(location))
- rfc2396_unreserved = "-_.!~*'()" # RFC 2396 section 2.3
+ rfc3986_unreserved = "-_.!~*'()" # RFC 3986 section 2.3
for idx, idx_safe in (
# authority
- (1, ";:@?/&=+$,"), # RFC 2396 section 3.2, 3.2.1, 3.2.3
+ (1, "[];:@?/&=+$,"), # RFC 3986 section 3.2, 3.2.1, 3.2.3
# path
- (2, "/;:@&=+$,"), # RFC 2396 section 3.3
+ (2, "/;:@&=+$,"), # RFC 3986 section 3.3
# params - actually part of path; empty in Python 3
- (3, "/;:@&=+$,"), # RFC 2396 section 3.3
+ (3, "/;:@&=+$,"), # RFC 3986 section 3.3
# query
- (4, ";/?:@&=+,$"), # RFC 2396 section 3.4
+ (4, ";/?:@&=+,$"), # RFC 3986 section 3.4
# fragment
- (5, ";/?:@&=+$,"), # RFC 2396 section 4
+ (5, ";/?:@&=+$,"), # RFC 3986 section 4
):
# Make a hacky guess whether the component is already
# URL-encoded by checking for %. If it is, we don't touch it.
if '%' not in parsed[idx]:
parsed[idx] = quote(parsed[idx],
- safe=rfc2396_unreserved + idx_safe)
+ safe=rfc3986_unreserved + idx_safe)
location = urlunparse(parsed)
self.setStatus(status, lock=lock)
diff --git a/src/ZPublisher/tests/testHTTPResponse.py b/src/ZPublisher/tests/testHTTPResponse.py
index a7f816c04..08a1674ba 100644
--- a/src/ZPublisher/tests/testHTTPResponse.py
+++ b/src/ZPublisher/tests/testHTTPResponse.py
@@ -767,15 +767,19 @@ class HTTPResponseTests(unittest.TestCase):
self._redirectURLCheck(ENC_URL)
def test_redirect_unreserved_chars(self):
- # RFC 2396 section 2.3, characters that should not be encoded
+ # RFC 3986 section 2.3, characters that should not be encoded
url = "http://example.com/-_.!~*'()"
self._redirectURLCheck(url)
def test_redirect_reserved_chars(self):
- # RFC 2396 section 3.3, characters with reserved meaning in a path
+ # RFC 3986 section 3.3, characters with reserved meaning in a path
url = 'http://example.com/+/$/;/,/=/?/&/@@index.html'
self._redirectURLCheck(url)
+ def test_redirect_ipv6(self):
+ url = "http://[fe80::1ff:fe23:4567:890a]:1234"
+ self._redirectURLCheck(url)
+
def test__encode_unicode_no_content_type_uses_default_encoding(self):
UNICODE = u'<h1>Tr\u0039s Bien</h1>'
response = self._makeOne()
--
2.42.0
...@@ -90,7 +90,6 @@ This software release assigns the following port ranges by default: ...@@ -90,7 +90,6 @@ This software release assigns the following port ranges by default:
balancer 2150-2199 balancer 2150-2199
zope 2200-* zope 2200-*
jupyter 8888 jupyter 8888
caucase 8890,8891
==================== ========== ==================== ==========
Non-zope partitions are unique in an ERP5 cluster, so you shouldn't have to Non-zope partitions are unique in an ERP5 cluster, so you shouldn't have to
......
...@@ -5,7 +5,7 @@ ...@@ -5,7 +5,7 @@
"additionalProperties": false, "additionalProperties": false,
"definitions": { "definitions": {
"routing-rule-list": { "routing-rule-list": {
"description": "Maps the path received in requests to given zope path. Rules are applied in the order they are given. This requires the path received from the outside world (typically: frontend) to have its root correspond to Zope's root (for frontend: 'path' parameter must be empty), with the customary VirtualHostMonster construct (for frontend: 'type' must be 'zope').", "description": "Maps the path received in requests to given zope path. Rules are applied in the order they are given, after 'internal-path' from 'frontend' parameter. This also supports legacy frontends, using Rapid CDN with \"zope\" type.",
"type": "array", "type": "array",
"default": [ "default": [
[ [
...@@ -111,7 +111,7 @@ ...@@ -111,7 +111,7 @@
"description": "Family-wide options, possibly overriding global options", "description": "Family-wide options, possibly overriding global options",
"default": {}, "default": {},
"patternProperties": { "patternProperties": {
".*": { "^[a-zA-Z0-9_-]+$": {
"default": {}, "default": {},
"properties": { "properties": {
"webdav": { "webdav": {
...@@ -164,7 +164,7 @@ ...@@ -164,7 +164,7 @@
"default": {} "default": {}
}, },
"patternProperties": { "patternProperties": {
".*": { "^[a-zA-Z0-9_-]+$": {
"required": [ "required": [
"zope-family" "zope-family"
], ],
...@@ -204,13 +204,14 @@ ...@@ -204,13 +204,14 @@
"1": {} "1": {}
}, },
"patternProperties": { "patternProperties": {
".*": { "^[a-zA-Z0-9_-]+$": {
"additionalProperties": false, "additionalProperties": false,
"properties": { "properties": {
"family": { "family": {
"description": "The family this partition is part of. For example: 'public', 'admin', 'backoffice', 'web-service'... Each family gets its own balancer entry. It has no special meaning for the system.", "description": "The family this partition is part of. For example: 'public', 'admin', 'backoffice', 'web-service'... Each family gets its own balancer entry. It has no special meaning for the system.",
"default": "default", "default": "default",
"type": "string" "type": "string",
"pattern": "^[a-zA-Z0-9_-]+$"
}, },
"instance-count": { "instance-count": {
"description": "Number of Zopes to setup on this partition", "description": "Number of Zopes to setup on this partition",
...@@ -513,7 +514,7 @@ ...@@ -513,7 +514,7 @@
"properties": { "properties": {
"url": { "url": {
"title": "Caucase URL", "title": "Caucase URL",
"description": "URL of existing caucase instance to use. If empty, a new caucase instance will be deployed. If not empty, other properties in this section will be ignored.", "description": "URL of existing caucase instance to use. If empty, caucase instances will be deployed inside partitions.",
"default": "", "default": "",
"type": "string", "type": "string",
"format": "uri" "format": "uri"
...@@ -720,15 +721,9 @@ ...@@ -720,15 +721,9 @@
}, },
"uniqueItems": true "uniqueItems": true
}, },
"caucase-url": {
"title": "Caucase URL",
"description": "URL of caucase service to use. If not set, global setting will be used.",
"type": "string",
"format": "uri"
},
"csr": { "csr": {
"title": "csr", "title": "csr",
"description": "PEM-encoded certificate signature request to request server certificate with. If not provided, HTTPS will be disabled.", "description": "PEM-encoded certificate signature request to request server certificate with.",
"type": "string" "type": "string"
}, },
"max-crl-update-delay": { "max-crl-update-delay": {
......
...@@ -69,7 +69,7 @@ ...@@ -69,7 +69,7 @@
"type": "string" "type": "string"
}, },
"caucase-http-url": { "caucase-http-url": {
"description": "Caucase url on HTTP. For HTTPS URL, uses https scheme, if port is explicitely specified in http URL, take that port and add 1 and use it as https port. If it is not specified.", "description": "Caucase url on HTTP. For HTTPS URL, uses https scheme, if port is explicitely specified in http URL, take that port and add 1 and use it as https port.",
"pattern": "^http://", "pattern": "^http://",
"type": "string" "type": "string"
} }
......
...@@ -314,7 +314,7 @@ class CaucaseCertificate(ManagedResource): ...@@ -314,7 +314,7 @@ class CaucaseCertificate(ManagedResource):
) )
return os.path.join(software_release_root_path, 'bin', 'caucase') return os.path.join(software_release_root_path, 'bin', 'caucase')
def request(self, common_name: str, caucase: CaucaseService) -> None: def request(self, common_name: str, caucase: CaucaseService, san: x509.SubjectAlternativeName=None) -> None:
"""Generate certificate and request signature to the caucase service. """Generate certificate and request signature to the caucase service.
This overwrite any previously requested certificate for this instance. This overwrite any previously requested certificate for this instance.
...@@ -345,11 +345,10 @@ class CaucaseCertificate(ManagedResource): ...@@ -345,11 +345,10 @@ class CaucaseCertificate(ManagedResource):
NameOID.COMMON_NAME, NameOID.COMMON_NAME,
common_name, common_name,
), ),
])).sign( ]))
key, if san:
hashes.SHA256(), csr = csr.add_extension(san, critical=True)
default_backend(), csr = csr.sign(key, hashes.SHA256(), default_backend())
)
with open(self.csr_file, 'wb') as f: with open(self.csr_file, 'wb') as f:
f.write(csr.public_bytes(serialization.Encoding.PEM)) f.write(csr.public_bytes(serialization.Encoding.PEM))
......
This diff is collapsed.
This diff is collapsed.
...@@ -14,7 +14,7 @@ ...@@ -14,7 +14,7 @@
# not need these here). # not need these here).
[template-erp5] [template-erp5]
filename = instance-erp5.cfg.in filename = instance-erp5.cfg.in
md5sum = ba46a66da1c834df14a80a20b21e4a96 md5sum = 6db19ee819a960a34012308e29c5bbfb
[template-balancer] [template-balancer]
filename = instance-balancer.cfg.in filename = instance-balancer.cfg.in
......
...@@ -435,6 +435,7 @@ return = ...@@ -435,6 +435,7 @@ return =
{% endfor -%} {% endfor -%}
{% do monitor_base_url_dict.__setitem__('request-balancer', '${' ~ 'request-balancer' ~ ':connection-monitor-base-url}') -%} {% do monitor_base_url_dict.__setitem__('request-balancer', '${' ~ 'request-balancer' ~ ':connection-monitor-base-url}') -%}
config-zope-family-dict = {{ dumps(zope_family_parameter_dict) }} config-zope-family-dict = {{ dumps(zope_family_parameter_dict) }}
config-frontend-parameter-dict = {{ dumps({}) }}
config-tcpv4-port = {{ dumps(balancer_dict.get('tcpv4-port', 2150)) }} config-tcpv4-port = {{ dumps(balancer_dict.get('tcpv4-port', 2150)) }}
{% for zope_section_id, name in zope_address_list_id_dict.items() -%} {% for zope_section_id, name in zope_address_list_id_dict.items() -%}
config-{{ name }} = {{ ' ${' ~ zope_section_id ~ ':connection-zope-address-list}' }} config-{{ name }} = {{ ' ${' ~ zope_section_id ~ ':connection-zope-address-list}' }}
......
...@@ -716,6 +716,7 @@ waitress-patches = ...@@ -716,6 +716,7 @@ waitress-patches =
waitress-patch-options = -p1 waitress-patch-options = -p1
Zope-patches = Zope-patches =
${:_profile_base_location_}/../../component/egg-patch/Zope/0001-WSGIPublisher-set-REMOTE_USER-even-in-case-of-error-.patch#a437f4da28975f94dd07db0b02954111 ${:_profile_base_location_}/../../component/egg-patch/Zope/0001-WSGIPublisher-set-REMOTE_USER-even-in-case-of-error-.patch#a437f4da28975f94dd07db0b02954111
${:_profile_base_location_}/../../component/egg-patch/Zope/0001-Fix-redirections-to-URLS-with-host-given-as-IP-litte.patch#093ad5755094d537c6a4deadc959ade0
Zope-patch-options = -p1 Zope-patch-options = -p1
# neoppod installs bin/coverage so we inject erp5 plugin here so that coverage script can use it in report # neoppod installs bin/coverage so we inject erp5 plugin here so that coverage script can use it in report
...@@ -757,7 +758,7 @@ pysvn = 1.9.15+SlapOSPatched001 ...@@ -757,7 +758,7 @@ pysvn = 1.9.15+SlapOSPatched001
python-ldap = 2.4.32+SlapOSPatched001 python-ldap = 2.4.32+SlapOSPatched001
python-magic = 0.4.12+SlapOSPatched001 python-magic = 0.4.12+SlapOSPatched001
waitress = 1.4.4+SlapOSPatched006 waitress = 1.4.4+SlapOSPatched006
Zope = 4.8.9+SlapOSPatched001 Zope = 4.8.9+SlapOSPatched002
## https://lab.nexedi.com/nexedi/slapos/merge_requests/648 ## https://lab.nexedi.com/nexedi/slapos/merge_requests/648
pylint = 1.4.4+SlapOSPatched002 pylint = 1.4.4+SlapOSPatched002
# astroid 1.4.1 breaks testDynamicClassGeneration # astroid 1.4.1 breaks testDynamicClassGeneration
......
...@@ -70,11 +70,11 @@ md5sum = b95084ae9eed95a68eada45e28ef0c04 ...@@ -70,11 +70,11 @@ md5sum = b95084ae9eed95a68eada45e28ef0c04
[template] [template]
filename = instance.cfg.in filename = instance.cfg.in
md5sum = 5e0e9565227fe190c420a7bbcd0f7b93 md5sum = 55463b0abdbe0118ef1c27e6b71c3324
[template-erp5] [template-erp5]
filename = instance-erp5.cfg.in filename = instance-erp5.cfg.in
md5sum = 46c5d2173ab91f56b32e6eb2b544bc26 md5sum = ae9c380ae04dde4f20e139c66ef7c22a
[template-zeo] [template-zeo]
filename = instance-zeo.cfg.in filename = instance-zeo.cfg.in
...@@ -86,15 +86,15 @@ md5sum = 0ac4b74436f554cd677f19275d18d880 ...@@ -86,15 +86,15 @@ md5sum = 0ac4b74436f554cd677f19275d18d880
[template-zope] [template-zope]
filename = instance-zope.cfg.in filename = instance-zope.cfg.in
md5sum = 41709f47e5a9051ca4a9c943859f589b md5sum = 6178ba7b42848f9e2412ab898a7b026c
[template-balancer] [template-balancer]
filename = instance-balancer.cfg.in filename = instance-balancer.cfg.in
md5sum = b0751d3d12cfcc8934cb1027190f5e5e md5sum = 0fad9497da12ed0186dca5236c23f3a7
[template-haproxy-cfg] [template-haproxy-cfg]
filename = haproxy.cfg.in filename = haproxy.cfg.in
md5sum = 85a8c0dadf7b648ef9748b6199dcfeb6 md5sum = 2cd76971b64b0bf7771978ad07bfc2e5
[template-rsyslogd-cfg] [template-rsyslogd-cfg]
filename = rsyslogd.cfg.in filename = rsyslogd.cfg.in
......
{# This file configures haproxy to redirect requests from ports to specific urls. {# This file configures haproxy to redirect requests from ports to specific urls.
# It provides TLS support for server and optionnaly for client. # It provides TLS support for server and optionnaly for client.
# #
...@@ -19,11 +18,11 @@ ...@@ -19,11 +18,11 @@
# "stats-socket": "<file_path>", # "stats-socket": "<file_path>",
# #
# # IPv4 to listen on # # IPv4 to listen on
# # All backends from `backend-dict` will listen on this IP. # # All frontends from `frontend-dict` will listen on this IP.
# "ipv4": "0.0.0.0", # "ipv4": "0.0.0.0",
# #
# # IPv6 to listen on # # IPv6 to listen on
# # All backends from `backend-dict` will listen on this IP. # # All frontends from `frontend-dict` will listen on this IP.
# "ipv6": "::1", # "ipv6": "::1",
# #
# # Certificate and key in PEM format. All ports will serve TLS using # # Certificate and key in PEM format. All ports will serve TLS using
...@@ -42,36 +41,59 @@ ...@@ -42,36 +41,59 @@
# # Path to use for HTTP health check on backends from `backend-dict`. # # Path to use for HTTP health check on backends from `backend-dict`.
# "server-check-path": "/", # "server-check-path": "/",
# #
# # The mapping of frontend, keyed by frontend name
# "frontend-dict": {
# "frontend-default": {
# "port": 8080,
# "client-cert-required": False,
# "backend-name": "family-default",
# "request-path-prepend": "/erp5",
# }
# "legacy-frontend-family-secure": {
# "port": 8000,
# "client-cert-required": False,
# "backend-name": "family-secure",
# "request-path-prepend": None, # None means do not rewrite the request path
# }
# "legacy-frontend-family-default": {
# "port": 8002,
# "client-cert-required": False,
# "backend-name": "family-default",
# "request-path-prepend": None, # None means do not rewrite the request path
# }
# }
# # The mapping of backends, keyed by family name # # The mapping of backends, keyed by family name
# "backend-dict": { # "backend-dict": {
# "family-secure": { # "family-secure": {
# ( 8000, # port int # "timeout": None, # in seconds
# 'https', # proto str # "backend-list": [
# True, # ssl_required bool # [
# None, # timeout (in seconds) int | None # '10.0.0.10:8001', # netloc str
# [ # backends # 1, # max_connection_count int
# '10.0.0.10:8001', # netloc str # False, # is_web_dav bool
# 1, # max_connection_count int # ]
# False, # is_web_dav bool # ]
# ], # },
# ), # "family-default": {
# }, # "timeout": None, # in seconds
# "family-default": { # "backend-list": [
# ( 8002, # port int # [
# 'https', # proto str # '10.0.0.10:8003', # netloc str
# False, # ssl_required bool # 1, # max_connection_count int
# None, # timeout (in seconds) int | None # False, # is_web_dav bool
# [ # backends # ],
# '10.0.0.10:8003', # netloc str # [
# 1, # max_connection_count int # '10.0.0.10:8004', # netloc str
# False, # is_web_dav bool # 1, # max_connection_count int
# ], # False, # is_web_dav bool
# ), # ],
# ]
# }, # },
# #
# # The mapping of zope paths. # # The mapping of zope paths.
# # This is a Zope specific feature. # # This is a Zope specific feature used only to provide https while running
# # `enable_authentication` has same meaning as for `backend-list`. # # ERP5 "unit test" suite.
# # `enable_authentication` has same meaning as for `backend-dict`.
# "zope-virtualhost-monster-backend-dict": { # "zope-virtualhost-monster-backend-dict": {
# # {(ip, port): ( enable_authentication, {frontend_path: ( internal_url ) }, ) } # # {(ip, port): ( enable_authentication, {frontend_path: ( internal_url ) }, ) }
# ('[::1]', 8004): ( # ('[::1]', 8004): (
...@@ -84,15 +106,20 @@ ...@@ -84,15 +106,20 @@
# } # }
# #
# This sample of `parameter_dict` will make haproxy listening to : # This sample of `parameter_dict` will make haproxy listening to :
# From to `backend-list`: # For "frontend-default":
# For "family-secure": # - 0.0.0.0:8080 redirecting internaly to http://10.0.0.10:8003 or http://10.0.0.10:8004
# - [::1]:8080 redirecting internaly to http://10.0.0.10:8003 or http://10.0.0.10:8004
# accepting requests from any client and rewriting the path to add a Zope rewrite rule
# so that the a request on https://0.0.0.0:8080/path is rewritten to serve a Zope object at
# path /erp5/path , visible as /path.
# For "legacy-frontend-family-secure":
# - 0.0.0.0:8000 redirecting internaly to http://10.0.0.10:8001 and # - 0.0.0.0:8000 redirecting internaly to http://10.0.0.10:8001 and
# - [::1]:8000 redirecting internaly to http://10.0.0.10:8001 # - [::1]:8000 redirecting internaly to http://10.0.0.10:8001
# only accepting requests from clients providing a verified TLS certificate # only accepting requests from clients providing a verified TLS certificate
# emitted by a CA from `ca-cert` and not revoked in `crl`. # emitted by a CA from `ca-cert` and not revoked in `crl`.
# For "family-default": # For "legacy-frontend-family-default":
# - 0.0.0.0:8002 redirecting internaly to http://10.0.0.10:8003 # - 0.0.0.0:8002 redirecting internaly to http://10.0.0.10:8003 or http://10.0.0.10:8004
# - [::1]:8002 redirecting internaly to http://10.0.0.10:8003 # - [::1]:8002 redirecting internaly to http://10.0.0.10:8003 or http://10.0.0.10:8004
# accepting requests from any client. # accepting requests from any client.
# #
# For both families, X-Forwarded-For header will be stripped unless # For both families, X-Forwarded-For header will be stripped unless
...@@ -105,7 +132,7 @@ ...@@ -105,7 +132,7 @@
# with some VirtualHostMonster rewrite rules so zope writes URLs with # with some VirtualHostMonster rewrite rules so zope writes URLs with
# [::1]:8004 as server name. # [::1]:8004 as server name.
# For more details, refer to # For more details, refer to
# https://docs.zope.org/zope2/zope2book/VirtualHosting.html#using-virtualhostroot-and-virtualhostbase-together # https://zope.readthedocs.io/en/latest/zopebook/VirtualHosting.html#using-virtualhostroot-and-virtualhostbase-together
-#} -#}
{% set server_check_path = parameter_dict['server-check-path'] -%} {% set server_check_path = parameter_dict['server-check-path'] -%}
...@@ -151,26 +178,17 @@ defaults ...@@ -151,26 +178,17 @@ defaults
{% set family_path_routing_dict = parameter_dict['family-path-routing-dict'] %} {% set family_path_routing_dict = parameter_dict['family-path-routing-dict'] %}
{% set path_routing_list = parameter_dict['path-routing-list'] %} {% set path_routing_list = parameter_dict['path-routing-list'] %}
{% for name, (port, _, certificate_authentication, timeout, backend_list) in sorted(six.iteritems(parameter_dict['backend-dict'])) -%}
listen family_{{ name }} {% for name, frontend in sorted(six.iteritems(parameter_dict['frontend-dict'])) %}
listen {{ name }}
{%- if parameter_dict.get('ca-cert') -%} {%- if parameter_dict.get('ca-cert') -%}
{%- set ssl_auth = ' ca-file ' ~ parameter_dict['ca-cert'] ~ ' verify' ~ ( ' required' if certificate_authentication else ' optional crt-ignore-err all' ) ~ ' crl-file ' ~ parameter_dict['crl'] %} {%- set ssl_auth = ' ca-file ' ~ parameter_dict['ca-cert'] ~ ' verify' ~ ( ' required' if frontend['client-cert-required'] else ' optional crt-ignore-err all' ) ~ ' crl-file ' ~ parameter_dict['crl'] %}
{%- else %} {%- else %}
{%- set ssl_auth = '' %} {%- set ssl_auth = '' %}
{%- endif %} {%- endif %}
bind {{ parameter_dict['ipv4'] }}:{{ port }} {{ bind_ssl_crt }} {{ ssl_auth }} bind {{ parameter_dict['ipv4'] }}:{{ frontend['port'] }} {{ bind_ssl_crt }} {{ ssl_auth }}
bind {{ parameter_dict['ipv6'] }}:{{ port }} {{ bind_ssl_crt }} {{ ssl_auth }} bind {{ parameter_dict['ipv6'] }}:{{ frontend['port'] }} {{ bind_ssl_crt }} {{ ssl_auth }}
cookie SERVERID rewrite
http-request set-header X-Balancer-Current-Cookie SERVERID
{% if timeout %}
{#
Apply a slightly longer timeout than the zope timeout so that clients can see the
TimeoutReachedError from zope, that is a bit more informative than the 504 error
page from haproxy.
#}
timeout server {{ timeout + 3 }}s
{%- endif %}
# remove X-Forwarded-For unless client presented a verified certificate # remove X-Forwarded-For unless client presented a verified certificate
http-request del-header X-Forwarded-For unless { ssl_c_verify 0 } { ssl_c_used 1 } http-request del-header X-Forwarded-For unless { ssl_c_verify 0 } { ssl_c_used 1 }
...@@ -178,18 +196,43 @@ listen family_{{ name }} ...@@ -178,18 +196,43 @@ listen family_{{ name }}
http-request del-header Remote-User http-request del-header Remote-User
http-request set-header Remote-User %{+Q}[ssl_c_s_dn(cn)] if { ssl_c_verify 0 } { ssl_c_used 1 } http-request set-header Remote-User %{+Q}[ssl_c_s_dn(cn)] if { ssl_c_verify 0 } { ssl_c_used 1 }
# reject invalid host header before using it in path
http-request deny deny_status 400 if { req.hdr(host) -m sub / }
# logs # logs
capture request header Referer len 512 capture request header Referer len 512
capture request header User-Agent len 512 capture request header User-Agent len 512
log-format "%{+Q}o %{-Q}ci - - [%trg] %r %ST %B %{+Q}[capture.req.hdr(0)] %{+Q}[capture.req.hdr(1)] %Ta" log-format "%{+Q}o %{-Q}ci - - [%trg] %r %ST %B %{+Q}[capture.req.hdr(0)] %{+Q}[capture.req.hdr(1)] %Ta"
{% for outer_prefix, inner_prefix in family_path_routing_dict.get(name, []) + path_routing_list %} {% if frontend['request-path-prepend'] is not none %}
http-request replace-path ^/(.*) /VirtualHostBase/https/%[req.hdr(Host)]{{ frontend['request-path-prepend'] }}/VirtualHostRoot/\1
{% endif %}
{% for outer_prefix, inner_prefix in family_path_routing_dict.get(frontend['backend-name'], []) + path_routing_list %}
{% set outer_prefix = outer_prefix.strip('/') -%} {% set outer_prefix = outer_prefix.strip('/') -%}
http-request replace-path ^(/+VirtualHostBase/+[^/]+/+[^/]+)/+VirtualHostRoot/+{% if outer_prefix %}{{ outer_prefix }}($|/.*){% else %}(.*){% endif %} \1/{{ inner_prefix.strip('/') }}/VirtualHostRoot/{% if outer_prefix %}_vh_{{ outer_prefix.replace('/', '/_vh_') }}{% endif %}\2 http-request replace-path ^(/+VirtualHostBase/+[^/]+/+[^/]+)/+VirtualHostRoot/+{% if outer_prefix %}{{ outer_prefix }}($|/.*){% else %}(.*){% endif %} \1/{{ inner_prefix.strip('/') }}/VirtualHostRoot/{% if outer_prefix %}_vh_{{ outer_prefix.replace('/', '/_vh_') }}{% endif %}\2
{% endfor %} {% endfor %}
use_backend {{ frontend['backend-name'] }}
{% endfor %}
{% for name, backend in sorted(six.iteritems(parameter_dict['backend-dict'])) %}
backend {{ name }}
cookie SERVERID rewrite
http-request set-header X-Balancer-Current-Cookie SERVERID
{% if backend['timeout'] %}
{#
Apply a slightly longer timeout than the zope timeout so that clients can see the
TimeoutReachedError from zope, that is a bit more informative than the 504 error
page from haproxy.
#}
timeout server {{ backend['timeout'] + 3 }}s
{%- endif %}
{% set has_webdav = [] -%} {% set has_webdav = [] -%}
{% for address, connection_count, webdav in backend_list -%} {% for address, connection_count, webdav in backend['backend-list'] -%}
{% if webdav %}{% do has_webdav.append(None) %}{% endif -%} {% if webdav %}{% do has_webdav.append(None) %}{% endif -%}
{% set server_name = name ~ '-' ~ loop.index0 %} {% set server_name = name ~ '-' ~ loop.index0 %}
server {{ server_name }} {{ address }} cookie {{ server_name }} check inter 3s rise 1 fall 2 maxqueue 5 maxconn {{ connection_count }} server {{ server_name }} {{ address }} cookie {{ server_name }} check inter 3s rise 1 fall 2 maxqueue 5 maxconn {{ connection_count }}
......
This diff is collapsed.
...@@ -79,35 +79,30 @@ bin = ${buildout:directory}/bin ...@@ -79,35 +79,30 @@ bin = ${buildout:directory}/bin
service-on-watch = ${buildout:directory}/etc/service service-on-watch = ${buildout:directory}/etc/service
srv = ${buildout:directory}/srv srv = ${buildout:directory}/srv
tmp = ${buildout:directory}/tmp tmp = ${buildout:directory}/tmp
backup-caucased = ${:srv}/backup/caucased
{% if not caucase_url -%}
{% if use_ipv6 -%}
{% set caucase_host = '[' ~ (ipv6_set | list)[0] ~ ']' %}
{%- else -%}
{% set caucase_host = (ipv4_set | list)[0] %}
{%- endif %}
{% set caucase_port = caucase_dict.get('base-port', 8890) -%}
{% set caucase_netloc = caucase_host ~ ':' ~ caucase_port -%}
{% set caucase_url = 'http://' ~ caucase_netloc -%}
{{ caucase.caucased(
prefix='caucased',
buildout_bin_directory=bin_directory,
caucased_path='${directory:service-on-watch}/caucased',
backup_dir='${directory:backup-caucased}',
data_dir='${directory:srv}/caucased',
netloc=caucase_netloc,
tmp='${directory:tmp}',
service_auto_approve_count=caucase_dict.get('service-auto-approve-amount', 1),
user_auto_approve_count=caucase_dict.get('user-auto-approve-amount', 0),
key_len=caucase_dict.get('key-length', 2048),
)}}
{% do root_common.section('caucased') -%}
{% do root_common.section('caucased-promise') -%}
{% endif -%}
{% do publish_dict.__setitem__('caucase-http-url', caucase_url) -%}
{% set balancer_dict = slapparameter_dict.setdefault('balancer', {}) -%} {% set balancer_dict = slapparameter_dict.setdefault('balancer', {}) -%}
{% do balancer_dict.setdefault('ssl', {}).setdefault('caucase-url', caucase_url) -%} {% do balancer_dict.setdefault('ssl', {}) %}
{% if caucase_url %}
{% do balancer_dict['ssl'].setdefault('caucase-url', caucase_url) -%}
[balancer-updated-caucase-url]
# Promise to wait for balancer partition to use the provided URL, for cases where the
# caucase URL is changed
recipe = slapos.cookbook:check_parameter
value = ${publish:caucase-http-url}
expected-not-value =
expected-value = {{ caucase_url }}
path = ${directory:bin}/${:_buildout_section_name_}
[balancer-updated-caucase-url-promise]
<= monitor-promise-base
promise = check_command_execute
name = ${:_buildout_section_name_}.py
config-command = ${balancer-updated-caucase-url:path}
{% do root_common.section("balancer-updated-caucase-url-promise") -%}
{% endif %}
{% do balancer_dict.setdefault('tcpv4-port', 2150) -%} {% do balancer_dict.setdefault('tcpv4-port', 2150) -%}
{% do balancer_dict.__setitem__('haproxy-server-check-path', balancer_dict.get('haproxy-server-check-path', '/') % {'site-id': site_id}) -%} {% do balancer_dict.__setitem__('haproxy-server-check-path', balancer_dict.get('haproxy-server-check-path', '/') % {'site-id': site_id}) -%}
{% set routing_path_template_field_dict = {"site-id": site_id} -%} {% set routing_path_template_field_dict = {"site-id": site_id} -%}
...@@ -241,7 +236,6 @@ return = ...@@ -241,7 +236,6 @@ return =
config-bt5 = {{ dumps(slapparameter_dict.get('bt5', ' '.join(bt5_default_list))) }} config-bt5 = {{ dumps(slapparameter_dict.get('bt5', ' '.join(bt5_default_list))) }}
config-bt5-repository-url = {{ dumps(slapparameter_dict.get('bt5-repository-url', local_bt5_repository)) }} config-bt5-repository-url = {{ dumps(slapparameter_dict.get('bt5-repository-url', local_bt5_repository)) }}
config-cloudooo-url-list = {{ dumps(slapparameter_dict.get('cloudooo-url-list', default_cloudooo_url_list)) }} config-cloudooo-url-list = {{ dumps(slapparameter_dict.get('cloudooo-url-list', default_cloudooo_url_list)) }}
config-caucase-url = {{ dumps(caucase_url) }}
config-deadlock-debugger-password = ${publish-early:deadlock-debugger-password} config-deadlock-debugger-password = ${publish-early:deadlock-debugger-password}
config-developer-list = {{ dumps(slapparameter_dict.get('developer-list', [inituser_login])) }} config-developer-list = {{ dumps(slapparameter_dict.get('developer-list', [inituser_login])) }}
config-selenium-server-configuration-dict = {{ dumps(slapparameter_dict.get('selenium-server-configuration-dict', {})) }} config-selenium-server-configuration-dict = {{ dumps(slapparameter_dict.get('selenium-server-configuration-dict', {})) }}
...@@ -275,7 +269,6 @@ software-type = zope ...@@ -275,7 +269,6 @@ software-type = zope
{% set global_activity_timeout = slapparameter_dict.get('activity-timeout') -%} {% set global_activity_timeout = slapparameter_dict.get('activity-timeout') -%}
{% set zope_family_dict = {} -%} {% set zope_family_dict = {} -%}
{% set zope_family_name_list = [] -%} {% set zope_family_name_list = [] -%}
{% set zope_backend_path_dict = {} -%}
{% set ssl_authentication_dict = {} -%} {% set ssl_authentication_dict = {} -%}
{% set balancer_timeout_dict = {} -%} {% set balancer_timeout_dict = {} -%}
{% set jupyter_zope_family_default = [] -%} {% set jupyter_zope_family_default = [] -%}
...@@ -288,14 +281,12 @@ software-type = zope ...@@ -288,14 +281,12 @@ software-type = zope
{% set promise_test_runner_url_section_name = 'promise-test-runner-url' ~ partition_name -%} {% set promise_test_runner_url_section_name = 'promise-test-runner-url' ~ partition_name -%}
{% set zope_family = zope_parameter_dict.get('family', 'default') -%} {% set zope_family = zope_parameter_dict.get('family', 'default') -%}
{% do zope_family_name_list.append(zope_family) %} {% do zope_family_name_list.append(zope_family) %}
{% set backend_path = zope_parameter_dict.get('backend-path', '') % {'site-id': site_id} %}
{# # default jupyter zope family is first zope family. -#} {# # default jupyter zope family is first zope family. -#}
{# # use list.append() to update it, because in jinja2 set changes only local scope. -#} {# # use list.append() to update it, because in jinja2 set changes only local scope. -#}
{% if not jupyter_zope_family_default -%} {% if not jupyter_zope_family_default -%}
{% do jupyter_zope_family_default.append(zope_family) -%} {% do jupyter_zope_family_default.append(zope_family) -%}
{% endif -%} {% endif -%}
{% do zope_family_dict.setdefault(zope_family, []).append(section_name) -%} {% do zope_family_dict.setdefault(zope_family, []).append(section_name) -%}
{% do zope_backend_path_dict.__setitem__(zope_family, backend_path) -%}
{% do ssl_authentication_dict.__setitem__(zope_family, zope_parameter_dict.get('ssl-authentication', False)) -%} {% do ssl_authentication_dict.__setitem__(zope_family, zope_parameter_dict.get('ssl-authentication', False)) -%}
{% set current_zope_family_override_dict = zope_family_override_dict.get(zope_family, {}) -%} {% set current_zope_family_override_dict = zope_family_override_dict.get(zope_family, {}) -%}
{% do balancer_timeout_dict.__setitem__(zope_family, current_zope_family_override_dict.get('publisher-timeout', global_publisher_timeout)) -%} {% do balancer_timeout_dict.__setitem__(zope_family, current_zope_family_override_dict.get('publisher-timeout', global_publisher_timeout)) -%}
...@@ -397,13 +388,13 @@ return = ...@@ -397,13 +388,13 @@ return =
{% set request_frontend_name = 'request-frontend-' ~ frontend_name -%} {% set request_frontend_name = 'request-frontend-' ~ frontend_name -%}
{% set frontend_software_url = frontend_parameters.get('software-url', 'http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg') -%} {% set frontend_software_url = frontend_parameters.get('software-url', 'http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg') -%}
{% set frontend_software_type = frontend_parameters.get('software-type', '') -%} {% set frontend_software_type = frontend_parameters.get('software-type', '') -%}
{% do frontend_parameters.__setitem__('internal-path', frontend_parameters.get('internal-path', '/%(site-id)s') % {'site-id': site_id}) %}
{% set frontend_instance_parameters = frontend_parameters.get('instance-parameters', {}) -%} {% set frontend_instance_parameters = frontend_parameters.get('instance-parameters', {}) -%}
{% if frontend_instance_parameters.setdefault('type', 'zope') == 'zope' -%} {% if frontend_instance_parameters.setdefault('type', '') == '' -%}
{% do frontend_instance_parameters.setdefault('authenticate-to-backend', 'true') -%} {% do frontend_instance_parameters.setdefault('authenticate-to-backend', 'true') -%}
{% set zope_family_name = frontend_parameters['zope-family'] -%} {% set zope_family_name = frontend_parameters['zope-family'] -%}
{% do assert(zope_family_name in zope_family_dict, 'Unknown family %s for frontend %s' % (zope_family_name, frontend_name)) -%} {% do assert(zope_family_name in zope_family_dict, 'Unknown family %s for frontend %s' % (zope_family_name, frontend_name)) -%}
{% do frontend_instance_parameters.setdefault('url', '${request-balancer:connection-' ~ zope_family_name ~ '-v6}') -%} {% do frontend_instance_parameters.setdefault('url', '${request-balancer:connection-url-backend-' ~ frontend_name ~ '}') -%}
{% do frontend_instance_parameters.setdefault('path', frontend_parameters.get('internal-path', '/%(site-id)s') % {'site-id': site_id}) -%}
{% endif %} {% endif %}
[{{ request_frontend_name }}] [{{ request_frontend_name }}]
<= request-frontend-base <= request-frontend-base
...@@ -447,7 +438,7 @@ config-allow-redirects = 0 ...@@ -447,7 +438,7 @@ config-allow-redirects = 0
{%- endif %} {%- endif %}
{% set balancer_ret_dict = {'monitor-base-url': False} -%} {% set balancer_ret_dict = {'monitor-base-url': False, 'caucase-http-url': False} -%}
{% for family in zope_family_dict -%} {% for family in zope_family_dict -%}
{% do balancer_ret_dict.__setitem__(family, False) -%} {% do balancer_ret_dict.__setitem__(family, False) -%}
{% do balancer_ret_dict.__setitem__(family + '-v6', False) -%} {% do balancer_ret_dict.__setitem__(family + '-v6', False) -%}
...@@ -455,6 +446,9 @@ config-allow-redirects = 0 ...@@ -455,6 +446,9 @@ config-allow-redirects = 0
{% do balancer_ret_dict.__setitem__(family + '-test-runner-url-list', False) -%} {% do balancer_ret_dict.__setitem__(family + '-test-runner-url-list', False) -%}
{% endif -%} {% endif -%}
{% endfor -%} {% endfor -%}
{% for frontend_name in frontend_parameter_dict -%}
{% do balancer_ret_dict.__setitem__('url-backend-' ~ frontend_name, False) -%}
{% endfor -%}
{% set balancer_key_config_dict = { {% set balancer_key_config_dict = {
'monitor-passwd': 'monitor-htpasswd:passwd', 'monitor-passwd': 'monitor-htpasswd:passwd',
} -%} } -%}
...@@ -476,7 +470,7 @@ config-allow-redirects = 0 ...@@ -476,7 +470,7 @@ config-allow-redirects = 0
config_key='balancer', config_key='balancer',
config={ config={
'zope-family-dict': zope_family_parameter_dict, 'zope-family-dict': zope_family_parameter_dict,
'backend-path-dict': zope_backend_path_dict, 'frontend-parameter-dict': frontend_parameter_dict,
'ssl-authentication-dict': ssl_authentication_dict, 'ssl-authentication-dict': ssl_authentication_dict,
'timeout-dict': balancer_timeout_dict, 'timeout-dict': balancer_timeout_dict,
'apachedex-promise-threshold': monitor_dict.get('apachedex-promise-threshold', 70), 'apachedex-promise-threshold': monitor_dict.get('apachedex-promise-threshold', 70),
...@@ -496,7 +490,7 @@ config-allow-redirects = 0 ...@@ -496,7 +490,7 @@ config-allow-redirects = 0
ret=balancer_ret_dict, ret=balancer_ret_dict,
key_config=balancer_key_config_dict, key_config=balancer_key_config_dict,
) }} ) }}
{% do publish_dict.__setitem__('caucase-http-url', '${request-balancer:connection-caucase-http-url}' ) -%}
{% endif -%}{# if zope_partition_dict -#} {% endif -%}{# if zope_partition_dict -#}
......
...@@ -81,7 +81,6 @@ environment += ...@@ -81,7 +81,6 @@ environment +=
MATPLOTLIBRC={{ parameter_dict['matplotlibrc'] }} MATPLOTLIBRC={{ parameter_dict['matplotlibrc'] }}
PYTHONUNBUFFERED=1 PYTHONUNBUFFERED=1
INSTANCE_HOME=${:instance-home} INSTANCE_HOME=${:instance-home}
CAUCASE={{ slapparameter_dict['caucase-url'] }}
FONTCONFIG_FILE=${fontconfig-conf:output} FONTCONFIG_FILE=${fontconfig-conf:output}
JUPYTER_PATH=${directory:jupyter-dir} JUPYTER_PATH=${directory:jupyter-dir}
JUPYTER_CONFIG_DIR=${directory:jupyter-config-dir} JUPYTER_CONFIG_DIR=${directory:jupyter-config-dir}
......
...@@ -91,8 +91,9 @@ url = {{ template_balancer }} ...@@ -91,8 +91,9 @@ url = {{ template_balancer }}
filename = instance-balancer.cfg filename = instance-balancer.cfg
extra-context = extra-context =
section parameter_dict dynamic-template-balancer-parameters section parameter_dict dynamic-template-balancer-parameters
import itertools itertools
import hashlib hashlib import hashlib hashlib
# XXX: only used in software/slapos-master:
import itertools itertools
import functools functools import functools functools
import-list = import-list =
file caucase context:caucase-jinja2-library file caucase context:caucase-jinja2-library
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment