Skip to content

slapos
slapos
Commit d45e9cdf authored by Łukasz Nowak's avatar Łukasz Nowak
Browse files
Options

caddy-frontend: Improve and stabilise Via header 

Via header for response and request is fully stabilized and present according
to the RFC now.

Important information, like protocol and protocol version are exposed.

Hops which were present in the request are stored and descriptively named.

Versions of components are exposed in hidden way and in the same time the
frontend administrator and frontend user are able to analyze the versions,
and the version history present on the node is transmitted back.

Additionally each node can be identified by frontend admins and users for
request and response headers, allowing more debugging.

Improve the tests by extending assertSlaveBase and reuse it instead of
duplicating the code everywhere.
parent e1143ed4
Hide whitespace changes
Inline Side-by-side
Showing with 425 additions and 888 deletions
software/caddy-frontend/buildout.hash.cfg
View file @ d45e9cdf
......@@ -22,19 +22,19 @@ md5sum = 5784bea3bd608913769ff9a8afcccb68
[profile-caddy-frontend]
filename = instance-apache-frontend.cfg.in
md5sum = 3e3021b86c3cfe93553489441da85496
md5sum = 04e550480d3057ca65d87c6fadbaed6e
[profile-caddy-replicate]
filename = instance-apache-replicate.cfg.in
md5sum = c028f1c5947494e7f25cf8266a3ecd2d
md5sum = 63b418626ef0f8ac54d6359fb6637371
[profile-slave-list]
_update_hash_filename_ = templates/apache-custom-slave-list.cfg.in
md5sum = 6b6ab13d82bf9ecff6a37c3402ddbf95
md5sum = e3ba0da5d137dcbd56c2604d200ac3b9
[profile-replicate-publish-slave-information]
_update_hash_filename_ = templates/replicate-publish-slave-information.cfg.in
md5sum = df304a8aee87b6f2425241016a48f7a5
md5sum = be54431846fe7f3cee65260eefc83d62
[profile-caddy-frontend-configuration]
_update_hash_filename_ = templates/Caddyfile.in
......@@ -46,11 +46,11 @@ md5sum = 88af61e7abbf30dc99a1a2526161128d
[template-default-slave-virtualhost]
_update_hash_filename_ = templates/default-virtualhost.conf.in
md5sum = 37475d79f28c5f126bc1947fdb938fdb
md5sum = 57c86795293b11300a036f5f8cf2c868
[template-backend-haproxy-configuration]
_update_hash_filename_ = templates/backend-haproxy.cfg.in
md5sum = ae4c9ce775ea003aa51eda5ecbbeec73
md5sum = 6d4ad68ac44ccc72fe9148bd8e05a6f0
[template-empty]
_update_hash_filename_ = templates/empty.in
......@@ -62,7 +62,7 @@ md5sum = 975177dedf677d24e14cede5d13187ce
[template-trafficserver-records-config]
_update_hash_filename_ = templates/trafficserver/records.config.jinja2
md5sum = e87238c53d080ef9ef90040e57bc1395
md5sum = 715baa302d562a7e4eddc3d1bf72f981
[template-trafficserver-storage-config]
_update_hash_filename_ = templates/trafficserver/storage.config.jinja2
......@@ -94,7 +94,7 @@ md5sum = 8c150e1e6c993708d31936742f3a7302
[caddyprofiledeps-setup]
filename = setup.py
md5sum = 6aad2b4c271294f524214192ee197c15
md5sum = f6f72d03af7d9dc29fb4d4fef1062e73
[caddyprofiledeps-dummy]
filename = caddyprofiledummy.py
......
software/caddy-frontend/instance-apache-frontend.cfg.in
View file @ d45e9cdf
......@@ -63,6 +63,75 @@ parts =
[caddyprofiledeps]
recipe = caddyprofiledeps
[frontend-node-id]
# Store id file in top of hierarchy, so it does not depend on directory creation
file = ${buildout:directory}/.frontend-node-id.txt
recipe = slapos.recipe.build
init =
import os
import secrets
if not os.path.exists(options['file']):
with open(options['file'], 'w') as fh:
fh.write(secrets.token_urlsafe(4))
with open(options['file'], 'r') as fh:
options['value'] = fh.read()
[frontend-node-private-salt]
# Private, not communicated, stable hash, which can be used to salt other
# hashes, so their values are connected to the node, but practicaly impossible
# to crack (until the node is hacked itself, but then those values are
# stolen anyway)
recipe = slapos.recipe.build
init =
import os
import uuid
if not os.path.exists(options['file']):
with open(options['file'], 'w') as fh:
fh.write(uuid.uuid4().hex)
with open(options['file'], 'r') as fh:
options['value'] = fh.read()
file = ${buildout:directory}/.frontend-node-private-salt.txt
[version-hash]
recipe = slapos.recipe.build
software-release-url = ${slap-connection:software-release-url}
hash-salt = ${frontend-node-private-salt:value}
init =
import hashlib
import base64
options['value'] = base64.urlsafe_b64encode(hashlib.md5(''.join([options['software-release-url'].strip(), options['hash-salt']])).digest())
[frontend-node-information]
recipe = slapos.recipe.build
file = ${buildout:directory}/.frontend-node-information.json
node-id = ${frontend-node-id:value}
current-hash = ${version-hash:value}
current-software-release-url = ${version-hash:software-release-url}
init =
import json
changed = False
try:
with open(options['file'], 'r') as fh:
data = json.load(fh)
except Exception:
changed = True
data = {
'node-id': options['node-id'],
'version-hash-history': {options['current-hash']: options['current-software-release-url']}
}
if 'node-id' not in data:
data['node-id'] = options['node-id']
changed = True
if 'version-hash-history' not in data:
data['version-hash-history'] = {}
changed = True
if options['current-hash'] not in data['version-hash-history']:
data['version-hash-history'][options['current-hash']] = options['current-software-release-url']
changed = True
if changed:
with open(options['file'], 'w') as fh:
json.dump(data, fh)
options['value'] = data
# Create all needed directories
[directory]
recipe = slapos.cookbook:mkdirectory
......@@ -305,6 +374,10 @@ extra-context =
key software_type :software_type
key frontend_lazy_graceful_reload frontend-caddy-lazy-graceful:rendered
key monitor_base_url monitor-instance-parameter:monitor-base-url
key node_id frontend-node-id:value
key version_hash version-hash:value
key software_release_url version-hash:software-release-url
key node_information frontend-node-information:value
key custom_ssl_directory caddy-directory:custom-ssl-directory
# BBB: SlapOS Master non-zero knowledge BEGIN
key apache_certificate apache-certificate:rendered
......@@ -462,6 +535,8 @@ disk-cache-size = ${configuration:disk-cache-size}
ram-cache-size = ${configuration:ram-cache-size}
templates-dir = {{ software_parameter_dict['trafficserver'] }}/etc/trafficserver/body_factory
request-timeout = ${configuration:request-timeout}
version-hash = ${version-hash:value}
node-id = ${frontend-node-id:value}
[trafficserver-configuration-directory]
recipe = plone.recipe.command
......
software/caddy-frontend/instance-apache-replicate.cfg.in
View file @ d45e9cdf
......@@ -286,7 +286,7 @@ config-monitor-username = ${monitor-instance-parameter:username}
config-monitor-password = ${monitor-htpasswd:passwd}
software-type = {{frontend_type}}
return = slave-instance-information-list monitor-base-url backend-client-csr-url kedifa-csr-url csr-certificate backend-haproxy-statistic-url
return = slave-instance-information-list monitor-base-url backend-client-csr-url kedifa-csr-url csr-certificate backend-haproxy-statistic-url node-information-json
{#- Send only needed parameters to frontend nodes #}
{%- set base_node_configuration_dict = {} %}
......@@ -376,6 +376,7 @@ kedifa-csr-certificate = ${request-kedifa:connection-csr-certificate}
{% for frontend in frontend_list %}
{% set section_part = '${request-' + frontend %}
{{ frontend }}-backend-haproxy-statistic-url = {{ section_part }}:connection-backend-haproxy-statistic-url}
{{ frontend }}-node-information-json = ${frontend-information:{{ frontend }}-node-information-json}
{% endfor %}
{% if not aibcc_enabled %}
{% for frontend in frontend_list %}
......@@ -461,6 +462,12 @@ warning-slave-dict = {{ dumps(warning_slave_dict) }}
{# sort_keys are important in order to avoid shuffling parameters on each run #}
active-slave-instance-list = {{ json_module.dumps(active_slave_instance_list, sort_keys=True) }}
[frontend-information]
{% for frontend in frontend_list %}
{% set section_part = '${request-' + frontend %}
{{ frontend }}-node-information-json = {{ section_part }}:connection-node-information-json}
{% endfor %}
[dynamic-publish-slave-information]
< = jinja2-template-base
template = {{ software_parameter_dict['profile_replicate_publish_slave_information'] }}
......@@ -468,6 +475,7 @@ filename = dynamic-publish-slave-information.cfg
extensions = jinja2.ext.do
extra-context =
section slave_information slave-information
section frontend_information frontend-information
section rejected_slave_information rejected-slave-information
section active_slave_instance_dict active-slave-instance
section warning_slave_information warning-slave-information
......
software/caddy-frontend/setup.py
View file @ d45e9cdf
......@@ -10,6 +10,7 @@ setup(
'furl',
'orderedmultidict',
'caucase',
'python2-secrets',
],
entry_points={
'zc.buildout': [
......
software/caddy-frontend/software.cfg
View file @ d45e9cdf
......@@ -214,6 +214,7 @@ kedifa = 0.0.6
# Modern KeDiFa requires zc.lockfile
zc.lockfile = 1.4
python2-secrets = 1.0.5
validators = 0.12.2
PyRSS2Gen = 1.1
cns.recipe.symlink = 0.2.3
......
software/caddy-frontend/templates/apache-custom-slave-list.cfg.in
View file @ d45e9cdf
......@@ -330,6 +330,8 @@ certificate = {{ certificate }}
https_port = {{ dumps('' ~ configuration['port']) }}
http_port = {{ dumps('' ~ configuration['plain_http_port']) }}
local_ipv4 = {{ dumps('' ~ instance_parameter_dict['ipv4-random']) }}
version-hash = {{ version_hash }}
node-id = {{ node_id }}
{%- for key, value in slave_instance.iteritems() %}
{%- if value is not none %}
{{ key }} = {{ dumps(value) }}
......@@ -463,6 +465,8 @@ csr-certificate = ${expose-csr-certificate-get:certificate}
{#- We unquote, as furl quotes automatically, but there is buildout value on purpose like ${...:...} in the passwod #}
{%- set statistic_url = urlparse_module.unquote(furled.tostr()) %}
backend-haproxy-statistic-url = {{ statistic_url }}
{#- sort_keys are important in order to avoid shuffling parameters on each run #}
node-information-json = {{ json_module.dumps(node_information, sort_keys=True) }}
[kedifa-updater]
recipe = slapos.cookbook:wrapper
......@@ -513,6 +517,8 @@ global-ipv6 = ${slap-configuration:ipv6-random}
request-timeout = {{ dumps('' ~ configuration['request-timeout']) }}
backend-connect-timeout = {{ dumps('' ~ configuration['backend-connect-timeout']) }}
backend-connect-retries = {{ dumps('' ~ configuration['backend-connect-retries']) }}
version-hash = {{ version_hash }}
node-id = {{ node_id }}
[template-expose-csr-link-csr]
recipe = plone.recipe.command
......
software/caddy-frontend/templates/backend-haproxy.cfg.in
View file @ d45e9cdf
......@@ -58,6 +58,8 @@ frontend statistic
frontend http-backend
bind {{ configuration['local-ipv4'] }}:{{ configuration['http-port'] }}
http-request add-header Via "%HV rapid-cdn-backend-{{ configuration['node-id'] }}-{{ configuration['version-hash'] }}"
http-response add-header Via "%HV rapid-cdn-backend-{{ configuration['node-id'] }}-{{ configuration['version-hash']}}"
{%- for slave_instance in backend_slave_list -%}
{{ frontend_entry(slave_instance, 'http', False) }}
{%- endfor %}
......
software/caddy-frontend/templates/default-virtualhost.conf.in
View file @ d45e9cdf
......@@ -21,6 +21,10 @@
# workaround for lost connection to haproxy by reconnecting
try_duration 3s
try_interval 250ms
header_upstream +Via "{proto} rapid-cdn-frontend-{{ slave_parameter['node-id'] }}-{{ slave_parameter['version-hash'] }}"
{%- if not slave_parameter['disable-via-header'] %}
header_downstream +Via "{proto} rapid-cdn-frontend-{{ slave_parameter['node-id'] }}-{{ slave_parameter['version-hash'] }}"
{%- endif %}
{%- endmacro %} {# proxy_header #}
{%- macro hsts_header(tls) %}
......
software/caddy-frontend/templates/replicate-publish-slave-information.cfg.in
View file @ d45e9cdf
......@@ -72,6 +72,9 @@ log-access-url = {{ dumps(json_module.dumps(log_access_url, sort_keys=True)) }}
{{ key }} = {{ dumps(value) }}
{% endfor %}
{% endif %}
{% for frontend_key, frontend_value in frontend_information.iteritems() %}
{{ frontend_key }} = {{ frontend_value }}
{% endfor %}
{% endfor %}
[buildout]
......
software/caddy-frontend/templates/trafficserver/records.config.jinja2
View file @ d45e9cdf
......@@ -18,6 +18,12 @@ LOCAL proxy.local.incoming_ip_to_bind STRING {{ ats_configuration['local-ip'] }}
CONFIG proxy.config.log.logfile_dir STRING {{ ats_directory['log'] }}
# Never change Server header
CONFIG proxy.config.http.response_server_enabled INT 0
# Handle Via header
CONFIG proxy.config.http.insert_request_via_str INT 1
CONFIG proxy.config.http.request_via_str STRING rapid-cdn-cache-{{ ats_configuration['node-id'] }}-{{ ats_configuration['version-hash'] }}
CONFIG proxy.config.http.insert_response_via_str INT 1
CONFIG proxy.config.http.response_via_str STRING rapid-cdn-cache-{{ ats_configuration['node-id'] }}-{{ ats_configuration['version-hash'] }}
# Implement RFC 5861 with core
CONFIG proxy.config.http.cache.open_write_fail_action INT 2
CONFIG proxy.config.body_factory.template_sets_dir STRING {{ ats_configuration['templates-dir'] }}
......@@ -53,13 +59,6 @@ CONFIG proxy.config.exec_thread.affinity INT 1
##############################################################################
CONFIG proxy.config.http.server_ports STRING {{ ats_configuration['local-ip'] + ':' + ats_configuration['input-port'] }}
##############################################################################
# Via: headers. Docs:
# https://docs.trafficserver.apache.org/records.config#proxy-config-http-insert-response-via-str
##############################################################################
CONFIG proxy.config.http.insert_request_via_str INT 1
CONFIG proxy.config.http.insert_response_via_str INT 0
##############################################################################
# Parent proxy configuration, in addition to these settings also see parent.config. Docs:
# https://docs.trafficserver.apache.org/records.config#parent-proxy-configuration
......
software/caddy-frontend/test/test.py
View file @ d45e9cdf
......@@ -95,12 +95,6 @@ KEDIFA_PORT = '15080'
# has to be not partition one
SOURCE_IP = '127.0.0.1'
# ATS version expectation in Via string
VIA_STRING = (
r'^http\/1.1 caddy-frontend-1\[.*\] '
r'\(ApacheTrafficServer\/9\.[0-9]\.[0-9]+\)$',
)[0]
# IP on which test run, in order to mimic HTTP[s] access
TEST_IP = os.environ['SLAPOS_TEST_IPV4']
......@@ -430,6 +424,8 @@ def fakeHTTPSResult(domain, path, port=HTTPS_PORT,
headers.setdefault('X-Forwarded-For', '192.168.0.1')
headers.setdefault('X-Forwarded-Proto', 'irc')
headers.setdefault('X-Forwarded-Port', '17')
# Expose some Via to show how nicely it arrives to the backend
headers.setdefault('Via', 'http/1.1 clientvia')
session = requests.Session()
if source_ip is not None:
......@@ -470,6 +466,8 @@ def fakeHTTPResult(domain, path, port=HTTP_PORT,
headers.setdefault('X-Forwarded-For', '192.168.0.1')
headers.setdefault('X-Forwarded-Proto', 'irc')
headers.setdefault('X-Forwarded-Port', '17')
# Expose some Via to show how nicely it arrives to the backend
headers.setdefault('Via', 'http/1.1 clientvia')
headers['Host'] = '%s:%s' % (domain, port)
session = requests.Session()
if source_ip is not None:
......@@ -604,6 +602,8 @@ class TestHandler(BaseHTTPRequestHandler):
self.send_header('Set-Cookie', 'secured=value;secure')
self.send_header('Set-Cookie', 'nonsecured=value')
if 'Via' not in drop_header_list:
self.send_header('Via', 'http/1.1 backendvia')
if compress:
self.send_header('Content-Encoding', 'gzip')
out = StringIO.StringIO()
......@@ -874,7 +874,8 @@ class HttpFrontendTestCase(SlapOSInstanceTestCase):
except Exception as e:
self.fail(e)
def assertResponseHeaders(self, result):
def assertResponseHeaders(
self, result, cached=False, via=True, backend_reached=True):
headers = result.headers.copy()
self.assertKeyWithPop('Date', headers)
# drop vary-keys
......@@ -883,8 +884,31 @@ class HttpFrontendTestCase(SlapOSInstanceTestCase):
headers.pop('Keep-Alive', None)
headers.pop('Transfer-Encoding', None)
self.assertEqual('TestBackend', headers.pop('Server', ''))
if backend_reached:
self.assertEqual('TestBackend', headers.pop('Server', ''))
via_id = '%s-%s' % (
self.node_information_dict['node-id'],
self.node_information_dict['version-hash-history'].keys()[0])
if via:
self.assertIn('Via', headers)
if cached:
self.assertEqual(
'http/1.1 backendvia, '
'HTTP/1.1 rapid-cdn-backend-%(via_id)s, '
'http/1.0 rapid-cdn-cache-%(via_id)s, '
'HTTP/1.1 rapid-cdn-frontend-%(via_id)s' % dict(via_id=via_id),
headers.pop('Via')
)
else:
self.assertEqual(
'http/1.1 backendvia, '
'HTTP/1.1 rapid-cdn-backend-%(via_id)s, '
'HTTP/1.1 rapid-cdn-frontend-%(via_id)s' % dict(via_id=via_id),
headers.pop('Via')
)
else:
self.assertNotIn('Via', headers)
return headers
def assertLogAccessUrlWithPop(self, parameter_dict):
......@@ -954,6 +978,23 @@ class HttpFrontendTestCase(SlapOSInstanceTestCase):
return generate_auth_url, upload_url
def assertNodeInformationWithPop(self, parameter_dict):
key = 'caddy-frontend-1-node-information-json'
node_information_json_dict = {}
for k in parameter_dict.keys():
if k.startswith('caddy-frontend') and k.endswith(
'node-information-json'):
node_information_json_dict[k] = parameter_dict.pop(k)
self.assertEqual(
[key],
node_information_json_dict.keys()
)
node_information_dict = json.loads(node_information_json_dict[key])
self.assertIn("node-id", node_information_dict)
self.assertIn("version-hash-history", node_information_dict)
self.node_information_dict = node_information_dict
def assertBackendHaproxyStatisticUrl(self, parameter_dict):
url_key = 'caddy-frontend-1-backend-haproxy-statistic-url'
backend_haproxy_statistic_url_dict = {}
......@@ -1276,20 +1317,27 @@ class SlaveHttpFrontendTestCase(HttpFrontendTestCase):
]
)
def assertSlaveBase(self, reference):
def assertSlaveBase(
self, reference, expected_parameter_dict=None, hostname=None):
if expected_parameter_dict is None:
expected_parameter_dict = {}
parameter_dict = self.parseSlaveParameterDict(reference)
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict, '')
hostname = reference.translate(None, '_-').lower()
self.current_generate_auth, self.current_upload_url = \
self.assertKedifaKeysWithPop(parameter_dict, '')
self.assertNodeInformationWithPop(parameter_dict)
if hostname is None:
hostname = reference.translate(None, '_-').lower()
expected_parameter_dict.update(**{
'domain': '%s.example.com' % (hostname,),
'replication_number': '1',
'url': 'http://%s.example.com' % (hostname, ),
'site_url': 'http://%s.example.com' % (hostname, ),
'secure_access': 'https://%s.example.com' % (hostname, ),
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
})
self.assertEqual(
{
'domain': '%s.example.com' % (hostname,),
'replication_number': '1',
'url': 'http://%s.example.com' % (hostname, ),
'site_url': 'http://%s.example.com' % (hostname, ),
'secure_access': 'https://%s.example.com' % (hostname, ),
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
expected_parameter_dict,
parameter_dict
)
......@@ -1323,6 +1371,7 @@ class TestMasterRequestDomain(HttpFrontendTestCase, TestDataMixin):
self.assertBackendHaproxyStatisticUrl(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict, 'master-')
self.assertRejectedSlavePromiseEmptyWithPop(parameter_dict)
self.assertNodeInformationWithPop(parameter_dict)
self.assertEqual(
{
......@@ -1354,6 +1403,7 @@ class TestMasterRequest(HttpFrontendTestCase, TestDataMixin):
self.assertBackendHaproxyStatisticUrl(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict, 'master-')
self.assertRejectedSlavePromiseEmptyWithPop(parameter_dict)
self.assertNodeInformationWithPop(parameter_dict)
self.assertEqual(
{
'monitor-base-url': 'https://[%s]:8401' % self._ipv6_address,
......@@ -1468,6 +1518,7 @@ class TestMasterAIKCDisabledAIBCCDisabledRequest(
'caddy-frontend-1-backend-client-csr-url', parameter_dict)
self.assertKeyWithPop(
'caddy-frontend-1-csr-certificate', parameter_dict)
self.assertNodeInformationWithPop(parameter_dict)
self.assertEqual(
{
'monitor-base-url': 'https://[%s]:8401' % self._ipv6_address,
......@@ -1924,6 +1975,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
self.assertBackendHaproxyStatisticUrl(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict, 'master-')
self.assertRejectedSlavePromiseEmptyWithPop(parameter_dict)
self.assertNodeInformationWithPop(parameter_dict)
expected_parameter_dict = {
'monitor-base-url': 'https://[%s]:8401' % self._ipv6_address,
......@@ -1955,6 +2007,40 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
os.path.join(
partition_path, 'etc', 'httpd-cors.cfg'), 'r').read().strip())
def test_node_information_json(self):
node_information_file_path = glob.glob(os.path.join(
self.instance_path, '*', '.frontend-node-information.json'))[0]
with open(node_information_file_path, 'r') as fh:
current_node_information = json.load(fh)
modified_node_information = current_node_information.copy()
modified_node_information['version-hash-history'] = {'testhash': 'testurl'}
def writeNodeInformation(node_information, path):
with open(path, 'w') as fh:
json.dump(node_information, fh, sort_keys=True)
self.waitForInstance()
self.waitForInstance()
self.waitForInstance()
self.addCleanup(
writeNodeInformation, current_node_information,
node_information_file_path)
# simulate that upgrade happened
writeNodeInformation(
modified_node_information,
node_information_file_path)
parameter_dict = self.parseConnectionParameterDict()
expected_node_information = {
'node-id': current_node_information['node-id'],
'version-hash-history': current_node_information['version-hash-history']
}
expected_node_information['version-hash-history']['testhash'] = 'testurl'
self.assertEqual(
json.loads(parameter_dict['caddy-frontend-1-node-information-json']),
expected_node_information
)
def test_slave_partition_state(self):
partition_path = self.getSlavePartitionPath()
self.assertTrue(
......@@ -2041,7 +2127,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
def assertBackendHeaders(
self, backend_header_dict, domain, source_ip=SOURCE_IP, port=HTTPS_PORT,
proto='https', ignore_header_list=None):
proto='https', ignore_header_list=None, cached=False):
if ignore_header_list is None:
ignore_header_list = []
if 'Host' not in ignore_header_list:
......@@ -2060,6 +2146,28 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
backend_header_dict['x-forwarded-proto'],
proto
)
via_id = '%s-%s' % (
self.node_information_dict['node-id'],
self.node_information_dict['version-hash-history'].keys()[0])
if cached:
self.assertEqual(
[
'http/1.1 clientvia',
'HTTP/1.1 rapid-cdn-frontend-%(via_id)s, '
'http/1.1 rapid-cdn-cache-%(via_id)s' % dict(via_id=via_id),
'HTTP/1.1 rapid-cdn-backend-%(via_id)s' % dict(via_id=via_id)
],
backend_header_dict['via']
)
else:
self.assertEqual(
[
'http/1.1 clientvia',
'HTTP/1.1 rapid-cdn-frontend-%(via_id)s' % dict(via_id=via_id),
'HTTP/1.1 rapid-cdn-backend-%(via_id)s' % dict(via_id=via_id)
],
backend_header_dict['via']
)
def test_telemetry_disabled(self):
# here we trust that telemetry not present in error log means it was
......@@ -2071,26 +2179,14 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
self.assertNotIn('Sending telemetry', fh.read(), 'Telemetry enabled')
def test_url(self):
reference = 'Url'
parameter_dict = self.parseSlaveParameterDict(reference)
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict, '')
hostname = reference.translate(None, '_-').lower()
self.assertEqual(
parameter_dict = self.assertSlaveBase(
'Url',
{
'domain': '%s.example.com' % (hostname,),
'replication_number': '1',
'url': 'http://%s.example.com' % (hostname, ),
'site_url': 'http://%s.example.com' % (hostname, ),
'secure_access': 'https://%s.example.com' % (hostname, ),
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
'warning-list': [
"slave url ' %s/?a=b&c= ' has been converted to '%s/?a=b&c='" % (
self.backend_url, self.backend_url)],
},
parameter_dict
}
)
result = fakeHTTPSResult(
parameter_dict['domain'],
'test-path/deep/.././deeper',
......@@ -2105,7 +2201,8 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
self.certificate_pem,
der2pem(result.peercert))
self.assertNotIn('Strict-Transport-Security', result.headers)
headers = self.assertResponseHeaders(result)
self.assertNotIn('Strict-Transport-Security', headers)
self.assertEqualResultJson(result, 'Path', '?a=b&c=/test-path/deeper')
try:
......@@ -2114,12 +2211,12 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
raise ValueError('JSON decode problem in:\n%s' % (result.text,))
self.assertEqual(j['Incoming Headers']['timeout'], '10')
self.assertFalse('Content-Encoding' in result.headers)
self.assertFalse('Content-Encoding' in headers)
self.assertBackendHeaders(j['Incoming Headers'], parameter_dict['domain'])
self.assertEqual(
'secured=value;secure, nonsecured=value',
result.headers['Set-Cookie']
headers['Set-Cookie']
)
self.assertLastLogLineRegexp(
......@@ -2150,9 +2247,11 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
result_http.status_code
)
headers = self.assertResponseHeaders(
result_http, via=False, backend_reached=False)
self.assertEqual(
'https://url.example.com:%s/test-path/deeper' % (HTTP_PORT,),
result_http.headers['Location']
headers['Location']
)
# check that timeouts are correctly set in the haproxy configuration
......@@ -2318,24 +2417,13 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
)
def test_compressed_result(self):
reference = 'Url'
parameter_dict = self.parseSlaveParameterDict(reference)
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict, '')
hostname = reference.translate(None, '_-').lower()
self.assertEqual(
parameter_dict = self.assertSlaveBase(
'Url',
{
'domain': '%s.example.com' % (hostname,),
'replication_number': '1',
'url': 'http://%s.example.com' % (hostname, ),
'site_url': 'http://%s.example.com' % (hostname, ),
'secure_access': 'https://%s.example.com' % (hostname, ),
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
'warning-list': [
"slave url ' %s/?a=b&c= ' has been converted to '%s/?a=b&c='" % (
self.backend_url, self.backend_url)],
},
parameter_dict
}
)
result_compressed = fakeHTTPSResult(
......@@ -2370,24 +2458,13 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
self.assertFalse('Content-Encoding' in result_not_compressed.headers)
def test_no_content_type_alter(self):
reference = 'Url'
parameter_dict = self.parseSlaveParameterDict(reference)
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict, '')
hostname = reference.translate(None, '_-').lower()
self.assertEqual(
parameter_dict = self.assertSlaveBase(
'Url',
{
'domain': '%s.example.com' % (hostname,),
'replication_number': '1',
'url': 'http://%s.example.com' % (hostname, ),
'site_url': 'http://%s.example.com' % (hostname, ),
'secure_access': 'https://%s.example.com' % (hostname, ),
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
'warning-list': [
"slave url ' %s/?a=b&c= ' has been converted to '%s/?a=b&c='" % (
self.backend_url, self.backend_url)],
},
parameter_dict
}
)
result = fakeHTTPSResult(
parameter_dict['domain'],
......@@ -2556,20 +2633,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
)
def test_server_alias_wildcard(self):
parameter_dict = self.parseSlaveParameterDict('server-alias-wildcard')
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict)
self.assertEqual(
{
'domain': 'serveraliaswildcard.example.com',
'replication_number': '1',
'url': 'http://serveraliaswildcard.example.com',
'site_url': 'http://serveraliaswildcard.example.com',
'secure_access': 'https://serveraliaswildcard.example.com',
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
parameter_dict = self.assertSlaveBase('server-alias-wildcard')
result = fakeHTTPSResult(
parameter_dict['domain'], 'test-path')
......@@ -2596,20 +2660,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
self.assertEqualResultJson(result, 'Path', '/test-path')
def test_server_alias_duplicated(self):
parameter_dict = self.parseSlaveParameterDict('server-alias-duplicated')
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict)
self.assertEqual(
{
'domain': 'serveraliasduplicated.example.com',
'replication_number': '1',
'url': 'http://serveraliasduplicated.example.com',
'site_url': 'http://serveraliasduplicated.example.com',
'secure_access': 'https://serveraliasduplicated.example.com',
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
parameter_dict = self.assertSlaveBase('server-alias-duplicated')
result = fakeHTTPSResult(
parameter_dict['domain'], 'test-path')
......@@ -2630,22 +2681,8 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
self.assertEqualResultJson(result, 'Path', '/test-path')
def test_server_alias_custom_domain_duplicated(self):
parameter_dict = self.parseSlaveParameterDict(
'server-alias_custom_domain-duplicated')
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict)
self.assertEqual(
{
'domain': 'alias4.example.com',
'replication_number': '1',
'url': 'http://alias4.example.com',
'site_url': 'http://alias4.example.com',
'secure_access': 'https://alias4.example.com',
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
parameter_dict = self.assertSlaveBase(
'server-alias_custom_domain-duplicated', hostname='alias4')
result = fakeHTTPSResult(
parameter_dict['domain'], 'test-path')
......@@ -2661,26 +2698,12 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
raise NotImplementedError(self.id())
def test_ssl_ca_crt(self):
parameter_dict = self.parseSlaveParameterDict(
parameter_dict = self.assertSlaveBase(
'custom_domain_ssl_crt_ssl_key_ssl_ca_crt')
self.assertLogAccessUrlWithPop(parameter_dict)
generate_auth, upload_url = self.assertKedifaKeysWithPop(parameter_dict)
self.assertEqual(
{
'domain': 'customdomainsslcrtsslkeysslcacrt.example.com',
'replication_number': '1',
'url': 'http://customdomainsslcrtsslkeysslcacrt.example.com',
'site_url': 'http://customdomainsslcrtsslkeysslcacrt.example.com',
'secure_access':
'https://customdomainsslcrtsslkeysslcacrt.example.com',
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
# as now the place to put the key is known put the key there
auth = requests.get(
generate_auth,
self.current_generate_auth,
verify=self.kedifa_caucase_ca_certificate_file)
self.assertEqual(httplib.CREATED, auth.status_code)
......@@ -2689,7 +2712,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
self.ca.certificate_pem
upload = requests.put(
upload_url + auth.text,
self.current_upload_url + auth.text,
data=data,
verify=self.kedifa_caucase_ca_certificate_file)
self.assertEqual(httplib.CREATED, upload.status_code)
......@@ -2713,31 +2736,17 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
self.assertEqual(data, out.read())
def test_ssl_ca_crt_only(self):
parameter_dict = self.parseSlaveParameterDict('ssl_ca_crt_only')
self.assertLogAccessUrlWithPop(parameter_dict)
generate_auth, upload_url = self.assertKedifaKeysWithPop(parameter_dict)
self.assertEqual(
{
'domain': 'sslcacrtonly.example.com',
'replication_number': '1',
'url': 'http://sslcacrtonly.example.com',
'site_url': 'http://sslcacrtonly.example.com',
'secure_access':
'https://sslcacrtonly.example.com',
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
self.assertSlaveBase('ssl_ca_crt_only')
# as now the place to put the key is known put the key there
auth = requests.get(
generate_auth,
self.current_generate_auth,
verify=self.kedifa_caucase_ca_certificate_file)
self.assertEqual(httplib.CREATED, auth.status_code)
data = self.ca.certificate_pem
upload = requests.put(
upload_url + auth.text,
self.current_upload_url + auth.text,
data=data,
verify=self.kedifa_caucase_ca_certificate_file)
......@@ -2745,25 +2754,11 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
self.assertEqual('Key incorrect', upload.text)
def test_ssl_ca_crt_garbage(self):
parameter_dict = self.parseSlaveParameterDict('ssl_ca_crt_garbage')
self.assertLogAccessUrlWithPop(parameter_dict)
generate_auth, upload_url = self.assertKedifaKeysWithPop(parameter_dict)
self.assertEqual(
{
'domain': 'sslcacrtgarbage.example.com',
'replication_number': '1',
'url': 'http://sslcacrtgarbage.example.com',
'site_url': 'http://sslcacrtgarbage.example.com',
'secure_access':
'https://sslcacrtgarbage.example.com',
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
parameter_dict = self.assertSlaveBase('ssl_ca_crt_garbage')
# as now the place to put the key is known put the key there
auth = requests.get(
generate_auth,
self.current_generate_auth,
verify=self.kedifa_caucase_ca_certificate_file)
self.assertEqual(httplib.CREATED, auth.status_code)
......@@ -2773,7 +2768,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
data = ca_certificate_pem + ca_key_pem + 'some garbage'
upload = requests.put(
upload_url + auth.text,
self.current_upload_url + auth.text,
data=data,
verify=self.kedifa_caucase_ca_certificate_file)
......@@ -2799,31 +2794,17 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
self.assertEqual(data, out.read())
def test_ssl_ca_crt_does_not_match(self):
parameter_dict = self.parseSlaveParameterDict('ssl_ca_crt_does_not_match')
self.assertLogAccessUrlWithPop(parameter_dict)
generate_auth, upload_url = self.assertKedifaKeysWithPop(parameter_dict)
self.assertEqual(
{
'domain': 'sslcacrtdoesnotmatch.example.com',
'replication_number': '1',
'url': 'http://sslcacrtdoesnotmatch.example.com',
'site_url': 'http://sslcacrtdoesnotmatch.example.com',
'secure_access':
'https://sslcacrtdoesnotmatch.example.com',
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
parameter_dict = self.assertSlaveBase('ssl_ca_crt_does_not_match')
# as now the place to put the key is known put the key there
auth = requests.get(
generate_auth,
self.current_generate_auth,
verify=self.kedifa_caucase_ca_certificate_file)
self.assertEqual(httplib.CREATED, auth.status_code)
data = self.certificate_pem + self.key_pem + self.ca.certificate_pem
upload = requests.put(
upload_url + auth.text,
self.current_upload_url + auth.text,
data=data,
verify=self.kedifa_caucase_ca_certificate_file)
......@@ -2867,22 +2848,8 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
self.assertEqualResultJson(result_http, 'Path', '/test-path/deeper')
def test_custom_domain(self):
reference = 'custom_domain'
hostname = 'mycustomdomain'
parameter_dict = self.parseSlaveParameterDict(reference)
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict, '')
self.assertEqual(
{
'domain': '%s.example.com' % (hostname,),
'replication_number': '1',
'url': 'http://%s.example.com' % (hostname, ),
'site_url': 'http://%s.example.com' % (hostname, ),
'secure_access': 'https://%s.example.com' % (hostname, ),
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
parameter_dict = self.assertSlaveBase(
'custom_domain', hostname='mycustomdomain')
result = fakeHTTPSResult(
parameter_dict['domain'], 'test-path')
......@@ -2894,22 +2861,8 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
self.assertEqualResultJson(result, 'Path', '/test-path')
def test_custom_domain_server_alias(self):
reference = 'custom_domain_server_alias'
hostname = 'mycustomdomainserveralias'
parameter_dict = self.parseSlaveParameterDict(reference)
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict, '')
self.assertEqual(
{
'domain': '%s.example.com' % (hostname,),
'replication_number': '1',
'url': 'http://%s.example.com' % (hostname, ),
'site_url': 'http://%s.example.com' % (hostname, ),
'secure_access': 'https://%s.example.com' % (hostname, ),
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
parameter_dict = self.assertSlaveBase(
'custom_domain_server_alias', hostname='mycustomdomainserveralias')
result = fakeHTTPSResult(
parameter_dict['domain'], 'test-path')
......@@ -2931,20 +2884,8 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
self.assertEqualResultJson(result, 'Path', '/test-path/deeper')
def test_custom_domain_wildcard(self):
parameter_dict = self.parseSlaveParameterDict('custom_domain_wildcard')
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict)
self.assertEqual(
{
'domain': '*.customdomain.example.com',
'replication_number': '1',
'url': 'http://*.customdomain.example.com',
'site_url': 'http://*.customdomain.example.com',
'secure_access': 'https://*.customdomain.example.com',
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
self.assertSlaveBase(
'custom_domain_wildcard', hostname='*.customdomain')
result = fakeHTTPSResult(
'wild.customdomain.example.com',
......@@ -2957,33 +2898,17 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
self.assertEqualResultJson(result, 'Path', '/test-path')
def test_custom_domain_ssl_crt_ssl_key(self):
reference = 'custom_domain_ssl_crt_ssl_key'
parameter_dict = self.parseSlaveParameterDict(reference)
self.assertLogAccessUrlWithPop(parameter_dict)
generate_auth, upload_url = self.assertKedifaKeysWithPop(parameter_dict)
hostname = reference.translate(None, '_-')
self.assertEqual(
{
'domain': '%s.example.com' % (hostname,),
'replication_number': '1',
'url': 'http://%s.example.com' % (hostname, ),
'site_url': 'http://%s.example.com' % (hostname, ),
'secure_access': 'https://%s.example.com' % (hostname, ),
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
parameter_dict = self.assertSlaveBase('custom_domain_ssl_crt_ssl_key')
# as now the place to put the key is known put the key there
auth = requests.get(
generate_auth,
self.current_generate_auth,
verify=self.kedifa_caucase_ca_certificate_file)
self.assertEqual(httplib.CREATED, auth.status_code)
data = self.customdomain_certificate_pem + \
self.customdomain_key_pem
upload = requests.put(
upload_url + auth.text,
self.current_upload_url + auth.text,
data=data,
verify=self.kedifa_caucase_ca_certificate_file)
self.assertEqual(httplib.CREATED, upload.status_code)
......@@ -3230,22 +3155,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
)
def test_type_notebook(self):
reference = 'type-notebook'
parameter_dict = self.parseSlaveParameterDict(reference)
self.assertLogAccessUrlWithPop(parameter_dict)
hostname = reference.translate(None, '_-')
self.assertKedifaKeysWithPop(parameter_dict)
self.assertEqual(
{
'domain': '%s.example.com' % (hostname,),
'replication_number': '1',
'url': 'http://%s.example.com' % (hostname, ),
'site_url': 'http://%s.example.com' % (hostname, ),
'secure_access': 'https://%s.example.com' % (hostname, ),
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
parameter_dict = self.assertSlaveBase('type-notebook')
result = fakeHTTPSResult(
parameter_dict['domain'],
......@@ -3504,22 +3414,8 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
)
def test_type_redirect_custom_domain(self):
reference = 'type-redirect-custom_domain'
hostname = 'customdomaintyperedirect'
parameter_dict = self.parseSlaveParameterDict(reference)
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict, '')
self.assertEqual(
{
'domain': '%s.example.com' % (hostname,),
'replication_number': '1',
'url': 'http://%s.example.com' % (hostname, ),
'site_url': 'http://%s.example.com' % (hostname, ),
'secure_access': 'https://%s.example.com' % (hostname, ),
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
parameter_dict = self.assertSlaveBase(
'type-redirect-custom_domain', hostname='customdomaintyperedirect')
result = fakeHTTPSResult(
parameter_dict['domain'],
......@@ -3540,25 +3436,9 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
)
def test_ssl_proxy_verify_ssl_proxy_ca_crt_unverified(self):
parameter_dict = self.parseSlaveParameterDict(
parameter_dict = self.assertSlaveBase(
'ssl-proxy-verify_ssl_proxy_ca_crt-unverified')
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict)
self.assertEqual(
{
'domain': 'sslproxyverifysslproxycacrtunverified.example.com',
'replication_number': '1',
'url': 'http://sslproxyverifysslproxycacrtunverified.example.com',
'site_url':
'http://sslproxyverifysslproxycacrtunverified.example.com',
'secure_access':
'https://sslproxyverifysslproxycacrtunverified.example.com',
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
result = fakeHTTPSResult(
parameter_dict['domain'], 'test-path')
......@@ -3749,22 +3629,9 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
)
def test_enable_cache_custom_domain(self):
reference = 'enable_cache_custom_domain'
hostname = 'customdomainenablecache'
parameter_dict = self.parseSlaveParameterDict(reference)
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict, '')
self.assertEqual(
{
'domain': '%s.example.com' % (hostname,),
'replication_number': '1',
'url': 'http://%s.example.com' % (hostname, ),
'site_url': 'http://%s.example.com' % (hostname, ),
'secure_access': 'https://%s.example.com' % (hostname, ),
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
parameter_dict = self.assertSlaveBase(
'enable_cache_custom_domain',
hostname='customdomainenablecache')
result = fakeHTTPSResult(
parameter_dict['domain'],
......@@ -3774,7 +3641,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
self.assertEqualResultJson(result, 'Path', '/test-path/deeper')
headers = self.assertResponseHeaders(result)
headers = self.assertResponseHeaders(result, True)
self.assertKeyWithPop('Age', headers)
self.assertEqual(
......@@ -3788,13 +3655,8 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
)
backend_headers = result.json()['Incoming Headers']
self.assertBackendHeaders(backend_headers, parameter_dict['domain'])
via = backend_headers.pop('via', None)
self.assertNotEqual(via, None)
self.assertRegexpMatches(
via,
VIA_STRING
)
self.assertBackendHeaders(
backend_headers, parameter_dict['domain'], cached=True)
def test_enable_cache_server_alias(self):
parameter_dict = self.assertSlaveBase('enable_cache_server_alias')
......@@ -3807,7 +3669,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
self.assertEqualResultJson(result, 'Path', '/test-path/deeper')
headers = self.assertResponseHeaders(result)
headers = self.assertResponseHeaders(result, cached=True)
self.assertKeyWithPop('Age', headers)
self.assertEqual(
......@@ -3821,13 +3683,8 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
)
backend_headers = result.json()['Incoming Headers']
self.assertBackendHeaders(backend_headers, parameter_dict['domain'])
via = backend_headers.pop('via', None)
self.assertNotEqual(via, None)
self.assertRegexpMatches(
via,
VIA_STRING
)
self.assertBackendHeaders(
backend_headers, parameter_dict['domain'], cached=True)
result = fakeHTTPResult(
'enablecacheserveralias1.example.com',
......@@ -3856,7 +3713,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
self.assertEqualResultJson(result, 'Path', '/test-path/deeper')
headers = self.assertResponseHeaders(result)
headers = self.assertResponseHeaders(result, cached=True)
self.assertKeyWithPop('Age', headers)
self.assertEqual(
{
......@@ -3877,17 +3734,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
self.assertEqual(httplib.OK, result.status_code)
self.assertEqualResultJson(result, 'Path', '/HTTPS/test')
self.assertResponseHeaders(result)
result = fakeHTTPSResult(
parameter_dict['domain'],
'HTTP/test', headers={
'X-Reply-Header-Cache-Control': 'max-age=1, stale-while-'
'revalidate=3600, stale-if-error=3600'})
self.assertEqual(httplib.OK, result.status_code)
self.assertEqualResultJson(result, 'Path', '/HTTP/test')
self.assertResponseHeaders(result)
self.assertResponseHeaders(result, cached=True)
def test_enable_cache(self):
parameter_dict = self.assertSlaveBase('enable_cache')
......@@ -3904,7 +3751,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
self.assertEqualResultJson(result, 'Path', '/test-path/deeper')
headers = self.assertResponseHeaders(result)
headers = self.assertResponseHeaders(result, cached=True)
self.assertKeyWithPop('Age', headers)
......@@ -3919,13 +3766,8 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
)
backend_headers = result.json()['Incoming Headers']
self.assertBackendHeaders(backend_headers, parameter_dict['domain'])
via = backend_headers.pop('via', None)
self.assertNotEqual(via, None)
self.assertRegexpMatches(
via,
VIA_STRING
)
self.assertBackendHeaders(
backend_headers, parameter_dict['domain'], cached=True)
# BEGIN: Check that squid.log is correctly filled in
ats_log_file_list = glob.glob(
......@@ -4113,13 +3955,8 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
)
backend_headers = result.json()['Incoming Headers']
self.assertBackendHeaders(backend_headers, parameter_dict['domain'])
via = backend_headers.pop('via', None)
self.assertNotEqual(via, None)
self.assertRegexpMatches(
via,
VIA_STRING
)
self.assertBackendHeaders(
backend_headers, parameter_dict['domain'], cached=True)
# check stale-if-error support is really respected if not present in the
# request
......@@ -4235,7 +4072,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
self.assertEqualResultJson(result, 'Path', '/test-path')
headers = self.assertResponseHeaders(result)
headers = self.assertResponseHeaders(result, cached=True)
self.assertKeyWithPop('Age', headers)
......@@ -4248,13 +4085,8 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
)
backend_headers = result.json()['Incoming Headers']
self.assertBackendHeaders(backend_headers, parameter_dict['domain'])
via = backend_headers.pop('via', None)
self.assertNotEqual(via, None)
self.assertRegexpMatches(
via,
VIA_STRING
)
self.assertBackendHeaders(
backend_headers, parameter_dict['domain'], cached=True)
try:
j = result.json()
......@@ -4274,7 +4106,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
self.assertEqualResultJson(result, 'Path', '/test-path')
headers = self.assertResponseHeaders(result)
headers = self.assertResponseHeaders(result, via=False)
self.assertKeyWithPop('Age', headers)
......@@ -4287,13 +4119,8 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
)
backend_headers = result.json()['Incoming Headers']
self.assertBackendHeaders(backend_headers, parameter_dict['domain'])
via = backend_headers.pop('via', None)
self.assertNotEqual(via, None)
self.assertRegexpMatches(
via,
VIA_STRING
)
self.assertBackendHeaders(
backend_headers, parameter_dict['domain'], cached=True)
def test_enable_http2_false(self):
parameter_dict = self.assertSlaveBase('enable-http2-false')
......@@ -4664,6 +4491,24 @@ class TestReplicateSlave(SlaveHttpFrontendTestCase, TestDataMixin):
parameter_dict = self.parseSlaveParameterDict('replicate')
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict)
key_list = [
'caddy-frontend-1-node-information-json',
'caddy-frontend-2-node-information-json'
]
node_information_json_dict = {}
for k in parameter_dict.keys():
if k.startswith('caddy-frontend') and k.endswith(
'node-information-json'):
node_information_json_dict[k] = parameter_dict.pop(k)
self.assertEqual(
key_list,
node_information_json_dict.keys()
)
node_information_dict = json.loads(node_information_json_dict[key_list[0]])
self.assertIn("node-id", node_information_dict)
self.assertIn("version-hash-history", node_information_dict)
self.node_information_dict = node_information_dict
self.assertEqual(
{
'domain': 'replicate.example.com',
......@@ -4792,61 +4637,19 @@ class TestEnableHttp2ByDefaultFalseSlave(SlaveHttpFrontendTestCase,
}
def test_enable_http2_default(self):
parameter_dict = self.parseSlaveParameterDict('enable-http2-default')
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict)
self.assertEqual(
{
'domain': 'enablehttp2default.example.com',
'replication_number': '1',
'url': 'http://enablehttp2default.example.com',
'site_url': 'http://enablehttp2default.example.com',
'secure_access':
'https://enablehttp2default.example.com',
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
parameter_dict = self.assertSlaveBase('enable-http2-default')
self.assertFalse(
isHTTP2(parameter_dict['domain']))
def test_enable_http2_false(self):
parameter_dict = self.parseSlaveParameterDict('enable-http2-false')
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict)
self.assertEqual(
{
'domain': 'enablehttp2false.example.com',
'replication_number': '1',
'url': 'http://enablehttp2false.example.com',
'site_url': 'http://enablehttp2false.example.com',
'secure_access':
'https://enablehttp2false.example.com',
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
parameter_dict = self.assertSlaveBase('enable-http2-false')
self.assertFalse(
isHTTP2(parameter_dict['domain']))
def test_enable_http2_true(self):
parameter_dict = self.parseSlaveParameterDict('enable-http2-true')
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict)
self.assertEqual(
{
'domain': 'enablehttp2true.example.com',
'replication_number': '1',
'url': 'http://enablehttp2true.example.com',
'site_url': 'http://enablehttp2true.example.com',
'secure_access':
'https://enablehttp2true.example.com',
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
parameter_dict = self.assertSlaveBase('enable-http2-true')
self.assertTrue(
isHTTP2(parameter_dict['domain']))
......@@ -4882,61 +4685,19 @@ class TestEnableHttp2ByDefaultDefaultSlave(SlaveHttpFrontendTestCase,
}
def test_enable_http2_default(self):
parameter_dict = self.parseSlaveParameterDict('enable-http2-default')
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict)
self.assertEqual(
{
'domain': 'enablehttp2default.example.com',
'replication_number': '1',
'url': 'http://enablehttp2default.example.com',
'site_url': 'http://enablehttp2default.example.com',
'secure_access':
'https://enablehttp2default.example.com',
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
parameter_dict = self.assertSlaveBase('enable-http2-default')
self.assertTrue(
isHTTP2(parameter_dict['domain']))
def test_enable_http2_false(self):
parameter_dict = self.parseSlaveParameterDict('enable-http2-false')
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict)
self.assertEqual(
{
'domain': 'enablehttp2false.example.com',
'replication_number': '1',
'url': 'http://enablehttp2false.example.com',
'site_url': 'http://enablehttp2false.example.com',
'secure_access':
'https://enablehttp2false.example.com',
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
parameter_dict = self.assertSlaveBase('enable-http2-false')
self.assertFalse(
isHTTP2(parameter_dict['domain']))
def test_enable_http2_true(self):
parameter_dict = self.parseSlaveParameterDict('enable-http2-true')
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict)
self.assertEqual(
{
'domain': 'enablehttp2true.example.com',
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
'replication_number': '1',
'url': 'http://enablehttp2true.example.com',
'site_url': 'http://enablehttp2true.example.com',
'secure_access':
'https://enablehttp2true.example.com',
},
parameter_dict
)
parameter_dict = self.assertSlaveBase('enable-http2-true')
self.assertTrue(
isHTTP2(parameter_dict['domain']))
......@@ -4947,6 +4708,7 @@ class TestRe6stVerificationUrlDefaultSlave(SlaveHttpFrontendTestCase,
@classmethod
def getInstanceParameterDict(cls):
return {
'domain': 'example.com',
'port': HTTPS_PORT,
'plain_http_port': HTTP_PORT,
'kedifa_port': KEDIFA_PORT,
......@@ -4968,20 +4730,7 @@ class TestRe6stVerificationUrlDefaultSlave(SlaveHttpFrontendTestCase,
return True
def test_default(self):
parameter_dict = self.parseSlaveParameterDict('default')
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict)
self.assertEqual(
{
'domain': 'default.None',
'replication_number': '1',
'url': 'http://default.None',
'site_url': 'http://default.None',
'secure_access': 'https://default.None',
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
self.assertSlaveBase('default')
re6st_connectivity_promise_list = glob.glob(
os.path.join(
......@@ -5034,20 +4783,7 @@ class TestRe6stVerificationUrlSlave(SlaveHttpFrontendTestCase,
except Exception:
pass
parameter_dict = self.parseSlaveParameterDict('default')
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict)
self.assertEqual(
{
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
'domain': 'default.example.com',
'replication_number': '1',
'url': 'http://default.example.com',
'site_url': 'http://default.example.com',
'secure_access': 'https://default.example.com',
},
parameter_dict
)
self.assertSlaveBase('default')
re6st_connectivity_promise_list = glob.glob(
os.path.join(
......@@ -5074,21 +4810,7 @@ class TestSlaveGlobalDisableHttp2(TestSlave):
return instance_parameter_dict
def test_enable_http2_default(self):
parameter_dict = self.parseSlaveParameterDict('enable-http2-default')
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict)
self.assertEqual(
{
'domain': 'enablehttp2default.example.com',
'replication_number': '1',
'url': 'http://enablehttp2default.example.com',
'site_url': 'http://enablehttp2default.example.com',
'secure_access':
'https://enablehttp2default.example.com',
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
parameter_dict = self.assertSlaveBase('enable-http2-default')
result = fakeHTTPSResult(
parameter_dict['domain'], 'test-path')
......@@ -5123,21 +4845,7 @@ class TestEnableHttp2ByDefaultFalseSlaveGlobalDisableHttp2(
return instance_parameter_dict
def test_enable_http2_true(self):
parameter_dict = self.parseSlaveParameterDict('enable-http2-true')
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict)
self.assertEqual(
{
'domain': 'enablehttp2true.example.com',
'replication_number': '1',
'url': 'http://enablehttp2true.example.com',
'site_url': 'http://enablehttp2true.example.com',
'secure_access':
'https://enablehttp2true.example.com',
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
parameter_dict = self.assertSlaveBase('enable-http2-true')
self.assertFalse(
isHTTP2(parameter_dict['domain']))
......@@ -5154,41 +4862,13 @@ class TestEnableHttp2ByDefaultDefaultSlaveGlobalDisableHttp2(
return instance_parameter_dict
def test_enable_http2_true(self):
parameter_dict = self.parseSlaveParameterDict('enable-http2-true')
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict)
self.assertEqual(
{
'domain': 'enablehttp2true.example.com',
'replication_number': '1',
'url': 'http://enablehttp2true.example.com',
'site_url': 'http://enablehttp2true.example.com',
'secure_access':
'https://enablehttp2true.example.com',
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
parameter_dict = self.assertSlaveBase('enable-http2-true')
self.assertFalse(
isHTTP2(parameter_dict['domain']))
def test_enable_http2_default(self):
parameter_dict = self.parseSlaveParameterDict('enable-http2-default')
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict)
self.assertEqual(
{
'domain': 'enablehttp2default.example.com',
'replication_number': '1',
'url': 'http://enablehttp2default.example.com',
'site_url': 'http://enablehttp2default.example.com',
'secure_access':
'https://enablehttp2default.example.com',
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
parameter_dict = self.assertSlaveBase('enable-http2-default')
self.assertFalse(
isHTTP2(parameter_dict['domain']))
......@@ -5229,22 +4909,8 @@ class TestSlaveSlapOSMasterCertificateCompatibilityOverrideMaster(
}
def test_ssl_from_master_kedifa_overrides_master_certificate(self):
reference = 'ssl_from_master_kedifa_overrides_master_certificate'
parameter_dict = self.parseSlaveParameterDict(reference)
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict)
hostname = reference.translate(None, '_-')
self.assertEqual(
{
'domain': '%s.example.com' % (hostname,),
'replication_number': '1',
'url': 'http://%s.example.com' % (hostname, ),
'site_url': 'http://%s.example.com' % (hostname, ),
'secure_access': 'https://%s.example.com' % (hostname, ),
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
parameter_dict = self.assertSlaveBase(
'ssl_from_master_kedifa_overrides_master_certificate')
result = fakeHTTPSResult(
parameter_dict['domain'], 'test-path')
......@@ -5426,6 +5092,7 @@ class TestSlaveSlapOSMasterCertificateCompatibility(
self.assertKeyWithPop('monitor-setup-url', parameter_dict)
self.assertBackendHaproxyStatisticUrl(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict, 'master-')
self.assertNodeInformationWithPop(parameter_dict)
self.assertRejectedSlavePromiseEmptyWithPop(parameter_dict)
expected_parameter_dict = {
......@@ -5493,21 +5160,7 @@ class TestSlaveSlapOSMasterCertificateCompatibility(
)
def test_ssl_from_master(self):
parameter_dict = self.parseSlaveParameterDict('ssl_from_master')
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict, '')
hostname = 'ssl_from_master'.translate(None, '_-')
self.assertEqual(
{
'domain': '%s.example.com' % (hostname,),
'replication_number': '1',
'url': 'http://%s.example.com' % (hostname, ),
'site_url': 'http://%s.example.com' % (hostname, ),
'secure_access': 'https://%s.example.com' % (hostname, ),
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
parameter_dict = self.assertSlaveBase('ssl_from_master')
result = fakeHTTPSResult(
parameter_dict['domain'], 'test-path')
......@@ -5519,22 +5172,7 @@ class TestSlaveSlapOSMasterCertificateCompatibility(
self.assertEqualResultJson(result, 'Path', '/test-path')
def test_ssl_from_master_kedifa_overrides(self):
reference = 'ssl_from_master_kedifa_overrides'
parameter_dict = self.parseSlaveParameterDict(reference)
self.assertLogAccessUrlWithPop(parameter_dict)
generate_auth, upload_url = self.assertKedifaKeysWithPop(parameter_dict)
hostname = reference.translate(None, '_-')
self.assertEqual(
{
'domain': '%s.example.com' % (hostname,),
'replication_number': '1',
'url': 'http://%s.example.com' % (hostname, ),
'site_url': 'http://%s.example.com' % (hostname, ),
'secure_access': 'https://%s.example.com' % (hostname, ),
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
parameter_dict = self.assertSlaveBase('ssl_from_master_kedifa_overrides')
result = fakeHTTPSResult(
parameter_dict['domain'], 'test-path')
......@@ -5550,14 +5188,14 @@ class TestSlaveSlapOSMasterCertificateCompatibility(
# as now the place to put the key is known put the key there
auth = requests.get(
generate_auth,
self.current_generate_auth,
verify=self.kedifa_caucase_ca_certificate_file)
self.assertEqual(httplib.CREATED, auth.status_code)
data = certificate_pem + key_pem
upload = requests.put(
upload_url + auth.text,
self.current_upload_url + auth.text,
data=data,
verify=self.kedifa_caucase_ca_certificate_file)
self.assertEqual(httplib.CREATED, upload.status_code)
......@@ -5573,27 +5211,14 @@ class TestSlaveSlapOSMasterCertificateCompatibility(
self.assertEqualResultJson(result, 'Path', '/test-path')
def test_ssl_from_slave(self):
reference = 'ssl_from_slave'
parameter_dict = self.parseSlaveParameterDict(reference)
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict)
hostname = reference.translate(None, '_-')
self.assertEqual(
{
'domain': '%s.example.com' % (hostname,),
'replication_number': '1',
'url': 'http://%s.example.com' % (hostname, ),
'site_url': 'http://%s.example.com' % (hostname, ),
'secure_access': 'https://%s.example.com' % (hostname, ),
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
parameter_dict = self.assertSlaveBase(
'ssl_from_slave',
expected_parameter_dict={
'warning-list': [
'ssl_crt is obsolete, please use key-upload-url',
'ssl_key is obsolete, please use key-upload-url',
]
},
parameter_dict
)
})
result = fakeHTTPSResult(
parameter_dict['domain'], 'test-path')
......@@ -5605,27 +5230,12 @@ class TestSlaveSlapOSMasterCertificateCompatibility(
self.assertEqualResultJson(result, 'Path', '/test-path')
def test_ssl_from_slave_kedifa_overrides(self):
reference = 'ssl_from_slave_kedifa_overrides'
parameter_dict = self.parseSlaveParameterDict(reference)
self.assertLogAccessUrlWithPop(parameter_dict)
generate_auth, upload_url = self.assertKedifaKeysWithPop(parameter_dict)
hostname = reference.translate(None, '_-')
self.assertEqual(
{
'domain': '%s.example.com' % (hostname,),
'replication_number': '1',
'url': 'http://%s.example.com' % (hostname, ),
'site_url': 'http://%s.example.com' % (hostname, ),
'secure_access': 'https://%s.example.com' % (hostname, ),
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
'warning-list': [
'ssl_crt is obsolete, please use key-upload-url',
'ssl_key is obsolete, please use key-upload-url',
]
},
parameter_dict
)
parameter_dict = self.assertSlaveBase(
'ssl_from_slave_kedifa_overrides',
expected_parameter_dict={
'warning-list': ['ssl_crt is obsolete, please use key-upload-url',
'ssl_key is obsolete, please use key-upload-url']
})
result = fakeHTTPSResult(
parameter_dict['domain'], 'test-path')
......@@ -5641,14 +5251,14 @@ class TestSlaveSlapOSMasterCertificateCompatibility(
# as now the place to put the key is known put the key there
auth = requests.get(
generate_auth,
self.current_generate_auth,
verify=self.kedifa_caucase_ca_certificate_file)
self.assertEqual(httplib.CREATED, auth.status_code)
data = certificate_pem + key_pem
upload = requests.put(
upload_url + auth.text,
self.current_upload_url + auth.text,
data=data,
verify=self.kedifa_caucase_ca_certificate_file)
self.assertEqual(httplib.CREATED, upload.status_code)
......@@ -5665,22 +5275,7 @@ class TestSlaveSlapOSMasterCertificateCompatibility(
self.assertEqualResultJson(result, 'Path', '/test-path')
def test_type_notebook_ssl_from_master(self):
reference = 'type-notebook-ssl_from_master'
parameter_dict = self.parseSlaveParameterDict(reference)
self.assertLogAccessUrlWithPop(parameter_dict)
hostname = reference.translate(None, '_-')
self.assertKedifaKeysWithPop(parameter_dict)
self.assertEqual(
{
'domain': '%s.example.com' % (hostname,),
'replication_number': '1',
'url': 'http://%s.example.com' % (hostname, ),
'site_url': 'http://%s.example.com' % (hostname, ),
'secure_access': 'https://%s.example.com' % (hostname, ),
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
parameter_dict = self.assertSlaveBase('type-notebook-ssl_from_master')
result = fakeHTTPSResult(
parameter_dict['domain'], 'test-path',
......@@ -5693,22 +5288,8 @@ class TestSlaveSlapOSMasterCertificateCompatibility(
self.assertEqualResultJson(result, 'Path', '/test-path')
def test_type_notebook_ssl_from_master_kedifa_overrides(self):
reference = 'type-notebook-ssl_from_master_kedifa_overrides'
parameter_dict = self.parseSlaveParameterDict(reference)
self.assertLogAccessUrlWithPop(parameter_dict)
generate_auth, upload_url = self.assertKedifaKeysWithPop(parameter_dict)
hostname = reference.translate(None, '_-')
self.assertEqual(
{
'domain': '%s.example.com' % (hostname,),
'replication_number': '1',
'url': 'http://%s.example.com' % (hostname, ),
'site_url': 'http://%s.example.com' % (hostname, ),
'secure_access': 'https://%s.example.com' % (hostname, ),
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
parameter_dict = self.assertSlaveBase(
'type-notebook-ssl_from_master_kedifa_overrides')
result = fakeHTTPSResult(
parameter_dict['domain'], 'test-path',
......@@ -5725,14 +5306,14 @@ class TestSlaveSlapOSMasterCertificateCompatibility(
# as now the place to put the key is known put the key there
auth = requests.get(
generate_auth,
self.current_generate_auth,
verify=self.kedifa_caucase_ca_certificate_file)
self.assertEqual(httplib.CREATED, auth.status_code)
data = certificate_pem + key_pem
upload = requests.put(
upload_url + auth.text,
self.current_upload_url + auth.text,
data=data,
verify=self.kedifa_caucase_ca_certificate_file)
self.assertEqual(httplib.CREATED, upload.status_code)
......@@ -5750,26 +5331,14 @@ class TestSlaveSlapOSMasterCertificateCompatibility(
self.assertEqualResultJson(result, 'Path', '/test-path')
def test_type_notebook_ssl_from_slave(self):
reference = 'type-notebook-ssl_from_slave'
parameter_dict = self.parseSlaveParameterDict(reference)
self.assertLogAccessUrlWithPop(parameter_dict)
hostname = reference.translate(None, '_-')
self.assertKedifaKeysWithPop(parameter_dict)
self.assertEqual(
{
'domain': '%s.example.com' % (hostname,),
'replication_number': '1',
'url': 'http://%s.example.com' % (hostname, ),
'site_url': 'http://%s.example.com' % (hostname, ),
'secure_access': 'https://%s.example.com' % (hostname, ),
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
parameter_dict = self.assertSlaveBase(
'type-notebook-ssl_from_slave',
expected_parameter_dict={
'warning-list': [
'ssl_crt is obsolete, please use key-upload-url',
'ssl_key is obsolete, please use key-upload-url',
]
},
parameter_dict
)
})
result = fakeHTTPSResult(
parameter_dict['domain'], 'test-path',
......@@ -5782,26 +5351,12 @@ class TestSlaveSlapOSMasterCertificateCompatibility(
self.assertEqualResultJson(result, 'Path', '/test-path')
def test_type_notebook_ssl_from_slave_kedifa_overrides(self):
reference = 'type-notebook-ssl_from_slave_kedifa_overrides'
parameter_dict = self.parseSlaveParameterDict(reference)
self.assertLogAccessUrlWithPop(parameter_dict)
generate_auth, upload_url = self.assertKedifaKeysWithPop(parameter_dict)
hostname = reference.translate(None, '_-')
self.assertEqual(
{
'domain': '%s.example.com' % (hostname,),
'replication_number': '1',
'url': 'http://%s.example.com' % (hostname, ),
'site_url': 'http://%s.example.com' % (hostname, ),
'secure_access': 'https://%s.example.com' % (hostname, ),
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
'warning-list': [
'ssl_crt is obsolete, please use key-upload-url',
'ssl_key is obsolete, please use key-upload-url',
]
},
parameter_dict
)
parameter_dict = self.assertSlaveBase(
'type-notebook-ssl_from_slave_kedifa_overrides',
expected_parameter_dict={
'warning-list': ['ssl_crt is obsolete, please use key-upload-url',
'ssl_key is obsolete, please use key-upload-url']
})
result = fakeHTTPSResult(
parameter_dict['domain'], 'test-path',
......@@ -5818,14 +5373,14 @@ class TestSlaveSlapOSMasterCertificateCompatibility(
# as now the place to put the key is known put the key there
auth = requests.get(
generate_auth,
self.current_generate_auth,
verify=self.kedifa_caucase_ca_certificate_file)
self.assertEqual(httplib.CREATED, auth.status_code)
data = certificate_pem + key_pem
upload = requests.put(
upload_url + auth.text,
self.current_upload_url + auth.text,
data=data,
verify=self.kedifa_caucase_ca_certificate_file)
self.assertEqual(httplib.CREATED, upload.status_code)
......@@ -5844,25 +5399,12 @@ class TestSlaveSlapOSMasterCertificateCompatibility(
@skip('Not implemented in new test system')
def test_custom_domain_ssl_crt_ssl_key(self):
reference = 'custom_domain_ssl_crt_ssl_key'
parameter_dict = self.parseSlaveParameterDict(reference)
self.assertLogAccessUrlWithPop(parameter_dict)
generate_auth, upload_url = self.assertKedifaKeysWithPop(parameter_dict)
hostname = reference.translate(None, '_-')
self.assertEqual(
{
'domain': '%s.example.com' % (hostname,),
'replication_number': '1',
'url': 'http://%s.example.com' % (hostname, ),
'site_url': 'http://%s.example.com' % (hostname, ),
'secure_access': 'https://%s.example.com' % (hostname, ),
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
parameter_dict = self.assertSlaveBase(
'custom_domain_ssl_crt_ssl_key',
expected_parameter_dict={
'warning-list': ['ssl_key is obsolete, please use key-upload-url',
'ssl_crt is obsolete, please use key-upload-url']
},
parameter_dict
)
})
result = fakeHTTPSResult(
parameter_dict['domain'], 'test-path')
......@@ -5874,27 +5416,15 @@ class TestSlaveSlapOSMasterCertificateCompatibility(
self.assertEqualResultJson(result, 'Path', '/test-path')
def test_ssl_ca_crt(self):
parameter_dict = self.parseSlaveParameterDict(
'custom_domain_ssl_crt_ssl_key_ssl_ca_crt')
self.assertLogAccessUrlWithPop(parameter_dict)
generate_auth, upload_url = self.assertKedifaKeysWithPop(parameter_dict)
self.assertEqual(
{
'domain': 'customdomainsslcrtsslkeysslcacrt.example.com',
'replication_number': '1',
'url': 'http://customdomainsslcrtsslkeysslcacrt.example.com',
'site_url': 'http://customdomainsslcrtsslkeysslcacrt.example.com',
'secure_access':
'https://customdomainsslcrtsslkeysslcacrt.example.com',
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
parameter_dict = self.assertSlaveBase(
'custom_domain_ssl_crt_ssl_key_ssl_ca_crt',
expected_parameter_dict={
'warning-list': [
'ssl_ca_crt is obsolete, please use key-upload-url',
'ssl_crt is obsolete, please use key-upload-url',
'ssl_key is obsolete, please use key-upload-url'
]
},
parameter_dict
)
})
result = fakeHTTPSResult(
parameter_dict['domain'], 'test-path')
......@@ -5963,25 +5493,14 @@ class TestSlaveSlapOSMasterCertificateCompatibility(
)
def test_ssl_ca_crt_garbage(self):
parameter_dict = self.parseSlaveParameterDict('ssl_ca_crt_garbage')
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict)
self.assertEqual(
{
'domain': 'sslcacrtgarbage.example.com',
'replication_number': '1',
'url': 'http://sslcacrtgarbage.example.com',
'site_url': 'http://sslcacrtgarbage.example.com',
'secure_access':
'https://sslcacrtgarbage.example.com',
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
parameter_dict = self.assertSlaveBase(
'ssl_ca_crt_garbage',
expected_parameter_dict={
'warning-list': [
'ssl_ca_crt is obsolete, please use key-upload-url',
'ssl_crt is obsolete, please use key-upload-url',
'ssl_key is obsolete, please use key-upload-url']
},
parameter_dict
)
})
result = fakeHTTPSResult(
parameter_dict['domain'], 'test-path')
......@@ -5994,26 +5513,15 @@ class TestSlaveSlapOSMasterCertificateCompatibility(
self.assertEqualResultJson(result, 'Path', '/test-path')
def test_ssl_ca_crt_does_not_match(self):
parameter_dict = self.parseSlaveParameterDict('ssl_ca_crt_does_not_match')
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict)
self.assertEqual(
{
'domain': 'sslcacrtdoesnotmatch.example.com',
'replication_number': '1',
'url': 'http://sslcacrtdoesnotmatch.example.com',
'site_url': 'http://sslcacrtdoesnotmatch.example.com',
'secure_access':
'https://sslcacrtdoesnotmatch.example.com',
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
parameter_dict = self.assertSlaveBase(
'ssl_ca_crt_does_not_match',
expected_parameter_dict={
'warning-list': [
'ssl_ca_crt is obsolete, please use key-upload-url',
'ssl_crt is obsolete, please use key-upload-url',
'ssl_key is obsolete, please use key-upload-url'
]
},
parameter_dict
)
})
result = fakeHTTPSResult(
parameter_dict['domain'], 'test-path')
......@@ -6080,6 +5588,7 @@ class TestSlaveSlapOSMasterCertificateCompatibilityUpdate(
self.assertKeyWithPop('monitor-setup-url', parameter_dict)
self.assertBackendHaproxyStatisticUrl(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict, 'master-')
self.assertNodeInformationWithPop(parameter_dict)
self.assertRejectedSlavePromiseEmptyWithPop(parameter_dict)
expected_parameter_dict = {
......@@ -6102,21 +5611,7 @@ class TestSlaveSlapOSMasterCertificateCompatibilityUpdate(
)
def test_apache_key_apache_certificate_update(self):
parameter_dict = self.parseSlaveParameterDict('ssl_from_master')
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict, '')
hostname = 'ssl_from_master'.translate(None, '_-')
self.assertEqual(
{
'domain': '%s.example.com' % (hostname,),
'replication_number': '1',
'url': 'http://%s.example.com' % (hostname, ),
'site_url': 'http://%s.example.com' % (hostname, ),
'secure_access': 'https://%s.example.com' % (hostname, ),
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
parameter_dict = self.assertSlaveBase('ssl_from_master')
result = fakeHTTPSResult(
parameter_dict['domain'], 'test-path')
......@@ -6185,6 +5680,7 @@ class TestSlaveCiphers(SlaveHttpFrontendTestCase, TestDataMixin):
self.assertKeyWithPop('monitor-setup-url', parameter_dict)
self.assertBackendHaproxyStatisticUrl(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict, 'master-')
self.assertNodeInformationWithPop(parameter_dict)
self.assertRejectedSlavePromiseEmptyWithPop(parameter_dict)
expected_parameter_dict = {
......@@ -6453,6 +5949,7 @@ class TestSlaveRejectReportUnsafeDamaged(SlaveHttpFrontendTestCase):
self.assertKeyWithPop('monitor-setup-url', parameter_dict)
self.assertBackendHaproxyStatisticUrl(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict, 'master-')
self.assertNodeInformationWithPop(parameter_dict)
self.assertRejectedSlavePromiseWithPop(parameter_dict)
expected_parameter_dict = {
......@@ -6542,6 +6039,7 @@ class TestSlaveRejectReportUnsafeDamaged(SlaveHttpFrontendTestCase):
def test_url(self):
parameter_dict = self.parseSlaveParameterDict('URL')
self.assertNodeInformationWithPop(parameter_dict)
self.assertEqual(
{
'request-error-list': [
......@@ -6552,6 +6050,7 @@ class TestSlaveRejectReportUnsafeDamaged(SlaveHttpFrontendTestCase):
def test_https_url(self):
parameter_dict = self.parseSlaveParameterDict('HTTPS-URL')
self.assertNodeInformationWithPop(parameter_dict)
self.assertEqual(
{
'request-error-list': [
......@@ -6563,6 +6062,7 @@ class TestSlaveRejectReportUnsafeDamaged(SlaveHttpFrontendTestCase):
def test_ssl_proxy_verify_ssl_proxy_ca_crt_damaged(self):
parameter_dict = self.parseSlaveParameterDict(
'SSL-PROXY-VERIFY_SSL_PROXY_CA_CRT_DAMAGED')
self.assertNodeInformationWithPop(parameter_dict)
self.assertEqual(
{'request-error-list': ["ssl_proxy_ca_crt is invalid"]},
parameter_dict
......@@ -6571,6 +6071,7 @@ class TestSlaveRejectReportUnsafeDamaged(SlaveHttpFrontendTestCase):
def test_ssl_proxy_verify_ssl_proxy_ca_crt_empty(self):
parameter_dict = self.parseSlaveParameterDict(
'SSL-PROXY-VERIFY_SSL_PROXY_CA_CRT_EMPTY')
self.assertNodeInformationWithPop(parameter_dict)
self.assertEqual(
{'request-error-list': ["ssl_proxy_ca_crt is invalid"]},
parameter_dict
......@@ -6579,6 +6080,7 @@ class TestSlaveRejectReportUnsafeDamaged(SlaveHttpFrontendTestCase):
def test_health_check_failover_ssl_proxy_ca_crt_damaged(self):
parameter_dict = self.parseSlaveParameterDict(
'health-check-failover-SSL-PROXY-VERIFY_SSL_PROXY_CA_CRT_DAMAGED')
self.assertNodeInformationWithPop(parameter_dict)
self.assertEqual(
{
'request-error-list': [
......@@ -6590,6 +6092,7 @@ class TestSlaveRejectReportUnsafeDamaged(SlaveHttpFrontendTestCase):
def test_health_check_failover_ssl_proxy_ca_crt_empty(self):
parameter_dict = self.parseSlaveParameterDict(
'health-check-failover-SSL-PROXY-VERIFY_SSL_PROXY_CA_CRT_EMPTY')
self.assertNodeInformationWithPop(parameter_dict)
self.assertEqual(
{
'request-error-list': [
......@@ -6599,20 +6102,8 @@ class TestSlaveRejectReportUnsafeDamaged(SlaveHttpFrontendTestCase):
)
def test_server_alias_same(self):
parameter_dict = self.parseSlaveParameterDict('SERVER-ALIAS-SAME')
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict)
self.assertEqual(
{
'domain': 'serveraliassame.example.com',
'replication_number': '1',
'url': 'http://serveraliassame.example.com',
'site_url': 'http://serveraliassame.example.com',
'secure_access': 'https://serveraliassame.example.com',
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
parameter_dict = self.assertSlaveBase(
'SERVER-ALIAS-SAME')
result = fakeHTTPSResult(
parameter_dict['domain'], 'test-path')
......@@ -6625,6 +6116,7 @@ class TestSlaveRejectReportUnsafeDamaged(SlaveHttpFrontendTestCase):
def test_custom_domain_unsafe(self):
parameter_dict = self.parseSlaveParameterDict('CUSTOM_DOMAIN-UNSAFE')
self.assertNodeInformationWithPop(parameter_dict)
self.assertEqual(
{
'request-error-list': [
......@@ -6636,6 +6128,7 @@ class TestSlaveRejectReportUnsafeDamaged(SlaveHttpFrontendTestCase):
def test_server_alias_unsafe(self):
parameter_dict = self.parseSlaveParameterDict('SERVER-ALIAS-UNSAFE')
self.assertNodeInformationWithPop(parameter_dict)
self.assertEqual(
{
'request-error-list': [
......@@ -6647,6 +6140,7 @@ class TestSlaveRejectReportUnsafeDamaged(SlaveHttpFrontendTestCase):
def test_bad_ciphers(self):
parameter_dict = self.parseSlaveParameterDict('BAD-CIPHERS')
self.assertNodeInformationWithPop(parameter_dict)
self.assertEqual(
{
'request-error-list': [
......@@ -6660,6 +6154,7 @@ class TestSlaveRejectReportUnsafeDamaged(SlaveHttpFrontendTestCase):
def test_virtualhostroot_http_port_unsafe(self):
parameter_dict = self.parseSlaveParameterDict(
'VIRTUALHOSTROOT-HTTP-PORT-UNSAFE')
self.assertNodeInformationWithPop(parameter_dict)
self.assertEqual(
{
'request-error-list': [
......@@ -6672,6 +6167,7 @@ class TestSlaveRejectReportUnsafeDamaged(SlaveHttpFrontendTestCase):
def test_virtualhostroot_https_port_unsafe(self):
parameter_dict = self.parseSlaveParameterDict(
'VIRTUALHOSTROOT-HTTPS-PORT-UNSAFE')
self.assertNodeInformationWithPop(parameter_dict)
self.assertEqual(
{
'request-error-list': [
......@@ -6685,6 +6181,7 @@ class TestSlaveRejectReportUnsafeDamaged(SlaveHttpFrontendTestCase):
parameter_dict = self.parseSlaveParameterDict('DEFAULT-PATH-UNSAFE')
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict, 'master-')
self.assertNodeInformationWithPop(parameter_dict)
self.assertEqual(
{
'domain': 'defaultpathunsafe.example.com',
......@@ -6716,20 +6213,7 @@ class TestSlaveRejectReportUnsafeDamaged(SlaveHttpFrontendTestCase):
)
def test_monitor_ipv4_test_unsafe(self):
parameter_dict = self.parseSlaveParameterDict('MONITOR-IPV4-TEST-UNSAFE')
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict)
self.assertEqual(
{
'domain': 'monitoripv4testunsafe.example.com',
'replication_number': '1',
'url': 'http://monitoripv4testunsafe.example.com',
'site_url': 'http://monitoripv4testunsafe.example.com',
'secure_access': 'https://monitoripv4testunsafe.example.com',
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
parameter_dict = self.assertSlaveBase('MONITOR-IPV4-TEST-UNSAFE')
result = fakeHTTPSResult(
parameter_dict['domain'], 'test-path')
......@@ -6760,20 +6244,7 @@ class TestSlaveRejectReportUnsafeDamaged(SlaveHttpFrontendTestCase):
)
def test_monitor_ipv6_test_unsafe(self):
parameter_dict = self.parseSlaveParameterDict('MONITOR-IPV6-TEST-UNSAFE')
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict)
self.assertEqual(
{
'domain': 'monitoripv6testunsafe.example.com',
'replication_number': '1',
'url': 'http://monitoripv6testunsafe.example.com',
'site_url': 'http://monitoripv6testunsafe.example.com',
'secure_access': 'https://monitoripv6testunsafe.example.com',
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
parameter_dict = self.assertSlaveBase('MONITOR-IPV6-TEST-UNSAFE')
result = fakeHTTPSResult(
parameter_dict['domain'], 'test-path')
......@@ -6802,23 +6273,11 @@ class TestSlaveRejectReportUnsafeDamaged(SlaveHttpFrontendTestCase):
)
def test_site_1(self):
parameter_dict = self.parseSlaveParameterDict('SITE_1')
self.assertLogAccessUrlWithPop(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict)
self.assertEqual(
{
'domain': 'duplicate.example.com',
'replication_number': '1',
'url': 'http://duplicate.example.com',
'site_url': 'http://duplicate.example.com',
'secure_access': 'https://duplicate.example.com',
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict
)
self.assertSlaveBase('SITE_1', hostname='duplicate')
def test_site_2(self):
parameter_dict = self.parseSlaveParameterDict('SITE_2')
self.assertNodeInformationWithPop(parameter_dict)
self.assertEqual(
{
'request-error-list': ["custom_domain 'duplicate.example.com' clashes"]
......@@ -6828,6 +6287,7 @@ class TestSlaveRejectReportUnsafeDamaged(SlaveHttpFrontendTestCase):
def test_site_3(self):
parameter_dict = self.parseSlaveParameterDict('SITE_3')
self.assertNodeInformationWithPop(parameter_dict)
self.assertEqual(
{
'request-error-list': ["server-alias 'duplicate.example.com' clashes"]
......@@ -6837,6 +6297,7 @@ class TestSlaveRejectReportUnsafeDamaged(SlaveHttpFrontendTestCase):
def test_site_4(self):
parameter_dict = self.parseSlaveParameterDict('SITE_4')
self.assertNodeInformationWithPop(parameter_dict)
self.assertEqual(
{
'request-error-list': ["custom_domain 'duplicate.example.com' clashes"]
......@@ -6846,7 +6307,7 @@ class TestSlaveRejectReportUnsafeDamaged(SlaveHttpFrontendTestCase):
def test_ssl_ca_crt_only(self):
parameter_dict = self.parseSlaveParameterDict('SSL_CA_CRT_ONLY')
self.assertNodeInformationWithPop(parameter_dict)
self.assertEqual(
parameter_dict,
{
......@@ -6860,6 +6321,7 @@ class TestSlaveRejectReportUnsafeDamaged(SlaveHttpFrontendTestCase):
def test_ssl_key_ssl_crt_unsafe(self):
parameter_dict = self.parseSlaveParameterDict('SSL_KEY-SSL_CRT-UNSAFE')
self.assertNodeInformationWithPop(parameter_dict)
self.assertEqual(
{
'request-error-list': ["slave ssl_key and ssl_crt does not match"],
......@@ -6872,6 +6334,7 @@ class TestSlaveRejectReportUnsafeDamaged(SlaveHttpFrontendTestCase):
def test_bad_backend(self):
parameter_dict = self.parseSlaveParameterDict('BAD-BACKEND')
self.assertNodeInformationWithPop(parameter_dict)
self.assertEqual(
{
'request-error-list': [
......@@ -6883,6 +6346,7 @@ class TestSlaveRejectReportUnsafeDamaged(SlaveHttpFrontendTestCase):
def test_empty_backend(self):
parameter_dict = self.parseSlaveParameterDict('EMPTY-BACKEND')
self.assertNodeInformationWithPop(parameter_dict)
self.assertEqual(
{
'request-error-list': [
......@@ -6925,36 +6389,10 @@ class TestSlaveHostHaproxyClash(SlaveHttpFrontendTestCase, TestDataMixin):
}
def test(self):
parameter_dict_wildcard = self.parseSlaveParameterDict('wildcard')
self.assertLogAccessUrlWithPop(parameter_dict_wildcard)
self.assertKedifaKeysWithPop(parameter_dict_wildcard, '')
hostname = '*.alias1'
self.assertEqual(
{
'domain': '%s.example.com' % (hostname,),
'replication_number': '1',
'url': 'http://%s.example.com' % (hostname, ),
'site_url': 'http://%s.example.com' % (hostname, ),
'secure_access': 'https://%s.example.com' % (hostname, ),
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict_wildcard
)
parameter_dict_specific = self.parseSlaveParameterDict('zspecific')
self.assertLogAccessUrlWithPop(parameter_dict_specific)
self.assertKedifaKeysWithPop(parameter_dict_specific, '')
hostname = 'zspecific.alias1'
self.assertEqual(
{
'domain': '%s.example.com' % (hostname,),
'replication_number': '1',
'url': 'http://%s.example.com' % (hostname, ),
'site_url': 'http://%s.example.com' % (hostname, ),
'secure_access': 'https://%s.example.com' % (hostname, ),
'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
},
parameter_dict_specific
)
self.assertSlaveBase(
'wildcard', hostname='*.alias1')
self.assertSlaveBase(
'zspecific', hostname='zspecific.alias1')
result_wildcard = fakeHTTPSResult(
'other.alias1.example.com',
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7