Commit 238008f6 authored by Łukasz Nowak's avatar Łukasz Nowak

caddy-frontend: Cover failover URL with cache

Configuring failover URL and being able to rely on cached content is not so
simple, thus document the requirement and prove in test, that such
configuration works correctly.
parent c3cec3e1
......@@ -233,6 +233,8 @@ Please be aware that the `health-check-timeout` is really short by default, so i
Thanks to using health-check it's possible to configure failover system. By providing `health-check-failover-url` or `health-check-failover-https-url` some special backend can be used to reply in case if original backend replies with error (codes like `5xx`). As a note one can setup this failover URL like `https://failover.example.com/?p=` so that the path from the incoming request will be passed as parameter. Additionally authentication to failover URL is supported with `health-check-authenticate-to-failover-backend` and SSL Proxy verification with `health-check-failover-ssl-proxy-verify` and `health-check-failover-ssl-proxy-ca-crt`.
**Note**: It's important to correctly configure failover URL response, especially in case if it's expected to use `stale-if-error` simulation available while `enable_cache` is used. In order to serve pages from cache the failover URL have to return error HTTP code (like 503 SERVICE_UNAVAILABLE), so that in such case cached page will have precedence over the reply from failover URL.
Examples
========
......
......@@ -275,6 +275,58 @@ def isHTTP2(domain):
return 'Using HTTP2, server supports'.encode() in err
class AtsMixin(object):
def _hack_ats(self, max_stale_age):
records_config = glob.glob(
os.path.join(
self.instance_path, '*', 'etc', 'trafficserver', 'records.config'
))
self.assertEqual(1, len(records_config))
self._hack_ats_records_config_path = records_config[0]
original_max_stale_age = \
'CONFIG proxy.config.http.cache.max_stale_age INT 604800\n'
new_max_stale_age = \
'CONFIG proxy.config.http.cache.max_stale_age INT %s\n' % (
max_stale_age,)
with open(self._hack_ats_records_config_path) as fh:
self._hack_ats_original_records_config = fh.readlines()
# sanity check - are we really do it?
self.assertIn(
original_max_stale_age,
self._hack_ats_original_records_config)
new_records_config = []
max_stale_age_changed = False
for line in self._hack_ats_original_records_config:
if line == original_max_stale_age:
line = new_max_stale_age
max_stale_age_changed = True
new_records_config.append(line)
self.assertTrue(max_stale_age_changed)
with open(self._hack_ats_records_config_path, 'w') as fh:
fh.write(''.join(new_records_config))
self._hack_ats_restart()
def _unhack_ats(self):
with open(self._hack_ats_records_config_path, 'w') as fh:
fh.write(''.join(self._hack_ats_original_records_config))
self._hack_ats_restart()
def _hack_ats_restart(self):
for process_info in self.callSupervisorMethod('getAllProcessInfo'):
if process_info['name'].startswith(
'trafficserver') and process_info['name'].endswith('-on-watch'):
self.callSupervisorMethod(
'stopProcess', '%(group)s:%(name)s' % process_info)
self.callSupervisorMethod(
'startProcess', '%(group)s:%(name)s' % process_info)
# give short time for the ATS to start back
time.sleep(5)
for process_info in self.callSupervisorMethod('getAllProcessInfo'):
if process_info['name'].startswith(
'trafficserver') and process_info['name'].endswith('-on-watch'):
self.assertEqual(process_info['statename'], 'RUNNING')
class TestDataMixin(object):
def getTrimmedProcessInfo(self):
return '\n'.join(sorted([
......@@ -1571,7 +1623,7 @@ class TestMasterAIKCDisabledAIBCCDisabledRequest(
)
class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin, AtsMixin):
@classmethod
def getInstanceParameterDict(cls):
return {
......@@ -3865,56 +3917,6 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
self.assertRegex(ats_log, direct_pattern)
# END: Check that squid.log is correctly filled in
def _hack_ats(self, max_stale_age):
records_config = glob.glob(
os.path.join(
self.instance_path, '*', 'etc', 'trafficserver', 'records.config'
))
self.assertEqual(1, len(records_config))
self._hack_ats_records_config_path = records_config[0]
original_max_stale_age = \
'CONFIG proxy.config.http.cache.max_stale_age INT 604800\n'
new_max_stale_age = \
'CONFIG proxy.config.http.cache.max_stale_age INT %s\n' % (
max_stale_age,)
with open(self._hack_ats_records_config_path) as fh:
self._hack_ats_original_records_config = fh.readlines()
# sanity check - are we really do it?
self.assertIn(
original_max_stale_age,
self._hack_ats_original_records_config)
new_records_config = []
max_stale_age_changed = False
for line in self._hack_ats_original_records_config:
if line == original_max_stale_age:
line = new_max_stale_age
max_stale_age_changed = True
new_records_config.append(line)
self.assertTrue(max_stale_age_changed)
with open(self._hack_ats_records_config_path, 'w') as fh:
fh.write(''.join(new_records_config))
self._hack_ats_restart()
def _unhack_ats(self):
with open(self._hack_ats_records_config_path, 'w') as fh:
fh.write(''.join(self._hack_ats_original_records_config))
self._hack_ats_restart()
def _hack_ats_restart(self):
for process_info in self.callSupervisorMethod('getAllProcessInfo'):
if process_info['name'].startswith(
'trafficserver') and process_info['name'].endswith('-on-watch'):
self.callSupervisorMethod(
'stopProcess', '%(group)s:%(name)s' % process_info)
self.callSupervisorMethod(
'startProcess', '%(group)s:%(name)s' % process_info)
# give short time for the ATS to start back
time.sleep(5)
for process_info in self.callSupervisorMethod('getAllProcessInfo'):
if process_info['name'].startswith(
'trafficserver') and process_info['name'].endswith('-on-watch'):
self.assertEqual(process_info['statename'], 'RUNNING')
def test_enable_cache_negative_revalidate(self):
parameter_dict = self.assertSlaveBase('enable_cache')
......@@ -4542,7 +4544,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
self.assertNotIn('X-Backend-Identification', result.headers)
class TestReplicateSlave(SlaveHttpFrontendTestCase, TestDataMixin):
class TestReplicateSlave(SlaveHttpFrontendTestCase, TestDataMixin, AtsMixin):
instance_parameter_dict = {
'domain': 'example.com',
'port': HTTPS_PORT,
......@@ -6763,7 +6765,7 @@ class TestPassedRequestParameter(HttpFrontendTestCase):
)
class TestSlaveHealthCheck(SlaveHttpFrontendTestCase, TestDataMixin):
class TestSlaveHealthCheck(SlaveHttpFrontendTestCase, TestDataMixin, AtsMixin):
@classmethod
def getInstanceParameterDict(cls):
return {
......@@ -6805,6 +6807,7 @@ class TestSlaveHealthCheck(SlaveHttpFrontendTestCase, TestDataMixin):
},
'health-check-failover-url': {
'https-only': False, # http and https access to check
'enable_cache': True,
'health-check-timeout': 1, # fail fast for test
'health-check-interval': 1, # fail fast for test
'url': cls.backend_url + 'url',
......@@ -6958,12 +6961,63 @@ backend _health-check-default-http
slave_parameter_dict = self.getSlaveParameterDictDict()[
'health-check-failover-url']
# check normal access
source_ip = '127.0.0.1'
max_stale_age = 30
max_age = int(max_stale_age / 2.)
body_200 = 'Body 200' * 500
body_failover = 'Failover response'
cached_path = self.id()
self.addCleanup(self._unhack_ats)
self._hack_ats(max_stale_age)
# Prerequisite for cache: setup failover backend with proper code
# for normal access (not cached, typical scenario) and cached access
# in order to check ATS behaviour
for path in ['/failoverpath', '/' + cached_path]:
for url in [
'failover-url?a=b&c=',
'failover-https-url?a=b&c='
]:
result = requests.put(
self.backend_url + url + path,
headers={
'X-Reply-Status-Code': '503',
'X-Reply-Body': base64.b64encode(body_failover.encode()),
})
self.assertEqual(result.status_code, http.client.CREATED)
def configureResult(status_code, body):
backend_url = self.getSlaveParameterDictDict()[
'health-check-failover-url']['https-url']
result = requests.put(
'/'.join([backend_url, cached_path]),
headers={
'X-Reply-Header-Cache-Control': 'max-age=%s, public' % (max_age,),
'X-Reply-Status-Code': status_code,
'X-Reply-Body': base64.b64encode(body.encode()),
# drop Content-Length header to ensure
# https://github.com/apache/trafficserver/issues/7880
'X-Drop-Header': 'Content-Length',
})
self.assertEqual(result.status_code, http.client.CREATED)
def checkResult(status_code, body):
result = fakeHTTPSResult(
parameter_dict['domain'], cached_path,
source_ip=source_ip
)
self.assertEqual(result.status_code, status_code)
self.assertEqual(result.text, body)
# check normal access...
result = fakeHTTPResult(parameter_dict['domain'], '/path')
self.assertEqualResultJson(result, 'Path', '/url/path')
result = fakeHTTPSResult(parameter_dict['domain'], '/path')
self.assertEqual(self.certificate_pem, der2pem(result.peercert))
self.assertEqualResultJson(result, 'Path', '/https-url/path')
# ...and cached result, also in order to store it in the cache
configureResult('200', body_200)
checkResult(http.client.OK, body_200)
# start replying with bad status code
result = requests.put(
......@@ -6982,10 +7036,11 @@ backend _health-check-default-http
time.sleep(3) # > health-check-timeout + health-check-interval
# check simple failover
result = fakeHTTPSResult(parameter_dict['domain'], '/failoverpath')
self.assertEqual(self.certificate_pem, der2pem(result.peercert))
self.assertEqualResultJson(
result, 'Path', '/failover-https-url?a=b&c=/failoverpath')
self.assertEqual(result.status_code, http.client.SERVICE_UNAVAILABLE)
self.assertEqual(result.text, body_failover)
self.assertLastLogLineRegexp(
'_health-check-failover-url_backend_log',
......@@ -6994,14 +7049,14 @@ backend _health-check-default-http
r'https-backend _health-check-failover-url-https-failover'
r'\/_health-check-failover-url-backend-https '
r'\d+/\d+\/\d+\/\d+\/\d+ '
r'200 \d+ - - ---- '
r'503 \d+ - - ---- '
r'\d+\/\d+\/\d+\/\d+\/\d+ \d+\/\d+ '
r'"GET /failoverpath HTTP/1.1"'
)
result = fakeHTTPResult(parameter_dict['domain'], '/failoverpath')
self.assertEqualResultJson(
result, 'Path', '/failover-url?a=b&c=/failoverpath')
self.assertEqual(result.status_code, http.client.SERVICE_UNAVAILABLE)
self.assertEqual(result.text, body_failover)
self.assertLastLogLineRegexp(
'_health-check-failover-url_backend_log',
r'^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d+ '
......@@ -7009,11 +7064,23 @@ backend _health-check-default-http
r'http-backend _health-check-failover-url-http-failover'
r'\/_health-check-failover-url-backend-http '
r'\d+/\d+\/\d+\/\d+\/\d+ '
r'200 \d+ - - ---- '
r'503 \d+ - - ---- '
r'\d+\/\d+\/\d+\/\d+\/\d+ \d+\/\d+ '
r'"GET /failoverpath HTTP/1.1"'
)
# It's time to check that ATS gives cached result, even if failover
# backend is used
checkResult(http.client.OK, body_200)
# interesting moment, time is between max_age and max_stale_age, triggers
# https://github.com/apache/trafficserver/issues/7880
# which is stale-if-error simulated by ATS while using failover backend
time.sleep(max_age + 1 - 3)
checkResult(http.client.OK, body_200)
# max_stale_age passed, time to return 502 with failover url
time.sleep(max_stale_age + 2 - 3)
checkResult(http.client.SERVICE_UNAVAILABLE, body_failover)
def test_health_check_failover_url_netloc_list(self):
parameter_dict = self.assertSlaveBase(
'health-check-failover-url-netloc-list')
......
......@@ -61,6 +61,7 @@
"url": "http://@@_ipv4_address@@:@@_server_http_port@@/"
},
{
"enable_cache": true,
"health-check": true,
"health-check-failover-https-url": "http://@@_ipv4_address@@:@@_server_http_port@@/failover-https-url?a=b&c=",
"health-check-failover-url": "http://@@_ipv4_address@@:@@_server_http_port@@/failover-url?a=b&c=",
......@@ -183,6 +184,7 @@
"url": "http://@@_ipv4_address@@:@@_server_http_port@@/"
},
{
"enable_cache": true,
"health-check": true,
"health-check-failover-https-url": "http://@@_ipv4_address@@:@@_server_http_port@@/failover-https-url?a=b&c=",
"health-check-failover-url": "http://@@_ipv4_address@@:@@_server_http_port@@/failover-url?a=b&c=",
......@@ -279,7 +281,7 @@
"backend-client-caucase-url": "http://[@@_ipv6_address@@]:8990",
"cluster-identification": "testing partition 0",
"domain": "example.com",
"extra_slave_instance_list": "[{\"health-check\": true, \"health-check-http-method\": \"CONNECT\", \"slave_reference\": \"_health-check-connect\", \"url\": \"http://@@_ipv4_address@@:@@_server_http_port@@/\"}, {\"health-check\": true, \"health-check-fall\": \"7\", \"health-check-http-method\": \"POST\", \"health-check-http-path\": \"/POST-path to be encoded\", \"health-check-http-version\": \"HTTP/1.0\", \"health-check-interval\": \"15\", \"health-check-rise\": \"3\", \"health-check-timeout\": \"7\", \"slave_reference\": \"_health-check-custom\", \"url\": \"http://@@_ipv4_address@@:@@_server_http_port@@/\"}, {\"health-check\": true, \"slave_reference\": \"_health-check-default\", \"url\": \"http://@@_ipv4_address@@:@@_server_http_port@@/\"}, {\"slave_reference\": \"_health-check-disabled\", \"url\": \"http://@@_ipv4_address@@:@@_server_http_port@@/\"}, {\"health-check\": true, \"health-check-failover-https-url\": \"http://@@_ipv4_address@@:@@_server_http_port@@/failover-https-url?a=b&c=\", \"health-check-failover-url\": \"http://@@_ipv4_address@@:@@_server_http_port@@/failover-url?a=b&c=\", \"health-check-http-path\": \"/health-check-failover-url\", \"health-check-interval\": 1, \"health-check-timeout\": 1, \"https-only\": false, \"https-url\": \"http://@@_ipv4_address@@:@@_server_http_port@@/https-url\", \"slave_reference\": \"_health-check-failover-url\", \"url\": \"http://@@_ipv4_address@@:@@_server_http_port@@/url\"}, {\"health-check\": true, \"health-check-authenticate-to-failover-backend\": true, \"health-check-failover-https-url\": \"https://@@_ipv4_address@@:@@_server_https_auth_port@@/failover-https-url?a=b&c=\", \"health-check-failover-url\": \"https://@@_ipv4_address@@:@@_server_https_auth_port@@/failover-url?a=b&c=\", \"health-check-http-path\": \"/health-check-failover-url-auth-to-backend\", \"health-check-interval\": 1, \"health-check-timeout\": 1, \"https-only\": false, \"https-url\": \"http://@@_ipv4_address@@:@@_server_http_port@@/https-url\", \"slave_reference\": \"_health-check-failover-url-auth-to-backend\", \"url\": \"http://@@_ipv4_address@@:@@_server_http_port@@/url\"}, {\"health-check\": true, \"health-check-failover-https-url\": \"http://@@_ipv4_address@@:@@_server_http_port@@/failover-https-url?a=b&c=\", \"health-check-failover-url\": \"http://@@_ipv4_address@@:@@_server_http_port@@/failover-url?a=b&c=\", \"health-check-failover-url-netloc-list\": \"@@_ipv4_address@@:@@_server_netloc_a_http_port@@ @@_ipv4_address@@:@@_server_netloc_b_http_port@@\", \"health-check-http-path\": \"/health-check-failover-url\", \"health-check-interval\": 1, \"health-check-timeout\": 1, \"https-only\": false, \"https-url\": \"http://@@_ipv4_address@@:@@_server_http_port@@/https-url\", \"slave_reference\": \"_health-check-failover-url-netloc-list\", \"url\": \"http://@@_ipv4_address@@:@@_server_http_port@@/url\"}, {\"health-check\": true, \"health-check-failover-ssl-proxy-ca-crt\": \"@@test_server_ca.certificate_pem_double@@\", \"health-check-failover-ssl-proxy-verify\": true, \"health-check-failover-url\": \"https://@@_ipv4_address@@:@@_server_https_port@@/\", \"health-check-http-path\": \"/health-check-failover-url-ssl-proxy-verified\", \"health-check-interval\": 1, \"health-check-timeout\": 1, \"slave_reference\": \"_health-check-failover-url-ssl-proxy-verified\", \"url\": \"http://@@_ipv4_address@@:@@_server_http_port@@/\"}, {\"health-check\": true, \"health-check-failover-ssl-proxy-verify\": true, \"health-check-failover-url\": \"https://@@_ipv4_address@@:@@_server_https_port@@/\", \"health-check-http-path\": \"/health-check-failover-url-ssl-proxy-verify-missing\", \"health-check-interval\": 1, \"health-check-timeout\": 1, \"slave_reference\": \"_health-check-failover-url-ssl-proxy-verify-missing\", \"url\": \"http://@@_ipv4_address@@:@@_server_http_port@@/\"}, {\"health-check\": true, \"health-check-failover-ssl-proxy-ca-crt\": \"@@another_server_ca.certificate_pem_double@@\", \"health-check-failover-ssl-proxy-verify\": true, \"health-check-failover-url\": \"https://@@_ipv4_address@@:@@_server_https_port@@/\", \"health-check-http-path\": \"/health-check-failover-url-ssl-proxy-verify-unverified\", \"health-check-interval\": 1, \"health-check-timeout\": 1, \"slave_reference\": \"_health-check-failover-url-ssl-proxy-verify-unverified\", \"url\": \"http://@@_ipv4_address@@:@@_server_http_port@@/\"}]",
"extra_slave_instance_list": "[{\"health-check\": true, \"health-check-http-method\": \"CONNECT\", \"slave_reference\": \"_health-check-connect\", \"url\": \"http://@@_ipv4_address@@:@@_server_http_port@@/\"}, {\"health-check\": true, \"health-check-fall\": \"7\", \"health-check-http-method\": \"POST\", \"health-check-http-path\": \"/POST-path to be encoded\", \"health-check-http-version\": \"HTTP/1.0\", \"health-check-interval\": \"15\", \"health-check-rise\": \"3\", \"health-check-timeout\": \"7\", \"slave_reference\": \"_health-check-custom\", \"url\": \"http://@@_ipv4_address@@:@@_server_http_port@@/\"}, {\"health-check\": true, \"slave_reference\": \"_health-check-default\", \"url\": \"http://@@_ipv4_address@@:@@_server_http_port@@/\"}, {\"slave_reference\": \"_health-check-disabled\", \"url\": \"http://@@_ipv4_address@@:@@_server_http_port@@/\"}, {\"enable_cache\": true, \"health-check\": true, \"health-check-failover-https-url\": \"http://@@_ipv4_address@@:@@_server_http_port@@/failover-https-url?a=b&c=\", \"health-check-failover-url\": \"http://@@_ipv4_address@@:@@_server_http_port@@/failover-url?a=b&c=\", \"health-check-http-path\": \"/health-check-failover-url\", \"health-check-interval\": 1, \"health-check-timeout\": 1, \"https-only\": false, \"https-url\": \"http://@@_ipv4_address@@:@@_server_http_port@@/https-url\", \"slave_reference\": \"_health-check-failover-url\", \"url\": \"http://@@_ipv4_address@@:@@_server_http_port@@/url\"}, {\"health-check\": true, \"health-check-authenticate-to-failover-backend\": true, \"health-check-failover-https-url\": \"https://@@_ipv4_address@@:@@_server_https_auth_port@@/failover-https-url?a=b&c=\", \"health-check-failover-url\": \"https://@@_ipv4_address@@:@@_server_https_auth_port@@/failover-url?a=b&c=\", \"health-check-http-path\": \"/health-check-failover-url-auth-to-backend\", \"health-check-interval\": 1, \"health-check-timeout\": 1, \"https-only\": false, \"https-url\": \"http://@@_ipv4_address@@:@@_server_http_port@@/https-url\", \"slave_reference\": \"_health-check-failover-url-auth-to-backend\", \"url\": \"http://@@_ipv4_address@@:@@_server_http_port@@/url\"}, {\"health-check\": true, \"health-check-failover-https-url\": \"http://@@_ipv4_address@@:@@_server_http_port@@/failover-https-url?a=b&c=\", \"health-check-failover-url\": \"http://@@_ipv4_address@@:@@_server_http_port@@/failover-url?a=b&c=\", \"health-check-failover-url-netloc-list\": \"@@_ipv4_address@@:@@_server_netloc_a_http_port@@ @@_ipv4_address@@:@@_server_netloc_b_http_port@@\", \"health-check-http-path\": \"/health-check-failover-url\", \"health-check-interval\": 1, \"health-check-timeout\": 1, \"https-only\": false, \"https-url\": \"http://@@_ipv4_address@@:@@_server_http_port@@/https-url\", \"slave_reference\": \"_health-check-failover-url-netloc-list\", \"url\": \"http://@@_ipv4_address@@:@@_server_http_port@@/url\"}, {\"health-check\": true, \"health-check-failover-ssl-proxy-ca-crt\": \"@@test_server_ca.certificate_pem_double@@\", \"health-check-failover-ssl-proxy-verify\": true, \"health-check-failover-url\": \"https://@@_ipv4_address@@:@@_server_https_port@@/\", \"health-check-http-path\": \"/health-check-failover-url-ssl-proxy-verified\", \"health-check-interval\": 1, \"health-check-timeout\": 1, \"slave_reference\": \"_health-check-failover-url-ssl-proxy-verified\", \"url\": \"http://@@_ipv4_address@@:@@_server_http_port@@/\"}, {\"health-check\": true, \"health-check-failover-ssl-proxy-verify\": true, \"health-check-failover-url\": \"https://@@_ipv4_address@@:@@_server_https_port@@/\", \"health-check-http-path\": \"/health-check-failover-url-ssl-proxy-verify-missing\", \"health-check-interval\": 1, \"health-check-timeout\": 1, \"slave_reference\": \"_health-check-failover-url-ssl-proxy-verify-missing\", \"url\": \"http://@@_ipv4_address@@:@@_server_http_port@@/\"}, {\"health-check\": true, \"health-check-failover-ssl-proxy-ca-crt\": \"@@another_server_ca.certificate_pem_double@@\", \"health-check-failover-ssl-proxy-verify\": true, \"health-check-failover-url\": \"https://@@_ipv4_address@@:@@_server_https_port@@/\", \"health-check-http-path\": \"/health-check-failover-url-ssl-proxy-verify-unverified\", \"health-check-interval\": 1, \"health-check-timeout\": 1, \"slave_reference\": \"_health-check-failover-url-ssl-proxy-verify-unverified\", \"url\": \"http://@@_ipv4_address@@:@@_server_http_port@@/\"}]",
"frontend-name": "caddy-frontend-1",
"kedifa-caucase-url": "http://[@@_ipv6_address@@]:15090",
"master-key-download-url": "https://[@@_ipv6_address@@]:15080/@@master-key-download-url_endpoint@@",
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment