caddy-frontend: Prove that stale-if-error works

Added assertion which proves that the ATS is serving stale content in case if
the backend does not work, according to RFC5861.

It is beleived that stale-while-revalidate will work the same way, but it is
much harder to test, thus it is not done directly.

software/caddy-frontend/test/test.py

@@ -588,6 +588,16 @@ class TestHandler(BaseHTTPRequestHandler):
     timeout = int(self.headers.dict.get('Timeout', '0'))
     time.sleep(timeout)
     self.send_response(200)
+
+    prefix = 'x-reply-header-'
+    lenght = len(prefix)
+    for key, value in self.headers.dict.items():
+      if key.startswith(prefix):
+        self.send_header(
+          '-'.join([q.capitalize() for q in key[lenght:].split('-')]),
+          value.strip()
+        )
+
     self.send_header("Content-type", "application/json")
     self.send_header('Set-Cookie', 'secured=value;secure')
     self.send_header('Set-Cookie', 'nonsecured=value')
@@ -2528,13 +2538,11 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
   def test_enable_cache(self):
     parameter_dict = self.assertSlaveBase('enable_cache')
 
-    result = self.fakeHTTPSResult(
+    result = self.fakeHTTPResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'],
-      'test-path/deep/.././deeper')
-
-    self.assertEqual(
-      self.certificate_pem,
-      der2pem(result.peercert))
+      'test-path/deep/.././deeper', headers={
+        'X-Reply-Header-Cache-Control': 'max-age=1, stale-while-'
+        'revalidate=3600, stale-if-error=3600'})
 
     self.assertEqualResultJson(result, 'Path', '/test-path/deeper')
 
@@ -2553,8 +2561,10 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
     self.assertEqual(
       {
         'Content-type': 'application/json',
-        'Set-Cookie': 'secured=value;secure, nonsecured=value'
-       },
+        'Set-Cookie': 'secured=value;secure, nonsecured=value',
+        'Cache-Control': 'max-age=1, stale-while-revalidate=3600, '
+                         'stale-if-error=3600'
+      },
       headers
     )
 
@@ -2566,6 +2576,71 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
       r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/7.1.6\)$'
     )
 
+    # check stale-if-error support (assumes stale-while-revalidate is same)
+    # wait a bit for max-age to expire
+    time.sleep(2)
+    # real check: cache access provides old data, access cache directly, as
+    # caddy has to be stopped
+    try:
+      # stop caddy, to have error on while connecting to the backend
+      caddy_process_name = [
+        ':'.join([q['group'], q['name']]) for q in
+        self.getSupervisorRPCServer().supervisor.getAllProcessInfo()
+        if 'caddy' in q['name'] and 'on-watch' in q['name']][0]
+      self.getSupervisorRPCServer().supervisor.stopProcess(caddy_process_name)
+
+      # sanity check: see that it is impossible to connect to caddy
+      with self.assertRaises(requests.ConnectionError):
+        self.fakeHTTPResult(
+          parameter_dict['domain'], parameter_dict['public-ipv4'],
+          'test-path/deep/.././deeper', headers={
+            'X-Reply-Header-Cache-Control': 'max-age=1, stale-while-'
+            'revalidate=3600, stale-if-error=3600'})
+
+      result = self.fakeHTTPResult(
+        parameter_dict['domain'], parameter_dict['public-ipv4'],
+        'test-path/deeper',  # simple path, as ATS can't change them
+        port=23432, headers={
+          'X-Reply-Header-Cache-Control': 'max-age=1, stale-while-'
+          'revalidate=3600, stale-if-error=3600'})
+      self.assertEqual(result.status_code, httplib.OK)
+      self.assertEqualResultJson(result, 'Path', '/test-path/deeper')
+      headers = result.headers.copy()
+      self.assertKeyWithPop('Server', headers)
+      self.assertKeyWithPop('Date', headers)
+      self.assertKeyWithPop('Age', headers)
+      # drop keys appearing randomly in headers
+      headers.pop('Transfer-Encoding', None)
+      headers.pop('Content-Length', None)
+      headers.pop('Connection', None)
+      headers.pop('Keep-Alive', None)
+
+      self.assertEqual(
+        {
+          'Content-type': 'application/json',
+          # ATS does not cache the cookied text content, see:
+          # https://docs.trafficserver.apache.org/en/7.1.x/admin-guide/\
+          # configuration/cache-basics.en.html#caching-cookied-objects
+          # 'Set-Cookie': 'secured=value;secure, nonsecured=value',
+          'Cache-Control': 'max-age=1, stale-while-revalidate=3600, '
+                           'stale-if-error=3600',
+          'Warning': '111 ApacheTrafficServer/7.1.6'
+        },
+        headers
+      )
+
+      backend_headers = result.json()['Incoming Headers']
+      via = backend_headers.pop('via', None)
+      self.assertNotEqual(via, None)
+      self.assertRegexpMatches(
+        via,
+        r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/7.1.6\)$'
+      )
+    finally:
+      self.getSupervisorRPCServer().supervisor.startProcess(caddy_process_name)
+      time.sleep(2)  # give few moments for caddy to start
+    # END: check stale-if-error support
+
     result_direct = self.fakeHTTPResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path',
       port=26011)