Store key only headers whose value is not stable.

src/surykatka/http.py

@@ -196,30 +196,27 @@ def checkHttpStatus(
     )
 
     # Blacklisted, because of non stability
-    # 'Date', 'Age', 'Expires'
+    # 'Date'
     header_list = [
         # Redirect
         "Location",
         # HTTP Range
         "Accept-Ranges",
         # HTTP Cache
-        "Etag",
-        "Last-Modified",
         "Vary",
         "Cache-Control",
-        "Set-Cookie",
-        "WWW-Authenticate"
+        "WWW-Authenticate",
         # gzip
         "Content-Type",
         "Content-Encoding",
-        "Content-Disposition"
+        "Content-Disposition",
         # Security
         "Content-Security-Policy",
         "Referrer-Policy",
-        "Strict-Transport-Policy",
+        "Strict-Transport-Security",
         "Feature-Policy",
         "X-Frame-Options",
-        "X-Content-Type-Options"
+        "X-Content-Type-Options",
         # CORS
         "Access-Control-Allow-Origin",
         "Access-Control-Allow-Methods",
@@ -233,6 +230,20 @@ def checkHttpStatus(
         if header_value is not None:
             header_dict[header_key] = header_value
 
+    # Store key only, because of non stability
+    # 'Etag', 'Last-Modified', 'Set-Cookie', 'Date', 'Age', 'Expires'
+    key_only_header_list = [
+        "Etag",
+        "Last-Modified",
+        "Set-Cookie",
+        "Age",
+        "Expires",
+    ]
+    for header_key in key_only_header_list:
+        header_value = response.headers.get(header_key, None)
+        if header_value is not None:
+            header_dict[header_key] = True
+
     logHttpStatus(
         db,
         ip,

tests/test_http.py

@@ -862,7 +862,7 @@ class SurykatkaHttpTestCase(unittest.TestCase):
             httpretty.GET,
             "http://127.0.0.1/foo?bar=1",
             status=418,
-            adding_headers={"Etag": "bar"},
+            adding_headers={"Etag": "bar", "Cache-Control": "public"},
         )
 
         status_id = logStatus(self.db, "foo")
@@ -884,7 +884,11 @@ class SurykatkaHttpTestCase(unittest.TestCase):
         assert self.db.HttpCodeChange.get().ip == ip
         assert self.db.HttpCodeChange.get().url == url
         assert self.db.HttpCodeChange.get().status_code == 418
-        assert self.db.HttpCodeChange.get().http_header_dict == {"Etag": "bar"}
+        assert self.db.HttpCodeChange.get().http_header_dict == {
+            "Cache-Control": "public",
+            "Content-Type": "text/plain; charset=utf-8",
+            "Etag": True,
+        }
         assert self.db.HttpCodeChange.get().status_id == status_id
 
     def test_checkHttpStatus_https(self):
@@ -894,7 +898,10 @@ class SurykatkaHttpTestCase(unittest.TestCase):
 
         status_id = logStatus(self.db, "foo")
         with mock.patch("surykatka.http.request") as mock_request:
-            mock_request.return_value.headers = {"Etag": "foobar"}
+            mock_request.return_value.headers = {
+                "Etag": "foobar",
+                "Cache-Control": "public",
+            }
 
             checkHttpStatus(self.db, status_id, url, ip, bot_version)
 
@@ -918,7 +925,8 @@ class SurykatkaHttpTestCase(unittest.TestCase):
         # XXX No idea how to mock SSL
         assert self.db.HttpCodeChange.get().status_code == 1
         assert self.db.HttpCodeChange.get().http_header_dict == {
-            "Etag": "foobar"
+            "Cache-Control": "public",
+            "Etag": True,
         }
         assert self.db.HttpCodeChange.get().status_id == status_id
 
@@ -935,10 +943,13 @@ class SurykatkaHttpTestCase(unittest.TestCase):
         else:
             raise NotImplementedError("Expected NotImplementedError")
 
-    def __generateHeaderDict(self, header_list):
+    def __generateHeaderDict(self, header_list, key_only=False):
         result_dict = {}
         for header in header_list:
-            result_dict[header] = header + " bar"
+            if key_only:
+                result_dict[header] = True
+            else:
+                result_dict[header] = header + " bar"
         return result_dict
 
     @httpretty.activate
@@ -952,23 +963,20 @@ class SurykatkaHttpTestCase(unittest.TestCase):
             # HTTP Range
             "Accept-Ranges",
             # HTTP Cache
-            "Etag",
-            "Last-Modified",
             "Vary",
             "Cache-Control",
-            "Set-Cookie",
-            "WWW-Authenticate"
+            "WWW-Authenticate",
             # gzip
             "Content-Type",
             "Content-Encoding",
-            "Content-Disposition"
+            "Content-Disposition",
             # Security
             "Content-Security-Policy",
             "Referrer-Policy",
-            "Strict-Transport-Policy",
+            "Strict-Transport-Security",
             "Feature-Policy",
             "X-Frame-Options",
-            "X-Content-Type-Options"
+            "X-Content-Type-Options",
             # CORS
             "Access-Control-Allow-Origin",
             "Access-Control-Allow-Methods",
@@ -976,13 +984,19 @@ class SurykatkaHttpTestCase(unittest.TestCase):
             "Access-Control-Allow-Headers",
             "Access-Control-Expose-Headers",
         ]
-        blacklist_header_list = [
+        key_only_header_list = [
+            "Etag",
+            "Last-Modified",
+            "Set-Cookie",
             "Age",
-            "Date",
             "Expires",
+        ]
+        blacklist_header_list = [
             "Foo",
+            "Date",
         ]
         header_dict = self.__generateHeaderDict(whitelist_header_list)
+        header_dict.update(self.__generateHeaderDict(key_only_header_list))
         header_dict.update(self.__generateHeaderDict(blacklist_header_list))
 
         httpretty.register_uri(
@@ -1011,9 +1025,16 @@ class SurykatkaHttpTestCase(unittest.TestCase):
         assert self.db.HttpCodeChange.get().ip == ip
         assert self.db.HttpCodeChange.get().url == url
         assert self.db.HttpCodeChange.get().status_code == 418
-        assert self.db.HttpCodeChange.get().http_header_dict == self.__generateHeaderDict(
+        expected_http_header_dict = self.__generateHeaderDict(
             whitelist_header_list
         )
+        expected_http_header_dict.update(
+            self.__generateHeaderDict(key_only_header_list, key_only=True)
+        )
+        assert (
+            self.db.HttpCodeChange.get().http_header_dict
+            == expected_http_header_dict
+        )
         assert self.db.HttpCodeChange.get().status_id == status_id