report missing http content-type / csp headers

src/surykatka/bot.py

@@ -649,6 +649,8 @@ class WebBot:
         for ip, url_list in ip_to_url_dict.items():
             query = reportHttp(self._db, ip=ip, url=url_list)
             for network_change in query.dicts().iterator():
+
+                # Confirm that redirection url are checked
                 if network_change["status_code"] in (301, 302, 303):
                     redirect_url = getRootUrl(
                         network_change["http_header_dict"]["Location"]
@@ -664,6 +666,39 @@ class WebBot:
                                 "date": rfc822(network_change["status"]),
                             }
                         )
+
+                # Check HTTP CSP header
+                if network_change["status_code"] != 524:
+                    # Skip timeout
+                    # check missing import headers
+                    if (
+                        "Content-Type"
+                        not in network_change["http_header_dict"]
+                    ):
+                        result_dict["warning"].append(
+                            {
+                                "text": "(No Content-Type header) %s"
+                                % (network_change["url"],),
+                                "date": result_dict["bot_status"][0]["date"],
+                            }
+                        )
+                    elif network_change["http_header_dict"][
+                        "Content-Type"
+                    ].startswith("text/html"):
+                        if (
+                            "Content-Security-Policy"
+                            not in network_change["http_header_dict"]
+                        ):
+                            result_dict["warning"].append(
+                                {
+                                    "text": "(No Content-Security-Policy header) %s"
+                                    % (network_change["url"],),
+                                    "date": result_dict["bot_status"][0][
+                                        "date"
+                                    ],
+                                }
+                            )
+
                 result_dict["http_query"].append(
                     {
                         "status_code": network_change["status_code"],