bot: Correctly group IPs per URLs and DOMAINs

src/surykatka/bot.py

@@ -249,31 +249,31 @@ class WebBot:
                     url_dict[url].append(network_change["ip"])
 
         # Report the SSL status
-        query = reportSslCertificate(
-            self._db,
-            ip=[x for x in server_ip_dict.keys()],
-            port=443,
-            hostname=domain_list,
-        )
         result_dict["ssl_certificate"] = []
-        for ssl_certificate in query.dicts().iterator():
-            result_dict["ssl_certificate"].append(
-                {
-                    "hostname": ssl_certificate["hostname"],
-                    "ip": ssl_certificate["ip"],
-                    "port": ssl_certificate["port"],
-                    "sha1_fingerprint": ssl_certificate["sha1_fingerprint"],
-                    "subject": ssl_certificate["subject"],
-                    "issuer": ssl_certificate["issuer"],
-                    "not_before": rfc822(ssl_certificate["not_before"])
-                    if (ssl_certificate["not_before"] is not None)
-                    else None,
-                    "not_after": rfc822(ssl_certificate["not_after"])
-                    if (ssl_certificate["not_after"] is not None)
-                    else None,
-                    "date": rfc822(ssl_certificate["status"]),
-                }
+        for ip_, domain_list_ in server_ip_dict.items():
+            query = reportSslCertificate(
+                self._db, ip=ip_, port=443, hostname=domain_list_,
             )
+            for ssl_certificate in query.dicts().iterator():
+                result_dict["ssl_certificate"].append(
+                    {
+                        "hostname": ssl_certificate["hostname"],
+                        "ip": ssl_certificate["ip"],
+                        "port": ssl_certificate["port"],
+                        "sha1_fingerprint": ssl_certificate[
+                            "sha1_fingerprint"
+                        ],
+                        "subject": ssl_certificate["subject"],
+                        "issuer": ssl_certificate["issuer"],
+                        "not_before": rfc822(ssl_certificate["not_before"])
+                        if (ssl_certificate["not_before"] is not None)
+                        else None,
+                        "not_after": rfc822(ssl_certificate["not_after"])
+                        if (ssl_certificate["not_after"] is not None)
+                        else None,
+                        "date": rfc822(ssl_certificate["status"]),
+                    }
+                )
 
         # XXX put back orignal url list
         for url in self.calculateUrlList():
@@ -282,23 +282,28 @@ class WebBot:
                 if root_url in url_dict:
                     url_dict[url] = url_dict[root_url]
 
+        # map IP to URLs for less queries during fetching results
+        ip_to_url_dict = {}
+        for url, ip_list in url_dict.items():
+            for ip in ip_list:
+                ip_to_url_dict.setdefault(ip, [])
+                if url not in ip_to_url_dict[ip]:
+                    ip_to_url_dict[ip].append(url)
+
         # Get the list of HTTP servers to check
-        query = reportHttp(
-            self._db,
-            ip=[x for x in server_ip_dict.keys()],
-            url=[x for x in url_dict.keys()],
-        )
         result_dict["http_query"] = []
-        for network_change in query.dicts().iterator():
-            result_dict["http_query"].append(
-                {
-                    "status_code": network_change["status_code"],
-                    "total_seconds": network_change["total_seconds"],
-                    "url": network_change["url"],
-                    "ip": network_change["ip"],
-                    "date": rfc822(network_change["status"]),
-                }
-            )
+        for ip, url_list in ip_to_url_dict.items():
+            query = reportHttp(self._db, ip=ip, url=url_list)
+            for network_change in query.dicts().iterator():
+                result_dict["http_query"].append(
+                    {
+                        "status_code": network_change["status_code"],
+                        "total_seconds": network_change["total_seconds"],
+                        "url": network_change["url"],
+                        "ip": network_change["ip"],
+                        "date": rfc822(network_change["status"]),
+                    }
+                )
 
         return result_dict