bot: Correctly group IPs per URLs and DOMAINs

src/surykatka/bot.py

@@ -249,20 +249,20 @@ class WebBot:
                     url_dict[url].append(network_change["ip"])
 
         # Report the SSL status
+        result_dict["ssl_certificate"] = []
+        for ip_, domain_list_ in server_ip_dict.items():
             query = reportSslCertificate(
-            self._db,
-            ip=[x for x in server_ip_dict.keys()],
-            port=443,
-            hostname=domain_list,
+                self._db, ip=ip_, port=443, hostname=domain_list_,
             )
-        result_dict["ssl_certificate"] = []
             for ssl_certificate in query.dicts().iterator():
                 result_dict["ssl_certificate"].append(
                     {
                         "hostname": ssl_certificate["hostname"],
                         "ip": ssl_certificate["ip"],
                         "port": ssl_certificate["port"],
-                    "sha1_fingerprint": ssl_certificate["sha1_fingerprint"],
+                        "sha1_fingerprint": ssl_certificate[
+                            "sha1_fingerprint"
+                        ],
                         "subject": ssl_certificate["subject"],
                         "issuer": ssl_certificate["issuer"],
                         "not_before": rfc822(ssl_certificate["not_before"])
@@ -282,13 +282,18 @@ class WebBot:
                 if root_url in url_dict:
                     url_dict[url] = url_dict[root_url]
 
+        # map IP to URLs for less queries during fetching results
+        ip_to_url_dict = {}
+        for url, ip_list in url_dict.items():
+            for ip in ip_list:
+                ip_to_url_dict.setdefault(ip, [])
+                if url not in ip_to_url_dict[ip]:
+                    ip_to_url_dict[ip].append(url)
+
         # Get the list of HTTP servers to check
-        query = reportHttp(
-            self._db,
-            ip=[x for x in server_ip_dict.keys()],
-            url=[x for x in url_dict.keys()],
-        )
         result_dict["http_query"] = []
+        for ip, url_list in ip_to_url_dict.items():
+            query = reportHttp(self._db, ip=ip, url=url_list)
             for network_change in query.dicts().iterator():
                 result_dict["http_query"].append(
                     {