Monitor the SSL certificate status

Cetificates are monitored until they expired.

examples/badssl.cfg

+[SURYKATKA]
+url =
+  https://sha1-2016.badssl.com/
+  https://sha1-2017.badssl.com/
+  https://expired.badssl.com/
+  https://wrong.host.badssl.com/
+  https://self-signed.badssl.com/
+  https://untrusted-root.badssl.com/
+  https://revoked.badssl.com/
+  https://pinning-test.badssl.com/
+  https://no-common-name.badssl.com/
+  https://no-subject.badssl.com/
+  https://no-subject.badssl.com/
+  https://incomplete-chain.badssl.com/
+  https://sha1-intermediate.badssl.com/
+  https://sha256.badssl.com/
+  https://sha384.badssl.com/
+  https://sha512.badssl.com/
+  https://1000-sans.badssl.com/
+  https://10000-sans.badssl.com/
+  https://ecc256.badssl.com/
+  https://ecc384.badssl.com/
+  https://rsa2048.badssl.com/
+  https://rsa4096.badssl.com/
+  https://rsa8192.badssl.com/
+  https://extended-validation.badssl.com/
+  https://cbc.badssl.com/
+  https://rc4-md5.badssl.com/
+  https://rc4.badssl.com/
+  https://3des.badssl.com/
+  https://null.badssl.com/
+  https://mozilla-old.badssl.com/
+  https://mozilla-intermediate.badssl.com/
+  https://mozilla-modern.badssl.com/
+  https://dh480.badssl.com/
+  https://dh512.badssl.com/
+  https://dh1024.badssl.com/
+  https://dh2048.badssl.com/
+  https://dh-small-subgroup.badssl.com/
+  https://dh-composite.badssl.com/
+  https://static-rsa.badssl.com/
+  https://tls-v1-0.badssl.com:1010/
+  https://tls-v1-1.badssl.com:1011/
+  https://tls-v1-2.badssl.com:1012/
+  https://invalid-expected-sct.badssl.com/
+  https://no-sct.badssl.com/
+  https://long-extended-subdomain-name-containing-many-letters-and-dashes.badssl.com/
+  https://longextendedsubdomainnamewithoutdashesinordertotestwordwrapping.badssl.com/
+  https://superfish.badssl.com/
+  https://edellroot.badssl.com/
+  https://dsdtestprovider.badssl.com/
+  https://preact-cli.badssl.com/
+  https://webpack-dev-server.badssl.com/
\ No newline at end of file

src/surykatka/bot.py

@@ -32,6 +32,7 @@ from .network import isTcpPortOpen, reportNetwork
 import json
 import email.utils
 from collections import OrderedDict
+from .ssl import hasValidSSLCertificate, reportSslCertificate
 
 
 __version__ = "0.1.1"
@@ -112,6 +113,19 @@ class WebBot:
                     self._db, server_ip, port, status_id, timeout
                 ):
                     for hostname in server_ip_dict[server_ip]:
+                        if port == 443:
+                            # Store certificate information
+                            if not hasValidSSLCertificate(
+                                self._db,
+                                server_ip,
+                                port,
+                                hostname,
+                                status_id,
+                                timeout,
+                            ):
+                                # If certificate is not valid,
+                                # no need to do another query
+                                continue
                         url = "%s://%s" % (protocol, hostname)
                         if url not in url_dict:
                             url_dict[url] = []
@@ -221,6 +235,33 @@ class WebBot:
                         url_dict[url] = []
                     url_dict[url].append(network_change["ip"])
 
+        # Report the SSL status
+        query = reportSslCertificate(
+            self._db,
+            ip=[x for x in server_ip_dict.keys()],
+            port=443,
+            hostname=domain_list,
+        )
+        result_dict["ssl_certificate"] = []
+        for ssl_certificate in query.dicts().iterator():
+            result_dict["ssl_certificate"].append(
+                {
+                    "hostname": ssl_certificate["hostname"],
+                    "ip": ssl_certificate["ip"],
+                    "port": ssl_certificate["port"],
+                    "sha1_fingerprint": ssl_certificate["sha1_fingerprint"],
+                    "subject": ssl_certificate["subject"],
+                    "issuer": ssl_certificate["issuer"],
+                    "not_before": rfc822(ssl_certificate["not_before"])
+                    if (ssl_certificate["not_before"] is not None)
+                    else None,
+                    "not_after": rfc822(ssl_certificate["not_after"])
+                    if (ssl_certificate["not_after"] is not None)
+                    else None,
+                    "date": rfc822(ssl_certificate["status"]),
+                }
+            )
+
         # XXX put back orignal url list
         for url in self.calculateUrlList():
             if url not in url_dict:

src/surykatka/db.py

@@ -18,7 +18,6 @@
 # See https://www.nexedi.com/licensing for rationale and options.
 
 import peewee
-from playhouse.migrate import migrate, SqliteMigrator
 from playhouse.sqlite_ext import SqliteExtDatabase
 import datetime
 
@@ -94,6 +93,22 @@ class LogDB:
                     "status", "resolver_ip", "domain", "rdtype"
                 )
 
+        class SslChange(BaseModel):
+            status = peewee.ForeignKeyField(Status)
+            ip = peewee.TextField()
+            port = peewee.IntegerField()
+            hostname = peewee.TextField()
+            sha1_fingerprint = peewee.TextField(null=True)
+            not_before = peewee.TimestampField(null=True, utc=True)
+            not_after = peewee.TimestampField(null=True, utc=True)
+            subject = peewee.TextField(null=True)
+            issuer = peewee.TextField(null=True)
+
+            class Meta:
+                primary_key = peewee.CompositeKey(
+                    "status", "ip", "port", "hostname"
+                )
+
         class HttpCodeChange(BaseModel):
             status = peewee.ForeignKeyField(Status)
             ip = peewee.TextField()
@@ -109,13 +124,17 @@ class LogDB:
         self.NetworkChange = NetworkChange
         self.DnsChange = DnsChange
         self.HttpCodeChange = HttpCodeChange
+        self.SslChange = SslChange
 
     def createTables(self):
         # http://www.sqlite.org/pragma.html#pragma_user_version
         db_version = self._db.pragma("user_version")
-        expected_version = 1
-        if db_version == 0:
+        expected_version = 2
+        if db_version != expected_version:
             with self._db.transaction():
+
+                if db_version == 0:
+                    # version 0 (no table)
                     self._db.create_tables(
                         [
                             self.Status,
@@ -126,21 +145,14 @@ class LogDB:
                             self.DnsChange,
                         ]
                     )
-                self._db.pragma("user_version", expected_version)
-        elif db_version != expected_version:
-            # migrator = SqliteMigrator(self._db)
-            migration_list = []
-            sql_query_list = []
 
-            if migration_list or sql_query_list:
-                with self._db.transaction():
-                    if migration_list:
-                        migrate(*migration_list)
-                    if sql_query_list:
-                        for sql_query in sql_query_list:
-                            self._db.execute_sql(
-                                sql_query, require_commit=False
-                            )
+                if db_version <= 1:
+                    # version 1 without SSL support
+                    self._db.create_tables([self.SslChange])
+
+                else:
+                    raise ValueError("Can not downgrade SQLite DB")
+
                 self._db.pragma("user_version", expected_version)
 
     def close(self):

src/surykatka/ssl.py

+# Copyright (C) 2019  Nexedi SA and Contributors.
+#                     Romain Courteaud <romain@nexedi.com>
+#
+# This program is free software: you can Use, Study, Modify and Redistribute
+# it under the terms of the GNU General Public License version 3, or (at your
+# option) any later version, as published by the Free Software Foundation.
+#
+# You can also Link and Combine this program with other software covered by
+# the terms of any of the Free Software licenses or any of the Open Source
+# Initiative approved licenses and Convey the resulting work. Corresponding
+# source of such a combination shall include the source code for all other
+# software used.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See COPYING file for full licensing terms.
+# See https://www.nexedi.com/licensing for rationale and options.
+
+from peewee import fn
+import socket
+import ssl
+import hashlib
+from binascii import hexlify
+import datetime
+
+
+TIMEOUT = 2
+
+
+def reportSslCertificate(db, ip=None, port=None, hostname=None):
+    query = (
+        db.SslChange.select(db.SslChange)
+        .group_by(db.SslChange.ip, db.SslChange.port, db.SslChange.hostname,)
+        .having(db.SslChange.status_id == fn.MAX(db.SslChange.status_id))
+    )
+
+    if hostname is not None:
+        if type(hostname) == list:
+            query = query.where(db.SslChange.hostname << hostname)
+        else:
+            query = query.where(db.SslChange.hostname == hostname)
+    if port is not None:
+        if type(port) == list:
+            query = query.where(db.SslChange.port << port)
+        else:
+            query = query.where(db.SslChange.port == port)
+    if ip is not None:
+        if type(ip) == list:
+            query = query.where(db.SslChange.ip << ip)
+        else:
+            query = query.where(db.SslChange.ip == ip)
+    return query
+
+
+def logSslCertificate(
+    db,
+    ip,
+    port,
+    hostname,
+    sha1_fingerprint,
+    not_before,
+    not_after,
+    subject,
+    issuer,
+    status_id,
+):
+
+    with db._db.atomic():
+        try:
+            # Check previous parameter value
+            previous_entry = reportSslCertificate(
+                db, ip=ip, port=port, hostname=hostname
+            ).get()
+        except db.SslChange.DoesNotExist:
+            previous_entry = None
+
+        if (previous_entry is None) or (
+            previous_entry.sha1_fingerprint != sha1_fingerprint
+        ):
+            previous_entry = db.SslChange.create(
+                status=status_id,
+                ip=ip,
+                port=port,
+                hostname=hostname,
+                sha1_fingerprint=sha1_fingerprint,
+                not_before=not_before,
+                not_after=not_after,
+                subject=subject,
+                issuer=issuer,
+            )
+        return previous_entry.status_id
+
+
+def hasValidSSLCertificate(db, ip, port, hostname, status_id, timeout=TIMEOUT):
+    ssl_context = ssl.create_default_context()
+    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    sock.settimeout(timeout)
+    wrapped_sock = ssl_context.wrap_socket(sock, server_hostname=hostname)
+
+    try:
+        wrapped_sock.connect((ip, port))
+        der = wrapped_sock.getpeercert(True)
+        # XXX How to extract info from the der directly?
+        ssl_info = wrapped_sock.getpeercert()
+
+    except (ssl.SSLError, ConnectionRefusedError, socket.timeout, OSError):
+        wrapped_sock.close()
+        # XXX Expired certificate can not be fetched with the builtin ssl lib
+        # pyOpenSSL is one way to fix this
+        # https://stackoverflow.com/a/52298575
+        logSslCertificate(
+            db, ip, port, hostname, None, None, None, None, None, status_id,
+        )
+        return False
+    except:
+        wrapped_sock.close()
+        raise
+
+    wrapped_sock.close()
+
+    sha1_fingerprint = hexlify(hashlib.sha1(der).digest())
+    ssl_date_fmt = "%b %d %H:%M:%S %Y %Z"
+    not_before = datetime.datetime.strptime(
+        ssl_info["notBefore"], ssl_date_fmt
+    )
+    not_after = datetime.datetime.strptime(ssl_info["notAfter"], ssl_date_fmt)
+    subject = dict([y for x in ssl_info["subject"] for y in x]).get(
+        "commonName", ""
+    )
+    issuer = dict([y for x in ssl_info["issuer"] for y in x]).get(
+        "commonName", ""
+    )
+    logSslCertificate(
+        db,
+        ip,
+        port,
+        hostname,
+        sha1_fingerprint.decode(),
+        not_before,
+        not_after,
+        subject,
+        issuer,
+        status_id,
+    )
+    return True

tests/test_db.py

@@ -28,7 +28,38 @@ class SurykatkaDBTestCase(unittest.TestCase):
     def test_createTable(self):
         assert self.db._db.pragma("user_version") == 0
         self.db.createTables()
-        assert self.db._db.pragma("user_version") == 1
+        assert self.db._db.pragma("user_version") == 2
+
+    def test_downgrade(self):
+        assert self.db._db.pragma("user_version") == 0
+        # Unpossible high version
+        self.db._db.pragma("user_version", 9999)
+        try:
+            self.db.createTables()
+        except ValueError:
+            assert self.db._db.pragma("user_version") == 9999
+        else:
+            raise NotImplementedError("Expected ValueError")
+
+    def test_migrationFromVersion1(self):
+        assert self.db._db.pragma("user_version") == 0
+        # Recreate version 1
+        with self.db._db.transaction():
+
+            self.db._db.create_tables(
+                [
+                    self.db.Status,
+                    self.db.ConfigurationChange,
+                    self.db.HttpCodeChange,
+                    self.db.NetworkChange,
+                    self.db.PlatformChange,
+                    self.db.DnsChange,
+                ]
+            )
+            self.db._db.pragma("user_version", 1)
+
+        self.db.createTables()
+        assert self.db._db.pragma("user_version") == 2
 
 
 def suite():