http: store http request elapsed time

The goal is to detect speed regression of website access.

The elapsed time value is updated in the DB only if an arbitrary threshold is modified (<200ms = fast, <500ms = moderate, >500ms = slow).

src/surykatka/bot.py

@@ -284,6 +284,7 @@ class WebBot:
             result_dict["http_query"].append(
                 {
                     "status_code": network_change["status_code"],
+                    "total_seconds": network_change["total_seconds"],
                     "url": network_change["url"],
                     "ip": network_change["ip"],
                     "date": rfc822(network_change["status"]),

src/surykatka/db.py

@@ -20,6 +20,7 @@
 import peewee
 from playhouse.sqlite_ext import SqliteExtDatabase
 import datetime
+from playhouse.migrate import migrate, SqliteMigrator
 
 
 class LogDB:
@@ -114,6 +115,7 @@ class LogDB:
             ip = peewee.TextField()
             url = peewee.TextField()
             status_code = peewee.IntegerField()
+            total_seconds = peewee.FloatField(default=0)
 
             class Meta:
                 primary_key = peewee.CompositeKey("status", "ip", "url")
@@ -129,7 +131,7 @@ class LogDB:
     def createTables(self):
         # http://www.sqlite.org/pragma.html#pragma_user_version
         db_version = self._db.pragma("user_version")
-        expected_version = 2
+        expected_version = 3
         if db_version != expected_version:
             with self._db.transaction():
 
@@ -150,7 +152,18 @@ class LogDB:
                     # version 1 without SSL support
                     self._db.create_tables([self.SslChange])
 
-                else:
+                if (0 < db_version) and (db_version <= 2):
+                    # version 2 without the http total_seconds column
+                    migrator = SqliteMigrator(self._db)
+                    migrate(
+                        migrator.add_column(
+                            "HttpCodeChange",
+                            "total_seconds",
+                            self.HttpCodeChange.total_seconds,
+                        )
+                    )
+
+                if db_version >= expected_version:
                     raise ValueError("Can not downgrade SQLite DB")
 
                 self._db.pragma("user_version", expected_version)

src/surykatka/http.py

@@ -78,7 +78,6 @@ def request(url, timeout=TIMEOUT, headers=None, session=requests, version=0):
     except requests.exceptions.TooManyRedirects:
         response = requests.models.Response()
         response.status_code = 520
-
     return response
 
 
@@ -106,7 +105,20 @@ def reportHttp(db, ip=None, url=None):
     return query
 
 
-def logHttpStatus(db, ip, url, code, status_id):
+def calculateSpeedRange(total_seconds):
+    # Prevent updating the DB by defining acceptable speed range
+    if total_seconds == 0:
+        # error cases
+        return "BAD"
+    elif total_seconds < 0.2:
+        return "FAST"
+    elif total_seconds < 0.5:
+        return "MODERATE"
+    else:
+        return "SLOW"
+
+
+def logHttpStatus(db, ip, url, code, total_seconds, status_id):
 
     with db._db.atomic():
         try:
@@ -115,9 +127,20 @@ def logHttpStatus(db, ip, url, code, status_id):
         except db.HttpCodeChange.DoesNotExist:
             previous_entry = None
 
-        if (previous_entry is None) or (previous_entry.status_code != code):
+        if (
+            (previous_entry is None)
+            or (previous_entry.status_code != code)
+            or (
+                calculateSpeedRange(previous_entry.total_seconds)
+                != calculateSpeedRange(total_seconds)
+            )
+        ):
             previous_entry = db.HttpCodeChange.create(
-                status=status_id, ip=ip, url=url, status_code=code
+                status=status_id,
+                ip=ip,
+                url=url,
+                status_code=code,
+                total_seconds=total_seconds,
             )
         return previous_entry.status_id
 
@@ -146,4 +169,11 @@ def checkHttpStatus(db, status_id, url, ip, bot_version, timeout=TIMEOUT):
     response = request(
         ip_url, headers={"Host": hostname}, version=bot_version, **request_kw
     )
-    logHttpStatus(db, ip, url, response.status_code, status_id)
+    logHttpStatus(
+        db,
+        ip,
+        url,
+        response.status_code,
+        response.elapsed.total_seconds(),
+        status_id,
+    )

tests/test_db.py

@@ -19,6 +19,42 @@
 
 import unittest
 from surykatka.db import LogDB
+from playhouse.migrate import migrate, SqliteMigrator
+from collections import namedtuple
+from playhouse.reflection import Introspector
+
+ValidationResult = namedtuple(
+    "ValidationResult",
+    ("valid", "table_exists", "add_fields", "remove_fields", "change_fields"),
+)
+
+
+def validate_schema(model):
+    db = model._meta.database
+    table = model._meta.table_name
+    if not db.table_exists(table):
+        return ValidationResult(False, False, None, None, None)
+
+    introspector = Introspector.from_database(db)
+    db_model = introspector.generate_models(table_names=[table])[table]
+
+    columns = set(model._meta.columns)
+    db_columns = set(db_model._meta.columns)
+
+    to_remove = [model._meta.columns[c] for c in columns - db_columns]
+    to_add = [db_model._meta.columns[c] for c in db_columns - columns]
+    to_change = []
+    intersect = columns & db_columns  # Take intersection and remove matches.
+    for column in intersect:
+        field = model._meta.columns[column]
+        db_field = db_model._meta.columns[column]
+        if (field.field_type != db_field.field_type) and (
+            not (field.field_type == "BIGINT")
+        ):
+            to_change.append((field, db_field))
+
+    is_valid = not any((to_remove, to_add, to_change))
+    return ValidationResult(is_valid, True, to_add, to_remove, to_change)
 
 
 class SurykatkaDBTestCase(unittest.TestCase):
@@ -28,7 +64,7 @@ class SurykatkaDBTestCase(unittest.TestCase):
     def test_createTable(self):
         assert self.db._db.pragma("user_version") == 0
         self.db.createTables()
-        assert self.db._db.pragma("user_version") == 2
+        assert self.db._db.pragma("user_version") == 3
 
     def test_downgrade(self):
         assert self.db._db.pragma("user_version") == 0
@@ -56,10 +92,47 @@ class SurykatkaDBTestCase(unittest.TestCase):
                     self.db.DnsChange,
                 ]
             )
+            migrate(
+                SqliteMigrator(self.db._db).drop_column(
+                    "HttpCodeChange", "total_seconds"
+                ),
+            )
             self.db._db.pragma("user_version", 1)
 
         self.db.createTables()
-        assert self.db._db.pragma("user_version") == 2
+        assert self.db._db.pragma("user_version") == 3
+        assert validate_schema(self.db.SslChange).valid, validate_schema(
+            self.db.SslChange
+        )
+
+    def test_migrationFromVersion2(self):
+        assert self.db._db.pragma("user_version") == 0
+        # Recreate version 2
+        with self.db._db.transaction():
+
+            self.db._db.create_tables(
+                [
+                    self.db.Status,
+                    self.db.ConfigurationChange,
+                    self.db.HttpCodeChange,
+                    self.db.NetworkChange,
+                    self.db.PlatformChange,
+                    self.db.DnsChange,
+                    self.db.SslChange,
+                ]
+            )
+            migrate(
+                SqliteMigrator(self.db._db).drop_column(
+                    "HttpCodeChange", "total_seconds"
+                ),
+            )
+            self.db._db.pragma("user_version", 2)
+
+        self.db.createTables()
+        assert self.db._db.pragma("user_version") == 3
+        assert validate_schema(self.db.HttpCodeChange).valid, validate_schema(
+            self.db.HttpCodeChange
+        )
 
 
 def suite():