http: Allow to configure elapsed time intervals

src/surykatka/bot.py

@@ -84,6 +84,8 @@ class WebBot:
             self.config = createConfiguration(**self.config_kw)
 
         timeout = int(self.config["TIMEOUT"])
+        elapsed_fast = float(self.config["ELAPSED_FAST"])
+        elapsed_moderate = float(self.config["ELAPSED_moderate"])
         # logPlatform(self._db, __version__, status_id)
 
         # Calculate the resolver list
@@ -142,7 +144,7 @@ class WebBot:
         for url in url_dict:
             for ip in url_dict[url]:
                 checkHttpStatus(
-                    self._db, status_id, url, ip, __version__, timeout
+                    self._db, status_id, url, ip, __version__, timeout, elapsed_fast, elapsed_moderate
                 )
                 # XXX Check location header and check new url recursively
                 # XXX Parse HTML, fetch found link, css, js, image

src/surykatka/configuration.py

@@ -56,6 +56,10 @@ def createConfiguration(
         config[CONFIG_SECTION]["FORMAT"] = "json"
     if "TIMEOUT" not in config[CONFIG_SECTION]:
         config[CONFIG_SECTION]["TIMEOUT"] = "1"
+    if "ELAPSED_FAST" not in config[CONFIG_SECTION]:
+        config[CONFIG_SECTION]["ELAPSED_FAST"] = "0.2"
+    if "ELAPSED_MODERATE" not in config[CONFIG_SECTION]:
+        config[CONFIG_SECTION]["ELAPSED_MODERATE"] = "0.5"
     if "RELOAD" not in config[CONFIG_SECTION]:
         config[CONFIG_SECTION]["RELOAD"] = str(False)
 

src/surykatka/http.py

@@ -25,6 +25,8 @@ from peewee import fn
 
 PREFERRED_TYPE = "text/html"
 TIMEOUT = 2
+ELAPSED_FAST = 0.2
+ELAPSED_MODERATE = 0.5
 
 
 def getUrlHostname(url):
@@ -105,20 +107,20 @@ def reportHttp(db, ip=None, url=None):
     return query
 
 
-def calculateSpeedRange(total_seconds):
+def calculateSpeedRange(total_seconds, fast, moderate):
     # Prevent updating the DB by defining acceptable speed range
     if total_seconds == 0:
         # error cases
         return "BAD"
-    elif total_seconds < 0.2:
+    elif total_seconds < fast:
         return "FAST"
-    elif total_seconds < 0.5:
+    elif total_seconds < moderate:
         return "MODERATE"
     else:
         return "SLOW"
 
 
-def logHttpStatus(db, ip, url, code, total_seconds, status_id):
+def logHttpStatus(db, ip, url, code, total_seconds, fast, moderate, status_id):
 
     with db._db.atomic():
         try:
@@ -131,8 +133,8 @@ def logHttpStatus(db, ip, url, code, total_seconds, status_id):
             (previous_entry is None)
             or (previous_entry.status_code != code)
             or (
-                calculateSpeedRange(previous_entry.total_seconds)
-                != calculateSpeedRange(total_seconds)
+                calculateSpeedRange(previous_entry.total_seconds, fast, moderate)
+                != calculateSpeedRange(total_seconds, fast, moderate)
             )
         ):
             previous_entry = db.HttpCodeChange.create(
@@ -145,7 +147,7 @@ def logHttpStatus(db, ip, url, code, total_seconds, status_id):
         return previous_entry.status_id
 
 
-def checkHttpStatus(db, status_id, url, ip, bot_version, timeout=TIMEOUT):
+def checkHttpStatus(db, status_id, url, ip, bot_version, timeout=TIMEOUT, elapsed_fast=ELAPSED_FAST, elapsed_moderate=ELAPSED_MODERATE):
     parsed_url = urlparse(url)
     hostname = parsed_url.hostname
     request_kw = {"timeout": timeout}
@@ -175,5 +177,7 @@ def checkHttpStatus(db, status_id, url, ip, bot_version, timeout=TIMEOUT):
         url,
         response.status_code,
         response.elapsed.total_seconds(),
+        elapsed_fast,
+        elapsed_moderate,
         status_id,
     )

tests/test_http.py

@@ -185,9 +185,11 @@ class SurykatkaHttpTestCase(unittest.TestCase):
         url = "http://example.org"
         status_code = 200
         total_seconds = 0.1
+        fast = 0.2
+        moderate = 0.5
         status_id = logStatus(self.db, "foo")
         result = logHttpStatus(
-            self.db, ip, url, status_code, total_seconds, status_id
+            self.db, ip, url, status_code, total_seconds, fast, moderate, status_id
         )
         assert self.db.HttpCodeChange.select().count() == 1
         assert self.db.HttpCodeChange.get().ip == ip
@@ -202,13 +204,15 @@ class SurykatkaHttpTestCase(unittest.TestCase):
         url = "http://example.org"
         status_code = 200
         total_seconds = 0.1
+        fast = 0.2
+        moderate = 0.5
         status_id = logStatus(self.db, "foo")
         result = logHttpStatus(
-            self.db, ip, url, status_code, total_seconds, status_id
+            self.db, ip, url, status_code, total_seconds, fast, moderate, status_id
         )
         try:
             logHttpStatus(
-                self.db, ip, url, status_code + 1, total_seconds + 1, status_id
+                self.db, ip, url, status_code + 1, total_seconds + 1, fast, moderate, status_id
             )
         except peewee.IntegrityError:
             assert self.db.HttpCodeChange.select().count() == 1
@@ -221,13 +225,15 @@ class SurykatkaHttpTestCase(unittest.TestCase):
         url = "http://example.org"
         status_code = 200
         total_seconds = 0.1
+        fast = 0.2
+        moderate = 0.5
         status_id = logStatus(self.db, "foo")
         result = logHttpStatus(
-            self.db, ip, url, status_code, total_seconds, status_id
+            self.db, ip, url, status_code, total_seconds, fast, moderate, status_id
         )
         status_id_2 = logStatus(self.db, "foo")
         result_2 = logHttpStatus(
-            self.db, ip, url, status_code, total_seconds, status_id_2
+            self.db, ip, url, status_code, total_seconds, fast, moderate, status_id_2
         )
         assert result_2 == result
         assert self.db.HttpCodeChange.select().count() == 1
@@ -242,14 +248,16 @@ class SurykatkaHttpTestCase(unittest.TestCase):
         url = "http://example.org"
         status_code = 200
         total_seconds = 0.1
+        fast = 0.2
+        moderate = 0.5
         status_code_2 = status_code + 1
         status_id = logStatus(self.db, "foo")
         result = logHttpStatus(
-            self.db, ip, url, status_code, total_seconds, status_id
+            self.db, ip, url, status_code, total_seconds, fast, moderate, status_id
         )
         status_id_2 = logStatus(self.db, "foo")
         result_2 = logHttpStatus(
-            self.db, ip, url, status_code_2, total_seconds, status_id_2
+            self.db, ip, url, status_code_2, total_seconds, fast, moderate, status_id_2
         )
         assert result_2 != result
         assert self.db.HttpCodeChange.select().count() == 2
@@ -302,6 +310,63 @@ class SurykatkaHttpTestCase(unittest.TestCase):
             == total_seconds
         )
 
+    def test_logHttpStatus_configurableRanges(self):
+        ip = "127.0.0.1"
+        url = "http://example.org"
+        status_code = 200
+        total_seconds_error = 0
+        total_seconds_fast = 4
+        total_seconds_moderate = 5
+        total_seconds_slow = 30
+        fast = 5
+        moderate = 30
+        status_id = logStatus(self.db, "foo")
+        result = logHttpStatus(
+            self.db, ip, url, status_code, total_seconds_error, fast, moderate, status_id
+        )
+        status_id_2 = logStatus(self.db, "foo")
+        result_2 = logHttpStatus(
+            self.db, ip, url, status_code, total_seconds_fast, fast, moderate, status_id_2
+        )
+        status_id_3 = logStatus(self.db, "foo")
+        result_3 = logHttpStatus(
+            self.db, ip, url, status_code, total_seconds_moderate, fast, moderate, status_id_3
+        )
+        status_id_4 = logStatus(self.db, "foo")
+        result_4 = logHttpStatus(
+            self.db, ip, url, status_code, total_seconds_slow, fast, moderate, status_id_4
+        )
+        assert status_id == result
+        assert status_id_2 == result_2
+        assert status_id_3 == result_3
+        assert status_id_4 == result_4
+        assert self.db.HttpCodeChange.select().count() == 4
+
+        assert (
+            self.db.HttpCodeChange.get(
+                self.db.HttpCodeChange.status == status_id
+            ).total_seconds
+            == total_seconds_error
+        )
+        assert (
+            self.db.HttpCodeChange.get(
+                self.db.HttpCodeChange.status == status_id_2
+            ).total_seconds
+            == total_seconds_fast
+        )
+        assert (
+            self.db.HttpCodeChange.get(
+                self.db.HttpCodeChange.status == status_id_3
+            ).total_seconds
+            == total_seconds_moderate
+        )
+        assert (
+            self.db.HttpCodeChange.get(
+                self.db.HttpCodeChange.status == status_id_4
+            ).total_seconds
+            == total_seconds_slow
+        )
+
     def test_logHttpStatus_insertWhenDifferentTotalSeconds(self):
         ip = "127.0.0.1"
         url = "http://example.org"
@@ -310,21 +375,23 @@ class SurykatkaHttpTestCase(unittest.TestCase):
         total_seconds_fast = 0.001
         total_seconds_moderate = 0.2
         total_seconds_slow = 0.5
+        fast = 0.2
+        moderate = 0.5
         status_id = logStatus(self.db, "foo")
         result = logHttpStatus(
-            self.db, ip, url, status_code, total_seconds_error, status_id
+            self.db, ip, url, status_code, total_seconds_error, fast, moderate, status_id
         )
         status_id_2 = logStatus(self.db, "foo")
         result_2 = logHttpStatus(
-            self.db, ip, url, status_code, total_seconds_fast, status_id_2
+            self.db, ip, url, status_code, total_seconds_fast, fast, moderate, status_id_2
         )
         status_id_3 = logStatus(self.db, "foo")
         result_3 = logHttpStatus(
-            self.db, ip, url, status_code, total_seconds_moderate, status_id_3
+            self.db, ip, url, status_code, total_seconds_moderate, fast, moderate, status_id_3
         )
         status_id_4 = logStatus(self.db, "foo")
         result_4 = logHttpStatus(
-            self.db, ip, url, status_code, total_seconds_slow, status_id_4
+            self.db, ip, url, status_code, total_seconds_slow, fast, moderate, status_id_4
         )
         assert status_id == result
         assert status_id_2 == result_2
@@ -363,13 +430,15 @@ class SurykatkaHttpTestCase(unittest.TestCase):
         status_code = 200
         total_seconds = 0.001
         total_seconds_2 = 0.199
+        fast = 0.2
+        moderate = 0.5
         status_id = logStatus(self.db, "foo")
         result = logHttpStatus(
-            self.db, ip, url, status_code, total_seconds, status_id
+            self.db, ip, url, status_code, total_seconds, fast, moderate, status_id
         )
         status_id_2 = logStatus(self.db, "foo")
         result_2 = logHttpStatus(
-            self.db, ip, url, status_code, total_seconds_2, status_id_2
+            self.db, ip, url, status_code, total_seconds_2, fast, moderate, status_id_2
         )
         assert result_2 == result
         assert self.db.HttpCodeChange.select().count() == 1
@@ -385,13 +454,15 @@ class SurykatkaHttpTestCase(unittest.TestCase):
         status_code = 200
         total_seconds = 0.2
         total_seconds_2 = 0.499
+        fast = 0.2
+        moderate = 0.5
         status_id = logStatus(self.db, "foo")
         result = logHttpStatus(
-            self.db, ip, url, status_code, total_seconds, status_id
+            self.db, ip, url, status_code, total_seconds, fast, moderate, status_id
         )
         status_id_2 = logStatus(self.db, "foo")
         result_2 = logHttpStatus(
-            self.db, ip, url, status_code, total_seconds_2, status_id_2
+            self.db, ip, url, status_code, total_seconds_2, fast, moderate, status_id_2
         )
         assert result_2 == result
         assert self.db.HttpCodeChange.select().count() == 1
@@ -407,13 +478,15 @@ class SurykatkaHttpTestCase(unittest.TestCase):
         status_code = 200
         total_seconds = 0.5
         total_seconds_2 = 99
+        fast = 0.2
+        moderate = 0.5
         status_id = logStatus(self.db, "foo")
         result = logHttpStatus(
-            self.db, ip, url, status_code, total_seconds, status_id
+            self.db, ip, url, status_code, total_seconds, fast, moderate, status_id
         )
         status_id_2 = logStatus(self.db, "foo")
         result_2 = logHttpStatus(
-            self.db, ip, url, status_code, total_seconds_2, status_id_2
+            self.db, ip, url, status_code, total_seconds_2, fast, moderate, status_id_2
         )
         assert result_2 == result
         assert self.db.HttpCodeChange.select().count() == 1
@@ -430,16 +503,18 @@ class SurykatkaHttpTestCase(unittest.TestCase):
         url_2 = url + "2"
         total_seconds = 0.1
         status_code = 200
+        fast = 0.2
+        moderate = 0.5
         status_id = logStatus(self.db, "foo")
-        logHttpStatus(self.db, ip, url, status_code, total_seconds, status_id)
+        logHttpStatus(self.db, ip, url, status_code, total_seconds, fast, moderate, status_id)
         logHttpStatus(
-            self.db, ip_2, url, status_code, total_seconds, status_id
+            self.db, ip_2, url, status_code, total_seconds, fast, moderate, status_id
         )
         logHttpStatus(
-            self.db, ip, url_2, status_code, total_seconds, status_id
+            self.db, ip, url_2, status_code, total_seconds, fast, moderate, status_id
         )
         logHttpStatus(
-            self.db, ip_2, url_2, status_code, total_seconds, status_id
+            self.db, ip_2, url_2, status_code, total_seconds, fast, moderate, status_id
         )
         assert self.db.HttpCodeChange.select().count() == 4