monitor: Implement check-maximum-elapsed-time

Allows to configure maximum time for which the site is considered correct.

For each class (the maximum elapsed time value) separate surykatka process
is started, in order to not impact fast sites checking by long timeouts on
slow sites.

software/monitor/buildout.hash.cfg

@@ -14,7 +14,7 @@
 # not need these here).
 [template]
 filename = instance.cfg
-md5sum = d778b6f436ae6864819eb2ff2d12a86f
+md5sum = 1e64cee5a2249d19f6241312f94d06b7
 
 [template-monitor]
 _update_hash_filename_ = instance-monitor.cfg.jinja2
@@ -30,7 +30,7 @@ md5sum = 9e237dbdda59e788202f0da194a57d41
 
 [template-monitor-edgebot]
 _update_hash_filename_ = instance-monitor-edgebot.cfg.jinja2
-md5sum = 8786e4245db0d27dfa4815222d970e52
+md5sum = b8f3c467cfd988e29be2ba70aa7a059e
 
 [network-bench-cfg]
 filename = network_bench.cfg.in
@@ -42,4 +42,4 @@ md5sum = cad2402bbd21907cfed6bc5af8c5d3ab
 
 [template-surykatka-ini]
 _update_hash_filename_ = surykatka.ini.jinja2
-md5sum = 40870921e05d93b5843ab34abd7e3902
+md5sum = a2de719a5a65438c8c3ee5195442beb6

software/monitor/instance-edgetest-input-schema.json

@@ -24,6 +24,12 @@
       "title": "Default certificate expiration days check",
       "description": "Default amount of days to consider certificate as being to-be-expired (default: 15).",
       "type": "string"
+    },
+    "check-maximum-elapsed-time": {
+      "default": "2",
+      "title": "Default maximum elapsed time for a site to reply (seconds)",
+      "description": "Default maximum elapsed time for a site to reply to be considered good (default: 2s).",
+      "type": "string"
     }
   }
 }

software/monitor/instance-edgetest-slave-input-schema.json

@@ -23,6 +23,12 @@
       "title": "Certificate expiration days check",
       "description": "Amount of days to consider certificate as being to-be-expired (default: comes from master partition).",
       "type": "string"
+    },
+    "check-maximum-elapsed-time": {
+      "default": "Master default",
+      "title": "Maximum elapsed time for a site to reply (seconds)",
+      "description": "Maximum elapsed time for a site to reply to be considered good.(default: comes from master partition).",
+      "type": "string"
     }
   }
 }

software/monitor/instance-monitor-edgebot.cfg.jinja2

@@ -5,7 +5,7 @@
 {%-     do CONFIGURATION.__setitem__(k[14:], v) %}
 {%-   endif %}
 {%- endfor %}
-{%- set slave_instance_list = [] %}
+{%- set slave_instance_dict = {} %}
 {%- set extra_slave_instance_list = slapparameter_dict.get('extra_slave_instance_list') %}
 {%- if extra_slave_instance_list %}
 {#- Create slaves to process with setting up defaults #}
@@ -16,19 +16,28 @@
 {%-   if 'check-certificate-expiration-days' not in slave %}
 {%-     do slave.__setitem__('check-certificate-expiration-days', CONFIGURATION['check-certificate-expiration-days']) %}
 {%-   endif %}
+{%-   if 'check-maximum-elapsed-time' not in slave %}
+{%-     do slave.__setitem__('check-maximum-elapsed-time', CONFIGURATION['check-maximum-elapsed-time']) %}
+{%-   endif %}
 {%-   if 'check-frontend-ip' not in slave %}
 {%-     do slave.__setitem__('check-frontend-ip', CONFIGURATION['check-frontend-ip']) %}
 {%-   endif %}
 {%-   if 'url' in slave %}
-{%-     do slave_instance_list.append(slave) %}
+{%-     set class = slave['check-maximum-elapsed-time'] %}
+{%-     if class not in slave_instance_dict %}
+{%-       do slave_instance_dict.__setitem__(class, []) %}
+{%-     endif %}
+{%-     do slave_instance_dict[class].append(slave) %}
 {%-   endif %}
 {%-   endfor %}
 {%- endif %}
 {%- set part_list = [] %}
-{%- for slave in sorted(slave_instance_list) %}
-{%-   set part_id = 'http-query-' ~ slave['slave_reference'] ~ '-promise' %}
-{%-   do part_list.append(part_id) %}
-{%-   set safe_name = part_id.replace('_', '').replace('.', '-').replace(' ', '-') %}
+{%- for class, slave_instance_list in slave_instance_dict.items() %}
+{#- class is used to separate surykatka with different timeouts #}
+{%-   for slave in sorted(slave_instance_list) %}
+{%-     set part_id = 'http-query-' ~ slave['slave_reference'] ~ '-promise' %}
+{%-     do part_list.append(part_id) %}
+{%-     set safe_name = part_id.replace('_', '').replace('.', '-').replace(' ', '-') %}
 [{{part_id}}]
 <= monitor-promise-base
 module = check_surykatka_json
@@ -37,75 +46,80 @@ config-report = http_query
 config-url = {{ slave['url'] }}
 config-status-code = {{ slave['check-status-code'] }}
 config-certificate-expiration-days = {{ slave['check-certificate-expiration-days'] }}
+config-maximum-elapsed-time = {{ slave['check-maximum-elapsed-time'] }}
 config-ip-list = {{ slave['check-frontend-ip'] }}
-config-json-file = ${surykatka-config:json}
-{% endfor %}
+config-json-file = ${surykatka-config-{{ class }}:json}
+{%-   endfor %}
 
-[surykatka-bot-promise]
+[surykatka-bot-promise-{{ class }}]
 <= monitor-promise-base
 module = check_surykatka_json
-name = surykatka-bot-promise.py
+name = surykatka-bot-promise-{{ class }}.py
 config-report = bot_status
-config-json-file = ${surykatka-config:json}
+config-json-file = ${surykatka-config-{{ class }}:json}
 
-[buildout]
-extends = {{ monitor_template_output }}
-
-parts =
-  cron
-  cron-entry-surykatka-status
-  monitor-base
-  publish-connection-information
-  surykatka
-  surykatka-bot-promise
-{% for part_id in sorted(part_list) %}
-  {{ part_id }}
-{% endfor %}
-
-eggs-directory = {{ eggs_directory }}
-develop-eggs-directory = {{ develop_eggs_directory }}
-offline = true
-
-[surykatka-config]
+[surykatka-config-{{ class }}]
 recipe = slapos.recipe.template:jinja2
-db = ${directory:srv}/surykatka.db
-rendered = ${directory:etc}/surykatka.ini
+db = ${directory:srv}/surykatka-{{ class }}.db
+rendered = ${directory:etc}/surykatka-{{ class }}.ini
 template = {{ template_surykatka_ini }}
 slave_instance_list = {{ dumps(slave_instance_list) }}
 nameserver = {{ dumps(CONFIGURATION['nameserver']) }}
-json = ${directory:srv}/surykatka.json
+json = ${directory:srv}/surykatka-{{ class }}.json
+{#- timeout is just a bit bigger than class time #}
+timeout = {{ int(class) + 2 }}
 
 context =
   import json_module json
   key db :db
   key nameserver :nameserver
   key slave_instance_list :slave_instance_list
+  key timeout :timeout
 
-[surykatka]
+[surykatka-{{ class }}]
 recipe = slapos.cookbook:wrapper
-config = ${surykatka-config:rendered}
+config = ${surykatka-config-{{ class }}:rendered}
 command-line =
   {{ surykatka_binary }} --run crawl --reload --configuration ${:config}
 wrapper-path = ${monitor-directory:service}/${:_buildout_section_name_}
 hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
 
-[surykatka-status-json]
+[surykatka-status-json-{{ class }}]
 recipe = slapos.recipe.template:jinja2
+json = ${surykatka-config-{{ class }}:json}
 template = inline:#!/bin/sh
-  if {{ surykatka_binary }} --run status --configuration ${surykatka:config} --output json > ${surykatka-config:json}.tmp ; then
-    mv -f ${surykatka-config:json}.tmp ${surykatka-config:json}
+  if {{ surykatka_binary }} --run status --configuration ${surykatka-{{ class }}:config} --output json > ${:json}.tmp ; then
+    mv -f ${:json}.tmp ${:json}
   else
-    rm -f ${surykatka-config:json}.tmp
+    rm -f ${:json}.tmp
   fi
 rendered = ${monitor-directory:bin}/${:_buildout_section_name_}
 mode = 0755
 
-[cron-entry-surykatka-status]
+[cron-entry-surykatka-status-{{ class }}]
 recipe = slapos.cookbook:cron.d
 cron-entries = ${directory:etc}/cron.d
-name = surykatka-status
+name = surykatka-status-{{ class }}
 frequency = */2 * * * *
-command = ${surykatka-status-json:rendered}
+command = ${surykatka-status-json-{{ class }}:rendered}
+{%- do part_list.append('surykatka-' + class) %}
+{%- do part_list.append('surykatka-bot-promise-' + class) %}
+{%- do part_list.append('cron-entry-surykatka-status-' + class) %}
+{%- endfor %}
+
+[buildout]
+extends = {{ monitor_template_output }}
+parts =
+  cron
+  monitor-base
+  publish-connection-information
+{% for part_id in sorted(part_list) %}
+  {{ part_id }}
+{% endfor %}
+
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
+offline = true
 
 [publish-connection-information]
 recipe = slapos.cookbook:publish.serialised

software/monitor/instance.cfg

@@ -79,6 +79,7 @@ configuration.check-status-code = 200
 configuration.nameserver =
 configuration.check-frontend-ip =
 configuration.check-certificate-expiration-days = 15
+configuration.check-maximum-elapsed-time = 2
 # use monitor-base-port to have monitor listening on each instance
 # on different port and also on different port than other services
 # it makes it possible to instantiate it correctly on signle IP, for

software/monitor/surykatka.ini.jinja2

 [SURYKATKA]
 INTERVAL = 120
+TIMEOUT = {{ timeout }}
 SQLITE = {{ db }}
 {%- set nameserver_list = nameserver.split() %}
 {%- if len(nameserver_list) > 0 %}

software/monitor/test/test.py

@@ -25,6 +25,7 @@
 #
 ##############################################################################
 
+import glob
 import json
 import os
 import re
@@ -150,14 +151,29 @@ class EdgeSlaveMixin(MonitorTestMixin):
       shared=True
     )
 
+  def updateSurykatkaDict(self):
+    class_list = self.surykatka_dict.keys()
+    for class_ in class_list:
+      update_dict = {}
+      update_dict['ini-file'] = os.path.join(
+        self.bot_partition_path, 'etc', 'surykatka-%s.ini' % (class_,))
+      update_dict['json-file'] = os.path.join(
+        self.bot_partition_path, 'srv', 'surykatka-%s.json' % (class_,))
+      update_dict['status-json'] = os.path.join(
+        self.bot_partition_path, 'bin', 'surykatka-status-json-%s' % (class_,))
+      update_dict['bot-promise'] = 'surykatka-bot-promise-%s.py' % (class_,)
+      update_dict['status-cron'] = os.path.join(
+        self.bot_partition_path, 'etc', 'cron.d', 'surykatka-status-%s' % (
+          class_,))
+      update_dict['db_file'] = os.path.join(
+        self.bot_partition_path, 'srv', 'surykatka-%s.db' % (class_,))
+      self.surykatka_dict[class_].update(update_dict)
+
   def setUp(self):
     self.bot_partition_path = os.path.join(
       self.slap.instance_directory,
       self.__partition_reference__ + '1')
-    self.surykatka_json = os.path.join(
-        self.bot_partition_path, 'srv', 'surykatka.json')
-    self.surykatka_status_json = os.path.join(
-        self.bot_partition_path, 'bin', 'surykatka-status-json')
+    self.updateSurykatkaDict()
     self.monitor_configuration_list = [
       {
         'xmlUrl': 'https://[%s]:9700/public/feed' % (self._ipv6_address,),
@@ -180,15 +196,17 @@ class EdgeSlaveMixin(MonitorTestMixin):
     ]
 
   def assertSurykatkaIni(self):
-    surykatka_ini = open(
-      os.path.join(
-        self.bot_partition_path, 'etc', 'surykatka.ini')).read().strip()
-
-    expected = self.surykatka_ini % dict(
-        partition_path=self.bot_partition_path)
     self.assertEqual(
-      expected.strip(),
-      surykatka_ini)
+      set(
+        glob.glob(
+          os.path.join(self.bot_partition_path, 'etc', 'surykatka*.ini'))),
+      set([q['ini-file'] for q in self.surykatka_dict.values()])
+    )
+    for info_dict in self.surykatka_dict.values():
+      self.assertEqual(
+        info_dict['expected_ini'].strip() % info_dict,
+        open(info_dict['ini-file']).read().strip()
+      )
 
   def assertPromiseContent(self, name, content):
     promise = open(
@@ -199,23 +217,21 @@ class EdgeSlaveMixin(MonitorTestMixin):
     self.assertTrue(content in promise)
 
   def assertSurykatkaBotPromise(self):
-    self.assertPromiseContent(
-      'surykatka-bot-promise.py',
-      "'report': 'bot_status'")
-    self.assertPromiseContent(
-      'surykatka-bot-promise.py',
-      "'json-file': '%s'" % (self.surykatka_json,)
-    )
+    for info_dict in self.surykatka_dict.values():
+      self.assertPromiseContent(
+        info_dict['bot-promise'],
+        "'report': 'bot_status'")
+      self.assertPromiseContent(
+        info_dict['bot-promise'],
+        "'json-file': '%s'" % (info_dict['json-file'],)
+      )
 
   def assertSurykatkaCron(self):
-    surykatka_cron = open(
-      os.path.join(
-        self.bot_partition_path, 'etc', 'cron.d', 'surykatka-status')
-        ).read().strip()
-    self.assertEqual(
-      '*/2 * * * * %s' % (self.surykatka_status_json,),
-      surykatka_cron
-    )
+    for info_dict in self.surykatka_dict.values():
+      self.assertEqual(
+        '*/2 * * * * %s' % (info_dict['status-json'],),
+        open(info_dict['status-cron']).read().strip()
+      )
 
   def initiateSurykatkaRun(self):
     try:
@@ -224,15 +240,20 @@ class EdgeSlaveMixin(MonitorTestMixin):
       pass
 
   def assertSurykatkaStatusJSON(self):
-    if os.path.exists(self.surykatka_json):
-      os.unlink(self.surykatka_json)
-    env = os.environ.copy()
-    env.pop('PYTHONPATH', None)
-    subprocess.check_call(self.surykatka_status_json, shell=True, env=env)
-    self.assertTrue(os.path.exists(self.surykatka_json))
-    with open(self.surykatka_json) as fh:
-      status_json = json.load(fh)
-    self.assertIn('bot_status', status_json)
+    for info_dict in self.surykatka_dict.values():
+      if os.path.exists(info_dict['json-file']):
+        os.unlink(info_dict['json-file'])
+      env = os.environ.copy()
+      env.pop('PYTHONPATH', None)
+      try:
+        subprocess.check_call(info_dict['status-json'], shell=True, env=env)
+      except subprocess.CalledProcessError as e:
+        self.fail('%s failed with code %s and message %s' % (
+          info_dict['status-json'], e.returncode, e.output))
+      self.assertTrue(os.path.exists(info_dict['json-file']))
+      with open(info_dict['json-file']) as fh:
+        status_json = json.load(fh)
+      self.assertIn('bot_status', status_json)
 
   def test(self):
     # Note: Those tests do not run surykatka and do not do real checks, as
@@ -252,12 +273,15 @@ class EdgeSlaveMixin(MonitorTestMixin):
 
 
 class TestEdge(EdgeSlaveMixin, SlapOSInstanceTestCase):
-  surykatka_ini = """[SURYKATKA]
+  surykatka_dict = {
+    2: {'expected_ini': """[SURYKATKA]
 INTERVAL = 120
-SQLITE = %(partition_path)s/srv/surykatka.db
+TIMEOUT = 4
+SQLITE = %(db_file)s
 URL =
   https://www.erp5.com/
-  https://www.erp5.org/"""
+  https://www.erp5.org/"""}
+  }
 
   def assertSurykatkaPromises(self):
     self.assertPromiseContent(
@@ -277,7 +301,7 @@ URL =
       "'url': 'https://www.erp5.org/'")
     self.assertPromiseContent(
       'http-query-backend-300-promise.py',
-      "'json-file': '%s'" % (self.surykatka_json,)
+      "'json-file': '%s'" % (self.surykatka_dict[2]['json-file'],)
     )
 
     self.assertPromiseContent(
@@ -297,7 +321,7 @@ URL =
       "'url': 'https://www.erp5.com/'")
     self.assertPromiseContent(
       'http-query-backend-promise.py',
-      "'json-file': '%s'" % (self.surykatka_json,)
+      "'json-file': '%s'" % (self.surykatka_dict[2]['json-file'],)
     )
 
   def requestEdgetestSlaves(self):
@@ -313,15 +337,18 @@ URL =
 
 class TestEdgeNameserverCheckFrontendIp(
   EdgeSlaveMixin, SlapOSInstanceTestCase):
-  surykatka_ini = """[SURYKATKA]
+  surykatka_dict = {
+    2: {'expected_ini': """[SURYKATKA]
 INTERVAL = 120
-SQLITE = %(partition_path)s/srv/surykatka.db
+TIMEOUT = 4
+SQLITE = %(db_file)s
 NAMESERVER =
   127.0.1.1
   127.0.1.2
 
 URL =
-  https://www.erp5.com/"""
+  https://www.erp5.com/"""}
+  }
 
   @classmethod
   def getInstanceParameterDict(cls):
@@ -348,7 +375,7 @@ URL =
       "'url': 'https://www.erp5.com/'")
     self.assertPromiseContent(
       'http-query-backend-promise.py',
-      "'json-file': '%s'" % (self.surykatka_json,)
+      "'json-file': '%s'" % (self.surykatka_dict[2]['json-file'],)
     )
 
   def requestEdgetestSlaves(self):
@@ -359,12 +386,15 @@ URL =
 
 
 class TestEdgeCheckStatusCode(EdgeSlaveMixin, SlapOSInstanceTestCase):
-  surykatka_ini = """[SURYKATKA]
+  surykatka_dict = {
+    2: {'expected_ini': """[SURYKATKA]
 INTERVAL = 120
-SQLITE = %(partition_path)s/srv/surykatka.db
+TIMEOUT = 4
+SQLITE = %(db_file)s
 URL =
   https://www.erp5.com/
-  https://www.erp5.org/"""
+  https://www.erp5.org/"""}
+  }
 
   @classmethod
   def getInstanceParameterDict(cls):
@@ -390,7 +420,7 @@ URL =
       "'url': 'https://www.erp5.org/'")
     self.assertPromiseContent(
       'http-query-backend-501-promise.py',
-      "'json-file': '%s'" % (self.surykatka_json,)
+      "'json-file': '%s'" % (self.surykatka_dict[2]['json-file'],)
     )
 
     self.assertPromiseContent(
@@ -410,7 +440,7 @@ URL =
       "'url': 'https://www.erp5.com/'")
     self.assertPromiseContent(
       'http-query-backend-promise.py',
-      "'json-file': '%s'" % (self.surykatka_json,)
+      "'json-file': '%s'" % (self.surykatka_dict[2]['json-file'],)
     )
 
   def requestEdgetestSlaves(self):
@@ -426,12 +456,15 @@ URL =
 
 class TestEdgeCheckCertificateExpirationDays(
   EdgeSlaveMixin, SlapOSInstanceTestCase):
-  surykatka_ini = """[SURYKATKA]
+  surykatka_dict = {
+    2: {'expected_ini': """[SURYKATKA]
 INTERVAL = 120
-SQLITE = %(partition_path)s/srv/surykatka.db
+TIMEOUT = 4
+SQLITE = %(db_file)s
 URL =
   https://www.erp5.com/
-  https://www.erp5.org/"""
+  https://www.erp5.org/"""}
+  }
 
   @classmethod
   def getInstanceParameterDict(cls):
@@ -457,7 +490,7 @@ URL =
       "'url': 'https://www.erp5.org/'")
     self.assertPromiseContent(
       'http-query-backend-20-promise.py',
-      "'json-file': '%s'" % (self.surykatka_json,)
+      "'json-file': '%s'" % (self.surykatka_dict[2]['json-file'],)
     )
 
     self.assertPromiseContent(
@@ -477,7 +510,7 @@ URL =
       "'url': 'https://www.erp5.com/'")
     self.assertPromiseContent(
       'http-query-backend-promise.py',
-      "'json-file': '%s'" % (self.surykatka_json,)
+      "'json-file': '%s'" % (self.surykatka_dict[2]['json-file'],)
     )
 
   def requestEdgetestSlaves(self):
@@ -490,3 +523,110 @@ URL =
       {'url': 'https://www.erp5.org/',
        'check-certificate-expiration-days': '20'},
     )
+
+
+class TestEdgeCheckMaximumElapsedTime(
+  EdgeSlaveMixin, SlapOSInstanceTestCase):
+  surykatka_dict = {
+    5: {'expected_ini': """[SURYKATKA]
+INTERVAL = 120
+TIMEOUT = 7
+SQLITE = %(db_file)s
+URL =
+  https://www.erp5.com/"""},
+    20: {'expected_ini': """[SURYKATKA]
+INTERVAL = 120
+TIMEOUT = 22
+SQLITE = %(db_file)s
+URL =
+  https://www.erp5.org/"""},
+    1: {'expected_ini': """[SURYKATKA]
+INTERVAL = 120
+TIMEOUT = 3
+SQLITE = %(db_file)s
+URL =
+  https://www.erp5.net/"""}
+  }
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {
+      'check-maximum-elapsed-time': '5',
+    }
+
+  def assertSurykatkaPromises(self):
+    self.assertPromiseContent(
+      'http-query-backend-20-promise.py',
+      "'ip-list': ''")
+    self.assertPromiseContent(
+      'http-query-backend-20-promise.py',
+      "'report': 'http_query'")
+    self.assertPromiseContent(
+      'http-query-backend-20-promise.py',
+      "'status-code': '200'")
+    self.assertPromiseContent(
+      'http-query-backend-20-promise.py',
+      "'maximum-elapsed-time': '20'")
+    self.assertPromiseContent(
+      'http-query-backend-20-promise.py',
+      "'url': 'https://www.erp5.org/'")
+    self.assertPromiseContent(
+      'http-query-backend-20-promise.py',
+      "'json-file': '%s'" % (self.surykatka_dict[20]['json-file'],)
+    )
+
+    self.assertPromiseContent(
+      'http-query-backend-default-promise.py',
+      "'ip-list': ''")
+    self.assertPromiseContent(
+      'http-query-backend-default-promise.py',
+      "'report': 'http_query'")
+    self.assertPromiseContent(
+      'http-query-backend-default-promise.py',
+      "'status-code': '200'")
+    self.assertPromiseContent(
+      'http-query-backend-default-promise.py',
+      "'maximum-elapsed-time': '5'")
+    self.assertPromiseContent(
+      'http-query-backend-default-promise.py',
+      "'url': 'https://www.erp5.com/'")
+    self.assertPromiseContent(
+      'http-query-backend-default-promise.py',
+      "'json-file': '%s'" % (self.surykatka_dict[5]['json-file'],)
+    )
+
+    self.assertPromiseContent(
+      'http-query-backend-1-promise.py',
+      "'ip-list': ''")
+    self.assertPromiseContent(
+      'http-query-backend-1-promise.py',
+      "'report': 'http_query'")
+    self.assertPromiseContent(
+      'http-query-backend-1-promise.py',
+      "'status-code': '200'")
+    self.assertPromiseContent(
+      'http-query-backend-1-promise.py',
+      "'maximum-elapsed-time': '1'")
+    self.assertPromiseContent(
+      'http-query-backend-1-promise.py',
+      "'url': 'https://www.erp5.net/'")
+    self.assertPromiseContent(
+      'http-query-backend-1-promise.py',
+      "'json-file': '%s'" % (self.surykatka_dict[1]['json-file'],)
+    )
+
+  def requestEdgetestSlaves(self):
+    self.requestEdgetestSlave(
+      'backend-default',
+      {'url': 'https://www.erp5.com/'},
+    )
+    self.requestEdgetestSlave(
+      'backend-20',
+      {'url': 'https://www.erp5.org/',
+       'check-maximum-elapsed-time': '20'},
+    )
+    self.requestEdgetestSlave(
+      'backend-1',
+      {'url': 'https://www.erp5.net/',
+       'check-maximum-elapsed-time': '1'},
+    )