monitor: Implement check-maximum-elapsed-time

Allows to configure maximum time for which the site is considered correct.

For each class (the maximum elapsed time value) separate surykatka process
is started, in order to not impact fast sites checking by long timeouts on
slow sites.

software/monitor/buildout.hash.cfg

@@ -14,7 +14,7 @@
 # not need these here).
 [template]
 filename = instance.cfg
-md5sum = d778b6f436ae6864819eb2ff2d12a86f
+md5sum = 1e64cee5a2249d19f6241312f94d06b7
 
 [template-monitor]
 _update_hash_filename_ = instance-monitor.cfg.jinja2
@@ -30,7 +30,7 @@ md5sum = 9e237dbdda59e788202f0da194a57d41
 
 [template-monitor-edgebot]
 _update_hash_filename_ = instance-monitor-edgebot.cfg.jinja2
-md5sum = 8786e4245db0d27dfa4815222d970e52
+md5sum = b8f3c467cfd988e29be2ba70aa7a059e
 
 [network-bench-cfg]
 filename = network_bench.cfg.in
@@ -42,4 +42,4 @@ md5sum = cad2402bbd21907cfed6bc5af8c5d3ab
 
 [template-surykatka-ini]
 _update_hash_filename_ = surykatka.ini.jinja2
-md5sum = 40870921e05d93b5843ab34abd7e3902
+md5sum = a2de719a5a65438c8c3ee5195442beb6

software/monitor/instance-edgetest-input-schema.json

@@ -24,6 +24,12 @@
       "title": "Default certificate expiration days check",
       "description": "Default amount of days to consider certificate as being to-be-expired (default: 15).",
       "type": "string"
+    },
+    "check-maximum-elapsed-time": {
+      "default": "2",
+      "title": "Default maximum elapsed time for a site to reply (seconds)",
+      "description": "Default maximum elapsed time for a site to reply to be considered good (default: 2s).",
+      "type": "string"
     }
   }
 }

software/monitor/instance-edgetest-slave-input-schema.json

@@ -23,6 +23,12 @@
       "title": "Certificate expiration days check",
       "description": "Amount of days to consider certificate as being to-be-expired (default: comes from master partition).",
       "type": "string"
+    },
+    "check-maximum-elapsed-time": {
+      "default": "Master default",
+      "title": "Maximum elapsed time for a site to reply (seconds)",
+      "description": "Maximum elapsed time for a site to reply to be considered good.(default: comes from master partition).",
+      "type": "string"
     }
   }
 }

software/monitor/instance-monitor-edgebot.cfg.jinja2

@@ -5,7 +5,7 @@
 {%-     do CONFIGURATION.__setitem__(k[14:], v) %}
 {%-   endif %}
 {%- endfor %}
-{%- set slave_instance_list = [] %}
+{%- set slave_instance_dict = {} %}
 {%- set extra_slave_instance_list = slapparameter_dict.get('extra_slave_instance_list') %}
 {%- if extra_slave_instance_list %}
 {#- Create slaves to process with setting up defaults #}
@@ -16,15 +16,24 @@
 {%-   if 'check-certificate-expiration-days' not in slave %}
 {%-     do slave.__setitem__('check-certificate-expiration-days', CONFIGURATION['check-certificate-expiration-days']) %}
 {%-   endif %}
+{%-   if 'check-maximum-elapsed-time' not in slave %}
+{%-     do slave.__setitem__('check-maximum-elapsed-time', CONFIGURATION['check-maximum-elapsed-time']) %}
+{%-   endif %}
 {%-   if 'check-frontend-ip' not in slave %}
 {%-     do slave.__setitem__('check-frontend-ip', CONFIGURATION['check-frontend-ip']) %}
 {%-   endif %}
 {%-   if 'url' in slave %}
-{%-     do slave_instance_list.append(slave) %}
+{%-     set class = slave['check-maximum-elapsed-time'] %}
+{%-     if class not in slave_instance_dict %}
+{%-       do slave_instance_dict.__setitem__(class, []) %}
+{%-     endif %}
+{%-     do slave_instance_dict[class].append(slave) %}
 {%-   endif %}
 {%-   endfor %}
 {%- endif %}
 {%- set part_list = [] %}
+{%- for class, slave_instance_list in slave_instance_dict.items() %}
+{#- class is used to separate surykatka with different timeouts #}
 {%-   for slave in sorted(slave_instance_list) %}
 {%-     set part_id = 'http-query-' ~ slave['slave_reference'] ~ '-promise' %}
 {%-     do part_list.append(part_id) %}
@@ -37,75 +46,80 @@ config-report = http_query
 config-url = {{ slave['url'] }}
 config-status-code = {{ slave['check-status-code'] }}
 config-certificate-expiration-days = {{ slave['check-certificate-expiration-days'] }}
+config-maximum-elapsed-time = {{ slave['check-maximum-elapsed-time'] }}
 config-ip-list = {{ slave['check-frontend-ip'] }}
-config-json-file = ${surykatka-config:json}
-{% endfor %}
+config-json-file = ${surykatka-config-{{ class }}:json}
+{%-   endfor %}
 
-[surykatka-bot-promise]
+[surykatka-bot-promise-{{ class }}]
 <= monitor-promise-base
 module = check_surykatka_json
-name = surykatka-bot-promise.py
+name = surykatka-bot-promise-{{ class }}.py
 config-report = bot_status
-config-json-file = ${surykatka-config:json}
-
-[buildout]
-extends = {{ monitor_template_output }}
-
-parts =
-  cron
-  cron-entry-surykatka-status
-  monitor-base
-  publish-connection-information
-  surykatka
-  surykatka-bot-promise
-{% for part_id in sorted(part_list) %}
-  {{ part_id }}
-{% endfor %}
-
-eggs-directory = {{ eggs_directory }}
-develop-eggs-directory = {{ develop_eggs_directory }}
-offline = true
+config-json-file = ${surykatka-config-{{ class }}:json}
 
-[surykatka-config]
+[surykatka-config-{{ class }}]
 recipe = slapos.recipe.template:jinja2
-db = ${directory:srv}/surykatka.db
-rendered = ${directory:etc}/surykatka.ini
+db = ${directory:srv}/surykatka-{{ class }}.db
+rendered = ${directory:etc}/surykatka-{{ class }}.ini
 template = {{ template_surykatka_ini }}
 slave_instance_list = {{ dumps(slave_instance_list) }}
 nameserver = {{ dumps(CONFIGURATION['nameserver']) }}
-json = ${directory:srv}/surykatka.json
+json = ${directory:srv}/surykatka-{{ class }}.json
+{#- timeout is just a bit bigger than class time #}
+timeout = {{ int(class) + 2 }}
 
 context =
   import json_module json
   key db :db
   key nameserver :nameserver
   key slave_instance_list :slave_instance_list
+  key timeout :timeout
 
-[surykatka]
+[surykatka-{{ class }}]
 recipe = slapos.cookbook:wrapper
-config = ${surykatka-config:rendered}
+config = ${surykatka-config-{{ class }}:rendered}
 command-line =
   {{ surykatka_binary }} --run crawl --reload --configuration ${:config}
 wrapper-path = ${monitor-directory:service}/${:_buildout_section_name_}
 hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
 
-[surykatka-status-json]
+[surykatka-status-json-{{ class }}]
 recipe = slapos.recipe.template:jinja2
+json = ${surykatka-config-{{ class }}:json}
 template = inline:#!/bin/sh
-  if {{ surykatka_binary }} --run status --configuration ${surykatka:config} --output json > ${surykatka-config:json}.tmp ; then
-    mv -f ${surykatka-config:json}.tmp ${surykatka-config:json}
+  if {{ surykatka_binary }} --run status --configuration ${surykatka-{{ class }}:config} --output json > ${:json}.tmp ; then
+    mv -f ${:json}.tmp ${:json}
   else
-    rm -f ${surykatka-config:json}.tmp
+    rm -f ${:json}.tmp
   fi
 rendered = ${monitor-directory:bin}/${:_buildout_section_name_}
 mode = 0755
 
-[cron-entry-surykatka-status]
+[cron-entry-surykatka-status-{{ class }}]
 recipe = slapos.cookbook:cron.d
 cron-entries = ${directory:etc}/cron.d
-name = surykatka-status
+name = surykatka-status-{{ class }}
 frequency = */2 * * * *
-command = ${surykatka-status-json:rendered}
+command = ${surykatka-status-json-{{ class }}:rendered}
+{%- do part_list.append('surykatka-' + class) %}
+{%- do part_list.append('surykatka-bot-promise-' + class) %}
+{%- do part_list.append('cron-entry-surykatka-status-' + class) %}
+{%- endfor %}
+
+[buildout]
+extends = {{ monitor_template_output }}
+parts =
+  cron
+  monitor-base
+  publish-connection-information
+{% for part_id in sorted(part_list) %}
+  {{ part_id }}
+{% endfor %}
+
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
+offline = true
 
 [publish-connection-information]
 recipe = slapos.cookbook:publish.serialised

software/monitor/instance.cfg

@@ -79,6 +79,7 @@ configuration.check-status-code = 200
 configuration.nameserver =
 configuration.check-frontend-ip =
 configuration.check-certificate-expiration-days = 15
+configuration.check-maximum-elapsed-time = 2
 # use monitor-base-port to have monitor listening on each instance
 # on different port and also on different port than other services
 # it makes it possible to instantiate it correctly on signle IP, for

software/monitor/surykatka.ini.jinja2

 [SURYKATKA]
 INTERVAL = 120
+TIMEOUT = {{ timeout }}
 SQLITE = {{ db }}
 {%- set nameserver_list = nameserver.split() %}
 {%- if len(nameserver_list) > 0 %}