software/monitor: edgetest-basic implementation

Instead of using complex mechanism with slaves (which are not monitored
anyway), allow the user to express monitoring with simple UI on normal
instance.

Additionally support for simple domains, not only URLs, is added.

software/monitor/buildout.hash.cfg

@@ -14,7 +14,7 @@
 # not need these here).
 [template]
 filename = instance.cfg
-md5sum = 2114ae8c1e92bd33ef1347f36f567c74
+md5sum = b6c2df0d4a62473d6dae26b10c0a4adc
 
 [template-monitor]
 _update_hash_filename_ = instance-monitor.cfg.jinja2
@@ -24,13 +24,17 @@ md5sum = 165a15672fc85981f68b9af2d6253254
 _update_hash_filename_ = json-test-template.json.in.jinja2
 md5sum = 2eb5596544d9c341acf653d4f7ce2680
 
+[template-monitor-edgetest-basic]
+_update_hash_filename_ = instance-monitor-edgetest-basic.cfg.jinja2
+md5sum = 61309a48f7b0135cba21b09247a2d8fd
+
 [template-monitor-edgetest]
 _update_hash_filename_ = instance-monitor-edgetest.cfg.jinja2
 md5sum = 3c8ab4e78f66c974eb95afc595a13514
 
 [template-monitor-edgebot]
 _update_hash_filename_ = instance-monitor-edgebot.cfg.jinja2
-md5sum = 4b19eea828dc1d9b393a1537f1d8d5f5
+md5sum = 2ac74559d6108ca0dbabb872f1071e44
 
 [network-bench-cfg]
 filename = network_bench.cfg.in
@@ -42,4 +46,4 @@ md5sum = d3cfa1f6760e3fa64ccd64acf213bdfb
 
 [template-surykatka-ini]
 _update_hash_filename_ = surykatka.ini.jinja2
-md5sum = 609c6cca763b73a80fa05ee56475eb20
+md5sum = 7e9b874c20faaa8190d2bf2b74caa727

software/monitor/instance-edgetest-basic-input-schema.json

+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "type": "object",
+  "properties": {
+    "nameserver-list": {
+      "default": [],
+      "title": "Nameservers",
+      "description": "List of nameservers IPs (like 1.2.3.4) to use.",
+      "type": "array"
+    },
+    "check-frontend-ip-list": {
+      "default": [],
+      "title": "Default Frontend IPs to check",
+      "description": "List of default frontend IPs (like 1.2.3.4) to check, if empty no constraint is used.",
+      "type": "array"
+    },
+    "check-dict": {
+      "title": "Configuration of checks",
+      "description": "Configure checks.",
+      "patternProperties": {
+        ".*": {
+          "properties": {
+            "url-list": {
+              "default": [],
+              "title": "URLs to check",
+              "description": "List of URLs to check, can be domain (example.com) or URL (https://example.com/, http://example.com/path)",
+              "type": "array"
+            },
+            "check-status-code": {
+              "title": "HTTP Code Check",
+              "description": "Expected response HTTP Code. Defaults to global configuration.",
+              "type": "number",
+              "minimum": 100,
+              "maximum": 599
+            },
+            "check-frontend-ip-list": {
+              "title": "Frontend IPs to check (optional)",
+              "description": "List of frontend IPs (like 1.2.3.4) to check. Defaults to global configuration.",
+              "type": "array"
+            },
+            "check-certificate-expiration-days": {
+              "title": "Certificate Expiration Check (days)",
+              "description": "Amount of days to consider certificate as being to-be-expired. Defaults to global configuration.",
+              "type": "number",
+              "minimum": 1
+            },
+            "check-maximum-elapsed-time": {
+              "title": "Maximum Elapsed Check (seconds)",
+              "description": "Maximum elapsed time for a site to reply to be considered good. Defaults to global configuration.",
+              "type": "number",
+              "minimum": 1
+            },
+            "failure-amount": {
+              "title": "Failure Amount",
+              "description": "Amount of failures to consider URL as in bad state, can be set to higher value for endpoints with accepted short outages. Defaults to global configuration.",
+              "type": "number",
+              "minimum": 1
+            },
+            "check-http-header-dict": {
+              "title": "[UI UNSUPPORTED] HTTP Header Check",
+              "description": "[UI UNSUPPORTED] JSON object of expected HTTP header, like {\"Cache-Control\": \"max-age=3600, public\", \"Vary\": \"Accept-Encoding\"}. Note: Shall be expressed directly as object, without any additional qouting.",
+              "type": "object",
+              "default": {}
+            }
+          },
+          "type": "object"
+        }
+      },
+      "type": "object",
+      "default": {}
+    },
+    "check-status-code": {
+      "title": "Default HTTP Code Check",
+      "description": "Default expected response HTTP Code.",
+      "type": "number",
+      "default": 200,
+      "minimum": 100,
+      "maximum": 599
+    },
+    "check-certificate-expiration-days": {
+      "title": "Default Certificate Expiration Check (days)",
+      "description": "Default amount of days to consider certificate as being to-be-expired.",
+      "type": "number",
+      "default": 15,
+      "minimum": 1
+    },
+    "check-maximum-elapsed-time": {
+      "title": "Default Maximum Elapsed Check (seconds)",
+      "description": "Default maximum elapsed time for a site to reply to be considered good.",
+      "type": "number",
+      "default": 2,
+      "minimum": 1
+    },
+    "failure-amount": {
+      "title": "Default Failure Amount",
+      "description": "Default amount of failures to consider URL as in bad state, can be set to higher value for endpoints with accepted short outages.",
+      "type": "number",
+      "default": 2,
+      "minimum": 1
+    }
+  }
+}

software/monitor/instance-monitor-edgebot.cfg.jinja2

@@ -28,7 +28,9 @@
 {%- set part_list = [] %}
 {%- for class, slave_instance_list in slave_instance_dict.items() %}
 {#- class is used to separate surykatka with different timeouts #}
+{%-   set URL_LIST = [] %}
 {%-   for slave in slave_instance_list | sort(attribute='-slave-title') %}
+{%-     do URL_LIST.append(slave['url']) %}
 {%-     set part_id = 'http-query-' ~ hashlib_module.md5(slave['-slave-reference'].encode('utf-8')).hexdigest() ~ '-promise' %}
 {%-     do part_list.append(part_id) %}
 {%-     set safe_name = part_id.replace('_', '').replace('.', '-').replace(' ', '-') %}
@@ -59,7 +61,7 @@ recipe = slapos.recipe.template:jinja2
 db = ${directory:srv}/surykatka-{{ class }}.db
 rendered = ${directory:etc}/surykatka-{{ class }}.ini
 template = {{ template_surykatka_ini }}
-slave_instance_list = {{ dumps(slave_instance_list) }}
+url_list = {{ dumps(URL_LIST) }}
 nameserver_list = {{ dumps(CONFIGURATION['nameserver-list']) }}
 json = ${directory:srv}/surykatka-{{ class }}.json
 {#- timeout is just a bit bigger than class time #}
@@ -69,7 +71,7 @@ context =
   import json_module json
   key db :db
   key nameserver_list :nameserver_list
-  key slave_instance_list :slave_instance_list
+  key url_list :url_list
   key timeout :timeout
 
 [surykatka-{{ class }}]

software/monitor/instance-monitor-edgetest-basic.cfg.jinja2

+{#- PREPARE #}
+{%- set DEFAULT_DICT = {
+  'check-status-code': 200,
+  'check-http-header-dict': {},
+  'check-certificate-expiration-days': 15,
+  'failure-amount': 2,
+  'check-maximum-elapsed-time': 2,
+  'check-frontend-ip-list': [],
+  'nameserver-list': []
+} %}
+{%- for default_key in ['check-frontend-ip-list', 'nameserver-list', 'check-status-code', 'check-certificate-expiration-days', 'check-maximum-elapsed-time', 'failure-amount'] %}
+{%-   if default_key in slapparameter_dict %}
+{%-     do DEFAULT_DICT.__setitem__(default_key, slapparameter_dict[default_key]) %}
+{%-   endif %}
+{%- endfor %}
+{%- set CHECK_DICT = {} %}
+{%- for check_name, check_definition in slapparameter_dict['check-dict'].items() %}
+{%-   if 'url-list' in check_definition %}
+{%-     set check = DEFAULT_DICT.copy()  %}
+{%-     set url_list = [] %}
+{%-     for url in check_definition.pop('url-list') %}
+{%-       if len(url.strip()) > 0 %}
+{#-         Support various URL entries #}
+{%-         if url.startswith('http:') or url.startswith('https:') %}
+{%-           if url not in url_list %}
+{%-             do url_list.append(url) %}
+{%-           endif %}
+{%-         else %}
+{%-           for schema in ['http://', 'https://'] %}
+{%-             set add_url = '%s%s' % (schema, url) %}
+{%-             if add_url not in url_list %}
+{%-               do url_list.append('%s%s' % (schema, url)) %}
+{%-             endif %}
+{%-           endfor %}
+{%-         endif %}
+{%-       endif %}
+{%-     endfor %}
+{%-     do check.update(check_definition) %}
+{%-     do check.__setitem__('url-list', url_list) %}
+{%-     set class = check['check-maximum-elapsed-time'] %}
+{%-     if class not in CHECK_DICT %}
+{%-       do CHECK_DICT.__setitem__(class, []) %}
+{%-     endif %}
+{%-     do check.__setitem__('-name', check_name) %}
+{%-     do CHECK_DICT[class].append(check) %}
+{%-   endif %}
+{%- endfor %}
+
+{%- set PART_LIST = [] %}
+{%- for class, class_check_list in CHECK_DICT.items() %}
+{#- class is used to separate surykatka with different timeouts #}
+{%-   set CLASS_URL_LIST = [] %}
+{%-   for check in class_check_list | sort(attribute='-name') %}
+{%-     do CLASS_URL_LIST.extend(check['url-list']) %}
+{%-     for url in check['url-list'] %}
+{%-       set promise_name = 'http-query-' ~ hashlib_module.md5(check['-name'].encode('utf-8')).hexdigest() ~ '-' ~ hashlib_module.md5(url.encode('utf-8')).hexdigest() %}
+{%-       set part_name = promise_name ~ '-promise' %}
+{%-       do PART_LIST.append(part_name) %}
+[{{part_name}}]
+<= monitor-promise-base
+module = check_surykatka_json
+name = {{ promise_name }}.py
+config-report = http_query
+config-url = {{ url }}
+config-status-code = {{ check['check-status-code'] }}
+config-http-header-dict = {{ json_module.dumps(check['check-http-header-dict']) }}
+config-certificate-expiration-days = {{ check['check-certificate-expiration-days'] }}
+config-failure-amount = {{ check['failure-amount'] }}
+config-maximum-elapsed-time = {{ check['check-maximum-elapsed-time'] }}
+config-ip-list = {{ ' '.join(check['check-frontend-ip-list']) }}
+config-json-file = ${surykatka-config-{{ class }}:json}
+{%-     endfor %} {#- for url in check['url-list'] #}
+{%-   endfor %} {#- for check in class_check_list | sort(attribute='-name') #}
+
+{%-     do PART_LIST.append('surykatka-bot-%i-promise' % (class,)) %}
+[surykatka-bot-{{ class }}-promise]
+<= monitor-promise-base
+module = check_surykatka_json
+name = surykatka-bot-{{ class }}.py
+config-report = bot_status
+config-json-file = ${surykatka-config-{{ class }}:json}
+
+[surykatka-config-{{ class }}]
+recipe = slapos.recipe.template:jinja2
+db = ${directory:srv}/surykatka-{{ class }}.db
+rendered = ${directory:etc}/surykatka-{{ class }}.ini
+template = {{ template_surykatka_ini }}
+url_list = {{ dumps(CLASS_URL_LIST) }}
+nameserver_list = {{ dumps(DEFAULT_DICT['nameserver-list']) }}
+json = ${directory:srv}/surykatka-{{ class }}.json
+{#- timeout is just a bit bigger than class time #}
+timeout = {{ int(class) + 2 }}
+
+context =
+  import json_module json
+  key db :db
+  key nameserver_list :nameserver_list
+  key url_list :url_list
+  key timeout :timeout
+
+{%-     do PART_LIST.append('surykatka-%i'% (class,)) %}
+[surykatka-{{ class }}]
+recipe = slapos.cookbook:wrapper
+config = ${surykatka-config-{{ class }}:rendered}
+command-line =
+  {{ surykatka_binary }} --run crawl --reload --configuration ${:config}
+wrapper-path = ${monitor-directory:service}/${:_buildout_section_name_}
+hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
+
+{%-     do PART_LIST.append('surykatka-json-%i-promise'% (class,)) %}
+[surykatka-json-{{ class }}-promise]
+<= monitor-promise-base
+module = check_file_state
+name = surykatka-json-{{ class }}.py
+config-filename = ${surykatka-config-{{ class }}:json}
+config-state = not-empty
+# workaround for bug in check_file_state
+config-url =
+
+[surykatka-status-json-{{ class }}]
+recipe = slapos.recipe.template:jinja2
+json = ${surykatka-config-{{ class }}:json}
+template = inline:#!/bin/sh
+  {{ surykatka_binary }} --run status --configuration ${surykatka-{{ class }}:config} --output json --stdout ${:json}
+rendered = ${monitor-directory:bin}/${:_buildout_section_name_}
+mode = 0755
+
+{%-     do PART_LIST.append('cron-entry-surykatka-status-%i' % (class,)) %}
+[cron-entry-surykatka-status-{{ class }}]
+recipe = slapos.cookbook:cron.d
+cron-entries = ${directory:etc}/cron.d
+name = surykatka-status-{{ class }}
+frequency = */2 * * * *
+command = ${surykatka-status-json-{{ class }}:rendered}
+{%- endfor %} {#- for class, class_check_list in CHECK_DICT.items() #}
+
+[buildout]
+extends = {{ instance_base_monitor }}
+parts +=
+{% for part_id in sorted(PART_LIST) %}
+  {{ part_id }}
+{% endfor %}

software/monitor/instance.cfg

@@ -10,6 +10,7 @@ recipe = slapos.cookbook:switch-softwaretype
 default = instance-base-monitor:rendered
 edgetest = instance-base-edgetest:rendered
 edgebot = instance-base-edgebot:rendered
+edgetest-basic = instance-edgetest-basic:rendered
 RootSoftwareInstance = $${:default}
 
 [instance-base-monitor]
@@ -27,6 +28,34 @@ context = key develop_eggs_directory buildout:develop-eggs-directory
           raw monitor_collect_csv_dump ${monitor-collect-csv-dump:output}
 mode = 0644
 
+[instance-template]
+recipe = slapos.recipe.template:jinja2
+extensions = jinja2.ext.do
+rendered = $${buildout:directory}/$${:_buildout_section_name_}.cfg
+context =
+  import json_module json
+  import hashlib_module hashlib
+  key develop_eggs_directory buildout:develop-eggs-directory
+  key eggs_directory buildout:eggs-directory
+  key slapparameter_dict slap-configuration:configuration
+  key slap_software_type slap-configuration:slap-software-type
+  key instance_base_monitor instance-base-monitor:rendered
+  raw buildout_bin ${buildout:bin-directory}
+  $${:extra-context}
+mode = 0644
+
+[surykatka]
+binary = ${buildout:bin-directory}/${surykatka:script-name}
+ini = ${template-surykatka-ini:target}
+
+[instance-edgetest-basic]
+<= instance-template
+template = ${template-monitor-edgetest-basic:target}
+extra-context =
+  raw software_type edgetest-basic
+  key template_surykatka_ini surykatka:ini
+  key surykatka_binary surykatka:binary
+
 [instance-base-edgetest]
 recipe = slapos.recipe.template:jinja2
 template = ${template-monitor-edgetest:target}
@@ -51,8 +80,6 @@ recipe = slapos.recipe.template:jinja2
 template = ${template-monitor-edgebot:target}
 rendered = $${buildout:directory}/template-monitor-edgebot.cfg
 extensions = jinja2.ext.do
-surykatka-binary = ${buildout:bin-directory}/${surykatka:script-name}
-template-surykatka-ini = ${template-surykatka-ini:target}
 
 context = import json_module json
           import hashlib_module hashlib
@@ -62,8 +89,8 @@ context = import json_module json
           key slapparameter_dict slap-configuration:configuration
           key slap_software_type slap-configuration:slap-software-type
           raw software_type edgebot
-          key surykatka_binary :surykatka-binary
-          key template_surykatka_ini :template-surykatka-ini
+          key surykatka_binary surykatka:binary
+          key template_surykatka_ini surykatka:ini
           raw buildout_bin ${buildout:bin-directory}
           raw monitor_template_output ${monitor-template:output}
           raw monitor_collect_csv_dump ${monitor-collect-csv-dump:output}

software/monitor/software.cfg

@@ -31,6 +31,11 @@ recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/${:_update_hash_filename_}
 mode = 0644
 
+[template-monitor-edgetest-basic]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/${:_update_hash_filename_}
+mode = 0644
+
 [template-monitor-edgetest]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/${:_update_hash_filename_}

software/monitor/software.cfg.json

@@ -10,23 +10,31 @@
       "response": "instance-default-output-schema.json",
       "index": 0
     },
+    "edgetest-basic": {
+      "title": "Edge Test Basic",
+      "description": "Basic URL monitoring configuration",
+      "request": "instance-edgetest-basic-input-schema.json",
+      "response": "instance-default-output-schema.json",
+      "serialisation": "json-in-xml",
+      "index": 1
+    },
     "edgetest": {
-      "title": "Edge Test",
-      "description": "Cluster of bots to perform a distributed monitoring ",
+      "title": "Edge Test [OBSOLETE]",
+      "description": "Cluster of bots to perform a distributed monitoring. OBSOLETE: Use Edge Test Basic.",
       "request": "instance-edgetest-input-schema.json",
       "response": "instance-default-output-schema.json",
       "serialisation": "json-in-xml",
-      "index": 1
+      "index": 2
     },
     "edgetest-slave": {
-      "title": "Edge Test Slave",
+      "title": "Edge Test Slave [OBSOLETE]",
       "shared": true,
       "software-type": "edgetest",
-      "description": "Cluster of bots to perform a distributed monitoring ",
+      "description": "Cluster of bots to perform a distributed monitoring. OBSOLETE: Use Edge Test Basic.",
       "request": "instance-edgetest-slave-input-schema.json",
       "response": "instance-default-output-schema.json",
       "serialisation": "json-in-xml",
-      "index": 2
+      "index": 3
     }
   }
 }

software/monitor/surykatka.ini.jinja2

@@ -9,9 +9,6 @@ NAMESERVER =
 {%-   endfor %}
 {% endif %}
 URL =
-{%- for slave in slave_instance_list | sort(attribute='-slave-title') %}
-{%-   if 'url' in slave %}
-  {{ slave['url'] }}
-{%-   endif -%}
+{%- for url in sorted(url_list) %}
+  {{ url -}}
 {% endfor %}
-