Commit a51cd296 authored by Łukasz Nowak's avatar Łukasz Nowak

software/monitor: edgetest-basic implementation

Instead of using complex mechanism with slaves (which are not monitored
anyway), allow the user to express monitoring with simple UI on normal
instance.

Additionally support for simple domains, not only URLs, is added.
parent 959a9404
......@@ -14,7 +14,7 @@
# not need these here).
[template]
filename = instance.cfg
md5sum = 2114ae8c1e92bd33ef1347f36f567c74
md5sum = b6c2df0d4a62473d6dae26b10c0a4adc
[template-monitor]
_update_hash_filename_ = instance-monitor.cfg.jinja2
......@@ -24,13 +24,17 @@ md5sum = 165a15672fc85981f68b9af2d6253254
_update_hash_filename_ = json-test-template.json.in.jinja2
md5sum = 2eb5596544d9c341acf653d4f7ce2680
[template-monitor-edgetest-basic]
_update_hash_filename_ = instance-monitor-edgetest-basic.cfg.jinja2
md5sum = c88c9e078a6c91d0fb92bfa0ca3db398
[template-monitor-edgetest]
_update_hash_filename_ = instance-monitor-edgetest.cfg.jinja2
md5sum = 3c8ab4e78f66c974eb95afc595a13514
[template-monitor-edgebot]
_update_hash_filename_ = instance-monitor-edgebot.cfg.jinja2
md5sum = 4b19eea828dc1d9b393a1537f1d8d5f5
md5sum = 2ac74559d6108ca0dbabb872f1071e44
[network-bench-cfg]
filename = network_bench.cfg.in
......@@ -42,4 +46,4 @@ md5sum = d3cfa1f6760e3fa64ccd64acf213bdfb
[template-surykatka-ini]
_update_hash_filename_ = surykatka.ini.jinja2
md5sum = 609c6cca763b73a80fa05ee56475eb20
md5sum = 7e9b874c20faaa8190d2bf2b74caa727
{
"$schema": "http://json-schema.org/draft-04/schema#",
"type": "object",
"properties": {
"nameserver-list": {
"default": [],
"title": "Nameservers",
"description": "List of nameservers IPs (like 1.2.3.4) to use.",
"type": "array"
},
"check-frontend-ip-list": {
"default": [],
"title": "Default Frontend IPs to check",
"description": "List of default frontend IPs (like 1.2.3.4) to check, if empty no constraint is used.",
"type": "array"
},
"check-dict": {
"title": "Configuration of checks",
"description": "Configure checks.",
"patternProperties": {
".*": {
"properties": {
"url-list": {
"default": [],
"title": "URLs to check",
"description": "List of URLs to check, can be domain (example.com) or URL (https://example.com/, http://example.com/path)",
"type": "array"
},
"check-status-code": {
"title": "HTTP Code Check",
"description": "Expected response HTTP Code. Defaults to global configuration.",
"type": "number",
"minimum": 100,
"maximum": 599
},
"check-frontend-ip-list": {
"default": [],
"title": "Frontend IPs to check (optional)",
"description": "List of frontend IPs (like 1.2.3.4) to check. Defaults to global configuration.",
"type": "array"
},
"check-certificate-expiration-days": {
"title": "Certificate Expiration Check (days)",
"description": "Amount of days to consider certificate as being to-be-expired. Defaults to global configuration.",
"type": "number",
"minimum": 1
},
"check-maximum-elapsed-time": {
"title": "Maximum Elapsed Check (seconds)",
"description": "Maximum elapsed time for a site to reply to be considered good. Defaults to global configuration.",
"type": "number",
"minimum": 1
},
"failure-amount": {
"title": "Failure Amount",
"description": "Amount of failures to consider URL as in bad state, can be set to higher value for endpoints with accepted short outages. Defaults to global configuration.",
"type": "number",
"minimum": 1
},
"check-http-header-dict": {
"title": "[UI UNSUPPORTED] HTTP Header Check",
"description": "[UI UNSUPPORTED] JSON object of expected HTTP header, like {\"Cache-Control\": \"max-age=3600, public\", \"Vary\": \"Accept-Encoding\"}. Note: Shall be expressed directly as object, without any additional qouting.",
"type": "object",
"default": {}
}
},
"type": "object"
}
},
"type": "object",
"default": {}
},
"check-status-code": {
"title": "Default HTTP Code Check",
"description": "Default expected response HTTP Code.",
"type": "number",
"default": 200,
"minimum": 100,
"maximum": 599
},
"check-certificate-expiration-days": {
"title": "Default Certificate Expiration Check (days)",
"description": "Default amount of days to consider certificate as being to-be-expired.",
"type": "number",
"default": 15,
"minimum": 1
},
"check-maximum-elapsed-time": {
"title": "Default Maximum Elapsed Check (seconds)",
"description": "Default maximum elapsed time for a site to reply to be considered good.",
"type": "number",
"default": 2,
"minimum": 1
},
"failure-amount": {
"title": "Default Failure Amount",
"description": "Default amount of failures to consider URL as in bad state, can be set to higher value for endpoints with accepted short outages.",
"type": "number",
"default": 2,
"minimum": 1
}
}
}
......@@ -28,7 +28,9 @@
{%- set part_list = [] %}
{%- for class, slave_instance_list in slave_instance_dict.items() %}
{#- class is used to separate surykatka with different timeouts #}
{%- set URL_LIST = [] %}
{%- for slave in slave_instance_list | sort(attribute='-slave-title') %}
{%- do URL_LIST.append(slave['url']) %}
{%- set part_id = 'http-query-' ~ hashlib_module.md5(slave['-slave-reference'].encode('utf-8')).hexdigest() ~ '-promise' %}
{%- do part_list.append(part_id) %}
{%- set safe_name = part_id.replace('_', '').replace('.', '-').replace(' ', '-') %}
......@@ -59,7 +61,7 @@ recipe = slapos.recipe.template:jinja2
db = ${directory:srv}/surykatka-{{ class }}.db
rendered = ${directory:etc}/surykatka-{{ class }}.ini
template = {{ template_surykatka_ini }}
slave_instance_list = {{ dumps(slave_instance_list) }}
url_list = {{ dumps(URL_LIST) }}
nameserver_list = {{ dumps(CONFIGURATION['nameserver-list']) }}
json = ${directory:srv}/surykatka-{{ class }}.json
{#- timeout is just a bit bigger than class time #}
......@@ -69,7 +71,7 @@ context =
import json_module json
key db :db
key nameserver_list :nameserver_list
key slave_instance_list :slave_instance_list
key url_list :url_list
key timeout :timeout
[surykatka-{{ class }}]
......
{#- PREPARE #}
{%- set DEFAULT_DICT = {
'check-status-code': 200,
'check-http-header-dict': {},
'check-certificate-expiration-days': 15,
'failure-amount': 2,
'check-maximum-elapsed-time': 2,
'check-frontend-ip-list': [],
'nameserver-list': []
} %}
{%- for default_key in ['check-frontend-ip-list', 'nameserver-list', 'check-status-code', 'check-certificate-expiration-days', 'check-maximum-elapsed-time', 'failure-amount'] %}
{%- if default_key in slapparameter_dict %}
{%- do DEFAULT_DICT.__setitem__(default_key, slapparameter_dict[default_key]) %}
{%- endif %}
{%- endfor %}
{%- set CHECK_DICT = {} %}
{%- for check_name, check_definition in slapparameter_dict['check-dict'].items() %}
{%- if 'url-list' in check_definition %}
{%- set check = DEFAULT_DICT.copy() %}
{%- set url_list = [] %}
{%- for url in check_definition.pop('url-list') %}
{%- if len(url.strip()) > 0 %}
{#- Support various URL entries #}
{%- if url.startswith('http:') or url.startswith('https:') %}
{%- if url not in url_list %}
{%- do url_list.append(url) %}
{%- endif %}
{%- else %}
{%- for schema in ['http://', 'https://'] %}
{%- set add_url = '%s%s' % (schema, url) %}
{%- if add_url not in url_list %}
{%- do url_list.append('%s%s' % (schema, url)) %}
{%- endif %}
{%- endfor %}
{%- endif %}
{%- endif %}
{%- endfor %}
{%- do check.update(check_definition) %}
{%- do check.__setitem__('url-list', url_list) %}
{%- set class = check['check-maximum-elapsed-time'] %}
{%- if class not in CHECK_DICT %}
{%- do CHECK_DICT.__setitem__(class, []) %}
{%- endif %}
{%- do check.__setitem__('-name', check_name) %}
{%- do CHECK_DICT[class].append(check) %}
{%- endif %}
{%- endfor %}
{%- set PART_LIST = [] %}
{%- for class, class_check_list in CHECK_DICT.items() %}
{#- class is used to separate surykatka with different timeouts #}
{%- set CLASS_URL_LIST = [] %}
{%- for check in class_check_list | sort(attribute='-name') %}
{%- do CLASS_URL_LIST.extend(check['url-list']) %}
{%- for url in check['url-list'] %}
{%- set promise_name = 'http-query-' ~ hashlib_module.md5(check['-name'].encode('utf-8')).hexdigest() ~ '-' ~ hashlib_module.md5(url.encode('utf-8')).hexdigest() %}
{%- set part_name = promise_name ~ '-promise' %}
{%- do PART_LIST.append(part_name) %}
[{{part_name}}]
<= monitor-promise-base
module = check_surykatka_json
name = {{ promise_name }}.py
config-report = http_query
config-url = {{ url }}
config-status-code = {{ check['check-status-code'] }}
config-http-header-dict = {{ json_module.dumps(check['check-http-header-dict']) }}
config-certificate-expiration-days = {{ check['check-certificate-expiration-days'] }}
config-failure-amount = {{ check['failure-amount'] }}
config-maximum-elapsed-time = {{ check['check-maximum-elapsed-time'] }}
config-ip-list = {{ ' '.join(check['check-frontend-ip-list']) }}
config-json-file = ${surykatka-config-{{ class }}:json}
{%- endfor %} {#- for url in check['url-list'] #}
{%- endfor %} {#- for check in class_check_list | sort(attribute='-name') #}
{%- do PART_LIST.append('surykatka-bot-%i-promise' % (class,)) %}
[surykatka-bot-{{ class }}-promise]
<= monitor-promise-base
module = check_surykatka_json
name = surykatka-bot-{{ class }}.py
config-report = bot_status
config-json-file = ${surykatka-config-{{ class }}:json}
[surykatka-config-{{ class }}]
recipe = slapos.recipe.template:jinja2
db = ${directory:srv}/surykatka-{{ class }}.db
rendered = ${directory:etc}/surykatka-{{ class }}.ini
template = {{ template_surykatka_ini }}
url_list = {{ dumps(CLASS_URL_LIST) }}
nameserver_list = {{ dumps(DEFAULT_DICT['nameserver-list']) }}
json = ${directory:srv}/surykatka-{{ class }}.json
{#- timeout is just a bit bigger than class time #}
timeout = {{ int(class) + 2 }}
context =
import json_module json
key db :db
key nameserver_list :nameserver_list
key url_list :url_list
key timeout :timeout
{%- do PART_LIST.append('surykatka-%i'% (class,)) %}
[surykatka-{{ class }}]
recipe = slapos.cookbook:wrapper
config = ${surykatka-config-{{ class }}:rendered}
command-line =
{{ surykatka_binary }} --run crawl --reload --configuration ${:config}
wrapper-path = ${monitor-directory:service}/${:_buildout_section_name_}
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
[surykatka-status-json-{{ class }}]
recipe = slapos.recipe.template:jinja2
json = ${surykatka-config-{{ class }}:json}
template = inline:#!/bin/sh
{{ surykatka_binary }} --run status --configuration ${surykatka-{{ class }}:config} --output json --stdout ${:json}
rendered = ${monitor-directory:bin}/${:_buildout_section_name_}
mode = 0755
{%- do PART_LIST.append('cron-entry-surykatka-status-%i' % (class,)) %}
[cron-entry-surykatka-status-{{ class }}]
recipe = slapos.cookbook:cron.d
cron-entries = ${directory:etc}/cron.d
name = surykatka-status-{{ class }}
frequency = */2 * * * *
command = ${surykatka-status-json-{{ class }}:rendered}
{%- endfor %} {#- for class, class_check_list in CHECK_DICT.items() #}
[buildout]
extends = {{ instance_base_monitor }}
parts +=
{% for part_id in sorted(PART_LIST) %}
{{ part_id }}
{% endfor %}
......@@ -10,6 +10,7 @@ recipe = slapos.cookbook:switch-softwaretype
default = instance-base-monitor:rendered
edgetest = instance-base-edgetest:rendered
edgebot = instance-base-edgebot:rendered
edgetest-basic = instance-edgetest-basic:rendered
RootSoftwareInstance = $${:default}
[instance-base-monitor]
......@@ -27,6 +28,34 @@ context = key develop_eggs_directory buildout:develop-eggs-directory
raw monitor_collect_csv_dump ${monitor-collect-csv-dump:output}
mode = 0644
[instance-template]
recipe = slapos.recipe.template:jinja2
extensions = jinja2.ext.do
rendered = $${buildout:directory}/$${:_buildout_section_name_}.cfg
context =
import json_module json
import hashlib_module hashlib
key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory
key slapparameter_dict slap-configuration:configuration
key slap_software_type slap-configuration:slap-software-type
key instance_base_monitor instance-base-monitor:rendered
raw buildout_bin ${buildout:bin-directory}
$${:extra-context}
mode = 0644
[surykatka]
binary = ${buildout:bin-directory}/${surykatka:script-name}
ini = ${template-surykatka-ini:target}
[instance-edgetest-basic]
<= instance-template
template = ${template-monitor-edgetest-basic:target}
extra-context =
raw software_type edgetest-basic
key template_surykatka_ini surykatka:ini
key surykatka_binary surykatka:binary
[instance-base-edgetest]
recipe = slapos.recipe.template:jinja2
template = ${template-monitor-edgetest:target}
......@@ -51,8 +80,6 @@ recipe = slapos.recipe.template:jinja2
template = ${template-monitor-edgebot:target}
rendered = $${buildout:directory}/template-monitor-edgebot.cfg
extensions = jinja2.ext.do
surykatka-binary = ${buildout:bin-directory}/${surykatka:script-name}
template-surykatka-ini = ${template-surykatka-ini:target}
context = import json_module json
import hashlib_module hashlib
......@@ -62,8 +89,8 @@ context = import json_module json
key slapparameter_dict slap-configuration:configuration
key slap_software_type slap-configuration:slap-software-type
raw software_type edgebot
key surykatka_binary :surykatka-binary
key template_surykatka_ini :template-surykatka-ini
key surykatka_binary surykatka:binary
key template_surykatka_ini surykatka:ini
raw buildout_bin ${buildout:bin-directory}
raw monitor_template_output ${monitor-template:output}
raw monitor_collect_csv_dump ${monitor-collect-csv-dump:output}
......
......@@ -31,6 +31,11 @@ recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_update_hash_filename_}
mode = 0644
[template-monitor-edgetest-basic]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_update_hash_filename_}
mode = 0644
[template-monitor-edgetest]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_update_hash_filename_}
......
......@@ -10,23 +10,31 @@
"response": "instance-default-output-schema.json",
"index": 0
},
"edgetest-basic": {
"title": "Edge Test Basic",
"description": "Basic URL monitoring configuration",
"request": "instance-edgetest-basic-input-schema.json",
"response": "instance-default-output-schema.json",
"serialisation": "json-in-xml",
"index": 1
},
"edgetest": {
"title": "Edge Test",
"description": "Cluster of bots to perform a distributed monitoring ",
"title": "Edge Test [OBSOLETE]",
"description": "Cluster of bots to perform a distributed monitoring. OBSOLETE: Use Edge Test Basic.",
"request": "instance-edgetest-input-schema.json",
"response": "instance-default-output-schema.json",
"serialisation": "json-in-xml",
"index": 1
"index": 2
},
"edgetest-slave": {
"title": "Edge Test Slave",
"title": "Edge Test Slave [OBSOLETE]",
"shared": true,
"software-type": "edgetest",
"description": "Cluster of bots to perform a distributed monitoring ",
"description": "Cluster of bots to perform a distributed monitoring. OBSOLETE: Use Edge Test Basic.",
"request": "instance-edgetest-slave-input-schema.json",
"response": "instance-default-output-schema.json",
"serialisation": "json-in-xml",
"index": 2
"index": 3
}
}
}
......@@ -9,9 +9,6 @@ NAMESERVER =
{%- endfor %}
{% endif %}
URL =
{%- for slave in slave_instance_list | sort(attribute='-slave-title') %}
{%- if 'url' in slave %}
{{ slave['url'] }}
{%- endif -%}
{%- for url in sorted(url_list) %}
{{ url -}}
{% endfor %}
This diff is collapsed.
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment