Skip to content

slapos
slapos
Commit 17cd7641 authored by Kirill Smelkov's avatar Kirill Smelkov
Browse files
Options

Merge branch 'master' into master+ZODB4-wc2 

* master:
  software/monitor: edgetest-basic implementation
  component/open62541: add open62541 1.2.2
  stack/slapos: Version up
  monitor: Use surykatka direct output to a file
  monitor: Pick up recent surykatka
  caddy-frontend: Raise promise only on slave errors impacting cluster
parents c87abe0a 6b4d1124
Hide whitespace changes
Inline Side-by-side
Showing with 786 additions and 134 deletions
component/open62541/buildout.cfg 0 → 100644
View file @ 17cd7641
# Implementation of OPC UA (OPC Unified Architecture). https://open62541.org/
[buildout]
parts = open62541
extends =
../cmake/buildout.cfg
[open62541]
recipe = slapos.recipe.cmmi
shared = true
url = https://github.com/open62541/open62541/archive/refs/tags/v1.2.2.tar.gz
md5sum = 2883bde165bc9bc3d459ccbb47acf7f4
configure-command = ${cmake:location}/bin/cmake
configure-options =
-Bbuild
-DCMAKE_INSTALL_PREFIX=@@LOCATION@@
make-options = -C build
software/caddy-frontend/buildout.hash.cfg
View file @ 17cd7641
......@@ -26,7 +26,7 @@ md5sum = 51087ac7615bd7cc01e60eb23701f625
[profile-caddy-replicate]
filename = instance-apache-replicate.cfg.in
md5sum = b6fc5a004a1235ffad3af0b4cb0e661f
md5sum = 99741e618b1c249bd17c9e02778d74ee
[profile-slave-list]
_update_hash_filename_ = templates/apache-custom-slave-list.cfg.in
......
software/caddy-frontend/instance-apache-replicate.cfg.in
View file @ 17cd7641
......@@ -103,10 +103,12 @@ context =
{% set authorized_slave_string_list = [] %}
{% set authorized_slave_list = [] %}
{% set rejected_slave_dict = {} %}
{% set critical_rejected_slave_dict = {} %}
{% set warning_slave_dict = {} %}
{% set used_host_list = [] %}
{% for slave in sorted(instance_parameter_dict['slave-instance-list']) %}
{% set slave_error_list = [] %}
{% set slave_critical_error_list = [] %}
{% set slave_warning_list = [] %}
{% set slave_server_alias_unclashed = [] %}
{% set slave_type = slave.get('type') %}
......@@ -165,7 +167,9 @@ context =
{% endif %}
{% set custom_domain = slave.get('custom_domain') %}
{% if custom_domain and custom_domain in used_host_list %}
{% do slave_error_list.append('custom_domain %r clashes' % (custom_domain,)) %}
{% set message = 'custom_domain %r clashes' % (custom_domain,) %}
{% do slave_error_list.append(message) %}
{% do slave_critical_error_list.append(message) %}
{% else %}
{% do used_host_list.append(custom_domain) %}
{% endif %}
......@@ -182,7 +186,9 @@ context =
{% if slave_alias in slave_server_alias_unclashed or slave_alias == custom_domain %}
{# optionally do something about reporting back that server-alias has been unclashed #}
{% elif slave_alias in used_host_list %}
{% do slave_error_list.append('server-alias \'%s\' clashes' % (slave_alias,)) %}
{% set message = 'server-alias \'%s\' clashes' % (slave_alias,) %}
{% do slave_error_list.append(message) %}
{% do slave_critical_error_list.append(message) %}
{% else %}
{% do slave_server_alias_unclashed.append(slave_alias) %}
{% do used_host_list.append(slave_alias) %}
......@@ -251,6 +257,9 @@ context =
{% else %}
{% do rejected_slave_dict.__setitem__(slave.get('slave_reference'), sorted(slave_error_list)) %}
{% endif %}
{% if len(slave_critical_error_list) > 0 %}
{% do critical_rejected_slave_dict.__setitem__(slave.get('slave_reference'), sorted(slave_critical_error_list)) %}
{% endif %}
{% if len(slave_warning_list) > 0 %}
{% do warning_slave_dict.__setitem__(slave.get('slave_reference'), sorted(slave_warning_list)) %}
{% endif %}
......@@ -752,9 +761,9 @@ filename = rejected-slave.json
directory = ${directory:promise-output}
rendered = ${:directory}/${:filename}
template = {{ software_parameter_dict['template_empty'] }}
{% if rejected_slave_dict %}
{% if critical_rejected_slave_dict %}
{# sort_keys are important in order to avoid shuffling parameters on each run #}
content = {{ dumps(json_module.dumps(rejected_slave_dict, indent=2, sort_keys=True)) }}
content = {{ dumps(json_module.dumps(critical_rejected_slave_dict, indent=2, sort_keys=True)) }}
{% else %}
content =
{% endif %}
......
software/caddy-frontend/test/test.py
View file @ 17cd7641
......@@ -795,7 +795,7 @@ class HttpFrontendTestCase(SlapOSInstanceTestCase):
with cls.slap.instance_supervisor_rpc as instance_supervisor:
return getattr(instance_supervisor, method)(*args, **kwargs)
def assertRejectedSlavePromiseWithPop(self, parameter_dict):
def assertRejectedSlavePromiseEmptyWithPop(self, parameter_dict):
rejected_slave_promise_url = parameter_dict.pop(
'rejected-slave-promise-url')
......@@ -806,7 +806,7 @@ class HttpFrontendTestCase(SlapOSInstanceTestCase):
else:
result_json = result.json()
self.assertEqual(
parameter_dict['rejected-slave-dict'],
{},
result_json
)
except AssertionError:
......@@ -1238,7 +1238,7 @@ class TestMasterRequestDomain(HttpFrontendTestCase, TestDataMixin):
self.assertKeyWithPop('monitor-setup-url', parameter_dict)
self.assertBackendHaproxyStatisticUrl(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict, 'master-')
self.assertRejectedSlavePromiseWithPop(parameter_dict)
self.assertRejectedSlavePromiseEmptyWithPop(parameter_dict)
self.assertEqual(
{
......@@ -1269,7 +1269,7 @@ class TestMasterRequest(HttpFrontendTestCase, TestDataMixin):
self.assertKeyWithPop('monitor-setup-url', parameter_dict)
self.assertBackendHaproxyStatisticUrl(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict, 'master-')
self.assertRejectedSlavePromiseWithPop(parameter_dict)
self.assertRejectedSlavePromiseEmptyWithPop(parameter_dict)
self.assertEqual(
{
'monitor-base-url': 'https://[%s]:8401' % self._ipv6_address,
......@@ -1707,7 +1707,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
self.assertKeyWithPop('monitor-setup-url', parameter_dict)
self.assertBackendHaproxyStatisticUrl(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict, 'master-')
self.assertRejectedSlavePromiseWithPop(parameter_dict)
self.assertRejectedSlavePromiseEmptyWithPop(parameter_dict)
expected_parameter_dict = {
'monitor-base-url': 'https://[%s]:8401' % self._ipv6_address,
......@@ -5286,7 +5286,7 @@ class TestSlaveSlapOSMasterCertificateCompatibility(
self.assertKeyWithPop('monitor-setup-url', parameter_dict)
self.assertBackendHaproxyStatisticUrl(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict, 'master-')
self.assertRejectedSlavePromiseWithPop(parameter_dict)
self.assertRejectedSlavePromiseEmptyWithPop(parameter_dict)
expected_parameter_dict = {
'monitor-base-url': 'https://[%s]:8401' % self._ipv6_address,
......@@ -5296,10 +5296,6 @@ class TestSlaveSlapOSMasterCertificateCompatibility(
'rejected-slave-amount': '0',
'slave-amount': '12',
'rejected-slave-dict': {
# u"_ssl_ca_crt_only":
# [u"ssl_ca_crt is present, so ssl_crt and ssl_key are required"],
# u"_ssl_key-ssl_crt-unsafe":
# [u"slave ssl_key and ssl_crt does not match"]
},
'warning-list': [
u'apache-certificate is obsolete, please use master-key-upload-url',
......@@ -5948,7 +5944,7 @@ class TestSlaveSlapOSMasterCertificateCompatibilityUpdate(
self.assertKeyWithPop('monitor-setup-url', parameter_dict)
self.assertBackendHaproxyStatisticUrl(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict, 'master-')
self.assertRejectedSlavePromiseWithPop(parameter_dict)
self.assertRejectedSlavePromiseEmptyWithPop(parameter_dict)
expected_parameter_dict = {
'monitor-base-url': 'https://[%s]:8401' % self._ipv6_address,
......@@ -6053,7 +6049,7 @@ class TestSlaveCiphers(SlaveHttpFrontendTestCase, TestDataMixin):
self.assertKeyWithPop('monitor-setup-url', parameter_dict)
self.assertBackendHaproxyStatisticUrl(parameter_dict)
self.assertKedifaKeysWithPop(parameter_dict, 'master-')
self.assertRejectedSlavePromiseWithPop(parameter_dict)
self.assertRejectedSlavePromiseEmptyWithPop(parameter_dict)
expected_parameter_dict = {
'monitor-base-url': 'https://[%s]:8401' % self._ipv6_address,
......@@ -6293,6 +6289,29 @@ class TestSlaveRejectReportUnsafeDamaged(SlaveHttpFrontendTestCase):
}
}
def assertRejectedSlavePromiseWithPop(self, parameter_dict):
rejected_slave_promise_url = parameter_dict.pop(
'rejected-slave-promise-url')
try:
result = requests.get(rejected_slave_promise_url, verify=False)
if result.text == '':
result_json = {}
else:
result_json = result.json()
self.assertEqual(
{
u'_SITE_4': [u"custom_domain 'duplicate.example.com' clashes"],
u'_SITE_2': [u"custom_domain 'duplicate.example.com' clashes"],
u'_SITE_3': [u"server-alias 'duplicate.example.com' clashes"]
},
result_json
)
except AssertionError:
raise
except Exception as e:
self.fail(e)
def test_master_partition_state(self):
parameter_dict = self.parseConnectionParameterDict()
self.assertKeyWithPop('monitor-setup-url', parameter_dict)
......
software/monitor/buildout.hash.cfg
View file @ 17cd7641
......@@ -14,7 +14,7 @@
# not need these here).
[template]
filename = instance.cfg
md5sum = 2114ae8c1e92bd33ef1347f36f567c74
md5sum = b6c2df0d4a62473d6dae26b10c0a4adc
[template-monitor]
_update_hash_filename_ = instance-monitor.cfg.jinja2
......@@ -24,13 +24,17 @@ md5sum = 165a15672fc85981f68b9af2d6253254
_update_hash_filename_ = json-test-template.json.in.jinja2
md5sum = 2eb5596544d9c341acf653d4f7ce2680
[template-monitor-edgetest-basic]
_update_hash_filename_ = instance-monitor-edgetest-basic.cfg.jinja2
md5sum = 61309a48f7b0135cba21b09247a2d8fd
[template-monitor-edgetest]
_update_hash_filename_ = instance-monitor-edgetest.cfg.jinja2
md5sum = 3c8ab4e78f66c974eb95afc595a13514
[template-monitor-edgebot]
_update_hash_filename_ = instance-monitor-edgebot.cfg.jinja2
md5sum = 365a6cc6831267a73fa5ebd56ad394ee
md5sum = 2ac74559d6108ca0dbabb872f1071e44
[network-bench-cfg]
filename = network_bench.cfg.in
......@@ -42,4 +46,4 @@ md5sum = d3cfa1f6760e3fa64ccd64acf213bdfb
[template-surykatka-ini]
_update_hash_filename_ = surykatka.ini.jinja2
md5sum = 609c6cca763b73a80fa05ee56475eb20
md5sum = 7e9b874c20faaa8190d2bf2b74caa727
software/monitor/instance-edgetest-basic-input-schema.json 0 → 100644
View file @ 17cd7641
{
"$schema": "http://json-schema.org/draft-04/schema#",
"type": "object",
"properties": {
"nameserver-list": {
"default": [],
"title": "Nameservers",
"description": "List of nameservers IPs (like 1.2.3.4) to use.",
"type": "array"
},
"check-frontend-ip-list": {
"default": [],
"title": "Default Frontend IPs to check",
"description": "List of default frontend IPs (like 1.2.3.4) to check, if empty no constraint is used.",
"type": "array"
},
"check-dict": {
"title": "Configuration of checks",
"description": "Configure checks.",
"patternProperties": {
".*": {
"properties": {
"url-list": {
"default": [],
"title": "URLs to check",
"description": "List of URLs to check, can be domain (example.com) or URL (https://example.com/, http://example.com/path)",
"type": "array"
},
"check-status-code": {
"title": "HTTP Code Check",
"description": "Expected response HTTP Code. Defaults to global configuration.",
"type": "number",
"minimum": 100,
"maximum": 599
},
"check-frontend-ip-list": {
"title": "Frontend IPs to check (optional)",
"description": "List of frontend IPs (like 1.2.3.4) to check. Defaults to global configuration.",
"type": "array"
},
"check-certificate-expiration-days": {
"title": "Certificate Expiration Check (days)",
"description": "Amount of days to consider certificate as being to-be-expired. Defaults to global configuration.",
"type": "number",
"minimum": 1
},
"check-maximum-elapsed-time": {
"title": "Maximum Elapsed Check (seconds)",
"description": "Maximum elapsed time for a site to reply to be considered good. Defaults to global configuration.",
"type": "number",
"minimum": 1
},
"failure-amount": {
"title": "Failure Amount",
"description": "Amount of failures to consider URL as in bad state, can be set to higher value for endpoints with accepted short outages. Defaults to global configuration.",
"type": "number",
"minimum": 1
},
"check-http-header-dict": {
"title": "[UI UNSUPPORTED] HTTP Header Check",
"description": "[UI UNSUPPORTED] JSON object of expected HTTP header, like {\"Cache-Control\": \"max-age=3600, public\", \"Vary\": \"Accept-Encoding\"}. Note: Shall be expressed directly as object, without any additional qouting.",
"type": "object",
"default": {}
}
},
"type": "object"
}
},
"type": "object",
"default": {}
},
"check-status-code": {
"title": "Default HTTP Code Check",
"description": "Default expected response HTTP Code.",
"type": "number",
"default": 200,
"minimum": 100,
"maximum": 599
},
"check-certificate-expiration-days": {
"title": "Default Certificate Expiration Check (days)",
"description": "Default amount of days to consider certificate as being to-be-expired.",
"type": "number",
"default": 15,
"minimum": 1
},
"check-maximum-elapsed-time": {
"title": "Default Maximum Elapsed Check (seconds)",
"description": "Default maximum elapsed time for a site to reply to be considered good.",
"type": "number",
"default": 2,
"minimum": 1
},
"failure-amount": {
"title": "Default Failure Amount",
"description": "Default amount of failures to consider URL as in bad state, can be set to higher value for endpoints with accepted short outages.",
"type": "number",
"default": 2,
"minimum": 1
}
}
}
software/monitor/instance-monitor-edgebot.cfg.jinja2
View file @ 17cd7641
......@@ -28,7 +28,9 @@
{%- set part_list = [] %}
{%- for class, slave_instance_list in slave_instance_dict.items() %}
{#- class is used to separate surykatka with different timeouts #}
{%- set URL_LIST = [] %}
{%- for slave in slave_instance_list | sort(attribute='-slave-title') %}
{%- do URL_LIST.append(slave['url']) %}
{%- set part_id = 'http-query-' ~ hashlib_module.md5(slave['-slave-reference'].encode('utf-8')).hexdigest() ~ '-promise' %}
{%- do part_list.append(part_id) %}
{%- set safe_name = part_id.replace('_', '').replace('.', '-').replace(' ', '-') %}
......@@ -59,7 +61,7 @@ recipe = slapos.recipe.template:jinja2
db = ${directory:srv}/surykatka-{{ class }}.db
rendered = ${directory:etc}/surykatka-{{ class }}.ini
template = {{ template_surykatka_ini }}
slave_instance_list = {{ dumps(slave_instance_list) }}
url_list = {{ dumps(URL_LIST) }}
nameserver_list = {{ dumps(CONFIGURATION['nameserver-list']) }}
json = ${directory:srv}/surykatka-{{ class }}.json
{#- timeout is just a bit bigger than class time #}
......@@ -69,7 +71,7 @@ context =
import json_module json
key db :db
key nameserver_list :nameserver_list
key slave_instance_list :slave_instance_list
key url_list :url_list
key timeout :timeout
[surykatka-{{ class }}]
......@@ -84,11 +86,7 @@ hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
recipe = slapos.recipe.template:jinja2
json = ${surykatka-config-{{ class }}:json}
template = inline:#!/bin/sh
if {{ surykatka_binary }} --run status --configuration ${surykatka-{{ class }}:config} --output json > ${:json}.tmp ; then
mv -f ${:json}.tmp ${:json}
else
rm -f ${:json}.tmp
fi
{{ surykatka_binary }} --run status --configuration ${surykatka-{{ class }}:config} --output json --stdout ${:json}
rendered = ${monitor-directory:bin}/${:_buildout_section_name_}
mode = 0755
......
software/monitor/instance-monitor-edgetest-basic.cfg.jinja2 0 → 100644
View file @ 17cd7641
{#- PREPARE #}
{%- set DEFAULT_DICT = {
'check-status-code': 200,
'check-http-header-dict': {},
'check-certificate-expiration-days': 15,
'failure-amount': 2,
'check-maximum-elapsed-time': 2,
'check-frontend-ip-list': [],
'nameserver-list': []
} %}
{%- for default_key in ['check-frontend-ip-list', 'nameserver-list', 'check-status-code', 'check-certificate-expiration-days', 'check-maximum-elapsed-time', 'failure-amount'] %}
{%- if default_key in slapparameter_dict %}
{%- do DEFAULT_DICT.__setitem__(default_key, slapparameter_dict[default_key]) %}
{%- endif %}
{%- endfor %}
{%- set CHECK_DICT = {} %}
{%- for check_name, check_definition in slapparameter_dict['check-dict'].items() %}
{%- if 'url-list' in check_definition %}
{%- set check = DEFAULT_DICT.copy() %}
{%- set url_list = [] %}
{%- for url in check_definition.pop('url-list') %}
{%- if len(url.strip()) > 0 %}
{#- Support various URL entries #}
{%- if url.startswith('http:') or url.startswith('https:') %}
{%- if url not in url_list %}
{%- do url_list.append(url) %}
{%- endif %}
{%- else %}
{%- for schema in ['http://', 'https://'] %}
{%- set add_url = '%s%s' % (schema, url) %}
{%- if add_url not in url_list %}
{%- do url_list.append('%s%s' % (schema, url)) %}
{%- endif %}
{%- endfor %}
{%- endif %}
{%- endif %}
{%- endfor %}
{%- do check.update(check_definition) %}
{%- do check.__setitem__('url-list', url_list) %}
{%- set class = check['check-maximum-elapsed-time'] %}
{%- if class not in CHECK_DICT %}
{%- do CHECK_DICT.__setitem__(class, []) %}
{%- endif %}
{%- do check.__setitem__('-name', check_name) %}
{%- do CHECK_DICT[class].append(check) %}
{%- endif %}
{%- endfor %}
{%- set PART_LIST = [] %}
{%- for class, class_check_list in CHECK_DICT.items() %}
{#- class is used to separate surykatka with different timeouts #}
{%- set CLASS_URL_LIST = [] %}
{%- for check in class_check_list | sort(attribute='-name') %}
{%- do CLASS_URL_LIST.extend(check['url-list']) %}
{%- for url in check['url-list'] %}
{%- set promise_name = 'http-query-' ~ hashlib_module.md5(check['-name'].encode('utf-8')).hexdigest() ~ '-' ~ hashlib_module.md5(url.encode('utf-8')).hexdigest() %}
{%- set part_name = promise_name ~ '-promise' %}
{%- do PART_LIST.append(part_name) %}
[{{part_name}}]
<= monitor-promise-base
module = check_surykatka_json
name = {{ promise_name }}.py
config-report = http_query
config-url = {{ url }}
config-status-code = {{ check['check-status-code'] }}
config-http-header-dict = {{ json_module.dumps(check['check-http-header-dict']) }}
config-certificate-expiration-days = {{ check['check-certificate-expiration-days'] }}
config-failure-amount = {{ check['failure-amount'] }}
config-maximum-elapsed-time = {{ check['check-maximum-elapsed-time'] }}
config-ip-list = {{ ' '.join(check['check-frontend-ip-list']) }}
config-json-file = ${surykatka-config-{{ class }}:json}
{%- endfor %} {#- for url in check['url-list'] #}
{%- endfor %} {#- for check in class_check_list | sort(attribute='-name') #}
{%- do PART_LIST.append('surykatka-bot-%i-promise' % (class,)) %}
[surykatka-bot-{{ class }}-promise]
<= monitor-promise-base
module = check_surykatka_json
name = surykatka-bot-{{ class }}.py
config-report = bot_status
config-json-file = ${surykatka-config-{{ class }}:json}
[surykatka-config-{{ class }}]
recipe = slapos.recipe.template:jinja2
db = ${directory:srv}/surykatka-{{ class }}.db
rendered = ${directory:etc}/surykatka-{{ class }}.ini
template = {{ template_surykatka_ini }}
url_list = {{ dumps(CLASS_URL_LIST) }}
nameserver_list = {{ dumps(DEFAULT_DICT['nameserver-list']) }}
json = ${directory:srv}/surykatka-{{ class }}.json
{#- timeout is just a bit bigger than class time #}
timeout = {{ int(class) + 2 }}
context =
import json_module json
key db :db
key nameserver_list :nameserver_list
key url_list :url_list
key timeout :timeout
{%- do PART_LIST.append('surykatka-%i'% (class,)) %}
[surykatka-{{ class }}]
recipe = slapos.cookbook:wrapper
config = ${surykatka-config-{{ class }}:rendered}
command-line =
{{ surykatka_binary }} --run crawl --reload --configuration ${:config}
wrapper-path = ${monitor-directory:service}/${:_buildout_section_name_}
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
{%- do PART_LIST.append('surykatka-json-%i-promise'% (class,)) %}
[surykatka-json-{{ class }}-promise]
<= monitor-promise-base
module = check_file_state
name = surykatka-json-{{ class }}.py
config-filename = ${surykatka-config-{{ class }}:json}
config-state = not-empty
# workaround for bug in check_file_state
config-url =
[surykatka-status-json-{{ class }}]
recipe = slapos.recipe.template:jinja2
json = ${surykatka-config-{{ class }}:json}
template = inline:#!/bin/sh
{{ surykatka_binary }} --run status --configuration ${surykatka-{{ class }}:config} --output json --stdout ${:json}
rendered = ${monitor-directory:bin}/${:_buildout_section_name_}
mode = 0755
{%- do PART_LIST.append('cron-entry-surykatka-status-%i' % (class,)) %}
[cron-entry-surykatka-status-{{ class }}]
recipe = slapos.cookbook:cron.d
cron-entries = ${directory:etc}/cron.d
name = surykatka-status-{{ class }}
frequency = */2 * * * *
command = ${surykatka-status-json-{{ class }}:rendered}
{%- endfor %} {#- for class, class_check_list in CHECK_DICT.items() #}
[buildout]
extends = {{ instance_base_monitor }}
parts +=
{% for part_id in sorted(PART_LIST) %}
{{ part_id }}
{% endfor %}
software/monitor/instance.cfg
View file @ 17cd7641
......@@ -10,6 +10,7 @@ recipe = slapos.cookbook:switch-softwaretype
default = instance-base-monitor:rendered
edgetest = instance-base-edgetest:rendered
edgebot = instance-base-edgebot:rendered
edgetest-basic = instance-edgetest-basic:rendered
RootSoftwareInstance = $${:default}
[instance-base-monitor]
......@@ -27,6 +28,34 @@ context = key develop_eggs_directory buildout:develop-eggs-directory
raw monitor_collect_csv_dump ${monitor-collect-csv-dump:output}
mode = 0644
[instance-template]
recipe = slapos.recipe.template:jinja2
extensions = jinja2.ext.do
rendered = $${buildout:directory}/$${:_buildout_section_name_}.cfg
context =
import json_module json
import hashlib_module hashlib
key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory
key slapparameter_dict slap-configuration:configuration
key slap_software_type slap-configuration:slap-software-type
key instance_base_monitor instance-base-monitor:rendered
raw buildout_bin ${buildout:bin-directory}
$${:extra-context}
mode = 0644
[surykatka]
binary = ${buildout:bin-directory}/${surykatka:script-name}
ini = ${template-surykatka-ini:target}
[instance-edgetest-basic]
<= instance-template
template = ${template-monitor-edgetest-basic:target}
extra-context =
raw software_type edgetest-basic
key template_surykatka_ini surykatka:ini
key surykatka_binary surykatka:binary
[instance-base-edgetest]
recipe = slapos.recipe.template:jinja2
template = ${template-monitor-edgetest:target}
......@@ -51,8 +80,6 @@ recipe = slapos.recipe.template:jinja2
template = ${template-monitor-edgebot:target}
rendered = $${buildout:directory}/template-monitor-edgebot.cfg
extensions = jinja2.ext.do
surykatka-binary = ${buildout:bin-directory}/${surykatka:script-name}
template-surykatka-ini = ${template-surykatka-ini:target}
context = import json_module json
import hashlib_module hashlib
......@@ -62,8 +89,8 @@ context = import json_module json
key slapparameter_dict slap-configuration:configuration
key slap_software_type slap-configuration:slap-software-type
raw software_type edgebot
key surykatka_binary :surykatka-binary
key template_surykatka_ini :template-surykatka-ini
key surykatka_binary surykatka:binary
key template_surykatka_ini surykatka:ini
raw buildout_bin ${buildout:bin-directory}
raw monitor_template_output ${monitor-template:output}
raw monitor_collect_csv_dump ${monitor-collect-csv-dump:output}
......
software/monitor/software.cfg
View file @ 17cd7641
......@@ -31,6 +31,11 @@ recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_update_hash_filename_}
mode = 0644
[template-monitor-edgetest-basic]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_update_hash_filename_}
mode = 0644
[template-monitor-edgetest]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_update_hash_filename_}
......@@ -77,11 +82,13 @@ scripts =
[versions]
surykatka = 0.5.0
surykatka = 0.7.0
# For surykatka 0.5.0
click = 7.0
dnspython = 1.16.0
# For surykatka 0.7.0
click = 8.0.1
dnspython = 2.1.0
forcediphttpsadapter = 1.0.1
miniupnpc = 2.0.2
peewee = 3.13.1
peewee = 3.14.4
python-whois = 0.7.3
future = 0.18.2
software/monitor/software.cfg.json
View file @ 17cd7641
......@@ -10,23 +10,31 @@
"response": "instance-default-output-schema.json",
"index": 0
},
"edgetest-basic": {
"title": "Edge Test Basic",
"description": "Basic URL monitoring configuration",
"request": "instance-edgetest-basic-input-schema.json",
"response": "instance-default-output-schema.json",
"serialisation": "json-in-xml",
"index": 1
},
"edgetest": {
"title": "Edge Test",
"description": "Cluster of bots to perform a distributed monitoring ",
"title": "Edge Test [OBSOLETE]",
"description": "Cluster of bots to perform a distributed monitoring. OBSOLETE: Use Edge Test Basic.",
"request": "instance-edgetest-input-schema.json",
"response": "instance-default-output-schema.json",
"serialisation": "json-in-xml",
"index": 1
"index": 2
},
"edgetest-slave": {
"title": "Edge Test Slave",
"title": "Edge Test Slave [OBSOLETE]",
"shared": true,
"software-type": "edgetest",
"description": "Cluster of bots to perform a distributed monitoring ",
"description": "Cluster of bots to perform a distributed monitoring. OBSOLETE: Use Edge Test Basic.",
"request": "instance-edgetest-slave-input-schema.json",
"response": "instance-default-output-schema.json",
"serialisation": "json-in-xml",
"index": 2
"index": 3
}
}
}
software/monitor/surykatka.ini.jinja2
View file @ 17cd7641
......@@ -9,9 +9,6 @@ NAMESERVER =
{%- endfor %}
{% endif %}
URL =
{%- for slave in slave_instance_list | sort(attribute='-slave-title') %}
{%- if 'url' in slave %}
{{ slave['url'] }}
{%- endif -%}
{%- for url in sorted(url_list) %}
{{ url -}}
{% endfor %}
software/monitor/test/test.py
View file @ 17cd7641
......@@ -137,89 +137,13 @@ class MonitorTestMixin:
)
class EdgeSlaveMixin(MonitorTestMixin):
class EdgeMixin(object):
__partition_reference__ = 'edge'
instance_max_retry = 20
expected_connection_parameter_dict = {}
@classmethod
def setUpClass(cls):
# XXX we run these tests with --all as a workaround for the fact that after
# requesting new shared instances we don't have promise to wait for the
# processing of these shared instances to be completed.
# The sequence is something like this:
# - `requestEdgetestSlaves` will request edgetest partition
# - first `waitForInstance` will process the edgetest partition, which will
# request a edgebot partition, but without promise to wait for the
# processing to be finished, so the first run of `slapos node instance`
# exits with success code and `waitForInstance` return.
# - second `waitForInstance` process the edgebot partition.
# Once we implement a promise (or something similar) here, we should not
# have to use --all
cls.slap._force_slapos_node_instance_all = True
return super(EdgeSlaveMixin, cls).setUpClass()
@classmethod
def getInstanceSoftwareType(cls):
return 'edgetest'
def requestEdgetestSlave(self, partition_reference, partition_parameter_kw):
software_url = self.getSoftwareURL()
return self.slap.request(
software_release=software_url,
software_type='edgetest',
partition_reference=partition_reference,
partition_parameter_kw={'_': json.dumps(partition_parameter_kw)},
shared=True
)
def updateSurykatkaDict(self):
for instance_reference in self.surykatka_dict:
for class_ in self.surykatka_dict[instance_reference]:
update_dict = {}
update_dict['ini-file'] = os.path.join(
self.slap.instance_directory, instance_reference, 'etc',
'surykatka-%s.ini' % (class_,))
update_dict['json-file'] = os.path.join(
self.slap.instance_directory, instance_reference, 'srv',
'surykatka-%s.json' % (class_,))
update_dict['status-json'] = os.path.join(
self.slap.instance_directory, instance_reference, 'bin',
'surykatka-status-json-%s' % (class_,))
update_dict['bot-promise'] = 'surykatka-bot-promise-%s.py' % (class_,)
update_dict['status-cron'] = os.path.join(
self.slap.instance_directory, instance_reference, 'etc',
'cron.d', 'surykatka-status-%s' % (class_,))
update_dict['db_file'] = os.path.join(
self.slap.instance_directory, instance_reference, 'srv',
'surykatka-%s.db' % (class_,))
self.surykatka_dict[instance_reference][class_].update(update_dict)
def setUpMonitorConfigurationList(self):
self.monitor_configuration_list = [
{
'xmlUrl': 'https://[%s]:9700/public/feed' % (self._ipv6_address,),
'version': 'RSS',
'title': 'testing partition 0',
'url': 'https://[%s]:9700/share/private/' % (self._ipv6_address,),
'text': 'testing partition 0',
'type': 'rss',
'htmlUrl': 'https://[%s]:9700/public/feed' % (self._ipv6_address,)
},
{
'xmlUrl': 'https://[%s]:9701/public/feed' % (self._ipv6_address,),
'version': 'RSS',
'title': 'edgebot-1',
'url': 'https://[%s]:9701/share/private/' % (self._ipv6_address,),
'text': 'edgebot-1',
'type': 'rss',
'htmlUrl': 'https://[%s]:9701/public/feed' % (self._ipv6_address,)
}
]
def setUp(self):
self.updateSurykatkaDict()
self.setUpMonitorConfigurationList()
def assertSurykatkaIni(self):
expected_init_path_list = []
......@@ -252,11 +176,6 @@ class EdgeSlaveMixin(MonitorTestMixin):
self.assertTrue(content in promise)
def assertHttpQueryPromiseContent(self, instance_reference, name, content):
hashed = 'http-query-%s-promise.py' % (
hashlib.md5(('_' + name).encode('utf-8')).hexdigest(),)
self.assertPromiseContent(instance_reference, hashed, content)
def assertSurykatkaBotPromise(self):
for instance_reference in self.surykatka_dict:
for info_dict in self.surykatka_dict[instance_reference].values():
......@@ -297,6 +216,25 @@ class EdgeSlaveMixin(MonitorTestMixin):
status_json = json.load(fh)
self.assertIn('bot_status', status_json)
class EdgeSlaveMixin(EdgeMixin, MonitorTestMixin):
@classmethod
def setUpClass(cls):
# XXX we run these tests with --all as a workaround for the fact that after
# requesting new shared instances we don't have promise to wait for the
# processing of these shared instances to be completed.
# The sequence is something like this:
# - `requestEdgetestSlaves` will request edgetest partition
# - first `waitForInstance` will process the edgetest partition, which
# will request a edgebot partition, but without promise to wait for the
# processing to be finished, so the first run of `slapos node instance`
# exits with success code and `waitForInstance` return.
# - second `waitForInstance` process the edgebot partition.
# Once we implement a promise (or something similar) here, we should not
# have to use --all
cls.slap._force_slapos_node_instance_all = True
return super().setUpClass()
def assertConnectionParameterDict(self):
serialised = self.requestDefaultInstance().getConnectionParameterDict()
connection_parameter_dict = json.loads(serialised['_'])
......@@ -309,6 +247,73 @@ class EdgeSlaveMixin(MonitorTestMixin):
connection_parameter_dict
)
@classmethod
def getInstanceSoftwareType(cls):
return 'edgetest'
def updateSurykatkaDict(self):
for instance_reference in self.surykatka_dict:
for class_ in self.surykatka_dict[instance_reference]:
update_dict = {}
update_dict['ini-file'] = os.path.join(
self.slap.instance_directory, instance_reference, 'etc',
'surykatka-%s.ini' % (class_,))
update_dict['json-file'] = os.path.join(
self.slap.instance_directory, instance_reference, 'srv',
'surykatka-%s.json' % (class_,))
update_dict['status-json'] = os.path.join(
self.slap.instance_directory, instance_reference, 'bin',
'surykatka-status-json-%s' % (class_,))
update_dict['bot-promise'] = 'surykatka-bot-promise-%s.py' % (class_,)
update_dict['status-cron'] = os.path.join(
self.slap.instance_directory, instance_reference, 'etc',
'cron.d', 'surykatka-status-%s' % (class_,))
update_dict['db_file'] = os.path.join(
self.slap.instance_directory, instance_reference, 'srv',
'surykatka-%s.db' % (class_,))
self.surykatka_dict[instance_reference][class_].update(update_dict)
def assertHttpQueryPromiseContent(self, instance_reference, name, content):
hashed = 'http-query-%s-promise.py' % (
hashlib.md5(('_' + name).encode('utf-8')).hexdigest(),)
self.assertPromiseContent(instance_reference, hashed, content)
def requestEdgetestSlave(self, partition_reference, partition_parameter_kw):
software_url = self.getSoftwareURL()
return self.slap.request(
software_release=software_url,
software_type='edgetest',
partition_reference=partition_reference,
partition_parameter_kw={'_': json.dumps(partition_parameter_kw)},
shared=True
)
def setUpMonitorConfigurationList(self):
self.monitor_configuration_list = [
{
'xmlUrl': 'https://[%s]:9700/public/feed' % (self._ipv6_address,),
'version': 'RSS',
'title': 'testing partition 0',
'url': 'https://[%s]:9700/share/private/' % (self._ipv6_address,),
'text': 'testing partition 0',
'type': 'rss',
'htmlUrl': 'https://[%s]:9700/public/feed' % (self._ipv6_address,)
},
{
'xmlUrl': 'https://[%s]:9701/public/feed' % (self._ipv6_address,),
'version': 'RSS',
'title': 'edgebot-1',
'url': 'https://[%s]:9701/share/private/' % (self._ipv6_address,),
'text': 'edgebot-1',
'type': 'rss',
'htmlUrl': 'https://[%s]:9701/public/feed' % (self._ipv6_address,)
}
]
def setUp(self):
super().setUp()
self.setUpMonitorConfigurationList()
def test(self):
# Note: Those tests do not run surykatka and do not do real checks, as
# this depends too much on the environment and is really hard to
......@@ -950,7 +955,7 @@ URL =
self.surykatka_dict['edge3'][2]['json-file'],))
def test(self):
super(TestEdgeRegion, self).test()
super().test()
self.assertSlaveConnectionParameterDict()
maxDiff = None
......@@ -1101,7 +1106,7 @@ class TestEdgeRegionDestroyed(TestEdgeRegion):
]
def test(self):
super(TestEdgeRegionDestroyed, self).test()
super().test()
# hack around @classmethod
self.__class__.instance_parameter_dict[
'region-dict']['Region Three']['state'] = 'destroyed'
......@@ -1256,7 +1261,7 @@ class TestEdgeRegionAdded(TestEdgeRegion):
]
def test(self):
super(TestEdgeRegionAdded, self).test()
super().test()
self.__class__.instance_parameter_dict['region-dict']['Region Four'] = {
'sla-computer_guid': 'local',
'state': 'started',
......@@ -1408,3 +1413,320 @@ URL =
'status-code': '200',
'url': 'https://www.all.org/'}""" % (
self.surykatka_dict['edge4'][2]['json-file'],))
class TestEdgeBasic(EdgeMixin, SlapOSInstanceTestCase):
surykatka_dict = {}
def assertConnectionParameterDict(self):
connection_parameter_dict = self.requestDefaultInstance(
).getConnectionParameterDict()
# tested elsewhere
connection_parameter_dict.pop('monitor-setup-url', None)
# comes from instance-monitor.cfg.jinja2, not needed here
connection_parameter_dict.pop('server_log_url', None)
self.assertEqual(
self.expected_connection_parameter_dict,
connection_parameter_dict
)
def assertHttpQueryPromiseContent(
self, instance_reference, name, url, content):
hashed = 'http-query-%s-%s.py' % (
hashlib.md5((name).encode('utf-8')).hexdigest(),
hashlib.md5((url).encode('utf-8')).hexdigest(),
)
self.assertPromiseContent(instance_reference, hashed, content)
def updateSurykatkaDict(self):
for instance_reference in self.surykatka_dict:
for class_ in self.surykatka_dict[instance_reference]:
update_dict = {}
update_dict['ini-file'] = os.path.join(
self.slap.instance_directory, instance_reference, 'etc',
'surykatka-%s.ini' % (class_,))
update_dict['json-file'] = os.path.join(
self.slap.instance_directory, instance_reference, 'srv',
'surykatka-%s.json' % (class_,))
update_dict['status-json'] = os.path.join(
self.slap.instance_directory, instance_reference, 'bin',
'surykatka-status-json-%s' % (class_,))
update_dict['bot-promise'] = 'surykatka-bot-%s.py' % (class_,)
update_dict['status-cron'] = os.path.join(
self.slap.instance_directory, instance_reference, 'etc',
'cron.d', 'surykatka-status-%s' % (class_,))
update_dict['db_file'] = os.path.join(
self.slap.instance_directory, instance_reference, 'srv',
'surykatka-%s.db' % (class_,))
self.surykatka_dict[instance_reference][class_].update(update_dict)
@classmethod
def getInstanceParameterDict(cls):
return {'_': json.dumps({
'nameserver-list': ['127.0.1.1', '127.0.1.2'],
'check-frontend-ip-list': ['127.0.0.1', '127.0.0.2'],
"check-maximum-elapsed-time": 5,
"check-certificate-expiration-days": 7,
"check-status-code": 201,
"failure-amount": 1,
"check-dict": {
"path-check": {
"url-list": [
"https://path.example.com/path",
]
},
"domain-check": {
"url-list": [
"domain.example.com",
]
},
"frontend-check": {
"url-list": [
"https://frontend.example.com",
],
"check-frontend-ip-list": ['127.0.0.3'],
},
"frontend-empty-check": {
"url-list": [
"https://frontendempty.example.com",
],
"check-frontend-ip-list": [],
},
"status-check": {
"url-list": [
"https://status.example.com",
],
"check-status-code": 202,
},
"certificate-check": {
"url-list": [
"https://certificate.example.com",
],
"check-certificate-expiration-days": 11,
},
"time-check": {
"url-list": [
"https://time.example.com",
],
"check-maximum-elapsed-time": 11,
},
"failure-check": {
"url-list": [
"https://failure.example.com",
],
"failure-amount": 3,
},
"header-check": {
"url-list": [
"https://header.example.com",
],
'check-http-header-dict': {"A": "AAA"},
},
}
})}
surykatka_dict = {
'edge0': {
5: {'expected_ini': """[SURYKATKA]
INTERVAL = 120
TIMEOUT = 7
SQLITE = %(db_file)s
NAMESERVER =
127.0.1.1
127.0.1.2
URL =
http://domain.example.com
https://certificate.example.com
https://domain.example.com
https://failure.example.com
https://frontend.example.com
https://frontendempty.example.com
https://header.example.com
https://path.example.com/path
https://status.example.com
"""},
11: {'expected_ini': """[SURYKATKA]
INTERVAL = 120
TIMEOUT = 13
SQLITE = %(db_file)s
NAMESERVER =
127.0.1.1
127.0.1.2
URL =
https://time.example.com
"""},
}
}
@classmethod
def getInstanceSoftwareType(cls):
return 'edgetest-basic'
def assertSurykatkaPromises(self):
self.assertHttpQueryPromiseContent(
'edge0',
'path-check',
'https://path.example.com/path',
"""extra_config_dict = { 'certificate-expiration-days': '7',
'failure-amount': '1',
'http-header-dict': '{}',
'ip-list': '127.0.0.1 127.0.0.2',
'json-file': '%s',
'maximum-elapsed-time': '5',
'report': 'http_query',
'status-code': '201',
'url': 'https://path.example.com/path'}""" % (
self.surykatka_dict['edge0'][5]['json-file'],))
self.assertHttpQueryPromiseContent(
'edge0',
'domain-check',
'https://domain.example.com',
"""extra_config_dict = { 'certificate-expiration-days': '7',
'failure-amount': '1',
'http-header-dict': '{}',
'ip-list': '127.0.0.1 127.0.0.2',
'json-file': '%s',
'maximum-elapsed-time': '5',
'report': 'http_query',
'status-code': '201',
'url': 'https://domain.example.com'}""" % (
self.surykatka_dict['edge0'][5]['json-file'],))
self.assertHttpQueryPromiseContent(
'edge0',
'domain-check',
'http://domain.example.com',
"""extra_config_dict = { 'certificate-expiration-days': '7',
'failure-amount': '1',
'http-header-dict': '{}',
'ip-list': '127.0.0.1 127.0.0.2',
'json-file': '%s',
'maximum-elapsed-time': '5',
'report': 'http_query',
'status-code': '201',
'url': 'http://domain.example.com'}""" % (
self.surykatka_dict['edge0'][5]['json-file'],))
self.assertHttpQueryPromiseContent(
'edge0',
'frontend-check',
'https://frontend.example.com',
"""extra_config_dict = { 'certificate-expiration-days': '7',
'failure-amount': '1',
'http-header-dict': '{}',
'ip-list': '127.0.0.3',
'json-file': '%s',
'maximum-elapsed-time': '5',
'report': 'http_query',
'status-code': '201',
'url': 'https://frontend.example.com'}""" % (
self.surykatka_dict['edge0'][5]['json-file'],))
self.assertHttpQueryPromiseContent(
'edge0',
'frontend-empty-check',
'https://frontendempty.example.com',
"""extra_config_dict = { 'certificate-expiration-days': '7',
'failure-amount': '1',
'http-header-dict': '{}',
'ip-list': '',
'json-file': '%s',
'maximum-elapsed-time': '5',
'report': 'http_query',
'status-code': '201',
'url': 'https://frontendempty.example.com'}""" % (
self.surykatka_dict['edge0'][5]['json-file'],))
self.assertHttpQueryPromiseContent(
'edge0',
'status-check',
'https://status.example.com',
"""extra_config_dict = { 'certificate-expiration-days': '7',
'failure-amount': '1',
'http-header-dict': '{}',
'ip-list': '127.0.0.1 127.0.0.2',
'json-file': '%s',
'maximum-elapsed-time': '5',
'report': 'http_query',
'status-code': '202',
'url': 'https://status.example.com'}""" % (
self.surykatka_dict['edge0'][5]['json-file'],))
self.assertHttpQueryPromiseContent(
'edge0',
'certificate-check',
'https://certificate.example.com',
"""extra_config_dict = { 'certificate-expiration-days': '11',
'failure-amount': '1',
'http-header-dict': '{}',
'ip-list': '127.0.0.1 127.0.0.2',
'json-file': '%s',
'maximum-elapsed-time': '5',
'report': 'http_query',
'status-code': '201',
'url': 'https://certificate.example.com'}""" % (
self.surykatka_dict['edge0'][5]['json-file'],))
self.assertHttpQueryPromiseContent(
'edge0',
'time-check',
'https://time.example.com',
"""extra_config_dict = { 'certificate-expiration-days': '7',
'failure-amount': '1',
'http-header-dict': '{}',
'ip-list': '127.0.0.1 127.0.0.2',
'json-file': '%s',
'maximum-elapsed-time': '11',
'report': 'http_query',
'status-code': '201',
'url': 'https://time.example.com'}""" % (
self.surykatka_dict['edge0'][11]['json-file'],))
self.assertHttpQueryPromiseContent(
'edge0',
'failure-check',
'https://failure.example.com',
"""extra_config_dict = { 'certificate-expiration-days': '7',
'failure-amount': '3',
'http-header-dict': '{}',
'ip-list': '127.0.0.1 127.0.0.2',
'json-file': '%s',
'maximum-elapsed-time': '5',
'report': 'http_query',
'status-code': '201',
'url': 'https://failure.example.com'}""" % (
self.surykatka_dict['edge0'][5]['json-file'],))
self.assertHttpQueryPromiseContent(
'edge0',
'header-check',
'https://header.example.com',
"""extra_config_dict = { 'certificate-expiration-days': '7',
'failure-amount': '1',
'http-header-dict': '{"A": "AAA"}',
'ip-list': '127.0.0.1 127.0.0.2',
'json-file': '%s',
'maximum-elapsed-time': '5',
'report': 'http_query',
'status-code': '201',
'url': 'https://header.example.com'}""" % (
self.surykatka_dict['edge0'][5]['json-file'],))
def test(self):
# Note: Those tests do not run surykatka and do not do real checks, as
# this depends too much on the environment and is really hard to
# mock
# So it is possible that some bugs might slip under the radar
# Nevertheless the surykatka and check_surykatka_json are heavily
# unit tested, and configuration created by the profiles is asserted
# here, so it shall be enough as reasonable status
self.initiateSurykatkaRun()
self.assertSurykatkaStatusJSON()
self.assertSurykatkaIni()
self.assertSurykatkaBotPromise()
self.assertSurykatkaPromises()
self.assertSurykatkaCron()
self.assertConnectionParameterDict()
stack/slapos.cfg
View file @ 17cd7641
......@@ -139,7 +139,7 @@ zc.recipe.egg = 2.0.3+slapos003
hexagonit.recipe.download = 1.7.post4
traitlets = 4.3.3
Jinja2 = 2.11.2
Jinja2 = 2.11.3
Importing = 1.10
MarkupSafe = 1.0
PyYAML = 5.4.1
......@@ -157,7 +157,7 @@ collective.recipe.shelloutput = 0.1
collective.recipe.template = 2.0
configparser = 4.0.2:whl
contextlib2 = 0.6.0.post1
cryptography = 3.3.1
cryptography = 3.3.2
dateparser = 0.7.6
decorator = 4.3.0
funcsigs = 1.0.2
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7