Skip to content

slapos
slapos
Commit 40cd2ab0 authored by Łukasz Nowak's avatar Łukasz Nowak
Browse files
Options

monitor: Implement check-maximum-elapsed-time 

Allows to configure maximum time for which the site is considered correct.

For each class (the maximum elapsed time value) separate surykatka process
is started, in order to not impact fast sites checking by long timeouts on
slow sites.
parent 98fac475
Show whitespace changes
Inline Side-by-side
Showing with 268 additions and 100 deletions
software/monitor/buildout.hash.cfg
View file @ 40cd2ab0
......@@ -14,7 +14,7 @@
# not need these here).
[template]
filename = instance.cfg
md5sum = d778b6f436ae6864819eb2ff2d12a86f
md5sum = 1e64cee5a2249d19f6241312f94d06b7
[template-monitor]
_update_hash_filename_ = instance-monitor.cfg.jinja2
......@@ -30,7 +30,7 @@ md5sum = 9e237dbdda59e788202f0da194a57d41
[template-monitor-edgebot]
_update_hash_filename_ = instance-monitor-edgebot.cfg.jinja2
md5sum = 8786e4245db0d27dfa4815222d970e52
md5sum = b8f3c467cfd988e29be2ba70aa7a059e
[network-bench-cfg]
filename = network_bench.cfg.in
......@@ -42,4 +42,4 @@ md5sum = cad2402bbd21907cfed6bc5af8c5d3ab
[template-surykatka-ini]
_update_hash_filename_ = surykatka.ini.jinja2
md5sum = 40870921e05d93b5843ab34abd7e3902
md5sum = a2de719a5a65438c8c3ee5195442beb6
software/monitor/instance-edgetest-input-schema.json
View file @ 40cd2ab0
......@@ -24,6 +24,12 @@
"title": "Default certificate expiration days check",
"description": "Default amount of days to consider certificate as being to-be-expired (default: 15).",
"type": "string"
},
"check-maximum-elapsed-time": {
"default": "2",
"title": "Default maximum elapsed time for a site to reply (seconds)",
"description": "Default maximum elapsed time for a site to reply to be considered good (default: 2s).",
"type": "string"
}
}
}
software/monitor/instance-edgetest-slave-input-schema.json
View file @ 40cd2ab0
......@@ -23,6 +23,12 @@
"title": "Certificate expiration days check",
"description": "Amount of days to consider certificate as being to-be-expired (default: comes from master partition).",
"type": "string"
},
"check-maximum-elapsed-time": {
"default": "Master default",
"title": "Maximum elapsed time for a site to reply (seconds)",
"description": "Maximum elapsed time for a site to reply to be considered good.(default: comes from master partition).",
"type": "string"
}
}
}
software/monitor/instance-monitor-edgebot.cfg.jinja2
View file @ 40cd2ab0
......@@ -5,7 +5,7 @@
{%- do CONFIGURATION.__setitem__(k[14:], v) %}
{%- endif %}
{%- endfor %}
{%- set slave_instance_list = [] %}
{%- set slave_instance_dict = {} %}
{%- set extra_slave_instance_list = slapparameter_dict.get('extra_slave_instance_list') %}
{%- if extra_slave_instance_list %}
{#- Create slaves to process with setting up defaults #}
......@@ -16,15 +16,24 @@
{%- if 'check-certificate-expiration-days' not in slave %}
{%- do slave.__setitem__('check-certificate-expiration-days', CONFIGURATION['check-certificate-expiration-days']) %}
{%- endif %}
{%- if 'check-maximum-elapsed-time' not in slave %}
{%- do slave.__setitem__('check-maximum-elapsed-time', CONFIGURATION['check-maximum-elapsed-time']) %}
{%- endif %}
{%- if 'check-frontend-ip' not in slave %}
{%- do slave.__setitem__('check-frontend-ip', CONFIGURATION['check-frontend-ip']) %}
{%- endif %}
{%- if 'url' in slave %}
{%- do slave_instance_list.append(slave) %}
{%- set class = slave['check-maximum-elapsed-time'] %}
{%- if class not in slave_instance_dict %}
{%- do slave_instance_dict.__setitem__(class, []) %}
{%- endif %}
{%- do slave_instance_dict[class].append(slave) %}
{%- endif %}
{%- endfor %}
{%- endif %}
{%- set part_list = [] %}
{%- for class, slave_instance_list in slave_instance_dict.items() %}
{#- class is used to separate surykatka with different timeouts #}
{%- for slave in sorted(slave_instance_list) %}
{%- set part_id = 'http-query-' ~ slave['slave_reference'] ~ '-promise' %}
{%- do part_list.append(part_id) %}
......@@ -37,75 +46,80 @@ config-report = http_query
config-url = {{ slave['url'] }}
config-status-code = {{ slave['check-status-code'] }}
config-certificate-expiration-days = {{ slave['check-certificate-expiration-days'] }}
config-maximum-elapsed-time = {{ slave['check-maximum-elapsed-time'] }}
config-ip-list = {{ slave['check-frontend-ip'] }}
config-json-file = ${surykatka-config:json}
{% endfor %}
config-json-file = ${surykatka-config-{{ class }}:json}
{%- endfor %}
[surykatka-bot-promise]
[surykatka-bot-promise-{{ class }}]
<= monitor-promise-base
module = check_surykatka_json
name = surykatka-bot-promise.py
name = surykatka-bot-promise-{{ class }}.py
config-report = bot_status
config-json-file = ${surykatka-config:json}
[buildout]
extends = {{ monitor_template_output }}
parts =
cron
cron-entry-surykatka-status
monitor-base
publish-connection-information
surykatka
surykatka-bot-promise
{% for part_id in sorted(part_list) %}
{{ part_id }}
{% endfor %}
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
config-json-file = ${surykatka-config-{{ class }}:json}
[surykatka-config]
[surykatka-config-{{ class }}]
recipe = slapos.recipe.template:jinja2
db = ${directory:srv}/surykatka.db
rendered = ${directory:etc}/surykatka.ini
db = ${directory:srv}/surykatka-{{ class }}.db
rendered = ${directory:etc}/surykatka-{{ class }}.ini
template = {{ template_surykatka_ini }}
slave_instance_list = {{ dumps(slave_instance_list) }}
nameserver = {{ dumps(CONFIGURATION['nameserver']) }}
json = ${directory:srv}/surykatka.json
json = ${directory:srv}/surykatka-{{ class }}.json
{#- timeout is just a bit bigger than class time #}
timeout = {{ int(class) + 2 }}
context =
import json_module json
key db :db
key nameserver :nameserver
key slave_instance_list :slave_instance_list
key timeout :timeout
[surykatka]
[surykatka-{{ class }}]
recipe = slapos.cookbook:wrapper
config = ${surykatka-config:rendered}
config = ${surykatka-config-{{ class }}:rendered}
command-line =
{{ surykatka_binary }} --run crawl --reload --configuration ${:config}
wrapper-path = ${monitor-directory:service}/${:_buildout_section_name_}
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
[surykatka-status-json]
[surykatka-status-json-{{ class }}]
recipe = slapos.recipe.template:jinja2
json = ${surykatka-config-{{ class }}:json}
template = inline:#!/bin/sh
if {{ surykatka_binary }} --run status --configuration ${surykatka:config} --output json > ${surykatka-config:json}.tmp ; then
mv -f ${surykatka-config:json}.tmp ${surykatka-config:json}
if {{ surykatka_binary }} --run status --configuration ${surykatka-{{ class }}:config} --output json > ${:json}.tmp ; then
mv -f ${:json}.tmp ${:json}
else
rm -f ${surykatka-config:json}.tmp
rm -f ${:json}.tmp
fi
rendered = ${monitor-directory:bin}/${:_buildout_section_name_}
mode = 0755
[cron-entry-surykatka-status]
[cron-entry-surykatka-status-{{ class }}]
recipe = slapos.cookbook:cron.d
cron-entries = ${directory:etc}/cron.d
name = surykatka-status
name = surykatka-status-{{ class }}
frequency = */2 * * * *
command = ${surykatka-status-json:rendered}
command = ${surykatka-status-json-{{ class }}:rendered}
{%- do part_list.append('surykatka-' + class) %}
{%- do part_list.append('surykatka-bot-promise-' + class) %}
{%- do part_list.append('cron-entry-surykatka-status-' + class) %}
{%- endfor %}
[buildout]
extends = {{ monitor_template_output }}
parts =
cron
monitor-base
publish-connection-information
{% for part_id in sorted(part_list) %}
{{ part_id }}
{% endfor %}
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
[publish-connection-information]
recipe = slapos.cookbook:publish.serialised
......
software/monitor/instance.cfg
View file @ 40cd2ab0
......@@ -79,6 +79,7 @@ configuration.check-status-code = 200
configuration.nameserver =
configuration.check-frontend-ip =
configuration.check-certificate-expiration-days = 15
configuration.check-maximum-elapsed-time = 2
# use monitor-base-port to have monitor listening on each instance
# on different port and also on different port than other services
# it makes it possible to instantiate it correctly on signle IP, for
......
software/monitor/surykatka.ini.jinja2
View file @ 40cd2ab0
[SURYKATKA]
INTERVAL = 120
TIMEOUT = {{ timeout }}
SQLITE = {{ db }}
{%- set nameserver_list = nameserver.split() %}
{%- if len(nameserver_list) > 0 %}
......
software/monitor/test/test.py
View file @ 40cd2ab0
......@@ -25,6 +25,7 @@
#
##############################################################################
import glob
import json
import os
import re
......@@ -150,14 +151,29 @@ class EdgeSlaveMixin(MonitorTestMixin):
shared=True
)
def updateSurykatkaDict(self):
class_list = self.surykatka_dict.keys()
for class_ in class_list:
update_dict = {}
update_dict['ini-file'] = os.path.join(
self.bot_partition_path, 'etc', 'surykatka-%s.ini' % (class_,))
update_dict['json-file'] = os.path.join(
self.bot_partition_path, 'srv', 'surykatka-%s.json' % (class_,))
update_dict['status-json'] = os.path.join(
self.bot_partition_path, 'bin', 'surykatka-status-json-%s' % (class_,))
update_dict['bot-promise'] = 'surykatka-bot-promise-%s.py' % (class_,)
update_dict['status-cron'] = os.path.join(
self.bot_partition_path, 'etc', 'cron.d', 'surykatka-status-%s' % (
class_,))
update_dict['db_file'] = os.path.join(
self.bot_partition_path, 'srv', 'surykatka-%s.db' % (class_,))
self.surykatka_dict[class_].update(update_dict)
def setUp(self):
self.bot_partition_path = os.path.join(
self.slap.instance_directory,
self.__partition_reference__ + '1')
self.surykatka_json = os.path.join(
self.bot_partition_path, 'srv', 'surykatka.json')
self.surykatka_status_json = os.path.join(
self.bot_partition_path, 'bin', 'surykatka-status-json')
self.updateSurykatkaDict()
self.monitor_configuration_list = [
{
'xmlUrl': 'https://[%s]:9700/public/feed' % (self._ipv6_address,),
......@@ -180,15 +196,17 @@ class EdgeSlaveMixin(MonitorTestMixin):
]
def assertSurykatkaIni(self):
surykatka_ini = open(
os.path.join(
self.bot_partition_path, 'etc', 'surykatka.ini')).read().strip()
expected = self.surykatka_ini % dict(
partition_path=self.bot_partition_path)
self.assertEqual(
expected.strip(),
surykatka_ini)
set(
glob.glob(
os.path.join(self.bot_partition_path, 'etc', 'surykatka*.ini'))),
set([q['ini-file'] for q in self.surykatka_dict.values()])
)
for info_dict in self.surykatka_dict.values():
self.assertEqual(
info_dict['expected_ini'].strip() % info_dict,
open(info_dict['ini-file']).read().strip()
)
def assertPromiseContent(self, name, content):
promise = open(
......@@ -199,22 +217,20 @@ class EdgeSlaveMixin(MonitorTestMixin):
self.assertTrue(content in promise)
def assertSurykatkaBotPromise(self):
for info_dict in self.surykatka_dict.values():
self.assertPromiseContent(
'surykatka-bot-promise.py',
info_dict['bot-promise'],
"'report': 'bot_status'")
self.assertPromiseContent(
'surykatka-bot-promise.py',
"'json-file': '%s'" % (self.surykatka_json,)
info_dict['bot-promise'],
"'json-file': '%s'" % (info_dict['json-file'],)
)
def assertSurykatkaCron(self):
surykatka_cron = open(
os.path.join(
self.bot_partition_path, 'etc', 'cron.d', 'surykatka-status')
).read().strip()
for info_dict in self.surykatka_dict.values():
self.assertEqual(
'*/2 * * * * %s' % (self.surykatka_status_json,),
surykatka_cron
'*/2 * * * * %s' % (info_dict['status-json'],),
open(info_dict['status-cron']).read().strip()
)
def initiateSurykatkaRun(self):
......@@ -224,13 +240,18 @@ class EdgeSlaveMixin(MonitorTestMixin):
pass
def assertSurykatkaStatusJSON(self):
if os.path.exists(self.surykatka_json):
os.unlink(self.surykatka_json)
for info_dict in self.surykatka_dict.values():
if os.path.exists(info_dict['json-file']):
os.unlink(info_dict['json-file'])
env = os.environ.copy()
env.pop('PYTHONPATH', None)
subprocess.check_call(self.surykatka_status_json, shell=True, env=env)
self.assertTrue(os.path.exists(self.surykatka_json))
with open(self.surykatka_json) as fh:
try:
subprocess.check_call(info_dict['status-json'], shell=True, env=env)
except subprocess.CalledProcessError as e:
self.fail('%s failed with code %s and message %s' % (
info_dict['status-json'], e.returncode, e.output))
self.assertTrue(os.path.exists(info_dict['json-file']))
with open(info_dict['json-file']) as fh:
status_json = json.load(fh)
self.assertIn('bot_status', status_json)
......@@ -252,12 +273,15 @@ class EdgeSlaveMixin(MonitorTestMixin):
class TestEdge(EdgeSlaveMixin, SlapOSInstanceTestCase):
surykatka_ini = """[SURYKATKA]
surykatka_dict = {
2: {'expected_ini': """[SURYKATKA]
INTERVAL = 120
SQLITE = %(partition_path)s/srv/surykatka.db
TIMEOUT = 4
SQLITE = %(db_file)s
URL =
https://www.erp5.com/
https://www.erp5.org/"""
https://www.erp5.org/"""}
}
def assertSurykatkaPromises(self):
self.assertPromiseContent(
......@@ -277,7 +301,7 @@ URL =
"'url': 'https://www.erp5.org/'")
self.assertPromiseContent(
'http-query-backend-300-promise.py',
"'json-file': '%s'" % (self.surykatka_json,)
"'json-file': '%s'" % (self.surykatka_dict[2]['json-file'],)
)
self.assertPromiseContent(
......@@ -297,7 +321,7 @@ URL =
"'url': 'https://www.erp5.com/'")
self.assertPromiseContent(
'http-query-backend-promise.py',
"'json-file': '%s'" % (self.surykatka_json,)
"'json-file': '%s'" % (self.surykatka_dict[2]['json-file'],)
)
def requestEdgetestSlaves(self):
......@@ -313,15 +337,18 @@ URL =
class TestEdgeNameserverCheckFrontendIp(
EdgeSlaveMixin, SlapOSInstanceTestCase):
surykatka_ini = """[SURYKATKA]
surykatka_dict = {
2: {'expected_ini': """[SURYKATKA]
INTERVAL = 120
SQLITE = %(partition_path)s/srv/surykatka.db
TIMEOUT = 4
SQLITE = %(db_file)s
NAMESERVER =
127.0.1.1
127.0.1.2
URL =
https://www.erp5.com/"""
https://www.erp5.com/"""}
}
@classmethod
def getInstanceParameterDict(cls):
......@@ -348,7 +375,7 @@ URL =
"'url': 'https://www.erp5.com/'")
self.assertPromiseContent(
'http-query-backend-promise.py',
"'json-file': '%s'" % (self.surykatka_json,)
"'json-file': '%s'" % (self.surykatka_dict[2]['json-file'],)
)
def requestEdgetestSlaves(self):
......@@ -359,12 +386,15 @@ URL =
class TestEdgeCheckStatusCode(EdgeSlaveMixin, SlapOSInstanceTestCase):
surykatka_ini = """[SURYKATKA]
surykatka_dict = {
2: {'expected_ini': """[SURYKATKA]
INTERVAL = 120
SQLITE = %(partition_path)s/srv/surykatka.db
TIMEOUT = 4
SQLITE = %(db_file)s
URL =
https://www.erp5.com/
https://www.erp5.org/"""
https://www.erp5.org/"""}
}
@classmethod
def getInstanceParameterDict(cls):
......@@ -390,7 +420,7 @@ URL =
"'url': 'https://www.erp5.org/'")
self.assertPromiseContent(
'http-query-backend-501-promise.py',
"'json-file': '%s'" % (self.surykatka_json,)
"'json-file': '%s'" % (self.surykatka_dict[2]['json-file'],)
)
self.assertPromiseContent(
......@@ -410,7 +440,7 @@ URL =
"'url': 'https://www.erp5.com/'")
self.assertPromiseContent(
'http-query-backend-promise.py',
"'json-file': '%s'" % (self.surykatka_json,)
"'json-file': '%s'" % (self.surykatka_dict[2]['json-file'],)
)
def requestEdgetestSlaves(self):
......@@ -426,12 +456,15 @@ URL =
class TestEdgeCheckCertificateExpirationDays(
EdgeSlaveMixin, SlapOSInstanceTestCase):
surykatka_ini = """[SURYKATKA]
surykatka_dict = {
2: {'expected_ini': """[SURYKATKA]
INTERVAL = 120
SQLITE = %(partition_path)s/srv/surykatka.db
TIMEOUT = 4
SQLITE = %(db_file)s
URL =
https://www.erp5.com/
https://www.erp5.org/"""
https://www.erp5.org/"""}
}
@classmethod
def getInstanceParameterDict(cls):
......@@ -457,7 +490,7 @@ URL =
"'url': 'https://www.erp5.org/'")
self.assertPromiseContent(
'http-query-backend-20-promise.py',
"'json-file': '%s'" % (self.surykatka_json,)
"'json-file': '%s'" % (self.surykatka_dict[2]['json-file'],)
)
self.assertPromiseContent(
......@@ -477,7 +510,7 @@ URL =
"'url': 'https://www.erp5.com/'")
self.assertPromiseContent(
'http-query-backend-promise.py',
"'json-file': '%s'" % (self.surykatka_json,)
"'json-file': '%s'" % (self.surykatka_dict[2]['json-file'],)
)
def requestEdgetestSlaves(self):
......@@ -490,3 +523,110 @@ URL =
{'url': 'https://www.erp5.org/',
'check-certificate-expiration-days': '20'},
)
class TestEdgeCheckMaximumElapsedTime(
EdgeSlaveMixin, SlapOSInstanceTestCase):
surykatka_dict = {
5: {'expected_ini': """[SURYKATKA]
INTERVAL = 120
TIMEOUT = 7
SQLITE = %(db_file)s
URL =
https://www.erp5.com/"""},
20: {'expected_ini': """[SURYKATKA]
INTERVAL = 120
TIMEOUT = 22
SQLITE = %(db_file)s
URL =
https://www.erp5.org/"""},
1: {'expected_ini': """[SURYKATKA]
INTERVAL = 120
TIMEOUT = 3
SQLITE = %(db_file)s
URL =
https://www.erp5.net/"""}
}
@classmethod
def getInstanceParameterDict(cls):
return {
'check-maximum-elapsed-time': '5',
}
def assertSurykatkaPromises(self):
self.assertPromiseContent(
'http-query-backend-20-promise.py',
"'ip-list': ''")
self.assertPromiseContent(
'http-query-backend-20-promise.py',
"'report': 'http_query'")
self.assertPromiseContent(
'http-query-backend-20-promise.py',
"'status-code': '200'")
self.assertPromiseContent(
'http-query-backend-20-promise.py',
"'maximum-elapsed-time': '20'")
self.assertPromiseContent(
'http-query-backend-20-promise.py',
"'url': 'https://www.erp5.org/'")
self.assertPromiseContent(
'http-query-backend-20-promise.py',
"'json-file': '%s'" % (self.surykatka_dict[20]['json-file'],)
)
self.assertPromiseContent(
'http-query-backend-default-promise.py',
"'ip-list': ''")
self.assertPromiseContent(
'http-query-backend-default-promise.py',
"'report': 'http_query'")
self.assertPromiseContent(
'http-query-backend-default-promise.py',
"'status-code': '200'")
self.assertPromiseContent(
'http-query-backend-default-promise.py',
"'maximum-elapsed-time': '5'")
self.assertPromiseContent(
'http-query-backend-default-promise.py',
"'url': 'https://www.erp5.com/'")
self.assertPromiseContent(
'http-query-backend-default-promise.py',
"'json-file': '%s'" % (self.surykatka_dict[5]['json-file'],)
)
self.assertPromiseContent(
'http-query-backend-1-promise.py',
"'ip-list': ''")
self.assertPromiseContent(
'http-query-backend-1-promise.py',
"'report': 'http_query'")
self.assertPromiseContent(
'http-query-backend-1-promise.py',
"'status-code': '200'")
self.assertPromiseContent(
'http-query-backend-1-promise.py',
"'maximum-elapsed-time': '1'")
self.assertPromiseContent(
'http-query-backend-1-promise.py',
"'url': 'https://www.erp5.net/'")
self.assertPromiseContent(
'http-query-backend-1-promise.py',
"'json-file': '%s'" % (self.surykatka_dict[1]['json-file'],)
)
def requestEdgetestSlaves(self):
self.requestEdgetestSlave(
'backend-default',
{'url': 'https://www.erp5.com/'},
)
self.requestEdgetestSlave(
'backend-20',
{'url': 'https://www.erp5.org/',
'check-maximum-elapsed-time': '20'},
)
self.requestEdgetestSlave(
'backend-1',
{'url': 'https://www.erp5.net/',
'check-maximum-elapsed-time': '1'},
)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7