Commit ac2a7e45 authored by Łukasz Nowak's avatar Łukasz Nowak

monitor: target Python 3

See merge request !712
parents d2453c40 d7e9ab1c
Pipeline #8453 failed with stage
in 0 seconds
[buildout]
extends =
buildout.hash.cfg
../python3/buildout.cfg
parts =
surykatka
[surykatka-requirements]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_update_hash_filename_}
[surykatka]
recipe = plone.recipe.command
command =
bash -c "${python3.7:executable} -m venv ${:location} && \
. ${:location}/bin/activate && \
pip install -r ${surykatka-requirements:target}"
location = ${buildout:parts-directory}/${:_buildout_section_name_}
stop-on-error = true
executable = ${:location}/bin/surykatka
# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
# The only allowed lines here are (regexes):
# - "^#" comments, copied verbatim
# - "^[" section beginings, copied verbatim
# - lines containing an "=" sign which must fit in the following categorie.
# - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
# Copied verbatim.
# - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
# by the re-generation script.
# Re-generated.
# - other lines are copied verbatim
# Substitution (${...:...}), extension ([buildout] extends = ...) and
# section inheritance (< = ...) are NOT supported (but you should really
# not need these here).
[surykatka-requirements]
_update_hash_filename_ = requirements.txt
md5sum = c4bd35bcc5c9c25efe1edff5dd022605
Click==7.0
certifi==2019.11.28
chardet==3.0.4
dnspython==1.16.0
forcediphttpsadapter==1.0.1
idna==2.8
miniupnpc==2.0.2
peewee==3.13.1
requests==2.22.0
surykatka==0.4.2
urllib3==1.25.8
......@@ -14,7 +14,7 @@
# not need these here).
[template]
filename = instance.cfg
md5sum = dc9770bacea2c504b92ad2162e58d222
md5sum = 07873511a15df74f0be761947819ac9b
[template-monitor]
_update_hash_filename_ = instance-monitor.cfg.jinja2
......@@ -26,11 +26,11 @@ md5sum = 2eb5596544d9c341acf653d4f7ce2680
[template-monitor-edgetest]
_update_hash_filename_ = instance-monitor-edgetest.cfg.jinja2
md5sum = 9e237dbdda59e788202f0da194a57d41
md5sum = a57106ee88ff3295b9ffce84105da79b
[template-monitor-edgebot]
_update_hash_filename_ = instance-monitor-edgebot.cfg.jinja2
md5sum = f28a329e830ed737d468abcb4e89e1a2
md5sum = c535f4df6388cdc3f6c1df2a91d6dd53
[network-bench-cfg]
filename = network_bench.cfg.in
......@@ -38,8 +38,8 @@ md5sum = cfcbf2002b8eff5153e2bf68ed24b720
[monitor-collect-csv-dump]
filename = script/collect_csv_dump.py
md5sum = cad2402bbd21907cfed6bc5af8c5d3ab
md5sum = d3cfa1f6760e3fa64ccd64acf213bdfb
[template-surykatka-ini]
_update_hash_filename_ = surykatka.ini.jinja2
md5sum = a2de719a5a65438c8c3ee5195442beb6
md5sum = 89545501f0e5bf11608978886429da3d
......@@ -9,7 +9,7 @@
{%- set extra_slave_instance_list = slapparameter_dict.get('extra_slave_instance_list') %}
{%- if extra_slave_instance_list %}
{#- Create slaves to process with setting up defaults #}
{%- for slave in sorted(json_module.loads(extra_slave_instance_list)) %}
{%- for slave in json_module.loads(extra_slave_instance_list) | sort(attribute='slave_title') %}
{%- if 'check-status-code' not in slave %}
{%- do slave.__setitem__('check-status-code', CONFIGURATION['check-status-code']) %}
{%- endif %}
......@@ -37,7 +37,7 @@
{%- set part_list = [] %}
{%- for class, slave_instance_list in slave_instance_dict.items() %}
{#- class is used to separate surykatka with different timeouts #}
{%- for slave in sorted(slave_instance_list) %}
{%- for slave in slave_instance_list | sort(attribute='slave_title') %}
{%- set part_id = 'http-query-' ~ slave['slave_reference'] ~ '-promise' %}
{%- do part_list.append(part_id) %}
{%- set safe_name = part_id.replace('_', '').replace('.', '-').replace(' ', '-') %}
......
......@@ -27,7 +27,7 @@ extra-context =
{%- set namebase = "edgebot" %}
{%- set authorized_slave_list = [] %}
{%- set monitor_base_url_dict = {} -%}
{%- for slave in sorted(slave_instance_list) %}
{%- for slave in slave_instance_list | sort(attribute='slave_title') %}
{%- do authorized_slave_list.append(slave) %}
{%- endfor %}
{%- set monitor_base_port = int(slap_configuration['configuration.monitor-base-port']) %}
......@@ -56,7 +56,7 @@ software-url = ${slap-connection:software-release-url}
software-type = {{edgebot_software_type}}
return = monitor-base-url
{% for section, edgebot_request in request_dict.iteritems() %}
{% for section, edgebot_request in request_dict.items() %}
[{{section}}]
<= replicate
name = {{ edgebot_request.get('name') }}
......@@ -66,11 +66,11 @@ state = {{ edgebot_request.get('state') }}
{%- set slave_configuration_dict = slapparameter_dict %}
{%- do slave_configuration_dict.update(edgebot_request.get('config')) %}
{%- do slave_configuration_dict.__setitem__(slave_list_name, json_module.dumps(authorized_slave_list)) %}
{%- for config_key, config_value in slave_configuration_dict.iteritems() %}
{%- for config_key, config_value in slave_configuration_dict.items() %}
config-{{ config_key }} = {{ dumps(config_value) }}
{% endfor -%}
{%- if edgebot_request.get('sla') %}
{%- for parameter, value in edgebot_request.get('sla').iteritems() %}
{%- for parameter, value in edgebot_request.get('sla').items() %}
sla-{{ parameter }} = {{ value }}
{%- endfor %}
{%- else %}
......
......@@ -44,17 +44,12 @@ context = import json_module json
raw template_json_edgetest_test ${json-test-template:target}
mode = 0644
[surykatka-wrapped]
recipe = slapos.cookbook:wrapper
wrapper-path = $${buildout:bin-directory}/surykatka-wrapped
command-line = ${surykatka:location}/bin/python3.7 ${surykatka:executable}
[instance-base-edgebot]
recipe = slapos.recipe.template:jinja2
template = ${template-monitor-edgebot:target}
rendered = $${buildout:directory}/template-monitor-edgebot.cfg
extensions = jinja2.ext.do
surykatka-binary = ${surykatka:executable}
surykatka-binary = ${buildout:bin-directory}/surykatka
template-surykatka-ini = ${template-surykatka-ini:target}
context = import json_module json
......@@ -64,7 +59,7 @@ context = import json_module json
key slapparameter_dict slap-configuration:configuration
key slap_software_type slap-configuration:slap-software-type
raw software_type edgebot
key surykatka_binary surykatka-wrapped:wrapper-path
key surykatka_binary :surykatka-binary
key template_surykatka_ini :template-surykatka-ini
raw buildout_bin ${buildout:bin-directory}
raw monitor_template_output ${monitor-template:output}
......
......@@ -27,6 +27,7 @@
#
##############################################################################
from __future__ import print_function
import os
import argparse
import csv
......@@ -53,15 +54,14 @@ def parseArguments():
return parser.parse_args()
def writeFile(name, folder, date_scope, rows):
if os.path.exists(
os.path.join(folder, "%s/dump_%s.csv" % (date_scope, name))):
folder = os.path.join(folder, date_scope)
f = os.path.join(folder, "dump_%s.csv" % name)
if os.path.exists(f):
# File already exists, no reason to recreate it.
return
mkdir_p(os.path.join(folder, date_scope), 0o755)
file_io = open(os.path.join(folder, "%s/dump_%s.csv" % (date_scope, name)), "w")
csv_output = csv.writer(file_io)
csv_output.writerows(rows)
file_io.close()
mkdir_p(folder, 0o755)
with open(f, "w") as file_io:
csv.writer(file_io).writerows(rows)
def dump_table_into_csv(db, folder):
db.connect()
......@@ -96,6 +96,6 @@ if __name__ == "__main__":
if not os.path.exists(parser.collector_db):
print "Collector database not found..."
print("Collector database not found...")
dump_table_into_csv(Database(parser.collector_db), parser.output_folder)
......@@ -4,9 +4,11 @@ extends =
buildout.hash.cfg
../../component/pycurl/buildout.cfg
../../component/python-cryptography/buildout.cfg
../../component/surykatka/buildout.cfg
../../stack/monitor/buildout.cfg
../../stack/slapos.cfg
../../component/python3/buildout.cfg
python = python3
parts =
slapos-cookbook
......@@ -68,6 +70,7 @@ eggs +=
${python-cryptography:egg}
slapos.toolbox
slapos.core
surykatka
scripts =
networkbench
......@@ -78,6 +81,7 @@ scripts =
monitor.genstatus
monitor.configwrite
check-computer-memory
surykatka
[monitor-eggs]
eggs +=
......@@ -88,3 +92,17 @@ eggs +=
[versions]
slapos.recipe.template = 4.4
surykatka = 0.4.2
# For surykatka 0.4.2
click = 7.0
certifi = 2019.11.28
chardet = 3.0.4
dnspython = 1.16.0
forcediphttpsadapter = 1.0.1
idna = 2.8
miniupnpc = 2.0.2
peewee = 3.13.1
requests = 2.22.0
urllib3 = 1.25.8
......@@ -10,7 +10,7 @@ NAMESERVER =
{%- endfor %}
{% endif %}
URL =
{%- for slave in sorted(slave_instance_list) %}
{%- for slave in slave_instance_list | sort(attribute='slave_title') %}
{%- if 'url' in slave %}
{{ slave['url'] }}
{%- endif -%}
......
......@@ -65,7 +65,7 @@ class ServicesTestCase(SlapOSInstanceTestCase):
self.assertIn(expected_process_name, process_names)
class MonitorTestMixin(object):
class MonitorTestMixin:
monitor_setup_url_key = 'monitor-setup-url'
def test_monitor_setup(self):
......@@ -152,8 +152,7 @@ class EdgeSlaveMixin(MonitorTestMixin):
)
def updateSurykatkaDict(self):
class_list = self.surykatka_dict.keys()
for class_ in class_list:
for class_ in self.surykatka_dict:
update_dict = {}
update_dict['ini-file'] = os.path.join(
self.bot_partition_path, 'etc', 'surykatka-%s.ini' % (class_,))
......@@ -200,7 +199,7 @@ class EdgeSlaveMixin(MonitorTestMixin):
set(
glob.glob(
os.path.join(self.bot_partition_path, 'etc', 'surykatka*.ini'))),
set([q['ini-file'] for q in self.surykatka_dict.values()])
{q['ini-file'] for q in self.surykatka_dict.values()}
)
for info_dict in self.surykatka_dict.values():
self.assertEqual(
......@@ -243,14 +242,11 @@ class EdgeSlaveMixin(MonitorTestMixin):
for info_dict in self.surykatka_dict.values():
if os.path.exists(info_dict['json-file']):
os.unlink(info_dict['json-file'])
env = os.environ.copy()
env.pop('PYTHONPATH', None)
try:
subprocess.check_call(info_dict['status-json'], shell=True, env=env)
subprocess.check_call(info_dict['status-json'])
except subprocess.CalledProcessError as e:
self.fail('%s failed with code %s and message %s' % (
info_dict['status-json'], e.returncode, e.output))
self.assertTrue(os.path.exists(info_dict['json-file']))
with open(info_dict['json-file']) as fh:
status_json = json.load(fh)
self.assertIn('bot_status', status_json)
......@@ -680,8 +676,8 @@ INTERVAL = 120
TIMEOUT = 4
SQLITE = %(db_file)s
URL =
https://www.erp5.org/
https://www.erp5.com/"""}
https://www.erp5.com/
https://www.erp5.org/"""}
}
@classmethod
......
......@@ -13,3 +13,4 @@ eggs -=
[template]
extra =
${slapos.test.monitor-setup:setup}
......@@ -217,7 +217,6 @@ extra =
${slapos.test.erp5-setup:setup}
${slapos.test.htmlvalidatorserver-setup:setup}
${slapos.test.slapos-master-setup:setup}
${slapos.test.monitor-setup:setup}
${slapos.test.plantuml-setup:setup}
${slapos.test.powerdns-setup:setup}
${slapos.test.proftpd-setup:setup}
......
......@@ -100,7 +100,7 @@ eggs =
slapos.libnetworkcache
[versions]
setuptools = 40.4.3
setuptools = 44.0.0
# Use SlapOS patched zc.buildout
zc.buildout = 2.7.1+slapos001
# Use SlapOS patched zc.recipe.egg (zc.recipe.egg 2.x is for Buildout 2)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment