monitor: target Python 3

See merge request !712

component/surykatka/buildout.cfg

-[buildout]
-extends =
-  buildout.hash.cfg
-  ../python3/buildout.cfg
-
-parts =
-  surykatka
-
-[surykatka-requirements]
-recipe = slapos.recipe.build:download 
-url = ${:_profile_base_location_}/${:_update_hash_filename_}
-
-[surykatka]
-recipe = plone.recipe.command
-command =
-  bash -c "${python3.7:executable} -m venv ${:location} && \
-    . ${:location}/bin/activate && \
-    pip install -r ${surykatka-requirements:target}"
-location = ${buildout:parts-directory}/${:_buildout_section_name_}
-stop-on-error = true
-executable = ${:location}/bin/surykatka

component/surykatka/buildout.hash.cfg

-# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
-# The only allowed lines here are (regexes):
-# - "^#" comments, copied verbatim
-# - "^[" section beginings, copied verbatim
-# - lines containing an "=" sign which must fit in the following categorie.
-#   - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
-#     Copied verbatim.
-#   - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
-#     by the re-generation script.
-#     Re-generated.
-# - other lines are copied verbatim
-# Substitution (${...:...}), extension ([buildout] extends = ...) and
-# section inheritance (< = ...) are NOT supported (but you should really
-# not need these here).
-[surykatka-requirements]
-_update_hash_filename_ = requirements.txt
-md5sum = c4bd35bcc5c9c25efe1edff5dd022605

component/surykatka/requirements.txt

-Click==7.0
-certifi==2019.11.28
-chardet==3.0.4
-dnspython==1.16.0
-forcediphttpsadapter==1.0.1
-idna==2.8
-miniupnpc==2.0.2
-peewee==3.13.1
-requests==2.22.0
-surykatka==0.4.2
-urllib3==1.25.8

software/monitor/buildout.hash.cfg

@@ -14,7 +14,7 @@
 # not need these here).
 [template]
 filename = instance.cfg
-md5sum = dc9770bacea2c504b92ad2162e58d222
+md5sum = 07873511a15df74f0be761947819ac9b
 
 [template-monitor]
 _update_hash_filename_ = instance-monitor.cfg.jinja2
@@ -26,11 +26,11 @@ md5sum = 2eb5596544d9c341acf653d4f7ce2680
 
 [template-monitor-edgetest]
 _update_hash_filename_ = instance-monitor-edgetest.cfg.jinja2
-md5sum = 9e237dbdda59e788202f0da194a57d41
+md5sum = a57106ee88ff3295b9ffce84105da79b
 
 [template-monitor-edgebot]
 _update_hash_filename_ = instance-monitor-edgebot.cfg.jinja2
-md5sum = f28a329e830ed737d468abcb4e89e1a2
+md5sum = c535f4df6388cdc3f6c1df2a91d6dd53
 
 [network-bench-cfg]
 filename = network_bench.cfg.in
@@ -38,8 +38,8 @@ md5sum = cfcbf2002b8eff5153e2bf68ed24b720
 
 [monitor-collect-csv-dump]
 filename = script/collect_csv_dump.py
-md5sum = cad2402bbd21907cfed6bc5af8c5d3ab
+md5sum = d3cfa1f6760e3fa64ccd64acf213bdfb
 
 [template-surykatka-ini]
 _update_hash_filename_ = surykatka.ini.jinja2
-md5sum = a2de719a5a65438c8c3ee5195442beb6
+md5sum = 89545501f0e5bf11608978886429da3d

software/monitor/instance-monitor-edgebot.cfg.jinja2

@@ -9,7 +9,7 @@
 {%- set extra_slave_instance_list = slapparameter_dict.get('extra_slave_instance_list') %}
 {%- if extra_slave_instance_list %}
 {#- Create slaves to process with setting up defaults #}
-{%-   for slave in sorted(json_module.loads(extra_slave_instance_list)) %}
+{%-   for slave in json_module.loads(extra_slave_instance_list) | sort(attribute='slave_title') %}
 {%-   if 'check-status-code' not in slave %}
 {%-     do slave.__setitem__('check-status-code', CONFIGURATION['check-status-code']) %}
 {%-   endif %}
@@ -37,7 +37,7 @@
 {%- set part_list = [] %}
 {%- for class, slave_instance_list in slave_instance_dict.items() %}
 {#- class is used to separate surykatka with different timeouts #}
-{%-   for slave in sorted(slave_instance_list) %}
+{%-   for slave in slave_instance_list | sort(attribute='slave_title') %}
 {%-     set part_id = 'http-query-' ~ slave['slave_reference'] ~ '-promise' %}
 {%-     do part_list.append(part_id) %}
 {%-     set safe_name = part_id.replace('_', '').replace('.', '-').replace(' ', '-') %}

software/monitor/instance-monitor-edgetest.cfg.jinja2

@@ -27,7 +27,7 @@ extra-context =
 {%- set namebase = "edgebot" %}
 {%- set authorized_slave_list = [] %}
 {%- set monitor_base_url_dict = {} -%}
-{%- for slave in sorted(slave_instance_list) %}
+{%- for slave in slave_instance_list | sort(attribute='slave_title') %}
 {%-   do authorized_slave_list.append(slave) %}
 {%- endfor %}
 {%- set monitor_base_port = int(slap_configuration['configuration.monitor-base-port']) %}
@@ -56,7 +56,7 @@ software-url = ${slap-connection:software-release-url}
 software-type = {{edgebot_software_type}}
 return = monitor-base-url
 
-{% for section, edgebot_request in request_dict.iteritems() %}
+{% for section, edgebot_request in request_dict.items() %}
 [{{section}}]
 <= replicate
 name = {{ edgebot_request.get('name') }}
@@ -66,11 +66,11 @@ state = {{ edgebot_request.get('state') }}
 {%-   set slave_configuration_dict = slapparameter_dict %}
 {%-   do slave_configuration_dict.update(edgebot_request.get('config')) %}
 {%-   do slave_configuration_dict.__setitem__(slave_list_name, json_module.dumps(authorized_slave_list)) %}
-{%-   for config_key, config_value in slave_configuration_dict.iteritems() %}
+{%-   for config_key, config_value in slave_configuration_dict.items() %}
 config-{{ config_key }} = {{ dumps(config_value) }}
 {%   endfor -%}
 {%-   if edgebot_request.get('sla') %}
-{%-     for parameter, value in edgebot_request.get('sla').iteritems() %}
+{%-     for parameter, value in edgebot_request.get('sla').items() %}
 sla-{{ parameter }} = {{ value }}
 {%-     endfor %}
 {%-   else %}

software/monitor/instance.cfg

@@ -44,17 +44,12 @@ context = import json_module json
           raw template_json_edgetest_test ${json-test-template:target}
 mode = 0644
 
-[surykatka-wrapped]
-recipe = slapos.cookbook:wrapper
-wrapper-path = $${buildout:bin-directory}/surykatka-wrapped
-command-line = ${surykatka:location}/bin/python3.7 ${surykatka:executable}
-
 [instance-base-edgebot]
 recipe = slapos.recipe.template:jinja2
 template = ${template-monitor-edgebot:target}
 rendered = $${buildout:directory}/template-monitor-edgebot.cfg
 extensions = jinja2.ext.do
-surykatka-binary = ${surykatka:executable}
+surykatka-binary = ${buildout:bin-directory}/surykatka
 template-surykatka-ini = ${template-surykatka-ini:target}
 
 context = import json_module json
@@ -64,7 +59,7 @@ context = import json_module json
           key slapparameter_dict slap-configuration:configuration
           key slap_software_type slap-configuration:slap-software-type
           raw software_type edgebot
-          key surykatka_binary surykatka-wrapped:wrapper-path
+          key surykatka_binary :surykatka-binary
           key template_surykatka_ini :template-surykatka-ini
           raw buildout_bin ${buildout:bin-directory}
           raw monitor_template_output ${monitor-template:output}

software/monitor/script/collect_csv_dump.py

@@ -27,6 +27,7 @@
 #
 ##############################################################################
 
+from __future__ import print_function
 import os
 import argparse
 import csv
@@ -53,15 +54,14 @@ def parseArguments():
   return parser.parse_args()
 
 def writeFile(name, folder, date_scope, rows):
-  if os.path.exists(
-      os.path.join(folder, "%s/dump_%s.csv" % (date_scope, name))):
+  folder = os.path.join(folder, date_scope)
+  f = os.path.join(folder, "dump_%s.csv" % name)
+  if os.path.exists(f):
     # File already exists, no reason to recreate it.
     return
-  mkdir_p(os.path.join(folder, date_scope), 0o755)
-  file_io = open(os.path.join(folder, "%s/dump_%s.csv" % (date_scope, name)), "w")
-  csv_output = csv.writer(file_io)
-  csv_output.writerows(rows)
-  file_io.close()
+  mkdir_p(folder, 0o755)
+  with open(f, "w") as file_io:
+    csv.writer(file_io).writerows(rows)
 
 def dump_table_into_csv(db, folder):
     db.connect()
@@ -96,6 +96,6 @@ if __name__ == "__main__":
 
   
   if not os.path.exists(parser.collector_db):
-    print "Collector database not found..."
+    print("Collector database not found...")
 
   dump_table_into_csv(Database(parser.collector_db), parser.output_folder)

software/monitor/software.cfg

@@ -4,9 +4,11 @@ extends =
   buildout.hash.cfg
   ../../component/pycurl/buildout.cfg
   ../../component/python-cryptography/buildout.cfg
-  ../../component/surykatka/buildout.cfg
   ../../stack/monitor/buildout.cfg
   ../../stack/slapos.cfg
+  ../../component/python3/buildout.cfg
+
+python = python3
 
 parts = 
   slapos-cookbook
@@ -68,6 +70,7 @@ eggs +=
   ${python-cryptography:egg}
   slapos.toolbox
   slapos.core
+  surykatka
 
 scripts =
   networkbench
@@ -78,6 +81,7 @@ scripts =
   monitor.genstatus
   monitor.configwrite
   check-computer-memory
+  surykatka
 
 [monitor-eggs]
 eggs += 
@@ -88,3 +92,17 @@ eggs +=
 
 [versions]
 slapos.recipe.template = 4.4
+
+surykatka = 0.4.2
+
+# For surykatka 0.4.2
+click = 7.0
+certifi = 2019.11.28
+chardet = 3.0.4
+dnspython = 1.16.0
+forcediphttpsadapter = 1.0.1
+idna = 2.8
+miniupnpc = 2.0.2
+peewee = 3.13.1
+requests = 2.22.0
+urllib3 = 1.25.8

software/monitor/surykatka.ini.jinja2

@@ -10,7 +10,7 @@ NAMESERVER =
 {%-   endfor %}
 {% endif %}
 URL =
-{%- for slave in sorted(slave_instance_list) %}
+{%- for slave in slave_instance_list | sort(attribute='slave_title') %}
 {%-   if 'url' in slave %}
   {{ slave['url'] }}
 {%-   endif -%}

software/monitor/test/test.py

@@ -65,7 +65,7 @@ class ServicesTestCase(SlapOSInstanceTestCase):
       self.assertIn(expected_process_name, process_names)
 
 
-class MonitorTestMixin(object):
+class MonitorTestMixin:
   monitor_setup_url_key = 'monitor-setup-url'
 
   def test_monitor_setup(self):
@@ -152,8 +152,7 @@ class EdgeSlaveMixin(MonitorTestMixin):
     )
 
   def updateSurykatkaDict(self):
-    class_list = self.surykatka_dict.keys()
-    for class_ in class_list:
+    for class_ in self.surykatka_dict:
       update_dict = {}
       update_dict['ini-file'] = os.path.join(
         self.bot_partition_path, 'etc', 'surykatka-%s.ini' % (class_,))
@@ -200,7 +199,7 @@ class EdgeSlaveMixin(MonitorTestMixin):
       set(
         glob.glob(
           os.path.join(self.bot_partition_path, 'etc', 'surykatka*.ini'))),
-      set([q['ini-file'] for q in self.surykatka_dict.values()])
+      {q['ini-file'] for q in self.surykatka_dict.values()}
     )
     for info_dict in self.surykatka_dict.values():
       self.assertEqual(
@@ -243,14 +242,11 @@ class EdgeSlaveMixin(MonitorTestMixin):
     for info_dict in self.surykatka_dict.values():
       if os.path.exists(info_dict['json-file']):
         os.unlink(info_dict['json-file'])
-      env = os.environ.copy()
-      env.pop('PYTHONPATH', None)
       try:
-        subprocess.check_call(info_dict['status-json'], shell=True, env=env)
+        subprocess.check_call(info_dict['status-json'])
       except subprocess.CalledProcessError as e:
         self.fail('%s failed with code %s and message %s' % (
           info_dict['status-json'], e.returncode, e.output))
-      self.assertTrue(os.path.exists(info_dict['json-file']))
       with open(info_dict['json-file']) as fh:
         status_json = json.load(fh)
       self.assertIn('bot_status', status_json)
@@ -680,8 +676,8 @@ INTERVAL = 120
 TIMEOUT = 4
 SQLITE = %(db_file)s
 URL =
-  https://www.erp5.org/
-  https://www.erp5.com/"""}
+  https://www.erp5.com/
+  https://www.erp5.org/"""}
   }
 
   @classmethod

software/slapos-sr-testing/software-py3.cfg

@@ -13,3 +13,4 @@ eggs -=
 
 [template]
 extra =
+  ${slapos.test.monitor-setup:setup}

software/slapos-sr-testing/software.cfg

@@ -217,7 +217,6 @@ extra =
   ${slapos.test.erp5-setup:setup}
   ${slapos.test.htmlvalidatorserver-setup:setup}
   ${slapos.test.slapos-master-setup:setup}
-  ${slapos.test.monitor-setup:setup}
   ${slapos.test.plantuml-setup:setup}
   ${slapos.test.powerdns-setup:setup}
   ${slapos.test.proftpd-setup:setup}

stack/slapos.cfg

@@ -100,7 +100,7 @@ eggs =
   slapos.libnetworkcache
 
 [versions]
-setuptools = 40.4.3
+setuptools = 44.0.0
 # Use SlapOS patched zc.buildout
 zc.buildout = 2.7.1+slapos001
 # Use SlapOS patched zc.recipe.egg (zc.recipe.egg 2.x is for Buildout 2)