Merge remote-tracking branch 'origin/master' into zope4py2

software/backupserver/test/test.py

@@ -45,7 +45,7 @@ class TestBackupServer(InstanceTestCase):
     # Check that there is a RSS feed
     self.assertTrue('rss' in parameter_dict)
     self.assertTrue(parameter_dict['rss'].startswith(
-      'https://[%s]:9443/' % (self._ipv6_address, )
+      f'https://[{self._ipv6_address}]:9443/'
     ))
 
     result = requests.get(

software/beremiz-ide/test/test.py

@@ -24,7 +24,6 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 ##############################################################################
-from __future__ import unicode_literals
 
 import os
 import requests

software/caddy-frontend/test/setup.py

@@ -51,7 +51,6 @@ setup(name=name,
         # caucase needed to connect to the KeDiFa caucase
         'caucase',
         'cryptography',
-        'backports.lzma',
       ],
       zip_safe=True,
       test_suite='test',

software/caddy-frontend/test/test.py

@@ -51,18 +51,13 @@ import urllib.parse
 import socket
 import sys
 import logging
+import lzma
 import random
 import string
 from slapos.slap.standalone import SlapOSNodeInstanceError
 import caucase.client
 import caucase.utils
 
-
-try:
-    import lzma
-except ImportError:
-    from backports import lzma
-
 import datetime
 
 from cryptography import x509

software/cloudooo/test/setup.py

@@ -45,7 +45,6 @@ setup(name=name,
         'slapos.cookbook',
         'slapos.libnetworkcache',
         'requests',
-        'six',
         'PyPDF2',
       ],
       zip_safe=True,

software/cloudooo/test/test.py

 ##############################################################################
-# coding: utf-8
 #
 # Copyright (c) 2020 Nexedi SA and Contributors. All Rights Reserved.
 #
@@ -31,8 +30,8 @@ import csv
 import multiprocessing
 import os
 import json
-import six.moves.xmlrpc_client as xmlrpclib
-import six.moves.urllib.parse as urllib_parse
+import xmlrpc.client as xmlrpclib
+import urllib.parse as urllib_parse
 import ssl
 import base64
 import io
@@ -63,6 +62,7 @@ class CloudOooTestCase(_CloudOooTestCase):
         context=ssl_context,
         allow_none=True,
     )
+    self.addCleanup(self.server('close'))
 
 
 def normalizeFontName(font_name):
@@ -111,16 +111,16 @@ class HTMLtoPDFConversionFontTestMixin:
   def test(self):
     actual_font_mapping_mapping = {}
     for font in self.expected_font_mapping:
-      src_html = '''
+      src_html = f'''
       <style>
           p {{ font-family: "{font}"; font-size: 20pt; }}
       </style>
       <p>the quick brown fox jumps over the lazy dog.</p>
       <p>THE QUICK BROWN FOX JUMPS OVER THE LAZY DOG.</p>
-      '''.format(**locals())
+      '''
 
       pdf_data = self._convert_html_to_pdf(src_html)
-      pdf_reader = PyPDF2.PdfFileReader(io.BytesIO((pdf_data)))
+      pdf_reader = PyPDF2.PdfFileReader(io.BytesIO(pdf_data))
       self.assertEqual(
           self.pdf_producer,
           pdf_reader.getDocumentInfo()['/Producer'])
@@ -165,7 +165,7 @@ class TestWkhtmlToPDF(HTMLtoPDFConversionFontTestMixin, CloudOooTestCase):
       'Liberation Sans Narrow': 'LiberationSansNarrow',
       'Liberation Serif': 'LiberationSerif',
       'Linux LibertineG': 'LiberationSans',
-      'OpenSymbol': set(['DejaVuSans', 'OpenSymbol']),
+      'OpenSymbol': {'DejaVuSans', 'OpenSymbol'},
       'Palatino': 'LiberationSans',
       'Roboto Black': 'LiberationSans',
       'Roboto Condensed Light': 'LiberationSans',
@@ -180,9 +180,9 @@ class TestWkhtmlToPDF(HTMLtoPDFConversionFontTestMixin, CloudOooTestCase):
   }
 
   def _convert_html_to_pdf(self, src_html):
-    return base64.decodestring(
+    return base64.decodebytes(
         self.server.convertFile(
-            base64.encodestring(src_html.encode()).decode(),
+            base64.encodebytes(src_html.encode()).decode(),
             'html',
             'pdf',
             False,
@@ -238,9 +238,9 @@ class TestLibreoffice(HTMLtoPDFConversionFontTestMixin, CloudOooTestCase):
   }
 
   def _convert_html_to_pdf(self, src_html):
-    return base64.decodestring(
+    return base64.decodebytes(
         self.server.convertFile(
-            base64.encodestring(src_html.encode()).decode(),
+            base64.encodebytes(src_html.encode()).decode(),
             'html',
             'pdf',
         ).encode())
@@ -251,10 +251,10 @@ class TestLibreOfficeTextConversion(CloudOooTestCase):
 
   def test_html_to_text(self):
     self.assertEqual(
-        base64.decodestring(
+        base64.decodebytes(
             self.server.convertFile(
-                base64.encodestring(
-                    u'<html>héhé</html>'.encode('utf-8')).decode(),
+                base64.encodebytes(
+                    '<html>héhé</html>'.encode()).decode(),
                 'html',
                 'txt',
             ).encode()),
@@ -274,19 +274,18 @@ class TestLibreOfficeCluster(CloudOooTestCase):
     global _convert_html_to_text
 
     def _convert_html_to_text(src_html):
-      return base64.decodestring(
+      return base64.decodebytes(
           self.server.convertFile(
-              base64.encodestring(src_html.encode()).decode(),
+              base64.encodebytes(src_html.encode()).decode(),
               'html',
               'txt',
           ).encode())
 
     pool = multiprocessing.Pool(5)
-    # TODO py3: use with pool
-    converted = pool.map(_convert_html_to_text,
-                         ['<html><body>hello</body></html>'] * 100)
-    pool.terminate()
-    pool.join()
+    with pool:
+      converted = pool.map(
+        _convert_html_to_text,
+        ['<html><body>hello</body></html>'] * 100)
 
     self.assertEqual(converted, [codecs.BOM_UTF8 + b'hello\n'] * 100)
 
@@ -294,9 +293,8 @@ class TestLibreOfficeCluster(CloudOooTestCase):
     res = requests.get(
         urllib_parse.urljoin(self.url, '/haproxy;csv'),
         verify=False,
-        stream=True,
     )
-    reader = csv.DictReader(res.raw)
+    reader = csv.DictReader(io.StringIO(res.text))
     line_list = list(reader)
     # requests have been balanced
     total_hrsp_2xx = {
@@ -309,8 +307,8 @@ class TestLibreOfficeCluster(CloudOooTestCase):
       # ideally there should be 25% of requests on each backend, because we use
       # round robin scheduling, but it can happen that some backend take longer
       # to start, so we are tolerant here and just check that each backend
-      # process at least 15% of requests. 
-      self.assertGreater(total_hrsp_2xx[backend], 15)
+      # process at least one request.
+      self.assertGreater(total_hrsp_2xx[backend], 0)
     # no errors
     total_eresp = {
         line['svname']: int(line['eresp'] or 0)

software/dream/test/test.py

 ##############################################################################
-# coding: utf-8
 #
 # Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
 #

software/erp5/test/setup.py

@@ -24,7 +24,6 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 ##############################################################################
-from __future__ import absolute_import
 from setuptools import setup, find_packages
 
 version = '0.0.1.dev0'
@@ -49,7 +48,6 @@ setup(name=name,
         'psutil',
         'requests',
         'mysqlclient',
-        'backports.lzma',
         'cryptography',
         'pexpect',
         'pyOpenSSL',

software/erp5/test/test/__init__.py

@@ -25,7 +25,6 @@
 #
 ##############################################################################
 
-from __future__ import absolute_import
 import json
 import os
 

software/erp5/test/test/test_mariadb.py

 ##############################################################################
-# coding: utf-8
 #
 # Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
 #
@@ -26,11 +25,10 @@
 #
 ##############################################################################
 
-from __future__ import absolute_import
 import os
 import json
 import glob
-import six.moves.urllib.parse
+import urllib.parse
 import socket
 import sys
 import time
@@ -39,7 +37,7 @@ import datetime
 import subprocess
 import gzip
 
-from backports import lzma
+import lzma
 import MySQLdb
 
 from slapos.testing.utils import CrontabMixin
@@ -84,7 +82,7 @@ class MariaDBTestCase(ERP5InstanceTestCase):
     # type: () -> MySQLdb.connections.Connection
     connection_parameter_dict = json.loads(
         self.computer_partition.getConnectionParameterDict()['_'])
-    db_url = six.moves.urllib.parse.urlparse(connection_parameter_dict['database-list'][0])
+    db_url = urllib.parse.urlparse(connection_parameter_dict['database-list'][0])
     self.assertEqual('mysql', db_url.scheme)
 
     self.assertTrue(db_url.path.startswith('/'))
@@ -208,7 +206,7 @@ class TestMariaDB(MariaDBTestCase):
           """
           select * from test_utf8_collation where col1 = "a"
           """)
-      self.assertEqual(((u'à',),), cnx.store_result().fetch_row(maxrows=2))
+      self.assertEqual((('à',),), cnx.store_result().fetch_row(maxrows=2))
 
 
 class TestMroonga(MariaDBTestCase):
@@ -232,7 +230,7 @@ class TestMroonga(MariaDBTestCase):
           SELECT mroonga_normalize("ABCDあぃうぇ㍑")
           """)
       # XXX this is returned as bytes by mroonga/mariadb (this might be a bug)
-      self.assertEqual(((u'abcdあぃうぇリットル'.encode('utf-8'),),),
+      self.assertEqual((('abcdあぃうぇリットル'.encode(),),),
                        cnx.store_result().fetch_row(maxrows=2))
 
       if 0:
@@ -245,7 +243,7 @@ class TestMroonga(MariaDBTestCase):
             """
             SELECT mroonga_normalize("aBｃＤあぃウェ㍑", "NormalizerMySQLUnicodeCIExceptKanaCIKanaWithVoicedSoundMark")
             """)
-        self.assertEqual(((u'ABCDあぃうぇ㍑'.encode('utf-8'),),),
+        self.assertEqual((('ABCDあぃうぇ㍑'.encode(),),),
                          cnx.store_result().fetch_row(maxrows=2))
 
   def test_mroonga_full_text_normalizer(self):
@@ -282,7 +280,7 @@ class TestMroonga(MariaDBTestCase):
            WHERE MATCH (content) AGAINST ("+ﾌﾞﾗｯｸ" IN BOOLEAN MODE)
           """)
       self.assertEqual(
-          ((datetime.date(2013, 4, 23), u'ブラックコーヒーを飲んだ。'),),
+          ((datetime.date(2013, 4, 23), 'ブラックコーヒーを飲んだ。'),),
           cnx.store_result().fetch_row(maxrows=2),
       )
 

software/erp5/test/test/test_wcfs.py

@@ -16,7 +16,6 @@
 # See COPYING file for full licensing terms.
 # See https://www.nexedi.com/licensing for rationale and options.
 
-from __future__ import absolute_import
 import json
 import os.path
 import unittest
@@ -76,5 +75,5 @@ def lookupMount(zurl):
 
 # readfile returns content of file @path.
 def readfile(path):
-  with open(path, 'r') as f:
+  with open(path) as f:
     return f.read()

software/erp5/upgrade_test/setup.py

@@ -44,7 +44,6 @@ setup(name=name,
         'slapos.core',
         'supervisor',
         'slapos.libnetworkcache',
-        'typing; python_version<"3"',
       ],
       test_suite='test',
     )

software/erp5/upgrade_test/test.py

@@ -35,8 +35,8 @@ import tempfile
 import time
 
 import requests
-import six.moves.urllib as urllib
-import six.moves.xmlrpc_client
+import urllib.parse
+import xmlrpc.client
 import urllib3
 
 from slapos.grid.utils import md5digest
@@ -83,8 +83,8 @@ class ERP5UpgradeTestCase(SlapOSInstanceTestCase):
 
   @classmethod
   def setUpClass(cls):
-    # request and instanciate with old software url
-    super(ERP5UpgradeTestCase, cls).setUpClass()
+    # request and instantiate with old software url
+    super().setUpClass()
 
     cls.setUpOldInstance()
 
@@ -155,7 +155,7 @@ class TestERP5Upgrade(ERP5UpgradeTestCase):
 
     # wait for old site creation
     cls.session.get(
-      '{zope_base_url}/person_module'.format(zope_base_url=cls.zope_base_url),
+      f'{cls.zope_base_url}/person_module',
       auth=requests.auth.HTTPBasicAuth(
         username=param_dict['inituser-login'],
         password=param_dict['inituser-password'],
@@ -171,16 +171,12 @@ class TestERP5Upgrade(ERP5UpgradeTestCase):
       ssl_context = ssl.create_default_context()
       ssl_context.check_hostname = False
       ssl_context.verify_mode = ssl.CERT_NONE
-      erp5_xmlrpc_client = six.moves.xmlrpc_client.ServerProxy(
+      erp5_xmlrpc_client = xmlrpc.client.ServerProxy(
         cls.authenticated_zope_base_url,
         context=ssl_context,
       )
-      # BBB use as a context manager only on python3
-      if sys.version_info < (3, ):
+      with erp5_xmlrpc_client:
         yield erp5_xmlrpc_client
-      else:
-        with erp5_xmlrpc_client:
-          yield erp5_xmlrpc_client
 
     def addPythonScript(script_id, params, body):
       with getXMLRPCClient() as erp5_xmlrpc_client:
@@ -188,7 +184,7 @@ class TestERP5Upgrade(ERP5UpgradeTestCase):
         try:
           custom.manage_addProduct.PythonScripts.manage_addPythonScript(
             script_id)
-        except six.moves.xmlrpc_client.ProtocolError as e:
+        except xmlrpc.client.ProtocolError as e:
           if e.errcode != 302:
             raise
         getattr(custom, script_id).ZPythonScriptHTML_editAction(

software/erp5testnode/testsuite/deploy-test/buildout.hash.cfg

@@ -18,4 +18,4 @@ md5sum = 8357771b70efd0740561b1cb46f6955e
 
 [template-deploy-test]
 filename = instance.cfg.in
-md5sum = c5db797980951b764b69aaa4b60a0380
+md5sum = 062e677990ca0cd0c0396993c58b46a2

software/erp5testnode/testsuite/deploy-test/instance-deploy-test.cfg.jinja2

@@ -22,7 +22,7 @@ parts =
 [request-kvm]
 <= slap-connection
 
-recipe = slapos.cookbook:request
+recipe = slapos.cookbook:request.serialised
 software-url = ${slap-connection:software-release-url}
 software-type = kvm
 name = test-kvm

software/erp5testnode/testsuite/deploy-test/instance.cfg.in

@@ -25,24 +25,22 @@ command =
 
 [runTestSuite]
 recipe = slapos.recipe.template:jinja2
-rendered = $${buildout:directory}/bin/$${:_buildout_section_name_}
-template = inline:
+output = $${buildout:directory}/bin/$${:_buildout_section_name_}
+inline =
   #!/bin/sh
-  export PATH=${python2.7-with-eggs:location}:$PATH
   exec ${buildout:bin-directory}/${runTestSuite_py:interpreter} ${:_profile_base_location_}/runTestSuite.py --partition_ipv4 {{ list(partition_ipv4)[0] }} --partition_path $${buildout:directory} --test_reference "{{ slapparameter_dict.get('image-to-test-url') }} {{ slapparameter_dict.get('script-to-test-url')}}" --test_location "${test-location:base}/{{ slapparameter_dict.get('test-relative-directory', 'playbook/roles/standalone-shared') }}" --python_interpreter=${buildout:bin-directory}/${runTestSuite_py:interpreter} "$@"
-mode = 0755
 context =
     key slapparameter_dict slap-configuration:configuration
     key partition_ipv4 slap-configuration:ipv4
 
 [switch_softwaretype]
 default = $${:deploy-test}
-deploy-test = dynamic-template-deploy-test:rendered
+deploy-test = dynamic-template-deploy-test:output
 
 [dynamic-template-deploy-test]
 recipe = slapos.recipe.template:jinja2
-template = ${:_profile_base_location_}/instance-deploy-test.cfg.jinja2
-rendered = $${buildout:directory}/template-deploy-test.cfg
+url = ${:_profile_base_location_}/instance-deploy-test.cfg.jinja2
+output = $${buildout:directory}/template-deploy-test.cfg
 context =
     key develop_eggs_directory buildout:develop-eggs-directory
     key eggs_directory buildout:eggs-directory
@@ -52,4 +50,3 @@ context =
     raw in_vm_test_script_md5 ${deploy-script-controller-script:md5sum}
     raw waittime ${deploy-script-controller-script:waittime}
     raw tries ${deploy-script-controller-script:tries}
-mode = 0644

software/erp5testnode/testsuite/deploy-test/software.cfg

 [buildout]
 extends =
   buildout.hash.cfg
-  https://lab.nexedi.com/nexedi/slapos/raw/1.0.264/software/kvm/software.cfg
+  https://lab.nexedi.com/nexedi/slapos/raw/1.0.271/software/kvm/software.cfg
 parts =
   python-with-eggs
   template-deploy-test
@@ -15,23 +15,8 @@ eggs =
   requests
 interpreter = ${:_buildout_section_name_}
 
-[python2.7-with-eggs]
-# create interpreters named "python" and "python2.7" so that
-# instance profile can use them in $PATH
-recipe = plone.recipe.command
-
-location = ${buildout:parts-directory}/${:_buildout_section_name_}
-stop-on-error = true
-command =
-  rm -fr ${:location} &&
-  mkdir -p ${:location} &&
-  ln -s ${buildout:bin-directory}/${python-with-eggs:interpreter} ${:location}/python &&
-  ln -s ${buildout:bin-directory}/${python-with-eggs:interpreter} ${:location}/python2.7
-update-command = ${:command}
-
 [playbook]
 recipe = plone.recipe.command
-
 stop-on-error = true
 environment = export PATH=${tar:location}/bin:${gzip:location}/bin:$PATH
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
@@ -70,7 +55,6 @@ location = ${:_profile_base_location_}/${:filename}
 recipe = slapos.recipe.template
 url = ${:_profile_base_location_}/${:filename}
 output = ${buildout:directory}/template.cfg
-mode = 0644
 
 [versions]
 erp5.util = 0.4.69

software/fluentd/software-py2.cfg

-[buildout]
-extends =
-  software.cfg
-
-[python]
-part = python2.7

software/fluentd/test/setup.py

@@ -46,7 +46,6 @@ setup(name=name,
           'slapos.libnetworkcache',
           'erp5.util',
           'supervisor',
-          'six',
       ],
       zip_safe=True,
       test_suite='test',

software/fluentd/test/test.py

@@ -34,11 +34,10 @@ import struct
 import subprocess
 import tempfile
 import time
-import six
 import sys
 
-from six.moves.SimpleHTTPServer import SimpleHTTPRequestHandler
-from six.moves.socketserver import StreamRequestHandler, TCPServer
+from http.server import SimpleHTTPRequestHandler
+from socketserver import StreamRequestHandler, TCPServer
 
 from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
 from slapos.testing.utils import findFreeTCPPort
@@ -49,8 +48,8 @@ FLUSH_INTERVAL = 1
 
 setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
     os.path.abspath(
-        os.path.join(os.path.dirname(__file__), '..',
-                     'software%s.cfg' % ("-py2" if six.PY2 else ""))))
+        os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
+
 
 class FluentdTestCase(SlapOSInstanceTestCase):
   __partition_reference__ = 'fluentd'
@@ -135,11 +134,11 @@ class WendelinTutorialTestCase(FluentdTestCase):
     return subprocess.check_output(
       [self._fluentd_bin, '-c', conf_path, '--dry-run'],
       env={'GEM_PATH': self._gem_path},
-      universal_newlines=True,
+      text=True,
     )
 
   def _test_configuration(self, expected_str):
-    self.assertRegexpMatches(
+    self.assertRegex(
       self.read_fluentd_conf(self._conf),
       expected_str,
     )
@@ -168,12 +167,12 @@ class SensorConfTestCase(WendelinTutorialTestCase):
 
   @classmethod
   def sensor_conf(cls, script_path):
-    return '''\
+    return f'''\
 <source>
   @type exec
   tag tag.name
-  command %s %s
-  run_interval %ss
+  command {sys.executable} {script_path}
+  run_interval {FLUSH_INTERVAL}s
   <parse>
     keys pressure, humidity, temperature
   </parse>
@@ -182,25 +181,25 @@ class SensorConfTestCase(WendelinTutorialTestCase):
   @type forward
   <server>
     name myserver1
-    host %s
+    host {cls._ipv6_address}
   </server>
   <buffer>
     flush_mode immediate
   </buffer>
-</match>''' % (sys.executable, script_path, FLUSH_INTERVAL, cls._ipv6_address)
+</match>'''
 
   @classmethod
   def sensor_script(cls, measurementList):
-    return '''\
-#!/usr/bin/python
+    measurement_text = "\t".join(measurementList)
+    return f'''\
+#!{sys.executable}
 # -*- coding: utf-8 -*-
 
-print("%s")''' % "\t".join(measurementList)
+print("{measurement_text}")'''
 
   def test_configuration(self):
     self._test_configuration(
-      r'adding forwarding server \'myserver1\' host="%s" port=%s weight=60'
-      % (self._ipv6_address, FLUENTD_PORT)
+      fr'adding forwarding server \'myserver1\' host="{self._ipv6_address}" port={FLUENTD_PORT} weight=60'
     )
 
   def test_send_data(self):
@@ -229,25 +228,24 @@ class GatewayConfTestCase(WendelinTutorialTestCase):
 
   @classmethod
   def gateway_conf(cls, fluentd_port, wendelin_port):
-    return '''\
+    return f'''\
 <source>
   @type forward
-  port %s
-  bind %s
+  port {fluentd_port}
+  bind {cls._ipv6_address}
 </source>
 <match tag.name>
   @type wendelin
-  streamtool_uri http://[%s]:%s/erp5/portal_ingestion_policies/default
+  streamtool_uri http://[{cls._ipv6_address}]:{wendelin_port}/erp5/portal_ingestion_policies/default
   user      foo
   password  bar
   <buffer>
     flush_mode interval
     @type file
     path fluentd-buffer-file/
-    flush_interval %ss
+    flush_interval {FLUSH_INTERVAL}s
   </buffer>
-</match>''' % (fluentd_port, cls._ipv6_address, cls._ipv6_address,
-               wendelin_port, FLUSH_INTERVAL)
+</match>'''
 
   @classmethod
   def get_configuration(cls):

software/gitlab/test/test.py

@@ -27,7 +27,7 @@
 
 import os
 import logging
-from six.moves.urllib.parse import urlparse
+from urllib.parse import urlparse
 
 import requests
 

software/kvm/test/setup.py

@@ -48,7 +48,6 @@ setup(name=name,
           'erp5.util',
           'supervisor',
           'psutil',
-          'six',
       ],
       zip_safe=True,
       test_suite='test',

software/metabase/test/setup.py

@@ -45,7 +45,6 @@ setup(name=name,
         'slapos.cookbook',
         'slapos.libnetworkcache',
         'supervisor',
-        'six',
         'requests'
         ],
       zip_safe=True,

software/metabase/test/test.py

 ##############################################################################
-# coding: utf-8
 #
 # Copyright (c) 2020 Nexedi SA and Contributors. All Rights Reserved.
 #
@@ -28,7 +27,7 @@
 
 import os
 import json
-from six.moves.urllib import parse
+from urllib import parse
 import requests
 
 from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass

software/slapos-master/test/test/test_mariadb.py

 ##############################################################################
-# coding: utf-8
 #
 # Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
 #
@@ -29,7 +28,7 @@
 import os
 import json
 import glob
-import urlparse
+import urllib.parse
 import socket
 import sys
 import time
@@ -38,7 +37,7 @@ import datetime
 import subprocess
 import gzip
 
-from backports import lzma
+import lzma
 import MySQLdb
 
 from slapos.testing.utils import CrontabMixin
@@ -80,7 +79,7 @@ class MariaDBTestCase(ERP5InstanceTestCase):
   def getDatabaseConnection(self):
     connection_parameter_dict = json.loads(
         self.computer_partition.getConnectionParameterDict()['_'])
-    db_url = urlparse.urlparse(connection_parameter_dict['database-list'][0])
+    db_url = urllib.parse.urlparse(connection_parameter_dict['database-list'][0])
     self.assertEqual('mysql', db_url.scheme)
 
     self.assertTrue(db_url.path.startswith('/'))
@@ -91,6 +90,8 @@ class MariaDBTestCase(ERP5InstanceTestCase):
         host=db_url.hostname,
         port=db_url.port,
         db=database_name,
+        use_unicode=True,
+        charset='utf8mb4'
     )
 
 
@@ -106,7 +107,7 @@ class TestCrontabs(MariaDBTestCase, CrontabMixin):
             'mariadb-full',
             '20500101000000.sql.gz',
         ),
-        'r') as dump:
+        'rt') as dump:
       self.assertIn('CREATE TABLE', dump.read())
 
   def test_logrotate_and_slow_query_digest(self):
@@ -148,7 +149,7 @@ class TestCrontabs(MariaDBTestCase, CrontabMixin):
         'slowquery_digest',
         'slowquery_digest.txt-2050-01-01.xz',
     )
-    with lzma.open(slow_query_report, 'r') as f:
+    with lzma.open(slow_query_report, 'rt') as f:
       # this is the hash for our "select sleep(n)" slow query
       self.assertIn("ID 0xF9A57DD5A41825CA", f.read())
 
@@ -170,7 +171,7 @@ class TestCrontabs(MariaDBTestCase, CrontabMixin):
       subprocess.check_output('faketime 2050-01-01 %s' % check_slow_query_promise_plugin['command'], shell=True)
     self.assertEqual(
         error_context.exception.output,
-"""\
+b"""\
 Threshold is lower than expected: 
 Expected total queries : 1.0 and current is: 2
 Expected slowest query : 0.1 and current is: 3
@@ -220,7 +221,7 @@ class TestMroonga(MariaDBTestCase):
           """
           SELECT mroonga_normalize("ABCDあぃうぇ㍑")
           """)
-      self.assertEqual((('abcdあぃうぇリットル',),),
+      self.assertEqual((('abcdあぃうぇリットル'.encode(),),),
                        cnx.store_result().fetch_row(maxrows=2))
 
       if 0:
@@ -233,7 +234,7 @@ class TestMroonga(MariaDBTestCase):
             """
             SELECT mroonga_normalize("aBｃＤあぃウェ㍑", "NormalizerMySQLUnicodeCIExceptKanaCIKanaWithVoicedSoundMark")
             """)
-        self.assertEqual((('ABCDあぃうぇ㍑',),),
+        self.assertEqual((('ABCDあぃうぇ㍑'.encode(),),),
                          cnx.store_result().fetch_row(maxrows=2))
 
   def test_mroonga_full_text_normalizer(self):
@@ -321,7 +322,7 @@ class TestMroonga(MariaDBTestCase):
     cnx = self.getDatabaseConnection()
     with contextlib.closing(cnx):
       cnx.query("SELECT mroonga_command('register token_filters/stem')")
-      self.assertEqual((('true',),), cnx.store_result().fetch_row(maxrows=2))
+      self.assertEqual(((b'true',),), cnx.store_result().fetch_row(maxrows=2))
       cnx.query(
           """
           CREATE TABLE memos (

software/slapos-master/test/test/test_wcfs.py

@@ -75,5 +75,5 @@ def lookupMount(zurl):
 
 # readfile returns content of file @path.
 def readfile(path):
-  with open(path, 'r') as f:
+  with open(path) as f:
     return f.read()

software/slapos-sr-testing/README.md

@@ -17,7 +17,6 @@ changes to the code, run tests and publish changes.
 
 ```bash
 # install this software release and request an instance
-# use software-py3.cfg instead of software.cfg if the SR you want to test is written in Python 3
 SR=https://lab.nexedi.com/nexedi/slapos/raw/1.0/software/slapos-sr-testing/software.cfg
 COMP=slaprunner
 INSTANCE_NAME=$COMP

software/slapos-sr-testing/software-py3.cfg

-[buildout]
-extends =
-  software.cfg
-
-[python]
-part = python3
-
-[python-interpreter]
-extra-eggs +=
-# plantuml 0.3.0 is only available for Python 3
-  ${slapos.test.plantuml-setup:egg}
-
-[template]
-extra =
-# The following list is for SR whose buildout runs only with Python 3.
-  caddy-frontend ${slapos.test.caddy-frontend-setup:setup}
-  caucase ${slapos.test.caucase-setup:setup}
-  erp5testnode ${slapos.test.erp5testnode-setup:setup}
-  galene ${slapos.test.galene-setup:setup}
-  grafana ${slapos.test.grafana-setup:setup}
-  headless-chromium ${slapos.test.headless-chromium-setup:setup}
-  helloworld ${slapos.test.helloworld-setup:setup}
-  html5as ${slapos.test.html5as-setup:setup}
-  html5as-base ${slapos.test.html5as-base-setup:setup}
-  htmlvalidatorserver ${slapos.test.htmlvalidatorserver-setup:setup}
-  hugo ${slapos.test.hugo-setup:setup}
-  jscrawler ${slapos.test.jscrawler-setup:setup}
-  jstestnode ${slapos.test.jstestnode-setup:setup}
-  jupyter ${slapos.test.jupyter-setup:setup}
-  kvm ${slapos.test.kvm-setup:setup}
-  matomo ${slapos.test.matomo-setup:setup}
-  metabase ${slapos.test.metabase-setup:setup}
-  monitor ${slapos.test.monitor-setup:setup}
-  nextcloud ${slapos.test.nextcloud-setup:setup}
-  nginx-push-stream ${slapos.test.nginx-push-stream-setup:setup}
-  ors-amarisoft ${slapos.test.ors-amarisoft-setup:setup}
-  plantuml ${slapos.test.plantuml-setup:setup}
-  powerdns ${slapos.test.powerdns-setup:setup}
-  proftpd ${slapos.test.proftpd-setup:setup}
-  repman ${slapos.test.repman-setup:setup}
-  restic-rest-server ${slapos.test.restic_rest_server-setup:setup}
-  seleniumserver ${slapos.test.seleniumserver-setup:setup}
-  theia ${slapos.test.theia-setup:setup}
-  turnserver ${slapos.test.turnserver-setup:setup}

software/slapos-sr-testing/software.cfg

@@ -27,9 +27,6 @@ parts =
 
 shared-part-list =
 
-[python]
-part = python2.7
-
 [setup-develop-egg]
 recipe = zc.recipe.egg:develop
 
@@ -250,8 +247,7 @@ egg = slapos.core
 setup = ${slapos.core-repository:location}
 
 [python-interpreter]
-eggs += ${:extra-eggs}
-extra-eggs =
+eggs +=
   ${lxml-python:egg}
   ${python-PyYAML:egg}
   ${slapos.core-setup:egg}
@@ -289,10 +285,11 @@ extra-eggs =
   ${slapos.test.kvm-setup:egg}
   ${slapos.test.matomo-setup:egg}
   ${slapos.test.metabase-setup:egg}
-  ${slapos.test.ors-amarisoft-setup:egg}
   ${slapos.test.monitor-setup:egg}
   ${slapos.test.nextcloud-setup:egg}
   ${slapos.test.nginx-push-stream-setup:egg}
+  ${slapos.test.ors-amarisoft-setup:egg}
+  ${slapos.test.plantuml-setup:egg}
   ${slapos.test.powerdns-setup:egg}
   ${slapos.test.proftpd-setup:egg}
   ${slapos.test.re6stnet-setup:egg}
@@ -352,27 +349,46 @@ context =
 
 tests =
   json-schemas ${slapos.cookbook-setup:setup}
-# The following list is for SR that work with either Python 2 and 3
-# (as main Python). The test egg must supply a URL which depends on
-# the version of Python that is used to run the test.
-# Due to a bug in the way promises are run, we may also list some Py3-only SR
-# here, to check there's no promise issue when slapos node runs with Python 2.
-  erp5 ${slapos.test.erp5-setup:setup}
-  fluentd ${slapos.test.fluentd-setup:setup}
-###
-  ${:extra}
 
-extra =
-# WARNING: This is for SR that only support Python 2.
-#          You should not add more lines here.
   backupserver ${slapos.test.backupserver-setup:setup}
   beremiz-ide ${slapos.test.beremiz-ide-setup:setup}
+  caddy-frontend ${slapos.test.caddy-frontend-setup:setup}
+  caucase ${slapos.test.caucase-setup:setup}
   cloudooo ${slapos.test.cloudooo-setup:setup}
   dream ${slapos.test.dream-setup:setup}
+  erp5 ${slapos.test.erp5-setup:setup}
+  erp5testnode ${slapos.test.erp5testnode-setup:setup}
+  fluentd ${slapos.test.fluentd-setup:setup}
+  galene ${slapos.test.galene-setup:setup}
   gitlab ${slapos.test.gitlab-setup:setup}
+  grafana ${slapos.test.grafana-setup:setup}
+  headless-chromium ${slapos.test.headless-chromium-setup:setup}
+  helloworld ${slapos.test.helloworld-setup:setup}
+  html5as ${slapos.test.html5as-setup:setup}
+  html5as-base ${slapos.test.html5as-base-setup:setup}
+  htmlvalidatorserver ${slapos.test.htmlvalidatorserver-setup:setup}
+  hugo ${slapos.test.hugo-setup:setup}
+  jscrawler ${slapos.test.jscrawler-setup:setup}
+  jstestnode ${slapos.test.jstestnode-setup:setup}
+  jupyter ${slapos.test.jupyter-setup:setup}
+  kvm ${slapos.test.kvm-setup:setup}
+  matomo ${slapos.test.matomo-setup:setup}
+  metabase ${slapos.test.metabase-setup:setup}
+  monitor ${slapos.test.monitor-setup:setup}
+  nextcloud ${slapos.test.nextcloud-setup:setup}
+  nginx-push-stream ${slapos.test.nginx-push-stream-setup:setup}
+  ors-amarisoft ${slapos.test.ors-amarisoft-setup:setup}
+  plantuml ${slapos.test.plantuml-setup:setup}
+  powerdns ${slapos.test.powerdns-setup:setup}
+  proftpd ${slapos.test.proftpd-setup:setup}
   re6stnet ${slapos.test.re6stnet-setup:setup}
+  repman ${slapos.test.repman-setup:setup}
+  restic-rest-server ${slapos.test.restic_rest_server-setup:setup}
+  seleniumserver ${slapos.test.seleniumserver-setup:setup}
   slapos-master ${slapos.test.slapos-master-setup:setup}
   slaprunner ${slapos.test.slaprunner-setup:setup}
+  theia ${slapos.test.theia-setup:setup}
+  turnserver ${slapos.test.turnserver-setup:setup}
   upgrade_erp5 ${slapos.test.upgrade_erp5-setup:setup}
 
 [versions]
@@ -397,11 +413,9 @@ PyPDF2 = 1.26.0+SlapOSPatched001
 # Django 1.11 is python 2 compatible
 Django = 1.11
 
-mock = 2.0.0:whl
 testfixtures = 6.11.0
 funcsigs = 1.0.2
 mysqlclient = 1.3.12
 pexpect = 4.8.0
 ptyprocess = 0.6.0
-typing = 3.7.4.3
 psycopg2 = 2.8.6

software/slaprunner/test/setup.py

@@ -48,7 +48,6 @@ setup(name=name,
           'supervisor',
           'psutil',
           'paramiko',
-          'six',
           'requests',
       ],
       zip_safe=True,

software/slaprunner/test/test.py

@@ -35,12 +35,11 @@ import subprocess
 import json
 import time
 
-from six.moves.urllib.parse import urlparse
-from six.moves.urllib.parse import quote
-from six.moves.urllib.parse import urljoin
-from six.moves.configparser import ConfigParser
+from urllib.parse import urlparse
+from urllib.parse import quote
+from urllib.parse import urljoin
+from configparser import ConfigParser
 import requests
-import six
 
 from slapos.recipe.librecipe import generateHashFromFiles
 from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
@@ -61,7 +60,7 @@ class SlaprunnerTestCase(SlapOSInstanceTestCase):
     data = {
       "path": "workspace/slapos/software/%s" % software_release,
     }
-    resp = self._postToSlaprunner(url, data) 
+    resp = self._postToSlaprunner(url, data)
     self.assertEqual(requests.codes.ok, resp.status_code)
     self.assertNotEqual(json.loads(resp.text)['code'], 0,
        'Unexpecting result in call to setCurrentProject: %s' % resp.text)
@@ -69,7 +68,7 @@ class SlaprunnerTestCase(SlapOSInstanceTestCase):
   def _buildSoftwareRelease(self):
     parameter_dict = self.computer_partition.getConnectionParameterDict()
     url = "%s/runSoftwareProfile" % parameter_dict['url']
-    resp = self._postToSlaprunner(url, {}) 
+    resp = self._postToSlaprunner(url, {})
     self.assertEqual(requests.codes.ok, resp.status_code)
     self.assertEqual(json.loads(resp.text)['result'], True,
        'Unexpecting result in call to runSoftwareProfile: %s' % resp.text)
@@ -77,7 +76,7 @@ class SlaprunnerTestCase(SlapOSInstanceTestCase):
   def _deployInstance(self):
     parameter_dict = self.computer_partition.getConnectionParameterDict()
     url = "%s/runInstanceProfile" % parameter_dict['url']
-    resp = self._postToSlaprunner(url, {}) 
+    resp = self._postToSlaprunner(url, {})
     self.assertEqual(requests.codes.ok, resp.status_code)
     self.assertEqual(json.loads(resp.text)['result'], True,
        'Unexpecting result in call to runSoftwareProfile: %s' % resp.text)
@@ -100,7 +99,7 @@ class SlaprunnerTestCase(SlapOSInstanceTestCase):
   def _isSoftwareReleaseReady(self):
     parameter_dict = self.computer_partition.getConnectionParameterDict()
     url = "%s/isSRReady" % parameter_dict['url']
-    resp = self._getFromSlaprunner(url) 
+    resp = self._getFromSlaprunner(url)
     if requests.codes.ok != resp.status_code:
       return -1
     return resp.text
@@ -125,7 +124,7 @@ class SlaprunnerTestCase(SlapOSInstanceTestCase):
       }
     while True:
       time.sleep(25)
-      resp = self._postToSlaprunner(url, data) 
+      resp = self._postToSlaprunner(url, data)
       if requests.codes.ok != resp.status_code:
         continue
       if json.loads(resp.text)["instance"]["state"] is False:
@@ -153,9 +152,9 @@ class SlaprunnerTestCase(SlapOSInstanceTestCase):
     url = "%s/getFileContent" % parameter_dict['url']
 
     data = {
-      "file": relative_path 
+      "file": relative_path
     }
-    resp = self._postToSlaprunner(url, data) 
+    resp = self._postToSlaprunner(url, data)
     self.assertEqual(requests.codes.ok, resp.status_code)
     self.assertNotEqual(json.loads(resp.text)['code'], 0,
        'Unexpecting result in call to getFileContent: %s' % resp.text)
@@ -189,9 +188,9 @@ class SlaprunnerTestCase(SlapOSInstanceTestCase):
     parameter_dict = self.computer_partition.getConnectionParameterDict()
     takeover_url = parameter_dict["takeover-%s-url" % scope]
     takeover_password = parameter_dict["takeover-%s-password" % scope]
-    
+
     resp = requests.get(
-      "%s?password=%s" % (takeover_url, takeover_password),
+      f"{takeover_url}?password={takeover_password}",
       verify=True)
     self.assertEqual(requests.codes.ok, resp.status_code)
     self.assertNotIn("Error", resp.text,
@@ -236,7 +235,7 @@ class TestWebRunnerAutorun(SlaprunnerTestCase):
   @classmethod
   def getInstanceParameterDict(cls):
     return {
-        # Auto deploy is required for the isSRReady works. 
+        # Auto deploy is required for the isSRReady works.
         'auto-deploy': 'true',
         'autorun': 'true',
         'software-root': os.path.join(cls.slap._instance_root, "..", "soft"),
@@ -363,7 +362,7 @@ class TestSSH(SlaprunnerTestCase):
     self.assertTrue(fingerprint_from_url.startswith('ssh-rsa-'), fingerprint_from_url)
     fingerprint_from_url = fingerprint_from_url[len('ssh-rsa-'):]
 
-    class KeyPolicy(object):
+    class KeyPolicy:
       """Accept server key and keep it in self.key for inspection
       """
       def missing_host_key(self, client, hostname, key):
@@ -507,7 +506,7 @@ class TestResilientInstance(SlaprunnerTestCase):
     # just check that keys returned on requested partition are for resilient
     self.assertSetEqual(
       set(self.computer_partition.getConnectionParameterDict().keys()),
-      set([
+      {
         'backend-url',
         'feed-url-runner-1-pull',
         'feed-url-runner-1-push',
@@ -520,7 +519,7 @@ class TestResilientInstance(SlaprunnerTestCase):
         'takeover-runner-1-password',
         'takeover-runner-1-url',
         'url',
-        'webdav-url']))
+        'webdav-url'})
 
 class TestResilientCustomFrontend(TestCustomFrontend):
   instance_max_retry = 20
@@ -589,7 +588,7 @@ class TestResilientDummyInstance(SlaprunnerTestCase):
 
     self._waitForCloneToBeReadyForTakeover()
     self._doTakeover()
-    self.slap.waitForInstance(20) 
+    self.slap.waitForInstance(20)
 
     previous_computer_partition = self.computer_partition
     self.computer_partition = self.requestDefaultInstance()
@@ -601,5 +600,5 @@ class TestResilientDummyInstance(SlaprunnerTestCase):
     self.assertTrue(result_after.startswith("Hello"), result_after)
 
     self.assertIn(result, result_after,
-            "%s not in %s" % (result, result_after))
+            f"{result} not in {result_after}")
 

software/theia/test/test.py

@@ -391,52 +391,49 @@ class TestTheiaEnv(TheiaTestCase):
     # Start a theia shell that inherits the environment of the theia process
     # This simulates the environment of a shell launched from the browser application
     theia_shell_process = pexpect.spawnu('{}/bin/theia-shell'.format(self.getPath()), env=theia_env)
-    try:
-      theia_shell_process.expect_exact('Standalone SlapOS for computer `slaprunner` activated')
-
-      # Launch slapos node software from theia shell
-      theia_shell_process.sendline('slapos node software')
-      theia_shell_process.expect('Installing software release %s' % self.dummy_software_path)
-      theia_shell_process.expect('Finished software releases.')
-
-      # Get the theia shell environment
-      with open(env_json_path) as f:
-        theia_shell_env = json.load(f)
-
-      # Remove the env.json file to later be sure that a new one has been generated
-      os.remove(env_json_path)
-
-      # Launch slapos node software service from the embedded supervisord.
-      # Note that we have two services, slapos-node-software and slapos-node-software-all
-      # The later uses --all which is what we want to use here, because the software
-      # is already installed and we want to install it again, this time from supervisor
-      embedded_run_path = self.getPath('srv', 'runner', 'var', 'run')
-      embedded_supervisord_socket_path = _getSupervisordSocketPath(embedded_run_path, self.logger)
-      with getSupervisorRPC(embedded_supervisord_socket_path) as embedded_supervisor:
-        previous_stop_time = embedded_supervisor.getProcessInfo('slapos-node-software-all')['stop']
-        embedded_supervisor.startProcess('slapos-node-software-all')
-        for _retries in range(20):
-          time.sleep(1)
-          if embedded_supervisor.getProcessInfo('slapos-node-software-all')['stop'] != previous_stop_time:
-            break
-        else:
-          self.fail("the supervisord service 'slapos-node-software-all' takes too long to finish")
-
-      # Get the supervisord environment
-      with open(env_json_path) as f:
-        supervisord_env = json.load(f)
-
-      # Compare relevant variables from both environments
-      self.maxDiff = None
-      self.assertEqual(theia_shell_env['PATH'].split(':'), supervisord_env['PATH'].split(':'))
-      self.assertEqual(theia_shell_env['SLAPOS_CONFIGURATION'], supervisord_env['SLAPOS_CONFIGURATION'])
-      self.assertEqual(theia_shell_env['SLAPOS_CLIENT_CONFIGURATION'], supervisord_env['SLAPOS_CLIENT_CONFIGURATION'])
-      self.assertEqual(theia_shell_env['HOME'], supervisord_env['HOME'])
-
-    finally:
-      # Cleanup the theia shell process
-      theia_shell_process.terminate()
-      theia_shell_process.wait()
+    self.addCleanup(theia_shell_process.wait)
+    self.addCleanup(theia_shell_process.terminate)
+
+    theia_shell_process.expect_exact('Standalone SlapOS for computer `slaprunner` activated')
+
+    # Launch slapos node software from theia shell
+    theia_shell_process.sendline('slapos node software')
+    theia_shell_process.expect('Installing software release %s' % self.dummy_software_path)
+    theia_shell_process.expect('Finished software releases.')
+
+    # Get the theia shell environment
+    with open(env_json_path) as f:
+      theia_shell_env = json.load(f)
+
+    # Remove the env.json file to later be sure that a new one has been generated
+    os.remove(env_json_path)
+
+    # Launch slapos node software service from the embedded supervisord.
+    # Note that we have two services, slapos-node-software and slapos-node-software-all
+    # The later uses --all which is what we want to use here, because the software
+    # is already installed and we want to install it again, this time from supervisor
+    embedded_run_path = self.getPath('srv', 'runner', 'var', 'run')
+    embedded_supervisord_socket_path = _getSupervisordSocketPath(embedded_run_path, self.logger)
+    with getSupervisorRPC(embedded_supervisord_socket_path) as embedded_supervisor:
+      previous_stop_time = embedded_supervisor.getProcessInfo('slapos-node-software-all')['stop']
+      embedded_supervisor.startProcess('slapos-node-software-all')
+      for _retries in range(20):
+        time.sleep(1)
+        if embedded_supervisor.getProcessInfo('slapos-node-software-all')['stop'] != previous_stop_time:
+          break
+      else:
+        self.fail("the supervisord service 'slapos-node-software-all' takes too long to finish")
+
+    # Get the supervisord environment
+    with open(env_json_path) as f:
+      supervisord_env = json.load(f)
+
+    # Compare relevant variables from both environments
+    self.maxDiff = None
+    self.assertEqual(theia_shell_env['PATH'].split(':'), supervisord_env['PATH'].split(':'))
+    self.assertEqual(theia_shell_env['SLAPOS_CONFIGURATION'], supervisord_env['SLAPOS_CONFIGURATION'])
+    self.assertEqual(theia_shell_env['SLAPOS_CLIENT_CONFIGURATION'], supervisord_env['SLAPOS_CLIENT_CONFIGURATION'])
+    self.assertEqual(theia_shell_env['HOME'], supervisord_env['HOME'])
 
 
 class ResilientTheiaMixin(object):