Use python3 only for software release tests

See merge request !1247

software/backupserver/test/test.py

@@ -45,7 +45,7 @@ class TestBackupServer(InstanceTestCase):
     # Check that there is a RSS feed
     self.assertTrue('rss' in parameter_dict)
     self.assertTrue(parameter_dict['rss'].startswith(
-      'https://[%s]:9443/' % (self._ipv6_address, )
+      f'https://[{self._ipv6_address}]:9443/'
     ))
 
     result = requests.get(

software/beremiz-ide/test/test.py

@@ -24,7 +24,6 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 ##############################################################################
-from __future__ import unicode_literals
 
 import os
 import requests

software/caddy-frontend/test/setup.py

@@ -51,7 +51,6 @@ setup(name=name,
         # caucase needed to connect to the KeDiFa caucase
         'caucase',
         'cryptography',
-        'backports.lzma',
       ],
       zip_safe=True,
       test_suite='test',

software/caddy-frontend/test/test.py

@@ -51,18 +51,13 @@ import urllib.parse
 import socket
 import sys
 import logging
+import lzma
 import random
 import string
 from slapos.slap.standalone import SlapOSNodeInstanceError
 import caucase.client
 import caucase.utils
 
-
-try:
-    import lzma
-except ImportError:
-    from backports import lzma
-
 import datetime
 
 from cryptography import x509

software/cloudooo/test/setup.py

@@ -45,7 +45,6 @@ setup(name=name,
         'slapos.cookbook',
         'slapos.libnetworkcache',
         'requests',
-        'six',
         'PyPDF2',
       ],
       zip_safe=True,

software/cloudooo/test/test.py

 ##############################################################################
-# coding: utf-8
 #
 # Copyright (c) 2020 Nexedi SA and Contributors. All Rights Reserved.
 #
@@ -31,8 +30,8 @@ import csv
 import multiprocessing
 import os
 import json
-import six.moves.xmlrpc_client as xmlrpclib
-import six.moves.urllib.parse as urllib_parse
+import xmlrpc.client as xmlrpclib
+import urllib.parse as urllib_parse
 import ssl
 import base64
 import io
@@ -63,6 +62,7 @@ class CloudOooTestCase(_CloudOooTestCase):
         context=ssl_context,
         allow_none=True,
     )
+    self.addCleanup(self.server('close'))
 
 
 def normalizeFontName(font_name):
@@ -111,16 +111,16 @@ class HTMLtoPDFConversionFontTestMixin:
   def test(self):
     actual_font_mapping_mapping = {}
     for font in self.expected_font_mapping:
-      src_html = '''
+      src_html = f'''
       <style>
           p {{ font-family: "{font}"; font-size: 20pt; }}
       </style>
       <p>the quick brown fox jumps over the lazy dog.</p>
       <p>THE QUICK BROWN FOX JUMPS OVER THE LAZY DOG.</p>
-      '''.format(**locals())
+      '''
 
       pdf_data = self._convert_html_to_pdf(src_html)
-      pdf_reader = PyPDF2.PdfFileReader(io.BytesIO((pdf_data)))
+      pdf_reader = PyPDF2.PdfFileReader(io.BytesIO(pdf_data))
       self.assertEqual(
           self.pdf_producer,
           pdf_reader.getDocumentInfo()['/Producer'])
@@ -165,7 +165,7 @@ class TestWkhtmlToPDF(HTMLtoPDFConversionFontTestMixin, CloudOooTestCase):
       'Liberation Sans Narrow': 'LiberationSansNarrow',
       'Liberation Serif': 'LiberationSerif',
       'Linux LibertineG': 'LiberationSans',
-      'OpenSymbol': set(['DejaVuSans', 'OpenSymbol']),
+      'OpenSymbol': {'DejaVuSans', 'OpenSymbol'},
       'Palatino': 'LiberationSans',
       'Roboto Black': 'LiberationSans',
       'Roboto Condensed Light': 'LiberationSans',
@@ -180,9 +180,9 @@ class TestWkhtmlToPDF(HTMLtoPDFConversionFontTestMixin, CloudOooTestCase):
   }
 
   def _convert_html_to_pdf(self, src_html):
-    return base64.decodestring(
+    return base64.decodebytes(
         self.server.convertFile(
-            base64.encodestring(src_html.encode()).decode(),
+            base64.encodebytes(src_html.encode()).decode(),
             'html',
             'pdf',
             False,
@@ -238,9 +238,9 @@ class TestLibreoffice(HTMLtoPDFConversionFontTestMixin, CloudOooTestCase):
   }
 
   def _convert_html_to_pdf(self, src_html):
-    return base64.decodestring(
+    return base64.decodebytes(
         self.server.convertFile(
-            base64.encodestring(src_html.encode()).decode(),
+            base64.encodebytes(src_html.encode()).decode(),
             'html',
             'pdf',
         ).encode())
@@ -251,10 +251,10 @@ class TestLibreOfficeTextConversion(CloudOooTestCase):
 
   def test_html_to_text(self):
     self.assertEqual(
-        base64.decodestring(
+        base64.decodebytes(
             self.server.convertFile(
-                base64.encodestring(
-                    u'<html>héhé</html>'.encode('utf-8')).decode(),
+                base64.encodebytes(
+                    '<html>héhé</html>'.encode()).decode(),
                 'html',
                 'txt',
             ).encode()),
@@ -274,19 +274,18 @@ class TestLibreOfficeCluster(CloudOooTestCase):
     global _convert_html_to_text
 
     def _convert_html_to_text(src_html):
-      return base64.decodestring(
+      return base64.decodebytes(
           self.server.convertFile(
-              base64.encodestring(src_html.encode()).decode(),
+              base64.encodebytes(src_html.encode()).decode(),
               'html',
               'txt',
           ).encode())
 
     pool = multiprocessing.Pool(5)
-    # TODO py3: use with pool
-    converted = pool.map(_convert_html_to_text,
+    with pool:
+      converted = pool.map(
+        _convert_html_to_text,
         ['<html><body>hello</body></html>'] * 100)
-    pool.terminate()
-    pool.join()
 
     self.assertEqual(converted, [codecs.BOM_UTF8 + b'hello\n'] * 100)
 
@@ -294,9 +293,8 @@ class TestLibreOfficeCluster(CloudOooTestCase):
     res = requests.get(
         urllib_parse.urljoin(self.url, '/haproxy;csv'),
         verify=False,
-        stream=True,
     )
-    reader = csv.DictReader(res.raw)
+    reader = csv.DictReader(io.StringIO(res.text))
     line_list = list(reader)
     # requests have been balanced
     total_hrsp_2xx = {
@@ -309,8 +307,8 @@ class TestLibreOfficeCluster(CloudOooTestCase):
       # ideally there should be 25% of requests on each backend, because we use
       # round robin scheduling, but it can happen that some backend take longer
       # to start, so we are tolerant here and just check that each backend
-      # process at least 15% of requests. 
-      self.assertGreater(total_hrsp_2xx[backend], 15)
+      # process at least one request.
+      self.assertGreater(total_hrsp_2xx[backend], 0)
     # no errors
     total_eresp = {
         line['svname']: int(line['eresp'] or 0)

software/dream/test/test.py

 ##############################################################################
-# coding: utf-8
 #
 # Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
 #

software/erp5/test/setup.py

@@ -24,7 +24,6 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 ##############################################################################
-from __future__ import absolute_import
 from setuptools import setup, find_packages
 
 version = '0.0.1.dev0'
@@ -49,7 +48,6 @@ setup(name=name,
         'psutil',
         'requests',
         'mysqlclient',
-        'backports.lzma',
         'cryptography',
         'pexpect',
         'pyOpenSSL',

software/erp5/test/test/__init__.py

@@ -25,7 +25,6 @@
 #
 ##############################################################################
 
-from __future__ import absolute_import
 import json
 import os
 

software/erp5/test/test/test_mariadb.py

 ##############################################################################
-# coding: utf-8
 #
 # Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
 #
@@ -26,11 +25,10 @@
 #
 ##############################################################################
 
-from __future__ import absolute_import
 import os
 import json
 import glob
-import six.moves.urllib.parse
+import urllib.parse
 import socket
 import sys
 import time
@@ -39,7 +37,7 @@ import datetime
 import subprocess
 import gzip
 
-from backports import lzma
+import lzma
 import MySQLdb
 
 from slapos.testing.utils import CrontabMixin
@@ -84,7 +82,7 @@ class MariaDBTestCase(ERP5InstanceTestCase):
     # type: () -> MySQLdb.connections.Connection
     connection_parameter_dict = json.loads(
         self.computer_partition.getConnectionParameterDict()['_'])
-    db_url = six.moves.urllib.parse.urlparse(connection_parameter_dict['database-list'][0])
+    db_url = urllib.parse.urlparse(connection_parameter_dict['database-list'][0])
     self.assertEqual('mysql', db_url.scheme)
 
     self.assertTrue(db_url.path.startswith('/'))
@@ -208,7 +206,7 @@ class TestMariaDB(MariaDBTestCase):
           """
           select * from test_utf8_collation where col1 = "a"
           """)
-      self.assertEqual(((u'à',),), cnx.store_result().fetch_row(maxrows=2))
+      self.assertEqual((('à',),), cnx.store_result().fetch_row(maxrows=2))
 
 
 class TestMroonga(MariaDBTestCase):
@@ -232,7 +230,7 @@ class TestMroonga(MariaDBTestCase):
           SELECT mroonga_normalize("ABCDあぃうぇ㍑")
           """)
       # XXX this is returned as bytes by mroonga/mariadb (this might be a bug)
-      self.assertEqual(((u'abcdあぃうぇリットル'.encode('utf-8'),),),
+      self.assertEqual((('abcdあぃうぇリットル'.encode(),),),
                        cnx.store_result().fetch_row(maxrows=2))
 
       if 0:
@@ -245,7 +243,7 @@ class TestMroonga(MariaDBTestCase):
             """
             SELECT mroonga_normalize("aBｃＤあぃウェ㍑", "NormalizerMySQLUnicodeCIExceptKanaCIKanaWithVoicedSoundMark")
             """)
-        self.assertEqual(((u'ABCDあぃうぇ㍑'.encode('utf-8'),),),
+        self.assertEqual((('ABCDあぃうぇ㍑'.encode(),),),
                          cnx.store_result().fetch_row(maxrows=2))
 
   def test_mroonga_full_text_normalizer(self):
@@ -282,7 +280,7 @@ class TestMroonga(MariaDBTestCase):
            WHERE MATCH (content) AGAINST ("+ﾌﾞﾗｯｸ" IN BOOLEAN MODE)
           """)
       self.assertEqual(
-          ((datetime.date(2013, 4, 23), u'ブラックコーヒーを飲んだ。'),),
+          ((datetime.date(2013, 4, 23), 'ブラックコーヒーを飲んだ。'),),
           cnx.store_result().fetch_row(maxrows=2),
       )
 

software/erp5/test/test/test_wcfs.py

@@ -16,7 +16,6 @@
 # See COPYING file for full licensing terms.
 # See https://www.nexedi.com/licensing for rationale and options.
 
-from __future__ import absolute_import
 import json
 import os.path
 import unittest
@@ -76,5 +75,5 @@ def lookupMount(zurl):
 
 # readfile returns content of file @path.
 def readfile(path):
-  with open(path, 'r') as f:
+  with open(path) as f:
     return f.read()

software/erp5/upgrade_test/setup.py

@@ -44,7 +44,6 @@ setup(name=name,
         'slapos.core',
         'supervisor',
         'slapos.libnetworkcache',
-        'typing; python_version<"3"',
       ],
       test_suite='test',
     )

software/erp5/upgrade_test/test.py

@@ -35,8 +35,8 @@ import tempfile
 import time
 
 import requests
-import six.moves.urllib as urllib
-import six.moves.xmlrpc_client
+import urllib.parse
+import xmlrpc.client
 import urllib3
 
 from slapos.grid.utils import md5digest
@@ -83,8 +83,8 @@ class ERP5UpgradeTestCase(SlapOSInstanceTestCase):
 
   @classmethod
   def setUpClass(cls):
-    # request and instanciate with old software url
-    super(ERP5UpgradeTestCase, cls).setUpClass()
+    # request and instantiate with old software url
+    super().setUpClass()
 
     cls.setUpOldInstance()
 
@@ -155,7 +155,7 @@ class TestERP5Upgrade(ERP5UpgradeTestCase):
 
     # wait for old site creation
     cls.session.get(
-      '{zope_base_url}/person_module'.format(zope_base_url=cls.zope_base_url),
+      f'{cls.zope_base_url}/person_module',
       auth=requests.auth.HTTPBasicAuth(
         username=param_dict['inituser-login'],
         password=param_dict['inituser-password'],
@@ -171,14 +171,10 @@ class TestERP5Upgrade(ERP5UpgradeTestCase):
       ssl_context = ssl.create_default_context()
       ssl_context.check_hostname = False
       ssl_context.verify_mode = ssl.CERT_NONE
-      erp5_xmlrpc_client = six.moves.xmlrpc_client.ServerProxy(
+      erp5_xmlrpc_client = xmlrpc.client.ServerProxy(
         cls.authenticated_zope_base_url,
         context=ssl_context,
       )
-      # BBB use as a context manager only on python3
-      if sys.version_info < (3, ):
-        yield erp5_xmlrpc_client
-      else:
       with erp5_xmlrpc_client:
         yield erp5_xmlrpc_client
 
@@ -188,7 +184,7 @@ class TestERP5Upgrade(ERP5UpgradeTestCase):
         try:
           custom.manage_addProduct.PythonScripts.manage_addPythonScript(
             script_id)
-        except six.moves.xmlrpc_client.ProtocolError as e:
+        except xmlrpc.client.ProtocolError as e:
           if e.errcode != 302:
             raise
         getattr(custom, script_id).ZPythonScriptHTML_editAction(

software/fluentd/software-py2.cfg

-[buildout]
-extends =
-  software.cfg
-
-[python]
-part = python2.7

software/fluentd/test/setup.py

@@ -46,7 +46,6 @@ setup(name=name,
           'slapos.libnetworkcache',
           'erp5.util',
           'supervisor',
-          'six',
       ],
       zip_safe=True,
       test_suite='test',

software/fluentd/test/test.py

@@ -34,11 +34,10 @@ import struct
 import subprocess
 import tempfile
 import time
-import six
 import sys
 
-from six.moves.SimpleHTTPServer import SimpleHTTPRequestHandler
-from six.moves.socketserver import StreamRequestHandler, TCPServer
+from http.server import SimpleHTTPRequestHandler
+from socketserver import StreamRequestHandler, TCPServer
 
 from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
 from slapos.testing.utils import findFreeTCPPort
@@ -49,8 +48,8 @@ FLUSH_INTERVAL = 1
 
 setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
     os.path.abspath(
-        os.path.join(os.path.dirname(__file__), '..',
-                     'software%s.cfg' % ("-py2" if six.PY2 else ""))))
+        os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
+
 
 class FluentdTestCase(SlapOSInstanceTestCase):
   __partition_reference__ = 'fluentd'
@@ -135,11 +134,11 @@ class WendelinTutorialTestCase(FluentdTestCase):
     return subprocess.check_output(
       [self._fluentd_bin, '-c', conf_path, '--dry-run'],
       env={'GEM_PATH': self._gem_path},
-      universal_newlines=True,
+      text=True,
     )
 
   def _test_configuration(self, expected_str):
-    self.assertRegexpMatches(
+    self.assertRegex(
       self.read_fluentd_conf(self._conf),
       expected_str,
     )
@@ -168,12 +167,12 @@ class SensorConfTestCase(WendelinTutorialTestCase):
 
   @classmethod
   def sensor_conf(cls, script_path):
-    return '''\
+    return f'''\
 <source>
   @type exec
   tag tag.name
-  command %s %s
-  run_interval %ss
+  command {sys.executable} {script_path}
+  run_interval {FLUSH_INTERVAL}s
   <parse>
     keys pressure, humidity, temperature
   </parse>
@@ -182,25 +181,25 @@ class SensorConfTestCase(WendelinTutorialTestCase):
   @type forward
   <server>
     name myserver1
-    host %s
+    host {cls._ipv6_address}
   </server>
   <buffer>
     flush_mode immediate
   </buffer>
-</match>''' % (sys.executable, script_path, FLUSH_INTERVAL, cls._ipv6_address)
+</match>'''
 
   @classmethod
   def sensor_script(cls, measurementList):
-    return '''\
-#!/usr/bin/python
+    measurement_text = "\t".join(measurementList)
+    return f'''\
+#!{sys.executable}
 # -*- coding: utf-8 -*-
 
-print("%s")''' % "\t".join(measurementList)
+print("{measurement_text}")'''
 
   def test_configuration(self):
     self._test_configuration(
-      r'adding forwarding server \'myserver1\' host="%s" port=%s weight=60'
-      % (self._ipv6_address, FLUENTD_PORT)
+      fr'adding forwarding server \'myserver1\' host="{self._ipv6_address}" port={FLUENTD_PORT} weight=60'
     )
 
   def test_send_data(self):
@@ -229,25 +228,24 @@ class GatewayConfTestCase(WendelinTutorialTestCase):
 
   @classmethod
   def gateway_conf(cls, fluentd_port, wendelin_port):
-    return '''\
+    return f'''\
 <source>
   @type forward
-  port %s
-  bind %s
+  port {fluentd_port}
+  bind {cls._ipv6_address}
 </source>
 <match tag.name>
   @type wendelin
-  streamtool_uri http://[%s]:%s/erp5/portal_ingestion_policies/default
+  streamtool_uri http://[{cls._ipv6_address}]:{wendelin_port}/erp5/portal_ingestion_policies/default
   user      foo
   password  bar
   <buffer>
     flush_mode interval
     @type file
     path fluentd-buffer-file/
-    flush_interval %ss
+    flush_interval {FLUSH_INTERVAL}s
   </buffer>
-</match>''' % (fluentd_port, cls._ipv6_address, cls._ipv6_address,
-               wendelin_port, FLUSH_INTERVAL)
+</match>'''
 
   @classmethod
   def get_configuration(cls):

software/gitlab/test/test.py

@@ -27,7 +27,7 @@
 
 import os
 import logging
-from six.moves.urllib.parse import urlparse
+from urllib.parse import urlparse
 
 import requests
 

software/kvm/test/setup.py

@@ -48,7 +48,6 @@ setup(name=name,
           'erp5.util',
           'supervisor',
           'psutil',
-          'six',
       ],
       zip_safe=True,
       test_suite='test',

software/metabase/test/setup.py

@@ -45,7 +45,6 @@ setup(name=name,
         'slapos.cookbook',
         'slapos.libnetworkcache',
         'supervisor',
-        'six',
         'requests'
         ],
       zip_safe=True,

software/metabase/test/test.py

 ##############################################################################
-# coding: utf-8
 #
 # Copyright (c) 2020 Nexedi SA and Contributors. All Rights Reserved.
 #
@@ -28,7 +27,7 @@
 
 import os
 import json
-from six.moves.urllib import parse
+from urllib import parse
 import requests
 
 from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass

software/slapos-master/test/test/test_erp5.py

@@ -28,11 +28,11 @@
 import os
 import json
 import glob
-import urlparse
+import urllib.parse
 import socket
 import time
 import re
-import BaseHTTPServer
+import http.server
 import multiprocessing
 import subprocess
 
@@ -44,7 +44,7 @@ from . import setUpModule
 setUpModule # pyflakes
 
 
-class TestPublishedURLIsReachableMixin(object):
+class TestPublishedURLIsReachableMixin:
   """Mixin that checks that default page of ERP5 is reachable.
   """
 
@@ -52,7 +52,7 @@ class TestPublishedURLIsReachableMixin(object):
     # We access ERP5 trough a "virtual host", which should make
     # ERP5 produce URLs using https://virtual-host-name:1234/virtual_host_root
     # as base.
-    virtual_host_url = urlparse.urljoin(
+    virtual_host_url = urllib.parse.urljoin(
         base_url,
         '/VirtualHostBase/https/virtual-host-name:1234/{}/VirtualHostRoot/_vh_virtual_host_root/'
         .format(site_id))
@@ -72,7 +72,7 @@ class TestPublishedURLIsReachableMixin(object):
                 total=60,
                 backoff_factor=.5,
                 status_forcelist=(404, 500, 503))))
-
+    with session:
       r = session.get(virtual_host_url, verify=verify, allow_redirects=False)
       self.assertEqual(r.status_code, requests.codes.found)
       # access on / are redirected to login form, with virtual host preserved
@@ -80,7 +80,7 @@ class TestPublishedURLIsReachableMixin(object):
 
       # login page can be rendered and contain the text "ERP5"
       r = session.get(
-        urlparse.urljoin(base_url, '{}/login_form'.format(site_id)),
+          urllib.parse.urljoin(base_url, f'{site_id}/login_form'),
           verify=verify,
           allow_redirects=False,
       )
@@ -134,7 +134,7 @@ class TestJupyter(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
     param_dict = self.getRootPartitionConnectionParameterDict()
 
     self.assertEqual(
-      'https://[%s]:8888/tree' % self._ipv6_address,
+      f'https://[{self._ipv6_address}]:8888/tree',
       param_dict['jupyter-url']
     )
 
@@ -172,7 +172,7 @@ class TestApacheBalancerPorts(ERP5InstanceTestCase):
     }
 
   def checkValidHTTPSURL(self, url):
-    parsed = urlparse.urlparse(url)
+    parsed = urllib.parse.urlparse(url)
     self.assertEqual(parsed.scheme, 'https')
     self.assertTrue(parsed.hostname)
     self.assertTrue(parsed.port)
@@ -182,16 +182,16 @@ class TestApacheBalancerPorts(ERP5InstanceTestCase):
     param_dict = self.getRootPartitionConnectionParameterDict()
     for family_name in ('family1', 'family2'):
       self.checkValidHTTPSURL(
-          param_dict['family-{family_name}'.format(family_name=family_name)])
+          param_dict[f'family-{family_name}'])
       self.checkValidHTTPSURL(
-          param_dict['family-{family_name}-v6'.format(family_name=family_name)])
+          param_dict[f'family-{family_name}-v6'])
 
   def test_published_test_runner_url(self):
     # each family's also a list of test test runner URLs, by default 3 per family
     param_dict = self.getRootPartitionConnectionParameterDict()
     for family_name in ('family1', 'family2'):
       family_test_runner_url_list = param_dict[
-          '{family_name}-test-runner-url-list'.format(family_name=family_name)]
+          f'{family_name}-test-runner-url-list']
       self.assertEqual(3, len(family_test_runner_url_list))
       for url in family_test_runner_url_list:
         self.checkValidHTTPSURL(url)
@@ -209,23 +209,23 @@ class TestApacheBalancerPorts(ERP5InstanceTestCase):
     # normal access on ipv4 and ipv6 and test runner access on ipv4 only
     with self.slap.instance_supervisor_rpc as supervisor:
       all_process_info = supervisor.getAllProcessInfo()
-    process_info, = [p for p in all_process_info if p['name'] == 'apache']
+    process_info, = (p for p in all_process_info if p['name'] == 'apache')
     apache_process = psutil.Process(process_info['pid'])
     self.assertEqual(
         sorted([socket.AF_INET] * 4 + [socket.AF_INET6] * 2),
-        sorted([
+        sorted(
             c.family
             for c in apache_process.connections()
             if c.status == 'LISTEN'
-        ]))
+        ))
 
   def test_haproxy_listen(self):
     # There is one haproxy per family
     with self.slap.instance_supervisor_rpc as supervisor:
       all_process_info = supervisor.getAllProcessInfo()
-    process_info, = [
+    process_info, = (
         p for p in all_process_info if p['name'].startswith('haproxy-')
-    ]
+    )
     haproxy_process = psutil.Process(process_info['pid'])
     self.assertEqual([socket.AF_INET, socket.AF_INET], [
         c.family for c in haproxy_process.connections() if c.status == 'LISTEN'
@@ -290,8 +290,8 @@ class TestZopeNodeParameterOverride(ERP5InstanceTestCase, TestPublishedURLIsReac
       zodb["pool-timeout"] = "10m"
       storage["storage"] = "root"
       storage["server"] = zeo_addr
-      with open('%s/etc/zope-%s.conf' % (partition, zope)) as f:
-        conf = map(str.strip, f.readlines())
+      with open(f'{partition}/etc/zope-{zope}.conf') as f:
+        conf = list(map(str.strip, f.readlines()))
       i = conf.index("<zodb_db root>") + 1
       conf = iter(conf[i:conf.index("</zodb_db>", i)])
       for line in conf:
@@ -300,23 +300,23 @@ class TestZopeNodeParameterOverride(ERP5InstanceTestCase, TestPublishedURLIsReac
             if line == '</zeoclient>':
               break
             checkParameter(line, storage)
-          for k, v in storage.iteritems():
+          for k, v in storage.items():
             self.assertIsNone(v, k)
           del storage
         else:
           checkParameter(line, zodb)
-      for k, v in zodb.iteritems():
+      for k, v in zodb.items():
         self.assertIsNone(v, k)
 
     partition = self.getComputerPartitionPath('zope-a')
-    for zope in xrange(3):
+    for zope in range(3):
       checkConf({
           "cache-size-bytes": "20MB",
         }, {
           "cache-size": "50MB",
         })
     partition = self.getComputerPartitionPath('zope-bb')
-    for zope in xrange(5):
+    for zope in range(5):
       checkConf({
           "cache-size-bytes": "500MB" if zope else 1<<20,
         }, {
@@ -332,19 +332,20 @@ def popenCommunicate(command_list, input_=None, **kwargs):
     popen.kill()
   if popen.returncode != 0:
     raise ValueError(
-      'Issue during calling %r, result was:\n%s' % (command_list, result))
+      f'Issue during calling {command_list!r}, result was:\n{result}')
   return result
 
 
-class TestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
+class TestHandler(http.server.BaseHTTPRequestHandler):
   def do_GET(self):
     self.send_response(200)
-
-    response = {
+    response = json.dumps(
+        {
             'Path': self.path,
-      'Incoming Headers': self.headers.dict
-    }
-    response = json.dumps(response, indent=2)
+            'Incoming Headers': {k.lower(): v for k, v in self.headers.items()},
+        },
+        indent=2,
+    ).encode('utf-8')
     self.end_headers()
     self.wfile.write(response)
 
@@ -352,7 +353,7 @@ class TestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
 class TestDeploymentScriptInstantiation(ERP5InstanceTestCase):
   """This check deployment script like instantiation
 
-  Low level assertions are done here in roder to assure that
+  Low level assertions are done here in order to assure that
   https://lab.nexedi.com/nexedi/slapos.package/blob/master/playbook/
   slapos-master-standalone.yml
   works correctly
@@ -426,8 +427,8 @@ class TestDeploymentScriptInstantiation(ERP5InstanceTestCase):
       1,
       len(backend_apache_configuration_list)
     )
-    backend_apache_configuration = open(
-      backend_apache_configuration_list[0]).read()
+    with open(backend_apache_configuration_list[0]) as f:
+      backend_apache_configuration = f.read()
     self.assertIn(
       'SSLVerifyClient require',
       backend_apache_configuration
@@ -452,7 +453,7 @@ class TestDeploymentScriptInstantiation(ERP5InstanceTestCase):
     common_name = 'TEST-SSL-AUTH'
     popenCommunicate([
       'openssl', 'req', '-utf8', '-nodes', '-config', openssl_config, '-new',
-      '-keyout', key, '-out', csr, '-days', '3650'], '%s\n' % (common_name,),
+      '-keyout', key, '-out', csr, '-days', '3650'], f'{common_name}\n'.encode(),
       stdin=subprocess.PIPE)
     popenCommunicate([
       'openssl', 'ca', '-utf8', '-days', '3650', '-batch', '-config',
@@ -464,11 +465,14 @@ class TestDeploymentScriptInstantiation(ERP5InstanceTestCase):
     ip, port = re.search(
       r'.*http:\/\/(.*):(\d*)\/.*', portal_slap_line).groups()
     port = int(port)
-    server = BaseHTTPServer.HTTPServer((ip, port), TestHandler)
+    server = http.server.HTTPServer((ip, port), TestHandler)
     server_process = multiprocessing.Process(
       target=server.serve_forever, name='HTTPServer')
     server_process.start()
-    try:
+    self.addCleanup(server_process.terminate)
+    self.addCleanup(server_process.join, 10)
+    server.socket.close()
+
     # assert that accessing the service endpoint results with certificate
     # authentication and proper information extraction
     result_json = requests.get(
@@ -482,6 +486,3 @@ class TestDeploymentScriptInstantiation(ERP5InstanceTestCase):
       '/erp5/portal_slap/',
       result_json['Path']
     )
-    finally:
-      server_process.join(10)
-      server_process.terminate()

software/slapos-master/test/test/test_mariadb.py

 ##############################################################################
-# coding: utf-8
 #
 # Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
 #
@@ -29,7 +28,7 @@
 import os
 import json
 import glob
-import urlparse
+import urllib.parse
 import socket
 import sys
 import time
@@ -38,7 +37,7 @@ import datetime
 import subprocess
 import gzip
 
-from backports import lzma
+import lzma
 import MySQLdb
 
 from slapos.testing.utils import CrontabMixin
@@ -80,7 +79,7 @@ class MariaDBTestCase(ERP5InstanceTestCase):
   def getDatabaseConnection(self):
     connection_parameter_dict = json.loads(
         self.computer_partition.getConnectionParameterDict()['_'])
-    db_url = urlparse.urlparse(connection_parameter_dict['database-list'][0])
+    db_url = urllib.parse.urlparse(connection_parameter_dict['database-list'][0])
     self.assertEqual('mysql', db_url.scheme)
 
     self.assertTrue(db_url.path.startswith('/'))
@@ -91,6 +90,8 @@ class MariaDBTestCase(ERP5InstanceTestCase):
         host=db_url.hostname,
         port=db_url.port,
         db=database_name,
+        use_unicode=True,
+        charset='utf8mb4'
     )
 
 
@@ -106,7 +107,7 @@ class TestCrontabs(MariaDBTestCase, CrontabMixin):
             'mariadb-full',
             '20500101000000.sql.gz',
         ),
-        'r') as dump:
+        'rt') as dump:
       self.assertIn('CREATE TABLE', dump.read())
 
   def test_logrotate_and_slow_query_digest(self):
@@ -148,7 +149,7 @@ class TestCrontabs(MariaDBTestCase, CrontabMixin):
         'slowquery_digest',
         'slowquery_digest.txt-2050-01-01.xz',
     )
-    with lzma.open(slow_query_report, 'r') as f:
+    with lzma.open(slow_query_report, 'rt') as f:
       # this is the hash for our "select sleep(n)" slow query
       self.assertIn("ID 0xF9A57DD5A41825CA", f.read())
 
@@ -170,7 +171,7 @@ class TestCrontabs(MariaDBTestCase, CrontabMixin):
       subprocess.check_output('faketime 2050-01-01 %s' % check_slow_query_promise_plugin['command'], shell=True)
     self.assertEqual(
         error_context.exception.output,
-"""\
+b"""\
 Threshold is lower than expected: 
 Expected total queries : 1.0 and current is: 2
 Expected slowest query : 0.1 and current is: 3
@@ -220,7 +221,7 @@ class TestMroonga(MariaDBTestCase):
           """
           SELECT mroonga_normalize("ABCDあぃうぇ㍑")
           """)
-      self.assertEqual((('abcdあぃうぇリットル',),),
+      self.assertEqual((('abcdあぃうぇリットル'.encode(),),),
                        cnx.store_result().fetch_row(maxrows=2))
 
       if 0:
@@ -233,7 +234,7 @@ class TestMroonga(MariaDBTestCase):
             """
             SELECT mroonga_normalize("aBｃＤあぃウェ㍑", "NormalizerMySQLUnicodeCIExceptKanaCIKanaWithVoicedSoundMark")
             """)
-        self.assertEqual((('ABCDあぃうぇ㍑',),),
+        self.assertEqual((('ABCDあぃうぇ㍑'.encode(),),),
                          cnx.store_result().fetch_row(maxrows=2))
 
   def test_mroonga_full_text_normalizer(self):
@@ -321,7 +322,7 @@ class TestMroonga(MariaDBTestCase):
     cnx = self.getDatabaseConnection()
     with contextlib.closing(cnx):
       cnx.query("SELECT mroonga_command('register token_filters/stem')")
-      self.assertEqual((('true',),), cnx.store_result().fetch_row(maxrows=2))
+      self.assertEqual(((b'true',),), cnx.store_result().fetch_row(maxrows=2))
       cnx.query(
           """
           CREATE TABLE memos (

software/slapos-master/test/test/test_wcfs.py

@@ -75,5 +75,5 @@ def lookupMount(zurl):
 
 # readfile returns content of file @path.
 def readfile(path):
-  with open(path, 'r') as f:
+  with open(path) as f:
     return f.read()

software/slapos-sr-testing/README.md

@@ -17,7 +17,6 @@ changes to the code, run tests and publish changes.
 
 ```bash
 # install this software release and request an instance
-# use software-py3.cfg instead of software.cfg if the SR you want to test is written in Python 3
 SR=https://lab.nexedi.com/nexedi/slapos/raw/1.0/software/slapos-sr-testing/software.cfg
 COMP=slaprunner
 INSTANCE_NAME=$COMP

software/slapos-sr-testing/software-py3.cfg

-[buildout]
-extends =
-  software.cfg
-
-[python]
-part = python3
-
-[python-interpreter]
-extra-eggs +=
-# plantuml 0.3.0 is only available for Python 3
-  ${slapos.test.plantuml-setup:egg}
-
-[template]
-extra =
-# The following list is for SR whose buildout runs only with Python 3.
-  caddy-frontend ${slapos.test.caddy-frontend-setup:setup}
-  caucase ${slapos.test.caucase-setup:setup}
-  erp5testnode ${slapos.test.erp5testnode-setup:setup}
-  galene ${slapos.test.galene-setup:setup}
-  grafana ${slapos.test.grafana-setup:setup}
-  headless-chromium ${slapos.test.headless-chromium-setup:setup}
-  helloworld ${slapos.test.helloworld-setup:setup}
-  html5as ${slapos.test.html5as-setup:setup}
-  html5as-base ${slapos.test.html5as-base-setup:setup}
-  htmlvalidatorserver ${slapos.test.htmlvalidatorserver-setup:setup}
-  hugo ${slapos.test.hugo-setup:setup}
-  jscrawler ${slapos.test.jscrawler-setup:setup}
-  jstestnode ${slapos.test.jstestnode-setup:setup}
-  jupyter ${slapos.test.jupyter-setup:setup}
-  kvm ${slapos.test.kvm-setup:setup}
-  matomo ${slapos.test.matomo-setup:setup}
-  metabase ${slapos.test.metabase-setup:setup}
-  monitor ${slapos.test.monitor-setup:setup}
-  nextcloud ${slapos.test.nextcloud-setup:setup}
-  nginx-push-stream ${slapos.test.nginx-push-stream-setup:setup}
-  ors-amarisoft ${slapos.test.ors-amarisoft-setup:setup}
-  plantuml ${slapos.test.plantuml-setup:setup}
-  powerdns ${slapos.test.powerdns-setup:setup}
-  proftpd ${slapos.test.proftpd-setup:setup}
-  repman ${slapos.test.repman-setup:setup}
-  restic-rest-server ${slapos.test.restic_rest_server-setup:setup}
-  seleniumserver ${slapos.test.seleniumserver-setup:setup}
-  theia ${slapos.test.theia-setup:setup}
-  turnserver ${slapos.test.turnserver-setup:setup}

software/slapos-sr-testing/software.cfg

@@ -27,9 +27,6 @@ parts =
 
 shared-part-list =
 
-[python]
-part = python2.7
-
 [setup-develop-egg]
 recipe = zc.recipe.egg:develop
 
@@ -250,8 +247,7 @@ egg = slapos.core
 setup = ${slapos.core-repository:location}
 
 [python-interpreter]
-eggs += ${:extra-eggs}
-extra-eggs =
+eggs +=
   ${lxml-python:egg}
   ${python-PyYAML:egg}
   ${slapos.core-setup:egg}
@@ -289,10 +285,11 @@ extra-eggs =
   ${slapos.test.kvm-setup:egg}
   ${slapos.test.matomo-setup:egg}
   ${slapos.test.metabase-setup:egg}
-  ${slapos.test.ors-amarisoft-setup:egg}
   ${slapos.test.monitor-setup:egg}
   ${slapos.test.nextcloud-setup:egg}
   ${slapos.test.nginx-push-stream-setup:egg}
+  ${slapos.test.ors-amarisoft-setup:egg}
+  ${slapos.test.plantuml-setup:egg}
   ${slapos.test.powerdns-setup:egg}
   ${slapos.test.proftpd-setup:egg}
   ${slapos.test.re6stnet-setup:egg}
@@ -352,27 +349,46 @@ context =
 
 tests =
   json-schemas ${slapos.cookbook-setup:setup}
-# The following list is for SR that work with either Python 2 and 3
-# (as main Python). The test egg must supply a URL which depends on
-# the version of Python that is used to run the test.
-# Due to a bug in the way promises are run, we may also list some Py3-only SR
-# here, to check there's no promise issue when slapos node runs with Python 2.
-  erp5 ${slapos.test.erp5-setup:setup}
-  fluentd ${slapos.test.fluentd-setup:setup}
-###
-  ${:extra}
 
-extra =
-# WARNING: This is for SR that only support Python 2.
-#          You should not add more lines here.
   backupserver ${slapos.test.backupserver-setup:setup}
   beremiz-ide ${slapos.test.beremiz-ide-setup:setup}
+  caddy-frontend ${slapos.test.caddy-frontend-setup:setup}
+  caucase ${slapos.test.caucase-setup:setup}
   cloudooo ${slapos.test.cloudooo-setup:setup}
   dream ${slapos.test.dream-setup:setup}
+  erp5 ${slapos.test.erp5-setup:setup}
+  erp5testnode ${slapos.test.erp5testnode-setup:setup}
+  fluentd ${slapos.test.fluentd-setup:setup}
+  galene ${slapos.test.galene-setup:setup}
   gitlab ${slapos.test.gitlab-setup:setup}
+  grafana ${slapos.test.grafana-setup:setup}
+  headless-chromium ${slapos.test.headless-chromium-setup:setup}
+  helloworld ${slapos.test.helloworld-setup:setup}
+  html5as ${slapos.test.html5as-setup:setup}
+  html5as-base ${slapos.test.html5as-base-setup:setup}
+  htmlvalidatorserver ${slapos.test.htmlvalidatorserver-setup:setup}
+  hugo ${slapos.test.hugo-setup:setup}
+  jscrawler ${slapos.test.jscrawler-setup:setup}
+  jstestnode ${slapos.test.jstestnode-setup:setup}
+  jupyter ${slapos.test.jupyter-setup:setup}
+  kvm ${slapos.test.kvm-setup:setup}
+  matomo ${slapos.test.matomo-setup:setup}
+  metabase ${slapos.test.metabase-setup:setup}
+  monitor ${slapos.test.monitor-setup:setup}
+  nextcloud ${slapos.test.nextcloud-setup:setup}
+  nginx-push-stream ${slapos.test.nginx-push-stream-setup:setup}
+  ors-amarisoft ${slapos.test.ors-amarisoft-setup:setup}
+  plantuml ${slapos.test.plantuml-setup:setup}
+  powerdns ${slapos.test.powerdns-setup:setup}
+  proftpd ${slapos.test.proftpd-setup:setup}
   re6stnet ${slapos.test.re6stnet-setup:setup}
+  repman ${slapos.test.repman-setup:setup}
+  restic-rest-server ${slapos.test.restic_rest_server-setup:setup}
+  seleniumserver ${slapos.test.seleniumserver-setup:setup}
   slapos-master ${slapos.test.slapos-master-setup:setup}
   slaprunner ${slapos.test.slaprunner-setup:setup}
+  theia ${slapos.test.theia-setup:setup}
+  turnserver ${slapos.test.turnserver-setup:setup}
   upgrade_erp5 ${slapos.test.upgrade_erp5-setup:setup}
 
 [versions]
@@ -397,11 +413,9 @@ PyPDF2 = 1.26.0+SlapOSPatched001
 # Django 1.11 is python 2 compatible
 Django = 1.11
 
-mock = 2.0.0:whl
 testfixtures = 6.11.0
 funcsigs = 1.0.2
 mysqlclient = 1.3.12
 pexpect = 4.8.0
 ptyprocess = 0.6.0
-typing = 3.7.4.3
 psycopg2 = 2.8.6

software/slaprunner/test/setup.py

@@ -48,7 +48,6 @@ setup(name=name,
           'supervisor',
           'psutil',
           'paramiko',
-          'six',
           'requests',
       ],
       zip_safe=True,

software/slaprunner/test/test.py

@@ -35,12 +35,11 @@ import subprocess
 import json
 import time
 
-from six.moves.urllib.parse import urlparse
-from six.moves.urllib.parse import quote
-from six.moves.urllib.parse import urljoin
-from six.moves.configparser import ConfigParser
+from urllib.parse import urlparse
+from urllib.parse import quote
+from urllib.parse import urljoin
+from configparser import ConfigParser
 import requests
-import six
 
 from slapos.recipe.librecipe import generateHashFromFiles
 from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
@@ -191,7 +190,7 @@ class SlaprunnerTestCase(SlapOSInstanceTestCase):
     takeover_password = parameter_dict["takeover-%s-password" % scope]
 
     resp = requests.get(
-      "%s?password=%s" % (takeover_url, takeover_password),
+      f"{takeover_url}?password={takeover_password}",
       verify=True)
     self.assertEqual(requests.codes.ok, resp.status_code)
     self.assertNotIn("Error", resp.text,
@@ -363,7 +362,7 @@ class TestSSH(SlaprunnerTestCase):
     self.assertTrue(fingerprint_from_url.startswith('ssh-rsa-'), fingerprint_from_url)
     fingerprint_from_url = fingerprint_from_url[len('ssh-rsa-'):]
 
-    class KeyPolicy(object):
+    class KeyPolicy:
       """Accept server key and keep it in self.key for inspection
       """
       def missing_host_key(self, client, hostname, key):
@@ -507,7 +506,7 @@ class TestResilientInstance(SlaprunnerTestCase):
     # just check that keys returned on requested partition are for resilient
     self.assertSetEqual(
       set(self.computer_partition.getConnectionParameterDict().keys()),
-      set([
+      {
         'backend-url',
         'feed-url-runner-1-pull',
         'feed-url-runner-1-push',
@@ -520,7 +519,7 @@ class TestResilientInstance(SlaprunnerTestCase):
         'takeover-runner-1-password',
         'takeover-runner-1-url',
         'url',
-        'webdav-url']))
+        'webdav-url'})
 
 class TestResilientCustomFrontend(TestCustomFrontend):
   instance_max_retry = 20
@@ -601,5 +600,5 @@ class TestResilientDummyInstance(SlaprunnerTestCase):
     self.assertTrue(result_after.startswith("Hello"), result_after)
 
     self.assertIn(result, result_after,
-            "%s not in %s" % (result, result_after))
+            f"{result} not in {result_after}")
 

software/theia/test/test.py

@@ -391,7 +391,9 @@ class TestTheiaEnv(TheiaTestCase):
     # Start a theia shell that inherits the environment of the theia process
     # This simulates the environment of a shell launched from the browser application
     theia_shell_process = pexpect.spawnu('{}/bin/theia-shell'.format(self.getPath()), env=theia_env)
-    try:
+    self.addCleanup(theia_shell_process.wait)
+    self.addCleanup(theia_shell_process.terminate)
+
     theia_shell_process.expect_exact('Standalone SlapOS for computer `slaprunner` activated')
 
     # Launch slapos node software from theia shell
@@ -433,11 +435,6 @@ class TestTheiaEnv(TheiaTestCase):
     self.assertEqual(theia_shell_env['SLAPOS_CLIENT_CONFIGURATION'], supervisord_env['SLAPOS_CLIENT_CONFIGURATION'])
     self.assertEqual(theia_shell_env['HOME'], supervisord_env['HOME'])
 
-    finally:
-      # Cleanup the theia shell process
-      theia_shell_process.terminate()
-      theia_shell_process.wait()
-
 
 class ResilientTheiaMixin(object):
   @classmethod