WIP type defintions

mypy.ini

+[mypy]
+ignore_missing_imports = True
+
+[mypy-slapos]
+ignore_missing_imports = True
+[mypy-slapos.slap]
+ignore_missing_imports = True
+[mypy-slapos.slap.grid]
+ignore_missing_imports = True
+
+
+# interfaces cause error: Method must have at least one argument
+[mypy-slapos.slap.interface]
+ignore_errors = True
+[mypy-slapos.slap.interface.slap]
+ignore_errors = True
+

setup.py

@@ -75,7 +75,8 @@ setup(name=name,
           'cachecontrol',
           'lockfile',
           'uritemplate', # used by hateoas navigator
-          'subprocess32; python_version<"3"'
+          'subprocess32; python_version<"3"',
+          'typing; python_version<"3"',
         ] + additional_install_requires,
       extras_require=extras_require,
       tests_require=extras_require['test'],

slapos/__init__.py

@@ -3,4 +3,4 @@ try:
     __import__('pkg_resources').declare_namespace(__name__)
 except ImportError:
     from pkgutil import extend_path
-    __path__ = extend_path(__path__, __name__)
+    __path__ = extend_path(__path__, __name__) # type: ignore

slapos/cli/proxy_show.py

@@ -44,7 +44,7 @@ from slapos.proxy import ProxyConfig
 from slapos.proxy.db_version import DB_VERSION
 from slapos.util import sqlite_connect, str2bytes
 
-if bytes is str:
+if sys.version_info[0] == 3:
   from io import BytesIO
   class StringIO(BytesIO):
     # Something between strict io.BytesIO and laxist/slow StringIO.StringIO

slapos/collect/temperature/__init__.py

 from __future__ import print_function
 
 from multiprocessing import Process, active_children, cpu_count, Pipe
-try:
+from typing import TYPE_CHECKING
+if TYPE_CHECKING:
+  import subprocess
+else:
+  try:
     import subprocess32 as subprocess
-except ImportError:
+  except ImportError:
     import subprocess
 import os
 import signal

slapos/grid/SlapObject.py

@@ -38,7 +38,8 @@ import subprocess
 import tarfile
 import tempfile
 import time
-from six.moves import xmlrpc_client as xmlrpclib, range
+from six.moves import xmlrpc_client as xmlrpclib # type: ignore
+from six.moves import range
 from six.moves.configparser import ConfigParser
 
 from supervisor import xmlrpc

slapos/grid/slapgrid.py

@@ -49,6 +49,10 @@ if sys.version_info < (2, 6):
   warnings.warn('Used python version (%s) is old and has problems with'
       ' IPv6 connections' % sys.version.split('\n')[0])
 
+from typing import List, Tuple, Sequence, TYPE_CHECKING
+if TYPE_CHECKING:
+  from ..slap.slap import ComputerPartition
+
 from lxml import etree
 
 from slapos import manager as slapmanager
@@ -537,6 +541,7 @@ stderr_logfile_backups=1
       launchSupervisord(instance_root=self.instance_root, logger=self.logger)
 
   def getComputerPartitionList(self):
+    # type: () -> Sequence[ComputerPartition]
     try:
       return self.computer.getComputerPartitionList()
     except socket.error as exc:
@@ -789,7 +794,7 @@ stderr_logfile_backups=1
       reload_process = FPopen(reload_cmd, universal_newlines=True)
       stdout, stderr = reload_process.communicate()
       if reload_process.returncode != 0:
-        raise Exception("Failed to load firewalld rules with command %s.\n%" % (
+        raise Exception("Failed to load firewalld rules with command %s.\n%s" % (
                         stderr, reload_cmd))
 
       with open(firewall_rules_path, 'w') as frules:
@@ -1254,6 +1259,7 @@ stderr_logfile_backups=1
         f.write(str(timestamp))
 
   def FilterComputerPartitionList(self, computer_partition_list):
+    # type: (Sequence[ComputerPartition]) -> List[ComputerPartition]
     """
     Try to filter valid partitions to be processed from free partitions.
     """
@@ -1392,12 +1398,12 @@ stderr_logfile_backups=1
     self.logger.info('=' * 80)
     if process_error_partition_list:
       self.logger.info('Error while processing the following partitions:')
-      for partition, exc in process_error_partition_list:
-        self.logger.info('  %s[%s]: %s', partition.getId(), getPartitionType(partition), exc)
+      for partition, error in process_error_partition_list:
+        self.logger.info('  %s[%s]: %s', partition.getId(), getPartitionType(partition), error)
     if promise_error_partition_list:
       self.logger.info('Error with promises for the following partitions:')
-      for partition, exc in promise_error_partition_list:
-        self.logger.info('  %s[%s]: %s', partition.getId(), getPartitionType(partition), exc)
+      for partition, error in promise_error_partition_list:
+        self.logger.info('  %s[%s]: %s', partition.getId(), getPartitionType(partition), error)
 
     # Return success value
     if not clean_run:
@@ -1420,7 +1426,7 @@ stderr_logfile_backups=1
     computer_partition_list = self.FilterComputerPartitionList(
         self.getComputerPartitionList())
 
-    promise_error_partition_list = []
+    promise_error_partition_list = [] # type: List[Tuple[ComputerPartition, Union[PromiseError, Exception]]]
     for computer_partition in computer_partition_list:
       try:
         # Process the partition itself
@@ -1444,8 +1450,8 @@ stderr_logfile_backups=1
 
     if promise_error_partition_list:
       self.logger.info('Finished computer partitions.')
-      for partition, exc in promise_error_partition_list:
-        self.logger.info('  %s[%s]: %s', partition.getId(), getPartitionType(partition), exc)
+      for partition, error in promise_error_partition_list:
+        self.logger.info('  %s[%s]: %s', partition.getId(), getPartitionType(partition), error)
 
     # Return success value
     if not clean_run_promise:

slapos/grid/svcbackend.py

@@ -35,7 +35,7 @@ import subprocess
 import stat
 import sys
 import time
-from six.moves import xmlrpc_client as xmlrpclib
+from six.moves import xmlrpc_client as xmlrpclib # type: ignore
 import contextlib
 
 from slapos.grid.utils import (createPrivateDirectory, SlapPopen, updateFile)

slapos/proxy/views.py

@@ -53,8 +53,11 @@ EMPTY_DICT_XML = dumps({})
 class UnauthorizedError(Exception):
   pass
 
+from typing import Dict, Union, no_type_check
 
+@no_type_check
 def partitiondict2partition(partition):
+  # type: (Dict[str, str]) -> ComputerPartition
   slap_partition = ComputerPartition(partition['computer_reference'],
       partition['reference'])
   slap_partition._software_release_document = None
@@ -365,6 +368,7 @@ def supplySupply():
 
 @app.route('/requestComputerPartition', methods=['POST'])
 def requestComputerPartition():
+  # type: () -> str
   parsed_request_dict = parseRequestComputerPartitionForm(request.form)
   # Is it a slave instance?
   slave = loads(request.form.get('shared_xml', EMPTY_DICT_XML).encode('utf-8'))
@@ -457,6 +461,7 @@ def requestComputerPartition():
   return dumps(software_instance)
 
 def parseRequestComputerPartitionForm(form):
+  # type: (Dict) -> Dict
   """
   Parse without intelligence a form from a request(), return it.
   """
@@ -469,7 +474,7 @@ def parseRequestComputerPartitionForm(form):
     'filter_kw': loads(form.get('filter_xml', EMPTY_DICT_XML).encode('utf-8')),
     # Note: currently ignored for slave instance (slave instances
     # are always started).
-    'requested_state': loads(form.get('state').encode('utf-8')),
+    'requested_state': loads(form['state'].encode('utf-8')),
   }
 
   return parsed_dict
@@ -543,10 +548,11 @@ def isRequestToBeForwardedToExternalMaster(parsed_request_dict):
     return None
 
 def forwardRequestToExternalMaster(master_url, request_form):
+  # type: (str, Dict) -> str
   """
   Forward instance request to external SlapOS Master.
   """
-  master_entry = app.config.get('multimaster').get(master_url, {})
+  master_entry = app.config['multimaster'].get(master_url, {})
   key_file = master_entry.get('key')
   cert_file = master_entry.get('cert')
   if master_url.startswith('https') and (not key_file or not cert_file):
@@ -568,7 +574,7 @@ def forwardRequestToExternalMaster(master_url, request_form):
              {'partition_reference':partition_reference, 'master_url': master_url})
 
   new_request_form = request_form.copy()
-  filter_kw = loads(new_request_form['filter_xml'].encode('utf-8'))
+  filter_kw = loads(request_form['filter_xml'].encode('utf-8'))
   filter_kw['source_instance_id'] = partition_reference
   new_request_form['filter_xml'] = dumps(filter_kw)
 
@@ -576,7 +582,7 @@ def forwardRequestToExternalMaster(master_url, request_form):
   partition = loads(xml)
 
   # XXX move to other end
-  partition._master_url = master_url
+  partition._master_url = master_url # type: ignore
 
   return dumps(partition)
 

slapos/slap/standalone.py

@@ -38,9 +38,15 @@ import shutil
 from six.moves import urllib
 from six.moves import http_client
 
-try:
+from typing import TYPE_CHECKING, Optional, Iterable, Dict
+if TYPE_CHECKING:
+  import subprocess
+  from ..slap.slap import Computer, ComputerPartition, SoftwareState, InstanceState, PartitionParameters, FilterParameters
+
+else:
+  try:
     import subprocess32 as subprocess
-except ImportError:
+  except ImportError:
     import subprocess
 
 import xml_marshaller
@@ -80,9 +86,11 @@ class ConfigWriter(object):
   """Base class for an object writing a config file or wrapper script.
   """
   def __init__(self, standalone_slapos):
+    # type: (StandaloneSlapOS) -> None
     self._standalone_slapos = standalone_slapos
 
   def writeConfig(self, path):
+    # type: (str) -> None
     NotImplemented
 
 
@@ -90,6 +98,7 @@ class SupervisorConfigWriter(ConfigWriter):
   """Write supervisor configuration at etc/supervisor.conf
   """
   def _getProgramConfig(self, program_name, command, stdout_logfile):
+    # type: (str, str, str) -> str
     """Format a supervisor program block.
     """
     return textwrap.dedent(
@@ -108,6 +117,7 @@ class SupervisorConfigWriter(ConfigWriter):
     """).format(**locals())
 
   def _getSupervisorConfigParts(self):
+    # type: () -> Iterable[str]
     """Iterator on parts of formatted config.
     """
     standalone_slapos = self._standalone_slapos
@@ -143,6 +153,7 @@ class SupervisorConfigWriter(ConfigWriter):
               'stdout_logfile', 'AUTO').format(self=standalone_slapos))
 
   def writeConfig(self, path):
+    # type: (str) -> None
     with open(path, 'w') as f:
       for part in self._getSupervisorConfigParts():
         f.write(part)
@@ -151,8 +162,10 @@ class SupervisorConfigWriter(ConfigWriter):
 class SlapOSConfigWriter(ConfigWriter):
   """Write slapos configuration at etc/slapos.cfg
   """
+
   def writeConfig(self, path):
-    standalone_slapos = self._standalone_slapos  # type: StandaloneSlapOS
+    # type: (str) -> None
+    standalone_slapos = self._standalone_slapos
     read_only_shared_part_list = '\n  '.join( #  pylint: disable=unused-variable; used in format()
         standalone_slapos._shared_part_list)
     with open(path, 'w') as f:
@@ -183,6 +196,7 @@ class SlapOSCommandWriter(ConfigWriter):
   """Write a bin/slapos wrapper.
   """
   def writeConfig(self, path):
+    # type: (str) -> None
     with open(path, 'w') as f:
       f.write(
           textwrap.dedent(
@@ -215,7 +229,9 @@ class StandaloneSlapOS(object):
       shared_part_list=(),
       software_root=None,
       instance_root=None,
-      shared_part_root=None):
+      shared_part_root=None,
+  ):
+    # type: (str, str, int, str, Iterable[str], Optional[str], Optional[str], Optional[str]) -> None
     """Constructor, creates a standalone slapos in `base_directory`.
 
     Arguments:
@@ -273,6 +289,7 @@ class StandaloneSlapOS(object):
     self._initBaseDirectory(software_root, instance_root, shared_part_root)
 
   def _initBaseDirectory(self, software_root, instance_root, shared_part_root):
+    # type: (Optional[str], Optional[str], Optional[str]) -> None
     """Create the directory after checking it's not too deep.
     """
     base_directory = self._base_directory
@@ -337,6 +354,7 @@ class StandaloneSlapOS(object):
 
   @property
   def computer(self):
+    # type: () -> Computer
     """Access the computer.
     """
     return self._slap.registerComputer(self._computer_id)
@@ -391,6 +409,7 @@ class StandaloneSlapOS(object):
       ipv4_address,
       ipv6_address,
       partition_base_name="slappart"):
+    # type: (int, str, str, str) -> None
     """Creates `partition_count` partitions.
 
     All partitions have the same `ipv4_address` and `ipv6_address` and
@@ -489,6 +508,7 @@ class StandaloneSlapOS(object):
         os.unlink(supervisor_conf)
 
   def supply(self, software_url, computer_guid=None, state="available"):
+    # type: (str, Optional[str], SoftwareState) -> None
     """Supply a software, see ISupply.supply
 
     Software can only be supplied on this embedded computer.
@@ -510,6 +530,7 @@ class StandaloneSlapOS(object):
       partition_parameter_kw=None,
       filter_kw=None,
       state=None):
+    # type: (str, str, Optional[str], bool, Optional[PartitionParameters], Optional[FilterParameters], Optional[InstanceState]) -> ComputerPartition
     """Request an instance, see IRequester.request
 
     Instance can only be requested on this embedded computer.
@@ -526,6 +547,7 @@ class StandaloneSlapOS(object):
         state=state)
 
   def start(self):
+    # type: () -> None
     """Start the system.
 
     If system was stopped, it will start partitions.
@@ -536,6 +558,7 @@ class StandaloneSlapOS(object):
     self._ensureSlapOSAvailable()
 
   def stop(self):
+    # type: () -> None
     """Stops all services.
 
     This methods blocks until services are stopped or a timeout is reached.
@@ -573,6 +596,7 @@ class StandaloneSlapOS(object):
               alive + instance_process_alive))
 
   def waitForSoftware(self, max_retry=0, debug=False, error_lines=30):
+    # type: (int, bool, int) -> None
     """Synchronously install or uninstall all softwares previously supplied/removed.
 
     This method retries on errors. If after `max_retry` times there's
@@ -594,6 +618,7 @@ class StandaloneSlapOS(object):
     )
 
   def waitForInstance(self, max_retry=0, debug=False, error_lines=30):
+    # type: (int, bool, int) -> None
     """Instantiate all partitions previously requested for start.
 
     This method retries on errors. If after `max_retry` times there's
@@ -615,6 +640,7 @@ class StandaloneSlapOS(object):
     )
 
   def waitForReport(self, max_retry=0, debug=False, error_lines=30):
+    # type: (int, bool, int) -> None
     """Destroy all partitions previously requested for destruction.
 
     This method retries on errors. If after `max_retry` times there's
@@ -637,17 +663,19 @@ class StandaloneSlapOS(object):
 
   def _runSlapOSCommand(
       self, command, max_retry=0, debug=False, error_lines=30):
+    # type: (str, int, bool, int) -> None
     if debug:
       prog = self._slapos_commands[command]
       # used in format(**locals()) below
       debug_args = prog.get('debug_args', '')  # pylint: disable=unused-variable
       command = prog['command'].format(**locals())
       try:
-        return subprocess.check_call(
+        subprocess.check_call(
             command,
             shell=True,
             env=self._getSubprocessEnvironment(),
         )
+        return
       except subprocess.CalledProcessError as e:
         if e.returncode == SLAPGRID_PROMISE_FAIL:
           self._logger.exception('Promise error when running %s', command)
@@ -687,6 +715,7 @@ class StandaloneSlapOS(object):
         retry += 1
 
   def _ensureSupervisordStarted(self):
+    # type: () -> None
     if os.path.exists(self._supervisor_pid):
       with open(self._supervisor_pid, 'r') as f:
         try:
@@ -715,6 +744,7 @@ class StandaloneSlapOS(object):
     self._logger.debug("Started new supervisor: %s", output)
 
   def _isSlapOSAvailable(self):
+    # type: () -> bool
     try:
       urllib.request.urlopen(self._master_url).close()
     except urllib.error.HTTPError as e:
@@ -723,6 +753,7 @@ class StandaloneSlapOS(object):
         return True
       raise
     except urllib.error.URLError as e:
+      assert isinstance(e.reason, OSError)
       if e.reason.errno == errno.ECONNREFUSED:
         return False
       raise
@@ -735,6 +766,7 @@ class StandaloneSlapOS(object):
     return True  # (if / becomes 200 OK)
 
   def _ensureSlapOSAvailable(self):
+    # type: () -> None
     # Wait for proxy to accept connections
     for i in range(2**8):
       if self._isSlapOSAvailable():
@@ -743,6 +775,7 @@ class StandaloneSlapOS(object):
     raise RuntimeError("SlapOS not started")
 
   def _getSubprocessEnvironment(self):
+    # type: () -> Optional[Dict[str, str]]
     # Running tests with `python setup.py test` sets a PYTHONPATH that
     # is suitable for current python, but problematic when this process
     # runs another version of python in subprocess.
@@ -752,3 +785,4 @@ class StandaloneSlapOS(object):
       env = os.environ.copy()
       del env['PYTHONPATH']
       return env
+    return None

slapos/tests/test_prune.py

@@ -32,10 +32,11 @@ import shutil
 import unittest
 import slapos.client
 
-try:
-  import mock
-except ImportError:
+import sys
+if sys.version_info[0] == 3:
   from unittest import mock
+else:
+  import mock
 
 from slapos.cli.prune import do_prune
 

slapos/tests/test_slapproxy.py

@@ -34,10 +34,15 @@ import os
 import logging
 import shutil
 import socket
-try:
+from typing import TYPE_CHECKING
+if TYPE_CHECKING:
+  import subprocess
+else:
+  try:
     import subprocess32 as subprocess
-except ImportError:
+  except ImportError:
     import subprocess
+
 import sys
 import tempfile
 import time