core: use mypy to check python code ( WIP experiment )

product/ERP5/bootstrap/erp5_core/ExtensionTemplateItem/portal_components/extension.erp5.PythonCodeUtils.py

@@ -5,12 +5,15 @@ from Products.ERP5Type.Utils import checkPythonSourceCode
 
 logger = logging.getLogger('extension.erp5.PythonCodeUtils')
 
+import erp5.portal_type
+import typing
+
+
 def checkPythonSourceCodeAsJSON(self, data, REQUEST=None):
+  #xtype: (erp5.portal_type.ERP5Site, str, str) -> str
   """
   Check Python source suitable for Source Code Editor and return a JSON object
   """
-  import json
-
   # XXX data is encoded as json, because jQuery serialize lists as []
   if isinstance(data, basestring):
     data = json.loads(data)
@@ -77,45 +80,94 @@ def checkPythonSourceCodeAsJSON(self, data, REQUEST=None):
   reporter = Reporter()
   pyflakes.api.check(code, script_name, reporter)
   logger.info(
-      'pyflake checked %d lines in %.2f',
-      len(code.splitlines()),
-      time.time() - start
-    )
+      'pyflakes checked %d lines in %.2f', len(code.splitlines()),
+      time.time() - start)
 
   message_list = reporter.message_list
 
   import lib2to3.refactor
   import lib2to3.pgen2.parse
-  refactoring_tool = lib2to3.refactor.RefactoringTool(fixer_names=('lib2to3.fixes.fix_except', ))
+  refactoring_tool = lib2to3.refactor.RefactoringTool(
+      fixer_names=('lib2to3.fixes.fix_except',))
   old_code = code.decode('utf-8')
   try:
     new_code = unicode(refactoring_tool.refactor_string(old_code, script_name))
   except lib2to3.pgen2.parse.ParseError as e:
     message, (row, column) = e.context
-    message_list.append(
-          dict(row=row, column=column, type='E', text=message))
+    message_list.append(dict(row=row, column=column, type='E', text=message))
   else:
     if new_code != old_code:
       i = 0
-      for new_line, old_line in zip(new_code.splitlines(), old_code.splitlines()):
+      for new_line, old_line in zip(new_code.splitlines(),
+                                    old_code.splitlines()):
         i += 1
         #print ('new_line', new_line, 'old_line', old_line)
         if new_line != old_line:
           message_list.append(
-            dict(row=i, column=0, type='W', text=u'-{}\n+{}'.format(old_line, new_line)))
+              dict(
+                  row=i,
+                  column=0,
+                  type='W',
+                  text=u'-{}\n+{}'.format(old_line, new_line)))
+
 
 #  import pdb; pdb.set_trace()
   pylint_message_list = []
-  if 1:
+  if 0:
     start = time.time()
     pylint_message_list = checkPythonSourceCode(code, data.get('portal_type'))
     logger.info(
-      'pylint checked %d lines in %.2f',
-      len(code.splitlines()),
-      time.time() - start
-    )
+        'pylint checked %d lines in %.2f', len(code.splitlines()),
+        time.time() - start)
     message_list = pylint_message_list
 
+  import subprocess
+  start = time.time()
+  mypy_process = subprocess.Popen(
+      [
+          "/srv/slapgrid/slappart3/srv/runner/software/7ded4ab7e8ec62a2b1ad4312c472eeea/parts/python-language-server/bin/mypy",
+          "--python-version=2.7",
+          "--allow-redefinition",
+          "--allow-untyped-globals",
+          "--ignore-missing-imports", # XXX
+        #  "--check-untyped-defs",
+          "--show-error-codes",
+         "-c",
+          code,
+      #    "--booom"
+      ],
+
+      env={
+        'MYPYPATH': '/tmp/ahaha/',
+        'MYPY_CACHE_DIR': '/tmp/ahaha/.mypy_cache/',
+      },
+      stdout=subprocess.PIPE,
+      stderr=subprocess.PIPE)
+
+  mypy_out, mypy_err = mypy_process.communicate()
+  logger.info(
+      'mypy checked %d lines in %.2f', len(code.splitlines()),
+      time.time() - start)
+
+  #import pdb
+  #  pdb.set_trace()
+  # '<string>:9: error: Type signature has too few arguments\nFound 1 error in 1 file (checked 1 source file)\n'
+
+  for line in mypy_out.splitlines():
+    try:
+      filename, line_number, error_type, message = line.split(':', 3)
+    except ValueError as e:
+      logger.info("oops %s / %s", e, line)
+    else:
+      if filename == '<string>':
+        message_list.append(
+            dict(
+                row=int(line_number),
+                column=0,
+                type='E',
+                text="mypy: " + message,
+            ))
+
   for message_dict in message_list:
     if is_script:
       message_dict['row'] = message_dict['row'] - 2

product/ERP5/bootstrap/erp5_core/ExtensionTemplateItem/portal_components/extension.erp5.PythonCodeUtils.xml

@@ -45,7 +45,12 @@
         <item>
             <key> <string>text_content_warning_message</string> </key>
             <value>
-              <tuple/>
+              <tuple>
+                <string>W:145, 12: Unused variable \'mypy_err\' (unused-variable)</string>
+                <string>W:156, 29: Unused variable \'error_type\' (unused-variable)</string>
+                <string>W:  8,  0: Unused import erp5.portal_type (unused-import)</string>
+                <string>W:  9,  0: Unused import typing (unused-import)</string>
+              </tuple>
             </value>
         </item>
         <item>