ERP5Type: more pylint / astroid fixes

product/ERP5Type/Tool/ComponentTool.py

@@ -163,8 +163,7 @@ class ComponentTool(BaseTool):
       if six.PY2:
         from astroid.builder import MANAGER
       else:
-        from astroid.builder import AstroidManager
-        MANAGER = AstroidManager()
+        from astroid.astroid_manager import MANAGER
       astroid_cache = MANAGER.astroid_cache
       for k in list(astroid_cache.keys()):
         if k.startswith('erp5.component.') and k not in component_package_list:

product/ERP5Type/Utils.py

@@ -467,10 +467,6 @@ _pylint_message_re = re.compile(
 def checkPythonSourceCode(source_code_str, portal_type=None):
   """
   Check source code with pylint or compile() builtin if not available.
-
-  TODO-arnau: Get rid of NamedTemporaryFile (require a patch on pylint to
-              allow passing a string) and this should probably return a proper
-              ERP5 object rather than a dict...
   """
   if not source_code_str:
     return []
@@ -555,7 +551,13 @@ def checkPythonSourceCode(source_code_str, portal_type=None):
            '--dummy-variables-rgx=_$|dummy|__traceback_info__|__traceback_supplement__',
       ]
       if six.PY3:
-        args.append("--msg-template='{C}: {line},{column}: {msg} ({symbol})'")
+        args.extend(
+          (
+            "--msg-template='{C}: {line},{column}: {msg} ({symbol})'",
+            # BBB until we drop compatibility with PY2
+            "--disable=redundant-u-string-prefix,raise-missing-from",
+          )
+        )
       if portal_type == 'Interface Component':
         # __init__ method from base class %r is not called
         args.append('--disable=W0231')
@@ -581,10 +583,9 @@ def checkPythonSourceCode(source_code_str, portal_type=None):
       finally:
         if six.PY2:
           from astroid.builder import MANAGER
-          astroid_cache = MANAGER.astroid_cache
         else:
-          from astroid.manager import AstroidManager
-          astroid_cache = AstroidManager().astroid_cache
+          from astroid.astroid_manager import MANAGER
+        astroid_cache = MANAGER.astroid_cache
         astroid_cache.pop(
           os.path.splitext(os.path.basename(input_file.name))[0],
           None)

product/ERP5Type/__init__.py

@@ -39,8 +39,7 @@ if  getZopeVersion()[0] == 2: # BBB Zope2
 else:
   IS_ZOPE2 = False
 import six
-if six.PY2:
-  from .patches import pylint
+from .patches import pylint
 from zLOG import LOG, INFO
 DISPLAY_BOOT_PROCESS = False
 

product/ERP5Type/patches/pylint.py

@@ -19,9 +19,11 @@
 # 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 
 from __future__ import absolute_import
+import importlib
+import six
 import sys
+import types
 import warnings
-import six
 from Products.ERP5Type import IS_ZOPE2
 
 # TODO: make sure that trying to use it does not import isort, because the
@@ -32,27 +34,21 @@ sys.modules.setdefault('isort', None)
 
 ## All arguments are passed as arguments and this needlessly outputs a 'No
 ## config file found, using default configuration' message on stderr.
-try:
+if six.PY2:
     from logilab.common.configuration import OptionsManagerMixIn
-except ImportError:
-    # pylint 2.x (python3)
-    from pylint.config import OptionsManagerMixIn
-OptionsManagerMixIn.read_config_file = lambda *args, **kw: None
+    OptionsManagerMixIn.read_config_file = lambda *args, **kw: None
 
 ## Pylint transforms and plugin to generate AST for ZODB Components
 from astroid.builder import AstroidBuilder
-from astroid.exceptions import AstroidBuildingException
-from astroid import node_classes
-
 if six.PY2:
-  from astroid import MANAGER
+    from astroid.exceptions import AstroidBuildingException as AstroidBuildingError
 else:
-  from astroid import AstroidManager
-  MANAGER = AstroidManager()
+    from astroid.exceptions import AstroidBuildingError
+from astroid import node_classes
+from astroid import MANAGER
 
-try:
+if six.PY2:
   from astroid.builder import _guess_encoding
-except ImportError:
   # XXX: With python3, tokenize.detect_encoding() is used instead. This
   # should do the same instead of copying/pasting legacy code...
   import re
@@ -67,7 +63,7 @@ except ImportError:
           match = _ENCODING_RGX.match(line)
           if match is not None:
               return match.group(1)
-def string_build(self, data, modname='', path=None):
+  def string_build(self, data, modname='', path=None):
     """
     build astroid from source code string and return rebuilded astroid
 
@@ -101,11 +97,11 @@ def string_build(self, data, modname='', path=None):
         LOG("Products.ERP5Type.patches.pylint", WARNING,
             "%s: Considered as not importable: Wrong encoding? (%r)" %
             (modname, exc))
-        raise AstroidBuildingException(exc)
+        raise AstroidBuildingError(exc)
     module = self._data_build(data, modname, path)
     module.file_bytes = data
     return self._post_build(module, encoding)
-AstroidBuilder.string_build = string_build
+  AstroidBuilder.string_build = string_build
 
 # patch node_classes.const_factory not to fail on LazyModules that e.g.
 # pygolang installs for pytest and ipython into sys.modules dict:
@@ -160,17 +156,17 @@ def _buildAstroidModuleFromComponentModuleName(modname):
             obj = getattr(component_tool,
                           component_id.replace('_version', '', 1))
         except AttributeError:
-            raise AstroidBuildingException()
+            raise AstroidBuildingError()
         if obj.getValidationState() in ('modified', 'validated'):
             component_obj = obj
         else:
-            raise AstroidBuildingException()
+            raise AstroidBuildingError()
 
     else:
         try:
             package, reference = component_id.split('.', 1)
         except ValueError:
-            raise AstroidBuildingException()
+            raise AstroidBuildingError()
         for version in portal.getVersionPriorityNameList():
             try:
                 obj = getattr(component_tool,
@@ -189,11 +185,12 @@ def _buildAstroidModuleFromComponentModuleName(modname):
                 return module
 
     if component_obj is None:
-        raise AstroidBuildingException()
+        raise AstroidBuildingError()
+
+    if six.PY3:
+        return AstroidBuilder(MANAGER).module_build(
+            importlib.import_module(modname))
 
-    # module_build() could also be used but this requires importing
-    # the ZODB Component and also monkey-patch it to support PEP-302
-    # for __file__ starting with '<'
     module = AstroidBuilder(MANAGER).string_build(
         component_obj.getTextContent(validated_only=True),
         modname)
@@ -201,7 +198,7 @@ def _buildAstroidModuleFromComponentModuleName(modname):
 
 def fail_hook_erp5_component(modname):
     if not modname.startswith('erp5.'):
-        raise AstroidBuildingException()
+        raise AstroidBuildingError()
 
     if (modname in ('erp5.portal_type',
                     'erp5.component',
@@ -229,8 +226,11 @@ MANAGER.register_failed_import_hook(fail_hook_erp5_component)
 ## transforms but this would require either checking dynamically which
 ## attributes has been added (much more complex than the current approach)
 ## or listing them statically (inconvenient).
-from astroid.exceptions import NotFoundError
-from astroid.scoped_nodes import Module
+from astroid.exceptions import AstroidError, NotFoundError
+if six.PY2:
+    from astroid.scoped_nodes import Module
+else:
+    from astroid.nodes import Module
 Module_getattr = Module.getattr
 def _getattr(self, name, *args, **kw):
     try:
@@ -238,8 +238,12 @@ def _getattr(self, name, *args, **kw):
     except NotFoundError as e:
         if self.name.startswith('erp5.'):
             raise
-
-        real_module = __import__(self.name, fromlist=[self.name], level=0)
+        if six.PY3 and self.name == 'numpy' or self.name.startswith('numpy.'):
+            raise
+        real_module = __import__(
+            self.name,
+            fromlist=[self.name] if six.PY2 else [name],
+            level=0)
         try:
             attr = getattr(real_module, name)
         except AttributeError:
@@ -255,13 +259,21 @@ def _getattr(self, name, *args, **kw):
         except AttributeError:
             from astroid import nodes
             if isinstance(attr, dict):
+                if six.PY2:
                     ast = nodes.Dict(attr)
+                else:
+                    ast = nodes.Dict(attr, 0, None, end_lineno=0, end_col_offset=0)
             elif isinstance(attr, list):
                 ast = nodes.List(attr)
             elif isinstance(attr, tuple):
                 ast = nodes.Tuple(attr)
             elif isinstance(attr, set):
+                if six.PY2:
                     ast = nodes.Set(attr)
+                else:
+                    ast = nodes.Set(attr, 0, None, end_lineno=0, end_col_offset=0)
+            elif isinstance(attr, types.ModuleType):
+                ast = MANAGER.ast_from_module(attr)
             else:
                 try:
                     ast = nodes.Const(attr)
@@ -272,16 +284,29 @@ def _getattr(self, name, *args, **kw):
                 raise
 
             # ast_from_class() actually works for any attribute of a Module
+            # (on py2 at least), but it raises some AssertionError when the class
+            # is defined dynamically, for example with zope.hookable.hookable,
+            # which (in version 6.0) is defined as:
+            #
+            #   if _PURE_PYTHON or _c_hookable is None:
+            #       hookable = _py_hookable
+            #   else:  # pragma: no cover
+            #       hookable = _c_hookable
             try:
                 ast = MANAGER.ast_from_class(attr)
-            except AstroidBuildingException:
+            except (AstroidError, AssertionError):
+                if six.PY2:
+                    raise e
+                try:
+                    ast = next(MANAGER.infer_ast_from_something(attr))
+                except AstroidError:
                     raise e
 
         self.locals[name] = [ast]
         return [ast]
 Module.getattr = _getattr
 
-if sys.version_info < (2, 8):
+if six.PY2:
     from astroid.node_classes import From
     def _absolute_import_activated(self):
         if (self.name.startswith('checkPythonSourceCode') or
@@ -428,7 +453,7 @@ _inspected_modules = {}
 def fail_hook_BTrees(modname):
     # Only consider BTrees.OOBTree pattern
     if not modname.startswith('BTrees.') or len(modname.split('.')) != 2:
-        raise AstroidBuildingException()
+        raise AstroidBuildingError()
     if modname not in _inspected_modules:
         try:
             modcode = build_stub(
@@ -444,7 +469,7 @@ def fail_hook_BTrees(modname):
     else:
         astng = _inspected_modules[modname]
     if astng is None:
-        raise AstroidBuildingException('Failed to import module %r' % modname)
+        raise AstroidBuildingError('Failed to import module %r' % modname)
     return astng
 MANAGER.register_failed_import_hook(fail_hook_BTrees)
 

product/ERP5Type/tests/testDynamicClassGeneration.py

@@ -1950,11 +1950,10 @@ class TestZodbModuleComponent(SecurityTestCase):
        [ComponentMixin._message_text_content_not_set],
        [],
        []),
-      ("""def foobar(*args, **kwargs)
-  return 42
+      ("""None()
 """ + valid_code,
-       ["Error in Source Code: E:  1,  0: invalid syntax (syntax-error)"],
-       ["E:  1,  0: invalid syntax (syntax-error)"],
+       ["Error in Source Code: E:  1,  0: None is not callable (not-callable)"],
+       ["E:  1,  0: None is not callable (not-callable)"],
        []),
       # Make sure that foobar NameError is at the end to make sure that after
       # defining foobar function, it is not available at all
@@ -2227,8 +2226,7 @@ def function_foo(*args, **kwargs):
     if six.PY2:
       from astroid.builder import MANAGER
     else:
-      from astroid.builder import AstroidManager
-      MANAGER = AstroidManager()
+      from astroid.astroid_manager import MANAGER
     should_not_be_in_cache_list = []
     for modname in MANAGER.astroid_cache:
       if (modname.startswith('checkPythonSourceCode') or
@@ -2284,10 +2282,10 @@ def hoge():
       """# -*- coding: utf-8 -*-
 # Source code with non-ASCII character should not fail: éàホゲ
 from %(namespace)s import %(reference1)s
-from %(namespace)s.erp5_version import %(reference1)s
+from %(namespace)s.erp5_version import %(reference1)s  # pylint:disable=reimported
 
 from %(module2)s import hoge
-from %(module2_with_version)s import hoge
+from %(module2_with_version)s import hoge  # pylint:disable=reimported
 
 import %(module2)s
 import %(module2_with_version)s
@@ -2312,7 +2310,8 @@ from AccessControl.PermissionRole import rolesForPermissionOn, PermissionRole, i
 
 # Monkey patch of astroid 1.3.8: it raised 'no-name-in-module' because
 # Shared.DC was not considered a namespace package
-from Shared.DC.ZRDB.Results import Results # pylint: disable=unused-import
+from Shared.DC.ZRDB.Results import Results
+_ = Results
 
 import lxml.etree
 lxml.etree.Element('test')
@@ -2377,9 +2376,11 @@ _ = ZBigArray
 
     self.tic()
     self.assertEqual(component.getValidationState(), 'modified')
+    if six.PY2:
       self.assertEqual(
         component.getTextContentErrorMessageList(),
-      ["E:  3,  0: No name '%s' in module '%s' (no-name-in-module)" %
+        [
+          "E:  3,  0: No name '%s' in module '%s' (no-name-in-module)" %
           (imported_reference1, namespace),
           "E:  4,  0: No name '%s' in module '%s.erp5_version' (no-name-in-module)" %
           (imported_reference1, namespace),
@@ -2403,9 +2404,41 @@ _ = ZBigArray
           "E: 10,  0: No name '%s' in module '%s.erp5_version' (no-name-in-module)" %
           (imported_reference2, namespace),
           "F: 10,  0: Unable to import '%s' (import-error)" %
-       imported_module2_with_version])
+          imported_module2_with_version,
+        ],
+      )
+    else:
+      self.assertEqual(
+        component.getTextContentErrorMessageList(),
+        [
+          "E:  3,  0: No name '%s' in module '%s' (no-name-in-module)" %
+          (imported_reference1, namespace),
+          "E:  4,  0: No name '%s' in module '%s.erp5_version' (no-name-in-module)" %
+          (imported_reference1, namespace),
+          "E:  6,  0: Unable to import '%s.%s' (import-error)" %
+          (namespace, imported_reference2),
+          # Spurious message but same as filesystem modules: 2 errors raised
+          # (no-name-in-module and import-error)
+          "E:  6,  0: No name '%s' in module '%s' (no-name-in-module)" %
+          (imported_reference2, namespace),
+          "E:  7,  0: Unable to import '%s' (import-error)" %
+          imported_module2_with_version,
+          # Spurious message (see above comment)
+          "E:  7,  0: No name '%s' in module '%s.erp5_version' (no-name-in-module)" %
+          (imported_reference2, namespace),
+          "E:  9,  0: Unable to import '%s.%s' (import-error)" %
+          (namespace, imported_reference2),
+          # Spurious message (see above comment)
+          "E:  9,  0: No name '%s' in module '%s' (no-name-in-module)" %
+          (imported_reference2, namespace),
+          "E: 10,  0: Unable to import '%s' (import-error)" %
+          imported_module2_with_version,
+          # Spurious message (see above comment)
+          "E: 10,  0: No name '%s' in module '%s.erp5_version' (no-name-in-module)" %
+          (imported_reference2, namespace),
+        ],
+      )
     self.assertEqual(component.getTextContentWarningMessageList(), [])
-
     ## Simulate user:
     # 1) First check and validate 'imported' Components
     self.portal.portal_workflow.doActionFor(imported_component1, 'validate_action')
@@ -2416,20 +2449,29 @@ _ = ZBigArray
 
     message_list = component.checkSourceCode()
     self.assertEqual(message_list, [])
-    self._assertAstroidCacheContent(
-      must_be_in_cache_set={'%s' % namespace,
-                            '%s.erp5_version' % namespace,
+    must_be_in_cache_set = {
       imported_module1,
       imported_module1_with_version,
       imported_module2,
-                            imported_module2_with_version},
+      imported_module2_with_version,
+    }
+    if six.PY2:
+      must_be_in_cache_set.update({
+        '%s' % namespace,
+        '%s.erp5_version' % namespace,
+      })
+    self._assertAstroidCacheContent(
+      must_be_in_cache_set=must_be_in_cache_set,
       must_not_be_in_cache_set=set())
 
     # 2) Then modify the main one so that it automatically 'validate'
     component.setTextContent(component.getTextContent() + '\n')
     self.tic()
+    must_be_in_cache_set = set()
+    if six.PY2:
+      must_be_in_cache_set.add(str(namespace))
     self._assertAstroidCacheContent(
-      must_be_in_cache_set={'%s' % namespace},
+      must_be_in_cache_set=must_be_in_cache_set,
       must_not_be_in_cache_set={'%s.erp5_version' % namespace,
                                 imported_module1,
                                 imported_module1_with_version,
@@ -2442,10 +2484,11 @@ _ = ZBigArray
     component.setTextContent(
       """# -*- coding: utf-8 -*-
 from %(module)s import undefined
-from %(module_with_version)s import undefined
+from %(module_with_version)s import undefined2
 
 # To avoid 'unused-import' warning...
 undefined()
+undefined2()
 
 """ % (dict(module=imported_module2,
             module_with_version=imported_module2_with_version)) +
@@ -2456,7 +2499,7 @@ undefined()
       component.getTextContentErrorMessageList(),
       ["E:  2,  0: No name 'undefined' in module '%s' (no-name-in-module)" %
        imported_module2_with_version,
-       "E:  3,  0: No name 'undefined' in module '%s' (no-name-in-module)" %
+       "E:  3,  0: No name 'undefined2' in module '%s' (no-name-in-module)" %
        imported_module2_with_version])
     self.assertEqual(component.getTextContentWarningMessageList(), [])
 
@@ -2492,8 +2535,8 @@ def hoge():
       component = self._newComponent(reference)
       component.setTextContent(component.getTextContent() + """
 from %(namespace)s import %(reference)s
-from %(namespace)s.bar_version import %(reference)s
-from %(namespace)s.erp5_version import %(reference)s
+from %(namespace)s.bar_version import %(reference)s  # pylint:disable=reimported
+from %(namespace)s.erp5_version import %(reference)s  # pylint:disable=reimported
 
 # To avoid 'unused-import' warning...
 %(reference)s.hoge()
@@ -2504,8 +2547,7 @@ from %(namespace)s.erp5_version import %(reference)s
       if six.PY2:
         from astroid.builder import MANAGER
       else:
-        from astroid.builder import AstroidManager
-        MANAGER = AstroidManager()
+        from astroid.astroid_manager import MANAGER
       imported_module = self._getComponentFullModuleName(imported_reference)
       self.assertEqual(
         MANAGER.astroid_cache[self._getComponentFullModuleName(imported_reference, version='bar')],