implement metaclass calculation/validation algorithm, make classes inherit their parents' metaclass

CHANGES.rst

@@ -52,6 +52,16 @@ Features added
 Bugs fixed
 ----------
 
+* The metaclass of a Python class was not inherited from its parent
+  class(es).  It is now extracted from the list of base classes if not
+  provided explicitly using the Py3 ``metaclass`` keyword argument.
+  In Py2 compilation mode, a ``__metaclass__`` entry in the class
+  dict will still take precedence if not using Py3 metaclass syntax,
+  but only *after* creating the class dict (which may have been done
+  by a metaclass of a base class, see PEP 3115).  It is generally
+  recommended to use the explicit Py3 syntax to define metaclasses
+  for Python types at compile time.
+
 * The automatic C switch statement generation behaves more safely for
   heterogeneous value types (e.g. mixing enum and char), allowing for
   a slightly wider application and reducing corner cases.  It now always
@@ -61,6 +71,9 @@ Bugs fixed
 Other changes
 -------------
 
+* In Py3 compilation mode, Python2-style metaclasses declared by a
+  ``__metaclass__`` class dict entry are ignored.
+
 * In Py3.4+, the Cython generator type uses ``tp_finalize()`` for safer
   cleanup instead of ``tp_del()``.
 

Cython/Compiler/ExprNodes.py

@@ -6782,6 +6782,8 @@ class Py3ClassNode(ExprNode):
     #  name         EncodedString      Name of the class
     #  dict         ExprNode           Class dict (not owned by this node)
     #  module_name  EncodedString      Name of defining module
+    #  calculate_metaclass  bool       should call CalculateMetaclass()
+    #  allow_py2_metaclass  bool       should look for Py2 metaclass
 
     subexprs = []
 
@@ -6798,14 +6800,20 @@ class Py3ClassNode(ExprNode):
     def generate_result_code(self, code):
         code.globalstate.use_utility_code(UtilityCode.load_cached("Py3ClassCreate", "ObjectHandling.c"))
         cname = code.intern_identifier(self.name)
+        if self.mkw:
+            mkw = self.mkw.py_result()
+        else:
+            mkw = 'NULL'
         code.putln(
-            '%s = __Pyx_Py3ClassCreate(%s, %s, %s, %s, %s); %s' % (
+            '%s = __Pyx_Py3ClassCreate(%s, %s, %s, %s, %s, %d, %d); %s' % (
                 self.result(),
                 self.metaclass.result(),
                 cname,
                 self.bases.py_result(),
                 self.dict.py_result(),
-                self.mkw.py_result(),
+                mkw,
+                self.calculate_metaclass,
+                self.allow_py2_metaclass,
                 code.error_goto_if_null(self.result(), self.pos)))
         code.put_gotref(self.py_result())
 
@@ -6941,12 +6949,20 @@ class PyClassMetaclassNode(ExprNode):
         return True
 
     def generate_result_code(self, code):
-        code.globalstate.use_utility_code(UtilityCode.load_cached("Py3MetaclassGet", "ObjectHandling.c"))
-        code.putln(
-            "%s = __Pyx_Py3MetaclassGet(%s, %s); %s" % (
-                self.result(),
+        if self.mkw:
+            code.globalstate.use_utility_code(
+                UtilityCode.load_cached("Py3MetaclassGet", "ObjectHandling.c"))
+            call = "__Pyx_Py3MetaclassGet(%s, %s)" % (
                 self.bases.result(),
-                self.mkw.result(),
+                self.mkw.result())
+        else:
+            code.globalstate.use_utility_code(
+                UtilityCode.load_cached("CalculateMetaclass", "ObjectHandling.c"))
+            call = "__Pyx_CalculateMetaclass(NULL, %s)" % (
+                self.bases.result())
+        code.putln(
+            "%s = %s; %s" % (
+                self.result(), call,
                 code.error_goto_if_null(self.result(), self.pos)))
         code.put_gotref(self.py_result())
 
@@ -6983,6 +6999,10 @@ class PyClassNamespaceNode(ExprNode, ModuleNameMixin):
             doc_code = self.doc.result()
         else:
             doc_code = '(PyObject *) NULL'
+        if self.mkw:
+            mkw = self.mkw.py_result()
+        else:
+            mkw = '(PyObject *) NULL'
         code.putln(
             "%s = __Pyx_Py3MetaclassPrepare(%s, %s, %s, %s, %s, %s, %s); %s" % (
                 self.result(),
@@ -6990,7 +7010,7 @@ class PyClassNamespaceNode(ExprNode, ModuleNameMixin):
                 self.bases.result(),
                 cname,
                 qualname,
-                self.mkw.result(),
+                mkw,
                 py_mod_name,
                 doc_code,
                 code.error_goto_if_null(self.result(), self.pos)))

Cython/Compiler/Nodes.py

@@ -3915,25 +3915,29 @@ class PyClassDefNode(ClassDefNode):
                    "target", "class_cell", "decorators"]
     decorators = None
     class_result = None
-    py3_style_class = False # Python3 style class (bases+kwargs)
+    is_py3_style_class = False  # Python3 style class (kwargs)
+    metaclass = None
+    mkw = None
 
-    def __init__(self, pos, name, bases, doc, body, decorators = None,
-                 keyword_args = None, starstar_arg = None):
+    def __init__(self, pos, name, bases, doc, body, decorators=None,
+                 keyword_args=None, starstar_arg=None, force_py3_semantics=False):
         StatNode.__init__(self, pos)
         self.name = name
         self.doc = doc
         self.body = body
         self.decorators = decorators
+        self.bases = bases
         import ExprNodes
         if self.doc and Options.docstrings:
             doc = embed_position(self.pos, self.doc)
-            doc_node = ExprNodes.StringNode(pos, value = doc)
+            doc_node = ExprNodes.StringNode(pos, value=doc)
         else:
             doc_node = None
+
+        allow_py2_metaclass = not force_py3_semantics
         if keyword_args or starstar_arg:
-            self.py3_style_class = True
-            self.bases = bases
-            self.metaclass = None
+            allow_py2_metaclass = False
+            self.is_py3_style_class = True
             if keyword_args and not starstar_arg:
                 for i, item in list(enumerate(keyword_args.key_value_pairs))[::-1]:
                     if item.key.value == 'metaclass':
@@ -3946,36 +3950,50 @@ class PyClassDefNode(ClassDefNode):
                         del keyword_args.key_value_pairs[i]
             if starstar_arg:
                 self.mkw = ExprNodes.KeywordArgsNode(
-                    pos, keyword_args = keyword_args and keyword_args.key_value_pairs or [],
-                    starstar_arg = starstar_arg)
-            elif keyword_args and keyword_args.key_value_pairs:
+                    pos, keyword_args=keyword_args and keyword_args.key_value_pairs or [],
+                    starstar_arg=starstar_arg)
+            elif keyword_args.key_value_pairs:
                 self.mkw = keyword_args
             else:
-                self.mkw = ExprNodes.NullNode(pos)
+                assert self.metaclass is not None
+
+        if force_py3_semantics or self.bases or self.mkw or self.metaclass:
             if self.metaclass is None:
+                if starstar_arg:
+                    # **kwargs may contain 'metaclass' arg
+                    mkdict = self.mkw
+                else:
+                    mkdict = None
                 self.metaclass = ExprNodes.PyClassMetaclassNode(
-                    pos, mkw = self.mkw, bases = self.bases)
-            self.dict = ExprNodes.PyClassNamespaceNode(pos, name = name,
-                        doc = doc_node, metaclass = self.metaclass, bases = self.bases,
-                        mkw = self.mkw)
-            self.classobj = ExprNodes.Py3ClassNode(pos, name = name,
-                    bases = self.bases, dict = self.dict, doc = doc_node,
-                    metaclass = self.metaclass, mkw = self.mkw)
-        else:
-            self.dict = ExprNodes.DictNode(pos, key_value_pairs = [])
-            self.metaclass = None
-            self.mkw = None
-            self.bases = None
-            self.classobj = ExprNodes.ClassNode(pos, name = name,
-                    bases = bases, dict = self.dict, doc = doc_node)
-        self.target = ExprNodes.NameNode(pos, name = name)
+                    pos, mkw=mkdict, bases=self.bases)
+                needs_metaclass_calculation = False
+            else:
+                needs_metaclass_calculation = True
+
+            self.dict = ExprNodes.PyClassNamespaceNode(
+                pos, name=name, doc=doc_node,
+                metaclass=self.metaclass, bases=self.bases, mkw=self.mkw)
+            self.classobj = ExprNodes.Py3ClassNode(
+                pos, name=name,
+                bases=self.bases, dict=self.dict, doc=doc_node,
+                metaclass=self.metaclass, mkw=self.mkw,
+                calculate_metaclass=needs_metaclass_calculation,
+                allow_py2_metaclass=allow_py2_metaclass)
+        else:
+            # no bases, no metaclass => old style class creation
+            self.dict = ExprNodes.DictNode(pos, key_value_pairs=[])
+            self.classobj = ExprNodes.ClassNode(
+                pos, name=name,
+                bases=bases, dict=self.dict, doc=doc_node)
+
+        self.target = ExprNodes.NameNode(pos, name=name)
         self.class_cell = ExprNodes.ClassCellInjectorNode(self.pos)
 
     def as_cclass(self):
         """
         Return this node as if it were declared as an extension class
         """
-        if self.py3_style_class:
+        if self.is_py3_style_class:
             error(self.classobj.pos, "Python3 style class could not be represented as C class")
             return
         bases = self.classobj.bases.args
@@ -4039,9 +4057,11 @@ class PyClassDefNode(ClassDefNode):
         self.body.analyse_declarations(cenv)
 
     def analyse_expressions(self, env):
-        if self.py3_style_class:
+        if self.bases:
             self.bases = self.bases.analyse_expressions(env)
+        if self.metaclass:
             self.metaclass = self.metaclass.analyse_expressions(env)
+        if self.mkw:
             self.mkw = self.mkw.analyse_expressions(env)
         self.dict = self.dict.analyse_expressions(env)
         self.class_result = self.class_result.analyse_expressions(env)
@@ -4059,9 +4079,11 @@ class PyClassDefNode(ClassDefNode):
     def generate_execution_code(self, code):
         code.pyclass_stack.append(self)
         cenv = self.scope
-        if self.py3_style_class:
+        if self.bases:
             self.bases.generate_evaluation_code(code)
+        if self.mkw:
             self.mkw.generate_evaluation_code(code)
+        if self.metaclass:
             self.metaclass.generate_evaluation_code(code)
         self.dict.generate_evaluation_code(code)
         cenv.namespace_cname = cenv.class_obj_cname = self.dict.result()
@@ -4075,11 +4097,13 @@ class PyClassDefNode(ClassDefNode):
         self.target.generate_assignment_code(self.class_result, code)
         self.dict.generate_disposal_code(code)
         self.dict.free_temps(code)
-        if self.py3_style_class:
-            self.mkw.generate_disposal_code(code)
-            self.mkw.free_temps(code)
+        if self.metaclass:
             self.metaclass.generate_disposal_code(code)
             self.metaclass.free_temps(code)
+        if self.mkw:
+            self.mkw.generate_disposal_code(code)
+            self.mkw.free_temps(code)
+        if self.bases:
             self.bases.generate_disposal_code(code)
             self.bases.free_temps(code)
         code.pyclass_stack.pop()

Cython/Compiler/Parsing.py

@@ -2962,12 +2962,13 @@ def p_class_statement(s, decorators):
         # XXX: empty arg_tuple
         arg_tuple = ExprNodes.TupleNode(pos, args=[])
     doc, body = p_suite_with_docstring(s, Ctx(level='class'))
-    return Nodes.PyClassDefNode(pos,
-        name = class_name,
-        bases = arg_tuple,
-        keyword_args = keyword_dict,
-        starstar_arg = starstar_arg,
-        doc = doc, body = body, decorators = decorators)
+    return Nodes.PyClassDefNode(
+        pos, name=class_name,
+        bases=arg_tuple,
+        keyword_args=keyword_dict,
+        starstar_arg=starstar_arg,
+        doc=doc, body=body, decorators=decorators,
+        force_py3_semantics=s.context.language_level >= 3)
 
 def p_c_class_definition(s, pos,  ctx):
     # s.sy == 'class'

tests/run/locals_T732.pyx

@@ -24,7 +24,7 @@ def test_class_locals_and_dir():
     >>> 'visible' in klass.locs and 'not_visible' not in klass.locs
     True
     >>> klass.names
-    ['visible']
+    ['__module__', '__qualname__', 'visible']
     """
     not_visible = 1234
     class Foo:

tests/run/metaclass.pyx

@@ -6,7 +6,7 @@ class Base(type):
         attrs['metaclass_was_here'] = True
         return type.__new__(cls, name, bases, attrs)
 
-@cython.test_fail_if_path_exists("//PyClassMetaclassNode", "//Py3ClassNode")
+@cython.test_assert_path_exists("//PyClassMetaclassNode", "//Py3ClassNode")
 class Foo(object):
     """
     >>> obj = Foo()
@@ -27,6 +27,7 @@ class ODict(dict):
 
 class Py3MetaclassPlusAttr(type):
     def __new__(cls, name, bases, attrs, **kwargs):
+        assert isinstance(attrs, ODict), str(type(attrs))
         for key, value in kwargs.items():
             attrs[key] = value
         attrs['metaclass_was_here'] = True
@@ -53,8 +54,21 @@ class Py3ClassMCOnly(object, metaclass=Py3MetaclassPlusAttr):
     """
     bar = 321
 
+class Py3InheritedMetaclass(Py3ClassMCOnly):
+    """
+    >>> obj = Py3InheritedMetaclass()
+    >>> obj.bar
+    345
+    >>> obj.metaclass_was_here
+    True
+    >>> obj._order
+    ['__module__', '__qualname__', '__doc__', 'bar', 'metaclass_was_here']
+    """
+    bar = 345
+
 class Py3Base(type):
     def __new__(cls, name, bases, attrs, **kwargs):
+        assert isinstance(attrs, ODict), str(type(attrs))
         for key, value in kwargs.items():
             attrs[key] = value
         return type.__new__(cls, name, bases, attrs)
@@ -80,6 +94,19 @@ class Py3Foo(object, metaclass=Py3Base, foo=123):
     """
     bar = 321
 
+@cython.test_assert_path_exists("//PyClassMetaclassNode", "//Py3ClassNode")
+class Py3FooInherited(Py3Foo, foo=567):
+    """
+    >>> obj = Py3FooInherited()
+    >>> obj.foo
+    567
+    >>> obj.bar
+    321
+    >>> obj._order
+    ['__module__', '__qualname__', '__doc__', 'bar', 'foo']
+    """
+    bar = 321
+
 kwargs = {'foo': 123, 'bar': 456}
 
 @cython.test_assert_path_exists("//PyClassMetaclassNode", "//Py3ClassNode")