Add directive "cpp_locals" to handle C++ variables using std::optional (GH-4225)

This avoids the need for default constructors of stack allocated variables and temps by allowing late initialisation.

Closes https://github.com/cython/cython/issues/4160

Cython/Compiler/Code.py

@@ -849,6 +849,8 @@ class FunctionState(object):
         elif type.is_cfunction:
             from . import PyrexTypes
             type = PyrexTypes.c_ptr_type(type)  # A function itself isn't an l-value
+        elif type.is_cpp_class and self.scope.directives['cpp_locals']:
+            self.scope.use_utility_code(UtilityCode.load_cached("OptionalLocals", "CppSupport.cpp"))
         if not type.is_pyobject and not type.is_memoryviewslice:
             # Make manage_ref canonical, so that manage_ref will always mean
             # a decref is needed.
@@ -2070,8 +2072,12 @@ class CCodeWriter(object):
             self.put("%s " % storage_class)
         if not entry.cf_used:
             self.put('CYTHON_UNUSED ')
-        self.put(entry.type.declaration_code(
-            entry.cname, dll_linkage=dll_linkage))
+        if entry.is_cpp_optional:
+            self.put(entry.type.cpp_optional_declaration_code(
+                entry.cname, dll_linkage=dll_linkage))
+        else:
+            self.put(entry.type.declaration_code(
+                entry.cname, dll_linkage=dll_linkage))
         if entry.init is not None:
             self.put_safe(" = %s" % entry.type.literal_code(entry.init))
         elif entry.type.is_pyobject:
@@ -2080,7 +2086,10 @@ class CCodeWriter(object):
 
     def put_temp_declarations(self, func_context):
         for name, type, manage_ref, static in func_context.temps_allocated:
-            decl = type.declaration_code(name)
+            if type.is_cpp_class and func_context.scope.directives['cpp_locals']:
+                decl = type.cpp_optional_declaration_code(name)
+            else:
+                decl = type.declaration_code(name)
             if type.is_pyobject:
                 self.putln("%s = NULL;" % decl)
             elif type.is_memoryviewslice:
@@ -2363,7 +2372,7 @@ class CCodeWriter(object):
         # return self.putln("if (unlikely(%s < 0)) %s" % (value, self.error_goto(pos)))
         return self.putln("if (%s < 0) %s" % (value, self.error_goto(pos)))
 
-    def put_error_if_unbound(self, pos, entry, in_nogil_context=False):
+    def put_error_if_unbound(self, pos, entry, in_nogil_context=False, unbound_check_code=None):
         if entry.from_closure:
             func = '__Pyx_RaiseClosureNameError'
             self.globalstate.use_utility_code(
@@ -2372,13 +2381,25 @@ class CCodeWriter(object):
             func = '__Pyx_RaiseUnboundMemoryviewSliceNogil'
             self.globalstate.use_utility_code(
                 UtilityCode.load_cached("RaiseUnboundMemoryviewSliceNogil", "ObjectHandling.c"))
+        elif entry.type.is_cpp_class and entry.is_cglobal:
+            func = '__Pyx_RaiseCppGlobalNameError'
+            self.globalstate.use_utility_code(
+                UtilityCode.load_cached("RaiseCppGlobalNameError", "ObjectHandling.c"))
+        elif entry.type.is_cpp_class and entry.is_variable and not entry.is_member and entry.scope.is_c_class_scope:
+            # there doesn't seem to be a good way to detecting an instance-attribute of a C class
+            # (is_member is only set for class attributes)
+            func = '__Pyx_RaiseCppAttributeError'
+            self.globalstate.use_utility_code(
+                UtilityCode.load_cached("RaiseCppAttributeError", "ObjectHandling.c"))
         else:
             func = '__Pyx_RaiseUnboundLocalError'
             self.globalstate.use_utility_code(
                 UtilityCode.load_cached("RaiseUnboundLocalError", "ObjectHandling.c"))
 
+        if not unbound_check_code:
+            unbound_check_code = entry.type.check_for_null_code(entry.cname)
         self.putln('if (unlikely(!%s)) { %s("%s"); %s }' % (
-                                entry.type.check_for_null_code(entry.cname),
+                                unbound_check_code,
                                 func,
                                 entry.name,
                                 self.error_goto(pos)))

Cython/Compiler/ExprNodes.py

@@ -2364,12 +2364,21 @@ class NameNode(AtomicExprNode):
             # Raise UnboundLocalError for objects and memoryviewslices
             raise_unbound = (
                 (self.cf_maybe_null or self.cf_is_null) and not self.allow_null)
-            null_code = entry.type.check_for_null_code(entry.cname)
 
             memslice_check = entry.type.is_memoryviewslice and self.initialized_check
+            optional_cpp_check = entry.is_cpp_optional and self.initialized_check
 
-            if null_code and raise_unbound and (entry.type.is_pyobject or memslice_check):
-                code.put_error_if_unbound(self.pos, entry, self.in_nogil_context)
+            if optional_cpp_check:
+                unbound_check_code = entry.type.cpp_optional_check_for_null_code(entry.cname)
+            else:
+                unbound_check_code = entry.type.check_for_null_code(entry.cname)
+
+            if unbound_check_code and raise_unbound and (entry.type.is_pyobject or memslice_check or optional_cpp_check):
+                code.put_error_if_unbound(self.pos, entry, self.in_nogil_context, unbound_check_code=unbound_check_code)
+
+        elif entry.is_cglobal and entry.is_cpp_optional and self.initialized_check:
+            unbound_check_code = entry.type.cpp_optional_check_for_null_code(entry.cname)
+            code.put_error_if_unbound(self.pos, entry, unbound_check_code=unbound_check_code)
 
     def generate_assignment_code(self, rhs, code, overloaded_assignment=False,
                                  exception_check=None, exception_value=None):
@@ -7152,6 +7161,9 @@ class AttributeNode(ExprNode):
             self.op = "->"
         elif obj_type.is_reference and obj_type.is_fake_reference:
             self.op = "->"
+        elif (obj_type.is_cpp_class and (self.obj.is_name or self.obj.is_attribute) and
+                self.obj.entry and self.obj.entry.is_cpp_optional):
+            self.op = "->"
         else:
             self.op = "."
         if obj_type.has_attributes:
@@ -7360,6 +7372,9 @@ class AttributeNode(ExprNode):
                                         '"Memoryview is not initialized");'
                         '%s'
                     '}' % (self.result(), code.error_goto(self.pos)))
+        elif self.entry.is_cpp_optional and self.initialized_check:
+            unbound_check_code = self.type.cpp_optional_check_for_null_code(self.result())
+            code.put_error_if_unbound(self.pos, self.entry, unbound_check_code=unbound_check_code)
         else:
             # result_code contains what is needed, but we may need to insert
             # a check and raise an exception
@@ -13664,6 +13679,7 @@ class CoerceToTempNode(CoercionNode):
                 code.put_incref_memoryviewslice(self.result(), self.type,
                                             have_gil=not self.in_nogil_context)
 
+
 class ProxyNode(CoercionNode):
     """
     A node that should not be replaced by transforms or other means,
@@ -13782,6 +13798,23 @@ class CloneNode(CoercionNode):
         pass
 
 
+class CppOptionalTempCoercion(CoercionNode):
+    """
+    Used only in CoerceCppTemps - handles cases the temp is actually a OptionalCppClassType (and thus needs dereferencing when on the rhs)
+    """
+    is_temp = False
+
+    @property
+    def type(self):
+        return self.arg.type
+
+    def calculate_result_code(self):
+        return "(*%s)" % self.arg.result()
+
+    def generate_result_code(self, code):
+        pass
+
+
 class CMethodSelfCloneNode(CloneNode):
     # Special CloneNode for the self argument of builtin C methods
     # that accepts subtypes of the builtin type.  This is safe only

Cython/Compiler/FlowControl.py

@@ -160,7 +160,7 @@ class ControlFlow(object):
                 (entry.type.is_struct_or_union or
                  entry.type.is_complex or
                  entry.type.is_array or
-                 entry.type.is_cpp_class)):
+                 (entry.type.is_cpp_class and not entry.is_cpp_optional))):
             # stack allocated structured variable => never uninitialised
             return True
         return False

Cython/Compiler/ModuleNode.py

@@ -1277,8 +1277,11 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
                 attr_type = py_object_type
             else:
                 attr_type = attr.type
-            code.putln(
-                "%s;" % attr_type.declaration_code(attr.cname))
+            if attr.is_cpp_optional:
+                decl = attr_type.cpp_optional_declaration_code(attr.cname)
+            else:
+                decl = attr_type.declaration_code(attr.cname)
+            code.putln("%s;" % decl)
         code.putln(footer)
         if type.objtypedef_cname is not None:
             # Only for exposing public typedef name.
@@ -1357,8 +1360,12 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
 
             if storage_class:
                 code.put("%s " % storage_class)
-            code.put(type.declaration_code(
-                cname, dll_linkage=dll_linkage))
+            if entry.is_cpp_optional:
+                code.put(type.cpp_optional_declaration_code(
+                    cname, dll_linkage=dll_linkage))
+            else:
+                code.put(type.declaration_code(
+                    cname, dll_linkage=dll_linkage))
             if init is not None:
                 code.put_safe(" = %s" % init)
             code.putln(";")
@@ -1471,7 +1478,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
             # internal classes (should) never need None inits, normal zeroing will do
             py_attrs = []
         cpp_constructable_attrs = [entry for entry in scope.var_entries
-                                   if entry.type.needs_cpp_construction]
+                                   if (entry.type.needs_cpp_construction and not entry.is_cpp_optional)]
 
         cinit_func_entry = scope.lookup_here("__cinit__")
         if cinit_func_entry and not cinit_func_entry.is_special:

Cython/Compiler/Options.py

@@ -217,6 +217,7 @@ _directive_defaults = {
     'old_style_globals': False,
     'np_pythran': False,
     'fast_gil': False,
+    'cpp_locals': False,  # uses std::optional for C++ locals, so that they work more like Python locals
 
     # set __file__ and/or __path__ to known source/target path at import time (instead of not having them available)
     'set_initial_path' : None,  # SOURCEFILE or "/full/path/to/module"
@@ -371,6 +372,7 @@ directive_scopes = {  # defaults to available everywhere
     'iterable_coroutine': ('module', 'function'),
     'trashcan' : ('cclass',),
     'total_ordering': ('cclass', ),
+    'cpp_locals': ('module', 'function', 'cclass'),  # I don't think they make sense in a with_statement
 }
 
 

Cython/Compiler/ParseTreeTransforms.py

@@ -3148,6 +3148,30 @@ class GilCheck(VisitorTransform):
         return node
 
 
+class CoerceCppTemps(EnvTransform, SkipDeclarations):
+    """
+    For temporary expression that are implemented using std::optional it's necessary the temps are
+    assigned using `__pyx_t_x = value;` but accessed using `something = (*__pyx_t_x)`. This transform
+    inserts a coercion node to take care of this, and runs absolutely last (once nothing else can be
+    inserted into the tree)
+
+    TODO: a possible alternative would be to split ExprNode.result() into ExprNode.rhs_rhs() and ExprNode.lhs_rhs()???
+    """
+    def visit_ModuleNode(self, node):
+        if self.current_env().cpp:
+            # skipping this makes it essentially free for C files
+            self.visitchildren(node)
+        return node
+
+    def visit_ExprNode(self, node):
+        self.visitchildren(node)
+        if (self.current_env().directives['cpp_locals'] and
+                node.is_temp and node.type.is_cpp_class):
+            node = ExprNodes.CppOptionalTempCoercion(node)
+
+        return node
+
+
 class TransformBuiltinMethods(EnvTransform):
     """
     Replace Cython's own cython.* builtins by the corresponding tree nodes.

Cython/Compiler/Pipeline.py

@@ -150,7 +150,7 @@ def create_pipeline(context, mode, exclude_classes=()):
     from .ParseTreeTransforms import CalculateQualifiedNamesTransform
     from .TypeInference import MarkParallelAssignments, MarkOverflowingArithmetic
     from .ParseTreeTransforms import AdjustDefByDirectives, AlignFunctionDefinitions, AutoCpdefFunctionDefinitions
-    from .ParseTreeTransforms import RemoveUnreachableCode, GilCheck
+    from .ParseTreeTransforms import RemoveUnreachableCode, GilCheck, CoerceCppTemps
     from .FlowControl import ControlFlowAnalysis
     from .AnalysedTreeTransforms import AutoTestDictTransform
     from .AutoDocTransforms import EmbedSignature
@@ -221,6 +221,7 @@ def create_pipeline(context, mode, exclude_classes=()):
         ConsolidateOverflowCheck(context),
         DropRefcountingTransform(),
         FinalOptimizePhase(context),
+        CoerceCppTemps(context),
         GilCheck(),
         ]
     filtered_stages = []

Cython/Compiler/PyrexTypes.py

@@ -186,6 +186,8 @@ class PyrexType(BaseType):
     #  is_cfunction          boolean     Is a C function type
     #  is_struct_or_union    boolean     Is a C struct or union type
     #  is_struct             boolean     Is a C struct type
+    #  is_cpp_class          boolean     Is a C++ class
+    #  is_optional_cpp_class boolean     Is a C++ class with variable lifetime handled with std::optional
     #  is_enum               boolean     Is a C enum type
     #  is_cpp_enum           boolean     Is a C++ scoped enum type
     #  is_typedef            boolean     Is a typedef type
@@ -253,6 +255,7 @@ class PyrexType(BaseType):
     is_cfunction = 0
     is_struct_or_union = 0
     is_cpp_class = 0
+    is_optional_cpp_class = 0
     is_cpp_string = 0
     is_struct = 0
     is_enum = 0
@@ -373,6 +376,10 @@ class PyrexType(BaseType):
                                       self)
         code.putln("1")
 
+    def cpp_optional_declaration_code(self, entity_code, dll_linkage=None):
+        # declares an std::optional c++ variable
+        raise NotImplementedError(
+            "cpp_optional_declaration_code only implemented for c++ classes and not type %s" % self)
 
 
 def public_decl(base_code, dll_linkage):
@@ -3799,10 +3806,13 @@ class CppClassType(CType):
                 'maybe_unordered': self.maybe_unordered(),
                 'type': self.cname,
             })
+            # Override directives that should not be inherited from user code.
+            # TODO: filter directives with an allow list to keep only those that are safe and relevant.
+            directives = dict(env.directives, cpp_locals=False)
             from .UtilityCode import CythonUtilityCode
             env.use_utility_code(CythonUtilityCode.load(
                 cls.replace('unordered_', '') + ".from_py", "CppConvert.pyx",
-                context=context, compiler_directives=env.directives))
+                context=context, compiler_directives=directives))
             self.from_py_function = cname
             return True
 
@@ -3844,10 +3854,13 @@ class CppClassType(CType):
                 'maybe_unordered': self.maybe_unordered(),
                 'type': self.cname,
             })
+            # Override directives that should not be inherited from user code.
+            # TODO: filter directives with an allow list to keep only those that are safe and relevant.
+            directives = dict(env.directives, cpp_locals=False)
             from .UtilityCode import CythonUtilityCode
             env.use_utility_code(CythonUtilityCode.load(
                 cls.replace('unordered_', '') + ".to_py", "CppConvert.pyx",
-                context=context, compiler_directives=env.directives))
+                context=context, compiler_directives=directives))
             self.to_py_function = cname
             return True
 
@@ -3986,6 +3999,12 @@ class CppClassType(CType):
             base_code = public_decl(base_code, dll_linkage)
         return self.base_declaration_code(base_code, entity_code)
 
+    def cpp_optional_declaration_code(self, entity_code, dll_linkage=None, template_params=None):
+        return "__Pyx_Optional_Type<%s> %s" % (
+                self.declaration_code("", False, dll_linkage, False,
+                                    template_params),
+                entity_code)
+
     def is_subclass(self, other_type):
         if self.same_as_resolved_type(other_type):
             return 1
@@ -4068,6 +4087,11 @@ class CppClassType(CType):
         if constructor is not None and best_match([], constructor.all_alternatives()) is None:
             error(pos, "C++ class must have a nullary constructor to be %s" % msg)
 
+    def cpp_optional_check_for_null_code(self, cname):
+        # only applies to c++ classes that are being declared as std::optional
+        return "(%s.has_value())" % cname
+
+
 class CppScopedEnumType(CType):
     # name    string
     # doc     string or None

Cython/Compiler/Symtab.py

@@ -158,6 +158,7 @@ class Entry(object):
     # is_fused_specialized boolean Whether this entry of a cdef or def function
     #                              is a specialization
     # is_cgetter       boolean    Is a c-level getter function
+    # is_cpp_optional  boolean    Entry should be declared as std::optional (cpp_locals directive)
 
     # TODO: utility_code and utility_code_definition serves the same purpose...
 
@@ -229,6 +230,7 @@ class Entry(object):
     cf_used = True
     outer_entry = None
     is_cgetter = False
+    is_cpp_optional = False
 
     def __init__(self, name, cname, type, pos = None, init = None):
         self.name = name
@@ -268,6 +270,12 @@ class Entry(object):
     def cf_is_reassigned(self):
         return len(self.cf_assignments) > 1
 
+    def make_cpp_optional(self):
+        assert self.type.is_cpp_class
+        self.is_cpp_optional = True
+        assert not self.utility_code  # we're not overwriting anything?
+        self.utility_code = Code.UtilityCode.load_cached("OptionalLocals", "CppSupport.cpp")
+
 
 class InnerEntry(Entry):
     """
@@ -724,10 +732,13 @@ class Scope(object):
                 cname = name
             else:
                 cname = self.mangle(Naming.var_prefix, name)
-        if type.is_cpp_class and visibility != 'extern':
-            type.check_nullary_constructor(pos)
         entry = self.declare(name, cname, type, pos, visibility)
         entry.is_variable = 1
+        if type.is_cpp_class and visibility != 'extern':
+            if self.directives['cpp_locals']:
+                entry.make_cpp_optional()
+            else:
+                type.check_nullary_constructor(pos)
         if in_pxd and visibility != 'extern':
             entry.defined_in_pxd = 1
             entry.used = 1
@@ -2234,11 +2245,14 @@ class CClassScope(ClassScope):
                 if visibility == 'private':
                     cname = c_safe_identifier(cname)
                 cname = punycodify_name(cname, Naming.unicode_structmember_prefix)
-            if type.is_cpp_class and visibility != 'extern':
-                type.check_nullary_constructor(pos)
             entry = self.declare(name, cname, type, pos, visibility)
             entry.is_variable = 1
             self.var_entries.append(entry)
+            if type.is_cpp_class and visibility != 'extern':
+                if self.directives['cpp_locals']:
+                    entry.make_cpp_optional()
+                else:
+                    type.check_nullary_constructor(pos)
             if type.is_memoryviewslice:
                 self.has_memoryview_attrs = True
             elif type.needs_cpp_construction:

Cython/Compiler/TypeInference.py

@@ -358,12 +358,17 @@ class SimpleAssignmentTypeInferer(object):
     Note: in order to support cross-closure type inference, this must be
     applies to nested scopes in top-down order.
     """
-    def set_entry_type(self, entry, entry_type):
+    def set_entry_type(self, entry, entry_type, scope):
         for e in entry.all_entries():
             e.type = entry_type
             if e.type.is_memoryviewslice:
                 # memoryview slices crash if they don't get initialized
                 e.init = e.type.default_value
+            if e.type.is_cpp_class:
+                if scope.directives['cpp_locals']:
+                    e.make_cpp_optional()
+                else:
+                    e.type.check_nullary_constructor(entry.pos)
 
     def infer_types(self, scope):
         enabled = scope.directives['infer_types']
@@ -376,7 +381,7 @@ class SimpleAssignmentTypeInferer(object):
         else:
             for entry in scope.entries.values():
                 if entry.type is unspecified_type:
-                    self.set_entry_type(entry, py_object_type)
+                    self.set_entry_type(entry, py_object_type, scope)
             return
 
         # Set of assignments
@@ -405,7 +410,7 @@ class SimpleAssignmentTypeInferer(object):
             else:
                 entry = node.entry
                 node_type = spanning_type(
-                    types, entry.might_overflow, entry.pos, scope)
+                    types, entry.might_overflow, scope)
             node.inferred_type = node_type
 
         def infer_name_node_type_partial(node):
@@ -414,7 +419,7 @@ class SimpleAssignmentTypeInferer(object):
             if not types:
                 return
             entry = node.entry
-            return spanning_type(types, entry.might_overflow, entry.pos, scope)
+            return spanning_type(types, entry.might_overflow, scope)
 
         def inferred_types(entry):
             has_none = False
@@ -489,9 +494,9 @@ class SimpleAssignmentTypeInferer(object):
                 types = inferred_types(entry)
                 if types and all(types):
                     entry_type = spanning_type(
-                        types, entry.might_overflow, entry.pos, scope)
+                        types, entry.might_overflow, scope)
                     inferred.add(entry)
-            self.set_entry_type(entry, entry_type)
+            self.set_entry_type(entry, entry_type, scope)
 
         def reinfer():
             dirty = False
@@ -499,9 +504,9 @@ class SimpleAssignmentTypeInferer(object):
                 for assmt in entry.cf_assignments:
                     assmt.infer_type()
                 types = inferred_types(entry)
-                new_type = spanning_type(types, entry.might_overflow, entry.pos, scope)
+                new_type = spanning_type(types, entry.might_overflow, scope)
                 if new_type != entry.type:
-                    self.set_entry_type(entry, new_type)
+                    self.set_entry_type(entry, new_type, scope)
                     dirty = True
             return dirty
 
@@ -531,22 +536,20 @@ def find_spanning_type(type1, type2):
         return PyrexTypes.c_double_type
     return result_type
 
-def simply_type(result_type, pos):
+def simply_type(result_type):
     if result_type.is_reference:
         result_type = result_type.ref_base_type
     if result_type.is_cv_qualified:
         result_type = result_type.cv_base_type
-    if result_type.is_cpp_class:
-        result_type.check_nullary_constructor(pos)
     if result_type.is_array:
         result_type = PyrexTypes.c_ptr_type(result_type.base_type)
     return result_type
 
-def aggressive_spanning_type(types, might_overflow, pos, scope):
-    return simply_type(reduce(find_spanning_type, types), pos)
+def aggressive_spanning_type(types, might_overflow, scope):
+    return simply_type(reduce(find_spanning_type, types))
 
-def safe_spanning_type(types, might_overflow, pos, scope):
-    result_type = simply_type(reduce(find_spanning_type, types), pos)
+def safe_spanning_type(types, might_overflow, scope):
+    result_type = simply_type(reduce(find_spanning_type, types))
     if result_type.is_pyobject:
         # In theory, any specific Python type is always safe to
         # infer. However, inferring str can cause some existing code

Cython/Utility/CppSupport.cpp

@@ -80,3 +80,19 @@ auto __Pyx_pythran_to_python(T &&value) -> decltype(to_python(
 #else
   #define __PYX_ENUM_CLASS_DECL enum
 #endif
+
+////////////// OptionalLocals.proto ////////////////
+//@proto_block: utility_code_proto_before_types
+
+#if defined(CYTHON_USE_BOOST_OPTIONAL)
+    // fallback mode - std::optional is preferred but this gives
+    // people with a less up-to-date compiler a chance
+    #include <boost/optional.hpp>
+    #define __Pyx_Optional_Type boost::optional
+#else
+    #include <optional>
+    // since std::optional is a C++17 features, a templated using declaration should be safe
+    // (although it could be replaced with a define)
+    template <typename T>
+    using __Pyx_Optional_Type = std::optional<T>;
+#endif

Cython/Utility/ObjectHandling.c

@@ -3082,3 +3082,21 @@ static void __Pyx_RaiseUnboundMemoryviewSliceNogil(const char *varname) {
     PyGILState_Release(gilstate);
     #endif
 }
+
+//////////////// RaiseCppGlobalNameError.proto ///////////////////////
+static CYTHON_INLINE void __Pyx_RaiseCppGlobalNameError(const char *varname); /*proto*/
+
+/////////////// RaiseCppGlobalNameError //////////////////////////////
+static CYTHON_INLINE void __Pyx_RaiseCppGlobalNameError(const char *varname) {
+    PyErr_Format(PyExc_NameError, "C++ global '%s' is not initialized", varname);
+}
+
+//////////////// RaiseCppAttributeError.proto ///////////////////////
+static CYTHON_INLINE void __Pyx_RaiseCppAttributeError(const char *varname); /*proto*/
+
+/////////////// RaiseCppAttributeError //////////////////////////////
+static CYTHON_INLINE void __Pyx_RaiseCppAttributeError(const char *varname) {
+    PyErr_Format(PyExc_AttributeError, "C++ attribute '%s' is not initialized", varname);
+}
+
+

docs/src/userguide/source_files_and_compilation.rst

@@ -778,9 +778,12 @@ Cython code.  Here is the list of currently supported directives:
     Default is True.
 
 ``initializedcheck`` (True / False)
-    If set to True, Cython checks that a memoryview is initialized
-    whenever its elements are accessed or assigned to. Setting this
-    to False disables these checks.
+    If set to True, Cython checks that 
+     - a memoryview is initialized whenever its elements are accessed 
+       or assigned to. 
+     - a C++ class is initialized when it is accessed 
+       (only when ``cpp_locals`` is on)
+    Setting this to False disables these checks.
     Default is True.
 
 ``nonecheck``  (True / False)
@@ -912,6 +915,11 @@ Cython code.  Here is the list of currently supported directives:
     Copy the original source code line by line into C code comments in the generated
     code file to help with understanding the output.
     This is also required for coverage analysis.
+    
+``cpp_locals`` (True / False)
+    Make C++ variables behave more like Python variables by allowing them to be
+    "unbound" instead of always default-constructing them at the start of a
+    function.  See :ref:`cpp_locals directive` for more detail.
 
 .. _configurable_optimisations:
 

docs/src/userguide/wrapping_CPlusPlus.rst

@@ -136,6 +136,9 @@ a "default" constructor::
     def func():
         cdef Foo foo
         ...
+        
+See the section on the :ref:`cpp_locals directive` for a way
+to avoid requiring a nullary/default constructor.
 
 Note that, like C++, if the class has only one constructor and it
 is a nullary one, it's not necessary to declare it.
@@ -162,7 +165,9 @@ attribute access, you could just implement some properties:
 
 Cython initializes C++ class attributes of a cdef class using the nullary constructor.
 If the class you're wrapping does not have a nullary constructor, you must store a pointer
-to the wrapped class and manually allocate and deallocate it.
+to the wrapped class and manually allocate and deallocate it.  Alternatively, the
+:ref:`cpp_locals directive` avoids the need for the pointer and only initializes the
+C++ class attribute when it is assigned to.
 A convenient and safe place to do so is in the `__cinit__` and `__dealloc__` methods
 which are guaranteed to be called exactly once upon creation and deletion of the Python
 instance.
@@ -629,6 +634,44 @@ utility can be used to generate a C++ ``.cpp`` file, and then compile it
 into a python extension.  C++ mode for the ``cython`` command is turned
 on with the ``--cplus`` option.
 
+.. _cpp_locals directive:
+
+``cpp_locals`` directive
+========================
+
+The ``cpp_locals`` compiler directive is an experimental feature that makes
+C++ variables behave like normal Python object variables.  With this
+directive they are only initialized at their first assignment, and thus
+they no longer require a nullary constructor to be stack-allocated.  Trying to
+access an uninitialized C++ variable will generate an ``UnboundLocalError``
+(or similar) in the same way as a Python variable would.  For example::
+
+    def function(dont_write):
+        cdef SomeCppClass c  # not initialized
+        if dont_write:
+            return c.some_cpp_function()  # UnboundLocalError
+        else:
+            c = SomeCppClass(...)  # initialized
+            return c.some_cpp_function()  # OK
+            
+Additionally, the directive avoids initializing temporary C++ objects before
+they are assigned, for cases where Cython needs to use such objects in its
+own code-generation (often for return values of functions that can throw
+exceptions).
+
+For extra speed, the ``initializedcheck`` directive disables the check for an
+unbound-local.  With this directive on, accessing a variable that has not
+been initialized will trigger undefined behaviour, and it is entirely the user's
+responsibility to avoid such access.
+
+The ``cpp_locals`` directive is currently implemented using ``std::optional``
+and thus requires a C++17 compatible compiler. Defining
+``CYTHON_USE_BOOST_OPTIONAL`` (as define for the C++ compiler) uses ``boost::optional``
+instead (but is even more experimental and untested).  The directive may
+come with a memory and performance cost due to the need to store and check 
+a boolean that tracks if a variable is initialized, but the C++ compiler should
+be able to eliminate the check in most cases.
+
 
 Caveats and Limitations
 ========================

runtests.py

@@ -327,6 +327,10 @@ def update_cpp17_extension(ext):
     clang_version = get_clang_version(ext.language)
     if clang_version:
         ext.extra_compile_args.append("-std=c++17")
+        if sys.version_info[0] < 3:
+            # The Python 2.7 headers contain the 'register' modifier
+            # which clang warns about in C++17 mode.
+            ext.extra_compile_args.append('-Wno-register')
         if sys.platform == "darwin":
           ext.extra_compile_args.append("-stdlib=libc++")
           ext.extra_compile_args.append("-mmacosx-version-min=10.13")

tests/run/cpp_optional_temps.pyx

+# mode: run
+# tag: cpp, cpp17
+
+# cython: cpp_locals=True
+
+cimport cython
+
+cdef extern from *:
+    """
+    class C {
+        public:
+            C() = delete; // look! No default constructor
+            C(int x) : x(x) {}
+
+            int getX() const { return x; }
+
+        private:
+            int x;
+    };
+
+    C make_C(int x) {
+        return C(x);
+    }
+    """
+    cdef cppclass C:
+        C(int)
+        int getX() const
+    C make_C(int) except +  # needs a temp to receive
+
+def maybe_assign_infer(assign, value):
+    """
+    >>> maybe_assign_infer(True, 5)
+    5
+    >>> maybe_assign_infer(False, 0)
+    Traceback (most recent call last):
+        ...
+    UnboundLocalError: local variable 'x' referenced before assignment
+    """
+    if assign:
+        x = C(value)
+    print(x.getX())
+
+def maybe_assign_cdef(assign, value):
+    """
+    >>> maybe_assign_cdef(True, 5)
+    5
+    >>> maybe_assign_cdef(False, 0)
+    Traceback (most recent call last):
+        ...
+    UnboundLocalError: local variable 'x' referenced before assignment
+    """
+    cdef C x
+    if assign:
+        x = C(value)
+    print(x.getX())
+
+def maybe_assign_annotation(assign, value):
+    """
+    >>> maybe_assign_annotation(True, 5)
+    5
+    >>> maybe_assign_annotation(False, 0)
+    Traceback (most recent call last):
+        ...
+    UnboundLocalError: local variable 'x' referenced before assignment
+    """
+    x: C
+    if assign:
+        x = C(value)
+    print(x.getX())
+
+def maybe_assign_directive1(assign, value):
+    """
+    >>> maybe_assign_directive1(True, 5)
+    5
+    >>> maybe_assign_directive1(False, 0)
+    Traceback (most recent call last):
+        ...
+    UnboundLocalError: local variable 'x' referenced before assignment
+    """
+    x = cython.declare(C)
+    if assign:
+        x = C(value)
+    print(x.getX())
+
+@cython.locals(x=C)
+def maybe_assign_directive2(assign, value):
+    """
+    >>> maybe_assign_directive2(True, 5)
+    5
+    >>> maybe_assign_directive2(False, 0)
+    Traceback (most recent call last):
+        ...
+    UnboundLocalError: local variable 'x' referenced before assignment
+    """
+    if assign:
+        x = C(value)
+    print(x.getX())
+
+def maybe_assign_nocheck(assign, value):
+    """
+    >>> maybe_assign_nocheck(True, 5)
+    5
+
+    # unfortunately it's quite difficult to test not assigning because there's a decent chance it'll crash
+    """
+    if assign:
+        x = C(value)
+    with cython.initializedcheck(False):
+        print(x.getX())
+
+def uses_temp(value):
+    """
+    needs a temp to handle the result of make_C - still doesn't use the default constructor
+    >>> uses_temp(10)
+    10
+    """
+
+    x = make_C(value)
+    print(x.getX())
+
+# c should not be optional - it isn't easy to check this, but we can at least check it compiles
+cdef void has_argument(C c):
+    print(c.getX())
+
+def call_has_argument():
+    """
+    >>> call_has_argument()
+    50
+    """
+    has_argument(C(50))
+
+cdef class HoldsC:
+    """
+    >>> inst = HoldsC(True)
+    >>> inst.getCX()
+    10
+    >>> inst = HoldsC(False)
+    >>> inst.getCX()
+    Traceback (most recent call last):
+        ...
+    AttributeError: C++ attribute 'value' is not initialized
+    """
+    cdef C value
+    def __cinit__(self, initialize):
+        if initialize:
+            self.value = C(10)
+
+    def getCX(self):
+        return self.value.getX()
+
+cdef C global_var
+
+def initialize_global_var():
+    global global_var
+    global_var = C(-1)
+
+def read_global_var():
+    """
+    >>> read_global_var()
+    Traceback (most recent call last):
+        ...
+    NameError: C++ global 'global_var' is not initialized
+    >>> initialize_global_var()
+    >>> read_global_var()
+    -1
+    """
+    print(global_var.getX())