Add support for forwarding references (GH-3821)

See, for example, https://isocpp.org/blog/2012/11/universal-references-in-c11-scott-meyers

Closes #3814

Cython/Compiler/ExprNodes.py

@@ -504,21 +504,21 @@ class ExprNode(Node):
         else:
             return self.calculate_result_code()
 
-    def _make_move_result_rhs(self, result, allow_move=True):
-        if (self.is_temp and allow_move and
-                self.type.is_cpp_class and not self.type.is_reference):
-            self.has_temp_moved = True
-            return "__PYX_STD_MOVE_IF_SUPPORTED({0})".format(result)
-        else:
+    def _make_move_result_rhs(self, result, optional=False):
+        if optional and not (self.is_temp and self.type.is_cpp_class and not self.type.is_reference):
             return result
+        self.has_temp_moved = True
+        return "{}({})".format("__PYX_STD_MOVE_IF_SUPPORTED" if optional else "std::move", result)
 
     def move_result_rhs(self):
-        return self._make_move_result_rhs(self.result())
+        return self._make_move_result_rhs(self.result(), optional=True)
 
     def move_result_rhs_as(self, type):
-        allow_move = (type and not type.is_reference and not type.needs_refcounting)
-        return self._make_move_result_rhs(self.result_as(type),
-                                          allow_move=allow_move)
+        result = self.result_as(type)
+        if not (type.is_reference or type.needs_refcounting):
+            requires_move = type.is_rvalue_reference and self.is_temp
+            result = self._make_move_result_rhs(result, optional=not requires_move)
+        return result
 
     def pythran_result(self, type_=None):
         if is_pythran_supported_node_or_none(self):
@@ -5784,8 +5784,18 @@ class SimpleCallNode(CallNode):
             else:
                 alternatives = overloaded_entry.all_alternatives()
 
-            entry = PyrexTypes.best_match(
-                [arg.type for arg in args], alternatives, self.pos, env, args)
+            # For any argument/parameter pair A/P, if P is a forwarding reference,
+            # use lvalue-reference-to-A for deduction in place of A when the
+            # function call argument is an lvalue. See:
+            # https://en.cppreference.com/w/cpp/language/template_argument_deduction#Deduction_from_a_function_call
+            arg_types = [arg.type for arg in args]
+            if func_type.is_cfunction:
+                for i, formal_arg in enumerate(func_type.args):
+                    if formal_arg.is_forwarding_reference():
+                        if self.args[i].is_lvalue():
+                            arg_types[i] = PyrexTypes.c_ref_type(arg_types[i])
+
+            entry = PyrexTypes.best_match(arg_types, alternatives, self.pos, env, args)
 
             if not entry:
                 self.type = PyrexTypes.error_type

Cython/Compiler/Lexicon.py

@@ -77,7 +77,7 @@ def make_lexicon():
     punct = Any(":,;+-*/|&<>=.%`~^?!@")
     diphthong = Str("==", "<>", "!=", "<=", ">=", "<<", ">>", "**", "//",
                     "+=", "-=", "*=", "/=", "%=", "|=", "^=", "&=",
-                    "<<=", ">>=", "**=", "//=", "->", "@=")
+                    "<<=", ">>=", "**=", "//=", "->", "@=", "&&")
     spaces = Rep1(Any(" \t\f"))
     escaped_newline = Str("\\\n")
     lineterm = Eol + Opt(Str("\n"))

Cython/Compiler/Nodes.py

@@ -547,9 +547,7 @@ class CPtrDeclaratorNode(CDeclaratorNode):
         return self.base.analyse(ptr_type, env, nonempty=nonempty, visibility=visibility, in_pxd=in_pxd)
 
 
-class CReferenceDeclaratorNode(CDeclaratorNode):
-    # base     CDeclaratorNode
-
+class _CReferenceDeclaratorBaseNode(CDeclaratorNode):
     child_attrs = ["base"]
 
     def declared_name(self):
@@ -558,6 +556,8 @@ class CReferenceDeclaratorNode(CDeclaratorNode):
     def analyse_templates(self):
         return self.base.analyse_templates()
 
+
+class CReferenceDeclaratorNode(_CReferenceDeclaratorBaseNode):
     def analyse(self, base_type, env, nonempty=0, visibility=None, in_pxd=False):
         if base_type.is_pyobject:
             error(self.pos, "Reference base type cannot be a Python object")
@@ -565,6 +565,14 @@ class CReferenceDeclaratorNode(CDeclaratorNode):
         return self.base.analyse(ref_type, env, nonempty=nonempty, visibility=visibility, in_pxd=in_pxd)
 
 
+class CppRvalueReferenceDeclaratorNode(_CReferenceDeclaratorBaseNode):
+    def analyse(self, base_type, env, nonempty=0, visibility=None, in_pxd=False):
+        if base_type.is_pyobject:
+            error(self.pos, "Rvalue-reference base type cannot be a Python object")
+        ref_type = PyrexTypes.cpp_rvalue_ref_type(base_type)
+        return self.base.analyse(ref_type, env, nonempty=nonempty, visibility=visibility, in_pxd=in_pxd)
+
+
 class CArrayDeclaratorNode(CDeclaratorNode):
     # base        CDeclaratorNode
     # dimension   ExprNode
@@ -779,6 +787,13 @@ class CFuncDeclaratorNode(CDeclaratorNode):
                 error(self.pos, "cannot have both '%s' and '%s' "
                       "calling conventions" % (current, callspec))
             func_type.calling_convention = callspec
+
+        if func_type.return_type.is_rvalue_reference:
+            warning(self.pos, "Rvalue-reference as function return type not supported", 1)
+        for arg in func_type.args:
+            if arg.type.is_rvalue_reference and not arg.is_forwarding_reference():
+                warning(self.pos, "Rvalue-reference as function argument not supported", 1)
+
         return self.base.analyse(func_type, env, visibility=visibility, in_pxd=in_pxd)
 
     def declare_optional_arg_struct(self, func_type, env, fused_cname=None):
@@ -1389,6 +1404,8 @@ class CVarDefNode(StatNode):
                 return
             if type.is_reference and self.visibility != 'extern':
                 error(declarator.pos, "C++ references cannot be declared; use a pointer instead")
+            if type.is_rvalue_reference and self.visibility != 'extern':
+                error(declarator.pos, "C++ rvalue-references cannot be declared")
             if type.is_cfunction:
                 if 'staticmethod' in env.directives:
                     type.is_static_method = True

Cython/Compiler/Parsing.py

@@ -2876,12 +2876,13 @@ def p_c_simple_declarator(s, ctx, empty, is_type, cmethod_flag,
         if is_ptrptr:
             base = Nodes.CPtrDeclaratorNode(pos, base=base)
         result = Nodes.CPtrDeclaratorNode(pos, base=base)
-    elif s.sy == '&':
+    elif s.sy == '&' or (s.sy == '&&' and s.context.cpp):
+        node_class = Nodes.CppRvalueReferenceDeclaratorNode if s.sy == '&&' else Nodes.CReferenceDeclaratorNode
         s.next()
-        base = p_c_declarator(s, ctx, empty = empty, is_type = is_type,
-                              cmethod_flag = cmethod_flag,
-                              assignable = assignable, nonempty = nonempty)
-        result = Nodes.CReferenceDeclaratorNode(pos, base = base)
+        base = p_c_declarator(s, ctx, empty=empty, is_type=is_type,
+                              cmethod_flag=cmethod_flag,
+                              assignable=assignable, nonempty=nonempty)
+        result = node_class(pos, base=base)
     else:
         rhs = None
         if s.sy == 'IDENT':

Cython/Compiler/PyrexTypes.py

@@ -175,6 +175,7 @@ class PyrexType(BaseType):
     #  is_ptr                boolean     Is a C pointer type
     #  is_null_ptr           boolean     Is the type of NULL
     #  is_reference          boolean     Is a C reference type
+    #  is_rvalue_reference   boolean     Is a C++ rvalue reference type
     #  is_const              boolean     Is a C const type
     #  is_volatile           boolean     Is a C volatile type
     #  is_cv_qualified       boolean     Is a C const or volatile type
@@ -241,6 +242,7 @@ class PyrexType(BaseType):
     is_ptr = 0
     is_null_ptr = 0
     is_reference = 0
+    is_rvalue_reference = 0
     is_const = 0
     is_volatile = 0
     is_cv_qualified = 0
@@ -2751,26 +2753,17 @@ class CNullPtrType(CPtrType):
     is_null_ptr = 1
 
 
-class CReferenceType(BaseType):
+class CReferenceBaseType(BaseType):
 
-    is_reference = 1
     is_fake_reference = 0
 
+    # Common base type for C reference and C++ rvalue reference types.
+
     def __init__(self, base_type):
         self.ref_base_type = base_type
 
     def __repr__(self):
-        return "<CReferenceType %s>" % repr(self.ref_base_type)
-
-    def __str__(self):
-        return "%s &" % self.ref_base_type
-
-    def declaration_code(self, entity_code,
-            for_display = 0, dll_linkage = None, pyrex = 0):
-        #print "CReferenceType.declaration_code: pointer to", self.base_type ###
-        return self.ref_base_type.declaration_code(
-            "&%s" % entity_code,
-            for_display, dll_linkage, pyrex)
+        return "<%s %s>" % repr(self.__class__.__name__, self.ref_base_type)
 
     def specialize(self, values):
         base_type = self.ref_base_type.specialize(values)
@@ -2786,13 +2779,25 @@ class CReferenceType(BaseType):
         return getattr(self.ref_base_type, name)
 
 
+class CReferenceType(CReferenceBaseType):
+
+    is_reference = 1
+
+    def __str__(self):
+        return "%s &" % self.ref_base_type
+
+    def declaration_code(self, entity_code,
+            for_display = 0, dll_linkage = None, pyrex = 0):
+        #print "CReferenceType.declaration_code: pointer to", self.base_type ###
+        return self.ref_base_type.declaration_code(
+            "&%s" % entity_code,
+            for_display, dll_linkage, pyrex)
+
+
 class CFakeReferenceType(CReferenceType):
 
     is_fake_reference = 1
 
-    def __repr__(self):
-        return "<CFakeReferenceType %s>" % repr(self.ref_base_type)
-
     def __str__(self):
         return "%s [&]" % self.ref_base_type
 
@@ -2802,6 +2807,20 @@ class CFakeReferenceType(CReferenceType):
         return "__Pyx_FakeReference<%s> %s" % (self.ref_base_type.empty_declaration_code(), entity_code)
 
 
+class CppRvalueReferenceType(CReferenceBaseType):
+
+    is_rvalue_reference = 1
+
+    def __str__(self):
+        return "%s &&" % self.ref_base_type
+
+    def declaration_code(self, entity_code,
+            for_display = 0, dll_linkage = None, pyrex = 0):
+        return self.ref_base_type.declaration_code(
+            "&&%s" % entity_code,
+            for_display, dll_linkage, pyrex)
+
+
 class CFuncType(CType):
     #  return_type      CType
     #  args             [CFuncTypeArg]
@@ -3434,6 +3453,12 @@ class CFuncTypeArg(BaseType):
     def specialize(self, values):
         return CFuncTypeArg(self.name, self.type.specialize(values), self.pos, self.cname)
 
+    def is_forwarding_reference(self):
+        if self.type.is_rvalue_reference:
+            if (isinstance(self.type.ref_base_type, TemplatePlaceholderType)
+                    and not self.type.ref_base_type.is_cv_qualified):
+                return True
+        return False
 
 class ToPyStructUtilityCode(object):
 
@@ -4925,6 +4950,10 @@ def c_ref_type(base_type):
     # Construct a C reference type
     return _construct_type_from_base(CReferenceType, base_type)
 
+def cpp_rvalue_ref_type(base_type):
+    # Construct a C++ rvalue reference type
+    return _construct_type_from_base(CppRvalueReferenceType, base_type)
+
 def c_const_type(base_type):
     # Construct a C const type.
     return _construct_type_from_base(CConstType, base_type)

tests/compile/cpp_rvalue_reference_binding.pyx

+# tag: cpp, cpp11
+# mode: compile
+
+cdef extern from *:
+    """
+    template <typename T>
+    void accept(T&& x) {}
+    """
+    cdef void accept[T](T&& x)
+
+cdef int make_int_py() except *:
+    # might raise Python exception (thus needs a temp)
+    return 1
+
+cdef int make_int_cpp() except +:
+    # might raise C++ exception (thus needs a temp)
+    return 1
+
+def test_func_arg():
+    # won't compile if move() isn't called on the temp:
+    accept(make_int_py())
+    accept(make_int_cpp())

tests/errors/cpp_rvalue_reference_support.pyx

+# mode: error
+# tag: werror, cpp, cpp11
+
+# These tests check for unsupported use of rvalue-references (&&)
+# and should be removed or cleaned up when support is added.
+
+cdef int&& x
+
+cdef void foo(int&& x):
+    pass
+
+cdef int&& bar():
+    pass
+
+cdef extern from *:
+    """
+    void baz(int x, int&& y) {}
+
+    template <typename T>
+    void qux(const T&& x) {}
+    """
+    cdef void baz(int x, int&& y)
+    cdef void qux[T](const T&& x)
+
+
+_ERRORS="""
+7:8: C++ rvalue-references cannot be declared
+9:13: Rvalue-reference as function argument not supported
+12:14: Rvalue-reference as function return type not supported
+22:17: Rvalue-reference as function argument not supported
+23:20: Rvalue-reference as function argument not supported
+"""

tests/errors/e_cpp_references.pyx

+# mode: error
+# tag: cpp, cpp11
+
+cdef foo(object& x): pass
+cdef bar(object&& x): pass
+
+_ERRORS="""
+4:15: Reference base type cannot be a Python object
+5:15: Rvalue-reference base type cannot be a Python object
+"""

tests/run/cpp_forwarding_ref.pyx

+# mode: run
+# tag: cpp, cpp11
+
+from libcpp.utility cimport move
+
+
+cdef extern from *:
+    """
+    #include <utility>
+
+    const char* f(int& x) {
+        return "lvalue-ref";
+    }
+
+    const char* f(int&& x) {
+        return "rvalue-ref";
+    }
+
+    template <typename T>
+    const char* foo(T&& x)
+    {
+        return f(std::forward<T>(x));
+    }
+    """
+    const char* foo[T](T&& x)
+
+
+def test_forwarding_ref():
+    """
+    >>> test_forwarding_ref()
+    """
+    cdef int x = 1
+    assert foo(x) == b"lvalue-ref"
+    assert foo(<int>(1)) == b"rvalue-ref"
+    assert foo(move(x)) == b"rvalue-ref"