fix explicit <bytes> etc. casts for C++ std::string when auto-decoding is enabled

CHANGES.rst

@@ -8,6 +8,9 @@ Latest
 Bugs fixed
 ----------
 
+* Casting C++ ``std::string`` to Python byte strings failed when auto-decoding
+  was enabled.
+
 * Fatal exceptions in global module init code could lead to crashes
   if the already created module was used later on (e.g. through a
   stale reference in sys.modules or elsewhere).

Cython/Compiler/PyrexTypes.py

@@ -3091,10 +3091,12 @@ class CppClassType(CType):
                          X[ix], T.to_py_function, X[ix]))
             if self.cname in cpp_string_conversions:
                 cls = 'string'
+                prefix = 'PyObject_'  # gets specialised by explicit type casts in CoerceToPyTypeNode
                 tags = self.cname.replace(':', '_'),
             else:
                 cls = self.cname[5:]
-            cname = "__pyx_convert_%s_to_py_%s" % (cls, "____".join(tags))
+                prefix = ''
+            cname = "__pyx_convert_%s%s_to_py_%s" % (prefix, cls, "____".join(tags))
             context = {
                 'template_type_declarations': '\n'.join(declarations),
                 'cname': cname,
@@ -3102,7 +3104,8 @@ class CppClassType(CType):
                 'type': self.cname,
             }
             from .UtilityCode import CythonUtilityCode
-            env.use_utility_code(CythonUtilityCode.load(cls.replace('unordered_', '') + ".to_py", "CppConvert.pyx", context=context))
+            env.use_utility_code(CythonUtilityCode.load(
+                cls.replace('unordered_', '') + ".to_py", "CppConvert.pyx", context=context))
             self.to_py_function = cname
             return True
 

Cython/Utility/CppConvert.pyx

@@ -25,11 +25,26 @@ cdef extern from *:
         char* data()
         size_t size()
     cdef object __Pyx_PyObject_FromStringAndSize(char*, size_t)
+    cdef object __Pyx_PyBytes_FromStringAndSize(char*, size_t)
+    cdef object __Pyx_PyByteArray_FromStringAndSize(char*, size_t)
+    cdef object __Pyx_PyUnicode_FromStringAndSize(char*, size_t)
 
 @cname("{{cname}}")
-cdef object {{cname}}(const string& s):
+cdef inline object {{cname}}(const string& s):
     return __Pyx_PyObject_FromStringAndSize(s.data(), s.size())
 
+@cname("{{cname.replace("PyObject", "PyUnicode")}}")
+cdef inline object {{cname}}__PyUnicode(const string& s):
+    return __Pyx_PyUnicode_FromStringAndSize(s.data(), s.size())
+
+@cname("{{cname.replace("PyObject", "PyBytes")}}")
+cdef inline object {{cname}}__PyBytes(const string& s):
+    return __Pyx_PyBytes_FromStringAndSize(s.data(), s.size())
+
+@cname("{{cname.replace("PyObject", "PyByteArray")}}")
+cdef inline object {{cname}}__PyByteArray(const string& s):
+    return __Pyx_PyByteArray_FromStringAndSize(s.data(), s.size())
+
 
 #################### vector.from_py ####################
 

tests/run/cpp_stl_string_ascii_auto_encoding.pyx

@@ -68,6 +68,7 @@ def test_clear(a):
     s.clear()
     return s
 
+
 def test_assign(char *a):
     """
     >>> test_assign(b_asdf) == 'ggg'
@@ -76,3 +77,64 @@ def test_assign(char *a):
     cdef string s = string(a)
     s.assign(<char *>"ggg")
     return s.c_str()
+
+
+def test_bytes_cast(a):
+    """
+    >>> b = test_bytes_cast(b'abc')
+    >>> isinstance(b, bytes)
+    True
+    >>> print(b.decode('ascii'))
+    abc
+    >>> b = test_bytes_cast(b'abc\\xe4\\xfc')
+    >>> isinstance(b, bytes)
+    True
+    >>> len(b)
+    5
+    >>> print(b[:3].decode('ascii'))
+    abc
+    >>> print(ord(b[3:4]))
+    228
+    >>> print(ord(b[4:5]))
+    252
+    """
+    cdef string s = a
+    assert s.length() == <size_t>len(a), "%d != %d" % (s.length(), len(a))
+    return <bytes>s
+
+
+def test_bytearray_cast(a):
+    """
+    >>> b = test_bytearray_cast(b'abc')
+    >>> isinstance(b, bytearray)
+    True
+    >>> print(b.decode('ascii'))
+    abc
+    >>> b = test_bytearray_cast(b'abc\\xe4\\xfc')
+    >>> isinstance(b, bytearray)
+    True
+    >>> len(b)
+    5
+    >>> print(b[:3].decode('ascii'))
+    abc
+    >>> print(ord(b[3:4]))
+    228
+    >>> print(ord(b[4:5]))
+    252
+    """
+    cdef string s = a
+    assert s.length() == <size_t>len(a), "%d != %d" % (s.length(), len(a))
+    return <bytearray>s
+
+
+def test_unicode_cast(a):
+    """
+    >>> u = test_unicode_cast(b'abc')
+    >>> type(u) is type(u_asdf) or type(u)
+    True
+    >>> print(u)
+    abc
+    """
+    cdef string s = a
+    assert s.length() == <size_t>len(a), "%d != %d" % (s.length(), len(a))
+    return <unicode>s