merged in latest cython-devel

Cython/Compiler/ExprNodes.py

@@ -1905,12 +1905,12 @@ class IndexNode(ExprNode):
         return self.base.type_dependencies(env)
     
     def infer_type(self, env):
-        if isinstance(self.base, StringNode): # FIXME: BytesNode?
+        if isinstance(self.base, BytesNode):
             return py_object_type
         base_type = self.base.infer_type(env)
         if base_type.is_ptr or base_type.is_array:
             return base_type.base_type
-        elif base_type is Builtin.unicode_type and self.index.infer_type(env).is_int:
+        elif base_type is unicode_type and self.index.infer_type(env).is_int:
             # Py_UNICODE will automatically coerce to a unicode string
             # if required, so this is safe. We only infer Py_UNICODE
             # when the index is a C integer type. Otherwise, we may
@@ -1919,6 +1919,9 @@ class IndexNode(ExprNode):
             # to receive it, throw it away, and potentially rebuild it
             # on a subsequent PyObject coercion.
             return PyrexTypes.c_py_unicode_type
+        elif base_type in (str_type, unicode_type):
+            # these types will always return themselves on Python indexing
+            return base_type
         else:
             # TODO: Handle buffers (hopefully without too much redundancy).
             return py_object_type
@@ -6353,7 +6356,9 @@ class CoerceToPyTypeNode(CoercionNode):
         if type is not py_object_type:
             self.type = py_object_type
         elif arg.type.is_string:
-            self.type = Builtin.bytes_type
+            self.type = bytes_type
+        elif arg.type is PyrexTypes.c_py_unicode_type:
+            self.type = unicode_type
 
     gil_message = "Converting to Python object"
 

Cython/Compiler/Optimize.py

@@ -1846,6 +1846,71 @@ class OptimizeBuiltinCalls(Visitor.EnvTransform):
 
     ### unicode type methods
 
+    PyUnicode_uchar_predicate_func_type = PyrexTypes.CFuncType(
+        PyrexTypes.c_bint_type, [
+            PyrexTypes.CFuncTypeArg("uchar", PyrexTypes.c_py_unicode_type, None),
+            ])
+
+    def _inject_unicode_predicate(self, node, args, is_unbound_method):
+        if is_unbound_method or len(args) != 1:
+            return node
+        ustring = args[0]
+        if not isinstance(ustring, ExprNodes.CoerceToPyTypeNode) or \
+               ustring.arg.type is not PyrexTypes.c_py_unicode_type:
+            return node
+        uchar = ustring.arg
+        method_name = node.function.attribute
+        if method_name == 'istitle':
+            # istitle() doesn't directly map to Py_UNICODE_ISTITLE()
+            utility_code = py_unicode_istitle_utility_code
+            function_name = '__Pyx_Py_UNICODE_ISTITLE'
+        else:
+            utility_code = None
+            function_name = 'Py_UNICODE_%s' % method_name.upper()
+        func_call = self._substitute_method_call(
+            node, function_name, self.PyUnicode_uchar_predicate_func_type,
+            method_name, is_unbound_method, [uchar],
+            utility_code = utility_code)
+        if node.type.is_pyobject:
+            func_call = func_call.coerce_to_pyobject(self.current_env)
+        return func_call
+
+    _handle_simple_method_unicode_isalnum   = _inject_unicode_predicate
+    _handle_simple_method_unicode_isalpha   = _inject_unicode_predicate
+    _handle_simple_method_unicode_isdecimal = _inject_unicode_predicate
+    _handle_simple_method_unicode_isdigit   = _inject_unicode_predicate
+    _handle_simple_method_unicode_islower   = _inject_unicode_predicate
+    _handle_simple_method_unicode_isnumeric = _inject_unicode_predicate
+    _handle_simple_method_unicode_isspace   = _inject_unicode_predicate
+    _handle_simple_method_unicode_istitle   = _inject_unicode_predicate
+    _handle_simple_method_unicode_isupper   = _inject_unicode_predicate
+
+    PyUnicode_uchar_conversion_func_type = PyrexTypes.CFuncType(
+        PyrexTypes.c_py_unicode_type, [
+            PyrexTypes.CFuncTypeArg("uchar", PyrexTypes.c_py_unicode_type, None),
+            ])
+
+    def _inject_unicode_character_conversion(self, node, args, is_unbound_method):
+        if is_unbound_method or len(args) != 1:
+            return node
+        ustring = args[0]
+        if not isinstance(ustring, ExprNodes.CoerceToPyTypeNode) or \
+               ustring.arg.type is not PyrexTypes.c_py_unicode_type:
+            return node
+        uchar = ustring.arg
+        method_name = node.function.attribute
+        function_name = 'Py_UNICODE_TO%s' % method_name.upper()
+        func_call = self._substitute_method_call(
+            node, function_name, self.PyUnicode_uchar_conversion_func_type,
+            method_name, is_unbound_method, [uchar])
+        if node.type.is_pyobject:
+            func_call = func_call.coerce_to_pyobject(self.current_env)
+        return func_call
+
+    _handle_simple_method_unicode_lower = _inject_unicode_character_conversion
+    _handle_simple_method_unicode_upper = _inject_unicode_character_conversion
+    _handle_simple_method_unicode_title = _inject_unicode_character_conversion
+
     PyUnicode_Splitlines_func_type = PyrexTypes.CFuncType(
         Builtin.list_type, [
             PyrexTypes.CFuncTypeArg("str", Builtin.unicode_type, None),
@@ -2306,6 +2371,18 @@ class OptimizeBuiltinCalls(Visitor.EnvTransform):
             args[arg_index] = args[arg_index].coerce_to_boolean(self.current_env())
 
 
+py_unicode_istitle_utility_code = UtilityCode(
+# Py_UNICODE_ISTITLE() doesn't match unicode.istitle() as the latter
+# additionally allows character that comply with Py_UNICODE_ISUPPER()
+proto = '''
+static CYTHON_INLINE int __Pyx_Py_UNICODE_ISTITLE(Py_UNICODE uchar); /* proto */
+''',
+impl = '''
+static CYTHON_INLINE int __Pyx_Py_UNICODE_ISTITLE(Py_UNICODE uchar) {
+    return Py_UNICODE_ISTITLE(uchar) || Py_UNICODE_ISUPPER(uchar);
+}
+''')
+
 unicode_tailmatch_utility_code = UtilityCode(
     # Python's unicode.startswith() and unicode.endswith() support a
     # tuple of prefixes/suffixes, whereas it's much more common to

tests/bugs.txt

@@ -9,6 +9,8 @@ missing_baseclass_in_predecl_T262
 cfunc_call_tuple_args_T408
 cascaded_list_unpacking_T467
 compile.cpp_operators
+cpp_templated_ctypedef
+cpp_structs
 genexpr_T491
 
 # CPython regression tests that don't current work:

tests/compile/cpp_structs.pyx

+cdef extern from "point.h" namespace "geometry":
+
+    cdef struct Point:
+        double x
+        double y
+        int color
+
+cdef Point p = Point(0.0, 0.0, 0)
+the_point = p

tests/compile/cpp_templated_ctypedef.pyx

+cdef extern from *:
+    cdef cppclass Foo[T]:
+        pass
+    ctypedef Foo[int] IntFoo

tests/compile/point.h

+#ifndef POINT_H
+#define POINT_H
+
+namespace geometry {
+    
+    struct Point
+    {
+        double x;
+        double y;
+        int color;
+    };
+}
+
+#endif

tests/run/py_unicode_type.pyx

@@ -77,3 +77,45 @@ def unicode_ordinal(Py_UNICODE i):
     ValueError: only single character unicode strings can be converted to Py_UNICODE, got length 2
     """
     return i
+
+@cython.test_assert_path_exists('//PythonCapiCallNode')
+@cython.test_fail_if_path_exists('//SimpleCallNode')
+def unicode_type_methods(Py_UNICODE uchar):
+    """
+    >>> unicode_type_methods(ord('A'))
+    [True, True, False, False, False, False, False, True, True]
+    >>> unicode_type_methods(ord('a'))
+    [True, True, False, False, True, False, False, False, False]
+    >>> unicode_type_methods(ord('8'))
+    [True, False, True, True, False, True, False, False, False]
+    >>> unicode_type_methods(ord('\\t'))
+    [False, False, False, False, False, False, True, False, False]
+    """
+    return [
+        # character types
+        uchar.isalnum(),
+        uchar.isalpha(),
+        uchar.isdecimal(),
+        uchar.isdigit(),
+        uchar.islower(),
+        uchar.isnumeric(),
+        uchar.isspace(),
+        uchar.istitle(),
+        uchar.isupper(),
+        ]
+
+@cython.test_assert_path_exists('//PythonCapiCallNode')
+@cython.test_fail_if_path_exists('//SimpleCallNode')
+def unicode_methods(Py_UNICODE uchar):
+    """
+    >>> unicode_methods(ord('A')) == ['a', 'A', 'A']
+    True
+    >>> unicode_methods(ord('a')) == ['a', 'A', 'A']
+    True
+    """
+    return [
+        # character conversion
+        uchar.lower(),
+        uchar.upper(),
+        uchar.title(),
+        ]

tests/run/type_inference.pyx

@@ -71,6 +71,27 @@ def slicing():
     t1 = t[1:2]
     assert typeof(t1) == "tuple object", typeof(t1)
 
+def indexing():
+    """
+    >>> indexing()
+    """
+    b = b"abc"
+    assert typeof(b) == "char *", typeof(b)
+    b1 = b[1]
+    assert typeof(b1) == "char", typeof(b1)  # FIXME: bytes object ??
+    u = u"xyz"
+    assert typeof(u) == "unicode object", typeof(u)
+    u1 = u[1]
+    assert typeof(u1) == "Py_UNICODE", typeof(u1)
+    L = [1,2,3]
+    assert typeof(L) == "list object", typeof(L)
+    L1 = L[1]
+    assert typeof(L1) == "Python object", typeof(L1)
+    t = (4,5,6)
+    assert typeof(t) == "tuple object", typeof(t)
+    t1 = t[1]
+    assert typeof(t1) == "Python object", typeof(t1)
+
 def multiple_assignments():
     """
     >>> multiple_assignments()
@@ -197,6 +218,15 @@ def loop_over_charptr():
         pass
     return typeof(c)
 
+def loop_over_bytes_literal():
+    """
+    >>> print( loop_over_bytes_literal() )
+    Python object
+    """
+    for c in b'abcdefg':
+        pass
+    return typeof(c)
+
 def loop_over_bytes():
     """
     >>> print( loop_over_bytes() )
@@ -207,6 +237,16 @@ def loop_over_bytes():
         pass
     return typeof(c)
 
+def loop_over_str():
+    """
+    >>> print( loop_over_str() )
+    str object
+    """
+    cdef str string = 'abcdefg'
+    for c in string:
+        pass
+    return typeof(c)
+
 def loop_over_unicode():
     """
     >>> print( loop_over_unicode() )