Merge remote branch 'upstream/master'

Cython/Compiler/CmdLine.py

@@ -10,15 +10,15 @@ usage = """\
 Cython (http://cython.org) is a compiler for code written in the
 Cython language.  Cython is based on Pyrex by Greg Ewing.
 
-Usage: cython [options] sourcefile.pyx ...
+Usage: cython [options] sourcefile.{pyx,py} ...
 
 Options:
   -V, --version                  Display version number of cython compiler
   -l, --create-listing           Write error messages to a listing file
   -I, --include-dir <directory>  Search for include files in named directory
-                                 (multiply include directories are allowed).
+                                 (multiple include directories are allowed).
   -o, --output-file <filename>   Specify name of generated C file
-  -t, --timestamps               Only compile newer source files (implied with -r)
+  -t, --timestamps               Only compile newer source files
   -f, --force                    Compile all source files (overrides implied -t)
   -q, --quiet                    Don't print module names in recursive mode
   -v, --verbose                  Be verbose, print file names on multiple compilation
@@ -30,11 +30,11 @@ Options:
                                  are searched from)
   --gdb                          Output debug information for cygdb
 
-  -D, --no-docstrings            Remove docstrings.
+  -D, --no-docstrings            Strip docstrings from the compiled module.
   -a, --annotate                 Produce a colorized HTML version of the source.
   --line-directives              Produce #line directives pointing to the .pyx source
-  --cplus                        Output a c++ rather than c file.
-  --embed                        Embed the Python interpreter in a main() method.
+  --cplus                        Output a C++ rather than C file.
+  --embed                        Generate a main() function that embeds the Python interpreter.
   -2                             Compile based on Python-2 syntax and code semantics.
   -3                             Compile based on Python-3 syntax and code semantics.
   --fast-fail                    Abort the compilation on the first error
@@ -42,7 +42,7 @@ Options:
 """
 
 # The following is broken http://trac.cython.org/cython_trac/ticket/379
-#  -r, --recursive                Recursively find and compile dependencies
+#  -r, --recursive                Recursively find and compile dependencies (implies -t)
 
 
 #The following experimental options are supported only on MacOSX:
@@ -143,6 +143,9 @@ def parse_command_line(args):
                 else:
                     sys.stderr.write("Unknown debug flag: %s\n" % option)
                     bad_usage()
+            elif option in ('-h', '--help'):
+                sys.stdout.write(usage)
+                sys.exit(0)
             else:
                 sys.stderr.write("Unknown compiler flag: %s\n" % option)
                 sys.exit(1)

Cython/Compiler/ExprNodes.py

@@ -954,7 +954,8 @@ class BytesNode(ConstNode):
     #
     # value      BytesLiteral
 
-    type = PyrexTypes.c_char_ptr_type
+    # start off as Python 'bytes' to support len() in O(1)
+    type = bytes_type
 
     def compile_time_value(self, denv):
         return self.value
@@ -975,11 +976,13 @@ class BytesNode(ConstNode):
         return len(self.value) == 1
 
     def coerce_to_boolean(self, env):
-        # This is special because we start off as a C char*.  Testing
-        # that for truth directly would yield the wrong result.
+        # This is special because testing a C char* for truth directly
+        # would yield the wrong result.
         return BoolNode(self.pos, value=bool(self.value))
 
     def coerce_to(self, dst_type, env):
+        if self.type == dst_type:
+            return self
         if dst_type.is_int:
             if not self.can_coerce_to_char_literal():
                 error(self.pos, "Only single-character string literals can be coerced into ints.")
@@ -990,21 +993,20 @@ class BytesNode(ConstNode):
             return CharNode(self.pos, value=self.value)
 
         node = BytesNode(self.pos, value=self.value)
-        if dst_type == PyrexTypes.c_char_ptr_type:
-            node.type = PyrexTypes.c_char_ptr_type
+        if dst_type.is_pyobject:
+            if dst_type in (py_object_type, Builtin.bytes_type):
+                node.type = Builtin.bytes_type
+            else:
+                self.check_for_coercion_error(dst_type, fail=True)
+                return node
+        elif dst_type == PyrexTypes.c_char_ptr_type:
+            node.type = dst_type
             return node
         elif dst_type == PyrexTypes.c_uchar_ptr_type:
             node.type = PyrexTypes.c_char_ptr_type
             return CastNode(node, PyrexTypes.c_uchar_ptr_type)
-
-        if not self.type.is_pyobject:
-            if dst_type in (py_object_type, Builtin.bytes_type):
-                node.type = Builtin.bytes_type
-            elif dst_type.is_pyobject:
-                self.fail_assignment(dst_type)
-                return self
-        elif dst_type.is_pyobject and dst_type is not py_object_type:
-            self.check_for_coercion_error(dst_type, fail=True)
+        elif dst_type.assignable_from(PyrexTypes.c_char_ptr_type):
+            node.type = dst_type
             return node
 
         # We still need to perform normal coerce_to processing on the
@@ -1012,11 +1014,6 @@ class BytesNode(ConstNode):
         # in which case a type test node will be needed.
         return ConstNode.coerce_to(node, dst_type, env)
 
-    def as_py_string_node(self, env):
-        # Return a new BytesNode with the same value as this node
-        # but whose type is a Python type instead of a C type.
-        return BytesNode(self.pos, value = self.value, type = Builtin.bytes_type)
-
     def generate_evaluation_code(self, code):
         if self.type.is_pyobject:
             self.result_code = code.get_py_string_const(self.value)
@@ -2969,9 +2966,14 @@ class SimpleCallNode(CallNode):
                 arg = arg.coerce_to_temp(env)
             self.args[i] = arg
         for i in range(max_nargs, actual_nargs):
-            if self.args[i].type.is_pyobject:
+            arg = self.args[i]
+            if arg.type.is_pyobject:
+                arg_ctype = arg.type.default_coerced_ctype()
+                if arg_ctype is None:
                     error(self.args[i].pos,
                           "Python object cannot be passed as a varargs parameter")
+                else:
+                    self.args[i] = arg.coerce_to(arg_ctype, env)
         # Calc result type and code fragment
         if isinstance(self.function, NewExprNode):
             self.type = PyrexTypes.CPtrType(self.function.class_type)

Cython/Compiler/ModuleNode.py

@@ -2051,12 +2051,12 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
                                        code.error_goto_if_null(type.typeptr_cname, pos))
         self.use_type_import_utility_code(env)
         if type.vtabptr_cname:
-            code.putln(
-                "if (__Pyx_GetVtable(%s->tp_dict, &%s) < 0) %s" % (
-                    type.typeptr_cname,
-                    type.vtabptr_cname,
-                    code.error_goto(pos)))
             env.use_utility_code(Nodes.get_vtable_utility_code)
+            code.putln("%s = (struct %s*)__Pyx_GetVtable(%s->tp_dict); %s" % (
+                type.vtabptr_cname,
+                type.vtabstruct_cname,
+                type.typeptr_cname,
+                code.error_goto_if_null(type.vtabptr_cname, pos)))
         env.types_imported[type] = 1
 
     py3_type_name_map = {'str' : 'bytes', 'unicode' : 'str'}
@@ -2109,24 +2109,24 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
                 # a significant performance hit. (See trac #561.)
                 for func in entry.type.scope.pyfunc_entries:
                     if func.is_special and Options.docstrings and func.wrapperbase_cname:
-                        code.putln("{");
+                        code.putln("{")
                         code.putln(
                             'PyObject *wrapper = __Pyx_GetAttrString((PyObject *)&%s, "%s"); %s' % (
                                 typeobj_cname,
                                 func.name,
-                                code.error_goto_if_null('wrapper', entry.pos)));
+                                code.error_goto_if_null('wrapper', entry.pos)))
                         code.putln(
-                            "if (Py_TYPE(wrapper) == &PyWrapperDescr_Type) {");
+                            "if (Py_TYPE(wrapper) == &PyWrapperDescr_Type) {")
                         code.putln(
                             "%s = *((PyWrapperDescrObject *)wrapper)->d_base;" % (
-                                func.wrapperbase_cname));
+                                func.wrapperbase_cname))
                         code.putln(
-                            "%s.doc = %s;" % (func.wrapperbase_cname, func.doc_cname));
+                            "%s.doc = %s;" % (func.wrapperbase_cname, func.doc_cname))
                         code.putln(
                             "((PyWrapperDescrObject *)wrapper)->d_base = &%s;" % (
-                                func.wrapperbase_cname));
-                        code.putln("}");
-                        code.putln("}");
+                                func.wrapperbase_cname))
+                        code.putln("}")
+                        code.putln("}")
                 if type.vtable_cname:
                     code.putln(
                         "if (__Pyx_SetVtable(%s.tp_dict, %s) < 0) %s" % (

Cython/Compiler/Nodes.py

@@ -2556,8 +2556,6 @@ class DefNode(FuncDefNode):
                     self.name, Naming.args_cname, self.error_value()))
             code.putln("}")
 
-        code.globalstate.use_utility_code(keyword_string_check_utility_code)
-
         if self.starstar_arg:
             if self.star_arg:
                 kwarg_check = "unlikely(%s)" % Naming.kwds_cname
@@ -2566,6 +2564,7 @@ class DefNode(FuncDefNode):
         else:
             kwarg_check = "unlikely(%s) && unlikely(PyDict_Size(%s) > 0)" % (
                 Naming.kwds_cname, Naming.kwds_cname)
+        code.globalstate.use_utility_code(keyword_string_check_utility_code)
         code.putln(
             "if (%s && unlikely(!__Pyx_CheckKeywordStrings(%s, \"%s\", %d))) return %s;" % (
                 kwarg_check, Naming.kwds_cname, self.name,
@@ -2629,8 +2628,6 @@ class DefNode(FuncDefNode):
         has_fixed_positional_count = not self.star_arg and \
             min_positional_args == max_positional_args
 
-        code.globalstate.use_utility_code(raise_double_keywords_utility_code)
-        code.globalstate.use_utility_code(raise_argtuple_invalid_utility_code)
         if self.num_required_kw_args:
             code.globalstate.use_utility_code(raise_keyword_required_utility_code)
 
@@ -2721,6 +2718,7 @@ class DefNode(FuncDefNode):
         if code.label_used(argtuple_error_label):
             code.put_goto(success_label)
             code.put_label(argtuple_error_label)
+            code.globalstate.use_utility_code(raise_argtuple_invalid_utility_code)
             code.put('__Pyx_RaiseArgtupleInvalid("%s", %d, %d, %d, PyTuple_GET_SIZE(%s)); ' % (
                     self.name, has_fixed_positional_count,
                     min_positional_args, max_positional_args,
@@ -2842,6 +2840,7 @@ class DefNode(FuncDefNode):
                             # kwargs) that were passed into positional
                             # arguments up to this point
                             code.putln('else {')
+                            code.globalstate.use_utility_code(raise_argtuple_invalid_utility_code)
                             code.put('__Pyx_RaiseArgtupleInvalid("%s", %d, %d, %d, %d); ' % (
                                     self.name, has_fixed_positional_count,
                                     min_positional_args, max_positional_args, i))
@@ -6237,7 +6236,8 @@ invalid_keyword:
 bad:
     return -1;
 }
-""")
+""",
+requires=[raise_double_keywords_utility_code])
 
 #------------------------------------------------------------------------------------
 
@@ -6381,25 +6381,26 @@ bad:
 
 get_vtable_utility_code = UtilityCode(
 proto = """
-static int __Pyx_GetVtable(PyObject *dict, void *vtabptr); /*proto*/
+static void* __Pyx_GetVtable(PyObject *dict); /*proto*/
 """,
 impl = r"""
-static int __Pyx_GetVtable(PyObject *dict, void *vtabptr) {
+static void* __Pyx_GetVtable(PyObject *dict) {
+    void* ptr;
     PyObject *ob = PyMapping_GetItemString(dict, (char *)"__pyx_vtable__");
     if (!ob)
         goto bad;
 #if PY_VERSION_HEX >= 0x02070000 && !(PY_MAJOR_VERSION==3&&PY_MINOR_VERSION==0)
-    *(void **)vtabptr = PyCapsule_GetPointer(ob, 0);
+    ptr = PyCapsule_GetPointer(ob, 0);
 #else
-    *(void **)vtabptr = PyCObject_AsVoidPtr(ob);
+    ptr = PyCObject_AsVoidPtr(ob);
 #endif
-    if (!*(void **)vtabptr)
-        goto bad;
+    if (!ptr && !PyErr_Occurred())
+        PyErr_SetString(PyExc_RuntimeError, "invalid vtable found for imported type");
     Py_DECREF(ob);
-    return 0;
+    return ptr;
 bad:
     Py_XDECREF(ob);
-    return -1;
+    return NULL;
 }
 """)
 

Cython/Compiler/PyrexTypes.py

@@ -353,6 +353,10 @@ class PyObjectType(PyrexType):
     def can_coerce_to_pyobject(self, env):
         return True
 
+    def default_coerced_ctype(self):
+        "The default C type that this Python type coerces to, or None."
+        return None
+
     def assignable_from(self, src_type):
         # except for pointers, conversion will be attempted
         return not src_type.is_ptr or src_type.is_string
@@ -404,6 +408,15 @@ class BuiltinObjectType(PyObjectType):
     def __repr__(self):
         return "<%s>"% self.cname
 
+    def default_coerced_ctype(self):
+        if self.name == 'bytes':
+            return c_char_ptr_type
+        elif self.name == 'bool':
+            return c_bint_type
+        elif self.name == 'float':
+            return c_double_type
+        return None
+
     def assignable_from(self, src_type):
         if isinstance(src_type, BuiltinObjectType):
             return src_type.name == self.name

tests/errors/string_assignments.pyx

@@ -58,18 +58,18 @@ _ERRORS = u"""
 30:22: Cannot convert Unicode string to 'bytes' implicitly, encoding required.
 31:22: Cannot convert 'str' to 'bytes' implicitly. This is not portable.
 
-33:17: Cannot assign type 'char *' to 'str object'
+33:17: Cannot convert 'bytes' object to str implicitly. This is not portable to Py3.
 34:19: Cannot convert 'bytes' object to str implicitly. This is not portable to Py3.
 35:17: Cannot convert Unicode string to 'str' implicitly. This is not portable and requires explicit encoding.
 36:19: Cannot convert Unicode string to 'str' implicitly. This is not portable and requires explicit encoding.
 
 38:20: str objects do not support coercion to unicode, use a unicode string literal instead (u'')
 39:22: str objects do not support coercion to unicode, use a unicode string literal instead (u'')
-40:20: Cannot assign type 'char *' to 'unicode object'
+40:20: Cannot convert 'bytes' object to unicode implicitly, decoding required
 41:22: Cannot convert 'bytes' object to unicode implicitly, decoding required
 42:22: Cannot convert 'char*' to unicode implicitly, decoding required
 
 44:19: Cannot assign type 'str object' to 'tuple object'
 45:18: Cannot assign type 'unicode object' to 'tuple object'
-46:18: Cannot assign type 'char *' to 'tuple object'
+46:18: Cannot assign type 'bytes object' to 'tuple object'
 """

tests/run/type_inference.pyx

@@ -25,7 +25,7 @@ def simple():
     xptrptr = &xptr
     assert typeof(xptrptr) == "double **", typeof(xptrptr)
     b = b"abc"
-    assert typeof(b) == "char *", typeof(b)
+    assert typeof(b) == "bytes object", typeof(b)
     s = "abc"
     assert typeof(s) == "str object", typeof(s)
     u = u"xyz"
@@ -57,7 +57,7 @@ def slicing():
     >>> slicing()
     """
     b = b"abc"
-    assert typeof(b) == "char *", typeof(b)
+    assert typeof(b) == "bytes object", typeof(b)
     b1 = b[1:2]
     assert typeof(b1) == "bytes object", typeof(b1)
     b2 = b[1:2:2]
@@ -92,9 +92,9 @@ def indexing():
     >>> indexing()
     """
     b = b"abc"
-    assert typeof(b) == "char *", typeof(b)
+    assert typeof(b) == "bytes object", typeof(b)
     b1 = b[1]
-    assert typeof(b1) == "char", typeof(b1)  # FIXME: Python object ??
+    assert typeof(b1) == "Python object", typeof(b1)
     u = u"xyz"
     assert typeof(u) == "unicode object", typeof(u)
     u1 = u[1]