Merge remote-tracking branch 'upstream/master'

.travis.yml

@@ -7,6 +7,7 @@ python:
   - 3.3
   - 3.4
   - 3.5
+  - 3.5-dev
   - pypy
   - pypy3
 
@@ -36,6 +37,7 @@ matrix:
   allow_failures:
     - python: pypy
     - python: pypy3
+    - python: 3.5-dev
   exclude:
     - python: pypy
       env: BACKEND=cpp

CHANGES.rst

@@ -28,6 +28,8 @@ Features added
 * Exception type tests have slightly lower overhead.
   This fixes ticket 868.
 
+* C++ classes can now be declared with default template parameters.
+
 Bugs fixed
 ----------
 

Cython/Build/Dependencies.py

@@ -45,10 +45,13 @@ except ImportError:
 from distutils.extension import Extension
 
 from .. import Utils
-from ..Utils import cached_function, cached_method, path_exists, find_root_package_dir, is_package_dir
+from ..Utils import (cached_function, cached_method, path_exists,
+    safe_makedirs, copy_file_to_dir_if_newer, is_package_dir)
 from ..Compiler.Main import Context, CompilationOptions, default_options
 
 join_path = cached_function(os.path.join)
+copy_once_if_newer = cached_function(copy_file_to_dir_if_newer)
+safe_makedirs_once = cached_function(safe_makedirs)
 
 if sys.version_info[0] < 3:
     # stupid Py2 distutils enforces str type in list of sources
@@ -122,6 +125,10 @@ def file_hash(filename):
 
 def parse_list(s):
     """
+    >>> parse_list("")
+    []
+    >>> parse_list("a")
+    ['a']
     >>> parse_list("a b c")
     ['a', 'b', 'c']
     >>> parse_list("[a, b, c]")
@@ -131,7 +138,7 @@ def parse_list(s):
     >>> parse_list('[a, ",a", "a,", ",", ]')
     ['a', ',a', 'a,', ',']
     """
-    if s[0] == '[' and s[-1] == ']':
+    if len(s) >= 2 and s[0] == '[' and s[-1] == ']':
         s = s[1:-1]
         delimiter = ','
     else:
@@ -642,6 +649,14 @@ def create_extension_list(patterns, exclude=None, ctx=None, aliases=None, quiet=
 
     module_list = []
     module_metadata = {}
+
+    # workaround for setuptools
+    if 'setuptools' in sys.modules:
+       Extension_setuptools = sys.modules['setuptools'].Extension
+    else:
+         # dummy class, in case we do not have setuptools
+         class Extension_setuptools(Extension): pass
+
     for pattern in patterns:
         if isinstance(pattern, str):
             filepattern = pattern
@@ -650,7 +665,7 @@ def create_extension_list(patterns, exclude=None, ctx=None, aliases=None, quiet=
             base = None
             exn_type = Extension
             ext_language = language
-        elif isinstance(pattern, Extension):
+        elif isinstance(pattern, (Extension, Extension_setuptools)):
             for filepattern in pattern.sources:
                 if os.path.splitext(filepattern)[1] in ('.py', '.pyx'):
                     break
@@ -664,7 +679,11 @@ def create_extension_list(patterns, exclude=None, ctx=None, aliases=None, quiet=
             exn_type = template.__class__
             ext_language = None  # do not override whatever the Extension says
         else:
-            raise TypeError(pattern)
+            msg = str("pattern is not of type str nor subclass of Extension (%s)"
+                      " but of type %s and class %s" % (repr(Extension),
+                                                        type(pattern),
+                                                        pattern.__class__))
+            raise TypeError(msg)
 
         for file in nonempty(sorted(extended_iglob(filepattern)), "'%s' doesn't match any files" % filepattern):
             if os.path.abspath(file) in to_exclude:
@@ -703,7 +722,7 @@ def create_extension_list(patterns, exclude=None, ctx=None, aliases=None, quiet=
                 else:
                     extra_sources = None
                 if 'depends' in kwds:
-                    depends = resolve_depends(kwds['depends'], (kwds.get('include_dirs') or []) + [find_root_package_dir(file)])
+                    depends = resolve_depends(kwds['depends'], (kwds.get('include_dirs') or []) + ["."])
                     if template is not None:
                         # Always include everything from the template.
                         depends = set(template.depends).union(depends)
@@ -762,8 +781,7 @@ def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False,
     if 'common_utility_include_dir' in options:
         if options.get('cache'):
             raise NotImplementedError("common_utility_include_dir does not yet work with caching")
-        if not os.path.exists(options['common_utility_include_dir']):
-            os.makedirs(options['common_utility_include_dir'])
+        safe_makedirs(options['common_utility_include_dir'])
     c_options = CompilationOptions(**options)
     cpp_options = CompilationOptions(**options); cpp_options.cplus = True
     ctx = c_options.create_context()
@@ -783,17 +801,15 @@ def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False,
     to_compile = []
     for m in module_list:
         if build_dir:
-            root = os.path.realpath(os.path.abspath(find_root_package_dir(m.sources[0])))
+            root = os.getcwd()  # distutil extension depends are relative to cwd
             def copy_to_build_dir(filepath, root=root):
-                filepath_abs = os.path.realpath(os.path.abspath(filepath))
+                filepath_abs = os.path.abspath(filepath)
                 if os.path.isabs(filepath):
                     filepath = filepath_abs
                 if filepath_abs.startswith(root):
-                    mod_dir = os.path.join(build_dir,
+                    mod_dir = join_path(build_dir,
                             os.path.dirname(_relpath(filepath, root)))
-                    if not os.path.isdir(mod_dir):
-                        os.makedirs(mod_dir)
-                    shutil.copy(filepath, mod_dir)
+                    copy_once_if_newer(filepath_abs, mod_dir)
             for dep in m.depends:
                 copy_to_build_dir(dep)
 
@@ -812,8 +828,7 @@ def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False,
                 if build_dir:
                     c_file = os.path.join(build_dir, c_file)
                     dir = os.path.dirname(c_file)
-                    if not os.path.isdir(dir):
-                        os.makedirs(dir)
+                    safe_makedirs_once(dir)
 
                 if os.path.exists(c_file):
                     c_timestamp = os.path.getmtime(c_file)

Cython/Compiler/Code.py

@@ -2145,8 +2145,9 @@ class CCodeWriter(object):
     def error_goto(self, pos):
         lbl = self.funcstate.error_label
         self.funcstate.use_label(lbl)
-        return "{%s goto %s;}" % (
-            self.set_error_info(pos),
+        return "__PYX_ERR(%s, %s, %s)" % (
+            self.lookup_filename(pos[0]),
+            pos[1],
             lbl)
 
     def error_goto_if(self, cond, pos):

Cython/Compiler/ExprNodes.py

@@ -888,7 +888,8 @@ class ExprNode(Node):
             # Added the string comparison, since for c types that
             # is enough, but Cython gets confused when the types are
             # in different pxi files.
-            if not (str(src.type) == str(dst_type) or dst_type.assignable_from(src_type)):
+            # TODO: Remove this hack and require shared declarations.
+            if not (src.type == dst_type or str(src.type) == str(dst_type) or dst_type.assignable_from(src_type)):
                 self.fail_assignment(dst_type)
         return src
 
@@ -11150,6 +11151,12 @@ class CondExprNode(ExprNode):
             self.type_error()
         return self
 
+    def coerce_to_integer(self, env):
+        self.true_val = self.true_val.coerce_to_integer(env)
+        self.false_val = self.false_val.coerce_to_integer(env)
+        self.result_ctype = None
+        return self.analyse_result_type(env)
+
     def coerce_to(self, dst_type, env):
         self.true_val = self.true_val.coerce_to(dst_type, env)
         self.false_val = self.false_val.coerce_to(dst_type, env)

Cython/Compiler/FlowControl.py

@@ -341,6 +341,14 @@ class NameAssignment(object):
             return self.entry.type
         return self.inferred_type
 
+    def __getstate__(self):
+        return (self.lhs, self.rhs, self.entry, self.pos,
+                self.refs, self.is_arg, self.is_deletion, self.inferred_type)
+
+    def __setstate__(self, state):
+        (self.lhs, self.rhs, self.entry, self.pos,
+         self.refs, self.is_arg, self.is_deletion, self.inferred_type) = state
+
 
 class StaticAssignment(NameAssignment):
     """Initialised at declaration time, e.g. stack allocation."""

Cython/Compiler/ModuleNode.py

@@ -40,6 +40,17 @@ def check_c_declarations(module_node):
     module_node.scope.check_c_functions()
     return module_node
 
+def generate_c_code_config(env, options):
+    if Options.annotate or options.annotate:
+        emit_linenums = False
+    else:
+        emit_linenums = options.emit_linenums
+        rootwriter = Code.CCodeWriter()
+
+    return Code.CCodeConfig(emit_linenums=emit_linenums,
+        emit_code_comments=env.directives['emit_code_comments'],
+        c_line_in_traceback=options.c_line_in_traceback)
+
 class ModuleNode(Nodes.Node, Nodes.BlockNode):
     #  doc       string or None
     #  body      StatListNode
@@ -117,7 +128,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         self.sort_cdef_classes(env)
         self.generate_c_code(env, options, result)
         self.generate_h_code(env, options, result)
-        self.generate_api_code(env, result)
+        self.generate_api_code(env, options, result)
 
     def has_imported_c_functions(self):
         for module in self.referenced_modules:
@@ -139,7 +150,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         if (h_types or  h_vars or h_funcs or h_extension_types):
             result.h_file = replace_suffix(result.c_file, ".h")
             h_code = Code.CCodeWriter()
-            Code.GlobalState(h_code, self, Code.CCodeConfig())  # FIXME: config?
+            c_code_config = generate_c_code_config(env, options)
+            Code.GlobalState(h_code, self, c_code_config)
             if options.generate_pxi:
                 result.i_file = replace_suffix(result.c_file, ".pxi")
                 i_code = Code.PyrexCodeWriter(result.i_file)
@@ -203,7 +215,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
     def api_name(self, env):
         return env.qualified_name.replace(".", "__")
 
-    def generate_api_code(self, env, result):
+    def generate_api_code(self, env, options, result):
         def api_entries(entries, pxd=0):
             return [entry for entry in entries
                     if entry.api or (pxd and entry.defined_in_pxd)]
@@ -213,7 +225,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         if api_vars or api_funcs or api_extension_types:
             result.api_file = replace_suffix(result.c_file, "_api.h")
             h_code = Code.CCodeWriter()
-            Code.GlobalState(h_code, self, Code.CCodeConfig())  # FIXME: config?
+            c_code_config = generate_c_code_config(env, options)
+            Code.GlobalState(h_code, self, c_code_config)
             h_code.put_generated_by()
             api_guard = Naming.api_guard_prefix + self.api_name(env)
             h_code.put_h_guard(api_guard)
@@ -308,17 +321,12 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         modules = self.referenced_modules
 
         if Options.annotate or options.annotate:
-            emit_linenums = False
             rootwriter = Annotate.AnnotationCCodeWriter()
         else:
-            emit_linenums = options.emit_linenums
             rootwriter = Code.CCodeWriter()
 
-        c_code_config = Code.CCodeConfig(
-            emit_linenums=emit_linenums,
-            emit_code_comments=env.directives['emit_code_comments'],
-            c_line_in_traceback=options.c_line_in_traceback,
-        )
+        c_code_config = generate_c_code_config(env, options)
+
         globalstate = Code.GlobalState(
             rootwriter, self,
             code_config=c_code_config,
@@ -327,7 +335,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         globalstate.initialize_main_c_code()
         h_code = globalstate['h_code']
 
-        self.generate_module_preamble(env, modules, result.embedded_metadata, h_code)
+        self.generate_module_preamble(env, options, modules, result.embedded_metadata, h_code)
 
         globalstate.module_pos = self.pos
         globalstate.directives = self.directives
@@ -582,7 +590,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
     def _put_setup_code(self, code, name):
         code.put(UtilityCode.load_as_string(name, "ModuleSetupCode.c")[1])
 
-    def generate_module_preamble(self, env, cimported_modules, metadata, code):
+    def generate_module_preamble(self, env, options, cimported_modules, metadata, code):
         code.put_generated_by()
         if metadata:
             code.putln("/* BEGIN: Cython Metadata")
@@ -610,6 +618,18 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
             self._put_setup_code(code, "CInitCode")
         self._put_setup_code(code, "MathInitCode")
 
+        if options.c_line_in_traceback:
+            cinfo = "%s = %s; " % (Naming.clineno_cname, Naming.line_c_macro)
+        else:
+            cinfo = ""
+        code.put("""
+#define __PYX_ERR(f_index, lineno, Ln_error) \\
+{ \\
+  %s = %s[f_index]; %s = lineno; %sgoto Ln_error; \\
+}
+""" % (Naming.filename_cname, Naming.filetable_cname, Naming.lineno_cname,
+       cinfo))
+
         code.put("""
 #if PY_MAJOR_VERSION >= 3
   #define __Pyx_PyNumber_Divide(x,y)         PyNumber_TrueDivide(x,y)

Cython/Compiler/Nodes.py

@@ -1403,7 +1403,7 @@ class CppClassNode(CStructOrUnionDefNode, BlockNode):
     #  attributes    [CVarDefNode] or None
     #  entry         Entry
     #  base_classes  [CBaseTypeNode]
-    #  templates     [string] or None
+    #  templates     [(string, bool)] or None
     #  decorators    [DecoratorNode] or None
 
     decorators = None
@@ -1412,25 +1412,31 @@ class CppClassNode(CStructOrUnionDefNode, BlockNode):
         if self.templates is None:
             template_types = None
         else:
-            template_types = [PyrexTypes.TemplatePlaceholderType(template_name) for template_name in self.templates]
+            template_types = [PyrexTypes.TemplatePlaceholderType(template_name, not required)
+                              for template_name, required in self.templates]
+            num_optional_templates = sum(not required for _, required in self.templates)
+            if num_optional_templates and not all(required for _, required in self.templates[:-num_optional_templates]):
+                error(self.pos, "Required template parameters must precede optional template parameters.")
         self.entry = env.declare_cpp_class(
             self.name, None, self.pos,
             self.cname, base_classes = [], visibility = self.visibility, templates = template_types)
 
     def analyse_declarations(self, env):
+        if self.templates is None:
+            template_types = template_names = None
+        else:
+            template_names = [template_name for template_name, _ in self.templates]
+            template_types = [PyrexTypes.TemplatePlaceholderType(template_name, not required)
+                              for template_name, required in self.templates]
         scope = None
         if self.attributes is not None:
-            scope = CppClassScope(self.name, env, templates = self.templates)
+            scope = CppClassScope(self.name, env, templates = template_names)
         def base_ok(base_class):
             if base_class.is_cpp_class or base_class.is_struct:
                 return True
             else:
                 error(self.pos, "Base class '%s' not a struct or class." % base_class)
         base_class_types = filter(base_ok, [b.analyse(scope or env) for b in self.base_classes])
-        if self.templates is None:
-            template_types = None
-        else:
-            template_types = [PyrexTypes.TemplatePlaceholderType(template_name) for template_name in self.templates]
         self.entry = env.declare_cpp_class(
             self.name, scope, self.pos,
             self.cname, base_class_types, visibility = self.visibility, templates = template_types)
@@ -1455,7 +1461,7 @@ class CppClassNode(CStructOrUnionDefNode, BlockNode):
             for func in func_attributes(self.attributes):
                 defined_funcs.append(func)
                 if self.templates is not None:
-                    func.template_declaration = "template <typename %s>" % ", typename ".join(self.templates)
+                    func.template_declaration = "template <typename %s>" % ", typename ".join(template_names)
         self.body = StatListNode(self.pos, stats=defined_funcs)
         self.scope = scope
 

Cython/Compiler/Optimize.py

@@ -2021,6 +2021,9 @@ class OptimizeBuiltinCalls(Visitor.NodeRefCleanupMixin,
                 if node.type.assignable_from(arg.arg.type):
                     # completely redundant C->Py->C coercion
                     return arg.arg.coerce_to(node.type, self.current_env())
+            elif arg.type is Builtin.unicode_type:
+                if arg.arg.type.is_unicode_char and node.type.is_unicode_char:
+                    return arg.arg.coerce_to(node.type, self.current_env())
         elif isinstance(arg, ExprNodes.SimpleCallNode):
             if node.type.is_int or node.type.is_float:
                 return self._optimise_numeric_cast_call(node, arg)

Cython/Compiler/ParseTreeTransforms.py

@@ -2191,6 +2191,8 @@ class AlignFunctionDefinitions(CythonTransform):
         if pxd_def is None:
             pxd_def = self.scope.lookup(node.class_name)
         if pxd_def:
+            if not pxd_def.defined_in_pxd:
+                return node
             outer_scope = self.scope
             self.scope = pxd_def.type.scope
         self.visitchildren(node)

Cython/Compiler/Parsing.pxd

@@ -187,5 +187,6 @@ cdef p_property_decl(PyrexScanner s)
 cdef p_doc_string(PyrexScanner s)
 cdef p_ignorable_statement(PyrexScanner s)
 cdef p_compiler_directive_comments(PyrexScanner s)
+cdef p_template_definition(PyrexScanner s)
 cdef p_cpp_class_definition(PyrexScanner s, pos, ctx)
 cdef p_cpp_class_attribute(PyrexScanner s, ctx)

Cython/Compiler/Parsing.py

@@ -3363,6 +3363,16 @@ def p_module(s, pxd, full_module_name, ctx=Ctx):
                       full_module_name = full_module_name,
                       directive_comments = directive_comments)
 
+def p_template_definition(s):
+    name = p_ident(s)
+    if s.sy == '=':
+        s.expect('=')
+        s.expect('*')
+        required = False
+    else:
+        required = True
+    return name, required
+
 def p_cpp_class_definition(s, pos,  ctx):
     # s.sy == 'cppclass'
     s.next()
@@ -3375,19 +3385,21 @@ def p_cpp_class_definition(s, pos,  ctx):
         error(pos, "Qualified class name not allowed C++ class")
     if s.sy == '[':
         s.next()
-        templates = [p_ident(s)]
+        templates = [p_template_definition(s)]
         while s.sy == ',':
             s.next()
-            templates.append(p_ident(s))
+            templates.append(p_template_definition(s))
         s.expect(']')
+        template_names = [name for name, required in templates]
     else:
         templates = None
+        template_names = None
     if s.sy == '(':
         s.next()
-        base_classes = [p_c_base_type(s, templates = templates)]
+        base_classes = [p_c_base_type(s, templates = template_names)]
         while s.sy == ',':
             s.next()
-            base_classes.append(p_c_base_type(s, templates = templates))
+            base_classes.append(p_c_base_type(s, templates = template_names))
         s.expect(')')
     else:
         base_classes = []
@@ -3400,7 +3412,7 @@ def p_cpp_class_definition(s, pos,  ctx):
         s.expect_indent()
         attributes = []
         body_ctx = Ctx(visibility = ctx.visibility, level='cpp_class', nogil=nogil or ctx.nogil)
-        body_ctx.templates = templates
+        body_ctx.templates = template_names
         while s.sy != 'DEDENT':
             if s.sy != 'pass':
                 attributes.append(p_cpp_class_attribute(s, body_ctx))

Cython/Compiler/PyrexTypes.py

@@ -16,7 +16,7 @@ from .Code import UtilityCode, LazyUtilityCode, TempitaUtilityCode
 from . import StringEncoding
 from . import Naming
 
-from .Errors import error
+from .Errors import error, warning
 
 
 class BaseType(object):
@@ -2061,7 +2061,8 @@ proto="""
     #define __Pyx_CIMAG(z) ((z).imag)
 #endif
 
-#if (defined(_WIN32) || defined(__clang__)) && defined(__cplusplus) && CYTHON_CCOMPLEX
+#if defined(__cplusplus) && CYTHON_CCOMPLEX \
+        && (defined(_WIN32) || defined(__clang__) || (defined(__GNUC__) && (__GNUC__ >= 5 || __GNUC__ == 4 && __GNUC_MINOR__ >= 4 )) || __cplusplus >= 201103)
     #define __Pyx_SET_CREAL(z,x) ((z).real(x))
     #define __Pyx_SET_CIMAG(z,y) ((z).imag(y))
 #else
@@ -3257,8 +3258,10 @@ class CStructOrUnionType(CType):
         self.scope = scope
         self.typedef_flag = typedef_flag
         self.is_struct = kind == 'struct'
-        self.to_py_function = "%s_to_py_%s" % (Naming.convert_func_prefix, self.cname)
-        self.from_py_function = "%s_from_py_%s" % (Naming.convert_func_prefix, self.cname)
+        self.to_py_function = "%s_to_py_%s" % (
+            Naming.convert_func_prefix, self.specialization_name())
+        self.from_py_function = "%s_from_py_%s" % (
+            Naming.convert_func_prefix, self.specialization_name())
         self.exception_check = True
         self._convert_to_py_code = None
         self._convert_from_py_code = None
@@ -3395,7 +3398,6 @@ builtin_cpp_conversions = ("std::pair",
                            "std::set", "std::unordered_set",
                            "std::map", "std::unordered_map")
 
-
 class CppClassType(CType):
     #  name          string
     #  cname         string
@@ -3422,6 +3424,7 @@ class CppClassType(CType):
         self.operators = []
         self.templates = templates
         self.template_type = template_type
+        self.num_optional_templates = sum(is_optional_template_param(T) for T in templates or ())
         self.specializations = {}
         self.is_cpp_string = cname in cpp_string_conversions
 
@@ -3551,6 +3554,21 @@ class CppClassType(CType):
         if not self.is_template_type():
             error(pos, "'%s' type is not a template" % self)
             return error_type
+        if len(self.templates) - self.num_optional_templates <= len(template_values) < len(self.templates):
+            num_defaults = len(self.templates) - len(template_values)
+            partial_specialization = self.declaration_code('', template_params=template_values)
+            # Most of the time we don't need to declare anything typed to these
+            # default template arguments, but when we do there's no way in C++
+            # to reference this directly.  However, it is common convention to
+            # provide a typedef in the template class that resolves to each
+            # template type.  For now, allow the user to specify this name as
+            # the template parameter.
+            # TODO: Allow typedefs in cpp classes and search for it in this
+            # classes scope as a concrete name we could use.
+            template_values = template_values + [
+                TemplatePlaceholderType(
+                    "%s::%s" % (partial_specialization, param.name), True)
+                for param in self.templates[-num_defaults:]]
         if len(self.templates) != len(template_values):
             error(pos, "%s templated type receives %d arguments, got %d" %
                   (self.name, len(self.templates), len(template_values)))
@@ -3598,10 +3616,14 @@ class CppClassType(CType):
             return None
 
     def declaration_code(self, entity_code,
-            for_display = 0, dll_linkage = None, pyrex = 0):
+            for_display = 0, dll_linkage = None, pyrex = 0,
+            template_params = None):
+        if template_params is None:
+            template_params = self.templates
         if self.templates:
             template_strings = [param.declaration_code('', for_display, None, pyrex)
-                                for param in self.templates]
+                                for param in template_params
+                                if not is_optional_template_param(param)]
             if for_display:
                 brackets = "[%s]"
             else:
@@ -3670,8 +3692,9 @@ class CppClassType(CType):
 
 class TemplatePlaceholderType(CType):
 
-    def __init__(self, name):
+    def __init__(self, name, optional=False):
         self.name = name
+        self.optional = optional
 
     def declaration_code(self, entity_code,
             for_display = 0, dll_linkage = None, pyrex = 0):
@@ -3710,6 +3733,9 @@ class TemplatePlaceholderType(CType):
         else:
             return False
 
+def is_optional_template_param(type):
+    return isinstance(type, TemplatePlaceholderType) and type.optional
+
 
 class CEnumType(CType):
     #  name           string
@@ -4250,6 +4276,10 @@ def merge_template_deductions(a, b):
 def widest_numeric_type(type1, type2):
     """Given two numeric types, return the narrowest type encompassing both of them.
     """
+    if type1.is_reference:
+        type1 = type1.ref_base_type
+    if type2.is_reference:
+        type2 = type2.ref_base_type
     if type1 == type2:
         widest_type = type1
     elif type1.is_complex or type2.is_complex:

Cython/Compiler/Scanning.py

@@ -158,8 +158,11 @@ class SourceDescriptor(object):
 
     def get_escaped_description(self):
         if self._escaped_description is None:
-            self._escaped_description = \
+            esc_desc = \
                 self.get_description().encode('ASCII', 'replace').decode("ASCII")
+            # Use foreward slashes on Windows since these paths
+            # will be used in the #line directives in the C/C++ files.
+            self._escaped_description = esc_desc.replace('\\', '/')
         return self._escaped_description
 
     def __gt__(self, other):

Cython/Compiler/Tests/TestFlowControl.py

+
+from __future__ import absolute_import
+
+from copy import deepcopy
+from unittest import TestCase
+
+from Cython.Compiler.FlowControl import (
+    NameAssignment, StaticAssignment, Argument, NameDeletion)
+
+
+class FakeType(object):
+    is_pyobject = True
+
+
+class FakeNode(object):
+    pos = ('filename.pyx', 1, 2)
+    cf_state = None
+    type = FakeType()
+
+    def infer_type(self, scope):
+        return self.type
+
+
+class FakeEntry(object):
+    type = FakeType()
+
+
+class TestGraph(TestCase):
+    def test_deepcopy(self):
+        lhs, rhs = FakeNode(), FakeNode()
+        entry = FakeEntry()
+        entry.pos = lhs.pos
+
+        name_ass = NameAssignment(lhs, rhs, entry)
+        ass = deepcopy(name_ass)
+        self.assertTrue(ass.lhs)
+        self.assertTrue(ass.rhs)
+        self.assertTrue(ass.entry)
+        self.assertEqual(ass.pos, name_ass.pos)
+        self.assertFalse(ass.is_arg)
+        self.assertFalse(ass.is_deletion)
+
+        static_ass = StaticAssignment(entry)
+        ass = deepcopy(static_ass)
+        self.assertTrue(ass.lhs)
+        self.assertTrue(ass.rhs)
+        self.assertTrue(ass.entry)
+        self.assertEqual(ass.pos, static_ass.pos)
+        self.assertFalse(ass.is_arg)
+        self.assertFalse(ass.is_deletion)
+
+        arg_ass = Argument(lhs, rhs, entry)
+        ass = deepcopy(arg_ass)
+        self.assertTrue(ass.lhs)
+        self.assertTrue(ass.rhs)
+        self.assertTrue(ass.entry)
+        self.assertEqual(ass.pos, arg_ass.pos)
+        self.assertTrue(ass.is_arg)
+        self.assertFalse(ass.is_deletion)
+
+        name_del = NameDeletion(lhs, entry)
+        ass = deepcopy(name_del)
+        self.assertTrue(ass.lhs)
+        self.assertTrue(ass.rhs)
+        self.assertTrue(ass.entry)
+        self.assertEqual(ass.pos, name_del.pos)
+        self.assertFalse(ass.is_arg)
+        self.assertTrue(ass.is_deletion)

Cython/Compiler/TypeInference.py

@@ -398,7 +398,7 @@ class SimpleAssignmentTypeInferer(object):
             else:
                 entry = node.entry
                 node_type = spanning_type(
-                    types, entry.might_overflow, entry.pos)
+                    types, entry.might_overflow, entry.pos, scope)
             node.inferred_type = node_type
 
         def infer_name_node_type_partial(node):
@@ -407,7 +407,7 @@ class SimpleAssignmentTypeInferer(object):
             if not types:
                 return
             entry = node.entry
-            return spanning_type(types, entry.might_overflow, entry.pos)
+            return spanning_type(types, entry.might_overflow, entry.pos, scope)
 
         def resolve_assignments(assignments):
             resolved = set()
@@ -464,7 +464,7 @@ class SimpleAssignmentTypeInferer(object):
                 types = [assmt.inferred_type for assmt in entry.cf_assignments]
                 if types and all(types):
                     entry_type = spanning_type(
-                        types, entry.might_overflow, entry.pos)
+                        types, entry.might_overflow, entry.pos, scope)
                     inferred.add(entry)
             self.set_entry_type(entry, entry_type)
 
@@ -473,7 +473,7 @@ class SimpleAssignmentTypeInferer(object):
             for entry in inferred:
                 types = [assmt.infer_type()
                          for assmt in entry.cf_assignments]
-                new_type = spanning_type(types, entry.might_overflow, entry.pos)
+                new_type = spanning_type(types, entry.might_overflow, entry.pos, scope)
                 if new_type != entry.type:
                     self.set_entry_type(entry, new_type)
                     dirty = True
@@ -516,10 +516,10 @@ def simply_type(result_type, pos):
         result_type = PyrexTypes.c_ptr_type(result_type.base_type)
     return result_type
 
-def aggressive_spanning_type(types, might_overflow, pos):
+def aggressive_spanning_type(types, might_overflow, pos, scope):
     return simply_type(reduce(find_spanning_type, types), pos)
 
-def safe_spanning_type(types, might_overflow, pos):
+def safe_spanning_type(types, might_overflow, pos, scope):
     result_type = simply_type(reduce(find_spanning_type, types), pos)
     if result_type.is_pyobject:
         # In theory, any specific Python type is always safe to
@@ -554,6 +554,8 @@ def safe_spanning_type(types, might_overflow, pos):
     # to make sure everything is supported.
     elif (result_type.is_int or result_type.is_enum) and not might_overflow:
         return result_type
+    elif not result_type.can_coerce_to_pyobject(scope):
+        return result_type
     return py_object_type
 
 

Cython/Coverage.py

@@ -8,6 +8,7 @@ from __future__ import absolute_import
 
 import re
 import os.path
+import sys
 from collections import defaultdict
 
 from coverage.plugin import CoveragePlugin, FileTracer, FileReporter  # requires coverage.py 4.0+
@@ -35,6 +36,12 @@ def _find_dep_file_path(main_file, file_path):
         pxi_file_path = os.path.join(os.path.dirname(main_file), file_path)
         if os.path.exists(pxi_file_path):
             abs_path = os.path.abspath(pxi_file_path)
+    # search sys.path for external locations if a valid file hasn't been found
+    if not os.path.exists(abs_path):
+        for sys_path in sys.path:
+            test_path = os.path.realpath(os.path.join(sys_path, file_path))
+            if os.path.exists(test_path):
+                return test_path
     return abs_path
 
 
@@ -132,7 +139,7 @@ class Plugin(CoveragePlugin):
             try:
                 with open(c_file, 'rb') as f:
                     if b'/* Generated by Cython ' not in f.read(30):
-                        return None  # not a Cython file
+                        return None, None  # not a Cython file
             except (IOError, OSError):
                 c_file = None
 
@@ -238,7 +245,7 @@ class CythonModuleTracer(FileTracer):
             return self._file_path_map[source_file]
         except KeyError:
             pass
-        abs_path = os.path.abspath(source_file)
+        abs_path = _find_dep_file_path(filename, source_file)
 
         if self.py_file and source_file[-3:].lower() == '.py':
             # always let coverage.py handle this case itself

Cython/Distutils/build_ext.py

@@ -86,7 +86,7 @@ class build_ext(_build_ext.build_ext):
          "generate debug information for cygdb"),
         ('cython-compile-time-env', None,
             "cython compile time environment"),
-            
+
         # For backwards compatibility.
         ('pyrex-cplus', None,
          "generate C++ source files"),
@@ -109,7 +109,7 @@ class build_ext(_build_ext.build_ext):
     boolean_options.extend([
         'cython-cplus', 'cython-create-listing', 'cython-line-directives',
         'cython-c-in-temp', 'cython-gdb',
-        
+
         # For backwards compatibility.
         'pyrex-cplus', 'pyrex-create-listing', 'pyrex-line-directives',
         'pyrex-c-in-temp', 'pyrex-gdb',
@@ -127,7 +127,7 @@ class build_ext(_build_ext.build_ext):
         self.cython_gdb = False
         self.no_c_in_traceback = 0
         self.cython_compile_time_env = None
-    
+
     def __getattr__(self, name):
         if name[:6] == 'pyrex_':
             return getattr(self, 'cython_' + name[6:])
@@ -236,6 +236,10 @@ class build_ext(_build_ext.build_ext):
                     includes.append(i)
         except AttributeError:
             pass
+
+        # In case extension.include_dirs is a generator, evaluate it and keep
+        # result
+        extension.include_dirs = list(extension.include_dirs)
         for i in extension.include_dirs:
             if not i in includes:
                 includes.append(i)

Cython/Includes/cpython/array.pxd

@@ -52,7 +52,7 @@ from libc.string cimport strcat, strncat, \
 from cpython.object cimport Py_SIZE
 from cpython.ref cimport PyTypeObject, Py_TYPE
 from cpython.exc cimport PyErr_BadArgument
-from cpython.mem cimport PyMem_Malloc, PyMem_Free
+from cpython.mem cimport PyObject_Malloc, PyObject_Free
 
 cdef extern from *:  # Hard-coded utility code hack.
     ctypedef class array.array [object arrayobject]
@@ -102,7 +102,7 @@ cdef extern from *:  # Hard-coded utility code hack.
             info.itemsize = self.ob_descr.itemsize   # e.g. sizeof(float)
             info.len = info.itemsize * item_count
 
-            info.shape = <Py_ssize_t*> PyMem_Malloc(sizeof(Py_ssize_t) + 2)
+            info.shape = <Py_ssize_t*> PyObject_Malloc(sizeof(Py_ssize_t) + 2)
             if not info.shape:
                 raise MemoryError()
             info.shape[0] = item_count      # constant regardless of resizing
@@ -114,7 +114,7 @@ cdef extern from *:  # Hard-coded utility code hack.
             info.obj = self
 
         def __releasebuffer__(self, Py_buffer* info):
-            PyMem_Free(info.shape)
+            PyObject_Free(info.shape)
 
     array newarrayobject(PyTypeObject* type, Py_ssize_t size, arraydescr *descr)
 

Cython/Includes/cpython/mem.pxd

@@ -73,3 +73,36 @@ cdef extern from "Python.h":
 
     # PyMem_MALLOC(), PyMem_REALLOC(), PyMem_FREE().
     # PyMem_NEW(), PyMem_RESIZE(), PyMem_DEL().
+
+
+    #####################################################################
+    # Raw object memory interface
+    #####################################################################
+
+    # Functions to call the same malloc/realloc/free as used by Python's
+    # object allocator.  If WITH_PYMALLOC is enabled, these may differ from
+    # the platform malloc/realloc/free.  The Python object allocator is
+    # designed for fast, cache-conscious allocation of many "small" objects,
+    # and with low hidden memory overhead.
+    #
+    # PyObject_Malloc(0) returns a unique non-NULL pointer if possible.
+    #
+    # PyObject_Realloc(NULL, n) acts like PyObject_Malloc(n).
+    # PyObject_Realloc(p != NULL, 0) does not return  NULL, or free the memory
+    # at p.
+    #
+    # Returned pointers must be checked for NULL explicitly; no action is
+    # performed on failure other than to return NULL (no warning it printed, no
+    # exception is set, etc).
+    #
+    # For allocating objects, use PyObject_{New, NewVar} instead whenever
+    # possible.  The PyObject_{Malloc, Realloc, Free} family is exposed
+    # so that you can exploit Python's small-block allocator for non-object
+    # uses.  If you must use these routines to allocate object memory, make sure
+    # the object gets initialized via PyObject_{Init, InitVar} after obtaining
+    # the raw memory.
+
+    void* PyObject_Malloc(size_t size)
+    void* PyObject_Calloc(size_t nelem, size_t elsize)
+    void* PyObject_Realloc(void *ptr, size_t new_size)
+    void PyObject_Free(void *ptr)

Cython/Includes/libcpp/iterator.pxd

+#Basic reference: http://www.cplusplus.com/reference/iterator/
+#Most of these classes are in fact empty structs
+
+cdef extern from "<iterator>" namespace "std" nogil:
+    cdef cppclass iterator[Category,T,Distance,Pointer,Reference]:
+        pass
+    cdef cppclass output_iterator_tag:
+        pass
+    cdef cppclass input_iterator_tag:
+        pass
+    cdef cppclass forward_iterator_tag(input_iterator_tag):
+        pass
+    cdef cppclass bidirectional_iterator_tag(forward_iterator_tag):
+        pass
+    cdef cppclass random_access_iterator_tag(bidirectional_iterator_tag):
+        pass
+    
+    cdef cppclass back_insert_iterator[T](iterator[output_iterator_tag,void,void,void,void]):
+        pass
+    cdef cppclass front_insert_iterator[T](iterator[output_iterator_tag,void,void,void,void]):
+        pass
+    cdef cppclass insert_iterator[T](iterator[output_iterator_tag,void,void,void,void]):
+        pass
+    back_insert_iterator[CONTAINER] back_inserter[CONTAINER](CONTAINER &)
+    front_insert_iterator[CONTAINER] front_inserter[CONTAINER](CONTAINER &)
+    ##Note: this is the C++98 version of inserter.
+    ##The C++11 versions's prototype relies on typedef members of classes, which Cython doesn't currently support:
+    ##template <class Container>
+    ##insert_iterator<Container> inserter (Container& x, typename Container::iterator it)
+    insert_iterator[CONTAINER] inserter[CONTAINER,ITERATOR](CONTAINER &, ITERATOR)
+
+    

Cython/Utility/Coroutine.c

@@ -566,8 +566,7 @@ static PyObject *__Pyx_Coroutine_Send(PyObject *self, PyObject *value) {
         #endif
         {
             if (value == Py_None)
-                // FIXME - is this the right thing to do?
-                ret = PyIter_Next(yf);
+                ret = Py_TYPE(yf)->tp_iternext(yf);
             else
                 ret = __Pyx_PyObject_CallMethod1(yf, PYIDENT("send"), value);
         }

Cython/Utility/CythonFunction.c

@@ -498,7 +498,7 @@ __Pyx_CyFunction_clear(__pyx_CyFunctionObject *m)
         for (i = 0; i < m->defaults_pyobjects; i++)
             Py_XDECREF(pydefaults[i]);
 
-        PyMem_Free(m->defaults);
+        PyObject_Free(m->defaults);
         m->defaults = NULL;
     }
 
@@ -708,7 +708,7 @@ static int __pyx_CyFunction_init(void) {
 static CYTHON_INLINE void *__Pyx_CyFunction_InitDefaults(PyObject *func, size_t size, int pyobjects) {
     __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func;
 
-    m->defaults = PyMem_Malloc(size);
+    m->defaults = PyObject_Malloc(size);
     if (!m->defaults)
         return PyErr_NoMemory();
     memset(m->defaults, 0, size);

Cython/Utility/ImportExport.c

@@ -394,14 +394,14 @@ static PyTypeObject *__Pyx_ImportType(const char *module_name, const char *class
 #endif
     if (!strict && (size_t)basicsize > size) {
         PyOS_snprintf(warning, sizeof(warning),
-            "%s.%s size changed, may indicate binary incompatibility",
-            module_name, class_name);
+            "%s.%s size changed, may indicate binary incompatibility. Expected %zd, got %zd",
+            module_name, class_name, basicsize, size);
         if (PyErr_WarnEx(NULL, warning, 0) < 0) goto bad;
     }
     else if ((size_t)basicsize != size) {
         PyErr_Format(PyExc_ValueError,
-            "%.200s.%.200s has the wrong size, try recompiling",
-            module_name, class_name);
+            "%.200s.%.200s has the wrong size, try recompiling. Expected %zd, got %zd",
+            module_name, class_name, basicsize, size);
         goto bad;
     }
     return (PyTypeObject *)result;

Cython/Utility/MemoryView.pyx

@@ -33,6 +33,8 @@ cdef extern from *:
 
     void* PyMem_Malloc(size_t n)
     void PyMem_Free(void *p)
+    void* PyObject_Malloc(size_t n)
+    void PyObject_Free(void *p)
 
     cdef struct __pyx_memoryview "__pyx_memoryview_obj":
         Py_buffer view
@@ -137,7 +139,7 @@ cdef class array:
         self.format = self._format
 
         # use single malloc() for both shape and strides
-        self._shape = <Py_ssize_t *> PyMem_Malloc(sizeof(Py_ssize_t)*self.ndim*2)
+        self._shape = <Py_ssize_t *> PyObject_Malloc(sizeof(Py_ssize_t)*self.ndim*2)
         self._strides = self._shape + self.ndim
 
         if not self._shape:
@@ -212,7 +214,7 @@ cdef class array:
                 refcount_objects_in_slice(self.data, self._shape,
                                           self._strides, self.ndim, False)
             free(self.data)
-        PyMem_Free(self._shape)
+        PyObject_Free(self._shape)
 
     property memview:
         @cname('get_memview')

Cython/Utils.py

@@ -15,6 +15,7 @@ import sys
 import re
 import io
 import codecs
+import shutil
 from contextlib import contextmanager
 
 modification_time = os.path.getmtime
@@ -84,6 +85,34 @@ def file_newer_than(path, time):
     ftime = modification_time(path)
     return ftime > time
 
+
+def safe_makedirs(path):
+    try:
+        os.makedirs(path)
+    except OSError:
+        if not os.path.isdir(path):
+            raise
+
+
+def copy_file_to_dir_if_newer(sourcefile, destdir):
+    """
+    Copy file sourcefile to directory destdir (creating it if needed),
+    preserving metadata. If the destination file exists and is not
+    older than the source file, the copying is skipped.
+    """
+    destfile = os.path.join(destdir, os.path.basename(sourcefile))
+    try:
+        desttime = modification_time(destfile)
+    except OSError:
+        # New file does not exist, destdir may or may not exist
+        safe_makedirs(destdir)
+    else:
+        # New file already exists
+        if not file_newer_than(sourcefile, desttime):
+            return
+    shutil.copy2(sourcefile, destfile)
+
+
 @cached_function
 def search_include_directories(dirs, qualified_name, suffix, pos,
                                include=False, sys_path=False):

docs/src/reference/extension_types.rst

@@ -36,7 +36,7 @@ Attributes
 * Are fixed at compile time.
 
  * You can't add attributes to an extension type instance at run time like in normal Python.
- * You can sub-class the extenstion type in Python to add attributes at run-time.
+ * You can sub-class the extension type in Python to add attributes at run-time.
 
 * There are two ways to access extension type attributes:
 

docs/src/tutorial/numpy.rst

@@ -126,7 +126,7 @@ compatibility. Consider this code (*read the comments!*) ::
             raise ValueError("Only odd dimensions on filter supported")
         assert f.dtype == DTYPE and g.dtype == DTYPE
         # The "cdef" keyword is also used within functions to type variables. It
-        # can only be used at the top indendation level (there are non-trivial
+        # can only be used at the top indentation level (there are non-trivial
         # problems with allowing them in other places, though we'd love to see
         # good and thought out proposals for it).
         #

docs/src/userguide/index.rst

@@ -22,6 +22,7 @@ Contents:
    buffer
    parallelism
    debugging
+   numpy_tutorial
 
 Indices and tables
 ------------------

docs/src/userguide/numpy_tutorial.rst

@@ -263,7 +263,7 @@ compatibility. Here's :file:`convolve2.pyx`. *Read the comments!*  ::
             raise ValueError("Only odd dimensions on filter supported")
         assert f.dtype == DTYPE and g.dtype == DTYPE
         # The "cdef" keyword is also used within functions to type variables. It
-        # can only be used at the top indendation level (there are non-trivial
+        # can only be used at the top indentation level (there are non-trivial
         # problems with allowing them in other places, though we'd love to see
         # good and thought out proposals for it).
         #
@@ -463,7 +463,7 @@ if someone is interested also under Python 2.x.
 
 There is some speed penalty to this though (as one makes more assumptions
 compile-time if the type is set to :obj:`np.ndarray`, specifically it is
-assumed that the data is stored in pure strided more and not in indirect
+assumed that the data is stored in pure strided mode and not in indirect
 mode).
 
 [:enhancements/buffer:More information]

docs/src/userguide/pxd_package.rst

-I think this is a result of a recent change to Pyrex that
-has been merged into Cython.
-
-If a directory contains an :file:`__init__.py` or :file:`__init__.pyx` file,
-it's now assumed to be a package directory. So, for example,
-if you have a directory structure::
-
-   foo/
-     __init__.py
-     shrubbing.pxd
-     shrubbing.pyx
-
-then the shrubbing module is assumed to belong to a package
-called 'foo', and its fully qualified module name is
-'foo.shrubbing'.
-
-So when Pyrex wants to find out whether there is a `.pxd` file for shrubbing,
-it looks for one corresponding to a module called `foo.shrubbing`. It
-does this by searching the include path for a top-level package directory
-called 'foo' containing a file called 'shrubbing.pxd'.
-
-However, if foo is the current directory you're running
-the compiler from, and you haven't added foo to the
-include path using a -I option, then it won't be on
-the include path, and the `.pxd` won't be found.
-
-What to do about this depends on whether you really
-intend the module to reside in a package.
-
-If you intend shrubbing to be a top-level module, you
-will have to move it somewhere else where there is
-no :file:`__init__.*` file.
-
-If you do intend it to reside in a package, then there
-are two alternatives:
-
-1. cd to the directory containing foo and compile
-   from there::
-
-      cd ..; cython foo/shrubbing.pyx
-
-2. arrange for the directory containing foo to be
-   passed as a -I option, e.g.::
-
-      cython -I .. shrubbing.pyx
-
-Arguably this behaviour is not very desirable, and I'll
-see if I can do something about it.
-

docs/src/userguide/wrapping_CPlusPlus.rst

@@ -3,7 +3,7 @@
 .. _wrapping-cplusplus:
 
 ********************************
-Using C++ in Cython
+Using C++ in Cythonp
 ********************************
 
 Overview
@@ -68,7 +68,7 @@ document. Let's assume it will be in a header file called
             void move(int dx, int dy);
         };
     }
-    
+
 and the implementation in the file called :file:`Rectangle.cpp`:
 
 .. sourcecode:: c++
@@ -104,7 +104,7 @@ and the implementation in the file called :file:`Rectangle.cpp`:
             x1 += dx;
             y1 += dy;
         }
-  
+
     }
 
 This is pretty dumb, but should suffice to demonstrate the steps involved.
@@ -145,7 +145,7 @@ is then handled by ``cythonize()`` as follows::
    from Cython.Build import cythonize
 
    setup(ext_modules = cythonize(Extension(
-              "rect",                                # the extesion name
+              "rect",                                # the extension name
               sources=["rect.pyx", "Rectangle.cpp"], # the Cython source and
                                                      # additional C++ source files
               language="c++",                        # generate and compile C++ code
@@ -196,7 +196,7 @@ class name from Rectangle.h and adjust for Cython syntax, so now it becomes::
 
     cdef extern from "Rectangle.h" namespace "shapes":
         cdef cppclass Rectangle:
-    
+
 Add public attributes
 ^^^^^^^^^^^^^^^^^^^^^^
 
@@ -346,9 +346,9 @@ nested in Cython::
             T& at(int)
             iterator begin()
             iterator end()
-            
+
     cdef vector[int].iterator iter  #iter is declared as being of type vector<int>::iterator
-            
+
 Note that the nested class is declared with a ``cppclass`` but without a ``cdef``.
 
 C++ operators not compatible with Python syntax
@@ -363,13 +363,18 @@ a special module ``cython.operator``. The functions provided are:
 * ``cython.operator.dereference`` for dereferencing. ``dereference(foo)``
   will produce the C++ code ``*(foo)``
 * ``cython.operator.preincrement`` for pre-incrementation. ``preincrement(foo)``
-  will produce the C++ code ``++(foo)``
-* ...
+  will produce the C++ code ``++(foo)``.
+  Similarly for ``predecrement``, ``postincrement`` and ``postdecrement``.
+* ``cython.operator.comma`` for the comma operator. ``comma(a, b)``
+  will produce the C++ code ``((a), (b))``.
 
 These functions need to be cimported. Of course, one can use a
 ``from ... cimport ... as`` to have shorter and more readable functions.
 For example: ``from cython.operator cimport dereference as deref``.
 
+For completeness, it's also worth mentioning ``cython.operator.address``
+which can also be written ``&foo``.
+
 Templates
 ----------
 
@@ -404,8 +409,15 @@ Cython uses a bracket syntax for templating. A simple example for wrapping C++ v
 
     del v
 
-Multiple template parameters can be defined as a list, such as [T, U, V]
-or [int, bool, char].  Template functions are defined similarly, with
+Multiple template parameters can be defined as a list, such as ``[T, U, V]``
+or ``[int, bool, char]``.  Optional template parameters can be indicated
+by writing ``[T, U, V=*]``.  In the event that Cython needs to explicitly
+reference a default template value for an incomplete template instantiation,
+it will write ``MyClass<T, U>::V``, so if the class provides a typedef
+for its template parameters it is preferable to use that name here.
+
+
+Template functions are defined similarly to class templates, with
 the template parameter list following the function name::
 
     cdef extern from "<algorithm>" namespace "std":
@@ -432,7 +444,7 @@ For example::
         vect.push_back(i)
     for i in range(10):
         print vect[i]
-        
+
 The pxd files in ``/Cython/Includes/libcpp`` also work as good examples on
 how to declare C++ classes.
 
@@ -596,7 +608,7 @@ module which:
 
 * includes the needed C headers in an extern "C" block
 * contains minimal forwarding functions in C++, each of which calls the
-  respective pure-C function 
+  respective pure-C function
 
 Declaring/Using References
 ---------------------------
@@ -609,5 +621,3 @@ C++ left-values
 C++ allows functions returning a reference to be left-values.  This is currently
 not supported in Cython. ``cython.operator.dereference(foo)`` is also not
 considered a left-value.
-
-

docs/welcome.rst

-.. highlight:: cython
-
-.. _overview:
-
-********
-Welcome!
-********
-
-===============
-What is Cython?
-===============
-
-Cython is a programming language based on Python
-with extra syntax to provide static type declarations.
-
-================
-What Does It Do?
-================
-
-It takes advantage of the benefits of Python while allowing one to achieve the speed of C.
-
-============================
-How Exactly Does It Do That?
-============================
-
-The source code gets translated into optimized C/C++
-code and compiled as Python extension modules.
-
-This allows for both very fast program execution and tight
-integration with external C libraries, while keeping
-up the high *programmer productivity* for which the
-Python language is well known.
-
-=============
-Tell Me More!
-=============
-
-The Python language is well known.
-
-The primary Python execution environment is commonly referred to as CPython, as it is written in
-C. Other major implementations use:
-
-:Java: Jython [#Jython]_
-:C#: IronPython [#IronPython]_)
-:Python itself: PyPy [#PyPy]_
-
-Written in C, CPython has been
-conducive to wrapping many external libraries that interface through the C language. It has, however, remained non trivial to write the necessary glue code in
-C, especially for programmers who are more fluent in a
-high-level language like Python than in a do-it-yourself
-language like C.
-
-Originally based on the well-known Pyrex [#Pyrex]_, the
-Cython project has approached this problem by means
-of a source code compiler that translates Python code
-to equivalent C code. This code is executed within the
-CPython runtime environment, but at the speed of
-compiled C and with the ability to call directly into C
-libraries.
-
-At the same time, it keeps the original interface of the Python source code, which makes it directly
-usable from Python code. These two-fold characteristics enable Cython’s two major use cases:
-
-#. Extending the CPython interpreter with fast binary modules, and
-#. Interfacing Python code with external C libraries.
-
-While Cython can compile (most) regular Python
-code, the generated C code usually gains major (and
-sometime impressive) speed improvements from optional static type declarations for both Python and
-C types. These allow Cython to assign C semantics to
-parts of the code, and to translate them into very efficient C code.
-
-Type declarations can therefore be used
-for two purposes:
-
-#. For moving code sections from dynamic Python semantics into static-and-fast C semantics, but also for..
-#. Directly manipulating types defined in external libraries. Cython thus merges the two worlds into a very broadly applicable programming language.
-
-==================
-Where Do I Get It?
-==================
-
-Well.. at `cython.org <http://cython.org>`_.. of course!
-
-======================
-How Do I Report a Bug?
-======================
-
-=================================
-I Want To Make A Feature Request!
-=================================
-
-============================================
-Is There a Mail List? How Do I Contact You?
-============================================
-
-
-
-.. rubric:: Footnotes
-
-.. [#Jython] **Jython:** \J. Huginin, B. Warsaw, F. Bock, et al., Jython: Python for the Java platform, http://www.jython.org
-
-.. [#IronPython] **IronPython:** Jim Hugunin et al., http://www.codeplex.com/IronPython.
-
-
-.. [#PyPy] **PyPy:** The PyPy Group, PyPy: a Python implementation written in Python, http://pypy.org
-
-.. [#Pyrex] **Pyrex:** G. Ewing, Pyrex: C-Extensions for Python, http://www.cosc.canterbury.ac.nz/greg.ewing/python/Pyrex/
-
-
-
-
-
-
-
-
-
-
-
-
-

tests/build/build_dir_src.srctree

+# Mostly the same test as build_dir.srctree but with everything inside
+# a common "src" directory. We don't use --inplace and don't actually
+# import the built modules.
+
+PYTHON shutil_copy.py src/subdir src/fake
+PYTHON setup.py build_ext
+PYTHON check_paths.py
+
+######## shutil_copy.py ########
+
+import shutil, sys
+shutil.copytree(sys.argv[1], sys.argv[2])
+
+######## setup.py ########
+
+from Cython.Build.Dependencies import cythonize
+from Cython.Distutils.extension import Extension
+
+from distutils.core import setup
+
+ext_modules = cythonize(
+        Extension("a", ["src/a.pyx"]), build_dir="scratchA")
+ext_modules += cythonize(
+        Extension("pkg.b", ["src/pkg/b.pyx"]), build_dir="scratchB")
+
+setup(ext_modules=ext_modules)
+
+######## src/a.pyx ########
+
+cdef extern from "helper.h":
+    int value1
+
+cdef extern from "subdir/helper.h":
+    int value2
+
+cdef extern from "pkg/pkg_helper.h":
+    int value3
+
+assert value1 == 100
+assert value2 == 200
+assert value3 == 300
+
+######## src/helper.h ########
+
+int value1 = 100;
+
+######## src/subdir/helper.h ########
+
+int value2 = 200;
+
+######## src/pkg/__init__.py ########
+
+######## src/pkg/b.pyx ########
+
+cdef extern from "../fake/helper.h":
+    int value2
+
+cdef extern from "pkg_helper.h":
+    int value3
+
+cdef extern from "subdir/pkg_helper.h":
+    int value4
+
+assert value2 == 200
+assert value3 == 300
+assert value4 == 400
+
+######## src/pkg/pkg_helper.h ########
+
+int value3 = 300;
+
+######## src/pkg/subdir/pkg_helper.h ########
+
+int value4 = 400;
+
+######## check_paths.py ########
+
+import os
+assert os.path.exists("scratchA/src/a.c")
+assert os.path.exists("scratchA/src/helper.h")
+assert os.path.exists("scratchA/src/subdir/helper.h")
+assert os.path.exists("scratchA/src/pkg/pkg_helper.h")
+assert not os.path.exists("src/a.c")
+
+assert os.path.exists("scratchB/src/pkg/b.c")
+assert os.path.exists("scratchB/src/pkg/pkg_helper.h")
+assert os.path.exists("scratchB/src/pkg/subdir/pkg_helper.h")
+assert os.path.exists("scratchB/src/fake/helper.h")
+assert not os.path.exists("src/b.c")
+assert not os.path.exists("src/pkg/b.c")

tests/errors/pure_cclass_without_body.pxd

+# mode: error
+
+cdef class Test

tests/errors/pure_cclass_without_body.py

+# mode: error
+
+class Test(object):
+    pass
+
+_ERRORS = u"""
+3:5: C class 'Test' is declared but not defined
+"""

tests/run/asyncio_generators.srctree

@@ -78,7 +78,7 @@ if ASYNCIO_SUPPORTS_COROUTINE:
         loop = asyncio.get_event_loop()
         task, events, expected = await_future(loop)
         result = loop.run_until_complete(task())
-        assert events == expected, events
+        assert events == expected, 'expected %s, got %s' % (expected, events)
 
     runloop()
 
@@ -207,6 +207,7 @@ async def wait3():
 
 
 ######## async_def_future.pyx ########
+# cython: binding=True
 
 import asyncio
 

tests/run/cpp_namespaces.pyx

@@ -3,6 +3,9 @@
 
 cdef extern from "cpp_namespaces_helper.h" namespace "A":
     ctypedef int A_t
+    cdef struct S:
+        double x
+        A_t k
     A_t A_func(A_t first, A_t)
     cdef void f(A_t)
 
@@ -36,3 +39,11 @@ def test_typedef(A_t a):
     3
     """
     return a
+
+def test_convert_struct(S s):
+    """
+    >>> py_value = {'x': 3.5, 'k': 10}
+    >>> test_convert_struct(py_value) == py_value
+    True
+    """
+    return s

tests/run/cpp_namespaces_helper.h

@@ -18,6 +18,11 @@ namespace A {
 
     typedef int A_t;
 
+    struct S {
+        A_t k;
+        double x;
+    };
+
     A_t A_func(A_t first, A_t second) {
         return first + second;
     }

tests/run/cpp_templates.pyx

@@ -3,12 +3,18 @@
 from cython.operator import dereference as deref
 
 cdef extern from "cpp_templates_helper.h":
-    cdef cppclass Wrap[T]:
+    cdef cppclass Wrap[T, AltType=*, UndeclarableAltType=*]:
         Wrap(T)
         void set(T)
         T get()
         bint operator==(Wrap[T])
 
+        AltType get_alt_type()
+        void set_alt_type(AltType)
+
+        UndeclarableAltType create()
+        bint accept(UndeclarableAltType)
+
     cdef cppclass Pair[T1,T2]:
         Pair(T1,T2)
         T1 first()
@@ -57,6 +63,31 @@ def test_double(double x, double y):
     finally:
         del a, b
 
+
+def test_default_template_arguments(double x):
+    """
+    >>> test_default_template_arguments(3.5)
+    (3.5, 3.0)
+    """
+    try:
+        a = new Wrap[double](x)
+        b = new Wrap[double, int, long](x)
+
+        ax = a.get_alt_type()
+        a.set_alt_type(ax)
+        assert a.accept(a.create())  # never declared
+
+        bx = b.get_alt_type()
+        b.set_alt_type(bx)
+
+        bc = b.create()              # declaration here is fine
+        assert b.accept(bc)
+
+        return a.get(), b.get()
+    finally:
+        del a
+
+
 def test_pair(int i, double x):
     """
     >>> test_pair(1, 1.5)

tests/run/cpp_templates_helper.h

-template <class T>
+template <typename T, typename S=T, typename U=T>
 class Wrap {
     T value;
 public:
+    typedef S AltType;
+
     Wrap(T v) : value(v) { }
     void set(T v) { value = v; }
     T get(void) { return value; }
     bool operator==(Wrap<T> other) { return value == other.value; }
+
+    S get_alt_type(void) { return (S) value; }
+    void set_alt_type(S v) { value = (T) v; }
+
+    U create(void) { return (U) value; }
+    bool accept(U v) { return v == (U) value; }
 };
 
 template <class T1, class T2>

tests/run/py_ucs4_type.pyx

@@ -145,6 +145,24 @@ def unicode_methods(Py_UCS4 uchar):
         uchar.title(),
         ]
 
+
+@cython.test_assert_path_exists('//PythonCapiCallNode')
+@cython.test_fail_if_path_exists(
+    '//SimpleCallNode',
+    '//CoerceFromPyTypeNode',
+)
+def unicode_method_return_type(Py_UCS4 uchar):
+    """
+    >>> unicode_method_return_type(ord('A'))
+    [True, False]
+    >>> unicode_method_return_type(ord('a'))
+    [False, True]
+    """
+    cdef Py_UCS4 uc, ul
+    uc, ul = uchar.upper(), uchar.lower()
+    return [uc == uchar, ul == uchar]
+
+
 @cython.test_assert_path_exists('//IntNode')
 @cython.test_fail_if_path_exists('//SimpleCallNode',
                                  '//PythonCapiCallNode')