Merge branch 'master' of git+ssh://github.com/cython/cython

Cython/Compiler/ExprNodes.py

@@ -43,9 +43,10 @@ from . import Future
 from ..Debugging import print_call_chain
 from .DebugFlags import debug_disposal_code, debug_temp_alloc, \
     debug_coercion
-from .Pythran import to_pythran, is_pythran_supported_type, is_pythran_supported_operation_type, \
-     is_pythran_expr, pythran_func_type, pythran_binop_type, pythran_unaryop_type, has_np_pythran, \
-     pythran_indexing_code, pythran_indexing_type, is_pythran_supported_node_or_none, pythran_type
+from .Pythran import (to_pythran, is_pythran_supported_type, is_pythran_supported_operation_type,
+     is_pythran_expr, pythran_func_type, pythran_binop_type, pythran_unaryop_type, has_np_pythran,
+     pythran_indexing_code, pythran_indexing_type, is_pythran_supported_node_or_none, pythran_type,
+     pythran_is_numpy_func_supported, pythran_get_func_include_file, pythran_functor)
 from .PyrexTypes import PythranExpr
 
 try:
@@ -5408,7 +5409,8 @@ class SimpleCallNode(CallNode):
 
         func_type = self.function_type()
         self.is_numpy_call_with_exprs = False
-        if has_np_pythran(env) and self.function.is_numpy_attribute:
+        if (has_np_pythran(env) and function.is_numpy_attribute and
+                pythran_is_numpy_func_supported(function)):
             has_pythran_args = True
             self.arg_tuple = TupleNode(self.pos, args = self.args)
             self.arg_tuple = self.arg_tuple.analyse_types(env)
@@ -5416,12 +5418,12 @@ class SimpleCallNode(CallNode):
                 has_pythran_args &= is_pythran_supported_node_or_none(arg)
             self.is_numpy_call_with_exprs = bool(has_pythran_args)
         if self.is_numpy_call_with_exprs:
-            env.add_include_file("pythonic/numpy/%s.hpp" % self.function.attribute)
+            env.add_include_file(pythran_get_func_include_file(function))
             return NumPyMethodCallNode.from_node(
                 self,
-                function=self.function,
+                function=function,
                 arg_tuple=self.arg_tuple,
-                type=PythranExpr(pythran_func_type(self.function.attribute, self.arg_tuple.args)),
+                type=PythranExpr(pythran_func_type(function, self.arg_tuple.args)),
             )
         elif func_type.is_pyobject:
             self.arg_tuple = TupleNode(self.pos, args = self.args)
@@ -5839,10 +5841,10 @@ class NumPyMethodCallNode(SimpleCallNode):
 
         code.putln("// function evaluation code for numpy function")
         code.putln("__Pyx_call_destructor(%s);" % self.result())
-        code.putln("new (&%s) decltype(%s){pythonic::numpy::functor::%s{}(%s)};" % (
+        code.putln("new (&%s) decltype(%s){%s{}(%s)};" % (
             self.result(),
             self.result(),
-            self.function.attribute,
+            pythran_functor(self.function),
             ", ".join(a.pythran_result() for a in args)))
 
 

Cython/Compiler/ParseTreeTransforms.py

@@ -613,7 +613,8 @@ class TrackNumpyAttributes(VisitorTransform, SkipDeclarations):
 
     def visit_AttributeNode(self, node):
         self.visitchildren(node)
-        if node.obj.is_name and node.obj.name in self.numpy_module_names:
+        if (node.obj.is_name and node.obj.name in self.numpy_module_names) or \
+            node.obj.is_numpy_attribute:
             node.is_numpy_attribute = True
         return node
 

Cython/Compiler/Pythran.py

@@ -6,16 +6,20 @@ from .PyrexTypes import CType, CTypedefType, CStructOrUnionType
 
 import cython
 
+try:
+    import pythran
+    _pythran_available = True
+except ImportError:
+    _pythran_available = False
+
 
 # Pythran/Numpy specific operations
 
 def has_np_pythran(env):
-    while env is not None:
-        directives = getattr(env, 'directives', None)
-        if directives and env.directives.get('np_pythran', False):
-            return True
-        env = env.outer_scope
-
+    if env is None:
+        return False
+    directives = getattr(env, 'directives', None)
+    return (directives and directives.get('np_pythran', False))
 
 @cython.ccall
 def is_pythran_supported_dtype(type_):
@@ -111,10 +115,32 @@ def pythran_indexing_type(type_, indices):
 def pythran_indexing_code(indices):
     return _index_access(_index_code, indices)
 
+def np_func_to_list(func):
+    if not func.is_numpy_attribute:
+        return []
+    return np_func_to_list(func.obj) + [func.attribute]
+
+if _pythran_available:
+    def pythran_is_numpy_func_supported(func):
+        CurF = pythran.tables.MODULES['numpy']
+        FL = np_func_to_list(func)
+        for F in FL:
+            CurF = CurF.get(F, None)
+            if CurF is None:
+                return False
+        return True
+else:
+    def pythran_is_numpy_func_supported(name):
+        return False
+
+def pythran_functor(func):
+    func = np_func_to_list(func)
+    submodules = "::".join(func[:-1] + ["functor"])
+    return "pythonic::numpy::%s::%s" % (submodules, func[-1])
 
 def pythran_func_type(func, args):
     args = ",".join(("std::declval<%s>()" % pythran_type(a.type) for a in args))
-    return "decltype(pythonic::numpy::functor::%s{}(%s))" % (func, args)
+    return "decltype(%s{}(%s))" % (pythran_functor(func), args)
 
 
 @cython.ccall
@@ -168,6 +194,9 @@ def is_pythran_buffer(type_):
     return (type_.is_numpy_buffer and is_pythran_supported_dtype(type_.dtype) and
             type_.mode in ("c", "strided") and not type_.cast)
 
+def pythran_get_func_include_file(func):
+    func = np_func_to_list(func)
+    return "pythonic/include/numpy/%s.hpp" % "/".join(func)
 
 def include_pythran_generic(env):
     # Generic files

docs/src/userguide/source_files_and_compilation.rst

@@ -6,14 +6,27 @@
 Source Files and Compilation
 ****************************
 
-.. note:: See :ref:`compilation-reference` reference section for more details
-
 Cython source file names consist of the name of the module followed by a
 ``.pyx`` extension, for example a module called primes would have a source
 file named :file:`primes.pyx`.
 
+Cython code, unlike Python, must be compiled.  This happens in two stages:
+
+  * A ``.pyx`` file is compiled by Cython to a ``.c`` file.
+
+  * The ``.c`` file is compiled by a C compiler to a ``.so`` file (or a
+    ``.pyd`` file on Windows)
+
 Once you have written your ``.pyx`` file, there are a couple of ways of turning it
-into an extension module. One way is to compile it manually with the Cython
+into an extension module.
+
+The following sub-sections describe several ways to build your
+extension modules, and how to pass directives to the Cython compiler.
+
+Compiling from the command line
+===============================
+
+One way is to compile it manually with the Cython
 compiler, e.g.:
 
 .. sourcecode:: text
@@ -54,6 +67,164 @@ current directory use:
 
     $ python setup.py build_ext --inplace
 
+Configuring the C-Build
+------------------------
+
+If you have include files in non-standard places you can pass an
+``include_path`` parameter to ``cythonize``::
+
+    from distutils.core import setup
+    from Cython.Build import cythonize
+
+    setup(
+        name="My hello app",
+        ext_modules=cythonize("src/*.pyx", include_path=[...]),
+    )
+
+Often, Python packages that offer a C-level API provide a way to find
+the necessary include files, e.g. for NumPy::
+
+    include_path = [numpy.get_include()]
+
+.. note::
+
+    Using memoryviews or importing NumPy with ``import numpy`` does not mean that
+    you have to add the path to NumPy include files. You need to add this path only
+    if you use ``cimport numpy``.
+
+Despite this, you will still get warnings like the
+following from the compiler, because Cython is using a deprecated Numpy API::
+
+   .../include/numpy/npy_1_7_deprecated_api.h:15:2: warning: #warning "Using deprecated NumPy API, disable it by " "#defining NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION" [-Wcpp]
+
+For the time being, it is just a warning that you can ignore.
+
+If you need to specify compiler options, libraries to link with or other
+linker options you will need to create ``Extension`` instances manually
+(note that glob syntax can still be used to specify multiple extensions
+in one line)::
+
+    from distutils.core import setup
+    from distutils.extension import Extension
+    from Cython.Build import cythonize
+
+    extensions = [
+        Extension("primes", ["primes.pyx"],
+            include_dirs=[...],
+            libraries=[...],
+            library_dirs=[...]),
+        # Everything but primes.pyx is included here.
+        Extension("*", ["*.pyx"],
+            include_dirs=[...],
+            libraries=[...],
+            library_dirs=[...]),
+    ]
+    setup(
+        name="My hello app",
+        ext_modules=cythonize(extensions),
+    )
+
+Note that when using setuptools, you should import it before Cython as
+setuptools may replace the ``Extension`` class in distutils.  Otherwise,
+both might disagree about the class to use here.
+
+Note also that if you use setuptools instead of distutils, the default
+action when running ``python setup.py install`` is to create a zipped
+``egg`` file which will not work with ``cimport`` for ``pxd`` files
+when you try to use them from a dependent package.
+To prevent this, include ``zip_safe=False`` in the arguments to ``setup()``.
+
+If your options are static (for example you do not need to call a tool like
+``pkg-config`` to determine them) you can also provide them directly in your
+.pyx or .pxd source file using a special comment block at the start of the file::
+
+    # distutils: libraries = spam eggs
+    # distutils: include_dirs = /opt/food/include
+
+If you cimport multiple .pxd files defining libraries, then Cython
+merges the list of libraries, so this works as expected (similarly
+with other options, like ``include_dirs`` above).
+
+If you have some C files that have been wrapped with Cython and you want to
+compile them into your extension, you can define the distutils ``sources``
+parameter::
+
+    # distutils: sources = helper.c, another_helper.c
+
+Note that these sources are added to the list of sources of the current
+extension module.  Spelling this out in the :file:`setup.py` file looks
+as follows::
+
+    from distutils.core import setup
+    from Cython.Build import cythonize
+    from distutils.extension import Extension
+
+    sourcefiles = ['example.pyx', 'helper.c', 'another_helper.c']
+
+    extensions = [Extension("example", sourcefiles)]
+
+    setup(
+        ext_modules=cythonize(extensions)
+    )
+
+The :class:`Extension` class takes many options, and a fuller explanation can
+be found in the `distutils documentation`_. Some useful options to know about
+are ``include_dirs``, ``libraries``, and ``library_dirs`` which specify where
+to find the ``.h`` and library files when linking to external libraries.
+
+.. _distutils documentation: https://docs.python.org/extending/building.html
+
+Sometimes this is not enough and you need finer customization of the
+distutils :class:`Extension`.
+To do this, you can provide a custom function ``create_extension``
+to create the final :class:`Extension` object after Cython has processed
+the sources, dependencies and ``# distutils`` directives but before the
+file is actually Cythonized.
+This function takes 2 arguments ``template`` and ``kwds``, where
+``template`` is the :class:`Extension` object given as input to Cython
+and ``kwds`` is a :class:`dict` with all keywords which should be used
+to create the :class:`Extension`.
+The function ``create_extension`` must return a 2-tuple
+``(extension, metadata)``, where ``extension`` is the created
+:class:`Extension` and ``metadata`` is metadata which will be written
+as JSON at the top of the generated C files. This metadata is only used
+for debugging purposes, so you can put whatever you want in there
+(as long as it can be converted to JSON).
+The default function (defined in ``Cython.Build.Dependencies``) is::
+
+    def default_create_extension(template, kwds):
+        if 'depends' in kwds:
+            include_dirs = kwds.get('include_dirs', []) + ["."]
+            depends = resolve_depends(kwds['depends'], include_dirs)
+            kwds['depends'] = sorted(set(depends + template.depends))
+
+        t = template.__class__
+        ext = t(**kwds)
+        metadata = dict(distutils=kwds, module_name=kwds['name'])
+        return ext, metadata
+
+In case that you pass a string instead of an :class:`Extension` to
+``cythonize()``, the ``template`` will be an :class:`Extension` without
+sources. For example, if you do ``cythonize("*.pyx")``,
+the ``template`` will be ``Extension(name="*.pyx", sources=[])``.
+
+Just as an example, this adds ``mylib`` as library to every extension::
+
+    from Cython.Build.Dependencies import default_create_extension
+
+    def my_create_extension(template, kwds):
+        libs = kwds.get('libraries', []) + ["mylib"]
+        kwds['libraries'] = libs
+        return default_create_extension(template, kwds)
+
+    ext_modules = cythonize(..., create_extension=my_create_extension)
+
+.. note::
+
+    If you Cythonize in parallel (using the ``nthreads`` argument),
+    then the argument to ``create_extension`` must be pickleable.
+    In particular, it cannot be a lambda function.
+
 
 Multiple Cython Files in a Package
 ===================================