Commit 8ccdda6d authored by scoder's avatar scoder Committed by GitHub

Merge pull request #1915 from jdemeyer/include_string

Verbatim C code using docstring syntax.
parents b8215f4e 637d6bd5
...@@ -80,6 +80,82 @@ modifier_output_mapper = { ...@@ -80,6 +80,82 @@ modifier_output_mapper = {
is_self_assignment = re.compile(r" *(\w+) = (\1);\s*$").match is_self_assignment = re.compile(r" *(\w+) = (\1);\s*$").match
class IncludeCode(object):
"""
An include file and/or verbatim C code to be included in the
generated sources.
"""
# attributes:
#
# pieces {order: unicode}: pieces of C code to be generated.
# For the included file, the key "order" is zero.
# For verbatim include code, the "order" is the "order"
# attribute of the original IncludeCode where this piece
# of C code was first added. This is needed to prevent
# duplication if the same include code is found through
# multiple cimports.
# location int: where to put this include in the C sources, one
# of the constants INITIAL, EARLY, LATE
# order int: sorting order (automatically set by increasing counter)
# Constants for location. If the same include occurs with different
# locations, the earliest one takes precedense.
INITIAL = 0
EARLY = 1
LATE = 2
counter = 1 # Counter for "order"
def __init__(self, include=None, verbatim=None, late=True, initial=False):
self.order = self.counter
type(self).counter += 1
self.pieces = {}
if include:
if include[0] == '<' and include[-1] == '>':
self.pieces[0] = u'#include {0}'.format(include)
late = False # system include is never late
else:
self.pieces[0] = u'#include "{0}"'.format(include)
if verbatim:
self.pieces[self.order] = verbatim
if initial:
self.location = self.INITIAL
elif late:
self.location = self.LATE
else:
self.location = self.EARLY
def dict_update(self, d, key):
"""
Insert `self` in dict `d` with key `key`. If that key already
exists, update the attributes of the existing value with `self`.
"""
if key in d:
other = d[key]
other.location = min(self.location, other.location)
other.pieces.update(self.pieces)
else:
d[key] = self
def sortkey(self):
return self.order
def mainpiece(self):
"""
Return the main piece of C code, corresponding to the include
file. If there was no include file, return None.
"""
return self.pieces.get(0)
def write(self, code):
# Write values of self.pieces dict, sorted by the keys
for k in sorted(self.pieces):
code.putln(self.pieces[k])
def get_utility_dir(): def get_utility_dir():
# make this a function and not global variables: # make this a function and not global variables:
# http://trac.cython.org/cython_trac/ticket/475 # http://trac.cython.org/cython_trac/ticket/475
......
...@@ -28,7 +28,7 @@ from . import Pythran ...@@ -28,7 +28,7 @@ from . import Pythran
from .Errors import error, warning from .Errors import error, warning
from .PyrexTypes import py_object_type from .PyrexTypes import py_object_type
from ..Utils import open_new_file, replace_suffix, decode_filename from ..Utils import open_new_file, replace_suffix, decode_filename
from .Code import UtilityCode from .Code import UtilityCode, IncludeCode
from .StringEncoding import EncodedString from .StringEncoding import EncodedString
from .Pythran import has_np_pythran from .Pythran import has_np_pythran
...@@ -86,16 +86,15 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): ...@@ -86,16 +86,15 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
self.scope.utility_code_list.extend(scope.utility_code_list) self.scope.utility_code_list.extend(scope.utility_code_list)
for inc in scope.c_includes.values():
self.scope.process_include(inc)
def extend_if_not_in(L1, L2): def extend_if_not_in(L1, L2):
for x in L2: for x in L2:
if x not in L1: if x not in L1:
L1.append(x) L1.append(x)
extend_if_not_in(self.scope.include_files_early, scope.include_files_early)
extend_if_not_in(self.scope.include_files_late, scope.include_files_late)
extend_if_not_in(self.scope.included_files, scope.included_files) extend_if_not_in(self.scope.included_files, scope.included_files)
extend_if_not_in(self.scope.python_include_files,
scope.python_include_files)
if merge_scope: if merge_scope:
# Ensure that we don't generate import code for these entries! # Ensure that we don't generate import code for these entries!
...@@ -621,8 +620,9 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): ...@@ -621,8 +620,9 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
code.putln("") code.putln("")
code.putln("#define PY_SSIZE_T_CLEAN") code.putln("#define PY_SSIZE_T_CLEAN")
for filename in env.python_include_files: for inc in sorted(env.c_includes.values(), key=IncludeCode.sortkey):
code.putln('#include "%s"' % filename) if inc.location == inc.INITIAL:
inc.write(code)
code.putln("#ifndef Py_PYTHON_H") code.putln("#ifndef Py_PYTHON_H")
code.putln(" #error Python headers needed to compile C extensions, " code.putln(" #error Python headers needed to compile C extensions, "
"please install development version of Python.") "please install development version of Python.")
...@@ -739,19 +739,13 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): ...@@ -739,19 +739,13 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
def generate_includes(self, env, cimported_modules, code, early=True, late=True): def generate_includes(self, env, cimported_modules, code, early=True, late=True):
includes = [] includes = []
for inc in sorted(env.c_includes.values(), key=IncludeCode.sortkey):
if inc.location == inc.EARLY:
if early: if early:
includes += env.include_files_early inc.write(code)
elif inc.location == inc.LATE:
if late: if late:
includes += [include for include in env.include_files_late inc.write(code)
if include not in env.include_files_early]
for filename in includes:
byte_decoded_filenname = str(filename)
if byte_decoded_filenname[0] == '<' and byte_decoded_filenname[-1] == '>':
code.putln('#include %s' % byte_decoded_filenname)
else:
code.putln('#include "%s"' % byte_decoded_filenname)
if early: if early:
code.putln_openmp("#include <omp.h>") code.putln_openmp("#include <omp.h>")
......
...@@ -471,6 +471,7 @@ class StatNode(Node): ...@@ -471,6 +471,7 @@ class StatNode(Node):
class CDefExternNode(StatNode): class CDefExternNode(StatNode):
# include_file string or None # include_file string or None
# verbatim_include string or None
# body StatListNode # body StatListNode
child_attrs = ["body"] child_attrs = ["body"]
...@@ -480,18 +481,16 @@ class CDefExternNode(StatNode): ...@@ -480,18 +481,16 @@ class CDefExternNode(StatNode):
env.in_cinclude = 1 env.in_cinclude = 1
self.body.analyse_declarations(env) self.body.analyse_declarations(env)
env.in_cinclude = old_cinclude_flag env.in_cinclude = old_cinclude_flag
inc = self.include_file
if inc: if self.include_file or self.verbatim_include:
# Determine whether include should be late
stats = self.body.stats stats = self.body.stats
if inc[0] == '<' and inc[-1] == '>': if not stats:
# System include => always early # Special case: empty 'cdef extern' blocks are early
env.add_include_file(inc) late = False
elif stats and all(isinstance(node, CVarDefNode) for node in stats): else:
# Generate a late include if the body is not empty and late = all(isinstance(node, CVarDefNode) for node in stats)
# all statements are variable or function declarations. env.add_include_file(self.include_file, self.verbatim_include, late)
env.add_include_file(inc, late=True)
else:
env.add_include_file(inc)
def analyse_expressions(self, env): def analyse_expressions(self, env):
return self return self
......
...@@ -3081,9 +3081,13 @@ def p_cdef_extern_block(s, pos, ctx): ...@@ -3081,9 +3081,13 @@ def p_cdef_extern_block(s, pos, ctx):
ctx.namespace = p_string_literal(s, 'u')[2] ctx.namespace = p_string_literal(s, 'u')[2]
if p_nogil(s): if p_nogil(s):
ctx.nogil = 1 ctx.nogil = 1
body = p_suite(s, ctx)
# Use "docstring" as verbatim string to include
verbatim_include, body = p_suite_with_docstring(s, ctx, True)
return Nodes.CDefExternNode(pos, return Nodes.CDefExternNode(pos,
include_file = include_file, include_file = include_file,
verbatim_include = verbatim_include,
body = body, body = body,
namespace = ctx.namespace) namespace = ctx.namespace)
......
...@@ -1068,9 +1068,8 @@ class ModuleScope(Scope): ...@@ -1068,9 +1068,8 @@ class ModuleScope(Scope):
# doc string Module doc string # doc string Module doc string
# doc_cname string C name of module doc string # doc_cname string C name of module doc string
# utility_code_list [UtilityCode] Queuing utility codes for forwarding to Code.py # utility_code_list [UtilityCode] Queuing utility codes for forwarding to Code.py
# python_include_files [string] Standard Python headers to be included # c_includes {key: IncludeCode} C headers or verbatim code to be generated
# include_files_early [string] C headers to be included before Cython decls # See process_include() for more documentation
# include_files_late [string] C headers to be included after Cython decls
# string_to_entry {string : Entry} Map string const to entry # string_to_entry {string : Entry} Map string const to entry
# identifier_to_entry {string : Entry} Map identifier string const to entry # identifier_to_entry {string : Entry} Map identifier string const to entry
# context Context # context Context
...@@ -1113,9 +1112,7 @@ class ModuleScope(Scope): ...@@ -1113,9 +1112,7 @@ class ModuleScope(Scope):
self.doc_cname = Naming.moddoc_cname self.doc_cname = Naming.moddoc_cname
self.utility_code_list = [] self.utility_code_list = []
self.module_entries = {} self.module_entries = {}
self.python_include_files = ["Python.h"] self.c_includes = {}
self.include_files_early = []
self.include_files_late = []
self.type_names = dict(outer_scope.type_names) self.type_names = dict(outer_scope.type_names)
self.pxd_file_loaded = 0 self.pxd_file_loaded = 0
self.cimported_modules = [] self.cimported_modules = []
...@@ -1129,6 +1126,7 @@ class ModuleScope(Scope): ...@@ -1129,6 +1126,7 @@ class ModuleScope(Scope):
for var_name in ['__builtins__', '__name__', '__file__', '__doc__', '__path__', for var_name in ['__builtins__', '__name__', '__file__', '__doc__', '__path__',
'__spec__', '__loader__', '__package__', '__cached__']: '__spec__', '__loader__', '__package__', '__cached__']:
self.declare_var(EncodedString(var_name), py_object_type, None) self.declare_var(EncodedString(var_name), py_object_type, None)
self.process_include(Code.IncludeCode("Python.h", initial=True))
def qualifying_scope(self): def qualifying_scope(self):
return self.parent_module return self.parent_module
...@@ -1251,24 +1249,50 @@ class ModuleScope(Scope): ...@@ -1251,24 +1249,50 @@ class ModuleScope(Scope):
module = module.lookup_submodule(submodule) module = module.lookup_submodule(submodule)
return module return module
def add_include_file(self, filename, late=False): def add_include_file(self, filename, verbatim_include=None, late=False):
if filename in self.python_include_files: """
return Add `filename` as include file. Add `verbatim_include` as
# Possibly, the same include appears both as early and as late verbatim text in the C file.
# include. We'll deal with this at code generation time. Both `filename` and `verbatim_include` can be `None` or empty.
if late: """
incs = self.include_files_late inc = Code.IncludeCode(filename, verbatim_include, late=late)
else: self.process_include(inc)
incs = self.include_files_early
if filename not in incs: def process_include(self, inc):
incs.append(filename) """
Add `inc`, which is an instance of `IncludeCode`, to this
`ModuleScope`. This either adds a new element to the
`c_includes` dict or it updates an existing entry.
In detail: the values of the dict `self.c_includes` are
instances of `IncludeCode` containing the code to be put in the
generated C file. The keys of the dict are needed to ensure
uniqueness in two ways: if an include file is specified in
multiple "cdef extern" blocks, only one `#include` statement is
generated. Second, the same include might occur multiple times
if we find it through multiple "cimport" paths. So we use the
generated code (of the form `#include "header.h"`) as dict key.
If verbatim code does not belong to any include file (i.e. it
was put in a `cdef extern from *` block), then we use a unique
dict key: namely, the `sortkey()`.
One `IncludeCode` object can contain multiple pieces of C code:
one optional "main piece" for the include file and several other
pieces for the verbatim code. The `IncludeCode.dict_update`
method merges the pieces of two different `IncludeCode` objects
if needed.
"""
key = inc.mainpiece()
if key is None:
key = inc.sortkey()
inc.dict_update(self.c_includes, key)
inc = self.c_includes[key]
def add_imported_module(self, scope): def add_imported_module(self, scope):
if scope not in self.cimported_modules: if scope not in self.cimported_modules:
for filename in scope.include_files_early: for inc in scope.c_includes.values():
self.add_include_file(filename, late=False) self.process_include(inc)
for filename in scope.include_files_late:
self.add_include_file(filename, late=True)
self.cimported_modules.append(scope) self.cimported_modules.append(scope)
for m in scope.cimported_modules: for m in scope.cimported_modules:
self.add_imported_module(m) self.add_imported_module(m)
......
...@@ -328,6 +328,41 @@ are entirely on your own with this feature. If you want to declare a name ...@@ -328,6 +328,41 @@ are entirely on your own with this feature. If you want to declare a name
the C file for it, you can do this using a C name declaration. Consider this the C file for it, you can do this using a C name declaration. Consider this
an advanced feature, only for the rare cases where everything else fails. an advanced feature, only for the rare cases where everything else fails.
Including verbatim C code
-------------------------
For advanced use cases, Cython allows you to directly write C code
as "docstring" of a ``cdef extern from`` block::
cdef extern from *:
"""
/* This is C code which will be put
* in the .c file output by Cython */
static long square(long x) {return x * x;}
#define assign(x, y) ((x) = (y))
"""
long square(long x)
void assign(long& x, long y)
The above is essentially equivalent to having the C code in a file
``header.h`` and writing ::
cdef extern from "header.h":
long square(long x)
void assign(long& x, long y)
It is also possible to combine a header file and verbatim C code::
cdef extern from "badheader.h":
"""
/* This macro breaks stuff */
#undef int
"""
# Stuff from badheader.h
In this case, the C code ``#undef int`` is put right after
``#include "badheader.h"`` in the C code generated by Cython.
Using Cython Declarations from C Using Cython Declarations from C
================================ ================================
......
# mode: compile # mode: compile
cdef extern from "cnamespec.h": cdef extern from *:
"""
int c_a, c_b;
"""
int a "c_a", b "c_b" int a "c_a", b "c_b"
cdef struct foo "c_foo": cdef struct foo "c_foo":
......
PYTHON setup.py build_ext --inplace
######## setup.py ########
from Cython.Build import cythonize
from distutils.core import setup
setup(
ext_modules = cythonize("*.pyx"),
)
######## test.pyx ########
from moda cimport DEFINE_A
from modb cimport DEFINE_B
######## moda.pxd ########
from verbatim cimport DEFINE_ONCE as DEFINE_A
######## modb.pxd ########
from verbatim cimport DEFINE_ONCE as DEFINE_B
######## verbatim.pxd ########
# Check that we include this only once
cdef extern from *:
"""
#ifdef DEFINE_ONCE
#error "DEFINE_ONCE already defined"
#endif
#define DEFINE_ONCE 1
"""
int DEFINE_ONCE
...@@ -26,7 +26,8 @@ def for_in_plain_ulong(): ...@@ -26,7 +26,8 @@ def for_in_plain_ulong():
print j print j
cdef extern from "for_from_pyvar_loop_T601_extern_def.h": cdef extern from *:
"""typedef unsigned long Ulong;"""
ctypedef unsigned long Ulong ctypedef unsigned long Ulong
cdef Ulong size(): cdef Ulong size():
......
static long cube(long x)
{
return x * x * x;
}
#define long broken_long
cdef extern from "verbatiminclude.h":
long cube(long)
cdef extern from *:
"""
static long square(long x)
{
return x * x;
}
"""
long square(long)
cdef extern from "verbatiminclude.h":
"typedef int myint;"
ctypedef int myint
cdef extern from "verbatiminclude.h":
"#undef long"
cdef class C:
cdef myint val
cdef extern from "Python.h":
"""
#define Py_SET_SIZE(obj, size) Py_SIZE((obj)) = (size)
"""
void Py_SET_SIZE(object, Py_ssize_t)
def test_square(x):
"""
>>> test_square(4)
16
"""
return square(x)
def test_cube(x):
"""
>>> test_cube(4)
64
"""
return cube(x)
def test_class():
"""
>>> test_class()
42
"""
cdef C x = C()
x.val = 42
return x.val
def test_set_size(x, size):
# This function manipulates Python objects in a bad way, so we
# do not call it. The real test is that it compiles.
Py_SET_SIZE(x, size)
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment