Commit dfa31de0 authored by Mark's avatar Mark

Merge pull request #61 from markflorisson88/_memview_rebase

Support for typed memoryviews, memoryview objects & cython.array
parents e8527c58 2a4ffd18
This diff is collapsed.
cimport cython
cdef class UtilityCode:
cdef class UtilityCodeBase(object):
cdef public object name
cdef class UtilityCode(UtilityCodeBase):
cdef public object proto
cdef public object impl
cdef public object init
......@@ -10,6 +13,7 @@ cdef class UtilityCode:
cdef public dict _cache
cdef public list specialize_list
cdef public object proto_block
cdef public object file
cpdef put_code(self, output)
......
This diff is collapsed.
......@@ -3,14 +3,19 @@ from PyrexTypes import *
from UtilityCode import CythonUtilityCode
from Errors import error
from Scanning import StringSourceDescriptor
import Options
import Buffer
import MemoryView
class CythonScope(ModuleScope):
is_cython_builtin = 1
def __init__(self):
def __init__(self, context):
ModuleScope.__init__(self, u'cython', None, None)
self.pxd_file_loaded = True
self.populate_cython_scope()
# The Main.Context object
self.context = context
def lookup_type(self, name):
# This function should go away when types are all first-level objects.
......@@ -66,28 +71,40 @@ class CythonScope(ModuleScope):
# self.test_cythonscope()
def test_cythonscope(self):
# A special function just to make it easy to test the scope and
# utility code functionality in isolation. It is available to
# "end-users" but nobody will know it is there anyway...
cython_testscope_utility_code.declare_in_scope(self)
cython_test_extclass_utility_code.declare_in_scope(self)
"""
Creates some entries for testing purposes and entries for
cython.array() and for cython.view.*.
"""
cython_testscope_utility_code.declare_in_scope(
self, cython_scope=self)
cython_test_extclass_utility_code.declare_in_scope(
self, cython_scope=self)
MemoryView.cython_array_utility_code.declare_in_scope(
self, cython_scope=self)
#
# The view sub-scope
#
self.viewscope = viewscope = ModuleScope(u'cython.view', self, None)
self.declare_module('view', viewscope, None)
self.viewscope = viewscope = ModuleScope(u'view', self, None)
self.declare_module('view', viewscope, None).as_module = viewscope
viewscope.is_cython_builtin = True
viewscope.pxd_file_loaded = True
cythonview_testscope_utility_code.declare_in_scope(viewscope)
cythonview_testscope_utility_code.declare_in_scope(
viewscope, cython_scope=self)
view_utility_scope = MemoryView.view_utility_code.declare_in_scope(
viewscope, cython_scope=self)
# MemoryView.memview_fromslice_utility_code.from_scope = view_utility_scope
# MemoryView.memview_fromslice_utility_code.declare_in_scope(viewscope)
def create_cython_scope(context, create_testscope):
# One could in fact probably make it a singleton,
# but not sure yet whether any code mutates it (which would kill reusing
# it across different contexts)
scope = CythonScope()
scope = CythonScope(context)
if create_testscope:
scope.test_cythonscope()
......@@ -95,80 +112,29 @@ def create_cython_scope(context, create_testscope):
return scope
cython_testscope_utility_code = CythonUtilityCode(u"""
@cname('__pyx_testscope')
cdef object _testscope(int value):
return "hello from cython scope, value=%d" % value
""")
# Load test utilities for the cython scope
def load_testscope_utility(cy_util_name, **kwargs):
return CythonUtilityCode.load(cy_util_name, "TestCythonScope.pyx", **kwargs)
undecorated_methods_protos = UtilityCode(proto=u"""
/* These methods are undecorated and have therefore no prototype */
static PyObject *__pyx_TestClass_cdef_method(
struct __pyx_TestClass *self, int value);
struct __pyx_TestClass_obj *self, int value);
static PyObject *__pyx_TestClass_cpdef_method(
struct __pyx_TestClass *self, int value, int skip_dispatch);
struct __pyx_TestClass_obj *self, int value, int skip_dispatch);
static PyObject *__pyx_TestClass_def_method(
PyObject *self, PyObject *value);
""")
test_cython_utility_dep = CythonUtilityCode(u"""
@cname('__pyx_test_dep')
cdef test_dep(obj):
print 'test_dep', obj
""")
cython_test_extclass_utility_code = CythonUtilityCode(
name="TestClassUtilityCode",
prefix="__pyx_prefix_TestClass_",
requires=[undecorated_methods_protos, test_cython_utility_dep],
impl=u"""
cdef extern from *:
cdef object __pyx_test_dep(object)
@cname('__pyx_TestClass')
cdef class TestClass(object):
cdef public int value
def __init__(self, int value):
self.value = value
def __str__(self):
return 'TestClass(%d)' % self.value
cython_testscope_utility_code = load_testscope_utility("TestScope")
cdef cdef_method(self, int value):
print 'Hello from cdef_method', value
test_cython_utility_dep = load_testscope_utility("TestDep")
cpdef cpdef_method(self, int value):
print 'Hello from cpdef_method', value
cython_test_extclass_utility_code = \
load_testscope_utility("TestClass", name="TestClass",
requires=[undecorated_methods_protos,
test_cython_utility_dep])
def def_method(self, int value):
print 'Hello from def_method', value
@cname('cdef_cname')
cdef cdef_cname_method(self, int value):
print "Hello from cdef_cname_method", value
@cname('cpdef_cname')
cpdef cpdef_cname_method(self, int value):
print "Hello from cpdef_cname_method", value
@cname('def_cname')
def def_cname_method(self, int value):
print "Hello from def_cname_method", value
@cname('__pyx_test_call_other_cy_util')
cdef test_call(obj):
print 'test_call'
__pyx_test_dep(obj)
@cname('__pyx_TestClass_New')
cdef _testclass_new(int value):
return TestClass(value)
""")
cythonview_testscope_utility_code = CythonUtilityCode(u"""
@cname('__pyx_view_testscope')
cdef object _testscope(int value):
return "hello from cython.view scope, value=%d" % value
""")
cythonview_testscope_utility_code = load_testscope_utility("View.TestScope")
\ No newline at end of file
This diff is collapsed.
......@@ -430,7 +430,7 @@ def create_default_resultobj(compilation_source, options):
def run_pipeline(source, options, full_module_name = None):
import Pipeline
# Set up context
context = options.create_context()
# Set up source object
......@@ -438,6 +438,7 @@ def run_pipeline(source, options, full_module_name = None):
abs_path = os.path.abspath(source)
source_ext = os.path.splitext(source)[1]
full_module_name = full_module_name or context.extract_module_name(source, options)
if options.relative_path_in_code_position_comments:
rel_path = full_module_name.replace('.', os.sep) + source_ext
if not abs_path.endswith(rel_path):
......@@ -584,7 +585,8 @@ def compile_multiple(sources, options):
a CompilationResultSet. Performs timestamp checking and/or recursion
if these are specified in the options.
"""
context = options.create_context()
# run_pipeline creates the context
# context = options.create_context()
sources = [os.path.abspath(source) for source in sources]
processed = set()
results = CompilationResultSet()
......
This diff is collapsed.
......@@ -65,10 +65,25 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
self.body.stats.extend(tree.stats)
else:
self.body.stats.append(tree)
selfscope = self.scope
selfscope.utility_code_list.extend(scope.utility_code_list)
self.scope.utility_code_list.extend(scope.utility_code_list)
def extend_if_not_in(L1, L2):
for x in L2:
if x not in L1:
L1.append(x)
extend_if_not_in(self.scope.include_files, scope.include_files)
extend_if_not_in(self.scope.included_files, scope.included_files)
extend_if_not_in(self.scope.python_include_files,
scope.python_include_files)
if merge_scope:
selfscope.merge_in(scope)
# Ensure that we don't generate import code for these entries!
for entry in scope.c_class_entries:
entry.type.module_name = self.full_module_name
self.scope.merge_in(scope)
def analyse_declarations(self, env):
if Options.embed_pos_in_docstring:
......@@ -126,7 +141,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
if (h_types or h_vars or h_funcs or h_extension_types):
result.h_file = replace_suffix(result.c_file, ".h")
h_code = Code.CCodeWriter()
Code.GlobalState(h_code)
Code.GlobalState(h_code, self)
if options.generate_pxi:
result.i_file = replace_suffix(result.c_file, ".pxi")
i_code = Code.PyrexCodeWriter(result.i_file)
......@@ -195,7 +210,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
if api_vars or api_funcs or api_extension_types:
result.api_file = replace_suffix(result.c_file, "_api.h")
h_code = Code.CCodeWriter()
Code.GlobalState(h_code)
Code.GlobalState(h_code, self)
api_guard = Naming.api_guard_prefix + self.api_name(env)
h_code.put_h_guard(api_guard)
h_code.putln('#include "Python.h"')
......@@ -293,7 +308,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
else:
emit_linenums = options.emit_linenums
rootwriter = Code.CCodeWriter(emit_linenums=emit_linenums, c_line_in_traceback=options.c_line_in_traceback)
globalstate = Code.GlobalState(rootwriter, emit_linenums)
globalstate = Code.GlobalState(rootwriter, self, emit_linenums)
globalstate.initialize_main_c_code()
h_code = globalstate['h_code']
......@@ -334,7 +349,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
self.generate_declarations_for_modules(env, modules, globalstate)
h_code.write('\n')
for utilcode in env.utility_code_list:
for utilcode in env.utility_code_list[:]:
globalstate.use_utility_code(utilcode)
globalstate.finalize_main_c_code()
......@@ -1185,10 +1200,13 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
type = scope.parent_type
base_type = type.base_type
py_attrs = []
memviewslice_attrs = []
for entry in scope.var_entries:
if entry.type.is_pyobject:
py_attrs.append(entry)
need_self_cast = type.vtabslot_cname or py_attrs
elif entry.type.is_memoryviewslice:
memviewslice_attrs.append(entry)
need_self_cast = type.vtabslot_cname or py_attrs or memviewslice_attrs
code.putln("")
code.putln(
"static PyObject *%s(PyTypeObject *t, PyObject *a, PyObject *k) {"
......@@ -1231,6 +1249,9 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
code.putln("p->%s = 0;" % entry.cname)
else:
code.put_init_var_to_py_none(entry, "p->%s", nanny=False)
for entry in memviewslice_attrs:
code.putln("p->%s.data = NULL;" % entry.cname)
code.putln("p->%s.memview = NULL;" % entry.cname)
entry = scope.lookup_here("__new__")
if entry and entry.is_special:
if entry.trivial_signature:
......@@ -2134,8 +2155,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
# variables to None.
for entry in env.var_entries:
if entry.visibility != 'extern':
if entry.type.is_pyobject and entry.used:
code.put_init_var_to_py_none(entry, nanny=False)
if entry.used:
entry.type.global_init_code(entry, code)
def generate_c_variable_export_code(self, env, code):
# Generate code to create PyCFunction wrappers for exported C functions.
......@@ -2237,7 +2258,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
# Generate type import code for extern extension types
# and type ready code for non-extern ones.
for entry in env.c_class_entries:
if entry.visibility == 'extern':
if entry.visibility == 'extern' and not entry.utility_code_definition:
self.generate_type_import_code(env, entry.type, entry.pos, code)
else:
self.generate_base_type_import_code(env, entry, code)
......@@ -2247,8 +2268,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
def generate_base_type_import_code(self, env, entry, code):
base_type = entry.type.base_type
if base_type and base_type.module_name != env.qualified_name \
and not base_type.is_builtin_type:
if (base_type and base_type.module_name != env.qualified_name and not
base_type.is_builtin_type and not entry.utility_code_definition):
self.generate_type_import_code(env, base_type, self.pos, code)
def use_type_import_utility_code(self, env):
......@@ -2419,8 +2440,9 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
type.typeptr_cname, type.typeobj_cname))
def generate_cfunction_declaration(entry, env, code, definition):
from_cy_utility = entry.used and entry.utility_code_definition
if entry.inline_func_in_pxd or (not entry.in_cinclude and (definition
or entry.defined_in_pxd or entry.visibility == 'extern')):
or entry.defined_in_pxd or entry.visibility == 'extern' or from_cy_utility)):
if entry.visibility == 'extern':
storage_class = "%s " % Naming.extern_c_macro
dll_linkage = "DL_IMPORT"
......
......@@ -38,10 +38,8 @@ typeobj_prefix = pyrex_prefix + "type_"
var_prefix = pyrex_prefix + "v_"
varptr_prefix = pyrex_prefix + "vp_"
wrapperbase_prefix= pyrex_prefix + "wrapperbase_"
bufstruct_prefix = pyrex_prefix + "bstruct_"
bufstride_prefix = pyrex_prefix + "bstride_"
bufshape_prefix = pyrex_prefix + "bshape_"
bufsuboffset_prefix = pyrex_prefix + "boffset_"
pybuffernd_prefix = pyrex_prefix + "pybuffernd_"
pybufferstruct_prefix = pyrex_prefix + "pybuffer_"
vtable_prefix = pyrex_prefix + "vtable_"
vtabptr_prefix = pyrex_prefix + "vtabptr_"
vtabstruct_prefix = pyrex_prefix + "vtabstruct_"
......
This diff is collapsed.
......@@ -67,11 +67,15 @@ old_style_globals = False
cimport_from_pyx = False
# max # of dims for buffers -- set to same value as max # of dims for numpy
# arrays.
buffer_max_dims = 32
# Declare compiler directives
directive_defaults = {
'boundscheck' : True,
'nonecheck' : False,
'initializedcheck' : True,
'embedsignature' : False,
'locals' : {},
'auto_cpdef': False,
......
......@@ -1037,7 +1037,8 @@ class ParallelRangeTransform(CythonTransform, SkipDeclarations):
directive = directive.rstrip('.')
cls = self.directive_to_node.get(directive)
if cls is None:
if cls is None and not (self.namenode_is_cython_module and
self.parallel_directive[0] != 'parallel'):
error(node.pos, "Invalid directive: %s" % directive)
self.namenode_is_cython_module = False
......@@ -1255,7 +1256,7 @@ class CnameDirectivesTransform(CythonTransform, SkipDeclarations):
"""
def handle_function(self, node):
if not node.decorators:
if not getattr(node, 'decorators', None):
return self.visit_Node(node)
for i, decorator in enumerate(node.decorators):
......@@ -1289,6 +1290,8 @@ class CnameDirectivesTransform(CythonTransform, SkipDeclarations):
visit_FuncDefNode = handle_function
visit_CClassDefNode = handle_function
visit_CEnumDefNode = handle_function
visit_CStructOrUnionDefNode = handle_function
class ForwardDeclareTypes(CythonTransform):
......@@ -1576,6 +1579,14 @@ if VALUE is not None:
self.visitchildren(node)
return None
def visit_CnameDecoratorNode(self, node):
self.visitchildren(node)
if not node.node:
return None
return node
def create_Property(self, entry):
if entry.visibility == 'public':
if entry.type.is_pyobject:
......@@ -2213,6 +2224,7 @@ class GilCheck(VisitorTransform):
if self.env_stack and self.nogil and node.nogil_check:
node.nogil_check(self.env_stack[-1])
self.visitchildren(node)
node.in_nogil_context = self.nogil
return node
......
......@@ -7,8 +7,8 @@ ctypedef object (*p_sub_expr_func)(object)
# entry points
cpdef p_module(PyrexScanner s, pxd, full_module_name)
cpdef p_code(PyrexScanner s, level= *)
cpdef p_module(PyrexScanner s, pxd, full_module_name, ctx=*)
cpdef p_code(PyrexScanner s, level= *, ctx=*)
# internal parser states
......@@ -131,6 +131,8 @@ cdef p_calling_convention(PyrexScanner s)
cdef p_c_complex_base_type(PyrexScanner s)
cpdef p_c_simple_base_type(PyrexScanner s, bint self_flag, bint nonempty, templates = *)
cdef p_buffer_or_template(PyrexScanner s, base_type_node, templates)
cdef is_memoryviewslice_access(PyrexScanner s)
cdef p_memoryviewslice_access(PyrexScanner s, base_type_node)
cdef bint looking_at_name(PyrexScanner s) except -2
cdef bint looking_at_expr(PyrexScanner s) except -2
cdef bint looking_at_base_type(PyrexScanner s) except -2
......
......@@ -33,6 +33,7 @@ class Ctx(object):
nogil = 0
namespace = None
templates = None
allow_struct_enum_decorator = False
def __init__(self, **kwds):
self.__dict__.update(kwds)
......@@ -296,7 +297,8 @@ def p_typecast(s):
pos = s.position()
s.next()
base_type = p_c_base_type(s)
if base_type.name is None:
is_memslice = isinstance(base_type, Nodes.MemoryViewSliceTypeNode)
if not is_memslice and base_type.name is None:
s.error("Unknown type")
declarator = p_c_declarator(s, empty = 1)
if s.sy == '?':
......@@ -306,6 +308,10 @@ def p_typecast(s):
typecheck = 0
s.expect(">")
operand = p_factor(s)
if is_memslice:
return ExprNodes.CythonArrayNode(pos, base_type_node=base_type,
operand=operand)
return ExprNodes.TypecastNode(pos,
base_type = base_type,
declarator = declarator,
......@@ -1754,7 +1760,8 @@ def p_statement(s, ctx, first_statement = 0):
s.error('decorator not allowed here')
s.level = ctx.level
decorators = p_decorators(s)
if s.sy not in ('def', 'cdef', 'cpdef', 'class'):
bad_toks = 'def', 'cdef', 'cpdef', 'class'
if not ctx.allow_struct_enum_decorator and s.sy not in bad_toks:
s.error("Decorators can only be followed by functions or classes")
elif s.sy == 'pass' and cdef_flag:
# empty cdef block
......@@ -1774,7 +1781,10 @@ def p_statement(s, ctx, first_statement = 0):
s.level = ctx.level
node = p_cdef_statement(s, ctx(overridable = overridable))
if decorators is not None:
if not isinstance(node, (Nodes.CFuncDefNode, Nodes.CVarDefNode, Nodes.CClassDefNode)):
tup = Nodes.CFuncDefNode, Nodes.CVarDefNode, Nodes.CClassDefNode
if ctx.allow_struct_enum_decorator:
tup += Nodes.CStructOrUnionDefNode, Nodes.CEnumDefNode
if not isinstance(node, tup):
s.error("Decorators can only be followed by functions or classes")
node.decorators = decorators
return node
......@@ -2002,7 +2012,11 @@ def p_c_simple_base_type(s, self_flag, nonempty, templates = None):
complex = complex, longness = longness,
is_self_arg = self_flag, templates = templates)
# declarations here.
if s.sy == '[':
if is_memoryviewslice_access(s):
type_node = p_memoryviewslice_access(s, type_node)
else:
type_node = p_buffer_or_template(s, type_node, templates)
if s.sy == '.':
......@@ -2034,6 +2048,63 @@ def p_buffer_or_template(s, base_type_node, templates):
base_type_node = base_type_node)
return result
def p_bracketed_base_type(s, base_type_node, nonempty, empty):
# s.sy == '['
if empty and not nonempty:
# sizeof-like thing. Only anonymous C arrays allowed (int[SIZE]).
return base_type_node
elif not empty and nonempty:
# declaration of either memoryview slice or buffer.
if is_memoryviewslice_access(s):
return p_memoryviewslice_access(s, base_type_node)
else:
return p_buffer_or_template(s, base_type_node, None)
# return p_buffer_access(s, base_type_node)
elif not empty and not nonempty:
# only anonymous C arrays and memoryview slice arrays here. We
# disallow buffer declarations for now, due to ambiguity with anonymous
# C arrays.
if is_memoryviewslice_access(s):
return p_memoryviewslice_access(s, base_type_node)
else:
return base_type_node
def is_memoryviewslice_access(s):
# s.sy == '['
# a memoryview slice declaration is distinguishable from a buffer access
# declaration by the first entry in the bracketed list. The buffer will
# not have an unnested colon in the first entry; the memoryview slice will.
saved = [(s.sy, s.systring)]
s.next()
retval = False
if s.systring == ':':
retval = True
elif s.sy == 'INT':
saved.append((s.sy, s.systring))
s.next()
if s.sy == ':':
retval = True
for sv in saved[::-1]:
s.put_back(*sv)
return retval
def p_memoryviewslice_access(s, base_type_node):
# s.sy == '['
pos = s.position()
s.next()
subscripts = p_subscript_list(s)
# make sure each entry in subscripts is a slice
for subscript in subscripts:
if len(subscript) < 2:
s.error("An axis specification in memoryview declaration does not have a ':'.")
s.expect(']')
indexes = make_slice_nodes(pos, subscripts)
result = Nodes.MemoryViewSliceTypeNode(pos,
base_type_node = base_type_node,
axes = indexes)
return result
def looking_at_name(s):
return s.sy == 'IDENT' and not s.systring in calling_convention_words
......@@ -2831,8 +2902,8 @@ def p_doc_string(s):
else:
return None
def p_code(s, level=None):
body = p_statement_list(s, Ctx(level = level), first_statement = 1)
def p_code(s, level=None, ctx=Ctx):
body = p_statement_list(s, ctx(level = level), first_statement = 1)
if s.sy != 'EOF':
s.error("Syntax error in statement [%s,%s]" % (
repr(s.sy), repr(s.systring)))
......@@ -2854,7 +2925,7 @@ def p_compiler_directive_comments(s):
s.next()
return result
def p_module(s, pxd, full_module_name):
def p_module(s, pxd, full_module_name, ctx=Ctx):
pos = s.position()
directive_comments = p_compiler_directive_comments(s)
......@@ -2869,7 +2940,7 @@ def p_module(s, pxd, full_module_name):
else:
level = 'module'
body = p_statement_list(s, Ctx(level = level), first_statement = 1)
body = p_statement_list(s, ctx(level=level), first_statement = 1)
if s.sy != 'EOF':
s.error("Syntax error in statement [%s,%s]" % (
repr(s.sy), repr(s.systring)))
......@@ -2977,4 +3048,3 @@ def print_parse_tree(f, node, level, key = None):
f.write("%s]\n" % ind)
return
f.write("%s%s\n" % (ind, node))
......@@ -4,6 +4,7 @@ from time import time
import Errors
import DebugFlags
import Options
from Visitor import CythonTransform
from Errors import PyrexError, CompileError, InternalError, AbortError, error
#
......@@ -72,22 +73,41 @@ def use_utility_code_definitions(scope, target):
def inject_utility_code_stage_factory(context):
def inject_utility_code_stage(module_node):
# First, make sure any utility code pulled in by using symbols in the cython
# scope is included
use_utility_code_definitions(context.cython_scope, module_node.scope)
added = []
# Note: the list might be extended inside the loop (if some utility code
# pulls in other utility code)
# pulls in other utility code, explicitly or implicitly)
for utilcode in module_node.scope.utility_code_list:
if utilcode in added: continue
added.append(utilcode)
if utilcode.requires:
for dep in utilcode.requires:
if not dep in added and not dep in module_node.scope.utility_code_list:
module_node.scope.utility_code_list.append(dep)
tree = utilcode.get_tree()
if tree:
module_node.merge_in(tree.body, tree.scope, merge_scope=True)
return module_node
return inject_utility_code_stage
class UseUtilityCodeDefinitions(CythonTransform):
# Temporary hack to use any utility code in nodes' "utility_code_definitions".
# This should be moved to the code generation phase of the relevant nodes once
# it is safe to generate CythonUtilityCode at code generation time.
def __call__(self, node):
self.scope = node.scope
return super(UseUtilityCodeDefinitions, self).__call__(node)
def visit_AttributeNode(self, node):
if node.entry and node.entry.utility_code_definition:
self.scope.use_utility_code(node.entry.utility_code_definition)
return node
def visit_NameNode(self, node):
for e in (node.entry, node.type_entry):
if e and e.utility_code_definition:
self.scope.use_utility_code(e.utility_code_definition)
return node
#
# Pipeline factories
#
......@@ -113,6 +133,8 @@ def create_pipeline(context, mode, exclude_classes=()):
from Optimize import DropRefcountingTransform
from Buffer import IntroduceBufferAuxiliaryVars
from ModuleNode import check_c_declarations, check_c_declarations_pxd
from ModuleNode import check_c_declarations
if mode == 'pxd':
_check_c_declarations = check_c_declarations_pxd
......@@ -165,6 +187,7 @@ def create_pipeline(context, mode, exclude_classes=()):
DropRefcountingTransform(),
FinalOptimizePhase(context),
GilCheck(),
UseUtilityCodeDefinitions(context),
]
filtered_stages = []
for s in stages:
......@@ -265,7 +288,9 @@ def insert_into_pipeline(pipeline, transform, before=None, after=None):
# Running a pipeline
#
def run_pipeline(pipeline, source):
def run_pipeline(pipeline, source, printtree=True):
from Cython.Compiler.Visitor import PrintTree
error = None
data = source
try:
......@@ -275,6 +300,8 @@ def run_pipeline(pipeline, source):
if DebugFlags.debug_verbose_pipeline:
t = time()
print "Entering pipeline phase %r" % phase
if not printtree and isinstance(phase, PrintTree):
continue
data = phase(data)
if DebugFlags.debug_verbose_pipeline:
print " %.3f seconds" % (time() - t)
......
This diff is collapsed.
......@@ -39,12 +39,9 @@ def c_safe_identifier(cname):
class BufferAux(object):
writable_needed = False
def __init__(self, buffer_info_var, stridevars, shapevars,
suboffsetvars):
self.buffer_info_var = buffer_info_var
self.stridevars = stridevars
self.shapevars = shapevars
self.suboffsetvars = suboffsetvars
def __init__(self, buflocal_nd_var, rcbuf_var):
self.buflocal_nd_var = buflocal_nd_var
self.rcbuf_var = rcbuf_var
def __repr__(self):
return "<BufferAux %r>" % self.__dict__
......@@ -127,6 +124,8 @@ class Entry(object):
# used uninitialized
# cf_used boolean Entry is used
# TODO: utility_code and utility_code_definition serves the same purpose...
inline_func_in_pxd = False
borrowed = 0
init = ""
......@@ -292,8 +291,6 @@ class Scope(object):
entries = [(name, entry)
for name, entry in other.entries.iteritems()
if entry.used or merge_unused]
# !@#$ py23
entries = dict(entries)
self.entries.update(entries)
......@@ -306,8 +303,9 @@ class Scope(object):
'cfunc_entries',
'c_class_entries'):
self_entries = getattr(self, attr)
names = set([e.name for e in self_entries])
for entry in getattr(other, attr):
if entry.used or merge_unused:
if (entry.used or merge_unused) and entry.name not in names:
self_entries.append(entry)
def __str__(self):
......@@ -383,6 +381,11 @@ class Scope(object):
# entries[name] = entry
if not shadow:
entries[name] = entry
if type.is_memoryviewslice:
import MemoryView
entry.init = MemoryView.memslice_entry_init
entry.scope = self
entry.visibility = visibility
return entry
......@@ -1163,7 +1166,7 @@ class ModuleScope(Scope):
def declare_c_class(self, name, pos, defining = 0, implementing = 0,
module_name = None, base_type = None, objstruct_cname = None,
typeobj_cname = None, visibility = 'private', typedef_flag = 0, api = 0,
typeobj_cname = None, typeptr_cname = None, visibility = 'private', typedef_flag = 0, api = 0,
buffer_defaults = None, shadow = 0):
# If this is a non-extern typedef class, expose the typedef, but use
# the non-typedef struct internally to avoid needing forward
......@@ -1205,6 +1208,9 @@ class ModuleScope(Scope):
type.module_name = module_name
else:
type.module_name = self.qualified_name
if typeptr_cname:
type.typeptr_cname = typeptr_cname
else:
type.typeptr_cname = self.mangle(Naming.typeptr_prefix, name)
entry = self.declare_type(name, type, pos, visibility = visibility,
defining = 0, shadow = shadow)
......
......@@ -98,3 +98,8 @@ class TestBufferOptions(CythonTest):
self.assert_(stats[1].base_type.ndim == 3)
# add exotic and impossible combinations as they come along...
if __name__ == '__main__':
import unittest
unittest.main()
from Cython.TestUtils import CythonTest
import Cython.Compiler.Errors as Errors
from Cython.Compiler.Nodes import *
from Cython.Compiler.ParseTreeTransforms import *
from Cython.Compiler.Buffer import *
class TestMemviewParsing(CythonTest):
def parse(self, s):
return self.should_not_fail(lambda: self.fragment(s)).root
def not_parseable(self, expected_error, s):
e = self.should_fail(lambda: self.fragment(s), Errors.CompileError)
self.assertEqual(expected_error, e.message_only)
def test_default_1dim(self):
self.parse(u"cdef int[:] x")
self.parse(u"cdef short int[:] x")
def test_default_ndim(self):
self.parse(u"cdef int[:,:,:,:,:] x")
self.parse(u"cdef unsigned long int[:,:,:,:,:] x")
self.parse(u"cdef unsigned int[:,:,:,:,:] x")
def test_zero_offset(self):
self.parse(u"cdef long double[0:] x")
self.parse(u"cdef int[0:] x")
def test_zero_offset_ndim(self):
self.parse(u"cdef int[0:,0:,0:,0:] x")
def test_def_arg(self):
self.parse(u"def foo(int[:,:] x): pass")
def test_cdef_arg(self):
self.parse(u"cdef foo(int[:,:] x): pass")
def test_general_slice(self):
self.parse(u'cdef float[::ptr, ::direct & contig, 0::full & strided] x')
def test_non_slice_memview(self):
self.not_parseable(u"An axis specification in memoryview declaration does not have a ':'.",
u"cdef double[:foo, bar] x")
self.not_parseable(u"An axis specification in memoryview declaration does not have a ':'.",
u"cdef double[0:foo, bar] x")
def test_basic(self):
t = self.parse(u"cdef int[:] x")
memv_node = t.stats[0].base_type
self.assert_(isinstance(memv_node, MemoryViewSliceTypeNode))
# we also test other similar declarations (buffers, anonymous C arrays)
# since the parsing has to distinguish between them.
def disable_test_no_buf_arg(self): # TODO
self.not_parseable(u"Expected ']'",
u"cdef extern foo(object[int, ndim=2])")
def disable_test_parse_sizeof(self): # TODO
self.parse(u"sizeof(int[NN])")
self.parse(u"sizeof(int[])")
self.parse(u"sizeof(int[][NN])")
self.not_parseable(u"Expected an identifier or literal",
u"sizeof(int[:NN])")
self.not_parseable(u"Expected ']'",
u"sizeof(foo[dtype=bar]")
if __name__ == '__main__':
import unittest
unittest.main()
import unittest
from Cython.Compiler import Code, UtilityCode
def strip_2tup(tup):
return tup[0] and tup[0].strip(), tup[1] and tup[1].strip()
class TestUtilityLoader(unittest.TestCase):
"""
Test loading UtilityCodes
"""
expected = "test {{loader}} prototype", "test {{loader}} impl"
expected_tempita = (expected[0].replace('{{loader}}', 'Loader'),
expected[1].replace('{{loader}}', 'Loader'))
required = "I am a dependency proto", "I am a dependency impl"
context = dict(loader='Loader')
name = "TestUtilityLoader"
filename = "TestUtilityLoader.c"
cls = Code.UtilityCode
def test_load_as_string(self):
got = strip_2tup(self.cls.load_as_string(self.name))
self.assertEquals(got, self.expected)
got = strip_2tup(self.cls.load_as_string(self.name, self.filename))
self.assertEquals(got, self.expected)
got = strip_2tup(self.cls.load_as_string(self.name, context=self.context))
self.assertEquals(got, self.expected_tempita)
def test_load(self):
utility = self.cls.load(self.name)
got = strip_2tup((utility.proto, utility.impl))
self.assertEquals(got, self.expected)
# Not implemented yet
#required, = utility.requires
#self.assertEquals((required.proto, required.impl), self.required)
utility = self.cls.load(self.name, from_file=self.filename)
got = strip_2tup((utility.proto, utility.impl))
self.assertEquals(got, self.expected)
class TestCythonUtilityLoader(TestUtilityLoader):
"""
Test loading CythonUtilityCodes
"""
# Just change the attributes and run the same tests
expected = None, "test {{cy_loader}} impl"
expected_tempita = None, "test CyLoader impl"
required = None, "I am a Cython dependency impl"
context = dict(cy_loader='CyLoader')
name = "TestCyUtilityLoader"
filename = "TestCyUtilityLoader.pyx"
cls = UtilityCode.CythonUtilityCode
# Small hack to pass our tests above
cls.proto = None
\ No newline at end of file
......@@ -32,7 +32,7 @@ class StringParseContext(Main.Context):
return ModuleScope(module_name, parent_module = None, context = self)
def parse_from_strings(name, code, pxds={}, level=None, initial_pos=None,
context=None):
context=None, allow_struct_enum_decorator=False):
"""
Utility method to parse a (unicode) string of code. This is mostly
used for internal Cython compiler purposes (creating code snippets
......@@ -66,12 +66,15 @@ def parse_from_strings(name, code, pxds={}, level=None, initial_pos=None,
scanner = PyrexScanner(buf, code_source, source_encoding = encoding,
scope = scope, context = context, initial_pos = initial_pos)
ctx = Parsing.Ctx(allow_struct_enum_decorator=allow_struct_enum_decorator)
if level is None:
tree = Parsing.p_module(scanner, 0, module_name)
tree = Parsing.p_module(scanner, 0, module_name, ctx=ctx)
tree.scope = scope
tree.is_pxd = False
else:
tree = Parsing.p_code(scanner, level=level)
tree = Parsing.p_code(scanner, level=level, ctx=ctx)
tree.scope = scope
return tree
......
......@@ -3,9 +3,15 @@ from Scanning import StringSourceDescriptor
import Symtab
import Naming
from Cython.Compiler import Visitor
import Code
class NonManglingModuleScope(Symtab.ModuleScope):
def __init__(self, prefix, *args, **kw):
self.prefix = prefix
self.cython_scope = None
Symtab.ModuleScope.__init__(self, *args, **kw)
def add_imported_entry(self, name, entry, pos):
entry.used = True
return super(NonManglingModuleScope, self).add_imported_entry(
......@@ -13,32 +19,36 @@ class NonManglingModuleScope(Symtab.ModuleScope):
def mangle(self, prefix, name=None):
if name:
if prefix in (Naming.typeobj_prefix, Naming.func_prefix):
if prefix in (Naming.typeobj_prefix, Naming.func_prefix, Naming.var_prefix, Naming.pyfunc_prefix):
# Functions, classes etc. gets a manually defined prefix easily
# manually callable instead (the one passed to CythonUtilityCode)
prefix = self.prefix
return "%s%s" % (prefix, name)
else:
return self.base.name
return Symtab.ModuleScope.mangle(self, prefix)
class CythonUtilityCodeContext(StringParseContext):
scope = None
def find_module(self, module_name, relative_to = None, pos = None,
need_pxd = 1):
if module_name != self.module_name:
raise AssertionError("Not yet supporting any cimports/includes "
"from string code snippets")
if module_name not in self.modules:
raise AssertionError("Only the cython cimport is supported.")
else:
return self.modules[module_name]
if self.scope is None:
self.scope = NonManglingModuleScope(
module_name, parent_module=None, context=self)
self.scope.prefix = self.prefix
self.scope = NonManglingModuleScope(self.prefix,
module_name,
parent_module=None,
context=self)
return self.scope
class CythonUtilityCode(object):
class CythonUtilityCode(Code.UtilityCodeBase):
"""
Utility code written in the Cython language itself.
......@@ -46,8 +56,8 @@ class CythonUtilityCode(object):
Functions decorated with @cname('c_func_name') get the given cname.
For cdef classes the rules are as follows:
obj struct -> <cname>
obj type ptr -> __pyx_ptype_<cname>
obj struct -> <cname>_obj
obj type ptr -> <cname>_type
methods -> <class_cname>_<method_cname>
For methods the cname decorator is optional, but without the decorator the
......@@ -57,19 +67,22 @@ class CythonUtilityCode(object):
is_cython_utility = True
def __init__(self, impl, name="CythonUtilityCode", prefix="", requires=None):
def __init__(self, impl, name="__pyxutil", prefix="", requires=None,
file=None, from_scope=None):
# 1) We need to delay the parsing/processing, so that all modules can be
# imported without import loops
# 2) The same utility code object can be used for multiple source files;
# while the generated node trees can be altered in the compilation of a
# single file.
# Hence, delay any processing until later.
self.pyx = impl
self.impl = impl
self.name = name
self.file = file
self.prefix = prefix
self.requires = requires or []
self.from_scope = from_scope
def get_tree(self):
def get_tree(self, entries_only=False, cython_scope=None):
from AnalysedTreeTransforms import AutoTestDictTransform
# The AutoTestDictTransform creates the statement "__test__ = {}",
# which when copied into the main ModuleNode overwrites
......@@ -79,10 +92,21 @@ class CythonUtilityCode(object):
import Pipeline, ParseTreeTransforms
context = CythonUtilityCodeContext(self.name)
context.prefix = self.prefix
context.cython_scope = cython_scope
#context = StringParseContext(self.name)
tree = parse_from_strings(self.name, self.pyx, context=context)
tree = parse_from_strings(self.name, self.impl, context=context,
allow_struct_enum_decorator=True)
pipeline = Pipeline.create_pipeline(context, 'pyx', exclude_classes=excludes)
if entries_only:
p = []
for t in pipeline:
p.append(t)
if isinstance(p, ParseTreeTransforms.AnalyseDeclarationsTransform):
break
pipeline = p
transform = ParseTreeTransforms.CnameDirectivesTransform(context)
# InterpretCompilerDirectives already does a cdef declarator check
#before = ParseTreeTransforms.DecoratorTransform
......@@ -90,21 +114,31 @@ class CythonUtilityCode(object):
pipeline = Pipeline.insert_into_pipeline(pipeline, transform,
before=before)
(err, tree) = Pipeline.run_pipeline(pipeline, tree)
if self.from_scope:
def scope_transform(module_node):
module_node.scope.merge_in(self.from_scope)
return module_node
transform = ParseTreeTransforms.AnalyseDeclarationsTransform
pipeline = Pipeline.insert_into_pipeline(pipeline, scope_transform,
before=transform)
(err, tree) = Pipeline.run_pipeline(pipeline, tree, printtree=False)
assert not err, err
return tree
def put_code(self, output):
pass
def declare_in_scope(self, dest_scope, used=False):
def declare_in_scope(self, dest_scope, used=False, cython_scope=None):
"""
Declare all entries from the utility code in dest_scope. Code will only
be included for used entries.
be included for used entries. If module_name is given, declare the
type entries with that name.
"""
self.tree = self.get_tree()
tree = self.get_tree(entries_only=True, cython_scope=cython_scope)
entries = self.tree.scope.entries
entries = tree.scope.entries
entries.pop('__name__')
entries.pop('__file__')
entries.pop('__builtins__')
......@@ -114,9 +148,12 @@ class CythonUtilityCode(object):
entry.utility_code_definition = self
entry.used = used
dest_scope.merge_in(self.tree.scope, merge_unused=True)
self.tree.scope = dest_scope
original_scope = tree.scope
dest_scope.merge_in(original_scope, merge_unused=True)
tree.scope = dest_scope
for dep in self.requires:
if dep.is_cython_utility:
dep.declare_in_scope(dest_scope)
return original_scope
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
########## TestCyUtilityLoader ##########
test {{cy_loader}} impl
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment