merged in latest cython-unstable

.hgtags

@@ -13,3 +13,7 @@ a4abf0156540db4d3ebaa95712b65811c43c5acb 0.11-beta
 4497f635d5fdbd38ebb841be4869fbfa2bbfdbb6 0.11.1.alpha
 7bc36a0f81723117a19f92ffde1676a0884fef65 0.11.1.beta
 6454db601984145f38e28d34176fca8a3a22329c 0.11.1
+af6f1bed8cd40a2edefb57d3eacbc9274a8788b4 0.11.2.rc1
+15ad532e2127840ae09dfbe46ccc80ac8c562f99 0.11.2
+eb00d00a73c13b6aa8b440fe07cd7acb52a060e8 0.11.3.rc0
+7c695fe49fd6912f52d995fe512d66baacf90ee6 0.11.3

Cython/Compiler/AnalysedTreeTransforms.py

+from Cython.Compiler.Visitor import VisitorTransform, ScopeTrackingTransform, TreeVisitor
+from Nodes import StatListNode, SingleAssignmentNode, CFuncDefNode
+from ExprNodes import (DictNode, DictItemNode, NameNode, UnicodeNode, NoneNode,
+                      ExprNode, AttributeNode, ModuleRefNode, DocstringRefNode)
+from PyrexTypes import py_object_type
+from Builtin import dict_type
+from StringEncoding import EncodedString
+import Naming
+
+class AutoTestDictTransform(ScopeTrackingTransform):
+    # Handles autotestdict directive
+
+    def visit_ModuleNode(self, node):
+        self.scope_type = 'module'
+        self.scope_node = node
+        if self.current_directives['autotestdict']:
+            assert isinstance(node.body, StatListNode)
+
+            # First see if __test__ is already created
+            if u'__test__' in node.scope.entries:
+                # Do nothing
+                return node
+            
+            pos = node.pos
+
+            self.tests = []
+            self.testspos = node.pos
+
+            test_dict_entry = node.scope.declare_var(EncodedString(u'__test__'),
+                                                     py_object_type,
+                                                     pos,
+                                                     visibility='public')
+            create_test_dict_assignment = SingleAssignmentNode(pos,
+                lhs=NameNode(pos, name=EncodedString(u'__test__'),
+                             entry=test_dict_entry),
+                rhs=DictNode(pos, key_value_pairs=self.tests))
+            self.visitchildren(node)
+            node.body.stats.append(create_test_dict_assignment)
+
+            
+        return node
+
+    def add_test(self, testpos, name, func_ref_node):
+        # func_ref_node must evaluate to the function object containing
+        # the docstring, BUT it should not be the function itself (which
+        # would lead to a new *definition* of the function)
+        pos = self.testspos
+        keystr = u'%s (line %d)' % (name, testpos[1])
+        key = UnicodeNode(pos, value=EncodedString(keystr))
+
+        value = DocstringRefNode(pos, func_ref_node)
+        self.tests.append(DictItemNode(pos, key=key, value=value))
+    
+    def visit_FuncDefNode(self, node):
+        if node.doc:
+            if isinstance(node, CFuncDefNode) and not node.py_func:
+                # skip non-cpdef cdef functions
+                return node
+            
+            pos = self.testspos
+            if self.scope_type == 'module':
+                parent = ModuleRefNode(pos)
+                name = node.entry.name
+            elif self.scope_type in ('pyclass', 'cclass'):
+                mod = ModuleRefNode(pos)
+                if self.scope_type == 'pyclass':
+                    clsname = self.scope_node.name
+                else:
+                    clsname = self.scope_node.class_name
+                parent = AttributeNode(pos, obj=mod,
+                                       attribute=clsname,
+                                       type=py_object_type,
+                                       is_py_attr=True,
+                                       is_temp=True)
+                name = "%s.%s" % (clsname, node.entry.name)
+            else:
+                assert False
+            getfunc = AttributeNode(pos, obj=parent,
+                                    attribute=node.entry.name,
+                                    type=py_object_type,
+                                    is_py_attr=True,
+                                    is_temp=True)
+            self.add_test(node.pos, name, getfunc)
+        return node
+

Cython/Compiler/AutoDocTransforms.py

@@ -82,7 +82,7 @@ class EmbedSignature(CythonTransform):
 
     def _embed_signature(self, signature, node_doc):
         if node_doc:
-            return signature + '\n' + node_doc
+            return "%s\n%s" % (signature, node_doc)
         else:
             return signature
 

Cython/Compiler/Builtin.py

@@ -3,7 +3,7 @@
 #
 
 from Symtab import BuiltinScope, StructOrUnionScope
-from Cython.Utils import UtilityCode
+from Code import UtilityCode
 from TypeSlots import Signature
 import PyrexTypes
 import Naming
@@ -50,6 +50,7 @@ builtin_function_table = [
     #('round',     "",     "",      ""),
     ('setattr',    "OOO",  "r",     "PyObject_SetAttr"),
     #('sum',       "",     "",      ""),
+    ('type',       "O",    "O",     "PyObject_Type"),
     #('unichr',    "",     "",      ""),
     #('unicode',   "",     "",      ""),
     #('vars',      "",     "",      ""),
@@ -159,6 +160,11 @@ bad:
 
 pyexec_utility_code = UtilityCode(
 proto = """
+#if PY_VERSION_HEX < 0x02040000
+#ifndef Py_EVAL_H
+#include "eval.h"
+#endif
+#endif
 static PyObject* __Pyx_PyRun(PyObject*, PyObject*, PyObject*);
 """,
 impl = """
@@ -167,18 +173,36 @@ static PyObject* __Pyx_PyRun(PyObject* o, PyObject* globals, PyObject* locals) {
     PyObject* s = 0;
     char *code = 0;
 
-    if (!locals && !globals) {
+    if (!globals || globals == Py_None) {
         globals = PyModule_GetDict(%s);""" % Naming.module_cname + """
         if (!globals)
             goto bad;
+    } else if (!PyDict_Check(globals)) {
+        PyErr_Format(PyExc_TypeError, "exec() arg 2 must be a dict, not %.100s",
+                     globals->ob_type->tp_name);
+        goto bad;
+    }
+    if (!locals || locals == Py_None) {
         locals = globals;
-    } else if (!locals) {
-        locals = globals;
-    } else if (!globals) {
-        globals = locals;
     }
 
+
+    if (PyDict_GetItemString(globals, "__builtins__") == NULL) {
+	PyDict_SetItemString(globals, "__builtins__", PyEval_GetBuiltins());
+    }
+
+    if (PyCode_Check(o)) {
+        if (PyCode_GetNumFree((PyCodeObject *)o) > 0) {
+            PyErr_SetString(PyExc_TypeError,
+                "code object passed to exec() may not contain free variables");
+            goto bad;
+        }
+	result = PyEval_EvalCode((PyCodeObject *)o, globals, locals);
+    } else {
+        PyCompilerFlags cf;
+        cf.cf_flags = 0;
 	if (PyUnicode_Check(o)) {
+            cf.cf_flags = PyCF_SOURCE_IS_UTF8;
     	    s = PyUnicode_AsUTF8String(o);
     	    if (!s) goto bad;
     	    o = s;
@@ -187,20 +211,23 @@ static PyObject* __Pyx_PyRun(PyObject* o, PyObject* globals, PyObject* locals) {
 	#else
 	} else if (!PyString_Check(o)) {
 	#endif
-        /* FIXME: support file objects and code objects */
     	    PyErr_SetString(PyExc_TypeError,
-            "exec currently requires a string as code input.");
+        	"exec: arg 1 must be string, bytes or code object");
     	    goto bad;
 	}
-
 	#if PY_MAJOR_VERSION >= 3
 	code = PyBytes_AS_STRING(o);
 	#else
 	code = PyString_AS_STRING(o);
 	#endif
+	if (PyEval_MergeCompilerFlags(&cf)) {
+	    result = PyRun_StringFlags(code, Py_file_input, globals, locals, &cf);
+        } else {
 	    result = PyRun_String(code, Py_file_input, globals, locals);
-
+        }
         Py_XDECREF(s);
+    }
+
     return result;
 bad:
     Py_XDECREF(s);

Cython/Compiler/CmdLine.py

@@ -126,7 +126,7 @@ def parse_command_line(args):
                 options.emit_linenums = True
             elif option in ("-X", "--directive"):
                 try:
-                    options.pragma_overrides = Options.parse_option_list(pop_arg())
+                    options.compiler_directives = Options.parse_option_list(pop_arg())
                 except ValueError, e:
                     sys.stderr.write("Error in compiler directive: %s\n" % e.message)
                     sys.exit(1)
@@ -153,5 +153,9 @@ def parse_command_line(args):
         sys.exit(1)
     if len(sources) == 0 and not options.show_version:
         bad_usage()
+    if Options.embed and len(sources) > 1:
+        sys.stderr.write(
+            "cython: Only one source file allowed when using -embed\n")
+        sys.exit(1)
     return options, sources
 

Cython/Compiler/ControlFlow.py

@@ -57,10 +57,36 @@ class ControlFlow(object):
             try:
                 return self.tip[item]
             except KeyError:
-                self.tip[item] = pos_state = self._get_pos_state(item, pos)
+                pass
+        pos_state = self._get_pos_state(item, pos)
+        if pos > self.end_pos:
+            self.tip[item] = pos_state
         return pos_state
-        else:
-            return self._get_pos_state(item, pos)
+
+    def _get_pos_state(self, item, pos):
+        current = self
+        while current is not None and pos <= current.start_pos:
+            current = current.incoming
+        if current is None:
+            return (None, None)
+        state = current._get_pos_state_local(item, pos)
+        while state is None and current.incoming is not None:
+            current = current.incoming
+            state = current._get_pos_state_local(item, pos)
+        if state is None:
+            return (None, None)
+        return state
+
+    def set_state(self, pos, item, state):
+        if item in self.tip:
+            del self.tip[item]
+        current = self
+        while pos < current.start_pos and current.incoming is not None:
+            current = current.incoming
+            if item in current.tip:
+                del current.tip[item]
+        current._set_state_local(pos, item, state)
+        
         
 class LinearControlFlow(ControlFlow):
 
@@ -68,13 +94,7 @@ class LinearControlFlow(ControlFlow):
         ControlFlow.__init__(self, start_pos, incoming, parent)
         self.events = {}
             
-    def set_state(self, pos, item, state):
-        if item in self.tip:
-            del self.tip[item]
-        if pos < self.start_pos:
-            if self.incoming is not None:
-                self.incoming.set_state(pos, item, state)
-        else:
+    def _set_state_local(self, pos, item, state):
         if item in self.events:
             event_list = self.events[item]
         else:
@@ -82,20 +102,13 @@ class LinearControlFlow(ControlFlow):
             self.events[item] = event_list
         bisect.insort(event_list, (pos, state))
 
-        
-    def _get_pos_state(self, item, pos):
-        if pos > self.start_pos:
+    def _get_pos_state_local(self, item, pos):
         if item in self.events:
             event_list = self.events[item]
             for event in event_list[::-1]:
                 if event[0] < pos:
                     return event
-                        
-        if self.incoming is not None:
-            return self.incoming.get_pos_state(item, pos)
-        
-        else:
-            return None, None
+        return None
 
     def to_string(self, indent='', limit=None):
     
@@ -122,37 +135,35 @@ class BranchingControlFlow(ControlFlow):
         self.branches = [LinearControlFlow(start_pos, incoming, parent=self)]
         self.branch_starts = [start_pos]
         
-    def set_state(self, pos, item, state):
-    
-        if item in self.tip:
-            del self.tip[item]
-        
-        if pos < self.start_pos:
-            self.incoming.set_state(pos, item, state)
-        else:
+    def _set_state_local(self, pos, item, state):
         for branch_pos, branch in zip(self.branch_starts[::-1], self.branches[::-1]):
             if pos >= branch_pos:
-                    branch.set_state(pos, item, state)
+                branch._set_state_local(pos, item, state)
                 return
     
-    def _get_pos_state(self, item, pos):
-        if pos <= self.start_pos:
-            return self.incoming.get_pos_state(item, pos)
-        elif pos < self.end_pos:
+    def _get_pos_state_local(self, item, pos, stop_at=None):
+        if pos < self.end_pos:
             for branch_pos, branch in zip(self.branch_starts[::-1], self.branches[::-1]):
                 if pos >= branch_pos:
-                    return branch.get_pos_state(item, pos)
+                    return branch._get_pos_state_local(item, pos)
         else:
-            last_pos, last_state = self.branches[0].get_pos_state(item, pos)
+            state = self.branches[0]._get_pos_state_local(item, pos)
+            if state is None:
+                return None, None
+            last_pos, last_state = state
             if last_state is None:
                 return None, None
             for branch in self.branches[1:]:
-                other_pos, other_state = branch.get_pos_state(item, pos)
-                if other_state is None or other_state != last_state:
+                state = branch._get_pos_state_local(item, pos)
+                if state is None:
+                    return None, None
+                other_pos, other_state = state
+                if other_state != last_state:
                     return None, None
                 elif last_pos is not other_pos:
                     last_pos = max(last_pos, other_pos)
             return last_pos, last_state
+        return None
 
     def new_branch(self, pos):
         self.branches.append(LinearControlFlow(pos, self.incoming, parent=self))

Cython/Compiler/DebugFlags.py

@@ -10,4 +10,7 @@ debug_temp_code_comments = 0
 debug_trace_code_generation = 0
 
 # Do not replace exceptions with user-friendly error messages
-debug_no_exception_intercept = 0
+debug_no_exception_intercept = 1
+
+# Print a message each time a new stage in the pipeline is entered
+debug_verbose_pipeline = 0

Cython/Compiler/Errors.py

@@ -21,28 +21,29 @@ def context(position):
         F = list(source.get_lines())
     except UnicodeDecodeError:
         # file has an encoding problem
-        s = "[unprintable code]\n"
+        s = u"[unprintable code]\n"
     else:
-        s =''.join(F[max(0, position[1]-6):position[1]])
-        s = '...\n' + s + ' '*(position[2]-1) + '^\n'
-    s = '-'*60 + '\n' + s + '-'*60 + '\n'
+        s = u''.join(F[max(0, position[1]-6):position[1]])
+        s = u'...\n%s%s^\n' % (s, u' '*(position[2]-1))
+    s = u'%s\n%s%s\n' % (u'-'*60, s, u'-'*60)
     return s
 
 class CompileError(PyrexError):
     
-    def __init__(self, position = None, message = ""):
+    def __init__(self, position = None, message = u""):
         self.position = position
         self.message_only = message
         self.reported = False
     # Deprecated and withdrawn in 2.6:
     #   self.message = message
         if position:
-            pos_str = "%s:%d:%d: " % (position[0].get_description(), position[1], position[2])
+            pos_str = u"%s:%d:%d: " % (position[0].get_description(), position[1], position[2])
             cont = context(position)
         else:
-            pos_str = ""
-            cont = ''
-        Exception.__init__(self, '\nError converting Pyrex file to C:\n' + cont + '\n' + pos_str + message )
+            pos_str = u""
+            cont = u''
+        Exception.__init__(self, u'\nError converting Pyrex file to C:\n%s\n%s%s' % (
+            cont, pos_str, message))
 
 class CompileWarning(PyrexWarning):
     
@@ -51,9 +52,9 @@ class CompileWarning(PyrexWarning):
     # Deprecated and withdrawn in 2.6:
     #   self.message = message
         if position:
-            pos_str = "%s:%d:%d: " % (position[0].get_description(), position[1], position[2])
+            pos_str = u"%s:%d:%d: " % (position[0].get_description(), position[1], position[2])
         else:
-            pos_str = ""
+            pos_str = u""
         Exception.__init__(self, pos_str + message)
 
 
@@ -61,7 +62,7 @@ class InternalError(Exception):
     # If this is ever raised, there is a bug in the compiler.
     
     def __init__(self, message):
-        Exception.__init__(self, "Internal compiler error: %s"
+        Exception.__init__(self, u"Internal compiler error: %s"
             % message)
 
 
@@ -73,7 +74,7 @@ class CompilerCrash(CompileError):
         else:
             message = u'\n'
         if context:
-            message = "Compiler crash in " + context + message
+            message = u"Compiler crash in %s%s" % (context, message)
         if stacktrace:
             import traceback
             message += (
@@ -118,11 +119,15 @@ def report_error(err):
         # See Main.py for why dual reporting occurs. Quick fix for now.
         if err.reported: return
         err.reported = True
-        line = "%s\n" % err
+        line = u"%s\n" % err
         if listing_file:
-            listing_file.write(line)
+            try: listing_file.write(line)
+            except UnicodeEncodeError:
+                listing_file.write(line.encode('ASCII', 'replace'))
         if echo_file:
-            echo_file.write(line)
+            try: echo_file.write(line)
+            except UnicodeEncodeError:
+                echo_file.write(line.encode('ASCII', 'replace'))
         num_errors = num_errors + 1
 
 def error(position, message):
@@ -145,6 +150,20 @@ def warning(position, message, level=0):
         echo_file.write(line)
     return warn
 
+_warn_once_seen = {}
+def warn_once(position, message, level=0):
+    if level < LEVEL or message in _warn_once_seen:
+        return
+    warn = CompileWarning(position, message)
+    line = "warning: %s\n" % warn
+    if listing_file:
+        listing_file.write(line)
+    if echo_file:
+        echo_file.write(line)
+    _warn_once_seen[message] = True
+    return warn
+
+
 # These functions can be used to momentarily suppress errors. 
 
 error_stack = []

Cython/Compiler/Main.py

@@ -24,6 +24,7 @@ from Symtab import BuiltinScope, ModuleScope
 from Cython import Utils
 from Cython.Utils import open_new_file, replace_suffix
 import CythonScope
+import DebugFlags
 
 module_name_pattern = re.compile(r"[A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_]*)*$")
 
@@ -34,6 +35,12 @@ def dumptree(t):
     print t.dump()
     return t
 
+def abort_on_errors(node):
+    # Stop the pipeline if there are any errors.
+    if Errors.num_errors != 0:
+        raise InternalError, "abort"
+    return node
+
 class CompilationData(object):
     #  Bundles the information that is passed from transform to transform.
     #  (For now, this is only)
@@ -59,14 +66,14 @@ class Context(object):
     #  include_directories   [string]
     #  future_directives     [object]
     
-    def __init__(self, include_directories, pragma_overrides):
+    def __init__(self, include_directories, compiler_directives):
         #self.modules = {"__builtin__" : BuiltinScope()}
         import Builtin, CythonScope
         self.modules = {"__builtin__" : Builtin.builtin_scope}
         self.modules["cython"] = CythonScope.create_cython_scope(self)
         self.include_directories = include_directories
         self.future_directives = set()
-        self.pragma_overrides = pragma_overrides
+        self.compiler_directives = compiler_directives
 
         self.pxds = {} # full name -> node tree
 
@@ -81,11 +88,12 @@ class Context(object):
         from ParseTreeTransforms import CreateClosureClasses, MarkClosureVisitor, DecoratorTransform
         from ParseTreeTransforms import InterpretCompilerDirectives, TransformBuiltinMethods
         from ParseTreeTransforms import AlignFunctionDefinitions, GilCheck
+        from AnalysedTreeTransforms import AutoTestDictTransform
         from AutoDocTransforms import EmbedSignature
         from Optimize import FlattenInListTransform, SwitchTransform, IterationTransform
         from Optimize import OptimizeBuiltinCalls, ConstantFolding, FinalOptimizePhase
         from Buffer import IntroduceBufferAuxiliaryVars
-        from ModuleNode import check_c_declarations
+        from ModuleNode import check_c_declarations, check_c_declarations_pxd
 
         # Temporary hack that can be used to ensure that all result_code's
         # are generated at code generation time.
@@ -97,7 +105,7 @@ class Context(object):
                 return node
 
         if pxd:
-            _check_c_declarations = None
+            _check_c_declarations = check_c_declarations_pxd
             _specific_post_parse = PxdPostParse(self)
         else:
             _check_c_declarations = check_c_declarations
@@ -112,7 +120,7 @@ class Context(object):
             NormalizeTree(self),
             PostParse(self),
             _specific_post_parse,
-            InterpretCompilerDirectives(self, self.pragma_overrides),
+            InterpretCompilerDirectives(self, self.compiler_directives),
             _align_function_definitions,
             MarkClosureVisitor(self),
             ConstantFolding(),
@@ -121,6 +129,7 @@ class Context(object):
             DecoratorTransform(self),
             AnalyseDeclarationsTransform(self),
             CreateClosureClasses(self),
+            AutoTestDictTransform(self),
             EmbedSignature(self),
             TransformBuiltinMethods(self),
             IntroduceBufferAuxiliaryVars(self),
@@ -156,10 +165,16 @@ class Context(object):
                  module_node.scope.utility_code_list.extend(scope.utility_code_list)
             return module_node
 
+        test_support = []
+        if options.evaluate_tree_assertions:
+            from Cython.TestUtils import TreeAssertVisitor
+            test_support.append(TreeAssertVisitor())
+
         return ([
                 create_parse(self),
-            ] + self.create_pipeline(pxd=False, py=py) + [
+            ] + self.create_pipeline(pxd=False, py=py) + test_support + [
                 inject_pxd_code,
+                abort_on_errors,
                 generate_pyx_code,
             ])
 
@@ -192,20 +207,24 @@ class Context(object):
         return Errors.report_error(exc)
 
     def run_pipeline(self, pipeline, source):
-        err = None
+        error = None
         data = source
         try:
             for phase in pipeline:
                 if phase is not None:
+                    if DebugFlags.debug_verbose_pipeline:
+                        print "Entering pipeline phase %r" % phase
                     data = phase(data)
         except CompileError, err:
             # err is set
             Errors.report_error(err)
+            error = err
         except InternalError, err:
             # Only raise if there was not an earlier error
             if Errors.num_errors == 0:
                 raise
-        return (err, data)
+            error = err
+        return (error, data)
 
     def find_module(self, module_name, 
             relative_to = None, pos = None, need_pxd = 1):
@@ -316,7 +335,9 @@ class Context(object):
             else:
                 dirs = [self.find_root_package_dir(file_desc.filename)] + dirs
 
-        dotted_filename = qualified_name + suffix
+        dotted_filename = qualified_name
+        if suffix:
+            dotted_filename += suffix
         if not include:
             names = qualified_name.split('.')
             package_names = names[:-1]
@@ -524,7 +545,7 @@ def create_default_resultobj(compilation_source, options):
 
 def run_pipeline(source, options, full_module_name = None):
     # Set up context
-    context = Context(options.include_path, options.pragma_overrides)
+    context = Context(options.include_path, options.compiler_directives)
 
     # Set up source object
     cwd = os.getcwd()
@@ -577,7 +598,8 @@ class CompilationOptions(object):
                                 defaults to true when recursive is true.
     verbose           boolean   Always print source names being compiled
     quiet             boolean   Don't print source names in recursive mode
-    pragma_overrides  dict      Overrides for pragma options (see Options.py)
+    compiler_directives  dict      Overrides for pragma options (see Options.py)
+    evaluate_tree_assertions boolean  Test support: evaluate parse tree assertions
     
     Following options are experimental and only used on MacOSX:
     
@@ -765,7 +787,8 @@ default_options = dict(
     timestamps = None,
     verbose = 0,
     quiet = 0,
-    pragma_overrides = {},
+    compiler_directives = {},
+    evaluate_tree_assertions = False,
     emit_linenums = False,
 )
 if sys.platform == "mac":

Cython/Compiler/Naming.py

@@ -47,6 +47,7 @@ convert_func_prefix = pyrex_prefix + "convert_"
 closure_scope_prefix = pyrex_prefix + "scope_"
 closure_class_prefix = pyrex_prefix + "scope_struct_"
 lambda_func_prefix = pyrex_prefix + "lambda_"
+module_is_main   = pyrex_prefix + "module_is_main_"
 
 args_cname       = pyrex_prefix + "args"
 pykwdlist_cname  = pyrex_prefix + "pyargnames"
@@ -77,6 +78,7 @@ c_api_tab_cname  = pyrex_prefix + "c_api_tab"
 gilstate_cname   = pyrex_prefix + "state"
 skip_dispatch_cname = pyrex_prefix + "skip_dispatch"
 empty_tuple      = pyrex_prefix + "empty_tuple"
+empty_bytes      = pyrex_prefix + "empty_bytes"
 print_function   = pyrex_prefix + "print"
 print_function_kwargs   = pyrex_prefix + "print_kwargs"
 cleanup_cname    = pyrex_prefix + "module_cleanup"
@@ -84,8 +86,11 @@ pymoduledef_cname = pyrex_prefix + "moduledef"
 optional_args_cname = pyrex_prefix + "optional_args"
 import_star      = pyrex_prefix + "import_star"
 import_star_set  = pyrex_prefix + "import_star_set"
-cur_scope_cname  = pyrex_prefix + "scope"
 outer_scope_cname= pyrex_prefix + "outer_scope"
+cur_scope_cname  = pyrex_prefix + "cur_scope"
+enc_scope_cname  = pyrex_prefix + "enc_scope"
+frame_cname      = pyrex_prefix + "frame"
+frame_code_cname = pyrex_prefix + "frame_code"
 
 line_c_macro = "__LINE__"
 

Cython/Compiler/Options.py

@@ -54,25 +54,41 @@ c_line_in_traceback = 1
 embed = False
 
 
-# Declare pragmas
+# Declare compiler directives
 option_defaults = {
     'boundscheck' : True,
     'nonecheck' : False,
     'embedsignature' : False,
     'locals' : {},
     'auto_cpdef': False,
-    'cdivision': True,  # Will be False in 0.12
+    'cdivision': False, # was True before 0.12
     'cdivision_warnings': False,
     'always_allow_keywords': False,
+    'wraparound' : True,
+    'c99_complex' : False, # Don't use macro wrappers for complex arith, not sure what to name this...
+    'callspec' : "",
+    'profile': False,
+    'autotestdict': True,
+
+# test support
+    'test_assert_path_exists' : [],
+    'test_fail_if_path_exists' : [],
 }
 
 # Override types possibilities above, if needed
-option_types = { }
+option_types = {}
 
 for key, val in option_defaults.items():
     if key not in option_types:
         option_types[key] = type(val)
 
+option_scopes = { # defaults to available everywhere
+    # 'module', 'function', 'class', 'with statement'
+    'autotestdict' : ('module',),
+    'test_assert_path_exists' : ('function',),
+    'test_fail_if_path_exists' : ('function',),
+}
+
 def parse_option_value(name, value):
     """
     Parses value as an option value for the given name and returns
@@ -94,6 +110,11 @@ def parse_option_value(name, value):
         if value == "True": return True
         elif value == "False": return False
         else: raise ValueError("%s directive must be set to True or False" % name)
+    elif type is int:
+        try:
+            return int(value)
+        except ValueError:
+            raise ValueError("%s directive must be set to an integer" % name)
     else:
         assert False
 

Cython/Compiler/Parsing.pxd

@@ -145,7 +145,7 @@ cpdef p_decorators(PyrexScanner s)
 cpdef p_def_statement(PyrexScanner s, list decorators = *)
 cpdef p_varargslist(PyrexScanner s, terminator=*)
 cpdef p_py_arg_decl(PyrexScanner s)
-cpdef p_class_statement(PyrexScanner s)
+cpdef p_class_statement(PyrexScanner s, decorators)
 cpdef p_c_class_definition(PyrexScanner s, pos,  ctx)
 cpdef p_c_class_options(PyrexScanner s)
 cpdef p_property_decl(PyrexScanner s)

Cython/Compiler/Parsing.py

@@ -14,7 +14,7 @@ from Cython.Compiler.Scanning import PyrexScanner, FileSourceDescriptor
 import Nodes
 import ExprNodes
 import StringEncoding
-from StringEncoding import EncodedString, BytesLiteral
+from StringEncoding import EncodedString, BytesLiteral, _str, _bytes
 from ModuleNode import ModuleNode
 from Errors import error, warning, InternalError
 from Cython import Utils
@@ -65,14 +65,23 @@ def p_ident_list(s):
 #
 #------------------------------------------
 
+def p_binop_operator(s):
+    pos = s.position()
+    op = s.sy
+    s.next()
+    return op, pos
+
 def p_binop_expr(s, ops, p_sub_expr):
     n1 = p_sub_expr(s)
     while s.sy in ops:
-        op = s.sy
-        pos = s.position()
-        s.next()
+        op, pos = p_binop_operator(s)
         n2 = p_sub_expr(s)
         n1 = ExprNodes.binop_node(pos, op, n1, n2)
+        if op == '/':
+            if Future.division in s.context.future_directives:
+                n1.truedivision = True
+            else:
+                n1.truedivision = None # unknown
     return n1
 
 #expression: or_test [if or_test else test] | lambda_form
@@ -601,7 +610,7 @@ def p_name(s, name):
             return ExprNodes.IntNode(pos, value = rep, longness = "L")
         elif isinstance(value, float):
             return ExprNodes.FloatNode(pos, value = rep)
-        elif isinstance(value, (str, unicode)):
+        elif isinstance(value, (_str, _bytes)):
             return ExprNodes.StringNode(pos, value = value)
         else:
             error(pos, "Invalid type for compile-time constant: %s"
@@ -633,7 +642,7 @@ def p_cat_string_literal(s):
         if kind == 'u':
             value = EncodedString( u''.join(strings) )
         else:
-            value = BytesLiteral( ''.join(strings) )
+            value = BytesLiteral( StringEncoding.join_bytes(strings) )
             value.encoding = s.source_encoding
     return kind, value
 
@@ -1460,6 +1469,7 @@ def p_include_statement(s, ctx):
     _, include_file_name = p_string_literal(s)
     s.expect_newline("Syntax error in include statement")
     if s.compile_time_eval:
+        include_file_name = include_file_name.decode(s.source_encoding)
         include_file_path = s.context.find_include_file(include_file_name, pos)
         if include_file_path:
             s.included_files.append(include_file_name)
@@ -1612,8 +1622,8 @@ def p_statement(s, ctx, first_statement = 0):
             s.error('decorator not allowed here')
         s.level = ctx.level
         decorators = p_decorators(s)
-        if s.sy not in ('def', 'cdef', 'cpdef'):
-            s.error("Decorators can only be followed by functions ")
+        if s.sy not in ('def', 'cdef', 'cpdef', 'class'):
+            s.error("Decorators can only be followed by functions or classes")
     elif s.sy == 'pass' and cdef_flag:
         # empty cdef block
         return p_pass_statement(s, with_newline = 1)
@@ -1633,7 +1643,7 @@ def p_statement(s, ctx, first_statement = 0):
         node = p_cdef_statement(s, ctx(overridable = overridable))
         if decorators is not None:
             if not isinstance(node, (Nodes.CFuncDefNode, Nodes.CVarDefNode)):
-                s.error("Decorators can only be followed by functions ")
+                s.error("Decorators can only be followed by functions or Python classes")
             node.decorators = decorators
         return node
     else:
@@ -1647,7 +1657,7 @@ def p_statement(s, ctx, first_statement = 0):
         elif s.sy == 'class':
             if ctx.level != 'module':
                 s.error("class definition not allowed here")
-            return p_class_statement(s)
+            return p_class_statement(s, decorators)
         elif s.sy == 'include':
             if ctx.level not in ('module', 'module_pxd'):
                 s.error("include statement not allowed here")
@@ -1791,7 +1801,7 @@ def p_calling_convention(s):
     else:
         return ""
 
-calling_convention_words = ("__stdcall", "__cdecl")
+calling_convention_words = ("__stdcall", "__cdecl", "__fastcall")
 
 def p_c_complex_base_type(s):
     # s.sy == '('
@@ -1808,6 +1818,7 @@ def p_c_simple_base_type(s, self_flag, nonempty):
     is_basic = 0
     signed = 1
     longness = 0
+    complex = 0
     module_path = []
     pos = s.position()
     if not s.sy == 'IDENT':
@@ -1826,6 +1837,9 @@ def p_c_simple_base_type(s, self_flag, nonempty):
                 s.next()
             else:
                 name = 'int'
+        if s.sy == 'IDENT' and s.systring == 'complex':
+            complex = 1
+            s.next()
     elif looking_at_dotted_name(s):
         #print "p_c_simple_base_type: looking_at_type_name at", s.position()
         name = s.systring
@@ -1854,7 +1868,8 @@ def p_c_simple_base_type(s, self_flag, nonempty):
     type_node = Nodes.CSimpleBaseTypeNode(pos, 
         name = name, module_path = module_path,
         is_basic_c_type = is_basic, signed = signed,
-        longness = longness, is_self_arg = self_flag)
+        complex = complex, longness = longness, 
+        is_self_arg = self_flag)
 
 
     # Treat trailing [] on type as buffer access if it appears in a context
@@ -1975,6 +1990,8 @@ def p_opt_cname(s):
     literal = p_opt_string_literal(s)
     if literal:
         _, cname = literal
+        cname = EncodedString(cname)
+        cname.encoding = s.source_encoding
     else:
         cname = None
     return cname
@@ -2462,7 +2479,7 @@ def p_py_arg_decl(s):
     name = p_ident(s)
     return Nodes.PyArgDeclNode(pos, name = name)
 
-def p_class_statement(s):
+def p_class_statement(s, decorators):
     # s.sy == 'class'
     pos = s.position()
     s.next()
@@ -2478,7 +2495,7 @@ def p_class_statement(s):
     return Nodes.PyClassDefNode(pos,
         name = class_name,
         bases = ExprNodes.TupleNode(pos, args = base_list),
-        doc = doc, body = body)
+        doc = doc, body = body, decorators = decorators)
 
 def p_c_class_definition(s, pos,  ctx):
     # s.sy == 'class'
@@ -2602,7 +2619,7 @@ def p_code(s, level=None):
             repr(s.sy), repr(s.systring)))
     return body
 
-COMPILER_DIRECTIVE_COMMENT_RE = re.compile(r"^#\s*cython:\s*([a-z_]+)\s*=(.*)$")
+COMPILER_DIRECTIVE_COMMENT_RE = re.compile(r"^#\s*cython:\s*(\w+)\s*=(.*)$")
 
 def p_compiler_directive_comments(s):
     result = {}
@@ -2612,10 +2629,10 @@ def p_compiler_directive_comments(s):
             name = m.group(1)
             try:
                 value = Options.parse_option_value(str(name), str(m.group(2).strip()))
-            except ValueError, e:
-                s.error(e.args[0], fatal=False)
                 if value is not None: # can be False!
                     result[name] = value
+            except ValueError, e:
+                s.error(e.args[0], fatal=False)
         s.next()
     return result
 
@@ -2654,7 +2671,7 @@ def print_parse_tree(f, node, level, key = None):
         if key:
             f.write("%s: " % key)
         t = type(node)
-        if t == TupleType:
+        if t is tuple:
             f.write("(%s @ %s\n" % (node[0], node[1]))
             for i in xrange(2, len(node)):
                 print_parse_tree(f, node[i], level+1)
@@ -2670,7 +2687,7 @@ def print_parse_tree(f, node, level, key = None):
                 if name != 'tag' and name != 'pos':
                     print_parse_tree(f, value, level+1, name)
             return
-        elif t == ListType:
+        elif t is list:
             f.write("[\n")
             for i in xrange(len(node)):
                 print_parse_tree(f, node[i], level+1)

Cython/Compiler/Scanning.py

@@ -65,7 +65,7 @@ def hash_source_file(path):
     # tabs by a single space.
     import re
     text = re.sub("[ \t]+", " ", text)
-    hash = new_md5(text).hexdigest()
+    hash = new_md5(text.encode("ASCII")).hexdigest()
     return hash
 
 def open_pickled_lexicon(expected_hash):
@@ -214,14 +214,18 @@ def initial_compile_time_env():
         'UNAME_VERSION', 'UNAME_MACHINE')
     for name, value in zip(names, platform.uname()):
         benv.declare(name, value)
-    import __builtin__
+    import __builtin__ as builtins
     names = ('False', 'True',
         'abs', 'bool', 'chr', 'cmp', 'complex', 'dict', 'divmod', 'enumerate',
         'float', 'hash', 'hex', 'int', 'len', 'list', 'long', 'map', 'max', 'min',
         'oct', 'ord', 'pow', 'range', 'reduce', 'repr', 'round', 'slice', 'str',
         'sum', 'tuple', 'xrange', 'zip')
     for name in names:
-        benv.declare(name, getattr(__builtin__, name))
+        try:
+            benv.declare(name, getattr(builtins, name))
+        except AttributeError:
+            # ignore, likely Py3
+            pass
     denv = CompileTimeScope(benv)
     return denv
 

Cython/Compiler/TreeFragment.py

@@ -3,7 +3,7 @@
 #
 
 import re
-from cStringIO import StringIO
+from StringIO import StringIO
 from Scanning import PyrexScanner, StringSourceDescriptor
 from Symtab import BuiltinScope, ModuleScope
 import Symtab
@@ -54,7 +54,7 @@ def parse_from_strings(name, code, pxds={}, level=None, initial_pos=None):
     context = StringParseContext([], name)
     scope = context.find_module(module_name, pos = initial_pos, need_pxd = 0)
 
-    buf = StringIO(code.encode(encoding))
+    buf = StringIO(code)
 
     scanner = PyrexScanner(buf, code_source, source_encoding = encoding,
                      scope = scope, context = context, initial_pos = initial_pos)

Cython/Compiler/Version.py

-version = '0.11.1'
+version = '0.11.3'

tests/errors/pxd_cdef_class_declaration_T286.pxd

+cdef class A