branch merge

.hgtags

@@ -24,3 +24,11 @@ e90c522631ae06f2170a751fb256cdea0e50fb21 0.12.1
 5ac2eaefcdc9c3a7a9c29a0bb8c3e4c6c016c64c 0.13.beta0
 14957f635a379c97d9966097276313e43491ed96 0.13.beta1
 32c957267b3ba3140fba4d1947fa98484d5e956b 0.13
+ef9d2c680684d0df7d81f529cda29e9e1741f575 0.10.1
+16a746d969e2654112fc0dc081690b891c496977 0.9.8
+16a746d969e2654112fc0dc081690b891c496977 Version-0.9.8
+0000000000000000000000000000000000000000 Version-0.9.8
+ef9d2c680684d0df7d81f529cda29e9e1741f575 cython-0.10.1
+0000000000000000000000000000000000000000 cython-0.10.1
+59c67af0674bd93c5fd8958e08c76a9dab9aae37 sage-cythonizes
+0000000000000000000000000000000000000000 sage-cythonizes

Cython/Compiler/Code.py

@@ -421,6 +421,7 @@ class GlobalState(object):
         'all_the_rest',
         'pystring_table',
         'cached_builtins',
+        'cached_constants',
         'init_globals',
         'init_module',
         'cleanup_globals',
@@ -462,7 +463,12 @@ class GlobalState(object):
             w.enter_cfunc_scope()
             w.putln("static int __Pyx_InitCachedBuiltins(void) {")
 
-        
+        w = self.parts['cached_constants']
+        w.enter_cfunc_scope()
+        w.putln("")
+        w.putln("static int __Pyx_InitCachedConstants(void) {")
+        w.put_setup_refcount_context("__Pyx_InitCachedConstants")
+
         w = self.parts['init_globals']
         w.enter_cfunc_scope()
         w.putln("")
@@ -509,15 +515,27 @@ class GlobalState(object):
         if Options.cache_builtins:
             w = self.parts['cached_builtins']
             w.putln("return 0;")
-            w.put_label(w.error_label)
-            w.putln("return -1;")
+            if w.label_used(w.error_label):
+                w.put_label(w.error_label)
+                w.putln("return -1;")
             w.putln("}")
             w.exit_cfunc_scope()
 
+        w = self.parts['cached_constants']
+        w.put_finish_refcount_context()
+        w.putln("return 0;")
+        if w.label_used(w.error_label):
+            w.put_label(w.error_label)
+            w.put_finish_refcount_context()
+            w.putln("return -1;")
+        w.putln("}")
+        w.exit_cfunc_scope()
+
         w = self.parts['init_globals']
         w.putln("return 0;")
-        w.put_label(w.error_label)
-        w.putln("return -1;")
+        if w.label_used(w.error_label):
+            w.put_label(w.error_label)
+            w.putln("return -1;")
         w.putln("}")
         w.exit_cfunc_scope()
 
@@ -536,6 +554,9 @@ class GlobalState(object):
 
     # constant handling at code generation time
 
+    def get_cached_constants_writer(self):
+        return self.parts['cached_constants']
+
     def get_int_const(self, str_value, longness=False):
         longness = bool(longness)
         try:
@@ -544,9 +565,14 @@ class GlobalState(object):
             c = self.new_int_const(str_value, longness)
         return c
 
-    def get_py_const(self, type):
+    def get_py_const(self, type, prefix='', cleanup_level=None):
         # create a new Python object constant
-        return self.new_py_const(type)
+        const = self.new_py_const(type, prefix)
+        if cleanup_level is not None \
+               and cleanup_level <= Options.generate_cleanup_code:
+            cleanup_writer = self.parts['cleanup_globals']
+            cleanup_writer.put_xdecref_clear(const.cname, type, nanny=False)
+        return const
 
     def get_string_const(self, text):
         # return a C string constant, creating a new one if necessary
@@ -581,8 +607,8 @@ class GlobalState(object):
         self.int_const_index[(value, longness)] = c
         return c
 
-    def new_py_const(self, type):
-        cname = self.new_const_cname()
+    def new_py_const(self, type, prefix=''):
+        cname = self.new_const_cname(prefix)
         c = PyObjectConst(cname, type)
         self.py_constants.append(c)
         return c
@@ -946,6 +972,9 @@ class CCodeWriter(object):
     def get_py_num(self, str_value, longness):
         return self.globalstate.get_int_const(str_value, longness).cname
 
+    def get_py_const(self, type, prefix='', cleanup_level=None):
+        return self.globalstate.get_py_const(type, prefix, cleanup_level).cname
+
     def get_string_const(self, text):
         return self.globalstate.get_string_const(text).cname
 
@@ -961,6 +990,9 @@ class CCodeWriter(object):
     def intern_identifier(self, text):
         return self.get_py_string_const(text, identifier=True)
 
+    def get_cached_constants_writer(self):
+        return self.globalstate.get_cached_constants_writer()
+
     # code generation
 
     def putln(self, code = "", safe=False):
@@ -1247,11 +1279,13 @@ class CCodeWriter(object):
 
     def put_pymethoddef(self, entry, term, allow_skip=True):
         if entry.is_special or entry.name == '__getattribute__':
-            if entry.name not in ['__cinit__', '__dealloc__', '__richcmp__', '__next__', '__getreadbuffer__', '__getwritebuffer__', '__getsegcount__', '__getcharbuffer__', '__getbuffer__', '__releasebuffer__', '__getattr__']:
+            if entry.name not in ['__cinit__', '__dealloc__', '__richcmp__', '__next__', '__getreadbuffer__', '__getwritebuffer__', '__getsegcount__', '__getcharbuffer__', '__getbuffer__', '__releasebuffer__']:
+                if entry.name == '__getattr__' and not self.globalstate.directives['fast_getattr']:
+                    pass
                 # Python's typeobject.c will automatically fill in our slot
                 # in add_operators() (called by PyType_Ready) with a value
                 # that's better than ours.
-                if allow_skip:
+                elif allow_skip:
                     return
         from TypeSlots import method_coexist
         if entry.doc:

Cython/Compiler/ModuleNode.py

@@ -1768,8 +1768,10 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
 
         if Options.cache_builtins:
             code.putln("/*--- Builtin init code ---*/")
-            code.putln(code.error_goto_if_neg("__Pyx_InitCachedBuiltins()",
-                                              self.pos))
+            code.putln(code.error_goto_if_neg("__Pyx_InitCachedBuiltins()", self.pos))
+
+        code.putln("/*--- Constants init code ---*/")
+        code.putln(code.error_goto_if_neg("__Pyx_InitCachedConstants()", self.pos))
 
         code.putln("/*--- Global init code ---*/")
         self.generate_global_init_code(env, code)

Cython/Compiler/Optimize.py

@@ -702,28 +702,25 @@ class SwitchTransform(Visitor.VisitorTransform):
                 break
 
         if isinstance(cond, ExprNodes.PrimaryCmpNode):
-            if cond.cascade is None and not cond.is_python_comparison():
+            if cond.cascade is not None:
+                return self.NO_MATCH
+            elif cond.is_c_string_contains() and \
+                   isinstance(cond.operand2, (ExprNodes.UnicodeNode, ExprNodes.BytesNode)):
+                not_in = cond.operator == 'not_in'
+                if not_in and not allow_not_in:
+                    return self.NO_MATCH
+                if isinstance(cond.operand2, ExprNodes.UnicodeNode) and \
+                       cond.operand2.contains_surrogates():
+                    # dealing with surrogates leads to different
+                    # behaviour on wide and narrow Unicode
+                    # platforms => refuse to optimise this case
+                    return self.NO_MATCH
+                return not_in, cond.operand1, self.extract_in_string_conditions(cond.operand2)
+            elif not cond.is_python_comparison():
                 if cond.operator == '==':
                     not_in = False
                 elif allow_not_in and cond.operator == '!=':
                     not_in = True
-                elif cond.is_c_string_contains() and \
-                         isinstance(cond.operand2, (ExprNodes.UnicodeNode, ExprNodes.BytesNode)):
-                    not_in = cond.operator == 'not_in'
-                    if not_in and not allow_not_in:
-                        return self.NO_MATCH
-                    if isinstance(cond.operand2, ExprNodes.UnicodeNode) and \
-                           cond.operand2.contains_surrogates():
-                        # dealing with surrogates leads to different
-                        # behaviour on wide and narrow Unicode
-                        # platforms => refuse to optimise this case
-                        return self.NO_MATCH
-                    # this looks somewhat silly, but it does the right
-                    # checks for NameNode and AttributeNode
-                    if is_common_value(cond.operand1, cond.operand1):
-                        return not_in, cond.operand1, self.extract_in_string_conditions(cond.operand2)
-                    else:
-                        return self.NO_MATCH
                 else:
                     return self.NO_MATCH
                 # this looks somewhat silly, but it does the right

Cython/Compiler/Options.py

@@ -71,6 +71,7 @@ directive_defaults = {
     'autotestdict.cdef': False,
     'autotestdict.all': False,
     'language_level': 2,
+    'fast_getattr': False, # Undocumented until we come up with a better way to handle this everywhere.
     
     'warn': None,
     'warn.undeclared': False,
@@ -101,8 +102,8 @@ directive_scopes = { # defaults to available everywhere
     'autotestdict' : ('module',),
     'autotestdict.all' : ('module',),
     'autotestdict.cdef' : ('module',),
-    'test_assert_path_exists' : ('function',),
-    'test_fail_if_path_exists' : ('function',),
+    'test_assert_path_exists' : ('function', 'class', 'cclass'),
+    'test_fail_if_path_exists' : ('function', 'class', 'cclass'),
 }
 
 def parse_directive_value(name, value, relaxed_bool=False):

Cython/Compiler/ParseTreeTransforms.py

@@ -787,6 +787,13 @@ class InterpretCompilerDirectives(CythonTransform, SkipDeclarations):
         body = StatListNode(node.pos, stats=[node])
         return self.visit_with_directives(body, directives)
 
+    def visit_PyClassDefNode(self, node):
+        directives = self._extract_directives(node, 'class')
+        if not directives:
+            return self.visit_Node(node)
+        body = StatListNode(node.pos, stats=[node])
+        return self.visit_with_directives(body, directives)
+
     def _extract_directives(self, node, scope_name):
         if not node.decorators:
             return {}
@@ -1060,21 +1067,17 @@ property NAME:
         self.seen_vars_stack.pop()
         return node
 
-    def visit_ComprehensionNode(self, node):
-        self.visitchildren(node)
-        node.analyse_declarations(self.env_stack[-1])
-        return node
-
     def visit_ScopedExprNode(self, node):
         node.analyse_declarations(self.env_stack[-1])
-        if self.seen_vars_stack:
+        # the node may or may not have a local scope
+        if node.expr_scope:
             self.seen_vars_stack.append(set(self.seen_vars_stack[-1]))
+            self.env_stack.append(node.expr_scope)
+            self.visitchildren(node)
+            self.env_stack.pop()
+            self.seen_vars_stack.pop()
         else:
-            self.seen_vars_stack.append(set())
-        self.env_stack.append(node.expr_scope)
-        self.visitchildren(node)
-        self.env_stack.pop()
-        self.seen_vars_stack.pop()
+            self.visitchildren(node)
         return node
 
     def visit_TempResultFromStatNode(self, node):
@@ -1187,6 +1190,7 @@ class ExpandInplaceOperators(EnvTransform):
             # There is code to handle this case.
             return node
 
+        env = self.current_env()
         def side_effect_free_reference(node, setting=False):
             if isinstance(node, NameNode):
                 return node, []
@@ -1215,12 +1219,18 @@ class ExpandInplaceOperators(EnvTransform):
                            operand1 = dup,
                            operand2 = rhs,
                            inplace=True)
-        node = SingleAssignmentNode(node.pos, lhs=lhs, rhs=binop)
+        # Manually analyse types for new node.
+        lhs.analyse_target_types(env)
+        dup.analyse_types(env)
+        binop.analyse_operation(env)
+        node = SingleAssignmentNode(
+                            node.pos, 
+                            lhs = lhs,
+                            rhs=binop.coerce_to(lhs.type, env))
         # Use LetRefNode to avoid side effects.
         let_ref_nodes.reverse()
         for t in let_ref_nodes:
             node = LetNode(t, node)
-        node.analyse_expressions(self.current_env())
         return node
 
     def visit_ExprNode(self, node):
@@ -1318,7 +1328,7 @@ class CreateClosureClasses(CythonTransform):
         return node
 
     def create_class_from_scope(self, node, target_module_scope):
-        as_name = "%s%s" % (Naming.closure_class_prefix, node.entry.cname)
+        as_name = '%s_%s' % (target_module_scope.next_id(Naming.closure_class_prefix), node.entry.cname)
         func_scope = node.local_scope
 
         entry = target_module_scope.declare_c_class(name = as_name,
@@ -1327,11 +1337,15 @@ class CreateClosureClasses(CythonTransform):
         class_scope = entry.type.scope
         class_scope.is_internal = True
         class_scope.directives = {'final': True}
-        if node.entry.scope.is_closure_scope:
+
+        cscope = node.entry.scope
+        while cscope.is_py_class_scope or cscope.is_c_class_scope:
+            cscope = cscope.outer_scope
+        if cscope.is_closure_scope:
             class_scope.declare_var(pos=node.pos,
                                     name=Naming.outer_scope_cname, # this could conflict?
                                     cname=Naming.outer_scope_cname,
-                                    type=node.entry.scope.scope_class.type,
+                                    type=cscope.scope_class.type,
                                     is_cdef=True)
         entries = func_scope.entries.items()
         entries.sort()

Cython/Compiler/Parsing.py

@@ -583,20 +583,7 @@ def p_atom(s):
         expect_ellipsis(s)
         return ExprNodes.EllipsisNode(pos)
     elif sy == 'INT':
-        value = s.systring
-        s.next()
-        unsigned = ""
-        longness = ""
-        while value[-1] in "UuLl":
-            if value[-1] in "Ll":
-                longness += "L"
-            else:
-                unsigned += "U"
-            value = value[:-1]
-        return ExprNodes.IntNode(pos, 
-                                 value = value,
-                                 unsigned = unsigned,
-                                 longness = longness)
+        return p_int_literal(s)
     elif sy == 'FLOAT':
         value = s.systring
         s.next()
@@ -631,6 +618,37 @@ def p_atom(s):
     else:
         s.error("Expected an identifier or literal")
 
+def p_int_literal(s):
+    pos = s.position()
+    value = s.systring
+    s.next()
+    unsigned = ""
+    longness = ""
+    while value[-1] in u"UuLl":
+        if value[-1] in u"Ll":
+            longness += "L"
+        else:
+            unsigned += "U"
+        value = value[:-1]
+    # '3L' is ambiguous in Py2 but not in Py3.  '3U' and '3LL' are
+    # illegal in Py2 Python files.  All suffixes are illegal in Py3
+    # Python files.
+    is_c_literal = None
+    if unsigned:
+        is_c_literal = True
+    elif longness:
+        if longness == 'LL' or s.context.language_level >= 3:
+            is_c_literal = True
+    if s.in_python_file:
+        if is_c_literal:
+            error(pos, "illegal integer literal syntax in Python source file")
+        is_c_literal = False
+    return ExprNodes.IntNode(pos,
+                             is_c_literal = is_c_literal,
+                             value = value,
+                             unsigned = unsigned,
+                             longness = longness)
+
 def p_name(s, name):
     pos = s.position()
     if not s.compile_time_expr and name in s.compile_time_env:
@@ -1722,7 +1740,7 @@ def p_statement(s, ctx, first_statement = 0):
             s.level = ctx.level
             return p_def_statement(s, decorators)
         elif s.sy == 'class':
-            if ctx.level != 'module':
+            if ctx.level not in ('module', 'function', 'class', 'other'):
                 s.error("class definition not allowed here")
             return p_class_statement(s, decorators)
         elif s.sy == 'include':
@@ -2166,7 +2184,7 @@ def p_c_simple_declarator(s, ctx, empty, is_type, cmethod_flag,
             cname = ctx.namespace + "::" + name
         if name == 'operator' and ctx.visibility == 'extern' and nonempty:
             op = s.sy
-            if [c in '+-*/<=>!%&|([^~,' for c in op]:
+            if [1 for c in op if c in '+-*/<=>!%&|([^~,']:
                 s.next()
                 # Handle diphthong operators.
                 if op == '(':

Cython/Compiler/Scanning.pxd

@@ -2,14 +2,21 @@ import cython
 
 from Cython.Plex.Scanners cimport Scanner
 
+cdef class Method:
+    cdef object name
+    cdef object __name__
+
 cdef class CompileTimeScope:
-    cdef public entries
-    cdef public outer
+    cdef public dict entries
+    cdef public CompileTimeScope outer
+    cdef declare(self, name, value)
+    cdef lookup_here(self, name)
+    cpdef lookup(self, name)
 
 cdef class PyrexScanner(Scanner):
     cdef public context
     cdef public list included_files
-    cdef public compile_time_env
+    cdef public CompileTimeScope compile_time_env
     cdef public bint compile_time_eval
     cdef public bint compile_time_expr
     cdef public bint parse_comments
@@ -23,9 +30,23 @@ cdef class PyrexScanner(Scanner):
     cdef public systring
 
     cdef long current_level(self)
-    cpdef begin(self, state)
-    cpdef next(self)
-    cpdef bint expect(self, what, message = *) except -2
-    
+    #cpdef commentline(self, text)
+    #cpdef open_bracket_action(self, text)
+    #cpdef close_bracket_action(self, text)
+    #cpdef newline_action(self, text)
+    #cpdef begin_string_action(self, text)
+    #cpdef end_string_action(self, text)
+    #cpdef unclosed_string_action(self, text)
     @cython.locals(current_level=cython.long, new_level=cython.long)
     cpdef indentation_action(self, text)
+    #cpdef eof_action(self, text)
+    cdef next(self)
+    cdef peek(self)
+    #cpdef put_back(self, sy, systring)
+    #cdef unread(self, token, value)
+    cdef bint expect(self, what, message = *) except -2
+    cdef expect_keyword(self, what, message = *)
+    cdef expected(self, what, message = *)
+    cdef expect_indent(self)
+    cdef expect_dedent(self)
+    cdef expect_newline(self, message = *)

Cython/Compiler/Scanning.py

@@ -358,10 +358,10 @@ class PyrexScanner(Scanner):
             self.error("Unrecognized character")
         if sy == IDENT:
             if systring in self.keywords:
-                if systring == 'print' and print_function in self.context.future_directives:
+                if systring == u'print' and print_function in self.context.future_directives:
                     self.keywords.remove('print')
                     systring = EncodedString(systring)
-                elif systring == 'exec' and self.context.language_level >= 3:
+                elif systring == u'exec' and self.context.language_level >= 3:
                     self.keywords.remove('exec')
                     systring = EncodedString(systring)
                 else:

Cython/Compiler/Symtab.py

@@ -1381,9 +1381,6 @@ class ClassScope(Scope):
         self.class_name = name
         self.doc = None
 
-    def add_string_const(self, value, identifier = False):
-        return self.outer_scope.add_string_const(value, identifier)
-
     def lookup(self, name):
         entry = Scope.lookup(self, name)
         if entry:

Cython/Compiler/Visitor.pxd

 cimport cython
 
-cdef class BasicVisitor:
+cdef class TreeVisitor:
+    cdef public list access_path
     cdef dict dispatch_table
+
     cpdef visit(self, obj)
     cdef _visit(self, obj)
     cdef find_handler(self, obj)
-
-cdef class TreeVisitor(BasicVisitor):
-    cdef public list access_path
     cdef _visitchild(self, child, parent, attrname, idx)
     @cython.locals(idx=int)
     cdef dict _visitchildren(self, parent, attrs)

Cython/Compiler/Visitor.py

@@ -11,49 +11,7 @@ import Naming
 import Errors
 import DebugFlags
 
-class BasicVisitor(object):
-    """A generic visitor base class which can be used for visiting any kind of object."""
-    # Note: If needed, this can be replaced with a more efficient metaclass
-    # approach, resolving the jump table at module load time rather than per visitor
-    # instance.
-    def __init__(self):
-        self.dispatch_table = {}
-
-    def visit(self, obj):
-        return self._visit(obj)
-
-    def _visit(self, obj):
-        try:
-            handler_method = self.dispatch_table[type(obj)]
-        except KeyError:
-            handler_method = self.find_handler(obj)
-            self.dispatch_table[type(obj)] = handler_method
-        return handler_method(obj)
-
-    def find_handler(self, obj):
-        cls = type(obj)
-        #print "Cache miss for class %s in visitor %s" % (
-        #    cls.__name__, type(self).__name__)
-        # Must resolve, try entire hierarchy
-        pattern = "visit_%s"
-        mro = inspect.getmro(cls)
-        handler_method = None
-        for mro_cls in mro:
-            if hasattr(self, pattern % mro_cls.__name__):
-                handler_method = getattr(self, pattern % mro_cls.__name__)
-                break
-        if handler_method is None:
-            print type(self), cls
-            if hasattr(self, 'access_path') and self.access_path:
-                print self.access_path
-                if self.access_path:
-                    print self.access_path[-1][0].pos
-                    print self.access_path[-1][0].__dict__
-            raise RuntimeError("Visitor %r does not accept object: %s" % (self, obj))
-        #print "Caching " + cls.__name__
-        return handler_method
-
-class TreeVisitor(BasicVisitor):
+class TreeVisitor(object):
     """
     Base class for writing visitors for a Cython tree, contains utilities for
     recursing such trees using visitors. Each node is
@@ -96,9 +54,9 @@ class TreeVisitor(BasicVisitor):
     out 3
     out 0
     """
-    
     def __init__(self):
         super(TreeVisitor, self).__init__()
+        self.dispatch_table = {}
         self.access_path = []
 
     def dump_node(self, node, indent=0):
@@ -176,10 +134,43 @@ class TreeVisitor(BasicVisitor):
             last_node.pos, self.__class__.__name__,
             u'\n'.join(trace), e, stacktrace)
 
+    def find_handler(self, obj):
+        # to resolve, try entire hierarchy
+        cls = type(obj)
+        pattern = "visit_%s"
+        mro = inspect.getmro(cls)
+        handler_method = None
+        for mro_cls in mro:
+            handler_method = getattr(self, pattern % mro_cls.__name__, None)
+            if handler_method is not None:
+                return handler_method
+        print type(self), cls
+        if self.access_path:
+            print self.access_path
+            print self.access_path[-1][0].pos
+            print self.access_path[-1][0].__dict__
+        raise RuntimeError("Visitor %r does not accept object: %s" % (self, obj))
+
+    def visit(self, obj):
+        return self._visit(obj)
+
+    def _visit(self, obj):
+        try:
+            handler_method = self.dispatch_table[type(obj)]
+        except KeyError:
+            handler_method = self.find_handler(obj)
+            self.dispatch_table[type(obj)] = handler_method
+        return handler_method(obj)
+
     def _visitchild(self, child, parent, attrname, idx):
         self.access_path.append((parent, attrname, idx))
         try:
-            result = self._visit(child)
+            try:
+                handler_method = self.dispatch_table[type(child)]
+            except KeyError:
+                handler_method = self.find_handler(child)
+                self.dispatch_table[type(child)] = handler_method
+            result = handler_method(child)
         except Errors.CompileError:
             raise
         except Exception, e:

Cython/Plex/Actions.pxd

+
+cdef class Action:
+  cdef perform(self, token_stream, text)
+  cpdef same_as(self, other)
+
+cdef class Return(Action):
+  cdef object value
+  cdef perform(self, token_stream, text)
+  cpdef same_as(self, other)
+
+cdef class Call(Action):
+  cdef object function
+  cdef perform(self, token_stream, text)
+  cpdef same_as(self, other)
+
+cdef class Begin(Action):
+  cdef object state_name
+  cdef perform(self, token_stream, text)
+  cpdef same_as(self, other)
+
+cdef class Ignore(Action):
+  cdef perform(self, token_stream, text)
+
+cdef class Text(Action):
+  cdef perform(self, token_stream, text)

Cython/Plex/Actions.py

@@ -8,6 +8,9 @@
 
 class Action(object):
 
+  def perform(self, token_stream, text):
+    pass # abstract
+
   def same_as(self, other):
     return self is other
 
@@ -18,8 +21,6 @@ class Return(Action):
   be returned as the value of the associated token
   """
 
-  value = None
-
   def __init__(self, value):
     self.value = value
 
@@ -38,8 +39,6 @@ class Call(Action):
   Internal Plex action which causes a function to be called.
   """
 
-  function = None
-
   def __init__(self, function):
     self.function = function
 
@@ -60,8 +59,6 @@ class Begin(Action):
   for more information.
   """
 
-  state_name = None
-
   def __init__(self, state_name):
     self.state_name = state_name
 
@@ -88,7 +85,7 @@ class Ignore(Action):
     return "IGNORE"
 
 IGNORE = Ignore()
-IGNORE.__doc__ = Ignore.__doc__
+#IGNORE.__doc__ = Ignore.__doc__
 
 class Text(Action):
   """
@@ -104,6 +101,6 @@ class Text(Action):
     return "TEXT"
 
 TEXT = Text()
-TEXT.__doc__ = Text.__doc__
+#TEXT.__doc__ = Text.__doc__
 
 

Cython/Plex/Scanners.pxd

 import cython
 
+from Cython.Plex.Actions cimport Action
+
 cdef class Scanner:
 
     cdef public lexicon
@@ -25,10 +27,11 @@ cdef class Scanner:
     cdef public level
 
     @cython.locals(input_state=long)
-    cpdef next_char(self)
-    cpdef tuple read(self)
+    cdef next_char(self)
+    @cython.locals(action=Action)
+    cdef tuple read(self)
     cdef tuple scan_a_token(self)
-    cpdef tuple position(self)
+    cdef tuple position(self)
 
     @cython.locals(cur_pos=long, cur_line=long, cur_line_start=long,
                    input_state=long, next_pos=long, state=dict,
@@ -36,5 +39,5 @@ cdef class Scanner:
                    trace=bint, discard=long, data=unicode, buffer=unicode)
     cdef run_machine_inlined(self)
     
-    cpdef begin(self, state)
-    cpdef produce(self, value, text = *)
+    cdef begin(self, state)
+    cdef produce(self, value, text = *)

Cython/Plex/Scanners.py

@@ -8,11 +8,13 @@
 #=======================================================================
 
 import cython
-cython.declare(BOL=object, EOL=object, EOF=object)
+cython.declare(BOL=object, EOL=object, EOF=object, NOT_FOUND=object)
 
 import Errors
 from Regexps import BOL, EOL, EOF
 
+NOT_FOUND = object()
+
 class Scanner(object):
   """
   A Scanner is used to read tokens from a stream of characters
@@ -179,8 +181,8 @@ class Scanner(object):
       # End inlined self.save_for_backup()
       c = cur_char
       #new_state = state.new_state(c) #@slow
-      new_state = state.get(c, -1) #@fast
-      if new_state == -1: #@fast
+      new_state = state.get(c, NOT_FOUND) #@fast
+      if new_state is NOT_FOUND: #@fast
         new_state = c and state.get('else') #@fast
       if new_state:
         if trace: #TRACE#

runtests.py

@@ -10,6 +10,7 @@ import unittest
 import doctest
 import operator
 import tempfile
+import traceback
 try:
     from StringIO import StringIO
 except ImportError:
@@ -20,6 +21,11 @@ try:
 except ImportError:
     import pickle
 
+try:
+    import threading
+except ImportError: # No threads, no problems
+    threading = None
+
 
 WITH_CYTHON = True
 
@@ -313,11 +319,10 @@ class CythonCompileTestCase(unittest.TestCase):
             if is_related(filename) and os.path.isfile(os.path.join(workdir, filename)) ]
 
     def split_source_and_output(self, test_directory, module, workdir):
-        source_file = os.path.join(test_directory, module) + '.pyx'
-        source_and_output = codecs.open(
-            self.find_module_source_file(source_file), 'rU', 'ISO-8859-1')
+        source_file = self.find_module_source_file(os.path.join(test_directory, module) + '.pyx')
+        source_and_output = codecs.open(source_file, 'rU', 'ISO-8859-1')
         try:
-            out = codecs.open(os.path.join(workdir, module + '.pyx'),
+            out = codecs.open(os.path.join(workdir, module + os.path.splitext(source_file)[1]),
                               'w', 'ISO-8859-1')
             for line in source_and_output:
                 last_line = line
@@ -434,9 +439,12 @@ class CythonRunTestCase(CythonCompileTestCase):
         result.startTest(self)
         try:
             self.setUp()
-            self.runCompileTest()
-            if not self.cython_only:
-                self.run_doctests(self.module, result)
+            try:
+                self.runCompileTest()
+                if not self.cython_only:
+                    self.run_doctests(self.module, result)
+            finally:
+                check_thread_termination()
         except Exception:
             result.addError(self, sys.exc_info())
             result.stopTest(self)
@@ -477,7 +485,6 @@ class CythonRunTestCase(CythonCompileTestCase):
                     output = open(result_file, 'wb')
                     pickle.dump(partial_result.data(), output)
                 except:
-                    import traceback
                     traceback.print_exc()
             finally:
                 try: output.close()
@@ -531,6 +538,7 @@ class PureDoctestTestCase(unittest.TestCase):
                 del m
                 if loaded_module_name in sys.modules:
                     del sys.modules[loaded_module_name]
+                check_thread_termination()
         except Exception:
             result.addError(self, sys.exc_info())
             result.stopTest(self)
@@ -602,8 +610,11 @@ class CythonUnitTestCase(CythonCompileTestCase):
         result.startTest(self)
         try:
             self.setUp()
-            self.runCompileTest()
-            unittest.defaultTestLoader.loadTestsFromName(self.module).run(result)
+            try:
+                self.runCompileTest()
+                unittest.defaultTestLoader.loadTestsFromName(self.module).run(result)
+            finally:
+                check_thread_termination()
         except Exception:
             result.addError(self, sys.exc_info())
             result.stopTest(self)
@@ -842,8 +853,38 @@ def refactor_for_py3(distdir, cy3_dir):
                      ''')
     sys.path.insert(0, cy3_dir)
 
+class PendingThreadsError(RuntimeError):
+    pass
 
-if __name__ == '__main__':
+threads_seen = []
+
+def check_thread_termination(ignore_seen=True):
+    if threading is None: # no threading enabled in CPython
+        return
+    current = threading.currentThread()
+    blocking_threads = []
+    for t in threading.enumerate():
+        if not t.isAlive() or t == current:
+            continue
+        t.join(timeout=2)
+        if t.isAlive():
+            if not ignore_seen:
+                blocking_threads.append(t)
+                continue
+            for seen in threads_seen:
+                if t is seen:
+                    break
+            else:
+                threads_seen.append(t)
+                blocking_threads.append(t)
+    if not blocking_threads:
+        return
+    sys.stderr.write("warning: left-over threads found after running test:\n")
+    for t in blocking_threads:
+        sys.stderr.write('...%s\n'  % repr(t))
+    raise PendingThreadsError("left-over threads found after running test")
+
+def main():
     from optparse import OptionParser
     parser = OptionParser()
     parser.add_option("--no-cleanup", dest="cleanup_workdir",
@@ -955,6 +996,7 @@ if __name__ == '__main__':
             coverage.start()
 
     if WITH_CYTHON:
+        global CompilationOptions, pyrex_default_options, cython_compile
         from Cython.Compiler.Main import \
             CompilationOptions, \
             default_options as pyrex_default_options, \
@@ -1097,7 +1139,27 @@ if __name__ == '__main__':
         import refnanny
         sys.stderr.write("\n".join([repr(x) for x in refnanny.reflog]))
 
+    print("ALL DONE")
+
     if options.exit_ok:
-        sys.exit(0)
+        return_code = 0
     else:
-        sys.exit(not result.wasSuccessful())
+        return_code = not result.wasSuccessful()
+
+    try:
+        check_thread_termination(ignore_seen=False)
+        sys.exit(return_code)
+    except PendingThreadsError:
+        # normal program exit won't kill the threads, do it the hard way here
+        os._exit(return_code)
+
+if __name__ == '__main__':
+    try:
+        main()
+    except Exception:
+        traceback.print_exc()
+        try:
+            check_thread_termination(ignore_seen=False)
+        except PendingThreadsError:
+            # normal program exit won't kill the threads, do it the hard way here
+            os._exit(1)

setup.py

@@ -87,6 +87,7 @@ else:
 def compile_cython_modules(profile=False):
     source_root = os.path.abspath(os.path.dirname(__file__))
     compiled_modules = ["Cython.Plex.Scanners",
+                        "Cython.Plex.Actions",
                         "Cython.Compiler.Scanning",
                         "Cython.Compiler.Parsing",
                         "Cython.Compiler.Visitor",

tests/bugs.txt

@@ -17,11 +17,16 @@ function_as_method_T494
 closure_inside_cdef_T554
 ipow_crash_T562
 pure_mode_cmethod_inheritance_T583
-
+closure_class_T596
 
 # CPython regression tests that don't current work:
 pyregr.test_threadsignals
 pyregr.test_module
+pyregr.test_capi
+pyregr.test_socket
+pyregr.test_threading
+pyregr.test_sys
 
 # CPython regression tests that don't make sense
 pyregr.test_gdb
+pyregr.test_support

tests/broken/specialfloatvals.pyx → tests/compile/specialfloatvals.pyx

@@ -7,3 +7,5 @@ cdef int f() except -1:
     x = nan
     y = inf
     z = minf
+
+f()

tests/errors/e_badexcvaltype.pyx

@@ -2,5 +2,6 @@ cdef char *spam() except -1:
 	pass
 
 _ERRORS = u"""
+1:26: Cannot assign type 'long' to 'char *'
 1:26: Exception value incompatible with function return type
 """

tests/errors/e_int_literals_py2.py

+# cython: language_level=2
+
+def int_literals():
+    a = 1L  # ok
+    b = 10000000000000L # ok
+    c = 1UL
+    d = 10000000000000UL
+    e = 10000000000000LL
+
+
+_ERRORS = """
+6:8: illegal integer literal syntax in Python source file
+7:8: illegal integer literal syntax in Python source file
+8:8: illegal integer literal syntax in Python source file
+"""

tests/errors/e_int_literals_py3.py

+# cython: language_level=3
+
+def int_literals():
+    a = 1L
+    b = 10000000000000L
+    c = 1UL
+    d = 10000000000000UL
+    e = 10000000000000LL
+
+
+_ERRORS = """
+4:8: illegal integer literal syntax in Python source file
+5:8: illegal integer literal syntax in Python source file
+6:8: illegal integer literal syntax in Python source file
+7:8: illegal integer literal syntax in Python source file
+8:8: illegal integer literal syntax in Python source file
+"""

tests/run/closure_class_T596.pyx

+def simple(a, b):
+    """
+    >>> kls = simple(1, 2)
+    >>> kls().result()
+    3
+    """
+    class Foo:
+        def result(self):
+            return a + b
+    return Foo
+
+def nested_classes(a, b):
+    """
+    >>> kls = nested_classes(1, 2)
+    >>> kls().result(-3)
+    0
+    """
+    class Foo:
+        class Bar:
+            def result(self, c):
+                return a + b + c
+    return Foo.Bar
+
+def staff(a, b):
+    """
+    >>> kls = staff(1, 2)
+    >>> kls.static()
+    (1, 2)
+    >>> kls.klass()
+    ('Foo', 1, 2)
+    >>> obj = kls()
+    >>> obj.member()
+    (1, 2)
+    """
+    class Foo:
+        def member(self):
+            return a, b
+        @staticmethod
+        def static():
+            return a, b
+        @classmethod
+        def klass(cls):
+            return cls.__name__, a, b
+    return Foo
+
+def nested2(a):
+    """
+    >>> obj = nested2(1)
+    >>> f = obj.run(2)
+    >>> f()
+    3
+    """
+    class Foo:
+        def run(self, b):
+            def calc():
+                return a + b
+            return calc
+    return Foo()

tests/run/closure_name_mangling_T537.pyx

+
+__doc__ = u"""
+>>> f1 = nested1()
+>>> f2 = nested2()
+>>> f1 == f2      # inner functions (f)
+False
+>>> f1() == f2()  # inner-inner functions (g)
+False
+"""
+
+def nested1():
+   def f():
+      def g():
+         pass
+      return g
+   return f
+
+def nested2():
+   def f():
+      def g():
+         pass
+      return g
+   return f

tests/run/cython3.pyx

@@ -64,6 +64,19 @@ def list_comp_unknown_type(l):
     """
     return [x*2 for x in l if x % 2 == 0]
 
+def listcomp_as_condition(sequence):
+    """
+    >>> listcomp_as_condition(['a', 'b', '+'])
+    True
+    >>> listcomp_as_condition('ab+')
+    True
+    >>> listcomp_as_condition('abc')
+    False
+    """
+    if [1 for c in sequence if c in '+-*/<=>!%&|([^~,']:
+        return True
+    return False
+
 def set_comp():
     """
     >>> sorted(set_comp())
@@ -106,3 +119,16 @@ def dict_iter(dict d):
     values = [ value for value in d.values() ]
     items = [ item for item in d.items() ]
     return keys, values, items
+
+def int_literals():
+    """
+    >>> int_literals()
+    long
+    long
+    unsigned long
+    unsigned long
+    """
+    print(cython.typeof(1L))
+    print(cython.typeof(10000000000000L))
+    print(cython.typeof(1UL))
+    print(cython.typeof(10000000000000UL))

tests/run/exceptionpropagation.pyx

@@ -28,3 +28,17 @@ def test_except_expr(bint fire):
     RuntimeError
     """
     except_expr(fire)
+
+cdef double except_big_result(bint fire) except 100000000000000000000000000000000:
+    if fire:
+        raise RuntimeError
+
+def test_except_big_result(bint fire):
+    """
+    >>> test_except_big_result(False)
+    >>> test_except_big_result(True)
+    Traceback (most recent call last):
+    ...
+    RuntimeError
+    """
+    except_big_result(fire)

tests/run/inplace.pyx

@@ -244,3 +244,13 @@ def double_deref_and_increment(o, key1, key2):
     v __setitem__ a 11
     """
     o[side_effect(key1)][side_effect(key2)] += 1
+
+def conditional_inplace(value, a, condition, b):
+    """
+    >>> conditional_inplace([1, 2, 3], [100], True, [200])
+    [1, 2, 3, 100]
+    >>> conditional_inplace([1, 2, 3], [100], False, [200])
+    [1, 2, 3, 200]
+    """
+    value += a if condition else b
+    return value

tests/run/listcomp.pyx

@@ -63,3 +63,16 @@ def nested_result():
     """
     result = [[a-1 for a in range(b)] for b in range(4)]
     return result
+
+def listcomp_as_condition(sequence):
+    """
+    >>> listcomp_as_condition(['a', 'b', '+'])
+    True
+    >>> listcomp_as_condition('ab+')
+    True
+    >>> listcomp_as_condition('abc')
+    False
+    """
+    if [1 for c in sequence if c in '+-*/<=>!%&|([^~,']:
+        return True
+    return False

tests/run/metaclass.pyx

 
+cimport cython
+
 class Base(type):
     def __new__(cls, name, bases, attrs):
         attrs['metaclass_was_here'] = True
         return type.__new__(cls, name, bases, attrs)
 
+@cython.test_fail_if_path_exists("//PyClassMetaclassNode", "//Py3ClassNode")
 class Foo(object):
     """
     >>> obj = Foo()
@@ -12,26 +15,111 @@ class Foo(object):
     """
     __metaclass__ = Base
 
+class ODict(dict):
+    def __init__(self):
+        dict.__init__(self)
+        self._order = []
+        dict.__setitem__(self, '_order', self._order)
+
+    def __setitem__(self, key, value):
+        dict.__setitem__(self, key, value)
+        self._order.append(key)
+
+class Py3MetaclassPlusAttr(type):
+    def __new__(cls, name, bases, attrs, **kwargs):
+        for key, value in kwargs.items():
+            attrs[key] = value
+        attrs['metaclass_was_here'] = True
+        return type.__new__(cls, name, bases, attrs)
+
+    def __init__(self, cls, attrs, obj, **kwargs):
+        pass
+
+    @staticmethod
+    def __prepare__(*args, **kwargs):
+        return ODict()
+
+@cython.test_fail_if_path_exists("//PyClassMetaclassNode")
+@cython.test_assert_path_exists("//Py3ClassNode")
+class Py3ClassMCOnly(object, metaclass=Py3MetaclassPlusAttr):
+    """
+    >>> obj = Py3ClassMCOnly()
+    >>> obj.bar
+    321
+    >>> obj.metaclass_was_here
+    True
+    >>> obj._order
+    ['__module__', '__doc__', 'bar', 'metaclass_was_here']
+    """
+    bar = 321
 
 class Py3Base(type):
-    def __new__(cls, name, bases, attrs, foo=None):
-        attrs['foo'] = foo
+    def __new__(cls, name, bases, attrs, **kwargs):
+        for key, value in kwargs.items():
+            attrs[key] = value
         return type.__new__(cls, name, bases, attrs)
 
-    def __init__(self, cls, attrs, obj, foo=None):
+    def __init__(self, cls, attrs, obj, **kwargs):
         pass
+
     @staticmethod
-    def __prepare__(name, bases, **kwargs):
-        return {'bar': 666, 'dirty': True}
+    def __prepare__(*args, **kwargs):
+        return ODict()
 
+@cython.test_fail_if_path_exists("//PyClassMetaclassNode")
+@cython.test_assert_path_exists("//Py3ClassNode")
 class Py3Foo(object, metaclass=Py3Base, foo=123):
     """
     >>> obj = Py3Foo()
     >>> obj.foo
     123
     >>> obj.bar
-    666
-    >>> obj.dirty
-    False
+    321
+    >>> obj._order
+    ['__module__', '__doc__', 'bar', 'foo']
+    """
+    bar = 321
+
+kwargs = {'foo': 123, 'bar': 456}
+
+@cython.test_assert_path_exists("//PyClassMetaclassNode", "//Py3ClassNode")
+class Py3Mixed(metaclass=Py3Base, **kwargs):
+    """
+    >>> Py3Mixed.foo
+    123
+    >>> Py3Mixed.bar
+    456
+    """
+
+kwargs['metaclass'] = Py3Base
+
+@cython.test_assert_path_exists("//PyClassMetaclassNode")
+class Py3Kwargs(**kwargs):
+    """
+    >>> Py3Kwargs.foo
+    123
+    >>> Py3Kwargs.bar
+    456
+    """
+
+class Base3(type):
+    def __new__(cls, name, bases, attrs, **kwargs):
+        kwargs['b'] = 2
+        return type.__new__(cls, name, bases, attrs)
+
+    def __init__(self, *args, **kwargs):
+        self.kwargs = kwargs
+
+    @staticmethod
+    def __prepare__(*args, **kwargs):
+        kwargs['a'] = 1
+        return {}
+
+kwargs = {'c': 0}
+
+@cython.test_assert_path_exists("//PyClassMetaclassNode", "//Py3ClassNode")
+class Foo3(metaclass=Base3, a=0, b=0, **kwargs):
+    """
+    >>> Foo3.kwargs
+    {'a': 0, 'c': 0, 'b': 0}
     """
-    dirty = False

tests/run/py_unicode_type.pyx

@@ -162,3 +162,36 @@ def count_lower_case_characters(unicode ustring):
          if uchar.islower():
              count += 1
     return count
+
+@cython.test_assert_path_exists('//SwitchStatNode',
+                                '//ForFromStatNode')
+@cython.test_fail_if_path_exists('//ForInStatNode')
+def iter_and_in():
+    """
+    >>> iter_and_in()
+    a
+    b
+    e
+    f
+    h
+    """
+    for c in u'abcdefgh':
+        if c in u'abCDefGh':
+            print c
+
+@cython.test_assert_path_exists('//SwitchStatNode',
+                                '//ForFromStatNode')
+@cython.test_fail_if_path_exists('//ForInStatNode')
+def index_and_in():
+    """
+    >>> index_and_in()
+    1
+    3
+    4
+    7
+    8
+    """
+    cdef int i
+    for i in range(1,9):
+        if u'abcdefgh'[-i] in u'abCDefGh':
+            print i

tests/run/tuple_constants.pyx

+
+cimport cython
+
+module_level_tuple = (1,2,3)
+
+def return_module_level_tuple():
+    """
+    >>> return_module_level_tuple()
+    (1, 2, 3)
+    """
+    return module_level_tuple
+
+@cython.test_assert_path_exists("//TupleNode",
+                                "//TupleNode[@is_literal = true]")
+@cython.test_fail_if_path_exists("//TupleNode[@is_literal = false]")
+def return_empty_tuple():
+    """
+    >>> return_empty_tuple()
+    ()
+    """
+    return ()
+
+@cython.test_assert_path_exists("//TupleNode",
+                                "//TupleNode[@is_literal = true]")
+@cython.test_fail_if_path_exists("//TupleNode[@is_literal = false]")
+def return_constant_tuple1():
+    """
+    >>> return_constant_tuple1()
+    (1,)
+    """
+    return (1,)
+
+@cython.test_assert_path_exists("//TupleNode",
+                                "//TupleNode[@is_literal = true]")
+@cython.test_fail_if_path_exists("//TupleNode[@is_literal = false]")
+def return_folded_tuple():
+    """
+    >>> return_folded_tuple()
+    (1, 2, 3)
+    """
+    return (1, 1+1, 1+1+1)
+
+@cython.test_assert_path_exists("//TupleNode",
+                                "//TupleNode[@is_literal = true]")
+@cython.test_fail_if_path_exists("//TupleNode[@is_literal = false]")
+def return_nested_tuple():
+    """
+    >>> return_nested_tuple()
+    (1, (2, 3), (3, (4, 5)))
+    """
+    return (1, (2, 3), (3, (4, 5)))
+
+@cython.test_assert_path_exists("//TupleNode",
+                                "//TupleNode[@is_literal = true]")
+@cython.test_fail_if_path_exists("//TupleNode[@is_literal = false]")
+def constant_tuple1():
+    """
+    >>> constant_tuple1()
+    (1,)
+    """
+    tuple1 = (1,)
+    return tuple1
+
+@cython.test_assert_path_exists("//TupleNode",
+                                "//TupleNode[@is_literal = true]")
+@cython.test_fail_if_path_exists("//TupleNode[@is_literal = false]")
+def return_constant_tuple2():
+    """
+    >>> return_constant_tuple2()
+    (1, 2)
+    """
+    return (1,2)
+
+@cython.test_assert_path_exists("//TupleNode",
+                                "//TupleNode[@is_literal = true]")
+@cython.test_fail_if_path_exists("//TupleNode[@is_literal = false]")
+def return_constant_tuple_strings():
+    """
+    >>> return_constant_tuple_strings()
+    ('tuple_1', 'bc', 'tuple_2')
+    """
+    return ('tuple_1', 'bc', 'tuple_2')
+
+@cython.test_assert_path_exists("//TupleNode",
+                                "//TupleNode[@is_literal = true]")
+@cython.test_fail_if_path_exists("//TupleNode[@is_literal = false]")
+def return_constant_tuples_string_types():
+    """
+    >>> a,b,c = return_constant_tuples_string_types()
+    >>> a is b
+    False
+    >>> a is c
+    False
+    >>> b is c
+    False
+    """
+    return ('a', 'bc'), (u'a', u'bc'), (b'a', b'bc')
+
+@cython.test_assert_path_exists("//ReturnStatNode//TupleNode",
+                                "//ReturnStatNode//TupleNode[@is_literal = false]")
+@cython.test_fail_if_path_exists("//ReturnStatNode//TupleNode[@is_literal = true]")
+def return_nonconstant_tuple():
+    """
+    >>> return_nonconstant_tuple()
+    ('a', 1, 'd')
+    """
+    a = eval("1")
+    return ('a', a, 'd')