Merge pull request #398 from rntz/with-exceptions

in merge:
- with_ctxclass_instance_attrs now succeeds due to d1387d74
- merge ~conflict in LangPrimitive opcodes

.gitignore

@@ -36,6 +36,7 @@ stdlib*.ll
 oprofile_data
 pprof.jit
 tags
+TAGS
 *.pyc
 
 perf.data

Makefile

@@ -37,6 +37,9 @@ VERBOSE := 0
 
 ENABLE_INTEL_JIT_EVENTS := 0
 
+CTAGS := ctags
+ETAGS := ctags-exuberant -e
+
 # Setting this to 1 will set the Makefile to use binaries from the trunk
 # directory, even if USE_TEST_LLVM is set to 1.
 # This is useful if clang isn't installed into the test directory, ex due
@@ -408,7 +411,11 @@ all: pyston_dbg pyston_release pyston_prof ext_python ext_pyston unittests
 ALL_HEADERS := $(wildcard src/*/*.h) $(wildcard src/*/*/*.h) $(wildcard from_cpython/Include/*.h)
 tags: $(SRCS) $(OPTIONAL_SRCS) $(FROM_CPYTHON_SRCS) $(ALL_HEADERS)
 	$(ECHO) Calculating tags...
-	$(VERB) ctags $^
+	$(VERB) $(CTAGS) $^
+
+TAGS: $(SRCS) $(OPTIONAL_SRCS) $(FROM_CPYTHON_SRCS) $(ALL_HEADERS)
+	$(ECHO) Calculating TAGS...
+	$(VERB) $(ETAGS) $^
 
 NON_ENTRY_OBJS := $(filter-out src/jit.o,$(OBJS))
 

src/analysis/type_analysis.cpp

@@ -511,7 +511,9 @@ private:
         }
     }
 
-    void visit_classdef(AST_ClassDef* node) override {
+    void* visit_makeclass(AST_MakeClass* mkclass) override {
+        AST_ClassDef* node = mkclass->class_def;
+
         for (auto d : node->decorator_list) {
             getType(d);
         }
@@ -521,9 +523,8 @@ private:
         }
 
         // TODO should we speculate that classdefs will generally return a class?
-        // CompilerType* t = typeFromClass(type_cls);
-        CompilerType* t = UNKNOWN;
-        _doSet(scope_info->mangleName(node->name), t);
+        // return typeFromClass(type_cls);
+        return UNKNOWN;
     }
 
     void visit_delete(AST_Delete* node) override {
@@ -551,7 +552,9 @@ private:
         }
     }
 
-    void visit_functiondef(AST_FunctionDef* node) override {
+    void* visit_makefunction(AST_MakeFunction* mkfn) override {
+        AST_FunctionDef* node = mkfn->function_def;
+
         for (auto d : node->decorator_list) {
             getType(d);
         }
@@ -563,7 +566,7 @@ private:
         CompilerType* t = UNKNOWN;
         if (node->decorator_list.empty())
             t = typeFromClass(function_cls);
-        _doSet(scope_info->mangleName(node->name), t);
+        return t;
     }
 
     void visit_global(AST_Global* node) override {}

src/codegen/ast_interpreter.cpp

@@ -88,10 +88,8 @@ private:
     Value visit_assign(AST_Assign* node);
     Value visit_binop(AST_BinOp* node);
     Value visit_call(AST_Call* node);
-    Value visit_classDef(AST_ClassDef* node);
     Value visit_compare(AST_Compare* node);
     Value visit_delete(AST_Delete* node);
-    Value visit_functionDef(AST_FunctionDef* node);
     Value visit_global(AST_Global* node);
     Value visit_module(AST_Module* node);
     Value visit_print(AST_Print* node);
@@ -117,6 +115,8 @@ private:
     Value visit_tuple(AST_Tuple* node);
     Value visit_yield(AST_Yield* node);
 
+    Value visit_makeClass(AST_MakeClass* node);
+    Value visit_makeFunction(AST_MakeFunction* node);
 
     // pseudo
     Value visit_augBinOp(AST_AugBinOp* node);
@@ -605,14 +605,10 @@ Value ASTInterpreter::visit_stmt(AST_stmt* node) {
             return visit_assert((AST_Assert*)node);
         case AST_TYPE::Assign:
             return visit_assign((AST_Assign*)node);
-        case AST_TYPE::ClassDef:
-            return visit_classDef((AST_ClassDef*)node);
         case AST_TYPE::Delete:
             return visit_delete((AST_Delete*)node);
         case AST_TYPE::Expr:
             return visit_expr((AST_Expr*)node);
-        case AST_TYPE::FunctionDef:
-            return visit_functionDef((AST_FunctionDef*)node);
         case AST_TYPE::Pass:
             return Value(); // nothing todo
         case AST_TYPE::Print:
@@ -690,7 +686,8 @@ Box* ASTInterpreter::createFunction(AST* node, AST_arguments* args, const std::v
     return boxCLFunction(cl, closure, is_generator, u.il);
 }
 
-Value ASTInterpreter::visit_functionDef(AST_FunctionDef* node) {
+Value ASTInterpreter::visit_makeFunction(AST_MakeFunction* mkfn) {
+    AST_FunctionDef* node = mkfn->function_def;
     AST_arguments* args = node->args;
 
     std::vector<Box*, StlCompatAllocator<Box*>> decorators;
@@ -702,11 +699,11 @@ Value ASTInterpreter::visit_functionDef(AST_FunctionDef* node) {
     for (int i = decorators.size() - 1; i >= 0; i--)
         func = runtimeCall(decorators[i], ArgPassSpec(1), func, 0, 0, 0, 0);
 
-    doStore(source_info->mangleName(node->name), func);
-    return Value();
+    return Value(func);
 }
 
-Value ASTInterpreter::visit_classDef(AST_ClassDef* node) {
+Value ASTInterpreter::visit_makeClass(AST_MakeClass* mkclass) {
+    AST_ClassDef* node = mkclass->class_def;
     ScopeInfo* scope_info = source_info->scoping->getScopeInfoForNode(node);
     assert(scope_info);
 
@@ -729,8 +726,7 @@ Value ASTInterpreter::visit_classDef(AST_ClassDef* node) {
     for (int i = decorators.size() - 1; i >= 0; i--)
         classobj = runtimeCall(decorators[i], ArgPassSpec(1), classobj, 0, 0, 0, 0);
 
-    doStore(source_info->mangleName(node->name), classobj);
-    return Value();
+    return Value(classobj);
 }
 
 Value ASTInterpreter::visit_raise(AST_Raise* node) {
@@ -896,6 +892,10 @@ Value ASTInterpreter::visit_expr(AST_expr* node) {
             return visit_clsAttribute((AST_ClsAttribute*)node);
         case AST_TYPE::LangPrimitive:
             return visit_langPrimitive((AST_LangPrimitive*)node);
+        case AST_TYPE::MakeClass:
+            return visit_makeClass((AST_MakeClass*)node);
+        case AST_TYPE::MakeFunction:
+            return visit_makeFunction((AST_MakeFunction*)node);
         default:
             RELEASE_ASSERT(0, "");
     };

src/codegen/compvars.cpp

@@ -757,6 +757,8 @@ CompilerVariable* makeFunction(IREmitter& emitter, CLFunction* f, CompilerVariab
 
     llvm::Value* isGenerator_v = llvm::ConstantInt::get(g.i1, isGenerator, false);
 
+    // We know this function call can't throw, so it's safe to use emitter.getBuilder()->CreateCall() rather than
+    // emitter.createCall().
     llvm::Value* boxed = emitter.getBuilder()->CreateCall(
         g.funcs.boxCLFunction,
         std::vector<llvm::Value*>{ embedConstantPtr(f, g.llvm_clfunction_type_ptr), closure_v, isGenerator_v, scratch,

src/codegen/compvars.h

@@ -291,7 +291,7 @@ private:
 
 protected:
     void drop(IREmitter& emitter) override { type->drop(emitter, this); }
-    void grab(IREmitter& emmitter) override { type->grab(emmitter, this); }
+    void grab(IREmitter& emitter) override { type->grab(emitter, this); }
 
 public:
     ValuedCompilerVariable(T* type, V value, bool grabbed) : CompilerVariable(grabbed), type(type), value(value) {

src/codegen/irgen/irgenerator.h

@@ -95,10 +95,14 @@ public:
     ParamNames* getParamNames() { return param_names; }
 };
 
+// turns CFGBlocks into LLVM IR
 class IRGenerator {
 private:
 public:
     struct EndingState {
+        // symbol_table records which Python variables are bound to what CompilerVariables at the end of this block.
+        // phi_symbol_table records the ones that will need to be `phi'd.
+        // both only record non-globals.
         SymbolTable* symbol_table;
         ConcreteSymbolTable* phi_symbol_table;
         llvm::BasicBlock* ending_block;
@@ -113,7 +117,7 @@ public:
 
     virtual void giveLocalSymbol(InternedString name, CompilerVariable* var) = 0;
     virtual void copySymbolsFrom(SymbolTable* st) = 0;
-    virtual void run(const CFGBlock* block) = 0;
+    virtual void run(const CFGBlock* block) = 0; // primary entry point
     virtual EndingState getEndingSymbolTable() = 0;
     virtual void doSafePoint() = 0;
     virtual void addFrameStackmapArgs(PatchpointInfo* pp, AST_stmt* current_stmt,

src/core/ast.cpp

@@ -953,7 +953,6 @@ void AST_Branch::accept_stmt(StmtVisitor* v) {
     v->visit_branch(this);
 }
 
-
 void AST_Jump::accept(ASTVisitor* v) {
     bool skip = v->visit_jump(this);
     if (skip)
@@ -976,7 +975,29 @@ void* AST_ClsAttribute::accept_expr(ExprVisitor* v) {
     return v->visit_clsattribute(this);
 }
 
+void AST_MakeFunction::accept(ASTVisitor* v) {
+    bool skip = v->visit_makefunction(this);
+    if (skip)
+        return;
+
+    function_def->accept(v);
+}
+
+void* AST_MakeFunction::accept_expr(ExprVisitor* v) {
+    return v->visit_makefunction(this);
+}
 
+void AST_MakeClass::accept(ASTVisitor* v) {
+    bool skip = v->visit_makeclass(this);
+    if (skip)
+        return;
+
+    class_def->accept(v);
+}
+
+void* AST_MakeClass::accept_expr(ExprVisitor* v) {
+    return v->visit_makeclass(this);
+}
 
 void print_ast(AST* ast) {
     PrintVisitor v;
@@ -1851,6 +1872,16 @@ bool PrintVisitor::visit_clsattribute(AST_ClsAttribute* node) {
     return true;
 }
 
+bool PrintVisitor::visit_makefunction(AST_MakeFunction* node) {
+    printf("make_");
+    return false;
+}
+
+bool PrintVisitor::visit_makeclass(AST_MakeClass* node) {
+    printf("make_");
+    return false;
+}
+
 class FlattenVisitor : public ASTVisitor {
 private:
     std::vector<AST*>* output;
@@ -2098,6 +2129,15 @@ public:
         output->push_back(node);
         return false;
     }
+
+    virtual bool visit_makeclass(AST_MakeClass* node) {
+        output->push_back(node);
+        return false;
+    }
+    virtual bool visit_makefunction(AST_MakeFunction* node) {
+        output->push_back(node);
+        return false;
+    }
 };
 
 void flatten(const std::vector<AST_stmt*>& roots, std::vector<AST*>& output, bool expand_scopes) {

src/core/ast.h

@@ -118,7 +118,7 @@ enum AST_TYPE {
     DictComp = 15,
     Set = 43,
     Ellipsis = 87,
-    Expression = 88,
+    Expression = 88, // like Module, but used for eval.
     SetComp = 89,
 
     // Pseudo-nodes that are specific to this compiler:
@@ -128,6 +128,8 @@ enum AST_TYPE {
     AugBinOp = 203,
     Invoke = 204,
     LangPrimitive = 205,
+    MakeClass = 206,    // wraps a ClassDef to make it an expr
+    MakeFunction = 207, // wraps a FunctionDef to make it an expr
 
     // These aren't real AST types, but since we use AST types to represent binexp types
     // and divmod+truediv are essentially types of binops, we add them here (at least for now):
@@ -951,6 +953,31 @@ public:
     static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::Yield;
 };
 
+class AST_MakeFunction : public AST_expr {
+public:
+    AST_FunctionDef* function_def;
+
+    virtual void accept(ASTVisitor* v);
+    virtual void* accept_expr(ExprVisitor* v);
+
+    AST_MakeFunction(AST_FunctionDef* fd)
+        : AST_expr(AST_TYPE::MakeFunction, fd->lineno, fd->col_offset), function_def(fd) {}
+
+    static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::MakeFunction;
+};
+
+class AST_MakeClass : public AST_expr {
+public:
+    AST_ClassDef* class_def;
+
+    virtual void accept(ASTVisitor* v);
+    virtual void* accept_expr(ExprVisitor* v);
+
+    AST_MakeClass(AST_ClassDef* cd) : AST_expr(AST_TYPE::MakeClass, cd->lineno, cd->col_offset), class_def(cd) {}
+
+    static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::MakeClass;
+};
+
 
 // AST pseudo-nodes that will get added during CFG-construction.  These don't exist in the input AST, but adding them in
 // lets us avoid creating a completely new IR for this phase
@@ -1019,14 +1046,14 @@ public:
 class AST_LangPrimitive : public AST_expr {
 public:
     enum Opcodes {
-        LANDINGPAD,
+        LANDINGPAD, // grabs the info about the last raised exception
         LOCALS,
         GET_ITER,
         IMPORT_FROM,
         IMPORT_NAME,
         IMPORT_STAR,
         NONE,
-        NONZERO,
+        NONZERO, // determines whether something is "true" for purposes of `if' and so forth
         CHECK_EXC_MATCH,
         SET_EXC_INFO,
         UNCACHE_EXC_INFO,
@@ -1115,6 +1142,8 @@ public:
     virtual bool visit_with(AST_With* node) { RELEASE_ASSERT(0, ""); }
     virtual bool visit_yield(AST_Yield* node) { RELEASE_ASSERT(0, ""); }
 
+    virtual bool visit_makeclass(AST_MakeClass* node) { RELEASE_ASSERT(0, ""); }
+    virtual bool visit_makefunction(AST_MakeFunction* node) { RELEASE_ASSERT(0, ""); }
     virtual bool visit_branch(AST_Branch* node) { RELEASE_ASSERT(0, ""); }
     virtual bool visit_jump(AST_Jump* node) { RELEASE_ASSERT(0, ""); }
 };
@@ -1187,6 +1216,8 @@ public:
 
     virtual bool visit_branch(AST_Branch* node) { return false; }
     virtual bool visit_jump(AST_Jump* node) { return false; }
+    virtual bool visit_makeclass(AST_MakeClass* node) { return false; }
+    virtual bool visit_makefunction(AST_MakeFunction* node) { return false; }
 };
 
 class ExprVisitor {
@@ -1223,6 +1254,8 @@ public:
     virtual void* visit_tuple(AST_Tuple* node) { RELEASE_ASSERT(0, ""); }
     virtual void* visit_unaryop(AST_UnaryOp* node) { RELEASE_ASSERT(0, ""); }
     virtual void* visit_yield(AST_Yield* node) { RELEASE_ASSERT(0, ""); }
+    virtual void* visit_makeclass(AST_MakeClass* node) { RELEASE_ASSERT(0, ""); }
+    virtual void* visit_makefunction(AST_MakeFunction* node) { RELEASE_ASSERT(0, ""); }
 };
 
 class StmtVisitor {
@@ -1332,6 +1365,8 @@ public:
 
     virtual bool visit_branch(AST_Branch* node);
     virtual bool visit_jump(AST_Jump* node);
+    virtual bool visit_makefunction(AST_MakeFunction* node);
+    virtual bool visit_makeclass(AST_MakeClass* node);
 };
 
 // Given an AST node, return a vector of the node plus all its descendents.

src/core/cfg.h

@@ -56,6 +56,7 @@ public:
     void unconnectFrom(CFGBlock* successor);
 
     void push_back(AST_stmt* node) { body.push_back(node); }
+    void print();
 };
 
 // Control Flow Graph
@@ -79,6 +80,9 @@ public:
         return block;
     }
 
+    // Creates a block which must be placed later, using placeBlock().
+    // Must be placed on same CFG it was created on.
+    // You can also safely delete it without placing it.
     CFGBlock* addDeferredBlock() {
         CFGBlock* block = new CFGBlock(this, -1);
         return block;

src/core/common.h

@@ -39,6 +39,8 @@
 #define _CAT(A, B) A##B
 #define CAT(A, B) _CAT(A, B)
 
+#define ARRAY_LEN(arr) (sizeof(arr) / sizeof((arr)[0]))
+
 // GCC and clang handle always_inline very differently;
 // we mostly only care about it for the stdlib, so just remove the attributes
 // if we're not in clang

src/core/util.h

@@ -55,7 +55,9 @@ bool endswith(const std::string& s, const std::string& pattern);
 
 void removeDirectoryIfExists(const std::string& path);
 
-template <class T1, class T2> void compareKeyset(T1* lhs, T2* rhs) {
+// Checks that lhs and rhs, which are iterables of InternedStrings, have the
+// same set of names in them.
+template <class T1, class T2> bool sameKeyset(T1* lhs, T2* rhs) {
     std::vector<InternedString> lv, rv;
     for (typename T1::iterator it = lhs->begin(); it != lhs->end(); it++) {
         lv.push_back(it->first);
@@ -92,7 +94,7 @@ template <class T1, class T2> void compareKeyset(T1* lhs, T2* rhs) {
         }
         good = false;
     }
-    assert(good);
+    return good;
 }
 }
 

src/runtime/file.cpp

@@ -800,10 +800,6 @@ Box* fileEnter(BoxedFile* self) {
 Box* fileExit(BoxedFile* self, Box* exc_type, Box* exc_val, Box** args) {
     Box* exc_tb = args[0];
     assert(self->cls == file_cls);
-    assert(exc_type == None);
-    assert(exc_val == None);
-    assert(exc_tb == None);
-
     fileClose(self);
     return None;
 }

src/runtime/types.cpp

@@ -390,6 +390,9 @@ std::string BoxedModule::name() {
     }
 }
 
+// This mustn't throw; our IR generator generates calls to it without "invoke" even when there are exception handlers /
+// finally-blocks in scope.
+// TODO: should we use C++11 `noexcept' here?
 extern "C" Box* boxCLFunction(CLFunction* f, BoxedClosure* closure, bool isGenerator,
                               std::initializer_list<Box*> defaults) {
     if (closure)

test/tests/exceptions_test.py

-# expected: fail
-# - with statements
-
 class TestException(Exception):
     pass
 
@@ -361,7 +358,11 @@ def f12():
         except Exception as l[0]:
             print "shouldnt get here"
     except Exception as e2:
-        print e2
+        # print it to stderr, so that our tester's error message substituter can
+        # deal with differences between error messages between different Python
+        # versions.
+        import traceback
+        traceback.print_exc()
 
     l = []
     try:

test/tests/file_closed.py

+# expected: fail
+import sys
+print sys.stdout.closed

test/tests/finally_except.py

+# these may seem pointless, but they exercise a family of corner cases in our
+# CFG generation pass (cfg.cpp).
+def returner():
+    try:
+        try:
+            print '2 + 2'
+            return
+        finally:
+            print 'finally'
+    except:
+        print 'exception'
+
+returner()
+
+def continuer():
+    for x in [1]:
+        try:
+            try:
+                print '2 + 2'
+                continue
+            finally:
+                print 'finally'
+        except:
+            print 'exception'
+
+continuer()
+
+def breaker():
+    for x in [1]:
+        try:
+            try:
+                print '2 + 2'
+                break
+            finally:
+                print 'finally'
+        except:
+            print 'exception'
+
+breaker()
+
+def raiser():
+    try:
+        try:
+            print '2 + 2'
+            raise Exception('blaaargh')
+        finally:
+            print 'finally'
+    except:
+        print 'exception'
+
+raiser()
+
+def alltogethernow():
+    for x in [1,2,3,4]:
+        try:
+            try:
+                print '2 + 2'
+                if x == 1: break
+                if x == 2: continue
+                if x == 3: raise Exception('blaargh')
+                if x == 4: return
+            finally:
+                print 'finally'
+        except:
+            print 'exception'
+
+alltogethernow()

test/tests/try_class.py

+# we have one test at global scope, another at local.
+# they behave differently in codegen; there have been points at which either was bugged when the other was not.
+try:
+    class C(object):
+        print 'here'
+finally:
+    print 'finally'
+
+def f():
+    try:
+        class C(object):
+            print 'here'
+    finally:
+        print 'finally'
+f()
+
+try:
+    class D(object):
+        print 'here'
+except:
+    print 'impossible'
+    raise
+
+def f2():
+    try:
+        class D(object):
+            print 'here'
+    except:
+        print 'impossible'
+        print D
+        raise
+f2()

test/tests/try_continue.py

+# expected: fail
+# Syntax error to have a continue outside a loop.
+def foo():
+    try: continue
+    finally: pass

test/tests/try_def.py

+def f():
+    try:
+        def foo(): return 0
+    except:
+        print 'except'
+    finally:
+        print 'finally'
+f()
+
+def f2():
+    try:
+        def foo(): return 0
+    except:
+        print 'except'
+f2()
+
+def f3():
+    try:
+        def foo(): return 0
+    finally:
+        print 'finally'
+f3()

test/tests/try_return_finally.py

+def f():
+    try:
+        # Looks like this returns from the function, but it needs to go to the finally block
+        return
+    finally:
+        pass
+f()

test/tests/with_class.py

+def f():
+    # originally this exposed a bug in our irgen phase, so even `with None`
+    # failed here; the bug happened before actual execution. Just to test more
+    # things, though, we use an actual contextmanager here.
+    with open('/dev/null'):
+        class C(object):
+            print 'hello'
+
+f()

test/tests/with_class_redefine.py

+def f():
+    C = 23
+    try:
+        class C(object): pass
+    except:
+        print 'except: C = %s' % C
+        raise
+    finally:
+        print 'finally: C = %s' % C
+    print 'end: C = %s' % C
+
+f()

test/tests/with_continue.py

+# should_error
+class Mgr(object):
+    def __enter__(self):
+        print 'entered!'
+    def __exit__(self, *args):
+        print 'exited!'
+
+with Mgr() as m:
+    continue

test/tests/with_ctxclass_instance_attrs.py

+class Mgr(object):
+    def __init__(self): self.__enter__ = 'sucks to be you'
+    def __enter__(self): pass
+    def __exit__(self, typ, val, tback): pass
+
+with Mgr() as m:
+    print 'boom boom boom boom'
+
+class Mgr2(object):
+    def __init__(self): self.__exit__ = 'screwed!'
+    def __enter__(self): pass
+    def __exit__(self, typ, val, tback): pass
+
+with Mgr2() as m:
+    print 'bang bang bang bang'

test/tests/with_ctxclass_order_of_access.py

+class Mgr(object):
+    def __init__(self): print 'Mgr.__init__()'
+
+    @property
+    def __enter__(self):
+        print 'Mgr.__enter__ accessed'
+        def enterer(*args):
+            print 'Mgr.__enter__ called'
+            return 23
+        return enterer
+
+    @property
+    def __exit__(self):
+        print 'Mgr.__exit__ accessed'
+        def exiter(*args):
+            print 'Mgr.__exit__ called'
+            return False
+        return exiter
+
+with Mgr() as m:
+    print 'hello I am a with block'
+    print 'm: %s' % m
+
+try:
+    with Mgr() as m:
+        1/0
+    print "you can't get there from here"
+except ZeroDivisionError, e:
+    print e

test/tests/with_file.py

+f = open('/dev/null')
+try:
+    with f:
+        1/0
+except ZeroDivisionError as e:
+    print e
+
+try:
+    f.readline()
+except ValueError as e:
+    print e

test/tests/with_functiondef.py

+def f():
+    # this exposes a bug in our irgen phase, so even `with None` bugs out here;
+    # the bug happens before actual execution. Just to test more things, though,
+    # we use an actual contextmanager here.
+    with open('/dev/null'):
+        def foo():
+            pass
+
+f()

tools/tester.py

@@ -115,6 +115,7 @@ def canonicalize_stderr(stderr):
             ("NameError: global name '", "NameError: name '"),
             ("AttributeError: '(\w+)' object attribute '(\w+)' is read-only", "AttributeError: \\2"),
             (r"TypeError: object.__new__\(\) takes no parameters", "TypeError: object() takes no parameters"),
+            ("IndexError: list assignment index out of range", "IndexError: list index out of range"),
             ]
 
     for pattern, subst_with in substitutions: