Merge branch 'master' of https://github.com/dropbox/pyston into generators

Conflicts:
	src/codegen/irgen/irgenerator.cpp
	src/codegen/runtime_hooks.h

src/analysis/fpc.h

@@ -59,7 +59,7 @@ typename BBAnalyzer<T>::AllMap computeFixedPoint(CFG* cfg, const BBAnalyzer<T>&
         CFGBlock* block = q.top();
         q.pop();
 
-        Map initial = states[block];
+        Map& initial = states[block];
         if (VERBOSITY("analysis") >= 2)
             printf("fpc on block %d - %ld entries\n", block->idx, initial.size());
 

src/analysis/function_analysis.cpp

@@ -95,6 +95,9 @@ public:
 };
 
 bool LivenessAnalysis::isLiveAtEnd(const std::string& name, CFGBlock* block) {
+    if (name[0] != '#')
+        return true;
+
     if (block->successors.size() == 0)
         return false;
 
@@ -292,19 +295,19 @@ DefinednessAnalysis::DefinednessAnalysis(const SourceInfo::ArgNames& arg_names,
             // printf("%d %s %d\n", p.first->idx, p2.first.c_str(), p2.second);
             required.insert(p2.first);
         }
-        defined.insert(make_pair(p.first, required));
+        defined_at_end.insert(make_pair(p.first, required));
     }
 }
 
-DefinednessAnalysis::DefinitionLevel DefinednessAnalysis::isDefinedAt(const std::string& name, CFGBlock* block) {
+DefinednessAnalysis::DefinitionLevel DefinednessAnalysis::isDefinedAtEnd(const std::string& name, CFGBlock* block) {
     std::unordered_map<std::string, DefinitionLevel>& map = results[block];
     if (map.count(name) == 0)
         return Undefined;
     return map[name];
 }
 
-const DefinednessAnalysis::RequiredSet& DefinednessAnalysis::getDefinedNamesAt(CFGBlock* block) {
-    return defined[block];
+const DefinednessAnalysis::RequiredSet& DefinednessAnalysis::getDefinedNamesAtEnd(CFGBlock* block) {
+    return defined_at_end[block];
 }
 
 PhiAnalysis::PhiAnalysis(const SourceInfo::ArgNames& arg_names, CFG* cfg, LivenessAnalysis* liveness,
@@ -313,19 +316,21 @@ PhiAnalysis::PhiAnalysis(const SourceInfo::ArgNames& arg_names, CFG* cfg, Livene
 
     for (CFGBlock* block : cfg->blocks) {
         RequiredSet required;
-        if (block->predecessors.size() < 2)
-            continue;
 
-        const RequiredSet& defined = definedness.getDefinedNamesAt(block);
-        if (defined.size())
-            assert(block->predecessors.size());
+        if (block->predecessors.size() > 1) {
+            for (CFGBlock* pred : block->predecessors) {
+                const RequiredSet& defined = definedness.getDefinedNamesAtEnd(pred);
                 for (const auto& s : defined) {
-            if (liveness->isLiveAtEnd(s, block->predecessors[0])) {
+                    if (required.count(s) == 0 && liveness->isLiveAtEnd(s, pred)) {
+                        // printf("%d-%d %s\n", pred->idx, block->idx, s.c_str());
+
                         required.insert(s);
                     }
                 }
+            }
+        }
 
-        required_phis.insert(make_pair(block, required));
+        required_phis.insert(make_pair(block, std::move(required)));
     }
 }
 
@@ -336,8 +341,8 @@ const PhiAnalysis::RequiredSet& PhiAnalysis::getAllRequiredAfter(CFGBlock* block
     return required_phis[block->successors[0]];
 }
 
-const PhiAnalysis::RequiredSet& PhiAnalysis::getAllDefinedAt(CFGBlock* block) {
-    return definedness.getDefinedNamesAt(block);
+const PhiAnalysis::RequiredSet& PhiAnalysis::getAllRequiredFor(CFGBlock* block) {
+    return required_phis[block];
 }
 
 bool PhiAnalysis::isRequired(const std::string& name, CFGBlock* block) {
@@ -358,11 +363,16 @@ bool PhiAnalysis::isRequiredAfter(const std::string& name, CFGBlock* block) {
 
 bool PhiAnalysis::isPotentiallyUndefinedAfter(const std::string& name, CFGBlock* block) {
     assert(!startswith(name, "!"));
-    assert(block->successors.size() > 0);
-    DefinednessAnalysis::DefinitionLevel dlevel = definedness.isDefinedAt(name, block->successors[0]);
-    ASSERT(dlevel != DefinednessAnalysis::Undefined, "%s %d", name.c_str(), block->idx);
 
-    return dlevel == DefinednessAnalysis::PotentiallyDefined;
+    if (block->successors.size() != 1)
+        return false;
+
+    for (CFGBlock* pred : block->successors[0]->predecessors) {
+        DefinednessAnalysis::DefinitionLevel dlevel = definedness.isDefinedAtEnd(name, pred);
+        if (dlevel != DefinednessAnalysis::Defined)
+            return true;
+    }
+    return false;
 }
 
 LivenessAnalysis* computeLivenessInfo(CFG*) {

src/analysis/function_analysis.h

@@ -49,14 +49,14 @@ public:
 
 private:
     std::unordered_map<CFGBlock*, std::unordered_map<std::string, DefinitionLevel> > results;
-    std::unordered_map<CFGBlock*, const RequiredSet> defined;
+    std::unordered_map<CFGBlock*, const RequiredSet> defined_at_end;
     ScopeInfo* scope_info;
 
 public:
     DefinednessAnalysis(const SourceInfo::ArgNames& args, CFG* cfg, ScopeInfo* scope_info);
 
-    DefinitionLevel isDefinedAt(const std::string& name, CFGBlock* block);
-    const RequiredSet& getDefinedNamesAt(CFGBlock* block);
+    DefinitionLevel isDefinedAtEnd(const std::string& name, CFGBlock* block);
+    const RequiredSet& getDefinedNamesAtEnd(CFGBlock* block);
 };
 class PhiAnalysis {
 public:
@@ -73,7 +73,7 @@ public:
     bool isRequired(const std::string& name, CFGBlock* block);
     bool isRequiredAfter(const std::string& name, CFGBlock* block);
     const RequiredSet& getAllRequiredAfter(CFGBlock* block);
-    const RequiredSet& getAllDefinedAt(CFGBlock* block);
+    const RequiredSet& getAllRequiredFor(CFGBlock* block);
     bool isPotentiallyUndefinedAfter(const std::string& name, CFGBlock* block);
 };
 

src/analysis/scoping_analysis.cpp

@@ -61,6 +61,8 @@ public:
 
     virtual bool takesClosure() { return false; }
 
+    bool passesThroughClosure() override { return false; }
+
     virtual bool refersToGlobal(const std::string& name) {
         if (isCompilerCreatedName(name))
             return false;
@@ -88,6 +90,7 @@ struct ScopingAnalysis::ScopeNameUsage {
     // Properties determined by looking at other scopes as well:
     StrSet referenced_from_nested;
     StrSet got_from_closure;
+    StrSet passthrough_accesses; // what names a child scope accesses a name from a parent scope
 
     ScopeNameUsage(AST* node, ScopeNameUsage* parent) : node(node), parent(parent) {
         if (node->type == AST_TYPE::ClassDef) {
@@ -123,7 +126,9 @@ public:
 
     virtual bool createsClosure() { return usage->referenced_from_nested.size() > 0; }
 
-    virtual bool takesClosure() { return usage->got_from_closure.size() > 0; }
+    virtual bool takesClosure() { return usage->got_from_closure.size() > 0 || usage->passthrough_accesses.size() > 0; }
+
+    bool passesThroughClosure() override { return usage->passthrough_accesses.size() > 0 && !createsClosure(); }
 
     virtual bool refersToGlobal(const std::string& name) {
         // HAX
@@ -235,6 +240,13 @@ public:
     virtual bool visit_jump(AST_Jump* node) { return false; }
 
 
+    virtual bool visit_delete(AST_Delete* node) {
+        for (auto t : node->targets) {
+            RELEASE_ASSERT(t->type != AST_TYPE::Name, "");
+        }
+        return false;
+    }
+
     virtual bool visit_global(AST_Global* node) {
         for (int i = 0; i < node->names.size(); i++) {
             const std::string& name = node->names[i];
@@ -364,6 +376,7 @@ void ScopingAnalysis::processNameUsages(ScopingAnalysis::NameUsageMap* usages) {
             ScopeNameUsage* parent = usage->parent;
             while (parent) {
                 if (parent->node->type == AST_TYPE::ClassDef) {
+                    intermediate_parents.push_back(parent);
                     parent = parent->parent;
                 } else if (parent->forced_globals.count(name)) {
                     break;
@@ -372,8 +385,7 @@ void ScopingAnalysis::processNameUsages(ScopingAnalysis::NameUsageMap* usages) {
                     parent->referenced_from_nested.insert(name);
 
                     for (ScopeNameUsage* iparent : intermediate_parents) {
-                        iparent->referenced_from_nested.insert(name);
-                        iparent->got_from_closure.insert(name);
+                        iparent->passthrough_accesses.insert(name);
                     }
 
                     break;

src/analysis/scoping_analysis.h

@@ -30,6 +30,7 @@ public:
 
     virtual bool createsClosure() = 0;
     virtual bool takesClosure() = 0;
+    virtual bool passesThroughClosure() = 0;
 
     virtual bool takesGenerator() { return isGeneratorValue; }
     virtual void setTakesGenerator(bool b = true) { isGeneratorValue = b; }

src/analysis/type_analysis.cpp

@@ -359,6 +359,8 @@ private:
                 return BOOL;
             case AST_LangPrimitive::LANDINGPAD:
                 return UNKNOWN;
+            case AST_LangPrimitive::LOCALS:
+                return DICT;
             default:
                 RELEASE_ASSERT(0, "%d", node->opcode);
         }
@@ -384,6 +386,10 @@ private:
             return UNKNOWN;
         }
 
+        if (scope_info->refersToClosure(node->id)) {
+            return UNKNOWN;
+        }
+
         CompilerType*& t = sym_table[node->id];
         if (t == NULL) {
             // if (VERBOSITY() >= 2) {

src/codegen/compvars.h

@@ -19,6 +19,7 @@
 #include <vector>
 
 #include "codegen/codegen.h"
+#include "core/ast.h"
 #include "core/types.h"
 
 namespace pyston {
@@ -47,6 +48,12 @@ public:
 
 typedef std::unordered_map<CompilerVariable*, CompilerVariable*> DupCache;
 
+enum BinExpType {
+    AugBinOp,
+    BinOp,
+    Compare,
+};
+
 template <class V> class _ValuedCompilerType : public CompilerType {
 public:
     typedef ValuedCompilerVariable<V> VAR;
@@ -63,11 +70,11 @@ public:
         printf("getBoxType not defined for %s\n", debugName().c_str());
         abort();
     }
-    virtual void drop(IREmitter& emmitter, VAR* value) {
+    virtual void drop(IREmitter& emmitter, VAR* var) {
         printf("drop not defined for %s\n", debugName().c_str());
         abort();
     }
-    virtual void grab(IREmitter& emmitter, VAR* value) {
+    virtual void grab(IREmitter& emmitter, VAR* var) {
         printf("grab not defined for %s\n", debugName().c_str());
         abort();
     }
@@ -75,53 +82,58 @@ public:
         printf("canConvertTo not defined for %s\n", debugName().c_str());
         abort();
     }
-    virtual ConcreteCompilerVariable* makeConverted(IREmitter& emitter, VAR* value, ConcreteCompilerType* other_type) {
+    virtual ConcreteCompilerVariable* makeConverted(IREmitter& emitter, VAR* var, ConcreteCompilerType* other_type) {
         printf("makeConverted not defined for %s\n", debugName().c_str());
         abort();
     }
-    virtual ConcreteCompilerVariable* nonzero(IREmitter& emitter, const OpInfo& info, VAR* value) {
+    virtual ConcreteCompilerVariable* nonzero(IREmitter& emitter, const OpInfo& info, VAR* var) {
         printf("nonzero not defined for %s\n", debugName().c_str());
         abort();
     }
-    virtual CompilerVariable* getattr(IREmitter& emitter, const OpInfo& info, VAR* value, const std::string* attr,
+    virtual CompilerVariable* getattr(IREmitter& emitter, const OpInfo& info, VAR* var, const std::string* attr,
                                       bool cls_only) {
         printf("getattr not defined for %s\n", debugName().c_str());
         abort();
     }
-    virtual void setattr(IREmitter& emitter, const OpInfo& info, VAR* value, const std::string* attr,
+    virtual void setattr(IREmitter& emitter, const OpInfo& info, VAR* var, const std::string* attr,
                          CompilerVariable* v) {
         printf("setattr not defined for %s\n", debugName().c_str());
         abort();
     }
-    virtual CompilerVariable* callattr(IREmitter& emitter, const OpInfo& info, VAR* value, const std::string* attr,
+    virtual CompilerVariable* callattr(IREmitter& emitter, const OpInfo& info, VAR* var, const std::string* attr,
                                        bool clsonly, struct ArgPassSpec argspec,
                                        const std::vector<CompilerVariable*>& args,
                                        const std::vector<const std::string*>* keyword_names) {
         printf("callattr not defined for %s\n", debugName().c_str());
         abort();
     }
-    virtual CompilerVariable* call(IREmitter& emitter, const OpInfo& info, VAR* value, struct ArgPassSpec argspec,
+    virtual CompilerVariable* call(IREmitter& emitter, const OpInfo& info, VAR* var, struct ArgPassSpec argspec,
                                    const std::vector<CompilerVariable*>& args,
                                    const std::vector<const std::string*>* keyword_names) {
         printf("call not defined for %s\n", debugName().c_str());
         abort();
     }
-    virtual void print(IREmitter& emitter, const OpInfo& info, VAR* value) {
+    virtual void print(IREmitter& emitter, const OpInfo& info, VAR* var) {
         printf("print not defined for %s\n", debugName().c_str());
         abort();
     }
-    virtual ConcreteCompilerVariable* len(IREmitter& emitter, const OpInfo& info, VAR* value) {
+    virtual ConcreteCompilerVariable* len(IREmitter& emitter, const OpInfo& info, VAR* var) {
         printf("len not defined for %s\n", debugName().c_str());
         abort();
     }
-    virtual CompilerVariable* getitem(IREmitter& emitter, const OpInfo& info, VAR* value, CompilerVariable* v) {
+    virtual CompilerVariable* getitem(IREmitter& emitter, const OpInfo& info, VAR* var, CompilerVariable* v) {
         // Can almost do this, except for error messages + types:
         // static const std::string attr("__getitem__");
-        // return callattr(emitter, info, value, &attr, true, ArgPassSpec(1, 0, 0, 0), {v}, NULL);
+        // return callattr(emitter, info, var, &attr, true, ArgPassSpec(1, 0, 0, 0), {v}, NULL);
         printf("getitem not defined for %s\n", debugName().c_str());
         abort();
     }
-    virtual llvm::Value* makeClassCheck(IREmitter& emitter, VAR* value, BoxedClass* c) {
+    virtual CompilerVariable* binexp(IREmitter& emitter, const OpInfo& info, VAR* var, CompilerVariable* rhs,
+                                     AST_TYPE::AST_TYPE op_type, BinExpType exp_type) {
+        printf("binexp not defined for %s\n", debugName().c_str());
+        abort();
+    }
+    virtual llvm::Value* makeClassCheck(IREmitter& emitter, VAR* var, BoxedClass* c) {
         printf("makeClassCheck not defined for %s\n", debugName().c_str());
         abort();
     }
@@ -189,7 +201,7 @@ public:
         }
     }
     void decvref(IREmitter& emitter) {
-        assert(vrefs > 0 && vrefs < (1 << 20));
+        ASSERT(vrefs > 0 && vrefs < (1 << 20), "%d", vrefs);
         // ASSERT(vrefs, "%s", getType()->debugName().c_str());
         vrefs--;
         if (vrefs == 0) {
@@ -229,6 +241,8 @@ public:
     virtual void print(IREmitter& emitter, const OpInfo& info) = 0;
     virtual ConcreteCompilerVariable* len(IREmitter& emitter, const OpInfo& info) = 0;
     virtual CompilerVariable* getitem(IREmitter& emitter, const OpInfo& info, CompilerVariable*) = 0;
+    virtual CompilerVariable* binexp(IREmitter& emitter, const OpInfo& info, CompilerVariable* rhs,
+                                     AST_TYPE::AST_TYPE op_type, BinExpType exp_type) = 0;
 };
 
 template <class V> class ValuedCompilerVariable : public CompilerVariable {
@@ -299,6 +313,12 @@ public:
     CompilerVariable* getitem(IREmitter& emitter, const OpInfo& info, CompilerVariable* slice) override {
         return type->getitem(emitter, info, this, slice);
     }
+
+    CompilerVariable* binexp(IREmitter& emitter, const OpInfo& info, CompilerVariable* rhs, AST_TYPE::AST_TYPE op_type,
+                             BinExpType exp_type) override {
+        return type->binexp(emitter, info, this, rhs, op_type, exp_type);
+    }
+
     llvm::Value* makeClassCheck(IREmitter& emitter, BoxedClass* cls) override {
         return type->makeClassCheck(emitter, this, cls);
     }

src/codegen/irgen.cpp

@@ -337,13 +337,18 @@ static void emitBBs(IRGenState* irstate, const char* bb_type, GuardList& out_gua
             arg_num++;
             if (arg_num < 3) {
                 from_arg = func_args[arg_num];
+                assert(from_arg->getType() == p.second->llvmType());
             } else {
                 ASSERT(func_args.size() == 4, "%ld", func_args.size());
                 llvm::Value* ptr = entry_emitter->getBuilder()->CreateConstGEP1_32(func_args[3], arg_num - 3);
                 if (p.second == INT) {
                     ptr = entry_emitter->getBuilder()->CreateBitCast(ptr, g.i64->getPointerTo());
+                } else if (p.second == BOOL) {
+                    ptr = entry_emitter->getBuilder()->CreateBitCast(ptr, g.i1->getPointerTo());
                 } else if (p.second == FLOAT) {
                     ptr = entry_emitter->getBuilder()->CreateBitCast(ptr, g.double_->getPointerTo());
+                } else {
+                    assert(p.second->llvmType() == g.llvm_value_type_ptr);
                 }
                 from_arg = entry_emitter->getBuilder()->CreateLoad(ptr);
                 assert(from_arg->getType() == p.second->llvmType());
@@ -599,12 +604,8 @@ static void emitBBs(IRGenState* irstate, const char* bb_type, GuardList& out_gua
                 into_hax.insert(b2);
             }
 
-            const PhiAnalysis::RequiredSet& names = source->phis->getAllDefinedAt(block);
+            const PhiAnalysis::RequiredSet& names = source->phis->getAllRequiredFor(block);
             for (const auto& s : names) {
-                // TODO the list from getAllDefinedAt should come filtered:
-                if (!source->liveness->isLiveAtEnd(s, block->predecessors[0]))
-                    continue;
-
                 // printf("adding guessed phi for %s\n", s.c_str());
                 ConcreteCompilerType* type = types->getTypeAtBlockStart(s, block);
                 llvm::PHINode* phi = emitter->getBuilder()->CreatePHI(type->llvmType(), block->predecessors.size(), s);

src/codegen/llvm_interpreter.cpp

@@ -102,6 +102,12 @@ Val fetch(llvm::Value* v, const llvm::DataLayout& dl, const SymMap& symbols) {
         case llvm::Value::ConstantExprVal: {
             llvm::ConstantExpr* ce = llvm::cast<llvm::ConstantExpr>(v);
             if (ce->isCast()) {
+                if (ce->getOpcode() == llvm::Instruction::IntToPtr && ce->getOperand(0)->getType() == g.i1) {
+                    // inttoptr is specified to zero-extend
+                    Val o = fetch(ce->getOperand(0), dl, symbols);
+                    return o.n & 0x1;
+                }
+
                 assert(width(ce->getOperand(0), dl) == 8 && width(ce, dl) == 8);
 
                 Val o = fetch(ce->getOperand(0), dl, symbols);
@@ -283,7 +289,7 @@ Box* interpretFunction(llvm::Function* f, int nargs, Box* closure, Box* generato
     int arg_num = -1;
     int closure_indicator = closure ? 1 : 0;
     int generator_indicator = generator ? 1 : 0;
-    int argOffset = closure_indicator + generator_indicator;
+    int arg_offset = closure_indicator + generator_indicator;
     for (llvm::Argument& arg : f->args()) {
         arg_num++;
 
@@ -291,15 +297,15 @@ Box* interpretFunction(llvm::Function* f, int nargs, Box* closure, Box* generato
             symbols.insert(std::make_pair(static_cast<llvm::Value*>(&arg), Val(closure)));
         else if ((arg_num == 0 || (arg_num == 1 && closure)) && generator)
             symbols.insert(std::make_pair(static_cast<llvm::Value*>(&arg), Val(generator)));
-        else if (arg_num == 0 + argOffset)
+        else if (arg_num == 0 + arg_offset)
             symbols.insert(std::make_pair(static_cast<llvm::Value*>(&arg), Val(arg1)));
-        else if (arg_num == 1 + argOffset)
+        else if (arg_num == 1 + arg_offset)
             symbols.insert(std::make_pair(static_cast<llvm::Value*>(&arg), Val(arg2)));
-        else if (arg_num == 2 + argOffset)
+        else if (arg_num == 2 + arg_offset)
             symbols.insert(std::make_pair(static_cast<llvm::Value*>(&arg), Val(arg3)));
         else {
-            assert(arg_num == 3 + argOffset);
-            assert(f->getArgumentList().size() == 4 + argOffset);
+            assert(arg_num == 3 + arg_offset);
+            assert(f->getArgumentList().size() == 4 + arg_offset);
             assert(f->getArgumentList().back().getType() == g.llvm_value_type_ptr->getPointerTo());
             symbols.insert(std::make_pair(static_cast<llvm::Value*>(&arg), Val((int64_t)args)));
             // printf("loading %%4 with %p\n", (void*)args);
@@ -310,6 +316,14 @@ Box* interpretFunction(llvm::Function* f, int nargs, Box* closure, Box* generato
     llvm::BasicBlock* prevblock = NULL;
     llvm::BasicBlock* curblock = &f->getEntryBlock();
 
+    // The symbol table at the end of the previous BB
+    // This is important for the following case:
+    // %a = phi [0, %l1], [1, %l2]
+    // %b = phi [0, %l1], [%a, %l2]
+    // The reference to %a in the definition of %b resolves to the *previous* value of %a,
+    // not the value of %a that we just set in the phi.
+    SymMap prev_symbols;
+
     struct {
         Box* exc_obj;
         int64_t exc_selector;
@@ -521,8 +535,12 @@ Box* interpretFunction(llvm::Function* f, int nargs, Box* closure, Box* generato
                 SET(fetch(bc->getOperand(0), dl, symbols));
                 continue;
             } else if (llvm::IntToPtrInst* bc = llvm::dyn_cast<llvm::IntToPtrInst>(inst)) {
+                if (bc->getOperand(0)->getType() == g.i1) {
+                    SET(fetch(bc->getOperand(0), dl, symbols).n & 0xff);
+                } else {
                     assert(width(bc->getOperand(0), dl) == 8);
                     SET(fetch(bc->getOperand(0), dl, symbols));
+                }
                 continue;
                 //} else if (llvm::CallInst* ci = llvm::dyn_cast<llvm::CallInst>(inst)) {
             } else if (llvm::isa<llvm::CallInst>(inst) || llvm::isa<llvm::InvokeInst>(inst)) {
@@ -615,14 +633,18 @@ Box* interpretFunction(llvm::Function* f, int nargs, Box* closure, Box* generato
                             r = reinterpret_cast<int64_t (*)(int64_t, double, double)>(f)(args[0].n, args[1].d,
                                                                                           args[2].d);
                             break;
-                        case 0b100000:
+                        case 0b100000: // 32
                             r = reinterpret_cast<int64_t (*)(int64_t, int64_t, int64_t, int64_t)>(f)(
                                 args[0].n, args[1].n, args[2].n, args[3].n);
                             break;
-                        case 0b100001:
+                        case 0b100001: // 33
                             r = reinterpret_cast<int64_t (*)(int64_t, int64_t, int64_t, double)>(f)(
                                 args[0].n, args[1].n, args[2].n, args[3].d);
                             break;
+                        case 0b100010: // 34
+                            r = reinterpret_cast<int64_t (*)(int64_t, int64_t, double, int64_t)>(f)(
+                                args[0].n, args[1].n, args[2].d, args[3].n);
+                            break;
                         case 0b100110:
                             r = reinterpret_cast<int64_t (*)(int64_t, double, double, int64_t)>(f)(
                                 args[0].n, args[1].d, args[2].d, args[3].n);
@@ -631,15 +653,15 @@ Box* interpretFunction(llvm::Function* f, int nargs, Box* closure, Box* generato
                             r = reinterpret_cast<int64_t (*)(double, int, double, int64_t)>(f)(args[0].d, args[1].n,
                                                                                                args[2].d, args[3].n);
                             break;
-                        case 0b1000000:
+                        case 0b1000000: // 64
                             r = reinterpret_cast<int64_t (*)(int64_t, int64_t, int64_t, int64_t, int64_t)>(f)(
                                 args[0].n, args[1].n, args[2].n, args[3].n, args[4].n);
                             break;
-                        case 0b10000000:
+                        case 0b10000000: // 128
                             r = reinterpret_cast<int64_t (*)(int64_t, int64_t, int64_t, int64_t, int64_t, int64_t)>(f)(
                                 args[0].n, args[1].n, args[2].n, args[3].n, args[4].n, args[5].n);
                             break;
-                        case 0b100000000:
+                        case 0b100000000: // 256
                             r = reinterpret_cast<int64_t (*)(int64_t, int64_t, int64_t, int64_t, int64_t, int64_t,
                                                              int64_t)>(f)(args[0].n, args[1].n, args[2].n, args[3].n,
                                                                           args[4].n, args[5].n, args[6].n);
@@ -667,6 +689,7 @@ Box* interpretFunction(llvm::Function* f, int nargs, Box* closure, Box* generato
                     if (invoke != nullptr) {
                         prevblock = curblock;
                         curblock = invoke->getNormalDest();
+                        prev_symbols = symbols;
                     }
                 } catch (Box* e) {
                     if (VERBOSITY("interpreter") >= 2) {
@@ -678,6 +701,7 @@ Box* interpretFunction(llvm::Function* f, int nargs, Box* closure, Box* generato
 
                     prevblock = curblock;
                     curblock = invoke->getUnwindDest();
+                    prev_symbols = symbols;
 
                     landingpad_value.exc_obj = e;
                     landingpad_value.exc_selector
@@ -700,7 +724,7 @@ Box* interpretFunction(llvm::Function* f, int nargs, Box* closure, Box* generato
                 continue;
             } else if (llvm::PHINode* phi = llvm::dyn_cast<llvm::PHINode>(inst)) {
                 assert(prevblock);
-                SET(fetch(phi->getIncomingValueForBlock(prevblock), dl, symbols));
+                SET(fetch(phi->getIncomingValueForBlock(prevblock), dl, prev_symbols));
                 continue;
             } else if (llvm::BranchInst* br = llvm::dyn_cast<llvm::BranchInst>(inst)) {
                 prevblock = curblock;
@@ -714,6 +738,7 @@ Box* interpretFunction(llvm::Function* f, int nargs, Box* closure, Box* generato
                 } else {
                     curblock = br->getSuccessor(0);
                 }
+                prev_symbols = symbols;
                 // if (VERBOSITY()) {
                 // printf("jumped to %s\n", curblock->getName().data());
                 //}

src/codegen/parser.cpp

@@ -881,7 +881,7 @@ AST_Module* parse(const char* fn) {
 
 #define MAGIC_STRING "a\nch"
 #define MAGIC_STRING_LENGTH 4
-#define LENGTH_SUFFIX_LENGTH 4
+#define CHECKSUM_LENGTH 4
 
 static void _reparse(const char* fn, const std::string& cache_fn) {
     FILE* parser = popen(getParserCommandLine(fn).c_str(), "r");
@@ -890,7 +890,14 @@ static void _reparse(const char* fn, const std::string& cache_fn) {
 
     fwrite(MAGIC_STRING, 1, MAGIC_STRING_LENGTH, cache_fp);
 
-    int bytes_written = 0;
+    int checksum_start = ftell(cache_fp);
+
+    int bytes_written = -1;
+    // Currently just use the length as the checksum
+    static_assert(sizeof(bytes_written) >= CHECKSUM_LENGTH, "");
+    fwrite(&bytes_written, 1, CHECKSUM_LENGTH, cache_fp);
+
+    bytes_written = 0;
     char buf[80];
     while (true) {
         int nread = fread(buf, 1, 80, parser);
@@ -900,13 +907,12 @@ static void _reparse(const char* fn, const std::string& cache_fn) {
         fwrite(buf, 1, nread, cache_fp);
     }
 
-    // Ideally this should be a full checksum rather than just a length.
-    // And maybe it should be put at the beginning?
-    static_assert(sizeof(bytes_written) >= LENGTH_SUFFIX_LENGTH, "");
-    fwrite(&bytes_written, 1, LENGTH_SUFFIX_LENGTH, cache_fp);
-
     int code = pclose(parser);
     assert(code == 0);
+
+    fseek(cache_fp, checksum_start, SEEK_SET);
+    fwrite(&bytes_written, 1, CHECKSUM_LENGTH, cache_fp);
+
     fclose(cache_fp);
 }
 
@@ -940,20 +946,25 @@ AST_Module* caching_parse(const char* fn) {
             char buf[MAGIC_STRING_LENGTH];
             int read = fread(buf, 1, MAGIC_STRING_LENGTH, fp);
             if (read != MAGIC_STRING_LENGTH || strncmp(buf, MAGIC_STRING, MAGIC_STRING_LENGTH) != 0) {
+                if (VERBOSITY()) {
+                    printf("Warning: corrupt or non-Pyston .pyc file found; ignoring\n");
+                }
                 good = false;
             }
         }
 
         if (good) {
             int length = 0;
-            fseek(fp, -LENGTH_SUFFIX_LENGTH, SEEK_END);
-            static_assert(sizeof(length) >= LENGTH_SUFFIX_LENGTH, "");
-            int read = fread(&length, 1, LENGTH_SUFFIX_LENGTH, fp);
             fseek(fp, MAGIC_STRING_LENGTH, SEEK_SET);
+            static_assert(sizeof(length) >= CHECKSUM_LENGTH, "");
+            int read = fread(&length, 1, CHECKSUM_LENGTH, fp);
 
-            int expected_length = MAGIC_STRING_LENGTH + LENGTH_SUFFIX_LENGTH + length;
+            int expected_total_length = MAGIC_STRING_LENGTH + CHECKSUM_LENGTH + length;
 
-            if (read != LENGTH_SUFFIX_LENGTH || expected_length != cache_stat.st_size) {
+            if (read != CHECKSUM_LENGTH || expected_total_length != cache_stat.st_size) {
+                if (VERBOSITY()) {
+                    printf("Warning: truncated .pyc file found; ignoring\n");
+                }
                 good = false;
             }
         }
@@ -974,7 +985,7 @@ AST_Module* caching_parse(const char* fn) {
     BufferedReader* reader = new BufferedReader(fp);
     AST* rtn = readASTMisc(reader);
     reader->fill();
-    assert(reader->bytesBuffered() == LENGTH_SUFFIX_LENGTH);
+    assert(reader->bytesBuffered() == 0);
     delete reader;
 
     assert(rtn->type == AST_TYPE::Module);

src/codegen/runtime_hooks.cpp

@@ -42,8 +42,6 @@
 
 extern "C" void* __cxa_begin_catch(void*);
 extern "C" void __cxa_end_catch();
-extern "C" void* __cxa_allocate_exception(size_t);
-extern "C" void __cxa_throw(void*, void*, void (*)(void*));
 
 namespace pyston {
 
@@ -135,9 +133,11 @@ void initGlobalFuncs(GlobalState& g) {
 
     g.llvm_str_type_ptr = lookupFunction("boxStringPtr")->arg_begin()->getType();
 
-    auto vector_type = g.stdlib_module->getTypeByName("class.std::vector");
-    assert(vector_type);
-    g.vector_ptr = vector_type->getPointerTo();
+    // The LLVM vector type for the arguments that we pass to runtimeCall and related functions.
+    // It will be a pointer to a type named something like class.std::vector or
+    // class.std::vector.##. We can figure out exactly what it is by looking at the last
+    // argument of runtimeCall.
+    g.vector_ptr = (--lookupFunction("runtimeCall")->getArgumentList().end())->getType();
 
     g.llvm_closure_type_ptr = g.stdlib_module->getTypeByName("class.pyston::BoxedClosure")->getPointerTo();
     assert(g.llvm_closure_type_ptr);
@@ -187,6 +187,7 @@ void initGlobalFuncs(GlobalState& g) {
     GET(getclsattr);
     GET(unaryop);
     GET(import);
+    GET(importFrom);
     GET(repr);
     GET(isinstance);
     GET(yield);
@@ -225,8 +226,8 @@ void initGlobalFuncs(GlobalState& g) {
 
     GET(__cxa_begin_catch);
     g.funcs.__cxa_end_catch = addFunc((void*)__cxa_end_catch, g.void_);
-    g.funcs.__cxa_allocate_exception = addFunc((void*)__cxa_allocate_exception, g.i8_ptr, g.i64);
-    g.funcs.__cxa_throw = addFunc((void*)__cxa_throw, g.void_, g.i8_ptr, g.i8_ptr, g.i8_ptr);
+    GET(raise0);
+    GET(raise1);
 
     g.funcs.div_i64_i64 = getFunc((void*)div_i64_i64, "div_i64_i64");
     g.funcs.mod_i64_i64 = getFunc((void*)mod_i64_i64, "mod_i64_i64");

src/codegen/runtime_hooks.h

@@ -34,7 +34,7 @@ struct GlobalFuncs {
         *boxInstanceMethod, *boxBool, *unboxBool, *createTuple, *createDict, *createList, *createSlice,
         *createUserClass, *createClosure, *createGenerator, *insideGenerator;
     llvm::Value* getattr, *setattr, *print, *nonzero, *binop, *compare, *augbinop, *unboxedLen, *getitem, *getclsattr,
-        *yield, *getGlobal, *setitem, *delitem, *unaryop, *import, *repr, *isinstance;
+        *getGlobal, *setitem, *delitem, *unaryop, *import, *importFrom, *repr, *isinstance, *yield;
 
     llvm::Value* checkUnpackingLength, *raiseAttributeError, *raiseAttributeErrorStr, *raiseNotIterableError,
         *assertNameDefined, *assertFail;
@@ -43,7 +43,8 @@ struct GlobalFuncs {
     llvm::Value* callattr0, *callattr1, *callattr2, *callattr3, *callattr;
     llvm::Value* reoptCompiledFunc, *compilePartialFunc;
 
-    llvm::Value* __cxa_begin_catch, *__cxa_end_catch, *__cxa_allocate_exception, *__cxa_throw;
+    llvm::Value* __cxa_begin_catch, *__cxa_end_catch;
+    llvm::Value* raise0, *raise1;
 
     llvm::Value* div_i64_i64, *mod_i64_i64, *pow_i64_i64;
     llvm::Value* div_float_float, *mod_float_float, *pow_float_float;

src/core/ast.cpp

@@ -1314,6 +1314,9 @@ bool PrintVisitor::visit_langprimitive(AST_LangPrimitive* node) {
         case AST_LangPrimitive::LANDINGPAD:
             printf("landingpad");
             break;
+        case AST_LangPrimitive::LOCALS:
+            printf("locals");
+            break;
         default:
             RELEASE_ASSERT(0, "%d", node->opcode);
     }

src/core/ast.h

@@ -879,6 +879,7 @@ public:
     enum Opcodes {
         ISINSTANCE,
         LANDINGPAD,
+        LOCALS,
     } opcode;
     std::vector<AST_expr*> args;
 

src/core/cfg.cpp

@@ -379,6 +379,8 @@ private:
     AST_expr* remapAttribute(AST_Attribute* node) {
         AST_Attribute* rtn = new AST_Attribute();
 
+        rtn->col_offset = node->col_offset;
+        rtn->lineno = node->lineno;
         rtn->ctx_type = node->ctx_type;
         rtn->attr = node->attr;
         rtn->value = remapExpr(node->value);
@@ -483,6 +485,8 @@ private:
     AST_expr* remapClsAttribute(AST_ClsAttribute* node) {
         AST_ClsAttribute* rtn = new AST_ClsAttribute();
 
+        rtn->col_offset = node->col_offset;
+        rtn->lineno = node->lineno;
         rtn->attr = node->attr;
         rtn->value = remapExpr(node->value);
         return rtn;
@@ -622,28 +626,20 @@ private:
     }
 
     AST_expr* remapLambda(AST_Lambda* node) {
-        if (node->args->defaults.empty()) {
-            return node;
-        }
+        // Remap in place: see note in visit_functiondef for why.
 
-        AST_Lambda* rtn = new AST_Lambda();
-        rtn->lineno = node->lineno;
-        rtn->col_offset = node->col_offset;
-
-        rtn->args = new AST_arguments();
-        rtn->args->args = node->args->args;
-        rtn->args->vararg = node->args->vararg;
-        rtn->args->kwarg = node->args->kwarg;
-        for (auto d : node->args->defaults) {
-            rtn->args->defaults.push_back(remapExpr(d));
+        for (int i = 0; i < node->args->defaults.size(); i++) {
+            node->args->defaults[i] = remapExpr(node->args->defaults[i]);
         }
 
-        rtn->body = node->body;
-        return rtn;
+        return node;
     }
 
     AST_expr* remapLangPrimitive(AST_LangPrimitive* node) {
         AST_LangPrimitive* rtn = new AST_LangPrimitive(node->opcode);
+        rtn->col_offset = node->col_offset;
+        rtn->lineno = node->lineno;
+
         for (AST_expr* arg : node->args) {
             rtn->args.push_back(remapExpr(arg));
         }
@@ -859,6 +855,8 @@ public:
         AST_Invoke* invoke = new AST_Invoke(node);
         invoke->normal_dest = normal_dest;
         invoke->exc_dest = exc_dest;
+        invoke->col_offset = node->col_offset;
+        invoke->lineno = node->lineno;
 
         curblock->push_back(invoke);
         curblock->connectTo(normal_dest);
@@ -878,36 +876,42 @@ public:
     }
 
     virtual bool visit_classdef(AST_ClassDef* node) {
+        // Remap in place: see note in visit_functiondef for why.
+
+        // Decorators are evaluated before the defaults:
+        for (int i = 0; i < node->decorator_list.size(); i++) {
+            node->decorator_list[i] = remapExpr(node->decorator_list[i]);
+        }
+
+        for (int i = 0; i < node->bases.size(); i++) {
+            node->bases[i] = remapExpr(node->bases[i]);
+        }
+
         push_back(node);
         return true;
     }
 
     virtual bool visit_functiondef(AST_FunctionDef* node) {
-        if (node->args->defaults.size() == 0 && node->decorator_list.size() == 0) {
-            push_back(node);
-        } else {
-            AST_FunctionDef* remapped = new AST_FunctionDef();
-
-            remapped->name = node->name;
-            remapped->lineno = node->lineno;
-            remapped->col_offset = node->col_offset;
-            remapped->args = new AST_arguments();
-            remapped->body = node->body; // hmm shouldnt have to copy this
+        // As much as I don't like it, for now we're remapping these in place.
+        // This is because we do certain analyses pre-remapping, and associate the
+        // results with the node.  We can either do some refactoring and have a way
+        // of associating the new node with the same results, or just do the remapping
+        // in-place.
+        // Doing it in-place seems ugly, but I can't think of anything it should break,
+        // so just do that for now.
+        // TODO If we remap these (functiondefs, lambdas, classdefs) in place, we should probably
+        // remap everything in place?
 
         // Decorators are evaluated before the defaults:
-            for (auto d : node->decorator_list) {
-                remapped->decorator_list.push_back(remapExpr(d));
+        for (int i = 0; i < node->decorator_list.size(); i++) {
+            node->decorator_list[i] = remapExpr(node->decorator_list[i]);
         }
 
-            remapped->args->args = node->args->args;
-            remapped->args->vararg = node->args->vararg;
-            remapped->args->kwarg = node->args->kwarg;
-            for (auto d : node->args->defaults) {
-                remapped->args->defaults.push_back(remapExpr(d));
+        for (int i = 0; i < node->args->defaults.size(); i++) {
+            node->args->defaults[i] = remapExpr(node->args->defaults[i]);
         }
 
-            push_back(remapped);
-        }
+        push_back(node);
         return true;
     }
 
@@ -1027,11 +1031,15 @@ public:
                 s_target->value = remapExpr(s->value);
                 s_target->slice = remapExpr(s->slice);
                 s_target->ctx_type = AST_TYPE::Store;
+                s_target->col_offset = s->col_offset;
+                s_target->lineno = s->lineno;
                 remapped_target = s_target;
 
                 AST_Subscript* s_lhs = new AST_Subscript();
                 s_lhs->value = s_target->value;
                 s_lhs->slice = s_target->slice;
+                s_lhs->col_offset = s->col_offset;
+                s_lhs->lineno = s->lineno;
                 s_lhs->ctx_type = AST_TYPE::Load;
                 remapped_lhs = remapExpr(s_lhs);
 
@@ -1045,12 +1053,16 @@ public:
                 a_target->value = remapExpr(a->value);
                 a_target->attr = a->attr;
                 a_target->ctx_type = AST_TYPE::Store;
+                a_target->col_offset = a->col_offset;
+                a_target->lineno = a->lineno;
                 remapped_target = a_target;
 
                 AST_Attribute* a_lhs = new AST_Attribute();
                 a_lhs->value = a_target->value;
                 a_lhs->attr = a->attr;
                 a_lhs->ctx_type = AST_TYPE::Load;
+                a_lhs->col_offset = a->col_offset;
+                a_lhs->lineno = a->lineno;
                 remapped_lhs = remapExpr(a_lhs);
 
                 break;
@@ -1063,6 +1075,8 @@ public:
         binop->op_type = node->op_type;
         binop->left = remapped_lhs;
         binop->right = remapExpr(node->value);
+        binop->col_offset = node->col_offset;
+        binop->lineno = node->lineno;
         AST_stmt* assign = makeAssign(remapped_target, binop);
         push_back(assign);
         return true;
@@ -1109,6 +1123,7 @@ public:
         int i = 0;
         for (auto v : node->values) {
             AST_Print* remapped = new AST_Print();
+            remapped->col_offset = node->col_offset;
             remapped->lineno = node->lineno;
             // TODO not good to reuse 'dest' like this
             remapped->dest = dest;
@@ -1128,6 +1143,8 @@ public:
             assert(node->nl);
 
             AST_Print* final = new AST_Print();
+            final->col_offset = node->col_offset;
+            final->lineno = node->lineno;
             // TODO not good to reuse 'dest' like this
             final->dest = dest;
             final->nl = node->nl;
@@ -1417,6 +1434,9 @@ public:
 
     bool visit_raise(AST_Raise* node) override {
         AST_Raise* remapped = new AST_Raise();
+        remapped->col_offset = node->col_offset;
+        remapped->lineno = node->lineno;
+
         if (node->arg0)
             remapped->arg0 = remapExpr(node->arg0);
         if (node->arg1)
@@ -1425,6 +1445,9 @@ public:
             remapped->arg2 = remapExpr(node->arg2);
         push_back(remapped);
 
+        if (!curblock)
+            return true;
+
         curblock->push_back(new AST_Unreachable());
         curblock = NULL;
 
@@ -1518,7 +1541,6 @@ public:
 
             if (!caught_all) {
                 AST_Raise* raise = new AST_Raise();
-                raise->arg0 = exc_obj;
                 push_back(raise);
                 curblock->push_back(new AST_Unreachable());
                 curblock = NULL;
@@ -1709,20 +1731,10 @@ CFG* computeCFG(SourceInfo* source, std::vector<AST_stmt*> body) {
     if (source->ast->type == AST_TYPE::ClassDef) {
         ScopeInfo* scope_info = source->scoping->getScopeInfoForNode(source->ast);
 
-        auto written_names = scope_info->getClassDefLocalNames();
-        AST_Dict* rtn_dict = new AST_Dict();
-
-        // Even if the user never explicitly wrote to __module__, there was an
-        // implicit write:
-        assert(written_names.count("__module__"));
-
-        for (auto s : written_names) {
-            rtn_dict->keys.push_back(new AST_Str(s));
-            rtn_dict->values.push_back(makeName(s, AST_TYPE::Load));
-        }
+        AST_LangPrimitive* locals = new AST_LangPrimitive(AST_LangPrimitive::LOCALS);
 
         AST_Return* rtn = new AST_Return();
-        rtn->value = rtn_dict;
+        rtn->value = locals;
         visitor.push_back(rtn);
     } else {
         // Put a fake "return" statement at the end of every function just to make sure they all have one;

src/gc/collector.cpp

@@ -42,6 +42,18 @@ void registerStaticRootObj(void* obj) {
     roots.push(obj);
 }
 
+static std::unordered_set<StaticRootHandle*>* getRootHandles() {
+    static std::unordered_set<StaticRootHandle*> root_handles;
+    return &root_handles;
+}
+
+StaticRootHandle::StaticRootHandle() {
+    getRootHandles()->insert(this);
+}
+StaticRootHandle::~StaticRootHandle() {
+    getRootHandles()->erase(this);
+}
+
 bool TraceStackGCVisitor::isValid(void* p) {
     return global_heap.getAllocationFromInteriorPointer(p);
 }
@@ -105,6 +117,10 @@ static void markPhase() {
 
     TraceStackGCVisitor visitor(&stack);
 
+    for (auto h : *getRootHandles()) {
+        visitor.visitPotential(h->value);
+    }
+
     // if (VERBOSITY()) printf("Found %d roots\n", stack.size());
     while (void* p = stack.pop()) {
         assert(((intptr_t)p) % 8 == 0);

src/gc/collector.h

@@ -83,6 +83,19 @@ public:
 // (that should be registerStaticRootPtr)
 void registerStaticRootObj(void* root_obj);
 void runCollection();
+
+// If you want to have a static root "location" where multiple values could be stored, use this:
+class StaticRootHandle {
+public:
+    Box* value;
+
+    StaticRootHandle();
+    ~StaticRootHandle();
+
+    void operator=(Box* b) { value = b; }
+
+    operator Box*() { return value; }
+};
 }
 }
 

src/runtime/builtin_modules/errno.cpp

+// Copyright (c) 2014 Dropbox, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <cmath>
+
+#include "core/types.h"
+#include "gc/collector.h"
+#include "runtime/gc_runtime.h"
+#include "runtime/inline/boxing.h"
+#include "runtime/types.h"
+#include "runtime/util.h"
+
+namespace pyston {
+
+BoxedModule* errno_module;
+
+void setupErrno() {
+    errno_module = createModule("errno", "__builtin__");
+}
+}

src/runtime/builtin_modules/posix.cpp

+// Copyright (c) 2014 Dropbox, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <cmath>
+
+#include "core/types.h"
+#include "gc/collector.h"
+#include "runtime/gc_runtime.h"
+#include "runtime/inline/boxing.h"
+#include "runtime/types.h"
+#include "runtime/util.h"
+
+namespace pyston {
+
+BoxedModule* posix_module;
+
+void setupPosix() {
+    posix_module = createModule("posix", "__builtin__");
+}
+}

src/runtime/builtin_modules/sys.cpp

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include <algorithm>
 #include <cmath>
 
 #include "core/types.h"
@@ -87,4 +88,16 @@ void setupSys() {
     sys_module->giveAttr("stdin", new BoxedFile(stdin));
     sys_module->giveAttr("stderr", new BoxedFile(stderr));
 }
+
+void setupSysEnd() {
+    BoxedTuple::GCVector builtin_module_names;
+    for (auto& p : sys_modules_dict->d) {
+        builtin_module_names.push_back(p.first);
+    }
+
+    std::sort<decltype(builtin_module_names)::iterator, PyLt>(builtin_module_names.begin(), builtin_module_names.end(),
+                                                              PyLt());
+
+    sys_module->giveAttr("builtin_module_names", new BoxedTuple(std::move(builtin_module_names)));
+}
 }

src/runtime/inline/link_forcer.cpp

@@ -76,6 +76,7 @@ void force() {
     FORCE(delitem);
     FORCE(unaryop);
     FORCE(import);
+    FORCE(importFrom);
     FORCE(repr);
     FORCE(isinstance);
     FORCE(yield);
@@ -93,6 +94,9 @@ void force() {
     FORCE(runtimeCall);
     FORCE(callattr);
 
+    FORCE(raise0);
+    FORCE(raise1);
+
     FORCE(div_i64_i64);
     FORCE(mod_i64_i64);
     FORCE(pow_i64_i64);

src/runtime/list.cpp

@@ -545,6 +545,7 @@ void setupList() {
     list_cls->giveAttr("pop", new BoxedFunction(boxRTFunction((void*)listPop, UNKNOWN, 2, 1, false, false), { None }));
 
     list_cls->giveAttr("append", new BoxedFunction(boxRTFunction((void*)listAppend, NONE, 2)));
+    list_cls->giveAttr("extend", new BoxedFunction(boxRTFunction((void*)listIAdd, NONE, 2)));
 
     CLFunction* setitem = createRTFunction(3, 0, false, false);
     addRTFunction(setitem, (void*)listSetitemInt, NONE, std::vector<ConcreteCompilerType*>{ LIST, BOXED_INT, UNKNOWN });

src/runtime/objmodel.cpp

@@ -1772,7 +1772,7 @@ Box* callFunc(BoxedFunction* func, CallRewriteArgs* rewrite_args, ArgPassSpec ar
             rewrite_args->rewriter->loadConst(0, (intptr_t)closure);
 
         if (func->isGenerator)
-            rewrite_args->rewriter->loadConst(0, (intptr_t)0 /*generator*/);
+            rewrite_args->rewriter->loadConst(0, (intptr_t)func->isGenerator);
 
         // We might have trouble if we have more output args than input args,
         // such as if we need more space to pass defaults.
@@ -1963,7 +1963,7 @@ Box* callFunc(BoxedFunction* func, CallRewriteArgs* rewrite_args, ArgPassSpec ar
         return createGenerator(func, oarg1, oarg2, oarg3, oargs);
     }
 
-    return callCLFunc(f, rewrite_args, num_output_args, closure, NULL, oarg1, oarg2, oarg3, oargs);
+    return callCLFunc(f, rewrite_args, num_output_args, closure, (BoxedGenerator*)func->isGenerator, oarg1, oarg2, oarg3, oargs);
 }
 
 Box* callCLFunc(CLFunction* f, CallRewriteArgs* rewrite_args, int num_output_args, BoxedClosure* closure,
@@ -2981,4 +2981,16 @@ extern "C" Box* import(const std::string* name) {
 
     raiseExcHelper(ImportError, "No module named %s", name->c_str());
 }
+
+extern "C" Box* importFrom(Box* _m, const std::string* name) {
+    assert(_m->cls == module_cls);
+
+    BoxedModule* m = static_cast<BoxedModule*>(_m);
+
+    Box* r = m->getattr(*name, NULL, NULL);
+    if (r)
+        return r;
+
+    raiseExcHelper(ImportError, "cannot import name %s", name->c_str());
+}
 }

src/runtime/objmodel.h

@@ -29,7 +29,14 @@ class BoxedList;
 class BoxedString;
 class BoxedGenerator;
 
-void raiseExc(Box*) __attribute__((__noreturn__));
+// user-level raise functions that implement python-level semantics
+extern "C" void raise0() __attribute__((__noreturn__));
+extern "C" void raise1(Box*) __attribute__((__noreturn__));
+extern "C" void raise2(Box*, Box*) __attribute__((__noreturn__));
+extern "C" void raise3(Box*, Box*, Box*) __attribute__((__noreturn__));
+void raiseExc(Box* exc_obj) __attribute__((__noreturn__));
+
+// helper function for raising from the runtime:
 void raiseExcHelper(BoxedClass*, const char* fmt, ...) __attribute__((__noreturn__));
 
 extern "C" const std::string* getTypeName(Box* o);
@@ -37,6 +44,7 @@ extern "C" const std::string* getNameOfClass(BoxedClass* cls);
 
 // TODO sort this
 extern "C" void my_assert(bool b);
+extern "C" Box* importFrom(Box* obj, const std::string* attr);
 extern "C" Box* getattr(Box* obj, const char* attr);
 extern "C" void setattr(Box* obj, const char* attr, Box* attr_val);
 extern "C" bool nonzero(Box* obj);

src/runtime/stacktrace.cpp

@@ -20,6 +20,7 @@
 #include "codegen/codegen.h"
 #include "codegen/llvm_interpreter.h"
 #include "core/options.h"
+#include "gc/collector.h"
 #include "runtime/objmodel.h"
 #include "runtime/types.h"
 #include "runtime/util.h"
@@ -99,17 +100,27 @@ void unwindExc(Box* exc_obj) {
     abort();
 }
 
-void raiseExc(Box* exc_obj) {
+static std::vector<const LineInfo*> getTracebackEntries();
+static gc::StaticRootHandle last_exc;
+static std::vector<const LineInfo*> last_tb;
+
+void raiseRaw(Box* exc_obj) __attribute__((__noreturn__));
+void raiseRaw(Box* exc_obj) {
     // Using libgcc:
     throw exc_obj;
 
     // Using libunwind
     // unwindExc(exc_obj);
+}
 
-    abort();
+void raiseExc(Box* exc_obj) {
+    auto entries = getTracebackEntries();
+    last_tb = std::move(entries);
+    last_exc = exc_obj;
+
+    raiseRaw(exc_obj);
 }
 
-static std::vector<const LineInfo*> last_tb;
 void printLastTraceback() {
     fprintf(stderr, "Traceback (most recent call last):\n");
 
@@ -183,10 +194,28 @@ static std::vector<const LineInfo*> getTracebackEntries() {
     return entries;
 }
 
-void raiseExcHelper(BoxedClass* cls, const char* msg, ...) {
-    auto entries = getTracebackEntries();
-    last_tb = std::move(entries);
+void raise0() {
+    raiseRaw(last_exc);
+}
 
+void raise1(Box* b) {
+    if (b->cls == type_cls) {
+        BoxedClass* c = static_cast<BoxedClass*>(b);
+        if (isSubclass(c, Exception)) {
+            auto exc_obj = exceptionNew1(c);
+            raiseExc(exc_obj);
+        } else {
+            raiseExcHelper(TypeError, "exceptions must be old-style classes or derived from BaseException, not %s",
+                           getTypeName(b)->c_str());
+        }
+    }
+
+    // TODO: should only allow throwing of old-style classes or things derived
+    // from BaseException:
+    raiseExc(b);
+}
+
+void raiseExcHelper(BoxedClass* cls, const char* msg, ...) {
     if (msg != NULL) {
         va_list ap;
         va_start(ap, msg);
@@ -223,6 +252,8 @@ std::string formatException(Box* b) {
 
     assert(r->cls == str_cls);
     const std::string* msg = &r->s;
+    if (msg->size())
         return *name + ": " + *msg;
+    return *name;
 }
 }

src/runtime/str.cpp

@@ -191,6 +191,46 @@ extern "C" BoxedString* strMul(BoxedString* lhs, BoxedInt* rhs) {
     return new BoxedString(buf);
 }
 
+extern "C" Box* strLt(BoxedString* lhs, Box* rhs) {
+    assert(lhs->cls == str_cls);
+
+    if (rhs->cls != str_cls)
+        return NotImplemented;
+
+    BoxedString* srhs = static_cast<BoxedString*>(rhs);
+    return boxBool(lhs->s < srhs->s);
+}
+
+extern "C" Box* strLe(BoxedString* lhs, Box* rhs) {
+    assert(lhs->cls == str_cls);
+
+    if (rhs->cls != str_cls)
+        return NotImplemented;
+
+    BoxedString* srhs = static_cast<BoxedString*>(rhs);
+    return boxBool(lhs->s <= srhs->s);
+}
+
+extern "C" Box* strGt(BoxedString* lhs, Box* rhs) {
+    assert(lhs->cls == str_cls);
+
+    if (rhs->cls != str_cls)
+        return NotImplemented;
+
+    BoxedString* srhs = static_cast<BoxedString*>(rhs);
+    return boxBool(lhs->s > srhs->s);
+}
+
+extern "C" Box* strGe(BoxedString* lhs, Box* rhs) {
+    assert(lhs->cls == str_cls);
+
+    if (rhs->cls != str_cls)
+        return NotImplemented;
+
+    BoxedString* srhs = static_cast<BoxedString*>(rhs);
+    return boxBool(lhs->s >= srhs->s);
+}
+
 extern "C" Box* strEq(BoxedString* lhs, Box* rhs) {
     assert(lhs->cls == str_cls);
 
@@ -201,6 +241,16 @@ extern "C" Box* strEq(BoxedString* lhs, Box* rhs) {
     return boxBool(lhs->s == srhs->s);
 }
 
+extern "C" Box* strNe(BoxedString* lhs, Box* rhs) {
+    assert(lhs->cls == str_cls);
+
+    if (rhs->cls != str_cls)
+        return boxBool(true);
+
+    BoxedString* srhs = static_cast<BoxedString*>(rhs);
+    return boxBool(lhs->s != srhs->s);
+}
+
 extern "C" Box* strLen(BoxedString* self) {
     assert(self->cls == str_cls);
 
@@ -572,7 +622,14 @@ void setupStr() {
     str_cls->giveAttr("__add__", new BoxedFunction(boxRTFunction((void*)strAdd, UNKNOWN, 2)));
     str_cls->giveAttr("__mod__", new BoxedFunction(boxRTFunction((void*)strMod, STR, 2)));
     str_cls->giveAttr("__mul__", new BoxedFunction(boxRTFunction((void*)strMul, UNKNOWN, 2)));
+
+    str_cls->giveAttr("__lt__", new BoxedFunction(boxRTFunction((void*)strLt, UNKNOWN, 2)));
+    str_cls->giveAttr("__le__", new BoxedFunction(boxRTFunction((void*)strLe, UNKNOWN, 2)));
+    str_cls->giveAttr("__gt__", new BoxedFunction(boxRTFunction((void*)strGt, UNKNOWN, 2)));
+    str_cls->giveAttr("__ge__", new BoxedFunction(boxRTFunction((void*)strGe, UNKNOWN, 2)));
     str_cls->giveAttr("__eq__", new BoxedFunction(boxRTFunction((void*)strEq, UNKNOWN, 2)));
+    str_cls->giveAttr("__ne__", new BoxedFunction(boxRTFunction((void*)strNe, UNKNOWN, 2)));
+
     str_cls->giveAttr("__getitem__", new BoxedFunction(boxRTFunction((void*)strGetitem, STR, 2)));
 
     str_cls->giveAttr("__iter__", new BoxedFunction(boxRTFunction((void*)strIter, typeFromClass(str_iterator_cls), 1)));

src/runtime/types.cpp

@@ -652,9 +652,13 @@ void setupRuntime() {
     setupMath();
     setupTime();
     setupThread();
+    setupErrno();
+    setupPosix();
 
     setupCAPI();
 
+    setupSysEnd();
+
     TRACK_ALLOCATIONS = true;
 }
 

src/runtime/types.h

@@ -61,6 +61,9 @@ void setupBuiltins();
 void setupMath();
 void setupTime();
 void setupThread();
+void setupErrno();
+void setupPosix();
+void setupSysEnd();
 
 BoxedDict* getSysModulesDict();
 BoxedList* getSysPath();

test/tests/scoping_classes.py → test/tests/class_scoping.py

-# expected: fail
-# - WIP
-
 X = 0
 Y = 0
 Z = 0
@@ -37,10 +34,10 @@ def wrapper():
         # The defaults for a and b should resolve to the classdef definitions, and the default
         # for c should resolve to the wrapper() definition
         def f(self, a=X, b=Y, c=Z, d=W):
-            print a, b, c, W # "2 2 0 1"
+            print a, b, c, d # "2 2 0 1"
             # These references should skip all of the classdef directives,
             # and hit the definitions in the wrapper() function
-            print X, Y # "1 1"
+            print X, Y, Z, W # "1 1 1 1"
     print "done with classdef"
 
     print hasattr(C, 'X')

test/tests/classdef_ordering.py

+# expected: fail
+# - decorators
+
+def f(o, msg):
+    print msg
+    return o
+
+@f(lambda c: f(c, "calling decorator"), "evaluating decorator object")
+class C(f(object, "evaluating base")):
+    print "in classdef"

test/tests/closure_name_error.py

+x3 = 0
+def bad_addr3(_x):
+    if 0:
+        x3 = _
+
+    def g(y3):
+        return x3 + y3
+    return g
+print bad_addr3(1)(2)
+

test/tests/closure_test.py

@@ -43,11 +43,29 @@ def make_addr3(x3):
     return g
 print make_addr3(10)(2)
 
-def bad_addr3(_x):
-    if 0:
-        x3 = _
+def f4(args):
+     def inner():
+       for a in args:
+         print a
+     return inner
+print f4([1, 2, 3])()
 
-    def g(y3):
-        return x3 + y3
-    return g
-print bad_addr3(1)(2)
+def f5():
+    x = 12039
+    def i1():
+        def i2():
+            print x
+        i2()
+    i1()
+f5()
+
+def f6():
+    x = 131
+    # Regression test:
+    # default args shouldn't mess with closure analysis
+    def inner(a=1):
+        print x
+    inner()
+
+    print (lambda a=1: x)()
+f6()

test/tests/del_name_scoping.py

+# expected: fail
+# - deletes on names
+
+x = 1
+def f():
+    if 0:
+        # the del marks 'x' as a name written to in this scope
+        del x
+    print x
+f()

test/tests/delitem.py

@@ -15,3 +15,12 @@ del a[:]
 print a
 a.append(1)
 print a
+
+# Make sure that del's work correctly in sub-scopes:
+x = 1
+def f1():
+    x = range(5)
+    def f2():
+        del x[1]
+    return f2
+f1()()

test/tests/import_failure_test.py

+try:
+    import non_existent_module
+    assert 0, "shouldn't get here"
+except ImportError:
+    pass
+
+try:
+    from non_existent_module import a
+    assert 0, "shouldn't get here"
+except ImportError:
+    pass
+
+try:
+    from sys import non_existent_attribute
+    assert 0, "shouldn't get here"
+except ImportError:
+    pass

test/tests/list.py

@@ -67,3 +67,7 @@ for i in xrange(100):
     l.append(i)
 while l:
     del l[0]
+
+l = range(5)
+l.extend(range(5))
+print l

test/tests/str_functions.py

@@ -39,3 +39,17 @@ for c in "hello world":
 for i in xrange(1, 10):
     for j in xrange(1, 4):
         print ("a"*i).count("a"*j)
+
+def test_comparisons(a, b):
+    print "%s < %s = " % (a, b), a < b
+    print "%s <= %s = " % (a, b), a <= b
+    print "%s > %s = " % (a, b), a > b
+    print "%s >= %s = " % (a, b), a >= b
+    print "%s == %s = " % (a, b), a == b
+    print "%s != %s = " % (a, b), a != b
+test_comparisons("a", "a")
+test_comparisons("a", "A")
+test_comparisons("a", "aa")
+test_comparisons("ab", "aa")
+
+print sorted([str(i) for i in xrange(25)])