WIP: AST Interpreter

src/codegen/ast_interpreter.h

+// Copyright (c) 2014 Dropbox, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef PYSTON_CODEGEN_ASTINTERPRETER_H
+#define PYSTON_CODEGEN_ASTINTERPRETER_H
+
+#include <llvm/ADT/StringMap.h>
+
+#include "codegen/compvars.h"
+#include "core/ast.h"
+#include "runtime/objmodel.h"
+
+namespace pyston {
+
+namespace gc {
+class GCVisitor;
+}
+
+class Box;
+class CLFunction;
+struct LineInfo;
+
+Box* astInterpretFunction(CompiledFunction* f, int nargs, Box* closure, Box* generator, Box* arg1, Box* arg2, Box* arg3,
+                          Box** args);
+
+const LineInfo* getLineInfoForInterpretedFrame(void* frame_ptr);
+BoxedModule* getModuleForInterpretedFrame(void* frame_ptr);
+
+union Value {
+    bool b;
+    int64_t n;
+    double d;
+    Box* o;
+
+    Value(bool b) : b(b) {}
+    Value(int64_t n = 0) : n(n) {}
+    Value(double d) : d(d) {}
+    Value(Box* o) : o(o) {}
+};
+
+class ASTInterpreter {
+public:
+    typedef llvm::StringMap<Box*> SymMap;
+
+    ASTInterpreter(CompiledFunction* compiled_function);
+    ~ASTInterpreter();
+
+    void initArguments(int nargs, BoxedClosure* closure, BoxedGenerator* generator, Box* arg1, Box* arg2, Box* arg3,
+                       Box** args);
+    Value execute(CFGBlock* block = 0);
+
+private:
+    Box* createFunction(AST* node, AST_arguments* args, const std::vector<AST_stmt*>& body);
+    Value doBinOp(Box* left, Box* right, int op, BinExpType exp_type);
+    void doStore(AST_expr* node, Value value);
+    void doStore(const std::string& name, Value value);
+    void eraseDeadSymbols();
+
+    Value visit_assert(AST_Assert* node);
+    Value visit_assign(AST_Assign* node);
+    Value visit_binop(AST_BinOp* node);
+    Value visit_call(AST_Call* node);
+    Value visit_classDef(AST_ClassDef* node);
+    Value visit_compare(AST_Compare* node);
+    Value visit_delete(AST_Delete* node);
+    Value visit_functionDef(AST_FunctionDef* node);
+    Value visit_global(AST_Global* node);
+    Value visit_module(AST_Module* node);
+    Value visit_print(AST_Print* node);
+    Value visit_raise(AST_Raise* node);
+    Value visit_return(AST_Return* node);
+    Value visit_stmt(AST_stmt* node);
+    Value visit_unaryop(AST_UnaryOp* node);
+
+    Value visit_attribute(AST_Attribute* node);
+    Value visit_dict(AST_Dict* node);
+    Value visit_expr(AST_expr* node);
+    Value visit_expr(AST_Expr* node);
+    Value visit_index(AST_Index* node);
+    Value visit_lambda(AST_Lambda* node);
+    Value visit_list(AST_List* node);
+    Value visit_name(AST_Name* node);
+    Value visit_num(AST_Num* node);
+    Value visit_repr(AST_Repr* node);
+    Value visit_set(AST_Set* node);
+    Value visit_slice(AST_Slice* node);
+    Value visit_str(AST_Str* node);
+    Value visit_subscript(AST_Subscript* node);
+    Value visit_tuple(AST_Tuple* node);
+    Value visit_yield(AST_Yield* node);
+
+
+    // pseudo
+    Value visit_augBinOp(AST_AugBinOp* node);
+    Value visit_branch(AST_Branch* node);
+    Value visit_clsAttribute(AST_ClsAttribute* node);
+    Value visit_invoke(AST_Invoke* node);
+    Value visit_jump(AST_Jump* node);
+    Value visit_langPrimitive(AST_LangPrimitive* node);
+
+public:
+    SourceInfo* source_info;
+    SymMap sym_table;
+    CFGBlock* next_block, *current_block;
+    AST* current_inst;
+    Box* last_exception;
+    BoxedClosure* passed_closure, *created_closure;
+    BoxedGenerator* generator;
+    ScopeInfo* scope_info;
+    CompiledFunction* compiled_func;
+    unsigned edgecount;
+};
+}
+
+#endif

src/codegen/irgen.cpp

@@ -986,7 +986,9 @@ static std::string getUniqueFunctionName(std::string nameprefix, EffortLevel::Ef
     os << nameprefix;
     os << "_e" << effort;
     if (entry) {
-        os << "_osr" << entry->backedge->target->idx << "_from_" << entry->cf->func->getName().data();
+        os << "_osr" << entry->backedge->target->idx;
+        if (entry->cf->func)
+            os << "_from_" << entry->cf->func->getName().data();
     }
     os << '_' << num_functions;
     num_functions++;

src/codegen/irgen/hooks.cpp

@@ -20,6 +20,7 @@
 #include "analysis/function_analysis.h"
 #include "analysis/scoping_analysis.h"
 #include "asm_writing/icinfo.h"
+#include "codegen/ast_interpreter.h"
 #include "codegen/codegen.h"
 #include "codegen/compvars.h"
 #include "codegen/irgen.h"
@@ -137,7 +138,6 @@ static std::unordered_map<std::string, CompiledFunction*> machine_name_to_cf;
 CompiledFunction* cfForMachineFunctionName(const std::string& machine_name) {
     assert(machine_name.size());
     auto r = machine_name_to_cf[machine_name];
-    ASSERT(r, "%s", machine_name.c_str());
     return r;
 }
 
@@ -185,17 +185,29 @@ CompiledFunction* compileFunction(CLFunction* f, FunctionSpecialization* spec, E
 
     // Do the analysis now if we had deferred it earlier:
     if (source->cfg == NULL) {
-        assert(source->ast);
         source->cfg = computeCFG(source, source->body);
-        source->liveness = computeLivenessInfo(source->cfg);
-        source->phis = computeRequiredPhis(source->arg_names, source->cfg, source->liveness, source->getScopeInfo());
     }
 
-    CompiledFunction* cf = doCompile(source, entry, effort, spec, name);
+    if (effort != EffortLevel::INTERPRETED) {
+        if (source->liveness == NULL)
+            source->liveness = computeLivenessInfo(source->cfg);
+
+        if (source->phis == NULL)
+            source->phis
+                = computeRequiredPhis(source->arg_names, source->cfg, source->liveness, source->getScopeInfo());
+    }
+
 
-    registerMachineName(cf->func->getName(), cf);
 
-    compileIR(cf, effort);
+    CompiledFunction* cf = 0;
+    if (effort == EffortLevel::INTERPRETED) {
+        cf = new CompiledFunction(0, spec, true, NULL, NULL, effort, 0);
+    } else {
+        cf = doCompile(source, entry, effort, spec, name);
+        registerMachineName(cf->func->getName(), cf);
+        compileIR(cf, effort);
+    }
+
     f->addVersion(cf);
     assert(f->versions.size());
 
@@ -252,10 +264,6 @@ void compileAndRunModule(AST_Module* m, BoxedModule* bm) {
         ScopingAnalysis* scoping = runScopingAnalysis(m);
 
         SourceInfo* si = new SourceInfo(bm, scoping, m, m->body);
-        si->cfg = computeCFG(si, m->body);
-        si->liveness = computeLivenessInfo(si->cfg);
-        si->phis = computeRequiredPhis(si->arg_names, si->cfg, si->liveness, si->getScopeInfo());
-
         CLFunction* cl_f = new CLFunction(0, 0, false, false, si);
 
         EffortLevel::EffortLevel effort = initialEffort();
@@ -265,7 +273,7 @@ void compileAndRunModule(AST_Module* m, BoxedModule* bm) {
     }
 
     if (cf->is_interpreted)
-        interpretFunction(cf->func, 0, NULL, NULL, NULL, NULL, NULL, NULL);
+        astInterpretFunction(cf, 0, NULL, NULL, NULL, NULL, NULL, NULL);
     else
         ((void (*)())cf->code)();
 }
@@ -311,7 +319,7 @@ static CompiledFunction* _doReopt(CompiledFunction* cf, EffortLevel::EffortLevel
 }
 
 static StatCounter stat_osrexits("OSR exits");
-void* compilePartialFunc(OSRExit* exit) {
+CompiledFunction* compilePartialFuncInternal(OSRExit* exit) {
     LOCK_REGION(codegen_rwlock.asWrite());
 
     assert(exit);
@@ -334,11 +342,16 @@ void* compilePartialFunc(OSRExit* exit) {
         assert(compiled = new_cf);
     }
 
-    return new_cf->code;
+    return new_cf;
 }
 
+void* compilePartialFunc(OSRExit* exit) {
+    return compilePartialFuncInternal(exit)->code;
+}
+
+
 static StatCounter stat_reopt("reopts");
-extern "C" char* reoptCompiledFunc(CompiledFunction* cf) {
+extern "C" CompiledFunction* reoptCompiledFuncInternal(CompiledFunction* cf) {
     if (VERBOSITY("irgen") >= 1)
         printf("In reoptCompiledFunc, %p, %ld\n", cf, cf->times_called);
     stat_reopt.log();
@@ -347,7 +360,12 @@ extern "C" char* reoptCompiledFunc(CompiledFunction* cf) {
     assert(cf->clfunc->versions.size());
     CompiledFunction* new_cf = _doReopt(cf, (EffortLevel::EffortLevel(cf->effort + 1)));
     assert(!new_cf->is_interpreted);
-    return (char*)new_cf->code;
+    return new_cf;
+}
+
+
+extern "C" char* reoptCompiledFunc(CompiledFunction* cf) {
+    return (char*)reoptCompiledFuncInternal(cf)->code;
 }
 
 CLFunction* createRTFunction(int num_args, int num_defaults, bool takes_varargs, bool takes_kwargs) {

src/codegen/irgen/hooks.h

@@ -23,7 +23,9 @@ struct CompiledFunction;
 class CLFunction;
 class OSRExit;
 
+CompiledFunction* compilePartialFuncInternal(OSRExit* exit);
 void* compilePartialFunc(OSRExit*);
+extern "C" CompiledFunction* reoptCompiledFuncInternal(CompiledFunction*);
 extern "C" char* reoptCompiledFunc(CompiledFunction*);
 
 class AST_Module;

src/codegen/irgen/irgenerator.cpp

@@ -1496,7 +1496,7 @@ private:
             decorators.push_back(evalExpr(d, exc_info));
         }
 
-        CLFunction* cl = _wrapFunction(node, nullptr, node->body);
+        CLFunction* cl = wrapFunction(node, nullptr, node->body, irstate->getSourceInfo());
 
         // TODO duplication with _createFunction:
         CompilerVariable* created_closure = NULL;
@@ -1628,27 +1628,9 @@ private:
         symbol_table.erase(target->id);
     }
 
-    CLFunction* _wrapFunction(AST* node, AST_arguments* args, const std::vector<AST_stmt*>& body) {
-        // Different compilations of the parent scope of a functiondef should lead
-        // to the same CLFunction* being used:
-        static std::unordered_map<AST*, CLFunction*> made;
-
-        CLFunction*& cl = made[node];
-        if (cl == NULL) {
-            SourceInfo* source = irstate->getSourceInfo();
-            SourceInfo* si = new SourceInfo(source->parent_module, source->scoping, node, body);
-            if (args)
-                cl = new CLFunction(args->args.size(), args->defaults.size(), args->vararg.size(), args->kwarg.size(),
-                                    si);
-            else
-                cl = new CLFunction(0, 0, 0, 0, si);
-        }
-        return cl;
-    }
-
     CompilerVariable* _createFunction(AST* node, ExcInfo exc_info, AST_arguments* args,
                                       const std::vector<AST_stmt*>& body) {
-        CLFunction* cl = this->_wrapFunction(node, args, body);
+        CLFunction* cl = wrapFunction(node, args, body, irstate->getSourceInfo());
 
         std::vector<ConcreteCompilerVariable*> defaults;
         for (auto d : args->defaults) {
@@ -2456,4 +2438,20 @@ IRGenerator* createIRGenerator(IRGenState* irstate, std::unordered_map<CFGBlock*
                                const GuardList& in_guards, bool is_partial) {
     return new IRGeneratorImpl(irstate, entry_blocks, myblock, types, out_guards, in_guards, is_partial);
 }
+
+CLFunction* wrapFunction(AST* node, AST_arguments* args, const std::vector<AST_stmt*>& body, SourceInfo* source) {
+    // Different compilations of the parent scope of a functiondef should lead
+    // to the same CLFunction* being used:
+    static std::unordered_map<AST*, CLFunction*> made;
+
+    CLFunction*& cl = made[node];
+    if (cl == NULL) {
+        SourceInfo* si = new SourceInfo(source->parent_module, source->scoping, node, body);
+        if (args)
+            cl = new CLFunction(args->args.size(), args->defaults.size(), args->vararg.size(), args->kwarg.size(), si);
+        else
+            cl = new CLFunction(0, 0, 0, 0, si);
+    }
+    return cl;
+}
 }

src/codegen/irgen/irgenerator.h

@@ -46,6 +46,7 @@ extern const std::string CREATED_CLOSURE_NAME;
 extern const std::string PASSED_CLOSURE_NAME;
 extern const std::string PASSED_GENERATOR_NAME;
 
+
 // Class that holds state of the current IR generation, that might not be local
 // to the specific phase or pass we're in.
 // TODO this probably shouldn't be here
@@ -205,6 +206,8 @@ IREmitter* createIREmitter(IRGenState* irstate, llvm::BasicBlock*& curblock, IRG
 IRGenerator* createIRGenerator(IRGenState* irstate, std::unordered_map<CFGBlock*, llvm::BasicBlock*>& entry_blocks,
                                CFGBlock* myblock, TypeAnalysis* types, GuardList& out_guards,
                                const GuardList& in_guards, bool is_partial);
+
+CLFunction* wrapFunction(AST* node, AST_arguments* args, const std::vector<AST_stmt*>& body, SourceInfo* source);
 }
 
 #endif

src/codegen/llvm_interpreter.cpp

@@ -219,8 +219,8 @@ static void set(SymMap& symbols, const llvm::BasicBlock::iterator& it, Val v) {
 static std::unordered_map<void*, llvm::Instruction*> cur_instruction_map;
 
 typedef std::vector<const SymMap*> root_stack_t;
-threading::PerThreadSet<root_stack_t> root_stack_set;
-
+static threading::PerThreadSet<root_stack_t> root_stack_set;
+/*
 void gatherInterpreterRoots(GCVisitor* visitor) {
     root_stack_set.forEachValue(std::function<void(root_stack_t*, GCVisitor*)>([](root_stack_t* v, GCVisitor* visitor) {
                                     for (const SymMap* sym_map : *v) {
@@ -231,7 +231,8 @@ void gatherInterpreterRoots(GCVisitor* visitor) {
                                 }),
                                 visitor);
 }
-
+*/
+#if 0
 BoxedDict* localsForInterpretedFrame(void* frame_ptr, bool only_user_visible) {
     llvm::Instruction* inst = cur_instruction_map[frame_ptr];
     assert(inst);
@@ -285,6 +286,7 @@ BoxedDict* localsForInterpretedFrame(void* frame_ptr, bool only_user_visible) {
 
     return rtn;
 }
+#endif
 
 class UnregisterHelper {
 private:
@@ -302,6 +304,7 @@ public:
 };
 
 static std::unordered_map<llvm::Instruction*, LineInfo*> line_infos;
+/*
 const LineInfo* getLineInfoForInterpretedFrame(void* frame_ptr) {
     llvm::Instruction* cur_instruction = cur_instruction_map[frame_ptr];
     assert(cur_instruction);
@@ -319,7 +322,7 @@ const LineInfo* getLineInfoForInterpretedFrame(void* frame_ptr) {
         return it->second;
     }
 }
-
+*/
 void dumpLLVM(llvm::Value* v) {
     v->dump();
 }

src/codegen/unwinding.cpp

@@ -23,6 +23,7 @@
 #include "llvm/ExecutionEngine/ObjectImage.h"
 #include "llvm/IR/DebugInfo.h"
 
+#include "codegen/ast_interpreter.h"
 #include "codegen/codegen.h"
 #include "codegen/compvars.h"
 #include "codegen/irgen/hooks.h"
@@ -307,7 +308,7 @@ public:
             int code = unw_get_proc_info(&this->cursor, &pip);
             RELEASE_ASSERT(code == 0, "%d", code);
 
-            if (pip.start_ip == (intptr_t)interpretFunction) {
+            if (pip.start_ip == (intptr_t)interpretFunction || pip.start_ip == (intptr_t)astInterpretFunction) {
                 unw_word_t bp;
                 unw_get_reg(&this->cursor, UNW_TDEP_BP, &bp);
 
@@ -401,12 +402,19 @@ CompiledFunction* getTopCompiledFunction() {
     assert(last_entry->func.size());
 
     CompiledFunction* cf = cfForMachineFunctionName(last_entry->func);
-    assert(cf);
     return cf;
 }
 
 BoxedModule* getCurrentModule() {
-    return getTopCompiledFunction()->clfunc->source->parent_module;
+    CompiledFunction* compiledFunction = getTopCompiledFunction();
+    if (compiledFunction)
+        return compiledFunction->clfunc->source->parent_module;
+    else {
+        std::unique_ptr<PythonFrameIterator> frame = getTopPythonFrame();
+        auto& id = frame->getId();
+        assert(id.type == PythonFrameId::INTERPRETED);
+        return getModuleForInterpretedFrame((void*)id.bp);
+    }
 }
 
 BoxedDict* getLocals(bool only_user_visible) {

src/core/types.h

@@ -288,7 +288,6 @@ public:
 
     void addVersion(CompiledFunction* compiled) {
         assert(compiled);
-        assert((source == NULL) == (compiled->func == NULL));
         assert(compiled->spec);
         assert(compiled->spec->arg_types.size() == num_args + (takes_varargs ? 1 : 0) + (takes_kwargs ? 1 : 0));
         assert(compiled->clfunc == NULL);

src/runtime/objmodel.cpp

@@ -23,6 +23,7 @@
 
 #include "asm_writing/icinfo.h"
 #include "asm_writing/rewriter.h"
+#include "codegen/ast_interpreter.h"
 #include "codegen/codegen.h"
 #include "codegen/compvars.h"
 #include "codegen/irgen/hooks.h"
@@ -2486,7 +2487,7 @@ Box* callCLFunc(CLFunction* f, CallRewriteArgs* rewrite_args, int num_output_arg
 
     assert(chosen_cf->is_interpreted == (chosen_cf->code == NULL));
     if (chosen_cf->is_interpreted) {
-        return interpretFunction(chosen_cf->func, num_output_args, closure, generator, oarg1, oarg2, oarg3, oargs);
+        return astInterpretFunction(chosen_cf, num_output_args, closure, generator, oarg1, oarg2, oarg3, oargs);
     }
 
     if (rewrite_args) {

src/runtime/set.h

@@ -31,9 +31,11 @@ extern "C" Box* createSet();
 
 class BoxedSet : public Box {
 public:
-    std::unordered_set<Box*, PyHasher, PyEq, StlCompatAllocator<Box*> > s;
+    typedef std::unordered_set<Box*, PyHasher, PyEq, StlCompatAllocator<Box*> > Set;
+    Set s;
 
     BoxedSet(BoxedClass* cls) __attribute__((visibility("default"))) : Box(cls) {}
+    BoxedSet(Set&& s, BoxedClass* cls) __attribute__((visibility("default"))) : Box(cls), s(s) {}
 };
 }