Merge pull request #1358 from dropbox/ast

Merge AST/BST work into master

Makefile

@@ -425,7 +425,6 @@ ARGS ?=
 ifneq ($(BR),)
 	override GDB_CMDS := --ex "break $(BR)" $(GDB_CMDS)
 endif
-$(call add_unittest,gc)
 $(call add_unittest,analysis)
 
 
@@ -923,8 +922,9 @@ wdbg_%:
 
 .PHONY: head_%
 HEAD := 40
+HEAD_SKIP := 6
 head_%:
-	@ bash -c "set -o pipefail; script -e -q -c '$(MAKE) $(dir $@)$(patsubst head_%,%,$(notdir $@))' /dev/null | head -n$(HEAD)"
+	@ bash -c "set -o pipefail; script -e -q -c '$(MAKE) $(dir $@)$(patsubst head_%,%,$(notdir $@))' /dev/null | tail -n+$(HEAD_SKIP) | head -n$(HEAD)"
 head: head_pyston_dbg
 .PHONY: hwatch_%
 hwatch_%:

src/CMakeLists.txt

@@ -65,6 +65,7 @@ add_library(PYSTON_OBJECTS OBJECT ${OPTIONAL_SRCS}
 		codegen/type_recording.cpp
 		codegen/unwinding.cpp
 		core/ast.cpp
+		core/bst.cpp
 		core/cfg.cpp
 		core/options.cpp
 		core/stats.cpp

src/analysis/function_analysis.h

@@ -26,12 +26,11 @@
 
 namespace pyston {
 
-class AST_arguments;
-class AST_Jump;
-class AST_Name;
+class BST_arguments;
+class BST_Jump;
+class BST_Name;
 class CFG;
 class CFGBlock;
-class ScopeInfo;
 class LivenessBBVisitor;
 
 class LivenessAnalysis {
@@ -49,7 +48,7 @@ public:
     ~LivenessAnalysis();
 
     // we don't keep track of node->parent_block relationships, so you have to pass both:
-    bool isKill(AST_Name* node, CFGBlock* parent_block);
+    bool isKill(BST_Name* node, CFGBlock* parent_block);
 
     bool isLiveAtEnd(int vreg, CFGBlock* block);
 };
@@ -72,7 +71,7 @@ private:
 public:
     DefinednessAnalysis() {}
 
-    void run(VRegMap<DefinitionLevel> initial_map, CFGBlock* initial_block, ScopeInfo* scope_info);
+    void run(VRegMap<DefinitionLevel> initial_map, CFGBlock* initial_block);
 
     DefinitionLevel isDefinedAtEnd(int vreg, CFGBlock* block);
     const VRegSet& getDefinedVregsAtEnd(CFGBlock* block);
@@ -94,7 +93,7 @@ public:
     // Initials_need_phis specifies that initial_map should count as an additional entry point
     // that may require phis.
     PhiAnalysis(VRegMap<DefinednessAnalysis::DefinitionLevel> initial_map, CFGBlock* initial_block,
-                bool initials_need_phis, LivenessAnalysis* liveness, ScopeInfo* scope_info);
+                bool initials_need_phis, LivenessAnalysis* liveness);
 
     bool isRequired(int vreg, CFGBlock* block);
     bool isRequiredAfter(int vreg, CFGBlock* block);
@@ -107,8 +106,8 @@ public:
 };
 
 std::unique_ptr<LivenessAnalysis> computeLivenessInfo(CFG*);
-std::unique_ptr<PhiAnalysis> computeRequiredPhis(const ParamNames&, CFG*, LivenessAnalysis*, ScopeInfo* scope_info);
-std::unique_ptr<PhiAnalysis> computeRequiredPhis(const OSREntryDescriptor*, LivenessAnalysis*, ScopeInfo* scope_info);
+std::unique_ptr<PhiAnalysis> computeRequiredPhis(const ParamNames&, CFG*, LivenessAnalysis*);
+std::unique_ptr<PhiAnalysis> computeRequiredPhis(const OSREntryDescriptor*, LivenessAnalysis*);
 }
 
 #endif

src/analysis/scoping_analysis.cpp

@@ -17,6 +17,7 @@
 #include "llvm/ADT/DenseSet.h"
 
 #include "core/ast.h"
+#include "core/bst.h"
 #include "core/common.h"
 #include "core/types.h"
 #include "core/util.h"
@@ -24,6 +25,28 @@
 
 namespace pyston {
 
+ScopingResults::ScopingResults(ScopeInfo* scope_info, bool globals_from_module)
+    : are_locals_from_module(scope_info->areLocalsFromModule()),
+      are_globals_from_module(globals_from_module),
+      creates_closure(scope_info->createsClosure()),
+      takes_closure(scope_info->takesClosure()),
+      passes_through_closure(scope_info->passesThroughClosure()),
+      uses_name_lookup(scope_info->usesNameLookup()),
+      closure_size(creates_closure ? scope_info->getClosureSize() : 0) {
+    deref_info = scope_info->getAllDerefVarsAndInfo();
+}
+
+DerefInfo ScopingResults::getDerefInfo(BST_Name* node) const {
+    assert(node->lookup_type == ScopeInfo::VarScopeType::DEREF);
+    assert(node->deref_info.offset != INT_MAX);
+    return node->deref_info;
+}
+size_t ScopingResults::getClosureOffset(BST_Name* node) const {
+    assert(node->lookup_type == ScopeInfo::VarScopeType::CLOSURE);
+    assert(node->closure_offset != -1);
+    return node->closure_offset;
+}
+
 class YieldVisitor : public NoopASTVisitor {
 public:
     AST* starting_node;
@@ -50,6 +73,13 @@ bool containsYield(AST* ast) {
     return visitor.contains_yield;
 }
 
+bool containsYield(llvm::ArrayRef<AST_stmt*> body) {
+    for (auto e : body)
+        if (containsYield(e))
+            return true;
+    return false;
+}
+
 // TODO
 // Combine this with the below? Basically the same logic with different string types...
 // Also should this go in this file?
@@ -576,8 +606,6 @@ public:
     bool visit_while(AST_While* node) override { return false; }
     bool visit_with(AST_With* node) override { return false; }
     bool visit_yield(AST_Yield* node) override { return false; }
-    bool visit_branch(AST_Branch* node) override { return false; }
-    bool visit_jump(AST_Jump* node) override { return false; }
     bool visit_delete(AST_Delete* node) override { return false; }
 
     bool visit_global(AST_Global* node) override {
@@ -903,13 +931,13 @@ void ScopingAnalysis::processNameUsages(ScopingAnalysis::NameUsageMap* usages) {
         ScopeNameUsage* usage = sorted_usages[i];
         AST* node = usage->node;
 
-        ScopeInfo* parent_info = this->scopes[(usage->parent == NULL) ? this->parent_module : usage->parent->node];
+        ScopeInfo* parent_info
+            = this->scopes[(usage->parent == NULL) ? this->parent_module : usage->parent->node].get();
 
         switch (node->type) {
             case AST_TYPE::ClassDef: {
-                ScopeInfoBase* scopeInfo = new ScopeInfoBase(parent_info, usage, usage->node, true /* usesNameLookup */,
-                                                             globals_from_module);
-                this->scopes[node] = scopeInfo;
+                this->scopes[node] = llvm::make_unique<ScopeInfoBase>(parent_info, usage, usage->node,
+                                                                      true /* usesNameLookup */, globals_from_module);
                 break;
             }
             case AST_TYPE::FunctionDef:
@@ -917,10 +945,9 @@ void ScopingAnalysis::processNameUsages(ScopingAnalysis::NameUsageMap* usages) {
             case AST_TYPE::GeneratorExp:
             case AST_TYPE::DictComp:
             case AST_TYPE::SetComp: {
-                ScopeInfoBase* scopeInfo
-                    = new ScopeInfoBase(parent_info, usage, usage->node,
-                                        usage->hasNameForcingSyntax() /* usesNameLookup */, globals_from_module);
-                this->scopes[node] = scopeInfo;
+                this->scopes[node] = llvm::make_unique<ScopeInfoBase>(
+                    parent_info, usage, usage->node, usage->hasNameForcingSyntax() /* usesNameLookup */,
+                    globals_from_module);
                 break;
             }
             default:
@@ -934,45 +961,22 @@ InternedStringPool& ScopingAnalysis::getInternedStrings() {
     return *interned_strings;
 }
 
-ScopeInfo* ScopingAnalysis::analyzeSubtree(AST* node) {
+void ScopingAnalysis::analyzeSubtree(AST* node) {
     NameUsageMap usages;
     usages[node] = new ScopeNameUsage(node, NULL, this);
     NameCollectorVisitor::collect(node, &usages, this);
 
     processNameUsages(&usages);
-
-    ScopeInfo* rtn = scopes[node];
-    assert(rtn);
-    return rtn;
-}
-
-void ScopingAnalysis::registerScopeReplacement(AST* original_node, AST* new_node) {
-    assert(scope_replacements.count(original_node) == 0);
-    assert(scope_replacements.count(new_node) == 0);
-    assert(scopes.count(new_node) == 0);
-
-#ifndef NDEBUG
-    // NULL this out just to make sure it doesn't get accessed:
-    scopes[new_node] = NULL;
-#endif
-
-    scope_replacements[new_node] = original_node;
 }
 
 ScopeInfo* ScopingAnalysis::getScopeInfoForNode(AST* node) {
     assert(node);
 
-    auto it = scope_replacements.find(node);
-    if (it != scope_replacements.end())
-        node = it->second;
-
-    auto rtn = scopes.find(node);
-    if (rtn != scopes.end()) {
-        assert(rtn->second);
-        return rtn->second;
-    }
+    if (!scopes.count(node))
+        analyzeSubtree(node);
 
-    return analyzeSubtree(node);
+    assert(scopes.count(node));
+    return scopes[node].get();
 }
 
 ScopingAnalysis::ScopingAnalysis(AST* ast, bool globals_from_module)
@@ -993,10 +997,10 @@ ScopingAnalysis::ScopingAnalysis(AST* ast, bool globals_from_module)
 
     if (globals_from_module) {
         assert(ast->type == AST_TYPE::Module);
-        scopes[ast] = new ModuleScopeInfo();
+        scopes[ast] = llvm::make_unique<ModuleScopeInfo>();
         parent_module = static_cast<AST_Module*>(ast);
     } else {
-        scopes[ast] = new EvalExprScopeInfo(ast, globals_from_module);
+        scopes[ast] = llvm::make_unique<EvalExprScopeInfo>(ast, globals_from_module);
     }
 }
 }

src/analysis/scoping_analysis.h

@@ -19,6 +19,7 @@
 
 #include "core/common.h"
 #include "core/stringpool.h"
+#include "core/types.h"
 
 namespace pyston {
 
@@ -27,16 +28,6 @@ class AST_Module;
 class AST_Expression;
 class AST_Suite;
 
-// Each closure has an array (fixed-size for that particular scope) of variables
-// and a parent pointer to a parent closure. To look up a variable from the passed-in
-// closure (i.e., DEREF), you just need to know (i) how many parents up to go and
-// (ii) what offset into the array to find the variable. This struct stores that
-// information. You can query the ScopeInfo with a name to get this info.
-struct DerefInfo {
-    size_t num_parents_from_passed_closure;
-    size_t offset;
-};
-
 class ScopeInfo {
 public:
     ScopeInfo() {}
@@ -74,7 +65,7 @@ public:
     //  import dis
     //  print dis.dis(g)
 
-    enum class VarScopeType {
+    enum class VarScopeType : unsigned char {
         FAST,
         GLOBAL,
         CLOSURE,
@@ -151,27 +142,16 @@ public:
     typedef llvm::DenseMap<AST*, ScopeNameUsage*> NameUsageMap;
 
 private:
-    llvm::DenseMap<AST*, ScopeInfo*> scopes;
+    llvm::DenseMap<AST*, std::unique_ptr<ScopeInfo>> scopes;
     AST_Module* parent_module;
     InternedStringPool* interned_strings;
 
-    llvm::DenseMap<AST*, AST*> scope_replacements;
-
-    ScopeInfo* analyzeSubtree(AST* node);
+    void analyzeSubtree(AST* node);
     void processNameUsages(NameUsageMap* usages);
 
     bool globals_from_module;
 
 public:
-    // The scope-analysis is done before any CFG-ization is done,
-    // but many of the queries will be done post-CFG-ization.
-    // The CFG process can replace scope AST nodes with others (ex:
-    // generator expressions with generator functions), so we need to
-    // have a way of mapping the original analysis with the new queries.
-    // This is a hook for the CFG process to register when it has replaced
-    // a scope-node with a different node.
-    void registerScopeReplacement(AST* original_node, AST* new_node);
-
     ScopingAnalysis(AST* ast, bool globals_from_module);
     ScopeInfo* getScopeInfoForNode(AST* node);
 
@@ -179,7 +159,9 @@ public:
     bool areGlobalsFromModule() { return globals_from_module; }
 };
 
+class AST_stmt;
 bool containsYield(AST* ast);
+bool containsYield(llvm::ArrayRef<AST_stmt*> ast);
 
 class BoxedString;
 BoxedString* mangleNameBoxedString(BoxedString* id, BoxedString* private_name);

src/analysis/type_analysis.h

@@ -23,11 +23,10 @@
 
 namespace pyston {
 
-class ScopeInfo;
 class CFGBlock;
 class BoxedClass;
-class AST_expr;
-class AST_slice;
+class BST_expr;
+class BST_slice;
 class OSREntryDescriptor;
 
 class TypeAnalysis {
@@ -41,15 +40,15 @@ public:
 
     virtual ConcreteCompilerType* getTypeAtBlockStart(int vreg, CFGBlock* block) = 0;
     virtual ConcreteCompilerType* getTypeAtBlockEnd(int vreg, CFGBlock* block) = 0;
-    virtual BoxedClass* speculatedExprClass(AST_expr*) = 0;
-    virtual BoxedClass* speculatedExprClass(AST_slice*) = 0;
+    virtual BoxedClass* speculatedExprClass(BST_expr*) = 0;
+    virtual BoxedClass* speculatedExprClass(BST_slice*) = 0;
 };
 
 TypeAnalysis* doTypeAnalysis(CFG* cfg, const ParamNames& param_names,
                              const std::vector<ConcreteCompilerType*>& arg_types, EffortLevel effort,
-                             TypeAnalysis::SpeculationLevel speculation, ScopeInfo* scope_info);
+                             TypeAnalysis::SpeculationLevel speculation);
 TypeAnalysis* doTypeAnalysis(const OSREntryDescriptor* entry_descriptor, EffortLevel effort,
-                             TypeAnalysis::SpeculationLevel speculation, ScopeInfo* scope_info);
+                             TypeAnalysis::SpeculationLevel speculation);
 }
 
 #endif

src/asm_writing/icinfo.cpp

@@ -336,7 +336,7 @@ ICSlotInfo* ICInfo::pickEntryForRewrite(const char* debug_name) {
 }
 
 static llvm::DenseMap<void*, ICInfo*> ics_by_return_addr;
-static llvm::DenseMap<AST*, ICInfo*> ics_by_ast_node;
+static llvm::DenseMap<BST*, ICInfo*> ics_by_ast_node;
 
 ICInfo::ICInfo(void* start_addr, void* slowpath_rtn_addr, void* continue_addr, StackInfo stack_info, int size,
                llvm::CallingConv::ID calling_conv, LiveOutSet _live_outs, assembler::GenericRegister return_register,
@@ -480,13 +480,13 @@ bool ICInfo::isMegamorphic() {
     return times_rewritten >= IC_MEGAMORPHIC_THRESHOLD;
 }
 
-ICInfo* ICInfo::getICInfoForNode(AST* node) {
+ICInfo* ICInfo::getICInfoForNode(BST* node) {
     auto&& it = ics_by_ast_node.find(node);
     if (it != ics_by_ast_node.end())
         return it->second;
     return NULL;
 }
-void ICInfo::associateNodeWithICInfo(AST* node, std::unique_ptr<TypeRecorder> type_recorder) {
+void ICInfo::associateNodeWithICInfo(BST* node, std::unique_ptr<TypeRecorder> type_recorder) {
     assert(!this->node);
     this->node = node;
     this->type_recorder = std::move(type_recorder);

src/asm_writing/icinfo.h

@@ -104,8 +104,8 @@ private:
     // global ones.
     std::vector<Location> ic_global_decref_locations;
 
-    // associated AST node for this IC
-    AST* node;
+    // associated BST node for this IC
+    BST* node;
 
     // for ICSlotRewrite:
     ICSlotInfo* pickEntryForRewrite(const char* debug_name);
@@ -145,8 +145,8 @@ public:
 
     friend class ICSlotRewrite;
 
-    static ICInfo* getICInfoForNode(AST* node);
-    void associateNodeWithICInfo(AST* node, std::unique_ptr<TypeRecorder> type_recorder);
+    static ICInfo* getICInfoForNode(BST* node);
+    void associateNodeWithICInfo(BST* node, std::unique_ptr<TypeRecorder> type_recorder);
 
     void appendDecrefInfosTo(std::vector<DecrefInfo>& dest_decref_infos);
 };

src/capi/typeobject.cpp

@@ -1972,12 +1972,12 @@ static const slotdef* update_one_slot(BoxedClass* type, const slotdef* p) noexce
                sanity checks.  I'll buy the first person to
                point out a bug in this reasoning a beer. */
         } else if (offset == offsetof(BoxedClass, tp_descr_get) && descr->cls == function_cls
-                   && !static_cast<BoxedFunction*>(descr)->md->always_use_version.empty()) {
-            auto md = static_cast<BoxedFunction*>(descr)->md;
-            if (md->always_use_version.get<CAPI>())
-                specific = md->always_use_version.get<CAPI>();
+                   && !static_cast<BoxedFunction*>(descr)->code->always_use_version.empty()) {
+            auto code = static_cast<BoxedFunction*>(descr)->code;
+            if (code->always_use_version.get<CAPI>())
+                specific = code->always_use_version.get<CAPI>();
             else {
-                type->tpp_descr_get = (descrgetfunc)md->always_use_version.get<CXX>()->code;
+                type->tpp_descr_get = (descrgetfunc)code->always_use_version.get<CXX>()->code;
                 specific = (void*)slot_tp_tpp_descr_get;
             }
         } else if (descr == Py_None && ptr == (void**)&type->tp_hash) {

src/codegen/ast_interpreter.h

@@ -23,13 +23,12 @@ namespace gc {
 class GCVisitor;
 }
 
-class AST_expr;
-class AST_stmt;
-class AST_Jump;
+class BST_expr;
+class BST_stmt;
+class BST_Jump;
 class Box;
 class BoxedClosure;
 class BoxedDict;
-struct FunctionMetadata;
 struct LineInfo;
 
 extern const void* interpreter_instr_addr;
@@ -47,11 +46,11 @@ struct ASTInterpreterJitInterface {
     static int getGlobalsOffset();
 
     static void delNameHelper(void* _interpreter, InternedString name);
-    static Box* derefHelper(void* interp, InternedString s);
+    static Box* derefHelper(void* interp, BST_Name* node);
     static Box* landingpadHelper(void* interp);
     static void pendingCallsCheckHelper();
     static void setExcInfoHelper(void* interp, STOLEN(Box*) type, STOLEN(Box*) value, STOLEN(Box*) traceback);
-    static void setLocalClosureHelper(void* interp, long vreg, InternedString id, Box* v);
+    static void setLocalClosureHelper(void* interp, BST_Name* name, Box* v);
     static void uncacheExcInfoHelper(void* interp);
     static void raise0Helper(void* interp) __attribute__((noreturn));
     static Box* yieldHelper(void* interp, STOLEN(Box*) value);
@@ -76,11 +75,11 @@ struct Value {
     Value(Box* o, RewriterVar* var) : o(o), var(var) {}
 };
 
-Box* astInterpretFunction(FunctionMetadata* f, Box* closure, Box* generator, Box* globals, Box* arg1, Box* arg2,
-                          Box* arg3, Box** args);
-Box* astInterpretFunctionEval(FunctionMetadata* cf, Box* globals, Box* boxedLocals);
+Box* astInterpretFunction(BoxedCode* f, Box* closure, Box* generator, Box* globals, Box* arg1, Box* arg2, Box* arg3,
+                          Box** args);
+Box* astInterpretFunctionEval(BoxedCode* cf, Box* globals, Box* boxedLocals);
 // this function is implemented in the src/codegen/ast_interpreter_exec.S assembler file
-extern "C" Box* astInterpretDeopt(FunctionMetadata* cf, AST_expr* after_expr, AST_stmt* enclosing_stmt, Box* expr_val,
+extern "C" Box* astInterpretDeopt(BoxedCode* cf, BST_expr* after_expr, BST_stmt* enclosing_stmt, Box* expr_val,
                                   STOLEN(FrameStackState) frame_state);
 
 struct FrameInfo;

src/codegen/baseline_jit.h

@@ -28,7 +28,7 @@ namespace pyston {
 #define ENABLE_BASELINEJIT_MAP_32BIT 1
 #define ENABLE_BASELINEJIT_ICS 1
 
-class AST_stmt;
+class BST_stmt;
 class Box;
 class BoxedDict;
 class BoxedList;
@@ -43,7 +43,7 @@ class JitFragmentWriter;
 // It operates on a basic block at a time (=CFGBLock*) and supports very fast switching between the
 // interpreter and the JITed code on every block start/end.
 //
-// To archive this it's tightly integrated with the AST Interpreter and always operates on an ASTInterpreter instance.
+// To archive this it's tightly integrated with the BST Interpreter and always operates on an ASTInterpreter instance.
 // The process works like this:
 //  - in the ASTInterpreter main loop we will check on every basic block start if we have already machine code for the
 //  basic block
@@ -53,7 +53,7 @@ class JitFragmentWriter;
 //  - create/reuse a JitCodeBlock for the function
 //  - create a new JitFragmentWriter for the basic block to JIT
 //  - interpret the basic block and in addition call into corresponding emit* functions of the JitFragmentWriter on
-//      every AST node encountered.
+//      every BST node encountered.
 //  - if a node is encountered which is not supported, abort JITing of the block and blacklist this block
 //  - if we reached the control flow changing node of the basic block (e.g. a branch, return or jump node) we finish
 //     JITing the block.
@@ -127,7 +127,7 @@ class JitFragmentWriter;
 
 // second_JitFragment:
 //      ...
-//    ; this shows how a AST_Return looks like
+//    ; this shows how a BST_Return looks like
 //      xor    %eax,%eax            ; rax contains the next block to interpret.
 //                                    in this case 0 which means we are finished
 //      movabs $0x1270014108,%rdx   ; rdx must contain the Box* value to return
@@ -168,7 +168,7 @@ private:
         uint8_t* get() { return addr; }
     };
 
-    FunctionMetadata* md;
+    BoxedCode* code;
     // the memory block contains the EH frame directly followed by the generated machine code.
     MemoryManager memory;
     int entry_offset;
@@ -183,7 +183,7 @@ private:
 
 
 public:
-    JitCodeBlock(FunctionMetadata* md, llvm::StringRef name);
+    JitCodeBlock(BoxedCode* code, llvm::StringRef name);
     ~JitCodeBlock();
 
     std::unique_ptr<JitFragmentWriter> newFragment(CFGBlock* block, int patch_jump_offset,
@@ -214,7 +214,7 @@ private:
 
     static constexpr int min_patch_size = 13;
 
-    FunctionMetadata* md;
+    BoxedCode* code;
     CFGBlock* block;
     int code_offset; // offset inside the JitCodeBlock to the start of this block
 
@@ -249,7 +249,7 @@ private:
         uint8_t* end_addr;
         std::unique_ptr<ICSetupInfo> ic;
         StackInfo stack_info;
-        AST* node;
+        BST* node;
         std::vector<Location> decref_infos;
         std::unique_ptr<TypeRecorder> type_recorder;
     };
@@ -257,7 +257,7 @@ private:
     llvm::SmallVector<PPInfo, 8> pp_infos;
 
 public:
-    JitFragmentWriter(FunctionMetadata* md, CFGBlock* block, std::unique_ptr<ICInfo> ic_info,
+    JitFragmentWriter(BoxedCode* code, CFGBlock* block, std::unique_ptr<ICInfo> ic_info,
                       std::unique_ptr<ICSlotRewrite> rewrite, int code_offset, int num_bytes_overlapping,
                       void* entry_code, JitCodeBlock& code_block, llvm::DenseSet<int> known_non_null_vregs);
     ~JitFragmentWriter();
@@ -266,29 +266,29 @@ public:
     RewriterVar* imm(uint64_t val);
     RewriterVar* imm(void* val);
 
-    RewriterVar* emitAugbinop(AST_expr* node, RewriterVar* lhs, RewriterVar* rhs, int op_type);
+    RewriterVar* emitAugbinop(BST_expr* node, RewriterVar* lhs, RewriterVar* rhs, int op_type);
     RewriterVar* emitApplySlice(RewriterVar* target, RewriterVar* lower, RewriterVar* upper);
-    RewriterVar* emitBinop(AST_expr* node, RewriterVar* lhs, RewriterVar* rhs, int op_type);
-    RewriterVar* emitCallattr(AST_expr* node, RewriterVar* obj, BoxedString* attr, CallattrFlags flags,
+    RewriterVar* emitBinop(BST_expr* node, RewriterVar* lhs, RewriterVar* rhs, int op_type);
+    RewriterVar* emitCallattr(BST_expr* node, RewriterVar* obj, BoxedString* attr, CallattrFlags flags,
                               const llvm::ArrayRef<RewriterVar*> args, std::vector<BoxedString*>* keyword_names);
-    RewriterVar* emitCompare(AST_expr* node, RewriterVar* lhs, RewriterVar* rhs, int op_type);
+    RewriterVar* emitCompare(BST_expr* node, RewriterVar* lhs, RewriterVar* rhs, int op_type);
     RewriterVar* emitCreateDict();
     void emitDictSet(RewriterVar* dict, RewriterVar* k, RewriterVar* v);
     RewriterVar* emitCreateList(const llvm::ArrayRef<STOLEN(RewriterVar*)> values);
     RewriterVar* emitCreateSet(const llvm::ArrayRef<RewriterVar*> values);
     RewriterVar* emitCreateSlice(RewriterVar* start, RewriterVar* stop, RewriterVar* step);
     RewriterVar* emitCreateTuple(const llvm::ArrayRef<RewriterVar*> values);
-    RewriterVar* emitDeref(InternedString s);
+    RewriterVar* emitDeref(BST_Name* name);
     RewriterVar* emitExceptionMatches(RewriterVar* v, RewriterVar* cls);
-    RewriterVar* emitGetAttr(RewriterVar* obj, BoxedString* s, AST_expr* node);
-    RewriterVar* emitGetBlockLocal(InternedString s, int vreg);
-    void emitKillTemporary(InternedString s, int vreg);
+    RewriterVar* emitGetAttr(RewriterVar* obj, BoxedString* s, BST_expr* node);
+    RewriterVar* emitGetBlockLocal(BST_Name* name);
+    void emitKillTemporary(BST_Name* name);
     RewriterVar* emitGetBoxedLocal(BoxedString* s);
     RewriterVar* emitGetBoxedLocals();
     RewriterVar* emitGetClsAttr(RewriterVar* obj, BoxedString* s);
     RewriterVar* emitGetGlobal(BoxedString* s);
-    RewriterVar* emitGetItem(AST_expr* node, RewriterVar* value, RewriterVar* slice);
-    RewriterVar* emitGetLocal(InternedString s, int vreg);
+    RewriterVar* emitGetItem(BST_expr* node, RewriterVar* value, RewriterVar* slice);
+    RewriterVar* emitGetLocal(BST_Name* name);
     RewriterVar* emitGetPystonIter(RewriterVar* v);
     RewriterVar* emitHasnext(RewriterVar* v);
     RewriterVar* emitImportFrom(RewriterVar* module, BoxedString* name);
@@ -298,7 +298,7 @@ public:
     RewriterVar* emitNonzero(RewriterVar* v);
     RewriterVar* emitNotNonzero(RewriterVar* v);
     RewriterVar* emitRepr(RewriterVar* v);
-    RewriterVar* emitRuntimeCall(AST_expr* node, RewriterVar* obj, ArgPassSpec argspec,
+    RewriterVar* emitRuntimeCall(BST_expr* node, RewriterVar* obj, ArgPassSpec argspec,
                                  const llvm::ArrayRef<RewriterVar*> args, std::vector<BoxedString*>* keyword_names);
     RewriterVar* emitUnaryop(RewriterVar* v, int op_type);
     std::vector<RewriterVar*> emitUnpackIntoArray(RewriterVar* v, uint64_t num);
@@ -311,20 +311,20 @@ public:
     void emitDelName(InternedString name);
     void emitExec(RewriterVar* code, RewriterVar* globals, RewriterVar* locals, FutureFlags flags);
     void emitJump(CFGBlock* b);
-    void emitOSRPoint(AST_Jump* node);
+    void emitOSRPoint(BST_Jump* node);
     void emitPendingCallsCheck();
     void emitPrint(RewriterVar* dest, RewriterVar* var, bool nl);
     void emitRaise0();
     void emitRaise3(RewriterVar* arg0, RewriterVar* arg1, RewriterVar* arg2);
     void emitReturn(RewriterVar* v);
-    void emitSetAttr(AST_expr* node, RewriterVar* obj, BoxedString* s, STOLEN(RewriterVar*) attr);
-    void emitSetBlockLocal(InternedString s, int vreg, STOLEN(RewriterVar*) v);
-    void emitSetCurrentInst(AST_stmt* node);
+    void emitSetAttr(BST_expr* node, RewriterVar* obj, BoxedString* s, STOLEN(RewriterVar*) attr);
+    void emitSetBlockLocal(BST_Name* name, STOLEN(RewriterVar*) v);
+    void emitSetCurrentInst(BST_stmt* node);
     void emitSetExcInfo(RewriterVar* type, RewriterVar* value, RewriterVar* traceback);
     void emitSetGlobal(BoxedString* s, STOLEN(RewriterVar*) v, bool are_globals_from_module);
     void emitSetItemName(BoxedString* s, RewriterVar* v);
     void emitSetItem(RewriterVar* target, RewriterVar* slice, RewriterVar* value);
-    void emitSetLocal(InternedString s, int vreg, bool set_closure, STOLEN(RewriterVar*) v);
+    void emitSetLocal(BST_Name* name, bool set_closure, STOLEN(RewriterVar*) v);
     // emitSideExit steals a full ref from v, not just a vref
     void emitSideExit(STOLEN(RewriterVar*) v, Box* cmp_value, CFGBlock* next_block);
     void emitUncacheExcInfo();
@@ -351,7 +351,7 @@ private:
                                            const llvm::ArrayRef<RewriterVar*> additional_uses);
     std::pair<RewriterVar*, RewriterAction*> emitPPCall(void* func_addr, llvm::ArrayRef<RewriterVar*> args,
                                                         unsigned short pp_size, bool should_record_type = false,
-                                                        AST* ast_node = NULL,
+                                                        BST* bst_node = NULL,
                                                         llvm::ArrayRef<RewriterVar*> additional_uses = {});
 
     static void assertNameDefinedHelper(const char* id);
@@ -371,7 +371,7 @@ private:
     void _emitJump(CFGBlock* b, RewriterVar* block_next, ExitInfo& exit_info);
     void _emitOSRPoint();
     void _emitPPCall(RewriterVar* result, void* func_addr, llvm::ArrayRef<RewriterVar*> args, unsigned short pp_size,
-                     AST* ast_node, llvm::ArrayRef<RewriterVar*> vars_to_bump);
+                     BST* bst_node, llvm::ArrayRef<RewriterVar*> vars_to_bump);
     void _emitRecordType(RewriterVar* obj_cls_var);
     void _emitReturn(RewriterVar* v);
     void _emitSideExit(STOLEN(RewriterVar*) var, RewriterVar* val_constant, CFGBlock* next_block,

src/codegen/codegen.cpp

@@ -29,50 +29,17 @@
 #include "analysis/scoping_analysis.h"
 #include "codegen/baseline_jit.h"
 #include "codegen/compvars.h"
-#include "core/ast.h"
+#include "core/bst.h"
 #include "core/cfg.h"
 #include "core/util.h"
-#include "runtime/code.h"
 #include "runtime/types.h"
 
 namespace pyston {
 
-FunctionMetadata::FunctionMetadata(int num_args, bool takes_varargs, bool takes_kwargs,
-                                   std::unique_ptr<SourceInfo> source)
-    : code_obj(NULL),
-      source(std::move(source)),
-      param_names(this->source->ast, this->source->getInternedStrings()),
-      takes_varargs(takes_varargs),
-      takes_kwargs(takes_kwargs),
-      num_args(num_args),
-      times_interpreted(0),
-      internal_callable(NULL, NULL) {
-}
-
-FunctionMetadata::FunctionMetadata(int num_args, bool takes_varargs, bool takes_kwargs, const ParamNames& param_names)
-    : code_obj(NULL),
-      source(nullptr),
-      param_names(param_names),
-      takes_varargs(takes_varargs),
-      takes_kwargs(takes_kwargs),
-      num_args(num_args),
-      times_interpreted(0),
-      internal_callable(NULL, NULL) {
-}
-
-BORROWED(BoxedCode*) FunctionMetadata::getCode() {
-    if (!code_obj) {
-        code_obj = new BoxedCode(this);
-        // FunctionMetadatas don't currently participate in GC.  They actually never get freed currently.
-        constants.push_back(code_obj);
-    }
-    return code_obj;
-}
-
-void FunctionMetadata::addVersion(CompiledFunction* compiled) {
+void BoxedCode::addVersion(CompiledFunction* compiled) {
     assert(compiled);
     assert((compiled->spec != NULL) + (compiled->entry_descriptor != NULL) == 1);
-    assert(compiled->md);
+    assert(compiled->code_obj);
     assert(compiled->code);
 
     if (compiled->entry_descriptor == NULL) {
@@ -91,31 +58,14 @@ void FunctionMetadata::addVersion(CompiledFunction* compiled) {
     }
 }
 
-SourceInfo::SourceInfo(BoxedModule* m, ScopingAnalysis* scoping, FutureFlags future_flags, AST* ast, BoxedString* fn)
-    : parent_module(m), scoping(scoping), scope_info(NULL), ast(ast), cfg(NULL), future_flags(future_flags) {
-    assert(fn);
-
-    // TODO: this is a very bad way of handling this:
-    incref(fn);
-    late_constants.push_back(fn);
-
-    this->fn = fn;
-
-    switch (ast->type) {
-        case AST_TYPE::ClassDef:
-        case AST_TYPE::Module:
-        case AST_TYPE::Expression:
-        case AST_TYPE::Suite:
-            is_generator = false;
-            break;
-        case AST_TYPE::FunctionDef:
-        case AST_TYPE::Lambda:
-            is_generator = containsYield(ast);
-            break;
-        default:
-            RELEASE_ASSERT(0, "Unknown type: %d", ast->type);
-            break;
-    }
+SourceInfo::SourceInfo(BoxedModule* m, ScopingResults scoping, FutureFlags future_flags, int ast_type,
+                       bool is_generator)
+    : parent_module(m),
+      scoping(std::move(scoping)),
+      cfg(NULL),
+      future_flags(future_flags),
+      is_generator(is_generator),
+      ast_type(ast_type) {
 }
 
 SourceInfo::~SourceInfo() {

src/codegen/codegen.h

@@ -73,9 +73,9 @@ struct GlobalState {
     llvm::Type* llvm_value_type, *llvm_value_type_ptr, *llvm_value_type_ptr_ptr;
     llvm::Type* llvm_class_type, *llvm_class_type_ptr;
     llvm::Type* llvm_opaque_type;
-    llvm::Type* llvm_boxedstring_type_ptr, *llvm_dict_type_ptr, *llvm_aststmt_type_ptr, *llvm_astexpr_type_ptr;
+    llvm::Type* llvm_boxedstring_type_ptr, *llvm_dict_type_ptr, *llvm_bststmt_type_ptr, *llvm_bstexpr_type_ptr;
     llvm::Type* llvm_frame_info_type;
-    llvm::Type* llvm_functionmetadata_type_ptr, *llvm_closure_type_ptr, *llvm_generator_type_ptr;
+    llvm::Type* llvm_code_type_ptr, *llvm_closure_type_ptr, *llvm_generator_type_ptr;
     llvm::Type* llvm_module_type_ptr, *llvm_bool_type_ptr;
     llvm::Type* llvm_excinfo_type;
     llvm::Type* i1, *i8, *i8_ptr, *i32, *i64, *void_, *double_;

src/codegen/compvars.cpp

@@ -26,6 +26,7 @@
 #include "codegen/irgen.h"
 #include "codegen/irgen/util.h"
 #include "codegen/patchpoints.h"
+#include "core/bst.h"
 #include "core/options.h"
 #include "core/types.h"
 #include "runtime/float.h"
@@ -837,9 +838,9 @@ ConcreteCompilerVariable* UnknownType::hasnext(IREmitter& emitter, const OpInfo&
     return boolFromI1(emitter, rtn_val);
 }
 
-CompilerVariable* makeFunction(IREmitter& emitter, FunctionMetadata* f, llvm::Value* closure, llvm::Value* globals,
+CompilerVariable* makeFunction(IREmitter& emitter, BoxedCode* code, llvm::Value* closure, llvm::Value* globals,
                                const std::vector<ConcreteCompilerVariable*>& defaults) {
-    // Unlike the FunctionMetadata*, which can be shared between recompilations, the Box* around it
+    // Unlike the BoxedCode*, which can be shared between recompilations, the Box* around it
     // should be created anew every time the functiondef is encountered
 
     ConcreteCompilerVariable* convertedClosure = NULL;
@@ -872,8 +873,8 @@ CompilerVariable* makeFunction(IREmitter& emitter, FunctionMetadata* f, llvm::Va
     // emitter.createCall().
     llvm::Instruction* boxed = emitter.getBuilder()->CreateCall(
         g.funcs.createFunctionFromMetadata,
-        std::vector<llvm::Value*>{ embedRelocatablePtr(f, g.llvm_functionmetadata_type_ptr), closure, globals, scratch,
-                                   getConstantInt(defaults.size(), g.i64) });
+        std::vector<llvm::Value*>{ emitter.setType(embedRelocatablePtr(code, g.llvm_code_type_ptr), RefType::BORROWED),
+                                   closure, globals, scratch, getConstantInt(defaults.size(), g.i64) });
     emitter.setType(boxed, RefType::OWNED);
 
     // The refcounter needs to know that this call "uses" the arguments that got passed via scratch.
@@ -946,12 +947,12 @@ public:
 
     static CompilerType* fromRT(BoxedFunction* rtfunc, bool stripfirst) {
         std::vector<Sig*> sigs;
-        FunctionMetadata* md = rtfunc->md;
+        BoxedCode* code = rtfunc->code;
 
         assert(!rtfunc->can_change_defaults);
 
-        for (int i = 0; i < md->versions.size(); i++) {
-            CompiledFunction* cf = md->versions[i];
+        for (int i = 0; i < code->versions.size(); i++) {
+            CompiledFunction* cf = code->versions[i];
 
             FunctionSpecialization* fspec = cf->spec;
 
@@ -1865,14 +1866,14 @@ public:
         // but I don't think we should be running into that case.
         RELEASE_ASSERT(!rtattr_func->can_change_defaults, "could handle this but unexpected");
 
-        FunctionMetadata* md = rtattr_func->md;
-        assert(md);
+        BoxedCode* code = rtattr_func->code;
+        assert(code);
 
         ParamReceiveSpec paramspec = rtattr_func->getParamspec();
-        if (md->takes_varargs || paramspec.takes_kwargs)
+        if (code->takes_varargs || paramspec.takes_kwargs)
             return NULL;
 
-        RELEASE_ASSERT(paramspec.num_args == md->numReceivedArgs(), "");
+        RELEASE_ASSERT(paramspec.num_args == code->numReceivedArgs(), "");
         RELEASE_ASSERT(args.size() + 1 >= paramspec.num_args - paramspec.num_defaults
                            && args.size() + 1 <= paramspec.num_args,
                        "%d", info.unw_info.current_stmt->lineno);
@@ -1881,9 +1882,9 @@ public:
         CompiledFunction* best_exception_mismatch = NULL;
         bool found = false;
         // TODO have to find the right version.. similar to resolveclfunc?
-        for (int i = 0; i < md->versions.size(); i++) {
-            cf = md->versions[i];
-            assert(cf->spec->arg_types.size() == md->numReceivedArgs());
+        for (int i = 0; i < code->versions.size(); i++) {
+            cf = code->versions[i];
+            assert(cf->spec->arg_types.size() == code->numReceivedArgs());
 
             bool fits = true;
             for (int j = 0; j < args.size(); j++) {
@@ -1915,7 +1916,7 @@ public:
         RELEASE_ASSERT(cf->code, "");
 
         std::vector<llvm::Type*> arg_types;
-        RELEASE_ASSERT(paramspec.num_args == md->numReceivedArgs(), "");
+        RELEASE_ASSERT(paramspec.num_args == code->numReceivedArgs(), "");
         for (int i = 0; i < paramspec.num_args; i++) {
             // TODO support passing unboxed values as arguments
             assert(cf->spec->arg_types[i]->llvmType() == g.llvm_value_type_ptr);
@@ -1930,7 +1931,7 @@ public:
         llvm::FunctionType* ft = llvm::FunctionType::get(cf->spec->rtn_type->llvmType(), arg_types, false);
 
         llvm::Value* linked_function;
-        if (cf->md->source) // for JITed functions we need to make the desination address relocatable.
+        if (cf->code_obj->source) // for JITed functions we need to make the desination address relocatable.
             linked_function = embedRelocatablePtr(cf->code, ft->getPointerTo());
         else
             linked_function = embedConstantPtr(cf->code, ft->getPointerTo());

src/codegen/compvars.h

@@ -362,7 +362,7 @@ UnboxedSlice extractSlice(CompilerVariable* slice);
 #if 0
 CompilerVariable* makeUnicode(IREmitter& emitter, llvm::StringRef);
 #endif
-CompilerVariable* makeFunction(IREmitter& emitter, FunctionMetadata*, llvm::Value* closure, llvm::Value* globals,
+CompilerVariable* makeFunction(IREmitter& emitter, BoxedCode*, llvm::Value* closure, llvm::Value* globals,
                                const std::vector<ConcreteCompilerVariable*>& defaults);
 ConcreteCompilerVariable* undefVariable();
 CompilerVariable* makeTuple(const std::vector<CompilerVariable*>& elts);

src/codegen/cpython_ast.h

@@ -25,7 +25,7 @@ namespace pyston {
 // Convert a CPython ast object to a Pyston ast object.
 // This will also check for certain kinds of "syntax errors" (ex continue not in loop) and will
 // throw them as C++ exceptions.
-AST* cpythonToPystonAST(mod_ty mod, llvm::StringRef fn);
+std::pair<AST*, std::unique_ptr<ASTAllocator>> cpythonToPystonAST(mod_ty mod, llvm::StringRef fn);
 }
 
 #endif

src/codegen/irgen.h

@@ -29,15 +29,15 @@
 
 namespace pyston {
 
-class AST_expr;
-class AST_stmt;
+class BST_expr;
+class BST_stmt;
 class CFGBlock;
 class GCBuilder;
 class IREmitter;
 
 struct UnwindInfo {
 public:
-    AST_stmt* current_stmt;
+    BST_stmt* current_stmt;
 
     llvm::BasicBlock* exc_dest;
 
@@ -46,7 +46,7 @@ public:
 
     bool hasHandler() const { return exc_dest != NULL; }
 
-    UnwindInfo(AST_stmt* current_stmt, llvm::BasicBlock* exc_dest, bool is_after_deopt = false)
+    UnwindInfo(BST_stmt* current_stmt, llvm::BasicBlock* exc_dest, bool is_after_deopt = false)
         : current_stmt(current_stmt), exc_dest(exc_dest), is_after_deopt(is_after_deopt) {}
 
     ExceptionStyle preferredExceptionStyle() const;
@@ -119,7 +119,7 @@ public:
     // virtual void checkAndPropagateCapiException(const UnwindInfo& unw_info, llvm::Value* returned_val,
     // llvm::Value* exc_val, bool double_check = false) = 0;
 
-    virtual llvm::Value* createDeopt(AST_stmt* current_stmt, AST_expr* node, llvm::Value* node_value) = 0;
+    virtual llvm::Value* createDeopt(BST_stmt* current_stmt, BST_expr* node, llvm::Value* node_value) = 0;
 
     virtual BORROWED(Box*) getIntConstant(int64_t n) = 0;
     virtual BORROWED(Box*) getFloatConstant(double d) = 0;
@@ -138,7 +138,7 @@ extern const std::string PASSED_GENERATOR_NAME;
 InternedString getIsDefinedName(InternedString name, InternedStringPool& interned_strings);
 bool isIsDefinedName(llvm::StringRef name);
 
-std::pair<CompiledFunction*, llvm::Function*> doCompile(FunctionMetadata* md, SourceInfo* source,
+std::pair<CompiledFunction*, llvm::Function*> doCompile(BoxedCode* code, SourceInfo* source,
                                                         const ParamNames* param_names,
                                                         const OSREntryDescriptor* entry_descriptor, EffortLevel effort,
                                                         ExceptionStyle exception_style, FunctionSpecialization* spec,

src/codegen/irgen/future.h

@@ -21,6 +21,8 @@
 
 namespace pyston {
 
+class AST_stmt;
+
 // Loop through import statements to find __future__ imports throwing errors for
 // bad __future__ imports. Returns the futures that are turned on. This is used
 // for irgeneration; the parser still has to handle some futures on its own,

src/codegen/irgen/hooks.h

@@ -22,7 +22,6 @@
 namespace pyston {
 
 struct CompiledFunction;
-class FunctionMetadata;
 class OSRExit;
 class Box;
 class BoxedDict;

src/codegen/parser.h

@@ -20,10 +20,11 @@
 namespace pyston {
 
 class AST_Module;
+class ASTAllocator;
 
-AST_Module* parse_string(const char* code, FutureFlags inherited_flags);
-AST_Module* parse_file(const char* fn, FutureFlags inherited_flags);
-AST_Module* caching_parse_file(const char* fn, FutureFlags inherited_flags);
+std::pair<AST_Module*, std::unique_ptr<ASTAllocator>> parse_string(const char* code, FutureFlags inherited_flags);
+std::pair<AST_Module*, std::unique_ptr<ASTAllocator>> parse_file(const char* fn, FutureFlags inherited_flags);
+std::pair<AST_Module*, std::unique_ptr<ASTAllocator>> caching_parse_file(const char* fn, FutureFlags inherited_flags);
 }
 
 #endif

src/codegen/patchpoints.cpp

@@ -95,12 +95,12 @@ void PatchpointInfo::parseLocationMap(StackMap::Record* r, LocationMap* map) {
                                                           .locations = std::move(locations) });
     };
 
-    auto&& source = parentFunction()->md->source;
+    auto&& source = parentFunction()->code_obj->source;
     if (source->is_generator)
         map->generator.locations.push_back(parse_type(GENERATOR));
-    if (source->getScopeInfo()->takesClosure())
+    if (source->scoping.takesClosure())
         map->passed_closure.locations.push_back(parse_type(CLOSURE));
-    if (source->getScopeInfo()->createsClosure())
+    if (source->scoping.createsClosure())
         map->created_closure.locations.push_back(parse_type(CLOSURE));
 
     for (FrameVarInfo& frame_var : frame_info_desc.vars) {