Create all FunctionMetadatas for a module at once

There are a bunch of interrelated changes in here that I couldn't really
separate out.  Mostly the relate to saving data on the AST nodes: we store
DerefInfo and closure info on the Name nodes, and store FunctionMetadata objects
on AST nodes that create scopes.  And those FunctionMetadata objects now also
store "ScopingResults", which is only the results from the ScopeInfo objects,
not all the extra info those keep around in order to do deferred analysis.

Once all this extra data is precomputed+stored, we can start freeing
the ScopingAnalysis objects and all the related ScopeInfos.

Future work:
- Name node shouldn't store scoing info; this should be embedded in choice of BST nodes
- Do pyc caching post-CFG
- Memory management.
- Continue to clean this all up.  I layered this change on top of the existing
  scoping and cfg systems, which reduced the amount of changes, but means that
  there is a potentially-unnecessary layer in the middle now.

src/analysis/function_analysis.cpp

@@ -214,10 +214,8 @@ class DefinednessBBAnalyzer : public BBAnalyzer<DefinednessAnalysis::DefinitionL
 private:
     typedef DefinednessAnalysis::DefinitionLevel DefinitionLevel;
 
-    ScopeInfo* scope_info;
-
 public:
-    DefinednessBBAnalyzer(ScopeInfo* scope_info) : scope_info(scope_info) {}
+    DefinednessBBAnalyzer() {}
 
     virtual DefinitionLevel merge(DefinitionLevel from, DefinitionLevel into) const {
         assert(from != DefinitionLevel::Unknown);
@@ -374,8 +372,7 @@ void DefinednessBBAnalyzer::processBB(Map& starting, CFGBlock* block) const {
     }
 }
 
-void DefinednessAnalysis::run(VRegMap<DefinednessAnalysis::DefinitionLevel> initial_map, CFGBlock* initial_block,
-                              ScopeInfo* scope_info) {
+void DefinednessAnalysis::run(VRegMap<DefinednessAnalysis::DefinitionLevel> initial_map, CFGBlock* initial_block) {
     Timer _t("DefinednessAnalysis()", 10);
 
     // Don't run this twice:
@@ -386,8 +383,8 @@ void DefinednessAnalysis::run(VRegMap<DefinednessAnalysis::DefinitionLevel> init
     assert(initial_map.numVregs() == nvregs);
 
     auto&& vreg_info = cfg->getVRegInfo();
-    computeFixedPoint(std::move(initial_map), initial_block, DefinednessBBAnalyzer(scope_info), false,
-                      defined_at_beginning, defined_at_end);
+    computeFixedPoint(std::move(initial_map), initial_block, DefinednessBBAnalyzer(), false, defined_at_beginning,
+                      defined_at_end);
 
     for (const auto& p : defined_at_end) {
         assert(p.second.numVregs() == nvregs);
@@ -398,14 +395,6 @@ void DefinednessAnalysis::run(VRegMap<DefinednessAnalysis::DefinitionLevel> init
         // required.resize(nvregs, /* value= */ false);
 
         for (int vreg = 0; vreg < nvregs; vreg++) {
-#ifndef NDEBUG
-            if (vreg_info.vregHasName(vreg)) {
-                ScopeInfo::VarScopeType vst = scope_info->getScopeTypeOfName(vreg_info.getName(vreg));
-                ASSERT(vst != ScopeInfo::VarScopeType::GLOBAL && vst != ScopeInfo::VarScopeType::NAME, "%s",
-                       vreg_info.getName(vreg).c_str());
-            }
-#endif
-
             auto status = p.second[vreg];
             // assert(p.second.count(name));
             // auto status = p.second.find(name)->second;
@@ -431,7 +420,7 @@ const VRegSet& DefinednessAnalysis::getDefinedVregsAtEnd(CFGBlock* block) {
 }
 
 PhiAnalysis::PhiAnalysis(VRegMap<DefinednessAnalysis::DefinitionLevel> initial_map, CFGBlock* initial_block,
-                         bool initials_need_phis, LivenessAnalysis* liveness, ScopeInfo* scope_info)
+                         bool initials_need_phis, LivenessAnalysis* liveness)
     : definedness(), empty_set(initial_map.numVregs()), liveness(liveness) {
     auto cfg = initial_block->cfg;
     auto&& vreg_info = cfg->getVRegInfo();
@@ -443,7 +432,7 @@ PhiAnalysis::PhiAnalysis(VRegMap<DefinednessAnalysis::DefinitionLevel> initial_m
     int num_vregs = initial_map.numVregs();
     assert(num_vregs == vreg_info.getTotalNumOfVRegs());
 
-    definedness.run(std::move(initial_map), initial_block, scope_info);
+    definedness.run(std::move(initial_map), initial_block);
 
     Timer _t("PhiAnalysis()", 10);
 
@@ -534,8 +523,7 @@ std::unique_ptr<LivenessAnalysis> computeLivenessInfo(CFG* cfg) {
     return std::unique_ptr<LivenessAnalysis>(new LivenessAnalysis(cfg));
 }
 
-std::unique_ptr<PhiAnalysis> computeRequiredPhis(const ParamNames& args, CFG* cfg, LivenessAnalysis* liveness,
-                                                 ScopeInfo* scope_info) {
+std::unique_ptr<PhiAnalysis> computeRequiredPhis(const ParamNames& args, CFG* cfg, LivenessAnalysis* liveness) {
     static StatCounter counter("num_phi_analysis");
     counter.log();
 
@@ -562,11 +550,11 @@ std::unique_ptr<PhiAnalysis> computeRequiredPhis(const ParamNames& args, CFG* cf
     assert(initial_map.numVregs() == vreg_info.getTotalNumOfVRegs());
 
     return std::unique_ptr<PhiAnalysis>(
-        new PhiAnalysis(std::move(initial_map), cfg->getStartingBlock(), false, liveness, scope_info));
+        new PhiAnalysis(std::move(initial_map), cfg->getStartingBlock(), false, liveness));
 }
 
-std::unique_ptr<PhiAnalysis> computeRequiredPhis(const OSREntryDescriptor* entry_descriptor, LivenessAnalysis* liveness,
-                                                 ScopeInfo* scope_info) {
+std::unique_ptr<PhiAnalysis> computeRequiredPhis(const OSREntryDescriptor* entry_descriptor,
+                                                 LivenessAnalysis* liveness) {
     static StatCounter counter("num_phi_analysis");
     counter.log();
 
@@ -588,6 +576,6 @@ std::unique_ptr<PhiAnalysis> computeRequiredPhis(const OSREntryDescriptor* entry
     }
 
     return std::unique_ptr<PhiAnalysis>(
-        new PhiAnalysis(std::move(initial_map), entry_descriptor->backedge->target, true, liveness, scope_info));
+        new PhiAnalysis(std::move(initial_map), entry_descriptor->backedge->target, true, liveness));
 }
 }

src/analysis/function_analysis.h

@@ -31,7 +31,6 @@ class AST_Jump;
 class AST_Name;
 class CFG;
 class CFGBlock;
-class ScopeInfo;
 class LivenessBBVisitor;
 
 class LivenessAnalysis {
@@ -72,7 +71,7 @@ private:
 public:
     DefinednessAnalysis() {}
 
-    void run(VRegMap<DefinitionLevel> initial_map, CFGBlock* initial_block, ScopeInfo* scope_info);
+    void run(VRegMap<DefinitionLevel> initial_map, CFGBlock* initial_block);
 
     DefinitionLevel isDefinedAtEnd(int vreg, CFGBlock* block);
     const VRegSet& getDefinedVregsAtEnd(CFGBlock* block);
@@ -94,7 +93,7 @@ public:
     // Initials_need_phis specifies that initial_map should count as an additional entry point
     // that may require phis.
     PhiAnalysis(VRegMap<DefinednessAnalysis::DefinitionLevel> initial_map, CFGBlock* initial_block,
-                bool initials_need_phis, LivenessAnalysis* liveness, ScopeInfo* scope_info);
+                bool initials_need_phis, LivenessAnalysis* liveness);
 
     bool isRequired(int vreg, CFGBlock* block);
     bool isRequiredAfter(int vreg, CFGBlock* block);
@@ -107,8 +106,8 @@ public:
 };
 
 std::unique_ptr<LivenessAnalysis> computeLivenessInfo(CFG*);
-std::unique_ptr<PhiAnalysis> computeRequiredPhis(const ParamNames&, CFG*, LivenessAnalysis*, ScopeInfo* scope_info);
-std::unique_ptr<PhiAnalysis> computeRequiredPhis(const OSREntryDescriptor*, LivenessAnalysis*, ScopeInfo* scope_info);
+std::unique_ptr<PhiAnalysis> computeRequiredPhis(const ParamNames&, CFG*, LivenessAnalysis*);
+std::unique_ptr<PhiAnalysis> computeRequiredPhis(const OSREntryDescriptor*, LivenessAnalysis*);
 }
 
 #endif

src/analysis/scoping_analysis.cpp

@@ -24,6 +24,28 @@
 
 namespace pyston {
 
+ScopingResults::ScopingResults(ScopeInfo* scope_info, bool globals_from_module)
+    : are_locals_from_module(scope_info->areLocalsFromModule()),
+      are_globals_from_module(globals_from_module),
+      creates_closure(scope_info->createsClosure()),
+      takes_closure(scope_info->takesClosure()),
+      passes_through_closure(scope_info->passesThroughClosure()),
+      uses_name_lookup(scope_info->usesNameLookup()),
+      closure_size(creates_closure ? scope_info->getClosureSize() : 0) {
+    deref_info = scope_info->getAllDerefVarsAndInfo();
+}
+
+DerefInfo ScopingResults::getDerefInfo(AST_Name* node) const {
+    assert(node->lookup_type == ScopeInfo::VarScopeType::DEREF);
+    assert(node->deref_info.offset != INT_MAX);
+    return node->deref_info;
+}
+size_t ScopingResults::getClosureOffset(AST_Name* node) const {
+    assert(node->lookup_type == ScopeInfo::VarScopeType::CLOSURE);
+    assert(node->closure_offset != -1);
+    return node->closure_offset;
+}
+
 class YieldVisitor : public NoopASTVisitor {
 public:
     AST* starting_node;
@@ -933,46 +955,22 @@ InternedStringPool& ScopingAnalysis::getInternedStrings() {
     return *interned_strings;
 }
 
-std::unique_ptr<ScopeInfo> ScopingAnalysis::analyzeSubtree(AST* node) {
+void ScopingAnalysis::analyzeSubtree(AST* node) {
     NameUsageMap usages;
     usages[node] = new ScopeNameUsage(node, NULL, this);
     NameCollectorVisitor::collect(node, &usages, this);
 
     processNameUsages(&usages);
-
-    std::unique_ptr<ScopeInfo> rtn = std::move(scopes[node]);
-    // scopes.erase(node);
-    assert(rtn);
-    return rtn;
 }
 
-void ScopingAnalysis::registerScopeReplacement(AST* original_node, AST* new_node) {
-    assert(scope_replacements.count(original_node) == 0);
-    assert(scope_replacements.count(new_node) == 0);
-    assert(scopes.count(new_node) == 0);
-
-#ifndef NDEBUG
-    // NULL this out just to make sure it doesn't get accessed:
-    scopes[new_node] = NULL;
-#endif
-
-    scope_replacements[new_node] = original_node;
-}
-
-std::unique_ptr<ScopeInfo> ScopingAnalysis::getScopeInfoForNode(AST* node) {
+ScopeInfo* ScopingAnalysis::getScopeInfoForNode(AST* node) {
     assert(node);
 
-    auto it = scope_replacements.find(node);
-    if (it != scope_replacements.end())
-        node = it->second;
-
-    if (scopes.count(node)) {
-        std::unique_ptr<ScopeInfo> rtn = std::move(scopes[node]);
-        assert(rtn);
-        return rtn;
-    }
+    if (!scopes.count(node))
+        analyzeSubtree(node);
 
-    return analyzeSubtree(node);
+    assert(scopes.count(node));
+    return scopes[node].get();
 }
 
 ScopingAnalysis::ScopingAnalysis(AST* ast, bool globals_from_module)

src/analysis/scoping_analysis.h

@@ -19,6 +19,7 @@
 
 #include "core/common.h"
 #include "core/stringpool.h"
+#include "core/types.h"
 
 namespace pyston {
 
@@ -27,16 +28,6 @@ class AST_Module;
 class AST_Expression;
 class AST_Suite;
 
-// Each closure has an array (fixed-size for that particular scope) of variables
-// and a parent pointer to a parent closure. To look up a variable from the passed-in
-// closure (i.e., DEREF), you just need to know (i) how many parents up to go and
-// (ii) what offset into the array to find the variable. This struct stores that
-// information. You can query the ScopeInfo with a name to get this info.
-struct DerefInfo {
-    size_t num_parents_from_passed_closure;
-    size_t offset;
-};
-
 class ScopeInfo {
 public:
     ScopeInfo() {}
@@ -155,25 +146,14 @@ private:
     AST_Module* parent_module;
     InternedStringPool* interned_strings;
 
-    llvm::DenseMap<AST*, AST*> scope_replacements;
-
-    std::unique_ptr<ScopeInfo> analyzeSubtree(AST* node);
+    void analyzeSubtree(AST* node);
     void processNameUsages(NameUsageMap* usages);
 
     bool globals_from_module;
 
 public:
-    // The scope-analysis is done before any CFG-ization is done,
-    // but many of the queries will be done post-CFG-ization.
-    // The CFG process can replace scope AST nodes with others (ex:
-    // generator expressions with generator functions), so we need to
-    // have a way of mapping the original analysis with the new queries.
-    // This is a hook for the CFG process to register when it has replaced
-    // a scope-node with a different node.
-    void registerScopeReplacement(AST* original_node, AST* new_node);
-
     ScopingAnalysis(AST* ast, bool globals_from_module);
-    std::unique_ptr<ScopeInfo> getScopeInfoForNode(AST* node);
+    ScopeInfo* getScopeInfoForNode(AST* node);
 
     InternedStringPool& getInternedStrings();
     bool areGlobalsFromModule() { return globals_from_module; }

src/analysis/type_analysis.cpp

@@ -93,17 +93,14 @@ private:
     ExprTypeMap& expr_types;
     TypeSpeculations& type_speculations;
     TypeAnalysis::SpeculationLevel speculation;
-    ScopeInfo* scope_info;
 
     BasicBlockTypePropagator(CFGBlock* block, TypeMap& initial, ExprTypeMap& expr_types,
-                             TypeSpeculations& type_speculations, TypeAnalysis::SpeculationLevel speculation,
-                             ScopeInfo* scope_info)
+                             TypeSpeculations& type_speculations, TypeAnalysis::SpeculationLevel speculation)
         : block(block),
           sym_table(initial),
           expr_types(expr_types),
           type_speculations(type_speculations),
-          speculation(speculation),
-          scope_info(scope_info) {}
+          speculation(speculation) {}
 
     void run() {
         for (int i = 0; i < block->body.size(); i++) {
@@ -429,8 +426,7 @@ private:
     }
 
     void* visit_name(AST_Name* node) override {
-        if (node->lookup_type == ScopeInfo::VarScopeType::UNKNOWN)
-            node->lookup_type = scope_info->getScopeTypeOfName(node->id);
+        assert(node->lookup_type != ScopeInfo::VarScopeType::UNKNOWN);
         auto name_scope = node->lookup_type;
 
         if (name_scope == ScopeInfo::VarScopeType::GLOBAL) {
@@ -671,10 +667,9 @@ private:
 
 public:
     static TypeMap propagate(CFGBlock* block, const TypeMap& starting, ExprTypeMap& expr_types,
-                             TypeSpeculations& type_speculations, TypeAnalysis::SpeculationLevel speculation,
-                             ScopeInfo* scope_info) {
+                             TypeSpeculations& type_speculations, TypeAnalysis::SpeculationLevel speculation) {
         TypeMap ending = starting;
-        BasicBlockTypePropagator(block, ending, expr_types, type_speculations, speculation, scope_info).run();
+        BasicBlockTypePropagator(block, ending, expr_types, type_speculations, speculation).run();
         return ending;
     }
 };
@@ -753,8 +748,8 @@ public:
         return changed;
     }
 
-    static PropagatingTypeAnalysis* doAnalysis(SpeculationLevel speculation, ScopeInfo* scope_info,
-                                               TypeMap&& initial_types, CFGBlock* initial_block) {
+    static PropagatingTypeAnalysis* doAnalysis(SpeculationLevel speculation, TypeMap&& initial_types,
+                                               CFGBlock* initial_block) {
         Timer _t("PropagatingTypeAnalysis::doAnalysis()");
 
         CFG* cfg = initial_block->cfg;
@@ -795,7 +790,7 @@ public:
             }
 
             TypeMap ending = BasicBlockTypePropagator::propagate(block, starting_types.find(block)->second, expr_types,
-                                                                 type_speculations, speculation, scope_info);
+                                                                 type_speculations, speculation);
 
             if (VERBOSITY("types") >= 3) {
                 printf("before (after):\n");
@@ -851,7 +846,7 @@ public:
 
 // public entry point:
 TypeAnalysis* doTypeAnalysis(CFG* cfg, const ParamNames& arg_names, const std::vector<ConcreteCompilerType*>& arg_types,
-                             EffortLevel effort, TypeAnalysis::SpeculationLevel speculation, ScopeInfo* scope_info) {
+                             EffortLevel effort, TypeAnalysis::SpeculationLevel speculation) {
     // if (effort == EffortLevel::INTERPRETED) {
     // return new NullTypeAnalysis();
     //}
@@ -870,12 +865,11 @@ TypeAnalysis* doTypeAnalysis(CFG* cfg, const ParamNames& arg_names, const std::v
 
     assert(i == arg_types.size());
 
-    return PropagatingTypeAnalysis::doAnalysis(speculation, scope_info, std::move(initial_types),
-                                               cfg->getStartingBlock());
+    return PropagatingTypeAnalysis::doAnalysis(speculation, std::move(initial_types), cfg->getStartingBlock());
 }
 
 TypeAnalysis* doTypeAnalysis(const OSREntryDescriptor* entry_descriptor, EffortLevel effort,
-                             TypeAnalysis::SpeculationLevel speculation, ScopeInfo* scope_info) {
+                             TypeAnalysis::SpeculationLevel speculation) {
     auto cfg = entry_descriptor->md->source->cfg;
     auto&& vreg_info = cfg->getVRegInfo();
     TypeMap initial_types(vreg_info.getTotalNumOfVRegs());
@@ -884,7 +878,7 @@ TypeAnalysis* doTypeAnalysis(const OSREntryDescriptor* entry_descriptor, EffortL
         initial_types[p.first] = p.second;
     }
 
-    return PropagatingTypeAnalysis::doAnalysis(speculation, scope_info, std::move(initial_types),
+    return PropagatingTypeAnalysis::doAnalysis(speculation, std::move(initial_types),
                                                entry_descriptor->backedge->target);
 }
 }

src/analysis/type_analysis.h

@@ -23,7 +23,6 @@
 
 namespace pyston {
 
-class ScopeInfo;
 class CFGBlock;
 class BoxedClass;
 class AST_expr;
@@ -47,9 +46,9 @@ public:
 
 TypeAnalysis* doTypeAnalysis(CFG* cfg, const ParamNames& param_names,
                              const std::vector<ConcreteCompilerType*>& arg_types, EffortLevel effort,
-                             TypeAnalysis::SpeculationLevel speculation, ScopeInfo* scope_info);
+                             TypeAnalysis::SpeculationLevel speculation);
 TypeAnalysis* doTypeAnalysis(const OSREntryDescriptor* entry_descriptor, EffortLevel effort,
-                             TypeAnalysis::SpeculationLevel speculation, ScopeInfo* scope_info);
+                             TypeAnalysis::SpeculationLevel speculation);
 }
 
 #endif

src/codegen/ast_interpreter.h

@@ -47,11 +47,11 @@ struct ASTInterpreterJitInterface {
     static int getGlobalsOffset();
 
     static void delNameHelper(void* _interpreter, InternedString name);
-    static Box* derefHelper(void* interp, InternedString s);
+    static Box* derefHelper(void* interp, AST_Name* node);
     static Box* landingpadHelper(void* interp);
     static void pendingCallsCheckHelper();
     static void setExcInfoHelper(void* interp, STOLEN(Box*) type, STOLEN(Box*) value, STOLEN(Box*) traceback);
-    static void setLocalClosureHelper(void* interp, long vreg, InternedString id, Box* v);
+    static void setLocalClosureHelper(void* interp, AST_Name* name, Box* v);
     static void uncacheExcInfoHelper(void* interp);
     static void raise0Helper(void* interp) __attribute__((noreturn));
     static Box* yieldHelper(void* interp, STOLEN(Box*) value);

src/codegen/baseline_jit.cpp

@@ -372,14 +372,8 @@ RewriterVar* JitFragmentWriter::emitCreateTuple(const llvm::ArrayRef<RewriterVar
     return r;
 }
 
-RewriterVar* JitFragmentWriter::emitDeref(InternedString s) {
-    return call(false, (void*)ASTInterpreterJitInterface::derefHelper, getInterp(),
-#ifndef NDEBUG
-                imm(asUInt(s).first), imm(asUInt(s).second))
-#else
-                imm(asUInt(s)))
-#endif
-        ->setType(RefType::OWNED);
+RewriterVar* JitFragmentWriter::emitDeref(AST_Name* name) {
+    return call(false, (void*)ASTInterpreterJitInterface::derefHelper, getInterp(), imm(name))->setType(RefType::OWNED);
 }
 
 RewriterVar* JitFragmentWriter::emitExceptionMatches(RewriterVar* v, RewriterVar* cls) {
@@ -391,20 +385,22 @@ RewriterVar* JitFragmentWriter::emitGetAttr(RewriterVar* obj, BoxedString* s, AS
         .first->setType(RefType::OWNED);
 }
 
-RewriterVar* JitFragmentWriter::emitGetBlockLocal(InternedString s, int vreg) {
+RewriterVar* JitFragmentWriter::emitGetBlockLocal(AST_Name* name) {
+    auto s = name->id;
+    auto vreg = name->vreg;
     auto it = local_syms.find(s);
     if (it == local_syms.end()) {
-        auto r = emitGetLocal(s, vreg);
+        auto r = emitGetLocal(name);
         assert(r->reftype == RefType::OWNED);
-        emitSetBlockLocal(s, vreg, r);
+        emitSetBlockLocal(name, r);
         return r;
     }
     return it->second;
 }
 
-void JitFragmentWriter::emitKillTemporary(InternedString s, int vreg) {
-    if (!local_syms.count(s))
-        emitSetLocal(s, vreg, false, imm(nullptr));
+void JitFragmentWriter::emitKillTemporary(AST_Name* name) {
+    if (!local_syms.count(name->id))
+        emitSetLocal(name, false, imm(nullptr));
 }
 
 RewriterVar* JitFragmentWriter::emitGetBoxedLocal(BoxedString* s) {
@@ -436,9 +432,11 @@ RewriterVar* JitFragmentWriter::emitGetItem(AST_expr* node, RewriterVar* value,
         .first->setType(RefType::OWNED);
 }
 
-RewriterVar* JitFragmentWriter::emitGetLocal(InternedString s, int vreg) {
+RewriterVar* JitFragmentWriter::emitGetLocal(AST_Name* name) {
     if (LOG_BJIT_ASSEMBLY)
         comment("BJIT: emitGetLocal start");
+    auto vreg = name->vreg;
+    auto s = name->id;
     assert(vreg >= 0);
     // TODO Can we use BORROWED here? Not sure if there are cases when we can't rely on borrowing the ref
     // from the vregs array.  Safer like this.
@@ -686,14 +684,16 @@ void JitFragmentWriter::emitSetAttr(AST_expr* node, RewriterVar* obj, BoxedStrin
     attr->refConsumed(rtn.second);
 }
 
-void JitFragmentWriter::emitSetBlockLocal(InternedString s, int vreg, STOLEN(RewriterVar*) v) {
+void JitFragmentWriter::emitSetBlockLocal(AST_Name* name, STOLEN(RewriterVar*) v) {
     if (LOG_BJIT_ASSEMBLY)
         comment("BJIT: emitSetBlockLocal() start");
-    RewriterVar* prev = local_syms[s];
+    auto vreg = name->vreg;
+    auto s = name->id;
+    RewriterVar* prev = local_syms[name->id];
     // if we never set this sym before in this BB and the symbol gets accessed in several blocks clear it because it
     // could have been set in a previous block.
     if (!prev && !block->cfg->getVRegInfo().isBlockLocalVReg(vreg))
-        emitSetLocal(s, vreg, false, imm(nullptr)); // clear out the vreg
+        emitSetLocal(name, false, imm(nullptr)); // clear out the vreg
     local_syms[s] = v;
     if (LOG_BJIT_ASSEMBLY)
         comment("BJIT: emitSetBlockLocal() end");
@@ -728,18 +728,13 @@ void JitFragmentWriter::emitSetItemName(BoxedString* s, RewriterVar* v) {
     emitSetItem(emitGetBoxedLocals(), imm(s), v);
 }
 
-void JitFragmentWriter::emitSetLocal(InternedString s, int vreg, bool set_closure, STOLEN(RewriterVar*) v) {
+void JitFragmentWriter::emitSetLocal(AST_Name* name, bool set_closure, STOLEN(RewriterVar*) v) {
     if (LOG_BJIT_ASSEMBLY)
         comment("BJIT: emitSetLocal() start");
+    auto vreg = name->vreg;
     assert(vreg >= 0);
     if (set_closure) {
-        call(false, (void*)ASTInterpreterJitInterface::setLocalClosureHelper, getInterp(), imm(vreg),
-#ifndef NDEBUG
-             imm(asUInt(s).first), imm(asUInt(s).second),
-#else
-             imm(asUInt(s)),
-#endif
-             v);
+        call(false, (void*)ASTInterpreterJitInterface::setLocalClosureHelper, getInterp(), imm(name), v);
         v->refConsumed();
     } else {
         // TODO With definedness analysis, we could know whether we needed to emit an decref/xdecref/neither.

src/codegen/baseline_jit.h

@@ -278,17 +278,17 @@ public:
     RewriterVar* emitCreateSet(const llvm::ArrayRef<RewriterVar*> values);
     RewriterVar* emitCreateSlice(RewriterVar* start, RewriterVar* stop, RewriterVar* step);
     RewriterVar* emitCreateTuple(const llvm::ArrayRef<RewriterVar*> values);
-    RewriterVar* emitDeref(InternedString s);
+    RewriterVar* emitDeref(AST_Name* name);
     RewriterVar* emitExceptionMatches(RewriterVar* v, RewriterVar* cls);
     RewriterVar* emitGetAttr(RewriterVar* obj, BoxedString* s, AST_expr* node);
-    RewriterVar* emitGetBlockLocal(InternedString s, int vreg);
-    void emitKillTemporary(InternedString s, int vreg);
+    RewriterVar* emitGetBlockLocal(AST_Name* name);
+    void emitKillTemporary(AST_Name* name);
     RewriterVar* emitGetBoxedLocal(BoxedString* s);
     RewriterVar* emitGetBoxedLocals();
     RewriterVar* emitGetClsAttr(RewriterVar* obj, BoxedString* s);
     RewriterVar* emitGetGlobal(BoxedString* s);
     RewriterVar* emitGetItem(AST_expr* node, RewriterVar* value, RewriterVar* slice);
-    RewriterVar* emitGetLocal(InternedString s, int vreg);
+    RewriterVar* emitGetLocal(AST_Name* name);
     RewriterVar* emitGetPystonIter(RewriterVar* v);
     RewriterVar* emitHasnext(RewriterVar* v);
     RewriterVar* emitImportFrom(RewriterVar* module, BoxedString* name);
@@ -318,13 +318,13 @@ public:
     void emitRaise3(RewriterVar* arg0, RewriterVar* arg1, RewriterVar* arg2);
     void emitReturn(RewriterVar* v);
     void emitSetAttr(AST_expr* node, RewriterVar* obj, BoxedString* s, STOLEN(RewriterVar*) attr);
-    void emitSetBlockLocal(InternedString s, int vreg, STOLEN(RewriterVar*) v);
+    void emitSetBlockLocal(AST_Name* name, STOLEN(RewriterVar*) v);
     void emitSetCurrentInst(AST_stmt* node);
     void emitSetExcInfo(RewriterVar* type, RewriterVar* value, RewriterVar* traceback);
     void emitSetGlobal(BoxedString* s, STOLEN(RewriterVar*) v, bool are_globals_from_module);
     void emitSetItemName(BoxedString* s, RewriterVar* v);
     void emitSetItem(RewriterVar* target, RewriterVar* slice, RewriterVar* value);
-    void emitSetLocal(InternedString s, int vreg, bool set_closure, STOLEN(RewriterVar*) v);
+    void emitSetLocal(AST_Name* name, bool set_closure, STOLEN(RewriterVar*) v);
     // emitSideExit steals a full ref from v, not just a vref
     void emitSideExit(STOLEN(RewriterVar*) v, Box* cmp_value, CFGBlock* next_block);
     void emitUncacheExcInfo();

src/codegen/codegen.cpp

@@ -38,10 +38,10 @@
 namespace pyston {
 
 FunctionMetadata::FunctionMetadata(int num_args, bool takes_varargs, bool takes_kwargs,
-                                   std::unique_ptr<SourceInfo> source)
+                                   std::unique_ptr<SourceInfo> source, ParamNames param_names)
     : code_obj(NULL),
       source(std::move(source)),
-      param_names(this->source->ast, this->source->getInternedStrings()),
+      param_names(std::move(param_names)),
       takes_varargs(takes_varargs),
       takes_kwargs(takes_kwargs),
       num_args(num_args),
@@ -91,14 +91,8 @@ void FunctionMetadata::addVersion(CompiledFunction* compiled) {
     }
 }
 
-SourceInfo::SourceInfo(BoxedModule* m, std::shared_ptr<ScopingAnalysis> scoping, FutureFlags future_flags, AST* ast,
-                       BoxedString* fn)
-    : parent_module(m),
-      scoping(scoping),
-      scope_info(scoping->getScopeInfoForNode(ast)),
-      ast(ast),
-      cfg(NULL),
-      future_flags(future_flags) {
+SourceInfo::SourceInfo(BoxedModule* m, ScopingResults scoping, FutureFlags future_flags, AST* ast, BoxedString* fn)
+    : parent_module(m), scoping(std::move(scoping)), ast(ast), cfg(NULL), future_flags(future_flags) {
     assert(fn);
 
     // TODO: this is a very bad way of handling this:

src/codegen/irgen.cpp

@@ -408,7 +408,7 @@ static void emitBBs(IRGenState* irstate, TypeAnalysis* types, const OSREntryDesc
         irstate->getRefcounts()->setType(osr_created_closure, RefType::BORROWED);
         if (source->is_generator)
             irstate->setPassedGenerator(osr_generator);
-        if (source->getScopeInfo()->createsClosure())
+        if (source->scoping.createsClosure())
             irstate->setCreatedClosure(osr_created_closure);
 
         int arg_num = -1;
@@ -730,13 +730,16 @@ static void emitBBs(IRGenState* irstate, TypeAnalysis* types, const OSREntryDesc
                         auto asgn = ast_cast<AST_Assign>(stmt);
                         assert(asgn->targets.size() == 1);
                         if (asgn->targets[0]->type == AST_TYPE::Name) {
-                            InternedString name = ast_cast<AST_Name>(asgn->targets[0])->id;
+                            auto asname = ast_cast<AST_Name>(asgn->targets[0]);
+                            assert(asname->lookup_type != ScopeInfo::VarScopeType::UNKNOWN);
+
+                            InternedString name = asname->id;
                             int vreg = ast_cast<AST_Name>(asgn->targets[0])->vreg;
                             assert(name.c_str()[0] == '#'); // it must be a temporary
                             // You might think I need to check whether `name' is being assigned globally or locally,
                             // since a global assign doesn't affect the symbol table. However, the CFG pass only
                             // generates invoke-assigns to temporary variables. Just to be sure, we assert:
-                            assert(source->getScopeInfo()->getScopeTypeOfName(name) != ScopeInfo::VarScopeType::GLOBAL);
+                            assert(asname->lookup_type != ScopeInfo::VarScopeType::GLOBAL);
 
                             // TODO: inefficient
                             sym_table = new SymbolTable(*sym_table);
@@ -1063,13 +1066,13 @@ std::pair<CompiledFunction*, llvm::Function*> doCompile(FunctionMetadata* md, So
         int nargs = param_names->totalParameters();
         ASSERT(nargs == spec->arg_types.size(), "%d %ld", nargs, spec->arg_types.size());
 
-        if (source->getScopeInfo()->takesClosure())
+        if (source->scoping.takesClosure())
             llvm_arg_types.push_back(g.llvm_closure_type_ptr);
 
         if (source->is_generator)
             llvm_arg_types.push_back(g.llvm_generator_type_ptr);
 
-        if (!source->scoping->areGlobalsFromModule())
+        if (!source->scoping.areGlobalsFromModule())
             llvm_arg_types.push_back(g.llvm_value_type_ptr);
 
         for (int i = 0; i < nargs; i++) {
@@ -1109,10 +1112,9 @@ std::pair<CompiledFunction*, llvm::Function*> doCompile(FunctionMetadata* md, So
         speculation_level = TypeAnalysis::SOME;
     TypeAnalysis* types;
     if (entry_descriptor)
-        types = doTypeAnalysis(entry_descriptor, effort, speculation_level, source->getScopeInfo());
+        types = doTypeAnalysis(entry_descriptor, effort, speculation_level);
     else
-        types = doTypeAnalysis(source->cfg, *param_names, spec->arg_types, effort, speculation_level,
-                               source->getScopeInfo());
+        types = doTypeAnalysis(source->cfg, *param_names, spec->arg_types, effort, speculation_level);
 
 
     _t2.split();
@@ -1131,9 +1133,9 @@ std::pair<CompiledFunction*, llvm::Function*> doCompile(FunctionMetadata* md, So
     std::unique_ptr<PhiAnalysis> phis;
 
     if (entry_descriptor)
-        phis = computeRequiredPhis(entry_descriptor, liveness, source->getScopeInfo());
+        phis = computeRequiredPhis(entry_descriptor, liveness);
     else
-        phis = computeRequiredPhis(*param_names, source->cfg, liveness, source->getScopeInfo());
+        phis = computeRequiredPhis(*param_names, source->cfg, liveness);
 
     RefcountTracker refcounter;
 

src/codegen/irgen/hooks.cpp

@@ -54,78 +54,6 @@
 
 namespace pyston {
 
-// TODO terrible place for these!
-ParamNames::ParamNames(AST* ast, InternedStringPool& pool)
-    : all_args_contains_names(1), takes_param_names(1), has_vararg_name(0), has_kwarg_name(0) {
-    if (ast->type == AST_TYPE::Module || ast->type == AST_TYPE::ClassDef || ast->type == AST_TYPE::Expression
-        || ast->type == AST_TYPE::Suite) {
-    } else if (ast->type == AST_TYPE::FunctionDef || ast->type == AST_TYPE::Lambda) {
-        AST_arguments* arguments = ast->type == AST_TYPE::FunctionDef ? ast_cast<AST_FunctionDef>(ast)->args
-                                                                      : ast_cast<AST_Lambda>(ast)->args;
-        for (int i = 0; i < arguments->args.size(); i++) {
-            AST_expr* arg = arguments->args[i];
-            if (arg->type == AST_TYPE::Name) {
-                AST_Name* name = ast_cast<AST_Name>(arg);
-                all_args.emplace_back(name);
-            } else {
-                InternedString dot_arg_name = pool.get("." + std::to_string(i));
-                all_args.emplace_back(new AST_Name(dot_arg_name, AST_TYPE::Param, arg->lineno, arg->col_offset));
-            }
-        }
-
-        auto vararg_name = arguments->vararg;
-        if (vararg_name) {
-            has_vararg_name = 1;
-            all_args.emplace_back(vararg_name);
-        }
-
-        auto kwarg_name = arguments->kwarg;
-        if (kwarg_name) {
-            has_kwarg_name = 1;
-            all_args.emplace_back(kwarg_name);
-        }
-    } else {
-        RELEASE_ASSERT(0, "%d", ast->type);
-    }
-}
-
-ParamNames::ParamNames(const std::vector<const char*>& args, const char* vararg, const char* kwarg)
-    : all_args_contains_names(0),
-      takes_param_names(1),
-      has_vararg_name(vararg && *vararg),
-      has_kwarg_name(kwarg && *kwarg) {
-    all_args.reserve(args.size() + has_vararg_name + has_kwarg_name);
-    for (auto&& arg : args) {
-        all_args.emplace_back(arg);
-    }
-    if (has_vararg_name)
-        all_args.emplace_back(vararg);
-    if (has_kwarg_name)
-        all_args.emplace_back(kwarg);
-}
-
-std::vector<const char*> ParamNames::allArgsAsStr() const {
-    std::vector<const char*> ret;
-    ret.reserve(all_args.size());
-    if (all_args_contains_names) {
-        for (auto&& arg : all_args) {
-            ret.push_back(arg.name->id.c_str());
-        }
-    } else {
-        for (auto&& arg : all_args) {
-            ret.push_back(arg.str);
-        }
-    }
-    return ret;
-}
-
-InternedString SourceInfo::mangleName(InternedString id) {
-    assert(ast);
-    if (ast->type == AST_TYPE::Module)
-        return id;
-    return getScopeInfo()->mangleName(id);
-}
-
 llvm::ArrayRef<AST_stmt*> SourceInfo::getBody() const {
     switch (ast->type) {
         case AST_TYPE::ClassDef:
@@ -141,10 +69,6 @@ llvm::ArrayRef<AST_stmt*> SourceInfo::getBody() const {
     };
 }
 
-InternedStringPool& SourceInfo::getInternedStrings() {
-    return scoping->getInternedStrings();
-}
-
 BORROWED(BoxedString*) SourceInfo::getFn() {
     assert(fn->ob_refcnt >= 1);
     return fn;
@@ -182,11 +106,6 @@ Box* SourceInfo::getDocString() {
     return incref(Py_None);
 }
 
-ScopeInfo* SourceInfo::getScopeInfo() {
-    assert(scope_info);
-    return scope_info.get();
-}
-
 LivenessAnalysis* SourceInfo::getLiveness() {
     if (!liveness_info)
         liveness_info = computeLivenessInfo(cfg);
@@ -309,10 +228,7 @@ CompiledFunction* compileFunction(FunctionMetadata* f, FunctionSpecialization* s
         printf("%s", ss.str().c_str());
     }
 
-    // Do the analysis now if we had deferred it earlier:
-    if (source->cfg == NULL) {
-        source->cfg = computeCFG(source, f->param_names);
-    }
+    assert(source->cfg);
 
 
     CompiledFunction* cf = NULL;
@@ -361,42 +277,32 @@ CompiledFunction* compileFunction(FunctionMetadata* f, FunctionSpecialization* s
 }
 
 void compileAndRunModule(AST_Module* m, BoxedModule* bm) {
-    FunctionMetadata* md;
-
     Timer _t("for compileModule()");
 
     const char* fn = PyModule_GetFilename(bm);
     RELEASE_ASSERT(fn, "");
 
     FutureFlags future_flags = getFutureFlags(m->body, fn);
-    std::shared_ptr<ScopingAnalysis> scoping = std::make_shared<ScopingAnalysis>(m, true);
+    computeAllCFGs(m, /* globals_from_module */ true, future_flags, autoDecref(boxString(fn)), bm);
 
-    auto fn_str = boxString(fn);
-    AUTO_DECREF(fn_str);
-    std::unique_ptr<SourceInfo> si(new SourceInfo(bm, std::move(scoping), future_flags, m, fn_str));
+    FunctionMetadata* md = metadataForAST(m);
+    assert(md);
 
     static BoxedString* doc_str = getStaticString("__doc__");
-    bm->setattr(doc_str, autoDecref(si->getDocString()), NULL);
+    bm->setattr(doc_str, autoDecref(md->source->getDocString()), NULL);
 
     static BoxedString* builtins_str = getStaticString("__builtins__");
     if (!bm->hasattr(builtins_str))
         bm->setattr(builtins_str, PyModule_GetDict(builtins_module), NULL);
 
-    md = new FunctionMetadata(0, false, false, std::move(si));
-
     UNAVOIDABLE_STAT_TIMER(t0, "us_timer_interpreted_module_toplevel");
     Box* r = astInterpretFunction(md, NULL, NULL, NULL, NULL, NULL, NULL, NULL);
     assert(r == Py_None);
     Py_DECREF(r);
-
-    // XXX for bjit testing
-    // r = astInterpretFunction(md, NULL, NULL, NULL, NULL, NULL, NULL, NULL);
-    // assert(r == None);
-    // Py_DECREF(r);
 }
 
 Box* evalOrExec(FunctionMetadata* md, Box* globals, Box* boxedLocals) {
-    RELEASE_ASSERT(!md->source->scoping->areGlobalsFromModule(), "");
+    RELEASE_ASSERT(!md->source->scoping.areGlobalsFromModule(), "");
 
     assert(globals && (globals->cls == module_cls || globals->cls == dict_cls));
 
@@ -415,8 +321,6 @@ static FunctionMetadata* compileForEvalOrExec(AST* source, llvm::ArrayRef<AST_st
                                               PyCompilerFlags* flags) {
     Timer _t("for evalOrExec()");
 
-    auto scoping = std::make_shared<ScopingAnalysis>(source, false);
-
     // `my_future_flags` are the future flags enabled in the exec's code.
     // `caller_future_flags` are the future flags of the source that the exec statement is in.
     // We need to enable features that are enabled in either.
@@ -428,9 +332,8 @@ static FunctionMetadata* compileForEvalOrExec(AST* source, llvm::ArrayRef<AST_st
         flags->cf_flags = future_flags;
     }
 
-    std::unique_ptr<SourceInfo> si(new SourceInfo(getCurrentModule(), std::move(scoping), future_flags, source, fn));
-
-    return new FunctionMetadata(0, false, false, std::move(si));
+    computeAllCFGs(source, /* globals_from_module */ false, future_flags, fn, getCurrentModule());
+    return metadataForAST(source);
 }
 
 static FunctionMetadata* compileExec(AST_Module* parsedModule, BoxedString* fn, PyCompilerFlags* flags) {

src/codegen/irgen/irgenerator.h

@@ -118,7 +118,7 @@ public:
     LivenessAnalysis* getLiveness() { return source_info->getLiveness(); }
     PhiAnalysis* getPhis() { return phis.get(); }
 
-    ScopeInfo* getScopeInfo();
+    const ScopingResults& getScopeInfo();
 
     llvm::MDNode* getFuncDbgInfo() { return func_dbg_info; }
 
@@ -204,7 +204,6 @@ IREmitter* createIREmitter(IRGenState* irstate, llvm::BasicBlock*& curblock, IRG
 IRGenerator* createIRGenerator(IRGenState* irstate, std::unordered_map<CFGBlock*, llvm::BasicBlock*>& entry_blocks,
                                CFGBlock* myblock, TypeAnalysis* types);
 
-FunctionMetadata* wrapFunction(AST* node, AST_arguments* args, SourceInfo* source);
 std::vector<BoxedString*>* getKeywordNameStorage(AST_Call* node);
 }
 

src/codegen/patchpoints.cpp

@@ -98,9 +98,9 @@ void PatchpointInfo::parseLocationMap(StackMap::Record* r, LocationMap* map) {
     auto&& source = parentFunction()->md->source;
     if (source->is_generator)
         map->generator.locations.push_back(parse_type(GENERATOR));
-    if (source->getScopeInfo()->takesClosure())
+    if (source->scoping.takesClosure())
         map->passed_closure.locations.push_back(parse_type(CLOSURE));
-    if (source->getScopeInfo()->createsClosure())
+    if (source->scoping.createsClosure())
         map->created_closure.locations.push_back(parse_type(CLOSURE));
 
     for (FrameVarInfo& frame_var : frame_info_desc.vars) {

src/codegen/unwinding.cpp

@@ -985,12 +985,12 @@ BORROWED(Box*) FrameInfo::updateBoxedLocals() {
 
     FrameInfo* frame_info = this;
     FunctionMetadata* md = frame_info->md;
-    ScopeInfo* scope_info = md->source->getScopeInfo();
+    const ScopingResults& scope_info = md->source->scoping;
 
-    if (scope_info->areLocalsFromModule()) {
+    if (scope_info.areLocalsFromModule()) {
         // TODO we should cache this in frame_info->locals or something so that locals()
         // (and globals() too) will always return the same dict
-        RELEASE_ASSERT(md->source->scoping->areGlobalsFromModule(), "");
+        RELEASE_ASSERT(md->source->scoping.areGlobalsFromModule(), "");
         return md->source->parent_module->getAttrWrapper();
     }
 
@@ -1000,7 +1000,7 @@ BORROWED(Box*) FrameInfo::updateBoxedLocals() {
     // Add the locals from the closure
     // TODO in a ClassDef scope, we aren't supposed to add these
     size_t depth = 0;
-    for (auto& p : scope_info->getAllDerefVarsAndInfo()) {
+    for (auto& p : scope_info.getAllDerefVarsAndInfo()) {
         InternedString name = p.first;
         DerefInfo derefInfo = p.second;
         while (depth < derefInfo.num_parents_from_passed_closure) {

src/core/ast.cpp

@@ -1639,8 +1639,24 @@ bool PrintVisitor::visit_suite(AST_Suite* node) {
 
 bool PrintVisitor::visit_name(AST_Name* node) {
     stream << node->id.s();
-    // Uncomment this line to see which names are kills:
-    // if (node->is_kill) stream << "<k>";
+#if 0
+    if (node->lookup_type == ScopeInfo::VarScopeType::UNKNOWN)
+        stream << "<U>";
+    else if (node->lookup_type == ScopeInfo::VarScopeType::FAST)
+        stream << "<F>";
+    else if (node->lookup_type == ScopeInfo::VarScopeType::DEREF)
+        stream << "<D>";
+    else if (node->lookup_type == ScopeInfo::VarScopeType::CLOSURE)
+        stream << "<C>";
+    else if (node->lookup_type == ScopeInfo::VarScopeType::GLOBAL)
+        stream << "<G>";
+    else
+        stream << "<?>";
+#endif
+
+#if 0
+    if (node->is_kill) stream << "<k>";
+#endif
     return false;
 }
 

src/core/ast.h

@@ -400,6 +400,8 @@ public:
     std::vector<AST_stmt*> body;
     InternedString name;
 
+    FunctionMetadata* md;
+
     AST_ClassDef() : AST_stmt(AST_TYPE::ClassDef) {}
 
     static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::ClassDef;
@@ -509,6 +511,8 @@ public:
     // this should be an expr but we convert it into a AST_Return(AST_expr) to make the code simpler
     AST_stmt* body;
 
+    FunctionMetadata* md;
+
     virtual void accept(ASTVisitor* v);
 
     AST_Expression(std::unique_ptr<InternedStringPool> interned_strings)
@@ -549,6 +553,8 @@ public:
     InternedString name; // if the name is not set this is a lambda
     AST_arguments* args;
 
+    FunctionMetadata* md;
+
     virtual void accept(ASTVisitor* v);
     virtual void accept_stmt(StmtVisitor* v);
 
@@ -704,6 +710,8 @@ public:
     // no lineno, col_offset attributes
     std::vector<AST_stmt*> body;
 
+    FunctionMetadata* md;
+
     virtual void accept(ASTVisitor* v);
 
     AST_Module(std::unique_ptr<InternedStringPool> interned_strings)
@@ -736,12 +744,17 @@ public:
     // different bytecodes.
     ScopeInfo::VarScopeType lookup_type;
 
+    // These are only valid for lookup_type == FAST or CLOSURE
     // The interpreter and baseline JIT store variables with FAST and CLOSURE scopes in an array (vregs) this specifies
     // the zero based index of this variable inside the vregs array. If uninitialized it's value is -1.
     int vreg;
-
     bool is_kill = false;
 
+    // Only valid for lookup_type == DEREF:
+    DerefInfo deref_info = DerefInfo({ INT_MAX, INT_MAX });
+    // Only valid for lookup_type == CLOSURE:
+    int closure_offset = -1;
+
     virtual void accept(ASTVisitor* v);
     virtual void* accept_expr(ExprVisitor* v);
 
@@ -1121,7 +1134,7 @@ public:
 };
 
 template <typename T> T* ast_cast(AST* node) {
-    assert(!node || node->type == T::TYPE);
+    ASSERT(!node || node->type == T::TYPE, "%d", node ? node->type : 0);
     return static_cast<T*>(node);
 }
 

src/core/cfg.h

@@ -169,7 +169,7 @@ public:
     int getNumOfCrossBlockVRegs() const { return num_vregs_cross_block; }
 
     bool hasVRegsAssigned() const { return num_vregs != -1; }
-    void assignVRegs(CFG* cfg, const ParamNames& param_names, ScopeInfo* scope_info);
+    void assignVRegs(CFG* cfg, const ParamNames& param_names);
 };
 
 // Control Flow Graph
@@ -330,8 +330,10 @@ public:
     iterator end() const { return iterator(*this, this->v.size()); }
 };
 
-class SourceInfo;
-CFG* computeCFG(SourceInfo* source, const ParamNames& param_names);
+FunctionMetadata*& metadataForAST(AST* ast);
+InternedStringPool& stringpoolForAST(AST* ast);
+
+void computeAllCFGs(AST* ast, bool globals_from_module, FutureFlags future_flags, BoxedString* fn, BoxedModule* bm);
 void printCFG(CFG* cfg);
 }
 

src/core/types.h

@@ -161,7 +161,6 @@ class AST_stmt;
 
 class PhiAnalysis;
 class LivenessAnalysis;
-class ScopingAnalysis;
 
 class FunctionMetadata;
 class OSREntryDescriptor;
@@ -430,10 +429,60 @@ public:
 typedef int FutureFlags;
 
 class BoxedModule;
-class ScopeInfo;
 class InternedStringPool;
 class LivenessAnalysis;
 
+// Each closure has an array (fixed-size for that particular scope) of variables
+// and a parent pointer to a parent closure. To look up a variable from the passed-in
+// closure (i.e., DEREF), you just need to know (i) how many parents up to go and
+// (ii) what offset into the array to find the variable. This struct stores that
+// information. You can query the ScopeInfo with a name to get this info.
+struct DerefInfo {
+    size_t num_parents_from_passed_closure;
+    size_t offset;
+};
+
+class ScopeInfo;
+// The results of our scoping analysis.
+// A ScopeInfo is a component of the analysis itself and contains a lot of other
+// metadata that is necessary during the analysis, after which we can throw it
+// away and only keep a ScopingResults object.
+struct ScopingResults {
+private:
+    bool are_locals_from_module : 1;
+    bool are_globals_from_module : 1;
+    bool creates_closure : 1;
+    bool takes_closure : 1;
+    bool passes_through_closure : 1;
+    bool uses_name_lookup : 1;
+
+    int closure_size;
+    std::vector<std::pair<InternedString, DerefInfo>> deref_info;
+
+public:
+    ScopingResults(ScopingResults&&) = default;
+    // Delete these just to make sure we avoid extra copies
+    ScopingResults(const ScopingResults&) = delete;
+    void operator=(const ScopingResults&) = delete;
+
+    bool areLocalsFromModule() const { return are_locals_from_module; }
+    bool areGlobalsFromModule() const { return are_globals_from_module; }
+    bool createsClosure() const { return creates_closure; }
+    bool takesClosure() const { return takes_closure; }
+    bool passesThroughClosure() const { return passes_through_closure; }
+    bool usesNameLookup() const { return uses_name_lookup; }
+
+    int getClosureSize() const {
+        assert(createsClosure());
+        return closure_size;
+    }
+    const std::vector<std::pair<InternedString, DerefInfo>>& getAllDerefVarsAndInfo() const { return deref_info; }
+    DerefInfo getDerefInfo(AST_Name*) const;
+    size_t getClosureOffset(AST_Name*) const;
+
+    ScopingResults(ScopeInfo* scope_info, bool globals_from_module);
+};
+
 // Data about a single textual function definition.
 class SourceInfo {
 private:
@@ -442,30 +491,23 @@ private:
 
 public:
     BoxedModule* parent_module;
-    std::shared_ptr<ScopingAnalysis> scoping;
-    std::unique_ptr<ScopeInfo> scope_info;
+    ScopingResults scoping;
     AST* ast;
     CFG* cfg;
     FutureFlags future_flags;
     bool is_generator;
 
-    InternedStringPool& getInternedStrings();
-
-    ScopeInfo* getScopeInfo();
     LivenessAnalysis* getLiveness();
 
     // does not throw CXX or CAPI exceptions:
     BORROWED(BoxedString*) getName() noexcept;
     BORROWED(BoxedString*) getFn();
 
-    InternedString mangleName(InternedString id);
-
     llvm::ArrayRef<AST_stmt*> getBody() const;
 
     Box* getDocString();
 
-    SourceInfo(BoxedModule* m, std::shared_ptr<ScopingAnalysis> scoping, FutureFlags future_flags, AST* ast,
-               BoxedString* fn);
+    SourceInfo(BoxedModule* m, ScopingResults scoping, FutureFlags future_flags, AST* ast, BoxedString* fn);
     ~SourceInfo();
 };
 
@@ -520,7 +562,8 @@ public:
                                         Box**, const std::vector<BoxedString*>*> InternalCallable;
     InternalCallable internal_callable;
 
-    FunctionMetadata(int num_args, bool takes_varargs, bool takes_kwargs, std::unique_ptr<SourceInfo> source);
+    FunctionMetadata(int num_args, bool takes_varargs, bool takes_kwargs, std::unique_ptr<SourceInfo> source,
+                     ParamNames param_names);
     FunctionMetadata(int num_args, bool takes_varargs, bool takes_kwargs,
                      const ParamNames& param_names = ParamNames::empty());
     ~FunctionMetadata();

src/runtime/code.cpp

@@ -206,7 +206,7 @@ extern "C" BORROWED(PyObject*) PyCode_GetName(PyCodeObject* op) noexcept {
 
 extern "C" int PyCode_HasFreeVars(PyCodeObject* _code) noexcept {
     BoxedCode* code = (BoxedCode*)_code;
-    return code->f->source->getScopeInfo()->takesClosure() ? 1 : 0;
+    return code->f->source->scoping.takesClosure() ? 1 : 0;
 }
 
 void setupCode() {

src/runtime/objmodel.cpp

@@ -4771,7 +4771,7 @@ static Box* callChosenCF(CompiledFunction* chosen_cf, BoxedClosure* closure, Box
         }
     }
 
-    assert((globals == NULL) == (!chosen_cf->md->source || chosen_cf->md->source->scoping->areGlobalsFromModule()));
+    assert((globals == NULL) == (!chosen_cf->md->source || chosen_cf->md->source->scoping.areGlobalsFromModule()));
 
     Box* maybe_args[3];
     int nmaybe_args = 0;

src/runtime/types.cpp

@@ -325,7 +325,7 @@ extern "C" BoxedFunctionBase::BoxedFunctionBase(FunctionMetadata* md, llvm::Arra
       modname(NULL),
       name(NULL),
       doc(NULL) {
-    assert((!globals) == (!md->source || md->source->scoping->areGlobalsFromModule()));
+    assert((!globals) == (!md->source || md->source->scoping.areGlobalsFromModule()));
 
     if (globals)
         ASSERT(globals->cls == dict_cls || globals->cls == module_cls, "%s", globals->cls->tp_name);
@@ -352,7 +352,7 @@ extern "C" BoxedFunctionBase::BoxedFunctionBase(FunctionMetadata* md, llvm::Arra
 
         Box* globals_for_name = globals;
         if (!globals_for_name) {
-            assert(md->source->scoping->areGlobalsFromModule());
+            assert(md->source->scoping.areGlobalsFromModule());
             globals_for_name = md->source->parent_module;
         }
 
@@ -1647,7 +1647,7 @@ static Box* function_new(BoxedClass* cls, Box* code, Box* globals, Box** _args)
 
     FunctionMetadata* md = static_cast<BoxedCode*>(code)->f;
     RELEASE_ASSERT(md->source, "");
-    if (md->source->scoping->areGlobalsFromModule()) {
+    if (md->source->scoping.areGlobalsFromModule()) {
         RELEASE_ASSERT(unwrapAttrWrapper(globals) == md->source->parent_module, "");
         globals = NULL;
     } else {
@@ -1729,13 +1729,13 @@ static Box* function_globals(Box* self, void*) noexcept {
     assert(self->cls == function_cls);
     BoxedFunction* func = static_cast<BoxedFunction*>(self);
     if (func->globals) {
-        assert(!func->md->source || !func->md->source->scoping->areGlobalsFromModule());
+        assert(!func->md->source || !func->md->source->scoping.areGlobalsFromModule());
         if (func->globals->cls == module_cls)
             return incref(func->globals->getAttrWrapper());
         return incref(func->globals);
     }
     assert(func->md->source);
-    assert(func->md->source->scoping->areGlobalsFromModule());
+    assert(func->md->source->scoping.areGlobalsFromModule());
 
     static BoxedString* dict_str = getStaticString("__dict__");
     return getattrInternal<CAPI>(func->md->source->parent_module, dict_str);

test/tests/name_scopes.py

@@ -14,3 +14,20 @@ def f2():
 
     f3(*[1, 2], **dict(a=1, b=2))
 f2()
+
+__module__ = 1
+__name__ = 2
+__doc__ = 3
+print __module__ # prints "1"
+print __name__ # prints "2"
+print __doc__ # prints "3"
+class C(object):
+    "hello world"
+
+    # C's implicit setting of "__module__" will end up propagating to the global scope:
+    global __module__
+    global __name__
+    global __doc__
+print __module__ # prints "2"
+print __name__ # prints "2"
+print __doc__ # prints "hello world"

test/unittests/analysis.cpp

@@ -31,21 +31,25 @@ TEST_F(AnalysisTest, augassign) {
     AST_Module* module = caching_parse_file(fn.c_str(), 0);
     assert(module);
 
+    FutureFlags future_flags = getFutureFlags(module->body, fn.c_str());
+
     auto scoping = std::make_shared<ScopingAnalysis>(module, true);
+    computeAllCFGs(module, true, future_flags, boxString(fn), NULL);
 
     assert(module->body[0]->type == AST_TYPE::FunctionDef);
     AST_FunctionDef* func = static_cast<AST_FunctionDef*>(module->body[0]);
 
-    FutureFlags future_flags = getFutureFlags(module->body, fn.c_str());
+    ScopeInfo* scope_info = scoping->getScopeInfoForNode(func);
 
-    SourceInfo* si = new SourceInfo(createModule(boxString("augassign"), fn.c_str()), scoping, future_flags, func, boxString(fn));
-
-    ScopeInfo* scope_info = si->getScopeInfo();
-    ASSERT_FALSE(scope_info->getScopeTypeOfName(module->interned_strings->get("a")) == ScopeInfo::VarScopeType::GLOBAL);
+    ASSERT_NE(scope_info->getScopeTypeOfName(module->interned_strings->get("a")), ScopeInfo::VarScopeType::GLOBAL);
     ASSERT_FALSE(scope_info->getScopeTypeOfName(module->interned_strings->get("b")) == ScopeInfo::VarScopeType::GLOBAL);
 
-    ParamNames param_names(si->ast, si->getInternedStrings());
-    CFG* cfg = computeCFG(si, param_names);
+    ParamNames param_names(func, *module->interned_strings.get());
+
+    // Hack to get at the cfg:
+    auto node = module->md->source->cfg->blocks[0]->body[0];
+    CFG* cfg = ast_cast<AST_MakeFunction>(ast_cast<AST_Assign>(node)->value)->function_def->md->source->cfg;
+
     std::unique_ptr<LivenessAnalysis> liveness = computeLivenessInfo(cfg);
     auto&& vregs = cfg->getVRegInfo();
 
@@ -57,7 +61,7 @@ TEST_F(AnalysisTest, augassign) {
             ASSERT_TRUE(liveness->isLiveAtEnd(vregs.getVReg(module->interned_strings->get("a")), block));
     }
 
-    std::unique_ptr<PhiAnalysis> phis = computeRequiredPhis(ParamNames(func, si->getInternedStrings()), cfg, liveness.get(), scope_info);
+    std::unique_ptr<PhiAnalysis> phis = computeRequiredPhis(ParamNames(func, *module->interned_strings.get()), cfg, liveness.get());
 }
 
 void doOsrTest(bool is_osr, bool i_maybe_undefined) {
@@ -65,20 +69,22 @@ void doOsrTest(bool is_osr, bool i_maybe_undefined) {
     AST_Module* module = caching_parse_file(fn.c_str(), 0);
     assert(module);
 
-    auto scoping = std::make_shared<ScopingAnalysis>(module, true);
+    ParamNames param_names(module, *module->interned_strings.get());
 
     assert(module->body[0]->type == AST_TYPE::FunctionDef);
     AST_FunctionDef* func = static_cast<AST_FunctionDef*>(module->body[0]);
 
+    auto scoping = std::make_shared<ScopingAnalysis>(module, true);
+    ScopeInfo* scope_info = scoping->getScopeInfoForNode(func);
+
     FutureFlags future_flags = getFutureFlags(module->body, fn.c_str());
 
-    std::unique_ptr<SourceInfo> si(new SourceInfo(createModule(boxString("osr" + std::to_string((is_osr << 1) + i_maybe_undefined)),
-                    fn.c_str()), scoping, future_flags, func, boxString(fn)));
-    ScopeInfo* scope_info = si->getScopeInfo();
-    FunctionMetadata* clfunc = new FunctionMetadata(0, false, false, std::move(si));
+    computeAllCFGs(module, true, future_flags, boxString(fn), NULL);
 
-    CFG* cfg = computeCFG(clfunc->source.get(), clfunc->param_names);
-    clfunc->source->cfg = cfg;
+    // Hack to get at the cfg:
+    auto node = module->md->source->cfg->blocks[0]->body[0];
+    auto md = ast_cast<AST_MakeFunction>(ast_cast<AST_Assign>(node)->value)->function_def->md;
+    CFG* cfg = md->source->cfg;
     std::unique_ptr<LivenessAnalysis> liveness = computeLivenessInfo(cfg);
 
     // cfg->print();
@@ -101,16 +107,16 @@ void doOsrTest(bool is_osr, bool i_maybe_undefined) {
 
     if (is_osr) {
         int vreg = vregs.getVReg(i_str);
-        OSREntryDescriptor* entry_descriptor = OSREntryDescriptor::create(clfunc, backedge, CXX);
+        OSREntryDescriptor* entry_descriptor = OSREntryDescriptor::create(md, backedge, CXX);
         // need to set it to non-null
         ConcreteCompilerType* fake_type = (ConcreteCompilerType*)1;
         entry_descriptor->args[vreg] = fake_type;
         if (i_maybe_undefined)
             entry_descriptor->potentially_undefined.set(vreg);
         entry_descriptor->args[vregs.getVReg(iter_str)] = fake_type;
-        phis = computeRequiredPhis(entry_descriptor, liveness.get(), scope_info);
+        phis = computeRequiredPhis(entry_descriptor, liveness.get());
     } else {
-        phis = computeRequiredPhis(ParamNames(func, clfunc->source->getInternedStrings()), cfg, liveness.get(), scope_info);
+        phis = computeRequiredPhis(ParamNames(func, *module->interned_strings), cfg, liveness.get());
     }
 
     // First, verify that we require phi nodes for the block we enter into.