Revert "Revert "Merge pull request #1349 from kmod/scoping""

This reverts commit 111ca525.

src/analysis/function_analysis.cpp

@@ -214,10 +214,8 @@ class DefinednessBBAnalyzer : public BBAnalyzer<DefinednessAnalysis::DefinitionL
 private:
     typedef DefinednessAnalysis::DefinitionLevel DefinitionLevel;
 
-    ScopeInfo* scope_info;
-
 public:
-    DefinednessBBAnalyzer(ScopeInfo* scope_info) : scope_info(scope_info) {}
+    DefinednessBBAnalyzer() {}
 
     virtual DefinitionLevel merge(DefinitionLevel from, DefinitionLevel into) const {
         assert(from != DefinitionLevel::Unknown);
@@ -374,8 +372,7 @@ void DefinednessBBAnalyzer::processBB(Map& starting, CFGBlock* block) const {
     }
 }
 
-void DefinednessAnalysis::run(VRegMap<DefinednessAnalysis::DefinitionLevel> initial_map, CFGBlock* initial_block,
-                              ScopeInfo* scope_info) {
+void DefinednessAnalysis::run(VRegMap<DefinednessAnalysis::DefinitionLevel> initial_map, CFGBlock* initial_block) {
     Timer _t("DefinednessAnalysis()", 10);
 
     // Don't run this twice:
@@ -386,8 +383,8 @@ void DefinednessAnalysis::run(VRegMap<DefinednessAnalysis::DefinitionLevel> init
     assert(initial_map.numVregs() == nvregs);
 
     auto&& vreg_info = cfg->getVRegInfo();
-    computeFixedPoint(std::move(initial_map), initial_block, DefinednessBBAnalyzer(scope_info), false,
-                      defined_at_beginning, defined_at_end);
+    computeFixedPoint(std::move(initial_map), initial_block, DefinednessBBAnalyzer(), false, defined_at_beginning,
+                      defined_at_end);
 
     for (const auto& p : defined_at_end) {
         assert(p.second.numVregs() == nvregs);
@@ -398,14 +395,6 @@ void DefinednessAnalysis::run(VRegMap<DefinednessAnalysis::DefinitionLevel> init
         // required.resize(nvregs, /* value= */ false);
 
         for (int vreg = 0; vreg < nvregs; vreg++) {
-#ifndef NDEBUG
-            if (vreg_info.vregHasName(vreg)) {
-                ScopeInfo::VarScopeType vst = scope_info->getScopeTypeOfName(vreg_info.getName(vreg));
-                ASSERT(vst != ScopeInfo::VarScopeType::GLOBAL && vst != ScopeInfo::VarScopeType::NAME, "%s",
-                       vreg_info.getName(vreg).c_str());
-            }
-#endif
-
             auto status = p.second[vreg];
             // assert(p.second.count(name));
             // auto status = p.second.find(name)->second;
@@ -431,7 +420,7 @@ const VRegSet& DefinednessAnalysis::getDefinedVregsAtEnd(CFGBlock* block) {
 }
 
 PhiAnalysis::PhiAnalysis(VRegMap<DefinednessAnalysis::DefinitionLevel> initial_map, CFGBlock* initial_block,
-                         bool initials_need_phis, LivenessAnalysis* liveness, ScopeInfo* scope_info)
+                         bool initials_need_phis, LivenessAnalysis* liveness)
     : definedness(), empty_set(initial_map.numVregs()), liveness(liveness) {
     auto cfg = initial_block->cfg;
     auto&& vreg_info = cfg->getVRegInfo();
@@ -443,7 +432,7 @@ PhiAnalysis::PhiAnalysis(VRegMap<DefinednessAnalysis::DefinitionLevel> initial_m
     int num_vregs = initial_map.numVregs();
     assert(num_vregs == vreg_info.getTotalNumOfVRegs());
 
-    definedness.run(std::move(initial_map), initial_block, scope_info);
+    definedness.run(std::move(initial_map), initial_block);
 
     Timer _t("PhiAnalysis()", 10);
 
@@ -534,8 +523,7 @@ std::unique_ptr<LivenessAnalysis> computeLivenessInfo(CFG* cfg) {
     return std::unique_ptr<LivenessAnalysis>(new LivenessAnalysis(cfg));
 }
 
-std::unique_ptr<PhiAnalysis> computeRequiredPhis(const ParamNames& args, CFG* cfg, LivenessAnalysis* liveness,
-                                                 ScopeInfo* scope_info) {
+std::unique_ptr<PhiAnalysis> computeRequiredPhis(const ParamNames& args, CFG* cfg, LivenessAnalysis* liveness) {
     static StatCounter counter("num_phi_analysis");
     counter.log();
 
@@ -562,11 +550,11 @@ std::unique_ptr<PhiAnalysis> computeRequiredPhis(const ParamNames& args, CFG* cf
     assert(initial_map.numVregs() == vreg_info.getTotalNumOfVRegs());
 
     return std::unique_ptr<PhiAnalysis>(
-        new PhiAnalysis(std::move(initial_map), cfg->getStartingBlock(), false, liveness, scope_info));
+        new PhiAnalysis(std::move(initial_map), cfg->getStartingBlock(), false, liveness));
 }
 
-std::unique_ptr<PhiAnalysis> computeRequiredPhis(const OSREntryDescriptor* entry_descriptor, LivenessAnalysis* liveness,
-                                                 ScopeInfo* scope_info) {
+std::unique_ptr<PhiAnalysis> computeRequiredPhis(const OSREntryDescriptor* entry_descriptor,
+                                                 LivenessAnalysis* liveness) {
     static StatCounter counter("num_phi_analysis");
     counter.log();
 
@@ -588,6 +576,6 @@ std::unique_ptr<PhiAnalysis> computeRequiredPhis(const OSREntryDescriptor* entry
     }
 
     return std::unique_ptr<PhiAnalysis>(
-        new PhiAnalysis(std::move(initial_map), entry_descriptor->backedge->target, true, liveness, scope_info));
+        new PhiAnalysis(std::move(initial_map), entry_descriptor->backedge->target, true, liveness));
 }
 }

src/analysis/function_analysis.h

@@ -31,7 +31,6 @@ class AST_Jump;
 class AST_Name;
 class CFG;
 class CFGBlock;
-class ScopeInfo;
 class LivenessBBVisitor;
 
 class LivenessAnalysis {
@@ -72,7 +71,7 @@ private:
 public:
     DefinednessAnalysis() {}
 
-    void run(VRegMap<DefinitionLevel> initial_map, CFGBlock* initial_block, ScopeInfo* scope_info);
+    void run(VRegMap<DefinitionLevel> initial_map, CFGBlock* initial_block);
 
     DefinitionLevel isDefinedAtEnd(int vreg, CFGBlock* block);
     const VRegSet& getDefinedVregsAtEnd(CFGBlock* block);
@@ -94,7 +93,7 @@ public:
     // Initials_need_phis specifies that initial_map should count as an additional entry point
     // that may require phis.
     PhiAnalysis(VRegMap<DefinednessAnalysis::DefinitionLevel> initial_map, CFGBlock* initial_block,
-                bool initials_need_phis, LivenessAnalysis* liveness, ScopeInfo* scope_info);
+                bool initials_need_phis, LivenessAnalysis* liveness);
 
     bool isRequired(int vreg, CFGBlock* block);
     bool isRequiredAfter(int vreg, CFGBlock* block);
@@ -107,8 +106,8 @@ public:
 };
 
 std::unique_ptr<LivenessAnalysis> computeLivenessInfo(CFG*);
-std::unique_ptr<PhiAnalysis> computeRequiredPhis(const ParamNames&, CFG*, LivenessAnalysis*, ScopeInfo* scope_info);
-std::unique_ptr<PhiAnalysis> computeRequiredPhis(const OSREntryDescriptor*, LivenessAnalysis*, ScopeInfo* scope_info);
+std::unique_ptr<PhiAnalysis> computeRequiredPhis(const ParamNames&, CFG*, LivenessAnalysis*);
+std::unique_ptr<PhiAnalysis> computeRequiredPhis(const OSREntryDescriptor*, LivenessAnalysis*);
 }
 
 #endif

src/analysis/scoping_analysis.cpp

@@ -24,6 +24,28 @@
 
 namespace pyston {
 
+ScopingResults::ScopingResults(ScopeInfo* scope_info, bool globals_from_module)
+    : are_locals_from_module(scope_info->areLocalsFromModule()),
+      are_globals_from_module(globals_from_module),
+      creates_closure(scope_info->createsClosure()),
+      takes_closure(scope_info->takesClosure()),
+      passes_through_closure(scope_info->passesThroughClosure()),
+      uses_name_lookup(scope_info->usesNameLookup()),
+      closure_size(creates_closure ? scope_info->getClosureSize() : 0) {
+    deref_info = scope_info->getAllDerefVarsAndInfo();
+}
+
+DerefInfo ScopingResults::getDerefInfo(AST_Name* node) const {
+    assert(node->lookup_type == ScopeInfo::VarScopeType::DEREF);
+    assert(node->deref_info.offset != INT_MAX);
+    return node->deref_info;
+}
+size_t ScopingResults::getClosureOffset(AST_Name* node) const {
+    assert(node->lookup_type == ScopeInfo::VarScopeType::CLOSURE);
+    assert(node->closure_offset != -1);
+    return node->closure_offset;
+}
+
 class YieldVisitor : public NoopASTVisitor {
 public:
     AST* starting_node;
@@ -903,13 +925,13 @@ void ScopingAnalysis::processNameUsages(ScopingAnalysis::NameUsageMap* usages) {
         ScopeNameUsage* usage = sorted_usages[i];
         AST* node = usage->node;
 
-        ScopeInfo* parent_info = this->scopes[(usage->parent == NULL) ? this->parent_module : usage->parent->node];
+        ScopeInfo* parent_info
+            = this->scopes[(usage->parent == NULL) ? this->parent_module : usage->parent->node].get();
 
         switch (node->type) {
             case AST_TYPE::ClassDef: {
-                ScopeInfoBase* scopeInfo = new ScopeInfoBase(parent_info, usage, usage->node, true /* usesNameLookup */,
-                                                             globals_from_module);
-                this->scopes[node] = scopeInfo;
+                this->scopes[node] = llvm::make_unique<ScopeInfoBase>(parent_info, usage, usage->node,
+                                                                      true /* usesNameLookup */, globals_from_module);
                 break;
             }
             case AST_TYPE::FunctionDef:
@@ -917,10 +939,9 @@ void ScopingAnalysis::processNameUsages(ScopingAnalysis::NameUsageMap* usages) {
             case AST_TYPE::GeneratorExp:
             case AST_TYPE::DictComp:
             case AST_TYPE::SetComp: {
-                ScopeInfoBase* scopeInfo
-                    = new ScopeInfoBase(parent_info, usage, usage->node,
-                                        usage->hasNameForcingSyntax() /* usesNameLookup */, globals_from_module);
-                this->scopes[node] = scopeInfo;
+                this->scopes[node] = llvm::make_unique<ScopeInfoBase>(
+                    parent_info, usage, usage->node, usage->hasNameForcingSyntax() /* usesNameLookup */,
+                    globals_from_module);
                 break;
             }
             default:
@@ -934,45 +955,22 @@ InternedStringPool& ScopingAnalysis::getInternedStrings() {
     return *interned_strings;
 }
 
-ScopeInfo* ScopingAnalysis::analyzeSubtree(AST* node) {
+void ScopingAnalysis::analyzeSubtree(AST* node) {
     NameUsageMap usages;
     usages[node] = new ScopeNameUsage(node, NULL, this);
     NameCollectorVisitor::collect(node, &usages, this);
 
     processNameUsages(&usages);
-
-    ScopeInfo* rtn = scopes[node];
-    assert(rtn);
-    return rtn;
-}
-
-void ScopingAnalysis::registerScopeReplacement(AST* original_node, AST* new_node) {
-    assert(scope_replacements.count(original_node) == 0);
-    assert(scope_replacements.count(new_node) == 0);
-    assert(scopes.count(new_node) == 0);
-
-#ifndef NDEBUG
-    // NULL this out just to make sure it doesn't get accessed:
-    scopes[new_node] = NULL;
-#endif
-
-    scope_replacements[new_node] = original_node;
 }
 
 ScopeInfo* ScopingAnalysis::getScopeInfoForNode(AST* node) {
     assert(node);
 
-    auto it = scope_replacements.find(node);
-    if (it != scope_replacements.end())
-        node = it->second;
-
-    auto rtn = scopes.find(node);
-    if (rtn != scopes.end()) {
-        assert(rtn->second);
-        return rtn->second;
-    }
+    if (!scopes.count(node))
+        analyzeSubtree(node);
 
-    return analyzeSubtree(node);
+    assert(scopes.count(node));
+    return scopes[node].get();
 }
 
 ScopingAnalysis::ScopingAnalysis(AST* ast, bool globals_from_module)
@@ -993,10 +991,10 @@ ScopingAnalysis::ScopingAnalysis(AST* ast, bool globals_from_module)
 
     if (globals_from_module) {
         assert(ast->type == AST_TYPE::Module);
-        scopes[ast] = new ModuleScopeInfo();
+        scopes[ast] = llvm::make_unique<ModuleScopeInfo>();
         parent_module = static_cast<AST_Module*>(ast);
     } else {
-        scopes[ast] = new EvalExprScopeInfo(ast, globals_from_module);
+        scopes[ast] = llvm::make_unique<EvalExprScopeInfo>(ast, globals_from_module);
     }
 }
 }

src/analysis/scoping_analysis.h

@@ -19,6 +19,7 @@
 
 #include "core/common.h"
 #include "core/stringpool.h"
+#include "core/types.h"
 
 namespace pyston {
 
@@ -27,16 +28,6 @@ class AST_Module;
 class AST_Expression;
 class AST_Suite;
 
-// Each closure has an array (fixed-size for that particular scope) of variables
-// and a parent pointer to a parent closure. To look up a variable from the passed-in
-// closure (i.e., DEREF), you just need to know (i) how many parents up to go and
-// (ii) what offset into the array to find the variable. This struct stores that
-// information. You can query the ScopeInfo with a name to get this info.
-struct DerefInfo {
-    size_t num_parents_from_passed_closure;
-    size_t offset;
-};
-
 class ScopeInfo {
 public:
     ScopeInfo() {}
@@ -151,27 +142,16 @@ public:
     typedef llvm::DenseMap<AST*, ScopeNameUsage*> NameUsageMap;
 
 private:
-    llvm::DenseMap<AST*, ScopeInfo*> scopes;
+    llvm::DenseMap<AST*, std::unique_ptr<ScopeInfo>> scopes;
     AST_Module* parent_module;
     InternedStringPool* interned_strings;
 
-    llvm::DenseMap<AST*, AST*> scope_replacements;
-
-    ScopeInfo* analyzeSubtree(AST* node);
+    void analyzeSubtree(AST* node);
     void processNameUsages(NameUsageMap* usages);
 
     bool globals_from_module;
 
 public:
-    // The scope-analysis is done before any CFG-ization is done,
-    // but many of the queries will be done post-CFG-ization.
-    // The CFG process can replace scope AST nodes with others (ex:
-    // generator expressions with generator functions), so we need to
-    // have a way of mapping the original analysis with the new queries.
-    // This is a hook for the CFG process to register when it has replaced
-    // a scope-node with a different node.
-    void registerScopeReplacement(AST* original_node, AST* new_node);
-
     ScopingAnalysis(AST* ast, bool globals_from_module);
     ScopeInfo* getScopeInfoForNode(AST* node);
 

src/analysis/type_analysis.cpp

@@ -93,17 +93,14 @@ private:
     ExprTypeMap& expr_types;
     TypeSpeculations& type_speculations;
     TypeAnalysis::SpeculationLevel speculation;
-    ScopeInfo* scope_info;
 
     BasicBlockTypePropagator(CFGBlock* block, TypeMap& initial, ExprTypeMap& expr_types,
-                             TypeSpeculations& type_speculations, TypeAnalysis::SpeculationLevel speculation,
-                             ScopeInfo* scope_info)
+                             TypeSpeculations& type_speculations, TypeAnalysis::SpeculationLevel speculation)
         : block(block),
           sym_table(initial),
           expr_types(expr_types),
           type_speculations(type_speculations),
-          speculation(speculation),
-          scope_info(scope_info) {}
+          speculation(speculation) {}
 
     void run() {
         for (int i = 0; i < block->body.size(); i++) {
@@ -429,8 +426,7 @@ private:
     }
 
     void* visit_name(AST_Name* node) override {
-        if (node->lookup_type == ScopeInfo::VarScopeType::UNKNOWN)
-            node->lookup_type = scope_info->getScopeTypeOfName(node->id);
+        assert(node->lookup_type != ScopeInfo::VarScopeType::UNKNOWN);
         auto name_scope = node->lookup_type;
 
         if (name_scope == ScopeInfo::VarScopeType::GLOBAL) {
@@ -671,10 +667,9 @@ private:
 
 public:
     static TypeMap propagate(CFGBlock* block, const TypeMap& starting, ExprTypeMap& expr_types,
-                             TypeSpeculations& type_speculations, TypeAnalysis::SpeculationLevel speculation,
-                             ScopeInfo* scope_info) {
+                             TypeSpeculations& type_speculations, TypeAnalysis::SpeculationLevel speculation) {
         TypeMap ending = starting;
-        BasicBlockTypePropagator(block, ending, expr_types, type_speculations, speculation, scope_info).run();
+        BasicBlockTypePropagator(block, ending, expr_types, type_speculations, speculation).run();
         return ending;
     }
 };
@@ -753,8 +748,8 @@ public:
         return changed;
     }
 
-    static PropagatingTypeAnalysis* doAnalysis(SpeculationLevel speculation, ScopeInfo* scope_info,
-                                               TypeMap&& initial_types, CFGBlock* initial_block) {
+    static PropagatingTypeAnalysis* doAnalysis(SpeculationLevel speculation, TypeMap&& initial_types,
+                                               CFGBlock* initial_block) {
         Timer _t("PropagatingTypeAnalysis::doAnalysis()");
 
         CFG* cfg = initial_block->cfg;
@@ -795,7 +790,7 @@ public:
             }
 
             TypeMap ending = BasicBlockTypePropagator::propagate(block, starting_types.find(block)->second, expr_types,
-                                                                 type_speculations, speculation, scope_info);
+                                                                 type_speculations, speculation);
 
             if (VERBOSITY("types") >= 3) {
                 printf("before (after):\n");
@@ -851,7 +846,7 @@ public:
 
 // public entry point:
 TypeAnalysis* doTypeAnalysis(CFG* cfg, const ParamNames& arg_names, const std::vector<ConcreteCompilerType*>& arg_types,
-                             EffortLevel effort, TypeAnalysis::SpeculationLevel speculation, ScopeInfo* scope_info) {
+                             EffortLevel effort, TypeAnalysis::SpeculationLevel speculation) {
     // if (effort == EffortLevel::INTERPRETED) {
     // return new NullTypeAnalysis();
     //}
@@ -870,12 +865,11 @@ TypeAnalysis* doTypeAnalysis(CFG* cfg, const ParamNames& arg_names, const std::v
 
     assert(i == arg_types.size());
 
-    return PropagatingTypeAnalysis::doAnalysis(speculation, scope_info, std::move(initial_types),
-                                               cfg->getStartingBlock());
+    return PropagatingTypeAnalysis::doAnalysis(speculation, std::move(initial_types), cfg->getStartingBlock());
 }
 
 TypeAnalysis* doTypeAnalysis(const OSREntryDescriptor* entry_descriptor, EffortLevel effort,
-                             TypeAnalysis::SpeculationLevel speculation, ScopeInfo* scope_info) {
+                             TypeAnalysis::SpeculationLevel speculation) {
     auto cfg = entry_descriptor->md->source->cfg;
     auto&& vreg_info = cfg->getVRegInfo();
     TypeMap initial_types(vreg_info.getTotalNumOfVRegs());
@@ -884,7 +878,7 @@ TypeAnalysis* doTypeAnalysis(const OSREntryDescriptor* entry_descriptor, EffortL
         initial_types[p.first] = p.second;
     }
 
-    return PropagatingTypeAnalysis::doAnalysis(speculation, scope_info, std::move(initial_types),
+    return PropagatingTypeAnalysis::doAnalysis(speculation, std::move(initial_types),
                                                entry_descriptor->backedge->target);
 }
 }

src/analysis/type_analysis.h

@@ -23,7 +23,6 @@
 
 namespace pyston {
 
-class ScopeInfo;
 class CFGBlock;
 class BoxedClass;
 class AST_expr;
@@ -47,9 +46,9 @@ public:
 
 TypeAnalysis* doTypeAnalysis(CFG* cfg, const ParamNames& param_names,
                              const std::vector<ConcreteCompilerType*>& arg_types, EffortLevel effort,
-                             TypeAnalysis::SpeculationLevel speculation, ScopeInfo* scope_info);
+                             TypeAnalysis::SpeculationLevel speculation);
 TypeAnalysis* doTypeAnalysis(const OSREntryDescriptor* entry_descriptor, EffortLevel effort,
-                             TypeAnalysis::SpeculationLevel speculation, ScopeInfo* scope_info);
+                             TypeAnalysis::SpeculationLevel speculation);
 }
 
 #endif

src/codegen/ast_interpreter.h

@@ -47,11 +47,11 @@ struct ASTInterpreterJitInterface {
     static int getGlobalsOffset();
 
     static void delNameHelper(void* _interpreter, InternedString name);
-    static Box* derefHelper(void* interp, InternedString s);
+    static Box* derefHelper(void* interp, AST_Name* node);
     static Box* landingpadHelper(void* interp);
     static void pendingCallsCheckHelper();
     static void setExcInfoHelper(void* interp, STOLEN(Box*) type, STOLEN(Box*) value, STOLEN(Box*) traceback);
-    static void setLocalClosureHelper(void* interp, long vreg, InternedString id, Box* v);
+    static void setLocalClosureHelper(void* interp, AST_Name* name, Box* v);
     static void uncacheExcInfoHelper(void* interp);
     static void raise0Helper(void* interp) __attribute__((noreturn));
     static Box* yieldHelper(void* interp, STOLEN(Box*) value);

src/codegen/baseline_jit.cpp

@@ -372,14 +372,8 @@ RewriterVar* JitFragmentWriter::emitCreateTuple(const llvm::ArrayRef<RewriterVar
     return r;
 }
 
-RewriterVar* JitFragmentWriter::emitDeref(InternedString s) {
-    return call(false, (void*)ASTInterpreterJitInterface::derefHelper, getInterp(),
-#ifndef NDEBUG
-                imm(asUInt(s).first), imm(asUInt(s).second))
-#else
-                imm(asUInt(s)))
-#endif
-        ->setType(RefType::OWNED);
+RewriterVar* JitFragmentWriter::emitDeref(AST_Name* name) {
+    return call(false, (void*)ASTInterpreterJitInterface::derefHelper, getInterp(), imm(name))->setType(RefType::OWNED);
 }
 
 RewriterVar* JitFragmentWriter::emitExceptionMatches(RewriterVar* v, RewriterVar* cls) {
@@ -391,20 +385,22 @@ RewriterVar* JitFragmentWriter::emitGetAttr(RewriterVar* obj, BoxedString* s, AS
         .first->setType(RefType::OWNED);
 }
 
-RewriterVar* JitFragmentWriter::emitGetBlockLocal(InternedString s, int vreg) {
+RewriterVar* JitFragmentWriter::emitGetBlockLocal(AST_Name* name) {
+    auto s = name->id;
+    auto vreg = name->vreg;
     auto it = local_syms.find(s);
     if (it == local_syms.end()) {
-        auto r = emitGetLocal(s, vreg);
+        auto r = emitGetLocal(name);
         assert(r->reftype == RefType::OWNED);
-        emitSetBlockLocal(s, vreg, r);
+        emitSetBlockLocal(name, r);
         return r;
     }
     return it->second;
 }
 
-void JitFragmentWriter::emitKillTemporary(InternedString s, int vreg) {
-    if (!local_syms.count(s))
-        emitSetLocal(s, vreg, false, imm(nullptr));
+void JitFragmentWriter::emitKillTemporary(AST_Name* name) {
+    if (!local_syms.count(name->id))
+        emitSetLocal(name, false, imm(nullptr));
 }
 
 RewriterVar* JitFragmentWriter::emitGetBoxedLocal(BoxedString* s) {
@@ -436,9 +432,11 @@ RewriterVar* JitFragmentWriter::emitGetItem(AST_expr* node, RewriterVar* value,
         .first->setType(RefType::OWNED);
 }
 
-RewriterVar* JitFragmentWriter::emitGetLocal(InternedString s, int vreg) {
+RewriterVar* JitFragmentWriter::emitGetLocal(AST_Name* name) {
     if (LOG_BJIT_ASSEMBLY)
         comment("BJIT: emitGetLocal start");
+    auto vreg = name->vreg;
+    auto s = name->id;
     assert(vreg >= 0);
     // TODO Can we use BORROWED here? Not sure if there are cases when we can't rely on borrowing the ref
     // from the vregs array.  Safer like this.
@@ -686,14 +684,16 @@ void JitFragmentWriter::emitSetAttr(AST_expr* node, RewriterVar* obj, BoxedStrin
     attr->refConsumed(rtn.second);
 }
 
-void JitFragmentWriter::emitSetBlockLocal(InternedString s, int vreg, STOLEN(RewriterVar*) v) {
+void JitFragmentWriter::emitSetBlockLocal(AST_Name* name, STOLEN(RewriterVar*) v) {
     if (LOG_BJIT_ASSEMBLY)
         comment("BJIT: emitSetBlockLocal() start");
-    RewriterVar* prev = local_syms[s];
+    auto vreg = name->vreg;
+    auto s = name->id;
+    RewriterVar* prev = local_syms[name->id];
     // if we never set this sym before in this BB and the symbol gets accessed in several blocks clear it because it
     // could have been set in a previous block.
     if (!prev && !block->cfg->getVRegInfo().isBlockLocalVReg(vreg))
-        emitSetLocal(s, vreg, false, imm(nullptr)); // clear out the vreg
+        emitSetLocal(name, false, imm(nullptr)); // clear out the vreg
     local_syms[s] = v;
     if (LOG_BJIT_ASSEMBLY)
         comment("BJIT: emitSetBlockLocal() end");
@@ -728,18 +728,13 @@ void JitFragmentWriter::emitSetItemName(BoxedString* s, RewriterVar* v) {
     emitSetItem(emitGetBoxedLocals(), imm(s), v);
 }
 
-void JitFragmentWriter::emitSetLocal(InternedString s, int vreg, bool set_closure, STOLEN(RewriterVar*) v) {
+void JitFragmentWriter::emitSetLocal(AST_Name* name, bool set_closure, STOLEN(RewriterVar*) v) {
     if (LOG_BJIT_ASSEMBLY)
         comment("BJIT: emitSetLocal() start");
+    auto vreg = name->vreg;
     assert(vreg >= 0);
     if (set_closure) {
-        call(false, (void*)ASTInterpreterJitInterface::setLocalClosureHelper, getInterp(), imm(vreg),
-#ifndef NDEBUG
-             imm(asUInt(s).first), imm(asUInt(s).second),
-#else
-             imm(asUInt(s)),
-#endif
-             v);
+        call(false, (void*)ASTInterpreterJitInterface::setLocalClosureHelper, getInterp(), imm(name), v);
         v->refConsumed();
     } else {
         // TODO With definedness analysis, we could know whether we needed to emit an decref/xdecref/neither.

src/codegen/baseline_jit.h

@@ -278,17 +278,17 @@ public:
     RewriterVar* emitCreateSet(const llvm::ArrayRef<RewriterVar*> values);
     RewriterVar* emitCreateSlice(RewriterVar* start, RewriterVar* stop, RewriterVar* step);
     RewriterVar* emitCreateTuple(const llvm::ArrayRef<RewriterVar*> values);
-    RewriterVar* emitDeref(InternedString s);
+    RewriterVar* emitDeref(AST_Name* name);
     RewriterVar* emitExceptionMatches(RewriterVar* v, RewriterVar* cls);
     RewriterVar* emitGetAttr(RewriterVar* obj, BoxedString* s, AST_expr* node);
-    RewriterVar* emitGetBlockLocal(InternedString s, int vreg);
-    void emitKillTemporary(InternedString s, int vreg);
+    RewriterVar* emitGetBlockLocal(AST_Name* name);
+    void emitKillTemporary(AST_Name* name);
     RewriterVar* emitGetBoxedLocal(BoxedString* s);
     RewriterVar* emitGetBoxedLocals();
     RewriterVar* emitGetClsAttr(RewriterVar* obj, BoxedString* s);
     RewriterVar* emitGetGlobal(BoxedString* s);
     RewriterVar* emitGetItem(AST_expr* node, RewriterVar* value, RewriterVar* slice);
-    RewriterVar* emitGetLocal(InternedString s, int vreg);
+    RewriterVar* emitGetLocal(AST_Name* name);
     RewriterVar* emitGetPystonIter(RewriterVar* v);
     RewriterVar* emitHasnext(RewriterVar* v);
     RewriterVar* emitImportFrom(RewriterVar* module, BoxedString* name);
@@ -318,13 +318,13 @@ public:
     void emitRaise3(RewriterVar* arg0, RewriterVar* arg1, RewriterVar* arg2);
     void emitReturn(RewriterVar* v);
     void emitSetAttr(AST_expr* node, RewriterVar* obj, BoxedString* s, STOLEN(RewriterVar*) attr);
-    void emitSetBlockLocal(InternedString s, int vreg, STOLEN(RewriterVar*) v);
+    void emitSetBlockLocal(AST_Name* name, STOLEN(RewriterVar*) v);
     void emitSetCurrentInst(AST_stmt* node);
     void emitSetExcInfo(RewriterVar* type, RewriterVar* value, RewriterVar* traceback);
     void emitSetGlobal(BoxedString* s, STOLEN(RewriterVar*) v, bool are_globals_from_module);
     void emitSetItemName(BoxedString* s, RewriterVar* v);
     void emitSetItem(RewriterVar* target, RewriterVar* slice, RewriterVar* value);
-    void emitSetLocal(InternedString s, int vreg, bool set_closure, STOLEN(RewriterVar*) v);
+    void emitSetLocal(AST_Name* name, bool set_closure, STOLEN(RewriterVar*) v);
     // emitSideExit steals a full ref from v, not just a vref
     void emitSideExit(STOLEN(RewriterVar*) v, Box* cmp_value, CFGBlock* next_block);
     void emitUncacheExcInfo();

src/codegen/codegen.cpp

@@ -38,10 +38,10 @@
 namespace pyston {
 
 FunctionMetadata::FunctionMetadata(int num_args, bool takes_varargs, bool takes_kwargs,
-                                   std::unique_ptr<SourceInfo> source)
+                                   std::unique_ptr<SourceInfo> source, ParamNames param_names)
     : code_obj(NULL),
       source(std::move(source)),
-      param_names(this->source->ast, this->source->getInternedStrings()),
+      param_names(std::move(param_names)),
       takes_varargs(takes_varargs),
       takes_kwargs(takes_kwargs),
       num_args(num_args),
@@ -91,8 +91,8 @@ void FunctionMetadata::addVersion(CompiledFunction* compiled) {
     }
 }
 
-SourceInfo::SourceInfo(BoxedModule* m, ScopingAnalysis* scoping, FutureFlags future_flags, AST* ast, BoxedString* fn)
-    : parent_module(m), scoping(scoping), scope_info(NULL), ast(ast), cfg(NULL), future_flags(future_flags) {
+SourceInfo::SourceInfo(BoxedModule* m, ScopingResults scoping, FutureFlags future_flags, AST* ast, BoxedString* fn)
+    : parent_module(m), scoping(std::move(scoping)), ast(ast), cfg(NULL), future_flags(future_flags) {
     assert(fn);
 
     // TODO: this is a very bad way of handling this:

src/codegen/irgen.cpp

@@ -408,7 +408,7 @@ static void emitBBs(IRGenState* irstate, TypeAnalysis* types, const OSREntryDesc
         irstate->getRefcounts()->setType(osr_created_closure, RefType::BORROWED);
         if (source->is_generator)
             irstate->setPassedGenerator(osr_generator);
-        if (source->getScopeInfo()->createsClosure())
+        if (source->scoping.createsClosure())
             irstate->setCreatedClosure(osr_created_closure);
 
         int arg_num = -1;
@@ -730,13 +730,16 @@ static void emitBBs(IRGenState* irstate, TypeAnalysis* types, const OSREntryDesc
                         auto asgn = ast_cast<AST_Assign>(stmt);
                         assert(asgn->targets.size() == 1);
                         if (asgn->targets[0]->type == AST_TYPE::Name) {
-                            InternedString name = ast_cast<AST_Name>(asgn->targets[0])->id;
+                            auto asname = ast_cast<AST_Name>(asgn->targets[0]);
+                            assert(asname->lookup_type != ScopeInfo::VarScopeType::UNKNOWN);
+
+                            InternedString name = asname->id;
                             int vreg = ast_cast<AST_Name>(asgn->targets[0])->vreg;
                             assert(name.c_str()[0] == '#'); // it must be a temporary
                             // You might think I need to check whether `name' is being assigned globally or locally,
                             // since a global assign doesn't affect the symbol table. However, the CFG pass only
                             // generates invoke-assigns to temporary variables. Just to be sure, we assert:
-                            assert(source->getScopeInfo()->getScopeTypeOfName(name) != ScopeInfo::VarScopeType::GLOBAL);
+                            assert(asname->lookup_type != ScopeInfo::VarScopeType::GLOBAL);
 
                             // TODO: inefficient
                             sym_table = new SymbolTable(*sym_table);
@@ -1063,13 +1066,13 @@ std::pair<CompiledFunction*, llvm::Function*> doCompile(FunctionMetadata* md, So
         int nargs = param_names->totalParameters();
         ASSERT(nargs == spec->arg_types.size(), "%d %ld", nargs, spec->arg_types.size());
 
-        if (source->getScopeInfo()->takesClosure())
+        if (source->scoping.takesClosure())
             llvm_arg_types.push_back(g.llvm_closure_type_ptr);
 
         if (source->is_generator)
             llvm_arg_types.push_back(g.llvm_generator_type_ptr);
 
-        if (!source->scoping->areGlobalsFromModule())
+        if (!source->scoping.areGlobalsFromModule())
             llvm_arg_types.push_back(g.llvm_value_type_ptr);
 
         for (int i = 0; i < nargs; i++) {
@@ -1109,10 +1112,9 @@ std::pair<CompiledFunction*, llvm::Function*> doCompile(FunctionMetadata* md, So
         speculation_level = TypeAnalysis::SOME;
     TypeAnalysis* types;
     if (entry_descriptor)
-        types = doTypeAnalysis(entry_descriptor, effort, speculation_level, source->getScopeInfo());
+        types = doTypeAnalysis(entry_descriptor, effort, speculation_level);
     else
-        types = doTypeAnalysis(source->cfg, *param_names, spec->arg_types, effort, speculation_level,
-                               source->getScopeInfo());
+        types = doTypeAnalysis(source->cfg, *param_names, spec->arg_types, effort, speculation_level);
 
 
     _t2.split();
@@ -1131,9 +1133,9 @@ std::pair<CompiledFunction*, llvm::Function*> doCompile(FunctionMetadata* md, So
     std::unique_ptr<PhiAnalysis> phis;
 
     if (entry_descriptor)
-        phis = computeRequiredPhis(entry_descriptor, liveness, source->getScopeInfo());
+        phis = computeRequiredPhis(entry_descriptor, liveness);
     else
-        phis = computeRequiredPhis(*param_names, source->cfg, liveness, source->getScopeInfo());
+        phis = computeRequiredPhis(*param_names, source->cfg, liveness);
 
     RefcountTracker refcounter;
 

src/codegen/irgen/hooks.cpp

@@ -54,78 +54,6 @@
 
 namespace pyston {
 
-// TODO terrible place for these!
-ParamNames::ParamNames(AST* ast, InternedStringPool& pool)
-    : all_args_contains_names(1), takes_param_names(1), has_vararg_name(0), has_kwarg_name(0) {
-    if (ast->type == AST_TYPE::Module || ast->type == AST_TYPE::ClassDef || ast->type == AST_TYPE::Expression
-        || ast->type == AST_TYPE::Suite) {
-    } else if (ast->type == AST_TYPE::FunctionDef || ast->type == AST_TYPE::Lambda) {
-        AST_arguments* arguments = ast->type == AST_TYPE::FunctionDef ? ast_cast<AST_FunctionDef>(ast)->args
-                                                                      : ast_cast<AST_Lambda>(ast)->args;
-        for (int i = 0; i < arguments->args.size(); i++) {
-            AST_expr* arg = arguments->args[i];
-            if (arg->type == AST_TYPE::Name) {
-                AST_Name* name = ast_cast<AST_Name>(arg);
-                all_args.emplace_back(name);
-            } else {
-                InternedString dot_arg_name = pool.get("." + std::to_string(i));
-                all_args.emplace_back(new AST_Name(dot_arg_name, AST_TYPE::Param, arg->lineno, arg->col_offset));
-            }
-        }
-
-        auto vararg_name = arguments->vararg;
-        if (vararg_name) {
-            has_vararg_name = 1;
-            all_args.emplace_back(vararg_name);
-        }
-
-        auto kwarg_name = arguments->kwarg;
-        if (kwarg_name) {
-            has_kwarg_name = 1;
-            all_args.emplace_back(kwarg_name);
-        }
-    } else {
-        RELEASE_ASSERT(0, "%d", ast->type);
-    }
-}
-
-ParamNames::ParamNames(const std::vector<const char*>& args, const char* vararg, const char* kwarg)
-    : all_args_contains_names(0),
-      takes_param_names(1),
-      has_vararg_name(vararg && *vararg),
-      has_kwarg_name(kwarg && *kwarg) {
-    all_args.reserve(args.size() + has_vararg_name + has_kwarg_name);
-    for (auto&& arg : args) {
-        all_args.emplace_back(arg);
-    }
-    if (has_vararg_name)
-        all_args.emplace_back(vararg);
-    if (has_kwarg_name)
-        all_args.emplace_back(kwarg);
-}
-
-std::vector<const char*> ParamNames::allArgsAsStr() const {
-    std::vector<const char*> ret;
-    ret.reserve(all_args.size());
-    if (all_args_contains_names) {
-        for (auto&& arg : all_args) {
-            ret.push_back(arg.name->id.c_str());
-        }
-    } else {
-        for (auto&& arg : all_args) {
-            ret.push_back(arg.str);
-        }
-    }
-    return ret;
-}
-
-InternedString SourceInfo::mangleName(InternedString id) {
-    assert(ast);
-    if (ast->type == AST_TYPE::Module)
-        return id;
-    return getScopeInfo()->mangleName(id);
-}
-
 llvm::ArrayRef<AST_stmt*> SourceInfo::getBody() const {
     switch (ast->type) {
         case AST_TYPE::ClassDef:
@@ -141,10 +69,6 @@ llvm::ArrayRef<AST_stmt*> SourceInfo::getBody() const {
     };
 }
 
-InternedStringPool& SourceInfo::getInternedStrings() {
-    return scoping->getInternedStrings();
-}
-
 BORROWED(BoxedString*) SourceInfo::getFn() {
     assert(fn->ob_refcnt >= 1);
     return fn;
@@ -182,12 +106,6 @@ Box* SourceInfo::getDocString() {
     return incref(Py_None);
 }
 
-ScopeInfo* SourceInfo::getScopeInfo() {
-    if (!scope_info)
-        scope_info = scoping->getScopeInfoForNode(ast);
-    return scope_info;
-}
-
 LivenessAnalysis* SourceInfo::getLiveness() {
     if (!liveness_info)
         liveness_info = computeLivenessInfo(cfg);
@@ -310,10 +228,7 @@ CompiledFunction* compileFunction(FunctionMetadata* f, FunctionSpecialization* s
         printf("%s", ss.str().c_str());
     }
 
-    // Do the analysis now if we had deferred it earlier:
-    if (source->cfg == NULL) {
-        source->cfg = computeCFG(source, f->param_names);
-    }
+    assert(source->cfg);
 
 
     CompiledFunction* cf = NULL;
@@ -362,42 +277,32 @@ CompiledFunction* compileFunction(FunctionMetadata* f, FunctionSpecialization* s
 }
 
 void compileAndRunModule(AST_Module* m, BoxedModule* bm) {
-    FunctionMetadata* md;
-
     Timer _t("for compileModule()");
 
     const char* fn = PyModule_GetFilename(bm);
     RELEASE_ASSERT(fn, "");
 
     FutureFlags future_flags = getFutureFlags(m->body, fn);
-    ScopingAnalysis* scoping = new ScopingAnalysis(m, true);
+    computeAllCFGs(m, /* globals_from_module */ true, future_flags, autoDecref(boxString(fn)), bm);
 
-    auto fn_str = boxString(fn);
-    AUTO_DECREF(fn_str);
-    std::unique_ptr<SourceInfo> si(new SourceInfo(bm, scoping, future_flags, m, fn_str));
+    FunctionMetadata* md = metadataForAST(m);
+    assert(md);
 
     static BoxedString* doc_str = getStaticString("__doc__");
-    bm->setattr(doc_str, autoDecref(si->getDocString()), NULL);
+    bm->setattr(doc_str, autoDecref(md->source->getDocString()), NULL);
 
     static BoxedString* builtins_str = getStaticString("__builtins__");
     if (!bm->hasattr(builtins_str))
         bm->setattr(builtins_str, PyModule_GetDict(builtins_module), NULL);
 
-    md = new FunctionMetadata(0, false, false, std::move(si));
-
     UNAVOIDABLE_STAT_TIMER(t0, "us_timer_interpreted_module_toplevel");
     Box* r = astInterpretFunction(md, NULL, NULL, NULL, NULL, NULL, NULL, NULL);
     assert(r == Py_None);
     Py_DECREF(r);
-
-    // XXX for bjit testing
-    // r = astInterpretFunction(md, NULL, NULL, NULL, NULL, NULL, NULL, NULL);
-    // assert(r == None);
-    // Py_DECREF(r);
 }
 
 Box* evalOrExec(FunctionMetadata* md, Box* globals, Box* boxedLocals) {
-    RELEASE_ASSERT(!md->source->scoping->areGlobalsFromModule(), "");
+    RELEASE_ASSERT(!md->source->scoping.areGlobalsFromModule(), "");
 
     assert(globals && (globals->cls == module_cls || globals->cls == dict_cls));
 
@@ -416,8 +321,6 @@ static FunctionMetadata* compileForEvalOrExec(AST* source, llvm::ArrayRef<AST_st
                                               PyCompilerFlags* flags) {
     Timer _t("for evalOrExec()");
 
-    ScopingAnalysis* scoping = new ScopingAnalysis(source, false);
-
     // `my_future_flags` are the future flags enabled in the exec's code.
     // `caller_future_flags` are the future flags of the source that the exec statement is in.
     // We need to enable features that are enabled in either.
@@ -429,10 +332,8 @@ static FunctionMetadata* compileForEvalOrExec(AST* source, llvm::ArrayRef<AST_st
         flags->cf_flags = future_flags;
     }
 
-    std::unique_ptr<SourceInfo> si(new SourceInfo(getCurrentModule(), scoping, future_flags, source, fn));
-
-    FunctionMetadata* md = new FunctionMetadata(0, false, false, std::move(si));
-    return md;
+    computeAllCFGs(source, /* globals_from_module */ false, future_flags, fn, getCurrentModule());
+    return metadataForAST(source);
 }
 
 static FunctionMetadata* compileExec(AST_Module* parsedModule, BoxedString* fn, PyCompilerFlags* flags) {

src/codegen/irgen/irgenerator.h

@@ -118,8 +118,7 @@ public:
     LivenessAnalysis* getLiveness() { return source_info->getLiveness(); }
     PhiAnalysis* getPhis() { return phis.get(); }
 
-    ScopeInfo* getScopeInfo();
-    ScopeInfo* getScopeInfoForNode(AST* node);
+    const ScopingResults& getScopeInfo();
 
     llvm::MDNode* getFuncDbgInfo() { return func_dbg_info; }
 
@@ -205,7 +204,6 @@ IREmitter* createIREmitter(IRGenState* irstate, llvm::BasicBlock*& curblock, IRG
 IRGenerator* createIRGenerator(IRGenState* irstate, std::unordered_map<CFGBlock*, llvm::BasicBlock*>& entry_blocks,
                                CFGBlock* myblock, TypeAnalysis* types);
 
-FunctionMetadata* wrapFunction(AST* node, AST_arguments* args, SourceInfo* source);
 std::vector<BoxedString*>* getKeywordNameStorage(AST_Call* node);
 }
 

src/codegen/patchpoints.cpp

@@ -98,9 +98,9 @@ void PatchpointInfo::parseLocationMap(StackMap::Record* r, LocationMap* map) {
     auto&& source = parentFunction()->md->source;
     if (source->is_generator)
         map->generator.locations.push_back(parse_type(GENERATOR));
-    if (source->getScopeInfo()->takesClosure())
+    if (source->scoping.takesClosure())
         map->passed_closure.locations.push_back(parse_type(CLOSURE));
-    if (source->getScopeInfo()->createsClosure())
+    if (source->scoping.createsClosure())
         map->created_closure.locations.push_back(parse_type(CLOSURE));
 
     for (FrameVarInfo& frame_var : frame_info_desc.vars) {

src/codegen/unwinding.cpp

@@ -985,12 +985,12 @@ BORROWED(Box*) FrameInfo::updateBoxedLocals() {
 
     FrameInfo* frame_info = this;
     FunctionMetadata* md = frame_info->md;
-    ScopeInfo* scope_info = md->source->getScopeInfo();
+    const ScopingResults& scope_info = md->source->scoping;
 
-    if (scope_info->areLocalsFromModule()) {
+    if (scope_info.areLocalsFromModule()) {
         // TODO we should cache this in frame_info->locals or something so that locals()
         // (and globals() too) will always return the same dict
-        RELEASE_ASSERT(md->source->scoping->areGlobalsFromModule(), "");
+        RELEASE_ASSERT(md->source->scoping.areGlobalsFromModule(), "");
         return md->source->parent_module->getAttrWrapper();
     }
 
@@ -1000,7 +1000,7 @@ BORROWED(Box*) FrameInfo::updateBoxedLocals() {
     // Add the locals from the closure
     // TODO in a ClassDef scope, we aren't supposed to add these
     size_t depth = 0;
-    for (auto& p : scope_info->getAllDerefVarsAndInfo()) {
+    for (auto& p : scope_info.getAllDerefVarsAndInfo()) {
         InternedString name = p.first;
         DerefInfo derefInfo = p.second;
         while (depth < derefInfo.num_parents_from_passed_closure) {

src/core/ast.cpp

@@ -1639,8 +1639,24 @@ bool PrintVisitor::visit_suite(AST_Suite* node) {
 
 bool PrintVisitor::visit_name(AST_Name* node) {
     stream << node->id.s();
-    // Uncomment this line to see which names are kills:
-    // if (node->is_kill) stream << "<k>";
+#if 0
+    if (node->lookup_type == ScopeInfo::VarScopeType::UNKNOWN)
+        stream << "<U>";
+    else if (node->lookup_type == ScopeInfo::VarScopeType::FAST)
+        stream << "<F>";
+    else if (node->lookup_type == ScopeInfo::VarScopeType::DEREF)
+        stream << "<D>";
+    else if (node->lookup_type == ScopeInfo::VarScopeType::CLOSURE)
+        stream << "<C>";
+    else if (node->lookup_type == ScopeInfo::VarScopeType::GLOBAL)
+        stream << "<G>";
+    else
+        stream << "<?>";
+#endif
+
+#if 0
+    if (node->is_kill) stream << "<k>";
+#endif
     return false;
 }
 

src/core/ast.h

@@ -400,6 +400,8 @@ public:
     std::vector<AST_stmt*> body;
     InternedString name;
 
+    FunctionMetadata* md;
+
     AST_ClassDef() : AST_stmt(AST_TYPE::ClassDef) {}
 
     static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::ClassDef;
@@ -509,6 +511,8 @@ public:
     // this should be an expr but we convert it into a AST_Return(AST_expr) to make the code simpler
     AST_stmt* body;
 
+    FunctionMetadata* md;
+
     virtual void accept(ASTVisitor* v);
 
     AST_Expression(std::unique_ptr<InternedStringPool> interned_strings)
@@ -549,6 +553,8 @@ public:
     InternedString name; // if the name is not set this is a lambda
     AST_arguments* args;
 
+    FunctionMetadata* md;
+
     virtual void accept(ASTVisitor* v);
     virtual void accept_stmt(StmtVisitor* v);
 
@@ -704,6 +710,8 @@ public:
     // no lineno, col_offset attributes
     std::vector<AST_stmt*> body;
 
+    FunctionMetadata* md;
+
     virtual void accept(ASTVisitor* v);
 
     AST_Module(std::unique_ptr<InternedStringPool> interned_strings)
@@ -736,12 +744,17 @@ public:
     // different bytecodes.
     ScopeInfo::VarScopeType lookup_type;
 
+    // These are only valid for lookup_type == FAST or CLOSURE
     // The interpreter and baseline JIT store variables with FAST and CLOSURE scopes in an array (vregs) this specifies
     // the zero based index of this variable inside the vregs array. If uninitialized it's value is -1.
     int vreg;
-
     bool is_kill = false;
 
+    // Only valid for lookup_type == DEREF:
+    DerefInfo deref_info = DerefInfo({ INT_MAX, INT_MAX });
+    // Only valid for lookup_type == CLOSURE:
+    int closure_offset = -1;
+
     virtual void accept(ASTVisitor* v);
     virtual void* accept_expr(ExprVisitor* v);
 
@@ -1121,7 +1134,7 @@ public:
 };
 
 template <typename T> T* ast_cast(AST* node) {
-    assert(!node || node->type == T::TYPE);
+    ASSERT(!node || node->type == T::TYPE, "%d", node ? node->type : 0);
     return static_cast<T*>(node);
 }
 

src/core/cfg.h

@@ -169,7 +169,7 @@ public:
     int getNumOfCrossBlockVRegs() const { return num_vregs_cross_block; }
 
     bool hasVRegsAssigned() const { return num_vregs != -1; }
-    void assignVRegs(CFG* cfg, const ParamNames& param_names, ScopeInfo* scope_info);
+    void assignVRegs(CFG* cfg, const ParamNames& param_names);
 };
 
 // Control Flow Graph
@@ -330,8 +330,10 @@ public:
     iterator end() const { return iterator(*this, this->v.size()); }
 };
 
-class SourceInfo;
-CFG* computeCFG(SourceInfo* source, const ParamNames& param_names);
+FunctionMetadata*& metadataForAST(AST* ast);
+InternedStringPool& stringpoolForAST(AST* ast);
+
+void computeAllCFGs(AST* ast, bool globals_from_module, FutureFlags future_flags, BoxedString* fn, BoxedModule* bm);
 void printCFG(CFG* cfg);
 }
 

src/core/options.cpp

@@ -80,7 +80,6 @@ bool ENABLE_PYSTON_PASSES = 0 && _GLOBAL_ENABLE;
 bool ENABLE_TYPE_FEEDBACK = 1 && _GLOBAL_ENABLE;
 bool ENABLE_RUNTIME_ICS = 1 && _GLOBAL_ENABLE;
 bool ENABLE_JIT_OBJECT_CACHE = 1 && _GLOBAL_ENABLE;
-bool LAZY_SCOPING_ANALYSIS = 1;
 
 bool ENABLE_FRAME_INTROSPECTION = 1;
 

src/core/options.h

@@ -44,8 +44,7 @@ extern bool LOG_IC_ASSEMBLY, LOG_BJIT_ASSEMBLY;
 extern bool ENABLE_ICS, ENABLE_ICGENERICS, ENABLE_ICGETITEMS, ENABLE_ICSETITEMS, ENABLE_ICDELITEMS, ENABLE_ICBINEXPS,
     ENABLE_ICNONZEROS, ENABLE_ICCALLSITES, ENABLE_ICSETATTRS, ENABLE_ICGETATTRS, ENALBE_ICDELATTRS, ENABLE_ICGETGLOBALS,
     ENABLE_SPECULATION, ENABLE_OSR, ENABLE_LLVMOPTS, ENABLE_INLINING, ENABLE_REOPT, ENABLE_PYSTON_PASSES,
-    ENABLE_TYPE_FEEDBACK, ENABLE_FRAME_INTROSPECTION, ENABLE_RUNTIME_ICS, ENABLE_JIT_OBJECT_CACHE,
-    LAZY_SCOPING_ANALYSIS;
+    ENABLE_TYPE_FEEDBACK, ENABLE_FRAME_INTROSPECTION, ENABLE_RUNTIME_ICS, ENABLE_JIT_OBJECT_CACHE;
 
 // Due to a temporary LLVM limitation, represent bools as i64's instead of i1's.
 #define BOOLS_AS_I64 1

src/core/types.h

@@ -161,7 +161,6 @@ class AST_stmt;
 
 class PhiAnalysis;
 class LivenessAnalysis;
-class ScopingAnalysis;
 
 class FunctionMetadata;
 class OSREntryDescriptor;
@@ -430,10 +429,60 @@ public:
 typedef int FutureFlags;
 
 class BoxedModule;
-class ScopeInfo;
 class InternedStringPool;
 class LivenessAnalysis;
 
+// Each closure has an array (fixed-size for that particular scope) of variables
+// and a parent pointer to a parent closure. To look up a variable from the passed-in
+// closure (i.e., DEREF), you just need to know (i) how many parents up to go and
+// (ii) what offset into the array to find the variable. This struct stores that
+// information. You can query the ScopeInfo with a name to get this info.
+struct DerefInfo {
+    size_t num_parents_from_passed_closure;
+    size_t offset;
+};
+
+class ScopeInfo;
+// The results of our scoping analysis.
+// A ScopeInfo is a component of the analysis itself and contains a lot of other
+// metadata that is necessary during the analysis, after which we can throw it
+// away and only keep a ScopingResults object.
+struct ScopingResults {
+private:
+    bool are_locals_from_module : 1;
+    bool are_globals_from_module : 1;
+    bool creates_closure : 1;
+    bool takes_closure : 1;
+    bool passes_through_closure : 1;
+    bool uses_name_lookup : 1;
+
+    int closure_size;
+    std::vector<std::pair<InternedString, DerefInfo>> deref_info;
+
+public:
+    ScopingResults(ScopingResults&&) = default;
+    // Delete these just to make sure we avoid extra copies
+    ScopingResults(const ScopingResults&) = delete;
+    void operator=(const ScopingResults&) = delete;
+
+    bool areLocalsFromModule() const { return are_locals_from_module; }
+    bool areGlobalsFromModule() const { return are_globals_from_module; }
+    bool createsClosure() const { return creates_closure; }
+    bool takesClosure() const { return takes_closure; }
+    bool passesThroughClosure() const { return passes_through_closure; }
+    bool usesNameLookup() const { return uses_name_lookup; }
+
+    int getClosureSize() const {
+        assert(createsClosure());
+        return closure_size;
+    }
+    const std::vector<std::pair<InternedString, DerefInfo>>& getAllDerefVarsAndInfo() const { return deref_info; }
+    DerefInfo getDerefInfo(AST_Name*) const;
+    size_t getClosureOffset(AST_Name*) const;
+
+    ScopingResults(ScopeInfo* scope_info, bool globals_from_module);
+};
+
 // Data about a single textual function definition.
 class SourceInfo {
 private:
@@ -442,29 +491,23 @@ private:
 
 public:
     BoxedModule* parent_module;
-    ScopingAnalysis* scoping;
-    ScopeInfo* scope_info;
+    ScopingResults scoping;
     AST* ast;
     CFG* cfg;
     FutureFlags future_flags;
     bool is_generator;
 
-    InternedStringPool& getInternedStrings();
-
-    ScopeInfo* getScopeInfo();
     LivenessAnalysis* getLiveness();
 
     // does not throw CXX or CAPI exceptions:
     BORROWED(BoxedString*) getName() noexcept;
     BORROWED(BoxedString*) getFn();
 
-    InternedString mangleName(InternedString id);
-
     llvm::ArrayRef<AST_stmt*> getBody() const;
 
     Box* getDocString();
 
-    SourceInfo(BoxedModule* m, ScopingAnalysis* scoping, FutureFlags future_flags, AST* ast, BoxedString* fn);
+    SourceInfo(BoxedModule* m, ScopingResults scoping, FutureFlags future_flags, AST* ast, BoxedString* fn);
     ~SourceInfo();
 };
 
@@ -519,7 +562,8 @@ public:
                                         Box**, const std::vector<BoxedString*>*> InternalCallable;
     InternalCallable internal_callable;
 
-    FunctionMetadata(int num_args, bool takes_varargs, bool takes_kwargs, std::unique_ptr<SourceInfo> source);
+    FunctionMetadata(int num_args, bool takes_varargs, bool takes_kwargs, std::unique_ptr<SourceInfo> source,
+                     ParamNames param_names);
     FunctionMetadata(int num_args, bool takes_varargs, bool takes_kwargs,
                      const ParamNames& param_names = ParamNames::empty());
     ~FunctionMetadata();

src/runtime/builtin_modules/pyston.cpp

@@ -45,7 +45,6 @@ static Box* setOption(Box* option, Box* value) {
     else CHECK(SPECULATION_THRESHOLD);
     else CHECK(ENABLE_ICS);
     else CHECK(ENABLE_ICGETATTRS);
-    else CHECK(LAZY_SCOPING_ANALYSIS);
     else raiseExcHelper(ValueError, "unknown option name '%s", option_string->data());
 
     Py_RETURN_NONE;

src/runtime/code.cpp

@@ -197,7 +197,7 @@ extern "C" BORROWED(PyObject*) PyCode_GetName(PyCodeObject* op) noexcept {
 
 extern "C" int PyCode_HasFreeVars(PyCodeObject* _code) noexcept {
     BoxedCode* code = (BoxedCode*)_code;
-    return code->f->source->getScopeInfo()->takesClosure() ? 1 : 0;
+    return code->f->source->scoping.takesClosure() ? 1 : 0;
 }
 
 void setupCode() {

src/runtime/objmodel.cpp

@@ -4771,7 +4771,7 @@ static Box* callChosenCF(CompiledFunction* chosen_cf, BoxedClosure* closure, Box
         }
     }
 
-    assert((globals == NULL) == (!chosen_cf->md->source || chosen_cf->md->source->scoping->areGlobalsFromModule()));
+    assert((globals == NULL) == (!chosen_cf->md->source || chosen_cf->md->source->scoping.areGlobalsFromModule()));
 
     Box* maybe_args[3];
     int nmaybe_args = 0;

src/runtime/types.cpp

@@ -326,7 +326,7 @@ extern "C" BoxedFunctionBase::BoxedFunctionBase(FunctionMetadata* md, llvm::Arra
       modname(NULL),
       name(NULL),
       doc(NULL) {
-    assert((!globals) == (!md->source || md->source->scoping->areGlobalsFromModule()));
+    assert((!globals) == (!md->source || md->source->scoping.areGlobalsFromModule()));
 
     if (globals)
         ASSERT(globals->cls == dict_cls || globals->cls == module_cls, "%s", globals->cls->tp_name);
@@ -353,7 +353,7 @@ extern "C" BoxedFunctionBase::BoxedFunctionBase(FunctionMetadata* md, llvm::Arra
 
         Box* globals_for_name = globals;
         if (!globals_for_name) {
-            assert(md->source->scoping->areGlobalsFromModule());
+            assert(md->source->scoping.areGlobalsFromModule());
             globals_for_name = md->source->parent_module;
         }
 
@@ -1648,7 +1648,7 @@ static Box* function_new(BoxedClass* cls, Box* code, Box* globals, Box** _args)
 
     FunctionMetadata* md = static_cast<BoxedCode*>(code)->f;
     RELEASE_ASSERT(md->source, "");
-    if (md->source->scoping->areGlobalsFromModule()) {
+    if (md->source->scoping.areGlobalsFromModule()) {
         RELEASE_ASSERT(unwrapAttrWrapper(globals) == md->source->parent_module, "");
         globals = NULL;
     } else {
@@ -1730,13 +1730,13 @@ static Box* function_globals(Box* self, void*) noexcept {
     assert(self->cls == function_cls);
     BoxedFunction* func = static_cast<BoxedFunction*>(self);
     if (func->globals) {
-        assert(!func->md->source || !func->md->source->scoping->areGlobalsFromModule());
+        assert(!func->md->source || !func->md->source->scoping.areGlobalsFromModule());
         if (func->globals->cls == module_cls)
             return incref(func->globals->getAttrWrapper());
         return incref(func->globals);
     }
     assert(func->md->source);
-    assert(func->md->source->scoping->areGlobalsFromModule());
+    assert(func->md->source->scoping.areGlobalsFromModule());
 
     static BoxedString* dict_str = getStaticString("__dict__");
     return getattrInternal<CAPI>(func->md->source->parent_module, dict_str);

test/tests/del_closure_var_syntax_error.py

-try:
-    import __pyston__
-    __pyston__.setOption("LAZY_SCOPING_ANALYSIS", 0)
-except ImportError:
-    pass
-
 cases = [
 
 """

test/tests/global_and_local.py

 # I would have expected this to be valid, but cPython and pypy err out saying "name 'x' is local and global"
-
-try:
-    import __pyston__
-    __pyston__.setOption("LAZY_SCOPING_ANALYSIS", 0)
-except ImportError:
-    pass
-
 try:
     exec """
 x = 1

test/tests/name_forcing_syntax_error.py

@@ -4,12 +4,6 @@
 
 # The logic beyond this error message is oddly complicated.
 
-try:
-    import __pyston__
-    __pyston__.setOption("LAZY_SCOPING_ANALYSIS", 0)
-except ImportError:
-    pass
-
 cases = [
 
 # protip: delete this first """ to get your editor to syntax-highlight the code

test/tests/name_scopes.py

@@ -14,3 +14,20 @@ def f2():
 
     f3(*[1, 2], **dict(a=1, b=2))
 f2()
+
+__module__ = 1
+__name__ = 2
+__doc__ = 3
+print __module__ # prints "1"
+print __name__ # prints "2"
+print __doc__ # prints "3"
+class C(object):
+    "hello world"
+
+    # C's implicit setting of "__module__" will end up propagating to the global scope:
+    global __module__
+    global __name__
+    global __doc__
+print __module__ # prints "2"
+print __name__ # prints "2"
+print __doc__ # prints "hello world"

test/unittests/analysis.cpp

@@ -31,21 +31,25 @@ TEST_F(AnalysisTest, augassign) {
     AST_Module* module = caching_parse_file(fn.c_str(), 0);
     assert(module);
 
-    ScopingAnalysis *scoping = new ScopingAnalysis(module, true);
+    FutureFlags future_flags = getFutureFlags(module->body, fn.c_str());
+
+    auto scoping = std::make_shared<ScopingAnalysis>(module, true);
+    computeAllCFGs(module, true, future_flags, boxString(fn), NULL);
 
     assert(module->body[0]->type == AST_TYPE::FunctionDef);
     AST_FunctionDef* func = static_cast<AST_FunctionDef*>(module->body[0]);
 
     ScopeInfo* scope_info = scoping->getScopeInfoForNode(func);
-    ASSERT_FALSE(scope_info->getScopeTypeOfName(module->interned_strings->get("a")) == ScopeInfo::VarScopeType::GLOBAL);
+
+    ASSERT_NE(scope_info->getScopeTypeOfName(module->interned_strings->get("a")), ScopeInfo::VarScopeType::GLOBAL);
     ASSERT_FALSE(scope_info->getScopeTypeOfName(module->interned_strings->get("b")) == ScopeInfo::VarScopeType::GLOBAL);
 
-    FutureFlags future_flags = getFutureFlags(module->body, fn.c_str());
+    ParamNames param_names(func, *module->interned_strings.get());
 
-    SourceInfo* si = new SourceInfo(createModule(boxString("augassign"), fn.c_str()), scoping, future_flags, func, boxString(fn));
+    // Hack to get at the cfg:
+    auto node = module->md->source->cfg->blocks[0]->body[0];
+    CFG* cfg = ast_cast<AST_MakeFunction>(ast_cast<AST_Assign>(node)->value)->function_def->md->source->cfg;
 
-    ParamNames param_names(si->ast, si->getInternedStrings());
-    CFG* cfg = computeCFG(si, param_names);
     std::unique_ptr<LivenessAnalysis> liveness = computeLivenessInfo(cfg);
     auto&& vregs = cfg->getVRegInfo();
 
@@ -57,7 +61,7 @@ TEST_F(AnalysisTest, augassign) {
             ASSERT_TRUE(liveness->isLiveAtEnd(vregs.getVReg(module->interned_strings->get("a")), block));
     }
 
-    std::unique_ptr<PhiAnalysis> phis = computeRequiredPhis(ParamNames(func, si->getInternedStrings()), cfg, liveness.get(), scope_info);
+    std::unique_ptr<PhiAnalysis> phis = computeRequiredPhis(ParamNames(func, *module->interned_strings.get()), cfg, liveness.get());
 }
 
 void doOsrTest(bool is_osr, bool i_maybe_undefined) {
@@ -65,20 +69,22 @@ void doOsrTest(bool is_osr, bool i_maybe_undefined) {
     AST_Module* module = caching_parse_file(fn.c_str(), 0);
     assert(module);
 
-    ScopingAnalysis *scoping = new ScopingAnalysis(module, true);
+    ParamNames param_names(module, *module->interned_strings.get());
 
     assert(module->body[0]->type == AST_TYPE::FunctionDef);
     AST_FunctionDef* func = static_cast<AST_FunctionDef*>(module->body[0]);
 
+    auto scoping = std::make_shared<ScopingAnalysis>(module, true);
+    ScopeInfo* scope_info = scoping->getScopeInfoForNode(func);
+
     FutureFlags future_flags = getFutureFlags(module->body, fn.c_str());
 
-    ScopeInfo* scope_info = scoping->getScopeInfoForNode(func);
-    std::unique_ptr<SourceInfo> si(new SourceInfo(createModule(boxString("osr" + std::to_string((is_osr << 1) + i_maybe_undefined)),
-                    fn.c_str()), scoping, future_flags, func, boxString(fn)));
-    FunctionMetadata* clfunc = new FunctionMetadata(0, false, false, std::move(si));
+    computeAllCFGs(module, true, future_flags, boxString(fn), NULL);
 
-    CFG* cfg = computeCFG(clfunc->source.get(), clfunc->param_names);
-    clfunc->source->cfg = cfg;
+    // Hack to get at the cfg:
+    auto node = module->md->source->cfg->blocks[0]->body[0];
+    auto md = ast_cast<AST_MakeFunction>(ast_cast<AST_Assign>(node)->value)->function_def->md;
+    CFG* cfg = md->source->cfg;
     std::unique_ptr<LivenessAnalysis> liveness = computeLivenessInfo(cfg);
 
     // cfg->print();
@@ -101,16 +107,16 @@ void doOsrTest(bool is_osr, bool i_maybe_undefined) {
 
     if (is_osr) {
         int vreg = vregs.getVReg(i_str);
-        OSREntryDescriptor* entry_descriptor = OSREntryDescriptor::create(clfunc, backedge, CXX);
+        OSREntryDescriptor* entry_descriptor = OSREntryDescriptor::create(md, backedge, CXX);
         // need to set it to non-null
         ConcreteCompilerType* fake_type = (ConcreteCompilerType*)1;
         entry_descriptor->args[vreg] = fake_type;
         if (i_maybe_undefined)
             entry_descriptor->potentially_undefined.set(vreg);
         entry_descriptor->args[vregs.getVReg(iter_str)] = fake_type;
-        phis = computeRequiredPhis(entry_descriptor, liveness.get(), scope_info);
+        phis = computeRequiredPhis(entry_descriptor, liveness.get());
     } else {
-        phis = computeRequiredPhis(ParamNames(func, clfunc->source->getInternedStrings()), cfg, liveness.get(), scope_info);
+        phis = computeRequiredPhis(ParamNames(func, *module->interned_strings), cfg, liveness.get());
     }
 
     // First, verify that we require phi nodes for the block we enter into.