CFG changes

Makefile

@@ -425,7 +425,6 @@ ARGS ?=
 ifneq ($(BR),)
 	override GDB_CMDS := --ex "break $(BR)" $(GDB_CMDS)
 endif
-$(call add_unittest,gc)
 $(call add_unittest,analysis)
 
 

src/analysis/scoping_analysis.cpp

@@ -73,6 +73,13 @@ bool containsYield(AST* ast) {
     return visitor.contains_yield;
 }
 
+bool containsYield(llvm::ArrayRef<AST_stmt*> body) {
+    for (auto e : body)
+        if (containsYield(e))
+            return true;
+    return false;
+}
+
 // TODO
 // Combine this with the below? Basically the same logic with different string types...
 // Also should this go in this file?

src/analysis/scoping_analysis.h

@@ -159,7 +159,9 @@ public:
     bool areGlobalsFromModule() { return globals_from_module; }
 };
 
+class AST_stmt;
 bool containsYield(AST* ast);
+bool containsYield(llvm::ArrayRef<AST_stmt*> ast);
 
 class BoxedString;
 BoxedString* mangleNameBoxedString(BoxedString* id, BoxedString* private_name);

src/codegen/codegen.cpp

@@ -58,24 +58,14 @@ void BoxedCode::addVersion(CompiledFunction* compiled) {
     }
 }
 
-SourceInfo::SourceInfo(BoxedModule* m, ScopingResults scoping, FutureFlags future_flags, AST* ast)
-    : parent_module(m), scoping(std::move(scoping)), cfg(NULL), future_flags(future_flags), ast_type(ast->type) {
-
-    switch (ast_type) {
-        case AST_TYPE::ClassDef:
-        case AST_TYPE::Module:
-        case AST_TYPE::Expression:
-        case AST_TYPE::Suite:
-            is_generator = false;
-            break;
-        case AST_TYPE::FunctionDef:
-        case AST_TYPE::Lambda:
-            is_generator = containsYield(ast);
-            break;
-        default:
-            RELEASE_ASSERT(0, "Unknown type: %d", ast_type);
-            break;
-    }
+SourceInfo::SourceInfo(BoxedModule* m, ScopingResults scoping, FutureFlags future_flags, int ast_type,
+                       bool is_generator)
+    : parent_module(m),
+      scoping(std::move(scoping)),
+      cfg(NULL),
+      future_flags(future_flags),
+      is_generator(is_generator),
+      ast_type(ast_type) {
 }
 
 SourceInfo::~SourceInfo() {

src/codegen/compvars.cpp

@@ -21,12 +21,12 @@
 #include "llvm/IR/Module.h"
 #include "llvm/Support/raw_ostream.h"
 
-#include "core/bst.h"
 #include "codegen/codegen.h"
 #include "codegen/gcbuilder.h"
 #include "codegen/irgen.h"
 #include "codegen/irgen/util.h"
 #include "codegen/patchpoints.h"
+#include "core/bst.h"
 #include "core/options.h"
 #include "core/types.h"
 #include "runtime/float.h"

src/codegen/cpython_ast.cpp

@@ -724,6 +724,7 @@ public:
         switch (mod->kind) {
             case Module_kind: {
                 AST_Module* rtn = new AST_Module(llvm::make_unique<InternedStringPool>());
+                rtn->lineno = 1;
                 assert(!this->pool);
                 this->pool = rtn->interned_strings.get();
                 convertAll<stmt_ty>(mod->v.Module.body, rtn->body);
@@ -732,6 +733,7 @@ public:
             case Interactive_kind: {
                 this->interactive = 1;
                 AST_Module* rtn = new AST_Module(llvm::make_unique<InternedStringPool>());
+                rtn->lineno = 1;
                 assert(!this->pool);
                 this->pool = rtn->interned_strings.get();
                 convertAll<stmt_ty>(mod->v.Interactive.body, rtn->body);
@@ -739,6 +741,7 @@ public:
             }
             case Expression_kind: {
                 AST_Expression* rtn = new AST_Expression(llvm::make_unique<InternedStringPool>());
+                rtn->lineno = 1;
                 this->pool = rtn->interned_strings.get();
 
                 // instead of storing the expression inside the AST node we convert it directly to a return statement

src/codegen/irgen/hooks.cpp

@@ -231,10 +231,8 @@ void compileAndRunModule(AST_Module* m, BoxedModule* bm) {
     RELEASE_ASSERT(fn, "");
 
     FutureFlags future_flags = getFutureFlags(m->body, fn);
-    computeAllCFGs(m, /* globals_from_module */ true, future_flags, autoDecref(boxString(fn)), bm);
-
-    BoxedCode* code = m->getCode();
-    assert(code);
+    BoxedCode* code = computeAllCFGs(m, /* globals_from_module */ true, future_flags, autoDecref(boxString(fn)), bm);
+    AUTO_DECREF(code);
 
     static BoxedString* doc_str = getStaticString("__doc__");
     bm->setattr(doc_str, code->_doc, NULL);
@@ -279,8 +277,7 @@ static BoxedCode* compileForEvalOrExec(AST* source, llvm::ArrayRef<AST_stmt*> bo
         flags->cf_flags = future_flags;
     }
 
-    computeAllCFGs(source, /* globals_from_module */ false, future_flags, fn, getCurrentModule());
-    return source->getCode();
+    return computeAllCFGs(source, /* globals_from_module */ false, future_flags, fn, getCurrentModule());
 }
 
 static BoxedCode* compileExec(AST_Module* parsedModule, BoxedString* fn, PyCompilerFlags* flags) {
@@ -319,7 +316,7 @@ extern "C" PyCodeObject* PyAST_Compile(struct _mod* _mod, const char* filename,
                 return NULL;
         }
 
-        return (PyCodeObject*)incref(code);
+        return (PyCodeObject*)code;
     } catch (ExcInfo e) {
         setCAPIException(e);
         return NULL;

src/core/ast.cpp

@@ -2138,22 +2138,6 @@ void flatten(AST_expr* root, std::vector<AST*>& output, bool expand_scopes) {
     root->accept(&visitor);
 }
 
-BoxedCode*& AST::getCode() {
-    switch (this->type) {
-        case AST_TYPE::Expression:
-            return ast_cast<AST_Expression>(this)->code;
-        case AST_TYPE::FunctionDef:
-            return ast_cast<AST_FunctionDef>(this)->code;
-        case AST_TYPE::ClassDef:
-            return ast_cast<AST_ClassDef>(this)->code;
-        case AST_TYPE::Module:
-            return ast_cast<AST_Module>(this)->code;
-        default:
-            break;
-    }
-    RELEASE_ASSERT(0, "%d", this->type);
-}
-
 InternedStringPool& AST::getStringpool() {
     switch (this->type) {
         case AST_TYPE::Expression:
@@ -2181,11 +2165,12 @@ llvm::ArrayRef<AST_stmt*> AST::getBody() {
     };
 }
 
-Box* AST::getDocString() {
-    auto body = this->getBody();
+Box* getDocString(llvm::ArrayRef<AST_stmt*> body) {
     if (body.size() > 0 && body[0]->type == AST_TYPE::Expr
         && static_cast<AST_Expr*>(body[0])->value->type == AST_TYPE::Str) {
-        return boxString(static_cast<AST_Str*>(static_cast<AST_Expr*>(body[0])->value)->str_data);
+        auto expr = static_cast<AST_Expr*>(body[0]);
+        auto str = static_cast<AST_Str*>(expr->value);
+        return boxString(str->str_data);
     }
 
     return incref(Py_None);

src/core/ast.h

@@ -190,12 +190,11 @@ public:
 
     // These could be virtual methods, but since we already keep track of the type use a switch statement
     // like everywhere else.
-    BoxedCode*& getCode();
     InternedStringPool& getStringpool();
     llvm::ArrayRef<AST_stmt*> getBody();
-    Box* getDocString();
     BORROWED(BoxedString*) getName() noexcept;
 };
+Box* getDocString(llvm::ArrayRef<AST_stmt*> body);
 
 class AST_expr : public AST {
 public:
@@ -398,8 +397,6 @@ public:
     std::vector<AST_stmt*> body;
     InternedString name;
 
-    BoxedCode* code;
-
     AST_ClassDef() : AST_stmt(AST_TYPE::ClassDef) {}
 
     static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::ClassDef;
@@ -506,8 +503,6 @@ public:
     // this should be an expr but we convert it into a AST_Return(AST_expr) to make the code simpler
     AST_stmt* body;
 
-    BoxedCode* code;
-
     virtual void accept(ASTVisitor* v);
 
     AST_Expression(std::unique_ptr<InternedStringPool> interned_strings)
@@ -547,8 +542,6 @@ public:
     InternedString name; // if the name is not set this is a lambda
     AST_arguments* args;
 
-    BoxedCode* code;
-
     virtual void accept(ASTVisitor* v);
     virtual void accept_stmt(ASTStmtVisitor* v);
 
@@ -698,8 +691,6 @@ public:
     // no lineno, col_offset attributes
     std::vector<AST_stmt*> body;
 
-    BoxedCode* code;
-
     virtual void accept(ASTVisitor* v);
 
     AST_Module(std::unique_ptr<InternedStringPool> interned_strings)

src/core/bst.cpp

@@ -43,6 +43,12 @@ template <class T> static void visitVector(const std::vector<T*>& vec, BSTVisito
     }
 }
 
+static void visitCFG(CFG* cfg, BSTVisitor* v) {
+    for (auto bb : cfg->blocks)
+        for (auto e : bb->body)
+            e->accept(v);
+}
+
 void BST_alias::accept(BSTVisitor* v) {
     bool skip = v->visit_alias(this);
     if (skip)
@@ -218,7 +224,7 @@ void BST_ClassDef::accept(BSTVisitor* v) {
 
     visitVector(this->bases, v);
     visitVector(this->decorator_list, v);
-    visitVector(this->body, v);
+    visitCFG(this->code->source->cfg, v);
 }
 
 void BST_ClassDef::accept_stmt(StmtVisitor* v) {
@@ -364,7 +370,7 @@ void BST_FunctionDef::accept(BSTVisitor* v) {
 
     visitVector(decorator_list, v);
     args->accept(v);
-    visitVector(body, v);
+    visitCFG(code->source->cfg, v);
 }
 
 void BST_FunctionDef::accept_stmt(StmtVisitor* v) {
@@ -1084,11 +1090,16 @@ bool PrintVisitor::visit_classdef(BST_ClassDef* node) {
     stream << ")";
 
     indent += 4;
+    stream << '\n';
+    printIndent();
+    stream << "...";
+#if 0
     for (int i = 0, n = node->body.size(); i < n; i++) {
         stream << "\n";
         printIndent();
         node->body[i]->accept(this);
     }
+#endif
     indent -= 4;
 
     return true;
@@ -1215,11 +1226,16 @@ bool PrintVisitor::visit_functiondef(BST_FunctionDef* node) {
     stream << ")";
 
     indent += 4;
+    stream << '\n';
+    printIndent();
+    stream << "...";
+#if 0
     for (int i = 0; i < node->body.size(); i++) {
         stream << "\n";
         printIndent();
         node->body[i]->accept(this);
     }
+#endif
     indent -= 4;
     return true;
 }

src/core/bst.h

@@ -398,7 +398,6 @@ public:
     virtual void accept_stmt(StmtVisitor* v);
 
     std::vector<BST_expr*> bases, decorator_list;
-    std::vector<BST_stmt*> body;
     InternedString name;
 
     BoxedCode* code;
@@ -512,8 +511,6 @@ public:
     // this should be an expr but we convert it into a BST_Return(BST_expr) to make the code simpler
     BST_stmt* body;
 
-    BoxedCode* code;
-
     virtual void accept(BSTVisitor* v);
 
     BST_Expression(std::unique_ptr<InternedStringPool> interned_strings)
@@ -549,7 +546,6 @@ public:
 
 class BST_FunctionDef : public BST_stmt {
 public:
-    std::vector<BST_stmt*> body;
     std::vector<BST_expr*> decorator_list;
     InternedString name; // if the name is not set this is a lambda
     BST_arguments* args;
@@ -711,8 +707,6 @@ public:
     // no lineno, col_offset attributes
     std::vector<BST_stmt*> body;
 
-    BoxedCode* code;
-
     virtual void accept(BSTVisitor* v);
 
     BST_Module(std::unique_ptr<InternedStringPool> interned_strings)

src/core/cfg.h

@@ -330,7 +330,8 @@ public:
     iterator end() const { return iterator(*this, this->v.size()); }
 };
 
-void computeAllCFGs(AST* ast, bool globals_from_module, FutureFlags future_flags, BoxedString* fn, BoxedModule* bm);
+BoxedCode* computeAllCFGs(AST* ast, bool globals_from_module, FutureFlags future_flags, BoxedString* fn,
+                          BoxedModule* bm);
 void printCFG(CFG* cfg);
 }
 

src/core/types.h

@@ -156,7 +156,7 @@ class CFG;
 class AST;
 class BST;
 class BST_FunctionDef;
-class BST_arguments;
+class AST_arguments;
 class BST_expr;
 class BST_Name;
 class BST_stmt;
@@ -234,7 +234,7 @@ struct ParamNames {
     unsigned char has_vararg_name : 1;
     unsigned char has_kwarg_name : 1;
 
-    explicit ParamNames(AST* ast, InternedStringPool& pool);
+    explicit ParamNames(AST_arguments* ast, InternedStringPool& pool);
     ParamNames(const std::vector<const char*>& args, const char* vararg, const char* kwarg);
     static ParamNames empty() { return ParamNames(); }
 
@@ -502,7 +502,7 @@ public:
 
     LivenessAnalysis* getLiveness();
 
-    SourceInfo(BoxedModule* m, ScopingResults scoping, FutureFlags future_flags, AST* ast);
+    SourceInfo(BoxedModule* m, ScopingResults scoping, FutureFlags future_flags, int ast_type, bool is_generator);
     ~SourceInfo();
 };
 

src/runtime/exceptions.cpp

@@ -16,8 +16,8 @@
 #include <cstdarg>
 #include <dlfcn.h>
 
-#include "core/ast.h"
 #include "codegen/unwinding.h"
+#include "core/ast.h"
 #include "core/options.h"
 #include "runtime/objmodel.h"
 #include "runtime/types.h"

test/tests/augassign_analysis.py

@@ -118,3 +118,11 @@ try:
     f10()
 except UnboundLocalError, e:
     print e
+
+def f11():
+    class C(object):
+        def __init__(self):
+            self.__x = 1
+            self.__x += 2
+    print C()._C__x
+f11()

test/unittests/CMakeLists.txt

 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-undef -D_PYSTON_API")
+set(CMAKE_CXX_FLAGS_DEBUG "-g -O0")
 set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
 
 add_definitions(-DGTEST_HAS_RTTI=0 ${LLVM_DEFINITIONS})

test/unittests/analysis.cpp

@@ -11,6 +11,7 @@
 #include "codegen/osrentry.h"
 #include "codegen/parser.h"
 #include "core/ast.h"
+#include "core/bst.h"
 #include "core/cfg.h"
 #include "runtime/types.h"
 #include "unittests.h"
@@ -34,7 +35,7 @@ TEST_F(AnalysisTest, augassign) {
     FutureFlags future_flags = getFutureFlags(module->body, fn.c_str());
 
     auto scoping = std::make_shared<ScopingAnalysis>(module, true);
-    computeAllCFGs(module, true, future_flags, boxString(fn), NULL);
+    auto module_code = computeAllCFGs(module, true, future_flags, boxString(fn), NULL);
 
     assert(module->body[0]->type == AST_TYPE::FunctionDef);
     AST_FunctionDef* func = static_cast<AST_FunctionDef*>(module->body[0]);
@@ -44,11 +45,12 @@ TEST_F(AnalysisTest, augassign) {
     ASSERT_NE(scope_info->getScopeTypeOfName(module->interned_strings->get("a")), ScopeInfo::VarScopeType::GLOBAL);
     ASSERT_FALSE(scope_info->getScopeTypeOfName(module->interned_strings->get("b")) == ScopeInfo::VarScopeType::GLOBAL);
 
-    ParamNames param_names(func, *module->interned_strings.get());
+    AST_arguments* args = new AST_arguments();
+    ParamNames param_names(args, *module->interned_strings.get());
 
     // Hack to get at the cfg:
-    auto node = module->code->source->cfg->blocks[0]->body[0];
-    CFG* cfg = ast_cast<AST_MakeFunction>(ast_cast<AST_Assign>(node)->value)->function_def->code->source->cfg;
+    auto node = module_code->source->cfg->blocks[0]->body[0];
+    CFG* cfg = bst_cast<BST_MakeFunction>(bst_cast<BST_Assign>(node)->value)->function_def->code->source->cfg;
 
     std::unique_ptr<LivenessAnalysis> liveness = computeLivenessInfo(cfg);
     auto&& vregs = cfg->getVRegInfo();
@@ -57,11 +59,11 @@ TEST_F(AnalysisTest, augassign) {
 
     for (CFGBlock* block : cfg->blocks) {
         //printf("%d\n", block->idx);
-        if (block->body.back()->type != AST_TYPE::Return)
+        if (block->body.back()->type != BST_TYPE::Return)
             ASSERT_TRUE(liveness->isLiveAtEnd(vregs.getVReg(module->interned_strings->get("a")), block));
     }
 
-    std::unique_ptr<PhiAnalysis> phis = computeRequiredPhis(ParamNames(func, *module->interned_strings.get()), cfg, liveness.get());
+    std::unique_ptr<PhiAnalysis> phis = computeRequiredPhis(ParamNames(args, *module->interned_strings.get()), cfg, liveness.get());
 }
 
 void doOsrTest(bool is_osr, bool i_maybe_undefined) {
@@ -69,7 +71,7 @@ void doOsrTest(bool is_osr, bool i_maybe_undefined) {
     AST_Module* module = caching_parse_file(fn.c_str(), 0);
     assert(module);
 
-    ParamNames param_names(module, *module->interned_strings.get());
+    ParamNames param_names(NULL, *module->interned_strings.get());
 
     assert(module->body[0]->type == AST_TYPE::FunctionDef);
     AST_FunctionDef* func = static_cast<AST_FunctionDef*>(module->body[0]);
@@ -79,11 +81,11 @@ void doOsrTest(bool is_osr, bool i_maybe_undefined) {
 
     FutureFlags future_flags = getFutureFlags(module->body, fn.c_str());
 
-    computeAllCFGs(module, true, future_flags, boxString(fn), NULL);
+    auto module_code = computeAllCFGs(module, true, future_flags, boxString(fn), NULL);
 
     // Hack to get at the cfg:
-    auto node = module->code->source->cfg->blocks[0]->body[0];
-    auto code = ast_cast<AST_MakeFunction>(ast_cast<AST_Assign>(node)->value)->function_def->code;
+    auto node = module_code->source->cfg->blocks[0]->body[0];
+    auto code = bst_cast<BST_MakeFunction>(bst_cast<BST_Assign>(node)->value)->function_def->code;
     CFG* cfg = code->source->cfg;
     std::unique_ptr<LivenessAnalysis> liveness = computeLivenessInfo(cfg);
 
@@ -99,8 +101,8 @@ void doOsrTest(bool is_osr, bool i_maybe_undefined) {
     ASSERT_EQ(6, loop_backedge->idx);
     ASSERT_EQ(1, loop_backedge->body.size());
 
-    ASSERT_EQ(AST_TYPE::Jump, loop_backedge->body[0]->type);
-    AST_Jump* backedge = ast_cast<AST_Jump>(loop_backedge->body[0]);
+    ASSERT_EQ(BST_TYPE::Jump, loop_backedge->body[0]->type);
+    BST_Jump* backedge = bst_cast<BST_Jump>(loop_backedge->body[0]);
     ASSERT_LE(backedge->target->idx, loop_backedge->idx);
 
     std::unique_ptr<PhiAnalysis> phis;
@@ -116,7 +118,7 @@ void doOsrTest(bool is_osr, bool i_maybe_undefined) {
         entry_descriptor->args[vregs.getVReg(iter_str)] = fake_type;
         phis = computeRequiredPhis(entry_descriptor, liveness.get());
     } else {
-        phis = computeRequiredPhis(ParamNames(func, *module->interned_strings), cfg, liveness.get());
+        phis = computeRequiredPhis(ParamNames(func->args, *module->interned_strings), cfg, liveness.get());
     }
 
     // First, verify that we require phi nodes for the block we enter into.