AST: deallocate nodes

Use a bumpptr allocator to keep track of the AST nodes.
I could not use the LLVM one because it does not call the destructor on allocated elements.

src/codegen/cpython_ast.h

@@ -25,7 +25,7 @@ namespace pyston {
 // Convert a CPython ast object to a Pyston ast object.
 // This will also check for certain kinds of "syntax errors" (ex continue not in loop) and will
 // throw them as C++ exceptions.
-AST* cpythonToPystonAST(mod_ty mod, llvm::StringRef fn);
+std::pair<AST*, std::unique_ptr<ASTAllocator>> cpythonToPystonAST(mod_ty mod, llvm::StringRef fn);
 }
 
 #endif

src/codegen/irgen/hooks.cpp

@@ -292,7 +292,9 @@ extern "C" PyCodeObject* PyAST_Compile(struct _mod* _mod, const char* filename,
                                        PyArena* arena) noexcept {
     try {
         mod_ty mod = _mod;
-        AST* parsed = cpythonToPystonAST(mod, filename);
+        std::unique_ptr<ASTAllocator> ast_allocator;
+        AST* parsed;
+        std::tie(parsed, ast_allocator) = cpythonToPystonAST(mod, filename);
         BoxedCode* code = NULL;
         switch (mod->kind) {
             case Module_kind:

src/codegen/parser.h

@@ -20,10 +20,11 @@
 namespace pyston {
 
 class AST_Module;
+class ASTAllocator;
 
-AST_Module* parse_string(const char* code, FutureFlags inherited_flags);
-AST_Module* parse_file(const char* fn, FutureFlags inherited_flags);
-AST_Module* caching_parse_file(const char* fn, FutureFlags inherited_flags);
+std::pair<AST_Module*, std::unique_ptr<ASTAllocator>> parse_string(const char* code, FutureFlags inherited_flags);
+std::pair<AST_Module*, std::unique_ptr<ASTAllocator>> parse_file(const char* fn, FutureFlags inherited_flags);
+std::pair<AST_Module*, std::unique_ptr<ASTAllocator>> caching_parse_file(const char* fn, FutureFlags inherited_flags);
 }
 
 #endif

src/jit.cpp

@@ -460,7 +460,9 @@ static int main(int argc, char** argv) noexcept {
         if (command != NULL) {
             try {
                 main_module = createModule(autoDecref(boxString("__main__")), "<string>");
-                AST_Module* m = parse_string(command, /* future_flags = */ 0);
+                AST_Module* m;
+                std::unique_ptr<ASTAllocator> ast_allocator;
+                std::tie(m, ast_allocator) = parse_string(command, /* future_flags = */ 0);
                 compileAndRunModule(m, main_module);
                 rtncode = 0;
             } catch (ExcInfo e) {
@@ -504,7 +506,10 @@ static int main(int argc, char** argv) noexcept {
                 free(real_path);
 
                 try {
-                    AST_Module* ast = parse_file(fn, /* future_flags = */ 0);
+                    std::unique_ptr<ASTAllocator> ast_allocator;
+                    AST_Module* ast;
+                    std::tie(ast, ast_allocator) = parse_file(fn, /* future_flags = */ 0);
+
                     compileAndRunModule(ast, main_module);
                     rtncode = 0;
                 } catch (ExcInfo e) {

src/runtime/capi.cpp

@@ -37,6 +37,7 @@
 #include "capi/types.h"
 #include "codegen/irgen/hooks.h"
 #include "codegen/unwinding.h"
+#include "core/ast.h"
 #include "core/threading.h"
 #include "core/types.h"
 #include "runtime/classobj.h"
@@ -847,8 +848,8 @@ extern "C" int PyRun_InteractiveOneFlags(FILE* fp, const char* filename, PyCompi
     bool failed = false;
     try {
         assert(mod->kind == Interactive_kind);
-        AST_Module* pyston_module = static_cast<AST_Module*>(cpythonToPystonAST(mod, filename));
-        compileAndRunModule(pyston_module, static_cast<BoxedModule*>(m));
+        auto res = cpythonToPystonAST(mod, filename);
+        compileAndRunModule((AST_Module*)res.first, static_cast<BoxedModule*>(m));
     } catch (ExcInfo e) {
         setCAPIException(e);
         failed = true;

src/runtime/import.cpp

@@ -25,6 +25,7 @@
 #include "codegen/irgen/hooks.h"
 #include "codegen/parser.h"
 #include "codegen/unwinding.h"
+#include "core/ast.h"
 #include "runtime/inline/list.h"
 #include "runtime/objmodel.h"
 #include "runtime/util.h"
@@ -87,7 +88,9 @@ extern "C" PyObject* load_source_module(char* name, char* pathname, FILE* fp) no
     AUTO_DECREF(name_boxed);
     try {
         BoxedModule* module = createModule(name_boxed, pathname);
-        AST_Module* ast = caching_parse_file(pathname, /* future_flags = */ 0);
+        std::unique_ptr<ASTAllocator> ast_allocator;
+        AST_Module* ast;
+        std::tie(ast, ast_allocator) = caching_parse_file(pathname, /* future_flags = */ 0);
         assert(ast);
         compileAndRunModule(ast, module);
         Box* r = getSysModulesDict()->getOrNull(name_boxed);
@@ -118,7 +121,9 @@ extern "C" PyObject* PyImport_ExecCodeModuleEx(const char* name, PyObject* co, c
 
         static BoxedString* file_str = getStaticString("__file__");
         module->setattr(file_str, autoDecref(boxString(pathname)), NULL);
-        AST_Module* ast = parse_string(code->data(), /* future_flags = */ 0);
+        AST_Module* ast;
+        std::unique_ptr<ASTAllocator> ast_allocator;
+        std::tie(ast, ast_allocator) = parse_string(code->data(), /* future_flags = */ 0);
         compileAndRunModule(ast, module);
         return incref(module);
     } catch (ExcInfo e) {

test/unittests/analysis.cpp

@@ -29,7 +29,9 @@ protected:
 #ifndef NDEBUG
 TEST_F(AnalysisTest, augassign) {
     const std::string fn("test/unittests/analysis_listcomp.py");
-    AST_Module* module = caching_parse_file(fn.c_str(), 0);
+    std::unique_ptr<ASTAllocator> ast_allocator;
+    AST_Module* module;
+    std::tie(module, ast_allocator) = caching_parse_file(fn.c_str(), 0);
     assert(module);
 
     FutureFlags future_flags = getFutureFlags(module->body, fn.c_str());
@@ -45,7 +47,7 @@ TEST_F(AnalysisTest, augassign) {
     ASSERT_NE(scope_info->getScopeTypeOfName(module->interned_strings->get("a")), ScopeInfo::VarScopeType::GLOBAL);
     ASSERT_FALSE(scope_info->getScopeTypeOfName(module->interned_strings->get("b")) == ScopeInfo::VarScopeType::GLOBAL);
 
-    AST_arguments* args = new AST_arguments();
+    AST_arguments* args = new (*ast_allocator) AST_arguments();
     ParamNames param_names(args, *module->interned_strings.get());
 
     // Hack to get at the cfg:
@@ -68,7 +70,9 @@ TEST_F(AnalysisTest, augassign) {
 
 void doOsrTest(bool is_osr, bool i_maybe_undefined) {
     const std::string fn("test/unittests/analysis_osr.py");
-    AST_Module* module = caching_parse_file(fn.c_str(), 0);
+    std::unique_ptr<ASTAllocator> ast_allocator;
+    AST_Module* module;
+    std::tie(module, ast_allocator) = caching_parse_file(fn.c_str(), 0);
     assert(module);
 
     ParamNames param_names(NULL, *module->interned_strings.get());