Skip to content

P
Pyston
Commit 6926ccff authored by Marius Wachtler's avatar Marius Wachtler
Browse files
Options

AST: deallocate nodes 

Use a bumpptr allocator to keep track of the AST nodes.
I could not use the LLVM one because it does not call the destructor on allocated elements.
parent 0dea31b9
Hide whitespace changes
Inline Side-by-side
Showing with 372 additions and 264 deletions
src/codegen/cpython_ast.cpp
View file @ 6926ccff
......@@ -41,7 +41,7 @@ while lines:
elif l.startswith("}"):
name = l[2:-1]
print "case %s_kind: {" % name
print "auto r = new AST_%s();" % name
print "auto r = new (getAlloc()) AST_%s();" % name
print "auto v = %s->v.%s;" % (type, name)
for n in names:
print "r->%s = convert(v.%s);" % (n, n)
......@@ -64,6 +64,9 @@ private:
int interactive = 0;
int nestlevel = 0;
llvm::StringRef fn;
std::unique_ptr<ASTAllocator> ast_allocator;
ASTAllocator& getAlloc() { return *ast_allocator; }
public:
Converter(llvm::StringRef fn) : fn(fn) {}
......@@ -100,11 +103,11 @@ public:
AST_Name* convertToName(identifier ident) {
if (!ident)
return NULL;
return new AST_Name(convert(ident), AST_TYPE::Store, -1, -1);
return new (getAlloc()) AST_Name(convert(ident), AST_TYPE::Store, -1, -1);
}
AST_arguments* convert(arguments_ty ident, AST* parent) {
auto r = new AST_arguments();
auto r = new (getAlloc()) AST_arguments();
convertAll<expr_ty>(ident->args, r->args);
convertAll<expr_ty>(ident->defaults, r->defaults);
......@@ -221,14 +224,14 @@ public:
#undef CASE
AST_keyword* convert(keyword_ty keyword) {
auto r = new AST_keyword();
auto r = new (getAlloc()) AST_keyword();
r->arg = convert(keyword->arg);
r->value = convert(keyword->value);
return r;
}
AST_comprehension* convert(comprehension_ty comprehension) {
auto r = new AST_comprehension();
auto r = new (getAlloc()) AST_comprehension();
r->target = convert(comprehension->target);
r->iter = convert(comprehension->iter);
r->ifs = convert<expr_ty, AST_expr*>(comprehension->ifs);
......@@ -238,7 +241,7 @@ public:
AST_slice* convert(slice_ty slice) {
switch (slice->kind) {
case Slice_kind: {
auto r = new AST_Slice();
auto r = new (getAlloc()) AST_Slice();
auto v = slice->v.Slice;
r->lower = convert(v.lower);
r->upper = convert(v.upper);
......@@ -246,19 +249,19 @@ public:
return r;
}
case ExtSlice_kind: {
auto r = new AST_ExtSlice();
auto r = new (getAlloc()) AST_ExtSlice();
auto v = slice->v.ExtSlice;
r->dims = convert<slice_ty, AST_slice*>(v.dims);
return r;
}
case Index_kind: {
auto r = new AST_Index();
auto r = new (getAlloc()) AST_Index();
auto v = slice->v.Index;
r->value = convert(v.value);
return r;
}
case Ellipsis_kind:
return new AST_Ellipsis();
return new (getAlloc()) AST_Ellipsis();
}
RELEASE_ASSERT(0, "invalid slice type: %d", slice->kind);
}
......@@ -266,14 +269,14 @@ public:
AST_expr* _convert(expr_ty expr) {
switch (expr->kind) {
case BoolOp_kind: {
auto r = new AST_BoolOp();
auto r = new (getAlloc()) AST_BoolOp();
auto v = expr->v.BoolOp;
r->op_type = convert(v.op);
r->values = convert<expr_ty, AST_expr*>(v.values);
return r;
}
case BinOp_kind: {
auto r = new AST_BinOp();
auto r = new (getAlloc()) AST_BinOp();
auto v = expr->v.BinOp;
r->left = convert(v.left);
r->op_type = convert(v.op);
......@@ -281,14 +284,14 @@ public:
return r;
}
case UnaryOp_kind: {
auto r = new AST_UnaryOp();
auto r = new (getAlloc()) AST_UnaryOp();
auto v = expr->v.UnaryOp;
r->op_type = convert(v.op);
r->operand = convert(v.operand);
return r;
}
case Lambda_kind: {
auto r = new AST_Lambda();
auto r = new (getAlloc()) AST_Lambda();
r->lineno = expr->lineno;
r->col_offset = expr->col_offset;
auto v = expr->v.Lambda;
......@@ -297,7 +300,7 @@ public:
return r;
}
case IfExp_kind: {
auto r = new AST_IfExp();
auto r = new (getAlloc()) AST_IfExp();
auto v = expr->v.IfExp;
r->test = convert(v.test);
r->body = convert(v.body);
......@@ -305,34 +308,34 @@ public:
return r;
}
case Dict_kind: {
auto r = new AST_Dict();
auto r = new (getAlloc()) AST_Dict();
auto v = expr->v.Dict;
r->keys = convert<expr_ty, AST_expr*>(v.keys);
r->values = convert<expr_ty, AST_expr*>(v.values);
return r;
}
case Set_kind: {
auto r = new AST_Set();
auto r = new (getAlloc()) AST_Set();
auto v = expr->v.Set;
r->elts = convert<expr_ty, AST_expr*>(v.elts);
return r;
}
case ListComp_kind: {
auto r = new AST_ListComp();
auto r = new (getAlloc()) AST_ListComp();
auto v = expr->v.ListComp;
r->elt = convert(v.elt);
r->generators = convert<comprehension_ty, AST_comprehension*>(v.generators);
return r;
}
case SetComp_kind: {
auto r = new AST_SetComp();
auto r = new (getAlloc()) AST_SetComp();
auto v = expr->v.SetComp;
r->elt = convert(v.elt);
r->generators = convert<comprehension_ty, AST_comprehension*>(v.generators);
return r;
}
case DictComp_kind: {
auto r = new AST_DictComp();
auto r = new (getAlloc()) AST_DictComp();
auto v = expr->v.DictComp;
r->key = convert(v.key);
r->value = convert(v.value);
......@@ -340,20 +343,20 @@ public:
return r;
}
case GeneratorExp_kind: {
auto r = new AST_GeneratorExp();
auto r = new (getAlloc()) AST_GeneratorExp();
auto v = expr->v.GeneratorExp;
r->elt = convert(v.elt);
r->generators = convert<comprehension_ty, AST_comprehension*>(v.generators);
return r;
}
case Yield_kind: {
auto r = new AST_Yield();
auto r = new (getAlloc()) AST_Yield();
auto v = expr->v.Yield;
r->value = convert(v.value);
return r;
}
case Compare_kind: {
auto r = new AST_Compare();
auto r = new (getAlloc()) AST_Compare();
auto v = expr->v.Compare;
r->left = convert(v.left);
r->ops = convert<cmpop_ty, AST_TYPE::AST_TYPE>(v.ops);
......@@ -361,7 +364,7 @@ public:
return r;
}
case Call_kind: {
auto r = new AST_Call();
auto r = new (getAlloc()) AST_Call();
auto v = expr->v.Call;
r->func = convert(v.func);
r->args = convert<expr_ty, AST_expr*>(v.args);
......@@ -371,13 +374,13 @@ public:
return r;
}
case Repr_kind: {
auto r = new AST_Repr();
auto r = new (getAlloc()) AST_Repr();
auto v = expr->v.Repr;
r->value = convert(v.value);
return r;
}
case Attribute_kind: {
auto r = new AST_Attribute();
auto r = new (getAlloc()) AST_Attribute();
auto v = expr->v.Attribute;
r->value = convert(v.value);
r->attr = convert(v.attr);
......@@ -385,7 +388,7 @@ public:
return r;
}
case Subscript_kind: {
auto r = new AST_Subscript();
auto r = new (getAlloc()) AST_Subscript();
auto v = expr->v.Subscript;
r->value = convert(v.value);
r->slice = convert(v.slice);
......@@ -394,18 +397,18 @@ public:
}
case Name_kind: {
auto v = expr->v.Name;
auto r = new AST_Name(convert(v.id), convert(v.ctx), 0);
auto r = new (getAlloc()) AST_Name(convert(v.id), convert(v.ctx), 0);
return r;
}
case List_kind: {
auto r = new AST_List();
auto r = new (getAlloc()) AST_List();
auto v = expr->v.List;
r->elts = convert<expr_ty, AST_expr*>(v.elts);
r->ctx_type = convert(v.ctx);
return r;
}
case Tuple_kind: {
auto r = new AST_Tuple();
auto r = new (getAlloc()) AST_Tuple();
auto v = expr->v.Tuple;
r->elts = convert<expr_ty, AST_expr*>(v.elts);
r->ctx_type = convert(v.ctx);
......@@ -414,19 +417,19 @@ public:
case Num_kind: {
PyObject* o = expr->v.Num.n;
if (o->cls == int_cls) {
auto r = new AST_Num();
auto r = new (getAlloc()) AST_Num();
r->num_type = AST_Num::INT;
r->n_int = unboxInt(o);
return r;
}
if (o->cls == float_cls) {
auto r = new AST_Num();
auto r = new (getAlloc()) AST_Num();
r->num_type = AST_Num::FLOAT;
r->n_float = unboxFloat(o);
return r;
}
if (o->cls == long_cls) {
auto r = new AST_Num();
auto r = new (getAlloc()) AST_Num();
r->num_type = AST_Num::LONG;
// XXX This is pretty silly:
auto s = _PyLong_Format(o, 10, 0, 0);
......@@ -436,7 +439,7 @@ public:
return r;
}
if (o->cls == complex_cls) {
auto r = new AST_Num();
auto r = new (getAlloc()) AST_Num();
r->num_type = AST_Num::COMPLEX;
double real = PyComplex_RealAsDouble(o);
......@@ -450,11 +453,11 @@ public:
return r;
// TODO very silly:
auto freal = new AST_Num();
auto freal = new (getAlloc()) AST_Num();
freal->n_float = real;
freal->num_type = AST_Num::FLOAT;
auto binop = new AST_BinOp();
auto binop = new (getAlloc()) AST_BinOp();
binop->op_type = AST_TYPE::Add;
binop->left = freal;
binop->right = r;
......@@ -470,14 +473,14 @@ public:
RELEASE_ASSERT(o, "");
AUTO_DECREF(o);
auto r = new AST_Str();
auto r = new (getAlloc()) AST_Str();
r->str_data = static_cast<BoxedString*>(o)->s();
r->str_type = AST_Str::UNICODE;
return r;
}
if (o->cls == str_cls) {
return new AST_Str(static_cast<BoxedString*>(o)->s());
return new (getAlloc()) AST_Str(static_cast<BoxedString*>(o)->s());
}
RELEASE_ASSERT(0, "unhandled str type: %s\n", o->cls->tp_name);
}
......@@ -500,7 +503,7 @@ public:
AST_ExceptHandler* convert(excepthandler_ty eh) {
assert(eh->kind == ExceptHandler_kind);
auto r = new AST_ExceptHandler();
auto r = new (getAlloc()) AST_ExceptHandler();
auto v = eh->v.ExceptHandler;
r->type = convert(v.type);
r->name = convert(v.name);
......@@ -510,12 +513,14 @@ public:
return r;
}
AST_alias* convert(alias_ty alias) { return new AST_alias(convert(alias->name), convert(alias->asname)); }
AST_alias* convert(alias_ty alias) {
return new (getAlloc()) AST_alias(convert(alias->name), convert(alias->asname));
}
AST_stmt* _convert(stmt_ty stmt) {
switch (stmt->kind) {
case FunctionDef_kind: {
auto r = new AST_FunctionDef();
auto r = new (getAlloc()) AST_FunctionDef();
r->lineno = stmt->lineno;
r->col_offset = stmt->col_offset;
auto v = stmt->v.FunctionDef;
......@@ -528,7 +533,7 @@ public:
return r;
}
case ClassDef_kind: {
auto r = new AST_ClassDef();
auto r = new (getAlloc()) AST_ClassDef();
auto v = stmt->v.ClassDef;
r->name = convert(v.name);
r->bases = convert<expr_ty, AST_expr*>(v.bases);
......@@ -539,26 +544,26 @@ public:
return r;
}
case Return_kind: {
auto r = new AST_Return();
auto r = new (getAlloc()) AST_Return();
auto v = stmt->v.Return;
r->value = convert(v.value);
return r;
}
case Delete_kind: {
auto r = new AST_Delete();
auto r = new (getAlloc()) AST_Delete();
auto v = stmt->v.Delete;
r->targets = convert<expr_ty, AST_expr*>(v.targets);
return r;
}
case Assign_kind: {
auto r = new AST_Assign();
auto r = new (getAlloc()) AST_Assign();
auto v = stmt->v.Assign;
r->targets = convert<expr_ty, AST_expr*>(v.targets);
r->value = convert(v.value);
return r;
}
case AugAssign_kind: {
auto r = new AST_AugAssign();
auto r = new (getAlloc()) AST_AugAssign();
auto v = stmt->v.AugAssign;
r->target = convert(v.target);
r->op_type = convert(v.op);
......@@ -566,7 +571,7 @@ public:
return r;
}
case Print_kind: {
auto r = new AST_Print();
auto r = new (getAlloc()) AST_Print();
auto v = stmt->v.Print;
r->dest = convert(v.dest);
r->values = convert<expr_ty, AST_expr*>(v.values);
......@@ -574,7 +579,7 @@ public:
return r;
}
case For_kind: {
auto r = new AST_For();
auto r = new (getAlloc()) AST_For();
auto v = stmt->v.For;
r->target = convert(v.target);
r->iter = convert(v.iter);
......@@ -588,7 +593,7 @@ public:
return r;
}
case While_kind: {
auto r = new AST_While();
auto r = new (getAlloc()) AST_While();
auto v = stmt->v.While;
r->test = convert(v.test);
auto fin = in_finally;
......@@ -601,7 +606,7 @@ public:
return r;
}
case If_kind: {
auto r = new AST_If();
auto r = new (getAlloc()) AST_If();
auto v = stmt->v.If;
r->test = convert(v.test);
r->body = convert<stmt_ty, AST_stmt*>(v.body);
......@@ -609,7 +614,7 @@ public:
return r;
}
case With_kind: {
auto r = new AST_With();
auto r = new (getAlloc()) AST_With();
auto v = stmt->v.With;
r->context_expr = convert(v.context_expr);
r->optional_vars = convert(v.optional_vars);
......@@ -617,7 +622,7 @@ public:
return r;
}
case Raise_kind: {
auto r = new AST_Raise();
auto r = new (getAlloc()) AST_Raise();
auto v = stmt->v.Raise;
r->arg0 = convert(v.type);
r->arg1 = convert(v.inst);
......@@ -625,7 +630,7 @@ public:
return r;
}
case TryExcept_kind: {
auto r = new AST_TryExcept();
auto r = new (getAlloc()) AST_TryExcept();
auto v = stmt->v.TryExcept;
r->body = convert<stmt_ty, AST_stmt*>(v.body);
r->handlers = convert<excepthandler_ty, AST_ExceptHandler*>(v.handlers);
......@@ -633,7 +638,7 @@ public:
return r;
}
case TryFinally_kind: {
auto r = new AST_TryFinally();
auto r = new (getAlloc()) AST_TryFinally();
auto v = stmt->v.TryFinally;
r->body = convert<stmt_ty, AST_stmt*>(v.body);
in_finally++;
......@@ -642,20 +647,20 @@ public:
return r;
}
case Assert_kind: {
auto r = new AST_Assert();
auto r = new (getAlloc()) AST_Assert();
auto v = stmt->v.Assert;
r->test = convert(v.test);
r->msg = convert(v.msg);
return r;
}
case Import_kind: {
auto r = new AST_Import();
auto r = new (getAlloc()) AST_Import();
auto v = stmt->v.Import;
r->names = convert<alias_ty, AST_alias*>(v.names);
return r;
}
case ImportFrom_kind: {
auto r = new AST_ImportFrom();
auto r = new (getAlloc()) AST_ImportFrom();
auto v = stmt->v.ImportFrom;
r->module = convert(v.module);
r->names = convert<alias_ty, AST_alias*>(v.names);
......@@ -663,7 +668,7 @@ public:
return r;
}
case Exec_kind: {
auto r = new AST_Exec();
auto r = new (getAlloc()) AST_Exec();
auto v = stmt->v.Exec;
r->body = convert(v.body);
r->globals = convert(v.globals);
......@@ -671,36 +676,36 @@ public:
return r;
}
case Global_kind: {
auto r = new AST_Global();
auto r = new (getAlloc()) AST_Global();
auto v = stmt->v.Global;
r->names = convert<identifier, InternedString>(v.names);
return r;
}
case Expr_kind: {
auto r = new AST_Expr();
auto r = new (getAlloc()) AST_Expr();
auto v = stmt->v.Expr;
r->value = convert(v.value);
if (interactive && nestlevel <= 0) {
auto print_expr = new AST_LangPrimitive(AST_LangPrimitive::PRINT_EXPR);
auto print_expr = new (getAlloc()) AST_LangPrimitive(AST_LangPrimitive::PRINT_EXPR);
print_expr->args.push_back(r->value);
r->value = print_expr;
}
return r;
}
case Pass_kind:
return new AST_Pass();
return new (getAlloc()) AST_Pass();
case Break_kind:
// This is not really the right place to be handling this, but this whole thing is temporary anyway.
if (loop_depth == 0)
raiseSyntaxError("'break' outside loop", stmt->lineno, stmt->col_offset, fn, "", true);
return new AST_Break();
return new (getAlloc()) AST_Break();
case Continue_kind:
if (loop_depth == 0)
raiseSyntaxError("'continue' not properly in loop", stmt->lineno, stmt->col_offset, fn, "", true);
if (in_finally)
raiseSyntaxError("'continue' not supported inside 'finally' clause", stmt->lineno, stmt->col_offset,
fn, "", true);
return new AST_Continue();
return new (getAlloc()) AST_Continue();
};
// GCC wants this:
RELEASE_ASSERT(0, "invalid statement type: %d", stmt->kind);
......@@ -720,39 +725,40 @@ public:
return r;
}
AST* convert(mod_ty mod) {
std::pair<AST*, std::unique_ptr<ASTAllocator>> convert(mod_ty mod) {
ast_allocator.reset(new ASTAllocator);
switch (mod->kind) {
case Module_kind: {
AST_Module* rtn = new AST_Module(llvm::make_unique<InternedStringPool>());
AST_Module* rtn = new (getAlloc()) AST_Module(llvm::make_unique<InternedStringPool>());
rtn->lineno = 1;
assert(!this->pool);
this->pool = rtn->interned_strings.get();
convertAll<stmt_ty>(mod->v.Module.body, rtn->body);
return rtn;
return std::make_pair(rtn, std::move(ast_allocator));
}
case Interactive_kind: {
this->interactive = 1;
AST_Module* rtn = new AST_Module(llvm::make_unique<InternedStringPool>());
AST_Module* rtn = new (getAlloc()) AST_Module(llvm::make_unique<InternedStringPool>());
rtn->lineno = 1;
assert(!this->pool);
this->pool = rtn->interned_strings.get();
convertAll<stmt_ty>(mod->v.Interactive.body, rtn->body);
return rtn;
return std::make_pair(rtn, std::move(ast_allocator));
}
case Expression_kind: {
AST_Expression* rtn = new AST_Expression(llvm::make_unique<InternedStringPool>());
AST_Expression* rtn = new (getAlloc()) AST_Expression(llvm::make_unique<InternedStringPool>());
rtn->lineno = 1;
this->pool = rtn->interned_strings.get();
// instead of storing the expression inside the AST node we convert it directly to a return statement
AST_expr* expr = this->convert(mod->v.Expression.body);
auto rtn_stmt = new AST_Return;
auto rtn_stmt = new (getAlloc()) AST_Return;
rtn_stmt->lineno = expr->lineno;
rtn_stmt->col_offset = expr->col_offset;
rtn_stmt->value = expr;
rtn->body = rtn_stmt;
return rtn;
return std::make_pair(rtn, std::move(ast_allocator));
}
default:
RELEASE_ASSERT(0, "unhandled kind: %d\n", mod->kind);
......@@ -760,7 +766,7 @@ public:
}
};
AST* cpythonToPystonAST(mod_ty mod, llvm::StringRef fn) {
std::pair<AST*, std::unique_ptr<ASTAllocator>> cpythonToPystonAST(mod_ty mod, llvm::StringRef fn) {
Converter c(fn);
return c.convert(mod);
}
......
src/codegen/cpython_ast.h
View file @ 6926ccff
......@@ -25,7 +25,7 @@ namespace pyston {
// Convert a CPython ast object to a Pyston ast object.
// This will also check for certain kinds of "syntax errors" (ex continue not in loop) and will
// throw them as C++ exceptions.
AST* cpythonToPystonAST(mod_ty mod, llvm::StringRef fn);
std::pair<AST*, std::unique_ptr<ASTAllocator>> cpythonToPystonAST(mod_ty mod, llvm::StringRef fn);
}
#endif
src/codegen/irgen/hooks.cpp
View file @ 6926ccff
......@@ -292,7 +292,9 @@ extern "C" PyCodeObject* PyAST_Compile(struct _mod* _mod, const char* filename,
PyArena* arena) noexcept {
try {
mod_ty mod = _mod;
AST* parsed = cpythonToPystonAST(mod, filename);
std::unique_ptr<ASTAllocator> ast_allocator;
AST* parsed;
std::tie(parsed, ast_allocator) = cpythonToPystonAST(mod, filename);
BoxedCode* code = NULL;
switch (mod->kind) {
case Module_kind:
......
src/codegen/parser.cpp
View file @ 6926ccff
......@@ -65,6 +65,8 @@ private:
}
public:
std::unique_ptr<ASTAllocator> ast_allocator;
void fill() {
if (unlikely(fp)) {
memmove(buf, buf + start, end - start);
......@@ -76,9 +78,15 @@ public:
}
}
BufferedReader(FILE* fp) : start(0), end(0), fp(fp), data(), intern_pool(NULL) {}
BufferedReader(FILE* fp)
: start(0), end(0), fp(fp), data(), intern_pool(NULL), ast_allocator(llvm::make_unique<ASTAllocator>()) {}
BufferedReader(std::vector<char> data, int start_offset = 0)
: start(start_offset), end(data.size()), fp(NULL), data(std::move(data)), intern_pool(NULL) {}
: start(start_offset),
end(data.size()),
fp(NULL),
data(std::move(data)),
intern_pool(NULL),
ast_allocator(llvm::make_unique<ASTAllocator>()) {}
int bytesBuffered() { return (end - start); }
......@@ -104,6 +112,8 @@ public:
return d;
}
ASTAllocator& getAlloc() { return *ast_allocator; }
std::unique_ptr<InternedStringPool> createInternedPool();
InternedString readAndInternString();
void readAndInternStringVector(std::vector<InternedString>& v);
......@@ -205,7 +215,7 @@ AST_alias* read_alias(BufferedReader* reader) {
InternedString asname = reader->readAndInternString();
InternedString name = reader->readAndInternString();
AST_alias* rtn = new AST_alias(name, asname);
AST_alias* rtn = new (reader->getAlloc()) AST_alias(name, asname);
rtn->col_offset = -1;
rtn->lineno = -1;
......@@ -216,7 +226,7 @@ AST_arguments* read_arguments(BufferedReader* reader) {
if (VERBOSITY("parsing") >= 3)
printf("reading arguments\n");
AST_arguments* rtn = new AST_arguments();
AST_arguments* rtn = new (reader->getAlloc()) AST_arguments();
readExprVector(rtn->args, reader);
rtn->col_offset = -1;
......@@ -228,7 +238,7 @@ AST_arguments* read_arguments(BufferedReader* reader) {
}
AST_Assert* read_assert(BufferedReader* reader) {
AST_Assert* rtn = new AST_Assert();
AST_Assert* rtn = new (reader->getAlloc()) AST_Assert();
rtn->col_offset = readColOffset(reader);
rtn->lineno = reader->readULL();
......@@ -238,7 +248,7 @@ AST_Assert* read_assert(BufferedReader* reader) {
}
AST_Assign* read_assign(BufferedReader* reader) {
AST_Assign* rtn = new AST_Assign();
AST_Assign* rtn = new (reader->getAlloc()) AST_Assign();
rtn->col_offset = readColOffset(reader);
rtn->lineno = reader->readULL();
......@@ -248,7 +258,7 @@ AST_Assign* read_assign(BufferedReader* reader) {
}
AST_AugAssign* read_augassign(BufferedReader* reader) {
AST_AugAssign* rtn = new AST_AugAssign();
AST_AugAssign* rtn = new (reader->getAlloc()) AST_AugAssign();
rtn->col_offset = readColOffset(reader);
rtn->lineno = reader->readULL();
......@@ -259,7 +269,7 @@ AST_AugAssign* read_augassign(BufferedReader* reader) {
}
AST_Attribute* read_attribute(BufferedReader* reader) {
AST_Attribute* rtn = new AST_Attribute();
AST_Attribute* rtn = new (reader->getAlloc()) AST_Attribute();
rtn->attr = reader->readAndInternString();
rtn->col_offset = readColOffset(reader);
......@@ -270,7 +280,7 @@ AST_Attribute* read_attribute(BufferedReader* reader) {
}
AST_expr* read_binop(BufferedReader* reader) {
AST_BinOp* rtn = new AST_BinOp();
AST_BinOp* rtn = new (reader->getAlloc()) AST_BinOp();
rtn->col_offset = readColOffset(reader);
rtn->left = readASTExpr(reader);
......@@ -282,7 +292,7 @@ AST_expr* read_binop(BufferedReader* reader) {
}
AST_expr* read_boolop(BufferedReader* reader) {
AST_BoolOp* rtn = new AST_BoolOp();
AST_BoolOp* rtn = new (reader->getAlloc()) AST_BoolOp();
rtn->col_offset = readColOffset(reader);
rtn->lineno = reader->readULL();
......@@ -293,7 +303,7 @@ AST_expr* read_boolop(BufferedReader* reader) {
}
AST_Break* read_break(BufferedReader* reader) {
AST_Break* rtn = new AST_Break();
AST_Break* rtn = new (reader->getAlloc()) AST_Break();
rtn->col_offset = readColOffset(reader);
rtn->lineno = reader->readULL();
......@@ -302,7 +312,7 @@ AST_Break* read_break(BufferedReader* reader) {
}
AST_Call* read_call(BufferedReader* reader) {
AST_Call* rtn = new AST_Call();
AST_Call* rtn = new (reader->getAlloc()) AST_Call();
readExprVector(rtn->args, reader);
rtn->col_offset = readColOffset(reader);
......@@ -317,7 +327,7 @@ AST_Call* read_call(BufferedReader* reader) {
}
AST_expr* read_compare(BufferedReader* reader) {
AST_Compare* rtn = new AST_Compare();
AST_Compare* rtn = new (reader->getAlloc()) AST_Compare();
rtn->col_offset = readColOffset(reader);
readExprVector(rtn->comparators, reader);
......@@ -334,7 +344,7 @@ AST_expr* read_compare(BufferedReader* reader) {
}
AST_comprehension* read_comprehension(BufferedReader* reader) {
AST_comprehension* rtn = new AST_comprehension();
AST_comprehension* rtn = new (reader->getAlloc()) AST_comprehension();
readExprVector(rtn->ifs, reader);
rtn->iter = readASTExpr(reader);
......@@ -347,7 +357,7 @@ AST_comprehension* read_comprehension(BufferedReader* reader) {
}
AST_ClassDef* read_classdef(BufferedReader* reader) {
AST_ClassDef* rtn = new AST_ClassDef();
AST_ClassDef* rtn = new (reader->getAlloc()) AST_ClassDef();
readExprVector(rtn->bases, reader);
readStmtVector(rtn->body, reader);
......@@ -360,7 +370,7 @@ AST_ClassDef* read_classdef(BufferedReader* reader) {
}
AST_Continue* read_continue(BufferedReader* reader) {
AST_Continue* rtn = new AST_Continue();
AST_Continue* rtn = new (reader->getAlloc()) AST_Continue();
rtn->col_offset = readColOffset(reader);
rtn->lineno = reader->readULL();
......@@ -369,7 +379,7 @@ AST_Continue* read_continue(BufferedReader* reader) {
}
AST_Delete* read_delete(BufferedReader* reader) {
AST_Delete* rtn = new AST_Delete();
AST_Delete* rtn = new (reader->getAlloc()) AST_Delete();
rtn->col_offset = readColOffset(reader);
rtn->lineno = reader->readULL();
......@@ -379,7 +389,7 @@ AST_Delete* read_delete(BufferedReader* reader) {
}
AST_Dict* read_dict(BufferedReader* reader) {
AST_Dict* rtn = new AST_Dict();
AST_Dict* rtn = new (reader->getAlloc()) AST_Dict();
rtn->col_offset = readColOffset(reader);
readExprVector(rtn->keys, reader);
......@@ -391,7 +401,7 @@ AST_Dict* read_dict(BufferedReader* reader) {
}
AST_DictComp* read_dictcomp(BufferedReader* reader) {
AST_DictComp* rtn = new AST_DictComp();
AST_DictComp* rtn = new (reader->getAlloc()) AST_DictComp();
rtn->col_offset = readColOffset(reader);
readMiscVector(rtn->generators, reader);
rtn->key = readASTExpr(reader);
......@@ -401,14 +411,14 @@ AST_DictComp* read_dictcomp(BufferedReader* reader) {
}
AST_Ellipsis* read_ellipsis(BufferedReader* reader) {
AST_Ellipsis* rtn = new AST_Ellipsis();
AST_Ellipsis* rtn = new (reader->getAlloc()) AST_Ellipsis();
rtn->col_offset = -1;
rtn->lineno = -1;
return rtn;
}
AST_ExceptHandler* read_excepthandler(BufferedReader* reader) {
AST_ExceptHandler* rtn = new AST_ExceptHandler();
AST_ExceptHandler* rtn = new (reader->getAlloc()) AST_ExceptHandler();
readStmtVector(rtn->body, reader);
rtn->col_offset = readColOffset(reader);
......@@ -420,7 +430,7 @@ AST_ExceptHandler* read_excepthandler(BufferedReader* reader) {
}
AST_Exec* read_exec(BufferedReader* reader) {
AST_Exec* rtn = new AST_Exec();
AST_Exec* rtn = new (reader->getAlloc()) AST_Exec();
rtn->body = readASTExpr(reader);
rtn->col_offset = readColOffset(reader);
......@@ -432,7 +442,7 @@ AST_Exec* read_exec(BufferedReader* reader) {
}
AST_Expr* read_expr(BufferedReader* reader) {
AST_Expr* rtn = new AST_Expr();
AST_Expr* rtn = new (reader->getAlloc()) AST_Expr();
rtn->col_offset = readColOffset(reader);
rtn->lineno = reader->readULL();
......@@ -441,7 +451,7 @@ AST_Expr* read_expr(BufferedReader* reader) {
}
AST_ExtSlice* read_extslice(BufferedReader* reader) {
AST_ExtSlice* rtn = new AST_ExtSlice();
AST_ExtSlice* rtn = new (reader->getAlloc()) AST_ExtSlice();
rtn->col_offset = -1;
rtn->lineno = -1;
......@@ -450,7 +460,7 @@ AST_ExtSlice* read_extslice(BufferedReader* reader) {
}
AST_For* read_for(BufferedReader* reader) {
AST_For* rtn = new AST_For();
AST_For* rtn = new (reader->getAlloc()) AST_For();
readStmtVector(rtn->body, reader);
rtn->col_offset = readColOffset(reader);
......@@ -464,7 +474,7 @@ AST_For* read_for(BufferedReader* reader) {
AST_FunctionDef* read_functiondef(BufferedReader* reader) {
if (VERBOSITY("parsing") >= 3)
printf("reading functiondef\n");
AST_FunctionDef* rtn = new AST_FunctionDef();
AST_FunctionDef* rtn = new (reader->getAlloc()) AST_FunctionDef();
rtn->args = ast_cast<AST_arguments>(readASTMisc(reader));
readStmtVector(rtn->body, reader);
......@@ -476,7 +486,7 @@ AST_FunctionDef* read_functiondef(BufferedReader* reader) {
}
AST_GeneratorExp* read_generatorexp(BufferedReader* reader) {
AST_GeneratorExp* rtn = new AST_GeneratorExp();
AST_GeneratorExp* rtn = new (reader->getAlloc()) AST_GeneratorExp();
rtn->col_offset = readColOffset(reader);
rtn->elt = readASTExpr(reader);
......@@ -486,7 +496,7 @@ AST_GeneratorExp* read_generatorexp(BufferedReader* reader) {
}
AST_Global* read_global(BufferedReader* reader) {
AST_Global* rtn = new AST_Global();
AST_Global* rtn = new (reader->getAlloc()) AST_Global();
rtn->col_offset = readColOffset(reader);
rtn->lineno = reader->readULL();
......@@ -495,7 +505,7 @@ AST_Global* read_global(BufferedReader* reader) {
}
AST_If* read_if(BufferedReader* reader) {
AST_If* rtn = new AST_If();
AST_If* rtn = new (reader->getAlloc()) AST_If();
readStmtVector(rtn->body, reader);
rtn->col_offset = readColOffset(reader);
......@@ -506,7 +516,7 @@ AST_If* read_if(BufferedReader* reader) {
}
AST_IfExp* read_ifexp(BufferedReader* reader) {
AST_IfExp* rtn = new AST_IfExp();
AST_IfExp* rtn = new (reader->getAlloc()) AST_IfExp();
rtn->body = readASTExpr(reader);
rtn->col_offset = readColOffset(reader);
......@@ -517,7 +527,7 @@ AST_IfExp* read_ifexp(BufferedReader* reader) {
}
AST_Import* read_import(BufferedReader* reader) {
AST_Import* rtn = new AST_Import();
AST_Import* rtn = new (reader->getAlloc()) AST_Import();
rtn->col_offset = readColOffset(reader);
rtn->lineno = reader->readULL();
......@@ -526,7 +536,7 @@ AST_Import* read_import(BufferedReader* reader) {
}
AST_ImportFrom* read_importfrom(BufferedReader* reader) {
AST_ImportFrom* rtn = new AST_ImportFrom();
AST_ImportFrom* rtn = new (reader->getAlloc()) AST_ImportFrom();
rtn->col_offset = readColOffset(reader);
rtn->level = reader->readULL();
......@@ -537,7 +547,7 @@ AST_ImportFrom* read_importfrom(BufferedReader* reader) {
}
AST_Index* read_index(BufferedReader* reader) {
AST_Index* rtn = new AST_Index();
AST_Index* rtn = new (reader->getAlloc()) AST_Index();
rtn->col_offset = -1;
rtn->lineno = -1;
......@@ -547,7 +557,7 @@ AST_Index* read_index(BufferedReader* reader) {
}
AST_keyword* read_keyword(BufferedReader* reader) {
AST_keyword* rtn = new AST_keyword();
AST_keyword* rtn = new (reader->getAlloc()) AST_keyword();
rtn->arg = reader->readAndInternString();
rtn->col_offset = -1;
......@@ -557,7 +567,7 @@ AST_keyword* read_keyword(BufferedReader* reader) {
}
AST_Lambda* read_lambda(BufferedReader* reader) {
AST_Lambda* rtn = new AST_Lambda();
AST_Lambda* rtn = new (reader->getAlloc()) AST_Lambda();
rtn->args = ast_cast<AST_arguments>(readASTMisc(reader));
rtn->body = readASTExpr(reader);
......@@ -567,7 +577,7 @@ AST_Lambda* read_lambda(BufferedReader* reader) {
}
AST_List* read_list(BufferedReader* reader) {
AST_List* rtn = new AST_List();
AST_List* rtn = new (reader->getAlloc()) AST_List();
rtn->col_offset = readColOffset(reader);
rtn->ctx_type = (AST_TYPE::AST_TYPE)reader->readByte();
......@@ -577,7 +587,7 @@ AST_List* read_list(BufferedReader* reader) {
}
AST_ListComp* read_listcomp(BufferedReader* reader) {
AST_ListComp* rtn = new AST_ListComp();
AST_ListComp* rtn = new (reader->getAlloc()) AST_ListComp();
rtn->col_offset = readColOffset(reader);
rtn->elt = readASTExpr(reader);
......@@ -590,7 +600,7 @@ AST_Module* read_module(BufferedReader* reader) {
if (VERBOSITY("parsing") >= 3)
printf("reading module\n");
AST_Module* rtn = new AST_Module(reader->createInternedPool());
AST_Module* rtn = new (reader->getAlloc()) AST_Module(reader->createInternedPool());
readStmtVector(rtn->body, reader);
rtn->col_offset = 0;
......@@ -604,11 +614,11 @@ AST_Name* read_name(BufferedReader* reader) {
auto id = reader->readAndInternString();
auto lineno = reader->readULL();
return new AST_Name(std::move(id), ctx_type, lineno, col_offset);
return new (reader->getAlloc()) AST_Name(std::move(id), ctx_type, lineno, col_offset);
}
AST_Num* read_num(BufferedReader* reader) {
AST_Num* rtn = new AST_Num();
AST_Num* rtn = new (reader->getAlloc()) AST_Num();
rtn->num_type = (AST_Num::NumType)reader->readByte();
......@@ -630,7 +640,7 @@ AST_Num* read_num(BufferedReader* reader) {
}
AST_Repr* read_repr(BufferedReader* reader) {
AST_Repr* rtn = new AST_Repr();
AST_Repr* rtn = new (reader->getAlloc()) AST_Repr();
rtn->col_offset = readColOffset(reader);
rtn->lineno = reader->readULL();
rtn->value = readASTExpr(reader);
......@@ -639,7 +649,7 @@ AST_Repr* read_repr(BufferedReader* reader) {
}
AST_Pass* read_pass(BufferedReader* reader) {
AST_Pass* rtn = new AST_Pass();
AST_Pass* rtn = new (reader->getAlloc()) AST_Pass();
rtn->col_offset = readColOffset(reader);
rtn->lineno = reader->readULL();
......@@ -647,7 +657,7 @@ AST_Pass* read_pass(BufferedReader* reader) {
}
AST_Print* read_print(BufferedReader* reader) {
AST_Print* rtn = new AST_Print();
AST_Print* rtn = new (reader->getAlloc()) AST_Print();
rtn->col_offset = readColOffset(reader);
rtn->dest = readASTExpr(reader);
......@@ -658,7 +668,7 @@ AST_Print* read_print(BufferedReader* reader) {
}
AST_Raise* read_raise(BufferedReader* reader) {
AST_Raise* rtn = new AST_Raise();
AST_Raise* rtn = new (reader->getAlloc()) AST_Raise();
// "arg0" "arg1" "arg2" are called "type", "inst", and "tback" in the python ast,
// so that's the order we have to read them:
......@@ -671,7 +681,7 @@ AST_Raise* read_raise(BufferedReader* reader) {
}
AST_Return* read_return(BufferedReader* reader) {
AST_Return* rtn = new AST_Return();
AST_Return* rtn = new (reader->getAlloc()) AST_Return();
rtn->col_offset = readColOffset(reader);
rtn->lineno = reader->readULL();
......@@ -680,7 +690,7 @@ AST_Return* read_return(BufferedReader* reader) {
}
AST_Set* read_set(BufferedReader* reader) {
AST_Set* rtn = new AST_Set();
AST_Set* rtn = new (reader->getAlloc()) AST_Set();
rtn->col_offset = readColOffset(reader);
readExprVector(rtn->elts, reader);
......@@ -690,7 +700,7 @@ AST_Set* read_set(BufferedReader* reader) {
}
AST_SetComp* read_setcomp(BufferedReader* reader) {
AST_SetComp* rtn = new AST_SetComp();
AST_SetComp* rtn = new (reader->getAlloc()) AST_SetComp();
rtn->col_offset = readColOffset(reader);
rtn->elt = readASTExpr(reader);
......@@ -700,7 +710,7 @@ AST_SetComp* read_setcomp(BufferedReader* reader) {
}
AST_Slice* read_slice(BufferedReader* reader) {
AST_Slice* rtn = new AST_Slice();
AST_Slice* rtn = new (reader->getAlloc()) AST_Slice();
rtn->col_offset = -1;
rtn->lineno = -1;
......@@ -712,7 +722,7 @@ AST_Slice* read_slice(BufferedReader* reader) {
}
AST_Str* read_str(BufferedReader* reader) {
AST_Str* rtn = new AST_Str();
AST_Str* rtn = new (reader->getAlloc()) AST_Str();
rtn->str_type = (AST_Str::StrType)reader->readByte();
......@@ -731,7 +741,7 @@ AST_Str* read_str(BufferedReader* reader) {
}
AST_Subscript* read_subscript(BufferedReader* reader) {
AST_Subscript* rtn = new AST_Subscript();
AST_Subscript* rtn = new (reader->getAlloc()) AST_Subscript();
rtn->col_offset = readColOffset(reader);
rtn->ctx_type = (AST_TYPE::AST_TYPE)reader->readByte();
......@@ -743,7 +753,7 @@ AST_Subscript* read_subscript(BufferedReader* reader) {
}
AST_TryExcept* read_tryexcept(BufferedReader* reader) {
AST_TryExcept* rtn = new AST_TryExcept();
AST_TryExcept* rtn = new (reader->getAlloc()) AST_TryExcept();
readStmtVector(rtn->body, reader);
rtn->col_offset = readColOffset(reader);
......@@ -754,7 +764,7 @@ AST_TryExcept* read_tryexcept(BufferedReader* reader) {
}
AST_TryFinally* read_tryfinally(BufferedReader* reader) {
AST_TryFinally* rtn = new AST_TryFinally();
AST_TryFinally* rtn = new (reader->getAlloc()) AST_TryFinally();
readStmtVector(rtn->body, reader);
rtn->col_offset = readColOffset(reader);
......@@ -764,7 +774,7 @@ AST_TryFinally* read_tryfinally(BufferedReader* reader) {
}
AST_Tuple* read_tuple(BufferedReader* reader) {
AST_Tuple* rtn = new AST_Tuple();
AST_Tuple* rtn = new (reader->getAlloc()) AST_Tuple();
rtn->col_offset = readColOffset(reader);
rtn->ctx_type = (AST_TYPE::AST_TYPE)reader->readByte();
......@@ -775,7 +785,7 @@ AST_Tuple* read_tuple(BufferedReader* reader) {
}
AST_UnaryOp* read_unaryop(BufferedReader* reader) {
AST_UnaryOp* rtn = new AST_UnaryOp();
AST_UnaryOp* rtn = new (reader->getAlloc()) AST_UnaryOp();
rtn->col_offset = readColOffset(reader);
rtn->lineno = reader->readULL();
......@@ -786,7 +796,7 @@ AST_UnaryOp* read_unaryop(BufferedReader* reader) {
}
AST_While* read_while(BufferedReader* reader) {
AST_While* rtn = new AST_While();
AST_While* rtn = new (reader->getAlloc()) AST_While();
readStmtVector(rtn->body, reader);
rtn->col_offset = readColOffset(reader);
......@@ -798,7 +808,7 @@ AST_While* read_while(BufferedReader* reader) {
}
AST_With* read_with(BufferedReader* reader) {
AST_With* rtn = new AST_With();
AST_With* rtn = new (reader->getAlloc()) AST_With();
readStmtVector(rtn->body, reader);
rtn->col_offset = readColOffset(reader);
......@@ -810,7 +820,7 @@ AST_With* read_with(BufferedReader* reader) {
}
AST_Yield* read_yield(BufferedReader* reader) {
AST_Yield* rtn = new AST_Yield();
AST_Yield* rtn = new (reader->getAlloc()) AST_Yield();
rtn->col_offset = readColOffset(reader);
rtn->lineno = reader->readULL();
......@@ -1001,7 +1011,7 @@ AST* readASTMisc(BufferedReader* reader) {
}
}
AST_Module* parse_string(const char* code, FutureFlags inherited_flags) {
std::pair<AST_Module*, std::unique_ptr<ASTAllocator>> parse_string(const char* code, FutureFlags inherited_flags) {
inherited_flags &= ~(CO_NESTED | CO_FUTURE_DIVISION);
PyCompilerFlags cf;
......@@ -1013,11 +1023,11 @@ AST_Module* parse_string(const char* code, FutureFlags inherited_flags) {
if (!mod)
throwCAPIException();
assert(mod->kind != Interactive_kind);
auto rtn = static_cast<AST_Module*>(cpythonToPystonAST(mod, fn));
return rtn;
auto t = cpythonToPystonAST(mod, fn);
return std::make_pair((AST_Module*)t.first, std::move(t.second));
}
AST_Module* parse_file(const char* fn, FutureFlags inherited_flags) {
std::pair<AST_Module*, std::unique_ptr<ASTAllocator>> parse_file(const char* fn, FutureFlags inherited_flags) {
Timer _t("parsing");
FileHandle fp(fn, "r");
......@@ -1029,13 +1039,13 @@ AST_Module* parse_file(const char* fn, FutureFlags inherited_flags) {
if (!mod)
throwCAPIException();
assert(mod->kind != Interactive_kind);
auto rtn = static_cast<AST_Module*>(cpythonToPystonAST(mod, fn));
return rtn;
auto t = cpythonToPystonAST(mod, fn);
return std::make_pair((AST_Module*)t.first, std::move(t.second));
}
// Does at least one of: returns a valid file_data vector, or fills in 'module'
static std::vector<char> _reparse(const char* fn, const std::string& cache_fn, AST_Module*& module,
FutureFlags inherited_flags) {
std::unique_ptr<ASTAllocator>& ast_allocator, FutureFlags inherited_flags) {
inherited_flags &= ~(CO_NESTED | CO_FUTURE_DIVISION);
FileHandle cache_fp(cache_fn.c_str(), "w");
......@@ -1069,7 +1079,9 @@ static std::vector<char> _reparse(const char* fn, const std::string& cache_fn, A
if (!mod)
throwCAPIException();
assert(mod->kind != Interactive_kind);
module = static_cast<AST_Module*>(cpythonToPystonAST(mod, fn));
auto t = cpythonToPystonAST(mod, fn);
module = static_cast<AST_Module*>(t.first);
ast_allocator = std::move(t.second);
if (!cache_fp)
return std::vector<char>();
......@@ -1092,7 +1104,7 @@ static std::vector<char> _reparse(const char* fn, const std::string& cache_fn, A
// Parsing the file is somewhat expensive since we have to shell out to cpython;
// it's not a huge deal right now, but this caching version can significantly cut down
// on the startup time (40ms -> 10ms).
AST_Module* caching_parse_file(const char* fn, FutureFlags inherited_flags) {
std::pair<AST_Module*, std::unique_ptr<ASTAllocator>> caching_parse_file(const char* fn, FutureFlags inherited_flags) {
std::ostringstream oss;
UNAVOIDABLE_STAT_TIMER(t0, "us_timer_caching_parse_file");
......@@ -1124,9 +1136,10 @@ AST_Module* caching_parse_file(const char* fn, FutureFlags inherited_flags) {
if (ferror(cache_fp)) {
oss << "encountered io error reading from the file\n";
AST_Module* mod = 0;
file_data = _reparse(fn, cache_fn, mod, inherited_flags);
std::unique_ptr<ASTAllocator> ast_allocator;
file_data = _reparse(fn, cache_fn, mod, ast_allocator, inherited_flags);
if (mod)
return mod;
return std::make_pair(mod, std::move(ast_allocator));
assert(file_data.size());
}
break;
......@@ -1204,7 +1217,7 @@ AST_Module* caching_parse_file(const char* fn, FutureFlags inherited_flags) {
if (rtn && reader->bytesBuffered() == 0) {
assert(rtn->type == AST_TYPE::Module);
return ast_cast<AST_Module>(rtn);
return std::make_pair(ast_cast<AST_Module>(rtn), std::move(reader->ast_allocator));
}
oss << "returned NULL module\n";
......@@ -1226,9 +1239,10 @@ AST_Module* caching_parse_file(const char* fn, FutureFlags inherited_flags) {
file_data.clear();
AST_Module* mod = 0;
file_data = _reparse(fn, cache_fn, mod, inherited_flags);
std::unique_ptr<ASTAllocator> ast_allocator;
file_data = _reparse(fn, cache_fn, mod, ast_allocator, inherited_flags);
if (mod)
return mod;
return std::make_pair(mod, std::move(ast_allocator));
assert(file_data.size());
}
}
......
src/codegen/parser.h
View file @ 6926ccff
......@@ -20,10 +20,11 @@
namespace pyston {
class AST_Module;
class ASTAllocator;
AST_Module* parse_string(const char* code, FutureFlags inherited_flags);
AST_Module* parse_file(const char* fn, FutureFlags inherited_flags);
AST_Module* caching_parse_file(const char* fn, FutureFlags inherited_flags);
std::pair<AST_Module*, std::unique_ptr<ASTAllocator>> parse_string(const char* code, FutureFlags inherited_flags);
std::pair<AST_Module*, std::unique_ptr<ASTAllocator>> parse_file(const char* fn, FutureFlags inherited_flags);
std::pair<AST_Module*, std::unique_ptr<ASTAllocator>> caching_parse_file(const char* fn, FutureFlags inherited_flags);
}
#endif
src/core/ast.h
View file @ 6926ccff
......@@ -23,6 +23,7 @@
#include <vector>
#include "llvm/ADT/StringRef.h"
#include "llvm/Support/Allocator.h"
#include "llvm/Support/raw_ostream.h"
#include "analysis/scoping_analysis.h"
......@@ -164,6 +165,44 @@ class ASTStmtVisitor;
class AST_keyword;
class AST_stmt;
class ASTAllocator {
private:
template <int slab_size, int alignment = 8> class ASTAllocatorSlab {
unsigned char data[slab_size];
int num_bytes_used;
public:
ASTAllocatorSlab() : num_bytes_used(0) {}
~ASTAllocatorSlab();
int numBytesFree() const { return slab_size - num_bytes_used; }
void* alloc(int num_bytes) {
assert(num_bytes_used % alignment == 0);
assert(numBytesFree() >= num_bytes);
void* ptr = &data[num_bytes_used];
num_bytes_used += llvm::RoundUpToAlignment(num_bytes, alignment);
return ptr;
}
};
// we subtract the size of the "num_bytes_used" field to generate a power of two allocation which I guess is more
// what the allocator is optimized for.
static constexpr int slab_size = 4096 - sizeof(int);
static_assert(sizeof(ASTAllocatorSlab<slab_size>) == 4096, "");
llvm::SmallVector<std::unique_ptr<ASTAllocatorSlab<slab_size>>, 4> slabs;
public:
ASTAllocator() = default;
ASTAllocator(ASTAllocator&&) = delete;
void* allocate(int num_bytes) {
if (slabs.empty() || slabs.back()->numBytesFree() < num_bytes)
slabs.emplace_back(llvm::make_unique<ASTAllocatorSlab<slab_size>>());
return slabs.back()->alloc(num_bytes);
}
};
class AST {
public:
virtual ~AST() {}
......@@ -172,6 +211,7 @@ public:
uint32_t lineno, col_offset;
virtual void accept(ASTVisitor* v) = 0;
virtual int getSize() const = 0; // returns size of AST node
// #define DEBUG_LINE_NUMBERS 1
#ifdef DEBUG_LINE_NUMBERS
......@@ -188,6 +228,10 @@ public:
AST(AST_TYPE::AST_TYPE type, uint32_t lineno, uint32_t col_offset = 0)
: type(type), lineno(lineno), col_offset(col_offset) {}
static void* operator new(size_t count, ASTAllocator& allocator) { return allocator.allocate(count); }
static void operator delete(void*) { RELEASE_ASSERT(0, "use the ASTAllocator instead"); }
// These could be virtual methods, but since we already keep track of the type use a switch statement
// like everywhere else.
InternedStringPool& getStringpool();
......@@ -196,6 +240,22 @@ public:
};
Box* getDocString(llvm::ArrayRef<AST_stmt*> body);
template <int slab_size, int alignment> ASTAllocator::ASTAllocatorSlab<slab_size, alignment>::~ASTAllocatorSlab() {
// find all AST* nodes and call the virtual destructor
for (int current_pos = 0; current_pos < num_bytes_used;) {
AST* node = (AST*)&data[current_pos];
int node_size = node->getSize();
node->~AST();
current_pos += llvm::RoundUpToAlignment(node_size, alignment);
}
}
#define DEFINE_AST_NODE(name) \
static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::name; \
virtual int getSize() const override { return sizeof(*this); }
class AST_expr : public AST {
public:
AST_expr(AST_TYPE::AST_TYPE type) : AST(type) {}
......@@ -223,7 +283,7 @@ public:
AST_alias(InternedString name, InternedString asname) : AST(AST_TYPE::alias), name(name), asname(asname) {}
static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::alias;
DEFINE_AST_NODE(alias)
};
class AST_Name;
......@@ -239,7 +299,7 @@ public:
AST_arguments() : AST(AST_TYPE::arguments) {}
static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::arguments;
DEFINE_AST_NODE(arguments)
};
class AST_Assert : public AST_stmt {
......@@ -251,7 +311,7 @@ public:
AST_Assert() : AST_stmt(AST_TYPE::Assert) {}
static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::Assert;
DEFINE_AST_NODE(Assert)
};
class AST_Assign : public AST_stmt {
......@@ -264,7 +324,7 @@ public:
AST_Assign() : AST_stmt(AST_TYPE::Assign) {}
static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::Assign;
DEFINE_AST_NODE(Assign)
};
class AST_AugAssign : public AST_stmt {
......@@ -278,7 +338,7 @@ public:
AST_AugAssign() : AST_stmt(AST_TYPE::AugAssign) {}
static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::AugAssign;
DEFINE_AST_NODE(AugAssign)
};
class AST_AugBinOp : public AST_expr {
......@@ -290,7 +350,7 @@ public:
AST_AugBinOp() : AST_expr(AST_TYPE::AugBinOp) {}
static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::AugBinOp;
DEFINE_AST_NODE(AugBinOp)
};
class AST_Attribute : public AST_expr {
......@@ -306,7 +366,7 @@ public:
AST_Attribute(AST_expr* value, AST_TYPE::AST_TYPE ctx_type, InternedString attr)
: AST_expr(AST_TYPE::Attribute), value(value), ctx_type(ctx_type), attr(attr) {}
static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::Attribute;
DEFINE_AST_NODE(Attribute)
};
class AST_BinOp : public AST_expr {
......@@ -318,7 +378,7 @@ public:
AST_BinOp() : AST_expr(AST_TYPE::BinOp) {}
static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::BinOp;
DEFINE_AST_NODE(BinOp)
};
class AST_BoolOp : public AST_expr {
......@@ -330,7 +390,7 @@ public:
AST_BoolOp() : AST_expr(AST_TYPE::BoolOp) {}
static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::BoolOp;
DEFINE_AST_NODE(BoolOp)
};
class AST_Break : public AST_stmt {
......@@ -340,7 +400,7 @@ public:
AST_Break() : AST_stmt(AST_TYPE::Break) {}
static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::Break;
DEFINE_AST_NODE(Break)
};
class AST_Call : public AST_expr {
......@@ -353,7 +413,7 @@ public:
AST_Call() : AST_expr(AST_TYPE::Call) {}
static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::Call;
DEFINE_AST_NODE(Call)
};
class AST_Compare : public AST_expr {
......@@ -366,7 +426,7 @@ public:
AST_Compare() : AST_expr(AST_TYPE::Compare) {}
static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::Compare;
DEFINE_AST_NODE(Compare)
};
class AST_comprehension : public AST {
......@@ -379,7 +439,7 @@ public:
AST_comprehension() : AST(AST_TYPE::comprehension) {}
static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::comprehension;
DEFINE_AST_NODE(comprehension)
};
class AST_ClassDef : public AST_stmt {
......@@ -393,7 +453,7 @@ public:
AST_ClassDef() : AST_stmt(AST_TYPE::ClassDef) {}
static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::ClassDef;
DEFINE_AST_NODE(ClassDef)
};
class AST_Continue : public AST_stmt {
......@@ -403,7 +463,7 @@ public:
AST_Continue() : AST_stmt(AST_TYPE::Continue) {}
static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::Continue;
DEFINE_AST_NODE(Continue)
};
class AST_Dict : public AST_expr {
......@@ -414,7 +474,7 @@ public:
AST_Dict() : AST_expr(AST_TYPE::Dict) {}
static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::Dict;
DEFINE_AST_NODE(Dict)
};
class AST_DictComp : public AST_expr {
......@@ -426,7 +486,7 @@ public:
AST_DictComp() : AST_expr(AST_TYPE::DictComp) {}
const static AST_TYPE::AST_TYPE TYPE = AST_TYPE::DictComp;
DEFINE_AST_NODE(DictComp)
};
class AST_Delete : public AST_stmt {
......@@ -437,7 +497,7 @@ public:
AST_Delete() : AST_stmt(AST_TYPE::Delete) {}
static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::Delete;
DEFINE_AST_NODE(Delete)
};
class AST_Ellipsis : public AST_slice {
......@@ -446,7 +506,7 @@ public:
AST_Ellipsis() : AST_slice(AST_TYPE::Ellipsis) {}
static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::Ellipsis;
DEFINE_AST_NODE(Ellipsis)
};
class AST_Expr : public AST_stmt {
......@@ -459,7 +519,7 @@ public:
AST_Expr() : AST_stmt(AST_TYPE::Expr) {}
AST_Expr(AST_expr* value) : AST_stmt(AST_TYPE::Expr), value(value) {}
static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::Expr;
DEFINE_AST_NODE(Expr)
};
class AST_ExceptHandler : public AST {
......@@ -472,7 +532,7 @@ public:
AST_ExceptHandler() : AST(AST_TYPE::ExceptHandler) {}
static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::ExceptHandler;
DEFINE_AST_NODE(ExceptHandler)
};
class AST_Exec : public AST_stmt {
......@@ -486,7 +546,7 @@ public:
AST_Exec() : AST_stmt(AST_TYPE::Exec) {}
static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::Exec;
DEFINE_AST_NODE(Exec)
};
// (Alternative to AST_Module, used for, e.g., eval)
......@@ -502,7 +562,7 @@ public:
AST_Expression(std::unique_ptr<InternedStringPool> interned_strings)
: AST(AST_TYPE::Expression), interned_strings(std::move(interned_strings)) {}
static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::Expression;
DEFINE_AST_NODE(Expression)
};
class AST_ExtSlice : public AST_slice {
......@@ -513,7 +573,7 @@ public:
AST_ExtSlice() : AST_slice(AST_TYPE::ExtSlice) {}
static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::ExtSlice;
DEFINE_AST_NODE(ExtSlice)
};
class AST_For : public AST_stmt {
......@@ -526,7 +586,7 @@ public:
AST_For() : AST_stmt(AST_TYPE::For) {}
static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::For;
DEFINE_AST_NODE(For)
};
class AST_FunctionDef : public AST_stmt {
......@@ -541,7 +601,7 @@ public:
AST_FunctionDef() : AST_stmt(AST_TYPE::FunctionDef) {}
static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::FunctionDef;
DEFINE_AST_NODE(FunctionDef)
};
class AST_GeneratorExp : public AST_expr {
......@@ -553,7 +613,7 @@ public:
AST_GeneratorExp() : AST_expr(AST_TYPE::GeneratorExp) {}
const static AST_TYPE::AST_TYPE TYPE = AST_TYPE::GeneratorExp;
DEFINE_AST_NODE(GeneratorExp)
};
class AST_Global : public AST_stmt {
......@@ -565,7 +625,7 @@ public:
AST_Global() : AST_stmt(AST_TYPE::Global) {}
static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::Global;
DEFINE_AST_NODE(Global)
};
class AST_If : public AST_stmt {
......@@ -578,7 +638,7 @@ public:
AST_If() : AST_stmt(AST_TYPE::If) {}
static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::If;
DEFINE_AST_NODE(If)
};
class AST_IfExp : public AST_expr {
......@@ -589,7 +649,7 @@ public:
AST_IfExp() : AST_expr(AST_TYPE::IfExp) {}
static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::IfExp;
DEFINE_AST_NODE(IfExp)
};
class AST_Import : public AST_stmt {
......@@ -601,7 +661,7 @@ public:
AST_Import() : AST_stmt(AST_TYPE::Import) {}
static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::Import;
DEFINE_AST_NODE(Import)
};
class AST_ImportFrom : public AST_stmt {
......@@ -615,7 +675,7 @@ public:
AST_ImportFrom() : AST_stmt(AST_TYPE::ImportFrom) {}
static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::ImportFrom;
DEFINE_AST_NODE(ImportFrom)
};
class AST_Index : public AST_slice {
......@@ -626,7 +686,7 @@ public:
AST_Index() : AST_slice(AST_TYPE::Index) {}
static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::Index;
DEFINE_AST_NODE(Index)
};
class AST_keyword : public AST {
......@@ -639,7 +699,7 @@ public:
AST_keyword() : AST(AST_TYPE::keyword) {}
static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::keyword;
DEFINE_AST_NODE(keyword)
};
class AST_Lambda : public AST_expr {
......@@ -651,7 +711,7 @@ public:
AST_Lambda() : AST_expr(AST_TYPE::Lambda) {}
static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::Lambda;
DEFINE_AST_NODE(Lambda)
};
class AST_List : public AST_expr {
......@@ -663,7 +723,7 @@ public:
AST_List() : AST_expr(AST_TYPE::List) {}
static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::List;
DEFINE_AST_NODE(List)
};
class AST_ListComp : public AST_expr {
......@@ -675,7 +735,7 @@ public:
AST_ListComp() : AST_expr(AST_TYPE::ListComp) {}
const static AST_TYPE::AST_TYPE TYPE = AST_TYPE::ListComp;
DEFINE_AST_NODE(ListComp)
};
class AST_Module : public AST {
......@@ -690,7 +750,7 @@ public:
AST_Module(std::unique_ptr<InternedStringPool> interned_strings)
: AST(AST_TYPE::Module), interned_strings(std::move(interned_strings)) {}
static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::Module;
DEFINE_AST_NODE(Module)
};
class AST_Suite : public AST {
......@@ -704,7 +764,7 @@ public:
AST_Suite(std::unique_ptr<InternedStringPool> interned_strings)
: AST(AST_TYPE::Suite), interned_strings(std::move(interned_strings)) {}
static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::Suite;
DEFINE_AST_NODE(Suite)
};
class AST_Name : public AST_expr {
......@@ -725,7 +785,7 @@ public:
id(id),
lookup_type(ScopeInfo::VarScopeType::UNKNOWN) {}
static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::Name;
DEFINE_AST_NODE(Name)
};
class AST_Num : public AST_expr {
......@@ -750,7 +810,7 @@ public:
AST_Num() : AST_expr(AST_TYPE::Num) {}
static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::Num;
DEFINE_AST_NODE(Num)
};
class AST_Repr : public AST_expr {
......@@ -761,7 +821,7 @@ public:
AST_Repr() : AST_expr(AST_TYPE::Repr) {}
static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::Repr;
DEFINE_AST_NODE(Repr)
};
class AST_Pass : public AST_stmt {
......@@ -771,7 +831,7 @@ public:
AST_Pass() : AST_stmt(AST_TYPE::Pass) {}
static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::Pass;
DEFINE_AST_NODE(Pass)
};
class AST_Print : public AST_stmt {
......@@ -785,7 +845,7 @@ public:
AST_Print() : AST_stmt(AST_TYPE::Print) {}
static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::Print;
DEFINE_AST_NODE(Print)
};
class AST_Raise : public AST_stmt {
......@@ -801,7 +861,7 @@ public:
AST_Raise() : AST_stmt(AST_TYPE::Raise), arg0(NULL), arg1(NULL), arg2(NULL) {}
static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::Raise;
DEFINE_AST_NODE(Raise)
};
class AST_Return : public AST_stmt {
......@@ -813,7 +873,7 @@ public:
AST_Return() : AST_stmt(AST_TYPE::Return) {}
static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::Return;
DEFINE_AST_NODE(Return)
};
class AST_Set : public AST_expr {
......@@ -824,7 +884,7 @@ public:
AST_Set() : AST_expr(AST_TYPE::Set) {}
static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::Set;
DEFINE_AST_NODE(Set)
};
class AST_SetComp : public AST_expr {
......@@ -836,7 +896,7 @@ public:
AST_SetComp() : AST_expr(AST_TYPE::SetComp) {}
const static AST_TYPE::AST_TYPE TYPE = AST_TYPE::SetComp;
DEFINE_AST_NODE(SetComp)
};
class AST_Slice : public AST_slice {
......@@ -847,7 +907,7 @@ public:
AST_Slice() : AST_slice(AST_TYPE::Slice) {}
static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::Slice;
DEFINE_AST_NODE(Slice)
};
class AST_Str : public AST_expr {
......@@ -867,7 +927,7 @@ public:
AST_Str() : AST_expr(AST_TYPE::Str), str_type(UNSET) {}
AST_Str(std::string s) : AST_expr(AST_TYPE::Str), str_type(STR), str_data(std::move(s)) {}
static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::Str;
DEFINE_AST_NODE(Str)
};
class AST_Subscript : public AST_expr {
......@@ -880,7 +940,7 @@ public:
AST_Subscript() : AST_expr(AST_TYPE::Subscript) {}
static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::Subscript;
DEFINE_AST_NODE(Subscript)
};
class AST_TryExcept : public AST_stmt {
......@@ -893,7 +953,7 @@ public:
AST_TryExcept() : AST_stmt(AST_TYPE::TryExcept) {}
static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::TryExcept;
DEFINE_AST_NODE(TryExcept)
};
class AST_TryFinally : public AST_stmt {
......@@ -905,7 +965,7 @@ public:
AST_TryFinally() : AST_stmt(AST_TYPE::TryFinally) {}
static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::TryFinally;
DEFINE_AST_NODE(TryFinally)
};
class AST_Tuple : public AST_expr {
......@@ -917,7 +977,7 @@ public:
AST_Tuple() : AST_expr(AST_TYPE::Tuple) {}
static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::Tuple;
DEFINE_AST_NODE(Tuple)
};
class AST_UnaryOp : public AST_expr {
......@@ -929,7 +989,7 @@ public:
AST_UnaryOp() : AST_expr(AST_TYPE::UnaryOp) {}
static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::UnaryOp;
DEFINE_AST_NODE(UnaryOp)
};
class AST_While : public AST_stmt {
......@@ -942,7 +1002,7 @@ public:
AST_While() : AST_stmt(AST_TYPE::While) {}
static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::While;
DEFINE_AST_NODE(While)
};
class AST_With : public AST_stmt {
......@@ -955,7 +1015,7 @@ public:
AST_With() : AST_stmt(AST_TYPE::With) {}
static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::With;
DEFINE_AST_NODE(With)
};
class AST_Yield : public AST_expr {
......@@ -966,7 +1026,7 @@ public:
AST_Yield() : AST_expr(AST_TYPE::Yield) {}
static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::Yield;
DEFINE_AST_NODE(Yield)
};
......@@ -984,7 +1044,7 @@ public:
AST_ClsAttribute() : AST_expr(AST_TYPE::ClsAttribute) {}
static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::ClsAttribute;
DEFINE_AST_NODE(ClsAttribute)
};
class AST_Invoke : public AST_stmt {
......@@ -998,7 +1058,7 @@ public:
AST_Invoke(AST_stmt* stmt) : AST_stmt(AST_TYPE::Invoke), stmt(stmt) {}
static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::Invoke;
DEFINE_AST_NODE(Invoke)
};
// "LangPrimitive" represents operations that "primitive" to the language,
......@@ -1028,7 +1088,7 @@ public:
AST_LangPrimitive(Opcodes opcode) : AST_expr(AST_TYPE::LangPrimitive), opcode(opcode) {}
static const AST_TYPE::AST_TYPE TYPE = AST_TYPE::LangPrimitive;
DEFINE_AST_NODE(LangPrimitive)
};
template <typename T> T* ast_cast(AST* node) {
......
src/core/cfg.cpp
View file @ 6926ccff
......@@ -367,8 +367,9 @@ private:
return stringpool.get(std::move(name));
}
AST_Name* makeASTName(InternedString id, AST_TYPE::AST_TYPE ctx_type, int lineno, int col_offset = 0) {
AST_Name* name = new AST_Name(id, ctx_type, lineno, col_offset);
AST_Name* makeASTName(ASTAllocator& allocator, InternedString id, AST_TYPE::AST_TYPE ctx_type, int lineno,
int col_offset = 0) {
AST_Name* name = new (allocator) AST_Name(id, ctx_type, lineno, col_offset);
return name;
}
......@@ -683,15 +684,15 @@ private:
curblock = nullptr;
}
AST_expr* makeASTLoadAttribute(AST_expr* base, InternedString name, bool clsonly) {
AST_expr* makeASTLoadAttribute(ASTAllocator& allocator, AST_expr* base, InternedString name, bool clsonly) {
AST_expr* rtn;
if (clsonly) {
AST_ClsAttribute* attr = new AST_ClsAttribute();
AST_ClsAttribute* attr = new (allocator) AST_ClsAttribute();
attr->value = base;
attr->attr = name;
rtn = attr;
} else {
AST_Attribute* attr = new AST_Attribute();
AST_Attribute* attr = new (allocator) AST_Attribute();
attr->ctx_type = AST_TYPE::Load;
attr->value = base;
attr->attr = name;
......@@ -720,8 +721,8 @@ private:
return rtn;
}
AST_Call* makeASTCall(AST_expr* func) {
AST_Call* call = new AST_Call();
AST_Call* makeASTCall(ASTAllocator& allocator, AST_expr* func) {
AST_Call* call = new (allocator) AST_Call();
call->starargs = NULL;
call->kwargs = NULL;
call->func = func;
......@@ -730,14 +731,14 @@ private:
return call;
}
AST_Call* makeASTCall(AST_expr* func, AST_expr* arg0) {
auto call = makeASTCall(func);
AST_Call* makeASTCall(ASTAllocator& allocator, AST_expr* func, AST_expr* arg0) {
auto call = makeASTCall(allocator, func);
call->args.push_back(arg0);
return call;
}
AST_Call* makeASTCall(AST_expr* func, AST_expr* arg0, AST_expr* arg1) {
auto call = makeASTCall(func);
AST_Call* makeASTCall(ASTAllocator& allocator, AST_expr* func, AST_expr* arg0, AST_expr* arg1) {
auto call = makeASTCall(allocator, func);
call->args.push_back(arg0);
call->args.push_back(arg1);
return call;
......@@ -880,8 +881,8 @@ private:
push_back(assign);
}
AST_stmt* makeASTExpr(AST_expr* expr) {
AST_Expr* stmt = new AST_Expr();
AST_stmt* makeASTExpr(ASTAllocator& allocator, AST_expr* expr) {
AST_Expr* stmt = new (allocator) AST_Expr();
stmt->value = expr;
stmt->lineno = expr->lineno;
stmt->col_offset = expr->col_offset;
......@@ -1188,13 +1189,13 @@ private:
// This is a helper function used for generator expressions and comprehensions.
// TODO(rntz): use this to handle unscoped (i.e. list) comprehensions as well?
void emitComprehensionLoops(int lineno, std::vector<AST_stmt*>* insert_point,
void emitComprehensionLoops(ASTAllocator& allocator, int lineno, std::vector<AST_stmt*>* insert_point,
const std::vector<AST_comprehension*>& comprehensions, AST_expr* first_generator,
std::function<void(std::vector<AST_stmt*>*)> do_yield) {
for (int i = 0; i < comprehensions.size(); i++) {
AST_comprehension* c = comprehensions[i];
AST_For* loop = new AST_For();
AST_For* loop = new (allocator) AST_For();
loop->target = c->target;
loop->iter = (i == 0) ? first_generator : c->iter;
loop->lineno = lineno;
......@@ -1203,7 +1204,7 @@ private:
insert_point = &loop->body;
for (AST_expr* if_condition : c->ifs) {
AST_If* if_block = new AST_If();
AST_If* if_block = new (allocator) AST_If();
if_block->lineno = if_condition->lineno;
// Note: don't call callNonzero here, since we are generating
// AST inside a new functiondef which will go through the CFG
......@@ -1216,7 +1217,6 @@ private:
}
do_yield(insert_point);
// TODO: have to delete these new ast nodes
}
BST_expr* remapGeneratorExp(AST_GeneratorExp* node) {
......@@ -1228,16 +1228,17 @@ private:
BST_expr* first = remapExpr(node->generators[0]->iter);
InternedString arg_name = internString("#arg");
AST_arguments* genexp_args = new AST_arguments();
genexp_args->args.push_back(makeASTName(arg_name, AST_TYPE::Param, node->lineno));
ASTAllocator allocator;
AST_arguments* genexp_args = new (allocator) AST_arguments();
genexp_args->args.push_back(makeASTName(allocator, arg_name, AST_TYPE::Param, node->lineno));
std::vector<AST_stmt*> new_body;
emitComprehensionLoops(node->lineno, &new_body, node->generators,
makeASTName(arg_name, AST_TYPE::Load, node->lineno, /* col_offset */ 0),
[this, node](std::vector<AST_stmt*>* insert_point) {
auto y = new AST_Yield();
emitComprehensionLoops(allocator, node->lineno, &new_body, node->generators,
makeASTName(allocator, arg_name, AST_TYPE::Load, node->lineno, /* col_offset */ 0),
[this, node, &allocator](std::vector<AST_stmt*>* insert_point) {
auto y = new (allocator) AST_Yield();
y->value = node->elt;
y->lineno = node->lineno;
insert_point->push_back(makeASTExpr(y));
insert_point->push_back(makeASTExpr(allocator, y));
});
// I'm not sure this actually gets used
......@@ -1258,18 +1259,21 @@ private:
return makeCall(makeLoad(func_var_name, node, true), first);
}
void emitComprehensionYield(AST_DictComp* node, InternedString dict_name, std::vector<AST_stmt*>* insert_point) {
void emitComprehensionYield(ASTAllocator& allocator, AST_DictComp* node, InternedString dict_name,
std::vector<AST_stmt*>* insert_point) {
// add entry to the dictionary
AST_expr* setitem = makeASTLoadAttribute(makeASTName(dict_name, AST_TYPE::Load, node->lineno),
internString("__setitem__"), true);
insert_point->push_back(makeASTExpr(makeASTCall(setitem, node->key, node->value)));
AST_expr* setitem
= makeASTLoadAttribute(allocator, makeASTName(allocator, dict_name, AST_TYPE::Load, node->lineno),
internString("__setitem__"), true);
insert_point->push_back(makeASTExpr(allocator, makeASTCall(allocator, setitem, node->key, node->value)));
}
void emitComprehensionYield(AST_SetComp* node, InternedString set_name, std::vector<AST_stmt*>* insert_point) {
void emitComprehensionYield(ASTAllocator& allocator, AST_SetComp* node, InternedString set_name,
std::vector<AST_stmt*>* insert_point) {
// add entry to the dictionary
AST_expr* add
= makeASTLoadAttribute(makeASTName(set_name, AST_TYPE::Load, node->lineno), internString("add"), true);
insert_point->push_back(makeASTExpr(makeASTCall(add, node->elt)));
AST_expr* add = makeASTLoadAttribute(allocator, makeASTName(allocator, set_name, AST_TYPE::Load, node->lineno),
internString("add"), true);
insert_point->push_back(makeASTExpr(allocator, makeASTCall(allocator, add, node->elt)));
}
template <typename ResultType, typename CompType> BST_expr* remapScopedComprehension(CompType* node) {
......@@ -1277,25 +1281,28 @@ private:
BST_expr* first = remapExpr(node->generators[0]->iter);
InternedString arg_name = internString("#arg");
AST_arguments* args = new AST_arguments();
args->args.push_back(makeASTName(arg_name, AST_TYPE::Param, node->lineno));
ASTAllocator allocator;
AST_arguments* args = new (allocator) AST_arguments();
args->args.push_back(makeASTName(allocator, arg_name, AST_TYPE::Param, node->lineno));
std::vector<AST_stmt*> new_body;
InternedString rtn_name = internString("#comp_rtn");
auto asgn = new AST_Assign();
asgn->targets.push_back(makeASTName(rtn_name, AST_TYPE::Store, node->lineno));
asgn->value = new ResultType();
auto asgn = new (allocator) AST_Assign();
asgn->targets.push_back(makeASTName(allocator, rtn_name, AST_TYPE::Store, node->lineno));
asgn->value = new (allocator) ResultType();
asgn->lineno = node->lineno;
new_body.push_back(asgn);
auto lambda =
[&](std::vector<AST_stmt*>* insert_point) { emitComprehensionYield(node, rtn_name, insert_point); };
AST_Name* first_name = makeASTName(internString("#arg"), AST_TYPE::Load, node->lineno, /* col_offset */ 0);
emitComprehensionLoops(node->lineno, &new_body, node->generators, first_name, lambda);
auto lambda = [&](std::vector<AST_stmt*>* insert_point) {
emitComprehensionYield(allocator, node, rtn_name, insert_point);
};
AST_Name* first_name
= makeASTName(allocator, internString("#arg"), AST_TYPE::Load, node->lineno, /* col_offset */ 0);
emitComprehensionLoops(allocator, node->lineno, &new_body, node->generators, first_name, lambda);
auto rtn = new AST_Return();
rtn->value = makeASTName(rtn_name, AST_TYPE::Load, node->lineno, /* col_offset */ 0);
auto rtn = new (allocator) AST_Return();
rtn->value = makeASTName(allocator, rtn_name, AST_TYPE::Load, node->lineno, /* col_offset */ 0);
rtn->lineno = node->lineno;
new_body.push_back(rtn);
......@@ -1363,7 +1370,8 @@ private:
}
BST_expr* remapLambda(AST_Lambda* node) {
auto stmt = new AST_Return;
ASTAllocator allocator;
auto stmt = new (allocator) AST_Return;
stmt->lineno = node->lineno;
stmt->value = node->body; // don't remap now; will be CFG'ed later
......@@ -2140,7 +2148,8 @@ public:
}
case AST_TYPE::List: {
AST_List* list = static_cast<AST_List*>(t);
AST_Delete* temp_ast_del = new AST_Delete();
ASTAllocator allocator;
AST_Delete* temp_ast_del = new (allocator) AST_Delete();
temp_ast_del->lineno = node->lineno;
temp_ast_del->col_offset = node->col_offset;
......@@ -2152,7 +2161,8 @@ public:
}
case AST_TYPE::Tuple: {
AST_Tuple* tuple = static_cast<AST_Tuple*>(t);
AST_Delete* temp_ast_del = new AST_Delete();
ASTAllocator allocator;
AST_Delete* temp_ast_del = new (allocator) AST_Delete();
temp_ast_del->lineno = node->lineno;
temp_ast_del->col_offset = node->col_offset;
......
src/jit.cpp
View file @ 6926ccff
......@@ -460,7 +460,9 @@ static int main(int argc, char** argv) noexcept {
if (command != NULL) {
try {
main_module = createModule(autoDecref(boxString("__main__")), "<string>");
AST_Module* m = parse_string(command, /* future_flags = */ 0);
AST_Module* m;
std::unique_ptr<ASTAllocator> ast_allocator;
std::tie(m, ast_allocator) = parse_string(command, /* future_flags = */ 0);
compileAndRunModule(m, main_module);
rtncode = 0;
} catch (ExcInfo e) {
......@@ -504,7 +506,10 @@ static int main(int argc, char** argv) noexcept {
free(real_path);
try {
AST_Module* ast = parse_file(fn, /* future_flags = */ 0);
std::unique_ptr<ASTAllocator> ast_allocator;
AST_Module* ast;
std::tie(ast, ast_allocator) = parse_file(fn, /* future_flags = */ 0);
compileAndRunModule(ast, main_module);
rtncode = 0;
} catch (ExcInfo e) {
......
src/runtime/capi.cpp
View file @ 6926ccff
......@@ -37,6 +37,7 @@
#include "capi/types.h"
#include "codegen/irgen/hooks.h"
#include "codegen/unwinding.h"
#include "core/ast.h"
#include "core/threading.h"
#include "core/types.h"
#include "runtime/classobj.h"
......@@ -847,8 +848,8 @@ extern "C" int PyRun_InteractiveOneFlags(FILE* fp, const char* filename, PyCompi
bool failed = false;
try {
assert(mod->kind == Interactive_kind);
AST_Module* pyston_module = static_cast<AST_Module*>(cpythonToPystonAST(mod, filename));
compileAndRunModule(pyston_module, static_cast<BoxedModule*>(m));
auto res = cpythonToPystonAST(mod, filename);
compileAndRunModule((AST_Module*)res.first, static_cast<BoxedModule*>(m));
} catch (ExcInfo e) {
setCAPIException(e);
failed = true;
......
src/runtime/import.cpp
View file @ 6926ccff
......@@ -25,6 +25,7 @@
#include "codegen/irgen/hooks.h"
#include "codegen/parser.h"
#include "codegen/unwinding.h"
#include "core/ast.h"
#include "runtime/inline/list.h"
#include "runtime/objmodel.h"
#include "runtime/util.h"
......@@ -87,7 +88,9 @@ extern "C" PyObject* load_source_module(char* name, char* pathname, FILE* fp) no
AUTO_DECREF(name_boxed);
try {
BoxedModule* module = createModule(name_boxed, pathname);
AST_Module* ast = caching_parse_file(pathname, /* future_flags = */ 0);
std::unique_ptr<ASTAllocator> ast_allocator;
AST_Module* ast;
std::tie(ast, ast_allocator) = caching_parse_file(pathname, /* future_flags = */ 0);
assert(ast);
compileAndRunModule(ast, module);
Box* r = getSysModulesDict()->getOrNull(name_boxed);
......@@ -118,7 +121,9 @@ extern "C" PyObject* PyImport_ExecCodeModuleEx(const char* name, PyObject* co, c
static BoxedString* file_str = getStaticString("__file__");
module->setattr(file_str, autoDecref(boxString(pathname)), NULL);
AST_Module* ast = parse_string(code->data(), /* future_flags = */ 0);
AST_Module* ast;
std::unique_ptr<ASTAllocator> ast_allocator;
std::tie(ast, ast_allocator) = parse_string(code->data(), /* future_flags = */ 0);
compileAndRunModule(ast, module);
return incref(module);
} catch (ExcInfo e) {
......
test/unittests/analysis.cpp
View file @ 6926ccff
......@@ -29,7 +29,9 @@ protected:
#ifndef NDEBUG
TEST_F(AnalysisTest, augassign) {
const std::string fn("test/unittests/analysis_listcomp.py");
AST_Module* module = caching_parse_file(fn.c_str(), 0);
std::unique_ptr<ASTAllocator> ast_allocator;
AST_Module* module;
std::tie(module, ast_allocator) = caching_parse_file(fn.c_str(), 0);
assert(module);
FutureFlags future_flags = getFutureFlags(module->body, fn.c_str());
......@@ -45,7 +47,7 @@ TEST_F(AnalysisTest, augassign) {
ASSERT_NE(scope_info->getScopeTypeOfName(module->interned_strings->get("a")), ScopeInfo::VarScopeType::GLOBAL);
ASSERT_FALSE(scope_info->getScopeTypeOfName(module->interned_strings->get("b")) == ScopeInfo::VarScopeType::GLOBAL);
AST_arguments* args = new AST_arguments();
AST_arguments* args = new (*ast_allocator) AST_arguments();
ParamNames param_names(args, *module->interned_strings.get());
// Hack to get at the cfg:
......@@ -68,7 +70,9 @@ TEST_F(AnalysisTest, augassign) {
void doOsrTest(bool is_osr, bool i_maybe_undefined) {
const std::string fn("test/unittests/analysis_osr.py");
AST_Module* module = caching_parse_file(fn.c_str(), 0);
std::unique_ptr<ASTAllocator> ast_allocator;
AST_Module* module;
std::tie(module, ast_allocator) = caching_parse_file(fn.c_str(), 0);
assert(module);
ParamNames param_names(NULL, *module->interned_strings.get());
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7