Skip to content

P
Pyston
Commit fc366564 authored by Marius Wachtler's avatar Marius Wachtler
Browse files
Options

Add new JIT tier between the interpreter and the LLVM JIT tiers. 

This JIT is tightly coupled to the ASTInterpreter, at every CFGBlock* entry/exit
on can switch between interpreting and directly executing the generated code without having
to do any translations.
Generating the code is pretty fast compared to the LLVM tier but the generated code is not as fast
as code generated by the higher LLVM tiers.
But because the JITed can use runtime ICs, avoids a lot of interpretation overhead and
stores CFGBlock locals sysbols inside register/stack slots, it's much faster than the interpreter.
parent 1ee49a67
Hide whitespace changes
Inline Side-by-side
Showing with 1817 additions and 340 deletions
src/CMakeLists.txt
View file @ fc366564
......@@ -34,6 +34,7 @@ add_library(PYSTON_OBJECTS OBJECT ${OPTIONAL_SRCS}
capi/object.cpp
capi/typeobject.cpp
codegen/ast_interpreter.cpp
codegen/baseline_jit.cpp
codegen/codegen.cpp
codegen/compvars.cpp
codegen/entry.cpp
......
src/asm_writing/icinfo.cpp
View file @ fc366564
......@@ -109,7 +109,6 @@ void ICSlotRewrite::commit(CommitHook* hook) {
if (!do_commit)
return;
assert(assembler->isExactlyFull());
assert(!assembler->hasFailed());
for (int i = 0; i < dependencies.size(); i++) {
......
src/asm_writing/icinfo.h
View file @ fc366564
......@@ -62,9 +62,8 @@ private:
ICSlotInfo* ic_entry;
ICSlotRewrite(ICInfo* ic, const char* debug_name);
public:
ICSlotRewrite(ICInfo* ic, const char* debug_name);
~ICSlotRewrite();
assembler::Assembler* getAssembler() { return assembler; }
......
src/asm_writing/rewriter.cpp
View file @ fc366564
......@@ -958,7 +958,7 @@ void Rewriter::_call(RewriterVar* result, bool has_side_effects, void* func_addr
assembler->callq(r);
} else {
assembler->call(assembler::Immediate(offset));
assert(asm_address == (uint64_t)assembler->curInstPointer());
assert(assembler->hasFailed() || asm_address == (uint64_t)assembler->curInstPointer());
}
assert(vars_by_location.count(assembler::RAX) == 0);
......
src/codegen/ast_interpreter.cpp
View file @ fc366564
......@@ -20,6 +20,7 @@
#include "analysis/function_analysis.h"
#include "analysis/scoping_analysis.h"
#include "codegen/baseline_jit.h"
#include "codegen/codegen.h"
#include "codegen/compvars.h"
#include "codegen/irgen.h"
......@@ -74,21 +75,6 @@ public:
static void deregister(void* frame_addr);
};
union Value {
bool b;
int64_t n;
double d;
Box* o;
Value(bool b) : b(b) {}
Value(int64_t n = 0) : n(n) {}
Value(double d) : d(d) {}
Value(Box* o) : o(o) {
if (DEBUG >= 2)
ASSERT(gc::isValidGCObject(o), "%p", o);
}
};
class ASTInterpreter : public Box {
public:
typedef ContiguousMap<InternedString, Box*> SymMap;
......@@ -108,9 +94,11 @@ public:
private:
Box* createFunction(AST* node, AST_arguments* args, const std::vector<AST_stmt*>& body);
Value doBinOp(Box* left, Box* right, int op, BinExpType exp_type);
Value doBinOp(Value left, Value right, int op, BinExpType exp_type);
void doStore(AST_expr* node, Value value);
void doStore(InternedString name, Value value);
Box* doOSR(AST_Jump* node);
Value getNone();
Value visit_assert(AST_Assert* node);
Value visit_assign(AST_Assign* node);
......@@ -156,6 +144,10 @@ private:
Value visit_jump(AST_Jump* node);
Value visit_langPrimitive(AST_LangPrimitive* node);
void startJITing(CFGBlock* block, int jump_offset = 0);
void abortJITing();
void finishJITing(CFGBlock* continue_block = NULL);
CompiledFunction* compiled_func;
SourceInfo* source_info;
ScopeInfo* scope_info;
......@@ -173,6 +165,7 @@ private:
// This is either a module or a dict
Box* globals;
void* frame_addr; // used to clear entry inside the s_interpreterMap on destruction
std::unique_ptr<JitFragmentWriter> jit;
public:
DEFAULT_CLASS_SIMPLE(astinterpreter_cls);
......@@ -211,6 +204,7 @@ public:
}
friend class RegisterHelper;
friend struct pyston::ASTInterpreterJitInterface;
};
void ASTInterpreter::addSymbol(InternedString name, Box* value, bool allow_duplicates) {
......@@ -305,15 +299,15 @@ void ASTInterpreter::initArguments(int nargs, BoxedClosure* _closure, BoxedGener
int i = 0;
for (auto& name : param_names.args) {
doStore(source_info->getInternedStrings().get(name), argsArray[i++]);
doStore(source_info->getInternedStrings().get(name), Value(argsArray[i++], 0));
}
if (!param_names.vararg.str().empty()) {
doStore(source_info->getInternedStrings().get(param_names.vararg), argsArray[i++]);
doStore(source_info->getInternedStrings().get(param_names.vararg), Value(argsArray[i++], 0));
}
if (!param_names.kwarg.str().empty()) {
doStore(source_info->getInternedStrings().get(param_names.kwarg), argsArray[i++]);
doStore(source_info->getInternedStrings().get(param_names.kwarg), Value(argsArray[i++], 0));
}
}
......@@ -342,17 +336,59 @@ void RegisterHelper::deregister(void* frame_addr) {
s_interpreterMap.erase(frame_addr);
}
void ASTInterpreter::startJITing(CFGBlock* block, int jump_offset) {
assert(ENABLE_BASELINEJIT);
assert(!jit);
auto& code_blocks = compiled_func->code_blocks;
JitCodeBlock* code_block = NULL;
if (!code_blocks.empty())
code_block = code_blocks[code_blocks.size() - 1].get();
if (!code_block || code_block->shouldCreateNewBlock()) {
code_blocks.push_back(std::unique_ptr<JitCodeBlock>(new JitCodeBlock(source_info->getName())));
code_block = code_blocks[code_blocks.size() - 1].get();
jump_offset = 0;
}
jit = code_block->newFragment(block, jump_offset);
}
void ASTInterpreter::abortJITing() {
if (jit) {
jit->abortCompilation();
jit.reset();
}
}
void ASTInterpreter::finishJITing(CFGBlock* continue_block) {
if (!jit)
return;
int jump_offset = jit->finishCompilation();
jit.reset();
if (continue_block && !continue_block->code)
startJITing(continue_block, jump_offset);
}
Value ASTInterpreter::executeInner(ASTInterpreter& interpreter, CFGBlock* start_block, AST_stmt* start_at,
RegisterHelper* reg) {
void* frame_addr = __builtin_frame_address(0);
reg->doRegister(frame_addr, &interpreter);
Value v;
bool should_jit = false;
bool from_start = start_block == NULL && start_at == NULL;
assert((start_block == NULL) == (start_at == NULL));
if (start_block == NULL) {
start_block = interpreter.source_info->cfg->getStartingBlock();
start_at = start_block->body[0];
if (ENABLE_BASELINEJIT && interpreter.compiled_func->times_called >= REOPT_THRESHOLD_INTERPRETER
&& !start_block->code)
should_jit = true;
}
// Important that this happens after RegisterHelper:
......@@ -360,25 +396,66 @@ Value ASTInterpreter::executeInner(ASTInterpreter& interpreter, CFGBlock* start_
threading::allowGLReadPreemption();
interpreter.current_inst = NULL;
interpreter.current_block = start_block;
bool started = false;
for (auto s : start_block->body) {
if (!started) {
if (s != start_at)
continue;
started = true;
if (!from_start) {
interpreter.current_block = start_block;
bool started = false;
for (auto s : start_block->body) {
if (!started) {
if (s != start_at)
continue;
started = true;
}
interpreter.current_inst = s;
v = interpreter.visit_stmt(s);
}
} else {
if (should_jit)
interpreter.startJITing(start_block);
interpreter.current_inst = s;
v = interpreter.visit_stmt(s);
interpreter.next_block = start_block;
}
while (interpreter.next_block) {
interpreter.current_block = interpreter.next_block;
interpreter.next_block = 0;
if (ENABLE_BASELINEJIT && !interpreter.jit) {
CFGBlock* b = interpreter.current_block;
if (b->entry_code) {
should_jit = true;
try {
UNAVOIDABLE_STAT_TIMER(t0, "us_timer_in_baseline_jitted_code");
std::pair<CFGBlock*, Box*> rtn = b->entry_code(&interpreter, b);
interpreter.next_block = rtn.first;
if (!interpreter.next_block)
return Value(rtn.second, 0);
} catch (ExcInfo e) {
AST_stmt* stmt = interpreter.getCurrentStatement();
if (stmt->type != AST_TYPE::Invoke)
throw e;
auto source = interpreter.getCF()->clfunc->source.get();
exceptionCaughtInInterpreter(
LineInfo(stmt->lineno, stmt->col_offset, source->fn, source->getName()), &e);
interpreter.next_block = ((AST_Invoke*)stmt)->exc_dest;
interpreter.last_exception = e;
}
continue;
}
}
if (ENABLE_BASELINEJIT && should_jit && !interpreter.jit) {
assert(!interpreter.current_block->code);
interpreter.startJITing(interpreter.current_block);
}
for (AST_stmt* s : interpreter.current_block->body) {
interpreter.current_inst = s;
if (interpreter.jit)
interpreter.jit->emitSetCurrentInst(s);
v = interpreter.visit_stmt(s);
}
}
......@@ -389,18 +466,17 @@ Value ASTInterpreter::execute(ASTInterpreter& interpreter, CFGBlock* start_block
UNAVOIDABLE_STAT_TIMER(t0, "us_timer_in_interpreter");
RegisterHelper frame_registerer;
return executeInner(interpreter, start_block, start_at, &frame_registerer);
}
Value ASTInterpreter::doBinOp(Box* left, Box* right, int op, BinExpType exp_type) {
Value ASTInterpreter::doBinOp(Value left, Value right, int op, BinExpType exp_type) {
switch (exp_type) {
case BinExpType::AugBinOp:
return augbinop(left, right, op);
return Value(augbinop(left.o, right.o, op), jit ? jit->emitAugbinop(left, right, op) : NULL);
case BinExpType::BinOp:
return binop(left, right, op);
return Value(binop(left.o, right.o, op), jit ? jit->emitBinop(left, right, op) : NULL);
case BinExpType::Compare:
return compare(left, right, op);
return Value(compare(left.o, right.o, op), jit ? jit->emitCompare(left, right, op) : NULL);
default:
RELEASE_ASSERT(0, "not implemented");
}
......@@ -410,14 +486,30 @@ Value ASTInterpreter::doBinOp(Box* left, Box* right, int op, BinExpType exp_type
void ASTInterpreter::doStore(InternedString name, Value value) {
ScopeInfo::VarScopeType vst = scope_info->getScopeTypeOfName(name);
if (vst == ScopeInfo::VarScopeType::GLOBAL) {
if (jit)
jit->emitSetGlobal(globals, name.getBox(), value);
setGlobal(globals, name.getBox(), value.o);
} else if (vst == ScopeInfo::VarScopeType::NAME) {
if (jit)
jit->emitSetItemName(name.getBox(), value);
assert(frame_info.boxedLocals != NULL);
// TODO should probably pre-box the names when it's a scope that usesNameLookup
setitem(frame_info.boxedLocals, name.getBox(), value.o);
} else {
bool closure = vst == ScopeInfo::VarScopeType::CLOSURE;
if (jit) {
if (!closure) {
bool is_live = source_info->getLiveness()->isLiveAtEnd(name, current_block);
if (is_live)
jit->emitSetLocal(name, closure, value);
else
jit->emitSetBlockLocal(name, value);
} else
jit->emitSetLocal(name, closure, value);
}
sym_table[name] = value.o;
if (vst == ScopeInfo::VarScopeType::CLOSURE) {
if (closure) {
created_closure->elts[scope_info->getClosureOffset(name)] = value.o;
}
}
......@@ -429,199 +521,263 @@ void ASTInterpreter::doStore(AST_expr* node, Value value) {
doStore(name->id, value);
} else if (node->type == AST_TYPE::Attribute) {
AST_Attribute* attr = (AST_Attribute*)node;
pyston::setattr(visit_expr(attr->value).o, attr->attr.getBox(), value.o);
Value o = visit_expr(attr->value);
if (jit)
jit->emitSetAttr(o, attr->attr.getBox(), value);
pyston::setattr(o.o, attr->attr.getBox(), value.o);
} else if (node->type == AST_TYPE::Tuple) {
AST_Tuple* tuple = (AST_Tuple*)node;
Box** array = unpackIntoArray(value.o, tuple->elts.size());
RewriterVar* array_var = NULL;
if (jit)
array_var = jit->emitUnpackIntoArray(value, tuple->elts.size());
unsigned i = 0;
for (AST_expr* e : tuple->elts)
doStore(e, array[i++]);
for (AST_expr* e : tuple->elts) {
doStore(e, Value(array[i], jit ? array_var->getAttr(i * sizeof(void*)) : NULL));
++i;
}
} else if (node->type == AST_TYPE::List) {
AST_List* list = (AST_List*)node;
Box** array = unpackIntoArray(value.o, list->elts.size());
RewriterVar* array_var = NULL;
if (jit)
array_var = jit->emitUnpackIntoArray(value, list->elts.size());
unsigned i = 0;
for (AST_expr* e : list->elts)
doStore(e, array[i++]);
for (AST_expr* e : list->elts) {
doStore(e, Value(array[i], jit ? array_var->getAttr(i * sizeof(void*)) : NULL));
++i;
}
} else if (node->type == AST_TYPE::Subscript) {
AST_Subscript* subscript = (AST_Subscript*)node;
Value target = visit_expr(subscript->value);
Value slice = visit_expr(subscript->slice);
if (jit)
jit->emitSetItem(target, slice, value);
setitem(target.o, slice.o, value.o);
} else {
RELEASE_ASSERT(0, "not implemented");
}
}
Value ASTInterpreter::getNone() {
return Value(None, jit ? jit->imm(None) : NULL);
}
Value ASTInterpreter::visit_unaryop(AST_UnaryOp* node) {
Value operand = visit_expr(node->operand);
if (node->op_type == AST_TYPE::Not)
return boxBool(!nonzero(operand.o));
return Value(boxBool(!nonzero(operand.o)), jit ? jit->emitNotNonzero(operand) : NULL);
else
return unaryop(operand.o, node->op_type);
return Value(unaryop(operand.o, node->op_type), jit ? jit->emitUnaryop(operand, node->op_type) : NULL);
}
Value ASTInterpreter::visit_binop(AST_BinOp* node) {
Value left = visit_expr(node->left);
Value right = visit_expr(node->right);
return doBinOp(left.o, right.o, node->op_type, BinExpType::BinOp);
return doBinOp(left, right, node->op_type, BinExpType::BinOp);
}
Value ASTInterpreter::visit_slice(AST_Slice* node) {
Value lower = node->lower ? visit_expr(node->lower) : None;
Value upper = node->upper ? visit_expr(node->upper) : None;
Value step = node->step ? visit_expr(node->step) : None;
return createSlice(lower.o, upper.o, step.o);
Value lower = node->lower ? visit_expr(node->lower) : getNone();
Value upper = node->upper ? visit_expr(node->upper) : getNone();
Value step = node->step ? visit_expr(node->step) : getNone();
Value v;
if (jit)
v.var = jit->emitCreateSlice(lower, upper, step);
v.o = createSlice(lower.o, upper.o, step.o);
return v;
}
Value ASTInterpreter::visit_extslice(AST_ExtSlice* node) {
llvm::SmallVector<RewriterVar*, 8> items;
int num_slices = node->dims.size();
BoxedTuple* rtn = BoxedTuple::create(num_slices);
for (int i = 0; i < num_slices; ++i)
rtn->elts[i] = visit_expr(node->dims[i]).o;
return rtn;
for (int i = 0; i < num_slices; ++i) {
Value v = visit_expr(node->dims[i]);
rtn->elts[i] = v.o;
items.push_back(v);
}
return Value(rtn, jit ? jit->emitCreateTuple(items) : NULL);
}
Value ASTInterpreter::visit_branch(AST_Branch* node) {
Value v = visit_expr(node->test);
ASSERT(v.o == True || v.o == False, "Should have called NONZERO before this branch");
if (jit)
jit->emitSideExit(v, v.o, v.o == True ? node->iffalse : node->iftrue);
if (v.o == True)
next_block = node->iftrue;
else
next_block = node->iffalse;
if (jit) {
jit->emitJump(next_block);
finishJITing(next_block);
}
return Value();
}
Value ASTInterpreter::visit_jump(AST_Jump* node) {
bool backedge = node->target->idx < current_block->idx && compiled_func;
if (backedge)
if (backedge) {
threading::allowGLReadPreemption();
if (ENABLE_OSR && backedge && edgecount++ == OSR_THRESHOLD_INTERPRETER) {
bool can_osr = !FORCE_INTERPRETER && source_info->scoping->areGlobalsFromModule();
if (can_osr) {
static StatCounter ast_osrs("num_ast_osrs");
ast_osrs.log();
// TODO: we will immediately want the liveness info again in the jit, we should pass
// it through.
std::unique_ptr<LivenessAnalysis> liveness = computeLivenessInfo(source_info->cfg);
std::unique_ptr<PhiAnalysis> phis
= computeRequiredPhis(compiled_func->clfunc->param_names, source_info->cfg, liveness.get(), scope_info);
std::vector<InternedString> dead_symbols;
for (auto& it : sym_table) {
if (!liveness->isLiveAtEnd(it.first, current_block)) {
dead_symbols.push_back(it.first);
} else if (phis->isRequiredAfter(it.first, current_block)) {
assert(scope_info->getScopeTypeOfName(it.first) != ScopeInfo::VarScopeType::GLOBAL);
} else {
}
}
for (auto&& dead : dead_symbols)
sym_table.erase(dead);
if (jit)
jit->call(false, (void*)threading::allowGLReadPreemption);
}
const OSREntryDescriptor* found_entry = nullptr;
for (auto& p : compiled_func->clfunc->osr_versions) {
if (p.first->cf != compiled_func)
continue;
if (p.first->backedge != node)
continue;
if (jit) {
if (backedge)
jit->emitOSRPoint(node);
jit->emitJump(node->target);
finishJITing(node->target);
}
found_entry = p.first;
}
if (backedge)
++edgecount;
std::map<InternedString, Box*> sorted_symbol_table;
if (ENABLE_BASELINEJIT && backedge && edgecount == OSR_THRESHOLD_INTERPRETER && !jit && !node->target->code)
startJITing(node->target);
for (auto& name : phis->definedness.getDefinedNamesAtEnd(current_block)) {
auto it = sym_table.find(name);
if (!liveness->isLiveAtEnd(name, current_block))
continue;
if (backedge && edgecount == OSR_THRESHOLD_BASELINE) {
Box* rtn = doOSR(node);
if (rtn)
return Value(rtn, NULL);
}
if (phis->isPotentiallyUndefinedAfter(name, current_block)) {
bool is_defined = it != sym_table.end();
// TODO only mangle once
sorted_symbol_table[getIsDefinedName(name, source_info->getInternedStrings())] = (Box*)is_defined;
if (is_defined)
assert(sym_table.getMapped(it->second) != NULL);
sorted_symbol_table[name] = is_defined ? sym_table.getMapped(it->second) : NULL;
} else {
ASSERT(it != sym_table.end(), "%s", name.c_str());
sorted_symbol_table[it->first] = sym_table.getMapped(it->second);
}
}
next_block = node->target;
return Value();
}
// Manually free these here, since we might not return from this scope for a long time.
liveness.reset(nullptr);
phis.reset(nullptr);
// LLVM has a limit on the number of operands a machine instruction can have (~255),
// in order to not hit the limit with the patchpoints cancel OSR when we have a high number of symbols.
if (sorted_symbol_table.size() > 225) {
static StatCounter times_osr_cancel("num_osr_cancel_too_many_syms");
times_osr_cancel.log();
next_block = node->target;
return Value();
}
Box* ASTInterpreter::doOSR(AST_Jump* node) {
bool can_osr = ENABLE_OSR && !FORCE_INTERPRETER && source_info->scoping->areGlobalsFromModule();
if (!can_osr)
return NULL;
if (generator)
sorted_symbol_table[source_info->getInternedStrings().get(PASSED_GENERATOR_NAME)] = generator;
static StatCounter ast_osrs("num_ast_osrs");
ast_osrs.log();
if (passed_closure)
sorted_symbol_table[source_info->getInternedStrings().get(PASSED_CLOSURE_NAME)] = passed_closure;
LivenessAnalysis* liveness = source_info->getLiveness();
std::unique_ptr<PhiAnalysis> phis
= computeRequiredPhis(compiled_func->clfunc->param_names, source_info->cfg, liveness, scope_info);
if (created_closure)
sorted_symbol_table[source_info->getInternedStrings().get(CREATED_CLOSURE_NAME)] = created_closure;
std::vector<InternedString> dead_symbols;
for (auto& it : sym_table) {
if (!liveness->isLiveAtEnd(it.first, current_block)) {
dead_symbols.push_back(it.first);
} else if (phis->isRequiredAfter(it.first, current_block)) {
assert(scope_info->getScopeTypeOfName(it.first) != ScopeInfo::VarScopeType::GLOBAL);
} else {
}
}
for (auto&& dead : dead_symbols)
sym_table.erase(dead);
sorted_symbol_table[source_info->getInternedStrings().get(FRAME_INFO_PTR_NAME)] = (Box*)&frame_info;
const OSREntryDescriptor* found_entry = nullptr;
for (auto& p : compiled_func->clfunc->osr_versions) {
if (p.first->cf != compiled_func)
continue;
if (p.first->backedge != node)
continue;
if (found_entry == nullptr) {
OSREntryDescriptor* entry = OSREntryDescriptor::create(compiled_func, node);
found_entry = p.first;
}
for (auto& it : sorted_symbol_table) {
if (isIsDefinedName(it.first))
entry->args[it.first] = BOOL;
else if (it.first.s() == PASSED_GENERATOR_NAME)
entry->args[it.first] = GENERATOR;
else if (it.first.s() == PASSED_CLOSURE_NAME || it.first.s() == CREATED_CLOSURE_NAME)
entry->args[it.first] = CLOSURE;
else if (it.first.s() == FRAME_INFO_PTR_NAME)
entry->args[it.first] = FRAME_INFO;
else {
assert(it.first.s()[0] != '!');
entry->args[it.first] = UNKNOWN;
}
}
std::map<InternedString, Box*> sorted_symbol_table;
found_entry = entry;
}
for (auto& name : phis->definedness.getDefinedNamesAtEnd(current_block)) {
auto it = sym_table.find(name);
if (!liveness->isLiveAtEnd(name, current_block))
continue;
if (phis->isPotentiallyUndefinedAfter(name, current_block)) {
bool is_defined = it != sym_table.end();
// TODO only mangle once
sorted_symbol_table[getIsDefinedName(name, source_info->getInternedStrings())] = (Box*)is_defined;
if (is_defined)
assert(sym_table.getMapped(it->second) != NULL);
sorted_symbol_table[name] = is_defined ? sym_table.getMapped(it->second) : NULL;
} else {
ASSERT(it != sym_table.end(), "%s", name.c_str());
sorted_symbol_table[it->first] = sym_table.getMapped(it->second);
}
}
OSRExit exit(compiled_func, found_entry);
// Manually free these here, since we might not return from this scope for a long time.
phis.reset(nullptr);
std::vector<Box*, StlCompatAllocator<Box*>> arg_array;
for (auto& it : sorted_symbol_table) {
arg_array.push_back(it.second);
}
// LLVM has a limit on the number of operands a machine instruction can have (~255),
// in order to not hit the limit with the patchpoints cancel OSR when we have a high number of symbols.
if (sorted_symbol_table.size() > 225) {
static StatCounter times_osr_cancel("num_osr_cancel_too_many_syms");
times_osr_cancel.log();
return nullptr;
}
if (generator)
sorted_symbol_table[source_info->getInternedStrings().get(PASSED_GENERATOR_NAME)] = generator;
if (passed_closure)
sorted_symbol_table[source_info->getInternedStrings().get(PASSED_CLOSURE_NAME)] = passed_closure;
if (created_closure)
sorted_symbol_table[source_info->getInternedStrings().get(CREATED_CLOSURE_NAME)] = created_closure;
UNAVOIDABLE_STAT_TIMER(t0, "us_timer_in_jitted_code");
CompiledFunction* partial_func = compilePartialFuncInternal(&exit);
auto arg_tuple = getTupleFromArgsArray(&arg_array[0], arg_array.size());
Box* r = partial_func->call(std::get<0>(arg_tuple), std::get<1>(arg_tuple), std::get<2>(arg_tuple),
std::get<3>(arg_tuple));
// This is one of the few times that we are allowed to have an invalid value in a Box* Value.
// Check for it, and return as an int so that we don't trigger a potential assert when
// creating the Value.
if (compiled_func->getReturnType() != VOID)
assert(r);
return (intptr_t)r;
sorted_symbol_table[source_info->getInternedStrings().get(FRAME_INFO_PTR_NAME)] = (Box*)&frame_info;
if (found_entry == nullptr) {
OSREntryDescriptor* entry = OSREntryDescriptor::create(compiled_func, node);
for (auto& it : sorted_symbol_table) {
if (isIsDefinedName(it.first))
entry->args[it.first] = BOOL;
else if (it.first.s() == PASSED_GENERATOR_NAME)
entry->args[it.first] = GENERATOR;
else if (it.first.s() == PASSED_CLOSURE_NAME || it.first.s() == CREATED_CLOSURE_NAME)
entry->args[it.first] = CLOSURE;
else if (it.first.s() == FRAME_INFO_PTR_NAME)
entry->args[it.first] = FRAME_INFO;
else {
assert(it.first.s()[0] != '!');
entry->args[it.first] = UNKNOWN;
}
}
found_entry = entry;
}
next_block = node->target;
return Value();
OSRExit exit(compiled_func, found_entry);
std::vector<Box*, StlCompatAllocator<Box*>> arg_array;
for (auto& it : sorted_symbol_table) {
arg_array.push_back(it.second);
}
UNAVOIDABLE_STAT_TIMER(t0, "us_timer_in_jitted_code");
CompiledFunction* partial_func = compilePartialFuncInternal(&exit);
auto arg_tuple = getTupleFromArgsArray(&arg_array[0], arg_array.size());
Box* r = partial_func->call(std::get<0>(arg_tuple), std::get<1>(arg_tuple), std::get<2>(arg_tuple),
std::get<3>(arg_tuple));
// This is one of the few times that we are allowed to have an invalid value in a Box* Value.
// Check for it, and return as an int so that we don't trigger a potential assert when
// creating the Value.
if (compiled_func->getReturnType() != VOID)
assert(r);
return r ? r : None;
}
Value ASTInterpreter::visit_invoke(AST_Invoke* node) {
......@@ -629,7 +785,13 @@ Value ASTInterpreter::visit_invoke(AST_Invoke* node) {
try {
v = visit_stmt(node->stmt);
next_block = node->normal_dest;
if (jit) {
jit->emitJump(next_block);
finishJITing(next_block);
}
} catch (ExcInfo e) {
abortJITing();
auto source = getCF()->clfunc->source.get();
exceptionCaughtInInterpreter(LineInfo(node->lineno, node->col_offset, source->fn, source->getName()), &e);
......@@ -642,7 +804,9 @@ Value ASTInterpreter::visit_invoke(AST_Invoke* node) {
}
Value ASTInterpreter::visit_clsAttribute(AST_ClsAttribute* node) {
return getclsattr(visit_expr(node->value).o, node->attr.getBox());
Value obj = visit_expr(node->value);
BoxedString* attr = node->attr.getBox();
return Value(getclsattr(obj.o, attr), jit ? jit->emitGetClsAttr(obj, attr) : NULL);
}
Value ASTInterpreter::visit_augBinOp(AST_AugBinOp* node) {
......@@ -650,15 +814,17 @@ Value ASTInterpreter::visit_augBinOp(AST_AugBinOp* node) {
Value left = visit_expr(node->left);
Value right = visit_expr(node->right);
return doBinOp(left.o, right.o, node->op_type, BinExpType::AugBinOp);
return doBinOp(left, right, node->op_type, BinExpType::AugBinOp);
}
Value ASTInterpreter::visit_langPrimitive(AST_LangPrimitive* node) {
Value v;
if (node->opcode == AST_LangPrimitive::GET_ITER) {
assert(node->args.size() == 1);
v = getPystonIter(visit_expr(node->args[0]).o);
Value val = visit_expr(node->args[0]);
v = Value(getPystonIter(val.o), jit ? jit->emitGetPystonIter(val) : NULL);
} else if (node->opcode == AST_LangPrimitive::IMPORT_FROM) {
abortJITing();
assert(node->args.size() == 2);
assert(node->args[0]->type == AST_TYPE::Name);
assert(node->args[1]->type == AST_TYPE::Str);
......@@ -669,8 +835,9 @@ Value ASTInterpreter::visit_langPrimitive(AST_LangPrimitive* node) {
const std::string& name = ast_str->str_data;
assert(name.size());
// TODO: shouldn't have to rebox here
v = importFrom(module.o, boxString(name));
v.o = importFrom(module.o, boxString(name));
} else if (node->opcode == AST_LangPrimitive::IMPORT_NAME) {
abortJITing();
assert(node->args.size() == 3);
assert(node->args[0]->type == AST_TYPE::Num);
assert(static_cast<AST_Num*>(node->args[0])->num_type == AST_Num::INT);
......@@ -681,8 +848,9 @@ Value ASTInterpreter::visit_langPrimitive(AST_LangPrimitive* node) {
auto ast_str = ast_cast<AST_Str>(node->args[2]);
assert(ast_str->str_type == AST_Str::STR);
const std::string& module_name = ast_str->str_data;
v = import(level, froms.o, module_name);
v.o = import(level, froms.o, module_name);
} else if (node->opcode == AST_LangPrimitive::IMPORT_STAR) {
abortJITing();
assert(node->args.size() == 1);
assert(node->args[0]->type == AST_TYPE::Name);
......@@ -691,30 +859,28 @@ Value ASTInterpreter::visit_langPrimitive(AST_LangPrimitive* node) {
Value module = visit_expr(node->args[0]);
v = importStar(module.o, globals);
v.o = importStar(module.o, globals);
} else if (node->opcode == AST_LangPrimitive::NONE) {
v = None;
v = getNone();
} else if (node->opcode == AST_LangPrimitive::LANDINGPAD) {
assert(last_exception.type);
Box* type = last_exception.type;
Box* value = last_exception.value ? last_exception.value : None;
Box* traceback = last_exception.traceback ? last_exception.traceback : None;
v = BoxedTuple::create({ type, value, traceback });
v = Value(BoxedTuple::create({ type, value, traceback }), jit ? jit->emitLandingpad() : NULL);
last_exception = ExcInfo(NULL, NULL, NULL);
} else if (node->opcode == AST_LangPrimitive::CHECK_EXC_MATCH) {
assert(node->args.size() == 2);
Value obj = visit_expr(node->args[0]);
Value cls = visit_expr(node->args[1]);
v = boxBool(exceptionMatches(obj.o, cls.o));
v = Value(boxBool(exceptionMatches(obj.o, cls.o)), jit ? jit->emitExceptionMatches(obj, cls) : NULL);
} else if (node->opcode == AST_LangPrimitive::LOCALS) {
assert(frame_info.boxedLocals != NULL);
v = frame_info.boxedLocals;
v = Value(frame_info.boxedLocals, jit ? jit->emitGetBoxedLocals() : NULL);
} else if (node->opcode == AST_LangPrimitive::NONZERO) {
assert(node->args.size() == 1);
Value obj = visit_expr(node->args[0]);
v = boxBool(nonzero(obj.o));
v = Value(boxBool(nonzero(obj.o)), jit ? jit->emitNonzero(obj) : NULL);
} else if (node->opcode == AST_LangPrimitive::SET_EXC_INFO) {
assert(node->args.size() == 3);
......@@ -725,25 +891,30 @@ Value ASTInterpreter::visit_langPrimitive(AST_LangPrimitive* node) {
Value traceback = visit_expr(node->args[2]);
assert(traceback.o);
if (jit)
jit->emitSetExcInfo(type, value, traceback);
getFrameInfo()->exc = ExcInfo(type.o, value.o, traceback.o);
v = None;
v = getNone();
} else if (node->opcode == AST_LangPrimitive::UNCACHE_EXC_INFO) {
assert(node->args.empty());
if (jit)
jit->emitUncacheExcInfo();
getFrameInfo()->exc = ExcInfo(NULL, NULL, NULL);
v = None;
v = getNone();
} else if (node->opcode == AST_LangPrimitive::HASNEXT) {
assert(node->args.size() == 1);
Value obj = visit_expr(node->args[0]);
v = boxBool(hasnext(obj.o));
v = Value(boxBool(hasnext(obj.o)), jit ? jit->emitHasnext(obj) : NULL);
} else
RELEASE_ASSERT(0, "unknown opcode %d", node->opcode);
return v;
}
Value ASTInterpreter::visit_yield(AST_Yield* node) {
Value value = node->value ? visit_expr(node->value) : None;
Value value = node->value ? visit_expr(node->value) : getNone();
assert(generator && generator->cls == generator_cls);
return yield(generator, value.o);
return Value(yield(generator, value.o), jit ? jit->emitYield(value) : NULL);
}
Value ASTInterpreter::visit_stmt(AST_stmt* node) {
......@@ -797,12 +968,19 @@ Value ASTInterpreter::visit_stmt(AST_stmt* node) {
}
Value ASTInterpreter::visit_return(AST_Return* node) {
Value s(node->value ? visit_expr(node->value) : None);
Value s = node->value ? visit_expr(node->value) : getNone();
if (jit) {
jit->emitReturn(s);
finishJITing();
}
next_block = 0;
return s;
}
Box* ASTInterpreter::createFunction(AST* node, AST_arguments* args, const std::vector<AST_stmt*>& body) {
abortJITing();
CLFunction* cl = wrapFunction(node, args, body, source_info);
std::vector<Box*, StlCompatAllocator<Box*>> defaults;
......@@ -852,6 +1030,7 @@ Box* ASTInterpreter::createFunction(AST* node, AST_arguments* args, const std::v
}
Value ASTInterpreter::visit_makeFunction(AST_MakeFunction* mkfn) {
abortJITing();
AST_FunctionDef* node = mkfn->function_def;
AST_arguments* args = node->args;
......@@ -864,10 +1043,11 @@ Value ASTInterpreter::visit_makeFunction(AST_MakeFunction* mkfn) {
for (int i = decorators.size() - 1; i >= 0; i--)
func = runtimeCall(decorators[i], ArgPassSpec(1), func, 0, 0, 0, 0);
return Value(func);
return Value(func, NULL);
}
Value ASTInterpreter::visit_makeClass(AST_MakeClass* mkclass) {
abortJITing();
AST_ClassDef* node = mkclass->class_def;
ScopeInfo* scope_info = source_info->scoping->getScopeInfoForNode(node);
assert(scope_info);
......@@ -902,22 +1082,37 @@ Value ASTInterpreter::visit_makeClass(AST_MakeClass* mkclass) {
for (int i = decorators.size() - 1; i >= 0; i--)
classobj = runtimeCall(decorators[i], ArgPassSpec(1), classobj, 0, 0, 0, 0);
return Value(classobj);
return Value(classobj, NULL);
}
Value ASTInterpreter::visit_raise(AST_Raise* node) {
if (node->arg0 == NULL) {
assert(!node->arg1);
assert(!node->arg2);
if (jit) {
jit->emitRaise0();
finishJITing();
}
raise0();
}
raise3(node->arg0 ? visit_expr(node->arg0).o : None, node->arg1 ? visit_expr(node->arg1).o : None,
node->arg2 ? visit_expr(node->arg2).o : None);
Value arg0 = node->arg0 ? visit_expr(node->arg0) : getNone();
Value arg1 = node->arg1 ? visit_expr(node->arg1) : getNone();
Value arg2 = node->arg2 ? visit_expr(node->arg2) : getNone();
if (jit) {
jit->emitRaise3(arg0, arg1, arg2);
finishJITing();
}
raise3(arg0.o, arg1.o, arg2.o);
return Value();
}
Value ASTInterpreter::visit_assert(AST_Assert* node) {
abortJITing();
#ifndef NDEBUG
// Currently we only generate "assert 0" statements
Value v = visit_expr(node->test);
......@@ -932,12 +1127,14 @@ Value ASTInterpreter::visit_assert(AST_Assert* node) {
}
Value ASTInterpreter::visit_global(AST_Global* node) {
abortJITing();
for (auto name : node->names)
sym_table.erase(name);
return Value();
}
Value ASTInterpreter::visit_delete(AST_Delete* node) {
abortJITing();
for (AST_expr* target_ : node->targets) {
switch (target_->type) {
case AST_TYPE::Subscript: {
......@@ -1003,49 +1200,39 @@ Value ASTInterpreter::visit_assign(AST_Assign* node) {
}
Value ASTInterpreter::visit_print(AST_Print* node) {
static BoxedString* write_str = static_cast<BoxedString*>(PyString_InternFromString("write"));
static BoxedString* newline_str = static_cast<BoxedString*>(PyString_InternFromString("\n"));
static BoxedString* space_str = static_cast<BoxedString*>(PyString_InternFromString(" "));
Box* dest = node->dest ? visit_expr(node->dest).o : getSysStdout();
int nvals = node->values.size();
assert(nvals <= 1 && "cfg should have lowered it to 0 or 1 values");
for (int i = 0; i < nvals; i++) {
Box* var = visit_expr(node->values[i]).o;
// begin code for handling of softspace
bool new_softspace = (i < nvals - 1) || (!node->nl);
if (softspace(dest, new_softspace)) {
callattrInternal(dest, write_str, CLASS_OR_INST, 0, ArgPassSpec(1), space_str, 0, 0, 0, 0);
}
assert(node->values.size() <= 1 && "cfg should have lowered it to 0 or 1 values");
Value dest = node->dest ? visit_expr(node->dest) : Value();
Value var = node->values.size() ? visit_expr(node->values[0]) : Value();
Box* str_or_unicode_var = (var->cls == unicode_cls) ? var : str(var);
callattrInternal(dest, write_str, CLASS_OR_INST, 0, ArgPassSpec(1), str_or_unicode_var, 0, 0, 0, 0);
}
if (jit)
jit->emitPrint(dest, var, node->nl);
if (node->dest)
printHelper(dest.o, var.o, node->nl);
else
printHelper(getSysStdout(), var.o, node->nl);
if (node->nl) {
callattrInternal(dest, write_str, CLASS_OR_INST, 0, ArgPassSpec(1), newline_str, 0, 0, 0, 0);
if (nvals == 0) {
softspace(dest, false);
}
}
return Value();
}
Value ASTInterpreter::visit_exec(AST_Exec* node) {
// TODO implement the locals and globals arguments
Box* code = visit_expr(node->body).o;
Box* globals = node->globals == NULL ? NULL : visit_expr(node->globals).o;
Box* locals = node->locals == NULL ? NULL : visit_expr(node->locals).o;
Value code = visit_expr(node->body);
Value globals = node->globals == NULL ? Value() : visit_expr(node->globals);
Value locals = node->locals == NULL ? Value() : visit_expr(node->locals);
exec(code, globals, locals, this->source_info->future_flags);
if (jit)
jit->emitExec(code, globals, locals, this->source_info->future_flags);
exec(code.o, globals.o, locals.o, this->source_info->future_flags);
return Value();
}
Value ASTInterpreter::visit_compare(AST_Compare* node) {
RELEASE_ASSERT(node->comparators.size() == 1, "not implemented");
return doBinOp(visit_expr(node->left).o, visit_expr(node->comparators[0]).o, node->ops[0], BinExpType::Compare);
Value left = visit_expr(node->left);
Value right = visit_expr(node->comparators[0]);
return doBinOp(left, right, node->ops[0], BinExpType::Compare);
}
Value ASTInterpreter::visit_expr(AST_expr* node) {
......@@ -1132,31 +1319,56 @@ Value ASTInterpreter::visit_call(AST_Call* node) {
}
std::vector<Box*, StlCompatAllocator<Box*>> args;
for (AST_expr* e : node->args)
args.push_back(visit_expr(e).o);
llvm::SmallVector<RewriterVar*, 8> args_vars;
for (AST_expr* e : node->args) {
Value v = visit_expr(e);
args.push_back(v.o);
args_vars.push_back(v);
}
std::vector<BoxedString*>* keyword_names = NULL;
if (node->keywords.size())
keyword_names = getKeywordNameStorage(node);
std::vector<BoxedString*> keywords;
for (AST_keyword* k : node->keywords) {
keywords.push_back(k->arg.getBox());
args.push_back(visit_expr(k->value).o);
Value v = visit_expr(k->value);
args.push_back(v.o);
args_vars.push_back(v);
}
if (node->starargs)
args.push_back(visit_expr(node->starargs).o);
if (node->starargs) {
Value v = visit_expr(node->starargs);
args.push_back(v.o);
args_vars.push_back(v);
}
if (node->kwargs)
args.push_back(visit_expr(node->kwargs).o);
if (node->kwargs) {
Value v = visit_expr(node->kwargs);
args.push_back(v.o);
args_vars.push_back(v);
}
ArgPassSpec argspec(node->args.size(), node->keywords.size(), node->starargs, node->kwargs);
if (is_callattr) {
CallattrFlags callattr_flags{.cls_only = callattr_clsonly, .null_on_nonexistent = false, .argspec = argspec };
return callattr(func.o, attr.getBox(), callattr_flags, args.size() > 0 ? args[0] : 0,
args.size() > 1 ? args[1] : 0, args.size() > 2 ? args[2] : 0, args.size() > 3 ? &args[3] : 0,
&keywords);
if (jit)
v.var = jit->emitCallattr(func, attr.getBox(), callattr_flags, args_vars, keyword_names);
v.o = callattr(func.o, attr.getBox(), callattr_flags, args.size() > 0 ? args[0] : 0,
args.size() > 1 ? args[1] : 0, args.size() > 2 ? args[2] : 0, args.size() > 3 ? &args[3] : 0,
keyword_names);
return v;
} else {
return runtimeCall(func.o, argspec, args.size() > 0 ? args[0] : 0, args.size() > 1 ? args[1] : 0,
args.size() > 2 ? args[2] : 0, args.size() > 3 ? &args[3] : 0, &keywords);
Value v;
if (jit)
v.var = jit->emitRuntimeCall(func, argspec, args_vars, keyword_names);
v.o = runtimeCall(func.o, argspec, args.size() > 0 ? args[0] : 0, args.size() > 1 ? args[1] : 0,
args.size() > 2 ? args[2] : 0, args.size() > 3 ? &args[3] : 0, keyword_names);
return v;
}
}
......@@ -1166,16 +1378,18 @@ Value ASTInterpreter::visit_expr(AST_Expr* node) {
}
Value ASTInterpreter::visit_num(AST_Num* node) {
if (node->num_type == AST_Num::INT)
return boxInt(node->n_int);
else if (node->num_type == AST_Num::FLOAT)
return boxFloat(node->n_float);
else if (node->num_type == AST_Num::LONG)
return createLong(node->n_long);
else if (node->num_type == AST_Num::COMPLEX)
return boxComplex(0.0, node->n_float);
RELEASE_ASSERT(0, "not implemented");
return Value();
Box* o = NULL;
if (node->num_type == AST_Num::INT) {
o = source_info->parent_module->getIntConstant(node->n_int);
} else if (node->num_type == AST_Num::FLOAT) {
o = source_info->parent_module->getFloatConstant(node->n_float);
} else if (node->num_type == AST_Num::LONG) {
o = source_info->parent_module->getLongConstant(node->n_long);
} else if (node->num_type == AST_Num::COMPLEX) {
o = source_info->parent_module->getPureImaginaryConstant(node->n_float);
} else
RELEASE_ASSERT(0, "not implemented");
return Value(o, jit ? jit->imm(o) : NULL);
}
Value ASTInterpreter::visit_index(AST_Index* node) {
......@@ -1183,45 +1397,61 @@ Value ASTInterpreter::visit_index(AST_Index* node) {
}
Value ASTInterpreter::visit_repr(AST_Repr* node) {
return repr(visit_expr(node->value).o);
Value v = visit_expr(node->value);
return Value(repr(v.o), jit ? jit->emitRepr(v) : NULL);
}
Value ASTInterpreter::visit_lambda(AST_Lambda* node) {
abortJITing();
AST_Return* expr = new AST_Return();
expr->value = node->body;
std::vector<AST_stmt*> body = { expr };
return createFunction(node, node->args, body);
return Value(createFunction(node, node->args, body), NULL);
}
Value ASTInterpreter::visit_dict(AST_Dict* node) {
RELEASE_ASSERT(node->keys.size() == node->values.size(), "not implemented");
llvm::SmallVector<RewriterVar*, 8> keys;
llvm::SmallVector<RewriterVar*, 8> values;
BoxedDict* dict = new BoxedDict();
for (size_t i = 0; i < node->keys.size(); ++i) {
Box* v = visit_expr(node->values[i]).o;
Box* k = visit_expr(node->keys[i]).o;
dict->d[k] = v;
Value v = visit_expr(node->values[i]);
Value k = visit_expr(node->keys[i]);
dict->d[k.o] = v.o;
values.push_back(v);
keys.push_back(k);
}
return dict;
return Value(dict, jit ? jit->emitCreateDict(keys, values) : NULL);
}
Value ASTInterpreter::visit_set(AST_Set* node) {
llvm::SmallVector<RewriterVar*, 8> items;
BoxedSet::Set set;
for (AST_expr* e : node->elts)
set.insert(visit_expr(e).o);
for (AST_expr* e : node->elts) {
Value v = visit_expr(e);
set.insert(v.o);
items.push_back(v);
}
return new BoxedSet(std::move(set));
return Value(new BoxedSet(std::move(set)), jit ? jit->emitCreateSet(items) : NULL);
}
Value ASTInterpreter::visit_str(AST_Str* node) {
Box* o = NULL;
if (node->str_type == AST_Str::STR) {
return source_info->parent_module->getStringConstant(node->str_data);
o = source_info->parent_module->getStringConstant(node->str_data);
} else if (node->str_type == AST_Str::UNICODE) {
return decodeUTF8StringPtr(node->str_data);
o = source_info->parent_module->getUnicodeConstant(node->str_data);
} else {
RELEASE_ASSERT(0, "%d", node->str_type);
}
return Value(o, jit ? jit->imm(o) : NULL);
}
Value ASTInterpreter::visit_name(AST_Name* node) {
......@@ -1230,67 +1460,195 @@ Value ASTInterpreter::visit_name(AST_Name* node) {
}
switch (node->lookup_type) {
case ScopeInfo::VarScopeType::GLOBAL:
return getGlobal(globals, node->id.getBox());
case ScopeInfo::VarScopeType::GLOBAL: {
Value v;
if (jit)
v.var = jit->emitGetGlobal(globals, node->id.getBox());
v.o = getGlobal(globals, node->id.getBox());
return v;
}
case ScopeInfo::VarScopeType::DEREF: {
DerefInfo deref_info = scope_info->getDerefInfo(node->id);
assert(passed_closure);
BoxedClosure* closure = passed_closure;
for (int i = 0; i < deref_info.num_parents_from_passed_closure; i++) {
closure = closure->parent;
}
Box* val = closure->elts[deref_info.offset];
if (val == NULL) {
raiseExcHelper(NameError, "free variable '%s' referenced before assignment in enclosing scope",
node->id.c_str());
}
return val;
return Value(ASTInterpreterJitInterface::derefHelper(this, node->id),
jit ? jit->emitDeref(node->id) : NULL);
}
case ScopeInfo::VarScopeType::FAST:
case ScopeInfo::VarScopeType::CLOSURE: {
SymMap::iterator it = sym_table.find(node->id);
if (it != sym_table.end())
return sym_table.getMapped(it->second);
Value v;
if (jit) {
bool is_live = false;
if (node->lookup_type == ScopeInfo::VarScopeType::FAST)
is_live = source_info->getLiveness()->isLiveAtEnd(node->id, current_block);
if (is_live)
v.var = jit->emitGetLocal(node->id);
else
v.var = jit->emitGetBlockLocal(node->id);
}
assertNameDefined(0, node->id.c_str(), UnboundLocalError, true);
return Value();
v.o = ASTInterpreterJitInterface::getLocalHelper(this, node->id);
return v;
}
case ScopeInfo::VarScopeType::NAME: {
return boxedLocalsGet(frame_info.boxedLocals, node->id.getBox(), globals);
Value v;
if (jit)
v.var = jit->emitGetBoxedLocal(node->id.getBox());
v.o = boxedLocalsGet(frame_info.boxedLocals, node->id.getBox(), globals);
return v;
}
default:
abort();
}
}
Value ASTInterpreter::visit_subscript(AST_Subscript* node) {
Value value = visit_expr(node->value);
Value slice = visit_expr(node->slice);
return getitem(value.o, slice.o);
return Value(getitem(value.o, slice.o), jit ? jit->emitGetItem(value, slice) : NULL);
}
Value ASTInterpreter::visit_list(AST_List* node) {
llvm::SmallVector<RewriterVar*, 8> items;
BoxedList* list = new BoxedList;
list->ensure(node->elts.size());
for (AST_expr* e : node->elts)
listAppendInternal(list, visit_expr(e).o);
return list;
for (AST_expr* e : node->elts) {
Value v = visit_expr(e);
items.push_back(v);
listAppendInternal(list, v.o);
}
return Value(list, jit ? jit->emitCreateList(items) : NULL);
}
Value ASTInterpreter::visit_tuple(AST_Tuple* node) {
llvm::SmallVector<RewriterVar*, 8> items;
BoxedTuple* rtn = BoxedTuple::create(node->elts.size());
int rtn_idx = 0;
for (AST_expr* e : node->elts)
rtn->elts[rtn_idx++] = visit_expr(e).o;
return rtn;
for (AST_expr* e : node->elts) {
Value v = visit_expr(e);
rtn->elts[rtn_idx++] = v.o;
items.push_back(v);
}
return Value(rtn, jit ? jit->emitCreateTuple(items) : NULL);
}
Value ASTInterpreter::visit_attribute(AST_Attribute* node) {
return pyston::getattr(visit_expr(node->value).o, node->attr.getBox());
Value v = visit_expr(node->value);
return Value(pyston::getattr(v.o, node->attr.getBox()), jit ? jit->emitGetAttr(v, node->attr.getBox()) : NULL);
}
}
int ASTInterpreterJitInterface::getCurrentBlockOffset() {
return offsetof(ASTInterpreter, current_block);
}
int ASTInterpreterJitInterface::getCurrentInstOffset() {
return offsetof(ASTInterpreter, current_inst);
}
Box* ASTInterpreterJitInterface::derefHelper(void* _interpreter, InternedString s) {
ASTInterpreter* interpreter = (ASTInterpreter*)_interpreter;
DerefInfo deref_info = interpreter->scope_info->getDerefInfo(s);
assert(interpreter->passed_closure);
BoxedClosure* closure = interpreter->passed_closure;
for (int i = 0; i < deref_info.num_parents_from_passed_closure; i++) {
closure = closure->parent;
}
Box* val = closure->elts[deref_info.offset];
if (val == NULL) {
raiseExcHelper(NameError, "free variable '%s' referenced before assignment in enclosing scope", s.c_str());
}
return val;
}
Box* ASTInterpreterJitInterface::doOSRHelper(void* _interpreter, AST_Jump* node) {
ASTInterpreter* interpreter = (ASTInterpreter*)_interpreter;
++interpreter->edgecount;
if (interpreter->edgecount >= OSR_THRESHOLD_BASELINE)
return interpreter->doOSR(node);
return NULL;
}
Box* ASTInterpreterJitInterface::getBoxedLocalHelper(void* _interpreter, BoxedString* s) {
ASTInterpreter* interpreter = (ASTInterpreter*)_interpreter;
return boxedLocalsGet(interpreter->frame_info.boxedLocals, s, interpreter->globals);
}
Box* ASTInterpreterJitInterface::getBoxedLocalsHelper(void* _interpreter) {
ASTInterpreter* interpreter = (ASTInterpreter*)_interpreter;
return interpreter->frame_info.boxedLocals;
}
Box* ASTInterpreterJitInterface::getLocalHelper(void* _interpreter, InternedString id) {
ASTInterpreter* interpreter = (ASTInterpreter*)_interpreter;
auto it = interpreter->sym_table.find(id);
if (it != interpreter->sym_table.end()) {
Box* v = interpreter->sym_table.getMapped(it->second);
assert(gc::isValidGCObject(v));
return v;
}
assertNameDefined(0, id.c_str(), UnboundLocalError, true);
return 0;
}
Box* ASTInterpreterJitInterface::landingpadHelper(void* _interpreter) {
ASTInterpreter* interpreter = (ASTInterpreter*)_interpreter;
auto& last_exception = interpreter->last_exception;
Box* type = last_exception.type;
Box* value = last_exception.value ? last_exception.value : None;
Box* traceback = last_exception.traceback ? last_exception.traceback : None;
Box* rtn = BoxedTuple::create({ type, value, traceback });
last_exception = ExcInfo(NULL, NULL, NULL);
return rtn;
}
Box* ASTInterpreterJitInterface::setExcInfoHelper(void* _interpreter, Box* type, Box* value, Box* traceback) {
ASTInterpreter* interpreter = (ASTInterpreter*)_interpreter;
interpreter->getFrameInfo()->exc = ExcInfo(type, value, traceback);
return None;
}
Box* ASTInterpreterJitInterface::uncacheExcInfoHelper(void* _interpreter) {
ASTInterpreter* interpreter = (ASTInterpreter*)_interpreter;
interpreter->getFrameInfo()->exc = ExcInfo(NULL, NULL, NULL);
return None;
}
Box* ASTInterpreterJitInterface::yieldHelper(void* _interpreter, Box* val) {
ASTInterpreter* interpreter = (ASTInterpreter*)_interpreter;
return yield(interpreter->generator, val);
}
void ASTInterpreterJitInterface::setItemNameHelper(void* _interpreter, Box* str, Box* val) {
ASTInterpreter* interpreter = (ASTInterpreter*)_interpreter;
assert(interpreter->frame_info.boxedLocals != NULL);
setitem(interpreter->frame_info.boxedLocals, str, val);
}
void ASTInterpreterJitInterface::setLocalClosureHelper(void* _interpreter, InternedString id, Box* v) {
ASTInterpreter* interpreter = (ASTInterpreter*)_interpreter;
assert(gc::isValidGCObject(v));
interpreter->sym_table[id] = v;
interpreter->created_closure->elts[interpreter->scope_info->getClosureOffset(id)] = v;
}
void ASTInterpreterJitInterface::setLocalHelper(void* _interpreter, InternedString id, Box* v) {
ASTInterpreter* interpreter = (ASTInterpreter*)_interpreter;
assert(gc::isValidGCObject(v));
interpreter->sym_table[id] = v;
}
const void* interpreter_instr_addr = (void*)&ASTInterpreter::executeInner;
Box* astInterpretFunction(CompiledFunction* cf, int nargs, Box* closure, Box* generator, Box* globals, Box* arg1,
......@@ -1299,7 +1657,9 @@ Box* astInterpretFunction(CompiledFunction* cf, int nargs, Box* closure, Box* ge
assert((!globals) == cf->clfunc->source->scoping->areGlobalsFromModule());
bool can_reopt = ENABLE_REOPT && !FORCE_INTERPRETER && (globals == NULL);
if (unlikely(can_reopt && cf->times_called > REOPT_THRESHOLD_INTERPRETER)) {
int num_blocks = cf->clfunc->source->cfg->blocks.size();
int threshold = num_blocks <= 20 ? (REOPT_THRESHOLD_BASELINE / 3) : REOPT_THRESHOLD_BASELINE;
if (unlikely(can_reopt && cf->times_called > threshold)) {
assert(!globals);
CompiledFunction* optimized = reoptCompiledFuncInternal(cf);
if (closure && generator)
......
src/codegen/ast_interpreter.h
View file @ fc366564
......@@ -25,6 +25,7 @@ class GCVisitor;
class AST_expr;
class AST_stmt;
class AST_Jump;
class Box;
class BoxedClosure;
class BoxedDict;
......@@ -33,6 +34,43 @@ struct LineInfo;
extern const void* interpreter_instr_addr;
struct ASTInterpreterJitInterface {
static int getCurrentBlockOffset();
static int getCurrentInstOffset();
static Box* derefHelper(void* interp, InternedString s);
static Box* doOSRHelper(void* interp, AST_Jump* node);
static Box* getBoxedLocalHelper(void* interp, BoxedString* s);
static Box* getBoxedLocalsHelper(void* interp);
static Box* getLocalHelper(void* interp, InternedString id);
static Box* landingpadHelper(void* interp);
static Box* setExcInfoHelper(void* interp, Box* type, Box* value, Box* traceback);
static Box* uncacheExcInfoHelper(void* interp);
static Box* yieldHelper(void* interp, Box* val);
static void setItemNameHelper(void* interp, Box* str, Box* val);
static void setLocalClosureHelper(void* interp, InternedString id, Box* v);
static void setLocalHelper(void* interp, InternedString id, Box* v);
};
class RewriterVar;
struct Value {
union {
bool b;
int64_t n;
double d;
Box* o;
};
RewriterVar* var;
operator RewriterVar*() { return var; }
Value() : o(0), var(0) {}
Value(bool b, RewriterVar* var) : b(b), var(var) {}
Value(int64_t n, RewriterVar* var) : n(n), var(var) {}
Value(double d, RewriterVar* var) : d(d), var(var) {}
Value(Box* o, RewriterVar* var) : o(o), var(var) {}
};
void setupInterpreter();
Box* astInterpretFunction(CompiledFunction* f, int nargs, Box* closure, Box* generator, Box* globals, Box* arg1,
Box* arg2, Box* arg3, Box** args);
......
src/codegen/baseline_jit.cpp 0 → 100644
View file @ fc366564
// Copyright (c) 2014-2015 Dropbox, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "codegen/baseline_jit.h"
#include <llvm/ADT/DenseMap.h>
#include <llvm/ADT/DenseSet.h>
#include "codegen/irgen/hooks.h"
#include "codegen/memmgr.h"
#include "core/cfg.h"
#include "runtime/objmodel.h"
#include "runtime/set.h"
#include "runtime/types.h"
namespace pyston {
static llvm::DenseSet<CFGBlock*> blocks_aborted;
JitCodeBlock::JitCodeBlock(llvm::StringRef name)
: frame_manager(false /* don't omit frame pointers */),
code(new uint8_t[code_size]),
entry_offset(0),
epilog_offset(0),
a(code.get(), code_size - epilog_size),
is_currently_writing(false),
asm_failed(false) {
static StatCounter num_jit_code_blocks("num_baselinejit_code_blocks");
num_jit_code_blocks.log();
static StatCounter num_jit_total_bytes("num_baselinejit_total_bytes");
num_jit_total_bytes.log(code_size);
// emit prolog
a.push(assembler::RBP);
a.mov(assembler::RSP, assembler::RBP);
static_assert(scratch_size % 16 == 0, "stack aligment code depends on this");
// subtract scratch size + 8bytes to align stack after the push.
a.sub(assembler::Immediate(scratch_size + 8), assembler::RSP);
a.push(assembler::RDI); // push interpreter pointer
a.jmp(assembler::Indirect(assembler::RSI, offsetof(CFGBlock, code))); // jump to block
entry_offset = a.bytesWritten();
// emit epilog
epilog_offset = code_size - epilog_size;
assembler::Assembler endAsm(code.get() + epilog_offset, epilog_size);
endAsm.leave();
endAsm.retq();
RELEASE_ASSERT(!endAsm.hasFailed(), "");
// generate eh frame...
frame_manager.writeAndRegister(code.get(), code_size);
g.func_addr_registry.registerFunction(("bjit: " + name).str(), code.get(), code_size, NULL);
}
std::unique_ptr<JitFragmentWriter> JitCodeBlock::newFragment(CFGBlock* block, int patch_jump_offset) {
if (is_currently_writing || blocks_aborted.count(block))
return std::unique_ptr<JitFragmentWriter>();
is_currently_writing = true;
StackInfo stack_info(scratch_size, 16);
std::unordered_set<int> live_outs;
void* fragment_start = a.curInstPointer() - patch_jump_offset;
long fragment_offset = a.bytesWritten() - patch_jump_offset;
long bytes_left = a.bytesLeft() + patch_jump_offset;
std::unique_ptr<ICInfo> ic_info(new ICInfo(fragment_start, nullptr, nullptr, stack_info, 1, bytes_left,
llvm::CallingConv::C, live_outs, assembler::RAX, 0));
std::unique_ptr<ICSlotRewrite> rewrite(new ICSlotRewrite(ic_info.get(), ""));
return std::unique_ptr<JitFragmentWriter>(new JitFragmentWriter(block, std::move(ic_info), std::move(rewrite),
fragment_offset, epilog_offset - fragment_offset,
patch_jump_offset, a.getStartAddr(), *this));
}
void JitCodeBlock::fragmentAbort(bool not_enough_space) {
asm_failed = not_enough_space;
is_currently_writing = false;
}
void JitCodeBlock::fragmentFinished(int bytes_written, int num_bytes_overlapping, void* next_fragment_start) {
assert(next_fragment_start == bytes_written + a.curInstPointer() - num_bytes_overlapping);
a.setCurInstPointer((uint8_t*)next_fragment_start);
asm_failed = false;
is_currently_writing = false;
}
JitFragmentWriter::JitFragmentWriter(CFGBlock* block, std::unique_ptr<ICInfo> ic_info,
std::unique_ptr<ICSlotRewrite> rewrite, int code_offset, int epilog_offset,
int num_bytes_overlapping, void* entry_code, JitCodeBlock& code_block)
: Rewriter(std::move(rewrite), 0, {}),
block(block),
code_offset(code_offset),
epilog_offset(epilog_offset),
num_bytes_overlapping(num_bytes_overlapping),
num_bytes_forward_jump(0),
entry_code(entry_code),
code_block(code_block),
interp(0),
ic_info(std::move(ic_info)) {
interp = createNewVar();
addLocationToVar(interp, Location(Location::Stack, 0));
interp->setAttr(ASTInterpreterJitInterface::getCurrentBlockOffset(), imm(block));
}
RewriterVar* JitFragmentWriter::imm(uint64_t val) {
return loadConst(val);
}
RewriterVar* JitFragmentWriter::imm(void* val) {
return loadConst((uint64_t)val);
}
RewriterVar* JitFragmentWriter::emitAugbinop(RewriterVar* lhs, RewriterVar* rhs, int op_type) {
#if ENABLE_BASELINEJIT_ICS
return call(false, (void*)augbinopICHelper, imm(new AugBinopIC), lhs, rhs, imm(op_type));
#else
return call(false, (void*)augbinop, lhs, rhs, imm(op_type));
#endif
}
RewriterVar* JitFragmentWriter::emitBinop(RewriterVar* lhs, RewriterVar* rhs, int op_type) {
#if ENABLE_BASELINEJIT_ICS
return call(false, (void*)binopICHelper, imm(new BinopIC), lhs, rhs, imm(op_type));
#else
return call(false, (void*)binop, lhs, rhs, imm(op_type));
#endif
}
RewriterVar* JitFragmentWriter::emitCallattr(RewriterVar* obj, BoxedString* attr, CallattrFlags flags,
const llvm::ArrayRef<RewriterVar*> args,
std::vector<BoxedString*>* keyword_names) {
// We could make this faster but for now: keep it simple, stupid...
RewriterVar* attr_var = imm(attr);
RewriterVar* flags_var = imm(flags.asInt());
RewriterVar* keyword_names_var = keyword_names ? imm(keyword_names) : nullptr;
RewriterVar* args_array = nullptr;
if (args.size())
args_array = allocArgs(args);
else
RELEASE_ASSERT(!keyword_names_var, "0 args but keyword names are set");
bool use_ic = false;
RewriterVar::SmallVector call_args;
call_args.push_back(obj);
call_args.push_back(attr_var);
call_args.push_back(flags_var);
#if ENABLE_BASELINEJIT_ICS
if (!keyword_names_var
&& flags.argspec.totalPassed() < 4) { // looks like runtime ICs with 7 or more args don't work right now..
use_ic = true;
call_args.push_back(imm(new CallattrIC));
}
#endif
if (args_array)
call_args.push_back(args_array);
if (keyword_names_var)
call_args.push_back(keyword_names_var);
#if ENABLE_BASELINEJIT_ICS
if (use_ic)
return call(false, (void*)callattrHelperIC, call_args);
#endif
return call(false, (void*)callattrHelper, call_args);
}
RewriterVar* JitFragmentWriter::emitCompare(RewriterVar* lhs, RewriterVar* rhs, int op_type) {
#if ENABLE_BASELINEJIT_ICS
return call(false, (void*)compareICHelper, imm(new CompareIC), lhs, rhs, imm(op_type));
#else
return call(false, (void*)compare, lhs, rhs, imm(op_type));
#endif
}
RewriterVar* JitFragmentWriter::emitCreateDict(const llvm::ArrayRef<RewriterVar*> keys,
const llvm::ArrayRef<RewriterVar*> values) {
assert(keys.size() == values.size());
if (keys.empty())
return call(false, (void*)createDict);
else
return call(false, (void*)createDictHelper, imm(keys.size()), allocArgs(keys), allocArgs(values));
}
RewriterVar* JitFragmentWriter::emitCreateList(const llvm::ArrayRef<RewriterVar*> values) {
auto num = values.size();
if (num == 0)
return call(false, (void*)createList);
else
return call(false, (void*)createListHelper, imm(num), allocArgs(values));
}
RewriterVar* JitFragmentWriter::emitCreateSet(const llvm::ArrayRef<RewriterVar*> values) {
return call(false, (void*)createSetHelper, imm(values.size()), allocArgs(values));
}
RewriterVar* JitFragmentWriter::emitCreateSlice(RewriterVar* start, RewriterVar* stop, RewriterVar* step) {
return call(false, (void*)createSlice, start, stop, step);
}
RewriterVar* JitFragmentWriter::emitCreateTuple(const llvm::ArrayRef<RewriterVar*> values) {
auto num = values.size();
if (num == 0)
return imm(EmptyTuple);
else if (num == 1)
return call(false, (void*)BoxedTuple::create1, values[0]);
else if (num == 2)
return call(false, (void*)BoxedTuple::create2, values[0], values[1]);
else if (num == 3)
return call(false, (void*)BoxedTuple::create3, values[0], values[1], values[2]);
else
return call(false, (void*)createTupleHelper, imm(num), allocArgs(values));
}
RewriterVar* JitFragmentWriter::emitDeref(InternedString s) {
return call(false, (void*)ASTInterpreterJitInterface::derefHelper, getInterp(),
#ifndef NDEBUG
imm(asUInt(s).first), imm(asUInt(s).second));
#else
imm(asUInt(s)));
#endif
}
RewriterVar* JitFragmentWriter::emitExceptionMatches(RewriterVar* v, RewriterVar* cls) {
return call(false, (void*)exceptionMatchesHelper, v, cls);
}
RewriterVar* JitFragmentWriter::emitGetAttr(RewriterVar* obj, BoxedString* s) {
#if ENABLE_BASELINEJIT_ICS
return call(false, (void*)getAttrICHelper, imm(new GetAttrIC), obj, imm(s));
#else
return call(false, (void*)getattr, obj, imm(s));
#endif
}
RewriterVar* JitFragmentWriter::emitGetBlockLocal(InternedString s) {
auto it = local_syms.find(s);
if (it == local_syms.end())
return emitGetLocal(s);
return it->second;
}
RewriterVar* JitFragmentWriter::emitGetBoxedLocal(BoxedString* s) {
return call(false, (void*)ASTInterpreterJitInterface::getBoxedLocalHelper, getInterp(), imm(s));
}
RewriterVar* JitFragmentWriter::emitGetBoxedLocals() {
return call(false, (void*)ASTInterpreterJitInterface::getBoxedLocalsHelper, getInterp());
}
RewriterVar* JitFragmentWriter::emitGetClsAttr(RewriterVar* obj, BoxedString* s) {
return call(false, (void*)getclsattr, obj, imm(s));
}
RewriterVar* JitFragmentWriter::emitGetGlobal(Box* global, BoxedString* s) {
#if ENABLE_BASELINEJIT_ICS
return call(false, (void*)getGlobalICHelper, imm(new GetGlobalIC), imm(global), imm(s));
#else
return call(false, (void*)getGlobal, imm(global), imm(s));
#endif
}
RewriterVar* JitFragmentWriter::emitGetItem(RewriterVar* value, RewriterVar* slice) {
#if ENABLE_BASELINEJIT_ICS
return call(false, (void*)getitemICHelper, imm(new GetItemIC), value, slice);
#else
return call(false, (void*)getitem, value, slice);
#endif
}
RewriterVar* JitFragmentWriter::emitGetLocal(InternedString s) {
return call(false, (void*)ASTInterpreterJitInterface::getLocalHelper, getInterp(),
#ifndef NDEBUG
imm(asUInt(s).first), imm(asUInt(s).second));
#else
imm(asUInt(s)));
#endif
}
RewriterVar* JitFragmentWriter::emitGetPystonIter(RewriterVar* v) {
return call(false, (void*)getPystonIter, v);
}
RewriterVar* JitFragmentWriter::emitHasnext(RewriterVar* v) {
return call(false, (void*)hasnextHelper, v);
}
RewriterVar* JitFragmentWriter::emitLandingpad() {
return call(false, (void*)ASTInterpreterJitInterface::landingpadHelper, getInterp());
}
RewriterVar* JitFragmentWriter::emitNonzero(RewriterVar* v) {
return call(false, (void*)nonzeroHelper, v);
}
RewriterVar* JitFragmentWriter::emitNotNonzero(RewriterVar* v) {
return call(false, (void*)notHelper, v);
}
RewriterVar* JitFragmentWriter::emitRepr(RewriterVar* v) {
return call(false, (void*)repr, v);
}
RewriterVar* JitFragmentWriter::emitRuntimeCall(RewriterVar* obj, ArgPassSpec argspec,
const llvm::ArrayRef<RewriterVar*> args,
std::vector<BoxedString*>* keyword_names) {
// We could make this faster but for now: keep it simple, stupid..
RewriterVar* argspec_var = imm(argspec.asInt());
RewriterVar* keyword_names_var = keyword_names ? imm(keyword_names) : nullptr;
RewriterVar* args_array = nullptr;
if (args.size()) {
args_array = allocArgs(args);
} else
RELEASE_ASSERT(!keyword_names_var, "0 args but keyword names are set");
bool use_ic = false;
RewriterVar::SmallVector call_args;
call_args.push_back(obj);
call_args.push_back(argspec_var);
#if ENABLE_BASELINEJIT_ICS
if (!keyword_names) { // looks like runtime ICs with 7 or more args don't work right now..
use_ic = true;
call_args.push_back(imm(new RuntimeCallIC));
}
#endif
if (args_array)
call_args.push_back(args_array);
if (keyword_names_var)
call_args.push_back(keyword_names_var);
#if ENABLE_BASELINEJIT_ICS
if (use_ic)
return call(false, (void*)runtimeCallHelperIC, call_args);
#endif
return call(false, (void*)runtimeCallHelper, call_args);
}
RewriterVar* JitFragmentWriter::emitUnaryop(RewriterVar* v, int op_type) {
#if ENABLE_BASELINEJIT_ICS
return call(false, (void*)unaryopICHelper, imm(new UnaryopIC), v, imm(op_type));
#else
return call(false, (void*)unaryop, v, imm(op_type));
#endif
}
RewriterVar* JitFragmentWriter::emitUnpackIntoArray(RewriterVar* v, uint64_t num) {
RewriterVar* array = call(false, (void*)unpackIntoArray, v, imm(num));
return array;
}
RewriterVar* JitFragmentWriter::emitYield(RewriterVar* v) {
return call(false, (void*)ASTInterpreterJitInterface::yieldHelper, getInterp(), v);
}
void JitFragmentWriter::emitExec(RewriterVar* code, RewriterVar* globals, RewriterVar* locals, FutureFlags flags) {
if (!globals)
globals = imm(0ul);
if (!locals)
locals = imm(0ul);
call(false, (void*)exec, code, globals, locals, imm(flags));
}
void JitFragmentWriter::emitJump(CFGBlock* b) {
RewriterVar* next = imm(b);
addAction([=]() { _emitJump(b, next, num_bytes_forward_jump); }, { next }, ActionType::NORMAL);
}
void JitFragmentWriter::emitOSRPoint(AST_Jump* node) {
RewriterVar* node_var = imm(node);
RewriterVar* result = createNewVar();
addAction([=]() { _emitOSRPoint(result, node_var); }, { result, node_var, getInterp() }, ActionType::NORMAL);
}
void JitFragmentWriter::emitPrint(RewriterVar* dest, RewriterVar* var, bool nl) {
if (!dest)
dest = call(false, (void*)getSysStdout);
if (!var)
var = imm(0ul);
call(false, (void*)printHelper, dest, var, imm(nl));
}
void JitFragmentWriter::emitRaise0() {
call(false, (void*)raise0);
}
void JitFragmentWriter::emitRaise3(RewriterVar* arg0, RewriterVar* arg1, RewriterVar* arg2) {
call(false, (void*)raise3, arg0, arg1, arg2);
}
void JitFragmentWriter::emitReturn(RewriterVar* v) {
addAction([=]() { _emitReturn(v); }, { v }, ActionType::NORMAL);
}
void JitFragmentWriter::emitSetAttr(RewriterVar* obj, BoxedString* s, RewriterVar* attr) {
#if ENABLE_BASELINEJIT_ICS
call(false, (void*)setAttrICHelper, imm(new SetAttrIC), obj, imm(s), attr);
#else
call(false, (void*)setattr, obj, imm(s), attr);
#endif
}
void JitFragmentWriter::emitSetBlockLocal(InternedString s, RewriterVar* v) {
local_syms[s] = v;
}
void JitFragmentWriter::emitSetCurrentInst(AST_stmt* node) {
getInterp()->setAttr(ASTInterpreterJitInterface::getCurrentInstOffset(), imm(node));
}
void JitFragmentWriter::emitSetExcInfo(RewriterVar* type, RewriterVar* value, RewriterVar* traceback) {
call(false, (void*)ASTInterpreterJitInterface::setExcInfoHelper, getInterp(), type, value, traceback);
}
void JitFragmentWriter::emitSetGlobal(Box* global, BoxedString* s, RewriterVar* v) {
#if ENABLE_BASELINEJIT_ICS
call(false, (void*)setGlobalICHelper, imm(new SetGlobalIC), imm(global), imm(s), v);
#else
call(false, (void*)setGlobal, imm(global), imm(s), v);
#endif
}
void JitFragmentWriter::emitSetItem(RewriterVar* target, RewriterVar* slice, RewriterVar* value) {
#if ENABLE_BASELINEJIT_ICS
call(false, (void*)setitemICHelper, imm(new SetItemIC), target, slice, value);
#else
call(false, (void*)setitem, target, slice, value);
#endif
}
void JitFragmentWriter::emitSetItemName(BoxedString* s, RewriterVar* v) {
call(false, (void*)ASTInterpreterJitInterface::setItemNameHelper, getInterp(), imm(s), v);
}
void JitFragmentWriter::emitSetLocal(InternedString s, bool set_closure, RewriterVar* v) {
void* func = set_closure ? (void*)ASTInterpreterJitInterface::setLocalClosureHelper
: (void*)ASTInterpreterJitInterface::setLocalHelper;
call(false, func, getInterp(),
#ifndef NDEBUG
imm(asUInt(s).first), imm(asUInt(s).second),
#else
imm(asUInt(s)),
#endif
v);
}
void JitFragmentWriter::emitSideExit(RewriterVar* v, Box* cmp_value, CFGBlock* next_block) {
RewriterVar* var = imm(cmp_value);
RewriterVar* next_block_var = imm(next_block);
addAction([=]() { _emitSideExit(v, var, next_block, next_block_var); }, { v, var, next_block_var },
ActionType::NORMAL);
}
void JitFragmentWriter::emitUncacheExcInfo() {
call(false, (void*)ASTInterpreterJitInterface::uncacheExcInfoHelper, getInterp());
}
void JitFragmentWriter::abortCompilation() {
blocks_aborted.insert(block);
code_block.fragmentAbort(false);
abort();
}
int JitFragmentWriter::finishCompilation() {
RELEASE_ASSERT(!assembler->hasFailed(), "");
commit();
if (failed) {
blocks_aborted.insert(block);
code_block.fragmentAbort(false);
return 0;
}
if (assembler->hasFailed()) {
code_block.fragmentAbort(true /* not_enough_space */);
return 0;
}
block->code = (void*)((uint64_t)entry_code + code_offset);
block->entry_code = (decltype(block->entry_code))entry_code;
void* next_fragment_start = (uint8_t*)block->code + assembler->bytesWritten();
code_block.fragmentFinished(assembler->bytesWritten(), num_bytes_overlapping, next_fragment_start);
return num_bytes_forward_jump;
}
bool JitFragmentWriter::finishAssembly(int continue_offset) {
return !assembler->hasFailed();
}
RewriterVar* JitFragmentWriter::allocArgs(const llvm::ArrayRef<RewriterVar*> args) {
auto num = args.size();
RewriterVar* array = allocate(num);
for (int i = 0; i < num; ++i)
array->setAttr(sizeof(void*) * i, args[i]);
return array;
}
#ifndef NDEBUG
std::pair<uint64_t, uint64_t> JitFragmentWriter::asUInt(InternedString s) {
static_assert(sizeof(InternedString) == sizeof(uint64_t) * 2, "");
union U {
U(InternedString is) : is(is) {}
InternedString is;
uint64_t u[2];
} u(s);
return std::make_pair(u.u[0], u.u[1]);
}
#else
uint64_t JitFragmentWriter::asUInt(InternedString s) {
static_assert(sizeof(InternedString) == sizeof(uint64_t), "");
union U {
U(InternedString is) : is(is) {}
InternedString is;
uint64_t u;
} u(s);
return u.u;
}
#endif
RewriterVar* JitFragmentWriter::getInterp() {
return interp;
}
Box* JitFragmentWriter::augbinopICHelper(AugBinopIC* ic, Box* lhs, Box* rhs, int op) {
return ic->call(lhs, rhs, op);
}
Box* JitFragmentWriter::binopICHelper(BinopIC* ic, Box* lhs, Box* rhs, int op) {
return ic->call(lhs, rhs, op);
}
#if ENABLE_BASELINEJIT_ICS
Box* JitFragmentWriter::callattrHelperIC(Box* obj, BoxedString* attr, CallattrFlags flags, CallattrIC* ic, Box** args) {
auto arg_tuple = getTupleFromArgsArray(&args[0], flags.argspec.totalPassed());
return ic->call(obj, attr, flags, std::get<0>(arg_tuple), std::get<1>(arg_tuple), std::get<2>(arg_tuple), NULL,
NULL);
}
#endif
Box* JitFragmentWriter::callattrHelper(Box* obj, BoxedString* attr, CallattrFlags flags, Box** args,
std::vector<BoxedString*>* keyword_names) {
auto arg_tuple = getTupleFromArgsArray(&args[0], flags.argspec.totalPassed());
Box* r = callattr(obj, attr, flags, std::get<0>(arg_tuple), std::get<1>(arg_tuple), std::get<2>(arg_tuple),
std::get<3>(arg_tuple), keyword_names);
assert(gc::isValidGCObject(r));
return r;
}
Box* JitFragmentWriter::compareICHelper(CompareIC* ic, Box* lhs, Box* rhs, int op) {
return ic->call(lhs, rhs, op);
}
Box* JitFragmentWriter::createDictHelper(uint64_t num, Box** keys, Box** values) {
BoxedDict* dict = (BoxedDict*)createDict();
for (uint64_t i = 0; i < num; ++i) {
assert(gc::isValidGCObject(keys[i]));
assert(gc::isValidGCObject(values[i]));
dict->d[keys[i]] = values[i];
}
return dict;
}
Box* JitFragmentWriter::createListHelper(uint64_t num, Box** data) {
BoxedList* list = (BoxedList*)createList();
list->ensure(num);
for (uint64_t i = 0; i < num; ++i) {
assert(gc::isValidGCObject(data[i]));
listAppendInternal(list, data[i]);
}
return list;
}
Box* JitFragmentWriter::createSetHelper(uint64_t num, Box** data) {
BoxedSet* set = (BoxedSet*)createSet();
for (int i = 0; i < num; ++i)
set->s.insert(data[i]);
return set;
}
Box* JitFragmentWriter::createTupleHelper(uint64_t num, Box** data) {
return BoxedTuple::create(num, data);
}
Box* JitFragmentWriter::exceptionMatchesHelper(Box* obj, Box* cls) {
return boxBool(exceptionMatches(obj, cls));
}
Box* JitFragmentWriter::getAttrICHelper(GetAttrIC* ic, Box* o, BoxedString* attr) {
return ic->call(o, attr);
}
Box* JitFragmentWriter::getGlobalICHelper(GetGlobalIC* ic, Box* o, BoxedString* s) {
return ic->call(o, s);
}
Box* JitFragmentWriter::getitemICHelper(GetItemIC* ic, Box* o, Box* attr) {
return ic->call(o, attr);
}
Box* JitFragmentWriter::hasnextHelper(Box* b) {
return boxBool(pyston::hasnext(b));
}
Box* JitFragmentWriter::nonzeroHelper(Box* b) {
return boxBool(b->nonzeroIC());
}
Box* JitFragmentWriter::notHelper(Box* b) {
return boxBool(!b->nonzeroIC());
}
#if ENABLE_BASELINEJIT_ICS
Box* JitFragmentWriter::runtimeCallHelperIC(Box* obj, ArgPassSpec argspec, RuntimeCallIC* ic, Box** args) {
auto arg_tuple = getTupleFromArgsArray(&args[0], argspec.totalPassed());
return ic->call(obj, argspec, std::get<0>(arg_tuple), std::get<1>(arg_tuple), std::get<2>(arg_tuple),
std::get<3>(arg_tuple));
}
#endif
Box* JitFragmentWriter::runtimeCallHelper(Box* obj, ArgPassSpec argspec, Box** args,
std::vector<BoxedString*>* keyword_names) {
auto arg_tuple = getTupleFromArgsArray(&args[0], argspec.totalPassed());
return runtimeCall(obj, argspec, std::get<0>(arg_tuple), std::get<1>(arg_tuple), std::get<2>(arg_tuple),
std::get<3>(arg_tuple), keyword_names);
}
Box* JitFragmentWriter::setAttrICHelper(SetAttrIC* ic, Box* o, BoxedString* attr, Box* value) {
return ic->call(o, attr, value);
}
Box* JitFragmentWriter::setGlobalICHelper(SetGlobalIC* ic, Box* o, BoxedString* s, Box* v) {
return ic->call(o, s, v);
}
Box* JitFragmentWriter::setitemICHelper(SetItemIC* ic, Box* o, Box* attr, Box* value) {
return ic->call(o, attr, value);
}
Box* JitFragmentWriter::unaryopICHelper(UnaryopIC* ic, Box* obj, int op) {
return ic->call(obj, op);
}
void JitFragmentWriter::_emitJump(CFGBlock* b, RewriterVar* block_next, int& size_of_indirect_jump) {
size_of_indirect_jump = 0;
if (b->code) {
int64_t offset = (uint64_t)b->code - ((uint64_t)entry_code + code_offset);
if (isLargeConstant(offset)) {
assembler->mov(assembler::Immediate(b->code), assembler::R11);
assembler->jmpq(assembler::R11);
} else
assembler->jmp(assembler::JumpDestination::fromStart(offset));
} else {
int num_bytes = assembler->bytesWritten();
block_next->getInReg(assembler::RAX, true);
assembler->mov(assembler::Indirect(assembler::RAX, 8), assembler::RSI);
assembler->test(assembler::RSI, assembler::RSI);
assembler->je(assembler::JumpDestination::fromStart(epilog_offset));
assembler->jmp(assembler::Indirect(assembler::RAX, offsetof(CFGBlock, code)));
size_of_indirect_jump = assembler->bytesWritten() - num_bytes;
}
block_next->bumpUse();
}
void JitFragmentWriter::_emitOSRPoint(RewriterVar* result, RewriterVar* node_var) {
RewriterVar::SmallVector args;
args.push_back(getInterp());
args.push_back(node_var);
_call(result, false, (void*)ASTInterpreterJitInterface::doOSRHelper, args, RewriterVar::SmallVector());
auto result_reg = result->getInReg(assembler::RDX);
result->bumpUse();
assembler->test(result_reg, result_reg);
{
assembler::ForwardJump je(*assembler, assembler::COND_EQUAL);
assembler->mov(assembler::Immediate(0ul), assembler::RAX); // TODO: use xor
assembler->jmp(assembler::JumpDestination::fromStart(epilog_offset));
}
assertConsistent();
}
void JitFragmentWriter::_emitReturn(RewriterVar* return_val) {
return_val->getInReg(assembler::RDX, true);
assembler->mov(assembler::Immediate(0ul), assembler::RAX); // TODO: use xor
assembler->jmp(assembler::JumpDestination::fromStart(epilog_offset));
return_val->bumpUse();
}
void JitFragmentWriter::_emitSideExit(RewriterVar* var, RewriterVar* val_constant, CFGBlock* next_block,
RewriterVar* next_block_var) {
assert(val_constant->is_constant);
assert(next_block_var->is_constant);
uint64_t val = val_constant->constant_value;
assembler::Register var_reg = var->getInReg();
if (isLargeConstant(val)) {
assembler::Register reg = val_constant->getInReg(Location::any(), true, /* otherThan */ var_reg);
assembler->cmp(var_reg, reg);
} else {
assembler->cmp(var_reg, assembler::Immediate(val));
}
{
assembler::ForwardJump jne(*assembler, assembler::COND_EQUAL);
int bytes = 0;
_emitJump(next_block, next_block_var, bytes);
if (bytes) {
// TODO: We generated an indirect jump.
// If we later on JIT the dest block we could patch this code to a direct jump to the dest.
}
}
var->bumpUse();
val_constant->bumpUse();
assertConsistent();
}
}
src/codegen/baseline_jit.h 0 → 100644
View file @ fc366564
// Copyright (c) 2014-2015 Dropbox, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef PYSTON_CODEGEN_BASELINEJIT_H
#define PYSTON_CODEGEN_BASELINEJIT_H
#include <llvm/ADT/ArrayRef.h>
#include "asm_writing/rewriter.h"
#include "codegen/ast_interpreter.h"
#include "gc/heap.h"
#include "runtime/ics.h"
namespace pyston {
#define ENABLE_BASELINEJIT_ICS 1
class AST_stmt;
class Box;
class BoxedDict;
class BoxedList;
class BoxedTuple;
class SetGlobalIC;
class GetGlobalIC;
class SetAttrIC;
class GetAttrIC;
class SetItemIC;
class GetItemIC;
class CompareIC;
class AugBinopIC;
class UnaryopIC;
class RuntimeCallIC;
class JitFragmentWriter;
// This JIT tier is designed as Pystons entry level JIT tier (executed after a only a few dozens runs of a basic block)
// which emits code very quick. It does not do any type specializations but can use inline caches.
// It operates on a basic block at a time (=CFGBLock*) and supports very fast switching between the
// interpreter and the JITed code on every block start/end.
//
// To archive this it's tightly integrated with the AST Interpreter and always operates on an ASTInterpreter instance.
// The process works like this:
// - in the ASTInterpreter main loop we will check on every basic block start if we have already machine code for the
// basic block
// - if we have, we will directly call it (='entry_code' pointer inside the CFGBlock)
// - if we don't have but determined that the block is hot (loop backedge threshold reached or function got called
// often enough) and and we haven't yet tried to JIT it we will start with the JITing process:
// - create/reuse a JitCodeBlock for the function
// - create a new JitFragmentWriter for the basic block to JIT
// - interpret the basic block and in addition call into corresponding emit* functions of the JitFragmentWriter on
// every AST node encountered.
// - if a node is encountered which is not supported, abort JITing of the block and blacklist this block
// - if we reached the control flow changing node of the basic block (e.g. a branch, return or jump node) we finish
// JITing the block.
//
// A JITed block is allowed to jump directly to another JITed basic block or if the block is not yet JITed (or we are
// unable to JIT it) we return from the function and the interpreter will immediatelly continue interpreting the next
// block.
// JitCodeBlock manages a fixed size memory block which stores JITed code.
// It can contain a variable number of blocks generated by JitFragmentWriter instances.
// Currently a JitFragment always contains the code of a single CFGBlock*.
// A JitFragment can get called from the Interpreter by calling 'entry_code' which will jump to the fragment start or
// it can get executed by a jump from another fragment.
// At every fragment end we can jump to another fragment, fallback to the Interpreter or exit.
// This means we are not allowed to assume that a register contains a specific value between JitFragments.
// This also means that we are allowed to store a Python variable which only lives in the current CFGBLock* inside a
// register or stack slot but we aren't if it outlives the block - we have to store it in the interpreter instance.
//
// To execute a specific CFGBlock one has to call:
// CFGBlock* block;
// block->entry_code(ast_interpreter_instance, block)
//
// Signature of a JitCodeBlock:
// std::pair<CFGBlock*, Box*>(*entry_code)(ASTInterpreter* interp, CFGBlock* block)
// args:
// interp: instance to the ASTInterpreter
// block: block to execute
//
// return value:
// first: next block to execute in the interpreter
// if 0, we executed a return node and should return second
// second: return value only used if first == 0
//
// Basic layout of generated code block is:
// entry_code:
// push %rbp ; setup frame pointer
// mov %rsp,%rbp
// sub $0x108,%rsp ; setup scratch, 0x108 = scratch_size + 8 (=stack alignment)
// push %rdi ; save the pointer to ASTInterpreter instance
// jmpq *0x8(%rsi) ; jump to block->code
// possible values: first_JitFragment, second_JitFragment,...
//
// first_JitFragment:
// ...
// ; Side exit (e.g. if <val> <block1> else <block2>
// movabs $0x1270014108,%rcx ; rcx = True
// cmp %rax,%rcx ; rax == True
// jne end_side_exit
// mov %rax,0x10(%rsp) ;
// movabs $0x215bb60,%rax ; rax = CFGBlock* to call next (rax is also the 1. return reg)
// mov 0x8(%rax),%rsi ; load CFGBlock->code
// test %rsi,%rsi ; CFGBlock->code == 0
// je epilog ; exit to interpreter if code == 0
// jmpq *0x8(%rax) ; jump to new jit fragment (e.g second_JitFragment)
// end_side_exit:
// ....
// second_JitFragment:
// ...
//
// nth_JitFragment:
// ... ; direct jump previous JITed block
// jmp first_JitFragment
//
// epilog: ; code which jumps to epilog has to make sure that
// ; rax contains the next block to execute
// ; or 0 if we are finished but then rdx must contain the Box* value to return
// leave
// ret
//
class JitCodeBlock {
private:
static constexpr int scratch_size = 256;
static constexpr int code_size = 4096 * 2;
static constexpr int epilog_size = 2; // size of [leave, ret] in bytes
EHFrameManager frame_manager;
std::unique_ptr<uint8_t[]> code;
int entry_offset;
int epilog_offset;
assembler::Assembler a;
bool is_currently_writing;
bool asm_failed;
public:
JitCodeBlock(llvm::StringRef name);
std::unique_ptr<JitFragmentWriter> newFragment(CFGBlock* block, int patch_jump_offset = 0);
bool shouldCreateNewBlock() const { return asm_failed || a.bytesLeft() < 128; }
void fragmentAbort(bool not_enough_space);
void fragmentFinished(int bytes_witten, int num_bytes_overlapping, void* next_fragment_start);
};
class JitFragmentWriter : public Rewriter {
private:
CFGBlock* block;
int code_offset; // offset inside the JitCodeBlock to the start of this block
int epilog_offset; // offset inside the JitCodeBlock to the epilog
int num_bytes_overlapping; // num of bytes this block overlaps with the prev. used to patch unessary forward jumps
int num_bytes_forward_jump; // number of bytes emited for the last forward jump to the next block. This is used to
// patch unessary forward jumps when the next fragment is emited (it becomes
// num_bytes_overlapping)
void* entry_code; // JitCodeBlock start address. Mmust have an offset of 0 into the code block
JitCodeBlock& code_block;
RewriterVar* interp;
llvm::DenseMap<InternedString, RewriterVar*> local_syms;
std::unique_ptr<ICInfo> ic_info;
public:
JitFragmentWriter(CFGBlock* block, std::unique_ptr<ICInfo> ic_info, std::unique_ptr<ICSlotRewrite> rewrite,
int code_offset, int epilog_offset, int num_bytes_overlapping, void* entry_code,
JitCodeBlock& code_block);
RewriterVar* imm(uint64_t val);
RewriterVar* imm(void* val);
RewriterVar* emitAugbinop(RewriterVar* lhs, RewriterVar* rhs, int op_type);
RewriterVar* emitBinop(RewriterVar* lhs, RewriterVar* rhs, int op_type);
RewriterVar* emitCallattr(RewriterVar* obj, BoxedString* attr, CallattrFlags flags,
const llvm::ArrayRef<RewriterVar*> args, std::vector<BoxedString*>* keyword_names);
RewriterVar* emitCompare(RewriterVar* lhs, RewriterVar* rhs, int op_type);
RewriterVar* emitCreateDict(const llvm::ArrayRef<RewriterVar*> keys, const llvm::ArrayRef<RewriterVar*> values);
RewriterVar* emitCreateList(const llvm::ArrayRef<RewriterVar*> values);
RewriterVar* emitCreateSet(const llvm::ArrayRef<RewriterVar*> values);
RewriterVar* emitCreateSlice(RewriterVar* start, RewriterVar* stop, RewriterVar* step);
RewriterVar* emitCreateTuple(const llvm::ArrayRef<RewriterVar*> values);
RewriterVar* emitDeref(InternedString s);
RewriterVar* emitExceptionMatches(RewriterVar* v, RewriterVar* cls);
RewriterVar* emitGetAttr(RewriterVar* obj, BoxedString* s);
RewriterVar* emitGetBlockLocal(InternedString s);
RewriterVar* emitGetBoxedLocal(BoxedString* s);
RewriterVar* emitGetBoxedLocals();
RewriterVar* emitGetClsAttr(RewriterVar* obj, BoxedString* s);
RewriterVar* emitGetGlobal(Box* global, BoxedString* s);
RewriterVar* emitGetItem(RewriterVar* value, RewriterVar* slice);
RewriterVar* emitGetLocal(InternedString s);
RewriterVar* emitGetPystonIter(RewriterVar* v);
RewriterVar* emitHasnext(RewriterVar* v);
RewriterVar* emitLandingpad();
RewriterVar* emitNonzero(RewriterVar* v);
RewriterVar* emitNotNonzero(RewriterVar* v);
RewriterVar* emitRepr(RewriterVar* v);
RewriterVar* emitRuntimeCall(RewriterVar* obj, ArgPassSpec argspec, const llvm::ArrayRef<RewriterVar*> args,
std::vector<BoxedString*>* keyword_names);
RewriterVar* emitUnaryop(RewriterVar* v, int op_type);
RewriterVar* emitUnpackIntoArray(RewriterVar* v, uint64_t num);
RewriterVar* emitYield(RewriterVar* v);
void emitExec(RewriterVar* code, RewriterVar* globals, RewriterVar* locals, FutureFlags flags);
void emitJump(CFGBlock* b);
void emitOSRPoint(AST_Jump* node);
void emitPrint(RewriterVar* dest, RewriterVar* var, bool nl);
void emitRaise0();
void emitRaise3(RewriterVar* arg0, RewriterVar* arg1, RewriterVar* arg2);
void emitReturn(RewriterVar* v);
void emitSetAttr(RewriterVar* obj, BoxedString* s, RewriterVar* attr);
void emitSetBlockLocal(InternedString s, RewriterVar* v);
void emitSetCurrentInst(AST_stmt* node);
void emitSetExcInfo(RewriterVar* type, RewriterVar* value, RewriterVar* traceback);
void emitSetGlobal(Box* global, BoxedString* s, RewriterVar* v);
void emitSetItemName(BoxedString* s, RewriterVar* v);
void emitSetItem(RewriterVar* target, RewriterVar* slice, RewriterVar* value);
void emitSetLocal(InternedString s, bool set_closure, RewriterVar* v);
void emitSideExit(RewriterVar* v, Box* cmp_value, CFGBlock* next_block);
void emitUncacheExcInfo();
void abortCompilation();
int finishCompilation();
bool finishAssembly(int continue_offset) override;
private:
RewriterVar* allocArgs(const llvm::ArrayRef<RewriterVar*> args);
#ifndef NDEBUG
std::pair<uint64_t, uint64_t> asUInt(InternedString s);
#else
uint64_t asUInt(InternedString s);
#endif
RewriterVar* getInterp();
static Box* augbinopICHelper(AugBinopIC* ic, Box* lhs, Box* rhs, int op);
static Box* binopICHelper(BinopIC* ic, Box* lhs, Box* rhs, int op);
static Box* callattrHelper(Box* obj, BoxedString* attr, CallattrFlags flags, Box** args,
std::vector<BoxedString*>* keyword_names);
static Box* compareICHelper(CompareIC* ic, Box* lhs, Box* rhs, int op);
static Box* createDictHelper(uint64_t num, Box** keys, Box** values);
static Box* createListHelper(uint64_t num, Box** data);
static Box* createSetHelper(uint64_t num, Box** data);
static Box* createTupleHelper(uint64_t num, Box** data);
static Box* exceptionMatchesHelper(Box* obj, Box* cls);
static Box* getAttrICHelper(GetAttrIC* ic, Box* o, BoxedString* attr);
static Box* getGlobalICHelper(GetGlobalIC* ic, Box* o, BoxedString* s);
static Box* getitemICHelper(GetItemIC* ic, Box* o, Box* attr);
static Box* hasnextHelper(Box* b);
static Box* nonzeroHelper(Box* b);
static Box* notHelper(Box* b);
static Box* runtimeCallHelper(Box* obj, ArgPassSpec argspec, Box** args, std::vector<BoxedString*>* keyword_names);
static Box* setAttrICHelper(SetAttrIC* ic, Box* o, BoxedString* attr, Box* value);
static Box* setGlobalICHelper(SetGlobalIC* ic, Box* o, BoxedString* s, Box* v);
static Box* setitemICHelper(SetItemIC* ic, Box* o, Box* attr, Box* value);
static Box* unaryopICHelper(UnaryopIC* ic, Box* obj, int op);
#if ENABLE_BASELINEJIT_ICS
static Box* callattrHelperIC(Box* obj, BoxedString* attr, CallattrFlags flags, CallattrIC* ic, Box** args);
static Box* runtimeCallHelperIC(Box* obj, ArgPassSpec argspec, RuntimeCallIC* ic, Box** args);
#endif
void _emitJump(CFGBlock* b, RewriterVar* block_next, int& size_of_indirect_jump);
void _emitOSRPoint(RewriterVar* result, RewriterVar* node_var);
void _emitReturn(RewriterVar* v);
void _emitSideExit(RewriterVar* var, RewriterVar* val_constant, CFGBlock* next_block, RewriterVar* false_path);
};
}
#endif
src/codegen/codegen.cpp
View file @ fc366564
......@@ -25,6 +25,7 @@
#include "llvm/Object/ObjectFile.h"
#include "llvm/Support/FileSystem.h"
#include "analysis/function_analysis.h"
#include "analysis/scoping_analysis.h"
#include "codegen/compvars.h"
#include "core/ast.h"
......@@ -62,6 +63,10 @@ SourceInfo::SourceInfo(BoxedModule* m, ScopingAnalysis* scoping, FutureFlags fut
}
}
SourceInfo::~SourceInfo() {
// TODO: release memory..
}
void FunctionAddressRegistry::registerFunction(const std::string& name, void* addr, int length,
llvm::Function* llvm_func) {
assert(addr);
......
src/codegen/irgen.cpp
View file @ fc366564
......@@ -544,9 +544,7 @@ static void emitBBs(IRGenState* irstate, TypeAnalysis* types, const OSREntryDesc
emitter->getBuilder()->CreateStore(new_call_count, call_count_ptr);
int reopt_threshold;
if (effort == EffortLevel::MINIMAL)
reopt_threshold = REOPT_THRESHOLD_BASELINE;
else if (effort == EffortLevel::MODERATE)
if (effort == EffortLevel::MODERATE)
reopt_threshold = REOPT_THRESHOLD_T2;
else
RELEASE_ASSERT(0, "Unknown effort: %d", (int)effort);
......@@ -1059,15 +1057,15 @@ CompiledFunction* doCompile(SourceInfo* source, ParamNames* param_names, const O
computeBlockSetClosure(blocks);
}
std::unique_ptr<LivenessAnalysis> liveness = computeLivenessInfo(source->cfg);
LivenessAnalysis* liveness = source->getLiveness();
std::unique_ptr<PhiAnalysis> phis;
if (entry_descriptor)
phis = computeRequiredPhis(entry_descriptor, liveness.get(), source->getScopeInfo());
phis = computeRequiredPhis(entry_descriptor, liveness, source->getScopeInfo());
else
phis = computeRequiredPhis(*param_names, source->cfg, liveness.get(), source->getScopeInfo());
phis = computeRequiredPhis(*param_names, source->cfg, liveness, source->getScopeInfo());
IRGenState irstate(cf, source, std::move(liveness), std::move(phis), param_names, getGCBuilder(), dbg_funcinfo);
IRGenState irstate(cf, source, std::move(phis), param_names, getGCBuilder(), dbg_funcinfo);
emitBBs(&irstate, types, entry_descriptor, blocks);
......
src/codegen/irgen/hooks.cpp
View file @ fc366564
......@@ -21,6 +21,7 @@
#include "analysis/scoping_analysis.h"
#include "asm_writing/icinfo.h"
#include "codegen/ast_interpreter.h"
#include "codegen/baseline_jit.h"
#include "codegen/codegen.h"
#include "codegen/compvars.h"
#include "codegen/irgen.h"
......@@ -120,6 +121,12 @@ ScopeInfo* SourceInfo::getScopeInfo() {
return scoping->getScopeInfoForNode(ast);
}
LivenessAnalysis* SourceInfo::getLiveness() {
if (!liveness_info)
liveness_info = computeLivenessInfo(cfg);
return liveness_info.get();
}
EffortLevel initialEffort() {
if (FORCE_INTERPRETER)
return EffortLevel::INTERPRETED;
......@@ -127,7 +134,7 @@ EffortLevel initialEffort() {
return EffortLevel::MAXIMAL;
if (ENABLE_INTERPRETER)
return EffortLevel::INTERPRETED;
return EffortLevel::MINIMAL;
return EffortLevel::MODERATE;
}
static void compileIR(CompiledFunction* cf, EffortLevel effort) {
......@@ -270,13 +277,6 @@ CompiledFunction* compileFunction(CLFunction* f, FunctionSpecialization* spec, E
num_compiles.log();
break;
}
case EffortLevel::MINIMAL: {
static StatCounter us_compiling("us_compiling_1_minimal");
us_compiling.log(us);
static StatCounter num_compiles("num_compiles_1_minimal");
num_compiles.log();
break;
}
case EffortLevel::MODERATE: {
static StatCounter us_compiling("us_compiling_2_moderate");
us_compiling.log(us);
......@@ -742,6 +742,21 @@ void CompiledFunction::speculationFailed() {
}
}
CompiledFunction::CompiledFunction(llvm::Function* func, FunctionSpecialization* spec, bool is_interpreted, void* code,
EffortLevel effort, const OSREntryDescriptor* entry_descriptor)
: clfunc(NULL),
func(func),
spec(spec),
entry_descriptor(entry_descriptor),
is_interpreted(is_interpreted),
code(code),
effort(effort),
times_called(0),
times_speculation_failed(0),
location_map(nullptr) {
assert((spec != NULL) + (entry_descriptor != NULL) == 1);
}
ConcreteCompilerType* CompiledFunction::getReturnType() {
if (spec)
return spec->rtn_type;
......@@ -803,7 +818,7 @@ CompiledFunction* compilePartialFuncInternal(OSRExit* exit) {
assert(exit->parent_cf->clfunc);
CompiledFunction*& new_cf = exit->parent_cf->clfunc->osr_versions[exit->entry];
if (new_cf == NULL) {
EffortLevel new_effort = exit->parent_cf->effort == EffortLevel::INTERPRETED ? EffortLevel::MINIMAL
EffortLevel new_effort = exit->parent_cf->effort == EffortLevel::INTERPRETED ? EffortLevel::MODERATE
: EffortLevel::MAXIMAL;
CompiledFunction* compiled = compileFunction(exit->parent_cf->clfunc, NULL, new_effort, exit->entry);
assert(compiled == new_cf);
......@@ -830,8 +845,6 @@ extern "C" CompiledFunction* reoptCompiledFuncInternal(CompiledFunction* cf) {
EffortLevel new_effort;
if (cf->effort == EffortLevel::INTERPRETED)
new_effort = EffortLevel::MINIMAL;
else if (cf->effort == EffortLevel::MINIMAL)
new_effort = EffortLevel::MODERATE;
else if (cf->effort == EffortLevel::MODERATE)
new_effort = EffortLevel::MAXIMAL;
......
src/codegen/irgen/irgenerator.cpp
View file @ fc366564
......@@ -42,12 +42,10 @@ extern "C" void dumpLLVM(llvm::Value* v) {
v->dump();
}
IRGenState::IRGenState(CompiledFunction* cf, SourceInfo* source_info, std::unique_ptr<LivenessAnalysis> liveness,
std::unique_ptr<PhiAnalysis> phis, ParamNames* param_names, GCBuilder* gc,
llvm::MDNode* func_dbg_info)
IRGenState::IRGenState(CompiledFunction* cf, SourceInfo* source_info, std::unique_ptr<PhiAnalysis> phis,
ParamNames* param_names, GCBuilder* gc, llvm::MDNode* func_dbg_info)
: cf(cf),
source_info(source_info),
liveness(std::move(liveness)),
phis(std::move(phis)),
param_names(param_names),
gc(gc),
......@@ -426,13 +424,22 @@ IREmitter* createIREmitter(IRGenState* irstate, llvm::BasicBlock*& curblock, IRG
}
static std::unordered_map<AST_expr*, std::vector<BoxedString*>*> made_keyword_storage;
static std::vector<BoxedString*>* getKeywordNameStorage(AST_Call* node) {
std::vector<BoxedString*>* getKeywordNameStorage(AST_Call* node) {
auto it = made_keyword_storage.find(node);
if (it != made_keyword_storage.end())
return it->second;
auto rtn = new std::vector<BoxedString*>();
made_keyword_storage.insert(it, std::make_pair(node, rtn));
// Only add the keywords to the array the first time, since
// the later times we will hit the cache which will have the
// keyword names already populated:
if (!rtn->size()) {
for (auto kw : node->keywords)
rtn->push_back(kw->arg.getBox());
}
return rtn;
}
......@@ -833,22 +840,10 @@ private:
}
std::vector<CompilerVariable*> args;
std::vector<BoxedString*>* keyword_names;
if (node->keywords.size()) {
std::vector<BoxedString*>* keyword_names = NULL;
if (node->keywords.size())
keyword_names = getKeywordNameStorage(node);
// Only add the keywords to the array the first time, since
// the later times we will hit the cache which will have the
// keyword names already populated:
if (!keyword_names->size()) {
for (auto kw : node->keywords) {
keyword_names->push_back(kw->arg.getBox());
}
}
} else {
keyword_names = NULL;
}
for (int i = 0; i < node->args.size(); i++) {
CompilerVariable* a = evalExpr(node->args[i], unw_info);
args.push_back(a);
......@@ -1887,6 +1882,7 @@ private:
static BoxedString* space_str = static_cast<BoxedString*>(PyString_InternFromString(" "));
// TODO: why are we inline-generating all this code instead of just emitting a call to some runtime function?
// (=printHelper())
int nvals = node->values.size();
for (int i = 0; i < nvals; i++) {
CompilerVariable* var = evalExpr(node->values[i], unw_info);
......@@ -2027,9 +2023,7 @@ private:
auto effort = irstate->getEffortLevel();
int osr_threshold;
if (effort == EffortLevel::MINIMAL)
osr_threshold = OSR_THRESHOLD_BASELINE;
else if (effort == EffortLevel::MODERATE)
if (effort == EffortLevel::MODERATE)
osr_threshold = OSR_THRESHOLD_T2;
else
RELEASE_ASSERT(0, "Unknown effort: %d", (int)effort);
......
src/codegen/irgen/irgenerator.h
View file @ fc366564
......@@ -56,7 +56,6 @@ class IRGenState {
private:
CompiledFunction* cf;
SourceInfo* source_info;
std::unique_ptr<LivenessAnalysis> liveness;
std::unique_ptr<PhiAnalysis> phis;
ParamNames* param_names;
GCBuilder* gc;
......@@ -70,8 +69,8 @@ private:
public:
IRGenState(CompiledFunction* cf, SourceInfo* source_info, std::unique_ptr<LivenessAnalysis> liveness,
std::unique_ptr<PhiAnalysis> phis, ParamNames* param_names, GCBuilder* gc, llvm::MDNode* func_dbg_info);
IRGenState(CompiledFunction* cf, SourceInfo* source_info, std::unique_ptr<PhiAnalysis> phis,
ParamNames* param_names, GCBuilder* gc, llvm::MDNode* func_dbg_info);
~IRGenState();
CompiledFunction* getCurFunction() { return cf; }
......@@ -90,7 +89,7 @@ public:
SourceInfo* getSourceInfo() { return source_info; }
LivenessAnalysis* getLiveness() { return liveness.get(); }
LivenessAnalysis* getLiveness() { return source_info->getLiveness(); }
PhiAnalysis* getPhis() { return phis.get(); }
ScopeInfo* getScopeInfo();
......@@ -133,11 +132,13 @@ public:
};
class IREmitter;
class AST_Call;
IREmitter* createIREmitter(IRGenState* irstate, llvm::BasicBlock*& curblock, IRGenerator* irgenerator = NULL);
IRGenerator* createIRGenerator(IRGenState* irstate, std::unordered_map<CFGBlock*, llvm::BasicBlock*>& entry_blocks,
CFGBlock* myblock, TypeAnalysis* types);
CLFunction* wrapFunction(AST* node, AST_arguments* args, const std::vector<AST_stmt*>& body, SourceInfo* source);
std::vector<BoxedString*>* getKeywordNameStorage(AST_Call* node);
}
#endif
src/core/cfg.h
View file @ fc366564
......@@ -36,6 +36,7 @@
namespace pyston {
class AST_stmt;
class Box;
class CFG;
class CFGBlock {
......@@ -43,6 +44,12 @@ private:
CFG* cfg;
public:
// Baseline JIT helper fields:
// contains address to the start of the code of this basic block
void* code;
// contains the address of the entry function
std::pair<CFGBlock*, Box*>(*entry_code)(void* interpeter, CFGBlock* block);
std::vector<AST_stmt*> body;
std::vector<CFGBlock*> predecessors, successors;
int idx; // index in the CFG
......@@ -50,7 +57,7 @@ public:
typedef std::vector<AST_stmt*>::iterator iterator;
CFGBlock(CFG* cfg, int idx) : cfg(cfg), idx(idx), info(NULL) {}
CFGBlock(CFG* cfg, int idx) : cfg(cfg), code(NULL), entry_code(NULL), idx(idx), info(NULL) {}
void connectTo(CFGBlock* successor, bool allow_backedge = false);
void unconnectFrom(CFGBlock* successor);
......
src/core/options.cpp
View file @ fc366564
......@@ -39,15 +39,16 @@ bool DUMPJIT = false;
bool TRAP = false;
bool USE_STRIPPED_STDLIB = true; // always true
bool ENABLE_INTERPRETER = true;
bool ENABLE_BASELINEJIT = true;
bool ENABLE_PYPA_PARSER = true;
bool USE_REGALLOC_BASIC = true;
bool PAUSE_AT_ABORT = false;
bool ENABLE_TRACEBACKS = true;
int OSR_THRESHOLD_INTERPRETER = 500;
int REOPT_THRESHOLD_INTERPRETER = 200;
int OSR_THRESHOLD_BASELINE = 10000;
int REOPT_THRESHOLD_BASELINE = 250;
int OSR_THRESHOLD_INTERPRETER = 25;
int REOPT_THRESHOLD_INTERPRETER = 25;
int OSR_THRESHOLD_BASELINE = 2500;
int REOPT_THRESHOLD_BASELINE = 1000;
int OSR_THRESHOLD_T2 = 10000;
int REOPT_THRESHOLD_T2 = 10000;
int SPECULATION_THRESHOLD = 100;
......
src/core/options.h
View file @ fc366564
......@@ -38,8 +38,8 @@ extern int SPECULATION_THRESHOLD;
extern int MAX_OBJECT_CACHE_ENTRIES;
extern bool SHOW_DISASM, FORCE_INTERPRETER, FORCE_OPTIMIZE, PROFILE, DUMPJIT, TRAP, USE_STRIPPED_STDLIB,
CONTINUE_AFTER_FATAL, ENABLE_INTERPRETER, ENABLE_PYPA_PARSER, USE_REGALLOC_BASIC, PAUSE_AT_ABORT, ENABLE_TRACEBACKS,
ASSEMBLY_LOGGING;
CONTINUE_AFTER_FATAL, ENABLE_INTERPRETER, ENABLE_BASELINEJIT, ENABLE_PYPA_PARSER, USE_REGALLOC_BASIC,
PAUSE_AT_ABORT, ENABLE_TRACEBACKS, ASSEMBLY_LOGGING;
extern bool ENABLE_ICS, ENABLE_ICGENERICS, ENABLE_ICGETITEMS, ENABLE_ICSETITEMS, ENABLE_ICDELITEMS, ENABLE_ICBINEXPS,
ENABLE_ICNONZEROS, ENABLE_ICCALLSITES, ENABLE_ICSETATTRS, ENABLE_ICGETATTRS, ENALBE_ICDELATTRS, ENABLE_ICGETGLOBALS,
......
src/core/types.h
View file @ fc366564
......@@ -65,7 +65,6 @@ using gc::GCVisitor;
enum class EffortLevel {
INTERPRETED = 0,
MINIMAL = 1,
MODERATE = 2,
MAXIMAL = 3,
};
......@@ -221,6 +220,7 @@ class BoxedClosure;
class BoxedGenerator;
class ICInfo;
class LocationMap;
class JitCodeBlock;
struct CompiledFunction {
private:
......@@ -249,21 +249,10 @@ public:
LocationMap* location_map; // only meaningful if this is a compiled frame
std::vector<ICInfo*> ics;
std::vector<std::unique_ptr<JitCodeBlock>> code_blocks;
CompiledFunction(llvm::Function* func, FunctionSpecialization* spec, bool is_interpreted, void* code,
EffortLevel effort, const OSREntryDescriptor* entry_descriptor)
: clfunc(NULL),
func(func),
spec(spec),
entry_descriptor(entry_descriptor),
is_interpreted(is_interpreted),
code(code),
effort(effort),
times_called(0),
times_speculation_failed(0),
location_map(nullptr) {
assert((spec != NULL) + (entry_descriptor != NULL) == 1);
}
EffortLevel effort, const OSREntryDescriptor* entry_descriptor);
ConcreteCompilerType* getReturnType();
......@@ -282,6 +271,7 @@ typedef int FutureFlags;
class BoxedModule;
class ScopeInfo;
class InternedStringPool;
class LivenessAnalysis;
class SourceInfo {
public:
BoxedModule* parent_module;
......@@ -295,6 +285,7 @@ public:
InternedStringPool& getInternedStrings();
ScopeInfo* getScopeInfo();
LivenessAnalysis* getLiveness();
// TODO we're currently copying the body of the AST into here, since lambdas don't really have a statement-based
// body and we have to create one. Ideally, we'd be able to avoid the space duplication for non-lambdas.
......@@ -307,6 +298,10 @@ public:
SourceInfo(BoxedModule* m, ScopingAnalysis* scoping, FutureFlags future_flags, AST* ast,
std::vector<AST_stmt*> body, std::string fn);
~SourceInfo();
private:
std::unique_ptr<LivenessAnalysis> liveness_info;
};
typedef std::vector<CompiledFunction*> FunctionList;
......
src/runtime/objmodel.cpp
View file @ fc366564
......@@ -226,6 +226,28 @@ extern "C" bool softspace(Box* b, bool newval) {
return r;
}
extern "C" void printHelper(Box* dest, Box* var, bool nl) {
static BoxedString* write_str = static_cast<BoxedString*>(PyString_InternFromString("write"));
static BoxedString* newline_str = static_cast<BoxedString*>(PyString_InternFromString("\n"));
static BoxedString* space_str = static_cast<BoxedString*>(PyString_InternFromString(" "));
if (var) {
// begin code for handling of softspace
bool new_softspace = !nl;
if (softspace(dest, new_softspace))
callattrInternal(dest, write_str, CLASS_OR_INST, 0, ArgPassSpec(1), space_str, 0, 0, 0, 0);
Box* str_or_unicode_var = (var->cls == unicode_cls) ? var : str(var);
callattrInternal(dest, write_str, CLASS_OR_INST, 0, ArgPassSpec(1), str_or_unicode_var, 0, 0, 0, 0);
}
if (nl) {
callattrInternal(dest, write_str, CLASS_OR_INST, 0, ArgPassSpec(1), newline_str, 0, 0, 0, 0);
if (!var)
softspace(dest, false);
}
}
extern "C" void my_assert(bool b) {
assert(b);
}
......
src/runtime/objmodel.h
View file @ fc366564
......@@ -48,6 +48,7 @@ BoxedModule* getCurrentModule();
// TODO sort this
extern "C" bool softspace(Box* b, bool newval);
extern "C" void printHelper(Box* dest, Box* var, bool nl);
extern "C" void my_assert(bool b);
extern "C" Box* getattr(Box* obj, BoxedString* attr);
extern "C" Box* getattrMaybeNonstring(Box* obj, Box* attr);
......
src/runtime/types.h
View file @ fc366564
......@@ -578,6 +578,23 @@ public:
rtn->elts[2] = elt2;
return rtn;
}
static BoxedTuple* create4(Box* elt0, Box* elt1, Box* elt2, Box* elt3) {
BoxedTuple* rtn = new (4) BoxedTuple(4);
rtn->elts[0] = elt0;
rtn->elts[1] = elt1;
rtn->elts[2] = elt2;
rtn->elts[3] = elt3;
return rtn;
}
static BoxedTuple* create5(Box* elt0, Box* elt1, Box* elt2, Box* elt3, Box* elt4) {
BoxedTuple* rtn = new (5) BoxedTuple(5);
rtn->elts[0] = elt0;
rtn->elts[1] = elt1;
rtn->elts[2] = elt2;
rtn->elts[3] = elt3;
rtn->elts[4] = elt4;
return rtn;
}
static BoxedTuple* create(std::initializer_list<Box*> members) { return new (members.size()) BoxedTuple(members); }
static BoxedTuple* create(int64_t size, BoxedClass* cls) {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7