Skip to content

P
Pyston
Commit 79505a1a authored by Kevin Modzelewski's avatar Kevin Modzelewski
Browse files
Options

Merge pull request #640 from kmod/perf5.2 

add tpp_call for faster calling of non-functions
parents 52924ca9 63d008f9
Hide whitespace changes
Inline Side-by-side
Showing with 904 additions and 761 deletions
from_cpython/Include/object.h
View file @ 79505a1a
......@@ -461,6 +461,7 @@ struct _typeobject {
bool _flags[3];
void* _tpp_descr_get;
void* _tpp_hasnext;
void* _tpp_call;
};
/* The *real* layout of a type object when allocated on the heap */
......
src/asm_writing/rewriter.cpp
View file @ 79505a1a
......@@ -694,6 +694,17 @@ RewriterVar* Rewriter::call(bool has_side_effects, void* func_addr, RewriterVar*
return call(has_side_effects, func_addr, args, args_xmm);
}
RewriterVar* Rewriter::call(bool has_side_effects, void* func_addr, RewriterVar* arg0, RewriterVar* arg1,
RewriterVar* arg2, RewriterVar* arg3) {
RewriterVar::SmallVector args;
RewriterVar::SmallVector args_xmm;
args.push_back(arg0);
args.push_back(arg1);
args.push_back(arg2);
args.push_back(arg3);
return call(has_side_effects, func_addr, args, args_xmm);
}
static const Location caller_save_registers[]{
assembler::RAX, assembler::RCX, assembler::RDX, assembler::RSI, assembler::RDI,
assembler::R8, assembler::R9, assembler::R10, assembler::R11, assembler::XMM0,
......@@ -1232,6 +1243,15 @@ int Rewriter::_allocate(RewriterVar* result, int n) {
if (consec == n) {
int a = i / 8 - n + 1;
int b = i / 8;
// Put placeholders in so the array space doesn't get re-allocated.
// This won't get collected, but that's fine.
// Note: make sure to do this marking before the initializeInReg call
for (int j = a; j <= b; j++) {
Location m(Location::Scratch, j * 8);
assert(vars_by_location.count(m) == 0);
vars_by_location[m] = LOCATION_PLACEHOLDER;
}
assembler::Register r = result->initializeInReg();
// TODO should be a LEA instruction
......@@ -1240,13 +1260,6 @@ int Rewriter::_allocate(RewriterVar* result, int n) {
assembler->mov(assembler::RSP, r);
assembler->add(assembler::Immediate(8 * a + rewrite->getScratchRspOffset()), r);
// Put placeholders in so the array space doesn't get re-allocated.
// This won't get collected, but that's fine.
for (int j = a; j <= b; j++) {
Location m(Location::Scratch, j * 8);
vars_by_location[m] = LOCATION_PLACEHOLDER;
}
assertConsistent();
result->releaseIfNoUses();
return a;
......@@ -1301,12 +1314,12 @@ RewriterVar* Rewriter::allocateAndCopyPlus1(RewriterVar* first_elem, RewriterVar
void Rewriter::_allocateAndCopyPlus1(RewriterVar* result, RewriterVar* first_elem, RewriterVar* rest_ptr, int n_rest) {
int offset = _allocate(result, n_rest + 1);
assembler::Register tmp = first_elem->getInReg();
assembler->mov(tmp, assembler::Indirect(assembler::RSP, 8 * offset + rewrite->getScratchRspOffset()));
assembler::Register first_reg = first_elem->getInReg();
assembler->mov(first_reg, assembler::Indirect(assembler::RSP, 8 * offset + rewrite->getScratchRspOffset()));
if (n_rest > 0) {
assembler::Register src_ptr = rest_ptr->getInReg();
// TODO if this triggers we'll need a way to allocate two distinct registers
assembler::Register tmp = allocReg(Location::any(), /* otherThan */ src_ptr);
assert(tmp != src_ptr);
for (int i = 0; i < n_rest; i++) {
......
src/asm_writing/rewriter.h
View file @ 79505a1a
......@@ -505,6 +505,8 @@ public:
RewriterVar* call(bool has_side_effects, void* func_addr, RewriterVar* arg0);
RewriterVar* call(bool has_side_effects, void* func_addr, RewriterVar* arg0, RewriterVar* arg1);
RewriterVar* call(bool has_side_effects, void* func_addr, RewriterVar* arg0, RewriterVar* arg1, RewriterVar* arg2);
RewriterVar* call(bool has_side_effects, void* func_addr, RewriterVar* arg0, RewriterVar* arg1, RewriterVar* arg2,
RewriterVar* arg3);
RewriterVar* add(RewriterVar* a, int64_t b, Location dest);
// Allocates n pointer-sized stack slots:
RewriterVar* allocate(int n);
......
src/codegen/ast_interpreter.cpp
View file @ 79505a1a
......@@ -752,7 +752,7 @@ Value ASTInterpreter::visit_stmt(AST_stmt* node) {
#endif
if (0) {
printf("%20s % 2d ", source_info->getName().c_str(), current_block->idx);
printf("%20s % 2d ", source_info->getName().data(), current_block->idx);
print_ast(node);
printf("\n");
}
......
src/codegen/compvars.cpp
View file @ 79505a1a
......@@ -828,7 +828,7 @@ public:
Sig* type_sig = new Sig();
type_sig->rtn_type = fspec->rtn_type;
type_sig->ndefaults = clf->num_defaults;
type_sig->ndefaults = clf->paramspec.num_defaults;
if (stripfirst) {
assert(fspec->arg_types.size() >= 1);
......@@ -1545,12 +1545,14 @@ public:
CLFunction* cl = rtattr_func->f;
assert(cl);
if (cl->takes_varargs || cl->takes_kwargs)
ParamReceiveSpec paramspec = cl->paramspec;
if (paramspec.takes_varargs || paramspec.takes_kwargs)
return NULL;
RELEASE_ASSERT(cl->num_args == cl->numReceivedArgs(), "");
RELEASE_ASSERT(args.size() + 1 >= cl->num_args - cl->num_defaults && args.size() + 1 <= cl->num_args, "%d",
info.unw_info.current_stmt->lineno);
RELEASE_ASSERT(paramspec.num_args == cl->numReceivedArgs(), "");
RELEASE_ASSERT(args.size() + 1 >= paramspec.num_args - paramspec.num_defaults
&& args.size() + 1 <= paramspec.num_args,
"%d", info.unw_info.current_stmt->lineno);
CompiledFunction* cf = NULL;
bool found = false;
......@@ -1578,8 +1580,8 @@ public:
assert(cf->code);
std::vector<llvm::Type*> arg_types;
RELEASE_ASSERT(cl->num_args == cl->numReceivedArgs(), "");
for (int i = 0; i < cl->num_args; i++) {
RELEASE_ASSERT(paramspec.num_args == cl->numReceivedArgs(), "");
for (int i = 0; i < paramspec.num_args; i++) {
// TODO support passing unboxed values as arguments
assert(cf->spec->arg_types[i]->llvmType() == g.llvm_value_type_ptr);
......@@ -1602,9 +1604,9 @@ public:
new_args.push_back(var);
new_args.insert(new_args.end(), args.begin(), args.end());
for (int i = args.size() + 1; i < cl->num_args; i++) {
for (int i = args.size() + 1; i < paramspec.num_args; i++) {
// TODO should _call() be able to take llvm::Value's directly?
auto value = rtattr_func->defaults->elts[i - cl->num_args + cl->num_defaults];
auto value = rtattr_func->defaults->elts[i - paramspec.num_args + paramspec.num_defaults];
llvm::Value* llvm_value;
if (value)
llvm_value = embedRelocatablePtr(value, g.llvm_value_type_ptr);
......
src/codegen/irgen/hooks.cpp
View file @ 79505a1a
......@@ -87,7 +87,7 @@ InternedStringPool& SourceInfo::getInternedStrings() {
return scoping->getInternedStrings();
}
const std::string SourceInfo::getName() {
llvm::StringRef SourceInfo::getName() {
assert(ast);
switch (ast->type) {
case AST_TYPE::ClassDef:
......
src/core/cfg.cpp
View file @ 79505a1a
......@@ -2595,7 +2595,7 @@ CFG* computeCFG(SourceInfo* source, std::vector<AST_stmt*> body) {
if (b->predecessors.size() == 0) {
if (b != rtn->getStartingBlock()) {
rtn->print();
printf("%s\n", source->getName().c_str());
printf("%s\n", source->getName().data());
}
ASSERT(b == rtn->getStartingBlock(), "%d", b->idx);
}
......
src/core/types.h
View file @ 79505a1a
......@@ -38,44 +38,6 @@ class Value;
namespace pyston {
struct ArgPassSpec {
bool has_starargs : 1;
bool has_kwargs : 1;
unsigned int num_keywords : 14;
unsigned int num_args : 16;
static const int MAX_ARGS = (1 << 16) - 1;
static const int MAX_KEYWORDS = (1 << 14) - 1;
explicit ArgPassSpec(int num_args) : has_starargs(false), has_kwargs(false), num_keywords(0), num_args(num_args) {
assert(num_args <= MAX_ARGS);
assert(num_keywords <= MAX_KEYWORDS);
}
explicit ArgPassSpec(int num_args, int num_keywords, bool has_starargs, bool has_kwargs)
: has_starargs(has_starargs), has_kwargs(has_kwargs), num_keywords(num_keywords), num_args(num_args) {
assert(num_args <= MAX_ARGS);
assert(num_keywords <= MAX_KEYWORDS);
}
bool operator==(ArgPassSpec rhs) {
return has_starargs == rhs.has_starargs && has_kwargs == rhs.has_kwargs && num_keywords == rhs.num_keywords
&& num_args == rhs.num_args;
}
bool operator!=(ArgPassSpec rhs) { return !(*this == rhs); }
int totalPassed() { return num_args + num_keywords + (has_starargs ? 1 : 0) + (has_kwargs ? 1 : 0); }
uint32_t asInt() const { return *reinterpret_cast<const uint32_t*>(this); }
void dump() {
printf("(has_starargs=%s, has_kwargs=%s, num_keywords=%d, num_args=%d)\n", has_starargs ? "true" : "false",
has_kwargs ? "true" : "false", num_keywords, num_args);
}
};
static_assert(sizeof(ArgPassSpec) <= sizeof(void*), "ArgPassSpec doesn't fit in register! (CC is probably wrong)");
static_assert(sizeof(ArgPassSpec) == sizeof(uint32_t), "ArgPassSpec::asInt needs to be updated");
namespace gc {
class TraceStack;
......@@ -144,6 +106,92 @@ class ScopingAnalysis;
class CLFunction;
class OSREntryDescriptor;
struct ArgPassSpec {
bool has_starargs : 1;
bool has_kwargs : 1;
unsigned int num_keywords : 14;
unsigned int num_args : 16;
static const int MAX_ARGS = (1 << 16) - 1;
static const int MAX_KEYWORDS = (1 << 14) - 1;
explicit ArgPassSpec(int num_args) : has_starargs(false), has_kwargs(false), num_keywords(0), num_args(num_args) {
assert(num_args <= MAX_ARGS);
assert(num_keywords <= MAX_KEYWORDS);
}
explicit ArgPassSpec(int num_args, int num_keywords, bool has_starargs, bool has_kwargs)
: has_starargs(has_starargs), has_kwargs(has_kwargs), num_keywords(num_keywords), num_args(num_args) {
assert(num_args <= MAX_ARGS);
assert(num_keywords <= MAX_KEYWORDS);
}
bool operator==(ArgPassSpec rhs) {
return has_starargs == rhs.has_starargs && has_kwargs == rhs.has_kwargs && num_keywords == rhs.num_keywords
&& num_args == rhs.num_args;
}
bool operator!=(ArgPassSpec rhs) { return !(*this == rhs); }
int totalPassed() { return num_args + num_keywords + (has_starargs ? 1 : 0) + (has_kwargs ? 1 : 0); }
uint32_t asInt() const { return *reinterpret_cast<const uint32_t*>(this); }
void dump() {
printf("(has_starargs=%s, has_kwargs=%s, num_keywords=%d, num_args=%d)\n", has_starargs ? "true" : "false",
has_kwargs ? "true" : "false", num_keywords, num_args);
}
};
static_assert(sizeof(ArgPassSpec) <= sizeof(void*), "ArgPassSpec doesn't fit in register! (CC is probably wrong)");
static_assert(sizeof(ArgPassSpec) == sizeof(uint32_t), "ArgPassSpec::asInt needs to be updated");
struct ParamNames {
bool takes_param_names;
std::vector<llvm::StringRef> args;
llvm::StringRef vararg, kwarg;
explicit ParamNames(AST* ast, InternedStringPool& pool);
ParamNames(const std::vector<llvm::StringRef>& args, llvm::StringRef vararg, llvm::StringRef kwarg);
static ParamNames empty() { return ParamNames(); }
int totalParameters() const {
return args.size() + (vararg.str().size() == 0 ? 0 : 1) + (kwarg.str().size() == 0 ? 0 : 1);
}
private:
ParamNames() : takes_param_names(false) {}
};
// Probably overkill to copy this from ArgPassSpec
struct ParamReceiveSpec {
bool takes_varargs : 1;
bool takes_kwargs : 1;
unsigned int num_defaults : 14;
unsigned int num_args : 16;
static const int MAX_ARGS = (1 << 16) - 1;
static const int MAX_DEFAULTS = (1 << 14) - 1;
explicit ParamReceiveSpec(int num_args)
: takes_varargs(false), takes_kwargs(false), num_defaults(0), num_args(num_args) {
assert(num_args <= MAX_ARGS);
assert(num_defaults <= MAX_DEFAULTS);
}
explicit ParamReceiveSpec(int num_args, int num_defaults, bool takes_varargs, bool takes_kwargs)
: takes_varargs(takes_varargs), takes_kwargs(takes_kwargs), num_defaults(num_defaults), num_args(num_args) {
assert(num_args <= MAX_ARGS);
assert(num_defaults <= MAX_DEFAULTS);
}
bool operator==(ParamReceiveSpec rhs) {
return takes_varargs == rhs.takes_varargs && takes_kwargs == rhs.takes_kwargs
&& num_defaults == rhs.num_defaults && num_args == rhs.num_args;
}
bool operator!=(ParamReceiveSpec rhs) { return !(*this == rhs); }
int totalReceived() { return num_args + (takes_varargs ? 1 : 0) + (takes_kwargs ? 1 : 0); }
};
class ICInvalidator {
private:
int64_t cur_version;
......@@ -229,23 +277,6 @@ public:
void speculationFailed();
};
struct ParamNames {
bool takes_param_names;
std::vector<llvm::StringRef> args;
llvm::StringRef vararg, kwarg;
explicit ParamNames(AST* ast, InternedStringPool& pool);
ParamNames(const std::vector<llvm::StringRef>& args, llvm::StringRef vararg, llvm::StringRef kwarg);
static ParamNames empty() { return ParamNames(); }
int totalParameters() const {
return args.size() + (vararg.str().size() == 0 ? 0 : 1) + (kwarg.str().size() == 0 ? 0 : 1);
}
private:
ParamNames() : takes_param_names(false) {}
};
typedef int FutureFlags;
class BoxedModule;
......@@ -269,7 +300,7 @@ public:
// body and we have to create one. Ideally, we'd be able to avoid the space duplication for non-lambdas.
const std::vector<AST_stmt*> body;
const std::string getName();
llvm::StringRef getName();
InternedString mangleName(InternedString id);
Box* getDocString();
......@@ -283,9 +314,7 @@ struct CallRewriteArgs;
class BoxedCode;
class CLFunction {
public:
int num_args;
int num_defaults;
bool takes_varargs, takes_kwargs;
ParamReceiveSpec paramspec;
std::unique_ptr<SourceInfo> source;
ParamNames param_names;
......@@ -308,10 +337,7 @@ public:
CLFunction(int num_args, int num_defaults, bool takes_varargs, bool takes_kwargs,
std::unique_ptr<SourceInfo> source)
: num_args(num_args),
num_defaults(num_defaults),
takes_varargs(takes_varargs),
takes_kwargs(takes_kwargs),
: paramspec(num_args, num_defaults, takes_varargs, takes_kwargs),
source(std::move(source)),
param_names(this->source->ast, this->source->getInternedStrings()),
always_use_version(NULL),
......@@ -319,10 +345,7 @@ public:
assert(num_args >= num_defaults);
}
CLFunction(int num_args, int num_defaults, bool takes_varargs, bool takes_kwargs, const ParamNames& param_names)
: num_args(num_args),
num_defaults(num_defaults),
takes_varargs(takes_varargs),
takes_kwargs(takes_kwargs),
: paramspec(num_args, num_defaults, takes_varargs, takes_kwargs),
source(nullptr),
param_names(param_names),
always_use_version(NULL),
......@@ -330,7 +353,7 @@ public:
assert(num_args >= num_defaults);
}
int numReceivedArgs() { return num_args + (takes_varargs ? 1 : 0) + (takes_kwargs ? 1 : 0); }
int numReceivedArgs() { return paramspec.totalReceived(); }
void addVersion(CompiledFunction* compiled) {
assert(compiled);
......@@ -344,7 +367,7 @@ public:
&& compiled->spec->boxed_return_value)
always_use_version = compiled;
assert(compiled->spec->arg_types.size() == num_args + (takes_varargs ? 1 : 0) + (takes_kwargs ? 1 : 0));
assert(compiled->spec->arg_types.size() == paramspec.totalReceived());
versions.push_back(compiled);
} else {
osr_versions[compiled->entry_descriptor] = compiled;
......
src/runtime/code.cpp
View file @ 79505a1a
......@@ -72,7 +72,7 @@ public:
static Box* argcount(Box* b, void*) {
RELEASE_ASSERT(b->cls == code_cls, "");
return boxInt(static_cast<BoxedCode*>(b)->f->num_args);
return boxInt(static_cast<BoxedCode*>(b)->f->paramspec.num_args);
}
static Box* varnames(Box* b, void*) {
......
src/runtime/descr.cpp
View file @ 79505a1a
......@@ -469,6 +469,81 @@ Box* BoxedWrapperObject::__call__(BoxedWrapperObject* self, Box* args, Box* kwds
return rtn;
}
Box* BoxedWrapperObject::tppCall(Box* _self, CallRewriteArgs* rewrite_args, ArgPassSpec argspec, Box* arg1, Box* arg2,
Box* arg3, Box** args, const std::vector<BoxedString*>* keyword_names) {
assert(_self->cls == wrapperobject_cls);
BoxedWrapperObject* self = static_cast<BoxedWrapperObject*>(_self);
int flags = self->descr->wrapper->flags;
wrapperfunc wrapper = self->descr->wrapper->wrapper;
assert(self->descr->wrapper->offset > 0);
if (rewrite_args && !rewrite_args->func_guarded) {
rewrite_args->obj->addAttrGuard(offsetof(BoxedWrapperObject, descr), (intptr_t)self->descr);
}
ParamReceiveSpec paramspec(0, 0, true, false);
if (flags == PyWrapperFlag_KEYWORDS) {
paramspec = ParamReceiveSpec(0, 0, true, true);
} else if (flags == PyWrapperFlag_PYSTON || flags == 0) {
paramspec = ParamReceiveSpec(0, 0, true, false);
} else {
RELEASE_ASSERT(0, "%d", flags);
}
Box* oarg1 = NULL;
Box* oarg2 = NULL;
Box* oarg3 = NULL;
Box** oargs = NULL;
bool rewrite_success = false;
rearrangeArguments(paramspec, NULL, self->descr->wrapper->name.data(), NULL, rewrite_args, rewrite_success, argspec,
arg1, arg2, arg3, args, keyword_names, oarg1, oarg2, oarg3, args);
assert(oarg1 && oarg1->cls == tuple_cls);
if (!paramspec.takes_kwargs)
assert(oarg2 == NULL);
assert(oarg3 == NULL);
assert(oargs == NULL);
if (!rewrite_success)
rewrite_args = NULL;
Box* rtn;
if (flags == PyWrapperFlag_KEYWORDS) {
wrapperfunc_kwds wk = (wrapperfunc_kwds)wrapper;
rtn = (*wk)(self->obj, oarg1, self->descr->wrapped, oarg2);
if (rewrite_args) {
auto rewriter = rewrite_args->rewriter;
auto r_obj = rewrite_args->obj->getAttr(offsetof(BoxedWrapperObject, obj), Location::forArg(0));
rewrite_args->out_rtn = rewriter->call(
true, (void*)wk, r_obj, rewrite_args->arg1,
rewriter->loadConst((intptr_t)self->descr->wrapped, Location::forArg(2)), rewrite_args->arg2);
rewriter->call(false, (void*)checkAndThrowCAPIException);
rewrite_args->out_success = true;
}
} else if (flags == PyWrapperFlag_PYSTON || flags == 0) {
rtn = (*wrapper)(self->obj, oarg1, self->descr->wrapped);
if (rewrite_args) {
auto rewriter = rewrite_args->rewriter;
auto r_obj = rewrite_args->obj->getAttr(offsetof(BoxedWrapperObject, obj), Location::forArg(0));
rewrite_args->out_rtn
= rewriter->call(true, (void*)wrapper, r_obj, rewrite_args->arg1,
rewriter->loadConst((intptr_t)self->descr->wrapped, Location::forArg(2)));
rewriter->call(false, (void*)checkAndThrowCAPIException);
rewrite_args->out_success = true;
}
} else {
RELEASE_ASSERT(0, "%d", flags);
}
checkAndThrowCAPIException();
assert(rtn && "should have set + thrown an exception!");
return rtn;
}
void BoxedWrapperObject::gcHandler(GCVisitor* v, Box* _o) {
assert(_o->cls == wrapperobject_cls);
BoxedWrapperObject* o = static_cast<BoxedWrapperObject*>(_o);
......@@ -535,6 +610,7 @@ void setupDescr() {
wrapperobject_cls->giveAttr(
"__call__", new BoxedFunction(boxRTFunction((void*)BoxedWrapperObject::__call__, UNKNOWN, 1, 0, true, true)));
wrapperobject_cls->tpp_call = BoxedWrapperObject::tppCall;
wrapperobject_cls->freeze();
}
......
src/runtime/generator.cpp
View file @ 79505a1a
......@@ -310,7 +310,7 @@ extern "C" BoxedGenerator::BoxedGenerator(BoxedFunctionBase* function, Box* arg1
#endif
{
int numArgs = function->f->num_args;
int numArgs = function->f->numReceivedArgs();
if (numArgs > 3) {
numArgs -= 3;
this->args = new (numArgs) GCdArray();
......@@ -384,7 +384,7 @@ extern "C" void generatorGCHandler(GCVisitor* v, Box* b) {
BoxedGenerator* g = (BoxedGenerator*)b;
v->visit(g->function);
int num_args = g->function->f->num_args;
int num_args = g->function->f->numReceivedArgs();
if (num_args >= 1)
v->visit(g->arg1);
if (num_args >= 2)
......
src/runtime/objmodel.cpp
View file @ 79505a1a
......@@ -2583,7 +2583,7 @@ extern "C" void dumpEx(void* p, int levels) {
CLFunction* cl = f->f;
if (cl->source) {
printf("User-defined function '%s'\n", cl->source->getName().c_str());
printf("User-defined function '%s'\n", cl->source->getName().data());
} else {
printf("A builtin function\n");
}
......@@ -2662,6 +2662,8 @@ extern "C" Box* callattrInternal(Box* obj, BoxedString* attr, LookupScope scope,
v->addAttrGuard(BOX_CLS_OFFSET, (intptr_t)args[i - 3]->cls);
}
}
rewrite_args->args_guarded = true;
}
// right now I don't think this is ever called with INST_ONLY?
......@@ -2670,7 +2672,7 @@ extern "C" Box* callattrInternal(Box* obj, BoxedString* attr, LookupScope scope,
// Look up the argument. Pass in the arguments to getattrInternalGeneral or getclsattr_general
// that will shortcut functions by not putting them into instancemethods
Box* bind_obj = NULL; // Initialize this to NULL to allow getattrInternalEx to ignore it
RewriterVar* r_bind_obj;
RewriterVar* r_bind_obj = NULL;
Box* val;
RewriterVar* r_val = NULL;
if (rewrite_args) {
......@@ -2692,114 +2694,28 @@ extern "C" Box* callattrInternal(Box* obj, BoxedString* attr, LookupScope scope,
}
if (bind_obj != NULL) {
Box** new_args = NULL;
if (npassed_args >= 3) {
new_args = (Box**)alloca(sizeof(Box*) * (npassed_args + 1 - 3));
}
if (rewrite_args) {
r_val->addGuard((int64_t)val);
rewrite_args->obj = r_val;
rewrite_args->func_guarded = true;
}
// TODO copy from runtimeCall
// TODO these two branches could probably be folded together (the first one is becoming
// a subset of the second)
if (npassed_args <= 2) {
Box* rtn;
if (rewrite_args) {
CallRewriteArgs srewrite_args(rewrite_args->rewriter, r_val, rewrite_args->destination);
srewrite_args.arg1 = r_bind_obj;
// should be no-ops:
if (npassed_args >= 1)
srewrite_args.arg2 = rewrite_args->arg1;
if (npassed_args >= 2)
srewrite_args.arg3 = rewrite_args->arg2;
srewrite_args.func_guarded = true;
srewrite_args.args_guarded = true;
rtn = runtimeCallInternal(val, &srewrite_args, ArgPassSpec(argspec.num_args + 1, argspec.num_keywords,
argspec.has_starargs, argspec.has_kwargs),
bind_obj, arg1, arg2, NULL, keyword_names);
if (!srewrite_args.out_success) {
rewrite_args = NULL;
} else {
rewrite_args->out_rtn = srewrite_args.out_rtn;
}
} else {
rtn = runtimeCallInternal(val, NULL, ArgPassSpec(argspec.num_args + 1, argspec.num_keywords,
argspec.has_starargs, argspec.has_kwargs),
bind_obj, arg1, arg2, NULL, keyword_names);
}
if (rewrite_args)
rewrite_args->out_success = true;
return rtn;
} else {
int alloca_size = sizeof(Box*) * (npassed_args + 1 - 3);
Box** new_args = (Box**)alloca(alloca_size);
new_args[0] = arg3;
memcpy(new_args + 1, args, (npassed_args - 3) * sizeof(Box*));
Box* rtn;
if (rewrite_args) {
CallRewriteArgs srewrite_args(rewrite_args->rewriter, r_val, rewrite_args->destination);
srewrite_args.arg1 = r_bind_obj;
srewrite_args.arg2 = rewrite_args->arg1;
srewrite_args.arg3 = rewrite_args->arg2;
srewrite_args.args = rewrite_args->rewriter->allocateAndCopyPlus1(
rewrite_args->arg3, npassed_args == 3 ? NULL : rewrite_args->args, npassed_args - 3);
srewrite_args.args_guarded = true;
srewrite_args.func_guarded = true;
rtn = runtimeCallInternal(val, &srewrite_args, ArgPassSpec(argspec.num_args + 1, argspec.num_keywords,
argspec.has_starargs, argspec.has_kwargs),
bind_obj, arg1, arg2, new_args, keyword_names);
if (!srewrite_args.out_success) {
rewrite_args = NULL;
} else {
rewrite_args->out_rtn = srewrite_args.out_rtn;
rewrite_args->out_success = true;
}
} else {
rtn = runtimeCallInternal(val, NULL, ArgPassSpec(argspec.num_args + 1, argspec.num_keywords,
argspec.has_starargs, argspec.has_kwargs),
bind_obj, arg1, arg2, new_args, keyword_names);
}
return rtn;
}
} else {
Box* rtn;
ArgPassSpec new_argspec
= bindObjIntoArgs(bind_obj, r_bind_obj, rewrite_args, argspec, arg1, arg2, arg3, args, new_args);
return runtimeCallInternal(val, rewrite_args, new_argspec, arg1, arg2, arg3, new_args, keyword_names);
} else {
if (rewrite_args) {
CallRewriteArgs srewrite_args(rewrite_args->rewriter, r_val, rewrite_args->destination);
if (npassed_args >= 1)
srewrite_args.arg1 = rewrite_args->arg1;
if (npassed_args >= 2)
srewrite_args.arg2 = rewrite_args->arg2;
if (npassed_args >= 3)
srewrite_args.arg3 = rewrite_args->arg3;
if (npassed_args >= 4)
srewrite_args.args = rewrite_args->args;
srewrite_args.args_guarded = true;
rtn = runtimeCallInternal(val, &srewrite_args, argspec, arg1, arg2, arg3, args, keyword_names);
if (!srewrite_args.out_success) {
rewrite_args = NULL;
} else {
rewrite_args->out_rtn = srewrite_args.out_rtn;
}
} else {
rtn = runtimeCallInternal(val, NULL, argspec, arg1, arg2, arg3, args, keyword_names);
rewrite_args->obj = r_val;
}
if (!rtn) {
raiseExcHelper(TypeError, "'%s' object is not callable", getTypeName(val));
}
if (rewrite_args)
rewrite_args->out_success = true;
Box* rtn = runtimeCallInternal(val, rewrite_args, argspec, arg1, arg2, arg3, args, keyword_names);
assert(rtn); // not sure why we have this here
return rtn;
}
}
......@@ -2876,16 +2792,6 @@ extern "C" Box* callattr(Box* obj, BoxedString* attr, CallattrFlags flags, ArgPa
return rtn;
}
static inline Box*& getArg(int idx, Box*& arg1, Box*& arg2, Box*& arg3, Box** args) {
if (idx == 0)
return arg1;
if (idx == 1)
return arg2;
if (idx == 2)
return arg3;
return args[idx - 3];
}
static inline RewriterVar* getArg(int idx, CallRewriteArgs* rewrite_args) {
if (idx == 0)
return rewrite_args->arg1;
......@@ -2962,13 +2868,14 @@ static CompiledFunction* pickVersion(CLFunction* f, int num_output_args, Box* oa
return compileFunction(f, spec, new_effort, NULL);
}
static std::string getFunctionName(CLFunction* f) {
static llvm::StringRef getFunctionName(CLFunction* f) {
if (f->source)
return f->source->getName();
else if (f->versions.size()) {
std::ostringstream oss;
oss << "<function at " << f->versions[0]->code << ">";
return oss.str();
return "<builtin function>";
// std::ostringstream oss;
// oss << "<function at " << f->versions[0]->code << ">";
// return oss.str();
}
return "<unknown function>";
}
......@@ -2977,19 +2884,19 @@ enum class KeywordDest {
POSITIONAL,
KWARGS,
};
static KeywordDest placeKeyword(const ParamNames& param_names, llvm::SmallVector<bool, 8>& params_filled,
static KeywordDest placeKeyword(const ParamNames* param_names, llvm::SmallVector<bool, 8>& params_filled,
BoxedString* kw_name, Box* kw_val, Box*& oarg1, Box*& oarg2, Box*& oarg3, Box** oargs,
BoxedDict* okwargs, CLFunction* cl) {
BoxedDict* okwargs, const char* func_name) {
assert(kw_val);
assert(gc::isValidGCObject(kw_val));
assert(kw_name);
assert(gc::isValidGCObject(kw_name));
for (int j = 0; j < param_names.args.size(); j++) {
if (param_names.args[j] == kw_name->s() && kw_name->size() > 0) {
for (int j = 0; j < param_names->args.size(); j++) {
if (param_names->args[j] == kw_name->s() && kw_name->size() > 0) {
if (params_filled[j]) {
raiseExcHelper(TypeError, "%.200s() got multiple values for keyword argument '%s'",
getFunctionName(cl).c_str(), kw_name->c_str());
raiseExcHelper(TypeError, "%.200s() got multiple values for keyword argument '%s'", func_name,
kw_name->c_str());
}
getArg(j, oarg1, oarg2, oarg3, oargs) = kw_val;
......@@ -3002,14 +2909,13 @@ static KeywordDest placeKeyword(const ParamNames& param_names, llvm::SmallVector
if (okwargs) {
Box*& v = okwargs->d[kw_name];
if (v) {
raiseExcHelper(TypeError, "%.200s() got multiple values for keyword argument '%s'",
getFunctionName(cl).c_str(), kw_name->c_str());
raiseExcHelper(TypeError, "%.200s() got multiple values for keyword argument '%s'", func_name,
kw_name->c_str());
}
v = kw_val;
return KeywordDest::KWARGS;
} else {
raiseExcHelper(TypeError, "%.200s() got an unexpected keyword argument '%s'", getFunctionName(cl).c_str(),
kw_name->c_str());
raiseExcHelper(TypeError, "%.200s() got an unexpected keyword argument '%s'", func_name, kw_name->c_str());
}
}
......@@ -3020,15 +2926,40 @@ static Box* _callFuncHelper(BoxedFunctionBase* func, ArgPassSpec argspec, Box* a
return callFunc(func, NULL, argspec, arg1, arg2, arg3, args, keyword_names);
}
static StatCounter slowpath_callfunc("slowpath_callfunc");
static StatCounter slowpath_callfunc_slowpath("slowpath_callfunc_slowpath");
Box* callFunc(BoxedFunctionBase* func, CallRewriteArgs* rewrite_args, ArgPassSpec argspec, Box* arg1, Box* arg2,
Box* arg3, Box** args, const std::vector<BoxedString*>* keyword_names) {
#if STAT_TIMERS
StatTimer::assertActive();
STAT_TIMER(t0, "us_timer_slowpath_callFunc", 0);
#endif
typedef std::function<Box*(int, int, RewriterVar*&)> GetDefaultFunc;
ArgPassSpec bindObjIntoArgs(Box* bind_obj, RewriterVar* r_bind_obj, CallRewriteArgs* rewrite_args, ArgPassSpec argspec,
Box*& arg1, Box*& arg2, Box*& arg3, Box** args, Box** new_args) {
int npassed_args = argspec.totalPassed();
assert((new_args != NULL) == (npassed_args >= 3));
if (npassed_args >= 3) {
new_args[0] = arg3;
memcpy(new_args + 1, args, (npassed_args - 3) * sizeof(Box*));
}
arg3 = arg2;
arg2 = arg1;
arg1 = bind_obj;
if (rewrite_args) {
if (npassed_args >= 3) {
rewrite_args->args = rewrite_args->rewriter->allocateAndCopyPlus1(
rewrite_args->arg3, npassed_args == 3 ? NULL : rewrite_args->args, npassed_args - 3);
}
rewrite_args->arg3 = rewrite_args->arg2;
rewrite_args->arg2 = rewrite_args->arg1;
rewrite_args->arg1 = r_bind_obj;
}
return ArgPassSpec(argspec.num_args + 1, argspec.num_keywords, argspec.has_starargs, argspec.has_kwargs);
}
void rearrangeArguments(ParamReceiveSpec paramspec, const ParamNames* param_names, const char* func_name,
Box** defaults, CallRewriteArgs* rewrite_args, bool& rewrite_success, ArgPassSpec argspec,
Box* arg1, Box* arg2, Box* arg3, Box** args, const std::vector<BoxedString*>* keyword_names,
Box*& oarg1, Box*& oarg2, Box*& oarg3, Box** oargs) {
/*
* Procedure:
* - First match up positional arguments; any extra go to varargs. error if too many.
......@@ -3037,12 +2968,7 @@ Box* callFunc(BoxedFunctionBase* func, CallRewriteArgs* rewrite_args, ArgPassSpe
* - error about missing parameters
*/
BoxedClosure* closure = func->closure;
CLFunction* f = func->f;
slowpath_callfunc.log();
int num_output_args = f->numReceivedArgs();
int num_output_args = paramspec.totalReceived();
int num_passed_args = argspec.totalPassed();
if (num_passed_args >= 1)
......@@ -3055,126 +2981,76 @@ Box* callFunc(BoxedFunctionBase* func, CallRewriteArgs* rewrite_args, ArgPassSpe
assert(gc::isValidGCObject(args[i - 3]) || args[i - 3] == NULL);
}
// TODO Should we guard on the CLFunction or the BoxedFunctionBase?
// A single CLFunction could end up forming multiple BoxedFunctionBases, and we
// could emit assembly that handles any of them. But doing this involves some
// extra indirection, and it's not clear if that's worth it, since it seems like
// the common case will be functions only ever getting a single set of default arguments.
bool guard_clfunc = false;
assert(!guard_clfunc && "I think there are users that expect the boxedfunction to be guarded");
assert((defaults != NULL) == (paramspec.num_defaults != 0));
if (rewrite_args) {
assert(rewrite_args->args_guarded && "need to guard args here");
if (!rewrite_args->func_guarded) {
if (guard_clfunc) {
rewrite_args->obj->addAttrGuard(offsetof(BoxedFunctionBase, f), (intptr_t)f);
} else {
rewrite_args->obj->addGuard((intptr_t)func);
}
rewrite_args->rewriter->addDependenceOn(func->dependent_ics);
}
rewrite_success = false; // default case
}
// Fast path: if it's a simple-enough call, we don't have to do anything special. On a simple
// django-admin test this covers something like 93% of all calls to callFunc.
if (!f->isGenerator()) {
if (argspec.num_keywords == 0 && argspec.has_starargs == f->takes_varargs && !argspec.has_kwargs
&& !f->takes_kwargs && argspec.num_args == f->num_args) {
// If the caller passed starargs, we can only pass those directly to the callee if it's a tuple,
// since otherwise modifications by the callee would be visible to the caller (hence why varargs
// received by the caller are always tuples).
// This is why we can't pass kwargs here.
if (argspec.has_starargs) {
Box* given_varargs = getArg(argspec.num_args + argspec.num_keywords, arg1, arg2, arg3, args);
if (given_varargs->cls == tuple_cls) {
if (rewrite_args) {
getArg(argspec.num_args + argspec.num_keywords, rewrite_args)
->addAttrGuard(offsetof(Box, cls), (intptr_t)tuple_cls);
}
return callCLFunc(f, rewrite_args, argspec.num_args + argspec.has_starargs + argspec.has_kwargs,
closure, NULL, func->globals, arg1, arg2, arg3, args);
if (argspec.num_keywords == 0 && argspec.has_starargs == paramspec.takes_varargs && !argspec.has_kwargs
&& !paramspec.takes_kwargs && argspec.num_args == paramspec.num_args) {
assert(num_output_args == num_passed_args);
// If the caller passed starargs, we can only pass those directly to the callee if it's a tuple,
// since otherwise modifications by the callee would be visible to the caller (hence why varargs
// received by the caller are always tuples).
// This is why we can't pass kwargs here.
if (argspec.has_starargs) {
Box* given_varargs = getArg(argspec.num_args + argspec.num_keywords, arg1, arg2, arg3, args);
if (given_varargs->cls == tuple_cls) {
if (rewrite_args) {
getArg(argspec.num_args + argspec.num_keywords, rewrite_args)
->addAttrGuard(offsetof(Box, cls), (intptr_t)tuple_cls);
}
} else {
return callCLFunc(f, rewrite_args, argspec.num_args + argspec.has_starargs + argspec.has_kwargs,
closure, NULL, func->globals, arg1, arg2, arg3, args);
rewrite_success = true;
oarg1 = arg1;
oarg2 = arg2;
oarg3 = arg3;
if (num_output_args > 3)
memcpy(oargs, args, sizeof(Box*) * (num_output_args - 3));
return;
}
} else {
rewrite_success = true;
oarg1 = arg1;
oarg2 = arg2;
oarg3 = arg3;
if (num_output_args > 3)
memcpy(oargs, args, sizeof(Box*) * (num_output_args - 3));
return;
}
}
slowpath_callfunc_slowpath.log();
if (argspec.has_starargs || argspec.has_kwargs || argspec.num_keywords || f->isGenerator()) {
// These are the cases that we won't be able to rewrite.
// So instead, just rewrite them to be a call to callFunc, which helps a little bit.
// TODO we should extract the rest of this function from the end of this block,
// put it in a different function, and have the rewrites target that.
static StatCounter slowpath_rearrangeargs_slowpath("slowpath_rearrangeargs_slowpath");
slowpath_rearrangeargs_slowpath.log();
// Note(kmod): I tried moving this section to runtimeCallInternal, ie to the place that calls
// callFunc. The thought was that this would let us apply this same optimization to other
// internal callables. It ended up hurting perf slightly; my theory is that it's because if
// an internal callable failed, it's better to call the non-internal version since it's lower
// overhead.
// To investigate the cases where we can't rewrite, enable this block.
// This also will also log the times that we call into callFunc directly
// from a rewrite.
#if 0
char buf[80];
snprintf(buf, sizeof(buf), "zzz_aborted_%d_args_%d_%d_%d_%d_params_%d_%d_%d_%d", f->isGenerator(),
argspec.num_args, argspec.num_keywords, argspec.has_starargs, argspec.has_kwargs, f->num_args,
f->num_defaults, f->takes_varargs, f->takes_kwargs);
uint64_t* counter = Stats::getStatCounter(buf);
Stats::log(counter);
#endif
if (argspec.has_starargs || argspec.has_kwargs || argspec.num_keywords) {
rewrite_args = NULL;
}
if (rewrite_args) {
Rewriter* rewriter = rewrite_args->rewriter;
// rewriter->trap();
RewriterVar* args_array = rewriter->allocate(2);
if (num_passed_args >= 4) {
RELEASE_ASSERT(rewrite_args->args, "");
args_array->setAttr(0, rewrite_args->args);
}
if (argspec.num_keywords)
args_array->setAttr(8, rewriter->loadConst((intptr_t)keyword_names));
else
args_array->setAttr(8, rewriter->loadConst(0));
if (paramspec.takes_varargs && argspec.num_args > paramspec.num_args + 3) {
// We currently only handle up to 3 arguments into the varargs tuple
rewrite_args = NULL;
}
RewriterVar::SmallVector arg_vec;
arg_vec.push_back(rewrite_args->obj);
arg_vec.push_back(rewriter->loadConst(argspec.asInt(), Location::forArg(1)));
if (num_passed_args >= 1)
arg_vec.push_back(rewrite_args->arg1);
else
arg_vec.push_back(rewriter->loadConst(0, Location::forArg(2)));
if (num_passed_args >= 2)
arg_vec.push_back(rewrite_args->arg2);
else
arg_vec.push_back(rewriter->loadConst(0, Location::forArg(3)));
if (num_passed_args >= 3)
arg_vec.push_back(rewrite_args->arg3);
else
arg_vec.push_back(rewriter->loadConst(0, Location::forArg(4)));
arg_vec.push_back(args_array);
for (auto v : arg_vec)
assert(v);
RewriterVar* r_rtn = rewriter->call(true, (void*)_callFuncHelper, arg_vec);
// At this point we are not allowed to abort the rewrite any more, since we will start
// modifying rewrite_args.
rewrite_args->out_success = true;
rewrite_args->out_rtn = r_rtn;
rewrite_args = NULL;
}
}
if (rewrite_args)
rewrite_success = true;
if (rewrite_args) {
// We might have trouble if we have more output args than input args,
// such as if we need more space to pass defaults.
if (num_output_args > 3 && num_output_args > argspec.totalPassed()) {
if (num_output_args > 3 && num_output_args > num_passed_args) {
int arg_bytes_required = (num_output_args - 3) * sizeof(Box*);
RewriterVar* new_args = NULL;
if (rewrite_args->args == NULL) {
// rewrite_args->args could be empty if there are not more than
// 3 input args.
assert((rewrite_args->args == NULL) == (num_passed_args <= 3));
if (num_passed_args <= 3) {
// we weren't passed args
new_args = rewrite_args->rewriter->allocate(num_output_args - 3);
} else {
new_args = rewrite_args->rewriter->allocateAndCopy(rewrite_args->args, num_output_args - 3);
......@@ -3186,35 +3062,21 @@ Box* callFunc(BoxedFunctionBase* func, CallRewriteArgs* rewrite_args, ArgPassSpe
std::vector<Box*, StlCompatAllocator<Box*>> varargs;
if (argspec.has_starargs) {
assert(!rewrite_args);
Box* given_varargs = getArg(argspec.num_args + argspec.num_keywords, arg1, arg2, arg3, args);
for (Box* e : given_varargs->pyElements()) {
varargs.push_back(e);
}
}
// The "output" args that we will pass to the called function:
Box* oarg1 = NULL, * oarg2 = NULL, * oarg3 = NULL;
Box** oargs = NULL;
if (num_output_args > 3) {
int size = (num_output_args - 3) * sizeof(Box*);
oargs = (Box**)alloca(size);
#ifndef NDEBUG
memset(&oargs[0], 0, size);
#endif
}
////
// First, match up positional parameters to positional/varargs:
int positional_to_positional = std::min((int)argspec.num_args, f->num_args);
int positional_to_positional = std::min(argspec.num_args, paramspec.num_args);
for (int i = 0; i < positional_to_positional; i++) {
getArg(i, oarg1, oarg2, oarg3, oargs) = getArg(i, arg1, arg2, arg3, args);
// we already moved the positional args into position
}
int varargs_to_positional = std::min((int)varargs.size(), f->num_args - positional_to_positional);
int varargs_to_positional = std::min((int)varargs.size(), paramspec.num_args - positional_to_positional);
for (int i = 0; i < varargs_to_positional; i++) {
assert(!rewrite_args && "would need to be handled here");
getArg(i + positional_to_positional, oarg1, oarg2, oarg3, oargs) = varargs[i];
......@@ -3241,13 +3103,12 @@ Box* callFunc(BoxedFunctionBase* func, CallRewriteArgs* rewrite_args, ArgPassSpe
}
}
for (int i = varargs_to_positional; i < varargs.size(); i++) {
rewrite_args = NULL;
REWRITE_ABORTED("");
assert(!rewrite_args);
unused_positional.push_back(varargs[i]);
}
if (f->takes_varargs) {
int varargs_idx = f->num_args;
if (paramspec.takes_varargs) {
int varargs_idx = paramspec.num_args;
if (rewrite_args) {
assert(!varargs.size());
assert(!argspec.has_starargs);
......@@ -3269,8 +3130,8 @@ Box* callFunc(BoxedFunctionBase* func, CallRewriteArgs* rewrite_args, ArgPassSpe
= rewrite_args->rewriter->call(false, (void*)BoxedTuple::create3, unused_positional_rvars[0],
unused_positional_rvars[1], unused_positional_rvars[2]);
} else {
varargs_val = NULL;
rewrite_args = NULL;
// This is too late to abort the rewrite (we should have checked this earlier)
abort();
}
if (varargs_val) {
......@@ -3288,17 +3149,16 @@ Box* callFunc(BoxedFunctionBase* func, CallRewriteArgs* rewrite_args, ArgPassSpe
Box* ovarargs = BoxedTuple::create(unused_positional.size(), &unused_positional[0]);
getArg(varargs_idx, oarg1, oarg2, oarg3, oargs) = ovarargs;
} else if (unused_positional.size()) {
raiseExcHelper(TypeError, "%s() takes at most %d argument%s (%d given)", getFunctionName(f).c_str(),
f->num_args, (f->num_args == 1 ? "" : "s"),
argspec.num_args + argspec.num_keywords + varargs.size());
raiseExcHelper(TypeError, "%s() takes at most %d argument%s (%d given)", func_name, paramspec.num_args,
(paramspec.num_args == 1 ? "" : "s"), argspec.num_args + argspec.num_keywords + varargs.size());
}
////
// Second, apply any keywords:
BoxedDict* okwargs = NULL;
if (f->takes_kwargs) {
int kwargs_idx = f->num_args + (f->takes_varargs ? 1 : 0);
if (paramspec.takes_kwargs) {
int kwargs_idx = paramspec.num_args + (paramspec.takes_varargs ? 1 : 0);
if (rewrite_args) {
RewriterVar* r_kwargs = rewrite_args->rewriter->call(true, (void*)createDict);
......@@ -3316,9 +3176,8 @@ Box* callFunc(BoxedFunctionBase* func, CallRewriteArgs* rewrite_args, ArgPassSpe
getArg(kwargs_idx, oarg1, oarg2, oarg3, oargs) = okwargs;
}
const ParamNames& param_names = f->param_names;
if (!param_names.takes_param_names && argspec.num_keywords && !f->takes_kwargs) {
raiseExcHelper(TypeError, "%s() doesn't take keyword arguments", getFunctionName(f).c_str());
if ((!param_names || !param_names->takes_param_names) && argspec.num_keywords && !paramspec.takes_kwargs) {
raiseExcHelper(TypeError, "%s() doesn't take keyword arguments", func_name);
}
if (argspec.num_keywords)
......@@ -3330,16 +3189,16 @@ Box* callFunc(BoxedFunctionBase* func, CallRewriteArgs* rewrite_args, ArgPassSpe
int arg_idx = i + argspec.num_args;
Box* kw_val = getArg(arg_idx, arg1, arg2, arg3, args);
if (!param_names.takes_param_names) {
if (!param_names || !param_names->takes_param_names) {
assert(okwargs);
rewrite_args = NULL; // would need to add it to r_kwargs
assert(!rewrite_args); // would need to add it to r_kwargs
okwargs->d[(*keyword_names)[i]] = kw_val;
continue;
}
auto dest = placeKeyword(param_names, params_filled, (*keyword_names)[i], kw_val, oarg1, oarg2, oarg3, oargs,
okwargs, f);
rewrite_args = NULL;
okwargs, func_name);
assert(!rewrite_args);
}
if (argspec.has_kwargs) {
......@@ -3360,92 +3219,180 @@ Box* callFunc(BoxedFunctionBase* func, CallRewriteArgs* rewrite_args, ArgPassSpe
auto k = coerceUnicodeToStr(p.first);
if (k->cls != str_cls)
raiseExcHelper(TypeError, "%s() keywords must be strings", getFunctionName(f).c_str());
raiseExcHelper(TypeError, "%s() keywords must be strings", func_name);
BoxedString* s = static_cast<BoxedString*>(k);
if (param_names.takes_param_names) {
if (param_names && param_names->takes_param_names) {
assert(!rewrite_args && "would need to make sure that this didn't need to go into r_kwargs");
placeKeyword(param_names, params_filled, s, p.second, oarg1, oarg2, oarg3, oargs, okwargs, f);
placeKeyword(param_names, params_filled, s, p.second, oarg1, oarg2, oarg3, oargs, okwargs, func_name);
} else {
assert(!rewrite_args && "would need to make sure that this didn't need to go into r_kwargs");
assert(okwargs);
Box*& v = okwargs->d[p.first];
if (v) {
raiseExcHelper(TypeError, "%s() got multiple values for keyword argument '%s'",
getFunctionName(f).c_str(), s->data());
raiseExcHelper(TypeError, "%s() got multiple values for keyword argument '%s'", func_name,
s->data());
}
v = p.second;
rewrite_args = NULL;
assert(!rewrite_args);
}
}
}
// Fill with defaults:
for (int i = 0; i < f->num_args - f->num_defaults; i++) {
for (int i = 0; i < paramspec.num_args - paramspec.num_defaults; i++) {
if (params_filled[i])
continue;
// TODO not right error message
raiseExcHelper(TypeError, "%s() did not get a value for positional argument %d", getFunctionName(f).c_str(), i);
raiseExcHelper(TypeError, "%s() did not get a value for positional argument %d", func_name, i);
}
RewriterVar* r_defaults_array = NULL;
if (guard_clfunc) {
r_defaults_array = rewrite_args->obj->getAttr(offsetof(BoxedFunctionBase, defaults), Location::any());
for (int arg_idx = paramspec.num_args - paramspec.num_defaults; arg_idx < paramspec.num_args; arg_idx++) {
if (params_filled[arg_idx])
continue;
int default_idx = arg_idx + paramspec.num_defaults - paramspec.num_args;
Box* default_obj = defaults[default_idx];
if (rewrite_args) {
if (arg_idx == 0)
rewrite_args->arg1 = rewrite_args->rewriter->loadConst((intptr_t)default_obj, Location::forArg(0));
else if (arg_idx == 1)
rewrite_args->arg2 = rewrite_args->rewriter->loadConst((intptr_t)default_obj, Location::forArg(1));
else if (arg_idx == 2)
rewrite_args->arg3 = rewrite_args->rewriter->loadConst((intptr_t)default_obj, Location::forArg(2));
else
rewrite_args->args->setAttr((arg_idx - 3) * sizeof(Box*),
rewrite_args->rewriter->loadConst((intptr_t)default_obj));
}
getArg(arg_idx, oarg1, oarg2, oarg3, oargs) = default_obj;
}
}
for (int i = f->num_args - f->num_defaults; i < f->num_args; i++) {
if (params_filled[i])
continue;
static StatCounter slowpath_callfunc("slowpath_callfunc");
Box* callFunc(BoxedFunctionBase* func, CallRewriteArgs* rewrite_args, ArgPassSpec argspec, Box* arg1, Box* arg2,
Box* arg3, Box** args, const std::vector<BoxedString*>* keyword_names) {
#if STAT_TIMERS
StatTimer::assertActive();
STAT_TIMER(t0, "us_timer_slowpath_callFunc", 0);
#endif
slowpath_callfunc.log();
CLFunction* f = func->f;
ParamReceiveSpec paramspec = f->paramspec;
if (rewrite_args) {
if (!rewrite_args->func_guarded) {
rewrite_args->obj->addGuard((intptr_t)func);
rewrite_args->rewriter->addDependenceOn(func->dependent_ics);
}
}
int default_idx = i + f->num_defaults - f->num_args;
Box* default_obj = func->defaults->elts[default_idx];
Box* oarg1, *oarg2, *oarg3, **oargs;
bool rewrite_success = false;
int num_output_args = paramspec.totalReceived();
int num_passed_args = argspec.totalPassed();
if (num_output_args > 3) {
int size = (num_output_args - 3) * sizeof(Box*);
oargs = (Box**)alloca(size);
#ifndef NDEBUG
memset(&oargs[0], 0, size);
#endif
} else {
// It won't get looked at, but the compiler wants this:
oargs = NULL;
}
rearrangeArguments(paramspec, &f->param_names, getFunctionName(f).data(),
paramspec.num_defaults ? func->defaults->elts : NULL, rewrite_args, rewrite_success, argspec,
arg1, arg2, arg3, args, keyword_names, oarg1, oarg2, oarg3, oargs);
#if 0
for (int i = 0; i < num_output_args; i++) {
auto arg = getArg(i, oarg1, oarg2, oarg3, oargs);
RELEASE_ASSERT(!arg || gc::isValidGCObject(arg), "%p", arg);
}
#endif
if (rewrite_args && !rewrite_success) {
// These are the cases that we weren't able to rewrite.
// So instead, just rewrite them to be a call to callFunc, which helps a little bit.
// TODO we should extract the rest of this function from the end of this block,
// put it in a different function, and have the rewrites target that.
// Note(kmod): I tried moving this section to runtimeCallInternal, ie to the place that calls
// callFunc. The thought was that this would let us apply this same optimization to other
// internal callables. It ended up hurting perf slightly; my theory is that it's because if
// an internal callable failed, it's better to call the non-internal version since it's lower
// overhead.
// To investigate the cases where we can't rewrite, enable this block.
// This also will also log the times that we call into callFunc directly
// from a rewrite.
#if 0
char buf[80];
snprintf(buf, sizeof(buf), "zzz_aborted_%d_args_%d_%d_%d_%d_params_%d_%d_%d_%d", f->isGenerator(),
argspec.num_args, argspec.num_keywords, argspec.has_starargs, argspec.has_kwargs, paramspec.num_args,
paramspec.num_defaults, paramspec.takes_varargs, paramspec.takes_kwargs);
uint64_t* counter = Stats::getStatCounter(buf);
Stats::log(counter);
#endif
if (rewrite_args) {
int offset = offsetof(std::remove_pointer<decltype(BoxedFunctionBase::defaults)>::type, elts)
+ sizeof(Box*) * default_idx;
if (guard_clfunc) {
// If we just guarded on the CLFunction, then we have to emit assembly
// to fetch the values from the defaults array:
if (i < 3) {
RewriterVar* r_default = r_defaults_array->getAttr(offset, Location::forArg(i));
if (i == 0)
rewrite_args->arg1 = r_default;
if (i == 1)
rewrite_args->arg2 = r_default;
if (i == 2)
rewrite_args->arg3 = r_default;
} else {
RewriterVar* r_default = r_defaults_array->getAttr(offset, Location::any());
rewrite_args->args->setAttr((i - 3) * sizeof(Box*), r_default);
}
} else {
// If we guarded on the BoxedFunctionBase, which has a constant set of defaults,
// we can embed the default arguments directly into the instructions.
if (i < 3) {
RewriterVar* r_default = rewrite_args->rewriter->loadConst((intptr_t)default_obj, Location::any());
if (i == 0)
rewrite_args->arg1 = r_default;
if (i == 1)
rewrite_args->arg2 = r_default;
if (i == 2)
rewrite_args->arg3 = r_default;
} else {
RewriterVar* r_default = rewrite_args->rewriter->loadConst((intptr_t)default_obj, Location::any());
rewrite_args->args->setAttr((i - 3) * sizeof(Box*), r_default);
}
Rewriter* rewriter = rewrite_args->rewriter;
// rewriter->trap();
RewriterVar* args_array = rewriter->allocate(2);
if (num_passed_args >= 4) {
RELEASE_ASSERT(rewrite_args->args, "");
args_array->setAttr(0, rewrite_args->args);
}
}
if (argspec.num_keywords)
args_array->setAttr(8, rewriter->loadConst((intptr_t)keyword_names));
else
args_array->setAttr(8, rewriter->loadConst(0));
RewriterVar::SmallVector arg_vec;
arg_vec.push_back(rewrite_args->obj);
arg_vec.push_back(rewriter->loadConst(argspec.asInt(), Location::forArg(1)));
if (num_passed_args >= 1)
arg_vec.push_back(rewrite_args->arg1);
else
arg_vec.push_back(rewriter->loadConst(0, Location::forArg(2)));
if (num_passed_args >= 2)
arg_vec.push_back(rewrite_args->arg2);
else
arg_vec.push_back(rewriter->loadConst(0, Location::forArg(3)));
if (num_passed_args >= 3)
arg_vec.push_back(rewrite_args->arg3);
else
arg_vec.push_back(rewriter->loadConst(0, Location::forArg(4)));
arg_vec.push_back(args_array);
for (auto v : arg_vec)
assert(v);
RewriterVar* r_rtn = rewriter->call(true, (void*)_callFuncHelper, arg_vec);
getArg(i, oarg1, oarg2, oarg3, oargs) = default_obj;
rewrite_args->out_success = true;
rewrite_args->out_rtn = r_rtn;
rewrite_args = NULL;
}
}
BoxedClosure* closure = func->closure;
// special handling for generators:
// the call to function containing a yield should just create a new generator object.
Box* res;
if (f->isGenerator()) {
// TODO: we might not have a lot to gain by rewriting into createGenerator, but we could at least
// rewrite up to the call to it:
res = createGenerator(func, oarg1, oarg2, oarg3, oargs);
} else {
res = callCLFunc(f, rewrite_args, num_output_args, closure, NULL, func->globals, oarg1, oarg2, oarg3, oargs);
......@@ -3567,6 +3514,21 @@ Box* runtimeCallInternal(Box* obj, CallRewriteArgs* rewrite_args, ArgPassSpec ar
int npassed_args = argspec.totalPassed();
if (obj->cls != function_cls && obj->cls != builtin_function_or_method_cls && obj->cls != instancemethod_cls) {
STAT_TIMER(t0, "us_timer_slowpath_runtimecall_nonfunction", 20);
// TODO: maybe eventually runtimeCallInternal should just be the default tpp_call?
if (obj->cls->tpp_call) {
return obj->cls->tpp_call(obj, rewrite_args, argspec, arg1, arg2, arg3, args, keyword_names);
}
#if 0
std::string per_name_stat_name = "zzz_runtimecall_nonfunction_" + std::string(obj->cls->tp_name);
uint64_t* counter = Stats::getStatCounter(per_name_stat_name);
Stats::log(counter);
if (obj->cls == wrapperobject_cls)
printf("");
#endif
Box* rtn;
if (DEBUG >= 2) {
......@@ -3633,6 +3595,7 @@ Box* runtimeCallInternal(Box* obj, CallRewriteArgs* rewrite_args, ArgPassSpec ar
if (rewrite_args && !rewrite_args->func_guarded) {
r_im_func->addGuard((intptr_t)im->func);
rewrite_args->func_guarded = true;
}
// Guard on which type of instancemethod (bound or unbound)
......@@ -3645,54 +3608,26 @@ Box* runtimeCallInternal(Box* obj, CallRewriteArgs* rewrite_args, ArgPassSpec ar
if (im->obj == NULL) {
Box* f = im->func;
if (rewrite_args) {
rewrite_args->func_guarded = true;
rewrite_args->args_guarded = true;
rewrite_args->obj = r_im_func;
}
Box* res = runtimeCallInternal(f, rewrite_args, argspec, arg1, arg2, arg3, args, keyword_names);
return res;
}
if (npassed_args <= 2) {
Box* rtn;
if (rewrite_args) {
CallRewriteArgs srewrite_args(rewrite_args->rewriter, r_im_func, rewrite_args->destination);
srewrite_args.arg1 = rewrite_args->obj->getAttr(INSTANCEMETHOD_OBJ_OFFSET, Location::any());
srewrite_args.func_guarded = true;
srewrite_args.args_guarded = true;
if (npassed_args >= 1)
srewrite_args.arg2 = rewrite_args->arg1;
if (npassed_args >= 2)
srewrite_args.arg3 = rewrite_args->arg2;
rtn = runtimeCallInternal(
im->func, &srewrite_args,
ArgPassSpec(argspec.num_args + 1, argspec.num_keywords, argspec.has_starargs, argspec.has_kwargs),
im->obj, arg1, arg2, NULL, keyword_names);
if (!srewrite_args.out_success) {
rewrite_args = NULL;
} else {
rewrite_args->out_rtn = srewrite_args.out_rtn;
}
} else {
rtn = runtimeCallInternal(im->func, NULL, ArgPassSpec(argspec.num_args + 1, argspec.num_keywords,
argspec.has_starargs, argspec.has_kwargs),
im->obj, arg1, arg2, NULL, keyword_names);
}
if (rewrite_args)
rewrite_args->out_success = true;
return rtn;
} else {
Box** new_args = (Box**)alloca(sizeof(Box*) * (npassed_args + 1 - 3));
new_args[0] = arg3;
memcpy(new_args + 1, args, (npassed_args - 3) * sizeof(Box*));
Box* rtn = runtimeCallInternal(im->func, NULL, ArgPassSpec(argspec.num_args + 1, argspec.num_keywords,
argspec.has_starargs, argspec.has_kwargs),
im->obj, arg1, arg2, new_args, keyword_names);
return rtn;
Box** new_args = NULL;
if (npassed_args >= 3) {
new_args = (Box**)alloca(sizeof(Box*) * (npassed_args + 1 - 3));
}
RewriterVar* r_bind_obj = NULL;
if (rewrite_args) {
r_bind_obj = rewrite_args->obj->getAttr(INSTANCEMETHOD_OBJ_OFFSET);
rewrite_args->obj = r_im_func;
}
ArgPassSpec new_argspec
= bindObjIntoArgs(im->obj, r_bind_obj, rewrite_args, argspec, arg1, arg2, arg3, args, new_args);
return runtimeCallInternal(im->func, rewrite_args, new_argspec, arg1, arg2, arg3, new_args, keyword_names);
}
assert(0);
abort();
......@@ -4574,13 +4509,6 @@ llvm::iterator_range<BoxIterator> Box::pyElements() {
return BoxIterator::getRange(this);
}
// For use on __init__ return values
static void assertInitNone(Box* obj) {
if (obj != None) {
raiseExcHelper(TypeError, "__init__() should return None, not '%s'", getTypeName(obj));
}
}
void assertValidSlotIdentifier(Box* s) {
// Ported from `valid_identifier` in cpython
......@@ -4874,296 +4802,6 @@ Box* typeNew(Box* _cls, Box* arg1, Box* arg2, Box** _args) {
return made;
}
Box* typeCallInternal(BoxedFunctionBase* f, CallRewriteArgs* rewrite_args, ArgPassSpec argspec, Box* arg1, Box* arg2,
Box* arg3, Box** args, const std::vector<BoxedString*>* keyword_names) {
int npassed_args = argspec.totalPassed();
if (rewrite_args)
assert(rewrite_args->func_guarded);
static StatCounter slowpath_typecall("slowpath_typecall");
slowpath_typecall.log();
if (argspec.has_starargs)
return callFunc(f, rewrite_args, argspec, arg1, arg2, arg3, args, keyword_names);
assert(argspec.num_args >= 1);
Box* _cls = arg1;
if (!isSubclass(_cls->cls, type_cls)) {
raiseExcHelper(TypeError, "descriptor '__call__' requires a 'type' object but received an '%s'",
getTypeName(_cls));
}
BoxedClass* cls = static_cast<BoxedClass*>(_cls);
RewriterVar* r_ccls = NULL;
RewriterVar* r_new = NULL;
RewriterVar* r_init = NULL;
Box* new_attr, *init_attr;
if (rewrite_args) {
assert(!argspec.has_starargs);
assert(argspec.num_args > 0);
r_ccls = rewrite_args->arg1;
// This is probably a duplicate, but it's hard to really convince myself of that.
// Need to create a clear contract of who guards on what
r_ccls->addGuard((intptr_t)arg1 /* = _cls */);
}
if (rewrite_args) {
GetattrRewriteArgs grewrite_args(rewrite_args->rewriter, r_ccls, rewrite_args->destination);
// TODO: if tp_new != Py_CallPythonNew, call that instead?
new_attr = typeLookup(cls, new_str, &grewrite_args);
if (!grewrite_args.out_success)
rewrite_args = NULL;
else {
assert(new_attr);
r_new = grewrite_args.out_rtn;
r_new->addGuard((intptr_t)new_attr);
}
// Special-case functions to allow them to still rewrite:
if (new_attr->cls != function_cls) {
Box* descr_r = processDescriptorOrNull(new_attr, None, cls);
if (descr_r) {
new_attr = descr_r;
rewrite_args = NULL;
REWRITE_ABORTED("");
}
}
} else {
new_attr = typeLookup(cls, new_str, NULL);
new_attr = processDescriptor(new_attr, None, cls);
}
assert(new_attr && "This should always resolve");
// typeCall is tricky to rewrite since it has complicated behavior: we are supposed to
// call the __init__ method of the *result of the __new__ call*, not of the original
// class. (And only if the result is an instance of the original class, but that's not
// even the tricky part here.)
//
// By the time we know the type of the result of __new__(), it's too late to add traditional
// guards. So, instead of doing that, we're going to add a guard that makes sure that __new__
// has the property that __new__(kls) always returns an instance of kls.
//
// Whitelist a set of __new__ methods that we know work like this. Most importantly: object.__new__.
//
// Most builtin classes behave this way, but not all!
// Notably, "type" itself does not. For instance, assuming M is a subclass of
// type, type.__new__(M, 1) will return the int class, which is not an instance of M.
// this is ok with not using StlCompatAllocator since we will manually register these objects with the GC
static std::vector<Box*> allowable_news;
if (allowable_news.empty()) {
for (BoxedClass* allowed_cls : { object_cls, enumerate_cls, xrange_cls, tuple_cls, list_cls, dict_cls }) {
auto new_obj = typeLookup(allowed_cls, new_str, NULL);
gc::registerPermanentRoot(new_obj);
allowable_news.push_back(new_obj);
}
}
bool type_new_special_case;
if (rewrite_args) {
bool ok = false;
for (auto b : allowable_news) {
if (b == new_attr) {
ok = true;
break;
}
}
if (!ok && (cls == int_cls || cls == float_cls || cls == long_cls)) {
if (npassed_args == 1)
ok = true;
else if (npassed_args == 2 && (arg2->cls == int_cls || arg2->cls == str_cls || arg2->cls == float_cls))
ok = true;
}
type_new_special_case = (cls == type_cls && argspec == ArgPassSpec(2));
if (!ok && !type_new_special_case) {
// Uncomment this to try to find __new__ functions that we could either white- or blacklist:
// ASSERT(cls->is_user_defined || cls == type_cls, "Does '%s' have a well-behaved __new__? if so, add to
// allowable_news, otherwise add to the blacklist in this assert", cls->tp_name);
rewrite_args = NULL;
REWRITE_ABORTED("");
}
}
if (rewrite_args) {
GetattrRewriteArgs grewrite_args(rewrite_args->rewriter, r_ccls, rewrite_args->destination);
init_attr = typeLookup(cls, init_str, &grewrite_args);
if (!grewrite_args.out_success)
rewrite_args = NULL;
else {
if (init_attr) {
r_init = grewrite_args.out_rtn;
r_init->addGuard((intptr_t)init_attr);
}
}
} else {
init_attr = typeLookup(cls, init_str, NULL);
}
// The init_attr should always resolve as well, but doesn't yet
Box* made;
RewriterVar* r_made = NULL;
ArgPassSpec new_argspec = argspec;
if (rewrite_args) {
if (cls->tp_new == object_cls->tp_new && cls->tp_init != object_cls->tp_init) {
// Fast case: if we are calling object_new, we normally doesn't look at the arguments at all.
// (Except in the case when init_attr != object_init, in which case object_new looks at the number
// of arguments and throws an exception.)
//
// Another option is to rely on rewriting to make this fast, which would probably require adding
// a custom internal callable to object.__new__
made = objectNewNoArgs(cls);
r_made = rewrite_args->rewriter->call(true, (void*)objectNewNoArgs, r_ccls);
} else {
CallRewriteArgs srewrite_args(rewrite_args->rewriter, r_new, rewrite_args->destination);
srewrite_args.args_guarded = true;
srewrite_args.func_guarded = true;
int new_npassed_args = new_argspec.totalPassed();
if (new_npassed_args >= 1)
srewrite_args.arg1 = r_ccls;
if (new_npassed_args >= 2)
srewrite_args.arg2 = rewrite_args->arg2;
if (new_npassed_args >= 3)
srewrite_args.arg3 = rewrite_args->arg3;
if (new_npassed_args >= 4)
srewrite_args.args = rewrite_args->args;
made = runtimeCallInternal(new_attr, &srewrite_args, new_argspec, cls, arg2, arg3, args, keyword_names);
if (!srewrite_args.out_success) {
rewrite_args = NULL;
} else {
r_made = srewrite_args.out_rtn;
}
}
ASSERT(made->cls == cls || type_new_special_case,
"We should only have allowed the rewrite to continue if we were guaranteed that made "
"would have class cls!");
} else {
made = runtimeCallInternal(new_attr, NULL, new_argspec, cls, arg2, arg3, args, keyword_names);
}
assert(made);
// Special-case (also a special case in CPython): if we just called type.__new__(arg), don't call __init__
if (cls == type_cls && argspec == ArgPassSpec(2)) {
if (rewrite_args) {
rewrite_args->out_success = true;
rewrite_args->out_rtn = r_made;
}
return made;
}
// If __new__ returns a subclass, supposed to call that subclass's __init__.
// If __new__ returns a non-subclass, not supposed to call __init__.
if (made->cls != cls) {
ASSERT(rewrite_args == NULL, "We should only have allowed the rewrite to continue if we were guaranteed that "
"made would have class cls!");
if (!isSubclass(made->cls, cls)) {
init_attr = NULL;
} else {
// We could have skipped the initial __init__ lookup
init_attr = typeLookup(made->cls, init_str, NULL);
}
}
if (init_attr && made->cls->tp_init != object_cls->tp_init) {
// TODO apply the same descriptor special-casing as in callattr?
Box* initrtn;
// Attempt to rewrite the basic case:
if (rewrite_args && init_attr->cls == function_cls) {
// Note: this code path includes the descriptor logic
CallRewriteArgs srewrite_args(rewrite_args->rewriter, r_init, rewrite_args->destination);
if (npassed_args >= 1)
srewrite_args.arg1 = r_made;
if (npassed_args >= 2)
srewrite_args.arg2 = rewrite_args->arg2;
if (npassed_args >= 3)
srewrite_args.arg3 = rewrite_args->arg3;
if (npassed_args >= 4)
srewrite_args.args = rewrite_args->args;
srewrite_args.args_guarded = true;
srewrite_args.func_guarded = true;
// initrtn = callattrInternal(cls, _init_str, INST_ONLY, &srewrite_args, argspec, made, arg2, arg3, args,
// keyword_names);
initrtn = runtimeCallInternal(init_attr, &srewrite_args, argspec, made, arg2, arg3, args, keyword_names);
if (!srewrite_args.out_success) {
rewrite_args = NULL;
} else {
rewrite_args->rewriter->call(true, (void*)assertInitNone, srewrite_args.out_rtn);
}
} else {
init_attr = processDescriptor(init_attr, made, cls);
ArgPassSpec init_argspec = argspec;
init_argspec.num_args--;
int passed = init_argspec.totalPassed();
// If we weren't passed the args array, it's not safe to index into it
if (passed <= 2)
initrtn = runtimeCallInternal(init_attr, NULL, init_argspec, arg2, arg3, NULL, NULL, keyword_names);
else
initrtn
= runtimeCallInternal(init_attr, NULL, init_argspec, arg2, arg3, args[0], &args[1], keyword_names);
}
assertInitNone(initrtn);
} else {
if (new_attr == NULL && npassed_args != 1) {
// TODO not npassed args, since the starargs or kwargs could be null
raiseExcHelper(TypeError, objectNewParameterTypeErrorMsg());
}
}
if (rewrite_args) {
rewrite_args->out_rtn = r_made;
rewrite_args->out_success = true;
}
return made;
}
Box* typeCall(Box* obj, BoxedTuple* vararg, BoxedDict* kwargs) {
assert(vararg->cls == tuple_cls);
bool pass_kwargs = (kwargs && kwargs->d.size());
int n = vararg->size();
int args_to_pass = n + 1 + (pass_kwargs ? 1 : 0); // 1 for obj, 1 for kwargs
Box** args = NULL;
if (args_to_pass > 3)
args = (Box**)alloca(sizeof(Box*) * (args_to_pass - 3));
Box* arg1, *arg2, *arg3;
arg1 = obj;
for (int i = 0; i < n; i++) {
getArg(i + 1, arg1, arg2, arg3, args) = vararg->elts[i];
}
if (pass_kwargs)
getArg(n + 1, arg1, arg2, arg3, args) = kwargs;
return typeCallInternal(NULL, NULL, ArgPassSpec(n + 1, 0, false, pass_kwargs), arg1, arg2, arg3, args, NULL);
}
extern "C" void delGlobal(Box* globals, BoxedString* name) {
if (globals->cls == module_cls) {
BoxedModule* m = static_cast<BoxedModule*>(globals);
......
src/runtime/objmodel.h
View file @ 79505a1a
......@@ -112,8 +112,6 @@ Box* runtimeCallInternal(Box* obj, CallRewriteArgs* rewrite_args, ArgPassSpec ar
Box* lenCallInternal(BoxedFunctionBase* f, CallRewriteArgs* rewrite_args, ArgPassSpec argspec, Box* arg1, Box* arg2,
Box* arg3, Box** args, const std::vector<BoxedString*>* keyword_names);
Box* typeCallInternal(BoxedFunctionBase* f, CallRewriteArgs* rewrite_args, ArgPassSpec argspec, Box* arg1, Box* arg2,
Box* arg3, Box** args, const std::vector<BoxedString*>* keyword_names);
Box* callFunc(BoxedFunctionBase* func, CallRewriteArgs* rewrite_args, ArgPassSpec argspec, Box* arg1, Box* arg2,
Box* arg3, Box** args, const std::vector<BoxedString*>* keyword_names);
......@@ -153,7 +151,11 @@ Box* typeCall(Box*, BoxedTuple*, BoxedDict*);
Box* typeNew(Box* cls, Box* arg1, Box* arg2, Box** _args);
bool isUserDefined(BoxedClass* cls);
// These process a potential descriptor, differing in their behavior if the object was not a descriptor.
// the OrNull variant returns NULL to signify it wasn't a descriptor, and the processDescriptor version
// returns obj.
Box* processDescriptor(Box* obj, Box* inst, Box* owner);
Box* processDescriptorOrNull(Box* obj, Box* inst, Box* owner);
Box* callCLFunc(CLFunction* f, CallRewriteArgs* rewrite_args, int num_output_args, BoxedClosure* closure,
BoxedGenerator* generator, Box* globals, Box* oarg1, Box* oarg2, Box* oarg3, Box** oargs);
......
src/runtime/rewrite_args.h
View file @ 79505a1a
......@@ -121,6 +121,21 @@ struct CompareRewriteArgs {
: rewriter(rewriter), lhs(lhs), rhs(rhs), destination(destination), out_success(false), out_rtn(NULL) {}
};
// Passes the output arguments back through oarg. Passes the rewrite success by setting rewrite_success.
// Directly modifies rewrite_args args in place, but only if rewrite_success got set.
// oargs needs to be pre-allocated by the caller, since it's assumed that they will want to use alloca.
// The caller is responsible for guarding for paramspec, argspec, param_names, and defaults.
// TODO Fix this function's signature. should we pass back out through args? the common case is that they
// match anyway. Or maybe it should call a callback function, which could save on the common case.
void rearrangeArguments(ParamReceiveSpec paramspec, const ParamNames* param_names, const char* func_name,
Box** defaults, CallRewriteArgs* rewrite_args, bool& rewrite_success, ArgPassSpec argspec,
Box* arg1, Box* arg2, Box* arg3, Box** args, const std::vector<BoxedString*>* keyword_names,
Box*& oarg1, Box*& oarg2, Box*& oarg3, Box** oargs);
// new_args should be allocated by the caller if at least three args get passed in.
// rewrite_args will get modified in place.
ArgPassSpec bindObjIntoArgs(Box* bind_obj, RewriterVar* r_bind_obj, CallRewriteArgs* rewrite_args, ArgPassSpec argspec,
Box*& arg1, Box*& arg2, Box*& arg3, Box** args, Box** new_args);
} // namespace pyston
#endif
src/runtime/types.cpp
View file @ 79505a1a
......@@ -41,6 +41,7 @@
#include "runtime/list.h"
#include "runtime/long.h"
#include "runtime/objmodel.h"
#include "runtime/rewrite_args.h"
#include "runtime/set.h"
#include "runtime/super.h"
#include "runtime/traceback.h"
......@@ -86,6 +87,9 @@ extern "C" void initstrop();
namespace pyston {
static const std::string init_str("__init__");
static const std::string new_str("__new__");
void setupGC();
bool IN_SHUTDOWN = false;
......@@ -316,7 +320,7 @@ extern "C" BoxedFunctionBase::BoxedFunctionBase(CLFunction* f)
this->doc = None;
}
assert(f->num_defaults == ndefaults);
assert(f->paramspec.num_defaults == ndefaults);
}
extern "C" BoxedFunctionBase::BoxedFunctionBase(CLFunction* f, std::initializer_list<Box*> defaults,
......@@ -338,7 +342,7 @@ extern "C" BoxedFunctionBase::BoxedFunctionBase(CLFunction* f, std::initializer_
this->doc = None;
}
assert(f->num_defaults == ndefaults);
assert(f->paramspec.num_defaults == ndefaults);
}
BoxedFunction::BoxedFunction(CLFunction* f) : BoxedFunction(f, {}) {
......@@ -526,6 +530,355 @@ extern "C" void boxGCHandler(GCVisitor* v, Box* b) {
}
}
static Box* typeCallInner(CallRewriteArgs* rewrite_args, ArgPassSpec argspec, Box* arg1, Box* arg2, Box* arg3,
Box** args, const std::vector<BoxedString*>* keyword_names);
static Box* typeTppCall(Box* self, CallRewriteArgs* rewrite_args, ArgPassSpec argspec, Box* arg1, Box* arg2, Box* arg3,
Box** args, const std::vector<BoxedString*>* keyword_names) {
int npassed_args = argspec.totalPassed();
if (argspec.has_starargs) {
// This would fail in typeCallInner
rewrite_args = NULL;
}
Box** new_args = NULL;
if (npassed_args >= 3) {
new_args = (Box**)alloca(sizeof(Box*) * (npassed_args + 1 - 3));
}
RewriterVar* r_bind_obj = NULL;
if (rewrite_args) {
r_bind_obj = rewrite_args->obj;
rewrite_args->obj = NULL;
}
ArgPassSpec new_argspec
= bindObjIntoArgs(self, r_bind_obj, rewrite_args, argspec, arg1, arg2, arg3, args, new_args);
return typeCallInner(rewrite_args, new_argspec, arg1, arg2, arg3, new_args, keyword_names);
}
static Box* typeCallInternal(BoxedFunctionBase* f, CallRewriteArgs* rewrite_args, ArgPassSpec argspec, Box* arg1,
Box* arg2, Box* arg3, Box** args, const std::vector<BoxedString*>* keyword_names) {
if (rewrite_args)
assert(rewrite_args->func_guarded);
static StatCounter slowpath_typecall("slowpath_typecall");
slowpath_typecall.log();
if (argspec.has_starargs)
return callFunc(f, rewrite_args, argspec, arg1, arg2, arg3, args, keyword_names);
return typeCallInner(rewrite_args, argspec, arg1, arg2, arg3, args, keyword_names);
}
// For use on __init__ return values
static void assertInitNone(Box* obj) {
if (obj != None) {
raiseExcHelper(TypeError, "__init__() should return None, not '%s'", getTypeName(obj));
}
}
static Box* typeCallInner(CallRewriteArgs* rewrite_args, ArgPassSpec argspec, Box* arg1, Box* arg2, Box* arg3,
Box** args, const std::vector<BoxedString*>* keyword_names) {
int npassed_args = argspec.totalPassed();
assert(argspec.num_args >= 1);
Box* _cls = arg1;
if (!isSubclass(_cls->cls, type_cls)) {
raiseExcHelper(TypeError, "descriptor '__call__' requires a 'type' object but received an '%s'",
getTypeName(_cls));
}
BoxedClass* cls = static_cast<BoxedClass*>(_cls);
RewriterVar* r_ccls = NULL;
RewriterVar* r_new = NULL;
RewriterVar* r_init = NULL;
Box* new_attr, *init_attr;
if (rewrite_args) {
assert(!argspec.has_starargs);
assert(argspec.num_args > 0);
r_ccls = rewrite_args->arg1;
// This is probably a duplicate, but it's hard to really convince myself of that.
// Need to create a clear contract of who guards on what
r_ccls->addGuard((intptr_t)arg1 /* = _cls */);
if (!rewrite_args->args_guarded) {
// TODO should know which args don't need to be guarded, ex if we're guaranteed that they
// already fit, either since the type inferencer could determine that,
// or because they only need to fit into an UNKNOWN slot.
if (npassed_args >= 1)
rewrite_args->arg1->addAttrGuard(offsetof(Box, cls), (intptr_t)arg1->cls);
if (npassed_args >= 2)
rewrite_args->arg2->addAttrGuard(offsetof(Box, cls), (intptr_t)arg2->cls);
if (npassed_args >= 3)
rewrite_args->arg3->addAttrGuard(offsetof(Box, cls), (intptr_t)arg3->cls);
for (int i = 3; i < npassed_args; i++) {
RewriterVar* v = rewrite_args->args->getAttr((i - 3) * sizeof(Box*), Location::any());
v->addAttrGuard(offsetof(Box, cls), (intptr_t)args[i - 3]->cls);
}
rewrite_args->args_guarded = true;
}
}
if (rewrite_args) {
GetattrRewriteArgs grewrite_args(rewrite_args->rewriter, r_ccls, rewrite_args->destination);
// TODO: if tp_new != Py_CallPythonNew, call that instead?
new_attr = typeLookup(cls, new_str, &grewrite_args);
if (!grewrite_args.out_success)
rewrite_args = NULL;
else {
assert(new_attr);
r_new = grewrite_args.out_rtn;
r_new->addGuard((intptr_t)new_attr);
}
// Special-case functions to allow them to still rewrite:
if (new_attr->cls != function_cls) {
Box* descr_r = processDescriptorOrNull(new_attr, None, cls);
if (descr_r) {
new_attr = descr_r;
rewrite_args = NULL;
}
}
} else {
new_attr = typeLookup(cls, new_str, NULL);
new_attr = processDescriptor(new_attr, None, cls);
}
assert(new_attr && "This should always resolve");
// typeCall is tricky to rewrite since it has complicated behavior: we are supposed to
// call the __init__ method of the *result of the __new__ call*, not of the original
// class. (And only if the result is an instance of the original class, but that's not
// even the tricky part here.)
//
// By the time we know the type of the result of __new__(), it's too late to add traditional
// guards. So, instead of doing that, we're going to add a guard that makes sure that __new__
// has the property that __new__(kls) always returns an instance of kls.
//
// Whitelist a set of __new__ methods that we know work like this. Most importantly: object.__new__.
//
// Most builtin classes behave this way, but not all!
// Notably, "type" itself does not. For instance, assuming M is a subclass of
// type, type.__new__(M, 1) will return the int class, which is not an instance of M.
// this is ok with not using StlCompatAllocator since we will manually register these objects with the GC
static std::vector<Box*> allowable_news;
if (allowable_news.empty()) {
for (BoxedClass* allowed_cls : { object_cls, enumerate_cls, xrange_cls, tuple_cls, list_cls, dict_cls }) {
auto new_obj = typeLookup(allowed_cls, new_str, NULL);
gc::registerPermanentRoot(new_obj);
allowable_news.push_back(new_obj);
}
}
bool type_new_special_case;
if (rewrite_args) {
bool ok = false;
for (auto b : allowable_news) {
if (b == new_attr) {
ok = true;
break;
}
}
if (!ok && (cls == int_cls || cls == float_cls || cls == long_cls)) {
if (npassed_args == 1)
ok = true;
else if (npassed_args == 2 && (arg2->cls == int_cls || arg2->cls == str_cls || arg2->cls == float_cls)) {
rewrite_args->arg2->addAttrGuard(offsetof(Box, cls), (intptr_t)arg2->cls);
ok = true;
}
}
type_new_special_case = (cls == type_cls && argspec == ArgPassSpec(2));
if (!ok && !type_new_special_case) {
// Uncomment this to try to find __new__ functions that we could either white- or blacklist:
// ASSERT(cls->is_user_defined || cls == type_cls, "Does '%s' have a well-behaved __new__? if so, add to
// allowable_news, otherwise add to the blacklist in this assert", cls->tp_name);
rewrite_args = NULL;
}
}
if (rewrite_args) {
GetattrRewriteArgs grewrite_args(rewrite_args->rewriter, r_ccls, rewrite_args->destination);
init_attr = typeLookup(cls, init_str, &grewrite_args);
if (!grewrite_args.out_success)
rewrite_args = NULL;
else {
if (init_attr) {
r_init = grewrite_args.out_rtn;
r_init->addGuard((intptr_t)init_attr);
}
}
} else {
init_attr = typeLookup(cls, init_str, NULL);
}
// The init_attr should always resolve as well, but doesn't yet
Box* made;
RewriterVar* r_made = NULL;
ArgPassSpec new_argspec = argspec;
if (rewrite_args) {
if (cls->tp_new == object_cls->tp_new && cls->tp_init != object_cls->tp_init) {
// Fast case: if we are calling object_new, we normally doesn't look at the arguments at all.
// (Except in the case when init_attr != object_init, in which case object_new looks at the number
// of arguments and throws an exception.)
//
// Another option is to rely on rewriting to make this fast, which would probably require adding
// a custom internal callable to object.__new__
made = objectNewNoArgs(cls);
r_made = rewrite_args->rewriter->call(true, (void*)objectNewNoArgs, r_ccls);
} else {
CallRewriteArgs srewrite_args(rewrite_args->rewriter, r_new, rewrite_args->destination);
srewrite_args.args_guarded = rewrite_args->args_guarded;
srewrite_args.func_guarded = true;
int new_npassed_args = new_argspec.totalPassed();
if (new_npassed_args >= 1)
srewrite_args.arg1 = r_ccls;
if (new_npassed_args >= 2)
srewrite_args.arg2 = rewrite_args->arg2;
if (new_npassed_args >= 3)
srewrite_args.arg3 = rewrite_args->arg3;
if (new_npassed_args >= 4)
srewrite_args.args = rewrite_args->args;
made = runtimeCallInternal(new_attr, &srewrite_args, new_argspec, cls, arg2, arg3, args, keyword_names);
if (!srewrite_args.out_success) {
rewrite_args = NULL;
} else {
r_made = srewrite_args.out_rtn;
}
}
ASSERT(made->cls == cls || type_new_special_case,
"We should only have allowed the rewrite to continue if we were guaranteed that made "
"would have class cls!");
} else {
made = runtimeCallInternal(new_attr, NULL, new_argspec, cls, arg2, arg3, args, keyword_names);
}
assert(made);
// Special-case (also a special case in CPython): if we just called type.__new__(arg), don't call __init__
if (cls == type_cls && argspec == ArgPassSpec(2)) {
if (rewrite_args) {
rewrite_args->out_success = true;
rewrite_args->out_rtn = r_made;
}
return made;
}
// If __new__ returns a subclass, supposed to call that subclass's __init__.
// If __new__ returns a non-subclass, not supposed to call __init__.
if (made->cls != cls) {
ASSERT(rewrite_args == NULL, "We should only have allowed the rewrite to continue if we were guaranteed that "
"made would have class cls!");
if (!isSubclass(made->cls, cls)) {
init_attr = NULL;
} else {
// We could have skipped the initial __init__ lookup
init_attr = typeLookup(made->cls, init_str, NULL);
}
}
if (init_attr && made->cls->tp_init != object_cls->tp_init) {
// TODO apply the same descriptor special-casing as in callattr?
Box* initrtn;
// Attempt to rewrite the basic case:
if (rewrite_args && init_attr->cls == function_cls) {
// Note: this code path includes the descriptor logic
CallRewriteArgs srewrite_args(rewrite_args->rewriter, r_init, rewrite_args->destination);
srewrite_args.arg1 = r_made;
if (npassed_args >= 2)
srewrite_args.arg2 = rewrite_args->arg2;
if (npassed_args >= 3)
srewrite_args.arg3 = rewrite_args->arg3;
if (npassed_args >= 4)
srewrite_args.args = rewrite_args->args;
srewrite_args.args_guarded = rewrite_args->args_guarded;
srewrite_args.func_guarded = true;
// initrtn = callattrInternal(cls, _init_str, INST_ONLY, &srewrite_args, argspec, made, arg2, arg3, args,
// keyword_names);
initrtn = runtimeCallInternal(init_attr, &srewrite_args, argspec, made, arg2, arg3, args, keyword_names);
if (!srewrite_args.out_success) {
rewrite_args = NULL;
} else {
rewrite_args->rewriter->call(true, (void*)assertInitNone, srewrite_args.out_rtn);
}
} else {
init_attr = processDescriptor(init_attr, made, cls);
ArgPassSpec init_argspec = argspec;
init_argspec.num_args--;
int passed = init_argspec.totalPassed();
// If we weren't passed the args array, it's not safe to index into it
if (passed <= 2)
initrtn = runtimeCallInternal(init_attr, NULL, init_argspec, arg2, arg3, NULL, NULL, keyword_names);
else
initrtn
= runtimeCallInternal(init_attr, NULL, init_argspec, arg2, arg3, args[0], &args[1], keyword_names);
}
assertInitNone(initrtn);
} else {
if (new_attr == NULL && npassed_args != 1) {
// TODO not npassed args, since the starargs or kwargs could be null
raiseExcHelper(TypeError, objectNewParameterTypeErrorMsg());
}
}
if (rewrite_args) {
rewrite_args->out_rtn = r_made;
rewrite_args->out_success = true;
}
return made;
}
Box* typeCall(Box* obj, BoxedTuple* vararg, BoxedDict* kwargs) {
assert(vararg->cls == tuple_cls);
bool pass_kwargs = (kwargs && kwargs->d.size());
int n = vararg->size();
int args_to_pass = n + 1 + (pass_kwargs ? 1 : 0); // 1 for obj, 1 for kwargs
Box** args = NULL;
if (args_to_pass > 3)
args = (Box**)alloca(sizeof(Box*) * (args_to_pass - 3));
Box* arg1, *arg2, *arg3;
arg1 = obj;
for (int i = 0; i < n; i++) {
getArg(i + 1, arg1, arg2, arg3, args) = vararg->elts[i];
}
if (pass_kwargs)
getArg(n + 1, arg1, arg2, arg3, args) = kwargs;
return typeCallInternal(NULL, NULL, ArgPassSpec(n + 1, 0, false, pass_kwargs), arg1, arg2, arg3, args, NULL);
}
extern "C" void typeGCHandler(GCVisitor* v, Box* b) {
boxGCHandler(v, b);
......@@ -2687,6 +3040,7 @@ void setupRuntime() {
type_cls->tp_richcompare = type_richcompare;
add_operators(type_cls);
type_cls->freeze();
type_cls->tpp_call = &typeTppCall;
none_cls->giveAttr("__repr__", new BoxedFunction(boxRTFunction((void*)noneRepr, STR, 1)));
none_cls->giveAttr("__nonzero__", new BoxedFunction(boxRTFunction((void*)noneNonzero, BOXED_BOOL, 1)));
......
src/runtime/types.h
View file @ 79505a1a
......@@ -201,6 +201,10 @@ public:
pyston_inquiry tpp_hasnext;
typedef Box* (*pyston_call)(Box*, CallRewriteArgs*, ArgPassSpec, Box*, Box*, Box*, Box**,
const std::vector<BoxedString*>*);
pyston_call tpp_call;
bool hasGenericGetattr() { return tp_getattr == NULL; }
void freeze();
......@@ -907,6 +911,8 @@ public:
DEFAULT_CLASS(wrapperobject_cls);
static Box* __call__(BoxedWrapperObject* self, Box* args, Box* kwds);
static Box* tppCall(Box* _self, CallRewriteArgs* rewrite_args, ArgPassSpec argspec, Box* arg1, Box* arg2, Box* arg3,
Box** args, const std::vector<BoxedString*>* keyword_names);
static void gcHandler(GCVisitor* v, Box* _o);
};
......@@ -1003,6 +1009,17 @@ extern "C" inline Box* boxInt(int64_t n) {
}
return new BoxedInt(n);
}
// Helper function: fetch an arg from our calling convention
inline Box*& getArg(int idx, Box*& arg1, Box*& arg2, Box*& arg3, Box** args) {
if (idx == 0)
return arg1;
if (idx == 1)
return arg2;
if (idx == 2)
return arg3;
return args[idx - 3];
}
}
#endif
test/extra/test_helper.py
View file @ 79505a1a
......@@ -78,7 +78,7 @@ def run_test(cmd, cwd, expected, env = None):
if expected == result:
print "Received expected output"
else:
print >> sys.stderr, output
print >> sys.stderr, '\n'.join(output.split('\n')[:200])
print >> sys.stderr, "WRONG output"
print >> sys.stderr, "is:", result
print >> sys.stderr, "expected:", expected
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7