Commit 4c2252ad authored by Kevin Modzelewski's avatar Kevin Modzelewski

Merge pull request #462 from kmod/compile_exec

initial compile() support
parents 11ebf1cf d4858920
......@@ -1023,8 +1023,11 @@ $(call make_target,_gcc)
nosearch_runpy_% nosearch_pyrun_%: %.py ext_python
$(VERB) PYTHONPATH=test/test_extension/build/lib.linux-x86_64-2.7 zsh -c 'time python $<'
nosearch_pypyrun_%: %.py ext_python
$(VERB) PYTHONPATH=test/test_extension/build/lib.linux-x86_64-2.7 zsh -c 'time python $<'
$(call make_search,runpy_%)
$(call make_search,pyrun_%)
$(call make_search,pypyrun_%)
nosearch_check_%: %.py ext_python ext_pyston
$(MAKE) check_dbg ARGS="$(patsubst %.py,%,$(notdir $<)) -K"
......
......@@ -359,7 +359,7 @@ void DefinednessBBAnalyzer::processBB(Map& starting, CFGBlock* block) const {
}
}
void DefinednessAnalysis::run(llvm::DenseMap<InternedString, DefinednessAnalysis::DefinitionLevel>&& initial_map,
void DefinednessAnalysis::run(llvm::DenseMap<InternedString, DefinednessAnalysis::DefinitionLevel> initial_map,
CFGBlock* initial_block, ScopeInfo* scope_info) {
Timer _t("DefinednessAnalysis()", 10);
......@@ -397,7 +397,7 @@ const DefinednessAnalysis::RequiredSet& DefinednessAnalysis::getDefinedNamesAtEn
return defined_at_end_sets[block];
}
PhiAnalysis::PhiAnalysis(llvm::DenseMap<InternedString, DefinednessAnalysis::DefinitionLevel>&& initial_map,
PhiAnalysis::PhiAnalysis(llvm::DenseMap<InternedString, DefinednessAnalysis::DefinitionLevel> initial_map,
CFGBlock* initial_block, bool initials_need_phis, LivenessAnalysis* liveness,
ScopeInfo* scope_info)
: definedness(), liveness(liveness) {
......
......@@ -70,7 +70,7 @@ private:
public:
DefinednessAnalysis() {}
void run(llvm::DenseMap<InternedString, DefinitionLevel>&& initial_map, CFGBlock* initial_block,
void run(llvm::DenseMap<InternedString, DefinitionLevel> initial_map, CFGBlock* initial_block,
ScopeInfo* scope_info);
DefinitionLevel isDefinedAtEnd(InternedString name, CFGBlock* block);
......@@ -92,7 +92,7 @@ private:
public:
// Initials_need_phis specifies that initial_map should count as an additional entry point
// that may require phis.
PhiAnalysis(llvm::DenseMap<InternedString, DefinednessAnalysis::DefinitionLevel>&& initial_map,
PhiAnalysis(llvm::DenseMap<InternedString, DefinednessAnalysis::DefinitionLevel> initial_map,
CFGBlock* initial_block, bool initials_need_phis, LivenessAnalysis* liveness, ScopeInfo* scope_info);
bool isRequired(InternedString name, CFGBlock* block);
......
......@@ -224,13 +224,13 @@ void ASTInterpreter::gcVisit(GCVisitor* visitor) {
}
ASTInterpreter::ASTInterpreter(CompiledFunction* compiled_function)
: compiled_func(compiled_function), source_info(compiled_function->clfunc->source), scope_info(0), phis(NULL),
: compiled_func(compiled_function), source_info(compiled_function->clfunc->source.get()), scope_info(0), phis(NULL),
current_block(0), current_inst(0), last_exception(NULL, NULL, NULL), passed_closure(0), created_closure(0),
generator(0), edgecount(0), frame_info(ExcInfo(NULL, NULL, NULL)) {
CLFunction* f = compiled_function->clfunc;
if (!source_info->cfg)
source_info->cfg = computeCFG(f->source, f->source->body);
source_info->cfg = computeCFG(f->source.get(), f->source->body);
scope_info = source_info->getScopeInfo();
......@@ -767,8 +767,11 @@ Box* ASTInterpreter::createFunction(AST* node, AST_arguments* args, const std::v
}
assert(closure);
}
return boxCLFunction(cl, closure, is_generator, globals->cls == dict_cls ? static_cast<BoxedDict*>(globals) : NULL,
u.il);
Box* passed_globals = NULL;
if (!getCF()->clfunc->source->scoping->areGlobalsFromModule())
passed_globals = globals;
return boxCLFunction(cl, closure, is_generator, passed_globals, u.il);
}
Value ASTInterpreter::visit_makeFunction(AST_MakeFunction* mkfn) {
......@@ -804,9 +807,11 @@ Value ASTInterpreter::visit_makeClass(AST_MakeClass* mkclass) {
BoxedClosure* closure = scope_info->takesClosure() ? created_closure : 0;
CLFunction* cl = wrapFunction(node, nullptr, node->body, source_info);
Box* attrDict = runtimeCall(
boxCLFunction(cl, closure, false, globals->cls == dict_cls ? static_cast<BoxedDict*>(globals) : NULL, {}),
ArgPassSpec(0), 0, 0, 0, 0, 0);
Box* passed_globals = NULL;
if (!getCF()->clfunc->source->scoping->areGlobalsFromModule())
passed_globals = globals;
Box* attrDict = runtimeCall(boxCLFunction(cl, closure, false, passed_globals, {}), ArgPassSpec(0), 0, 0, 0, 0, 0);
Box* classobj = createUserClass(&node->name.str(), basesTuple, attrDict);
......@@ -1202,8 +1207,9 @@ Value ASTInterpreter::visit_attribute(AST_Attribute* node) {
const void* interpreter_instr_addr = (void*)&ASTInterpreter::execute;
Box* astInterpretFunction(CompiledFunction* cf, int nargs, Box* closure, Box* generator, BoxedDict* globals, Box* arg1,
Box* astInterpretFunction(CompiledFunction* cf, int nargs, Box* closure, Box* generator, Box* globals, Box* arg1,
Box* arg2, Box* arg3, Box** args) {
assert((!globals) == cf->clfunc->source->scoping->areGlobalsFromModule());
bool can_reopt = ENABLE_REOPT && !FORCE_INTERPRETER && (globals == NULL);
if (unlikely(can_reopt && cf->times_called > REOPT_THRESHOLD_INTERPRETER)) {
assert(!globals);
......@@ -1222,7 +1228,7 @@ Box* astInterpretFunction(CompiledFunction* cf, int nargs, Box* closure, Box* ge
ASTInterpreter interpreter(cf);
ScopeInfo* scope_info = cf->clfunc->source->getScopeInfo();
SourceInfo* source_info = cf->clfunc->source;
SourceInfo* source_info = cf->clfunc->source.get();
if (unlikely(scope_info->usesNameLookup())) {
interpreter.setBoxedLocals(new BoxedDict());
}
......@@ -1240,7 +1246,7 @@ Box* astInterpretFunction(CompiledFunction* cf, int nargs, Box* closure, Box* ge
return v.o ? v.o : None;
}
Box* astInterpretFunctionEval(CompiledFunction* cf, BoxedDict* globals, Box* boxedLocals) {
Box* astInterpretFunctionEval(CompiledFunction* cf, Box* globals, Box* boxedLocals) {
++cf->times_called;
ASTInterpreter interpreter(cf);
......@@ -1248,14 +1254,11 @@ Box* astInterpretFunctionEval(CompiledFunction* cf, BoxedDict* globals, Box* box
interpreter.setBoxedLocals(boxedLocals);
ScopeInfo* scope_info = cf->clfunc->source->getScopeInfo();
SourceInfo* source_info = cf->clfunc->source;
if (cf->clfunc->source->scoping->areGlobalsFromModule()) {
assert(!globals);
interpreter.setGlobals(source_info->parent_module);
} else {
assert(globals);
interpreter.setGlobals(globals);
}
SourceInfo* source_info = cf->clfunc->source.get();
assert(!cf->clfunc->source->scoping->areGlobalsFromModule());
assert(globals);
interpreter.setGlobals(globals);
Value v = ASTInterpreter::execute(interpreter);
......@@ -1273,7 +1276,7 @@ Box* astInterpretFrom(CompiledFunction* cf, AST_expr* after_expr, AST_stmt* encl
ASTInterpreter interpreter(cf);
ScopeInfo* scope_info = cf->clfunc->source->getScopeInfo();
SourceInfo* source_info = cf->clfunc->source;
SourceInfo* source_info = cf->clfunc->source.get();
assert(cf->clfunc->source->scoping->areGlobalsFromModule());
interpreter.setGlobals(source_info->parent_module);
......
......@@ -33,9 +33,9 @@ struct LineInfo;
extern const void* interpreter_instr_addr;
Box* astInterpretFunction(CompiledFunction* f, int nargs, Box* closure, Box* generator, BoxedDict* globals, Box* arg1,
Box* astInterpretFunction(CompiledFunction* f, int nargs, Box* closure, Box* generator, Box* globals, Box* arg1,
Box* arg2, Box* arg3, Box** args);
Box* astInterpretFunctionEval(CompiledFunction* cf, BoxedDict* globals, Box* boxedLocals);
Box* astInterpretFunctionEval(CompiledFunction* cf, Box* globals, Box* boxedLocals);
Box* astInterpretFrom(CompiledFunction* cf, AST_expr* after_expr, AST_stmt* enclosing_stmt, Box* expr_val,
FrameStackState frame_state);
......
......@@ -34,8 +34,11 @@ namespace pyston {
DS_DEFINE_RWLOCK(codegen_rwlock);
SourceInfo::SourceInfo(BoxedModule* m, ScopingAnalysis* scoping, AST* ast, const std::vector<AST_stmt*>& body)
: parent_module(m), scoping(scoping), ast(ast), cfg(NULL), liveness(NULL), body(body) {
SourceInfo::SourceInfo(BoxedModule* m, ScopingAnalysis* scoping, AST* ast, const std::vector<AST_stmt*>& body,
std::string fn)
: parent_module(m), scoping(scoping), ast(ast), cfg(NULL), liveness(NULL), fn(std::move(fn)), body(body) {
assert(this->fn.size());
switch (ast->type) {
case AST_TYPE::ClassDef:
case AST_TYPE::Lambda:
......
......@@ -721,7 +721,7 @@ ConcreteCompilerVariable* UnknownType::hasnext(IREmitter& emitter, const OpInfo&
}
CompilerVariable* makeFunction(IREmitter& emitter, CLFunction* f, CompilerVariable* closure, bool isGenerator,
BoxedDict* globals, const std::vector<ConcreteCompilerVariable*>& defaults) {
Box* globals, const std::vector<ConcreteCompilerVariable*>& defaults) {
// Unlike the CLFunction*, which can be shared between recompilations, the Box* around it
// should be created anew every time the functiondef is encountered
......@@ -752,7 +752,7 @@ CompilerVariable* makeFunction(IREmitter& emitter, CLFunction* f, CompilerVariab
llvm::Value* isGenerator_v = llvm::ConstantInt::get(g.i1, isGenerator, false);
assert(globals == NULL);
llvm::Value* globals_v = getNullPtr(g.llvm_dict_type_ptr);
llvm::Value* globals_v = getNullPtr(g.llvm_value_type_ptr);
// We know this function call can't throw, so it's safe to use emitter.getBuilder()->CreateCall() rather than
// emitter.createCall().
......
......@@ -399,7 +399,7 @@ ConcreteCompilerVariable* makePureImaginary(IREmitter& emitter, double imag);
CompilerVariable* makeStr(const std::string*);
CompilerVariable* makeUnicode(IREmitter& emitter, const std::string*);
CompilerVariable* makeFunction(IREmitter& emitter, CLFunction*, CompilerVariable* closure, bool isGenerator,
BoxedDict* globals, const std::vector<ConcreteCompilerVariable*>& defaults);
Box* globals, const std::vector<ConcreteCompilerVariable*>& defaults);
ConcreteCompilerVariable* undefVariable();
CompilerVariable* makeTuple(const std::vector<CompilerVariable*>& elts);
......
......@@ -129,6 +129,13 @@ public:
if (error_code)
return false;
int uncompressed_size = data.size();
// Write the uncompressed size to the beginning of the file as a simple checksum.
// It looks like each lz4 block has its own data checksum, but we need to also
// make sure that we have all the blocks that we expected.
// In particular, without this, an empty file seems to be a valid lz4 stream.
file.write(reinterpret_cast<const char*>(&uncompressed_size), 4);
LZ4F_preferences_t preferences;
memset(&preferences, 0, sizeof(preferences));
preferences.frameInfo.contentChecksumFlag = contentChecksumEnabled;
......@@ -158,8 +165,14 @@ public:
const char* start = (*compressed_content)->getBufferStart();
size_t pos = 0;
size_t compressed_size = (*compressed_content)->getBufferSize();
if (compressed_size < 4)
return std::unique_ptr<llvm::MemoryBuffer>();
int orig_uncompressed_size = *reinterpret_cast<const int*>(start);
pos += 4;
size_t remaining = compressed_size - pos;
LZ4F_getFrameInfo(context, &frame_info, start, &remaining);
LZ4F_getFrameInfo(context, &frame_info, start + pos, &remaining);
pos += remaining;
std::vector<char> uncompressed;
......@@ -182,6 +195,9 @@ public:
if (uncompressed.size() != frame_info.contentSize)
return std::unique_ptr<llvm::MemoryBuffer>();
if (uncompressed.size() != orig_uncompressed_size)
return std::unique_ptr<llvm::MemoryBuffer>();
return llvm::MemoryBuffer::getMemBufferCopy(llvm::StringRef(uncompressed.data(), uncompressed.size()));
}
};
......
......@@ -918,7 +918,7 @@ static llvm::MDNode* setupDebugInfo(SourceInfo* source, llvm::Function* f, std::
llvm::DIBuilder builder(*g.cur_module);
std::string fn = source->parent_module->fn;
const std::string& fn = source->fn;
std::string dir = "";
std::string producer = "pyston; git rev " STRINGIFY(GITREV);
......
......@@ -186,7 +186,7 @@ CompiledFunction* compileFunction(CLFunction* f, FunctionSpecialization* spec, E
assert((entry_descriptor != NULL) + (spec != NULL) == 1);
SourceInfo* source = f->source;
SourceInfo* source = f->source.get();
assert(source);
std::string name = source->getName();
......@@ -198,7 +198,7 @@ CompiledFunction* compileFunction(CLFunction* f, FunctionSpecialization* spec, E
llvm::raw_string_ostream ss(s);
if (spec) {
ss << "\033[34;1mJIT'ing " << source->parent_module->fn << ":" << name << " with signature (";
ss << "\033[34;1mJIT'ing " << source->fn << ":" << name << " with signature (";
for (int i = 0; i < spec->arg_types.size(); i++) {
if (i > 0)
ss << ", ";
......@@ -208,7 +208,7 @@ CompiledFunction* compileFunction(CLFunction* f, FunctionSpecialization* spec, E
ss << ") -> ";
ss << spec->rtn_type->debugName();
} else {
ss << "\033[34;1mDoing OSR-entry partial compile of " << source->parent_module->fn << ":" << name
ss << "\033[34;1mDoing OSR-entry partial compile of " << source->fn << ":" << name
<< ", starting with backedge to block " << entry_descriptor->backedge->target->idx;
}
ss << " at effort level " << (int)effort;
......@@ -259,7 +259,7 @@ CompiledFunction* compileFunction(CLFunction* f, FunctionSpecialization* spec, E
static StatCounter us_compiling("us_compiling");
us_compiling.log(us);
if (VERBOSITY() >= 1 && us > 100000) {
printf("Took %ldms to compile %s::%s!\n", us / 1000, source->parent_module->fn.c_str(), name.c_str());
printf("Took %ldms to compile %s::%s!\n", us / 1000, source->fn.c_str(), name.c_str());
}
static StatCounter num_compiles("num_compiles");
......@@ -309,15 +309,17 @@ void compileAndRunModule(AST_Module* m, BoxedModule* bm) {
Timer _t("for compileModule()");
bm->future_flags = getFutureFlags(m, bm->fn.c_str());
const char* fn = PyModule_GetFilename(bm);
RELEASE_ASSERT(fn, "");
bm->future_flags = getFutureFlags(m, fn);
ScopingAnalysis* scoping = new ScopingAnalysis(m);
SourceInfo* si = new SourceInfo(bm, scoping, m, m->body);
CLFunction* cl_f = new CLFunction(0, 0, false, false, si);
std::unique_ptr<SourceInfo> si(new SourceInfo(bm, scoping, m, m->body, fn));
bm->setattr("__doc__", si->getDocString(), NULL);
CLFunction* cl_f = new CLFunction(0, 0, false, false, std::move(si));
EffortLevel effort = initialEffort();
assert(scoping->areGlobalsFromModule());
......@@ -332,48 +334,108 @@ void compileAndRunModule(AST_Module* m, BoxedModule* bm) {
((void (*)())cf->code)();
}
Box* evalOrExec(CLFunction* cl, Box* globals, Box* boxedLocals) {
RELEASE_ASSERT(!cl->source->scoping->areGlobalsFromModule(), "");
assert(globals && (globals->cls == module_cls || globals->cls == dict_cls));
// TODO Right now we only support going into an exec or eval through the
// intepretter, since the interpretter has a special function which lets
// us set the locals object. We should probably support it for optimized
// code as well, so we could use initialEffort() here instead of hard-coding
// INTERPRETED. This could actually be useful if we actually cache the parse
// results (since sometimes eval or exec might be called on constant strings).
EffortLevel effort = EffortLevel::INTERPRETED;
CompiledFunction* cf = compileFunction(cl, new FunctionSpecialization(VOID), effort, NULL);
assert(cf->clfunc->versions.size());
return astInterpretFunctionEval(cf, globals, boxedLocals);
}
template <typename AST_Type>
Box* evalOrExec(AST_Type* source, std::vector<AST_stmt*>& body, BoxedModule* bm, BoxedDict* globals, Box* boxedLocals) {
CompiledFunction* cf;
CLFunction* compileForEvalOrExec(AST_Type* source, std::vector<AST_stmt*>& body, std::string fn) {
LOCK_REGION(codegen_rwlock.asWrite());
assert(!globals || globals->cls == dict_cls);
Timer _t("for evalOrExec()");
{ // scope for limiting the locked region:
LOCK_REGION(codegen_rwlock.asWrite());
ScopingAnalysis* scoping = new ScopingAnalysis(source, false);
Timer _t("for evalOrExec()");
std::unique_ptr<SourceInfo> si(new SourceInfo(getCurrentModule(), scoping, source, body, std::move(fn)));
CLFunction* cl_f = new CLFunction(0, 0, false, false, std::move(si));
ScopingAnalysis* scoping = new ScopingAnalysis(source, globals == NULL);
return cl_f;
}
SourceInfo* si = new SourceInfo(bm, scoping, source, body);
CLFunction* cl_f = new CLFunction(0, 0, false, false, si);
static CLFunction* compileExec(llvm::StringRef source, llvm::StringRef fn) {
// TODO error message if parse fails or if it isn't an expr
// TODO should have a cleaner interface that can parse the Expression directly
// TODO this memory leaks
const char* code = source.data();
AST_Module* parsedModule = parse_string(code);
AST_Suite* parsedSuite = new AST_Suite(std::move(parsedModule->interned_strings));
parsedSuite->body = parsedModule->body;
return compileForEvalOrExec(parsedSuite, parsedSuite->body, fn);
}
// TODO Right now we only support going into an exec or eval through the
// intepretter, since the interpretter has a special function which lets
// us set the locals object. We should probably support it for optimized
// code as well, so we could use initialEffort() here instead of hard-coding
// INTERPRETED. This could actually be useful if we actually cache the parse
// results (since sometimes eval or exec might be called on constant strings).
EffortLevel effort = EffortLevel::INTERPRETED;
Box* compile(Box* source, Box* fn, Box* type, Box** _args) {
Box* flags = _args[0];
Box* dont_inherit = _args[0];
RELEASE_ASSERT(flags == boxInt(0), "");
RELEASE_ASSERT(dont_inherit == boxInt(0), "");
cf = compileFunction(cl_f, new FunctionSpecialization(VOID), effort, NULL);
assert(cf->clfunc->versions.size());
// source is allowed to be an AST, unicode, or anything that supports the buffer protocol
if (source->cls == unicode_cls) {
source = PyUnicode_AsUTF8String(source);
if (!source)
throwCAPIException();
// cf.cf_flags |= PyCF_SOURCE_IS_UTF8
}
return astInterpretFunctionEval(cf, globals, boxedLocals);
if (isSubclass(fn->cls, unicode_cls)) {
fn = _PyUnicode_AsDefaultEncodedString(fn, NULL);
if (!fn)
throwCAPIException();
}
RELEASE_ASSERT(isSubclass(fn->cls, str_cls), "");
if (isSubclass(type->cls, unicode_cls)) {
type = _PyUnicode_AsDefaultEncodedString(type, NULL);
if (!type)
throwCAPIException();
}
RELEASE_ASSERT(isSubclass(type->cls, str_cls), "");
llvm::StringRef filename_str = static_cast<BoxedString*>(fn)->s;
llvm::StringRef type_str = static_cast<BoxedString*>(type)->s;
RELEASE_ASSERT(isSubclass(source->cls, str_cls), "");
llvm::StringRef source_str = static_cast<BoxedString*>(source)->s;
CLFunction* cl;
if (type_str == "exec") {
cl = compileExec(source_str, filename_str);
} else if (type_str == "eval") {
fatalOrError(NotImplemented, "unimplemented");
throwCAPIException();
} else if (type_str == "single") {
fatalOrError(NotImplemented, "unimplemented");
throwCAPIException();
} else {
raiseExcHelper(ValueError, "compile() arg 3 must be 'exec', 'eval' or 'single'");
}
return codeForCLFunction(cl);
}
// Main entrypoints for eval and exec.
Box* eval(Box* boxedCode) {
Box* boxedLocals = fastLocalsToBoxedLocals();
BoxedModule* module = getCurrentModule();
Box* globals = getGlobals();
if (globals == module)
globals = NULL;
if (globals && globals->cls == attrwrapper_cls && unwrapAttrWrapper(globals) == module)
globals = NULL;
globals = module;
if (boxedCode->cls == unicode_cls) {
boxedCode = PyUnicode_AsUTF8String(boxedCode);
......@@ -408,9 +470,10 @@ Box* eval(Box* boxedCode) {
stmt->value = parsedExpr->body;
std::vector<AST_stmt*> body = { stmt };
assert(!globals || globals->cls == dict_cls);
assert(globals && (globals->cls == module_cls || globals->cls == dict_cls));
return evalOrExec<AST_Expression>(parsedExpr, body, module, static_cast<BoxedDict*>(globals), boxedLocals);
CLFunction* cl = compileForEvalOrExec(parsedExpr, body, "<string>");
return evalOrExec(cl, globals, boxedLocals);
}
Box* exec(Box* boxedCode, Box* globals, Box* locals) {
......@@ -432,8 +495,6 @@ Box* exec(Box* boxedCode, Box* globals, Box* locals) {
if (locals == None)
locals = NULL;
// TODO boxedCode is allowed to be a tuple
// TODO need to handle passing in globals
if (locals == NULL) {
locals = globals;
}
......@@ -446,19 +507,18 @@ Box* exec(Box* boxedCode, Box* globals, Box* locals) {
globals = getGlobals();
BoxedModule* module = getCurrentModule();
if (globals == module)
globals = NULL;
if (globals && globals->cls == attrwrapper_cls && unwrapAttrWrapper(globals) == module)
globals = NULL;
globals = module;
ASSERT(!globals || globals->cls == dict_cls, "%s", globals->cls->tp_name);
assert(globals && (globals->cls == module_cls || globals->cls == dict_cls));
if (globals) {
// From CPython (they set it to be f->f_builtins):
if (PyDict_GetItemString(globals, "__builtins__") == NULL)
PyDict_SetItemString(globals, "__builtins__", builtins_module);
Box* globals_dict = globals;
if (globals->cls == module_cls)
globals_dict = makeAttrWrapper(globals);
if (PyDict_GetItemString(globals_dict, "__builtins__") == NULL)
PyDict_SetItemString(globals_dict, "__builtins__", builtins_module);
}
if (boxedCode->cls == unicode_cls) {
......@@ -468,14 +528,17 @@ Box* exec(Box* boxedCode, Box* globals, Box* locals) {
// cf.cf_flags |= PyCF_SOURCE_IS_UTF8
}
// TODO same issues as in `eval`
RELEASE_ASSERT(boxedCode->cls == str_cls, "%s", boxedCode->cls->tp_name);
const char* code = static_cast<BoxedString*>(boxedCode)->s.data();
AST_Module* parsedModule = parse_string(code);
AST_Suite* parsedSuite = new AST_Suite(std::move(parsedModule->interned_strings));
parsedSuite->body = parsedModule->body;
CLFunction* cl;
if (boxedCode->cls == str_cls) {
cl = compileExec(static_cast<BoxedString*>(boxedCode)->s, "<string>");
} else if (boxedCode->cls == code_cls) {
cl = clfunctionFromCode(boxedCode);
} else {
abort();
}
assert(cl);
return evalOrExec<AST_Suite>(parsedSuite, parsedSuite->body, module, static_cast<BoxedDict*>(globals), locals);
return evalOrExec(cl, globals, locals);
}
// If a function version keeps failing its speculations, kill it (remove it
......
......@@ -39,6 +39,7 @@ CompiledFunction* cfForMachineFunctionName(const std::string&);
extern "C" Box* exec(Box* boxedCode, Box* globals, Box* locals);
extern "C" Box* eval(Box* boxedCode);
extern "C" Box* compile(Box* source, Box* filename, Box* mode, Box** _args /* flags, dont_inherit */);
}
#endif
......@@ -2591,12 +2591,12 @@ CLFunction* wrapFunction(AST* node, AST_arguments* args, const std::vector<AST_s
CLFunction*& cl = made[node];
if (cl == NULL) {
SourceInfo* si = new SourceInfo(source->parent_module, source->scoping, node, body);
std::unique_ptr<SourceInfo> si(new SourceInfo(source->parent_module, source->scoping, node, body, source->fn));
if (args)
cl = new CLFunction(args->args.size(), args->defaults.size(), args->vararg.str().size(),
args->kwarg.str().size(), si);
args->kwarg.str().size(), std::move(si));
else
cl = new CLFunction(0, 0, 0, 0, si);
cl = new CLFunction(0, 0, 0, 0, std::move(si));
}
return cl;
}
......
......@@ -387,7 +387,7 @@ static unw_word_t getFunctionEnd(unw_word_t ip) {
// the stack frame that they were created in, so we need to use this approach (as opposed to
// C++11 range loops, for example).
// Return true from the handler to stop iteration at that frame.
void unwindPythonStack(std::function<bool(std::unique_ptr<PythonFrameIteratorImpl>&&)> func) {
void unwindPythonStack(std::function<bool(std::unique_ptr<PythonFrameIteratorImpl>)> func) {
static unw_word_t interpreter_instr_end = getFunctionEnd((unw_word_t)interpreter_instr_addr);
static unw_word_t generator_entry_end = getFunctionEnd((unw_word_t)generatorEntry);
......@@ -485,7 +485,7 @@ void unwindPythonStack(std::function<bool(std::unique_ptr<PythonFrameIteratorImp
static std::unique_ptr<PythonFrameIteratorImpl> getTopPythonFrame() {
std::unique_ptr<PythonFrameIteratorImpl> rtn(nullptr);
unwindPythonStack([&](std::unique_ptr<PythonFrameIteratorImpl>&& iter) {
unwindPythonStack([&](std::unique_ptr<PythonFrameIteratorImpl> iter) {
rtn = std::move(iter);
return true;
});
......@@ -497,18 +497,9 @@ static const LineInfo* lineInfoForFrame(PythonFrameIteratorImpl& frame_it) {
auto* cf = frame_it.getCF();
assert(cf);
auto source = cf->clfunc->source;
auto source = cf->clfunc->source.get();
// Hack: the "filename" for eval and exec statements is "<string>", not the filename
// of the parent module. We can't currently represent this the same way that CPython does
// (but we probably should), so just check that here:
const std::string* fn = &source->parent_module->fn;
if (source->ast->type == AST_TYPE::Suite /* exec */ || source->ast->type == AST_TYPE::Expression /* eval */) {
static const std::string string_str("<string>");
fn = &string_str;
}
return new LineInfo(current_stmt->lineno, current_stmt->col_offset, *fn, source->getName());
return new LineInfo(current_stmt->lineno, current_stmt->col_offset, source->fn, source->getName());
}
static StatCounter us_gettraceback("us_gettraceback");
......@@ -525,7 +516,7 @@ BoxedTraceback* getTraceback() {
Timer _t("getTraceback", 1000);
std::vector<const LineInfo*> entries;
unwindPythonStack([&](std::unique_ptr<PythonFrameIteratorImpl>&& frame_iter) {
unwindPythonStack([&](std::unique_ptr<PythonFrameIteratorImpl> frame_iter) {
const LineInfo* line_info = lineInfoForFrame(*frame_iter.get());
if (line_info)
entries.push_back(line_info);
......@@ -545,7 +536,7 @@ ExcInfo* getFrameExcInfo() {
ExcInfo* copy_from_exc = NULL;
ExcInfo* cur_exc = NULL;
unwindPythonStack([&](std::unique_ptr<PythonFrameIteratorImpl>&& frame_iter) {
unwindPythonStack([&](std::unique_ptr<PythonFrameIteratorImpl> frame_iter) {
FrameInfo* frame_info = frame_iter->getFrameInfo();
copy_from_exc = &frame_info->exc;
......@@ -609,7 +600,7 @@ BoxedModule* getCurrentModule() {
PythonFrameIterator getPythonFrame(int depth) {
std::unique_ptr<PythonFrameIteratorImpl> rtn(nullptr);
unwindPythonStack([&](std::unique_ptr<PythonFrameIteratorImpl>&& frame_iter) {
unwindPythonStack([&](std::unique_ptr<PythonFrameIteratorImpl> frame_iter) {
if (depth == 0) {
rtn = std::move(frame_iter);
return true;
......@@ -631,7 +622,7 @@ void PythonFrameIterator::operator=(PythonFrameIterator&& rhs) {
std::swap(this->impl, rhs.impl);
}
PythonFrameIterator::PythonFrameIterator(std::unique_ptr<PythonFrameIteratorImpl>&& impl) {
PythonFrameIterator::PythonFrameIterator(std::unique_ptr<PythonFrameIteratorImpl> impl) {
std::swap(this->impl, impl);
}
......@@ -641,7 +632,7 @@ PythonFrameIterator::PythonFrameIterator(std::unique_ptr<PythonFrameIteratorImpl
FrameStackState getFrameStackState() {
FrameStackState rtn(NULL, NULL);
bool found = false;
unwindPythonStack([&](std::unique_ptr<PythonFrameIteratorImpl>&& frame_iter) {
unwindPythonStack([&](std::unique_ptr<PythonFrameIteratorImpl> frame_iter) {
BoxedDict* d;
BoxedClosure* closure;
CompiledFunction* cf;
......@@ -897,7 +888,7 @@ FrameInfo* PythonFrameIterator::getFrameInfo() {
PythonFrameIterator PythonFrameIterator::getCurrentVersion() {
std::unique_ptr<PythonFrameIteratorImpl> rtn(nullptr);
auto& impl = this->impl;
unwindPythonStack([&](std::unique_ptr<PythonFrameIteratorImpl>&& frame_iter) {
unwindPythonStack([&](std::unique_ptr<PythonFrameIteratorImpl> frame_iter) {
if (frame_iter->pointsToTheSameAs(*impl.get())) {
rtn = std::move(frame_iter);
return true;
......
......@@ -64,7 +64,7 @@ public:
PythonFrameIterator(PythonFrameIterator&& rhs);
void operator=(PythonFrameIterator&& rhs);
PythonFrameIterator(std::unique_ptr<PythonFrameIteratorImpl>&& impl);
PythonFrameIterator(std::unique_ptr<PythonFrameIteratorImpl> impl);
~PythonFrameIterator();
};
......
......@@ -246,6 +246,7 @@ public:
LivenessAnalysis* liveness;
std::unordered_map<const OSREntryDescriptor*, PhiAnalysis*> phis;
bool is_generator;
std::string fn; // equivalent of code.co_filename
InternedStringPool& getInternedStrings();
......@@ -260,7 +261,7 @@ public:
Box* getDocString();
SourceInfo(BoxedModule* m, ScopingAnalysis* scoping, AST* ast, const std::vector<AST_stmt*>& body);
SourceInfo(BoxedModule* m, ScopingAnalysis* scoping, AST* ast, const std::vector<AST_stmt*>& body, std::string fn);
};
typedef std::vector<CompiledFunction*> FunctionList;
......@@ -271,7 +272,7 @@ public:
int num_defaults;
bool takes_varargs, takes_kwargs;
SourceInfo* source;
std::unique_ptr<SourceInfo> source;
ParamNames param_names;
FunctionList
......@@ -287,14 +288,15 @@ public:
const std::vector<const std::string*>*);
InternalCallable internal_callable = NULL;
CLFunction(int num_args, int num_defaults, bool takes_varargs, bool takes_kwargs, SourceInfo* source)
CLFunction(int num_args, int num_defaults, bool takes_varargs, bool takes_kwargs,
std::unique_ptr<SourceInfo> source)
: num_args(num_args), num_defaults(num_defaults), takes_varargs(takes_varargs), takes_kwargs(takes_kwargs),
source(source), param_names(source->ast), always_use_version(NULL) {
source(std::move(source)), param_names(this->source->ast), always_use_version(NULL) {
assert(num_args >= num_defaults);
}
CLFunction(int num_args, int num_defaults, bool takes_varargs, bool takes_kwargs, const ParamNames& param_names)
: num_args(num_args), num_defaults(num_defaults), takes_varargs(takes_varargs), takes_kwargs(takes_kwargs),
source(NULL), param_names(param_names), always_use_version(NULL) {
source(nullptr), param_names(param_names), always_use_version(NULL) {
assert(num_args >= num_defaults);
}
......
......@@ -283,7 +283,7 @@ static int main(int argc, char** argv) {
if (!main_module) {
main_module = createModule("__main__", "<stdin>");
} else {
main_module->fn = "<stdin>";
// main_module->fn = "<stdin>";
}
for (;;) {
......
......@@ -1149,6 +1149,10 @@ void setupBuiltins() {
builtins_module->giveAttr("execfile",
new BoxedBuiltinFunctionOrMethod(boxRTFunction((void*)execfile, UNKNOWN, 1), "execfile"));
builtins_module->giveAttr(
"compile", new BoxedBuiltinFunctionOrMethod(boxRTFunction((void*)compile, UNKNOWN, 5, 2, false, false),
"compile", { boxInt(0), boxInt(0) }));
builtins_module->giveAttr(
"map", new BoxedBuiltinFunctionOrMethod(boxRTFunction((void*)map, LIST, 1, 0, true, false), "map"));
builtins_module->giveAttr(
......
......@@ -1273,8 +1273,31 @@ extern "C" void PyEval_InitThreads(void) noexcept {
}
extern "C" char* PyModule_GetName(PyObject* m) noexcept {
assert(m->cls == module_cls);
return &static_cast<BoxedModule*>(m)->fn[0];
PyObject* d;
PyObject* nameobj;
if (!PyModule_Check(m)) {
PyErr_BadArgument();
return NULL;
}
if ((nameobj = m->getattr("__name__")) == NULL || !PyString_Check(nameobj)) {
PyErr_SetString(PyExc_SystemError, "nameless module");
return NULL;
}
return PyString_AsString(nameobj);
}
extern "C" char* PyModule_GetFilename(PyObject* m) noexcept {
PyObject* d;
PyObject* fileobj;
if (!PyModule_Check(m)) {
PyErr_BadArgument();
return NULL;
}
if ((fileobj = m->getattr("__file__")) == NULL || !PyString_Check(fileobj)) {
PyErr_SetString(PyExc_SystemError, "module filename missing");
return NULL;
}
return PyString_AsString(fileobj);
}
BoxedModule* importCExtension(const std::string& full_name, const std::string& last_name, const std::string& path) {
......@@ -1315,7 +1338,6 @@ BoxedModule* importCExtension(const std::string& full_name, const std::string& l
BoxedModule* m = static_cast<BoxedModule*>(_m);
m->setattr("__file__", boxString(path), NULL);
m->fn = path;
return m;
}
......
......@@ -48,7 +48,7 @@ public:
static Box* filename(Box* b, void*) {
RELEASE_ASSERT(b->cls == code_cls, "");
return boxString(static_cast<BoxedCode*>(b)->f->source->parent_module->fn);
return boxString(static_cast<BoxedCode*>(b)->f->source->fn);
}
static Box* argcount(Box* b, void*) {
......@@ -96,6 +96,11 @@ Box* codeForCLFunction(CLFunction* f) {
return new BoxedCode(f);
}
CLFunction* clfunctionFromCode(Box* code) {
assert(code->cls == code_cls);
return static_cast<BoxedCode*>(code)->f;
}
extern "C" PyCodeObject* PyCode_New(int, int, int, int, PyObject*, PyObject*, PyObject*, PyObject*, PyObject*,
PyObject*, PyObject*, PyObject*, int, PyObject*) noexcept {
RELEASE_ASSERT(0, "not implemented");
......
......@@ -2975,7 +2975,7 @@ Box* callFunc(BoxedFunctionBase* func, CallRewriteArgs* rewrite_args, ArgPassSpe
}
Box* callCLFunc(CLFunction* f, CallRewriteArgs* rewrite_args, int num_output_args, BoxedClosure* closure,
BoxedGenerator* generator, BoxedDict* globals, Box* oarg1, Box* oarg2, Box* oarg3, Box** oargs) {
BoxedGenerator* generator, Box* globals, Box* oarg1, Box* oarg2, Box* oarg3, Box** oargs) {
CompiledFunction* chosen_cf = pickVersion(f, num_output_args, oarg1, oarg2, oarg3, oargs);
assert(chosen_cf->is_interpreted == (chosen_cf->code == NULL));
......
......@@ -150,7 +150,7 @@ bool isUserDefined(BoxedClass* cls);
Box* processDescriptor(Box* obj, Box* inst, Box* owner);
Box* callCLFunc(CLFunction* f, CallRewriteArgs* rewrite_args, int num_output_args, BoxedClosure* closure,
BoxedGenerator* generator, BoxedDict* globals, Box* oarg1, Box* oarg2, Box* oarg3, Box** oargs);
BoxedGenerator* generator, Box* globals, Box* oarg1, Box* oarg2, Box* oarg3, Box** oargs);
static const char* objectNewParameterTypeErrorMsg() {
if (PYTHON_VERSION_HEX >= version_hex(2, 7, 4)) {
......
......@@ -30,7 +30,7 @@ public:
std::vector<const LineInfo*> lines;
Box* py_lines;
BoxedTraceback(std::vector<const LineInfo*>&& lines) : lines(std::move(lines)), py_lines(NULL) {}
BoxedTraceback(std::vector<const LineInfo*> lines) : lines(std::move(lines)), py_lines(NULL) {}
BoxedTraceback() : py_lines(NULL) {}
DEFAULT_CLASS(traceback_cls);
......
......@@ -311,9 +311,10 @@ BoxedFunction::BoxedFunction(CLFunction* f) : BoxedFunction(f, {}) {
}
BoxedFunction::BoxedFunction(CLFunction* f, std::initializer_list<Box*> defaults, BoxedClosure* closure,
bool isGenerator, BoxedDict* globals)
bool isGenerator, Box* globals)
: BoxedFunctionBase(f, defaults, closure, isGenerator) {
assert((!globals) == (!f->source || f->source->scoping->areGlobalsFromModule()));
this->globals = globals;
// TODO eventually we want this to assert(f->source), I think, but there are still
......@@ -380,7 +381,8 @@ static void functionDtor(Box* b) {
self->dependent_ics.~ICInvalidator();
}
BoxedModule::BoxedModule(const std::string& name, const std::string& fn, const char* doc) : fn(fn) {
// TODO(kmod): builtin modules are not supposed to have a __file__ attribute
BoxedModule::BoxedModule(const std::string& name, const std::string& fn, const char* doc) {
this->giveAttr("__name__", boxString(name));
this->giveAttr("__file__", boxString(fn));
this->giveAttr("__doc__", doc ? boxStrConstant(doc) : None);
......@@ -418,12 +420,10 @@ extern "C" void moduleGCHandler(GCVisitor* v, Box* b) {
// This mustn't throw; our IR generator generates calls to it without "invoke" even when there are exception handlers /
// finally-blocks in scope.
// TODO: should we use C++11 `noexcept' here?
extern "C" Box* boxCLFunction(CLFunction* f, BoxedClosure* closure, bool isGenerator, BoxedDict* globals,
extern "C" Box* boxCLFunction(CLFunction* f, BoxedClosure* closure, bool isGenerator, Box* globals,
std::initializer_list<Box*> defaults) {
if (closure)
assert(closure->cls == closure_cls);
if (globals)
assert(globals->cls == dict_cls);
return new BoxedFunction(f, defaults, closure, isGenerator, globals);
}
......@@ -1095,10 +1095,13 @@ Box* moduleRepr(BoxedModule* m) {
os << "<module '" << m->name() << "' ";
if (m->fn == "__builtin__") {
const char* filename = PyModule_GetFilename((PyObject*)m);
// TODO(kmod): builtin modules are not supposed to have a __file__ attribute
if (!filename || !strcmp(filename, "__builtin__")) {
PyErr_Clear();
os << "(built-in)>";
} else {
os << "from '" << m->fn << "'>";
os << "from '" << filename << "'>";
}
return boxString(os.str());
}
......
......@@ -86,7 +86,7 @@ extern BoxedClass* object_cls, *type_cls, *bool_cls, *int_cls, *long_cls, *float
*none_cls, *instancemethod_cls, *list_cls, *slice_cls, *module_cls, *dict_cls, *tuple_cls, *file_cls,
*enumerate_cls, *xrange_cls, *member_cls, *method_cls, *closure_cls, *generator_cls, *complex_cls, *basestring_cls,
*property_cls, *staticmethod_cls, *classmethod_cls, *attrwrapper_cls, *pyston_getset_cls, *capi_getset_cls,
*builtin_function_or_method_cls, *set_cls, *frozenset_cls;
*builtin_function_or_method_cls, *set_cls, *frozenset_cls, *code_cls;
}
#define unicode_cls (&PyUnicode_Type)
#define memoryview_cls (&PyMemoryView_Type)
......@@ -128,7 +128,7 @@ char* getWriteableStringContents(BoxedString* s);
extern "C" void listAppendInternal(Box* self, Box* v);
extern "C" void listAppendArrayInternal(Box* self, Box** v, int nelts);
extern "C" Box* boxCLFunction(CLFunction* f, BoxedClosure* closure, bool isGenerator, BoxedDict* globals,
extern "C" Box* boxCLFunction(CLFunction* f, BoxedClosure* closure, bool isGenerator, Box* globals,
std::initializer_list<Box*> defaults);
extern "C" CLFunction* unboxCLFunction(Box* b);
extern "C" Box* createUserClass(const std::string* name, Box* base, Box* attr_dict);
......@@ -591,7 +591,7 @@ public:
// garbage values when the GC is run (BoxedFunctionBase's constructor might call the GC).
// So ick... needs to be fixed.
BoxedClosure* closure;
BoxedDict* globals;
Box* globals;
bool isGenerator;
int ndefaults;
......@@ -615,7 +615,7 @@ public:
BoxedFunction(CLFunction* f);
BoxedFunction(CLFunction* f, std::initializer_list<Box*> defaults, BoxedClosure* closure = NULL,
bool isGenerator = false, BoxedDict* globals = NULL);
bool isGenerator = false, Box* globals = NULL);
DEFAULT_CLASS(function_cls);
};
......@@ -633,8 +633,6 @@ class BoxedModule : public Box {
public:
HCAttrs attrs;
// for traceback purposes; not the same as __file__. This corresponds to co_filename
std::string fn;
FutureFlags future_flags;
BoxedModule(const std::string& name, const std::string& fn, const char* doc = NULL);
......@@ -823,6 +821,7 @@ extern Box* dict_descr;
Box* codeForFunction(BoxedFunction*);
Box* codeForCLFunction(CLFunction*);
CLFunction* clfunctionFromCode(Box* code);
Box* getFrame(int depth);
}
......
c = compile("a = 1; print a", "test.py", "exec")
print type(c), c.co_filename, c.co_name
print
a = 0
exec c
print a
print
a = 0
g = {}
exec c in g
print a, sorted(g.keys())
print
g = {}
exec """
c = compile("a = 1; print a", "test.py", "exec")
""" in g
a = 0
exec g['c']
print a, sorted(g.keys())
print
a = 0
g = {'_c':c}
exec "exec _c" in g
print a, sorted(g.keys())
from __future__ import division
# compile() inherits the future flags of the parent module
print 1 / 2
exec "print 1 / 2"
exec compile("print 1 / 2", "<string>", "exec")
# But you can explicitly request that they not be inherited:
# exec compile("print 1 / 2", "<string>", "exec", flags=0, dont_inherit=True)
# expected: fail
# - not currently supported
def f():
global b
b = 1
print b
print
b = 0
g = {}
exec f.func_code in g
print b, sorted(g.keys())
print
b = 0
g = {'f':f}
exec "f()" in g
print b, sorted(g.keys())
from __future__ import unicode_literals
print type("hello world")
......@@ -36,7 +36,7 @@ TEST_F(AnalysisTest, augassign) {
ASSERT_FALSE(scope_info->getScopeTypeOfName(module->interned_strings->get("a")) == ScopeInfo::VarScopeType::GLOBAL);
ASSERT_FALSE(scope_info->getScopeTypeOfName(module->interned_strings->get("b")) == ScopeInfo::VarScopeType::GLOBAL);
SourceInfo* si = new SourceInfo(createModule("augassign", fn), scoping, func, func->body);
SourceInfo* si = new SourceInfo(createModule("augassign", fn), scoping, func, func->body, fn);
CFG* cfg = computeCFG(si, func->body);
LivenessAnalysis* liveness = computeLivenessInfo(cfg);
......@@ -64,7 +64,7 @@ void doOsrTest(bool is_osr, bool i_maybe_undefined) {
ScopeInfo* scope_info = scoping->getScopeInfoForNode(func);
SourceInfo* si = new SourceInfo(createModule("osr" + std::to_string((is_osr << 1) + i_maybe_undefined), fn),
scoping, func, func->body);
scoping, func, func->body, fn);
CFG* cfg = computeCFG(si, func->body);
LivenessAnalysis* liveness = computeLivenessInfo(cfg);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment