Convert parts of the import system to use BoxedStrings

Reduces boxing during the import process

src/capi/modsupport.cpp

@@ -417,7 +417,7 @@ extern "C" PyObject* Py_InitModule4(const char* name, PyMethodDef* methods, cons
         }
     }
 
-    BoxedModule* module = createModule(name, NULL, doc);
+    BoxedModule* module = createModule(boxString(name), NULL, doc);
 
     // Pass self as is, even if NULL we are not allowed to change it to None
     Box* passthrough = static_cast<Box*>(self);

src/core/types.h

@@ -711,8 +711,8 @@ class BoxedClass;
 // TODO these shouldn't be here
 void setupRuntime();
 void teardownRuntime();
-Box* createAndRunModule(const std::string& name, const std::string& fn);
-BoxedModule* createModule(const std::string& name, const char* fn = NULL, const char* doc = NULL);
+Box* createAndRunModule(BoxedString* name, const std::string& fn);
+BoxedModule* createModule(BoxedString* name, const char* fn = NULL, const char* doc = NULL);
 Box* moduleInit(BoxedModule* self, Box* name, Box* doc = NULL);
 
 // TODO where to put this

src/jit.cpp

@@ -417,7 +417,7 @@ static int main(int argc, char** argv) {
         // if the user invoked `pyston -c command`
         if (command != NULL) {
             try {
-                main_module = createModule("__main__", "<string>");
+                main_module = createModule(boxString("__main__"), "<string>");
                 AST_Module* m = parse_string(command);
                 compileAndRunModule(m, main_module);
                 rtncode = 0;
@@ -428,7 +428,7 @@ static int main(int argc, char** argv) {
             }
         } else if (module != NULL) {
             // TODO: CPython uses the same main module for all code paths
-            main_module = createModule("__main__", "<string>");
+            main_module = createModule(boxString("__main__"), "<string>");
             rtncode = (RunModule(module, 1) != 0);
         } else {
             rtncode = 0;
@@ -455,7 +455,7 @@ static int main(int argc, char** argv) {
                 prependToSysPath(real_path);
                 free(real_path);
 
-                main_module = createModule("__main__", fn);
+                main_module = createModule(boxString("__main__"), fn);
                 try {
                     AST_Module* ast = caching_parse_file(fn);
                     compileAndRunModule(ast, main_module);
@@ -474,7 +474,7 @@ static int main(int argc, char** argv) {
             Py_InspectFlag = 0;
 
             if (!main_module) {
-                main_module = createModule("__main__", "<stdin>");
+                main_module = createModule(boxString("__main__"), "<stdin>");
             } else {
                 // main_module->fn = "<stdin>";
             }

src/runtime/builtin_modules/ast.cpp

@@ -66,7 +66,7 @@ extern "C" int PyAST_Check(PyObject* o) noexcept {
 }
 
 void setupAST() {
-    BoxedModule* ast_module = createModule("_ast", "__builtin__");
+    BoxedModule* ast_module = createModule(boxString("_ast"), "__builtin__");
 
     ast_module->giveAttr("PyCF_ONLY_AST", boxInt(PyCF_ONLY_AST));
 

src/runtime/builtin_modules/builtins.cpp

@@ -1449,9 +1449,9 @@ Box* builtinFormat(Box* value, Box* format_spec) {
 }
 
 void setupBuiltins() {
-    builtins_module
-        = createModule("__builtin__", NULL, "Built-in functions, exceptions, and other objects.\n\nNoteworthy: None is "
-                                            "the `nil' object; Ellipsis represents `...' in slices.");
+    builtins_module = createModule(boxString("__builtin__"), NULL,
+                                   "Built-in functions, exceptions, and other objects.\n\nNoteworthy: None is "
+                                   "the `nil' object; Ellipsis represents `...' in slices.");
 
     BoxedHeapClass* ellipsis_cls
         = BoxedHeapClass::create(type_cls, object_cls, NULL, 0, 0, sizeof(Box), false, "ellipsis");

src/runtime/builtin_modules/gc.cpp

@@ -43,7 +43,7 @@ static Box* enable() {
 }
 
 void setupGC() {
-    BoxedModule* gc_module = createModule("gc");
+    BoxedModule* gc_module = createModule(boxString("gc"));
 
     gc_module->giveAttr("collect",
                         new BoxedBuiltinFunctionOrMethod(boxRTFunction((void*)gcCollect, NONE, 0), "collect"));

src/runtime/builtin_modules/pyston.cpp

@@ -64,7 +64,7 @@ static Box* dumpStats(Box* includeZeros) {
 }
 
 void setupPyston() {
-    pyston_module = createModule("__pyston__");
+    pyston_module = createModule(boxString("__pyston__"));
 
     pyston_module->giveAttr("setOption",
                             new BoxedBuiltinFunctionOrMethod(boxRTFunction((void*)setOption, UNKNOWN, 2), "setOption"));

src/runtime/builtin_modules/sys.cpp

@@ -586,7 +586,7 @@ void setupSys() {
     gc::registerPermanentRoot(sys_modules_dict);
 
     // This is ok to call here because we've already created the sys_modules_dict
-    sys_module = createModule("sys");
+    sys_module = createModule(boxString("sys"));
 
     sys_module->giveAttr("modules", sys_modules_dict);
 

src/runtime/import.cpp

@@ -35,13 +35,12 @@ static const std::string name_str("__name__");
 static const std::string package_str("__package__");
 static BoxedClass* null_importer_cls;
 
-static void removeModule(const std::string& name) {
+static void removeModule(BoxedString* name) {
     BoxedDict* d = getSysModulesDict();
-    Box* b_name = boxString(name);
-    d->d.erase(b_name);
+    d->d.erase(name);
 }
 
-Box* createAndRunModule(const std::string& name, const std::string& fn) {
+Box* createAndRunModule(BoxedString* name, const std::string& fn) {
     BoxedModule* module = createModule(name, fn.c_str());
 
     AST_Module* ast = caching_parse_file(fn.c_str());
@@ -53,13 +52,13 @@ Box* createAndRunModule(const std::string& name, const std::string& fn) {
         throw e;
     }
 
-    Box* r = getSysModulesDict()->getOrNull(boxString(name));
+    Box* r = getSysModulesDict()->getOrNull(name);
     if (!r)
-        raiseExcHelper(ImportError, "Loaded module %.200s not found in sys.modules", name.c_str());
+        raiseExcHelper(ImportError, "Loaded module %.200s not found in sys.modules", name->c_str());
     return r;
 }
 
-static Box* createAndRunModule(const std::string& name, const std::string& fn, const std::string& module_path) {
+static Box* createAndRunModule(BoxedString* name, const std::string& fn, const std::string& module_path) {
     BoxedModule* module = createModule(name, fn.c_str());
 
     Box* b_path = boxString(module_path);
@@ -79,9 +78,9 @@ static Box* createAndRunModule(const std::string& name, const std::string& fn, c
         throw e;
     }
 
-    Box* r = getSysModulesDict()->getOrNull(boxString(name));
+    Box* r = getSysModulesDict()->getOrNull(name);
     if (!r)
-        raiseExcHelper(ImportError, "Loaded module %.200s not found in sys.modules", name.c_str());
+        raiseExcHelper(ImportError, "Loaded module %.200s not found in sys.modules", name->c_str());
     return r;
 }
 
@@ -184,7 +183,7 @@ struct SearchResult {
     SearchResult(Box* loader) : loader(loader), type(IMP_HOOK) {}
 };
 
-SearchResult findModule(const std::string& name, const std::string& full_name, BoxedList* path_list) {
+SearchResult findModule(const std::string& name, BoxedString* full_name, BoxedList* path_list) {
     static BoxedString* meta_path_str = internStringImmortal("meta_path");
     BoxedList* meta_path = static_cast<BoxedList*>(sys_module->getattr(meta_path_str));
     if (!meta_path || meta_path->cls != list_cls)
@@ -196,8 +195,7 @@ SearchResult findModule(const std::string& name, const std::string& full_name, B
 
         auto path_pass = path_list ? path_list : None;
         CallattrFlags callattr_flags{.cls_only = false, .null_on_nonexistent = false, .argspec = ArgPassSpec(2) };
-        Box* loader
-            = callattr(finder, findmodule_str, callattr_flags, boxString(full_name), path_pass, NULL, NULL, NULL);
+        Box* loader = callattr(finder, findmodule_str, callattr_flags, full_name, path_pass, NULL, NULL, NULL);
 
         if (loader != None)
             return SearchResult(loader);
@@ -240,8 +238,7 @@ SearchResult findModule(const std::string& name, const std::string& full_name, B
 
         if (importer != None) {
             CallattrFlags callattr_flags{.cls_only = false, .null_on_nonexistent = false, .argspec = ArgPassSpec(1) };
-            Box* loader
-                = callattr(importer, findmodule_str, callattr_flags, boxString(full_name), NULL, NULL, NULL, NULL);
+            Box* loader = callattr(importer, findmodule_str, callattr_flags, full_name, NULL, NULL, NULL, NULL);
             if (loader != None)
                 return SearchResult(loader);
         }
@@ -371,11 +368,10 @@ static Box* getParent(Box* globals, int level, std::string& buf) {
 }
 
 
-static Box* importSub(const std::string& name, const std::string& full_name, Box* parent_module) {
-    Box* boxed_name = boxString(full_name);
+static Box* importSub(const std::string& name, BoxedString* full_name, Box* parent_module) {
     BoxedDict* sys_modules = getSysModulesDict();
-    if (sys_modules->d.find(boxed_name) != sys_modules->d.end()) {
-        return sys_modules->d[boxed_name];
+    if (sys_modules->d.find(full_name) != sys_modules->d.end()) {
+        return sys_modules->d[full_name];
     }
 
     BoxedList* path_list;
@@ -406,12 +402,11 @@ static Box* importSub(const std::string& name, const std::string& full_name, Box
                 CallattrFlags callattr_flags{.cls_only = false,
                                              .null_on_nonexistent = false,
                                              .argspec = ArgPassSpec(1) };
-                module
-                    = callattr(sr.loader, loadmodule_str, callattr_flags, boxString(full_name), NULL, NULL, NULL, NULL);
+                module = callattr(sr.loader, loadmodule_str, callattr_flags, full_name, NULL, NULL, NULL, NULL);
             } else
                 RELEASE_ASSERT(0, "%d", sr.type);
         } catch (ExcInfo e) {
-            removeModule(name);
+            removeModule(full_name);
             throw e;
         }
 
@@ -464,10 +459,10 @@ static bool loadNext(Box* mod, Box* altmod, std::string& name, std::string& buf,
     std::string subname(local_name.substr(0, len));
     buf += subname;
 
-    result = importSub(subname, buf, mod);
+    result = importSub(subname, boxString(buf), mod);
     if (result == None && altmod != mod) {
         /* Here, altmod must be None and mod must not be None */
-        result = importSub(subname, subname, altmod);
+        result = importSub(subname, boxString(subname), altmod);
         if (result != NULL && result != None) {
             markMiss(buf);
 
@@ -597,7 +592,7 @@ static void ensureFromlist(Box* module, Box* fromlist, std::string& buf, bool re
             continue;
 
         // Just want to import it and add it to the modules list for now:
-        importSub(s->s(), (llvm::Twine(buf) + "." + s->s()).str(), module);
+        importSub(s->s(), boxStringTwine(llvm::Twine(buf) + "." + s->s()), module);
     }
 }
 
@@ -626,11 +621,12 @@ extern "C" PyObject* PyImport_ImportModule(const char* name) noexcept {
 extern "C" PyObject* PyImport_AddModule(const char* name) noexcept {
     try {
         PyObject* modules = getSysModulesDict();
-        PyObject* m = PyDict_GetItemString(modules, name);
+        BoxedString* s = boxString(name);
+        PyObject* m = PyDict_GetItem(modules, s);
 
         if (m != NULL && m->cls == module_cls)
             return m;
-        return createModule(name);
+        return createModule(s);
     } catch (ExcInfo e) {
         setCAPIException(e);
         return NULL;
@@ -638,6 +634,7 @@ extern "C" PyObject* PyImport_AddModule(const char* name) noexcept {
 }
 
 extern "C" PyObject* PyImport_ExecCodeModuleEx(char* name, PyObject* co, char* pathname) noexcept {
+    BoxedString* s = boxString(name);
     try {
         RELEASE_ASSERT(co->cls == str_cls, "");
         BoxedString* code = (BoxedString*)co;
@@ -652,7 +649,7 @@ extern "C" PyObject* PyImport_ExecCodeModuleEx(char* name, PyObject* co, char* p
         compileAndRunModule(ast, module);
         return module;
     } catch (ExcInfo e) {
-        removeModule(name);
+        removeModule(s);
         setCAPIException(e);
         return NULL;
     }
@@ -694,7 +691,7 @@ Box* impFindModule(Box* _name, BoxedList* path) {
     BoxedString* name = static_cast<BoxedString*>(_name);
     BoxedList* path_list = path && path != None ? path : getSysPath();
 
-    SearchResult sr = findModule(name->s(), name->s(), path_list);
+    SearchResult sr = findModule(name->s(), name, path_list);
     if (sr.type == SearchResult::SEARCH_ERROR)
         raiseExcHelper(ImportError, "%s", name->data());
 
@@ -747,10 +744,10 @@ Box* impLoadModule(Box* _name, Box* _file, Box* _pathname, Box** args) {
         RELEASE_ASSERT(suffix->s().empty(), "");
         RELEASE_ASSERT(mode->s().empty(), "");
         RELEASE_ASSERT(_file == None, "");
-        return createAndRunModule(name->s(), (llvm::Twine(pathname->s()) + "/__init__.py").str(), pathname->s());
+        return createAndRunModule(name, (llvm::Twine(pathname->s()) + "/__init__.py").str(), pathname->s());
     } else if (type->n == SearchResult::PY_SOURCE) {
         RELEASE_ASSERT(_file->cls == file_cls, "");
-        return createAndRunModule(name->s(), pathname->s());
+        return createAndRunModule(name, pathname->s());
     }
 
     Py_FatalError("unimplemented");
@@ -762,7 +759,7 @@ Box* impLoadSource(Box* _name, Box* _pathname, Box* _file) {
     RELEASE_ASSERT(_name->cls == str_cls, "");
     RELEASE_ASSERT(_pathname->cls == str_cls, "");
 
-    return createAndRunModule(static_cast<BoxedString*>(_name)->s(), static_cast<BoxedString*>(_pathname)->s());
+    return createAndRunModule(static_cast<BoxedString*>(_name), static_cast<BoxedString*>(_pathname)->s());
 }
 
 Box* impLoadDynamic(Box* _name, Box* _pathname, Box* _file) {
@@ -782,10 +779,10 @@ Box* impLoadDynamic(Box* _name, Box* _pathname, Box* _file) {
         shortname = lastdot + 1;
     }
 
-    return importCExtension(name->s(), shortname, pathname->s());
+    return importCExtension(name, shortname, pathname->s());
 }
 
-BoxedModule* importCExtension(const std::string& full_name, const std::string& last_name, const std::string& path) {
+BoxedModule* importCExtension(BoxedString* full_name, const std::string& last_name, const std::string& path) {
     void* handle = dlopen(path.c_str(), RTLD_NOW);
     if (!handle) {
         // raiseExcHelper(ImportError, "%s", dlerror());
@@ -815,7 +812,7 @@ BoxedModule* importCExtension(const std::string& full_name, const std::string& l
     bss_end -= bss_end % sizeof(void*);
     gc::registerPotentialRootRange((void*)bss_start, (void*)bss_end);
 
-    char* packagecontext = strdup(full_name.c_str());
+    char* packagecontext = strdup(full_name->c_str());
     char* oldcontext = _Py_PackageContext;
     _Py_PackageContext = packagecontext;
     (*init)();
@@ -825,8 +822,7 @@ BoxedModule* importCExtension(const std::string& full_name, const std::string& l
     checkAndThrowCAPIException();
 
     BoxedDict* sys_modules = getSysModulesDict();
-    Box* s = boxString(full_name);
-    Box* _m = sys_modules->d[s];
+    Box* _m = sys_modules->d[full_name];
     RELEASE_ASSERT(_m, "dynamic module not initialized properly");
     assert(_m->cls == module_cls);
 
@@ -886,8 +882,8 @@ Box* impIsFrozen(Box* name) {
 
 void setupImport() {
     BoxedModule* imp_module
-        = createModule("imp", NULL, "'This module provides the components needed to build your own\n"
-                                    "__import__ function.  Undocumented functions are obsolete.'");
+        = createModule(boxString("imp"), NULL, "'This module provides the components needed to build your own\n"
+                                               "__import__ function.  Undocumented functions are obsolete.'");
 
     imp_module->giveAttr("PY_SOURCE", boxInt(SearchResult::PY_SOURCE));
     imp_module->giveAttr("PY_COMPILED", boxInt(SearchResult::PY_COMPILED));

src/runtime/import.h

@@ -21,7 +21,7 @@ namespace pyston {
 
 extern "C" Box* import(int level, Box* from_imports, llvm::StringRef module_name);
 extern Box* importModuleLevel(llvm::StringRef module_name, Box* globals, Box* from_imports, int level);
-BoxedModule* importCExtension(const std::string& full_name, const std::string& last_name, const std::string& path);
+BoxedModule* importCExtension(BoxedString* full_name, const std::string& last_name, const std::string& path);
 }
 
 #endif

src/runtime/types.cpp

@@ -3816,26 +3816,25 @@ void setupRuntime() {
     TRACK_ALLOCATIONS = true;
 }
 
-BoxedModule* createModule(const std::string& name, const char* fn, const char* doc) {
+BoxedModule* createModule(BoxedString* name, const char* fn, const char* doc) {
     assert((!fn || strlen(fn)) && "probably wanted to set the fn to <stdin>?");
 
     BoxedDict* d = getSysModulesDict();
-    Box* b_name = boxString(name);
 
     // Surprisingly, there are times that we need to return the existing module if
     // one exists:
-    Box* existing = d->getOrNull(b_name);
+    Box* existing = d->getOrNull(name);
     if (existing && PyModule_Check(existing)) {
         return static_cast<BoxedModule*>(existing);
     }
 
     BoxedModule* module = new BoxedModule();
-    moduleInit(module, boxString(name), boxString(doc ? doc : ""));
+    moduleInit(module, name, boxString(doc ? doc : ""));
     if (fn)
         module->giveAttr("__file__", boxString(fn));
 
-    d->d[b_name] = module;
-    if (name == "__main__")
+    d->d[name] = module;
+    if (name->s() == "__main__")
         module->giveAttr("__builtins__", builtins_module);
     return module;
 }

test/unittests/analysis.cpp

@@ -40,7 +40,7 @@ TEST_F(AnalysisTest, augassign) {
 
     FutureFlags future_flags = getFutureFlags(module->body, fn.c_str());
 
-    SourceInfo* si = new SourceInfo(createModule("augassign", fn.c_str()), scoping, future_flags, func,
+    SourceInfo* si = new SourceInfo(createModule(boxString("augassign"), fn.c_str()), scoping, future_flags, func,
             func->body, boxString(fn));
 
     CFG* cfg = computeCFG(si, func->body);
@@ -70,7 +70,7 @@ void doOsrTest(bool is_osr, bool i_maybe_undefined) {
     FutureFlags future_flags = getFutureFlags(module->body, fn.c_str());
 
     ScopeInfo* scope_info = scoping->getScopeInfoForNode(func);
-    std::unique_ptr<SourceInfo> si(new SourceInfo(createModule("osr" + std::to_string((is_osr << 1) + i_maybe_undefined),
+    std::unique_ptr<SourceInfo> si(new SourceInfo(createModule(boxString("osr" + std::to_string((is_osr << 1) + i_maybe_undefined)),
                     fn.c_str()), scoping, future_flags, func, func->body, boxString(fn)));
     CLFunction* clfunc = new CLFunction(0, 0, false, false, std::move(si));