Misc changes from trying to run metaserver

- generalize dynamic loading support
- copy include files to cmake build directory: we don't need it for our build,
  but we need them to be there for building extension modules
- some beginning hacks for supporting Cython

Makefile

@@ -960,35 +960,35 @@ check$1 test$1: $(PYTHON_EXE_DEPS) pyston$1 ext_pyston
 
 .PHONY: run$1 dbg$1
 run$1: pyston$1 $$(RUN_DEPS)
-	./pyston$1 $$(ARGS)
+	PYTHONPATH=test/test_extension ./pyston$1 $$(ARGS)
 dbg$1: pyston$1 $$(RUN_DEPS)
-	zsh -c 'ulimit -v $$(MAX_DBG_MEM_KB); $$(GDB) $$(GDB_CMDS) --args ./pyston$1 $$(ARGS)'
+	PYTHONPATH=test/test_extension zsh -c 'ulimit -v $$(MAX_DBG_MEM_KB); $$(GDB) $$(GDB_CMDS) --args ./pyston$1 $$(ARGS)'
 nosearch_run$1_%: %.py pyston$1 $$(RUN_DEPS)
-	$(VERB) zsh -c 'ulimit -v $$(MAX_MEM_KB); ulimit -d $$(MAX_MEM_KB); time ./pyston$1 $$(ARGS) $$<'
+	$(VERB) PYTHONPATH=test/test_extension zsh -c 'ulimit -v $$(MAX_MEM_KB); ulimit -d $$(MAX_MEM_KB); time ./pyston$1 $$(ARGS) $$<'
 $$(call make_search,run$1_%)
 nosearch_dbg$1_%: %.py pyston$1 $$(RUN_DEPS)
-	$(VERB) zsh -c 'ulimit -v $$(MAX_DBG_MEM_KB); $$(GDB) $$(GDB_CMDS) --args ./pyston$1 $$(ARGS) $$<'
+	$(VERB) PYTHONPATH=test/test_extension zsh -c 'ulimit -v $$(MAX_DBG_MEM_KB); $$(GDB) $$(GDB_CMDS) --args ./pyston$1 $$(ARGS) $$<'
 $$(call make_search,dbg$1_%)
 
 ifneq ($$(ENABLE_VALGRIND),0)
 nosearch_memcheck$1_%: %.py pyston$1 $$(RUN_DEPS)
-	$$(VALGRIND) --tool=memcheck --leak-check=no --db-attach=yes ./pyston$1 $$(ARGS) $$<
+	PYTHONPATH=test/test_extension $$(VALGRIND) --tool=memcheck --leak-check=no --db-attach=yes ./pyston$1 $$(ARGS) $$<
 $$(call make_search,memcheck$1_%)
 nosearch_memcheck_gdb$1_%: %.py pyston$1 $$(RUN_DEPS)
-	set +e; $$(VALGRIND) -v -v -v -v -v --tool=memcheck --leak-check=no --track-origins=yes --vgdb=yes --vgdb-error=0 ./pyston$1 $$(ARGS) $$< & export PID=$$$$! ; \
+	set +e; PYTHONPATH=test/test_extension $$(VALGRIND) -v -v -v -v -v --tool=memcheck --leak-check=no --track-origins=yes --vgdb=yes --vgdb-error=0 ./pyston$1 $$(ARGS) $$< & export PID=$$$$! ; \
 	$$(GDB) --ex "set confirm off" --ex "target remote | $$(DEPS_DIR)/valgrind-3.10.0-install/bin/vgdb" --ex "continue" --ex "bt" ./pyston$1; kill -9 $$$$PID
 $$(call make_search,memcheck_gdb$1_%)
 nosearch_memleaks$1_%: %.py pyston$1 $$(RUN_DEPS)
-	$$(VALGRIND) --tool=memcheck --leak-check=full --leak-resolution=low --show-reachable=yes ./pyston$1 $$(ARGS) $$<
+	PYTHONPATH=test/test_extension $$(VALGRIND) --tool=memcheck --leak-check=full --leak-resolution=low --show-reachable=yes ./pyston$1 $$(ARGS) $$<
 $$(call make_search,memleaks$1_%)
 nosearch_cachegrind$1_%: %.py pyston$1 $$(RUN_DEPS)
-	$$(VALGRIND) --tool=cachegrind ./pyston$1 $$(ARGS) $$<
+	PYTHONPATH=test/test_extension $$(VALGRIND) --tool=cachegrind ./pyston$1 $$(ARGS) $$<
 $$(call make_search,cachegrind$1_%)
 endif
 
 .PHONY: perf$1_%
 nosearch_perf$1_%: %.py pyston$1
-	perf record -g -- ./pyston$1 -q -p $$(ARGS) $$<
+	PYTHONPATH=test/test_extension perf record -g -- ./pyston$1 -q -p $$(ARGS) $$<
 	@$(MAKE) perf_report
 $$(call make_search,perf$1_%)
 

from_cpython/CMakeLists.txt

 # Copy any changed stdlib files to the destination:
 file(GLOB_RECURSE STDLIB_SRCS Lib/ "*.py")
+file(GLOB_RECURSE STD_INCLUDES Include/ "*.h")
 set(STDLIB_TARGETS "")
-foreach(STDLIB_FILE ${STDLIB_SRCS})
+foreach(STDLIB_FILE ${STDLIB_SRCS} ${STD_INCLUDES})
     file(RELATIVE_PATH FN_REL ${CMAKE_SOURCE_DIR} ${STDLIB_FILE})
     set(TARGET ${CMAKE_BINARY_DIR}/${FN_REL})
 

from_cpython/Include/Python.h

@@ -15,6 +15,12 @@
 #ifndef PYSTON_EXTINCLUDE_PYTHON_H
 #define PYSTON_EXTINCLUDE_PYTHON_H
 
+// Cython depends on having this define set:
+#define Py_PYTHON_H
+
+// Cython has some "not targeting CPython" support that is triggered by having PYPY_VERSION defined:
+#define PYPY_VERSION "Pyston"
+
 // These include orders come from CPython:
 #include "patchlevel.h"
 #include "pyconfig.h"

src/runtime/capi.cpp

@@ -1275,27 +1275,15 @@ extern "C" char* PyModule_GetName(PyObject* m) noexcept {
     return &static_cast<BoxedModule*>(m)->fn[0];
 }
 
-BoxedModule* importTestExtension(const std::string& name) {
-    llvm::SmallString<128> pathname_str;
-    // TODO supposed to pass argv0, main_addr to this function:
-    pathname_str = llvm::sys::fs::getMainExecutable(NULL, NULL);
-    assert(pathname_str.size() && "could not find the path to the pyston src dir");
-
-    // Start by removing the binary name
-    llvm::sys::path::remove_filename(pathname_str);
-
-    llvm::sys::path::append(pathname_str, "test/test_extension");
-    llvm::sys::path::append(pathname_str, name + ".pyston.so");
-
-    const char* pathname = pathname_str.str().str().c_str();
-    void* handle = dlopen(pathname, RTLD_NOW);
+BoxedModule* importCExtension(const std::string& full_name, const std::string& last_name, const std::string& path) {
+    void* handle = dlopen(path.c_str(), RTLD_NOW);
     if (!handle) {
         fprintf(stderr, "%s\n", dlerror());
         exit(1);
     }
     assert(handle);
 
-    std::string initname = "init" + name;
+    std::string initname = "init" + last_name;
     void (*init)() = (void (*)())dlsym(handle, initname.c_str());
 
     char* error;
@@ -1308,14 +1296,14 @@ BoxedModule* importTestExtension(const std::string& name) {
     (*init)();
 
     BoxedDict* sys_modules = getSysModulesDict();
-    Box* s = boxStrConstant(name.c_str());
+    Box* s = boxStrConstant(full_name.c_str());
     Box* _m = sys_modules->d[s];
     RELEASE_ASSERT(_m, "module failed to initialize properly?");
     assert(_m->cls == module_cls);
 
     BoxedModule* m = static_cast<BoxedModule*>(_m);
-    m->setattr("__file__", boxStrConstant(pathname), NULL);
-    m->fn = pathname;
+    m->setattr("__file__", boxString(path), NULL);
+    m->fn = path;
     return m;
 }
 

src/runtime/capi.h

@@ -21,7 +21,7 @@ namespace pyston {
 
 class Box;
 class BoxedModule;
-BoxedModule* importTestExtension(const std::string&);
+BoxedModule* importCExtension(const std::string& full_name, const std::string& last_name, const std::string& path);
 
 void checkAndThrowCAPIException();
 void throwCAPIException() __attribute__((noreturn));

src/runtime/import.cpp

@@ -223,6 +223,13 @@ SearchResult findModule(const std::string& name, const std::string& full_name, B
 
         if (pathExists(fn))
             return SearchResult(std::move(fn), SearchResult::PY_SOURCE);
+
+        joined_path.clear();
+        llvm::sys::path::append(joined_path, p->s, name + ".pyston.so");
+        fn = joined_path.str();
+
+        if (pathExists(fn))
+            return SearchResult(std::move(fn), SearchResult::C_EXTENSION);
     }
 
     return SearchResult("", SearchResult::SEARCH_ERROR);
@@ -353,6 +360,8 @@ static Box* importSub(const std::string& name, const std::string& full_name, Box
             module = createAndRunModule(full_name, sr.path);
         else if (sr.type == SearchResult::PKG_DIRECTORY)
             module = createAndRunModule(full_name, sr.path + "/__init__.py", sr.path);
+        else if (sr.type == SearchResult::C_EXTENSION)
+            module = importCExtension(full_name, name, sr.path);
         else if (sr.type == SearchResult::IMP_HOOK) {
             const static std::string load_module_str("load_module");
             module = callattr(sr.loader, &load_module_str,
@@ -366,14 +375,6 @@ static Box* importSub(const std::string& name, const std::string& full_name, Box
         return module;
     }
 
-    // TODO: these are a crude hack to help our tests, find a better way
-    if (name == "basic_test")
-        return importTestExtension("basic_test");
-    if (name == "descr_test")
-        return importTestExtension("descr_test");
-    if (name == "slots_test")
-        return importTestExtension("slots_test");
-
     return None;
 }
 

tools/tester.py

@@ -57,6 +57,7 @@ def set_ulimits():
     resource.setrlimit(resource.RLIMIT_RSS, (MAX_MEM_MB * 1024 * 1024, MAX_MEM_MB * 1024 * 1024))
 
 EXTMODULE_DIR = os.path.abspath(os.path.dirname(os.path.realpath(__file__)) + "/../test/test_extension/build/lib.linux-x86_64-2.7/")
+EXTMODULE_DIR_PYSTON = os.path.abspath(os.path.dirname(os.path.realpath(__file__)) + "/../test/test_extension/")
 THIS_FILE = os.path.abspath(__file__)
 
 _global_mtime = None
@@ -141,7 +142,9 @@ def run_test(fn, check_stats, run_memcheck):
 
     run_args = [os.path.abspath(IMAGE)] + opts.jit_args + [fn]
     start = time.time()
-    p = subprocess.Popen(run_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=open("/dev/null"), preexec_fn=set_ulimits)
+    env = dict(os.environ)
+    env["PYTHONPATH"] = EXTMODULE_DIR_PYSTON
+    p = subprocess.Popen(run_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=open("/dev/null"), preexec_fn=set_ulimits, env=env)
     out, stderr = p.communicate()
     code = p.wait()
     elapsed = time.time() - start