Add sys.prefix, sys.exec_prefix, site.py and PYTHONPATH support

CMakeLists.txt

@@ -186,9 +186,9 @@ add_test(NAME lint COMMAND ${PYTHON_EXE} ${CMAKE_SOURCE_DIR}/tools/lint.py WORKI
 add_test(NAME check-format COMMAND ${CMAKE_SOURCE_DIR}/tools/check_format.sh ${LLVM_TOOLS_BINARY_DIR}/clang-format WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/src)
 add_test(NAME gc_unittest COMMAND gc_unittest)
 add_test(NAME analysis_unittest COMMAND analysis_unittest)
-add_test(NAME pyston_defaults COMMAND ${PYTHON_EXE} ${CMAKE_SOURCE_DIR}/tools/tester.py -R ./pyston -j${TEST_THREADS} -k ${CMAKE_SOURCE_DIR}/test/tests)
-add_test(NAME pyston_max_compilation_tier COMMAND ${PYTHON_EXE} ${CMAKE_SOURCE_DIR}/tools/tester.py -R ./pyston -j${TEST_THREADS} -a=-O -k ${CMAKE_SOURCE_DIR}/test/tests)
-add_test(NAME pyston_experimental_pypa_parser COMMAND ${PYTHON_EXE} ${CMAKE_SOURCE_DIR}/tools/tester.py -a=-x -R ./pyston -j${TEST_THREADS} -a=-n -k ${CMAKE_SOURCE_DIR}/test/tests)
+add_test(NAME pyston_defaults COMMAND ${PYTHON_EXE} ${CMAKE_SOURCE_DIR}/tools/tester.py -R ./pyston -j${TEST_THREADS} -a=-S -k ${CMAKE_SOURCE_DIR}/test/tests)
+add_test(NAME pyston_max_compilation_tier COMMAND ${PYTHON_EXE} ${CMAKE_SOURCE_DIR}/tools/tester.py -R ./pyston -j${TEST_THREADS} -a=-O -a=-S -k ${CMAKE_SOURCE_DIR}/test/tests)
+add_test(NAME pyston_experimental_pypa_parser COMMAND ${PYTHON_EXE} ${CMAKE_SOURCE_DIR}/tools/tester.py -a=-x -R ./pyston -j${TEST_THREADS} -a=-n -a=-S -k ${CMAKE_SOURCE_DIR}/test/tests)
 
 # format
 file(GLOB_RECURSE FORMAT_FILES ${CMAKE_SOURCE_DIR}/src/*.h ${CMAKE_SOURCE_DIR}/src/*.cpp)

Makefile

@@ -292,7 +292,7 @@ STDLIB_OBJS := stdlib.bc.o stdlib.stripped.bc.o
 STDLIB_RELEASE_OBJS := stdlib.release.bc.o
 ASM_SRCS := $(wildcard src/runtime/*.S)
 
-STDMODULE_SRCS := errnomodule.c shamodule.c sha256module.c sha512module.c _math.c mathmodule.c md5.c md5module.c _randommodule.c _sre.c operator.c binascii.c pwdmodule.c posixmodule.c _struct.c datetimemodule.c _functoolsmodule.c _collectionsmodule.c itertoolsmodule.c resource.c signalmodule.c selectmodule.c fcntlmodule.c timemodule.c arraymodule.c zlibmodule.c _codecsmodule.c socketmodule.c unicodedata.c _weakref.c cStringIO.c _io/bufferedio.c _io/bytesio.c _io/fileio.c _io/iobase.c _io/_iomodule.c _io/stringio.c _io/textio.c $(EXTRA_STDMODULE_SRCS)
+STDMODULE_SRCS := errnomodule.c shamodule.c sha256module.c sha512module.c _math.c mathmodule.c md5.c md5module.c _randommodule.c _sre.c operator.c binascii.c pwdmodule.c posixmodule.c _struct.c datetimemodule.c _functoolsmodule.c _collectionsmodule.c itertoolsmodule.c resource.c signalmodule.c selectmodule.c fcntlmodule.c timemodule.c arraymodule.c zlibmodule.c _codecsmodule.c socketmodule.c unicodedata.c _weakref.c cStringIO.c _io/bufferedio.c _io/bytesio.c _io/fileio.c _io/iobase.c _io/_iomodule.c _io/stringio.c _io/textio.c getpath.c $(EXTRA_STDMODULE_SRCS)
 STDOBJECT_SRCS := structseq.c capsule.c stringobject.c exceptions.c unicodeobject.c unicodectype.c bytearrayobject.c bytes_methods.c weakrefobject.c memoryobject.c iterobject.c $(EXTRA_STDOBJECT_SRCS)
 STDPYTHON_SRCS := pyctype.c getargs.c formatter_string.c pystrtod.c dtoa.c formatter_unicode.c structmember.c $(EXTRA_STDPYTHON_SRCS)
 FROM_CPYTHON_SRCS := $(addprefix from_cpython/Modules/,$(STDMODULE_SRCS)) $(addprefix from_cpython/Objects/,$(STDOBJECT_SRCS)) $(addprefix from_cpython/Python/,$(STDPYTHON_SRCS))
@@ -403,7 +403,7 @@ check:
 	$(MAKE) run_unittests
 
 	$(MAKE) pyston_gcc
-	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston_gcc -j$(TEST_THREADS) -k $(TESTS_DIR) $(ARGS)
+	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston_gcc -j$(TEST_THREADS) -k -a=-S $(TESTS_DIR) $(ARGS)
 
 	@# It can be useful to test release mode, since it actually exposes different functionality
 	@# since we can make different decisions about which internal functions to inline or not.
@@ -834,9 +834,9 @@ define make_target
 $(eval \
 .PHONY: test$1 check$1
 check$1 test$1: $(PYTHON_EXE_DEPS) pyston$1 ext_pyston
-	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston$1 -j$(TEST_THREADS) -k $(TESTS_DIR) $(ARGS)
-	$(PYTHON) $(TOOLS_DIR)/tester.py -a=-x -R pyston$1 -j$(TEST_THREADS) -a=-n -k $(TESTS_DIR) $(ARGS)
-	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston$1 -j$(TEST_THREADS) -a=-O -k $(TESTS_DIR) $(ARGS)
+	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston$1 -j$(TEST_THREADS) -a=-S -k $(TESTS_DIR) $(ARGS)
+	$(PYTHON) $(TOOLS_DIR)/tester.py -a=-x -R pyston$1 -j$(TEST_THREADS) -a=-n -a=-S -k $(TESTS_DIR) $(ARGS)
+	$(PYTHON) $(TOOLS_DIR)/tester.py -R pyston$1 -j$(TEST_THREADS) -a=-O -a=-S -k $(TESTS_DIR) $(ARGS)
 
 .PHONY: run$1 dbg$1
 run$1: pyston$1 $$(RUN_DEPS)

from_cpython/CMakeLists.txt

@@ -15,7 +15,7 @@ endforeach(STDLIB_FILE)
 add_custom_target(copy_stdlib ALL DEPENDS ${STDLIB_TARGETS})
 
 # compile specified files in from_cpython/Modules
-file(GLOB_RECURSE STDMODULE_SRCS Modules errnomodule.c shamodule.c sha256module.c sha512module.c _math.c mathmodule.c md5.c md5module.c _randommodule.c _sre.c operator.c binascii.c pwdmodule.c posixmodule.c _struct.c datetimemodule.c _functoolsmodule.c _collectionsmodule.c itertoolsmodule.c resource.c signalmodule.c selectmodule.c fcntlmodule.c timemodule.c arraymodule.c zlibmodule.c _codecsmodule.c socketmodule.c unicodedata.c _weakref.c cStringIO.c bufferedio.c bytesio.c fileio.c iobase.c _iomodule.c stringio.c textio.c)
+file(GLOB_RECURSE STDMODULE_SRCS Modules errnomodule.c shamodule.c sha256module.c sha512module.c _math.c mathmodule.c md5.c md5module.c _randommodule.c _sre.c operator.c binascii.c pwdmodule.c posixmodule.c _struct.c datetimemodule.c _functoolsmodule.c _collectionsmodule.c itertoolsmodule.c resource.c signalmodule.c selectmodule.c fcntlmodule.c timemodule.c arraymodule.c zlibmodule.c _codecsmodule.c socketmodule.c unicodedata.c _weakref.c cStringIO.c bufferedio.c bytesio.c fileio.c iobase.c _iomodule.c stringio.c textio.c getpath.c)
 
 # compile specified files in from_cpython/Objects
 file(GLOB_RECURSE STDOBJECT_SRCS Objects structseq.c capsule.c stringobject.c exceptions.c unicodeobject.c unicodectype.c bytearrayobject.c bytes_methods.c weakrefobject.c memoryobject.c iterobject.c)

from_cpython/Include/pythonrun.h

@@ -21,7 +21,7 @@ typedef struct {
 } PyCompilerFlags;
 
 PyAPI_FUNC(void) Py_SetProgramName(char *) PYSTON_NOEXCEPT;
-PyAPI_FUNC(char *) Py_GetProgramName(void) PYSTON_NOEXCEPT;
+PyAPI_FUNC(const char *) Py_GetProgramName(void) PYSTON_NOEXCEPT;
 
 PyAPI_FUNC(void) Py_SetPythonHome(char *) PYSTON_NOEXCEPT;
 PyAPI_FUNC(char *) Py_GetPythonHome(void) PYSTON_NOEXCEPT;

from_cpython/Modules/getpath.c

@@ -3,6 +3,8 @@
 #include "Python.h"
 #include "osdefs.h"
 
+#include <sys/stat.h> // Pyston change
+
 #include <sys/types.h>
 #include <string.h>
 
@@ -117,8 +119,9 @@
 #endif
 
 #ifndef PYTHONPATH
-#define PYTHONPATH PREFIX "/lib/python" VERSION ":" \
-              EXEC_PREFIX "/lib/python" VERSION "/lib-dynload"
+// Pyston change
+#define PYTHONPATH PREFIX "/from_cpython/Lib:" \
+              EXEC_PREFIX "/from_cpython/Lib/lib-dynload"
 #endif
 
 #ifndef LANDMARK
@@ -129,7 +132,7 @@ static char prefix[MAXPATHLEN+1];
 static char exec_prefix[MAXPATHLEN+1];
 static char progpath[MAXPATHLEN+1];
 static char *module_search_path = NULL;
-static char lib_python[] = "lib/python" VERSION;
+static char lib_python[] = "from_cpython/Lib"; // Pyston change
 
 static void
 reduce(char *dir)
@@ -205,7 +208,7 @@ isdir(char *filename)                   /* Is directory */
    stuff as fits will be appended.
 */
 static void
-joinpath(char *buffer, char *stuff)
+joinpath(char *buffer, const char *stuff)
 {
     size_t n, k;
     if (stuff[0] == SEP)
@@ -385,7 +388,7 @@ search_for_exec_prefix(char *argv0_path, char *home)
 static void
 calculate_path(void)
 {
-    extern char *Py_GetProgramName(void);
+    extern const char *Py_GetProgramName(void);
 
     static char delimiter[2] = {DELIM, '\0'};
     static char separator[2] = {SEP, '\0'};
@@ -393,9 +396,10 @@ calculate_path(void)
     char *rtpypath = Py_GETENV("PYTHONPATH");
     char *home = Py_GetPythonHome();
     char *path = getenv("PATH");
-    char *prog = Py_GetProgramName();
+    const char *prog = Py_GetProgramName();
     char argv0_path[MAXPATHLEN+1];
     char zip_path[MAXPATHLEN+1];
+    char lib_pyston_path[MAXPATHLEN+1]; // Pyston change
     int pfound, efound; /* 1 if found; -1 if found build directory */
     char *buf;
     size_t bufsz;
@@ -589,6 +593,16 @@ calculate_path(void)
     bufsz += strlen(zip_path) + 1;
     bufsz += strlen(exec_prefix) + 1;
 
+    // Pyston change: add from_cpython/Lib and lib_pyston
+    // Prefix contains at this point the full path to 'from_cpython/Lib'
+    strcpy(lib_pyston_path, prefix);
+    // go from ./from_cpython/Lib to ./lib_pyston
+    reduce(lib_pyston_path);
+    reduce(lib_pyston_path);
+    joinpath(lib_pyston_path, "lib_pyston");
+    bufsz += strlen(lib_pyston_path) + 1;
+    bufsz += strlen(prefix) + 1;
+
     /* This is the only malloc call in this file */
     buf = (char *)PyMem_Malloc(bufsz);
 
@@ -609,7 +623,17 @@ calculate_path(void)
 
         /* Next is the default zip path */
         strcat(buf, zip_path);
+
+        // Pyston change
+        // add from_cpython/Lib
         strcat(buf, delimiter);
+        strcat(buf, prefix);
+
+        // add lib_pyston
+        strcat(buf, delimiter);
+        strcat(buf, lib_pyston_path);
+        strcat(buf, delimiter);
+
 
         /* Next goes merge of compile-time $PYTHONPATH with
          * dynamically located prefix.

src/core/options.cpp

@@ -74,6 +74,9 @@ bool ENABLE_FRAME_INTROSPECTION = 1;
 bool BOOLS_AS_I64 = ENABLE_FRAME_INTROSPECTION;
 
 extern "C" {
-int Py_IgnoreEnvironmentFlag = 1;
+int Py_FrozenFlag = 1;
+int Py_IgnoreEnvironmentFlag = 0;
+int Py_NoSiteFlag = 0;
+int Py_OptimizeFlag = 0;
 }
 }

src/jit.cpp

@@ -27,6 +27,8 @@
 #include "llvm/Support/Path.h"
 #include "llvm/Support/Signals.h"
 
+#include "osdefs.h"
+
 #include "codegen/entry.h"
 #include "codegen/irgen/hooks.h"
 #include "codegen/parser.h"
@@ -37,6 +39,7 @@
 #include "core/threading.h"
 #include "core/types.h"
 #include "core/util.h"
+#include "runtime/import.h"
 #include "runtime/objmodel.h"
 #include "runtime/types.h"
 
@@ -76,7 +79,7 @@ static int main(int argc, char** argv) {
     bool force_repl = false;
     bool stats = false;
     const char* command = NULL;
-    while ((code = getopt(argc, argv, "+OqdIibpjtrsvnxc:")) != -1) {
+    while ((code = getopt(argc, argv, "+OqdIibpjtrsSvnxc:")) != -1) {
         if (code == 'O')
             FORCE_OPTIMIZE = true;
         else if (code == 't')
@@ -99,6 +102,8 @@ static int main(int argc, char** argv) {
             DUMPJIT = true;
         } else if (code == 's') {
             stats = true;
+        } else if (code == 'S') {
+            Py_NoSiteFlag = 1;
         } else if (code == 'r') {
             USE_STRIPPED_STDLIB = true;
         } else if (code == 'b') {
@@ -118,6 +123,8 @@ static int main(int argc, char** argv) {
     threading::registerMainThread();
     threading::acquireGLRead();
 
+    Py_SetProgramName(argv[0]);
+
     {
         Timer _t("for initCodegen");
         initCodegen();
@@ -142,20 +149,24 @@ static int main(int argc, char** argv) {
         addToSysArgv(argv[i]);
     }
 
-    std::string self_path = llvm::sys::fs::getMainExecutable(argv[0], (void*)main);
-    assert(self_path.size());
-
-    llvm::SmallString<128> stdlib_dir(self_path);
-    llvm::sys::path::remove_filename(stdlib_dir); // executable name
-    llvm::sys::path::append(stdlib_dir, "from_cpython");
-    llvm::sys::path::append(stdlib_dir, "Lib");
-    appendToSysPath(stdlib_dir.c_str());
+    llvm::StringRef module_search_path = Py_GetPath();
+    while (true) {
+        std::pair<llvm::StringRef, llvm::StringRef> split_str = module_search_path.split(DELIM);
+        if (split_str.first == module_search_path)
+            break; // could not find the delimiter
+        appendToSysPath(split_str.first);
+        module_search_path = split_str.second;
+    }
 
-    // go from ./from_cpython/Lib to ./lib_pyston
-    llvm::sys::path::remove_filename(stdlib_dir);
-    llvm::sys::path::remove_filename(stdlib_dir);
-    llvm::sys::path::append(stdlib_dir, "lib_pyston");
-    appendToSysPath(stdlib_dir.c_str());
+    if (!Py_NoSiteFlag) {
+        try {
+            std::string module_name = "site";
+            importModuleLevel(&module_name, None, None, 0);
+        } catch (ExcInfo e) {
+            e.printExcAndTraceback();
+            return 1;
+        }
+    }
 
     // end of argument parsing
 

src/runtime/builtin_modules/sys.cpp

@@ -276,9 +276,8 @@ void setupSys() {
 
     sys_module->giveAttr("meta_path", new BoxedList());
 
-    // TODO: should configure this in a better way
-    sys_module->giveAttr("prefix", boxStrConstant("/usr"));
-    sys_module->giveAttr("exec_prefix", boxStrConstant("/usr"));
+    sys_module->giveAttr("prefix", boxStrConstant(Py_GetPrefix()));
+    sys_module->giveAttr("exec_prefix", boxStrConstant(Py_GetExecPrefix()));
 
     sys_module->giveAttr("copyright",
                          boxStrConstant("Copyright 2014-2015 Dropbox.\nAll Rights Reserved.\n\nCopyright (c) 2001-2014 "

src/runtime/capi.cpp

@@ -981,6 +981,28 @@ static int dev_urandom_python(char* buffer, Py_ssize_t size) noexcept {
 }
 }
 
+static const char* progname = "pyston";
+extern "C" void Py_SetProgramName(char* pn) noexcept {
+    if (pn && *pn)
+        progname = pn;
+}
+
+extern "C" const char* Py_GetProgramName(void) noexcept {
+    return progname;
+}
+
+static char* default_home = NULL;
+extern "C" void Py_SetPythonHome(char* home) noexcept {
+    default_home = home;
+}
+
+extern "C" char* Py_GetPythonHome(void) noexcept {
+    char* home = default_home;
+    if (home == NULL && !Py_IgnoreEnvironmentFlag)
+        home = Py_GETENV("PYTHONHOME");
+    return home;
+}
+
 extern "C" PyObject* PyThreadState_GetDict(void) noexcept {
     Box* dict = cur_thread_state.dict;
     if (!dict) {

test/tests/sys_test.py

@@ -3,7 +3,6 @@ import os.path
 
 print sys.version[:3]
 print os.path.exists(sys.executable)
-print sys.prefix, sys.exec_prefix
 print sys.copyright[-200:]
 print sys.byteorder
 print sys.getdefaultencoding()