Be able to run an unmodified virtualenv

Well, really an unmodified pip+setuptools

The last piece is py_compile.  Just shim that back into our parser
to get the parser to write out a pyc file.

It seems like we don't even need this -- most callers of py_compile
(or at least, virtualenv), don't seem to care if the compile step
passed or failed.  But while we're here, try to support it.

from_cpython/Lib/compileall.py

@@ -82,6 +82,8 @@ def compile_file(fullname, ddir=None, force=0, rx=None, quiet=0):
     if os.path.isfile(fullname):
         head, tail = name[:-3], name[-3:]
         if tail == '.py':
+            # Pyston change: leave this to the parser
+            '''
             if not force:
                 try:
                     mtime = int(os.stat(fullname).st_mtime)
@@ -93,10 +95,11 @@ def compile_file(fullname, ddir=None, force=0, rx=None, quiet=0):
                         return success
                 except IOError:
                     pass
+            '''
             if not quiet:
                 print 'Compiling', fullname, '...'
             try:
-                ok = py_compile.compile(fullname, None, dfile, True)
+                ok = py_compile.compile(fullname, None, dfile, True, force=force)
             except py_compile.PyCompileError,err:
                 if quiet:
                     print 'Compiling', fullname, '...'

from_cpython/Lib/py_compile.py

@@ -3,15 +3,10 @@
 This module has intimate knowledge of the format of .pyc files.
 """
 
-import __builtin__
-import imp
-import marshal
-import os
+import __pyston__
 import sys
 import traceback
 
-MAGIC = imp.get_magic()
-
 __all__ = ["compile", "main", "PyCompileError"]
 
 
@@ -61,14 +56,7 @@ class PyCompileError(Exception):
         return self.msg
 
 
-def wr_long(f, x):
-    """Internal; write a 32-bit int to a file in little-endian order."""
-    f.write(chr( x        & 0xff))
-    f.write(chr((x >> 8)  & 0xff))
-    f.write(chr((x >> 16) & 0xff))
-    f.write(chr((x >> 24) & 0xff))
-
-def compile(file, cfile=None, dfile=None, doraise=False):
+def compile(file, cfile=None, dfile=None, doraise=False, force=False):
     """Byte-compile one Python source file to Python bytecode.
 
     Arguments:
@@ -103,14 +91,13 @@ def compile(file, cfile=None, dfile=None, doraise=False):
     directories).
 
     """
-    with open(file, 'U') as f:
-        try:
-            timestamp = long(os.fstat(f.fileno()).st_mtime)
-        except AttributeError:
-            timestamp = long(os.stat(file).st_mtime)
-        codestring = f.read()
+
+    # Pyston restrictions for things that we don't yet support
+    assert cfile is None or cfile == file + "c"
+    assert dfile is None
+
     try:
-        codeobject = __builtin__.compile(codestring, dfile or file,'exec')
+        __pyston__.py_compile(file, force)
     except Exception,err:
         py_exc = PyCompileError(err.__class__, err, dfile or file)
         if doraise:
@@ -118,15 +105,6 @@ def compile(file, cfile=None, dfile=None, doraise=False):
         else:
             sys.stderr.write(py_exc.msg + '\n')
             return
-    if cfile is None:
-        cfile = file + (__debug__ and 'c' or 'o')
-    with open(cfile, 'wb') as fc:
-        fc.write('\0\0\0\0')
-        wr_long(fc, timestamp)
-        marshal.dump(codeobject, fc)
-        fc.flush()
-        fc.seek(0, 0)
-        fc.write(MAGIC)
 
 def main(args=None):
     """Compile several source files.

from_cpython/Lib/test/test_py_compile.py

-# expected: fail
 import imp
 import os
 import py_compile

from_cpython/Lib/test/test_sundry.py

-# expected: fail
 """Do a minimal test of all the modules that aren't otherwise tested."""
 
 from test import test_support

from_cpython/Python/import.c

@@ -2949,15 +2949,13 @@ PyImport_Import(PyObject *module_name)
    importing modules.
 */
 
+// Pyston change: we don't support get_magic
+#if 0
 static PyObject *
 imp_get_magic(PyObject *self, PyObject *noargs)
 {
     char buf[4];
 
-    // Pyston change: we have different pyc
-    assert(0);
-    abort();
-
     buf[0] = (char) ((pyc_magic >>  0) & 0xff);
     buf[1] = (char) ((pyc_magic >>  8) & 0xff);
     buf[2] = (char) ((pyc_magic >> 16) & 0xff);
@@ -2965,6 +2963,7 @@ imp_get_magic(PyObject *self, PyObject *noargs)
 
     return PyString_FromStringAndSize(buf, 4);
 }
+#endif
 
 static PyObject *
 imp_get_suffixes(PyObject *self, PyObject *noargs)
@@ -3330,7 +3329,8 @@ On platforms without threads, this function does nothing.");
 static PyMethodDef imp_methods[] = {
     {"reload",           imp_reload,       METH_O,       doc_reload},
     {"find_module",      imp_find_module,  METH_VARARGS, doc_find_module},
-    {"get_magic",        imp_get_magic,    METH_NOARGS,  doc_get_magic},
+    // Pyston change: we don't support this function
+    // {"get_magic",        imp_get_magic,    METH_NOARGS,  doc_get_magic},
     {"get_suffixes", imp_get_suffixes, METH_NOARGS,  doc_get_suffixes},
     {"load_module",      imp_load_module,  METH_VARARGS, doc_load_module},
     {"new_module",       imp_new_module,   METH_VARARGS, doc_new_module},

src/codegen/parser.cpp

@@ -1104,7 +1104,8 @@ static std::vector<char> _reparse(const char* fn, const std::string& cache_fn, A
 // Parsing the file is somewhat expensive since we have to shell out to cpython;
 // it's not a huge deal right now, but this caching version can significantly cut down
 // on the startup time (40ms -> 10ms).
-std::pair<AST_Module*, std::unique_ptr<ASTAllocator>> caching_parse_file(const char* fn, FutureFlags inherited_flags) {
+std::pair<AST_Module*, std::unique_ptr<ASTAllocator>> caching_parse_file(const char* fn, FutureFlags inherited_flags,
+                                                                         bool force_reparse) {
     std::ostringstream oss;
 
     UNAVOIDABLE_STAT_TIMER(t0, "us_timer_caching_parse_file");

src/codegen/parser.h

@@ -15,6 +15,7 @@
 #ifndef PYSTON_CODEGEN_PARSER_H
 #define PYSTON_CODEGEN_PARSER_H
 
+#include "core/ast.h"
 #include "core/types.h"
 
 namespace pyston {
@@ -24,7 +25,8 @@ class ASTAllocator;
 
 std::pair<AST_Module*, std::unique_ptr<ASTAllocator>> parse_string(const char* code, FutureFlags inherited_flags);
 std::pair<AST_Module*, std::unique_ptr<ASTAllocator>> parse_file(const char* fn, FutureFlags inherited_flags);
-std::pair<AST_Module*, std::unique_ptr<ASTAllocator>> caching_parse_file(const char* fn, FutureFlags inherited_flags);
+std::pair<AST_Module*, std::unique_ptr<ASTAllocator>> caching_parse_file(const char* fn, FutureFlags inherited_flags,
+                                                                         bool force_reparse = false);
 }
 
 #endif

src/runtime/builtin_modules/builtins.cpp

@@ -2388,7 +2388,7 @@ void setupBuiltins() {
     builtins_module->giveAttrBorrowed("Ellipsis", Ellipsis);
     builtins_module->giveAttrBorrowed("None", Py_None);
 
-    builtins_module->giveAttrBorrowed("__debug__", Py_False);
+    builtins_module->giveAttrBorrowed("__debug__", Py_True);
 
     notimplemented_cls = BoxedClass::create(type_cls, object_cls, 0, 0, sizeof(Box), false, "NotImplementedType", false,
                                             NULL, NULL, false);

src/runtime/builtin_modules/pyston.cpp

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include "codegen/parser.h"
 #include "core/types.h"
 #include "runtime/objmodel.h"
 #include "runtime/types.h"
@@ -63,6 +64,18 @@ static Box* dumpStats(Box* includeZeros) {
     Py_RETURN_NONE;
 }
 
+static Box* pyCompile(Box* fname, Box* force) {
+    if (fname->cls != str_cls)
+        raiseExcHelper(TypeError, "py_compile takes a string for the filename");
+
+    if (force->cls != bool_cls)
+        raiseExcHelper(TypeError, "py_compile takes a bool for 'force' argument");
+
+    caching_parse_file(static_cast<BoxedString*>(fname)->c_str(), /* future flags */ 0, force == Py_True);
+
+    Py_RETURN_NONE;
+}
+
 void setupPyston() {
     pyston_module = createModule(autoDecref(boxString("__pyston__")));
 
@@ -74,5 +87,8 @@ void setupPyston() {
     pyston_module->giveAttr("dumpStats",
                             new BoxedBuiltinFunctionOrMethod(
                                 BoxedCode::create((void*)dumpStats, NONE, 1, false, false, "dumpStats"), { Py_False }));
+
+    pyston_module->giveAttr(
+        "py_compile", new BoxedBuiltinFunctionOrMethod(BoxedCode::create((void*)pyCompile, UNKNOWN, 2, "pyCompile")));
 }
 }

test/CPYTHON_TEST_NOTES.md

@@ -51,7 +51,7 @@ test_codeop             [unknown]
 test_code               [unknown]
 test_coding             works when run from inside the from_cpython dir
 test_coercion           1**1L, divmod(1, 1L); some unknown bug
-test_compileall         [unknown]
+test_compileall         Not sure if this test makes sense for us (wants to check the details of pyc files)
 test_compiler           [unknown]
 test_compile            [unknown]
 test_copy               Please debug this test in VM.
@@ -131,7 +131,6 @@ test_pprint             [unknown]
 test_profile            [unknown]
 test_py3kwarn           [unknown]
 test_pyclbr             This test passes but takes a very long time in debug mode (60s vs 5s for release mode).
-test_py_compile         [unknown]
 test_pydoc              [unknown]
 test_random             long("invalid number")
 test_repr               complex.__hash__; some unknown issues
@@ -148,7 +147,6 @@ test_structmembers      [unknown]
 test_subprocess         exit code 141 [sigpipe?], no error message
 test_sunaudiodev        [unknown]
 test_sunau              [unknown]
-test_sundry             [unknown]
 test_support            [unknown]
 test_symtable           [unknown]
 test_syntax             [unknown]

test/integration/virtualenv_test.py

@@ -5,19 +5,21 @@ import shutil
 
 VIRTUALENV_SCRIPT = os.path.dirname(__file__) + "/../lib/virtualenv/virtualenv.py"
 
-if os.path.exists("test_env"):
-    print "Removing the existing 'test_env/' directory"
-    subprocess.check_call(["rm", "-rf", "test_env"])
+ENV_NAME = "test_env_" + os.path.basename(sys.executable)
+
+if os.path.exists(ENV_NAME):
+    print "Removing the existing '%s/' directory" % ENV_NAME
+    subprocess.check_call(["rm", "-rf", ENV_NAME])
     # shutil follows symlinks to directories, and deletes whatever those contain.
-    # shutil.rmtree("test_env")
+    # shutil.rmtree(ENV_NAME)
 
-args = [sys.executable, VIRTUALENV_SCRIPT, "-p", sys.executable, "test_env"]
+args = [sys.executable, VIRTUALENV_SCRIPT, "-p", sys.executable, ENV_NAME]
 print "Running", args
 subprocess.check_call(args)
 
-sh_script = """
+sh_script = ("""
 set -e
-. test_env/bin/activate
+. %s/bin/activate
 set -ux
 python -c 'import __future__'
 python -c 'import sys; print sys.executable'
@@ -36,7 +38,7 @@ python -c 'import sqlalchemy; print "sqlalchemy imports"'
 python -c 'from PIL import Image; print "Pillow imports"'
 python -c 'import decorator; print "decorator imports"'
 python -c 'import oauth2client; print "oauth2client imports"'
-""".strip()
+""" % ENV_NAME).strip()
 
 # print sh_script
 subprocess.check_call(["sh", "-c", sh_script], stdout=sys.stderr)

virtualenv @ b112626d

-Subproject commit db70fc5084483b636b4bbf05519f4d69a2c741fa
+Subproject commit b112626df31a8788c7986c0bc7c2b470ac36ff49