Commit 64d40697 authored by Kevin Modzelewski's avatar Kevin Modzelewski

Be able to run an unmodified virtualenv

Well, really an unmodified pip+setuptools

The last piece is py_compile.  Just shim that back into our parser
to get the parser to write out a pyc file.

It seems like we don't even need this -- most callers of py_compile
(or at least, virtualenv), don't seem to care if the compile step
passed or failed.  But while we're here, try to support it.
parent 6681b73e
......@@ -82,6 +82,8 @@ def compile_file(fullname, ddir=None, force=0, rx=None, quiet=0):
if os.path.isfile(fullname):
head, tail = name[:-3], name[-3:]
if tail == '.py':
# Pyston change: leave this to the parser
'''
if not force:
try:
mtime = int(os.stat(fullname).st_mtime)
......@@ -93,10 +95,11 @@ def compile_file(fullname, ddir=None, force=0, rx=None, quiet=0):
return success
except IOError:
pass
'''
if not quiet:
print 'Compiling', fullname, '...'
try:
ok = py_compile.compile(fullname, None, dfile, True)
ok = py_compile.compile(fullname, None, dfile, True, force=force)
except py_compile.PyCompileError,err:
if quiet:
print 'Compiling', fullname, '...'
......
......@@ -3,15 +3,10 @@
This module has intimate knowledge of the format of .pyc files.
"""
import __builtin__
import imp
import marshal
import os
import __pyston__
import sys
import traceback
MAGIC = imp.get_magic()
__all__ = ["compile", "main", "PyCompileError"]
......@@ -61,14 +56,7 @@ class PyCompileError(Exception):
return self.msg
def wr_long(f, x):
"""Internal; write a 32-bit int to a file in little-endian order."""
f.write(chr( x & 0xff))
f.write(chr((x >> 8) & 0xff))
f.write(chr((x >> 16) & 0xff))
f.write(chr((x >> 24) & 0xff))
def compile(file, cfile=None, dfile=None, doraise=False):
def compile(file, cfile=None, dfile=None, doraise=False, force=False):
"""Byte-compile one Python source file to Python bytecode.
Arguments:
......@@ -103,14 +91,13 @@ def compile(file, cfile=None, dfile=None, doraise=False):
directories).
"""
with open(file, 'U') as f:
try:
timestamp = long(os.fstat(f.fileno()).st_mtime)
except AttributeError:
timestamp = long(os.stat(file).st_mtime)
codestring = f.read()
# Pyston restrictions for things that we don't yet support
assert cfile is None or cfile == file + "c"
assert dfile is None
try:
codeobject = __builtin__.compile(codestring, dfile or file,'exec')
__pyston__.py_compile(file, force)
except Exception,err:
py_exc = PyCompileError(err.__class__, err, dfile or file)
if doraise:
......@@ -118,15 +105,6 @@ def compile(file, cfile=None, dfile=None, doraise=False):
else:
sys.stderr.write(py_exc.msg + '\n')
return
if cfile is None:
cfile = file + (__debug__ and 'c' or 'o')
with open(cfile, 'wb') as fc:
fc.write('\0\0\0\0')
wr_long(fc, timestamp)
marshal.dump(codeobject, fc)
fc.flush()
fc.seek(0, 0)
fc.write(MAGIC)
def main(args=None):
"""Compile several source files.
......
# expected: fail
import imp
import os
import py_compile
......
# expected: fail
"""Do a minimal test of all the modules that aren't otherwise tested."""
from test import test_support
......
......@@ -2949,15 +2949,13 @@ PyImport_Import(PyObject *module_name)
importing modules.
*/
// Pyston change: we don't support get_magic
#if 0
static PyObject *
imp_get_magic(PyObject *self, PyObject *noargs)
{
char buf[4];
// Pyston change: we have different pyc
assert(0);
abort();
buf[0] = (char) ((pyc_magic >> 0) & 0xff);
buf[1] = (char) ((pyc_magic >> 8) & 0xff);
buf[2] = (char) ((pyc_magic >> 16) & 0xff);
......@@ -2965,6 +2963,7 @@ imp_get_magic(PyObject *self, PyObject *noargs)
return PyString_FromStringAndSize(buf, 4);
}
#endif
static PyObject *
imp_get_suffixes(PyObject *self, PyObject *noargs)
......@@ -3330,7 +3329,8 @@ On platforms without threads, this function does nothing.");
static PyMethodDef imp_methods[] = {
{"reload", imp_reload, METH_O, doc_reload},
{"find_module", imp_find_module, METH_VARARGS, doc_find_module},
{"get_magic", imp_get_magic, METH_NOARGS, doc_get_magic},
// Pyston change: we don't support this function
// {"get_magic", imp_get_magic, METH_NOARGS, doc_get_magic},
{"get_suffixes", imp_get_suffixes, METH_NOARGS, doc_get_suffixes},
{"load_module", imp_load_module, METH_VARARGS, doc_load_module},
{"new_module", imp_new_module, METH_VARARGS, doc_new_module},
......
......@@ -1104,7 +1104,8 @@ static std::vector<char> _reparse(const char* fn, const std::string& cache_fn, A
// Parsing the file is somewhat expensive since we have to shell out to cpython;
// it's not a huge deal right now, but this caching version can significantly cut down
// on the startup time (40ms -> 10ms).
std::pair<AST_Module*, std::unique_ptr<ASTAllocator>> caching_parse_file(const char* fn, FutureFlags inherited_flags) {
std::pair<AST_Module*, std::unique_ptr<ASTAllocator>> caching_parse_file(const char* fn, FutureFlags inherited_flags,
bool force_reparse) {
std::ostringstream oss;
UNAVOIDABLE_STAT_TIMER(t0, "us_timer_caching_parse_file");
......
......@@ -15,6 +15,7 @@
#ifndef PYSTON_CODEGEN_PARSER_H
#define PYSTON_CODEGEN_PARSER_H
#include "core/ast.h"
#include "core/types.h"
namespace pyston {
......@@ -24,7 +25,8 @@ class ASTAllocator;
std::pair<AST_Module*, std::unique_ptr<ASTAllocator>> parse_string(const char* code, FutureFlags inherited_flags);
std::pair<AST_Module*, std::unique_ptr<ASTAllocator>> parse_file(const char* fn, FutureFlags inherited_flags);
std::pair<AST_Module*, std::unique_ptr<ASTAllocator>> caching_parse_file(const char* fn, FutureFlags inherited_flags);
std::pair<AST_Module*, std::unique_ptr<ASTAllocator>> caching_parse_file(const char* fn, FutureFlags inherited_flags,
bool force_reparse = false);
}
#endif
......@@ -2388,7 +2388,7 @@ void setupBuiltins() {
builtins_module->giveAttrBorrowed("Ellipsis", Ellipsis);
builtins_module->giveAttrBorrowed("None", Py_None);
builtins_module->giveAttrBorrowed("__debug__", Py_False);
builtins_module->giveAttrBorrowed("__debug__", Py_True);
notimplemented_cls = BoxedClass::create(type_cls, object_cls, 0, 0, sizeof(Box), false, "NotImplementedType", false,
NULL, NULL, false);
......
......@@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include "codegen/parser.h"
#include "core/types.h"
#include "runtime/objmodel.h"
#include "runtime/types.h"
......@@ -63,6 +64,18 @@ static Box* dumpStats(Box* includeZeros) {
Py_RETURN_NONE;
}
static Box* pyCompile(Box* fname, Box* force) {
if (fname->cls != str_cls)
raiseExcHelper(TypeError, "py_compile takes a string for the filename");
if (force->cls != bool_cls)
raiseExcHelper(TypeError, "py_compile takes a bool for 'force' argument");
caching_parse_file(static_cast<BoxedString*>(fname)->c_str(), /* future flags */ 0, force == Py_True);
Py_RETURN_NONE;
}
void setupPyston() {
pyston_module = createModule(autoDecref(boxString("__pyston__")));
......@@ -74,5 +87,8 @@ void setupPyston() {
pyston_module->giveAttr("dumpStats",
new BoxedBuiltinFunctionOrMethod(
BoxedCode::create((void*)dumpStats, NONE, 1, false, false, "dumpStats"), { Py_False }));
pyston_module->giveAttr(
"py_compile", new BoxedBuiltinFunctionOrMethod(BoxedCode::create((void*)pyCompile, UNKNOWN, 2, "pyCompile")));
}
}
......@@ -51,7 +51,7 @@ test_codeop [unknown]
test_code [unknown]
test_coding works when run from inside the from_cpython dir
test_coercion 1**1L, divmod(1, 1L); some unknown bug
test_compileall [unknown]
test_compileall Not sure if this test makes sense for us (wants to check the details of pyc files)
test_compiler [unknown]
test_compile [unknown]
test_copy Please debug this test in VM.
......@@ -131,7 +131,6 @@ test_pprint [unknown]
test_profile [unknown]
test_py3kwarn [unknown]
test_pyclbr This test passes but takes a very long time in debug mode (60s vs 5s for release mode).
test_py_compile [unknown]
test_pydoc [unknown]
test_random long("invalid number")
test_repr complex.__hash__; some unknown issues
......@@ -148,7 +147,6 @@ test_structmembers [unknown]
test_subprocess exit code 141 [sigpipe?], no error message
test_sunaudiodev [unknown]
test_sunau [unknown]
test_sundry [unknown]
test_support [unknown]
test_symtable [unknown]
test_syntax [unknown]
......
......@@ -5,19 +5,21 @@ import shutil
VIRTUALENV_SCRIPT = os.path.dirname(__file__) + "/../lib/virtualenv/virtualenv.py"
if os.path.exists("test_env"):
print "Removing the existing 'test_env/' directory"
subprocess.check_call(["rm", "-rf", "test_env"])
ENV_NAME = "test_env_" + os.path.basename(sys.executable)
if os.path.exists(ENV_NAME):
print "Removing the existing '%s/' directory" % ENV_NAME
subprocess.check_call(["rm", "-rf", ENV_NAME])
# shutil follows symlinks to directories, and deletes whatever those contain.
# shutil.rmtree("test_env")
# shutil.rmtree(ENV_NAME)
args = [sys.executable, VIRTUALENV_SCRIPT, "-p", sys.executable, "test_env"]
args = [sys.executable, VIRTUALENV_SCRIPT, "-p", sys.executable, ENV_NAME]
print "Running", args
subprocess.check_call(args)
sh_script = """
sh_script = ("""
set -e
. test_env/bin/activate
. %s/bin/activate
set -ux
python -c 'import __future__'
python -c 'import sys; print sys.executable'
......@@ -36,7 +38,7 @@ python -c 'import sqlalchemy; print "sqlalchemy imports"'
python -c 'from PIL import Image; print "Pillow imports"'
python -c 'import decorator; print "decorator imports"'
python -c 'import oauth2client; print "oauth2client imports"'
""".strip()
""" % ENV_NAME).strip()
# print sh_script
subprocess.check_call(["sh", "-c", sh_script], stdout=sys.stderr)
......
Subproject commit db70fc5084483b636b4bbf05519f4d69a2c741fa
Subproject commit b112626df31a8788c7986c0bc7c2b470ac36ff49
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment