Merge pull request #984 from kmod/dist

Support building distribution packages

CMakeLists.txt

@@ -123,6 +123,25 @@ ExternalProject_Add(libjemalloc
                     LOG_BUILD ON
                     LOG_INSTALL ON)
 
+#
+# CMake (<3.3) has no way of knowing that an ExternalProject creates specific output files.  This is a problem for ninja,
+# which will not know how to build the generated file.
+# Here are a couple hacks to get around it:
+#
+# Add a copy step.  This just hides the dependency but it seems to work.
+# add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/jemalloc/lib/libjemalloc_copied.a DEPENDS libjemalloc COMMAND cp "${CMAKE_BINARY_DIR}/jemalloc/lib/libjemalloc.a" ${CMAKE_BINARY_DIR}/jemalloc/lib/libjemalloc_copied.a)
+# add_custom_target(libjemalloc_copied DEPENDS ${CMAKE_BINARY_DIR}/jemalloc/lib/libjemalloc_copied.a)
+#
+# Hack option #2: the existence of the custom target tells ninja that libjemalloc.a will get built somehow.
+# The name of the target doesn't matter.
+add_custom_target(libjemalloc_byproducts DEPENDS ${CMAKE_BINARY_DIR}/jemalloc/lib/libjemalloc.a)
+#
+# Hack option #3: delete the .so's and use `-ljemalloc` on the link line so that ninja doesn't know about the dependency.
+# ExternalProject_Add_Step(libjemalloc disable_shared
+# DEPENDEES install
+# COMMAND sh -c "rm -v ${CMAKE_BINARY_DIR}/jemalloc/lib/*.so*"
+# )
+
 execute_process(COMMAND cat llvm_revision.txt WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} OUTPUT_VARIABLE LLVMREV OUTPUT_STRIP_TRAILING_WHITESPACE)
 
 # llvm, clang, and libunwind patches
@@ -261,7 +280,7 @@ add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/linkdeps_dummy.c COMMAND ${CMAKE_C
 
 add_executable(pyston $<TARGET_OBJECTS:PYSTON_MAIN_OBJECT> $<TARGET_OBJECTS:PYSTON_OBJECTS> $<TARGET_OBJECTS:FROM_CPYTHON> linkdeps_dummy.c)
 # Wrap the stdlib in --whole-archive to force all the symbols to be included and eventually exported
-target_link_libraries(pyston -Wl,--whole-archive stdlib -Wl,--no-whole-archive pthread m z readline sqlite3 gmp ssl crypto unwind pypa liblz4 double-conversion util ${LLVM_LIBS} ${LIBLZMA_LIBRARIES} ${OPTIONAL_LIBRARIES} -L${CMAKE_BINARY_DIR}/jemalloc/lib -Wl,-rpath,${CMAKE_BINARY_DIR}/jemalloc/lib jemalloc)
+target_link_libraries(pyston -Wl,--whole-archive stdlib -Wl,--no-whole-archive pthread m z readline sqlite3 gmp ssl crypto unwind pypa liblz4 double-conversion util ${LLVM_LIBS} ${LIBLZMA_LIBRARIES} ${OPTIONAL_LIBRARIES} ${CMAKE_BINARY_DIR}/jemalloc/lib/libjemalloc.a)
 add_dependencies(pyston libjemalloc)
 
 # copy src/codegen/parse_ast.py to the build directory
@@ -335,5 +354,27 @@ else()
   add_custom_target(docs COMMAND ${CMAKE_COMMAND} -E echo "Can't create docs, doxygen not installed \(try sudo apt-get install doxygen grpahviz on Ubuntu and then rerun cmake\)" VERBATIM)
 endif()
 
+set(CMAKE_EXECUTABLE_FORMAT "ELF")
+install(TARGETS pyston DESTINATION ".")
+
+set(CPACK_GENERATOR "TGZ")
+
+set(CPACK_PACKAGE_VERSION_MAJOR "0")
+set(CPACK_PACKAGE_VERSION_MINOR "4")
+set(CPACK_PACKAGE_VERSION_PATCH "0")
+
+set(CPACK_SYSTEM_NAME "linux64")
+
+set(CPACK_RESOURCE_FILE_LICENSE ${CMAKE_SOURCE_DIR}/LICENSE)
+set(CPACK_PACKAGE_DESCRIPTION_FILE ${CMAKE_SOURCE_DIR}/README.md)
+
+install(FILES LICENSE README.md DESTINATION ".")
+
+install(FILES test/lib/virtualenv/virtualenv.py DESTINATION virtualenv)
+install(FILES test/lib/virtualenv/virtualenv_support/pip-6.0.8-py2.py3-none-any.whl DESTINATION virtualenv/virtualenv_support)
+install(FILES test/lib/virtualenv/virtualenv_support/setuptools-12.0.5-py2.py3-none-any.whl DESTINATION virtualenv/virtualenv_support)
+
+include(CPack)
+
 # last file added (need to change this if we add a file that is added via a glob):
 # from_cpython/Lib/test/test_zipimport.py

Makefile

@@ -31,10 +31,6 @@ GTEST_DIR := $(DEPS_DIR)/gtest-1.7.0
 
 USE_DEBUG_LIBUNWIND := 0
 
-PYTHON_MAJOR_VERSION := 2
-PYTHON_MINOR_VERSION := 7
-PYTHON_MICRO_VERSION := 3
-
 MAX_MEM_KB := 500000
 MAX_DBG_MEM_KB := 500000
 
@@ -147,7 +143,6 @@ else
 endif
 
 COMMON_CXXFLAGS += -DGITREV=$(shell git rev-parse HEAD | head -c 12) -DLLVMREV=$(LLVM_REVISION)
-COMMON_CXXFLAGS += -DDEFAULT_PYTHON_MAJOR_VERSION=$(PYTHON_MAJOR_VERSION) -DDEFAULT_PYTHON_MINOR_VERSION=$(PYTHON_MINOR_VERSION) -DDEFAULT_PYTHON_MICRO_VERSION=$(PYTHON_MICRO_VERSION)
 
 # Use our "custom linker" that calls gold if available
 COMMON_LDFLAGS := -B$(TOOLS_DIR)/build_system -L/usr/local/lib -lpthread -lm -lunwind -llzma -L$(DEPS_DIR)/gcc-4.8.2-install/lib64 -lreadline -lgmp -lssl -lcrypto -lsqlite3
@@ -476,7 +471,6 @@ quick_check:
 
 Makefile.local:
 	echo "Creating default Makefile.local"
-	python -c 'import sys; v = sys.version_info; print "PYTHON_MAJOR_VERSION:=%d\nPYTHON_MINOR_VERSION:=%d\nPYTHON_MICRO_VERSION:=%d" % (v[0], v[1], v[2])' > Makefile.local || (rm $@; false)
 	which ninja-build >/dev/null && echo "NINJA := ninja-build" >> Makefile.local
 
 llvm_up:
@@ -1079,3 +1073,13 @@ lint_%: %.cpp plugins/clang_linter.so
 
 .PHONY: clang_lint
 clang_lint: $(foreach FN,$(MAIN_SRCS),$(dir $(FN))lint_$(notdir $(FN:.cpp=)))
+
+# 'make package' will build a package using the pgo build, since that's the
+# configuration with the best performance.  Testing that is a pain since it
+# requires rerunning the pgo build, so there's also 'make package_nonpgo' mostly
+# for testing.
+package: pyston_pgo
+	$(NINJA) -C $(CMAKE_DIR_RELEASE_GCC_PGO) package
+
+package_nonpgo:
+	$(NINJA) -C $(CMAKE_DIR_RELEASE) package

cmake/pyconfig.cmake

@@ -64,3 +64,8 @@ set(CMAKE_REQUIRED_LIBRARIES util)
 check_symbol_exists(openpty "pty.h" HAVE_OPENPTY)
 
 configure_file(from_cpython/Include/pyconfig.h.in from_cpython/Include/pyconfig.h)
+
+# CMake sucks: it has no idea that pyconfig.h is something that can be installed.
+# Just tell it to install whatever file is at that particular location, and rely on
+# the rest of the build rules to ensure that it's made in time.
+install(FILES ${CMAKE_BINARY_DIR}/from_cpython/Include/pyconfig.h DESTINATION from_cpython/Include)

from_cpython/CMakeLists.txt

@@ -12,6 +12,9 @@ foreach(STDLIB_FILE ${STDLIB_SRCS} ${STD_INCLUDES})
                        COMMENT "Copying ${FN_REL}"
                        )
     set(STDLIB_TARGETS ${STDLIB_TARGETS} ${TARGET})
+
+    get_filename_component(DIR ${FN_REL} DIRECTORY)
+    install(FILES ${STDLIB_FILE} DESTINATION ${DIR})
 endforeach(STDLIB_FILE)
 add_custom_target(copy_stdlib ALL DEPENDS ${STDLIB_TARGETS})
 
@@ -126,19 +129,22 @@ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-missing-field-initializers -Wno-tautolo
 add_library(FROM_CPYTHON OBJECT ${STDMODULE_SRCS} ${STDOBJECT_SRCS} ${STDPYTHON_SRCS} ${STDPARSER_SRCS})
 add_dependencies(FROM_CPYTHON copy_stdlib)
 
-add_custom_command(OUTPUT
-                       ${CMAKE_BINARY_DIR}/lib_pyston/future_builtins.pyston.so
-                       ${CMAKE_BINARY_DIR}/lib_pyston/_multiprocessing.pyston.so
-                       ${CMAKE_BINARY_DIR}/lib_pyston/pyexpat.pyston.so
-                       ${CMAKE_BINARY_DIR}/lib_pyston/_elementtree.pyston.so
-                       ${CMAKE_BINARY_DIR}/lib_pyston/bz2.pyston.so
-                       ${CMAKE_BINARY_DIR}/lib_pyston/_ctypes.pyston.so
-                       ${CMAKE_BINARY_DIR}/lib_pyston/grp.pyston.so
-                       ${CMAKE_BINARY_DIR}/lib_pyston/readline.pyston.so
-                       ${CMAKE_BINARY_DIR}/lib_pyston/termios.pyston.so
-                       ${CMAKE_BINARY_DIR}/lib_pyston/_curses.pyston.so
-                       ${CMAKE_BINARY_DIR}/lib_pyston/mmap.pyston.so
-                   COMMAND ${CMAKE_BINARY_DIR}/pyston setup.py build --build-lib ${CMAKE_BINARY_DIR}/lib_pyston
+set(STDMODULES
+    ${CMAKE_BINARY_DIR}/from_cpython/Lib/future_builtins.pyston.so
+    ${CMAKE_BINARY_DIR}/from_cpython/Lib/_multiprocessing.pyston.so
+    ${CMAKE_BINARY_DIR}/from_cpython/Lib/pyexpat.pyston.so
+    ${CMAKE_BINARY_DIR}/from_cpython/Lib/_elementtree.pyston.so
+    ${CMAKE_BINARY_DIR}/from_cpython/Lib/bz2.pyston.so
+    ${CMAKE_BINARY_DIR}/from_cpython/Lib/_ctypes.pyston.so
+    ${CMAKE_BINARY_DIR}/from_cpython/Lib/grp.pyston.so
+    ${CMAKE_BINARY_DIR}/from_cpython/Lib/readline.pyston.so
+    ${CMAKE_BINARY_DIR}/from_cpython/Lib/termios.pyston.so
+    ${CMAKE_BINARY_DIR}/from_cpython/Lib/_curses.pyston.so
+    ${CMAKE_BINARY_DIR}/from_cpython/Lib/mmap.pyston.so
+)
+
+add_custom_command(OUTPUT ${STDMODULES}
+    COMMAND ${CMAKE_BINARY_DIR}/pyston setup.py build --build-lib ${CMAKE_BINARY_DIR}/from_cpython/Lib
                    DEPENDS
                        pyston
                        copy_stdlib
@@ -166,4 +172,6 @@ add_custom_command(OUTPUT
 		       Modules/_cursesmodule.c
 		       Modules/mmapmodule.c
                    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
-add_custom_target(sharedmods DEPENDS ${CMAKE_BINARY_DIR}/lib_pyston/_multiprocessing.pyston.so)
+add_custom_target(sharedmods ALL DEPENDS ${CMAKE_BINARY_DIR}/from_cpython/Lib/_multiprocessing.pyston.so)
+
+install(FILES ${STDMODULES} DESTINATION from_cpython/Lib)

lib_pyston/CMakeLists.txt

@@ -13,3 +13,5 @@ foreach(STDLIB_FILE ${LIBPYSTON_SRCS})
     set(LIBPYSTON_TARGETS ${LIBPYSTON_TARGETS} ${TARGET})
 endforeach(STDLIB_FILE)
 add_custom_target(copy_libpyston ALL DEPENDS ${LIBPYSTON_TARGETS})
+
+install(FILES ${LIBPYSTON_SRCS} DESTINATION lib_pyston)

src/analysis/scoping_analysis.cpp

@@ -780,7 +780,7 @@ static void raiseNameForcingSyntaxError(const char* msg, ScopingAnalysis::ScopeN
         syntaxElemMsg = "import * is not allowed in function '%s' because it %s";
         lineno = usage->nameForcingNodeImportStar->lineno;
     } else {
-        if (PYTHON_VERSION_MAJOR == 2 && PYTHON_VERSION_MINOR == 7 && PYTHON_VERSION_MICRO < 8)
+        if (PY_MAJOR_VERSION == 2 && PY_MINOR_VERSION == 7 && PY_MICRO_VERSION < 8)
             syntaxElemMsg = "unqualified exec is not allowed in function '%.100s' it %s";
         else
             syntaxElemMsg = "unqualified exec is not allowed in function '%.100s' because it %s";

src/codegen/irgen/future.cpp

@@ -72,7 +72,7 @@ FutureFlags getFutureFlags(std::vector<AST_stmt*> const& body, const char* file)
 
     // Set the defaults for the future flags depending on what version we are
     for (const std::pair<std::string, FutureOption>& p : future_options) {
-        if (PYTHON_VERSION_HEX >= p.second.mandatory_version_hex) {
+        if (PY_VERSION_HEX >= p.second.mandatory_version_hex) {
             ff |= p.second.ff_mask;
         }
     }
@@ -101,7 +101,7 @@ FutureFlags getFutureFlags(std::vector<AST_stmt*> const& body, const char* file)
                         raiseFutureImportErrorNotFound(file, alias, option_name.c_str());
                     } else {
                         const FutureOption& fo = iter->second;
-                        if (PYTHON_VERSION_HEX >= fo.optional_version_hex) {
+                        if (PY_VERSION_HEX >= fo.optional_version_hex) {
                             ff |= fo.ff_mask;
                         } else {
                             raiseFutureImportErrorNotFound(file, alias, option_name.c_str());

src/core/options.cpp

@@ -19,13 +19,8 @@ namespace pyston {
 int GLOBAL_VERBOSITY = 0;
 
 int PYSTON_VERSION_MAJOR = 0;
-int PYSTON_VERSION_MINOR = 3;
-
-int PYTHON_VERSION_MAJOR = DEFAULT_PYTHON_MAJOR_VERSION;
-int PYTHON_VERSION_MINOR = DEFAULT_PYTHON_MINOR_VERSION;
-int PYTHON_VERSION_MICRO = DEFAULT_PYTHON_MICRO_VERSION;
-
-int PYTHON_VERSION_HEX = version_hex(PYTHON_VERSION_MAJOR, PYTHON_VERSION_MINOR, PYTHON_VERSION_MICRO);
+int PYSTON_VERSION_MINOR = 4;
+int PYSTON_VERSION_MICRO = 0;
 
 int MAX_OPT_ITERATIONS = 1;
 

src/core/options.h

@@ -21,9 +21,7 @@ extern "C" {
 
 extern int GLOBAL_VERBOSITY;
 #define VERBOSITY(x) pyston::GLOBAL_VERBOSITY
-extern int PYSTON_VERSION_MAJOR, PYSTON_VERSION_MINOR;
-// Version number we're targeting:
-extern int PYTHON_VERSION_MAJOR, PYTHON_VERSION_MINOR, PYTHON_VERSION_MICRO, PYTHON_VERSION_HEX;
+extern int PYSTON_VERSION_MAJOR, PYSTON_VERSION_MINOR, PYSTON_VERSION_MICRO;
 
 inline int version_hex(int major, int minor, int micro, int level = 0, int serial = 0) {
     return (major << 24) | (minor << 16) | (micro << 8) | (level << 4) | (serial << 0);

src/jit.cpp

@@ -31,6 +31,7 @@
 #include "llvm/Support/Signals.h"
 
 #include "osdefs.h"
+#include "patchlevel.h"
 
 #include "asm_writing/disassemble.h"
 #include "capi/types.h"
@@ -507,8 +508,9 @@ static int main(int argc, char** argv) {
             if (!v)
                 PyErr_Clear();
 
-            printf("Pyston v%d.%d (rev " STRINGIFY(GITREV) ")", PYSTON_VERSION_MAJOR, PYSTON_VERSION_MINOR);
-            printf(", targeting Python %d.%d.%d\n", PYTHON_VERSION_MAJOR, PYTHON_VERSION_MINOR, PYTHON_VERSION_MICRO);
+            printf("Pyston v%d.%d.%d (rev " STRINGIFY(GITREV) ")", PYSTON_VERSION_MAJOR, PYSTON_VERSION_MINOR,
+                   PYSTON_VERSION_MICRO);
+            printf(", targeting Python %d.%d.%d\n", PY_MAJOR_VERSION, PY_MINOR_VERSION, PY_MICRO_VERSION);
 
             Py_InspectFlag = 0;
 

src/runtime/builtin_modules/sys.cpp

@@ -256,9 +256,9 @@ public:
 
 static std::string generateVersionString() {
     std::ostringstream oss;
-    oss << PYTHON_VERSION_MAJOR << '.' << PYTHON_VERSION_MINOR << '.' << PYTHON_VERSION_MICRO;
+    oss << PY_MAJOR_VERSION << '.' << PY_MINOR_VERSION << '.' << PY_MICRO_VERSION;
     oss << '\n';
-    oss << "[Pyston " << PYSTON_VERSION_MAJOR << '.' << PYSTON_VERSION_MINOR << "]";
+    oss << "[Pyston " << PYSTON_VERSION_MAJOR << '.' << PYSTON_VERSION_MINOR << '.' << PYSTON_VERSION_MICRO << "]";
     return oss.str();
 }
 

src/runtime/objmodel.cpp

@@ -2398,6 +2398,10 @@ void setattrGeneric(Box* obj, BoxedString* attr, Box* val, SetattrRewriteArgs* r
     }
 }
 
+// force template instantiation:
+template void setattrGeneric<NOT_REWRITABLE>(Box* obj, BoxedString* attr, Box* val, SetattrRewriteArgs* rewrite_args);
+template void setattrGeneric<REWRITABLE>(Box* obj, BoxedString* attr, Box* val, SetattrRewriteArgs* rewrite_args);
+
 extern "C" void setattr(Box* obj, BoxedString* attr, Box* attr_val) {
     STAT_TIMER(t0, "us_timer_slowpath_setattr", 10);
 
@@ -5343,12 +5347,12 @@ Box* getitemInternal(Box* target, Box* slice, GetitemRewriteArgs* rewrite_args)
         rewrite_args = NULL;
 
         // different versions of python give different error messages for this:
-        if (PYTHON_VERSION_MAJOR == 2 && PYTHON_VERSION_MINOR < 7) {
+        if (PY_MAJOR_VERSION == 2 && PY_MINOR_VERSION < 7) {
             if (S == CAPI)
                 PyErr_Format(TypeError, "'%s' object is unsubscriptable", getTypeName(target)); // tested on 2.6.6
             else
                 raiseExcHelper(TypeError, "'%s' object is unsubscriptable", getTypeName(target)); // tested on 2.6.6
-        } else if (PYTHON_VERSION_MAJOR == 2 && PYTHON_VERSION_MINOR == 7 && PYTHON_VERSION_MICRO < 3) {
+        } else if (PY_MAJOR_VERSION == 2 && PY_MINOR_VERSION == 7 && PY_MICRO_VERSION < 3) {
             if (S == CAPI)
                 PyErr_Format(TypeError, "'%s' object is not subscriptable", getTypeName(target)); // tested on 2.7.1
             else

src/runtime/objmodel.h

@@ -204,7 +204,7 @@ Box* callCLFunc(CLFunction* f, CallRewriteArgs* rewrite_args, int num_output_arg
                 Box** oargs) noexcept(S == CAPI);
 
 static const char* objectNewParameterTypeErrorMsg() {
-    if (PYTHON_VERSION_HEX >= version_hex(2, 7, 4)) {
+    if (PY_VERSION_HEX >= version_hex(2, 7, 4)) {
         return "object() takes no parameters";
     } else {
         return "object.__new__() takes no parameters";

test/extra/cheetah_test.py

@@ -4,7 +4,7 @@ sys.path.append(os.path.dirname(__file__) + "/../lib")
 from test_helper import create_virtenv, run_test
 
 ENV_NAME = os.path.abspath("cheetah_test_env_" + os.path.basename(sys.executable))
-create_virtenv(ENV_NAME, ["cheetah==2.4.4"], force_create = True)
+create_virtenv(ENV_NAME, ["cheetah==2.4.4", "Markdown==2.0.1"], force_create = True)
 
 cheetah_exe = os.path.join(ENV_NAME, "bin", "cheetah")
 env = os.environ