enable PGO for gcc.

instead of building twice in the same build dir (which if there's an error can lead
to using the -fprofile-generate thinking it's the -fprofile-use build), we build the
-fprofile-generate build as pyston_release_gcc_pgo.  The resulting -fprofile-use build
is the normal pyston_release_gcc build.

Deps aren't quite there yet, but this should also allows you to re-train the pgo build later
and then apply the new data to the release build, e.g.:

```
   $ rm -f ./build/Release-gcc
   $ ./pyston_release_gcc_pgo ${training.py}
   $ make pyston_release_gcc
```

.gitignore

@@ -24,6 +24,7 @@ pyston_grwl_dbg
 pyston_nosync
 pyston_gcc
 pyston_release_gcc
+pyston_release_gcc_pgo
 pystontmp*/
 /*_unittest
 

CMakeLists.txt

@@ -29,6 +29,9 @@ option(ENABLE_OPROFILE "enable oprofile support" OFF)
 option(ENABLE_SELF_HOST "use pyston to test pyston" OFF)
 option(ENABLE_VALGRIND "pyston valgrind support" OFF)
 
+option(ENABLE_PGO "enable -fprofile-generate/-fprofile-use" OFF)
+option(ENABLE_LTO "enable -flto (only if ENABLE_PGO=ON as well)" OFF)
+
 # automatically use ccache if found
 if(ENABLE_CCACHE)
   find_program(CCACHE ccache)
@@ -46,7 +49,10 @@ if(ENABLE_GOLD)
     message(STATUS "found the gold linker ${GOLD_LINKER}")
     set(CMAKE_LINKER "${GOLD_LINKER}")
     set(CMAKE_CXX_LINK_FLAGS "${CMAKE_CXX_LINK_FLAGS} -B${CMAKE_SOURCE_DIR}/tools/build_system")
-    set(CMAKE_CXX_LINK_FLAGS "${CMAKE_CXX_LINK_FLAGS} -Wl,--section-ordering-file,${CMAKE_CURRENT_SOURCE_DIR}/section_ordering.txt")
+    if(NOT ENABLE_PGO)
+      # let pgo determine the ordering
+      set(CMAKE_CXX_LINK_FLAGS "${CMAKE_CXX_LINK_FLAGS} -Wl,--section-ordering-file,${CMAKE_CURRENT_SOURCE_DIR}/section_ordering.txt")
+    endif()
   endif()
 endif()
 
@@ -59,14 +65,41 @@ else()
 endif()
 
 # initial clang flags (set here so they're used when building llvm)
-set(CLANG_FLAGS "-Qunused-arguments -fcolor-diagnostics" CACHE STRING "Clang specific C and CXX flags")
+set(CLANG_FLAGS "-Qunused-arguments -fcolor-diagnostics -Wno-profile-instr-unprofiled" CACHE STRING "Clang specific C and CXX flags")
 if("${CMAKE_C_COMPILER_ID}" STREQUAL "Clang")
   set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${CLANG_FLAGS}")
+else()
 endif()
 if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CLANG_FLAGS}")
 endif()
 
+macro(ADD_PROFILE_FLAGS)
+  if(ENABLE_LTO)
+    set(LTO_FLAGS "-flto")
+  endif()
+  if(NOT "${PROFILE_DIR}" STREQUAL "")
+    set(PROFILE_STATE "${PROFILE_STATE}=${PROFILE_DIR}")
+  endif()
+  set(PROFILE_C_FLAGS "-fprofile-${PROFILE_STATE} ${LTO_FLAGS}")
+  set(PROFILE_CXX_FLAGS "-fprofile-${PROFILE_STATE} ${LTO_FLAGS}")
+  set(PROFILE_LINK_FLAGS "-fprofile-${PROFILE_STATE} ${LTO_FLAGS}")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${PROFILE_CXX_FLAGS} ${LTO_FLAGS}")
+endmacro()
+
+# used in src/runtime/inline/CMakeLists.txt to remove the flags we added above
+macro(REMOVE_PROFILE_FLAGS)
+  string(REPLACE "${PROFILE_C_FLAGS}"   "" CMAKE_C_FLAGS "${CMAKE_C_FLAGS}")
+  string(REPLACE "${PROFILE_CXX_FLAGS}" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
+  string(REPLACE "${PROFILE_CXX_FLAGS}" "" CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE}")
+  string(REPLACE "${PROFILE_CXX_FLAGS}" "" CMAKE_CXX_LINK_FLAGS "${CMAKE_CXX_LINK_FLAGS}")
+  string(REPLACE "${PROFILE_CXX_FLAGS}" "" CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS}")
+endmacro()
+
+if(${CMAKE_BUILD_TYPE} STREQUAL "Release" AND ENABLE_PGO)
+  ADD_PROFILE_FLAGS()
+endif()
+
 # llvm disable debug info unless ENABLE_LLVM_DEBUG is ON
 if(${CMAKE_BUILD_TYPE} STREQUAL "Debug" AND NOT ENABLE_LLVM_DEBUG)
   set(CMAKE_CXX_FLAGS_DEBUG "-O3" CACHE STRING "" FORCE)
@@ -200,6 +233,7 @@ endif()
 
 if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-long-long -Wno-aggregate-return -Wno-inline -Wno-redundant-decls -Wno-strict-overflow")
+  set(CMAKE_CXX_LINK_FLAGS "${CMAKE_CXX_LINK_FLAGS} -Wl,--no-as-needed") # without this, we don't seem to get jemalloc on gcc builds
 endif()
 
 add_definitions(${LLVM_DEFINITIONS})

Makefile

@@ -65,6 +65,7 @@ CMAKE_DIR_DBG := $(BUILD_DIR)/Debug
 CMAKE_DIR_RELEASE := $(BUILD_DIR)/Release
 CMAKE_DIR_GCC := $(BUILD_DIR)/Debug-gcc
 CMAKE_DIR_RELEASE_GCC := $(BUILD_DIR)/Release-gcc
+CMAKE_DIR_RELEASE_GCC_PGO := $(BUILD_DIR)/Release-gcc-pgo
 CMAKE_SETUP_DBG := $(CMAKE_DIR_DBG)/build.ninja
 CMAKE_SETUP_RELEASE := $(CMAKE_DIR_RELEASE)/build.ninja
 
@@ -912,12 +913,29 @@ CMAKE_SETUP_RELEASE_GCC := $(CMAKE_DIR_RELEASE_GCC)/build.ninja
 $(CMAKE_SETUP_RELEASE_GCC):
 	@$(MAKE) cmake_check
 	@mkdir -p $(CMAKE_DIR_RELEASE_GCC)
-	cd $(CMAKE_DIR_RELEASE_GCC); CC='$(GCC)' CXX='$(GPP)' cmake -GNinja $(SRC_DIR) -DCMAKE_BUILD_TYPE=Release $(CMAKE_VALGRIND)
+	cd $(CMAKE_DIR_RELEASE_GCC); CC='$(GCC)' CXX='$(GPP)' cmake -GNinja $(SRC_DIR) -DCMAKE_BUILD_TYPE=Release $(CMAKE_VALGRIND) -DENABLE_PGO=ON -DPROFILE_STATE=use
 .PHONY: pyston_release_gcc
-pyston_release_gcc: $(CMAKE_SETUP_RELEASE_GCC)
+pyston_release_gcc: $(CMAKE_SETUP_RELEASE_GCC) $(CMAKE_DIR_RELEASE_GCC)/.trained
 	$(NINJA) -C $(CMAKE_DIR_RELEASE_GCC) pyston copy_stdlib copy_libpyston $(CMAKE_SHAREDMODS) ext_cpython $(NINJAFLAGS)
 	ln -sf $(CMAKE_DIR_RELEASE_GCC)/pyston pyston_release_gcc
 
+
+# GCC PGO build
+CMAKE_SETUP_RELEASE_GCC_PGO := $(CMAKE_DIR_RELEASE_GCC_PGO)/build.ninja
+$(CMAKE_SETUP_RELEASE_GCC_PGO):
+	@$(MAKE) cmake_check
+	@mkdir -p $(CMAKE_DIR_RELEASE_GCC_PGO)
+	cd $(CMAKE_DIR_RELEASE_GCC_PGO); CC='$(GCC)' CXX='$(GPP)' cmake -GNinja $(SRC_DIR) -DCMAKE_BUILD_TYPE=Release $(CMAKE_VALGRIND) -DENABLE_PGO=ON -DPROFILE_STATE=generate -DPROFILE_DIR=$(CMAKE_DIR_RELEASE_GCC)
+
+.PHONY: pyston_release_gcc_pgo
+pyston_release_gcc_pgo: $(CMAKE_SETUP_RELEASE_GCC_PGO)
+	$(NINJA) -C $(CMAKE_DIR_RELEASE_GCC_PGO) -f $(CMAKE_SETUP_RELEASE_GCC_PGO) pyston copy_stdlib copy_libpyston $(CMAKE_SHAREDMODS) ext_cpython $(NINJAFLAGS)
+	ln -sf $(CMAKE_DIR_RELEASE_GCC_PGO)/pyston $@
+
+$(CMAKE_DIR_RELEASE_GCC)/.trained: pyston_release_gcc_pgo
+	@echo "Training pgo"
+	( cd $(CMAKE_DIR_RELEASE_GCC_PGO) && ./pyston $(SRC_DIR)/minibenchmarks/combined.py && ./pyston $(SRC_DIR)/minibenchmarks/combined.py ) && touch $(CMAKE_DIR_RELEASE_GCC)/.trained
+
 .PHONY: format check_format
 format: $(CMAKE_SETUP_RELEASE)
 	$(NINJA) -C $(CMAKE_DIR_RELEASE) format

src/runtime/inline/CMakeLists.txt

@@ -36,6 +36,10 @@
 
 # adapted from https://github.com/shadow/shadow/blob/master/cmake/LLVMTools.cmake
 
+# don't do pgo here, as our gcc command line flags are incompatible with clang, and we
+# don't want to bake the instrumentation into the resulting bitcode anyway.
+REMOVE_PROFILE_FLAGS()
+
 set(BC_INLINE_SRCS boxing.cpp dict.cpp link_forcer.cpp list.cpp tuple.cpp xrange.cpp)
 
 include_directories(${CMAKE_SOURCE_DIR}/src)