Laying some groundwork for rebasing to LLVM trunk

Bunch of fairly-simple API changes to contend with.

Main problem is we're still running into those debuginfo issues.

clang_patches/0001-Rename-one-of-scan-builds-internal-environment-varia.patch

@@ -12,9 +12,7 @@ diff --git a/tools/scan-build/ccc-analyzer b/tools/scan-build/ccc-analyzer
 index b5445e6..65f8455 100755
 --- a/tools/scan-build/ccc-analyzer
 +++ b/tools/scan-build/ccc-analyzer
-@@ -43,7 +43,7 @@ if ($FindBin::Script =~ /c\+\+-analyzer/) {
-   $Compiler = $ENV{'CCC_CXX'};
-   if (!defined $Compiler || ! -x $Compiler) { $Compiler = $DefaultCXXCompiler; }
+@@ -43,5 +43,5 @@ if ($FindBin::Script =~ /c\+\+-analyzer/) {
  
 -  $Clang = $ENV{'CLANG_CXX'};
 +  $Clang = $ENV{'CCC_CLANG_CXX'};

llvm_patches/0001-Support-emitting-stackmap-sections-for-ELF-object-fi.patch

@@ -12,25 +12,6 @@ Subject: [PATCH] Support emitting stackmap sections for ELF object files as
  4 files changed, 138 insertions(+), 1 deletion(-)
  create mode 100644 test/CodeGen/X86/stackmap-elf.ll
 
-diff --git a/lib/IR/Function.cpp b/lib/IR/Function.cpp
-index c1fa372..d11727d 100644
---- a/lib/IR/Function.cpp
-+++ b/lib/IR/Function.cpp
-@@ -681,7 +681,13 @@ FunctionType *Intrinsic::getType(LLVMContext &Context,
-   while (!TableRef.empty())
-     ArgTys.push_back(DecodeFixedType(TableRef, Tys, Context));
- 
--  return FunctionType::get(ResultTy, ArgTys, false);
-+  bool variadic = false;
-+  if (ArgTys.size() && ArgTys.back() == Type::getVoidTy(Context)) {
-+    variadic = true;
-+    ArgTys.pop_back();
-+  }
-+
-+  return FunctionType::get(ResultTy, ArgTys, variadic);
- }
- 
- bool Intrinsic::isOverloaded(ID id) {
 diff --git a/lib/MC/MCObjectFileInfo.cpp b/lib/MC/MCObjectFileInfo.cpp
 index 931b354..40899c1 100644
 --- a/lib/MC/MCObjectFileInfo.cpp

llvm_patches/0002-support-varargs-intrinsics.patch

+From b370f4621298ec557e356e21fa0fa675a21d2331 Mon Sep 17 00:00:00 2001
+From: Kevin Modzelewski <kmod@dropbox.com>
+Date: Fri, 21 Mar 2014 16:06:58 -0700
+Subject: [PATCH] Support varargs intrinsics
+
+diff --git a/lib/IR/Function.cpp b/lib/IR/Function.cpp
+index c1fa372..d11727d 100644
+--- a/lib/IR/Function.cpp
++++ b/lib/IR/Function.cpp
+@@ -681,7 +681,13 @@ FunctionType *Intrinsic::getType(LLVMContext &Context,
+   while (!TableRef.empty())
+     ArgTys.push_back(DecodeFixedType(TableRef, Tys, Context));
+ 
+-  return FunctionType::get(ResultTy, ArgTys, false);
++  bool variadic = false;
++  if (ArgTys.size() && ArgTys.back() == Type::getVoidTy(Context)) {
++    variadic = true;
++    ArgTys.pop_back();
++  }
++
++  return FunctionType::get(ResultTy, ArgTys, variadic);
+ }
+ 
+ bool Intrinsic::isOverloaded(ID id) {
+
+-- 
+1.7.9.5
+
+

src/codegen/codegen.cpp

@@ -41,7 +41,7 @@ void FunctionAddressRegistry::dumpPerfMap() {
     std::string out_path = "perf_map";
     removeDirectoryIfExists(out_path);
 
-    llvm::error_code code;
+    llvm_error_code code;
     code = llvm::sys::fs::create_directory(out_path, false);
     assert(!code);
 
@@ -139,7 +139,7 @@ public:
         static StatCounter code_bytes("code_bytes");
         code_bytes.log(Obj.getData().size());
 
-        llvm::error_code code;
+        llvm_error_code code;
         for (llvm::object::symbol_iterator I = Obj.begin_symbols(), E = Obj.end_symbols(); I != E;
 #if LLVMREV < 200442
              I = I.increment(code)

src/codegen/dis.cpp

@@ -14,6 +14,8 @@
 
 #include "codegen/dis.h"
 
+#if LLVMREV < 216983
+
 #include <cstdio>
 #include <iostream>
 #include <unordered_map>
@@ -225,3 +227,5 @@ void PystonJITEventListener::NotifyObjectEmitted(const llvm::ObjectImage& Obj) {
     llvm::outs().flush();
 }
 }
+
+#endif

src/codegen/entry.cpp

@@ -92,10 +92,14 @@ static llvm::Module* loadStdlib() {
         data = llvm::StringRef(STRIPPED_STDLIB_BC_START, size);
     }
 
+#if LLVMREV < 216583
     llvm::MemoryBuffer* buffer = llvm::MemoryBuffer::getMemBuffer(data, "", false);
+#else
+    std::unique_ptr<llvm::MemoryBuffer> buffer = llvm::MemoryBuffer::getMemBuffer(data, "", false);
+#endif
 
     // llvm::ErrorOr<llvm::Module*> m_or = llvm::parseBitcodeFile(buffer, g.context);
-    llvm::ErrorOr<llvm::Module*> m_or = llvm::getLazyBitcodeModule(buffer, g.context);
+    llvm::ErrorOr<llvm::Module*> m_or = llvm::getLazyBitcodeModule(std::move(buffer), g.context);
     RELEASE_ASSERT(m_or, "");
     llvm::Module* m = m_or.get();
     assert(m);
@@ -115,13 +119,21 @@ private:
 public:
     MyObjectCache() : loaded(false) {}
 
+#if LLVMREV < 216002
     virtual void notifyObjectCompiled(const llvm::Module* M, const llvm::MemoryBuffer* Obj) {}
+#else
+    virtual void notifyObjectCompiled(const llvm::Module* M, llvm::MemoryBufferRef Obj) {}
+#endif
 
+#if LLVMREV < 215566
     virtual llvm::MemoryBuffer* getObject(const llvm::Module* M) {
+#else
+    virtual std::unique_ptr<llvm::MemoryBuffer> getObject(const llvm::Module* M) {
+#endif
         assert(!loaded);
-        loaded = true;
-        g.engine->setObjectCache(NULL);
-        std::unique_ptr<MyObjectCache> del_at_end(this);
+    loaded = true;
+    g.engine->setObjectCache(NULL);
+    std::unique_ptr<MyObjectCache> del_at_end(this);
 
 #if 0
             if (!USE_STRIPPED_STDLIB) {
@@ -137,17 +149,17 @@ public:
         intptr_t size = 0;
 #endif
 
-        // Make sure the stdlib got linked in correctly; check the magic number at the beginning:
-        assert(start[0] == 0x7f);
-        assert(start[1] == 'E');
-        assert(start[2] == 'L');
-        assert(start[3] == 'F');
+    // Make sure the stdlib got linked in correctly; check the magic number at the beginning:
+    assert(start[0] == 0x7f);
+    assert(start[1] == 'E');
+    assert(start[2] == 'L');
+    assert(start[3] == 'F');
 
-        assert(size > 0 && size < 1 << 30); // make sure the size is being loaded correctly
+    assert(size > 0 && size < 1 << 30); // make sure the size is being loaded correctly
 
-        llvm::StringRef data(start, size);
-        return llvm::MemoryBuffer::getMemBufferCopy(data, "");
-    }
+    llvm::StringRef data(start, size);
+    return llvm::MemoryBuffer::getMemBufferCopy(data, "");
+}
 };
 
 static void handle_sigfpe(int signum) {
@@ -171,9 +183,17 @@ void initCodegen() {
 
     g.stdlib_module = loadStdlib();
 
+#if LLVMREV < 215967
     llvm::EngineBuilder eb(new llvm::Module("empty_initial_module", g.context));
-    eb.setEngineKind(llvm::EngineKind::JIT); // specify we only want the JIT, and not the interpreter fallback
+#else
+    jlvm::EngineBuilder eb(std::unique_ptr<llvm::Module>(new llvm::Module("empty_initial_module", g.context)));
+#endif
+
+#if LLVMREV < 216982
     eb.setUseMCJIT(true);
+#endif
+
+    eb.setEngineKind(llvm::EngineKind::JIT); // specify we only want the JIT, and not the interpreter fallback
     eb.setMCJITMemoryManager(createMemoryManager());
     // eb.setOptLevel(llvm::CodeGenOpt::None); // -O0
     // eb.setOptLevel(llvm::CodeGenOpt::Less); // -O1
@@ -225,9 +245,14 @@ void initCodegen() {
     g.engine->RegisterJITEventListener(tracebacks_listener);
 
     if (SHOW_DISASM) {
+#if LLVMREV < 216983
         llvm::JITEventListener* listener = new PystonJITEventListener();
         g.jit_listeners.push_back(listener);
         g.engine->RegisterJITEventListener(listener);
+#else
+        fprintf(stderr, "The LLVM disassembler has been removed\n");
+        abort();
+#endif
     }
 
     initGlobalFuncs(g);
@@ -240,7 +265,7 @@ void initCodegen() {
     // There are some parts of llvm that are only configurable through command line args,
     // so construct a fake argc/argv pair and pass it to the llvm command line machinery:
     const char* llvm_args[] = {
-        "fake_name", "--enable-stackmap-liveness", "--enable-patchpoint-liveness",
+        "fake_name", "--enable-patchpoint-liveness",
 
 // Enabling and debugging fast-isel:
 //"--fast-isel",

src/codegen/irgen.cpp

@@ -26,6 +26,7 @@
 #include "llvm/PassManager.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Target/TargetMachine.h"
+#include "llvm/Target/TargetSubtargetInfo.h"
 #include "llvm/Transforms/Instrumentation.h"
 #include "llvm/Transforms/Scalar.h"
 
@@ -65,7 +66,7 @@ void MyInserter::InsertHelper(llvm::Instruction* I, const llvm::Twine& Name, llv
 static void addIRDebugSymbols(llvm::Function* f) {
     llvm::legacy::PassManager mpm;
 
-    llvm::error_code code = llvm::sys::fs::create_directory(".debug_ir", true);
+    llvm_error_code code = llvm::sys::fs::create_directory(".debug_ir", true);
     assert(!code);
 
     mpm.add(llvm::createDebugIRPass(false, false, ".debug_ir", f->getName()));
@@ -83,8 +84,11 @@ static void optimizeIR(llvm::Function* f, EffortLevel::EffortLevel effort) {
 
     llvm::FunctionPassManager fpm(g.cur_module);
 
-    // TODO: using this as a pass is a legacy cludge that shouldn't be necessary any more; can it be updated?
+#if LLVMREV < 217548
     fpm.add(new llvm::DataLayoutPass(*g.tm->getDataLayout()));
+#else
+    fpm.add(new llvm::DataLayoutPass());
+#endif
 
     if (ENABLE_INLINING && effort >= EffortLevel::MAXIMAL)
         fpm.add(makeFPInliner(275));
@@ -918,7 +922,11 @@ static llvm::MDNode* setupDebugInfo(SourceInfo* source, llvm::Function* f, std::
     std::string producer = "pyston; git rev " STRINGIFY(GITREV);
 
     llvm::DIFile file = builder.createFile(fn, dir);
+#if LLVMREV < 214132
     llvm::DIArray param_types = builder.getOrCreateArray(llvm::None);
+#else
+    llvm::DITypeArray param_types = builder.getOrCreateTypeArray(llvm::None);
+#endif
     llvm::DICompositeType func_type = builder.createSubroutineType(file, param_types);
     llvm::DISubprogram func_info = builder.createFunction(file, f->getName(), f->getName(), file, lineno, func_type,
                                                           false, true, lineno + 1, 0, true, f);
@@ -967,7 +975,11 @@ CompiledFunction* doCompile(SourceInfo* source, const OSREntryDescriptor* entry_
     assert(g.cur_module == NULL);
     std::string name = getUniqueFunctionName(nameprefix, effort, entry_descriptor);
     g.cur_module = new llvm::Module(name, g.context);
+#if LLVMREV < 217070 // not sure if this is the right rev
     g.cur_module->setDataLayout(g.tm->getDataLayout()->getStringRepresentation());
+#else
+    g.cur_module->setDataLayout(g.tm->getSubtargetImpl()->getDataLayout());
+#endif
     // g.engine->addModule(g.cur_module);
 
     ////

src/codegen/irgen/hooks.cpp

@@ -102,7 +102,11 @@ static void compileIR(CompiledFunction* cf, EffortLevel::EffortLevel effort) {
     void* compiled = NULL;
     if (effort > EffortLevel::INTERPRETED) {
         Timer _t("to jit the IR");
+#if LLVMREV < 215967
         g.engine->addModule(cf->func->getParent());
+#else
+        g.engine->addModule(std::unique_ptr<llvm::Module>(cf->func->getParent()));
+#endif
         compiled = (void*)g.engine->getFunctionAddress(cf->func->getName());
         assert(compiled);
         cf->llvm_code = embedConstantPtr(compiled, cf->func->getType());

src/codegen/memmgr.cpp

@@ -53,7 +53,7 @@ private:
 
     uint8_t* allocateSection(MemoryGroup& MemGroup, uintptr_t Size, unsigned Alignment);
 
-    error_code applyMemoryGroupPermissions(MemoryGroup& MemGroup, unsigned Permissions);
+    llvm_error_code applyMemoryGroupPermissions(MemoryGroup& MemGroup, unsigned Permissions);
 
     virtual uint64_t getSymbolAddress(const std::string& Name);
 
@@ -110,7 +110,7 @@ uint8_t* PystonMemoryManager::allocateSection(MemoryGroup& MemGroup, uintptr_t S
     //
     // FIXME: Initialize the Near member for each memory group to avoid
     // interleaving.
-    error_code ec;
+    llvm_error_code ec;
     sys::MemoryBlock MB = sys::Memory::allocateMappedMemory(RequiredSize, &MemGroup.Near,
                                                             sys::Memory::MF_READ | sys::Memory::MF_WRITE, ec);
     if (ec) {
@@ -140,7 +140,7 @@ uint8_t* PystonMemoryManager::allocateSection(MemoryGroup& MemGroup, uintptr_t S
 
 bool PystonMemoryManager::finalizeMemory(std::string* ErrMsg) {
     // FIXME: Should in-progress permissions be reverted if an error occurs?
-    error_code ec;
+    llvm_error_code ec;
 
     // Don't allow free memory blocks to be used after setting protection flags.
     CodeMem.FreeMem.clear();
@@ -177,10 +177,10 @@ bool PystonMemoryManager::finalizeMemory(std::string* ErrMsg) {
     return false;
 }
 
-error_code PystonMemoryManager::applyMemoryGroupPermissions(MemoryGroup& MemGroup, unsigned Permissions) {
+llvm_error_code PystonMemoryManager::applyMemoryGroupPermissions(MemoryGroup& MemGroup, unsigned Permissions) {
 
     for (int i = 0, e = MemGroup.AllocatedMem.size(); i != e; ++i) {
-        error_code ec;
+        llvm_error_code ec;
         ec = sys::Memory::protectMappedMemory(MemGroup.AllocatedMem[i], Permissions);
         if (ec) {
             return ec;
@@ -188,9 +188,9 @@ error_code PystonMemoryManager::applyMemoryGroupPermissions(MemoryGroup& MemGrou
     }
 
 #if LLVMREV < 209952
-    return error_code::success();
+    return llvm_error_code::success();
 #else
-    return error_code();
+    return llvm_error_code();
 #endif
 }
 

src/codegen/opt/const_classes.cpp

@@ -25,6 +25,7 @@
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Target/TargetMachine.h"
+#include "llvm/Target/TargetSubtargetInfo.h"
 
 #include "codegen/codegen.h"
 #include "codegen/irgen/util.h"
@@ -104,7 +105,11 @@ private:
             }
 
             APInt ap_offset(64, 0, true);
+#if LLVMREV < 214781
             bool success = gep->accumulateConstantOffset(*g.tm->getDataLayout(), ap_offset);
+#else
+            bool success = gep->accumulateConstantOffset(*g.tm->getSubtargetImpl()->getDataLayout(), ap_offset);
+#endif
             assert(success);
             int64_t offset = ap_offset.getSExtValue();
 

src/codegen/profiling/dumprof.cpp

@@ -31,7 +31,7 @@ public:
 
 static int num = 0;
 void DumpJITEventListener::NotifyObjectEmitted(const llvm::ObjectImage& Obj) {
-    llvm::error_code code;
+    llvm_error_code code;
 
     std::ostringstream os("");
     os << "jit" << ++num << ".o";

src/codegen/runtime_hooks.cpp

@@ -19,7 +19,6 @@
 
 #include "llvm/Analysis/Passes.h"
 #include "llvm/ExecutionEngine/ExecutionEngine.h"
-#include "llvm/ExecutionEngine/JIT.h"
 #include "llvm/IR/Module.h"
 #include "llvm/PassManager.h"
 #include "llvm/Support/DynamicLibrary.h"

src/codegen/stackmaps.cpp

@@ -60,7 +60,7 @@ extern void _force_link() {
 void StackmapJITEventListener::NotifyObjectEmitted(const llvm::ObjectImage& Obj) {
     // llvm::outs() << "An object has been emitted:\n";
 
-    llvm::error_code code;
+    llvm_error_code code;
 
     for (llvm::object::symbol_iterator I = Obj.begin_symbols(), E = Obj.end_symbols(); I != E;) {
         llvm::StringRef name;

src/codegen/unwinding.cpp

@@ -111,9 +111,13 @@ const LineInfo* getLineInfoFor(uint64_t addr) {
 class TracebacksEventListener : public llvm::JITEventListener {
 public:
     void NotifyObjectEmitted(const llvm::ObjectImage& Obj) {
+#if LLVMREV < 214433
         llvm::DIContext* Context = llvm::DIContext::getDWARFContext(Obj.getObjectFile());
+#else
+        llvm::DIContext* Context = llvm::DIContext::getDWARFContext(*Obj.getObjectFile());
+#endif
 
-        llvm::error_code ec;
+        llvm_error_code ec;
         for (llvm::object::symbol_iterator I = Obj.begin_symbols(), E = Obj.end_symbols(); I != E && !ec; ++I) {
             llvm::object::SymbolRef::Type SymType;
             if (I->getType(SymType))
@@ -151,7 +155,7 @@ public:
         delete Context;
 
         // Currently-unused libunwind support:
-        llvm::error_code code;
+        llvm_error_code code;
         bool found_text = false, found_eh_frame = false;
         uint64_t text_addr, text_size;
         uint64_t eh_frame_addr, eh_frame_size;

src/core/common.h

@@ -47,6 +47,12 @@
 #define NOINLINE
 #endif
 
+#if LLVMREV < 210783
+#define llvm_error_code llvm::error_code
+#else
+#define llvm_error_code std::error_code
+#endif
+
 // From http://stackoverflow.com/questions/3767869/adding-message-to-assert, modified to use fprintf and give a Python
 // stacktrace
 #define RELEASE_ASSERT(condition, fmt, ...)                                                                            \

src/core/util.cpp

@@ -22,6 +22,7 @@
 
 #include "core/common.h"
 #include "core/options.h"
+#include "core/types.h"
 
 namespace pyston {
 
@@ -92,7 +93,7 @@ bool endswith(const std::string& s, const std::string& pattern) {
 }
 
 void removeDirectoryIfExists(const std::string& path) {
-    llvm::error_code code;
+    llvm_error_code code;
 
     llvm::sys::fs::file_status status;
     code = llvm::sys::fs::status(path, status);

src/runtime/builtin_modules/builtins.cpp

@@ -586,13 +586,20 @@ Box* execfile(Box* _fn) {
 
     BoxedString* fn = static_cast<BoxedString*>(_fn);
 
+#if LLVMREV < 217625
     bool exists;
-    llvm::error_code code = llvm::sys::fs::exists(fn->s, exists);
+    llvm_error_code code = llvm::sys::fs::exists(fn->s, exists);
+
 #if LLVMREV < 210072
     ASSERT(code == 0, "%s: %s", code.message().c_str(), fn->s.c_str());
 #else
     assert(!code);
 #endif
+
+#else
+    bool exists = llvm::sys::fs::exists(fn->s);
+#endif
+
     if (!exists)
         raiseExcHelper(IOError, "No such file or directory: '%s'", fn->s.c_str());
 

src/runtime/import.cpp

@@ -66,9 +66,13 @@ static Box* importSub(const std::string* name, Box* parent_module) {
         if (VERBOSITY() >= 2)
             printf("Searching for %s at %s...\n", name->c_str(), fn.c_str());
 
+#if LLVMREV < 217625
         bool exists;
-        llvm::error_code code = llvm::sys::fs::exists(joined_path.str(), exists);
+        llvm_error_code code = llvm::sys::fs::exists(joined_path.str(), exists);
         assert(LLVM_SYS_FS_EXISTS_CODE_OKAY(code));
+#else
+        bool exists = llvm::sys::fs::exists(joined_path.str());
+#endif
 
         if (!exists)
             continue;

tools/git_svn_gotorev.py

@@ -81,6 +81,10 @@ if __name__ == "__main__":
             continue
         if "Update-TailCallElim" in patch_fn and svn_rev >= 208017:
             continue
+        if "Update-IntelJITEvents" in patch_fn and svn_rev >= 209989:
+            continue
+        if "stackmap-sections-for-ELF" in patch_fn and svn_rev >= 214538:
+            continue
 
         patch_fn = os.path.abspath(os.path.join(patch_dir, patch_fn))
         code = subprocess.call(["git", "am", patch_fn], cwd=repo)

tools/publicize.cpp

@@ -165,7 +165,11 @@ int main(int argc, char **argv) {
 
     SMDiagnostic Err;
 
+#if LLVMREV < 216466
     std::unique_ptr<Module> M(ParseIRFile(InputFilename, Err, Context));
+#else
+    std::unique_ptr<Module> M(parseIRFile(InputFilename, Err, Context));
+#endif
 
     if (M.get() == 0) {
         Err.print(argv[0], errs());
@@ -187,12 +191,21 @@ int main(int argc, char **argv) {
     if (OutputFilename.empty())
         OutputFilename = "-";
 
+#if LLVMREV < 216393
     std::string ErrorInfo;
     tool_output_file out(OutputFilename.c_str(), ErrorInfo, sys::fs::F_None);
     if (!ErrorInfo.empty()) {
         errs() << ErrorInfo << '\n';
         return 1;
     }
+#else
+    std::error_code EC;
+    tool_output_file out(OutputFilename, EC, sys::fs::F_None);
+    if (EC) {
+        errs() << "error opening file for writing\n";
+        return 1;
+    }
+#endif
 
     WriteBitcodeToFile(M.get(), out.os());