On Sunday, February 4, 2018 12:45:50 AM CET Andreas Karlsson wrote: > On 02/02/2018 10:48 AM, Pierre Ducroquet wrote: > > I have successfully built the JIT branch against LLVM 4.0.1 on Debian > > testing. This is not enough for Debian stable (LLVM 3.9 is the latest > > available there), but it's a first step. > > I've split the patch in four files. The first three fix the build issues, > > the last one fixes a runtime issue. > > I think they are small enough to not be a burden for you in your > > developments. But if you don't want to carry these ifdefs right now, I > > maintain them in a branch on a personal git and rebase as frequently as I > > can. > > I tested these patches and while the code built for me and passed the > test suite on Debian testing I have a weird bug where the very first > query fails to JIT while the rest work as they should. I think I need to > dig into LLVM's codebase to see what it is, but can you reproduce this > bug at your machine? > > Code to reproduce: > > SET jit_expressions = true; > SET jit_above_cost = 0; > SELECT 1; > SELECT 1; > > Output: > > postgres=# SELECT 1; > ERROR: failed to jit module > postgres=# SELECT 1; > ?column? > ---------- > 1 > (1 row) > > Config: > > Version: You patches applied on top of > 302b7a284d30fb0e00eb5f0163aa933d4d9bea10 > OS: Debian testing > llvm/clang: 4.0.1-8 > > Andreas
I have fixed the patches, I was wrong on 'guessing' the migration of the API for one function. I have rebuilt the whole patch set. It is still based on 302b7a284d and has been tested with both LLVM 3.9 and 4.0 on Debian testing. Thanks for your feedback !
>From ebf577fc6b1e74ddb2f6b9aef1c2632ab807dd70 Mon Sep 17 00:00:00 2001 From: Pierre <pierre.ducroq...@people-doc.com> Date: Fri, 2 Feb 2018 09:11:55 +0100 Subject: [PATCH 1/6] Add support for LLVM4 in llvmjit.c --- src/backend/lib/llvmjit.c | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/src/backend/lib/llvmjit.c b/src/backend/lib/llvmjit.c index 8e5ba94c98..046cd53ef3 100644 --- a/src/backend/lib/llvmjit.c +++ b/src/backend/lib/llvmjit.c @@ -230,12 +230,19 @@ llvm_get_function(LLVMJitContext *context, const char *funcname) addr = 0; if (LLVMOrcGetSymbolAddressIn(handle->stack, &addr, handle->orc_handle, mangled)) - elog(ERROR, "failed to lookup symbol"); + elog(ERROR, "failed to lookup symbol %s", mangled); if (addr) return (void *) addr; } #endif +#if LLVM_VERSION_MAJOR < 5 + if ((addr = LLVMOrcGetSymbolAddress(llvm_opt0_orc, mangled))) + return (void *) addr; + if ((addr = LLVMOrcGetSymbolAddress(llvm_opt3_orc, mangled))) + return (void *) addr; + elog(ERROR, "failed to lookup symbol %s for %s", mangled, funcname); +#else if (LLVMOrcGetSymbolAddress(llvm_opt0_orc, &addr, mangled)) elog(ERROR, "failed to lookup symbol"); if (addr) @@ -244,7 +251,7 @@ llvm_get_function(LLVMJitContext *context, const char *funcname) elog(ERROR, "failed to lookup symbol"); if (addr) return (void *) addr; - +#endif elog(ERROR, "failed to JIT: %s", funcname); return NULL; @@ -380,11 +387,18 @@ llvm_compile_module(LLVMJitContext *context) * faster instruction selection mechanism is used. */ { - LLVMSharedModuleRef smod; instr_time tb, ta; /* emit the code */ INSTR_TIME_SET_CURRENT(ta); +#if LLVM_VERSION_MAJOR < 5 + orc_handle = LLVMOrcAddEagerlyCompiledIR(compile_orc, context->module, + llvm_resolve_symbol, NULL); + // It seems there is no error return from that function in LLVM < 5. +#else + LLVMSharedModuleRef smod; + + LLVMSharedModuleRef smod; smod = LLVMOrcMakeSharedModule(context->module); if (LLVMOrcAddEagerlyCompiledIR(compile_orc, &orc_handle, smod, llvm_resolve_symbol, NULL)) @@ -392,6 +406,7 @@ llvm_compile_module(LLVMJitContext *context) elog(ERROR, "failed to jit module"); } LLVMOrcDisposeSharedModuleRef(smod); +#endif INSTR_TIME_SET_CURRENT(tb); INSTR_TIME_SUBTRACT(tb, ta); ereport(DEBUG1, (errmsg("time to emit: %.3fs", -- 2.15.1
>From f90c156c9b26eda38d3a1312c3de87ffba50340a Mon Sep 17 00:00:00 2001 From: Pierre <pierre.ducroq...@people-doc.com> Date: Fri, 2 Feb 2018 09:13:40 +0100 Subject: [PATCH 2/6] Add LLVM4 support in llvmjit_error.cpp --- src/backend/lib/llvmjit_error.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/backend/lib/llvmjit_error.cpp b/src/backend/lib/llvmjit_error.cpp index 70cecd114b..04e51b2a31 100644 --- a/src/backend/lib/llvmjit_error.cpp +++ b/src/backend/lib/llvmjit_error.cpp @@ -56,7 +56,9 @@ llvm_enter_fatal_on_oom(void) if (fatal_new_handler_depth == 0) { old_new_handler = std::set_new_handler(fatal_system_new_handler); +#if LLVM_VERSION_MAJOR > 4 llvm::install_bad_alloc_error_handler(fatal_llvm_new_handler); +#endif llvm::install_fatal_error_handler(fatal_llvm_error_handler); } fatal_new_handler_depth++; @@ -72,7 +74,9 @@ llvm_leave_fatal_on_oom(void) if (fatal_new_handler_depth == 0) { std::set_new_handler(old_new_handler); +#if LLVM_VERSION_MAJOR > 4 llvm::remove_bad_alloc_error_handler(); +#endif llvm::remove_fatal_error_handler(); } } @@ -87,7 +91,9 @@ llvm_reset_fatal_on_oom(void) if (fatal_new_handler_depth != 0) { std::set_new_handler(old_new_handler); +#if LLVM_VERSION_MAJOR > 4 llvm::remove_bad_alloc_error_handler(); +#endif llvm::remove_fatal_error_handler(); } fatal_new_handler_depth = 0; -- 2.15.1
>From 5d1270b25a80e8a25097916381f8abe67432c51f Mon Sep 17 00:00:00 2001 From: Pierre <pierre.ducroq...@people-doc.com> Date: Fri, 2 Feb 2018 09:23:56 +0100 Subject: [PATCH 3/6] Add LLVM4 support in llvmjit_inline.cpp --- src/backend/lib/llvmjit_inline.cpp | 36 ++++++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/src/backend/lib/llvmjit_inline.cpp b/src/backend/lib/llvmjit_inline.cpp index 151198547a..8a747cbfc0 100644 --- a/src/backend/lib/llvmjit_inline.cpp +++ b/src/backend/lib/llvmjit_inline.cpp @@ -100,6 +100,13 @@ llvm_inline(LLVMModuleRef M) llvm_execute_inline_plan(mod, globalsToInline.get()); } +#if LLVM_VERSION_MAJOR < 5 +bool operator!(const llvm::ValueInfo &vi) { + return !( (vi.Kind == llvm::ValueInfo::VI_GUID && vi.TheValue.Id) + || (vi.Kind == llvm::ValueInfo::VI_Value && vi.TheValue.GV)); +} +#endif + /* * Build information necessary for inlining external function references in * mod. @@ -146,7 +153,14 @@ llvm_build_inline_plan(llvm::Module *mod) if (threshold == -1) continue; +#if LLVM_VERSION_MAJOR > 4 llvm::ValueInfo funcVI = llvm_index->getValueInfo(funcGUID); +#else + const llvm::const_gvsummary_iterator &I = llvm_index->findGlobalValueSummaryList(funcGUID); + if (I == llvm_index->end()) + continue; + llvm::ValueInfo funcVI = llvm::ValueInfo(I->first); +#endif /* if index doesn't know function, we don't have a body, continue */ if (!funcVI) @@ -157,7 +171,12 @@ llvm_build_inline_plan(llvm::Module *mod) * look up module(s), check if function actually is defined (there * could be hash conflicts). */ +#if LLVM_VERSION_MAJOR > 4 for (const auto &gvs : funcVI.getSummaryList()) +#else + auto it_gvs = llvm_index->findGlobalValueSummaryList(funcVI.getGUID()); + for (const auto &gvs: it_gvs->second) +#endif { const llvm::FunctionSummary *fs; llvm::StringRef modPath = gvs->modulePath(); @@ -318,9 +337,14 @@ llvm_execute_inline_plan(llvm::Module *mod, ImportMapTy *globalsToInline) } +#if LLVM_VERSION_MAJOR > 4 +#define IRMOVE_PARAMS , /*IsPerformingImport=*/false +#else +#define IRMOVE_PARAMS , /*LinkModuleInlineAsm=*/false, /*IsPerformingImport=*/false +#endif if (Mover.move(std::move(importMod), GlobalsToImport.getArrayRef(), - [](llvm::GlobalValue &, llvm::IRMover::ValueAdder) {}, - /*IsPerformingImport=*/false)) + [](llvm::GlobalValue &, llvm::IRMover::ValueAdder) {} + IRMOVE_PARAMS)) elog(ERROR, "function import failed with linker error"); } } @@ -619,9 +643,17 @@ llvm_load_index(void) elog(ERROR, "failed to open %s: %s", subpath, EC.message().c_str()); llvm::MemoryBufferRef ref(*MBOrErr.get().get()); +#if LLVM_VERSION_MAJOR > 4 llvm::Error e = llvm::readModuleSummaryIndex(ref, *index, 0); if (e) elog(ERROR, "could not load summary at %s", subpath); +#else + std::unique_ptr<llvm::ModuleSummaryIndex> subindex = std::move(llvm::getModuleSummaryIndex(ref).get()); + if (!subindex) + elog(ERROR, "could not load summary at %s", subpath); + else + index->mergeFrom(std::move(subindex), 0); +#endif } } -- 2.15.1
>From db4d0a862f0fbf0f802bc6f83f42831240a0dc53 Mon Sep 17 00:00:00 2001 From: Pierre <pierre.ducroq...@people-doc.com> Date: Fri, 2 Feb 2018 10:34:09 +0100 Subject: [PATCH 4/6] Don't emit bitcode depending on an LLVM 5+ function --- src/backend/executor/execExprCompile.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/backend/executor/execExprCompile.c b/src/backend/executor/execExprCompile.c index 4d6304f748..d129ea7828 100644 --- a/src/backend/executor/execExprCompile.c +++ b/src/backend/executor/execExprCompile.c @@ -173,7 +173,11 @@ get_LifetimeEnd(LLVMModuleRef mod) LLVMTypeRef sig; LLVMValueRef fn; LLVMTypeRef param_types[2]; +#if LLVM_VERSION_MAJOR > 4 const char *nm = "llvm.lifetime.end.p0i8"; +#else + const char *nm = "llvm.lifetime.end"; +#endif fn = LLVMGetNamedFunction(mod, nm); if (fn) -- 2.15.1
>From 54e137d21d6ef46604a59c2ec68fb4ac1a08cc58 Mon Sep 17 00:00:00 2001 From: Pierre <pierre.ducroq...@people-doc.com> Date: Fri, 2 Feb 2018 11:29:57 +0100 Subject: [PATCH 6/6] Fix segfault with LLVM 3.9 --- src/backend/lib/llvmjit.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend/lib/llvmjit.c b/src/backend/lib/llvmjit.c index 046cd53ef3..6102b741b0 100644 --- a/src/backend/lib/llvmjit.c +++ b/src/backend/lib/llvmjit.c @@ -459,12 +459,12 @@ llvm_session_initialize(void) cpu = LLVMGetHostCPUName(); llvm_opt0_targetmachine = - LLVMCreateTargetMachine(llvm_targetref, llvm_triple, cpu, NULL, + LLVMCreateTargetMachine(llvm_targetref, llvm_triple, cpu, "", LLVMCodeGenLevelNone, LLVMRelocDefault, LLVMCodeModelJITDefault); llvm_opt3_targetmachine = - LLVMCreateTargetMachine(llvm_targetref, llvm_triple, cpu, NULL, + LLVMCreateTargetMachine(llvm_targetref, llvm_triple, cpu, "", LLVMCodeGenLevelAggressive, LLVMRelocDefault, LLVMCodeModelJITDefault); -- 2.15.1
>From f0e377d93ef0b39f5edd9b609018bfe9083f4f65 Mon Sep 17 00:00:00 2001 From: Pierre <pierre.ducroq...@people-doc.com> Date: Fri, 2 Feb 2018 11:29:45 +0100 Subject: [PATCH 5/6] Fix building with LLVM 3.9 --- src/backend/lib/llvmjit_inline.cpp | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/src/backend/lib/llvmjit_inline.cpp b/src/backend/lib/llvmjit_inline.cpp index 8a747cbfc0..a785261bea 100644 --- a/src/backend/lib/llvmjit_inline.cpp +++ b/src/backend/lib/llvmjit_inline.cpp @@ -37,7 +37,12 @@ extern "C" #include <llvm/ADT/StringSet.h> #include <llvm/ADT/StringMap.h> #include <llvm/Analysis/ModuleSummaryAnalysis.h> +#if LLVM_MAJOR_VERSION > 3 #include <llvm/Bitcode/BitcodeReader.h> +#else +#include "llvm/Bitcode/ReaderWriter.h" +#include "llvm/Support/Error.h" +#endif #include <llvm/IR/CallSite.h> #include <llvm/IR/DebugInfo.h> #include <llvm/IR/IntrinsicInst.h> @@ -100,7 +105,12 @@ llvm_inline(LLVMModuleRef M) llvm_execute_inline_plan(mod, globalsToInline.get()); } -#if LLVM_VERSION_MAJOR < 5 +#if LLVM_VERSION_MAJOR < 4 +bool operator!(const llvm::ValueInfo &vi) { + return !( (vi.Kind == llvm::ValueInfo::VI_GUID && vi.TheValue.Id) + || (vi.Kind == llvm::ValueInfo::VI_Value && vi.TheValue.V)); +} +#elif LLVM_VERSION_MAJOR < 5 bool operator!(const llvm::ValueInfo &vi) { return !( (vi.Kind == llvm::ValueInfo::VI_GUID && vi.TheValue.Id) || (vi.Kind == llvm::ValueInfo::VI_Value && vi.TheValue.GV)); @@ -188,12 +198,15 @@ llvm_build_inline_plan(llvm::Module *mod) funcName.data(), modPath.data()); +// XXX Missing in LLVM < 4.0 ? +#if LLVM_VERSION_MAJOR > 3 if (gvs->notEligibleToImport()) { elog(DEBUG1, "uneligible to import %s due to summary", funcName.data()); continue; } +#endif if ((int) fs->instCount() > threshold) { @@ -339,8 +352,10 @@ llvm_execute_inline_plan(llvm::Module *mod, ImportMapTy *globalsToInline) #if LLVM_VERSION_MAJOR > 4 #define IRMOVE_PARAMS , /*IsPerformingImport=*/false -#else +#elif LLVM_VERSION_MAJOR > 3 #define IRMOVE_PARAMS , /*LinkModuleInlineAsm=*/false, /*IsPerformingImport=*/false +#else +#define IRMOVE_PARAMS #endif if (Mover.move(std::move(importMod), GlobalsToImport.getArrayRef(), [](llvm::GlobalValue &, llvm::IRMover::ValueAdder) {} @@ -648,7 +663,11 @@ llvm_load_index(void) if (e) elog(ERROR, "could not load summary at %s", subpath); #else +#if LLVM_VERSION_MAJOR > 3 std::unique_ptr<llvm::ModuleSummaryIndex> subindex = std::move(llvm::getModuleSummaryIndex(ref).get()); +#else + std::unique_ptr<llvm::ModuleSummaryIndex> subindex = std::move(llvm::getModuleSummaryIndex(ref, [](const llvm::DiagnosticInfo &) {}).get()); +#endif if (!subindex) elog(ERROR, "could not load summary at %s", subpath); else -- 2.15.1