https://github.com/vgvassilev updated https://github.com/llvm/llvm-project/pull/175322
>From 4943f8a1df70a0e83bc5f62cf1db8b27f495a398 Mon Sep 17 00:00:00 2001 From: Vassil Vassilev <[email protected]> Date: Sat, 10 Jan 2026 15:17:04 +0000 Subject: [PATCH] [clang-repl] Simplify the logic around out of process execution. NFC --- clang/include/clang/Interpreter/Interpreter.h | 89 +++++--- clang/lib/Interpreter/IncrementalExecutor.cpp | 3 +- clang/lib/Interpreter/IncrementalExecutor.h | 6 +- clang/lib/Interpreter/Interpreter.cpp | 212 +++++++++--------- clang/tools/clang-repl/ClangRepl.cpp | 31 +-- .../Interpreter/InterpreterExtensionsTest.cpp | 39 ---- .../OutOfProcessInterpreterTests.cpp | 98 ++++---- 7 files changed, 235 insertions(+), 243 deletions(-) diff --git a/clang/include/clang/Interpreter/Interpreter.h b/clang/include/clang/Interpreter/Interpreter.h index 078d70b3b1749..c4ddfb067be0f 100644 --- a/clang/include/clang/Interpreter/Interpreter.h +++ b/clang/include/clang/Interpreter/Interpreter.h @@ -38,7 +38,7 @@ class ThreadSafeContext; namespace clang { namespace driver { -class ToolChain; +class Compilation; } // namespace driver class CompilerInstance; @@ -50,6 +50,8 @@ class IncrementalCUDADeviceParser; /// Create a pre-configured \c CompilerInstance for incremental processing. class IncrementalCompilerBuilder { + using DriverCompilationFn = llvm::Error(const driver::Compilation &); + public: IncrementalCompilerBuilder() {} @@ -68,11 +70,16 @@ class IncrementalCompilerBuilder { // CUDA specific void SetCudaSDK(llvm::StringRef path) { CudaSDKPath = path; }; + // Hand over the compilation. + void SetDriverCompilationCallback(std::function<DriverCompilationFn> C) { + CompilationCB = C; + } + llvm::Expected<std::unique_ptr<CompilerInstance>> CreateCudaHost(); llvm::Expected<std::unique_ptr<CompilerInstance>> CreateCudaDevice(); private: - static llvm::Expected<std::unique_ptr<CompilerInstance>> + llvm::Expected<std::unique_ptr<CompilerInstance>> create(std::string TT, std::vector<const char *> &ClangArgv); llvm::Expected<std::unique_ptr<CompilerInstance>> createCuda(bool device); @@ -82,6 +89,44 @@ class IncrementalCompilerBuilder { llvm::StringRef OffloadArch; llvm::StringRef CudaSDKPath; + + std::optional<std::function<DriverCompilationFn>> CompilationCB; +}; + +// FIXME: Consider deriving from the LLJITBuilder into a common interpreter +// creation configuraion class. +class IncrementalExecutorBuilder { +public: + /// Indicates whether out-of-process JIT execution is enabled. + bool IsOutOfProcess = false; + /// Path to the out-of-process JIT executor. + std::string OOPExecutor = ""; + std::string OOPExecutorConnect = ""; + /// Indicates whether to use shared memory for communication. + bool UseSharedMemory = false; + /// Representing the slab allocation size for memory management in kb. + unsigned SlabAllocateSize = 0; + /// Path to the ORC runtime library. + std::string OrcRuntimePath = ""; + /// PID of the out-of-process JIT executor. + uint32_t ExecutorPID = 0; + /// Custom lambda to be executed inside child process/executor + std::function<void()> CustomizeFork = nullptr; + /// An optional code model to provide to the JITTargetMachineBuilder + std::optional<llvm::CodeModel::Model> CM = std::nullopt; + std::function<llvm::Error(const driver::Compilation &)> + UpdateOrcRuntimePathCB = [this](const driver::Compilation &C) { + return UpdateOrcRuntimePath(C); + }; + + ~IncrementalExecutorBuilder(); + + llvm::Expected<std::unique_ptr<IncrementalExecutor>> + create(llvm::orc::ThreadSafeContext &TSC, + llvm::orc::LLJITBuilder &JITBuilder); + +private: + llvm::Error UpdateOrcRuntimePath(const driver::Compilation &C); }; class IncrementalAction; @@ -120,42 +165,16 @@ class Interpreter { /// An optional compiler instance for CUDA offloading std::unique_ptr<CompilerInstance> DeviceCI; -public: - struct JITConfig { - /// Indicates whether out-of-process JIT execution is enabled. - bool IsOutOfProcess = false; - /// Path to the out-of-process JIT executor. - std::string OOPExecutor = ""; - std::string OOPExecutorConnect = ""; - /// Indicates whether to use shared memory for communication. - bool UseSharedMemory = false; - /// Representing the slab allocation size for memory management in kb. - unsigned SlabAllocateSize = 0; - /// Path to the ORC runtime library. - std::string OrcRuntimePath = ""; - /// PID of the out-of-process JIT executor. - uint32_t ExecutorPID = 0; - /// Custom lambda to be executed inside child process/executor - std::function<void()> CustomizeFork = nullptr; - /// An optional code model to provide to the JITTargetMachineBuilder - std::optional<llvm::CodeModel::Model> CM = std::nullopt; - - JITConfig() - : IsOutOfProcess(false), OOPExecutor(""), OOPExecutorConnect(""), - UseSharedMemory(false), SlabAllocateSize(0), OrcRuntimePath(""), - ExecutorPID(0), CustomizeFork(nullptr), CM(std::nullopt) {} - }; - protected: // Derived classes can use an extended interface of the Interpreter. Interpreter(std::unique_ptr<CompilerInstance> Instance, llvm::Error &Err, std::unique_ptr<llvm::orc::LLJITBuilder> JITBuilder = nullptr, std::unique_ptr<clang::ASTConsumer> Consumer = nullptr, - JITConfig Config = JITConfig()); + std::unique_ptr<IncrementalExecutorBuilder> IEB = nullptr); // Create the internal IncrementalExecutor, or re-create it after calling // ResetExecutor(). - llvm::Error CreateExecutor(JITConfig Config = JITConfig()); + llvm::Error CreateExecutor(); // Delete the internal IncrementalExecutor. This causes a hard shutdown of the // JIT engine. In particular, it doesn't run cleanup or destructors. @@ -164,18 +183,14 @@ class Interpreter { public: virtual ~Interpreter(); static llvm::Expected<std::unique_ptr<Interpreter>> - create(std::unique_ptr<CompilerInstance> CI, JITConfig Config = {}); + create(std::unique_ptr<CompilerInstance> CI, + std::unique_ptr<IncrementalExecutorBuilder> IEB = nullptr); static llvm::Expected<std::unique_ptr<Interpreter>> createWithCUDA(std::unique_ptr<CompilerInstance> CI, std::unique_ptr<CompilerInstance> DCI); static llvm::Expected<std::unique_ptr<llvm::orc::LLJITBuilder>> createLLJITBuilder(std::unique_ptr<llvm::orc::ExecutorProcessControl> EPC, llvm::StringRef OrcRuntimePath); - static llvm::Expected< - std::pair<std::unique_ptr<llvm::orc::LLJITBuilder>, uint32_t>> - outOfProcessJITBuilder(JITConfig Config); - static llvm::Expected<std::string> - getOrcRuntimePath(const driver::ToolChain &TC); const ASTContext &getASTContext() const; ASTContext &getASTContext(); @@ -221,6 +236,8 @@ class Interpreter { std::unique_ptr<llvm::orc::LLJITBuilder> JITBuilder; + std::unique_ptr<IncrementalExecutorBuilder> IncrExecutorBuilder; + /// @} /// @name Value and pretty printing support /// @{ diff --git a/clang/lib/Interpreter/IncrementalExecutor.cpp b/clang/lib/Interpreter/IncrementalExecutor.cpp index 11ab2cfaac17a..f0069c4924f7a 100644 --- a/clang/lib/Interpreter/IncrementalExecutor.cpp +++ b/clang/lib/Interpreter/IncrementalExecutor.cpp @@ -69,9 +69,8 @@ IncrementalExecutor::createDefaultJITBuilder( IncrementalExecutor::IncrementalExecutor(llvm::orc::ThreadSafeContext &TSC, llvm::orc::LLJITBuilder &JITBuilder, - Interpreter::JITConfig Config, llvm::Error &Err) - : TSCtx(TSC), OutOfProcessChildPid(Config.ExecutorPID) { + : TSCtx(TSC) { using namespace llvm::orc; llvm::ErrorAsOutParameter EAO(&Err); diff --git a/clang/lib/Interpreter/IncrementalExecutor.h b/clang/lib/Interpreter/IncrementalExecutor.h index bb1ec33452515..c11c99b9bff8d 100644 --- a/clang/lib/Interpreter/IncrementalExecutor.h +++ b/clang/lib/Interpreter/IncrementalExecutor.h @@ -46,7 +46,6 @@ class IncrementalExecutor { using CtorDtorIterator = llvm::orc::CtorDtorIterator; std::unique_ptr<llvm::orc::LLJIT> Jit; llvm::orc::ThreadSafeContext &TSCtx; - uint32_t OutOfProcessChildPid = -1; llvm::DenseMap<const PartialTranslationUnit *, llvm::orc::ResourceTrackerSP> ResourceTrackers; @@ -58,8 +57,7 @@ class IncrementalExecutor { enum SymbolNameKind { IRName, LinkerName }; IncrementalExecutor(llvm::orc::ThreadSafeContext &TSC, - llvm::orc::LLJITBuilder &JITBuilder, - Interpreter::JITConfig Config, llvm::Error &Err); + llvm::orc::LLJITBuilder &JITBuilder, llvm::Error &Err); virtual ~IncrementalExecutor(); virtual llvm::Error addModule(PartialTranslationUnit &PTU); @@ -71,8 +69,6 @@ class IncrementalExecutor { llvm::orc::LLJIT &GetExecutionEngine() { return *Jit; } - uint32_t getOutOfProcessChildPid() const { return OutOfProcessChildPid; } - static llvm::Expected<std::unique_ptr<llvm::orc::LLJITBuilder>> createDefaultJITBuilder(llvm::orc::JITTargetMachineBuilder JTMB); diff --git a/clang/lib/Interpreter/Interpreter.cpp b/clang/lib/Interpreter/Interpreter.cpp index 1a6f6ea0813b7..0d9f66c4ff07b 100644 --- a/clang/lib/Interpreter/Interpreter.cpp +++ b/clang/lib/Interpreter/Interpreter.cpp @@ -182,6 +182,10 @@ IncrementalCompilerBuilder::create(std::string TT, llvm::ArrayRef<const char *> RF = llvm::ArrayRef(ClangArgv); std::unique_ptr<driver::Compilation> Compilation(Driver.BuildCompilation(RF)); + if (CompilationCB) + if (auto Err = (*CompilationCB)(*Compilation.get())) + return std::move(Err); + if (Compilation->getArgs().hasArg(options::OPT_v)) Compilation->getJobs().Print(llvm::errs(), "\n", /*Quote=*/false); @@ -247,12 +251,80 @@ IncrementalCompilerBuilder::CreateCudaHost() { return IncrementalCompilerBuilder::createCuda(false); } +IncrementalExecutorBuilder::~IncrementalExecutorBuilder() = default; + +llvm::Expected<std::unique_ptr<IncrementalExecutor>> +IncrementalExecutorBuilder::create(llvm::orc::ThreadSafeContext &TSC, + llvm::orc::LLJITBuilder &JITBuilder) { + llvm::Error Err = llvm::Error::success(); + std::unique_ptr<IncrementalExecutor> Executor; +#ifdef __EMSCRIPTEN__ + Executor = std::make_unique<WasmIncrementalExecutor>(TSC); +#else + Executor = std::make_unique<IncrementalExecutor>(TSC, JITBuilder, Err); +#endif + + if (Err) + return std::move(Err); + + return std::move(Executor); +} + +llvm::Error +IncrementalExecutorBuilder::UpdateOrcRuntimePath(const driver::Compilation &C) { + if (!IsOutOfProcess) + return llvm::Error::success(); + + const std::array<const char *, 3> OrcRTLibNames = { + "liborc_rt.a", "liborc_rt_osx.a", "liborc_rt-x86_64.a"}; + + auto findInDir = [&](llvm::StringRef Base) -> std::optional<std::string> { + for (const char *LibName : OrcRTLibNames) { + llvm::SmallString<256> CandidatePath(Base); + llvm::sys::path::append(CandidatePath, LibName); + if (llvm::sys::fs::exists(CandidatePath)) + return std::string(CandidatePath.str()); + } + return std::nullopt; + }; + + const driver::ToolChain &TC = C.getDefaultToolChain(); + std::string SearchedPaths; + if (std::optional<std::string> CompilerRTPath = TC.getCompilerRTPath()) { + if (auto Found = findInDir(*CompilerRTPath)) { + OrcRuntimePath = *Found; + return llvm::Error::success(); + } + SearchedPaths += *CompilerRTPath; + } else { + return llvm::make_error<llvm::StringError>("CompilerRT path not found", + std::error_code()); + } + + if (std::optional<std::string> ResourceDir = TC.getRuntimePath()) { + if (auto Found = findInDir(*ResourceDir)) { + OrcRuntimePath = *Found; + return llvm::Error::success(); + } + if (!SearchedPaths.empty()) + SearchedPaths += "; "; + SearchedPaths += *ResourceDir; + } else { + return llvm::make_error<llvm::StringError>("ResourceDir path not found", + std::error_code()); + } + + return llvm::make_error<llvm::StringError>( + llvm::Twine("OrcRuntime library not found in: ") + SearchedPaths, + std::error_code()); +} + Interpreter::Interpreter(std::unique_ptr<CompilerInstance> Instance, llvm::Error &ErrOut, std::unique_ptr<llvm::orc::LLJITBuilder> JITBuilder, std::unique_ptr<clang::ASTConsumer> Consumer, - JITConfig Config) - : JITBuilder(std::move(JITBuilder)) { + std::unique_ptr<IncrementalExecutorBuilder> IEB) + : JITBuilder(std::move(JITBuilder)), IncrExecutorBuilder(std::move(IEB)) { CI = std::move(Instance); llvm::ErrorAsOutParameter EAO(&ErrOut); auto LLVMCtx = std::make_unique<llvm::LLVMContext>(); @@ -286,7 +358,7 @@ Interpreter::Interpreter(std::unique_ptr<CompilerInstance> Instance, ASTContext &C = CI->getASTContext(); IncrParser->RegisterPTU(C.getTranslationUnitDecl(), std::move(M)); } - if (llvm::Error Err = CreateExecutor(Config)) { + if (llvm::Error Err = CreateExecutor()) { ErrOut = joinErrors(std::move(ErrOut), std::move(Err)); return; } @@ -348,25 +420,28 @@ const char *const Runtimes = R"( EXTERN_C void __clang_Interpreter_SetValueNoAlloc(void *This, void *OutVal, void *OpaqueType, ...); )"; -llvm::Expected<std::pair<std::unique_ptr<llvm::orc::LLJITBuilder>, uint32_t>> -Interpreter::outOfProcessJITBuilder(JITConfig Config) { +static llvm::Expected< + std::pair<std::unique_ptr<llvm::orc::LLJITBuilder>, uint32_t>> +outOfProcessJITBuilder(const IncrementalExecutorBuilder &IncrExecutorBuilder) { std::unique_ptr<llvm::orc::ExecutorProcessControl> EPC; uint32_t childPid = -1; - if (!Config.OOPExecutor.empty()) { + if (!IncrExecutorBuilder.OOPExecutor.empty()) { // Launch an out-of-process executor locally in a child process. auto ResultOrErr = IncrementalExecutor::launchExecutor( - Config.OOPExecutor, Config.UseSharedMemory, Config.SlabAllocateSize, - Config.CustomizeFork); + IncrExecutorBuilder.OOPExecutor, IncrExecutorBuilder.UseSharedMemory, + IncrExecutorBuilder.SlabAllocateSize, + IncrExecutorBuilder.CustomizeFork); if (!ResultOrErr) return ResultOrErr.takeError(); childPid = ResultOrErr->second; auto EPCOrErr = std::move(ResultOrErr->first); EPC = std::move(EPCOrErr); - } else if (Config.OOPExecutorConnect != "") { + } else if (IncrExecutorBuilder.OOPExecutorConnect != "") { #if LLVM_ON_UNIX && LLVM_ENABLE_THREADS auto EPCOrErr = IncrementalExecutor::connectTCPSocket( - Config.OOPExecutorConnect, Config.UseSharedMemory, - Config.SlabAllocateSize); + IncrExecutorBuilder.OOPExecutorConnect, + IncrExecutorBuilder.UseSharedMemory, + IncrExecutorBuilder.SlabAllocateSize); if (!EPCOrErr) return EPCOrErr.takeError(); EPC = std::move(*EPCOrErr); @@ -380,7 +455,7 @@ Interpreter::outOfProcessJITBuilder(JITConfig Config) { std::unique_ptr<llvm::orc::LLJITBuilder> JB; if (EPC) { auto JBOrErr = clang::Interpreter::createLLJITBuilder( - std::move(EPC), Config.OrcRuntimePath); + std::move(EPC), IncrExecutorBuilder.OrcRuntimePath); if (!JBOrErr) return JBOrErr.takeError(); JB = std::move(*JBOrErr); @@ -389,84 +464,14 @@ Interpreter::outOfProcessJITBuilder(JITConfig Config) { return std::make_pair(std::move(JB), childPid); } -llvm::Expected<std::string> -Interpreter::getOrcRuntimePath(const driver::ToolChain &TC) { - const std::array<const char *, 3> OrcRTLibNames = { - "liborc_rt.a", "liborc_rt_osx.a", "liborc_rt-x86_64.a"}; - - auto findInDir = [&](llvm::StringRef Base) -> std::optional<std::string> { - for (const char *LibName : OrcRTLibNames) { - llvm::SmallString<256> CandidatePath(Base); - llvm::sys::path::append(CandidatePath, LibName); - if (llvm::sys::fs::exists(CandidatePath)) - return std::string(CandidatePath.str()); - } - return std::nullopt; - }; - - std::string SearchedPaths; - - if (std::optional<std::string> CompilerRTPath = TC.getCompilerRTPath()) { - if (auto Found = findInDir(*CompilerRTPath)) - return *Found; - SearchedPaths += *CompilerRTPath; - } else { - return llvm::make_error<llvm::StringError>("CompilerRT path not found", - std::error_code()); - } - - if (std::optional<std::string> ResourceDir = TC.getRuntimePath()) { - if (auto Found = findInDir(*ResourceDir)) - return *Found; - if (!SearchedPaths.empty()) - SearchedPaths += "; "; - SearchedPaths += *ResourceDir; - } else { - return llvm::make_error<llvm::StringError>("ResourceDir path not found", - std::error_code()); - } - - return llvm::make_error<llvm::StringError>( - llvm::Twine("OrcRuntime library not found in: ") + SearchedPaths, - std::error_code()); -} - -llvm::Expected<std::unique_ptr<Interpreter>> -Interpreter::create(std::unique_ptr<CompilerInstance> CI, JITConfig Config) { - - if (Config.IsOutOfProcess) { - const TargetInfo &TI = CI->getTarget(); - const llvm::Triple &Triple = TI.getTriple(); - - DiagnosticsEngine &Diags = CI->getDiagnostics(); - std::string BinaryName = llvm::sys::fs::getMainExecutable(nullptr, nullptr); - driver::Driver Driver(BinaryName, Triple.str(), Diags); - // Need fake args to get the driver to create a compilation. - std::vector<const char *> Args = {"clang", "--version"}; - std::unique_ptr<clang::driver::Compilation> C( - Driver.BuildCompilation(Args)); - if (!C) { - return llvm::make_error<llvm::StringError>( - "Failed to create driver compilation for out-of-process JIT", - std::error_code()); - } - if (Config.OrcRuntimePath == "") { - const clang::driver::ToolChain &TC = C->getDefaultToolChain(); - - auto OrcRuntimePathOrErr = getOrcRuntimePath(TC); - if (!OrcRuntimePathOrErr) { - return OrcRuntimePathOrErr.takeError(); - } - - Config.OrcRuntimePath = *OrcRuntimePathOrErr; - } - } - +llvm::Expected<std::unique_ptr<Interpreter>> Interpreter::create( + std::unique_ptr<CompilerInstance> CI, + std::unique_ptr<IncrementalExecutorBuilder> IEB /*=nullptr*/) { llvm::Error Err = llvm::Error::success(); std::unique_ptr<llvm::orc::LLJITBuilder> JB; auto Interp = std::unique_ptr<Interpreter>(new Interpreter( - std::move(CI), Err, std::move(JB), /*Consumer=*/nullptr, Config)); + std::move(CI), Err, std::move(JB), /*Consumer=*/nullptr, std::move(IEB))); if (auto E = std::move(Err)) return std::move(E); @@ -560,7 +565,7 @@ size_t Interpreter::getEffectivePTUSize() const { uint32_t Interpreter::getOutOfProcessExecutorPID() const { if (IncrExecutor) - return IncrExecutor->getOutOfProcessChildPid(); + return IncrExecutorBuilder->ExecutorPID; return -1; } @@ -627,7 +632,7 @@ Interpreter::createLLJITBuilder( return std::move(*JB); } -llvm::Error Interpreter::CreateExecutor(JITConfig Config) { +llvm::Error Interpreter::CreateExecutor() { if (IncrExecutor) return llvm::make_error<llvm::StringError>("Operation failed. " "Execution engine exists", @@ -641,13 +646,16 @@ llvm::Error Interpreter::CreateExecutor(JITConfig Config) { llvm::Triple TargetTriple(TT); bool IsWindowsTarget = TargetTriple.isOSWindows(); - if (!IsWindowsTarget && Config.IsOutOfProcess) { + if (!IncrExecutorBuilder) + IncrExecutorBuilder = std::make_unique<IncrementalExecutorBuilder>(); + + if (!IsWindowsTarget && IncrExecutorBuilder->IsOutOfProcess) { if (!JITBuilder) { - auto ResOrErr = outOfProcessJITBuilder(Config); + auto ResOrErr = outOfProcessJITBuilder(*IncrExecutorBuilder); if (!ResOrErr) return ResOrErr.takeError(); JITBuilder = std::move(ResOrErr->first); - Config.ExecutorPID = ResOrErr->second; + IncrExecutorBuilder->ExecutorPID = ResOrErr->second; } if (!JITBuilder) return llvm::make_error<llvm::StringError>( @@ -659,29 +667,19 @@ llvm::Error Interpreter::CreateExecutor(JITConfig Config) { auto JTMB = createJITTargetMachineBuilder(TT); if (!JTMB) return JTMB.takeError(); - if (Config.CM) - JTMB->setCodeModel(Config.CM); + if (IncrExecutorBuilder->CM) + JTMB->setCodeModel(IncrExecutorBuilder->CM); auto JB = IncrementalExecutor::createDefaultJITBuilder(std::move(*JTMB)); if (!JB) return JB.takeError(); JITBuilder = std::move(*JB); } - llvm::Error Err = llvm::Error::success(); - - // Fix: Declare Executor as the appropriate unique_ptr type - std::unique_ptr<IncrementalExecutor> Executor; - -#ifdef __EMSCRIPTEN__ - Executor = std::make_unique<WasmIncrementalExecutor>(*TSCtx); -#else - Executor = - std::make_unique<IncrementalExecutor>(*TSCtx, *JITBuilder, Config, Err); -#endif - if (!Err) - IncrExecutor = std::move(Executor); + auto ExecutorOrErr = IncrExecutorBuilder->create(*TSCtx, *JITBuilder); + if (ExecutorOrErr) + IncrExecutor = std::move(*ExecutorOrErr); - return Err; + return ExecutorOrErr.takeError(); } void Interpreter::ResetExecutor() { IncrExecutor.reset(); } diff --git a/clang/tools/clang-repl/ClangRepl.cpp b/clang/tools/clang-repl/ClangRepl.cpp index 066f526cba9ae..e94749555ad1a 100644 --- a/clang/tools/clang-repl/ClangRepl.cpp +++ b/clang/tools/clang-repl/ClangRepl.cpp @@ -291,6 +291,22 @@ int main(int argc, const char **argv) { clang::IncrementalCompilerBuilder CB; CB.SetCompilerArgs(ClangArgv); + auto IEB = std::make_unique<clang::IncrementalExecutorBuilder>(); + IEB->IsOutOfProcess = !OOPExecutor.empty() || !OOPExecutorConnect.empty(); + IEB->OOPExecutor = OOPExecutor; + if (!OrcRuntimePath.empty()) + IEB->OrcRuntimePath = OrcRuntimePath; + else + CB.SetDriverCompilationCallback(IEB->UpdateOrcRuntimePathCB); + + auto SizeOrErr = getSlabAllocSize(SlabAllocateSizeString); + if (!SizeOrErr) { + llvm::logAllUnhandledErrors(SizeOrErr.takeError(), llvm::errs(), "error: "); + return EXIT_FAILURE; + } + IEB->SlabAllocateSize = *SizeOrErr; + IEB->UseSharedMemory = UseSharedMemory; + std::unique_ptr<clang::CompilerInstance> DeviceCI; if (CudaEnabled) { if (!CudaPath.empty()) @@ -306,18 +322,6 @@ int main(int argc, const char **argv) { ExitOnErr(sanitizeOopArguments(argv[0])); - clang::Interpreter::JITConfig Config; - Config.IsOutOfProcess = !OOPExecutor.empty() || !OOPExecutorConnect.empty(); - Config.OOPExecutor = OOPExecutor; - Config.OrcRuntimePath = OrcRuntimePath; - auto SizeOrErr = getSlabAllocSize(SlabAllocateSizeString); - if (!SizeOrErr) { - llvm::logAllUnhandledErrors(SizeOrErr.takeError(), llvm::errs(), "error: "); - return EXIT_FAILURE; - } - Config.SlabAllocateSize = *SizeOrErr; - Config.UseSharedMemory = UseSharedMemory; - // FIXME: Investigate if we could use runToolOnCodeWithArgs from tooling. It // can replace the boilerplate code for creation of the compiler instance. std::unique_ptr<clang::CompilerInstance> CI; @@ -350,7 +354,8 @@ int main(int argc, const char **argv) { ExitOnErr(Interp->LoadDynamicLibrary(CudaRuntimeLibPath.c_str())); } } else { - Interp = ExitOnErr(clang::Interpreter::create(std::move(CI), Config)); + Interp = + ExitOnErr(clang::Interpreter::create(std::move(CI), std::move(IEB))); } bool HasError = false; diff --git a/clang/unittests/Interpreter/InterpreterExtensionsTest.cpp b/clang/unittests/Interpreter/InterpreterExtensionsTest.cpp index f50f6e320776d..07a4a224f16bc 100644 --- a/clang/unittests/Interpreter/InterpreterExtensionsTest.cpp +++ b/clang/unittests/Interpreter/InterpreterExtensionsTest.cpp @@ -65,45 +65,6 @@ class InterpreterExtensionsTest : public InterpreterTestBase { } }; -struct OutOfProcInterpreter : public Interpreter { - OutOfProcInterpreter( - std::unique_ptr<CompilerInstance> CI, llvm::Error &ErrOut, - std::unique_ptr<clang::ASTConsumer> Consumer, - std::unique_ptr<llvm::orc::LLJITBuilder> JITBuilder = nullptr) - : Interpreter(std::move(CI), ErrOut, std::move(JITBuilder), - std::move(Consumer)) {} -}; - -TEST_F(InterpreterExtensionsTest, FindRuntimeInterface) { -// FIXME : WebAssembly doesn't currently support Jit (see -// https: // github.com/llvm/llvm-project/pull/150977#discussion_r2237521095). -// so this check of HostSupportsJIT has been skipped -// over until support is added, and HostSupportsJIT can return true. -#ifndef __EMSCRIPTEN__ - if (!HostSupportsJIT()) - GTEST_SKIP(); -#endif - clang::IncrementalCompilerBuilder CB; - llvm::Error ErrOut = llvm::Error::success(); - auto CI = cantFail(CB.CreateCpp()); - // Do not attach the default consumer which is specialized for in-process. - class NoopConsumer : public ASTConsumer {}; - std::unique_ptr<ASTConsumer> C = std::make_unique<NoopConsumer>(); - OutOfProcInterpreter I(std::move(CI), ErrOut, std::move(C), - /*JITBuilder=*/nullptr); - cantFail(std::move(ErrOut)); - cantFail(I.Parse("int a = 1; a")); - cantFail(I.Parse("int b = 2; b")); - cantFail(I.Parse("int c = 3; c")); - - // Make sure no clang::Value logic is attached by the Interpreter. - Value V1; - llvm::cantFail(I.ParseAndExecute("int x = 42;")); - llvm::cantFail(I.ParseAndExecute("x", &V1)); - EXPECT_FALSE(V1.isValid()); - EXPECT_FALSE(V1.hasValue()); -} - class CustomJBInterpreter : public Interpreter { using CustomJITBuilderCreatorFunction = std::function<llvm::Expected<std::unique_ptr<llvm::orc::LLJITBuilder>>()>; diff --git a/clang/unittests/Interpreter/OutOfProcessInterpreterTests.cpp b/clang/unittests/Interpreter/OutOfProcessInterpreterTests.cpp index 704ddc37e642e..d33005244d8da 100644 --- a/clang/unittests/Interpreter/OutOfProcessInterpreterTests.cpp +++ b/clang/unittests/Interpreter/OutOfProcessInterpreterTests.cpp @@ -27,6 +27,7 @@ #include "llvm/TargetParser/Host.h" #include "gmock/gmock.h" #include "gtest/gtest.h" + #include <memory> #include <signal.h> #include <sstream> @@ -101,47 +102,34 @@ static std::string getExecutorPath() { return ExecutorPath.str().str(); } -static std::string getOrcRuntimePath() { - llvm::SmallString<256> RuntimePath(llvm::sys::fs::getMainExecutable( - nullptr, reinterpret_cast<void *>(&getOrcRuntimePath))); - removePathComponent(5, RuntimePath); - llvm::sys::path::append(RuntimePath, CLANG_INSTALL_LIBDIR_BASENAME, "clang", - CLANG_VERSION_MAJOR_STRING, "lib"); - - llvm::Triple SystemTriple(llvm::sys::getProcessTriple()); - if (SystemTriple.isOSBinFormatMachO()) { - llvm::sys::path::append(RuntimePath, "darwin", "liborc_rt_osx.a"); - } else if (SystemTriple.isOSBinFormatELF()) { - llvm::sys::path::append(RuntimePath, "x86_64-unknown-linux-gnu", - "liborc_rt.a"); - } - return RuntimePath.str().str(); -} +struct OutOfProcessInterpreterInfo { + std::string OrcRuntimePath; + std::unique_ptr<Interpreter> Interpreter; +}; -static std::unique_ptr<Interpreter> +static OutOfProcessInterpreterInfo createInterpreterWithRemoteExecution(std::shared_ptr<IOContext> io_ctx, const Args &ExtraArgs = {}) { Args ClangArgs = {"-Xclang", "-emit-llvm-only"}; llvm::append_range(ClangArgs, ExtraArgs); - auto CB = clang::IncrementalCompilerBuilder(); - CB.SetCompilerArgs(ClangArgs); - auto CI = cantFail(CB.CreateCpp()); - clang::Interpreter::JITConfig Config; + auto Config = std::make_unique<IncrementalExecutorBuilder>(); llvm::Triple SystemTriple(llvm::sys::getProcessTriple()); if (SystemTriple.isOSBinFormatELF() || SystemTriple.isOSBinFormatMachO()) { - Config.IsOutOfProcess = true; - Config.OOPExecutor = getExecutorPath(); - Config.UseSharedMemory = false; - Config.SlabAllocateSize = 0; - Config.OrcRuntimePath = getOrcRuntimePath(); - + Config->IsOutOfProcess = true; + Config->OOPExecutor = getExecutorPath(); + Config->UseSharedMemory = false; + Config->SlabAllocateSize = 0; + + // Capture the raw file descriptors by value explicitly. This lambda will + // be invoked in the child process after fork(), so capturing the fd ints is + // safe and avoids capturing FILE* pointers or outer 'this'. int stdin_fd = fileno(io_ctx->stdin_file.get()); int stdout_fd = fileno(io_ctx->stdout_file.get()); int stderr_fd = fileno(io_ctx->stderr_file.get()); - Config.CustomizeFork = [=] { + Config->CustomizeFork = [stdin_fd, stdout_fd, stderr_fd]() { auto redirect = [](int from, int to) { if (from != to) { dup2(from, to); @@ -153,15 +141,21 @@ createInterpreterWithRemoteExecution(std::shared_ptr<IOContext> io_ctx, redirect(stdout_fd, STDOUT_FILENO); redirect(stderr_fd, STDERR_FILENO); + // Unbuffer the stdio in the child; useful for deterministic tests. setvbuf(stdout, nullptr, _IONBF, 0); setvbuf(stderr, nullptr, _IONBF, 0); + // Helpful marker for the unit-test to assert that fork customization ran. printf("CustomizeFork executed\n"); fflush(stdout); }; } - - return cantFail(clang::Interpreter::create(std::move(CI), Config)); + auto CB = IncrementalCompilerBuilder(); + CB.SetCompilerArgs(ClangArgs); + CB.SetDriverCompilationCallback(Config->UpdateOrcRuntimePathCB); + auto CI = cantFail(CB.CreateCpp()); + return {Config->OrcRuntimePath, + cantFail(Interpreter::create(std::move(CI), std::move(Config)))}; } static size_t DeclsSize(TranslationUnitDecl *PTUDecl) { @@ -172,20 +166,19 @@ TEST_F(InterpreterTestBase, SanityWithRemoteExecution) { if (!HostSupportsJIT()) GTEST_SKIP(); - std::string OrcRuntimePath = getOrcRuntimePath(); - std::string ExecutorPath = getExecutorPath(); - - if (!llvm::sys::fs::exists(OrcRuntimePath) || - !llvm::sys::fs::exists(ExecutorPath)) - GTEST_SKIP(); - auto io_ctx = std::make_shared<IOContext>(); ASSERT_TRUE(io_ctx->initializeTempFiles()); - std::unique_ptr<Interpreter> Interp = + OutOfProcessInterpreterInfo Info = createInterpreterWithRemoteExecution(io_ctx); + Interpreter *Interp = Info.Interpreter.get(); ASSERT_TRUE(Interp); + std::string ExecutorPath = getExecutorPath(); + if (!llvm::sys::fs::exists(Info.OrcRuntimePath) || + !llvm::sys::fs::exists(ExecutorPath)) + GTEST_SKIP(); + using PTU = PartialTranslationUnit; PTU &R1(cantFail(Interp->Parse("void g(); void g() {}"))); EXPECT_EQ(2U, DeclsSize(R1.TUPart)); @@ -196,8 +189,31 @@ TEST_F(InterpreterTestBase, SanityWithRemoteExecution) { std::string captured_stdout = io_ctx->readStdoutContent(); std::string captured_stderr = io_ctx->readStderrContent(); - EXPECT_TRUE(captured_stdout.find("CustomizeFork executed") != - std::string::npos); + EXPECT_NE(std::string::npos, captured_stdout.find("CustomizeFork executed")); +} + +TEST_F(InterpreterTestBase, FindRuntimeInterface) { + if (!HostSupportsJIT()) + GTEST_SKIP(); + + // make a fresh io context for this test + auto io_ctx = std::make_shared<IOContext>(); + ASSERT_TRUE(io_ctx->initializeTempFiles()); + + OutOfProcessInterpreterInfo I = createInterpreterWithRemoteExecution(io_ctx); + ASSERT_TRUE(I.Interpreter); + + // FIXME: Not yet supported. + // cantFail(I->Parse("int a = 1; a")); + // cantFail(I->Parse("int b = 2; b")); + // cantFail(I->Parse("int c = 3; c")); + + // // Make sure no clang::Value logic is attached by the Interpreter. + // Value V1; + // llvm::cantFail(I->ParseAndExecute("int x = 42;")); + // llvm::cantFail(I->ParseAndExecute("x", &V1)); + // EXPECT_FALSE(V1.isValid()); + // EXPECT_FALSE(V1.hasValue()); } -} // end anonymous namespace \ No newline at end of file +} // end anonymous namespace _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
