https://github.com/ayartsev7 updated https://github.com/llvm/llvm-project/pull/236610
>From ef51ca08203e2f3bcaaf8ee5874ee71ed9aa6c79 Mon Sep 17 00:00:00 2001 From: Anton Yartsev <[email protected]> Date: Thu, 29 Oct 2026 02:55:09 +0300 Subject: [PATCH 1/7] ThinLTO: read the index and imported modules through the VFS --- clang/lib/CodeGen/BackendUtil.cpp | 8 ++++---- llvm/include/llvm/Bitcode/BitcodeReader.h | 10 ++++++++-- llvm/include/llvm/LTO/LTOBackend.h | 10 ++++++++-- llvm/lib/Bitcode/Reader/BitcodeReader.cpp | 10 ++++++++-- llvm/lib/LTO/LTOBackend.cpp | 8 ++++++-- 5 files changed, 34 insertions(+), 12 deletions(-) diff --git a/clang/lib/CodeGen/BackendUtil.cpp b/clang/lib/CodeGen/BackendUtil.cpp index fe1454026f569d..6eb1ca23890202 100644 --- a/clang/lib/CodeGen/BackendUtil.cpp +++ b/clang/lib/CodeGen/BackendUtil.cpp @@ -1335,7 +1335,7 @@ static void runThinLTOBackend(CompilerInstance &CI, ModuleSummaryIndex *CombinedIndex, llvm::Module *M, std::unique_ptr<raw_pwrite_stream> OS, std::string SampleProfile, std::string ProfileRemapping, - BackendAction Action) { + BackendAction Action, llvm::vfs::FileSystem &FS) { DiagnosticsEngine &Diags = CI.getDiagnostics(); const auto &CGOpts = CI.getCodeGenOpts(); const auto &TOpts = CI.getTargetOpts(); @@ -1446,7 +1446,7 @@ runThinLTOBackend(CompilerInstance &CI, ModuleSummaryIndex *CombinedIndex, Conf, -1, AddStream, *M, *CombinedIndex, ImportList, ModuleToDefinedGVSummaries[M->getModuleIdentifier()], /*ModuleMap=*/nullptr, Conf.CodeGenOnly, /*BitcodeLibFuncs=*/{}, - /*IRAddStream=*/nullptr, CGOpts.CmdArgs)) { + /*IRAddStream=*/nullptr, CGOpts.CmdArgs, &FS)) { handleAllErrors(std::move(E), [&](ErrorInfoBase &EIB) { errs() << "Error running ThinLTO backend: " << EIB.message() << '\n'; }); @@ -1556,7 +1556,7 @@ void clang::emitBackendOutput(CompilerInstance &CI, CodeGenOptions &CGOpts, std::unique_ptr<ModuleSummaryIndex> CombinedIndex; if (Error E = llvm::getModuleSummaryIndexForFile( CGOpts.ThinLTOIndexFile, - /*IgnoreEmptyThinLTOIndexFile*/ true) + /*IgnoreEmptyThinLTOIndexFile*/ true, VFS.get()) .moveInto(CombinedIndex)) { logAllUnhandledErrors(std::move(E), errs(), "Error loading index file '" + @@ -1571,7 +1571,7 @@ void clang::emitBackendOutput(CompilerInstance &CI, CodeGenOptions &CGOpts, if (!CombinedIndex->skipModuleByDistributedBackend()) { runThinLTOBackend(CI, CombinedIndex.get(), M, std::move(OS), CGOpts.SampleProfileFile, CGOpts.ProfileRemappingFile, - Action); + Action, *VFS); return; } // Distributed indexing detected that nothing from the module is needed diff --git a/llvm/include/llvm/Bitcode/BitcodeReader.h b/llvm/include/llvm/Bitcode/BitcodeReader.h index df30463f556197..32d00fb05ef653 100644 --- a/llvm/include/llvm/Bitcode/BitcodeReader.h +++ b/llvm/include/llvm/Bitcode/BitcodeReader.h @@ -39,6 +39,10 @@ class Type; class Value; struct ValueInfo; +namespace vfs { +class FileSystem; +} // namespace vfs + // Callback to override the data layout string of an imported bitcode module. // The first argument is the target triple, the second argument the data layout // string from the input, or a default string. It will be used if the callback @@ -249,10 +253,12 @@ struct ParserCallbacks { /// Parse the module summary index out of an IR file and return the module /// summary index object if found, or an empty summary if not. If Path refers /// to an empty file and IgnoreEmptyThinLTOIndexFile is true, then - /// this function will return nullptr. + /// this function will return nullptr. If FS is not nullptr, the file is read + /// through it instead of the real file system. LLVM_ABI Expected<std::unique_ptr<ModuleSummaryIndex>> getModuleSummaryIndexForFile(StringRef Path, - bool IgnoreEmptyThinLTOIndexFile = false); + bool IgnoreEmptyThinLTOIndexFile = false, + vfs::FileSystem *FS = nullptr); /// isBitcodeWrapper - Return true if the given bytes are the magic bytes /// for an LLVM IR bitcode wrapper. diff --git a/llvm/include/llvm/LTO/LTOBackend.h b/llvm/include/llvm/LTO/LTOBackend.h index 4bb38529ec7540..b623cafb9e6f3b 100644 --- a/llvm/include/llvm/LTO/LTOBackend.h +++ b/llvm/include/llvm/LTO/LTOBackend.h @@ -32,6 +32,10 @@ class Error; class Module; class Target; +namespace vfs { +class FileSystem; +} // namespace vfs + namespace lto { /// Runs middle-end LTO optimizations on \p Mod. @@ -57,7 +61,8 @@ LLVM_ABI Error backend(const Config &C, AddStreamFn AddStream, /// one source module will be kept open at the most. If \p CodeGenOnly is true, /// the backend will skip optimization and only perform code generation. If /// \p IRAddStream is not nullptr, it will be called just before code generation -/// to serialize the optimized IR. +/// to serialize the optimized IR. If \p FS is not nullptr, module files to be +/// imported on demand are read through it instead of the real file system. LLVM_ABI Error thinBackend( const Config &C, unsigned Task, AddStreamFn AddStream, Module &M, const ModuleSummaryIndex &CombinedIndex, @@ -65,7 +70,8 @@ LLVM_ABI Error thinBackend( const GVSummaryMapTy &DefinedGlobals, MapVector<StringRef, BitcodeModule> *ModuleMap, bool CodeGenOnly, ArrayRef<StringRef> BitcodeLibFuncs, AddStreamFn IRAddStream = nullptr, - const std::vector<uint8_t> &CmdArgs = std::vector<uint8_t>()); + const std::vector<uint8_t> &CmdArgs = std::vector<uint8_t>(), + vfs::FileSystem *FS = nullptr); LLVM_ABI Error finalizeOptimizationRemarks(LLVMRemarkFileHandle DiagOutputFile); diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp index 0535d89057671c..b72209f9c29bd2 100644 --- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp +++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp @@ -74,6 +74,7 @@ #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/ModRef.h" #include "llvm/Support/SwapByteOrder.h" +#include "llvm/Support/VirtualFileSystem.h" #include "llvm/Support/raw_ostream.h" #include "llvm/TargetParser/Triple.h" #include <algorithm> @@ -9133,9 +9134,14 @@ Expected<BitcodeLTOInfo> llvm::getBitcodeLTOInfo(MemoryBufferRef Buffer) { Expected<std::unique_ptr<ModuleSummaryIndex>> llvm::getModuleSummaryIndexForFile(StringRef Path, - bool IgnoreEmptyThinLTOIndexFile) { + bool IgnoreEmptyThinLTOIndexFile, + vfs::FileSystem *FS) { ErrorOr<std::unique_ptr<MemoryBuffer>> FileOrErr = - MemoryBuffer::getFileOrSTDIN(Path); + FS && Path != "-" + ? FS->getBufferForFile(Path, /*FileSize=*/-1, + /*RequiresNullTerminator=*/true, + /*IsVolatile=*/false, /*IsText=*/false) + : MemoryBuffer::getFileOrSTDIN(Path); if (!FileOrErr) return errorCodeToError(FileOrErr.getError()); if (IgnoreEmptyThinLTOIndexFile && !(*FileOrErr)->getBufferSize()) diff --git a/llvm/lib/LTO/LTOBackend.cpp b/llvm/lib/LTO/LTOBackend.cpp index ed431be770f593..d06d10f6013e19 100644 --- a/llvm/lib/LTO/LTOBackend.cpp +++ b/llvm/lib/LTO/LTOBackend.cpp @@ -660,7 +660,8 @@ Error lto::thinBackend(const Config &Conf, unsigned Task, AddStreamFn AddStream, MapVector<StringRef, BitcodeModule> *ModuleMap, bool CodeGenOnly, ArrayRef<StringRef> BitcodeLibFuncs, AddStreamFn IRAddStream, - const std::vector<uint8_t> &CmdArgs) { + const std::vector<uint8_t> &CmdArgs, + vfs::FileSystem *FS) { llvm::TimeTraceScope timeScope("Thin backend", Mod.getModuleIdentifier()); Expected<const Target *> TOrErr = initAndLookupTarget(Conf, Mod); if (!TOrErr) @@ -750,7 +751,10 @@ Error lto::thinBackend(const Config &Conf, unsigned Task, AddStreamFn AddStream, } ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> MBOrErr = - llvm::MemoryBuffer::getFile(Identifier); + FS ? FS->getBufferForFile(Identifier, /*FileSize=*/-1, + /*RequiresNullTerminator=*/true, + /*IsVolatile=*/false, /*IsText=*/false) + : llvm::MemoryBuffer::getFile(Identifier); if (!MBOrErr) return Expected<std::unique_ptr<llvm::Module>>(make_error<StringError>( Twine("Error loading imported file ") + Identifier + " : ", >From 5223a36bc651c370f51c710279e77d3453769f3c Mon Sep 17 00:00:00 2001 From: Anton Yartsev <[email protected]> Date: Thu, 29 Oct 2026 02:55:10 +0300 Subject: [PATCH 2/7] Apply VFS overlays so the driver sees the same files as -cc1 --- clang/lib/Driver/Driver.cpp | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp index e43bd6b1049b81..e72bceaabca249 100644 --- a/clang/lib/Driver/Driver.cpp +++ b/clang/lib/Driver/Driver.cpp @@ -1589,6 +1589,22 @@ Compilation *Driver::BuildCompilation(ArrayRef<const char *> ArgList) { if (VFS->setCurrentWorkingDirectory(WD->getValue())) Diag(diag::err_drv_unable_to_set_working_directory) << WD->getValue(); + // Apply VFS overlays so the driver sees the same files as -cc1. + for (const Arg *A : + Args.filtered(options::OPT_ivfsoverlay, options::OPT_vfsoverlay)) { + StringRef File = A->getValue(); + llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> Buffer = + VFS->getBufferForFile(File); + if (!Buffer) + continue; + if (IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS = + llvm::vfs::getVFSFromYAML( + std::move(*Buffer), + [](const llvm::SMDiagnostic &, void *) {}, File, + /*DiagContext=*/nullptr, VFS)) + VFS = std::move(FS); + } + // Check for missing include directories. if (!Diags.isIgnored(diag::warn_missing_include_dirs, SourceLocation())) { for (auto IncludeDir : Args.getAllArgValues(options::OPT_I_Group)) { >From 4a3e6c30f84de9e46305c6655b5124562b0e7e90 Mon Sep 17 00:00:00 2001 From: Anton Yartsev <[email protected]> Date: Thu, 29 Oct 2026 02:55:12 +0300 Subject: [PATCH 3/7] Set module identifier to the command-line path for ThinLTO --- clang/lib/CodeGen/CodeGenAction.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/clang/lib/CodeGen/CodeGenAction.cpp b/clang/lib/CodeGen/CodeGenAction.cpp index c54cc093218c91..aa12396a4d40b2 100644 --- a/clang/lib/CodeGen/CodeGenAction.cpp +++ b/clang/lib/CodeGen/CodeGenAction.cpp @@ -455,6 +455,12 @@ CodeGenAction::loadModule(MemoryBufferRef MBRef) { Bm->parseModule(*VMContext); if (!MOrErr) return DiagErrors(MOrErr.takeError()); + // Set module identifier to the command-line path. + // The ThinLTO index identifies modules by their command-line paths, a VFS + // overlay can change this module's identifier, and then it cannot be found + // in the index. + if (getCurrentInput().isFile() && getCurrentFile() != "-") + (*MOrErr)->setModuleIdentifier(getCurrentFile()); return std::move(*MOrErr); } >From fd69b4a763fb0180c8016693c424671afcbc954c Mon Sep 17 00:00:00 2001 From: Anton Yartsev <[email protected]> Date: Thu, 29 Oct 2026 02:55:13 +0300 Subject: [PATCH 4/7] ThinLTO backend reads its inputs through -vfsoverlay --- clang/test/CodeGen/thinlto-distributed-vfs.ll | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 clang/test/CodeGen/thinlto-distributed-vfs.ll diff --git a/clang/test/CodeGen/thinlto-distributed-vfs.ll b/clang/test/CodeGen/thinlto-distributed-vfs.ll new file mode 100644 index 00000000000000..8d3923d08b3b95 --- /dev/null +++ b/clang/test/CodeGen/thinlto-distributed-vfs.ll @@ -0,0 +1,46 @@ +; REQUIRES: x86-registered-target + +; Test that the ThinLTO backend reads its files through -vfsoverlay if provided. + +; RUN: rm -rf %t && mkdir -p %t/orig %t/remote +; RUN: opt -module-summary -o %t/orig/1.o %s +; RUN: opt -module-summary -o %t/orig/2.o %S/Inputs/thinlto_backend.ll +; RUN: llvm-lto -thinlto -o %t/orig/all %t/orig/1.o %t/orig/2.o +; RUN: mv %t/orig/1.o %t/orig/2.o %t/orig/all.thinlto.bc %t/remote + +; No overlay provided, the index file is not found. +; RUN: %clang -target x86_64-unknown-linux-gnu -O2 -c -x ir %t/remote/1.o \ +; RUN: -fthinlto-index=%t/orig/all.thinlto.bc -o %t/none.o 2>&1 \ +; RUN: | FileCheck %s --check-prefix=CHECK-NONE +; CHECK-NONE: Error loading index file + +; Overlay of the index only, the import is still missing. +; RUN: echo "{'version': 0, 'use-external-names': false, 'roots': [{'type': 'file', 'name': '%/t/orig/all.thinlto.bc', 'external-contents': '%/t/remote/all.thinlto.bc'}]}" > %t/one-file.yaml +; RUN: %clang -target x86_64-unknown-linux-gnu -O2 -c -x ir %t/remote/1.o \ +; RUN: -fthinlto-index=%t/orig/all.thinlto.bc -vfsoverlay %t/one-file.yaml \ +; RUN: -o %t/one-file.o 2>&1 | FileCheck %s --check-prefix=CHECK-ONE +; CHECK-ONE: Error loading imported file + +; Remap the directory so f2 is imported and f1 stays for either external names. +; RUN: echo "{'version': 0, 'use-external-names': false, 'roots': [{'type': 'directory-remap', 'name': '%/t/orig', 'external-contents': '%/t/remote'}]}" > %t/vfs.yaml +; RUN: %clang -target x86_64-unknown-linux-gnu -O2 -c -x ir %t/orig/1.o \ +; RUN: -fthinlto-index=%t/orig/all.thinlto.bc -vfsoverlay %t/vfs.yaml \ +; RUN: -o %t/remapped.o +; RUN: llvm-nm %t/remapped.o | FileCheck %s --check-prefix=CHECK-OBJ +; RUN: echo "{'version': 0, 'use-external-names': true, 'roots': [{'type': 'directory-remap', 'name': '%/t/orig', 'external-contents': '%/t/remote'}]}" > %t/vfs-external.yaml +; RUN: %clang -target x86_64-unknown-linux-gnu -O2 -c -x ir %t/orig/1.o \ +; RUN: -fthinlto-index=%t/orig/all.thinlto.bc -vfsoverlay %t/vfs-external.yaml \ +; RUN: -o %t/remapped-external.o +; RUN: llvm-nm %t/remapped-external.o | FileCheck %s --check-prefix=CHECK-OBJ +; CHECK-OBJ: T f1 +; CHECK-OBJ-NOT: U f2 + +target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +declare void @f2() + +define void @f1() { + call void @f2() + ret void +} >From 9078b235fb1edb1f69123aeed3cfd3b76f3b8c93 Mon Sep 17 00:00:00 2001 From: Anton Yartsev <[email protected]> Date: Thu, 29 Oct 2026 02:55:14 +0300 Subject: [PATCH 5/7] Test that the driver resolves its inputs through -vfsoverlay --- clang/test/Driver/vfsoverlay.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/clang/test/Driver/vfsoverlay.c b/clang/test/Driver/vfsoverlay.c index 694cf623c4476d..152c9c332a4ea6 100644 --- a/clang/test/Driver/vfsoverlay.c +++ b/clang/test/Driver/vfsoverlay.c @@ -6,3 +6,18 @@ // RUN: not %clang -ivfsoverlay foo.h %s 2>&1 | FileCheck -check-prefix=CHECK-MISSING %s // CHECK-MISSING: virtual filesystem overlay file 'foo.h' not found + +// The driver applies overlays. +// RUN: rm -rf %t && mkdir -p %t/real +// RUN: echo "int a;" > %t/real/a.c +// RUN: echo "{'version': 0, 'roots': [{'type': 'directory-remap', 'name': '%/t/virtual', 'external-contents': '%/t/real'}]}" > %t/vfs.yaml +// RUN: not %clang -fsyntax-only %t/virtual/a.c 2>&1 | FileCheck %s --check-prefix=CHECK-NO-OVERLAY +// CHECK-NO-OVERLAY: error: no such file or directory: '{{.*}}a.c' +// RUN: %clang -fsyntax-only -ivfsoverlay %t/vfs.yaml %t/virtual/a.c +// RUN: %clang -fsyntax-only -vfsoverlay %t/vfs.yaml %t/virtual/a.c + +// The driver leaves invalid overlays for -cc1 to diagnose. +// RUN: echo "invalid" > %t/invalid.yaml +// RUN: not %clang -fsyntax-only -ivfsoverlay %t/invalid.yaml %t/real/a.c 2>&1 | FileCheck -check-prefix=CHECK-INVALID %s +// RUN: not %clang -fsyntax-only -vfsoverlay %t/invalid.yaml %t/real/a.c 2>&1 | FileCheck -check-prefix=CHECK-INVALID %s +// CHECK-INVALID: invalid virtual filesystem overlay file '{{.*}}invalid.yaml' >From 144c8787d5e67a740327d760b858be819a904aa3 Mon Sep 17 00:00:00 2001 From: Anton Yartsev <[email protected]> Date: Thu, 29 Oct 2026 02:55:16 +0300 Subject: [PATCH 6/7] clang-format --- clang/lib/Driver/Driver.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp index e72bceaabca249..77486304501e29 100644 --- a/clang/lib/Driver/Driver.cpp +++ b/clang/lib/Driver/Driver.cpp @@ -1599,8 +1599,8 @@ Compilation *Driver::BuildCompilation(ArrayRef<const char *> ArgList) { continue; if (IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS = llvm::vfs::getVFSFromYAML( - std::move(*Buffer), - [](const llvm::SMDiagnostic &, void *) {}, File, + std::move(*Buffer), [](const llvm::SMDiagnostic &, void *) {}, + File, /*DiagContext=*/nullptr, VFS)) VFS = std::move(FS); } >From f6df8261d49f5b0eac47e43ae51dda1fda2b5e71 Mon Sep 17 00:00:00 2001 From: Anton Yartsev <[email protected]> Date: Tue, 3 Nov 2026 02:47:40 +0300 Subject: [PATCH 7/7] Use the requested path as the ThinLTO module identifier --- clang/lib/CodeGen/CodeGenAction.cpp | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/clang/lib/CodeGen/CodeGenAction.cpp b/clang/lib/CodeGen/CodeGenAction.cpp index aa12396a4d40b2..53732709f919a3 100644 --- a/clang/lib/CodeGen/CodeGenAction.cpp +++ b/clang/lib/CodeGen/CodeGenAction.cpp @@ -455,12 +455,16 @@ CodeGenAction::loadModule(MemoryBufferRef MBRef) { Bm->parseModule(*VMContext); if (!MOrErr) return DiagErrors(MOrErr.takeError()); - // Set module identifier to the command-line path. - // The ThinLTO index identifies modules by their command-line paths, a VFS - // overlay can change this module's identifier, and then it cannot be found - // in the index. - if (getCurrentInput().isFile() && getCurrentFile() != "-") - (*MOrErr)->setModuleIdentifier(getCurrentFile()); + // The ThinLTO index identifies this module by the path given on the + // command line. With a VFS overlay that uses 'use-external-names', the + // FileManager renames the main file to its external path (see the FIXME in + // FileManager::getFileRefImpl), and BitcodeReader takes the module + // identifier from it. As a result, the module does not match its index + // entry, and the ThinLTO backend mishandles it. + // We restore the requested name so the module matches its index entry. + if (OptionalFileEntryRef FE = SM.getFileEntryRefForID(SM.getMainFileID())) + if (FE->getName() != FE->getNameAsRequested()) + (*MOrErr)->setModuleIdentifier(FE->getNameAsRequested()); return std::move(*MOrErr); } _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
