Author: Ben Dunbobbin Date: 2026-01-20T12:55:32Z New Revision: c6418780ab22b0dc64bedc399d640df608d6668c
URL: https://github.com/llvm/llvm-project/commit/c6418780ab22b0dc64bedc399d640df608d6668c DIFF: https://github.com/llvm/llvm-project/commit/c6418780ab22b0dc64bedc399d640df608d6668c.diff LOG: [DTLTO] Add DTLTO-specific LTO input handling time-trace scopes (#175799) Add time-trace scopes to the DTLTO-specific input-handling code to improve observability and debugging. These scopes are tested via LLD, as the primary purpose of this code is to support member files of non-thin archives as DTLTO inputs. `llvm-lto2` does not currently support archives. Adding archive support to `llvm-lto2` solely for testing these scopes does not appear to be worthwhile. As part of this change, the deletion of temporary DTLTO input files has been moved. Cleanup now occurs after LTO has completed, rather than during destruction of the LTO object. This is required since by the time the LTO object is destroyed, time-traces have already been finalized, so no additional trace data can be recorded. Recording time-trace data for temporary file deletion is important, as this has been a source of performance issues in the past and an area where we expect to make further performance improvements if supported by the data. SIE internal tracker: TOOLCHAIN-21021 (cherry picked from commit 31f5be4ce54107ed8adc79d773a7dbf5c7640938) Added: lld/test/ELF/dtlto/timetrace.test Modified: llvm/include/llvm/DTLTO/DTLTO.h llvm/include/llvm/LTO/LTO.h llvm/lib/DTLTO/DTLTO.cpp llvm/lib/LTO/LTO.cpp Removed: ################################################################################ diff --git a/lld/test/ELF/dtlto/timetrace.test b/lld/test/ELF/dtlto/timetrace.test new file mode 100644 index 0000000000000..639ad36f8019f --- /dev/null +++ b/lld/test/ELF/dtlto/timetrace.test @@ -0,0 +1,65 @@ +REQUIRES: x86 + +## Test that DTLTO-specific LTO input file handling time-trace output is +## produced as expected. + +RUN: rm -rf %t && split-file %s %t && cd %t + +RUN: sed 's/@t1/@t2/g' t1.ll > t2.ll + +## Generate ThinLTO bitcode files. +RUN: opt -thinlto-bc t1.ll -o t1.bc +RUN: opt -thinlto-bc t2.ll -o t2.bc + +## Create archives. +RUN: llvm-ar rcs t1.a t1.bc +RUN: llvm-ar rcsT t2.thin.a t2.bc + +## Generate object files for mock.py to return. +RUN: llc t1.ll --filetype=obj -o t1.o +RUN: llc t2.ll --filetype=obj -o t2.o + +## Link and generate a time-trace. +## Note: mock.py doesn't compile; it copies the specified object files to the +## outputs in job order. +RUN: ld.lld --whole-archive t1.a t2.thin.a -o my.elf \ +RUN: --thinlto-distributor=%python \ +RUN: --thinlto-distributor-arg=%llvm_src_root/utils/dtlto/mock.py \ +RUN: --thinlto-distributor-arg=t1.o --thinlto-distributor-arg=t2.o \ +RUN: --time-trace-granularity=0 --time-trace=%t.json +RUN: %python filter_order_and_pprint.py %t.json | FileCheck %s + +## Check that DTLTO add input file events are recorded. +CHECK: "name": "Add input for DTLTO" +CHECK: "name": "Add input for DTLTO" +CHECK: "name": "Remove temporary inputs for DTLTO" +CHECK: "name": "Save input archive member for DTLTO" +CHECK-SAME: "detail": "t1.a(t1.bc at [[#ARCHIVE_OFFSET:]]).1.[[PID:[A-F0-9]+]].o" +CHECK: "name": "Total Add input for DTLTO" +CHECK-SAME: "count": 2, +CHECK: "name": "Total Remove temporary inputs for DTLTO" +CHECK-SAME: "count": 1, +CHECK: "name": "Total Save input archive member for DTLTO" +CHECK-SAME: "count": 1, + +#--- t1.ll +target triple = "x86_64-unknown-linux-gnu" +target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" + +define void @t1() { + ret void +} + +#--- filter_order_and_pprint.py +import json, sys + +data = json.load(open(sys.argv[1], "r", encoding="utf-8")) + +# Get DTLTO events. +events = [e for e in data["traceEvents"] if "DTLTO" in e["name"]] +events.sort(key=lambda e: (e["name"], str(e.get("args", {}).get("detail", "")))) + +# Print an event per line. Ensure 'name' is the first key. +for ev in events: + name = ev.pop("name") + print(json.dumps({"name": name, **ev})) diff --git a/llvm/include/llvm/DTLTO/DTLTO.h b/llvm/include/llvm/DTLTO/DTLTO.h index f9fad743e721c..14f1f5fd00e30 100644 --- a/llvm/include/llvm/DTLTO/DTLTO.h +++ b/llvm/include/llvm/DTLTO/DTLTO.h @@ -16,19 +16,27 @@ namespace llvm { namespace lto { class DTLTO : public LTO { + using Base = LTO; + public: - // Inherit contructors from LTO base class. - using LTO::LTO; - ~DTLTO() { removeTempFiles(); } + // Inherit constructors. + using Base::Base; + ~DTLTO() override = default; + + // Add an input file and prepare it for distribution. + LLVM_ABI Expected<std::shared_ptr<InputFile>> + addInput(std::unique_ptr<InputFile> InputPtr) override; + +protected: + LLVM_ABI llvm::Error handleArchiveInputs() override; + + LLVM_ABI void cleanup() override; private: // Bump allocator for a purpose of saving updated module IDs. BumpPtrAllocator PtrAlloc; StringSaver Saver{PtrAlloc}; - // Removes temporary files. - LLVM_ABI void removeTempFiles(); - // Determines if a file at the given path is a thin archive file. Expected<bool> isThinArchive(const StringRef ArchivePath); @@ -44,17 +52,8 @@ class DTLTO : public LTO { // A cache to avoid repeatedly reading the same archive file. StringMap<bool> ArchiveFiles; - -public: - // Adds the input file to the LTO object's list of input files. - // For archive members, generates a new module ID which is a path to a real - // file on a filesystem. - LLVM_ABI virtual Expected<std::shared_ptr<lto::InputFile>> - addInput(std::unique_ptr<lto::InputFile> InputPtr) override; - - // Entry point for DTLTO archives support. - LLVM_ABI virtual llvm::Error handleArchiveInputs() override; }; + } // namespace lto } // namespace llvm diff --git a/llvm/include/llvm/LTO/LTO.h b/llvm/include/llvm/LTO/LTO.h index ea80a25ac4ed0..cba5cf7eb9e62 100644 --- a/llvm/include/llvm/LTO/LTO.h +++ b/llvm/include/llvm/LTO/LTO.h @@ -445,6 +445,13 @@ class LTO { LLVM_ABI static SmallVector<const char *> getRuntimeLibcallSymbols(const Triple &TT); +protected: + // Called at the start of run(). + virtual Error handleArchiveInputs() { return Error::success(); } + + // Called before returning from run(). + virtual void cleanup() {} + private: Config Conf; @@ -622,8 +629,6 @@ class LTO { addInput(std::unique_ptr<lto::InputFile> InputPtr) { return std::shared_ptr<lto::InputFile>(InputPtr.release()); } - - virtual llvm::Error handleArchiveInputs() { return llvm::Error::success(); } }; /// The resolution for a symbol. The linker must provide a SymbolResolution for diff --git a/llvm/lib/DTLTO/DTLTO.cpp b/llvm/lib/DTLTO/DTLTO.cpp index d41544277dd45..b9a5cd3a062e2 100644 --- a/llvm/lib/DTLTO/DTLTO.cpp +++ b/llvm/lib/DTLTO/DTLTO.cpp @@ -25,6 +25,7 @@ #include "llvm/Support/MemoryBufferRef.h" #include "llvm/Support/Path.h" #include "llvm/Support/Process.h" +#include "llvm/Support/TimeProfiler.h" #include "llvm/Support/raw_ostream.h" #include <iostream> @@ -116,15 +117,6 @@ Expected<bool> lto::DTLTO::isThinArchive(const StringRef ArchivePath) { return IsThin; } -// Removes any temporary regular archive member files that were created during -// processing. -void lto::DTLTO::removeTempFiles() { - for (auto &Input : InputFiles) { - if (Input->isMemberOfArchive()) - sys::fs::remove(Input->getName(), /*IgnoreNonExisting=*/true); - } -} - // This function performs the following tasks: // 1. Adds the input file to the LTO object's list of input files. // 2. For thin archive members, generates a new module ID which is a path to a @@ -133,6 +125,7 @@ void lto::DTLTO::removeTempFiles() { // 4. Updates the bitcode module's identifier. Expected<std::shared_ptr<lto::InputFile>> lto::DTLTO::addInput(std::unique_ptr<lto::InputFile> InputPtr) { + TimeTraceScope TimeScope("Add input for DTLTO"); // Add the input file to the LTO object. InputFiles.emplace_back(InputPtr.release()); @@ -180,6 +173,7 @@ lto::DTLTO::addInput(std::unique_ptr<lto::InputFile> InputPtr) { Error lto::DTLTO::saveInputArchiveMember(lto::InputFile *Input) { StringRef ModuleId = Input->getName(); if (Input->isMemberOfArchive()) { + TimeTraceScope TimeScope("Save input archive member for DTLTO", ModuleId); MemoryBufferRef MemoryBufferRef = Input->getFileBuffer(); if (Error EC = saveBuffer(MemoryBufferRef.getBuffer(), ModuleId)) return EC; @@ -210,3 +204,14 @@ llvm::Error lto::DTLTO::handleArchiveInputs() { return EC; return Error::success(); } + +// Remove temporary archive member files created to enable distribution. +void lto::DTLTO::cleanup() { + { + TimeTraceScope TimeScope("Remove temporary inputs for DTLTO"); + for (auto &Input : InputFiles) + if (Input->isMemberOfArchive()) + sys::fs::remove(Input->getName(), /*IgnoreNonExisting=*/true); + } + Base::cleanup(); +} diff --git a/llvm/lib/LTO/LTO.cpp b/llvm/lib/LTO/LTO.cpp index 4d69ffeda1280..955a19db48607 100644 --- a/llvm/lib/LTO/LTO.cpp +++ b/llvm/lib/LTO/LTO.cpp @@ -1217,6 +1217,8 @@ Error LTO::checkPartiallySplit() { } Error LTO::run(AddStreamFn AddStream, FileCache Cache) { + llvm::scope_exit CleanUp([this]() { cleanup(); }); + if (Error EC = handleArchiveInputs()) return EC; _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
