kazu created this revision. kazu added reviewers: davidxl, mtrofin, wmi. kazu added a project: LLVM. Herald added subscribers: cfe-commits, rupprecht, MaskRay, hiraditya, mgorny. Herald added a reviewer: jhenderson. Herald added a project: clang.
This patch adds a utility called DumpAccumulator. DumpAccumulator allows you to dump arbitrary text messages into a special section called .llvm_dump. The linker then concatenates these messages into the identically named section in the final executable. This utility makes it easy to collect information from optimization passes of interest in a build environment that caches compilation. See llvm/include/llvm/Analysis/DumpAccumulator.h for usage. The original implementation is from Mircea Trofin, and I generalized it. Repository: rG LLVM Github Monorepo https://reviews.llvm.org/D84473 Files: clang/lib/CodeGen/BackendUtil.cpp llvm/include/llvm/Analysis/DumpAccumulator.h llvm/include/llvm/InitializePasses.h llvm/lib/Analysis/CMakeLists.txt llvm/lib/Analysis/DumpAccumulator.cpp llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp llvm/lib/Passes/PassBuilder.cpp llvm/lib/Passes/PassRegistry.def llvm/test/Other/new-pm-defaults.ll llvm/tools/llvm-readobj/ObjDumper.cpp llvm/tools/llvm-readobj/ObjDumper.h llvm/tools/llvm-readobj/llvm-readobj.cpp
Index: llvm/tools/llvm-readobj/llvm-readobj.cpp =================================================================== --- llvm/tools/llvm-readobj/llvm-readobj.cpp +++ llvm/tools/llvm-readobj/llvm-readobj.cpp @@ -362,6 +362,10 @@ clEnumVal(GNU, "GNU readelf style")), cl::init(LLVM)); + // --llvm-dump + cl::opt<bool> LLVMDumpSection("llvm-dump", cl::desc("Print .llvm-dump"), + cl::init(false)); + cl::extrahelp HelpResponse("\nPass @FILE as argument to read options from FILE.\n"); } // namespace opts @@ -487,6 +491,8 @@ Dumper->printSymbols(opts::Symbols, opts::DynamicSymbols); if (!opts::StringDump.empty()) Dumper->printSectionsAsString(Obj, opts::StringDump); + if (opts::LLVMDumpSection) + Dumper->printLLVMDumpSection(Obj); if (!opts::HexDump.empty()) Dumper->printSectionsAsHex(Obj, opts::HexDump); if (opts::HashTable) Index: llvm/tools/llvm-readobj/ObjDumper.h =================================================================== --- llvm/tools/llvm-readobj/ObjDumper.h +++ llvm/tools/llvm-readobj/ObjDumper.h @@ -100,6 +100,7 @@ void printSectionsAsString(const object::ObjectFile *Obj, ArrayRef<std::string> Sections); + void printLLVMDumpSection(const object::ObjectFile *Obj); void printSectionsAsHex(const object::ObjectFile *Obj, ArrayRef<std::string> Sections); Index: llvm/tools/llvm-readobj/ObjDumper.cpp =================================================================== --- llvm/tools/llvm-readobj/ObjDumper.cpp +++ llvm/tools/llvm-readobj/ObjDumper.cpp @@ -15,8 +15,10 @@ #include "Error.h" #include "llvm-readobj.h" #include "llvm/Object/ObjectFile.h" +#include "llvm/Support/Compression.h" #include "llvm/Support/Error.h" #include "llvm/Support/FormatVariadic.h" +#include "llvm/Support/LEB128.h" #include "llvm/Support/ScopedPrinter.h" #include "llvm/Support/raw_ostream.h" #include <map> @@ -80,6 +82,45 @@ return Ret; } +void ObjDumper::printLLVMDumpSection(const object::ObjectFile *Obj) { + auto Sections = getSectionRefsByNameOrIndex(Obj, {".llvm_dump"}); + if (Sections.size() != 1) { + reportWarning(createError(".llvm_dump section not found"), + Obj->getFileName()); + return; + } + auto Section = Sections[0]; + StringRef SectionContent = + unwrapOrError(Obj->getFileName(), Section.getContents()); + + StringRef Current = SectionContent; + unsigned Read = 0; + auto ReadNumber = [&]() { + uint64_t V = + decodeULEB128(reinterpret_cast<const uint8_t *>(Current.data()), &Read); + Current = Current.substr(Read); + return V; + }; + while (Current.size() > 0) { + uint64_t UncompressedSize = ReadNumber(); + std::unique_ptr<char[]> Uncompressed(new char[UncompressedSize]); + uint64_t CompressedSize = ReadNumber(); + if (CompressedSize > Current.size()) { + reportWarning( + createError( + "Expecting a larger compressed buffer than available data"), + Obj->getFileName()); + } + if (auto E = + zlib::uncompress(Current, Uncompressed.get(), UncompressedSize)) { + reportWarning(createError("Error decompressing"), Obj->getFileName()); + } + Current = Current.substr(CompressedSize); + StringRef Message(Uncompressed.get(), UncompressedSize); + W.startLine() << Message << "\n"; + } +} + void ObjDumper::printSectionsAsString(const object::ObjectFile *Obj, ArrayRef<std::string> Sections) { bool First = true; Index: llvm/test/Other/new-pm-defaults.ll =================================================================== --- llvm/test/Other/new-pm-defaults.ll +++ llvm/test/Other/new-pm-defaults.ll @@ -92,6 +92,8 @@ ; CHECK-O-NEXT: Starting llvm::Module pass manager run. ; CHECK-O-NEXT: Running pass: PassManager<{{.*}}Module{{.*}}> ; CHECK-O-NEXT: Starting llvm::Module pass manager run. +; CHECK-O-NEXT: RequireAnalysisPass<llvm::DumpAccumulator, llvm::Module> +; CHECK-O-NEXT: Running analysis: DumpAccumulator ; CHECK-O-NEXT: Running pass: ForceFunctionAttrsPass ; CHECK-EP-PIPELINE-START-NEXT: Running pass: NoOpModulePass ; CHECK-O-NEXT: Running pass: PassManager<{{.*}}Module{{.*}}> Index: llvm/lib/Passes/PassRegistry.def =================================================================== --- llvm/lib/Passes/PassRegistry.def +++ llvm/lib/Passes/PassRegistry.def @@ -28,6 +28,7 @@ MODULE_ANALYSIS("pass-instrumentation", PassInstrumentationAnalysis(PIC)) MODULE_ANALYSIS("asan-globals-md", ASanGlobalsMetadataAnalysis()) MODULE_ANALYSIS("inline-advisor", InlineAdvisorAnalysis()) +MODULE_ANALYSIS("dump-accumulator", DumpAccumulator()) #ifndef MODULE_ALIAS_ANALYSIS #define MODULE_ALIAS_ANALYSIS(NAME, CREATE_PASS) \ Index: llvm/lib/Passes/PassBuilder.cpp =================================================================== --- llvm/lib/Passes/PassBuilder.cpp +++ llvm/lib/Passes/PassBuilder.cpp @@ -31,6 +31,7 @@ #include "llvm/Analysis/DemandedBits.h" #include "llvm/Analysis/DependenceAnalysis.h" #include "llvm/Analysis/DominanceFrontier.h" +#include "llvm/Analysis/DumpAccumulator.h" #include "llvm/Analysis/FunctionPropertiesAnalysis.h" #include "llvm/Analysis/GlobalsModRef.h" #include "llvm/Analysis/IVUsers.h" @@ -255,6 +256,10 @@ cl::Hidden, cl::desc("Enable inline deferral during PGO")); +cl::opt<bool> EnableDumpAccumulator("enable-dump-accumulator", cl::init(true), + cl::Hidden, + cl::desc("Enable the dump accumulator.")); + PipelineTuningOptions::PipelineTuningOptions() { LoopInterleaving = true; LoopVectorization = true; @@ -1241,6 +1246,8 @@ ModulePassManager MPM(DebugLogging); + MPM.addPass(RequireAnalysisPass<DumpAccumulator, Module>()); + // Force any function attributes we want the rest of the pipeline to observe. MPM.addPass(ForceFunctionAttrsPass()); Index: llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp =================================================================== --- llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp +++ llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp @@ -29,6 +29,7 @@ #include "llvm/ADT/Triple.h" #include "llvm/ADT/Twine.h" #include "llvm/Analysis/ConstantFolding.h" +#include "llvm/Analysis/DumpAccumulator.h" #include "llvm/Analysis/EHPersonalities.h" #include "llvm/Analysis/OptimizationRemarkEmitter.h" #include "llvm/BinaryFormat/COFF.h" @@ -106,6 +107,7 @@ #include "llvm/Support/Casting.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Compiler.h" +#include "llvm/Support/Compression.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/Format.h" #include "llvm/Support/MathExtras.h" @@ -246,6 +248,7 @@ MachineFunctionPass::getAnalysisUsage(AU); AU.addRequired<MachineOptimizationRemarkEmitterPass>(); AU.addRequired<GCModuleInfo>(); + AU.addUsedIfAvailable<DumpAccumulatorWrapper>(); } bool AsmPrinter::doInitialization(Module &M) { @@ -1757,6 +1760,20 @@ } } + if (auto *DumpAccumulatorAnalysis = + getAnalysisIfAvailable<DumpAccumulatorWrapper>()) { + auto *DS = OutStreamer->getContext().getELFSection( + ".llvm_dump", ELF::SHT_NOTE, ELF::SHF_STRINGS); + OutStreamer->SwitchSection(DS); + std::string &Out = DumpAccumulatorAnalysis->GetResult(M)->Message; + SmallVector<char, 128> Compressed; + Error E = zlib::compress(Out.c_str(), Compressed); + assert(!E); + OutStreamer->emitULEB128IntValue(Out.size()); + OutStreamer->emitULEB128IntValue(Compressed.size()); + OutStreamer->emitBytes({Compressed.data(), Compressed.size()}); + } + // Allow the target to emit any magic that it wants at the end of the file, // after everything else has gone out. emitEndOfAsmFile(M); Index: llvm/lib/Analysis/DumpAccumulator.cpp =================================================================== --- /dev/null +++ llvm/lib/Analysis/DumpAccumulator.cpp @@ -0,0 +1,30 @@ +//===- DumpAccumulator.cpp - Dumping infrastructure -----------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This implements feature and label extraction for offline supervised learning +// of a IR to native size model. +// +//===----------------------------------------------------------------------===// + +#include "llvm/Analysis/DumpAccumulator.h" +#include "llvm/InitializePasses.h" + +using namespace llvm; + +AnalysisKey DumpAccumulator::Key; + +char DumpAccumulatorWrapper::ID = 0; +INITIALIZE_PASS(DumpAccumulatorWrapper, "dumpaccumulator", "Dump Accumulator", + false, true) + +DumpAccumulatorWrapper::DumpAccumulatorWrapper( + std::function<DumpAccumulator::Result *(Module &M)> GetFFA) + : ImmutablePass(ID), GetFFA(GetFFA) { + initializeDumpAccumulatorWrapperPass(*PassRegistry::getPassRegistry()); +} Index: llvm/lib/Analysis/CMakeLists.txt =================================================================== --- llvm/lib/Analysis/CMakeLists.txt +++ llvm/lib/Analysis/CMakeLists.txt @@ -48,6 +48,7 @@ DomPrinter.cpp DomTreeUpdater.cpp DominanceFrontier.cpp + DumpAccumulator.cpp EHPersonalities.cpp FunctionPropertiesAnalysis.cpp GlobalsModRef.cpp Index: llvm/include/llvm/InitializePasses.h =================================================================== --- llvm/include/llvm/InitializePasses.h +++ llvm/include/llvm/InitializePasses.h @@ -138,6 +138,7 @@ void initializeDomViewerPass(PassRegistry&); void initializeDominanceFrontierWrapperPassPass(PassRegistry&); void initializeDominatorTreeWrapperPassPass(PassRegistry&); +void initializeDumpAccumulatorWrapperPass(PassRegistry &); void initializeDwarfEHPreparePass(PassRegistry&); void initializeEarlyCSELegacyPassPass(PassRegistry&); void initializeEarlyCSEMemSSALegacyPassPass(PassRegistry&); Index: llvm/include/llvm/Analysis/DumpAccumulator.h =================================================================== --- /dev/null +++ llvm/include/llvm/Analysis/DumpAccumulator.h @@ -0,0 +1,97 @@ +//===- llvm/Analysis/DumpAccumulator.h - Dump Accumulator -------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file contains the declarations for the dump accumulator. +// +// DumpAccumulator allows you to dump arbitrary text messages into a special +// section called .llvm_dump. The linker then concatenates these messages into +// the identically named section in the final executable. +// +// This utility makes it easy to collect information from optimization passes of +// interest in a build environment that caches compilation. +// +// Suppose you wish to dump all occurrences of inlining. Then: +// +// Step 1: Build your compiler with the following modifications. +// +// Add: +// +// #include "llvm/Analysis/DumpAccumulator.h" +// +// Declare in the inlining pass: +// +// DumpAccumulator::Result *DAR = MAMProxy.getCachedResult<DumpAccumulator>(M); +// +// Add code like so that a message is accumulated every time inlining happens: +// +// if (DAR) { +// DAR->Message += F.getName(); +// DAR->Message += ","; +// DAR->Message += Callee.getName(); +// DAR->Message += "\n"; +// } +// +// Step 2: Build your testcase with: +// +// -mllvm -enable-dump-accumulator +// +// Note that ThinLTO is not supported yet. +// +// Step 3: Dump the messages like so: +// +// $ llvm-readobj --llvm-dump a.out +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_ANALYSIS_DUMPACCUMULATOR_H +#define LLVM_ANALYSIS_DUMPACCUMULATOR_H + +#include "llvm/IR/PassManager.h" + +#include <string> + +namespace llvm { +class DumpAccumulator : public AnalysisInfoMixin<DumpAccumulator> { +public: + struct Result { + bool invalidate(Module &, const PreservedAnalyses &, + llvm::ModuleAnalysisManager::Invalidator &) { + return false; + } + + Result() = default; + Result(const Result &) = delete; + Result(Result &&Other) = default; + ~Result() = default; + + std::string Message; + }; + + DumpAccumulator() = default; + + Result run(Module &M, ModuleAnalysisManager &MAM) { return {}; } + +private: + static AnalysisKey Key; + friend AnalysisInfoMixin<DumpAccumulator>; +}; + +class DumpAccumulatorWrapper : public ImmutablePass { + std::function<DumpAccumulator::Result *(Module &M)> GetFFA; + +public: + static char ID; + explicit DumpAccumulatorWrapper( + std::function<DumpAccumulator::Result *(Module &M)> GetFFA); + DumpAccumulatorWrapper() : ImmutablePass(ID){}; + DumpAccumulator::Result *GetResult(Module &M) { return GetFFA(M); } +}; + +} // namespace llvm + +#endif // LLVM_ANALYSIS_DUMPACCUMULATOR_H Index: clang/lib/CodeGen/BackendUtil.cpp =================================================================== --- clang/lib/CodeGen/BackendUtil.cpp +++ clang/lib/CodeGen/BackendUtil.cpp @@ -18,6 +18,7 @@ #include "llvm/ADT/StringExtras.h" #include "llvm/ADT/StringSwitch.h" #include "llvm/ADT/Triple.h" +#include "llvm/Analysis/DumpAccumulator.h" #include "llvm/Analysis/StackSafetyAnalysis.h" #include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/Analysis/TargetTransformInfo.h" @@ -89,6 +90,8 @@ llvm::PassPluginLibraryInfo get##Ext##PluginInfo(); #include "llvm/Support/Extension.def" +extern cl::opt<bool> EnableDumpAccumulator; + namespace { // Default filename used for profile generation. @@ -1436,6 +1439,10 @@ NeedCodeGen = true; CodeGenPasses.add( createTargetTransformInfoWrapperPass(getTargetIRAnalysis())); + if (EnableDumpAccumulator) + CodeGenPasses.add(new DumpAccumulatorWrapper([&MAM](Module &M) { + return MAM.getCachedResult<DumpAccumulator>(M); + })); if (!CodeGenOpts.SplitDwarfOutput.empty()) { DwoOS = openOutputFile(CodeGenOpts.SplitDwarfOutput); if (!DwoOS)
_______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits