Author: Hongtao Yu Date: 2020-11-30T10:16:54-08:00 New Revision: 64fa8cce225f7d8bd499e3a99caa850b764ff109
URL: https://github.com/llvm/llvm-project/commit/64fa8cce225f7d8bd499e3a99caa850b764ff109 DIFF: https://github.com/llvm/llvm-project/commit/64fa8cce225f7d8bd499e3a99caa850b764ff109.diff LOG: [CSSPGO] Pseudo probe instrumentation pass This change introduces a pseudo probe instrumentation pass for block instrumentation. Please refer to https://reviews.llvm.org/D86193 for the whole story. Given the following LLVM IR: ``` define internal void @foo2(i32 %x, void (i32)* %f) !dbg !4 { bb0: %cmp = icmp eq i32 %x, 0 br i1 %cmp, label %bb1, label %bb2 bb1: br label %bb3 bb2: br label %bb3 bb3: ret void } ``` The instrumented IR will look like below. Note that each llvm.pseudoprobe intrinsic call represents a pseudo probe at a block, of which the first parameter is the GUID of the probe’s owner function and the second parameter is the probe’s ID. ``` define internal void @foo2(i32 %x, void (i32)* %f) !dbg !4 { bb0: %cmp = icmp eq i32 %x, 0 call void @llvm.pseudoprobe(i64 837061429793323041, i64 1) br i1 %cmp, label %bb1, label %bb2 bb1: call void @llvm.pseudoprobe(i64 837061429793323041, i64 2) br label %bb3 bb2: call void @llvm.pseudoprobe(i64 837061429793323041, i64 3) br label %bb3 bb3: call void @llvm.pseudoprobe(i64 837061429793323041, i64 4) ret void } ``` Reviewed By: wmi Differential Revision: https://reviews.llvm.org/D86499 Added: llvm/include/llvm/Transforms/IPO/SampleProfileProbe.h llvm/lib/Transforms/IPO/SampleProfileProbe.cpp llvm/test/Transforms/SampleProfile/pseudo-probe-emit.ll Modified: llvm/lib/Passes/PassBuilder.cpp llvm/lib/Passes/PassRegistry.def llvm/lib/Transforms/IPO/CMakeLists.txt Removed: ################################################################################ diff --git a/llvm/include/llvm/Transforms/IPO/SampleProfileProbe.h b/llvm/include/llvm/Transforms/IPO/SampleProfileProbe.h new file mode 100644 index 0000000000000..d0165a1cbc71a --- /dev/null +++ b/llvm/include/llvm/Transforms/IPO/SampleProfileProbe.h @@ -0,0 +1,65 @@ +//===- Transforms/IPO/SampleProfileProbe.h ----------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +/// \file +/// This file provides the interface for the pseudo probe implementation for +/// AutoFDO. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TRANSFORMS_IPO_SAMPLEPROFILEPROBE_H +#define LLVM_TRANSFORMS_IPO_SAMPLEPROFILEPROBE_H + +#include "llvm/ADT/DenseMap.h" +#include "llvm/IR/PassManager.h" +#include "llvm/Target/TargetMachine.h" +#include <unordered_map> + +namespace llvm { + +class Module; + +using BlockIdMap = std::unordered_map<BasicBlock *, uint32_t>; + +enum class PseudoProbeReservedId { Invalid = 0, Last = Invalid }; + +enum class PseudoProbeType { Block = 0 }; + +/// Sample profile pseudo prober. +/// +/// Insert pseudo probes for block sampling and value sampling. +class SampleProfileProber { +public: + // Give an empty module id when the prober is not used for instrumentation. + SampleProfileProber(Function &F); + void instrumentOneFunc(Function &F, TargetMachine *TM); + +private: + Function *getFunction() const { return F; } + uint32_t getBlockId(const BasicBlock *BB) const; + void computeProbeIdForBlocks(); + + Function *F; + + /// Map basic blocks to the their pseudo probe ids. + BlockIdMap BlockProbeIds; + + /// The ID of the last probe, Can be used to number a new probe. + uint32_t LastProbeId; +}; + +class SampleProfileProbePass : public PassInfoMixin<SampleProfileProbePass> { + TargetMachine *TM; + +public: + SampleProfileProbePass(TargetMachine *TM) : TM(TM) {} + PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); +}; + +} // end namespace llvm +#endif // LLVM_TRANSFORMS_IPO_SAMPLEPROFILEPROBE_H diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp index 2a9b0c1e4f6ba..6d6ddd0012fa9 100644 --- a/llvm/lib/Passes/PassBuilder.cpp +++ b/llvm/lib/Passes/PassBuilder.cpp @@ -110,6 +110,7 @@ #include "llvm/Transforms/IPO/PartialInlining.h" #include "llvm/Transforms/IPO/SCCP.h" #include "llvm/Transforms/IPO/SampleProfile.h" +#include "llvm/Transforms/IPO/SampleProfileProbe.h" #include "llvm/Transforms/IPO/StripDeadPrototypes.h" #include "llvm/Transforms/IPO/StripSymbols.h" #include "llvm/Transforms/IPO/SyntheticCountsPropagation.h" diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def index 7f0f51ad09773..2871282be639b 100644 --- a/llvm/lib/Passes/PassRegistry.def +++ b/llvm/lib/Passes/PassRegistry.def @@ -101,6 +101,7 @@ MODULE_PASS("oz-module-optimizer", buildModuleOptimizationPipeline(OptimizationLevel::Oz, /*LTOPreLink*/false)) MODULE_PASS("strip", StripSymbolsPass()) MODULE_PASS("strip-dead-debug-info", StripDeadDebugInfoPass()) +MODULE_PASS("pseudo-probe", SampleProfileProbePass(TM)) MODULE_PASS("strip-dead-prototypes", StripDeadPrototypesPass()) MODULE_PASS("strip-debug-declare", StripDebugDeclarePass()) MODULE_PASS("strip-nondebug", StripNonDebugSymbolsPass()) diff --git a/llvm/lib/Transforms/IPO/CMakeLists.txt b/llvm/lib/Transforms/IPO/CMakeLists.txt index 66e772c61b0a4..05e6f4355efe7 100644 --- a/llvm/lib/Transforms/IPO/CMakeLists.txt +++ b/llvm/lib/Transforms/IPO/CMakeLists.txt @@ -32,6 +32,7 @@ add_llvm_component_library(LLVMipo PassManagerBuilder.cpp PruneEH.cpp SampleProfile.cpp + SampleProfileProbe.cpp SCCP.cpp StripDeadPrototypes.cpp StripSymbols.cpp diff --git a/llvm/lib/Transforms/IPO/SampleProfileProbe.cpp b/llvm/lib/Transforms/IPO/SampleProfileProbe.cpp new file mode 100644 index 0000000000000..e158bc9c148fa --- /dev/null +++ b/llvm/lib/Transforms/IPO/SampleProfileProbe.cpp @@ -0,0 +1,117 @@ +//===- SampleProfileProbe.cpp - Pseudo probe Instrumentation -------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the SampleProfileProber transformation. +// +//===----------------------------------------------------------------------===// + +#include "llvm/Transforms/IPO/SampleProfileProbe.h" +#include "llvm/ADT/Statistic.h" +#include "llvm/Analysis/TargetLibraryInfo.h" +#include "llvm/IR/BasicBlock.h" +#include "llvm/IR/CFG.h" +#include "llvm/IR/Constant.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/DebugInfoMetadata.h" +#include "llvm/IR/GlobalValue.h" +#include "llvm/IR/GlobalVariable.h" +#include "llvm/IR/IRBuilder.h" +#include "llvm/IR/Instruction.h" +#include "llvm/IR/MDBuilder.h" +#include "llvm/ProfileData/SampleProf.h" +#include "llvm/Support/CRC.h" +#include "llvm/Transforms/Instrumentation.h" +#include "llvm/Transforms/Utils/ModuleUtils.h" +#include <vector> + +using namespace llvm; +#define DEBUG_TYPE "sample-profile-probe" + +STATISTIC(ArtificialDbgLine, + "Number of probes that have an artificial debug line"); + +SampleProfileProber::SampleProfileProber(Function &Func) : F(&Func) { + BlockProbeIds.clear(); + LastProbeId = (uint32_t)PseudoProbeReservedId::Last; + computeProbeIdForBlocks(); +} + +void SampleProfileProber::computeProbeIdForBlocks() { + for (auto &BB : *F) { + BlockProbeIds[&BB] = ++LastProbeId; + } +} + +uint32_t SampleProfileProber::getBlockId(const BasicBlock *BB) const { + auto I = BlockProbeIds.find(const_cast<BasicBlock *>(BB)); + return I == BlockProbeIds.end() ? 0 : I->second; +} + +void SampleProfileProber::instrumentOneFunc(Function &F, TargetMachine *TM) { + Module *M = F.getParent(); + MDBuilder MDB(F.getContext()); + // Compute a GUID without considering the function's linkage type. This is + // fine since function name is the only key in the profile database. + uint64_t Guid = Function::getGUID(F.getName()); + + // Probe basic blocks. + for (auto &I : BlockProbeIds) { + BasicBlock *BB = I.first; + uint32_t Index = I.second; + // Insert a probe before an instruction with a valid debug line number which + // will be assigned to the probe. The line number will be used later to + // model the inline context when the probe is inlined into other functions. + // Debug instructions, phi nodes and lifetime markers do not have an valid + // line number. Real instructions generated by optimizations may not come + // with a line number either. + auto HasValidDbgLine = [](Instruction *J) { + return !isa<PHINode>(J) && !isa<DbgInfoIntrinsic>(J) && + !J->isLifetimeStartOrEnd() && J->getDebugLoc(); + }; + + Instruction *J = &*BB->getFirstInsertionPt(); + while (J != BB->getTerminator() && !HasValidDbgLine(J)) { + J = J->getNextNode(); + } + + IRBuilder<> Builder(J); + assert(Builder.GetInsertPoint() != BB->end() && + "Cannot get the probing point"); + Function *ProbeFn = + llvm::Intrinsic::getDeclaration(M, Intrinsic::pseudoprobe); + Value *Args[] = {Builder.getInt64(Guid), Builder.getInt64(Index), + Builder.getInt32(0)}; + auto *Probe = Builder.CreateCall(ProbeFn, Args); + // Assign an artificial debug line to a probe that doesn't come with a real + // line. A probe not having a debug line will get an incomplete inline + // context. This will cause samples collected on the probe to be counted + // into the base profile instead of a context profile. The line number + // itself is not important though. + if (!Probe->getDebugLoc()) { + if (auto *SP = F.getSubprogram()) { + auto DIL = DebugLoc::get(0, 0, SP); + Probe->setDebugLoc(DIL); + ArtificialDbgLine++; + LLVM_DEBUG(dbgs() << "\nIn Function " << F.getName() << " Probe " + << Index << " gets an artificial debug line\n";); + } + } + } +} + +PreservedAnalyses SampleProfileProbePass::run(Module &M, + ModuleAnalysisManager &AM) { + for (auto &F : M) { + if (F.isDeclaration()) + continue; + SampleProfileProber ProbeManager(F); + ProbeManager.instrumentOneFunc(F, TM); + } + + return PreservedAnalyses::none(); +} diff --git a/llvm/test/Transforms/SampleProfile/pseudo-probe-emit.ll b/llvm/test/Transforms/SampleProfile/pseudo-probe-emit.ll new file mode 100644 index 0000000000000..0d3579dda3bc6 --- /dev/null +++ b/llvm/test/Transforms/SampleProfile/pseudo-probe-emit.ll @@ -0,0 +1,48 @@ +; RUN: opt < %s -passes=pseudo-probe -function-sections -S -o %t +; RUN: FileCheck %s < %t --check-prefix=CHECK-IL +; RUN: llc %t -stop-after=instruction-select -o - | FileCheck %s --check-prefix=CHECK-MIR +; +;; Check the generation of pseudoprobe intrinsic call. + +define void @foo(i32 %x) !dbg !3 { +bb0: + %cmp = icmp eq i32 %x, 0 +; CHECK-IL: call void @llvm.pseudoprobe(i64 [[#GUID:]], i64 1, i32 0), !dbg ![[#FAKELINE:]] +; CHECK-MIR: PSEUDO_PROBE [[#GUID:]], 1, 0, 0 + br i1 %cmp, label %bb1, label %bb2 + +bb1: +; CHECK-IL: call void @llvm.pseudoprobe(i64 [[#GUID:]], i64 2, i32 0), !dbg ![[#FAKELINE]] +; CHECK-MIR: PSEUDO_PROBE [[#GUID]], 3, 0, 0 +; CHECK-MIR: PSEUDO_PROBE [[#GUID]], 4, 0, 0 + br label %bb3 + +bb2: +; CHECK-IL: call void @llvm.pseudoprobe(i64 [[#GUID:]], i64 3, i32 0), !dbg ![[#FAKELINE]] +; CHECK-MIR: PSEUDO_PROBE [[#GUID]], 2, 0, 0 +; CHECK-MIR: PSEUDO_PROBE [[#GUID]], 4, 0, 0 + br label %bb3 + +bb3: +; CHECK-IL: call void @llvm.pseudoprobe(i64 [[#GUID]], i64 4, i32 0), !dbg ![[#REALLINE:]] + ret void, !dbg !12 +} + +; CHECK-IL: ![[#FOO:]] = distinct !DISubprogram(name: "foo" +; CHECK-IL: ![[#FAKELINE]] = !DILocation(line: 0, scope: ![[#FOO]]) +; CHECK-IL: ![[#REALLINE]] = !DILocation(line: 2, scope: ![[#FOO]]) + +!llvm.dbg.cu = !{!0} +!llvm.module.flags = !{!9, !10} + +!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1) +!1 = !DIFile(filename: "test.c", directory: "") +!2 = !{} +!3 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 1, type: !5, unit: !0, retainedNodes: !2) +!5 = !DISubroutineType(types: !6) +!6 = !{!7} +!7 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed) +!9 = !{i32 2, !"Dwarf Version", i32 4} +!10 = !{i32 2, !"Debug Info Version", i32 3} +!11 = !{!"clang version 3.9.0"} +!12 = !DILocation(line: 2, scope: !3) \ No newline at end of file _______________________________________________ llvm-branch-commits mailing list llvm-branch-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits