https://github.com/yxsamliu updated 
https://github.com/llvm/llvm-project/pull/225911

>From f8eb0d81988646496a72c61413d23a315b6e758b Mon Sep 17 00:00:00 2001
From: "Yaxun (Sam) Liu" <[email protected]>
Date: Thu, 29 Oct 2026 01:58:06 +0000
Subject: [PATCH] [PGO] Load GPU wave counts into IR metadata

GPU profiles contain wave counts alongside lane counts, but profile use
does not expose them to optimizations. Wave visits do not obey scalar
flow conservation, so unmeasured blocks cannot use counts reconstructed
from neighboring blocks or ordinary branch weights.

Map wave-counter indices to the blocks selected by PGO instrumentation,
after reproducing its critical-edge splits. For dense profiles, also map the
appended wave-only slots to eligible blocks in function order, after the
original block/select prefix. Select the layout from the profile flags.
Attach measured counts using wave.profile metadata, retaining measured
zeros and marking blocks without measurements unavailable. Require a
measured entry count for normalization and exclude select-counter slots
from the block mapping. Repeated instrumentation sites for the same block
represent one measurement and are not summed.

Validate the wave-counter layout against the accepted lane profile.
Clear old wave metadata when loading a replacement profile, including
when a function has no usable record. Do not emit wave metadata for
previously profiled functions: their branch weights may change counter
placement without changing the CFG hash. Keep ordinary lane-count
reconstruction and branch weights unchanged.

The hidden `-pgo-wave-metadata` option is enabled by default. Set it to
`false` (with Clang, `-mllvm -pgo-wave-metadata=false`) to suppress both
`wave.profile` and `wave.profile.block` during profile use. Existing wave
annotations are cleared, while ordinary counts and uniformity hints remain
available. This provides a control for debugging and performance comparisons,
and an escape hatch for suspected wave-profile regressions when recompiling.
Profile collection is unaffected, so both builds can use the same profile.

RFC: 
https://discourse.llvm.org/t/rfc-gpu-wave-counts-for-spill-placement-costs/91927
---
 .../Instrumentation/PGOInstrumentation.cpp    |  56 ++++
 .../Transforms/PGOProfile/dense-wave-cfg.ll   |  16 +-
 .../Transforms/PGOProfile/wave-profile-use.ll | 291 ++++++++++++++++++
 3 files changed, 362 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/Transforms/PGOProfile/wave-profile-use.ll

diff --git a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp 
b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
index 0e4c3eac6b7682..7270be90a67cea 100644
--- a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
+++ b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
@@ -52,6 +52,7 @@
 #include "ValueProfileCollector.h"
 #include "llvm/ADT/APInt.h"
 #include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/BitVector.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
@@ -164,6 +165,10 @@ static cl::opt<bool> PGOUniformityMetadata(
     "pgo-uniformity-metadata", cl::init(true), cl::Hidden,
     cl::desc("Enable uniformity profile metadata during PGO use"));
 
+static cl::opt<bool> PGOWaveMetadata(
+    "pgo-wave-metadata", cl::init(true), cl::Hidden,
+    cl::desc("Enable wave count profile metadata during PGO use"));
+
 namespace llvm {
 
 // Command line option to enable/disable the warning about a hash mismatch in
@@ -1150,6 +1155,9 @@ class PGOUseFunc {
   // Find the Instrumented BB and set the value. Return false on error.
   bool setInstrumentedCounts(const std::vector<uint64_t> &CountFromProfile);
 
+  void setWaveCounts(ArrayRef<BasicBlock *> InstrumentBBs,
+                     ArrayRef<BasicBlock *> ExtraWaveBBs);
+
   // Set the edge counter value for the unknown edge -- there should be only
   // one unknown edge.
   void setEdgeCount(DirectEdges &Edges, uint64_t Value);
@@ -1185,6 +1193,50 @@ static void setupBBInfoEdges(
   }
 }
 
+// Wave slots use the same indices as lane counters, including the trailing
+// select counters, followed by any dense block measurements. Select slots do
+// not describe block-entry events.
+void PGOUseFunc::setWaveCounts(ArrayRef<BasicBlock *> InstrumentBBs,
+                               ArrayRef<BasicBlock *> ExtraWaveBBs) {
+  if (!PGOWaveMetadata || ProfileRecord.WaveCounts.empty() ||
+      !isGPUProfTarget(*M) || IsCS)
+    return;
+  if (ProfileRecord.WaveCounts.size() != ProfileRecord.Counts.size()) {
+    F.getContext().diagnose(DiagnosticInfoPGOProfile(
+        M->getName().data(),
+        Twine("Inconsistent number of wave counts in ") + F.getName() +
+            "; ignoring wave profile",
+        DS_Warning));
+    return;
+  }
+
+  // Earlier profile annotations can change the instrumentation MST without
+  // changing the CFG hash. Do not reuse its counter indices for wave counts.
+  if (F.getMetadata(LLVMContext::MD_prof))
+    return;
+
+  DenseMap<const BasicBlock *, uint64_t> MeasuredCounts;
+  for (auto [Index, BB] : enumerate(InstrumentBBs))
+    MeasuredCounts.try_emplace(BB, ProfileRecord.WaveCounts[Index]);
+  unsigned Index =
+      InstrumentBBs.size() + FuncInfo.SIVisitor.getNumOfSelectInsts();
+  for (BasicBlock *BB : ExtraWaveBBs)
+    MeasuredCounts.try_emplace(BB, ProfileRecord.WaveCounts[Index++]);
+  // Lane-flow reconstruction cannot provide a missing wave normalization 
count.
+  if (!MeasuredCounts.contains(&F.getEntryBlock()))
+    return;
+
+  SmallVector<uint64_t> Counts;
+  BitVector HasCounts;
+  for (const BasicBlock &BB : F) {
+    auto It = MeasuredCounts.find(&BB);
+    bool HasCount = It != MeasuredCounts.end();
+    Counts.push_back(HasCount ? It->second : 0);
+    HasCounts.push_back(HasCount);
+  }
+  setBlockWaveCounts(F, Counts, HasCounts);
+}
+
 // Visit all the edges and assign the count value for the instrumented
 // edges and the BB. Return false on error.
 bool PGOUseFunc::setInstrumentedCounts(
@@ -1220,6 +1272,7 @@ bool PGOUseFunc::setInstrumentedCounts(
     });
     return false;
   }
+  setWaveCounts(InstrumentBBs, ExtraWaveBBs);
   auto *FuncEntry = &*F.begin();
 
   // Set the profile count to the Instrumented BBs.
@@ -2197,6 +2250,9 @@ static bool annotateAllFunctions(
         TI->setMetadata(LLVMContext::MD_branch_uniformity_profile, nullptr);
       }
     }
+    // A replacement profile must not leave an earlier wave mapping live when
+    // its record is absent, mismatched, or contains only lane counts.
+    clearBlockWaveCounts(F);
     if (skipPGOUse(Opts, F))
       continue;
     TargetLibraryInfo &TLI = LookupTLI(F);
diff --git a/llvm/test/Transforms/PGOProfile/dense-wave-cfg.ll 
b/llvm/test/Transforms/PGOProfile/dense-wave-cfg.ll
index b6f4924fc66320..6dc679e474278d 100644
--- a/llvm/test/Transforms/PGOProfile/dense-wave-cfg.ll
+++ b/llvm/test/Transforms/PGOProfile/dense-wave-cfg.ll
@@ -3,7 +3,8 @@
 ; RUN: %python %t/raw.py > %t/loop.raw
 ; RUN: llvm-profdata merge %t/loop.raw -o %t/loop.profdata
 ; RUN: opt -passes=pgo-instr-use,verify 
-pgo-test-profile-file=%t/loop.profdata -S %t/loop.ll -o %t/use.ll
-; RUN: FileCheck %s --check-prefix=COUNTS < %t/use.ll
+; RUN: FileCheck %s --check-prefixes=COUNTS,WAVE < %t/use.ll
+; RUN: opt -passes=pgo-instr-use,verify 
-pgo-test-profile-file=%t/loop.profdata -pgo-wave-metadata=false -S %t/loop.ll 
| FileCheck %s --check-prefix=COUNTS --implicit-check-not=wave.profile
 ; RUN: opt -passes=pgo-instr-gen,verify -pgo-instrument-entry -S %t/eh.ll | 
FileCheck %s --check-prefix=EH --implicit-check-not='call void 
@llvm.instrprof.increment'
 
 ; Dense instrumentation preserves the entry and split backedge counters,
@@ -27,9 +28,22 @@
 ; header has four times the entry wave count; sparse data cannot supply it.
 ; COUNTS-LABEL: define void @loop(
 ; COUNTS-SAME: !prof ![[ENTRY_COUNT:[0-9]+]]
+; WAVE-SAME: !wave.profile ![[WAVES:[0-9]+]]
+; WAVE: entry:
+; WAVE-NEXT: br label %header, {{.*}}!wave.profile.block ![[ENTRY:[0-9]+]]
 ; COUNTS: br i1 %done, label %exit, label %header.header_crit_edge, !prof 
![[WEIGHTS:[0-9]+]]
+; WAVE-SAME: !wave.profile.block ![[HEADER:[0-9]+]]
+; WAVE: header.header_crit_edge:
+; WAVE-NEXT: br label %header, {{.*}}!wave.profile.block ![[BACKEDGE:[0-9]+]]
+; WAVE: exit:
+; WAVE-NEXT: ret void, !wave.profile.block ![[EXIT:[0-9]+]]
 ; COUNTS-DAG: ![[ENTRY_COUNT]] = !{!"function_entry_count", i64 192}
 ; COUNTS-DAG: ![[WEIGHTS]] = !{!"branch_weights", i32 192, i32 576}
+; WAVE-DAG: ![[WAVES]] = distinct !{i64 2, i64 [[FID:-?[0-9]+]], i64 3, i64 
12, i64 9, i64 3}
+; WAVE-DAG: ![[ENTRY]] = !{i64 2, i64 [[FID]], i64 0, i64 1, i64 1}
+; WAVE-DAG: ![[HEADER]] = !{i64 2, i64 [[FID]], i64 1, i64 1, i64 3, i64 2}
+; WAVE-DAG: ![[BACKEDGE]] = !{i64 2, i64 [[FID]], i64 2, i64 1, i64 1}
+; WAVE-DAG: ![[EXIT]] = !{i64 2, i64 [[FID]], i64 3, i64 1}
 
 ; A catchswitch has no legal insertion point. Skip dispatch and append only
 ; exit; the sparse catch counter must remain after its catchpad.
diff --git a/llvm/test/Transforms/PGOProfile/wave-profile-use.ll 
b/llvm/test/Transforms/PGOProfile/wave-profile-use.ll
new file mode 100644
index 00000000000000..cc56ae02614e79
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/wave-profile-use.ll
@@ -0,0 +1,291 @@
+; RUN: split-file %s %t
+; RUN: opt -passes=pgo-instr-gen -pgo-instrument-dense-wave-counts=false 
-pgo-instrument-entry -S %t/input.ll | FileCheck %s --check-prefix=GEN
+; RUN: %python %t/raw.py wave > %t/wave.raw
+; RUN: llvm-profdata merge %t/wave.raw -o %t/wave.profdata
+; RUN: opt -passes=pgo-instr-use,verify 
-pgo-test-profile-file=%t/wave.profdata -S %t/input.ll -o %t/used.ll
+; RUN: FileCheck %s --check-prefixes=WAVE,COUNTS < %t/used.ll
+; RUN: opt -passes=pgo-instr-use,verify 
-pgo-test-profile-file=%t/wave.profdata -pgo-wave-metadata=true -S %t/input.ll 
-o %t/enabled.ll
+; RUN: diff %t/used.ll %t/enabled.ll
+; RUN: opt -passes=pgo-instr-use,verify 
-pgo-test-profile-file=%t/wave.profdata -pgo-wave-metadata=false -S %t/input.ll 
| FileCheck %s --check-prefixes=NONE,COUNTS --implicit-check-not=wave.profile
+; RUN: opt -passes=pgo-instr-use,verify 
-pgo-test-profile-file=%t/wave.profdata -pgo-wave-metadata=false -S %t/used.ll 
| FileCheck %s --check-prefixes=NONE,COUNTS --implicit-check-not=wave.profile
+; RUN: opt -passes=pgo-instr-use,verify 
-pgo-test-profile-file=%t/wave.profdata -S %t/used.ll | FileCheck %s 
--check-prefixes=NONE,COUNTS --implicit-check-not=wave.profile
+; RUN: %python %t/raw.py lane > %t/lane.raw
+; RUN: llvm-profdata merge %t/lane.raw -o %t/lane.profdata
+; RUN: opt -passes=pgo-instr-use,verify 
-pgo-test-profile-file=%t/lane.profdata -S %t/used.ll | FileCheck %s 
--check-prefixes=NONE,COUNTS --implicit-check-not=wave.profile
+; RUN: %python %t/raw.py bad-wave > %t/bad-wave.raw
+; RUN: llvm-profdata merge %t/bad-wave.raw -o %t/bad-wave.profdata
+; RUN: opt -passes=pgo-instr-use,verify 
-pgo-test-profile-file=%t/bad-wave.profdata -S %t/used.ll -o %t/bad-wave.ll 
2>&1 | FileCheck %s --check-prefix=WARN
+; RUN: FileCheck %s --check-prefixes=NONE,COUNTS 
--implicit-check-not=wave.profile < %t/bad-wave.ll
+; RUN: %python %t/raw.py zero > %t/zero.raw
+; RUN: llvm-profdata merge %t/zero.raw -o %t/zero.profdata
+; RUN: opt -passes=pgo-instr-use,verify 
-pgo-test-profile-file=%t/zero.profdata -pgo-wave-metadata=false -S %t/input.ll 
| FileCheck %s --check-prefix=ZERO-OFF --implicit-check-not=wave.profile
+; RUN: opt -passes=pgo-instr-use,verify 
-pgo-test-profile-file=%t/zero.profdata -pgo-wave-metadata=false -S %t/used.ll 
| FileCheck %s --check-prefix=ZERO-OFF --implicit-check-not=wave.profile
+; RUN: opt -passes=pgo-instr-use,verify 
-pgo-test-profile-file=%t/zero.profdata -S %t/input.ll | FileCheck %s 
--check-prefix=ZERO
+; RUN: opt -passes=pgo-instr-use,verify 
-pgo-test-profile-file=%t/zero.profdata -S %t/used.ll | FileCheck %s 
--check-prefix=NONE --implicit-check-not=wave.profile
+; RUN: %python %t/raw.py no-entry > %t/no-entry.raw
+; RUN: llvm-profdata merge %t/no-entry.raw -o %t/no-entry.profdata
+; RUN: opt -passes=pgo-instr-use,verify 
-pgo-test-profile-file=%t/no-entry.profdata -S %t/input.ll | FileCheck %s 
--check-prefix=NONE --implicit-check-not=wave.profile
+; RUN: opt -passes=pgo-instr-use,verify 
-pgo-test-profile-file=%t/no-entry.profdata -S %t/used.ll | FileCheck %s 
--check-prefix=NONE --implicit-check-not=wave.profile
+; RUN: %python %t/raw.py missing > %t/missing.raw
+; RUN: llvm-profdata merge %t/missing.raw -o %t/missing.profdata
+; RUN: opt -passes=pgo-instr-use,verify 
-pgo-test-profile-file=%t/missing.profdata -pgo-wave-metadata=false -S 
%t/used.ll | FileCheck %s --check-prefixes=NONE,COUNTS 
--implicit-check-not=wave.profile
+; RUN: opt -passes=pgo-instr-use,verify 
-pgo-test-profile-file=%t/missing.profdata -S %t/used.ll | FileCheck %s 
--check-prefix=NONE --implicit-check-not=wave.profile
+; RUN: %python %t/raw.py hash > %t/hash.raw
+; RUN: llvm-profdata merge %t/hash.raw -o %t/hash.profdata
+; RUN: opt -passes=pgo-instr-use,verify 
-pgo-test-profile-file=%t/hash.profdata -pgo-wave-metadata=false -S %t/used.ll 
| FileCheck %s --check-prefixes=NONE,COUNTS --implicit-check-not=wave.profile
+; RUN: opt -passes=pgo-instr-use,verify 
-pgo-test-profile-file=%t/hash.profdata -S %t/used.ll -o %t/hash.ll 2>&1 | 
FileCheck %s --check-prefix=MISMATCH
+; RUN: FileCheck %s --check-prefix=NONE --implicit-check-not=wave.profile < 
%t/hash.ll
+; RUN: %python %t/raw.py bad-lane > %t/bad-lane.raw
+; RUN: llvm-profdata merge %t/bad-lane.raw -o %t/bad-lane.profdata
+; RUN: opt -passes=pgo-instr-use,verify 
-pgo-test-profile-file=%t/bad-lane.profdata -S %t/used.ll -o %t/bad-lane.ll 
2>&1 | FileCheck %s --check-prefix=BAD-LANE
+; RUN: FileCheck %s --check-prefix=NONE --implicit-check-not=wave.profile < 
%t/bad-lane.ll
+; RUN: opt -mtriple=x86_64-unknown-linux-gnu -passes=pgo-instr-use,verify 
-pgo-test-profile-file=%t/wave.profdata -S %t/used.ll | FileCheck %s 
--check-prefix=NONE --implicit-check-not=wave.profile
+; RUN: opt -passes=pgo-instr-gen -pgo-instrument-dense-wave-counts=false 
-pgo-instrument-entry -S %t/critical.ll | FileCheck %s --check-prefix=CGEN
+; RUN: %python %t/raw.py critical > %t/critical.raw
+; RUN: llvm-profdata merge %t/critical.raw -o %t/critical.profdata
+; RUN: opt -passes=pgo-instr-use,verify 
-pgo-test-profile-file=%t/critical.profdata -S %t/critical.ll | FileCheck %s 
--check-prefix=CRIT
+
+; RUN: %python %t/raw.py skew > %t/skew.raw
+; RUN: llvm-profdata merge %t/skew.raw -o %t/skew.profdata
+; RUN: opt -passes=pgo-instr-use,verify 
-pgo-test-profile-file=%t/skew.profdata -S %t/input.ll -o %t/skew.ll
+; RUN: FileCheck %s --check-prefix=SKEW < %t/skew.ll
+; RUN: opt -passes=pgo-instr-use,verify 
-pgo-test-profile-file=%t/skew.profdata -S %t/skew.ll -o %t/reapplied.ll
+; RUN: FileCheck %s --check-prefix=NONE --implicit-check-not=wave.profile < 
%t/reapplied.ll
+; RUN: opt -passes=pgo-instr-use,verify 
-pgo-test-profile-file=%t/skew.profdata -S %t/reapplied.ll | FileCheck %s 
--check-prefix=NONE --implicit-check-not=wave.profile
+; RUN: opt -passes=pgo-instr-use,verify 
-pgo-test-profile-file=%t/wave.profdata -S %t/skew.ll | FileCheck %s 
--check-prefix=NONE --implicit-check-not=wave.profile
+; RUN: %python %t/raw.py skew-lane > %t/skew-lane.raw
+; RUN: llvm-profdata merge %t/skew-lane.raw -o %t/skew-lane.profdata
+; RUN: opt -passes=pgo-instr-use,verify 
-pgo-test-profile-file=%t/skew-lane.profdata -S %t/input.ll -o %t/skew-lane.ll
+; RUN: opt -passes=pgo-instr-use,verify 
-pgo-test-profile-file=%t/skew.profdata -S %t/skew-lane.ll | FileCheck %s 
--check-prefix=NONE --implicit-check-not=wave.profile
+
+; RUN: %python %t/raw.py uniform-wave > %t/uniform.raw
+; RUN: llvm-profdata merge %t/uniform.raw -o %t/uniform.profdata
+; RUN: opt -passes=pgo-instr-use,verify 
-pgo-test-profile-file=%t/uniform.profdata -S %t/input.ll -o %t/uniform.ll
+; RUN: FileCheck %s --check-prefixes=UNIFORM-WAVE,COUNTS < %t/uniform.ll
+; RUN: FileCheck %s --check-prefix=UNIFORM < %t/uniform.ll
+; RUN: opt -passes=pgo-instr-use,verify 
-pgo-test-profile-file=%t/uniform.profdata -pgo-wave-metadata=false -S 
%t/input.ll -o %t/uniform-no-wave.ll
+; RUN: FileCheck %s --check-prefixes=NONE,COUNTS 
--implicit-check-not=wave.profile < %t/uniform-no-wave.ll
+; RUN: FileCheck %s --check-prefix=UNIFORM < %t/uniform-no-wave.ll
+
+; Dense layout is selected from the profile, even with generation disabled.
+; RUN: %python %t/raw.py dense > %t/dense.raw
+; RUN: llvm-profdata merge %t/dense.raw -o %t/dense.profdata
+; RUN: opt -passes=pgo-instr-use,verify 
-pgo-test-profile-file=%t/dense.profdata -S %t/input.ll -o %t/dense.ll
+; RUN: FileCheck %s --check-prefix=DENSE < %t/dense.ll
+; RUN: FileCheck %s --check-prefix=COUNTS < %t/dense.ll
+; RUN: FileCheck %s --check-prefix=UNIFORM < %t/dense.ll
+; RUN: opt -passes=pgo-instr-use,verify 
-pgo-test-profile-file=%t/dense.profdata 
-pgo-instrument-dense-wave-counts=false -S %t/input.ll -o 
%t/dense-disabled-gen.ll
+; RUN: diff %t/dense.ll %t/dense-disabled-gen.ll
+; RUN: opt -passes=pgo-instr-use,verify 
-pgo-test-profile-file=%t/dense.profdata -pgo-wave-metadata=false -S 
%t/input.ll -o %t/dense-off.ll
+; RUN: FileCheck %s --check-prefix=COUNTS --implicit-check-not=wave.profile < 
%t/dense-off.ll
+; RUN: FileCheck %s --check-prefix=UNIFORM < %t/dense-off.ll
+; RUN: %python %t/raw.py dense-no-entry > %t/dense-no-entry.raw
+; RUN: llvm-profdata merge %t/dense-no-entry.raw -o %t/dense-no-entry.profdata
+; RUN: opt -passes=pgo-instr-use,verify 
-pgo-test-profile-file=%t/dense-no-entry.profdata -S %t/input.ll -o 
%t/dense-no-entry.ll
+; RUN: FileCheck %s --check-prefix=DENSE < %t/dense-no-entry.ll
+; RUN: FileCheck %s --check-prefix=COUNTS < %t/dense-no-entry.ll
+; RUN: %python %t/raw.py dense-zero > %t/dense-zero.raw
+; RUN: llvm-profdata merge %t/dense-zero.raw -o %t/dense-zero.profdata
+; RUN: opt -passes=pgo-instr-use,verify 
-pgo-test-profile-file=%t/dense-zero.profdata -S %t/input.ll | FileCheck %s 
--check-prefix=DENSE-ZERO
+
+; All four blocks are measured. Appended zero lane steps do not change select
+; counts or expand uniformity coverage beyond its sparse measurements.
+; DENSE-LABEL: define void @diamond
+; DENSE-SAME: !wave.profile ![[DW:[0-9]+]]
+; DENSE: a:
+; DENSE-NEXT: store volatile i32 1, ptr %p
+; DENSE-NEXT: br label %exit, !wave.profile.block ![[DA:[0-9]+]]
+; DENSE: ret void, !wave.profile.block ![[DX:[0-9]+]]
+; DENSE-DAG: ![[DW]] = distinct !{i64 2, i64 [[DFID:-?[0-9]+]], i64 100, i64 
100, i64 50, i64 100}
+; DENSE-DAG: ![[DA]] = !{i64 2, i64 [[DFID]], i64 1, i64 1, i64 3}
+; DENSE-DAG: ![[DX]] = !{i64 2, i64 [[DFID]], i64 3, i64 1}
+; DENSE-ZERO: distinct !{i64 2, i64 [[ZFID:-?[0-9]+]], i64 0, i64 0, i64 0, 
i64 0}
+; DENSE-ZERO-DAG: !{i64 2, i64 [[ZFID]], i64 1, i64 1, i64 3}
+; DENSE-ZERO-DAG: !{i64 2, i64 [[ZFID]], i64 3, i64 1}
+
+; Wave slots follow instrumentation indices, not IR block order. Only entry and
+; b have block counters. The trailing select counter must not make exit 
measured.
+; Generation checks keep the raw fixture aligned with the actual producer.
+; GEN-LABEL: define void @diamond
+; GEN: entry:
+; GEN: call void @llvm.instrprof.increment({{.*}}i64 
[[HASH:942389667449461396]], i32 3, i32 0)
+; GEN: b:
+; GEN: call void @llvm.instrprof.increment({{.*}}i64 [[HASH]], i32 3, i32 1)
+; GEN: exit:
+; GEN: call void @llvm.instrprof.increment.step({{.*}}i64 [[HASH]], i32 3, i32 
2,
+
+; A measured zero is retained even when all lane counters are zero. Blocks with
+; no measurements remain unavailable. Replacing a profile clears stale metadata
+; for absent, incompatible, lane-only, and unsupported-target records.
+; NONE: define void @diamond
+; UNIFORM-WAVE: define void @diamond{{.*}}!wave.profile
+; WAVE-LABEL: define void @diamond
+; WAVE-SAME: !wave.profile ![[WAVES:[0-9]+]]
+; WAVE: br i1 %cond, label %a, label %b, !prof 
![[WEIGHTS:[0-9]+]]{{.*}}!wave.profile.block ![[ENTRY:[0-9]+]]
+; WAVE: br label %exit, !wave.profile.block ![[A:[0-9]+]]
+; WAVE: br label %exit, {{.*}}!wave.profile.block ![[B:[0-9]+]]
+; WAVE: ret void, !wave.profile.block ![[EXIT:[0-9]+]]
+; WAVE-DAG: ![[WAVES]] = distinct !{i64 2, i64 [[FID:-?[0-9]+]], i64 100, i64 
0, i64 100, i64 0}
+; WAVE-DAG: ![[ENTRY]] = !{i64 2, i64 [[FID]], i64 0, i64 1, i64 1, i64 2}
+; WAVE-DAG: ![[A]] = !{i64 2, i64 [[FID]], i64 1, i64 0, i64 3}
+; WAVE-DAG: ![[B]] = !{i64 2, i64 [[FID]], i64 2, i64 1, i64 3}
+; WAVE-DAG: ![[EXIT]] = !{i64 2, i64 [[FID]], i64 3, i64 0}
+; UNIFORM-WAVE-DAG: !{i64 2, i64 {{-?[0-9]+}}, i64 100, i64 0, i64 50, i64 0}
+; COUNTS-DAG: !{!"function_entry_count", i64 6400}
+; COUNTS-DAG: !{!"branch_weights", i32 3200, i32 3200}
+; COUNTS-DAG: !{!"branch_weights", i32 1600, i32 4800}
+; ZERO-LABEL: define void @diamond
+; ZERO-SAME: !wave.profile ![[WAVES:[0-9]+]]
+; ZERO: br i1 %cond, label %a, label %b, {{.*}}!wave.profile.block 
![[ENTRY:[0-9]+]]
+; ZERO: br label %exit, !wave.profile.block ![[A:[0-9]+]]
+; ZERO: br label %exit, {{.*}}!wave.profile.block ![[B:[0-9]+]]
+; ZERO: ret void, !wave.profile.block ![[EXIT:[0-9]+]]
+; ZERO-DAG: ![[WAVES]] = distinct !{i64 2, i64 [[FID:-?[0-9]+]], i64 0, i64 0, 
i64 0, i64 0}
+; ZERO-DAG: ![[ENTRY]] = !{i64 2, i64 [[FID]], i64 0, i64 1, i64 1, i64 2}
+; ZERO-DAG: ![[A]] = !{i64 2, i64 [[FID]], i64 1, i64 0, i64 3}
+; ZERO-DAG: ![[B]] = !{i64 2, i64 [[FID]], i64 2, i64 1, i64 3}
+; ZERO-DAG: ![[EXIT]] = !{i64 2, i64 [[FID]], i64 3, i64 0}
+; ZERO-OFF: define void @diamond
+; ZERO-OFF: !{!"function_entry_count", i64 0}
+
+; The wave metadata switch leaves function, block and branch uniformity hints
+; available, together with the ordinary entry, branch and select counts.
+; UNIFORM: define void @diamond({{.*}}!uniformity.profile
+; UNIFORM: br i1 %cond, 
{{.*}}!block.uniformity.profile{{.*}}!branch.uniformity.profile
+; UNIFORM: b:
+; UNIFORM: br label %exit, {{.*}}!block.uniformity.profile
+
+; WARN: Inconsistent number of wave counts in diamond; ignoring wave profile
+; MISMATCH: function control flow change detected (hash mismatch) diamond
+; BAD-LANE: Inconsistent number of counts in diamond
+
+; Existing branch weights can change the instrumentation MST without changing
+; the CFG hash. Reapplying this skewed profile would move counter 1 from b to 
a.
+; Drop wave metadata on previously profiled functions, including after a 
lane-only
+; load, and do not recreate it on a subsequent load.
+; SKEW-LABEL: define void @diamond
+; SKEW-SAME: !wave.profile ![[WAVES:[0-9]+]]
+; SKEW: br i1 %cond, label %a, label %b, !prof ![[WEIGHTS:[0-9]+]]
+; SKEW: a:
+; SKEW: br label %exit, !wave.profile.block ![[A:[0-9]+]]
+; SKEW: b:
+; SKEW: br label %exit, {{.*}}!wave.profile.block ![[B:[0-9]+]]
+; SKEW-DAG: ![[WAVES]] = distinct !{i64 2, i64 [[FID:-?[0-9]+]], i64 100, i64 
0, i64 90, i64 0}
+; SKEW-DAG: ![[WEIGHTS]] = !{!"branch_weights", i32 640, i32 5760}
+; SKEW-DAG: ![[A]] = !{i64 2, i64 [[FID]], i64 1, i64 0, i64 3}
+; SKEW-DAG: ![[B]] = !{i64 2, i64 [[FID]], i64 2, i64 1, i64 3}
+
+; Critical edges are split before mapping counter indices to block identities.
+; CGEN-LABEL: define void @critical
+; CGEN: entry:
+; CGEN: call void @llvm.instrprof.increment({{.*}}i64 
[[CHASH:784007059655560962]], i32 2, i32 0)
+; CGEN: entry.exit_crit_edge:
+; CGEN: call void @llvm.instrprof.increment({{.*}}i64 [[CHASH]], i32 2, i32 1)
+; CRIT-LABEL: define void @critical
+; CRIT-SAME: !wave.profile ![[CWAVES:[0-9]+]]
+; CRIT: br i1 %cond, label %a, label %entry.exit_crit_edge, 
{{.*}}!wave.profile.block ![[CENT:[0-9]+]]
+; CRIT: entry.exit_crit_edge:
+; CRIT: br label %exit, {{.*}}!wave.profile.block ![[CEDGE:[0-9]+]]
+; CRIT: a:
+; CRIT: br label %exit, !wave.profile.block ![[CA:[0-9]+]]
+; CRIT: ret void, !wave.profile.block ![[CEXIT:[0-9]+]]
+; CRIT-DAG: ![[CWAVES]] = distinct !{i64 2, i64 [[CFID:-?[0-9]+]], i64 100, 
i64 25, i64 0, i64 0}
+; CRIT-DAG: ![[CENT]] = !{i64 2, i64 [[CFID]], i64 0, i64 1, i64 2, i64 1}
+; CRIT-DAG: ![[CEDGE]] = !{i64 2, i64 [[CFID]], i64 1, i64 1, i64 3}
+; CRIT-DAG: ![[CA]] = !{i64 2, i64 [[CFID]], i64 2, i64 0, i64 3}
+; CRIT-DAG: ![[CEXIT]] = !{i64 2, i64 [[CFID]], i64 3, i64 0}
+
+;--- input.ll
+source_filename = "wave-profile-use.ll"
+target triple = "amdgcn-amd-amdhsa"
+define void @diamond(i1 %cond, i1 %select_cond, ptr %p) {
+entry:
+  br i1 %cond, label %a, label %b
+a:
+  store volatile i32 1, ptr %p
+  br label %exit
+b:
+  store volatile i32 2, ptr %p
+  br label %exit
+exit:
+  %value = select i1 %select_cond, i32 1, i32 2
+  store volatile i32 %value, ptr %p
+  ret void
+}
+
+;--- critical.ll
+source_filename = "wave-profile-critical.ll"
+target triple = "amdgcn-amd-amdhsa"
+define void @critical(i1 %cond, ptr %p) {
+entry:
+  br i1 %cond, label %a, label %exit, !prof !0
+a:
+  store volatile i32 1, ptr %p
+  br label %exit
+exit:
+  ret void
+}
+
+!0 = !{!"branch_weights", i32 1000000, i32 1}
+
+;--- raw.py
+import hashlib
+import struct
+import sys
+
+mode = sys.argv[1]
+name = {"missing": b"missing", "critical": b"critical"}.get(mode, b"diamond")
+name_ref = int.from_bytes(hashlib.md5(name).digest()[:8], "little")
+# Hash emitted by pgo-instr-gen for input.ll; raw v12 has an 80-byte record.
+func_hash = 942389667449461396
+if mode == "hash":
+    func_hash += 1
+lanes = [6400, 3200, 1600]
+waves = [100, 100, 100]
+if mode == "uniform-wave":
+    waves[1] = 50
+if mode == "critical":
+    func_hash = 784007059655560962
+    lanes = [6400, 800]
+    waves = [100, 25]
+if mode in ("skew", "skew-lane"):
+    lanes = [6400, 5760, 1600]
+    waves = [100, 90, 100]
+if mode in ("lane", "skew-lane"):
+    waves = []
+elif mode == "bad-wave":
+    waves.pop()
+elif mode == "bad-lane":
+    lanes.pop()
+elif mode == "zero":
+    lanes = [0, 0, 0]
+    waves = [0, 0, 0]
+version = 12 | (1 << 56)
+if mode != "no-entry":
+    version |= 1 << 58
+if mode.startswith("dense"):
+    version |= 1 << 54
+    lanes = [6400, 3200, 1600, 0, 0]
+    waves = [100, 50, 100, 100, 100]
+    if mode == "dense-no-entry":
+        version &= ~(1 << 58)
+        lanes[0] = 3200
+    if mode == "dense-zero":
+        lanes = [0] * 5
+        waves = [0] * 5
+num_counters = len(lanes) + len(waves)
+counter_delta = 80
+uniform_delta = counter_delta + num_counters * 8
+names_delta = uniform_delta + len(lanes) * 8
+names = bytes([len(name), 0]) + name
+header = [0xff6c70726f667281, version, 0, 1, 0, num_counters, 0,
+          0, 0, len(lanes), 0, uniform_delta, len(names), counter_delta,
+          uniform_delta, names_delta, 0, 0, 2]
+record = struct.pack("<7QI4HII4x", name_ref, func_hash, counter_delta,
+                     uniform_delta, 0, 0, 0, num_counters, 0, 0, 0, 64, 0,
+                     len(waves))
+uniform = lanes if mode in ("uniform-wave", "dense") else [0] * len(lanes)
+counts = lanes + waves + uniform
+sys.stdout.buffer.write(struct.pack("<19Q", *header) + record +
+                        struct.pack("<" + "Q" * len(counts), *counts) +
+                        names + bytes((-len(names)) % 8))

_______________________________________________
llvm-branch-commits mailing list
[email protected]
https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits

Reply via email to