https://github.com/yxsamliu updated 
https://github.com/llvm/llvm-project/pull/221517

>From b5fc30b4d6bf6fd2dcdef7bf2aeb872706544cb7 Mon Sep 17 00:00:00 2001
From: "Yaxun (Sam) Liu" <[email protected]>
Date: Wed, 28 Oct 2026 16:47:54 +0000
Subject: [PATCH] [CHR] Use branch uniformity profiles to select scopes

Lane counts can make a divergent GPU branch look strongly biased. For
example, if 63 of 64 lanes take one path in every wave, that path has a
high count even though every wave also executes the other path. CHR
may duplicate control flow and increase register pressure without
avoiding the work on the other path.

When a function has uniformity.profile, require branch.uniformity.profile
on each biased branch used to form a candidate CHR scope. Skip the
scope if any such branch lacks the hint. Missing hints can also mean
missing observations, so this is a conservative profitability decision.

Functions without uniformity data and scopes containing only selects
retain their existing behavior.
---
 .../ControlHeightReduction.cpp                |  37 ++++
 .../chr-uniformity-profile-amdgpu.ll          |   5 +
 .../PGOProfile/chr-uniformity-profile.ll      | 174 ++++++++++++++++++
 3 files changed, 216 insertions(+)
 create mode 100644 
llvm/test/Transforms/PGOProfile/chr-uniformity-profile-amdgpu.ll
 create mode 100644 llvm/test/Transforms/PGOProfile/chr-uniformity-profile.ll

diff --git a/llvm/lib/Transforms/Instrumentation/ControlHeightReduction.cpp 
b/llvm/lib/Transforms/Instrumentation/ControlHeightReduction.cpp
index 8984e6cbdc6f62..67f4ffa543b9c4 100644
--- a/llvm/lib/Transforms/Instrumentation/ControlHeightReduction.cpp
+++ b/llvm/lib/Transforms/Instrumentation/ControlHeightReduction.cpp
@@ -356,6 +356,11 @@ class CHR {
   OptimizationRemarkEmitter &ORE;
   CHRStats Stats;
 
+  // A function attachment indicates that uniformity profile data is available.
+  // This preserves CHR's existing behavior when no uniformity profile was
+  // collected.
+  bool HasUniformityProfile = 
F.getMetadata(LLVMContext::MD_uniformity_profile);
+
   // All the true-biased regions in the function
   DenseSet<Region *> TrueBiasedRegionsGlobal;
   // All the false-biased regions in the function
@@ -1307,9 +1312,41 @@ static bool hasAtLeastTwoBiasedBranches(const 
InstrumentationOptions &Opts,
   return NumBiased >= Opts.chr_merge_threshold;
 }
 
+static Instruction *
+findBranchWithoutUniformityProfile(const DenseSet<Region *> &Regions) {
+  for (Region *R : Regions) {
+    Instruction *Branch = R->getEntry()->getTerminator();
+    if (!Branch->getMetadata(LLVMContext::MD_branch_uniformity_profile))
+      return Branch;
+  }
+  return nullptr;
+}
+
 void CHR::filterScopes(SmallVectorImpl<CHRScope *> &Input,
                        SmallVectorImpl<CHRScope *> &Output) {
   for (CHRScope *Scope : Input) {
+    // Branch weights count individual lanes. A divergent branch can therefore
+    // look strongly biased even when every wave executes both paths. Avoid
+    // applying CHR to such a scope because duplicating its control flow may
+    // increase live ranges and register pressure without eliminating control
+    // flow for the wave.
+    if (HasUniformityProfile) {
+      Instruction *BranchWithoutUniformityProfile =
+          findBranchWithoutUniformityProfile(Scope->TrueBiasedRegions);
+      if (!BranchWithoutUniformityProfile)
+        BranchWithoutUniformityProfile =
+            findBranchWithoutUniformityProfile(Scope->FalseBiasedRegions);
+      if (BranchWithoutUniformityProfile) {
+        ORE.emit([&]() {
+          return OptimizationRemarkMissed(DEBUG_TYPE, "BranchNotKnownUniform",
+                                          BranchWithoutUniformityProfile)
+                 << "drop scope containing a branch that is not known to be "
+                    "uniform";
+        });
+        continue;
+      }
+    }
+
     // Filter out the ones with only one region and no subs.
     if (!hasAtLeastTwoBiasedBranches(Opts, Scope)) {
       CHR_DEBUG(dbgs() << "Filtered out by biased branches truthy-regions "
diff --git a/llvm/test/Transforms/PGOProfile/chr-uniformity-profile-amdgpu.ll 
b/llvm/test/Transforms/PGOProfile/chr-uniformity-profile-amdgpu.ll
new file mode 100644
index 00000000000000..f8552e2ce64196
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/chr-uniformity-profile-amdgpu.ll
@@ -0,0 +1,5 @@
+; REQUIRES: amdgpu-registered-target
+; RUN: opt < %S/chr-uniformity-profile.ll -mtriple=amdgpu9.50-amd-amdhsa 
-passes='default<O3>' -S | FileCheck %S/chr-uniformity-profile.ll
+
+; Run the shared scope checks with the AMDGPU O3 pipeline. Keep this separately
+; gated so the target-independent checks also run in builds without AMDGPU.
diff --git a/llvm/test/Transforms/PGOProfile/chr-uniformity-profile.ll 
b/llvm/test/Transforms/PGOProfile/chr-uniformity-profile.ll
new file mode 100644
index 00000000000000..056637ffca88b0
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/chr-uniformity-profile.ll
@@ -0,0 +1,174 @@
+; RUN: opt < %s 
-passes='require<profile-summary>,function(chr,instcombine,simplifycfg)' -S | 
FileCheck %s
+; RUN: opt < %s -passes='default<O3>' -S | FileCheck %s
+
+declare void @foo()
+
+; Preserve the existing behavior when a function has no uniformity profile.
+define void @no_profile(ptr %ptr) !prof !14 {
+; CHECK-LABEL: define void @no_profile(
+; CHECK: entry.split.nonchr:
+entry:
+  %value = load i32, ptr %ptr
+  %bit0 = and i32 %value, 1
+  %cond0 = icmp eq i32 %bit0, 0
+  br i1 %cond0, label %bb1, label %bb0, !prof !15
+
+bb0:
+  call void @foo()
+  br label %bb1
+
+bb1:
+  %bit1 = and i32 %value, 2
+  %cond1 = icmp eq i32 %bit1, 0
+  br i1 %cond1, label %exit, label %bb2, !prof !15
+
+bb2:
+  call void @foo()
+  br label %exit
+
+exit:
+  ret void
+}
+
+; Uniform branches remain eligible for CHR when uniformity profile is
+; present.
+define void @uniform(ptr %ptr) !prof !14 !uniformity.profile !16 {
+; CHECK-LABEL: define void @uniform(
+; CHECK: entry.split.nonchr:
+entry:
+  %value = load i32, ptr %ptr
+  %bit0 = and i32 %value, 1
+  %cond0 = icmp eq i32 %bit0, 0
+  br i1 %cond0, label %bb1, label %bb0, !prof !15,
+      !branch.uniformity.profile !16
+
+bb0:
+  call void @foo()
+  br label %bb1
+
+bb1:
+  %bit1 = and i32 %value, 2
+  %cond1 = icmp eq i32 %bit1, 0
+  br i1 %cond1, label %exit, label %bb2, !prof !15,
+      !branch.uniformity.profile !16
+
+bb2:
+  call void @foo()
+  br label %exit
+
+exit:
+  ret void
+}
+
+; With uniformity profile, a branch without branch-uniformity metadata
+; is not known to be uniform. Do not apply CHR to a scope containing one.
+define void @not_known_uniform(ptr %ptr) !prof !14 !uniformity.profile !16 {
+; CHECK-LABEL: define void @not_known_uniform(
+; CHECK-NOT: split
+; CHECK: ret void
+entry:
+  %value = load i32, ptr %ptr
+  %bit0 = and i32 %value, 1
+  %cond0 = icmp eq i32 %bit0, 0
+  br i1 %cond0, label %bb1, label %bb0, !prof !15,
+      !branch.uniformity.profile !16
+
+bb0:
+  call void @foo()
+  br label %bb1
+
+bb1:
+  %bit1 = and i32 %value, 2
+  %cond1 = icmp eq i32 %bit1, 0
+  br i1 %cond1, label %exit, label %bb2, !prof !15
+
+bb2:
+  call void @foo()
+  br label %exit
+
+exit:
+  ret void
+}
+
+; A scope with a branch that is not known uniform does not disable a separate
+; uniform scope in the same function.
+define void @per_scope(ptr %uniform_ptr, ptr %divergent_ptr) !prof !14 
!uniformity.profile !16 {
+; CHECK-LABEL: define void @per_scope(
+; CHECK: entry.split.nonchr:
+; CHECK-NOT: after.uniform.split.nonchr:
+; CHECK: after.uniform:
+entry:
+  %uniform_value = load i32, ptr %uniform_ptr
+  %uniform_bit0 = and i32 %uniform_value, 1
+  %uniform_cond0 = icmp eq i32 %uniform_bit0, 0
+  br i1 %uniform_cond0, label %uniform.bb1, label %uniform.bb0, !prof !15,
+      !branch.uniformity.profile !16
+
+uniform.bb0:
+  call void @foo()
+  br label %uniform.bb1
+
+uniform.bb1:
+  %uniform_bit1 = and i32 %uniform_value, 2
+  %uniform_cond1 = icmp eq i32 %uniform_bit1, 0
+  br i1 %uniform_cond1, label %after.uniform, label %uniform.bb2, !prof !15,
+      !branch.uniformity.profile !16
+
+uniform.bb2:
+  call void @foo()
+  br label %after.uniform
+
+after.uniform:
+  %divergent_value = load i32, ptr %divergent_ptr
+  %divergent_bit0 = and i32 %divergent_value, 1
+  %divergent_cond0 = icmp eq i32 %divergent_bit0, 0
+  br i1 %divergent_cond0, label %divergent.bb1, label %divergent.bb0,
+      !prof !15
+
+divergent.bb0:
+  call void @foo()
+  br label %divergent.bb1
+
+divergent.bb1:
+  %divergent_bit1 = and i32 %divergent_value, 2
+  %divergent_cond1 = icmp eq i32 %divergent_bit1, 0
+  br i1 %divergent_cond1, label %exit, label %divergent.bb2, !prof !15
+
+divergent.bb2:
+  call void @foo()
+  br label %exit
+
+exit:
+  ret void
+}
+
+; Scopes containing only non-Boolean selects keep their existing behavior.
+define i32 @select_only(i32 %value) !prof !14 !uniformity.profile !16 {
+; CHECK-LABEL: define i32 @select_only(
+; CHECK: entry.split.nonchr:
+entry:
+  %bit0 = and i32 %value, 1
+  %cond0 = icmp eq i32 %bit0, 0
+  %sum0 = select i1 %cond0, i32 %value, i32 42, !prof !15
+  %bit1 = and i32 %value, 2
+  %cond1 = icmp eq i32 %bit1, 0
+  %sum1 = select i1 %cond1, i32 %sum0, i32 43, !prof !15
+  ret i32 %sum1
+}
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 1, !"ProfileSummary", !1}
+!1 = !{!2, !3, !4, !5, !6, !7, !8, !9}
+!2 = !{!"ProfileFormat", !"InstrProf"}
+!3 = !{!"TotalCount", i64 10000}
+!4 = !{!"MaxCount", i64 10}
+!5 = !{!"MaxInternalCount", i64 1}
+!6 = !{!"MaxFunctionCount", i64 1000}
+!7 = !{!"NumCounts", i64 1}
+!8 = !{!"NumFunctions", i64 1}
+!9 = !{!"DetailedSummary", !10}
+!10 = !{!11}
+!11 = !{i32 999999, i64 1, i32 1}
+!14 = !{!"function_entry_count", i64 100}
+!15 = !{!"branch_weights", i32 0, i32 1}
+!16 = !{}

_______________________________________________
llvm-branch-commits mailing list
[email protected]
https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits

Reply via email to