https://github.com/vpykhtin updated https://github.com/llvm/llvm-project/pull/71556
>From 1d71de2784084e75ed2b2f3bd1ad042d8801c226 Mon Sep 17 00:00:00 2001 From: Valery Pykhtin <valery.pykh...@gmail.com> Date: Mon, 20 Nov 2023 15:20:03 +0100 Subject: [PATCH 1/6] move tests to the appropriate place --- .../GlobalISel/llvm.amdgcn.ballot.i32.ll | 133 ++++---------- .../CodeGen/AMDGPU/llvm.amdgcn.ballot.i32.ll | 173 ++++++------------ .../AMDGPU/llvm.amdgcn.ballot.i64.wave32.ll | 70 +++++++ 3 files changed, 169 insertions(+), 207 deletions(-) diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ballot.i32.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ballot.i32.ll index d641063984eb810..34ff84d5aac36a4 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ballot.i32.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.ballot.i32.ll @@ -3,7 +3,6 @@ ; RUN: llc -march=amdgcn -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 -mattr=+wavefrontsize32,-wavefrontsize64 -global-isel -verify-machineinstrs < %s | FileCheck %s declare i32 @llvm.amdgcn.ballot.i32(i1) -declare i64 @llvm.amdgcn.ballot.i64(i1) declare i32 @llvm.ctpop.i32(i32) ; Test ballot(0) @@ -204,30 +203,6 @@ false: ret i32 33 } -define amdgpu_cs i32 @branch_divergent_ballot64_ne_zero_compare(i32 %v) { -; CHECK-LABEL: branch_divergent_ballot64_ne_zero_compare: -; CHECK: ; %bb.0: -; CHECK-NEXT: v_cmp_gt_u32_e64 s0, 12, v0 -; CHECK-NEXT: s_mov_b32 s1, 0 -; CHECK-NEXT: s_cmp_eq_u64 s[0:1], 0 -; CHECK-NEXT: s_cbranch_scc1 .LBB12_2 -; CHECK-NEXT: ; %bb.1: ; %true -; CHECK-NEXT: s_mov_b32 s0, 42 -; CHECK-NEXT: s_branch .LBB12_3 -; CHECK-NEXT: .LBB12_2: ; %false -; CHECK-NEXT: s_mov_b32 s0, 33 -; CHECK-NEXT: s_branch .LBB12_3 -; CHECK-NEXT: .LBB12_3: - %c = icmp ult i32 %v, 12 - %ballot = call i64 @llvm.amdgcn.ballot.i64(i1 %c) - %ballot_ne_zero = icmp ne i64 %ballot, 0 - br i1 %ballot_ne_zero, label %true, label %false -true: - ret i32 42 -false: - ret i32 33 -} - define amdgpu_cs i32 @branch_uniform_ballot_ne_zero_compare(i32 inreg %v) { ; CHECK-LABEL: branch_uniform_ballot_ne_zero_compare: ; CHECK: ; %bb.0: @@ -236,14 +211,14 @@ define amdgpu_cs i32 @branch_uniform_ballot_ne_zero_compare(i32 inreg %v) { ; CHECK-NEXT: s_and_b32 s0, 1, s0 ; CHECK-NEXT: v_cmp_ne_u32_e64 s0, 0, s0 ; CHECK-NEXT: s_cmp_eq_u32 s0, 0 -; CHECK-NEXT: s_cbranch_scc1 .LBB13_2 +; CHECK-NEXT: s_cbranch_scc1 .LBB12_2 ; CHECK-NEXT: ; %bb.1: ; %true ; CHECK-NEXT: s_mov_b32 s0, 42 -; CHECK-NEXT: s_branch .LBB13_3 -; CHECK-NEXT: .LBB13_2: ; %false +; CHECK-NEXT: s_branch .LBB12_3 +; CHECK-NEXT: .LBB12_2: ; %false ; CHECK-NEXT: s_mov_b32 s0, 33 -; CHECK-NEXT: s_branch .LBB13_3 -; CHECK-NEXT: .LBB13_3: +; CHECK-NEXT: s_branch .LBB12_3 +; CHECK-NEXT: .LBB12_3: %c = icmp ult i32 %v, 12 %ballot = call i32 @llvm.amdgcn.ballot.i32(i1 %c) %ballot_ne_zero = icmp ne i32 %ballot, 0 @@ -259,14 +234,14 @@ define amdgpu_cs i32 @branch_divergent_ballot_eq_zero_compare(i32 %v) { ; CHECK: ; %bb.0: ; CHECK-NEXT: v_cmp_gt_u32_e32 vcc_lo, 12, v0 ; CHECK-NEXT: s_cmp_lg_u32 vcc_lo, 0 -; CHECK-NEXT: s_cbranch_scc0 .LBB14_2 +; CHECK-NEXT: s_cbranch_scc0 .LBB13_2 ; CHECK-NEXT: ; %bb.1: ; %false ; CHECK-NEXT: s_mov_b32 s0, 33 -; CHECK-NEXT: s_branch .LBB14_3 -; CHECK-NEXT: .LBB14_2: ; %true +; CHECK-NEXT: s_branch .LBB13_3 +; CHECK-NEXT: .LBB13_2: ; %true ; CHECK-NEXT: s_mov_b32 s0, 42 -; CHECK-NEXT: s_branch .LBB14_3 -; CHECK-NEXT: .LBB14_3: +; CHECK-NEXT: s_branch .LBB13_3 +; CHECK-NEXT: .LBB13_3: %c = icmp ult i32 %v, 12 %ballot = call i32 @llvm.amdgcn.ballot.i32(i1 %c) %ballot_eq_zero = icmp eq i32 %ballot, 0 @@ -285,14 +260,14 @@ define amdgpu_cs i32 @branch_uniform_ballot_eq_zero_compare(i32 inreg %v) { ; CHECK-NEXT: s_and_b32 s0, 1, s0 ; CHECK-NEXT: v_cmp_ne_u32_e64 s0, 0, s0 ; CHECK-NEXT: s_cmp_lg_u32 s0, 0 -; CHECK-NEXT: s_cbranch_scc0 .LBB15_2 +; CHECK-NEXT: s_cbranch_scc0 .LBB14_2 ; CHECK-NEXT: ; %bb.1: ; %false ; CHECK-NEXT: s_mov_b32 s0, 33 -; CHECK-NEXT: s_branch .LBB15_3 -; CHECK-NEXT: .LBB15_2: ; %true +; CHECK-NEXT: s_branch .LBB14_3 +; CHECK-NEXT: .LBB14_2: ; %true ; CHECK-NEXT: s_mov_b32 s0, 42 -; CHECK-NEXT: s_branch .LBB15_3 -; CHECK-NEXT: .LBB15_3: +; CHECK-NEXT: s_branch .LBB14_3 +; CHECK-NEXT: .LBB14_3: %c = icmp ult i32 %v, 12 %ballot = call i32 @llvm.amdgcn.ballot.i32(i1 %c) %ballot_eq_zero = icmp eq i32 %ballot, 0 @@ -310,14 +285,14 @@ define amdgpu_cs i32 @branch_divergent_ballot_ne_zero_and(i32 %v1, i32 %v2) { ; CHECK-NEXT: v_cmp_lt_u32_e64 s0, 34, v1 ; CHECK-NEXT: s_and_b32 s0, vcc_lo, s0 ; CHECK-NEXT: s_cmp_eq_u32 s0, 0 -; CHECK-NEXT: s_cbranch_scc1 .LBB16_2 +; CHECK-NEXT: s_cbranch_scc1 .LBB15_2 ; CHECK-NEXT: ; %bb.1: ; %true ; CHECK-NEXT: s_mov_b32 s0, 42 -; CHECK-NEXT: s_branch .LBB16_3 -; CHECK-NEXT: .LBB16_2: ; %false +; CHECK-NEXT: s_branch .LBB15_3 +; CHECK-NEXT: .LBB15_2: ; %false ; CHECK-NEXT: s_mov_b32 s0, 33 -; CHECK-NEXT: s_branch .LBB16_3 -; CHECK-NEXT: .LBB16_3: +; CHECK-NEXT: s_branch .LBB15_3 +; CHECK-NEXT: .LBB15_3: %v1c = icmp ult i32 %v1, 12 %v2c = icmp ugt i32 %v2, 34 %c = and i1 %v1c, %v2c @@ -330,34 +305,6 @@ false: ret i32 33 } -define amdgpu_cs i32 @branch_divergent_ballot64_ne_zero_and(i32 %v1, i32 %v2) { -; CHECK-LABEL: branch_divergent_ballot64_ne_zero_and: -; CHECK: ; %bb.0: -; CHECK-NEXT: v_cmp_gt_u32_e32 vcc_lo, 12, v0 -; CHECK-NEXT: v_cmp_lt_u32_e64 s0, 34, v1 -; CHECK-NEXT: s_mov_b32 s1, 0 -; CHECK-NEXT: s_and_b32 s0, vcc_lo, s0 -; CHECK-NEXT: s_cmp_eq_u64 s[0:1], 0 -; CHECK-NEXT: s_cbranch_scc1 .LBB17_2 -; CHECK-NEXT: ; %bb.1: ; %true -; CHECK-NEXT: s_mov_b32 s0, 42 -; CHECK-NEXT: s_branch .LBB17_3 -; CHECK-NEXT: .LBB17_2: ; %false -; CHECK-NEXT: s_mov_b32 s0, 33 -; CHECK-NEXT: s_branch .LBB17_3 -; CHECK-NEXT: .LBB17_3: - %v1c = icmp ult i32 %v1, 12 - %v2c = icmp ugt i32 %v2, 34 - %c = and i1 %v1c, %v2c - %ballot = call i64 @llvm.amdgcn.ballot.i64(i1 %c) - %ballot_ne_zero = icmp ne i64 %ballot, 0 - br i1 %ballot_ne_zero, label %true, label %false -true: - ret i32 42 -false: - ret i32 33 -} - define amdgpu_cs i32 @branch_uniform_ballot_ne_zero_and(i32 inreg %v1, i32 inreg %v2) { ; CHECK-LABEL: branch_uniform_ballot_ne_zero_and: ; CHECK: ; %bb.0: @@ -369,14 +316,14 @@ define amdgpu_cs i32 @branch_uniform_ballot_ne_zero_and(i32 inreg %v1, i32 inreg ; CHECK-NEXT: s_and_b32 s0, 1, s0 ; CHECK-NEXT: v_cmp_ne_u32_e64 s0, 0, s0 ; CHECK-NEXT: s_cmp_eq_u32 s0, 0 -; CHECK-NEXT: s_cbranch_scc1 .LBB18_2 +; CHECK-NEXT: s_cbranch_scc1 .LBB16_2 ; CHECK-NEXT: ; %bb.1: ; %true ; CHECK-NEXT: s_mov_b32 s0, 42 -; CHECK-NEXT: s_branch .LBB18_3 -; CHECK-NEXT: .LBB18_2: ; %false +; CHECK-NEXT: s_branch .LBB16_3 +; CHECK-NEXT: .LBB16_2: ; %false ; CHECK-NEXT: s_mov_b32 s0, 33 -; CHECK-NEXT: s_branch .LBB18_3 -; CHECK-NEXT: .LBB18_3: +; CHECK-NEXT: s_branch .LBB16_3 +; CHECK-NEXT: .LBB16_3: %v1c = icmp ult i32 %v1, 12 %v2c = icmp ugt i32 %v2, 34 %c = and i1 %v1c, %v2c @@ -396,14 +343,14 @@ define amdgpu_cs i32 @branch_divergent_ballot_eq_zero_and(i32 %v1, i32 %v2) { ; CHECK-NEXT: v_cmp_lt_u32_e64 s0, 34, v1 ; CHECK-NEXT: s_and_b32 s0, vcc_lo, s0 ; CHECK-NEXT: s_cmp_lg_u32 s0, 0 -; CHECK-NEXT: s_cbranch_scc0 .LBB19_2 +; CHECK-NEXT: s_cbranch_scc0 .LBB17_2 ; CHECK-NEXT: ; %bb.1: ; %false ; CHECK-NEXT: s_mov_b32 s0, 33 -; CHECK-NEXT: s_branch .LBB19_3 -; CHECK-NEXT: .LBB19_2: ; %true +; CHECK-NEXT: s_branch .LBB17_3 +; CHECK-NEXT: .LBB17_2: ; %true ; CHECK-NEXT: s_mov_b32 s0, 42 -; CHECK-NEXT: s_branch .LBB19_3 -; CHECK-NEXT: .LBB19_3: +; CHECK-NEXT: s_branch .LBB17_3 +; CHECK-NEXT: .LBB17_3: %v1c = icmp ult i32 %v1, 12 %v2c = icmp ugt i32 %v2, 34 %c = and i1 %v1c, %v2c @@ -427,14 +374,14 @@ define amdgpu_cs i32 @branch_uniform_ballot_eq_zero_and(i32 inreg %v1, i32 inreg ; CHECK-NEXT: s_and_b32 s0, 1, s0 ; CHECK-NEXT: v_cmp_ne_u32_e64 s0, 0, s0 ; CHECK-NEXT: s_cmp_lg_u32 s0, 0 -; CHECK-NEXT: s_cbranch_scc0 .LBB20_2 +; CHECK-NEXT: s_cbranch_scc0 .LBB18_2 ; CHECK-NEXT: ; %bb.1: ; %false ; CHECK-NEXT: s_mov_b32 s0, 33 -; CHECK-NEXT: s_branch .LBB20_3 -; CHECK-NEXT: .LBB20_2: ; %true +; CHECK-NEXT: s_branch .LBB18_3 +; CHECK-NEXT: .LBB18_2: ; %true ; CHECK-NEXT: s_mov_b32 s0, 42 -; CHECK-NEXT: s_branch .LBB20_3 -; CHECK-NEXT: .LBB20_3: +; CHECK-NEXT: s_branch .LBB18_3 +; CHECK-NEXT: .LBB18_3: %v1c = icmp ult i32 %v1, 12 %v2c = icmp ugt i32 %v2, 34 %c = and i1 %v1c, %v2c @@ -455,14 +402,14 @@ define amdgpu_cs i32 @branch_uniform_ballot_sgt_N_compare(i32 inreg %v) { ; CHECK-NEXT: s_and_b32 s0, 1, s0 ; CHECK-NEXT: v_cmp_ne_u32_e64 s0, 0, s0 ; CHECK-NEXT: s_cmp_le_i32 s0, 22 -; CHECK-NEXT: s_cbranch_scc1 .LBB21_2 +; CHECK-NEXT: s_cbranch_scc1 .LBB19_2 ; CHECK-NEXT: ; %bb.1: ; %true ; CHECK-NEXT: s_mov_b32 s0, 42 -; CHECK-NEXT: s_branch .LBB21_3 -; CHECK-NEXT: .LBB21_2: ; %false +; CHECK-NEXT: s_branch .LBB19_3 +; CHECK-NEXT: .LBB19_2: ; %false ; CHECK-NEXT: s_mov_b32 s0, 33 -; CHECK-NEXT: s_branch .LBB21_3 -; CHECK-NEXT: .LBB21_3: +; CHECK-NEXT: s_branch .LBB19_3 +; CHECK-NEXT: .LBB19_3: %c = icmp ult i32 %v, 12 %ballot = call i32 @llvm.amdgcn.ballot.i32(i1 %c) %bc = icmp sgt i32 %ballot, 22 diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i32.ll index 3337d053eb930b9..857ef99e32618ac 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i32.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i32.ll @@ -3,7 +3,6 @@ ; RUN: llc -march=amdgcn -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 -mattr=+wavefrontsize32,-wavefrontsize64 < %s | FileCheck %s declare i32 @llvm.amdgcn.ballot.i32(i1) -declare i64 @llvm.amdgcn.ballot.i64(i1) declare i32 @llvm.ctpop.i32(i32) ; Test ballot(0) @@ -199,13 +198,11 @@ false: ret i32 33 } -define amdgpu_cs i32 @branch_divergent_ballot64_ne_zero_compare(i32 %v) { -; CHECK-LABEL: branch_divergent_ballot64_ne_zero_compare: +define amdgpu_cs i32 @branch_uniform_ballot_ne_zero_compare(i32 inreg %v) { +; CHECK-LABEL: branch_uniform_ballot_ne_zero_compare: ; CHECK: ; %bb.0: -; CHECK-NEXT: v_cmp_gt_u32_e64 s0, 12, v0 -; CHECK-NEXT: s_mov_b32 s1, 0 -; CHECK-NEXT: s_cmp_eq_u64 s[0:1], 0 -; CHECK-NEXT: s_cbranch_scc1 .LBB12_2 +; CHECK-NEXT: v_cmp_lt_u32_e64 vcc_lo, s0, 12 +; CHECK-NEXT: s_cbranch_vccz .LBB12_2 ; CHECK-NEXT: ; %bb.1: ; %true ; CHECK-NEXT: s_mov_b32 s0, 42 ; CHECK-NEXT: s_branch .LBB12_3 @@ -213,28 +210,6 @@ define amdgpu_cs i32 @branch_divergent_ballot64_ne_zero_compare(i32 %v) { ; CHECK-NEXT: s_mov_b32 s0, 33 ; CHECK-NEXT: s_branch .LBB12_3 ; CHECK-NEXT: .LBB12_3: - %c = icmp ult i32 %v, 12 - %ballot = call i64 @llvm.amdgcn.ballot.i64(i1 %c) - %ballot_ne_zero = icmp ne i64 %ballot, 0 - br i1 %ballot_ne_zero, label %true, label %false -true: - ret i32 42 -false: - ret i32 33 -} - -define amdgpu_cs i32 @branch_uniform_ballot_ne_zero_compare(i32 inreg %v) { -; CHECK-LABEL: branch_uniform_ballot_ne_zero_compare: -; CHECK: ; %bb.0: -; CHECK-NEXT: v_cmp_lt_u32_e64 vcc_lo, s0, 12 -; CHECK-NEXT: s_cbranch_vccz .LBB13_2 -; CHECK-NEXT: ; %bb.1: ; %true -; CHECK-NEXT: s_mov_b32 s0, 42 -; CHECK-NEXT: s_branch .LBB13_3 -; CHECK-NEXT: .LBB13_2: ; %false -; CHECK-NEXT: s_mov_b32 s0, 33 -; CHECK-NEXT: s_branch .LBB13_3 -; CHECK-NEXT: .LBB13_3: %c = icmp ult i32 %v, 12 %ballot = call i32 @llvm.amdgcn.ballot.i32(i1 %c) %ballot_ne_zero = icmp ne i32 %ballot, 0 @@ -249,14 +224,14 @@ define amdgpu_cs i32 @branch_divergent_ballot_eq_zero_compare(i32 %v) { ; CHECK-LABEL: branch_divergent_ballot_eq_zero_compare: ; CHECK: ; %bb.0: ; CHECK-NEXT: v_cmp_gt_u32_e32 vcc_lo, 12, v0 -; CHECK-NEXT: s_cbranch_vccz .LBB14_2 +; CHECK-NEXT: s_cbranch_vccz .LBB13_2 ; CHECK-NEXT: ; %bb.1: ; %false ; CHECK-NEXT: s_mov_b32 s0, 33 -; CHECK-NEXT: s_branch .LBB14_3 -; CHECK-NEXT: .LBB14_2: ; %true +; CHECK-NEXT: s_branch .LBB13_3 +; CHECK-NEXT: .LBB13_2: ; %true ; CHECK-NEXT: s_mov_b32 s0, 42 -; CHECK-NEXT: s_branch .LBB14_3 -; CHECK-NEXT: .LBB14_3: +; CHECK-NEXT: s_branch .LBB13_3 +; CHECK-NEXT: .LBB13_3: %c = icmp ult i32 %v, 12 %ballot = call i32 @llvm.amdgcn.ballot.i32(i1 %c) %ballot_eq_zero = icmp eq i32 %ballot, 0 @@ -271,14 +246,14 @@ define amdgpu_cs i32 @branch_uniform_ballot_eq_zero_compare(i32 inreg %v) { ; CHECK-LABEL: branch_uniform_ballot_eq_zero_compare: ; CHECK: ; %bb.0: ; CHECK-NEXT: v_cmp_lt_u32_e64 vcc_lo, s0, 12 -; CHECK-NEXT: s_cbranch_vccz .LBB15_2 +; CHECK-NEXT: s_cbranch_vccz .LBB14_2 ; CHECK-NEXT: ; %bb.1: ; %false ; CHECK-NEXT: s_mov_b32 s0, 33 -; CHECK-NEXT: s_branch .LBB15_3 -; CHECK-NEXT: .LBB15_2: ; %true +; CHECK-NEXT: s_branch .LBB14_3 +; CHECK-NEXT: .LBB14_2: ; %true ; CHECK-NEXT: s_mov_b32 s0, 42 -; CHECK-NEXT: s_branch .LBB15_3 -; CHECK-NEXT: .LBB15_3: +; CHECK-NEXT: s_branch .LBB14_3 +; CHECK-NEXT: .LBB14_3: %c = icmp ult i32 %v, 12 %ballot = call i32 @llvm.amdgcn.ballot.i32(i1 %c) %ballot_eq_zero = icmp eq i32 %ballot, 0 @@ -295,14 +270,14 @@ define amdgpu_cs i32 @branch_divergent_ballot_ne_zero_and(i32 %v1, i32 %v2) { ; CHECK-NEXT: v_cmp_gt_u32_e32 vcc_lo, 12, v0 ; CHECK-NEXT: v_cmp_lt_u32_e64 s0, 34, v1 ; CHECK-NEXT: s_and_b32 vcc_lo, vcc_lo, s0 -; CHECK-NEXT: s_cbranch_vccz .LBB16_2 +; CHECK-NEXT: s_cbranch_vccz .LBB15_2 ; CHECK-NEXT: ; %bb.1: ; %true ; CHECK-NEXT: s_mov_b32 s0, 42 -; CHECK-NEXT: s_branch .LBB16_3 -; CHECK-NEXT: .LBB16_2: ; %false +; CHECK-NEXT: s_branch .LBB15_3 +; CHECK-NEXT: .LBB15_2: ; %false ; CHECK-NEXT: s_mov_b32 s0, 33 -; CHECK-NEXT: s_branch .LBB16_3 -; CHECK-NEXT: .LBB16_3: +; CHECK-NEXT: s_branch .LBB15_3 +; CHECK-NEXT: .LBB15_3: %v1c = icmp ult i32 %v1, 12 %v2c = icmp ugt i32 %v2, 34 %c = and i1 %v1c, %v2c @@ -315,36 +290,6 @@ false: ret i32 33 } -define amdgpu_cs i32 @branch_divergent_ballot64_ne_zero_and(i32 %v1, i32 %v2) { -; CHECK-LABEL: branch_divergent_ballot64_ne_zero_and: -; CHECK: ; %bb.0: -; CHECK-NEXT: v_cmp_gt_u32_e32 vcc_lo, 12, v0 -; CHECK-NEXT: v_cmp_lt_u32_e64 s0, 34, v1 -; CHECK-NEXT: s_mov_b32 s1, 0 -; CHECK-NEXT: s_and_b32 s0, vcc_lo, s0 -; CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0 -; CHECK-NEXT: v_cmp_ne_u32_e64 s0, 0, v0 -; CHECK-NEXT: s_cmp_eq_u64 s[0:1], 0 -; CHECK-NEXT: s_cbranch_scc1 .LBB17_2 -; CHECK-NEXT: ; %bb.1: ; %true -; CHECK-NEXT: s_mov_b32 s0, 42 -; CHECK-NEXT: s_branch .LBB17_3 -; CHECK-NEXT: .LBB17_2: ; %false -; CHECK-NEXT: s_mov_b32 s0, 33 -; CHECK-NEXT: s_branch .LBB17_3 -; CHECK-NEXT: .LBB17_3: - %v1c = icmp ult i32 %v1, 12 - %v2c = icmp ugt i32 %v2, 34 - %c = and i1 %v1c, %v2c - %ballot = call i64 @llvm.amdgcn.ballot.i64(i1 %c) - %ballot_ne_zero = icmp ne i64 %ballot, 0 - br i1 %ballot_ne_zero, label %true, label %false -true: - ret i32 42 -false: - ret i32 33 -} - define amdgpu_cs i32 @branch_uniform_ballot_ne_zero_and(i32 inreg %v1, i32 inreg %v2) { ; CHECK-LABEL: branch_uniform_ballot_ne_zero_and: ; CHECK: ; %bb.0: @@ -354,14 +299,14 @@ define amdgpu_cs i32 @branch_uniform_ballot_ne_zero_and(i32 inreg %v1, i32 inreg ; CHECK-NEXT: s_cselect_b32 s1, -1, 0 ; CHECK-NEXT: s_and_b32 s0, s0, s1 ; CHECK-NEXT: s_and_b32 s0, s0, exec_lo -; CHECK-NEXT: s_cbranch_scc0 .LBB18_2 +; CHECK-NEXT: s_cbranch_scc0 .LBB16_2 ; CHECK-NEXT: ; %bb.1: ; %true ; CHECK-NEXT: s_mov_b32 s0, 42 -; CHECK-NEXT: s_branch .LBB18_3 -; CHECK-NEXT: .LBB18_2: ; %false +; CHECK-NEXT: s_branch .LBB16_3 +; CHECK-NEXT: .LBB16_2: ; %false ; CHECK-NEXT: s_mov_b32 s0, 33 -; CHECK-NEXT: s_branch .LBB18_3 -; CHECK-NEXT: .LBB18_3: +; CHECK-NEXT: s_branch .LBB16_3 +; CHECK-NEXT: .LBB16_3: %v1c = icmp ult i32 %v1, 12 %v2c = icmp ugt i32 %v2, 34 %c = and i1 %v1c, %v2c @@ -380,14 +325,14 @@ define amdgpu_cs i32 @branch_divergent_ballot_eq_zero_and(i32 %v1, i32 %v2) { ; CHECK-NEXT: v_cmp_gt_u32_e32 vcc_lo, 12, v0 ; CHECK-NEXT: v_cmp_lt_u32_e64 s0, 34, v1 ; CHECK-NEXT: s_and_b32 vcc_lo, vcc_lo, s0 -; CHECK-NEXT: s_cbranch_vccz .LBB19_2 +; CHECK-NEXT: s_cbranch_vccz .LBB17_2 ; CHECK-NEXT: ; %bb.1: ; %false ; CHECK-NEXT: s_mov_b32 s0, 33 -; CHECK-NEXT: s_branch .LBB19_3 -; CHECK-NEXT: .LBB19_2: ; %true +; CHECK-NEXT: s_branch .LBB17_3 +; CHECK-NEXT: .LBB17_2: ; %true ; CHECK-NEXT: s_mov_b32 s0, 42 -; CHECK-NEXT: s_branch .LBB19_3 -; CHECK-NEXT: .LBB19_3: +; CHECK-NEXT: s_branch .LBB17_3 +; CHECK-NEXT: .LBB17_3: %v1c = icmp ult i32 %v1, 12 %v2c = icmp ugt i32 %v2, 34 %c = and i1 %v1c, %v2c @@ -409,14 +354,14 @@ define amdgpu_cs i32 @branch_uniform_ballot_eq_zero_and(i32 inreg %v1, i32 inreg ; CHECK-NEXT: s_cselect_b32 s1, -1, 0 ; CHECK-NEXT: s_and_b32 s0, s0, s1 ; CHECK-NEXT: s_and_b32 s0, s0, exec_lo -; CHECK-NEXT: s_cbranch_scc0 .LBB20_2 +; CHECK-NEXT: s_cbranch_scc0 .LBB18_2 ; CHECK-NEXT: ; %bb.1: ; %false ; CHECK-NEXT: s_mov_b32 s0, 33 -; CHECK-NEXT: s_branch .LBB20_3 -; CHECK-NEXT: .LBB20_2: ; %true +; CHECK-NEXT: s_branch .LBB18_3 +; CHECK-NEXT: .LBB18_2: ; %true ; CHECK-NEXT: s_mov_b32 s0, 42 -; CHECK-NEXT: s_branch .LBB20_3 -; CHECK-NEXT: .LBB20_3: +; CHECK-NEXT: s_branch .LBB18_3 +; CHECK-NEXT: .LBB18_3: %v1c = icmp ult i32 %v1, 12 %v2c = icmp ugt i32 %v2, 34 %c = and i1 %v1c, %v2c @@ -434,14 +379,14 @@ define amdgpu_cs i32 @branch_uniform_ballot_sgt_N_compare(i32 inreg %v) { ; CHECK: ; %bb.0: ; CHECK-NEXT: v_cmp_lt_u32_e64 s0, s0, 12 ; CHECK-NEXT: s_cmp_lt_i32 s0, 23 -; CHECK-NEXT: s_cbranch_scc1 .LBB21_2 +; CHECK-NEXT: s_cbranch_scc1 .LBB19_2 ; CHECK-NEXT: ; %bb.1: ; %true ; CHECK-NEXT: s_mov_b32 s0, 42 -; CHECK-NEXT: s_branch .LBB21_3 -; CHECK-NEXT: .LBB21_2: ; %false +; CHECK-NEXT: s_branch .LBB19_3 +; CHECK-NEXT: .LBB19_2: ; %false ; CHECK-NEXT: s_mov_b32 s0, 33 -; CHECK-NEXT: s_branch .LBB21_3 -; CHECK-NEXT: .LBB21_3: +; CHECK-NEXT: s_branch .LBB19_3 +; CHECK-NEXT: .LBB19_3: %c = icmp ult i32 %v, 12 %ballot = call i32 @llvm.amdgcn.ballot.i32(i1 %c) %bc = icmp sgt i32 %ballot, 22 @@ -460,14 +405,14 @@ define amdgpu_cs i32 @branch_divergent_simulated_negated_ballot_ne_zero_and(i32 ; CHECK-NEXT: v_cmp_gt_u32_e32 vcc_lo, 12, v0 ; CHECK-NEXT: v_cmp_lt_u32_e64 s0, 34, v1 ; CHECK-NEXT: s_and_b32 vcc_lo, vcc_lo, s0 -; CHECK-NEXT: s_cbranch_vccnz .LBB22_2 +; CHECK-NEXT: s_cbranch_vccnz .LBB20_2 ; CHECK-NEXT: ; %bb.1: ; %true ; CHECK-NEXT: s_mov_b32 s0, 42 -; CHECK-NEXT: s_branch .LBB22_3 -; CHECK-NEXT: .LBB22_2: ; %false +; CHECK-NEXT: s_branch .LBB20_3 +; CHECK-NEXT: .LBB20_2: ; %false ; CHECK-NEXT: s_mov_b32 s0, 33 -; CHECK-NEXT: s_branch .LBB22_3 -; CHECK-NEXT: .LBB22_3: +; CHECK-NEXT: s_branch .LBB20_3 +; CHECK-NEXT: .LBB20_3: %v1c = icmp ult i32 %v1, 12 %v2c = icmp ugt i32 %v2, 34 %c = and i1 %v1c, %v2c @@ -503,14 +448,14 @@ define amdgpu_cs i32 @branch_uniform_simulated_negated_ballot_ne_zero_and(i32 in ; CHECK-NEXT: s_cselect_b32 s1, -1, 0 ; CHECK-NEXT: s_and_b32 s0, s0, s1 ; CHECK-NEXT: s_and_b32 s0, s0, exec_lo -; CHECK-NEXT: s_cbranch_scc1 .LBB23_2 +; CHECK-NEXT: s_cbranch_scc1 .LBB21_2 ; CHECK-NEXT: ; %bb.1: ; %true ; CHECK-NEXT: s_mov_b32 s0, 42 -; CHECK-NEXT: s_branch .LBB23_3 -; CHECK-NEXT: .LBB23_2: ; %false +; CHECK-NEXT: s_branch .LBB21_3 +; CHECK-NEXT: .LBB21_2: ; %false ; CHECK-NEXT: s_mov_b32 s0, 33 -; CHECK-NEXT: s_branch .LBB23_3 -; CHECK-NEXT: .LBB23_3: +; CHECK-NEXT: s_branch .LBB21_3 +; CHECK-NEXT: .LBB21_3: %v1c = icmp ult i32 %v1, 12 %v2c = icmp ugt i32 %v2, 34 %c = and i1 %v1c, %v2c @@ -529,14 +474,14 @@ define amdgpu_cs i32 @branch_divergent_simulated_negated_ballot_eq_zero_and(i32 ; CHECK-NEXT: v_cmp_gt_u32_e32 vcc_lo, 12, v0 ; CHECK-NEXT: v_cmp_lt_u32_e64 s0, 34, v1 ; CHECK-NEXT: s_and_b32 vcc_lo, vcc_lo, s0 -; CHECK-NEXT: s_cbranch_vccnz .LBB24_2 +; CHECK-NEXT: s_cbranch_vccnz .LBB22_2 ; CHECK-NEXT: ; %bb.1: ; %false ; CHECK-NEXT: s_mov_b32 s0, 33 -; CHECK-NEXT: s_branch .LBB24_3 -; CHECK-NEXT: .LBB24_2: ; %true +; CHECK-NEXT: s_branch .LBB22_3 +; CHECK-NEXT: .LBB22_2: ; %true ; CHECK-NEXT: s_mov_b32 s0, 42 -; CHECK-NEXT: s_branch .LBB24_3 -; CHECK-NEXT: .LBB24_3: +; CHECK-NEXT: s_branch .LBB22_3 +; CHECK-NEXT: .LBB22_3: %v1c = icmp ult i32 %v1, 12 %v2c = icmp ugt i32 %v2, 34 %c = and i1 %v1c, %v2c @@ -558,14 +503,14 @@ define amdgpu_cs i32 @branch_uniform_simulated_negated_ballot_eq_zero_and(i32 in ; CHECK-NEXT: s_cselect_b32 s1, -1, 0 ; CHECK-NEXT: s_and_b32 s0, s0, s1 ; CHECK-NEXT: s_and_b32 s0, s0, exec_lo -; CHECK-NEXT: s_cbranch_scc1 .LBB25_2 +; CHECK-NEXT: s_cbranch_scc1 .LBB23_2 ; CHECK-NEXT: ; %bb.1: ; %false ; CHECK-NEXT: s_mov_b32 s0, 33 -; CHECK-NEXT: s_branch .LBB25_3 -; CHECK-NEXT: .LBB25_2: ; %true +; CHECK-NEXT: s_branch .LBB23_3 +; CHECK-NEXT: .LBB23_2: ; %true ; CHECK-NEXT: s_mov_b32 s0, 42 -; CHECK-NEXT: s_branch .LBB25_3 -; CHECK-NEXT: .LBB25_3: +; CHECK-NEXT: s_branch .LBB23_3 +; CHECK-NEXT: .LBB23_3: %v1c = icmp ult i32 %v1, 12 %v2c = icmp ugt i32 %v2, 34 %c = and i1 %v1c, %v2c diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i64.wave32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i64.wave32.ll index 109a345d7a2c884..04a993eac82cd5e 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i64.wave32.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i64.wave32.ll @@ -104,3 +104,73 @@ define amdgpu_cs i64 @ctpop_of_ballot(float %x, float %y) { %bcnt = call i64 @llvm.ctpop.i64(i64 %ballot) ret i64 %bcnt } + +define amdgpu_cs i32 @branch_divergent_ballot64_ne_zero_compare(i32 %v) { +; CHECK-LABEL: branch_divergent_ballot64_ne_zero_compare: +; CHECK: ; %bb.0: +; CHECK-NEXT: v_cmp_gt_u32_e64 s0, 12, v0 +; CHECK-NEXT: s_mov_b32 s1, 0 +; CHECK-NEXT: s_cmp_eq_u64 s[0:1], 0 +; CHECK-NEXT: s_cbranch_scc1 .LBB7_2 +; CHECK-NEXT: ; %bb.1: ; %true +; CHECK-NEXT: s_mov_b32 s0, 42 +; CHECK-NEXT: s_branch .LBB7_3 +; CHECK-NEXT: .LBB7_2: ; %false +; CHECK-NEXT: s_mov_b32 s0, 33 +; CHECK-NEXT: s_branch .LBB7_3 +; CHECK-NEXT: .LBB7_3: + %c = icmp ult i32 %v, 12 + %ballot = call i64 @llvm.amdgcn.ballot.i64(i1 %c) + %ballot_ne_zero = icmp ne i64 %ballot, 0 + br i1 %ballot_ne_zero, label %true, label %false +true: + ret i32 42 +false: + ret i32 33 +} + +define amdgpu_cs i32 @branch_divergent_ballot64_ne_zero_and(i32 %v1, i32 %v2) { +; DAGISEL-LABEL: branch_divergent_ballot64_ne_zero_and: +; DAGISEL: ; %bb.0: +; DAGISEL-NEXT: v_cmp_gt_u32_e32 vcc_lo, 12, v0 +; DAGISEL-NEXT: v_cmp_lt_u32_e64 s0, 34, v1 +; DAGISEL-NEXT: s_mov_b32 s1, 0 +; DAGISEL-NEXT: s_and_b32 s0, vcc_lo, s0 +; DAGISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0 +; DAGISEL-NEXT: v_cmp_ne_u32_e64 s0, 0, v0 +; DAGISEL-NEXT: s_cmp_eq_u64 s[0:1], 0 +; DAGISEL-NEXT: s_cbranch_scc1 .LBB8_2 +; DAGISEL-NEXT: ; %bb.1: ; %true +; DAGISEL-NEXT: s_mov_b32 s0, 42 +; DAGISEL-NEXT: s_branch .LBB8_3 +; DAGISEL-NEXT: .LBB8_2: ; %false +; DAGISEL-NEXT: s_mov_b32 s0, 33 +; DAGISEL-NEXT: s_branch .LBB8_3 +; DAGISEL-NEXT: .LBB8_3: +; +; GISEL-LABEL: branch_divergent_ballot64_ne_zero_and: +; GISEL: ; %bb.0: +; GISEL-NEXT: v_cmp_gt_u32_e32 vcc_lo, 12, v0 +; GISEL-NEXT: v_cmp_lt_u32_e64 s0, 34, v1 +; GISEL-NEXT: s_mov_b32 s1, 0 +; GISEL-NEXT: s_and_b32 s0, vcc_lo, s0 +; GISEL-NEXT: s_cmp_eq_u64 s[0:1], 0 +; GISEL-NEXT: s_cbranch_scc1 .LBB8_2 +; GISEL-NEXT: ; %bb.1: ; %true +; GISEL-NEXT: s_mov_b32 s0, 42 +; GISEL-NEXT: s_branch .LBB8_3 +; GISEL-NEXT: .LBB8_2: ; %false +; GISEL-NEXT: s_mov_b32 s0, 33 +; GISEL-NEXT: s_branch .LBB8_3 +; GISEL-NEXT: .LBB8_3: + %v1c = icmp ult i32 %v1, 12 + %v2c = icmp ugt i32 %v2, 34 + %c = and i1 %v1c, %v2c + %ballot = call i64 @llvm.amdgcn.ballot.i64(i1 %c) + %ballot_ne_zero = icmp ne i64 %ballot, 0 + br i1 %ballot_ne_zero, label %true, label %false +true: + ret i32 42 +false: + ret i32 33 +} >From 5c3e6711ef9e683bb271416510e9525561820845 Mon Sep 17 00:00:00 2001 From: Valery Pykhtin <valery.pykh...@gmail.com> Date: Mon, 20 Nov 2023 15:22:16 +0100 Subject: [PATCH 2/6] run opt instcombine pass in the test --- .../AMDGPU/llvm.amdgcn.ballot.i64.wave32.ll | 44 ++++++++++--------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i64.wave32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i64.wave32.ll index 04a993eac82cd5e..bd319c6fc39ce62 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i64.wave32.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i64.wave32.ll @@ -1,8 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -march=amdgcn -global-isel=0 -mcpu=gfx1010 < %s | FileCheck %s --check-prefixes=CHECK,DAGISEL -; RUN: llc -march=amdgcn -global-isel=0 -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 < %s | FileCheck %s --check-prefixes=CHECK,DAGISEL -; RUN: llc -march=amdgcn -global-isel -mcpu=gfx1010 < %s | FileCheck %s --check-prefixes=CHECK,GISEL -; RUN: llc -march=amdgcn -global-isel -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 < %s | FileCheck %s --check-prefixes=CHECK,GISEL +; RUN: opt -mtriple=amdgcn-- -mcpu=gfx1010 -mattr=+wavefrontsize32,-wavefrontsize64 -passes=instcombine -o - < %s | llc -march=amdgcn -global-isel=0 -mcpu=gfx1010 - | FileCheck %s --check-prefixes=CHECK,DAGISEL +; RUN: opt -mtriple=amdgcn-- -mcpu=gfx1100 -mattr=+wavefrontsize32,-wavefrontsize64 -passes=instcombine -o - < %s | llc -march=amdgcn -global-isel=0 -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 - | FileCheck %s --check-prefixes=CHECK,DAGISEL +; RUN: opt -mtriple=amdgcn-- -mcpu=gfx1010 -mattr=+wavefrontsize32,-wavefrontsize64 -passes=instcombine -o - < %s | llc -march=amdgcn -global-isel -mcpu=gfx1010 - | FileCheck %s --check-prefixes=CHECK,GISEL +; RUN: opt -mtriple=amdgcn-- -mcpu=gfx1100 -mattr=+wavefrontsize32,-wavefrontsize64 -passes=instcombine -o - < %s | llc -march=amdgcn -global-isel -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 - | FileCheck %s --check-prefixes=CHECK,GISEL declare i64 @llvm.amdgcn.ballot.i64(i1) declare i64 @llvm.ctpop.i64(i64) @@ -40,12 +40,20 @@ define amdgpu_cs i64 @constant_true() { ; Test ballot of a non-comparison operation define amdgpu_cs i64 @non_compare(i32 %x) { -; CHECK-LABEL: non_compare: -; CHECK: ; %bb.0: -; CHECK-NEXT: v_and_b32_e32 v0, 1, v0 -; CHECK-NEXT: s_mov_b32 s1, 0 -; CHECK-NEXT: v_cmp_ne_u32_e64 s0, 0, v0 -; CHECK-NEXT: ; return to shader part epilog +; DAGISEL-LABEL: non_compare: +; DAGISEL: ; %bb.0: +; DAGISEL-NEXT: v_and_b32_e32 v0, 1, v0 +; DAGISEL-NEXT: s_mov_b32 s1, 0 +; DAGISEL-NEXT: v_cmp_ne_u32_e64 s0, 0, v0 +; DAGISEL-NEXT: ; return to shader part epilog +; +; GISEL-LABEL: non_compare: +; GISEL: ; %bb.0: +; GISEL-NEXT: v_and_b32_e32 v0, 1, v0 +; GISEL-NEXT: s_mov_b32 s1, 0 +; GISEL-NEXT: v_and_b32_e32 v0, 1, v0 +; GISEL-NEXT: v_cmp_ne_u32_e64 s0, 0, v0 +; GISEL-NEXT: ; return to shader part epilog %trunc = trunc i32 %x to i1 %ballot = call i64 @llvm.amdgcn.ballot.i64(i1 %trunc) ret i64 %ballot @@ -65,17 +73,11 @@ define amdgpu_cs i64 @compare_ints(i32 %x, i32 %y) { } define amdgpu_cs i64 @compare_int_with_constant(i32 %x) { -; DAGISEL-LABEL: compare_int_with_constant: -; DAGISEL: ; %bb.0: -; DAGISEL-NEXT: v_cmp_lt_i32_e64 s0, 0x62, v0 -; DAGISEL-NEXT: s_mov_b32 s1, 0 -; DAGISEL-NEXT: ; return to shader part epilog -; -; GISEL-LABEL: compare_int_with_constant: -; GISEL: ; %bb.0: -; GISEL-NEXT: v_cmp_le_i32_e64 s0, 0x63, v0 -; GISEL-NEXT: s_mov_b32 s1, 0 -; GISEL-NEXT: ; return to shader part epilog +; CHECK-LABEL: compare_int_with_constant: +; CHECK: ; %bb.0: +; CHECK-NEXT: v_cmp_lt_i32_e64 s0, 0x62, v0 +; CHECK-NEXT: s_mov_b32 s1, 0 +; CHECK-NEXT: ; return to shader part epilog %cmp = icmp sge i32 %x, 99 %ballot = call i64 @llvm.amdgcn.ballot.i64(i1 %cmp) ret i64 %ballot >From 4cf6ac7b7cb716f2a9184b8dfb0d17dca7bcfd9b Mon Sep 17 00:00:00 2001 From: Valery Pykhtin <valery.pykh...@gmail.com> Date: Mon, 20 Nov 2023 15:29:31 +0100 Subject: [PATCH 3/6] add instcombine rule --- .../AMDGPU/AMDGPUInstCombineIntrinsic.cpp | 12 ++++ .../AMDGPU/llvm.amdgcn.ballot.i64.wave32.ll | 68 +++++++++---------- .../InstCombine/AMDGPU/amdgcn-intrinsics.ll | 10 +-- 3 files changed, 52 insertions(+), 38 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp index 5296415ab4c36da..6b7bc2e1ed02ae6 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp @@ -961,6 +961,18 @@ GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const { return IC.replaceInstUsesWith(II, Constant::getNullValue(II.getType())); } } + if (ST->isWave32() && II.getType()->getIntegerBitWidth() == 64) { + // %b64 = call i64 ballot.i64(...) + // => + // %b32 = call i32 ballot.i32(...) + // %b64 = zext i32 %b32 to i64 + Function *NewF = Intrinsic::getDeclaration( + II.getModule(), Intrinsic::amdgcn_ballot, {IC.Builder.getInt32Ty()}); + CallInst *NewCall = IC.Builder.CreateCall(NewF, {II.getArgOperand(0)}); + NewCall->takeName(&II); + Value *CastedCall = IC.Builder.CreateZExtOrBitCast(NewCall, II.getType()); + return IC.replaceInstUsesWith(II, CastedCall); + } break; } case Intrinsic::amdgcn_wqm_vote: { diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i64.wave32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i64.wave32.ll index bd319c6fc39ce62..8647718d7a77b98 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i64.wave32.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i64.wave32.ll @@ -22,17 +22,11 @@ define amdgpu_cs i64 @constant_false() { ; Test ballot(1) define amdgpu_cs i64 @constant_true() { -; DAGISEL-LABEL: constant_true: -; DAGISEL: ; %bb.0: -; DAGISEL-NEXT: s_mov_b32 s0, exec_lo -; DAGISEL-NEXT: s_mov_b32 s1, exec_hi -; DAGISEL-NEXT: ; return to shader part epilog -; -; GISEL-LABEL: constant_true: -; GISEL: ; %bb.0: -; GISEL-NEXT: s_mov_b32 s0, exec_lo -; GISEL-NEXT: s_mov_b32 s1, 0 -; GISEL-NEXT: ; return to shader part epilog +; CHECK-LABEL: constant_true: +; CHECK: ; %bb.0: +; CHECK-NEXT: s_mov_b32 s0, exec_lo +; CHECK-NEXT: s_mov_b32 s1, 0 +; CHECK-NEXT: ; return to shader part epilog %ballot = call i64 @llvm.amdgcn.ballot.i64(i1 1) ret i64 %ballot } @@ -97,9 +91,9 @@ define amdgpu_cs i64 @compare_floats(float %x, float %y) { define amdgpu_cs i64 @ctpop_of_ballot(float %x, float %y) { ; CHECK-LABEL: ctpop_of_ballot: ; CHECK: ; %bb.0: -; CHECK-NEXT: v_cmp_gt_f32_e64 s0, v0, v1 +; CHECK-NEXT: v_cmp_gt_f32_e32 vcc_lo, v0, v1 ; CHECK-NEXT: s_mov_b32 s1, 0 -; CHECK-NEXT: s_bcnt1_i32_b64 s0, s[0:1] +; CHECK-NEXT: s_bcnt1_i32_b32 s0, vcc_lo ; CHECK-NEXT: ; return to shader part epilog %cmp = fcmp ogt float %x, %y %ballot = call i64 @llvm.amdgcn.ballot.i64(i1 %cmp) @@ -108,19 +102,30 @@ define amdgpu_cs i64 @ctpop_of_ballot(float %x, float %y) { } define amdgpu_cs i32 @branch_divergent_ballot64_ne_zero_compare(i32 %v) { -; CHECK-LABEL: branch_divergent_ballot64_ne_zero_compare: -; CHECK: ; %bb.0: -; CHECK-NEXT: v_cmp_gt_u32_e64 s0, 12, v0 -; CHECK-NEXT: s_mov_b32 s1, 0 -; CHECK-NEXT: s_cmp_eq_u64 s[0:1], 0 -; CHECK-NEXT: s_cbranch_scc1 .LBB7_2 -; CHECK-NEXT: ; %bb.1: ; %true -; CHECK-NEXT: s_mov_b32 s0, 42 -; CHECK-NEXT: s_branch .LBB7_3 -; CHECK-NEXT: .LBB7_2: ; %false -; CHECK-NEXT: s_mov_b32 s0, 33 -; CHECK-NEXT: s_branch .LBB7_3 -; CHECK-NEXT: .LBB7_3: +; DAGISEL-LABEL: branch_divergent_ballot64_ne_zero_compare: +; DAGISEL: ; %bb.0: +; DAGISEL-NEXT: v_cmp_gt_u32_e32 vcc_lo, 12, v0 +; DAGISEL-NEXT: s_cbranch_vccz .LBB7_2 +; DAGISEL-NEXT: ; %bb.1: ; %true +; DAGISEL-NEXT: s_mov_b32 s0, 42 +; DAGISEL-NEXT: s_branch .LBB7_3 +; DAGISEL-NEXT: .LBB7_2: ; %false +; DAGISEL-NEXT: s_mov_b32 s0, 33 +; DAGISEL-NEXT: s_branch .LBB7_3 +; DAGISEL-NEXT: .LBB7_3: +; +; GISEL-LABEL: branch_divergent_ballot64_ne_zero_compare: +; GISEL: ; %bb.0: +; GISEL-NEXT: v_cmp_gt_u32_e32 vcc_lo, 12, v0 +; GISEL-NEXT: s_cmp_eq_u32 vcc_lo, 0 +; GISEL-NEXT: s_cbranch_scc1 .LBB7_2 +; GISEL-NEXT: ; %bb.1: ; %true +; GISEL-NEXT: s_mov_b32 s0, 42 +; GISEL-NEXT: s_branch .LBB7_3 +; GISEL-NEXT: .LBB7_2: ; %false +; GISEL-NEXT: s_mov_b32 s0, 33 +; GISEL-NEXT: s_branch .LBB7_3 +; GISEL-NEXT: .LBB7_3: %c = icmp ult i32 %v, 12 %ballot = call i64 @llvm.amdgcn.ballot.i64(i1 %c) %ballot_ne_zero = icmp ne i64 %ballot, 0 @@ -136,12 +141,8 @@ define amdgpu_cs i32 @branch_divergent_ballot64_ne_zero_and(i32 %v1, i32 %v2) { ; DAGISEL: ; %bb.0: ; DAGISEL-NEXT: v_cmp_gt_u32_e32 vcc_lo, 12, v0 ; DAGISEL-NEXT: v_cmp_lt_u32_e64 s0, 34, v1 -; DAGISEL-NEXT: s_mov_b32 s1, 0 -; DAGISEL-NEXT: s_and_b32 s0, vcc_lo, s0 -; DAGISEL-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0 -; DAGISEL-NEXT: v_cmp_ne_u32_e64 s0, 0, v0 -; DAGISEL-NEXT: s_cmp_eq_u64 s[0:1], 0 -; DAGISEL-NEXT: s_cbranch_scc1 .LBB8_2 +; DAGISEL-NEXT: s_and_b32 vcc_lo, vcc_lo, s0 +; DAGISEL-NEXT: s_cbranch_vccz .LBB8_2 ; DAGISEL-NEXT: ; %bb.1: ; %true ; DAGISEL-NEXT: s_mov_b32 s0, 42 ; DAGISEL-NEXT: s_branch .LBB8_3 @@ -154,9 +155,8 @@ define amdgpu_cs i32 @branch_divergent_ballot64_ne_zero_and(i32 %v1, i32 %v2) { ; GISEL: ; %bb.0: ; GISEL-NEXT: v_cmp_gt_u32_e32 vcc_lo, 12, v0 ; GISEL-NEXT: v_cmp_lt_u32_e64 s0, 34, v1 -; GISEL-NEXT: s_mov_b32 s1, 0 ; GISEL-NEXT: s_and_b32 s0, vcc_lo, s0 -; GISEL-NEXT: s_cmp_eq_u64 s[0:1], 0 +; GISEL-NEXT: s_cmp_eq_u32 s0, 0 ; GISEL-NEXT: s_cbranch_scc1 .LBB8_2 ; GISEL-NEXT: ; %bb.1: ; %true ; GISEL-NEXT: s_mov_b32 s0, 42 diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-intrinsics.ll b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-intrinsics.ll index 804283cc20cd6a3..a11b8fe37b4b3f0 100644 --- a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-intrinsics.ll +++ b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-intrinsics.ll @@ -2599,8 +2599,9 @@ declare i32 @llvm.amdgcn.ballot.i32(i1) nounwind readnone convergent define i64 @ballot_nocombine_64(i1 %i) { ; CHECK-LABEL: @ballot_nocombine_64( -; CHECK-NEXT: [[B:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 [[I:%.*]]) -; CHECK-NEXT: ret i64 [[B]] +; CHECK-NEXT: [[B:%.*]] = call i32 @llvm.amdgcn.ballot.i32(i1 [[I:%.*]]) +; CHECK-NEXT: [[TMP1:%.*]] = zext i32 [[B]] to i64 +; CHECK-NEXT: ret i64 [[TMP1]] ; %b = call i64 @llvm.amdgcn.ballot.i64(i1 %i) ret i64 %b @@ -2616,8 +2617,9 @@ define i64 @ballot_zero_64() { define i64 @ballot_one_64() { ; CHECK-LABEL: @ballot_one_64( -; CHECK-NEXT: [[B:%.*]] = call i64 @llvm.amdgcn.ballot.i64(i1 true) -; CHECK-NEXT: ret i64 [[B]] +; CHECK-NEXT: [[B:%.*]] = call i32 @llvm.amdgcn.ballot.i32(i1 true) +; CHECK-NEXT: [[TMP1:%.*]] = zext i32 [[B]] to i64 +; CHECK-NEXT: ret i64 [[TMP1]] ; %b = call i64 @llvm.amdgcn.ballot.i64(i1 1) ret i64 %b >From da182a8fcede6cbfeaa95633277ff1d7e79780de Mon Sep 17 00:00:00 2001 From: Valery Pykhtin <valery.pykh...@gmail.com> Date: Mon, 20 Nov 2023 16:24:25 +0100 Subject: [PATCH 4/6] improve IR variable naming --- .../lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp | 2 +- .../InstCombine/AMDGPU/amdgcn-intrinsics.ll | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp index 6b7bc2e1ed02ae6..510f0a59719ecdb 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp @@ -969,8 +969,8 @@ GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const { Function *NewF = Intrinsic::getDeclaration( II.getModule(), Intrinsic::amdgcn_ballot, {IC.Builder.getInt32Ty()}); CallInst *NewCall = IC.Builder.CreateCall(NewF, {II.getArgOperand(0)}); - NewCall->takeName(&II); Value *CastedCall = IC.Builder.CreateZExtOrBitCast(NewCall, II.getType()); + CastedCall->takeName(&II); return IC.replaceInstUsesWith(II, CastedCall); } break; diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-intrinsics.ll b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-intrinsics.ll index a11b8fe37b4b3f0..94c32e3cbe99f72 100644 --- a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-intrinsics.ll +++ b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-intrinsics.ll @@ -2599,9 +2599,9 @@ declare i32 @llvm.amdgcn.ballot.i32(i1) nounwind readnone convergent define i64 @ballot_nocombine_64(i1 %i) { ; CHECK-LABEL: @ballot_nocombine_64( -; CHECK-NEXT: [[B:%.*]] = call i32 @llvm.amdgcn.ballot.i32(i1 [[I:%.*]]) -; CHECK-NEXT: [[TMP1:%.*]] = zext i32 [[B]] to i64 -; CHECK-NEXT: ret i64 [[TMP1]] +; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.amdgcn.ballot.i32(i1 [[I:%.*]]) +; CHECK-NEXT: [[B:%.*]] = zext i32 [[TMP1]] to i64 +; CHECK-NEXT: ret i64 [[B]] ; %b = call i64 @llvm.amdgcn.ballot.i64(i1 %i) ret i64 %b @@ -2617,9 +2617,9 @@ define i64 @ballot_zero_64() { define i64 @ballot_one_64() { ; CHECK-LABEL: @ballot_one_64( -; CHECK-NEXT: [[B:%.*]] = call i32 @llvm.amdgcn.ballot.i32(i1 true) -; CHECK-NEXT: [[TMP1:%.*]] = zext i32 [[B]] to i64 -; CHECK-NEXT: ret i64 [[TMP1]] +; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.amdgcn.ballot.i32(i1 true) +; CHECK-NEXT: [[B:%.*]] = zext i32 [[TMP1]] to i64 +; CHECK-NEXT: ret i64 [[B]] ; %b = call i64 @llvm.amdgcn.ballot.i64(i1 1) ret i64 %b >From 1efd3dfd459be1d275288feb9e73617e7b020159 Mon Sep 17 00:00:00 2001 From: Valery Pykhtin <valery.pykh...@gmail.com> Date: Mon, 20 Nov 2023 17:17:02 +0100 Subject: [PATCH 5/6] implement a TODO --- llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp index ead3f51d6acdc5a..a0ff1bd579b14b2 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp @@ -2314,9 +2314,8 @@ void AMDGPUDAGToDAGISel::SelectBRCOND(SDNode *N) { SDValue VCMP = Cond->getOperand(0); auto CC = cast<CondCodeSDNode>(Cond->getOperand(2))->get(); auto *CRHS = dyn_cast<ConstantSDNode>(Cond->getOperand(1)); - if ((CC == ISD::SETEQ || CC == ISD::SETNE) && CRHS && CRHS->isZero() && - // TODO: make condition below an assert after fixing ballot bitwidth. - VCMP.getValueType().getSizeInBits() == ST->getWavefrontSize()) { + if ((CC == ISD::SETEQ || CC == ISD::SETNE) && CRHS && CRHS->isZero()) { + assert(VCMP.getValueType().getSizeInBits() == ST->getWavefrontSize()); // %VCMP = i(WaveSize) AMDGPUISD::SETCC ... // %C = i1 ISD::SETCC %VCMP, 0, setne/seteq // BRCOND i1 %C, %BB >From 5543d14bb0014809629965d8a412f47ef39ac2ba Mon Sep 17 00:00:00 2001 From: Valery Pykhtin <valery.pykh...@gmail.com> Date: Mon, 20 Nov 2023 18:07:41 +0100 Subject: [PATCH 6/6] update clang/test/CodeGenOpenCL/builtins-amdgcn-wave32.cl --- clang/test/CodeGenOpenCL/builtins-amdgcn-wave32.cl | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-wave32.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-wave32.cl index 43553131f63c549..a0e27ce22fe7d9c 100644 --- a/clang/test/CodeGenOpenCL/builtins-amdgcn-wave32.cl +++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-wave32.cl @@ -24,13 +24,11 @@ void test_ballot_wave32_target_attr(global uint* out, int a, int b) } // CHECK-LABEL: @test_read_exec( -// CHECK: call i64 @llvm.amdgcn.ballot.i64(i1 true) +// CHECK: call i32 @llvm.amdgcn.ballot.i32(i1 true) void test_read_exec(global uint* out) { *out = __builtin_amdgcn_read_exec(); } -// CHECK: declare i64 @llvm.amdgcn.ballot.i64(i1) #[[$NOUNWIND_READONLY:[0-9]+]] - // CHECK-LABEL: @test_read_exec_lo( // CHECK: call i32 @llvm.amdgcn.ballot.i32(i1 true) void test_read_exec_lo(global uint* out) { @@ -38,9 +36,7 @@ void test_read_exec_lo(global uint* out) { } // CHECK-LABEL: @test_read_exec_hi( -// CHECK: call i64 @llvm.amdgcn.ballot.i64(i1 true) -// CHECK: lshr i64 [[A:%.*]], 32 -// CHECK: trunc i64 [[B:%.*]] to i32 +// CHECK: store i32 0, ptr addrspace(1) %out void test_read_exec_hi(global uint* out) { *out = __builtin_amdgcn_read_exec_hi(); } _______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits