https://github.com/saxlungs updated https://github.com/llvm/llvm-project/pull/209889
>From 20aaed567e4924464d6d3ce331d48fef6748a7ae Mon Sep 17 00:00:00 2001 From: Domenic Nutile <[email protected]> Date: Wed, 15 Jul 2026 14:38:30 -0400 Subject: [PATCH] [AMDGPU] Clean up check prefixes and regenerate checklines for some tests that were out of date --- .../CodeGen/AMDGPU/llvm.amdgcn.cvt.fp8.dpp.ll | 50 +- .../AMDGPU/llvm.amdgcn.raw.buffer.load.ll | 42 +- llvm/test/CodeGen/AMDGPU/llvm.floor.f16.ll | 76 +- llvm/test/CodeGen/AMDGPU/vector-reduce-mul.ll | 800 ++++++++++++------ ...wmma-gfx12-w32-f16-f32-matrix-modifiers.ll | 39 +- 5 files changed, 676 insertions(+), 331 deletions(-) diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.fp8.dpp.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.fp8.dpp.ll index 67acf16f70f3e..ae8b16d1a9479 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.fp8.dpp.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.fp8.dpp.ll @@ -1,8 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc -mtriple=amdgpu11.70 < %s | FileCheck -check-prefixes=GFX1170PLUS,GFX1170 %s ; RUN: llc -mtriple=amdgpu12.00 < %s | FileCheck -check-prefixes=GFX1170PLUS,GFX12 %s +; RUN: llc -mtriple=amdgpu12.50 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX1170PLUS,GFX1250,GFX1250-TRUE16 %s ; RUN: llc -mtriple=amdgpu12.50 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX1170PLUS,GFX1250,GFX1250-FAKE16 %s -; RUN: llc -mtriple=amdgpu12.50 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX1170PLUS,GFX1250,GFX1250-REAL16 %s define amdgpu_cs float @test_cvt_f32_bf8_byte0(i32 %a) { ; GFX1170-LABEL: test_cvt_f32_bf8_byte0: @@ -113,6 +113,18 @@ define amdgpu_cs void @test_cvt_pk_bf8_f32_word0(i32 %a, float %y, i32 %old, ptr ; GFX12-NEXT: global_store_b32 v[3:4], v2, off ; GFX12-NEXT: s_endpgm ; +; GFX1250-TRUE16-LABEL: test_cvt_pk_bf8_f32_word0: +; GFX1250-TRUE16: ; %bb.0: +; GFX1250-TRUE16-NEXT: global_wb +; GFX1250-TRUE16-NEXT: v_nop +; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0 +; GFX1250-TRUE16-NEXT: v_mov_b32_dpp v0, v0 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf bound_ctrl:1 +; GFX1250-TRUE16-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3 +; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1250-TRUE16-NEXT: v_cvt_pk_bf8_f32 v2.l, v0, v1 +; GFX1250-TRUE16-NEXT: global_store_b32 v[4:5], v2, off +; GFX1250-TRUE16-NEXT: s_endpgm +; ; GFX1250-FAKE16-LABEL: test_cvt_pk_bf8_f32_word0: ; GFX1250-FAKE16: ; %bb.0: ; GFX1250-FAKE16-NEXT: global_wb @@ -122,18 +134,6 @@ define amdgpu_cs void @test_cvt_pk_bf8_f32_word0(i32 %a, float %y, i32 %old, ptr ; GFX1250-FAKE16-NEXT: v_cvt_pk_bf8_f32_e64_dpp v2, v0, v1 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf bound_ctrl:1 ; GFX1250-FAKE16-NEXT: global_store_b32 v[4:5], v2, off ; GFX1250-FAKE16-NEXT: s_endpgm -; -; GFX1250-REAL16-LABEL: test_cvt_pk_bf8_f32_word0: -; GFX1250-REAL16: ; %bb.0: -; GFX1250-REAL16-NEXT: global_wb -; GFX1250-REAL16-NEXT: v_nop -; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0 -; GFX1250-REAL16-NEXT: v_mov_b32_dpp v0, v0 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf bound_ctrl:1 -; GFX1250-REAL16-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3 -; GFX1250-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX1250-REAL16-NEXT: v_cvt_pk_bf8_f32 v2.l, v0, v1 -; GFX1250-REAL16-NEXT: global_store_b32 v[4:5], v2, off -; GFX1250-REAL16-NEXT: s_endpgm %tmp0 = call i32 @llvm.amdgcn.mov.dpp.i32(i32 %a, i32 228, i32 15, i32 15, i1 1) %tmp1 = bitcast i32 %tmp0 to float %ret = tail call i32 @llvm.amdgcn.cvt.pk.bf8.f32(float %tmp1, float %y, i32 %old, i1 false) @@ -158,6 +158,18 @@ define amdgpu_cs void @test_cvt_pk_fp8_f32_word1(i32 %a, float %y, i32 %old, ptr ; GFX12-NEXT: global_store_b32 v[3:4], v2, off ; GFX12-NEXT: s_endpgm ; +; GFX1250-TRUE16-LABEL: test_cvt_pk_fp8_f32_word1: +; GFX1250-TRUE16: ; %bb.0: +; GFX1250-TRUE16-NEXT: global_wb +; GFX1250-TRUE16-NEXT: v_nop +; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0 +; GFX1250-TRUE16-NEXT: v_mov_b32_dpp v0, v0 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf bound_ctrl:1 +; GFX1250-TRUE16-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3 +; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1250-TRUE16-NEXT: v_cvt_pk_fp8_f32 v2.h, v0, v1 +; GFX1250-TRUE16-NEXT: global_store_b32 v[4:5], v2, off +; GFX1250-TRUE16-NEXT: s_endpgm +; ; GFX1250-FAKE16-LABEL: test_cvt_pk_fp8_f32_word1: ; GFX1250-FAKE16: ; %bb.0: ; GFX1250-FAKE16-NEXT: global_wb @@ -169,18 +181,6 @@ define amdgpu_cs void @test_cvt_pk_fp8_f32_word1(i32 %a, float %y, i32 %old, ptr ; GFX1250-FAKE16-NEXT: v_cvt_pk_fp8_f32 v2, v0, v1 op_sel:[0,0,1] ; GFX1250-FAKE16-NEXT: global_store_b32 v[4:5], v2, off ; GFX1250-FAKE16-NEXT: s_endpgm -; -; GFX1250-REAL16-LABEL: test_cvt_pk_fp8_f32_word1: -; GFX1250-REAL16: ; %bb.0: -; GFX1250-REAL16-NEXT: global_wb -; GFX1250-REAL16-NEXT: v_nop -; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0 -; GFX1250-REAL16-NEXT: v_mov_b32_dpp v0, v0 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf bound_ctrl:1 -; GFX1250-REAL16-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3 -; GFX1250-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX1250-REAL16-NEXT: v_cvt_pk_fp8_f32 v2.h, v0, v1 -; GFX1250-REAL16-NEXT: global_store_b32 v[4:5], v2, off -; GFX1250-REAL16-NEXT: s_endpgm %tmp0 = call i32 @llvm.amdgcn.mov.dpp.i32(i32 %a, i32 228, i32 15, i32 15, i1 1) %tmp1 = bitcast i32 %tmp0 to float %ret = tail call i32 @llvm.amdgcn.cvt.pk.fp8.f32(float %tmp1, float %y, i32 %old, i1 true) diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.load.ll index 6837c3fcb293d..6f2a5774ed691 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.load.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.load.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -;RUN: llc < %s -mtriple=amdgpu6.01 | FileCheck %s --check-prefixes=PREGFX10 -;RUN: llc < %s -mtriple=amdgpu8.02 | FileCheck %s --check-prefixes=PREGFX10 +;RUN: llc < %s -mtriple=amdgpu6.01 | FileCheck %s --check-prefixes=PREGFX10,GFX6 +;RUN: llc < %s -mtriple=amdgpu8.02 | FileCheck %s --check-prefixes=PREGFX10,GFX8 ;RUN: llc < %s -mtriple=amdgpu10.10 | FileCheck %s --check-prefixes=GFX10 ;RUN: llc < %s -mtriple=amdgpu11.00 -amdgpu-enable-delay-alu=0 | FileCheck %s --check-prefixes=GFX11 ;RUN: llc < %s -mtriple=amdgpu12.00 -amdgpu-enable-delay-alu=0 | FileCheck %s --check-prefixes=GFX12,GFX12-SDAG @@ -435,6 +435,20 @@ main_body: } define amdgpu_ps <4 x float> @buffer_load_negative_offset(<4 x i32> inreg, i32 %ofs) { +; GFX6-LABEL: buffer_load_negative_offset: +; GFX6: ; %bb.0: ; %main_body +; GFX6-NEXT: v_add_i32_e32 v0, vcc, -16, v0 +; GFX6-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen +; GFX6-NEXT: s_waitcnt vmcnt(0) +; GFX6-NEXT: ; return to shader part epilog +; +; GFX8-LABEL: buffer_load_negative_offset: +; GFX8: ; %bb.0: ; %main_body +; GFX8-NEXT: v_add_u32_e32 v0, vcc, -16, v0 +; GFX8-NEXT: buffer_load_dwordx4 v[0:3], v0, s[0:3], 0 offen +; GFX8-NEXT: s_waitcnt vmcnt(0) +; GFX8-NEXT: ; return to shader part epilog +; ; GFX10-LABEL: buffer_load_negative_offset: ; GFX10: ; %bb.0: ; %main_body ; GFX10-NEXT: v_add_nc_u32_e32 v0, -16, v0 @@ -462,6 +476,30 @@ main_body: } define amdgpu_ps float @buffer_load_mmo(<4 x i32> inreg %rsrc, ptr addrspace(3) %lds) { +; GFX6-LABEL: buffer_load_mmo: +; GFX6: ; %bb.0: ; %entry +; GFX6-NEXT: buffer_load_dword v1, off, s[0:3], 0 +; GFX6-NEXT: v_mov_b32_e32 v2, 0 +; GFX6-NEXT: s_mov_b32 m0, -1 +; GFX6-NEXT: ds_write_b32 v0, v2 +; GFX6-NEXT: v_add_i32_e32 v0, vcc, 16, v0 +; GFX6-NEXT: ds_write_b32 v0, v2 +; GFX6-NEXT: s_waitcnt vmcnt(0) +; GFX6-NEXT: v_mov_b32_e32 v0, v1 +; GFX6-NEXT: s_waitcnt lgkmcnt(0) +; GFX6-NEXT: ; return to shader part epilog +; +; GFX8-LABEL: buffer_load_mmo: +; GFX8: ; %bb.0: ; %entry +; GFX8-NEXT: buffer_load_dword v1, off, s[0:3], 0 +; GFX8-NEXT: v_mov_b32_e32 v2, 0 +; GFX8-NEXT: s_mov_b32 m0, -1 +; GFX8-NEXT: ds_write2_b32 v0, v2, v2 offset1:4 +; GFX8-NEXT: s_waitcnt vmcnt(0) +; GFX8-NEXT: v_mov_b32_e32 v0, v1 +; GFX8-NEXT: s_waitcnt lgkmcnt(0) +; GFX8-NEXT: ; return to shader part epilog +; ; GFX10-LABEL: buffer_load_mmo: ; GFX10: ; %bb.0: ; %entry ; GFX10-NEXT: buffer_load_dword v1, off, s[0:3], 0 diff --git a/llvm/test/CodeGen/AMDGPU/llvm.floor.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.floor.f16.ll index c9d2623aca41c..0c42d730bd8cf 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.floor.f16.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.floor.f16.ll @@ -1,7 +1,7 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2 ; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgpu6.00 < %s | FileCheck -check-prefixes=SI %s ; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgpu8.03 -mattr=-flat-for-global < %s | FileCheck -check-prefixes=VI %s -; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgpu11.00 -mattr=+real-true16,-flat-for-global < %s | FileCheck -check-prefixes=GFX11 %s +; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgpu11.00 -mattr=+real-true16,-flat-for-global < %s | FileCheck -check-prefixes=GFX11-TRUE16 %s ; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgpu11.00 -mattr=-real-true16,-flat-for-global < %s | FileCheck -check-prefixes=GFX11-FAKE16 %s declare half @llvm.floor.f16(half %a) @@ -46,23 +46,23 @@ define amdgpu_kernel void @floor_f16( ; VI-NEXT: buffer_store_short v0, off, s[4:7], 0 ; VI-NEXT: s_endpgm ; -; GFX11-LABEL: floor_f16: -; GFX11: ; %bb.0: ; %entry -; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 -; GFX11-NEXT: s_mov_b32 s6, -1 -; GFX11-NEXT: s_mov_b32 s7, 0x31016000 -; GFX11-NEXT: s_mov_b32 s10, s6 -; GFX11-NEXT: s_mov_b32 s11, s7 -; GFX11-NEXT: s_waitcnt lgkmcnt(0) -; GFX11-NEXT: s_mov_b32 s8, s2 -; GFX11-NEXT: s_mov_b32 s9, s3 -; GFX11-NEXT: s_mov_b32 s4, s0 -; GFX11-NEXT: buffer_load_d16_b16 v0, off, s[8:11], 0 -; GFX11-NEXT: s_mov_b32 s5, s1 -; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: v_floor_f16_e32 v0.l, v0.l -; GFX11-NEXT: buffer_store_b16 v0, off, s[4:7], 0 -; GFX11-NEXT: s_endpgm +; GFX11-TRUE16-LABEL: floor_f16: +; GFX11-TRUE16: ; %bb.0: ; %entry +; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 +; GFX11-TRUE16-NEXT: s_mov_b32 s6, -1 +; GFX11-TRUE16-NEXT: s_mov_b32 s7, 0x31016000 +; GFX11-TRUE16-NEXT: s_mov_b32 s10, s6 +; GFX11-TRUE16-NEXT: s_mov_b32 s11, s7 +; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-TRUE16-NEXT: s_mov_b32 s8, s2 +; GFX11-TRUE16-NEXT: s_mov_b32 s9, s3 +; GFX11-TRUE16-NEXT: s_mov_b32 s4, s0 +; GFX11-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[8:11], 0 +; GFX11-TRUE16-NEXT: s_mov_b32 s5, s1 +; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) +; GFX11-TRUE16-NEXT: v_floor_f16_e32 v0.l, v0.l +; GFX11-TRUE16-NEXT: buffer_store_b16 v0, off, s[4:7], 0 +; GFX11-TRUE16-NEXT: s_endpgm ; ; GFX11-FAKE16-LABEL: floor_f16: ; GFX11-FAKE16: ; %bb.0: ; %entry @@ -145,26 +145,26 @@ define amdgpu_kernel void @floor_v2f16( ; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0 ; VI-NEXT: s_endpgm ; -; GFX11-LABEL: floor_v2f16: -; GFX11: ; %bb.0: ; %entry -; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 -; GFX11-NEXT: s_mov_b32 s6, -1 -; GFX11-NEXT: s_mov_b32 s7, 0x31016000 -; GFX11-NEXT: s_mov_b32 s10, s6 -; GFX11-NEXT: s_mov_b32 s11, s7 -; GFX11-NEXT: s_waitcnt lgkmcnt(0) -; GFX11-NEXT: s_mov_b32 s8, s2 -; GFX11-NEXT: s_mov_b32 s9, s3 -; GFX11-NEXT: s_mov_b32 s4, s0 -; GFX11-NEXT: buffer_load_b32 v0, off, s[8:11], 0 -; GFX11-NEXT: s_mov_b32 s5, s1 -; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: v_lshrrev_b32_e32 v1, 16, v0 -; GFX11-NEXT: v_floor_f16_e32 v0.l, v0.l -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) -; GFX11-NEXT: v_floor_f16_e32 v0.h, v1.l -; GFX11-NEXT: buffer_store_b32 v0, off, s[4:7], 0 -; GFX11-NEXT: s_endpgm +; GFX11-TRUE16-LABEL: floor_v2f16: +; GFX11-TRUE16: ; %bb.0: ; %entry +; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 +; GFX11-TRUE16-NEXT: s_mov_b32 s6, -1 +; GFX11-TRUE16-NEXT: s_mov_b32 s7, 0x31016000 +; GFX11-TRUE16-NEXT: s_mov_b32 s10, s6 +; GFX11-TRUE16-NEXT: s_mov_b32 s11, s7 +; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-TRUE16-NEXT: s_mov_b32 s8, s2 +; GFX11-TRUE16-NEXT: s_mov_b32 s9, s3 +; GFX11-TRUE16-NEXT: s_mov_b32 s4, s0 +; GFX11-TRUE16-NEXT: buffer_load_b32 v0, off, s[8:11], 0 +; GFX11-TRUE16-NEXT: s_mov_b32 s5, s1 +; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) +; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0 +; GFX11-TRUE16-NEXT: v_floor_f16_e32 v0.l, v0.l +; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX11-TRUE16-NEXT: v_floor_f16_e32 v0.h, v1.l +; GFX11-TRUE16-NEXT: buffer_store_b32 v0, off, s[4:7], 0 +; GFX11-TRUE16-NEXT: s_endpgm ; ; GFX11-FAKE16-LABEL: floor_v2f16: ; GFX11-FAKE16: ; %bb.0: ; %entry diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-mul.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-mul.ll index afaf446c583b9..eab785f75438a 100644 --- a/llvm/test/CodeGen/AMDGPU/vector-reduce-mul.ll +++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-mul.ll @@ -9,12 +9,11 @@ ; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu10.10 < %s | FileCheck -check-prefixes=GFX10,GFX10-GISEL %s ; RUN: llc -global-isel=0 -mtriple=amdgpu11.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-SDAG,GFX11-SDAG-TRUE16 %s ; RUN: llc -global-isel=0 -mtriple=amdgpu11.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-SDAG,GFX11-SDAG-FAKE16 %s -; FIXME-TRUE16. enable gisel -; XUN: llc -global-isel=1 -mtriple=amdgpu11.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL,GFX11-GISEL-TRUE16 %s +; RUN: llc -global-isel=1 -mtriple=amdgpu11.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL,GFX11-GISEL-TRUE16 %s ; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu11.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL,GFX11-GISEL-FAKE16 %s ; RUN: llc -global-isel=0 -mtriple=amdgpu12.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-SDAG,GFX12-SDAG-TRUE16 %s ; RUN: llc -global-isel=0 -mtriple=amdgpu12.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-SDAG,GFX12-SDAG-FAKE16 %s -; XUN: llc -global-isel=1 -mtriple=amdgpu12.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL,GFX12-GISEL-TRUE16 %s +; RUN: llc -global-isel=1 -mtriple=amdgpu12.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL,GFX12-GISEL-TRUE16 %s ; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL,GFX12-GISEL-FAKE16 %s define i8 @test_vector_reduce_mul_v2i8(<2 x i8> %v) { @@ -102,11 +101,17 @@ define i8 @test_vector_reduce_mul_v2i8(<2 x i8> %v) { ; GFX11-SDAG-FAKE16-NEXT: v_or_b32_e32 v0, v0, v1 ; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-GISEL-LABEL: test_vector_reduce_mul_v2i8: -; GFX11-GISEL: ; %bb.0: ; %entry -; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1 -; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31] +; GFX11-GISEL-TRUE16-LABEL: test_vector_reduce_mul_v2i8: +; GFX11-GISEL-TRUE16: ; %bb.0: ; %entry +; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v1.l +; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-GISEL-FAKE16-LABEL: test_vector_reduce_mul_v2i8: +; GFX11-GISEL-FAKE16: ; %bb.0: ; %entry +; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_mul_v2i8: ; GFX12-SDAG-TRUE16: ; %bb.0: ; %entry @@ -142,15 +147,25 @@ define i8 @test_vector_reduce_mul_v2i8(<2 x i8> %v) { ; GFX12-SDAG-FAKE16-NEXT: v_or_b32_e32 v0, v0, v1 ; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31] ; -; GFX12-GISEL-LABEL: test_vector_reduce_mul_v2i8: -; GFX12-GISEL: ; %bb.0: ; %entry -; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-GISEL-NEXT: s_wait_expcnt 0x0 -; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0 -; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0 -; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0 -; GFX12-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1 -; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31] +; GFX12-GISEL-TRUE16-LABEL: test_vector_reduce_mul_v2i8: +; GFX12-GISEL-TRUE16: ; %bb.0: ; %entry +; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-GISEL-TRUE16-NEXT: s_wait_expcnt 0x0 +; GFX12-GISEL-TRUE16-NEXT: s_wait_samplecnt 0x0 +; GFX12-GISEL-TRUE16-NEXT: s_wait_bvhcnt 0x0 +; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0 +; GFX12-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v1.l +; GFX12-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX12-GISEL-FAKE16-LABEL: test_vector_reduce_mul_v2i8: +; GFX12-GISEL-FAKE16: ; %bb.0: ; %entry +; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-GISEL-FAKE16-NEXT: s_wait_expcnt 0x0 +; GFX12-GISEL-FAKE16-NEXT: s_wait_samplecnt 0x0 +; GFX12-GISEL-FAKE16-NEXT: s_wait_bvhcnt 0x0 +; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0 +; GFX12-GISEL-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v1 +; GFX12-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31] entry: %res = call i8 @llvm.vector.reduce.mul.v2i8(<2 x i8> %v) ret i8 %res @@ -229,13 +244,21 @@ define i8 @test_vector_reduce_mul_v3i8(<3 x i8> %v) { ; GFX11-SDAG-FAKE16-NEXT: v_mul_lo_u16 v0, v1, v0 ; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-GISEL-LABEL: test_vector_reduce_mul_v3i8: -; GFX11-GISEL: ; %bb.0: ; %entry -; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1 -; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v2 -; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31] +; GFX11-GISEL-TRUE16-LABEL: test_vector_reduce_mul_v3i8: +; GFX11-GISEL-TRUE16: ; %bb.0: ; %entry +; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v1.l +; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v2.l +; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-GISEL-FAKE16-LABEL: test_vector_reduce_mul_v3i8: +; GFX11-GISEL-FAKE16: ; %bb.0: ; %entry +; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-GISEL-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v2 +; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_mul_v3i8: ; GFX12-SDAG-TRUE16: ; %bb.0: ; %entry @@ -261,17 +284,29 @@ define i8 @test_vector_reduce_mul_v3i8(<3 x i8> %v) { ; GFX12-SDAG-FAKE16-NEXT: v_mul_lo_u16 v0, v1, v0 ; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31] ; -; GFX12-GISEL-LABEL: test_vector_reduce_mul_v3i8: -; GFX12-GISEL: ; %bb.0: ; %entry -; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-GISEL-NEXT: s_wait_expcnt 0x0 -; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0 -; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0 -; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0 -; GFX12-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1 -; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-GISEL-NEXT: v_mul_lo_u16 v0, v0, v2 -; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31] +; GFX12-GISEL-TRUE16-LABEL: test_vector_reduce_mul_v3i8: +; GFX12-GISEL-TRUE16: ; %bb.0: ; %entry +; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-GISEL-TRUE16-NEXT: s_wait_expcnt 0x0 +; GFX12-GISEL-TRUE16-NEXT: s_wait_samplecnt 0x0 +; GFX12-GISEL-TRUE16-NEXT: s_wait_bvhcnt 0x0 +; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0 +; GFX12-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v1.l +; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v2.l +; GFX12-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX12-GISEL-FAKE16-LABEL: test_vector_reduce_mul_v3i8: +; GFX12-GISEL-FAKE16: ; %bb.0: ; %entry +; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-GISEL-FAKE16-NEXT: s_wait_expcnt 0x0 +; GFX12-GISEL-FAKE16-NEXT: s_wait_samplecnt 0x0 +; GFX12-GISEL-FAKE16-NEXT: s_wait_bvhcnt 0x0 +; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0 +; GFX12-GISEL-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v1 +; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-GISEL-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v2 +; GFX12-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31] entry: %res = call i8 @llvm.vector.reduce.mul.v3i8(<3 x i8> %v) ret i8 %res @@ -370,15 +405,25 @@ define i8 @test_vector_reduce_mul_v4i8(<4 x i8> %v) { ; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0 ; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-GISEL-LABEL: test_vector_reduce_mul_v4i8: -; GFX11-GISEL: ; %bb.0: ; %entry -; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v2 -; GFX11-GISEL-NEXT: v_mul_lo_u16 v1, v1, v3 -; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1 -; GFX11-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 8 -; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31] +; GFX11-GISEL-TRUE16-LABEL: test_vector_reduce_mul_v4i8: +; GFX11-GISEL-TRUE16: ; %bb.0: ; %entry +; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v2.l +; GFX11-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.h, v1.l, v3.l +; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v0.h +; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v0, v0, 0, 8 +; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-GISEL-FAKE16-LABEL: test_vector_reduce_mul_v4i8: +; GFX11-GISEL-FAKE16: ; %bb.0: ; %entry +; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v2 +; GFX11-GISEL-FAKE16-NEXT: v_mul_lo_u16 v1, v1, v3 +; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-GISEL-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-FAKE16-NEXT: v_bfe_u32 v0, v0, 0, 8 +; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_mul_v4i8: ; GFX12-SDAG-TRUE16: ; %bb.0: ; %entry @@ -408,19 +453,33 @@ define i8 @test_vector_reduce_mul_v4i8(<4 x i8> %v) { ; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0 ; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31] ; -; GFX12-GISEL-LABEL: test_vector_reduce_mul_v4i8: -; GFX12-GISEL: ; %bb.0: ; %entry -; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-GISEL-NEXT: s_wait_expcnt 0x0 -; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0 -; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0 -; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0 -; GFX12-GISEL-NEXT: v_mul_lo_u16 v0, v0, v2 -; GFX12-GISEL-NEXT: v_mul_lo_u16 v1, v1, v3 -; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1 -; GFX12-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 8 -; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31] +; GFX12-GISEL-TRUE16-LABEL: test_vector_reduce_mul_v4i8: +; GFX12-GISEL-TRUE16: ; %bb.0: ; %entry +; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-GISEL-TRUE16-NEXT: s_wait_expcnt 0x0 +; GFX12-GISEL-TRUE16-NEXT: s_wait_samplecnt 0x0 +; GFX12-GISEL-TRUE16-NEXT: s_wait_bvhcnt 0x0 +; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0 +; GFX12-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v2.l +; GFX12-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.h, v1.l, v3.l +; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v0.h +; GFX12-GISEL-TRUE16-NEXT: v_bfe_u32 v0, v0, 0, 8 +; GFX12-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX12-GISEL-FAKE16-LABEL: test_vector_reduce_mul_v4i8: +; GFX12-GISEL-FAKE16: ; %bb.0: ; %entry +; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-GISEL-FAKE16-NEXT: s_wait_expcnt 0x0 +; GFX12-GISEL-FAKE16-NEXT: s_wait_samplecnt 0x0 +; GFX12-GISEL-FAKE16-NEXT: s_wait_bvhcnt 0x0 +; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0 +; GFX12-GISEL-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v2 +; GFX12-GISEL-FAKE16-NEXT: v_mul_lo_u16 v1, v1, v3 +; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-GISEL-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v1 +; GFX12-GISEL-FAKE16-NEXT: v_bfe_u32 v0, v0, 0, 8 +; GFX12-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31] entry: %res = call i8 @llvm.vector.reduce.mul.v4i8(<4 x i8> %v) ret i8 %res @@ -561,20 +620,35 @@ define i8 @test_vector_reduce_mul_v8i8(<8 x i8> %v) { ; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0 ; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-GISEL-LABEL: test_vector_reduce_mul_v8i8: -; GFX11-GISEL: ; %bb.0: ; %entry -; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v4 -; GFX11-GISEL-NEXT: v_mul_lo_u16 v1, v1, v5 -; GFX11-GISEL-NEXT: v_mul_lo_u16 v2, v2, v6 -; GFX11-GISEL-NEXT: v_mul_lo_u16 v3, v3, v7 -; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v2 -; GFX11-GISEL-NEXT: v_mul_lo_u16 v1, v1, v3 -; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1 -; GFX11-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 8 -; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31] +; GFX11-GISEL-TRUE16-LABEL: test_vector_reduce_mul_v8i8: +; GFX11-GISEL-TRUE16: ; %bb.0: ; %entry +; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v4.l +; GFX11-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.h, v1.l, v5.l +; GFX11-GISEL-TRUE16-NEXT: v_mul_lo_u16 v1.l, v2.l, v6.l +; GFX11-GISEL-TRUE16-NEXT: v_mul_lo_u16 v1.h, v3.l, v7.l +; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v1.l +; GFX11-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.h, v0.h, v1.h +; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v0.h +; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v0, v0, 0, 8 +; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-GISEL-FAKE16-LABEL: test_vector_reduce_mul_v8i8: +; GFX11-GISEL-FAKE16: ; %bb.0: ; %entry +; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v4 +; GFX11-GISEL-FAKE16-NEXT: v_mul_lo_u16 v1, v1, v5 +; GFX11-GISEL-FAKE16-NEXT: v_mul_lo_u16 v2, v2, v6 +; GFX11-GISEL-FAKE16-NEXT: v_mul_lo_u16 v3, v3, v7 +; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v2 +; GFX11-GISEL-FAKE16-NEXT: v_mul_lo_u16 v1, v1, v3 +; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-GISEL-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-FAKE16-NEXT: v_bfe_u32 v0, v0, 0, 8 +; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_mul_v8i8: ; GFX12-SDAG-TRUE16: ; %bb.0: ; %entry @@ -614,24 +688,43 @@ define i8 @test_vector_reduce_mul_v8i8(<8 x i8> %v) { ; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0 ; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31] ; -; GFX12-GISEL-LABEL: test_vector_reduce_mul_v8i8: -; GFX12-GISEL: ; %bb.0: ; %entry -; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-GISEL-NEXT: s_wait_expcnt 0x0 -; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0 -; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0 -; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0 -; GFX12-GISEL-NEXT: v_mul_lo_u16 v0, v0, v4 -; GFX12-GISEL-NEXT: v_mul_lo_u16 v1, v1, v5 -; GFX12-GISEL-NEXT: v_mul_lo_u16 v2, v2, v6 -; GFX12-GISEL-NEXT: v_mul_lo_u16 v3, v3, v7 -; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX12-GISEL-NEXT: v_mul_lo_u16 v0, v0, v2 -; GFX12-GISEL-NEXT: v_mul_lo_u16 v1, v1, v3 -; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1 -; GFX12-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 8 -; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31] +; GFX12-GISEL-TRUE16-LABEL: test_vector_reduce_mul_v8i8: +; GFX12-GISEL-TRUE16: ; %bb.0: ; %entry +; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-GISEL-TRUE16-NEXT: s_wait_expcnt 0x0 +; GFX12-GISEL-TRUE16-NEXT: s_wait_samplecnt 0x0 +; GFX12-GISEL-TRUE16-NEXT: s_wait_bvhcnt 0x0 +; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0 +; GFX12-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v4.l +; GFX12-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.h, v1.l, v5.l +; GFX12-GISEL-TRUE16-NEXT: v_mul_lo_u16 v1.l, v2.l, v6.l +; GFX12-GISEL-TRUE16-NEXT: v_mul_lo_u16 v1.h, v3.l, v7.l +; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX12-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v1.l +; GFX12-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.h, v0.h, v1.h +; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v0.h +; GFX12-GISEL-TRUE16-NEXT: v_bfe_u32 v0, v0, 0, 8 +; GFX12-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX12-GISEL-FAKE16-LABEL: test_vector_reduce_mul_v8i8: +; GFX12-GISEL-FAKE16: ; %bb.0: ; %entry +; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-GISEL-FAKE16-NEXT: s_wait_expcnt 0x0 +; GFX12-GISEL-FAKE16-NEXT: s_wait_samplecnt 0x0 +; GFX12-GISEL-FAKE16-NEXT: s_wait_bvhcnt 0x0 +; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0 +; GFX12-GISEL-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v4 +; GFX12-GISEL-FAKE16-NEXT: v_mul_lo_u16 v1, v1, v5 +; GFX12-GISEL-FAKE16-NEXT: v_mul_lo_u16 v2, v2, v6 +; GFX12-GISEL-FAKE16-NEXT: v_mul_lo_u16 v3, v3, v7 +; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX12-GISEL-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v2 +; GFX12-GISEL-FAKE16-NEXT: v_mul_lo_u16 v1, v1, v3 +; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-GISEL-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v1 +; GFX12-GISEL-FAKE16-NEXT: v_bfe_u32 v0, v0, 0, 8 +; GFX12-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31] entry: %res = call i8 @llvm.vector.reduce.mul.v8i8(<8 x i8> %v) ret i8 %res @@ -854,30 +947,55 @@ define i8 @test_vector_reduce_mul_v16i8(<16 x i8> %v) { ; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0 ; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-GISEL-LABEL: test_vector_reduce_mul_v16i8: -; GFX11-GISEL: ; %bb.0: ; %entry -; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v8 -; GFX11-GISEL-NEXT: v_mul_lo_u16 v1, v1, v9 -; GFX11-GISEL-NEXT: v_mul_lo_u16 v2, v2, v10 -; GFX11-GISEL-NEXT: v_mul_lo_u16 v3, v3, v11 -; GFX11-GISEL-NEXT: v_mul_lo_u16 v4, v4, v12 -; GFX11-GISEL-NEXT: v_mul_lo_u16 v5, v5, v13 -; GFX11-GISEL-NEXT: v_mul_lo_u16 v6, v6, v14 -; GFX11-GISEL-NEXT: v_mul_lo_u16 v7, v7, v15 -; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v4 -; GFX11-GISEL-NEXT: v_mul_lo_u16 v1, v1, v5 -; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX11-GISEL-NEXT: v_mul_lo_u16 v2, v2, v6 -; GFX11-GISEL-NEXT: v_mul_lo_u16 v3, v3, v7 -; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v2 -; GFX11-GISEL-NEXT: v_mul_lo_u16 v1, v1, v3 -; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1 -; GFX11-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 8 -; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31] +; GFX11-GISEL-TRUE16-LABEL: test_vector_reduce_mul_v16i8: +; GFX11-GISEL-TRUE16: ; %bb.0: ; %entry +; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v8.l +; GFX11-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.h, v1.l, v9.l +; GFX11-GISEL-TRUE16-NEXT: v_mul_lo_u16 v1.l, v2.l, v10.l +; GFX11-GISEL-TRUE16-NEXT: v_mul_lo_u16 v1.h, v3.l, v11.l +; GFX11-GISEL-TRUE16-NEXT: v_mul_lo_u16 v2.l, v4.l, v12.l +; GFX11-GISEL-TRUE16-NEXT: v_mul_lo_u16 v2.h, v5.l, v13.l +; GFX11-GISEL-TRUE16-NEXT: v_mul_lo_u16 v3.l, v6.l, v14.l +; GFX11-GISEL-TRUE16-NEXT: v_mul_lo_u16 v3.h, v7.l, v15.l +; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v2.l +; GFX11-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.h, v0.h, v2.h +; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-GISEL-TRUE16-NEXT: v_mul_lo_u16 v1.l, v1.l, v3.l +; GFX11-GISEL-TRUE16-NEXT: v_mul_lo_u16 v1.h, v1.h, v3.h +; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v1.l +; GFX11-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.h, v0.h, v1.h +; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v0.h +; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v0, v0, 0, 8 +; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-GISEL-FAKE16-LABEL: test_vector_reduce_mul_v16i8: +; GFX11-GISEL-FAKE16: ; %bb.0: ; %entry +; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v8 +; GFX11-GISEL-FAKE16-NEXT: v_mul_lo_u16 v1, v1, v9 +; GFX11-GISEL-FAKE16-NEXT: v_mul_lo_u16 v2, v2, v10 +; GFX11-GISEL-FAKE16-NEXT: v_mul_lo_u16 v3, v3, v11 +; GFX11-GISEL-FAKE16-NEXT: v_mul_lo_u16 v4, v4, v12 +; GFX11-GISEL-FAKE16-NEXT: v_mul_lo_u16 v5, v5, v13 +; GFX11-GISEL-FAKE16-NEXT: v_mul_lo_u16 v6, v6, v14 +; GFX11-GISEL-FAKE16-NEXT: v_mul_lo_u16 v7, v7, v15 +; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-GISEL-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v4 +; GFX11-GISEL-FAKE16-NEXT: v_mul_lo_u16 v1, v1, v5 +; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-GISEL-FAKE16-NEXT: v_mul_lo_u16 v2, v2, v6 +; GFX11-GISEL-FAKE16-NEXT: v_mul_lo_u16 v3, v3, v7 +; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v2 +; GFX11-GISEL-FAKE16-NEXT: v_mul_lo_u16 v1, v1, v3 +; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-GISEL-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-FAKE16-NEXT: v_bfe_u32 v0, v0, 0, 8 +; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_mul_v16i8: ; GFX12-SDAG-TRUE16: ; %bb.0: ; %entry @@ -935,34 +1053,63 @@ define i8 @test_vector_reduce_mul_v16i8(<16 x i8> %v) { ; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0 ; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31] ; -; GFX12-GISEL-LABEL: test_vector_reduce_mul_v16i8: -; GFX12-GISEL: ; %bb.0: ; %entry -; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-GISEL-NEXT: s_wait_expcnt 0x0 -; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0 -; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0 -; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0 -; GFX12-GISEL-NEXT: v_mul_lo_u16 v0, v0, v8 -; GFX12-GISEL-NEXT: v_mul_lo_u16 v1, v1, v9 -; GFX12-GISEL-NEXT: v_mul_lo_u16 v2, v2, v10 -; GFX12-GISEL-NEXT: v_mul_lo_u16 v3, v3, v11 -; GFX12-GISEL-NEXT: v_mul_lo_u16 v4, v4, v12 -; GFX12-GISEL-NEXT: v_mul_lo_u16 v5, v5, v13 -; GFX12-GISEL-NEXT: v_mul_lo_u16 v6, v6, v14 -; GFX12-GISEL-NEXT: v_mul_lo_u16 v7, v7, v15 -; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX12-GISEL-NEXT: v_mul_lo_u16 v0, v0, v4 -; GFX12-GISEL-NEXT: v_mul_lo_u16 v1, v1, v5 -; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX12-GISEL-NEXT: v_mul_lo_u16 v2, v2, v6 -; GFX12-GISEL-NEXT: v_mul_lo_u16 v3, v3, v7 -; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX12-GISEL-NEXT: v_mul_lo_u16 v0, v0, v2 -; GFX12-GISEL-NEXT: v_mul_lo_u16 v1, v1, v3 -; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1 -; GFX12-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 8 -; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31] +; GFX12-GISEL-TRUE16-LABEL: test_vector_reduce_mul_v16i8: +; GFX12-GISEL-TRUE16: ; %bb.0: ; %entry +; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-GISEL-TRUE16-NEXT: s_wait_expcnt 0x0 +; GFX12-GISEL-TRUE16-NEXT: s_wait_samplecnt 0x0 +; GFX12-GISEL-TRUE16-NEXT: s_wait_bvhcnt 0x0 +; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0 +; GFX12-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v8.l +; GFX12-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.h, v1.l, v9.l +; GFX12-GISEL-TRUE16-NEXT: v_mul_lo_u16 v1.l, v2.l, v10.l +; GFX12-GISEL-TRUE16-NEXT: v_mul_lo_u16 v1.h, v3.l, v11.l +; GFX12-GISEL-TRUE16-NEXT: v_mul_lo_u16 v2.l, v4.l, v12.l +; GFX12-GISEL-TRUE16-NEXT: v_mul_lo_u16 v2.h, v5.l, v13.l +; GFX12-GISEL-TRUE16-NEXT: v_mul_lo_u16 v3.l, v6.l, v14.l +; GFX12-GISEL-TRUE16-NEXT: v_mul_lo_u16 v3.h, v7.l, v15.l +; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX12-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v2.l +; GFX12-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.h, v0.h, v2.h +; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX12-GISEL-TRUE16-NEXT: v_mul_lo_u16 v1.l, v1.l, v3.l +; GFX12-GISEL-TRUE16-NEXT: v_mul_lo_u16 v1.h, v1.h, v3.h +; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX12-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v1.l +; GFX12-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.h, v0.h, v1.h +; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v0.h +; GFX12-GISEL-TRUE16-NEXT: v_bfe_u32 v0, v0, 0, 8 +; GFX12-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX12-GISEL-FAKE16-LABEL: test_vector_reduce_mul_v16i8: +; GFX12-GISEL-FAKE16: ; %bb.0: ; %entry +; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-GISEL-FAKE16-NEXT: s_wait_expcnt 0x0 +; GFX12-GISEL-FAKE16-NEXT: s_wait_samplecnt 0x0 +; GFX12-GISEL-FAKE16-NEXT: s_wait_bvhcnt 0x0 +; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0 +; GFX12-GISEL-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v8 +; GFX12-GISEL-FAKE16-NEXT: v_mul_lo_u16 v1, v1, v9 +; GFX12-GISEL-FAKE16-NEXT: v_mul_lo_u16 v2, v2, v10 +; GFX12-GISEL-FAKE16-NEXT: v_mul_lo_u16 v3, v3, v11 +; GFX12-GISEL-FAKE16-NEXT: v_mul_lo_u16 v4, v4, v12 +; GFX12-GISEL-FAKE16-NEXT: v_mul_lo_u16 v5, v5, v13 +; GFX12-GISEL-FAKE16-NEXT: v_mul_lo_u16 v6, v6, v14 +; GFX12-GISEL-FAKE16-NEXT: v_mul_lo_u16 v7, v7, v15 +; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX12-GISEL-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v4 +; GFX12-GISEL-FAKE16-NEXT: v_mul_lo_u16 v1, v1, v5 +; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX12-GISEL-FAKE16-NEXT: v_mul_lo_u16 v2, v2, v6 +; GFX12-GISEL-FAKE16-NEXT: v_mul_lo_u16 v3, v3, v7 +; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX12-GISEL-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v2 +; GFX12-GISEL-FAKE16-NEXT: v_mul_lo_u16 v1, v1, v3 +; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-GISEL-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v1 +; GFX12-GISEL-FAKE16-NEXT: v_bfe_u32 v0, v0, 0, 8 +; GFX12-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31] entry: %res = call i8 @llvm.vector.reduce.mul.v16i8(<16 x i8> %v) ret i8 %res @@ -1013,13 +1160,21 @@ define i16 @test_vector_reduce_mul_v2i16(<2 x i16> %v) { ; GFX11-SDAG-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1 ; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-GISEL-LABEL: test_vector_reduce_mul_v2i16: -; GFX11-GISEL: ; %bb.0: ; %entry -; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-GISEL-NEXT: v_alignbit_b32 v1, s0, v0, 16 -; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1 -; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31] +; GFX11-GISEL-TRUE16-LABEL: test_vector_reduce_mul_v2i16: +; GFX11-GISEL-TRUE16: ; %bb.0: ; %entry +; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h +; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-GISEL-TRUE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-GISEL-FAKE16-LABEL: test_vector_reduce_mul_v2i16: +; GFX11-GISEL-FAKE16: ; %bb.0: ; %entry +; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-FAKE16-NEXT: v_alignbit_b32 v1, s0, v0, 16 +; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-GISEL-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_mul_v2i16: ; GFX12-SDAG-TRUE16: ; %bb.0: ; %entry @@ -1045,17 +1200,29 @@ define i16 @test_vector_reduce_mul_v2i16(<2 x i16> %v) { ; GFX12-SDAG-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1 ; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31] ; -; GFX12-GISEL-LABEL: test_vector_reduce_mul_v2i16: -; GFX12-GISEL: ; %bb.0: ; %entry -; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-GISEL-NEXT: s_wait_expcnt 0x0 -; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0 -; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0 -; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0 -; GFX12-GISEL-NEXT: v_alignbit_b32 v1, s0, v0, 16 -; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1 -; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31] +; GFX12-GISEL-TRUE16-LABEL: test_vector_reduce_mul_v2i16: +; GFX12-GISEL-TRUE16: ; %bb.0: ; %entry +; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-GISEL-TRUE16-NEXT: s_wait_expcnt 0x0 +; GFX12-GISEL-TRUE16-NEXT: s_wait_samplecnt 0x0 +; GFX12-GISEL-TRUE16-NEXT: s_wait_bvhcnt 0x0 +; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0 +; GFX12-GISEL-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h +; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-GISEL-TRUE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1 +; GFX12-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX12-GISEL-FAKE16-LABEL: test_vector_reduce_mul_v2i16: +; GFX12-GISEL-FAKE16: ; %bb.0: ; %entry +; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-GISEL-FAKE16-NEXT: s_wait_expcnt 0x0 +; GFX12-GISEL-FAKE16-NEXT: s_wait_samplecnt 0x0 +; GFX12-GISEL-FAKE16-NEXT: s_wait_bvhcnt 0x0 +; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0 +; GFX12-GISEL-FAKE16-NEXT: v_alignbit_b32 v1, s0, v0, 16 +; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-GISEL-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1 +; GFX12-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31] entry: %res = call i16 @llvm.vector.reduce.mul.v2i16(<2 x i16> %v) ret i16 %res @@ -1121,14 +1288,22 @@ define i16 @test_vector_reduce_mul_v3i16(<3 x i16> %v) { ; GFX11-SDAG-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v2 ; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-GISEL-LABEL: test_vector_reduce_mul_v3i16: -; GFX11-GISEL: ; %bb.0: ; %entry -; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0 -; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1 -; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v2 -; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31] +; GFX11-GISEL-TRUE16-LABEL: test_vector_reduce_mul_v3i16: +; GFX11-GISEL-TRUE16: ; %bb.0: ; %entry +; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v0.h +; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v1.l +; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-GISEL-FAKE16-LABEL: test_vector_reduce_mul_v3i16: +; GFX11-GISEL-FAKE16: ; %bb.0: ; %entry +; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0 +; GFX11-GISEL-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-GISEL-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v2 +; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_mul_v3i16: ; GFX12-SDAG-TRUE16: ; %bb.0: ; %entry @@ -1155,18 +1330,30 @@ define i16 @test_vector_reduce_mul_v3i16(<3 x i16> %v) { ; GFX12-SDAG-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v2 ; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31] ; -; GFX12-GISEL-LABEL: test_vector_reduce_mul_v3i16: -; GFX12-GISEL: ; %bb.0: ; %entry -; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-GISEL-NEXT: s_wait_expcnt 0x0 -; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0 -; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0 -; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0 -; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0 -; GFX12-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1 -; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-GISEL-NEXT: v_mul_lo_u16 v0, v0, v2 -; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31] +; GFX12-GISEL-TRUE16-LABEL: test_vector_reduce_mul_v3i16: +; GFX12-GISEL-TRUE16: ; %bb.0: ; %entry +; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-GISEL-TRUE16-NEXT: s_wait_expcnt 0x0 +; GFX12-GISEL-TRUE16-NEXT: s_wait_samplecnt 0x0 +; GFX12-GISEL-TRUE16-NEXT: s_wait_bvhcnt 0x0 +; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0 +; GFX12-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v0.h +; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v1.l +; GFX12-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX12-GISEL-FAKE16-LABEL: test_vector_reduce_mul_v3i16: +; GFX12-GISEL-FAKE16: ; %bb.0: ; %entry +; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-GISEL-FAKE16-NEXT: s_wait_expcnt 0x0 +; GFX12-GISEL-FAKE16-NEXT: s_wait_samplecnt 0x0 +; GFX12-GISEL-FAKE16-NEXT: s_wait_bvhcnt 0x0 +; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0 +; GFX12-GISEL-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0 +; GFX12-GISEL-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1 +; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-GISEL-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v2 +; GFX12-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31] entry: %res = call i16 @llvm.vector.reduce.mul.v3i16(<3 x i16> %v) ret i16 %res @@ -1238,14 +1425,23 @@ define i16 @test_vector_reduce_mul_v4i16(<4 x i16> %v) { ; GFX11-SDAG-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1 ; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-GISEL-LABEL: test_vector_reduce_mul_v4i16: -; GFX11-GISEL: ; %bb.0: ; %entry -; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1 -; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-GISEL-NEXT: v_alignbit_b32 v1, s0, v0, 16 -; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1 -; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31] +; GFX11-GISEL-TRUE16-LABEL: test_vector_reduce_mul_v4i16: +; GFX11-GISEL-TRUE16: ; %bb.0: ; %entry +; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-TRUE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h +; GFX11-GISEL-TRUE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-GISEL-FAKE16-LABEL: test_vector_reduce_mul_v4i16: +; GFX11-GISEL-FAKE16: ; %bb.0: ; %entry +; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-GISEL-FAKE16-NEXT: v_alignbit_b32 v1, s0, v0, 16 +; GFX11-GISEL-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_mul_v4i16: ; GFX12-SDAG-TRUE16: ; %bb.0: ; %entry @@ -1273,18 +1469,31 @@ define i16 @test_vector_reduce_mul_v4i16(<4 x i16> %v) { ; GFX12-SDAG-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1 ; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31] ; -; GFX12-GISEL-LABEL: test_vector_reduce_mul_v4i16: -; GFX12-GISEL: ; %bb.0: ; %entry -; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-GISEL-NEXT: s_wait_expcnt 0x0 -; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0 -; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0 -; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0 -; GFX12-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1 -; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-GISEL-NEXT: v_alignbit_b32 v1, s0, v0, 16 -; GFX12-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1 -; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31] +; GFX12-GISEL-TRUE16-LABEL: test_vector_reduce_mul_v4i16: +; GFX12-GISEL-TRUE16: ; %bb.0: ; %entry +; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-GISEL-TRUE16-NEXT: s_wait_expcnt 0x0 +; GFX12-GISEL-TRUE16-NEXT: s_wait_samplecnt 0x0 +; GFX12-GISEL-TRUE16-NEXT: s_wait_bvhcnt 0x0 +; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0 +; GFX12-GISEL-TRUE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1 +; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-GISEL-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h +; GFX12-GISEL-TRUE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1 +; GFX12-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX12-GISEL-FAKE16-LABEL: test_vector_reduce_mul_v4i16: +; GFX12-GISEL-FAKE16: ; %bb.0: ; %entry +; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-GISEL-FAKE16-NEXT: s_wait_expcnt 0x0 +; GFX12-GISEL-FAKE16-NEXT: s_wait_samplecnt 0x0 +; GFX12-GISEL-FAKE16-NEXT: s_wait_bvhcnt 0x0 +; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0 +; GFX12-GISEL-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1 +; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-GISEL-FAKE16-NEXT: v_alignbit_b32 v1, s0, v0, 16 +; GFX12-GISEL-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1 +; GFX12-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31] entry: %res = call i16 @llvm.vector.reduce.mul.v4i16(<4 x i16> %v) ret i16 %res @@ -1405,17 +1614,29 @@ define i16 @test_vector_reduce_mul_v8i16(<8 x i16> %v) { ; GFX11-SDAG-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1 ; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-GISEL-LABEL: test_vector_reduce_mul_v8i16: -; GFX11-GISEL: ; %bb.0: ; %entry -; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2 -; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v3 -; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1 -; GFX11-GISEL-NEXT: v_alignbit_b32 v1, s0, v0, 16 -; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1 -; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31] +; GFX11-GISEL-TRUE16-LABEL: test_vector_reduce_mul_v8i16: +; GFX11-GISEL-TRUE16: ; %bb.0: ; %entry +; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-TRUE16-NEXT: v_pk_mul_lo_u16 v0, v0, v2 +; GFX11-GISEL-TRUE16-NEXT: v_pk_mul_lo_u16 v1, v1, v3 +; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-GISEL-TRUE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h +; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-GISEL-TRUE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-GISEL-FAKE16-LABEL: test_vector_reduce_mul_v8i16: +; GFX11-GISEL-FAKE16: ; %bb.0: ; %entry +; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v2 +; GFX11-GISEL-FAKE16-NEXT: v_pk_mul_lo_u16 v1, v1, v3 +; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-GISEL-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-FAKE16-NEXT: v_alignbit_b32 v1, s0, v0, 16 +; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-GISEL-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_mul_v8i16: ; GFX12-SDAG-TRUE16: ; %bb.0: ; %entry @@ -1449,21 +1670,37 @@ define i16 @test_vector_reduce_mul_v8i16(<8 x i16> %v) { ; GFX12-SDAG-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1 ; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31] ; -; GFX12-GISEL-LABEL: test_vector_reduce_mul_v8i16: -; GFX12-GISEL: ; %bb.0: ; %entry -; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-GISEL-NEXT: s_wait_expcnt 0x0 -; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0 -; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0 -; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0 -; GFX12-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2 -; GFX12-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v3 -; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1 -; GFX12-GISEL-NEXT: v_alignbit_b32 v1, s0, v0, 16 -; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1 -; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31] +; GFX12-GISEL-TRUE16-LABEL: test_vector_reduce_mul_v8i16: +; GFX12-GISEL-TRUE16: ; %bb.0: ; %entry +; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-GISEL-TRUE16-NEXT: s_wait_expcnt 0x0 +; GFX12-GISEL-TRUE16-NEXT: s_wait_samplecnt 0x0 +; GFX12-GISEL-TRUE16-NEXT: s_wait_bvhcnt 0x0 +; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0 +; GFX12-GISEL-TRUE16-NEXT: v_pk_mul_lo_u16 v0, v0, v2 +; GFX12-GISEL-TRUE16-NEXT: v_pk_mul_lo_u16 v1, v1, v3 +; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-GISEL-TRUE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1 +; GFX12-GISEL-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h +; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-GISEL-TRUE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1 +; GFX12-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX12-GISEL-FAKE16-LABEL: test_vector_reduce_mul_v8i16: +; GFX12-GISEL-FAKE16: ; %bb.0: ; %entry +; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-GISEL-FAKE16-NEXT: s_wait_expcnt 0x0 +; GFX12-GISEL-FAKE16-NEXT: s_wait_samplecnt 0x0 +; GFX12-GISEL-FAKE16-NEXT: s_wait_bvhcnt 0x0 +; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0 +; GFX12-GISEL-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v2 +; GFX12-GISEL-FAKE16-NEXT: v_pk_mul_lo_u16 v1, v1, v3 +; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-GISEL-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1 +; GFX12-GISEL-FAKE16-NEXT: v_alignbit_b32 v1, s0, v0, 16 +; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-GISEL-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1 +; GFX12-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31] entry: %res = call i16 @llvm.vector.reduce.mul.v8i16(<8 x i16> %v) ret i16 %res @@ -1642,22 +1879,39 @@ define i16 @test_vector_reduce_mul_v16i16(<16 x i16> %v) { ; GFX11-SDAG-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1 ; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31] ; -; GFX11-GISEL-LABEL: test_vector_reduce_mul_v16i16: -; GFX11-GISEL: ; %bb.0: ; %entry -; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v4 -; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v5 -; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v2, v2, v6 -; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v3, v3, v7 -; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2 -; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v3 -; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1 -; GFX11-GISEL-NEXT: v_alignbit_b32 v1, s0, v0, 16 -; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1 -; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31] +; GFX11-GISEL-TRUE16-LABEL: test_vector_reduce_mul_v16i16: +; GFX11-GISEL-TRUE16: ; %bb.0: ; %entry +; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-TRUE16-NEXT: v_pk_mul_lo_u16 v0, v0, v4 +; GFX11-GISEL-TRUE16-NEXT: v_pk_mul_lo_u16 v1, v1, v5 +; GFX11-GISEL-TRUE16-NEXT: v_pk_mul_lo_u16 v2, v2, v6 +; GFX11-GISEL-TRUE16-NEXT: v_pk_mul_lo_u16 v3, v3, v7 +; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-TRUE16-NEXT: v_pk_mul_lo_u16 v0, v0, v2 +; GFX11-GISEL-TRUE16-NEXT: v_pk_mul_lo_u16 v1, v1, v3 +; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-GISEL-TRUE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h +; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-GISEL-TRUE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-GISEL-FAKE16-LABEL: test_vector_reduce_mul_v16i16: +; GFX11-GISEL-FAKE16: ; %bb.0: ; %entry +; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v4 +; GFX11-GISEL-FAKE16-NEXT: v_pk_mul_lo_u16 v1, v1, v5 +; GFX11-GISEL-FAKE16-NEXT: v_pk_mul_lo_u16 v2, v2, v6 +; GFX11-GISEL-FAKE16-NEXT: v_pk_mul_lo_u16 v3, v3, v7 +; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v2 +; GFX11-GISEL-FAKE16-NEXT: v_pk_mul_lo_u16 v1, v1, v3 +; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-GISEL-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-FAKE16-NEXT: v_alignbit_b32 v1, s0, v0, 16 +; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-GISEL-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31] ; ; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_mul_v16i16: ; GFX12-SDAG-TRUE16: ; %bb.0: ; %entry @@ -1701,26 +1955,47 @@ define i16 @test_vector_reduce_mul_v16i16(<16 x i16> %v) { ; GFX12-SDAG-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1 ; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31] ; -; GFX12-GISEL-LABEL: test_vector_reduce_mul_v16i16: -; GFX12-GISEL: ; %bb.0: ; %entry -; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0 -; GFX12-GISEL-NEXT: s_wait_expcnt 0x0 -; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0 -; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0 -; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0 -; GFX12-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v4 -; GFX12-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v5 -; GFX12-GISEL-NEXT: v_pk_mul_lo_u16 v2, v2, v6 -; GFX12-GISEL-NEXT: v_pk_mul_lo_u16 v3, v3, v7 -; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX12-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2 -; GFX12-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v3 -; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX12-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1 -; GFX12-GISEL-NEXT: v_alignbit_b32 v1, s0, v0, 16 -; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX12-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1 -; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31] +; GFX12-GISEL-TRUE16-LABEL: test_vector_reduce_mul_v16i16: +; GFX12-GISEL-TRUE16: ; %bb.0: ; %entry +; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-GISEL-TRUE16-NEXT: s_wait_expcnt 0x0 +; GFX12-GISEL-TRUE16-NEXT: s_wait_samplecnt 0x0 +; GFX12-GISEL-TRUE16-NEXT: s_wait_bvhcnt 0x0 +; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0 +; GFX12-GISEL-TRUE16-NEXT: v_pk_mul_lo_u16 v0, v0, v4 +; GFX12-GISEL-TRUE16-NEXT: v_pk_mul_lo_u16 v1, v1, v5 +; GFX12-GISEL-TRUE16-NEXT: v_pk_mul_lo_u16 v2, v2, v6 +; GFX12-GISEL-TRUE16-NEXT: v_pk_mul_lo_u16 v3, v3, v7 +; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX12-GISEL-TRUE16-NEXT: v_pk_mul_lo_u16 v0, v0, v2 +; GFX12-GISEL-TRUE16-NEXT: v_pk_mul_lo_u16 v1, v1, v3 +; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-GISEL-TRUE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1 +; GFX12-GISEL-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h +; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-GISEL-TRUE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1 +; GFX12-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31] +; +; GFX12-GISEL-FAKE16-LABEL: test_vector_reduce_mul_v16i16: +; GFX12-GISEL-FAKE16: ; %bb.0: ; %entry +; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-GISEL-FAKE16-NEXT: s_wait_expcnt 0x0 +; GFX12-GISEL-FAKE16-NEXT: s_wait_samplecnt 0x0 +; GFX12-GISEL-FAKE16-NEXT: s_wait_bvhcnt 0x0 +; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0 +; GFX12-GISEL-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v4 +; GFX12-GISEL-FAKE16-NEXT: v_pk_mul_lo_u16 v1, v1, v5 +; GFX12-GISEL-FAKE16-NEXT: v_pk_mul_lo_u16 v2, v2, v6 +; GFX12-GISEL-FAKE16-NEXT: v_pk_mul_lo_u16 v3, v3, v7 +; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX12-GISEL-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v2 +; GFX12-GISEL-FAKE16-NEXT: v_pk_mul_lo_u16 v1, v1, v3 +; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX12-GISEL-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1 +; GFX12-GISEL-FAKE16-NEXT: v_alignbit_b32 v1, s0, v0, 16 +; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-GISEL-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1 +; GFX12-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31] entry: %res = call i16 @llvm.vector.reduce.mul.v16i16(<16 x i16> %v) ret i16 %res @@ -4228,6 +4503,3 @@ declare i64 @llvm.vector.reduce.mul.v3i64(<3 x i64>) declare i64 @llvm.vector.reduce.mul.v4i64(<4 x i64>) declare i64 @llvm.vector.reduce.mul.v8i64(<8 x i64>) declare i64 @llvm.vector.reduce.mul.v16i64(<16 x i64>) -;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: -; GFX11-GISEL-FAKE16: {{.*}} -; GFX12-GISEL-FAKE16: {{.*}} diff --git a/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w32-f16-f32-matrix-modifiers.ll b/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w32-f16-f32-matrix-modifiers.ll index a29c5fc84c70c..a3107fb580395 100644 --- a/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w32-f16-f32-matrix-modifiers.ll +++ b/llvm/test/CodeGen/AMDGPU/wmma-gfx12-w32-f16-f32-matrix-modifiers.ll @@ -1,7 +1,7 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc -mtriple=amdgpu11.70 < %s | FileCheck %s --check-prefixes=GCN,GFX1170 -; RUN: llc -mtriple=amdgpu12.00 -mattr=+real-true16 < %s | FileCheck %s --check-prefixes=GCN,GFX12 -; RUN: llc -mtriple=amdgpu12.00 -mattr=-real-true16 < %s | FileCheck %s --check-prefixes=GCN,GFX12 +; RUN: llc -mtriple=amdgpu12.00 -mattr=+real-true16 < %s | FileCheck %s --check-prefixes=GCN,GFX12,GFX12-TRUE16 +; RUN: llc -mtriple=amdgpu12.00 -mattr=-real-true16 < %s | FileCheck %s --check-prefixes=GCN,GFX12,GFX12-FAKE16 define amdgpu_ps void @test_wmma_f32_16x16x16_f16_negA(<8 x half> %A, <8 x half> %B, <8 x float> %C, ptr addrspace(1) %out) { ; GCN-LABEL: test_wmma_f32_16x16x16_f16_negA: @@ -424,6 +424,41 @@ define amdgpu_ps void @test_wmma_f16_16x16x16_f16_negC_pack(<8 x half> %A, <8 x ; GFX1170-NEXT: v_wmma_f16_16x16x16_f16 v[12:15], v[0:3], v[4:7], v[12:15] neg_lo:[0,0,1] ; GFX1170-NEXT: global_store_b128 v[10:11], v[12:15], off ; GFX1170-NEXT: s_endpgm +; +; GFX12-TRUE16-LABEL: test_wmma_f16_16x16x16_f16_negC_pack: +; GFX12-TRUE16: ; %bb.0: ; %bb +; GFX12-TRUE16-NEXT: s_clause 0x1 +; GFX12-TRUE16-NEXT: flat_load_b128 v[12:15], v[8:9] +; GFX12-TRUE16-NEXT: flat_load_b128 v[16:19], v[8:9] offset:16 +; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x101 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v14.h, v15.l +; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.h, v17.l +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v18.h, v19.l +; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.h, v13.l +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX12-TRUE16-NEXT: v_dual_mov_b32 v13, v14 :: v_dual_mov_b32 v14, v16 +; GFX12-TRUE16-NEXT: v_mov_b32_e32 v15, v18 +; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-TRUE16-NEXT: v_wmma_f16_16x16x16_f16 v[12:15], v[0:3], v[4:7], v[12:15] neg_lo:[0,0,1] +; GFX12-TRUE16-NEXT: global_store_b128 v[10:11], v[12:15], off +; GFX12-TRUE16-NEXT: s_endpgm +; +; GFX12-FAKE16-LABEL: test_wmma_f16_16x16x16_f16_negC_pack: +; GFX12-FAKE16: ; %bb.0: ; %bb +; GFX12-FAKE16-NEXT: s_clause 0x1 +; GFX12-FAKE16-NEXT: flat_load_b128 v[12:15], v[8:9] offset:16 +; GFX12-FAKE16-NEXT: flat_load_b128 v[16:19], v[8:9] +; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x101 +; GFX12-FAKE16-NEXT: v_perm_b32 v15, v15, v14, 0x5040100 +; GFX12-FAKE16-NEXT: v_perm_b32 v14, v13, v12, 0x5040100 +; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0 +; GFX12-FAKE16-NEXT: v_perm_b32 v13, v19, v18, 0x5040100 +; GFX12-FAKE16-NEXT: v_perm_b32 v12, v17, v16, 0x5040100 +; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX12-FAKE16-NEXT: v_wmma_f16_16x16x16_f16 v[12:15], v[0:3], v[4:7], v[12:15] neg_lo:[0,0,1] +; GFX12-FAKE16-NEXT: global_store_b128 v[10:11], v[12:15], off +; GFX12-FAKE16-NEXT: s_endpgm bb: %C = load <16 x half>, ptr %Caddr %C_shuffle = shufflevector <16 x half> %C, <16 x half> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14> _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
