https://github.com/jrbyrnes updated https://github.com/llvm/llvm-project/pull/239855
>From e5b40e7ea6bae38bc075f60078403ec750c5330e Mon Sep 17 00:00:00 2001 From: Jeffrey Byrnes <[email protected]> Date: Fri, 6 Nov 2026 15:42:55 -0800 Subject: [PATCH 1/2] [AMDGPU] Replace getRepeatRate with getBlockingCycles Change-Id: I5598664f64fc010bc1b8fed0c3a1b53477ef208b --- .../lib/Target/AMDGPU/GCNHazardRecognizer.cpp | 16 +-- llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 18 --- .../CodeGen/AMDGPU/GlobalISel/fptrunc.bf16.ll | 10 +- llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll | 53 ++++---- llvm/test/CodeGen/AMDGPU/addrspacecast-gas.ll | 4 +- llvm/test/CodeGen/AMDGPU/bf16-conversions.ll | 8 +- llvm/test/CodeGen/AMDGPU/bf16.ll | 89 +++++++------- .../CodeGen/AMDGPU/calling-conventions.ll | 5 +- llvm/test/CodeGen/AMDGPU/clmul.ll | 3 +- llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll | 26 ++-- .../test/CodeGen/AMDGPU/flat-saddr-atomics.ll | 12 +- .../AMDGPU/float-to-arbitrary-fp-fp8-hw.ll | 29 +++-- .../AMDGPU/float-to-arbitrary-fp-widen.ll | 2 +- .../CodeGen/AMDGPU/global-atomicrmw-fadd.ll | 4 +- llvm/test/CodeGen/AMDGPU/global-load-xcnt.ll | 46 +++---- .../CodeGen/AMDGPU/integer-mad-patterns.ll | 47 ++++---- .../AMDGPU/llvm.amdgcn.cvt.scale.pk.ll | 36 +++--- .../CodeGen/AMDGPU/llvm.amdgcn.update.dpp.ll | 2 +- llvm/test/CodeGen/AMDGPU/lshl-add-u64.ll | 10 +- llvm/test/CodeGen/AMDGPU/mad_64_32.ll | 12 +- llvm/test/CodeGen/AMDGPU/mul.ll | 6 +- llvm/test/CodeGen/AMDGPU/packed-fp32.ll | 6 +- .../AMDGPU/reassoc-mul-add-1-to-mad.ll | 27 +++-- llvm/test/CodeGen/AMDGPU/shl.v2i64.ll | 114 ++++++++++++------ llvm/test/CodeGen/AMDGPU/slp-int-to-fp.ll | 45 ++++--- .../CodeGen/AMDGPU/strict_fptrunc_bf16.ll | 18 +-- 26 files changed, 326 insertions(+), 322 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp index b91ef681cf6722..ccc34018fdc52b 100644 --- a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp +++ b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp @@ -279,7 +279,7 @@ void GCNHazardRecognizer::updateTRANSState(const MachineInstr &MI) { void GCNHazardRecognizer::updateMultiCycleVALUState(const MachineInstr &MI) { if (!hasCoExecWindowModel()) return; - // Multi-cycle VALU (CVT, etc.) blocks subsequent VALU for repeat rate cycles. + // Multi-cycle VALU (CVT, etc.) blocks subsequent VALU for BlockingCycles cycles. if (!SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true)) return; @@ -288,13 +288,13 @@ void GCNHazardRecognizer::updateMultiCycleVALUState(const MachineInstr &MI) { SIInstrInfo::isMFMA(MI) || SIInstrInfo::isTRANS(MI)) return; - unsigned RepeatRate = TII.getRepeatRate(MI); - if (RepeatRate > 1) { + unsigned BlockingCycles = TII.getBlockingCycles(MI); + if (BlockingCycles > 1) { // bumpCycle's AdvanceCycle decrements once before the next pick's // hazard check (same convention as CyclesUntilTRANS), so to expose - // RepeatRate-1 cycles of shadow we must seed with RepeatRate. - CyclesUntilVALU = RepeatRate; - LLVM_DEBUG(dbgs() << " Multi-cycle VALU: repeat=" << RepeatRate + // BlockingCycles-1 cycles of shadow we must seed with BlockingCycles. + CyclesUntilVALU = BlockingCycles; + LLVM_DEBUG(dbgs() << " Multi-cycle VALU: BlockingCycles=" << BlockingCycles << ", CyclesUntilVALU=" << CyclesUntilVALU << "\n"); } } @@ -315,7 +315,7 @@ unsigned GCNHazardRecognizer::checkTRANSHazard(const MachineInstr &MI) const { if (SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) && !SIInstrInfo::isWMMA(MI) && !SIInstrInfo::isSWMMAC(MI) && - TII.getRepeatRate(MI) > 1) + TII.getBlockingCycles(MI) > 1) return CyclesUntilTRANS; return 0; @@ -327,7 +327,7 @@ GCNHazardRecognizer::checkMultiCycleVALUHazard(const MachineInstr &MI) const { return 0; // Multi-cycle VALU blocks anything on the VALU pipe - VALU, WMMA, SWMMAC, - // and TRANS - for RepeatRate-1 cycles. Only off-pipe instructions (MEM, + // and TRANS - for BlockingCycles-1 cycles. Only off-pipe instructions (MEM, // SALU, control) can fill the shadow. if (!SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true) && !SIInstrInfo::isWMMA(MI) && !SIInstrInfo::isSWMMAC(MI) && diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp index 7895c1c8a6a8fe..fe4957a9a5218f 100644 --- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp +++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp @@ -11974,24 +11974,6 @@ void SIInstrInfo::enforceOperandRCAlignment(MachineInstr &MI, MI.addOperand(MachineOperand::CreateReg(NewVR, false, true)); } -unsigned SIInstrInfo::getRepeatRate(const MachineInstr &MI) const { - if (!SchedModel.hasInstrSchedModel()) - return 0; - - // The repeat rate is the throughput-limiting resource occupancy: the largest - // number of cycles any written processor resource is held. - const MCSchedClassDesc *SCDesc = SchedModel.resolveSchedClass(&MI); - unsigned RepeatRate = 0; - for (TargetSchedModel::ProcResIter - PI = SchedModel.getWriteProcResBegin(SCDesc), - PE = SchedModel.getWriteProcResEnd(SCDesc); - PI != PE; ++PI) { - RepeatRate = std::max(RepeatRate, (unsigned)PI->ReleaseAtCycle); - } - - return RepeatRate; -} - bool SIInstrInfo::isGlobalMemoryObject(const MachineInstr *MI) const { if (isIGLP(*MI)) return false; diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fptrunc.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fptrunc.bf16.ll index 1c4a9b928f75bb..64888d85c6d775 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fptrunc.bf16.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fptrunc.bf16.ll @@ -221,28 +221,28 @@ define amdgpu_ps bfloat @fptrunc_f64_to_bf16_s(double inreg %a) { ; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX1250-NEXT: v_cvt_f64_f32_e32 v[0:1], v2 ; GFX1250-NEXT: v_cmp_nlg_f64_e32 vcc_lo, s[0:1], v[0:1] -; GFX1250-NEXT: v_cmp_ngt_f64_e64 s0, |s[0:1]|, |v[0:1]| -; GFX1250-NEXT: v_readfirstlane_b32 s1, v2 ; GFX1250-NEXT: s_cmp_lg_u32 vcc_lo, 0 +; GFX1250-NEXT: v_cmp_ngt_f64_e64 s0, |s[0:1]|, |v[0:1]| ; GFX1250-NEXT: s_cselect_b32 s2, 1, 0 -; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1) +; GFX1250-NEXT: v_readfirstlane_b32 s1, v2 ; GFX1250-NEXT: s_or_b32 s2, s2, s1 ; GFX1250-NEXT: s_cmp_lg_u32 s0, 0 ; GFX1250-NEXT: s_cselect_b32 s0, -1, 0 ; GFX1250-NEXT: s_and_b32 s2, s2, 1 ; GFX1250-NEXT: s_or_b32 s0, s0, 1 +; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1) ; GFX1250-NEXT: s_add_co_i32 s0, s1, s0 ; GFX1250-NEXT: s_cmp_lg_u32 s2, 0 ; GFX1250-NEXT: s_cselect_b32 s0, s1, s0 -; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1) ; GFX1250-NEXT: s_bfe_u32 s1, s0, 0x10010 ; GFX1250-NEXT: s_or_b32 s2, s0, 0x400000 ; GFX1250-NEXT: s_add_co_i32 s1, s1, s0 +; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1) ; GFX1250-NEXT: s_addk_co_i32 s1, 0x7fff ; GFX1250-NEXT: s_cmp_u_f32 s0, 0 ; GFX1250-NEXT: s_cselect_b32 s0, s2, s1 -; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1) ; GFX1250-NEXT: s_lshr_b32 s0, s0, 16 +; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) ; GFX1250-NEXT: v_mov_b32_e32 v0, s0 ; GFX1250-NEXT: ; return to shader part epilog %result = fptrunc double %a to bfloat diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll index e0bcc5da36b58d..ee4af399d247a8 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll @@ -3602,70 +3602,65 @@ define i256 @v_mul_i256(i256 %num, i256 %den) { ; GFX1250-NEXT: s_wait_kmcnt 0x0 ; GFX1250-NEXT: v_dual_mov_b32 v16, v0 :: v_dual_mov_b32 v17, v1 ; GFX1250-NEXT: v_mul_lo_u32 v29, v5, v10 -; GFX1250-NEXT: v_mul_lo_u32 v30, v4, v11 -; GFX1250-NEXT: v_mul_lo_u32 v31, v3, v12 -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) ; GFX1250-NEXT: v_mad_nc_u64_u32 v[0:1], v16, v14, 0 +; GFX1250-NEXT: v_mul_lo_u32 v30, v4, v11 ; GFX1250-NEXT: v_mad_nc_u64_u32 v[18:19], v17, v13, v[0:1] ; GFX1250-NEXT: v_mad_nc_u64_u32 v[0:1], v16, v12, 0 ; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX1250-NEXT: v_mad_nc_u64_u32 v[20:21], v2, v12, v[18:19] ; GFX1250-NEXT: v_mad_co_u64_u32 v[18:19], s0, v17, v11, v[0:1] -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1250-NEXT: v_cndmask_b32_e64 v22, 0, 1, s0 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX1250-NEXT: v_mad_nc_u64_u32 v[0:1], v3, v11, v[20:21] -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX1250-NEXT: v_mad_co_u64_u32 v[20:21], vcc_lo, v2, v10, v[18:19] -; GFX1250-NEXT: v_add_co_ci_u32_e64 v22, null, 0, v22, vcc_lo -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2) ; GFX1250-NEXT: v_mad_nc_u64_u32 v[18:19], v4, v10, v[0:1] +; GFX1250-NEXT: v_cndmask_b32_e64 v22, 0, 1, s0 +; GFX1250-NEXT: v_mul_lo_u32 v31, v3, v12 +; GFX1250-NEXT: v_add_co_ci_u32_e64 v22, null, 0, v22, vcc_lo ; GFX1250-NEXT: v_mad_co_u64_u32 v[0:1], vcc_lo, v3, v9, v[20:21] -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1250-NEXT: v_add_co_ci_u32_e64 v26, null, 0, v22, vcc_lo ; GFX1250-NEXT: v_mad_nc_u64_u32 v[20:21], v5, v9, v[18:19] ; GFX1250-NEXT: v_mad_nc_u64_u32 v[18:19], v16, v10, 0 -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX1250-NEXT: v_add_co_ci_u32_e64 v26, null, 0, v22, vcc_lo ; GFX1250-NEXT: v_mad_co_u64_u32 v[22:23], vcc_lo, v4, v8, v[0:1] -; GFX1250-NEXT: v_add_co_ci_u32_e64 v28, null, 0, v26, vcc_lo ; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX1250-NEXT: v_mad_nc_u64_u32 v[24:25], v6, v8, v[20:21] ; GFX1250-NEXT: v_mad_co_u64_u32 v[0:1], s0, v17, v9, v[18:19] -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) ; GFX1250-NEXT: v_dual_mov_b32 v18, v23 :: v_dual_mov_b32 v19, v24 +; GFX1250-NEXT: v_add_co_ci_u32_e64 v28, null, 0, v26, vcc_lo ; GFX1250-NEXT: v_mad_co_u64_u32 v[20:21], vcc_lo, v2, v8, v[0:1] ; GFX1250-NEXT: v_mul_lo_u32 v24, v6, v9 ; GFX1250-NEXT: v_cndmask_b32_e64 v6, 0, 1, s0 -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) ; GFX1250-NEXT: v_mad_co_u64_u32 v[0:1], s0, v16, v13, v[18:19] -; GFX1250-NEXT: v_mov_b32_e32 v19, v22 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX1250-NEXT: v_dual_mov_b32 v18, v21 :: v_dual_mov_b32 v19, v22 ; GFX1250-NEXT: v_add_co_ci_u32_e64 v6, null, 0, v6, vcc_lo -; GFX1250-NEXT: v_mov_b32_e32 v18, v21 -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) ; GFX1250-NEXT: v_mad_co_u64_u32 v[26:27], vcc_lo, v17, v12, v[0:1] -; GFX1250-NEXT: v_mad_nc_u64_u32 v[0:1], v16, v8, 0 ; GFX1250-NEXT: v_mad_co_u64_u32 v[22:23], s2, v16, v11, v[18:19] -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX1250-NEXT: v_cndmask_b32_e64 v21, 0, 1, s2 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4) ; GFX1250-NEXT: v_mad_co_u64_u32 v[18:19], s1, v2, v11, v[26:27] -; GFX1250-NEXT: v_mov_b32_e32 v11, v20 -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX1250-NEXT: v_mad_nc_u64_u32 v[0:1], v16, v8, 0 +; GFX1250-NEXT: v_cndmask_b32_e64 v21, 0, 1, s2 ; GFX1250-NEXT: v_mad_co_u64_u32 v[26:27], s2, v17, v10, v[22:23] +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX1250-NEXT: v_mad_co_u64_u32 v[22:23], s3, v3, v10, v[18:19] -; GFX1250-NEXT: v_mov_b32_e32 v10, v1 +; GFX1250-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v11, v20 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX1250-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v21, s2 -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) ; GFX1250-NEXT: v_mad_co_u64_u32 v[18:19], s2, v2, v9, v[26:27] ; GFX1250-NEXT: v_mul_lo_u32 v2, v2, v13 +; GFX1250-NEXT: v_mad_co_u64_u32 v[12:13], s4, v4, v9, v[22:23] ; GFX1250-NEXT: v_mad_co_u64_u32 v[20:21], s5, v16, v9, v[10:11] -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) ; GFX1250-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, s2 -; GFX1250-NEXT: v_mad_co_u64_u32 v[12:13], s4, v4, v9, v[22:23] -; GFX1250-NEXT: v_mul_lo_u32 v22, v16, v15 -; GFX1250-NEXT: v_mul_lo_u32 v9, v17, v14 ; GFX1250-NEXT: v_mad_co_u64_u32 v[10:11], s2, v3, v8, v[18:19] ; GFX1250-NEXT: v_cndmask_b32_e64 v3, 0, 1, s5 -; GFX1250-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, s2 +; GFX1250-NEXT: v_mul_lo_u32 v22, v16, v15 +; GFX1250-NEXT: v_mul_lo_u32 v9, v17, v14 ; GFX1250-NEXT: v_mad_co_u64_u32 v[14:15], s5, v5, v8, v[12:13] +; GFX1250-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, s2 ; GFX1250-NEXT: v_mad_co_u64_u32 v[12:13], s2, v17, v8, v[20:21] ; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX1250-NEXT: v_add_co_ci_u32_e64 v3, s2, v3, v10, s2 diff --git a/llvm/test/CodeGen/AMDGPU/addrspacecast-gas.ll b/llvm/test/CodeGen/AMDGPU/addrspacecast-gas.ll index 208c8530781d5f..e478fa00bcf2f1 100644 --- a/llvm/test/CodeGen/AMDGPU/addrspacecast-gas.ll +++ b/llvm/test/CodeGen/AMDGPU/addrspacecast-gas.ll @@ -34,12 +34,12 @@ define amdgpu_kernel void @use_private_to_flat_addrspacecast(ptr addrspace(5) %p ; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE ; GFX1250-GISEL-NEXT: s_load_b32 s0, s[4:5], 0x24 nv ; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], src_flat_scratch_base_lo -; GFX1250-GISEL-NEXT: v_mbcnt_lo_u32_b32 v2, -1, 0 ; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0 +; GFX1250-GISEL-NEXT: s_cmp_lg_u32 s0, -1 +; GFX1250-GISEL-NEXT: v_mbcnt_lo_u32_b32 v2, -1, 0 ; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX1250-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, s0, v0 ; GFX1250-GISEL-NEXT: v_lshlrev_b32_e32 v2, 20, v2 -; GFX1250-GISEL-NEXT: s_cmp_lg_u32 s0, -1 ; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1) ; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, v2, v1, vcc_lo ; GFX1250-GISEL-NEXT: s_cselect_b32 vcc_lo, exec_lo, 0 diff --git a/llvm/test/CodeGen/AMDGPU/bf16-conversions.ll b/llvm/test/CodeGen/AMDGPU/bf16-conversions.ll index 0450b6cf7401ad..bd829a29457295 100644 --- a/llvm/test/CodeGen/AMDGPU/bf16-conversions.ll +++ b/llvm/test/CodeGen/AMDGPU/bf16-conversions.ll @@ -235,15 +235,15 @@ define amdgpu_ps float @v_test_cvt_v2f64_v2bf16_v(<2 x double> %src) { ; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX1250-NEXT: v_cvt_f64_f32_e32 v[4:5], v8 ; GFX1250-NEXT: v_cvt_f64_f32_e32 v[6:7], v9 -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) ; GFX1250-NEXT: v_cmp_gt_f64_e64 s1, |v[2:3]|, |v[4:5]| ; GFX1250-NEXT: v_cmp_nlg_f64_e32 vcc_lo, v[2:3], v[4:5] -; GFX1250-NEXT: v_cmp_nlg_f64_e64 s0, v[0:1], v[6:7] -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) ; GFX1250-NEXT: v_cndmask_b32_e64 v2, -1, 1, s1 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3) ; GFX1250-NEXT: v_cmp_gt_f64_e64 s1, |v[0:1]|, |v[6:7]| +; GFX1250-NEXT: v_cmp_nlg_f64_e64 s0, v[0:1], v[6:7] ; GFX1250-NEXT: v_dual_add_nc_u32 v1, v8, v2 :: v_dual_bitop2_b32 v10, 1, v8 bitop3:0x40 -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3) ; GFX1250-NEXT: v_cndmask_b32_e64 v0, -1, 1, s1 ; GFX1250-NEXT: v_and_b32_e32 v11, 1, v9 ; GFX1250-NEXT: v_cmp_ne_u32_e64 s1, 0, v10 diff --git a/llvm/test/CodeGen/AMDGPU/bf16.ll b/llvm/test/CodeGen/AMDGPU/bf16.ll index 5da163b24d948d..45e6859c61cba2 100644 --- a/llvm/test/CodeGen/AMDGPU/bf16.ll +++ b/llvm/test/CodeGen/AMDGPU/bf16.ll @@ -9105,54 +9105,45 @@ define <32 x double> @global_extload_v32bf16_to_v32f64(ptr addrspace(1) %ptr) #0 ; GFX1250-NEXT: v_cvt_f64_f32_e32 v[2:3], v3 ; GFX1250-NEXT: v_cvt_f64_f32_e32 v[8:9], v7 ; GFX1250-NEXT: v_cvt_f64_f32_e32 v[6:7], v11 -; GFX1250-NEXT: s_wait_loadcnt 0x11 -; GFX1250-NEXT: v_dual_lshlrev_b32 v69, 16, v17 :: v_dual_lshlrev_b32 v37, 16, v16 -; GFX1250-NEXT: s_wait_loadcnt 0xe +; GFX1250-NEXT: s_clause 0x1 +; GFX1250-NEXT: scratch_store_b128 v0, v[2:5], off offset:240 +; GFX1250-NEXT: scratch_store_b128 v0, v[6:9], off offset:224 +; GFX1250-NEXT: s_wait_loadcnt 0x0 +; GFX1250-NEXT: v_dual_lshlrev_b32 v34, 16, v34 :: v_dual_lshlrev_b32 v66, 16, v32 +; GFX1250-NEXT: v_dual_lshlrev_b32 v49, 16, v29 :: v_dual_lshlrev_b32 v52, 16, v30 +; GFX1250-NEXT: v_dual_lshlrev_b32 v64, 16, v25 :: v_dual_lshlrev_b32 v53, 16, v24 +; GFX1250-NEXT: v_dual_lshlrev_b32 v54, 16, v31 :: v_dual_lshlrev_b32 v65, 16, v33 +; GFX1250-NEXT: v_dual_lshlrev_b32 v23, 16, v23 :: v_dual_lshlrev_b32 v48, 16, v28 +; GFX1250-NEXT: v_dual_lshlrev_b32 v27, 16, v27 :: v_dual_lshlrev_b32 v26, 16, v26 ; GFX1250-NEXT: v_dual_lshlrev_b32 v36, 16, v20 :: v_dual_lshlrev_b32 v82, 16, v18 -; GFX1250-NEXT: s_wait_loadcnt 0xc +; GFX1250-NEXT: v_dual_lshlrev_b32 v69, 16, v17 :: v_dual_lshlrev_b32 v37, 16, v16 +; GFX1250-NEXT: s_wait_xcnt 0x0 +; GFX1250-NEXT: v_cvt_f64_f32_e32 v[8:9], v34 +; GFX1250-NEXT: v_cvt_f64_f32_e32 v[6:7], v66 ; GFX1250-NEXT: v_dual_lshlrev_b32 v68, 16, v21 :: v_dual_lshlrev_b32 v22, 16, v22 -; GFX1250-NEXT: s_wait_loadcnt 0x9 -; GFX1250-NEXT: v_dual_lshlrev_b32 v53, 16, v24 :: v_dual_lshlrev_b32 v64, 16, v25 -; GFX1250-NEXT: v_lshlrev_b32_e32 v23, 16, v23 -; GFX1250-NEXT: s_wait_loadcnt 0x7 -; GFX1250-NEXT: v_dual_lshlrev_b32 v27, 16, v27 :: v_dual_lshlrev_b32 v26, 16, v26 -; GFX1250-NEXT: s_wait_loadcnt 0x4 -; GFX1250-NEXT: v_dual_lshlrev_b32 v49, 16, v29 :: v_dual_lshlrev_b32 v52, 16, v30 -; GFX1250-NEXT: s_wait_loadcnt 0x3 -; GFX1250-NEXT: v_dual_lshlrev_b32 v54, 16, v31 :: v_dual_lshlrev_b32 v48, 16, v28 -; GFX1250-NEXT: s_wait_loadcnt 0x0 -; GFX1250-NEXT: v_dual_lshlrev_b32 v66, 16, v32 :: v_dual_lshlrev_b32 v34, 16, v34 -; GFX1250-NEXT: v_lshlrev_b32_e32 v65, 16, v33 -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX1250-NEXT: v_cvt_f64_f32_e32 v[4:5], v52 +; GFX1250-NEXT: v_cvt_f64_f32_e32 v[2:3], v53 ; GFX1250-NEXT: v_cvt_f64_f32_e32 v[54:55], v54 +; GFX1250-NEXT: v_cvt_f64_f32_e32 v[52:53], v65 +; GFX1250-NEXT: v_cvt_f64_f32_e32 v[66:67], v49 +; GFX1250-NEXT: v_cvt_f64_f32_e32 v[64:65], v64 ; GFX1250-NEXT: v_dual_lshlrev_b32 v11, 16, v14 :: v_dual_lshlrev_b32 v14, 16, v15 ; GFX1250-NEXT: v_lshlrev_b32_e32 v15, 16, v19 ; GFX1250-NEXT: v_cvt_f64_f32_e32 v[50:51], v48 +; GFX1250-NEXT: v_cvt_f64_f32_e32 v[48:49], v26 ; GFX1250-NEXT: v_cvt_f64_f32_e32 v[16:17], v35 ; GFX1250-NEXT: v_cvt_f64_f32_e32 v[28:29], v36 ; GFX1250-NEXT: v_cvt_f64_f32_e32 v[30:31], v37 ; GFX1250-NEXT: v_cvt_f64_f32_e32 v[36:37], v23 +; GFX1250-NEXT: v_cvt_f64_f32_e32 v[34:35], v27 ; GFX1250-NEXT: v_cvt_f64_f32_e32 v[32:33], v22 ; GFX1250-NEXT: v_cvt_f64_f32_e32 v[70:71], v68 ; GFX1250-NEXT: v_cvt_f64_f32_e32 v[68:69], v69 +; GFX1250-NEXT: v_cvt_f64_f32_e32 v[26:27], v82 ; GFX1250-NEXT: v_cvt_f64_f32_e32 v[24:25], v14 ; GFX1250-NEXT: v_cvt_f64_f32_e32 v[22:23], v15 ; GFX1250-NEXT: v_cvt_f64_f32_e32 v[20:21], v11 ; GFX1250-NEXT: v_cvt_f64_f32_e32 v[18:19], v10 -; GFX1250-NEXT: scratch_store_b128 v0, v[2:5], off offset:240 -; GFX1250-NEXT: s_wait_xcnt 0x0 -; GFX1250-NEXT: v_cvt_f64_f32_e32 v[4:5], v52 -; GFX1250-NEXT: scratch_store_b128 v0, v[6:9], off offset:224 -; GFX1250-NEXT: s_wait_xcnt 0x0 -; GFX1250-NEXT: v_cvt_f64_f32_e32 v[8:9], v34 -; GFX1250-NEXT: v_cvt_f64_f32_e32 v[6:7], v66 -; GFX1250-NEXT: v_cvt_f64_f32_e32 v[2:3], v53 -; GFX1250-NEXT: v_cvt_f64_f32_e32 v[52:53], v65 -; GFX1250-NEXT: v_cvt_f64_f32_e32 v[66:67], v49 -; GFX1250-NEXT: v_cvt_f64_f32_e32 v[64:65], v64 -; GFX1250-NEXT: v_cvt_f64_f32_e32 v[48:49], v26 -; GFX1250-NEXT: v_cvt_f64_f32_e32 v[34:35], v27 -; GFX1250-NEXT: v_cvt_f64_f32_e32 v[26:27], v82 ; GFX1250-NEXT: v_cvt_f64_f32_e32 v[82:83], v80 ; GFX1250-NEXT: v_cvt_f64_f32_e32 v[80:81], v81 ; GFX1250-NEXT: v_cvt_f64_f32_e32 v[14:15], v39 @@ -37372,22 +37363,23 @@ define <4 x bfloat> @v_sitofp_v4i64_to_v4bf16(<4 x i64> %x) #0 { ; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX1250-NEXT: v_min_u32_e32 v9, v12, v9 ; GFX1250-NEXT: v_min_u32_e32 v11, v13, v14 -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX1250-NEXT: v_lshlrev_b64_e32 v[4:5], v10, v[4:5] -; GFX1250-NEXT: v_lshlrev_b64_e32 v[2:3], v9, v[2:3] ; GFX1250-NEXT: v_min_u32_e32 v6, 1, v6 ; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX1250-NEXT: v_lshlrev_b64_e32 v[2:3], v9, v[2:3] ; GFX1250-NEXT: v_lshlrev_b64_e32 v[0:1], v11, v[0:1] -; GFX1250-NEXT: v_min_u32_e32 v4, 1, v4 ; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX1250-NEXT: v_min_u32_e32 v2, 1, v2 +; GFX1250-NEXT: v_min_u32_e32 v4, 1, v4 ; GFX1250-NEXT: v_or_b32_e32 v6, v7, v6 -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4) +; GFX1250-NEXT: v_min_u32_e32 v2, 1, v2 +; GFX1250-NEXT: v_sub_nc_u32_e32 v7, 32, v10 ; GFX1250-NEXT: v_min_u32_e32 v0, 1, v0 ; GFX1250-NEXT: v_dual_sub_nc_u32 v5, 32, v9 :: v_dual_bitop2_b32 v4, v5, v4 bitop3:0x54 -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) ; GFX1250-NEXT: v_or_b32_e32 v2, v3, v2 -; GFX1250-NEXT: v_dual_sub_nc_u32 v7, 32, v10 :: v_dual_bitop2_b32 v0, v1, v0 bitop3:0x54 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX1250-NEXT: v_or_b32_e32 v0, v1, v0 ; GFX1250-NEXT: v_sub_nc_u32_e32 v1, 32, v8 ; GFX1250-NEXT: v_cvt_f32_i32_e32 v3, v6 ; GFX1250-NEXT: v_cvt_f32_i32_e32 v4, v4 @@ -38886,8 +38878,8 @@ define <2 x bfloat> @v_uitofp_v2i64_to_v2bf16(<2 x i64> %x) #0 { ; GFX1250-NEXT: v_min_u32_e32 v2, 1, v2 ; GFX1250-NEXT: v_min_u32_e32 v0, 1, v0 ; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1250-NEXT: v_or_b32_e32 v2, v3, v2 -; GFX1250-NEXT: v_dual_sub_nc_u32 v3, 32, v5 :: v_dual_bitop2_b32 v0, v1, v0 bitop3:0x54 +; GFX1250-NEXT: v_dual_sub_nc_u32 v3, 32, v5 :: v_dual_bitop2_b32 v2, v3, v2 bitop3:0x54 +; GFX1250-NEXT: v_or_b32_e32 v0, v1, v0 ; GFX1250-NEXT: v_sub_nc_u32_e32 v1, 32, v4 ; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) ; GFX1250-NEXT: v_cvt_f32_u32_e32 v2, v2 @@ -39231,12 +39223,13 @@ define <3 x bfloat> @v_uitofp_v3i64_to_v3bf16(<3 x i64> %x) #0 { ; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) ; GFX1250-NEXT: v_lshlrev_b64_e32 v[0:1], v7, v[0:1] ; GFX1250-NEXT: v_lshlrev_b64_e32 v[4:5], v8, v[4:5] -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) ; GFX1250-NEXT: v_min_u32_e32 v2, 1, v2 +; GFX1250-NEXT: v_sub_nc_u32_e32 v8, 32, v8 ; GFX1250-NEXT: v_min_u32_e32 v0, 1, v0 -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX1250-NEXT: v_min_u32_e32 v4, 1, v4 -; GFX1250-NEXT: v_dual_sub_nc_u32 v8, 32, v8 :: v_dual_bitop2_b32 v2, v3, v2 bitop3:0x54 +; GFX1250-NEXT: v_or_b32_e32 v2, v3, v2 ; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) ; GFX1250-NEXT: v_dual_sub_nc_u32 v3, 32, v6 :: v_dual_bitop2_b32 v0, v1, v0 bitop3:0x54 ; GFX1250-NEXT: v_dual_sub_nc_u32 v4, 32, v7 :: v_dual_bitop2_b32 v1, v5, v4 bitop3:0x54 @@ -39670,17 +39663,17 @@ define <4 x bfloat> @v_uitofp_v4i64_to_v4bf16(<4 x i64> %x) #0 { ; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX1250-NEXT: v_min_u32_e32 v6, 1, v6 ; GFX1250-NEXT: v_min_u32_e32 v2, 1, v2 -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) ; GFX1250-NEXT: v_min_u32_e32 v0, 1, v0 +; GFX1250-NEXT: v_sub_nc_u32_e32 v8, 32, v8 ; GFX1250-NEXT: v_min_u32_e32 v4, 1, v4 -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX1250-NEXT: v_dual_sub_nc_u32 v8, 32, v8 :: v_dual_bitop2_b32 v6, v7, v6 bitop3:0x54 +; GFX1250-NEXT: v_or_b32_e32 v6, v7, v6 ; GFX1250-NEXT: v_dual_sub_nc_u32 v3, 32, v11 :: v_dual_bitop2_b32 v2, v3, v2 bitop3:0x54 -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX1250-NEXT: v_or_b32_e32 v0, v1, v0 -; GFX1250-NEXT: v_or_b32_e32 v1, v5, v4 ; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX1250-NEXT: v_or_b32_e32 v1, v5, v4 ; GFX1250-NEXT: v_cvt_f32_u32_e32 v4, v6 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) ; GFX1250-NEXT: v_cvt_f32_u32_e32 v2, v2 ; GFX1250-NEXT: v_sub_nc_u32_e32 v5, 32, v9 ; GFX1250-NEXT: v_cvt_f32_u32_e32 v0, v0 diff --git a/llvm/test/CodeGen/AMDGPU/calling-conventions.ll b/llvm/test/CodeGen/AMDGPU/calling-conventions.ll index 17aa94a597c566..4c7d91053e3bfd 100644 --- a/llvm/test/CodeGen/AMDGPU/calling-conventions.ll +++ b/llvm/test/CodeGen/AMDGPU/calling-conventions.ll @@ -1802,16 +1802,15 @@ define amdgpu_kernel void @amd_kernel_v3i8(<3 x i8> %arg0) { ; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE ; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x24 nv ; GFX1250-NEXT: v_mov_b64_e32 v[0:1], 2 -; GFX1250-NEXT: v_mov_b64_e32 v[2:3], 0 ; GFX1250-NEXT: s_wait_kmcnt 0x0 ; GFX1250-NEXT: s_lshr_b32 s1, s0, 16 ; GFX1250-NEXT: s_bfe_u32 s2, s0, 0x80008 ; GFX1250-NEXT: s_add_co_i32 s1, s1, s1 -; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) +; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2) ; GFX1250-NEXT: v_dual_mov_b32 v4, 0x6050400 :: v_dual_mov_b32 v5, s1 ; GFX1250-NEXT: s_add_co_i32 s0, s0, s0 ; GFX1250-NEXT: s_add_co_i32 s2, s2, s2 -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1) +; GFX1250-NEXT: v_mov_b64_e32 v[2:3], 0 ; GFX1250-NEXT: v_perm_b32 v4, s2, s0, v4 ; GFX1250-NEXT: s_clause 0x1 ; GFX1250-NEXT: global_store_b8 v[0:1], v5, off diff --git a/llvm/test/CodeGen/AMDGPU/clmul.ll b/llvm/test/CodeGen/AMDGPU/clmul.ll index 08bd7ccda07745..0e0ced47809309 100644 --- a/llvm/test/CodeGen/AMDGPU/clmul.ll +++ b/llvm/test/CodeGen/AMDGPU/clmul.ll @@ -368,8 +368,9 @@ define amdgpu_kernel void @test_clmul_i32(ptr addrspace(1) %out, ptr addrspace(1 ; GFX1250-NEXT: v_and_b32_e32 v8, 0x88888888, v0 ; GFX1250-NEXT: v_mul_lo_u32 v9, v3, v2 ; GFX1250-NEXT: v_mul_lo_u32 v13, v3, v6 -; GFX1250-NEXT: v_mul_lo_u32 v15, v7, v2 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) ; GFX1250-NEXT: v_mul_lo_u32 v14, v5, v1 +; GFX1250-NEXT: v_mul_lo_u32 v15, v7, v2 ; GFX1250-NEXT: v_mul_lo_u32 v16, v5, v6 ; GFX1250-NEXT: v_mul_lo_u32 v17, v7, v1 ; GFX1250-NEXT: v_mul_lo_u32 v10, v5, v4 diff --git a/llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll b/llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll index 50f3858e83f9a9..cbf9c42a39591a 100644 --- a/llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll +++ b/llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll @@ -746,9 +746,9 @@ define amdgpu_kernel void @ds_wmma_block_carried(ptr addrspace(3) %base, ptr add ; GCN-NEXT: v_mov_b64_e32 v[70:71], s[14:15] ; GCN-NEXT: s_add_co_i32 s0, s0, s1 ; GCN-NEXT: v_mov_b64_e32 v[68:69], s[12:13] -; GCN-NEXT: v_mov_b64_e32 v[66:67], s[10:11] ; GCN-NEXT: s_wait_dscnt 0x0 ; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[40:47], v[56:63], v[16:23] +; GCN-NEXT: v_mov_b64_e32 v[66:67], s[10:11] ; GCN-NEXT: v_mov_b64_e32 v[64:65], s[8:9] ; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[48:55], v[24:31], v[32:39], v[48:55] ; GCN-NEXT: ds_load_tr16_b128 v[24:27], v72 @@ -1006,25 +1006,25 @@ define amdgpu_kernel void @ds_wmma_loop_carried(ptr addrspace(3) %base, ptr addr ; GCN-NEXT: s_mov_b32 s14, s8 ; GCN-NEXT: s_mov_b32 s15, s8 ; GCN-NEXT: v_mov_b64_e32 v[68:69], s[12:13] -; GCN-NEXT: v_mov_b64_e32 v[70:71], s[14:15] -; GCN-NEXT: v_mov_b64_e32 v[66:67], s[10:11] -; GCN-NEXT: v_mov_b64_e32 v[64:65], s[8:9] ; GCN-NEXT: s_wait_kmcnt 0x0 -; GCN-NEXT: v_mov_b32_e32 v80, s2 ; GCN-NEXT: s_bitcmp1_b32 s0, 0 +; GCN-NEXT: v_mov_b32_e32 v80, s2 +; GCN-NEXT: v_mov_b64_e32 v[70:71], s[14:15] +; GCN-NEXT: s_cselect_b32 s3, -1, 0 ; GCN-NEXT: ds_load_tr16_b128 v[32:35], v80 offset:512 ; GCN-NEXT: ds_load_tr16_b128 v[36:39], v80 offset:576 ; GCN-NEXT: ds_load_tr16_b128 v[48:51], v80 offset:768 ; GCN-NEXT: ds_load_tr16_b128 v[52:55], v80 offset:832 ; GCN-NEXT: ds_load_tr16_b128 v[72:75], v80 offset:640 ; GCN-NEXT: ds_load_tr16_b128 v[76:79], v80 offset:704 +; GCN-NEXT: v_mov_b64_e32 v[66:67], s[10:11] ; GCN-NEXT: ds_load_tr16_b128 v[8:11], v80 ; GCN-NEXT: ds_load_tr16_b128 v[12:15], v80 offset:64 +; GCN-NEXT: v_mov_b64_e32 v[64:65], s[8:9] ; GCN-NEXT: ds_load_tr16_b128 v[40:43], v80 offset:256 ; GCN-NEXT: ds_load_tr16_b128 v[44:47], v80 offset:320 ; GCN-NEXT: ds_load_tr16_b128 v[56:59], v80 offset:384 ; GCN-NEXT: ds_load_tr16_b128 v[60:63], v80 offset:448 -; GCN-NEXT: s_cselect_b32 s3, -1, 0 ; GCN-NEXT: s_add_co_i32 s0, s2, s1 ; GCN-NEXT: s_and_b32 vcc_lo, exec_lo, s3 ; GCN-NEXT: s_wait_dscnt 0xa @@ -1059,9 +1059,9 @@ define amdgpu_kernel void @ds_wmma_loop_carried(ptr addrspace(3) %base, ptr addr ; GCN-NEXT: v_mov_b64_e32 v[70:71], s[14:15] ; GCN-NEXT: s_add_co_i32 s0, s0, s1 ; GCN-NEXT: v_mov_b64_e32 v[68:69], s[12:13] -; GCN-NEXT: v_mov_b64_e32 v[66:67], s[10:11] ; GCN-NEXT: s_wait_dscnt 0x0 ; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[40:47], v[56:63], v[16:23] +; GCN-NEXT: v_mov_b64_e32 v[66:67], s[10:11] ; GCN-NEXT: v_mov_b64_e32 v[64:65], s[8:9] ; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[8:15], v[48:55], v[0:7] ; GCN-NEXT: ds_load_tr16_b128 v[8:11], v72 @@ -1308,12 +1308,11 @@ define amdgpu_kernel void @ds_wmma_no_block_carried(ptr addrspace(3) %base, ptr ; GCN-NEXT: s_mov_b32 s14, s8 ; GCN-NEXT: s_mov_b32 s15, s8 ; GCN-NEXT: v_mov_b64_e32 v[100:101], s[12:13] -; GCN-NEXT: v_mov_b64_e32 v[102:103], s[14:15] -; GCN-NEXT: v_mov_b64_e32 v[98:99], s[10:11] -; GCN-NEXT: v_mov_b64_e32 v[96:97], s[8:9] ; GCN-NEXT: s_wait_kmcnt 0x0 -; GCN-NEXT: v_mov_b32_e32 v92, s0 ; GCN-NEXT: s_bitcmp1_b32 s1, 0 +; GCN-NEXT: v_mov_b32_e32 v92, s0 +; GCN-NEXT: v_mov_b64_e32 v[102:103], s[14:15] +; GCN-NEXT: s_cselect_b32 s0, -1, 0 ; GCN-NEXT: ds_load_tr16_b128 v[0:3], v92 ; GCN-NEXT: ds_load_tr16_b128 v[4:7], v92 offset:64 ; GCN-NEXT: ds_load_tr16_b128 v[40:43], v92 offset:256 @@ -1330,10 +1329,11 @@ define amdgpu_kernel void @ds_wmma_no_block_carried(ptr addrspace(3) %base, ptr ; GCN-NEXT: ds_load_tr16_b128 v[84:87], v92 offset:704 ; GCN-NEXT: ds_load_tr16_b128 v[88:91], v92 offset:896 ; GCN-NEXT: ds_load_tr16_b128 v[92:95], v92 offset:960 -; GCN-NEXT: s_cselect_b32 s0, -1, 0 +; GCN-NEXT: v_mov_b64_e32 v[98:99], s[10:11] +; GCN-NEXT: s_and_b32 vcc_lo, exec_lo, s0 +; GCN-NEXT: v_mov_b64_e32 v[96:97], s[8:9] ; GCN-NEXT: s_wait_dscnt 0xe ; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[8:15], v[0:7], v[96:103], 0 -; GCN-NEXT: s_and_b32 vcc_lo, exec_lo, s0 ; GCN-NEXT: s_wait_dscnt 0xc ; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[16:23], v[40:47], v[96:103], 0 ; GCN-NEXT: s_wait_dscnt 0xa diff --git a/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll b/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll index e215c80b472ed5..4cfbee75f05487 100644 --- a/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll +++ b/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll @@ -12511,7 +12511,7 @@ define amdgpu_ps <2 x float> @flat_dec_saddr_i64_rtn(ptr inreg %sbase, i32 %voff ; GFX1250-SDAG-NEXT: v_sub_co_u32 v5, s0, v0, 1 ; GFX1250-SDAG-NEXT: v_subrev_co_ci_u32_e64 v6, s0, 0, v1, s0 ; GFX1250-SDAG-NEXT: s_or_b32 vcc_lo, s0, vcc_lo -; GFX1250-SDAG-NEXT: v_dual_cndmask_b32 v3, v6, v3 :: v_dual_cndmask_b32 v2, v5, v2 +; GFX1250-SDAG-NEXT: v_dual_cndmask_b32 v2, v5, v2 :: v_dual_cndmask_b32 v3, v6, v3 ; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[2:3], off ; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0 ; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s1 @@ -12718,7 +12718,7 @@ define amdgpu_ps <2 x float> @flat_dec_saddr_i64_rtn_neg128(ptr inreg %sbase, i3 ; GFX1250-SDAG-NEXT: v_sub_co_u32 v5, s0, v0, 1 ; GFX1250-SDAG-NEXT: v_subrev_co_ci_u32_e64 v6, s0, 0, v1, s0 ; GFX1250-SDAG-NEXT: s_or_b32 vcc_lo, s0, vcc_lo -; GFX1250-SDAG-NEXT: v_dual_cndmask_b32 v3, v6, v3 :: v_dual_cndmask_b32 v2, v5, v2 +; GFX1250-SDAG-NEXT: v_dual_cndmask_b32 v2, v5, v2 :: v_dual_cndmask_b32 v3, v6, v3 ; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[2:3], off ; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0 ; GFX1250-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s1 @@ -12927,7 +12927,7 @@ define amdgpu_ps void @flat_dec_saddr_i64_nortn(ptr inreg %sbase, i32 %voffset, ; GFX1250-SDAG-NEXT: v_sub_co_u32 v0, s0, v0, 1 ; GFX1250-SDAG-NEXT: v_subrev_co_ci_u32_e64 v1, s0, 0, v1, s0 ; GFX1250-SDAG-NEXT: s_or_b32 vcc_lo, s0, vcc_lo -; GFX1250-SDAG-NEXT: v_dual_cndmask_b32 v1, v1, v3 :: v_dual_cndmask_b32 v0, v0, v2 +; GFX1250-SDAG-NEXT: v_dual_cndmask_b32 v0, v0, v2 :: v_dual_cndmask_b32 v1, v1, v3 ; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[0:1], off ; GFX1250-SDAG-NEXT: s_endpgm ; @@ -13102,7 +13102,7 @@ define amdgpu_ps void @flat_dec_saddr_i64_nortn_neg128(ptr inreg %sbase, i32 %vo ; GFX1250-SDAG-NEXT: v_sub_co_u32 v0, s0, v0, 1 ; GFX1250-SDAG-NEXT: v_subrev_co_ci_u32_e64 v1, s0, 0, v1, s0 ; GFX1250-SDAG-NEXT: s_or_b32 vcc_lo, s0, vcc_lo -; GFX1250-SDAG-NEXT: v_dual_cndmask_b32 v1, v1, v3 :: v_dual_cndmask_b32 v0, v0, v2 +; GFX1250-SDAG-NEXT: v_dual_cndmask_b32 v0, v0, v2 :: v_dual_cndmask_b32 v1, v1, v3 ; GFX1250-SDAG-NEXT: scratch_store_b64 v4, v[0:1], off ; GFX1250-SDAG-NEXT: s_endpgm ; @@ -13276,8 +13276,8 @@ define double @flat_atomic_fadd_f64_saddr_rtn(ptr inreg %ptr, double %data) { ; GFX1250-SDAG-NEXT: scratch_load_b64 v[2:3], off, s0 ; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 ; GFX1250-SDAG-NEXT: v_add_f64_e32 v[4:5], v[2:3], v[0:1] -; GFX1250-SDAG-NEXT: v_mov_b64_e32 v[0:1], v[2:3] ; GFX1250-SDAG-NEXT: scratch_store_b64 off, v[4:5], s0 +; GFX1250-SDAG-NEXT: v_mov_b64_e32 v[0:1], v[2:3] ; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31] ; GFX1250-SDAG-NEXT: .LBB110_4: ; %atomicrmw.global ; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, 0 @@ -13317,8 +13317,8 @@ define double @flat_atomic_fadd_f64_saddr_rtn(ptr inreg %ptr, double %data) { ; GFX1250-GISEL-NEXT: scratch_load_b64 v[2:3], off, s0 ; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 ; GFX1250-GISEL-NEXT: v_add_f64_e32 v[4:5], v[2:3], v[0:1] -; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], v[2:3] ; GFX1250-GISEL-NEXT: scratch_store_b64 off, v[4:5], s0 +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[0:1], v[2:3] ; GFX1250-GISEL-NEXT: s_set_pc_i64 s[30:31] ; GFX1250-GISEL-NEXT: .LBB110_4: ; %atomicrmw.global ; GFX1250-GISEL-NEXT: v_mov_b32_e32 v2, 0 diff --git a/llvm/test/CodeGen/AMDGPU/float-to-arbitrary-fp-fp8-hw.ll b/llvm/test/CodeGen/AMDGPU/float-to-arbitrary-fp-fp8-hw.ll index 954d6a3f54cec8..97f14d353b7fa5 100644 --- a/llvm/test/CodeGen/AMDGPU/float-to-arbitrary-fp-fp8-hw.ll +++ b/llvm/test/CodeGen/AMDGPU/float-to-arbitrary-fp-fp8-hw.ll @@ -1337,16 +1337,16 @@ define i8 @to_fp8_f64(double %x) { ; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 ; GFX1250-NEXT: s_wait_kmcnt 0x0 ; GFX1250-NEXT: v_frexp_exp_i32_f64_e32 v2, v[0:1] -; GFX1250-NEXT: v_frexp_mant_f64_e32 v[4:5], v[0:1] ; GFX1250-NEXT: s_mov_b32 s0, 0xfffff -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1250-NEXT: v_frexp_mant_f64_e32 v[4:5], v[0:1] +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX1250-NEXT: v_ashrrev_i32_e32 v3, 31, v2 +; GFX1250-NEXT: v_sub_nc_u64_e32 v[6:7], 37, v[2:3] +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX1250-NEXT: v_mov_b32_e32 v10, v4 ; GFX1250-NEXT: v_and_or_b32 v11, v5, s0, 0x100000 ; GFX1250-NEXT: s_mov_b32 s0, 0 -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1250-NEXT: v_mov_b32_e32 v10, v4 -; GFX1250-NEXT: v_sub_nc_u64_e32 v[6:7], 37, v[2:3] -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1) ; GFX1250-NEXT: v_cmp_gt_u64_e32 vcc_lo, 63, v[6:7] ; GFX1250-NEXT: v_dual_cndmask_b32 v7, 0, v7 :: v_dual_cndmask_b32 v6, 63, v6 ; GFX1250-NEXT: v_cmp_lt_u64_e32 vcc_lo, 1, v[6:7] @@ -1374,11 +1374,10 @@ define i8 @to_fp8_f64(double %x) { ; GFX1250-NEXT: v_and_b32_e32 v9, 0x1ffff, v5 ; GFX1250-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[8:9] ; GFX1250-NEXT: v_mov_b32_e32 v9, 0 -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3) -; GFX1250-NEXT: v_mov_b32_e32 v5, v9 ; GFX1250-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc_lo ; GFX1250-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7] -; GFX1250-NEXT: v_cndmask_b32_e32 v8, 0, v10, vcc_lo +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX1250-NEXT: v_dual_mov_b32 v5, v9 :: v_dual_cndmask_b32 v8, 0, v10 ; GFX1250-NEXT: v_and_or_b32 v10, v4, 1, v11 ; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX1250-NEXT: v_add_nc_u64_e32 v[6:7], v[12:13], v[8:9] @@ -1401,26 +1400,26 @@ define i8 @to_fp8_f64(double %x) { ; GFX1250-NEXT: v_lshlrev_b64_e32 v[8:9], 2, v[2:3] ; GFX1250-NEXT: v_cndmask_b32_e64 v9, 0, 4, s0 ; GFX1250-NEXT: v_cmp_eq_u64_e64 s0, 30, v[2:3] -; GFX1250-NEXT: v_cmp_lt_i64_e64 s1, 30, v[2:3] -; GFX1250-NEXT: v_cmp_gt_i64_e64 s2, 1, v[2:3] ; GFX1250-NEXT: s_and_b32 s0, s0, vcc_lo -; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX1250-NEXT: v_cmp_lt_i64_e64 s1, 30, v[2:3] ; GFX1250-NEXT: s_or_b32 vcc_lo, s1, s0 ; GFX1250-NEXT: v_and_b32_e32 v7, 0x80, v10 +; GFX1250-NEXT: v_cmp_gt_i64_e64 s2, 1, v[2:3] +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3) ; GFX1250-NEXT: v_or_b32_e32 v8, v8, v7 ; GFX1250-NEXT: v_or3_b32 v6, v7, v9, v6 ; GFX1250-NEXT: v_or_b32_e32 v3, 0x7c, v7 -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX1250-NEXT: v_or_b32_e32 v8, v8, v4 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX1250-NEXT: v_cndmask_b32_e64 v2, v8, v6, s2 -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) ; GFX1250-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc_lo ; GFX1250-NEXT: v_cmp_eq_f64_e32 vcc_lo, 0, v[0:1] +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) ; GFX1250-NEXT: v_cndmask_b32_e32 v2, v2, v7, vcc_lo ; GFX1250-NEXT: v_cmp_class_f64_e64 vcc_lo, v[0:1], 0x204 -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) ; GFX1250-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc_lo ; GFX1250-NEXT: v_cmp_o_f64_e32 vcc_lo, v[0:1], v[0:1] +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) ; GFX1250-NEXT: v_cndmask_b32_e32 v0, 0x7e, v2, vcc_lo ; GFX1250-NEXT: s_set_pc_i64 s[30:31] ; diff --git a/llvm/test/CodeGen/AMDGPU/float-to-arbitrary-fp-widen.ll b/llvm/test/CodeGen/AMDGPU/float-to-arbitrary-fp-widen.ll index 93bb7db3755126..0f428ad8aefbb6 100644 --- a/llvm/test/CodeGen/AMDGPU/float-to-arbitrary-fp-widen.ll +++ b/llvm/test/CodeGen/AMDGPU/float-to-arbitrary-fp-widen.ll @@ -2823,7 +2823,7 @@ define <7 x i8> @to_fp8_v7f16(<7 x half> %x) { ; GFX1250-FAKE16-NEXT: v_lshrrev_b64 v[8:9], 24, v[8:9] ; GFX1250-FAKE16-NEXT: v_lshl_or_b32 v1, v2, 16, v3 ; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX1250-FAKE16-NEXT: v_dual_lshrrev_b32 v1, 8, v1 :: v_dual_mov_b32 v3, v8 +; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v3, v8 :: v_dual_lshrrev_b32 v1, 8, v1 ; GFX1250-FAKE16-NEXT: s_set_pc_i64 s[30:31] ; ; GFX1310-LABEL: to_fp8_v7f16: diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll index 4a16912a14589f..3b8005a51a3a35 100644 --- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll +++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll @@ -15728,8 +15728,8 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_ ; GFX1250-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 ; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0 ; GFX1250-TRUE16-NEXT: v_add_nc_u64_e32 v[4:5], 0x7fe, v[0:1] -; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l ; GFX1250-TRUE16-NEXT: s_mov_b32 s0, 0 +; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l ; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1) ; GFX1250-TRUE16-NEXT: v_dual_mov_b32 v1, v5 :: v_dual_bitop2_b32 v0, -4, v4 bitop3:0x40 ; GFX1250-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4 @@ -18087,8 +18087,8 @@ define void @global_system_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine ; GFX1250-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0 ; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0 ; GFX1250-TRUE16-NEXT: v_add_nc_u64_e32 v[4:5], 0x7fe, v[0:1] -; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l ; GFX1250-TRUE16-NEXT: s_mov_b32 s0, 0 +; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l ; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1) ; GFX1250-TRUE16-NEXT: v_dual_mov_b32 v1, v5 :: v_dual_bitop2_b32 v0, -4, v4 bitop3:0x40 ; GFX1250-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4 diff --git a/llvm/test/CodeGen/AMDGPU/global-load-xcnt.ll b/llvm/test/CodeGen/AMDGPU/global-load-xcnt.ll index 20e8e1e7371bda..a61d1a3ff167ab 100644 --- a/llvm/test/CodeGen/AMDGPU/global-load-xcnt.ll +++ b/llvm/test/CodeGen/AMDGPU/global-load-xcnt.ll @@ -84,9 +84,9 @@ define i16 @test_v7i16_load_store(ptr addrspace(1) %ptr1, ptr addrspace(1) %ptr2 ; GCN-SDAG-FAKE16-NEXT: v_pk_add_u16 v14, v7, v11 ; GCN-SDAG-FAKE16-NEXT: v_mov_b64_e32 v[6:7], 12 ; GCN-SDAG-FAKE16-NEXT: v_mov_b64_e32 v[10:11], 8 -; GCN-SDAG-FAKE16-NEXT: v_pk_add_u16 v4, v4, v8 ; GCN-SDAG-FAKE16-NEXT: s_wait_xcnt 0x1 ; GCN-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v9 +; GCN-SDAG-FAKE16-NEXT: v_pk_add_u16 v4, v4, v8 ; GCN-SDAG-FAKE16-NEXT: s_clause 0x2 ; GCN-SDAG-FAKE16-NEXT: global_store_b16 v[6:7], v14, off ; GCN-SDAG-FAKE16-NEXT: global_store_b32 v[10:11], v9, off @@ -324,30 +324,30 @@ define i64 @test_v16i64_load_store(ptr addrspace(1) %ptr_a, ptr addrspace(1) %pt ; GCN-SDAG-NEXT: v_mov_b64_e32 v[0:1], 0x70 ; GCN-SDAG-NEXT: v_mov_b64_e32 v[54:55], 48 ; GCN-SDAG-NEXT: v_mov_b64_e32 v[2:3], 0x60 -; GCN-SDAG-NEXT: v_mov_b64_e32 v[64:65], 32 -; GCN-SDAG-NEXT: v_mov_b64_e32 v[66:67], 16 -; GCN-SDAG-NEXT: v_mov_b64_e32 v[68:69], 0 -; GCN-SDAG-NEXT: v_mov_b64_e32 v[38:39], 0x50 -; GCN-SDAG-NEXT: v_dual_mov_b32 v49, 0 :: v_dual_mov_b32 v48, 0xc8 -; GCN-SDAG-NEXT: v_mov_b64_e32 v[52:53], 64 ; GCN-SDAG-NEXT: s_wait_loadcnt 0x7 ; GCN-SDAG-NEXT: global_store_b128 v[0:1], v[6:9], off ; GCN-SDAG-NEXT: s_wait_loadcnt 0x6 ; GCN-SDAG-NEXT: global_store_b128 v[2:3], v[10:13], off -; GCN-SDAG-NEXT: s_wait_loadcnt 0x5 -; GCN-SDAG-NEXT: v_dual_mov_b32 v50, v16 :: v_dual_mov_b32 v51, v17 -; GCN-SDAG-NEXT: s_wait_xcnt 0x0 -; GCN-SDAG-NEXT: v_add_nc_u64_e32 v[12:13], v[12:13], v[12:13] -; GCN-SDAG-NEXT: s_wait_loadcnt 0x2 -; GCN-SDAG-NEXT: v_add_nc_u64_e32 v[2:3], v[28:29], v[28:29] -; GCN-SDAG-NEXT: s_clause 0x2 +; GCN-SDAG-NEXT: v_mov_b64_e32 v[64:65], 32 +; GCN-SDAG-NEXT: v_mov_b64_e32 v[66:67], 16 +; GCN-SDAG-NEXT: v_mov_b64_e32 v[68:69], 0 +; GCN-SDAG-NEXT: s_wait_loadcnt 0x4 ; GCN-SDAG-NEXT: global_store_b128 v[54:55], v[18:21], off +; GCN-SDAG-NEXT: s_wait_loadcnt 0x3 ; GCN-SDAG-NEXT: global_store_b128 v[64:65], v[22:25], off +; GCN-SDAG-NEXT: s_wait_loadcnt 0x2 ; GCN-SDAG-NEXT: global_store_b128 v[66:67], v[26:29], off ; GCN-SDAG-NEXT: s_wait_loadcnt 0x1 ; GCN-SDAG-NEXT: global_store_b128 v[68:69], v[30:33], off +; GCN-SDAG-NEXT: v_mov_b64_e32 v[38:39], 0x50 +; GCN-SDAG-NEXT: s_wait_xcnt 0x4 +; GCN-SDAG-NEXT: v_add_nc_u64_e32 v[2:3], v[28:29], v[28:29] ; GCN-SDAG-NEXT: s_wait_xcnt 0x1 ; GCN-SDAG-NEXT: v_add_nc_u64_e32 v[28:29], v[32:33], v[32:33] +; GCN-SDAG-NEXT: v_dual_mov_b32 v49, 0 :: v_dual_mov_b32 v48, 0xc8 +; GCN-SDAG-NEXT: v_mov_b64_e32 v[52:53], 64 +; GCN-SDAG-NEXT: v_dual_mov_b32 v50, v16 :: v_dual_mov_b32 v51, v17 +; GCN-SDAG-NEXT: v_add_nc_u64_e32 v[12:13], v[12:13], v[12:13] ; GCN-SDAG-NEXT: v_add_nc_u64_e32 v[10:11], v[10:11], v[10:11] ; GCN-SDAG-NEXT: v_add_nc_u64_e32 v[8:9], v[8:9], v[8:9] ; GCN-SDAG-NEXT: v_add_nc_u64_e32 v[6:7], v[6:7], v[6:7] @@ -399,9 +399,6 @@ define i64 @test_v16i64_load_store(ptr addrspace(1) %ptr_a, ptr addrspace(1) %pt ; GCN-GISEL-NEXT: v_mov_b64_e32 v[52:53], 48 ; GCN-GISEL-NEXT: v_mov_b64_e32 v[66:67], 0x60 ; GCN-GISEL-NEXT: v_mov_b64_e32 v[68:69], 0x70 -; GCN-GISEL-NEXT: v_mov_b64_e32 v[54:55], 64 -; GCN-GISEL-NEXT: v_mov_b64_e32 v[48:49], 0xc8 -; GCN-GISEL-NEXT: v_mov_b64_e32 v[64:65], 0x50 ; GCN-GISEL-NEXT: s_wait_loadcnt 0x6 ; GCN-GISEL-NEXT: global_store_b128 v[0:1], v[10:13], off ; GCN-GISEL-NEXT: s_wait_loadcnt 0x5 @@ -414,8 +411,11 @@ define i64 @test_v16i64_load_store(ptr addrspace(1) %ptr_a, ptr addrspace(1) %pt ; GCN-GISEL-NEXT: global_store_b128 v[66:67], v[26:29], off ; GCN-GISEL-NEXT: s_wait_loadcnt 0x1 ; GCN-GISEL-NEXT: global_store_b128 v[68:69], v[30:33], off +; GCN-GISEL-NEXT: v_mov_b64_e32 v[54:55], 64 ; GCN-GISEL-NEXT: s_wait_xcnt 0x4 ; GCN-GISEL-NEXT: v_add_nc_u64_e32 v[2:3], v[12:13], v[12:13] +; GCN-GISEL-NEXT: v_mov_b64_e32 v[48:49], 0xc8 +; GCN-GISEL-NEXT: v_mov_b64_e32 v[64:65], 0x50 ; GCN-GISEL-NEXT: v_mov_b64_e32 v[50:51], v[8:9] ; GCN-GISEL-NEXT: v_add_nc_u64_e32 v[0:1], v[10:11], v[10:11] ; GCN-GISEL-NEXT: v_add_nc_u64_e32 v[10:11], v[14:15], v[14:15] @@ -476,12 +476,12 @@ define amdgpu_kernel void @test_v7i16_load_store_kernel(ptr addrspace(1) %ptr1, ; GCN-SDAG-NEXT: s_wait_xcnt 0x0 ; GCN-SDAG-NEXT: s_load_b64 s[4:5], s[4:5], 0x10 nv ; GCN-SDAG-NEXT: v_mov_b64_e32 v[8:9], 12 -; GCN-SDAG-NEXT: v_mov_b64_e32 v[10:11], 8 -; GCN-SDAG-NEXT: v_mov_b64_e32 v[12:13], 0 ; GCN-SDAG-NEXT: s_wait_kmcnt 0x0 ; GCN-SDAG-NEXT: s_clause 0x1 ; GCN-SDAG-NEXT: global_load_b128 v[0:3], v14, s[0:1] scale_offset ; GCN-SDAG-NEXT: global_load_b128 v[4:7], v14, s[2:3] scale_offset +; GCN-SDAG-NEXT: v_mov_b64_e32 v[10:11], 8 +; GCN-SDAG-NEXT: v_mov_b64_e32 v[12:13], 0 ; GCN-SDAG-NEXT: s_wait_loadcnt 0x0 ; GCN-SDAG-NEXT: v_pk_add_u16 v3, v3, v7 ; GCN-SDAG-NEXT: v_pk_add_u16 v2, v2, v6 @@ -506,16 +506,16 @@ define amdgpu_kernel void @test_v7i16_load_store_kernel(ptr addrspace(1) %ptr1, ; GCN-GISEL-NEXT: s_wait_xcnt 0x0 ; GCN-GISEL-NEXT: s_load_b64 s[4:5], s[4:5], 0x10 nv ; GCN-GISEL-NEXT: v_mov_b64_e32 v[8:9], 0 +; GCN-GISEL-NEXT: s_wait_kmcnt 0x0 +; GCN-GISEL-NEXT: s_clause 0x1 +; GCN-GISEL-NEXT: global_load_b128 v[0:3], v22, s[0:1] scale_offset +; GCN-GISEL-NEXT: global_load_b128 v[4:7], v22, s[2:3] scale_offset ; GCN-GISEL-NEXT: v_mov_b64_e32 v[10:11], 2 ; GCN-GISEL-NEXT: v_mov_b64_e32 v[12:13], 4 ; GCN-GISEL-NEXT: v_mov_b64_e32 v[14:15], 6 ; GCN-GISEL-NEXT: v_mov_b64_e32 v[16:17], 8 ; GCN-GISEL-NEXT: v_mov_b64_e32 v[18:19], 10 ; GCN-GISEL-NEXT: v_mov_b64_e32 v[20:21], 12 -; GCN-GISEL-NEXT: s_wait_kmcnt 0x0 -; GCN-GISEL-NEXT: s_clause 0x1 -; GCN-GISEL-NEXT: global_load_b128 v[0:3], v22, s[0:1] scale_offset -; GCN-GISEL-NEXT: global_load_b128 v[4:7], v22, s[2:3] scale_offset ; GCN-GISEL-NEXT: s_wait_loadcnt 0x0 ; GCN-GISEL-NEXT: v_pk_add_u16 v0, v0, v4 ; GCN-GISEL-NEXT: v_pk_add_u16 v1, v1, v5 diff --git a/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll b/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll index 1a4f06cfcf694f..2a71b2118b588e 100644 --- a/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll +++ b/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll @@ -4174,16 +4174,16 @@ define <3 x i32> @clpeak_imad_pat_v3i32(<3 x i32> %x, <3 x i32> %y) { ; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) ; GFX1250-SDAG-NEXT: v_add_nc_u32_e32 v2, v8, v2 ; GFX1250-SDAG-NEXT: v_mul_lo_u32 v1, v1, v4 -; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1250-SDAG-NEXT: v_mad_u32 v3, v0, v6, v0 +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX1250-SDAG-NEXT: v_mul_lo_u32 v2, v2, v5 +; GFX1250-SDAG-NEXT: v_mad_u32 v3, v0, v6, v0 ; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) ; GFX1250-SDAG-NEXT: v_mad_u32 v4, v1, v7, v1 -; GFX1250-SDAG-NEXT: v_mad_u32 v0, v3, v0, v3 -; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) ; GFX1250-SDAG-NEXT: v_mad_u32 v5, v2, v8, v2 +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX1250-SDAG-NEXT: v_mad_u32 v0, v3, v0, v3 ; GFX1250-SDAG-NEXT: v_mad_u32 v1, v4, v1, v4 -; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) ; GFX1250-SDAG-NEXT: v_mad_u32 v2, v5, v2, v5 ; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31] ; @@ -4202,23 +4202,22 @@ define <3 x i32> @clpeak_imad_pat_v3i32(<3 x i32> %x, <3 x i32> %y) { ; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) ; GFX1250-GISEL-FAKE16-NEXT: v_add_nc_u32_e32 v1, v7, v1 ; GFX1250-GISEL-FAKE16-NEXT: v_mul_lo_u32 v0, v0, v3 -; GFX1250-GISEL-FAKE16-NEXT: v_add_nc_u32_e32 v3, 1, v6 -; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) ; GFX1250-GISEL-FAKE16-NEXT: v_add_nc_u32_e32 v2, v8, v2 ; GFX1250-GISEL-FAKE16-NEXT: v_mul_lo_u32 v1, v1, v4 -; GFX1250-GISEL-FAKE16-NEXT: v_add_nc_u32_e32 v4, 1, v7 -; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX1250-GISEL-FAKE16-NEXT: v_mul_lo_u32 v3, v0, v3 +; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2) ; GFX1250-GISEL-FAKE16-NEXT: v_mul_lo_u32 v2, v2, v5 +; GFX1250-GISEL-FAKE16-NEXT: v_dual_add_nc_u32 v3, 1, v6 :: v_dual_add_nc_u32 v4, 1, v7 ; GFX1250-GISEL-FAKE16-NEXT: v_add_nc_u32_e32 v5, 1, v8 -; GFX1250-GISEL-FAKE16-NEXT: v_add_nc_u32_e32 v0, 1, v0 +; GFX1250-GISEL-FAKE16-NEXT: v_mul_lo_u32 v3, v0, v3 +; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) ; GFX1250-GISEL-FAKE16-NEXT: v_mul_lo_u32 v4, v1, v4 ; GFX1250-GISEL-FAKE16-NEXT: v_add_nc_u32_e32 v1, 1, v1 -; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4) ; GFX1250-GISEL-FAKE16-NEXT: v_mul_lo_u32 v5, v2, v5 -; GFX1250-GISEL-FAKE16-NEXT: v_add_nc_u32_e32 v2, 1, v2 -; GFX1250-GISEL-FAKE16-NEXT: v_mul_lo_u32 v0, v3, v0 +; GFX1250-GISEL-FAKE16-NEXT: v_dual_add_nc_u32 v0, 1, v0 :: v_dual_add_nc_u32 v2, 1, v2 +; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX1250-GISEL-FAKE16-NEXT: v_mul_lo_u32 v1, v4, v1 +; GFX1250-GISEL-FAKE16-NEXT: v_mul_lo_u32 v0, v3, v0 ; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) ; GFX1250-GISEL-FAKE16-NEXT: v_mul_lo_u32 v2, v5, v2 ; GFX1250-GISEL-FAKE16-NEXT: s_set_pc_i64 s[30:31] @@ -4238,23 +4237,22 @@ define <3 x i32> @clpeak_imad_pat_v3i32(<3 x i32> %x, <3 x i32> %y) { ; GFX1250-GISEL-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) ; GFX1250-GISEL-REAL16-NEXT: v_add_nc_u32_e32 v1, v7, v1 ; GFX1250-GISEL-REAL16-NEXT: v_mul_lo_u32 v0, v0, v3 -; GFX1250-GISEL-REAL16-NEXT: v_add_nc_u32_e32 v3, 1, v6 -; GFX1250-GISEL-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX1250-GISEL-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) ; GFX1250-GISEL-REAL16-NEXT: v_add_nc_u32_e32 v2, v8, v2 ; GFX1250-GISEL-REAL16-NEXT: v_mul_lo_u32 v1, v1, v4 -; GFX1250-GISEL-REAL16-NEXT: v_add_nc_u32_e32 v4, 1, v7 -; GFX1250-GISEL-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX1250-GISEL-REAL16-NEXT: v_mul_lo_u32 v3, v0, v3 +; GFX1250-GISEL-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2) ; GFX1250-GISEL-REAL16-NEXT: v_mul_lo_u32 v2, v2, v5 +; GFX1250-GISEL-REAL16-NEXT: v_dual_add_nc_u32 v3, 1, v6 :: v_dual_add_nc_u32 v4, 1, v7 ; GFX1250-GISEL-REAL16-NEXT: v_add_nc_u32_e32 v5, 1, v8 -; GFX1250-GISEL-REAL16-NEXT: v_add_nc_u32_e32 v0, 1, v0 +; GFX1250-GISEL-REAL16-NEXT: v_mul_lo_u32 v3, v0, v3 +; GFX1250-GISEL-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) ; GFX1250-GISEL-REAL16-NEXT: v_mul_lo_u32 v4, v1, v4 ; GFX1250-GISEL-REAL16-NEXT: v_add_nc_u32_e32 v1, 1, v1 -; GFX1250-GISEL-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4) ; GFX1250-GISEL-REAL16-NEXT: v_mul_lo_u32 v5, v2, v5 -; GFX1250-GISEL-REAL16-NEXT: v_add_nc_u32_e32 v2, 1, v2 -; GFX1250-GISEL-REAL16-NEXT: v_mul_lo_u32 v0, v3, v0 +; GFX1250-GISEL-REAL16-NEXT: v_dual_add_nc_u32 v0, 1, v0 :: v_dual_add_nc_u32 v2, 1, v2 +; GFX1250-GISEL-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX1250-GISEL-REAL16-NEXT: v_mul_lo_u32 v1, v4, v1 +; GFX1250-GISEL-REAL16-NEXT: v_mul_lo_u32 v0, v3, v0 ; GFX1250-GISEL-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_3) ; GFX1250-GISEL-REAL16-NEXT: v_mul_lo_u32 v2, v5, v2 ; GFX1250-GISEL-REAL16-NEXT: s_set_pc_i64 s[30:31] @@ -11984,9 +11982,8 @@ define i32 @other_use_mul_mad_i32_var(i32 %x, i32 %y, i32 %z, ptr addrspace(3) % ; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0 ; GFX1250-NEXT: s_wait_kmcnt 0x0 ; GFX1250-NEXT: v_mul_lo_u32 v1, v0, v1 -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX1250-NEXT: v_add_nc_u32_e32 v0, v1, v2 ; GFX1250-NEXT: ds_store_b32 v3, v1 +; GFX1250-NEXT: v_add_nc_u32_e32 v0, v1, v2 ; GFX1250-NEXT: s_wait_dscnt 0x0 ; GFX1250-NEXT: s_set_pc_i64 s[30:31] entry: diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scale.pk.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scale.pk.ll index 91b38c7f421354..8dd9c3f13e60d1 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scale.pk.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.scale.pk.ll @@ -309,9 +309,9 @@ define void @test_cvt_scale_pk8_f32_fp8_ss(<2 x i32> inreg %src, i32 inreg %scal ; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s7, v9 ; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3] ; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1] -; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[4:5] -; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) ; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[6:7] +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[4:5] ; GFX1250-GISEL-NEXT: s_clause 0x1 ; GFX1250-GISEL-NEXT: global_store_b128 v[0:1], v[2:5], off ; GFX1250-GISEL-NEXT: global_store_b128 v[0:1], v[6:9], off offset:16 @@ -380,9 +380,9 @@ define void @test_cvt_scale_pk8_f32_bf8_ss(<2 x i32> inreg %src, i32 inreg %scal ; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s7, v9 ; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3] ; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1] -; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[4:5] -; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) ; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[6:7] +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[4:5] ; GFX1250-GISEL-NEXT: s_clause 0x1 ; GFX1250-GISEL-NEXT: global_store_b128 v[0:1], v[2:5], off ; GFX1250-GISEL-NEXT: global_store_b128 v[0:1], v[6:9], off offset:16 @@ -449,9 +449,9 @@ define void @test_cvt_scale_pk8_f32_fp4_ss(i32 inreg %src, i32 inreg %scale, ptr ; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s7, v9 ; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3] ; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1] -; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[4:5] -; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) ; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[6:7] +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[4:5] ; GFX1250-GISEL-NEXT: s_clause 0x1 ; GFX1250-GISEL-NEXT: global_store_b128 v[0:1], v[2:5], off ; GFX1250-GISEL-NEXT: global_store_b128 v[0:1], v[6:9], off offset:16 @@ -520,9 +520,9 @@ define void @test_cvt_scale_pk16_f16_fp6_ss(<3 x i32> inreg %src, i32 inreg %sca ; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s7, v9 ; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3] ; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1] -; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[4:5] -; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) ; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[6:7] +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[4:5] ; GFX1250-GISEL-NEXT: s_clause 0x1 ; GFX1250-GISEL-NEXT: global_store_b128 v[0:1], v[2:5], off ; GFX1250-GISEL-NEXT: global_store_b128 v[0:1], v[6:9], off offset:16 @@ -566,9 +566,9 @@ define void @test_cvt_scale_pk16_f16_fp6_sl(<3 x i32> inreg %src, ptr addrspace( ; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s7, v9 ; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3] ; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1] -; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[4:5] -; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) ; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[6:7] +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[4:5] ; GFX1250-GISEL-NEXT: s_clause 0x1 ; GFX1250-GISEL-NEXT: global_store_b128 v[0:1], v[2:5], off ; GFX1250-GISEL-NEXT: global_store_b128 v[0:1], v[6:9], off offset:16 @@ -637,9 +637,9 @@ define void @test_cvt_scale_pk16_bf16_fp6_sl(<3 x i32> inreg %src, ptr addrspace ; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s7, v9 ; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3] ; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1] -; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[4:5] -; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) ; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[6:7] +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[4:5] ; GFX1250-GISEL-NEXT: s_clause 0x1 ; GFX1250-GISEL-NEXT: global_store_b128 v[0:1], v[2:5], off ; GFX1250-GISEL-NEXT: global_store_b128 v[0:1], v[6:9], off offset:16 @@ -708,9 +708,9 @@ define void @test_cvt_scale_pk16_f16_bf6_ss(<3 x i32> inreg %src, i32 inreg %sca ; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s7, v9 ; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3] ; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1] -; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[4:5] -; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) ; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[6:7] +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[4:5] ; GFX1250-GISEL-NEXT: s_clause 0x1 ; GFX1250-GISEL-NEXT: global_store_b128 v[0:1], v[2:5], off ; GFX1250-GISEL-NEXT: global_store_b128 v[0:1], v[6:9], off offset:16 @@ -754,9 +754,9 @@ define void @test_cvt_scale_pk16_f16_bf6_sl(<3 x i32> inreg %src, ptr addrspace( ; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s7, v9 ; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3] ; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1] -; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[4:5] -; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) ; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[6:7] +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[4:5] ; GFX1250-GISEL-NEXT: s_clause 0x1 ; GFX1250-GISEL-NEXT: global_store_b128 v[0:1], v[2:5], off ; GFX1250-GISEL-NEXT: global_store_b128 v[0:1], v[6:9], off offset:16 @@ -825,9 +825,9 @@ define void @test_cvt_scale_pk16_bf16_bf6_sl(<3 x i32> inreg %src, ptr addrspace ; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s7, v9 ; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3] ; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1] -; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[4:5] -; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) ; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[6:7] +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[4:5] ; GFX1250-GISEL-NEXT: s_clause 0x1 ; GFX1250-GISEL-NEXT: global_store_b128 v[0:1], v[2:5], off ; GFX1250-GISEL-NEXT: global_store_b128 v[0:1], v[6:9], off offset:16 diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.update.dpp.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.update.dpp.ll index 22abc40fb4404e..a3c8950224b2b7 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.update.dpp.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.update.dpp.ll @@ -342,9 +342,9 @@ define weak_odr amdgpu_kernel void @dpp_test1(ptr %arg) local_unnamed_addr { ; GFX1250-NEXT: s_wait_dscnt 0x0 ; GFX1250-NEXT: v_add_nc_u32_e32 v1, v1, v1 ; GFX1250-NEXT: v_mov_b32_dpp v2, v1 quad_perm:[1,0,3,2] row_mask:0xf bank_mask:0xf +; GFX1250-NEXT: s_barrier_wait -1 ; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX1250-NEXT: v_add_nc_u32_e32 v1, v2, v1 -; GFX1250-NEXT: s_barrier_wait -1 ; GFX1250-NEXT: s_wait_kmcnt 0x0 ; GFX1250-NEXT: flat_store_b32 v0, v1, s[0:1] ; GFX1250-NEXT: s_endpgm diff --git a/llvm/test/CodeGen/AMDGPU/lshl-add-u64.ll b/llvm/test/CodeGen/AMDGPU/lshl-add-u64.ll index 3f3cde65dfdf7a..1a3d978e3c7e9b 100644 --- a/llvm/test/CodeGen/AMDGPU/lshl-add-u64.ll +++ b/llvm/test/CodeGen/AMDGPU/lshl-add-u64.ll @@ -754,11 +754,10 @@ define <4 x i64> @lshl_add_u64_vvv_and_2_v4(<4 x i64> %v, <4 x i64> %a, <4 x i64 ; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX1250-NEXT: v_lshlrev_b64_e32 v[6:7], v17, v[6:7] ; GFX1250-NEXT: v_lshl_add_u64 v[0:1], v[0:1], v16, v[8:9] -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX1250-NEXT: v_lshl_add_u64 v[4:5], v[4:5], v19, v[12:13] +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) ; GFX1250-NEXT: v_lshl_add_u64 v[2:3], v[2:3], v18, v[10:11] -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) ; GFX1250-NEXT: v_add_nc_u64_e32 v[6:7], v[6:7], v[14:15] +; GFX1250-NEXT: v_lshl_add_u64 v[4:5], v[4:5], v19, v[12:13] ; GFX1250-NEXT: s_set_pc_i64 s[30:31] ; ; GFX942-GISEL-LABEL: lshl_add_u64_vvv_and_2_v4: @@ -797,11 +796,10 @@ define <4 x i64> @lshl_add_u64_vvv_and_2_v4(<4 x i64> %v, <4 x i64> %a, <4 x i64 ; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX1250-GISEL-NEXT: v_lshlrev_b64_e32 v[6:7], v17, v[6:7] ; GFX1250-GISEL-NEXT: v_lshl_add_u64 v[0:1], v[0:1], v16, v[8:9] -; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) ; GFX1250-GISEL-NEXT: v_lshl_add_u64 v[2:3], v[2:3], v18, v[10:11] -; GFX1250-GISEL-NEXT: v_lshl_add_u64 v[4:5], v[4:5], v19, v[12:13] -; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) ; GFX1250-GISEL-NEXT: v_add_nc_u64_e32 v[6:7], v[6:7], v[14:15] +; GFX1250-GISEL-NEXT: v_lshl_add_u64 v[4:5], v[4:5], v19, v[12:13] ; GFX1250-GISEL-NEXT: s_set_pc_i64 s[30:31] %and = and <4 x i64> %s, <i64 1, i64 2, i64 3, i64 5> %shl = shl <4 x i64> %v, %and diff --git a/llvm/test/CodeGen/AMDGPU/mad_64_32.ll b/llvm/test/CodeGen/AMDGPU/mad_64_32.ll index 68767c5183c3ab..416d0e5130b10f 100644 --- a/llvm/test/CodeGen/AMDGPU/mad_64_32.ll +++ b/llvm/test/CodeGen/AMDGPU/mad_64_32.ll @@ -428,10 +428,10 @@ define i128 @mad_i64_i32_sextops_i32_i128(i32 %arg0, i32 %arg1, i128 %arg2) #0 { ; GFX1250-NEXT: s_wait_kmcnt 0x0 ; GFX1250-NEXT: v_mad_nc_u64_u32 v[6:7], v0, v1, 0 ; GFX1250-NEXT: v_dual_mov_b32 v9, 0 :: v_dual_ashrrev_i32 v16, 31, v0 -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1250-NEXT: v_mad_nc_i64_i32 v[14:15], v1, v16, 0 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX1250-NEXT: v_dual_mov_b32 v8, v7 :: v_dual_ashrrev_i32 v7, 31, v1 -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-NEXT: v_mad_nc_i64_i32 v[14:15], v1, v16, 0 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX1250-NEXT: v_mad_nc_u64_u32 v[10:11], v16, v1, v[8:9] ; GFX1250-NEXT: v_dual_mov_b32 v8, v11 :: v_dual_mov_b32 v11, v9 ; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3) @@ -2318,9 +2318,9 @@ define <2 x i64> @lshr_mad_i64_vec(<2 x i64> %arg0) #0 { ; GFX1250-NEXT: s_wait_kmcnt 0x0 ; GFX1250-NEXT: v_mad_nc_u64_u32 v[4:5], v1, 0xffff1c18, v[0:1] ; GFX1250-NEXT: v_mad_nc_u64_u32 v[6:7], v3, 0xffff1118, v[2:3] -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1250-NEXT: v_dual_sub_nc_u32 v1, v5, v1 :: v_dual_sub_nc_u32 v3, v7, v3 -; GFX1250-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v2, v6 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1250-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_sub_nc_u32 v1, v5, v1 +; GFX1250-NEXT: v_dual_mov_b32 v2, v6 :: v_dual_sub_nc_u32 v3, v7, v3 ; GFX1250-NEXT: s_set_pc_i64 s[30:31] %lsh = lshr <2 x i64> %arg0, <i64 32, i64 32> %mul = mul <2 x i64> %lsh, <i64 s0xffffffffffff1c18, i64 s0xffffffffffff1118> diff --git a/llvm/test/CodeGen/AMDGPU/mul.ll b/llvm/test/CodeGen/AMDGPU/mul.ll index a0b580a3a774cc..14e1aa694352ee 100644 --- a/llvm/test/CodeGen/AMDGPU/mul.ll +++ b/llvm/test/CodeGen/AMDGPU/mul.ll @@ -3937,11 +3937,11 @@ define amdgpu_kernel void @v_mul_i128(ptr addrspace(1) %out, ptr addrspace(1) %a ; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX1250-NEXT: v_mad_nc_u64_u32 v[2:3], v0, v5, v[14:15] ; GFX1250-NEXT: v_mad_u32 v0, v7, v0, v11 -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX1250-NEXT: v_dual_mov_b32 v14, v3 :: v_dual_mov_b32 v9, v2 -; GFX1250-NEXT: v_add_nc_u64_e32 v[12:13], v[12:13], v[14:15] -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX1250-NEXT: v_mad_u32 v11, v6, v1, v0 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1250-NEXT: v_add_nc_u64_e32 v[12:13], v[12:13], v[14:15] ; GFX1250-NEXT: v_mad_nc_u64_u32 v[0:1], v1, v5, v[12:13] ; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX1250-NEXT: v_add_nc_u64_e32 v[10:11], v[0:1], v[10:11] diff --git a/llvm/test/CodeGen/AMDGPU/packed-fp32.ll b/llvm/test/CodeGen/AMDGPU/packed-fp32.ll index 79d6e37c7f21e6..8fecac089f08c2 100644 --- a/llvm/test/CodeGen/AMDGPU/packed-fp32.ll +++ b/llvm/test/CodeGen/AMDGPU/packed-fp32.ll @@ -3425,10 +3425,11 @@ define amdgpu_kernel void @fma_v2_v_unfoldable_lit(ptr addrspace(1) %a) { ; GFX1250-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv ; GFX1250-SDAG-NEXT: v_and_b32_e32 v6, 0x3ff, v0 ; GFX1250-SDAG-NEXT: v_mov_b64_e32 v[2:3], 0x4040000040800000 -; GFX1250-SDAG-NEXT: v_mov_b64_e32 v[4:5], 0x400000003f800000 ; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0 ; GFX1250-SDAG-NEXT: global_load_b64 v[0:1], v6, s[0:1] scale_offset +; GFX1250-SDAG-NEXT: v_mov_b64_e32 v[4:5], 0x400000003f800000 ; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0 +; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX1250-SDAG-NEXT: v_pk_fma_f32 v[0:1], v[0:1], v[2:3], v[4:5] ; GFX1250-SDAG-NEXT: global_store_b64 v6, v[0:1], s[0:1] scale_offset ; GFX1250-SDAG-NEXT: s_endpgm @@ -3445,10 +3446,11 @@ define amdgpu_kernel void @fma_v2_v_unfoldable_lit(ptr addrspace(1) %a) { ; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0 ; GFX1250-GISEL-NEXT: s_mov_b64 s[4:5], 0x400000003f800000 ; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3] -; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5] ; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0 ; GFX1250-GISEL-NEXT: global_load_b64 v[0:1], v6, s[0:1] scale_offset +; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5] ; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0 +; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX1250-GISEL-NEXT: v_pk_fma_f32 v[0:1], v[0:1], v[2:3], v[4:5] ; GFX1250-GISEL-NEXT: global_store_b64 v6, v[0:1], s[0:1] scale_offset ; GFX1250-GISEL-NEXT: s_endpgm diff --git a/llvm/test/CodeGen/AMDGPU/reassoc-mul-add-1-to-mad.ll b/llvm/test/CodeGen/AMDGPU/reassoc-mul-add-1-to-mad.ll index 8bf811e335b13e..6e968c757e827d 100644 --- a/llvm/test/CodeGen/AMDGPU/reassoc-mul-add-1-to-mad.ll +++ b/llvm/test/CodeGen/AMDGPU/reassoc-mul-add-1-to-mad.ll @@ -5236,37 +5236,38 @@ define amdgpu_kernel void @compute_mad(ptr addrspace(4) %i18, ptr addrspace(4) % ; GFX1250-NEXT: s_and_b32 s4, ttmp6, 15 ; GFX1250-NEXT: s_getreg_b32 s5, hwreg(HW_REG_IB_STS2, 6, 4) ; GFX1250-NEXT: v_mul_lo_u32 v2, v2, v0 -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1) -; GFX1250-NEXT: v_mul_lo_u32 v3, v2, v1 ; GFX1250-NEXT: s_wait_kmcnt 0x0 ; GFX1250-NEXT: s_load_b32 s2, s[2:3], 0x4 nv -; GFX1250-NEXT: v_add_nc_u32_e32 v1, v3, v1 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1) +; GFX1250-NEXT: v_mul_lo_u32 v3, v2, v1 ; GFX1250-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 nv ; GFX1250-NEXT: s_wait_xcnt 0x0 ; GFX1250-NEXT: s_bfe_u32 s3, ttmp6, 0x4000c -; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1) ; GFX1250-NEXT: s_add_co_i32 s3, s3, 1 -; GFX1250-NEXT: v_mul_lo_u32 v1, v1, v2 -; GFX1250-NEXT: v_add_nc_u32_e32 v2, 1, v3 +; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX1250-NEXT: s_mul_i32 s3, ttmp9, s3 +; GFX1250-NEXT: v_add_nc_u32_e32 v1, v3, v1 ; GFX1250-NEXT: s_add_co_i32 s4, s4, s3 -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) -; GFX1250-NEXT: v_mul_lo_u32 v3, v1, v2 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-NEXT: v_mul_lo_u32 v1, v1, v2 ; GFX1250-NEXT: s_wait_kmcnt 0x0 ; GFX1250-NEXT: s_and_b32 s2, s2, 0xffff +; GFX1250-NEXT: v_add_nc_u32_e32 v2, 1, v3 ; GFX1250-NEXT: s_cmp_eq_u32 s5, 0 -; GFX1250-NEXT: v_add_nc_u32_e32 v2, v3, v2 ; GFX1250-NEXT: s_cselect_b32 s3, ttmp9, s4 -; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX1250-NEXT: v_mul_lo_u32 v3, v1, v2 ; GFX1250-NEXT: v_mad_u32 v0, s3, s2, v0 +; GFX1250-NEXT: v_add_nc_u32_e32 v2, v3, v2 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) ; GFX1250-NEXT: v_mul_lo_u32 v2, v2, v1 ; GFX1250-NEXT: v_mov_b32_e32 v1, 0 -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1250-NEXT: v_add_nc_u64_e32 v[0:1], s[0:1], v[0:1] ; GFX1250-NEXT: v_mad_u32 v3, v2, v3, v2 ; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) -; GFX1250-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 2, s[8:9] +; GFX1250-NEXT: v_add_nc_u64_e32 v[0:1], s[0:1], v[0:1] ; GFX1250-NEXT: v_mad_u32 v2, v3, v2, v3 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX1250-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 2, s[8:9] ; GFX1250-NEXT: global_store_b32 v[0:1], v2, off ; GFX1250-NEXT: s_endpgm ; diff --git a/llvm/test/CodeGen/AMDGPU/shl.v2i64.ll b/llvm/test/CodeGen/AMDGPU/shl.v2i64.ll index 067255d621c2c0..359ec532408fae 100644 --- a/llvm/test/CodeGen/AMDGPU/shl.v2i64.ll +++ b/llvm/test/CodeGen/AMDGPU/shl.v2i64.ll @@ -334,44 +334,82 @@ define amdgpu_kernel void @shl_v_imm_v4i64(ptr addrspace(1) %out, ptr addrspace( } define amdgpu_kernel void @v_shl_v8i64(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 { -; GCN-LABEL: v_shl_v8i64: -; GCN: ; %bb.0: -; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0 -; GCN-NEXT: s_mov_b64 s[64:65], 0 -; GCN-NEXT: v_nop -; GCN-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE -; GCN-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv -; GCN-NEXT: v_and_b32_e32 v0, 0x3ff, v0 -; GCN-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GCN-NEXT: v_lshlrev_b32_e32 v32, 6, v0 -; GCN-NEXT: s_wait_kmcnt 0x0 -; GCN-NEXT: s_clause 0x7 -; GCN-NEXT: global_load_b128 v[0:3], v32, s[2:3] -; GCN-NEXT: global_load_b128 v[4:7], v32, s[2:3] offset:16 -; GCN-NEXT: global_load_b128 v[8:11], v32, s[2:3] offset:64 -; GCN-NEXT: global_load_b128 v[12:15], v32, s[2:3] offset:80 -; GCN-NEXT: global_load_b128 v[16:19], v32, s[2:3] offset:32 -; GCN-NEXT: global_load_b128 v[20:23], v32, s[2:3] offset:48 -; GCN-NEXT: global_load_b128 v[24:27], v32, s[2:3] offset:112 -; GCN-NEXT: global_load_b128 v[28:31], v32, s[2:3] offset:96 -; GCN-NEXT: s_wait_loadcnt 0x5 -; GCN-NEXT: v_lshlrev_b64_e32 v[2:3], v10, v[2:3] -; GCN-NEXT: v_lshlrev_b64_e32 v[0:1], v8, v[0:1] -; GCN-NEXT: s_wait_loadcnt 0x4 -; GCN-NEXT: v_lshlrev_b64_e32 v[6:7], v14, v[6:7] -; GCN-NEXT: v_lshlrev_b64_e32 v[4:5], v12, v[4:5] -; GCN-NEXT: s_wait_loadcnt 0x1 -; GCN-NEXT: v_lshlrev_b64_e32 v[22:23], v26, v[22:23] -; GCN-NEXT: s_wait_loadcnt 0x0 -; GCN-NEXT: v_lshlrev_b64_e32 v[18:19], v30, v[18:19] -; GCN-NEXT: v_lshlrev_b64_e32 v[16:17], v28, v[16:17] -; GCN-NEXT: v_lshlrev_b64_e32 v[20:21], v24, v[20:21] -; GCN-NEXT: s_clause 0x3 -; GCN-NEXT: global_store_b128 v32, v[16:19], s[0:1] offset:32 -; GCN-NEXT: global_store_b128 v32, v[20:23], s[0:1] offset:48 -; GCN-NEXT: global_store_b128 v32, v[0:3], s[0:1] -; GCN-NEXT: global_store_b128 v32, v[4:7], s[0:1] offset:16 -; GCN-NEXT: s_endpgm +; GFX1250-LABEL: v_shl_v8i64: +; GFX1250: ; %bb.0: +; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0 +; GFX1250-NEXT: s_mov_b64 s[64:65], 0 +; GFX1250-NEXT: v_nop +; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE +; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv +; GFX1250-NEXT: v_and_b32_e32 v0, 0x3ff, v0 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1250-NEXT: v_lshlrev_b32_e32 v32, 6, v0 +; GFX1250-NEXT: s_wait_kmcnt 0x0 +; GFX1250-NEXT: s_clause 0x7 +; GFX1250-NEXT: global_load_b128 v[0:3], v32, s[2:3] +; GFX1250-NEXT: global_load_b128 v[4:7], v32, s[2:3] offset:16 +; GFX1250-NEXT: global_load_b128 v[8:11], v32, s[2:3] offset:64 +; GFX1250-NEXT: global_load_b128 v[12:15], v32, s[2:3] offset:80 +; GFX1250-NEXT: global_load_b128 v[16:19], v32, s[2:3] offset:32 +; GFX1250-NEXT: global_load_b128 v[20:23], v32, s[2:3] offset:48 +; GFX1250-NEXT: global_load_b128 v[24:27], v32, s[2:3] offset:112 +; GFX1250-NEXT: global_load_b128 v[28:31], v32, s[2:3] offset:96 +; GFX1250-NEXT: s_wait_loadcnt 0x5 +; GFX1250-NEXT: v_lshlrev_b64_e32 v[2:3], v10, v[2:3] +; GFX1250-NEXT: v_lshlrev_b64_e32 v[0:1], v8, v[0:1] +; GFX1250-NEXT: s_wait_loadcnt 0x1 +; GFX1250-NEXT: v_lshlrev_b64_e32 v[22:23], v26, v[22:23] +; GFX1250-NEXT: s_wait_loadcnt 0x0 +; GFX1250-NEXT: v_lshlrev_b64_e32 v[18:19], v30, v[18:19] +; GFX1250-NEXT: v_lshlrev_b64_e32 v[16:17], v28, v[16:17] +; GFX1250-NEXT: v_lshlrev_b64_e32 v[20:21], v24, v[20:21] +; GFX1250-NEXT: v_lshlrev_b64_e32 v[6:7], v14, v[6:7] +; GFX1250-NEXT: v_lshlrev_b64_e32 v[4:5], v12, v[4:5] +; GFX1250-NEXT: s_clause 0x3 +; GFX1250-NEXT: global_store_b128 v32, v[16:19], s[0:1] offset:32 +; GFX1250-NEXT: global_store_b128 v32, v[20:23], s[0:1] offset:48 +; GFX1250-NEXT: global_store_b128 v32, v[0:3], s[0:1] +; GFX1250-NEXT: global_store_b128 v32, v[4:7], s[0:1] offset:16 +; GFX1250-NEXT: s_endpgm +; +; GFX1251-LABEL: v_shl_v8i64: +; GFX1251: ; %bb.0: +; GFX1251-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0 +; GFX1251-NEXT: s_mov_b64 s[64:65], 0 +; GFX1251-NEXT: v_nop +; GFX1251-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE +; GFX1251-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv +; GFX1251-NEXT: v_and_b32_e32 v0, 0x3ff, v0 +; GFX1251-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1251-NEXT: v_lshlrev_b32_e32 v32, 6, v0 +; GFX1251-NEXT: s_wait_kmcnt 0x0 +; GFX1251-NEXT: s_clause 0x7 +; GFX1251-NEXT: global_load_b128 v[0:3], v32, s[2:3] +; GFX1251-NEXT: global_load_b128 v[4:7], v32, s[2:3] offset:16 +; GFX1251-NEXT: global_load_b128 v[8:11], v32, s[2:3] offset:64 +; GFX1251-NEXT: global_load_b128 v[12:15], v32, s[2:3] offset:80 +; GFX1251-NEXT: global_load_b128 v[16:19], v32, s[2:3] offset:32 +; GFX1251-NEXT: global_load_b128 v[20:23], v32, s[2:3] offset:48 +; GFX1251-NEXT: global_load_b128 v[24:27], v32, s[2:3] offset:112 +; GFX1251-NEXT: global_load_b128 v[28:31], v32, s[2:3] offset:96 +; GFX1251-NEXT: s_wait_loadcnt 0x5 +; GFX1251-NEXT: v_lshlrev_b64_e32 v[2:3], v10, v[2:3] +; GFX1251-NEXT: v_lshlrev_b64_e32 v[0:1], v8, v[0:1] +; GFX1251-NEXT: s_wait_loadcnt 0x4 +; GFX1251-NEXT: v_lshlrev_b64_e32 v[6:7], v14, v[6:7] +; GFX1251-NEXT: v_lshlrev_b64_e32 v[4:5], v12, v[4:5] +; GFX1251-NEXT: s_wait_loadcnt 0x1 +; GFX1251-NEXT: v_lshlrev_b64_e32 v[22:23], v26, v[22:23] +; GFX1251-NEXT: s_wait_loadcnt 0x0 +; GFX1251-NEXT: v_lshlrev_b64_e32 v[18:19], v30, v[18:19] +; GFX1251-NEXT: v_lshlrev_b64_e32 v[16:17], v28, v[16:17] +; GFX1251-NEXT: v_lshlrev_b64_e32 v[20:21], v24, v[20:21] +; GFX1251-NEXT: s_clause 0x3 +; GFX1251-NEXT: global_store_b128 v32, v[16:19], s[0:1] offset:32 +; GFX1251-NEXT: global_store_b128 v32, v[20:23], s[0:1] offset:48 +; GFX1251-NEXT: global_store_b128 v32, v[0:3], s[0:1] +; GFX1251-NEXT: global_store_b128 v32, v[4:7], s[0:1] offset:16 +; GFX1251-NEXT: s_endpgm %tid = call i32 @llvm.amdgcn.workitem.id.x() %tid.ext = sext i32 %tid to i64 %in.gep = getelementptr inbounds <8 x i64>, ptr addrspace(1) %in, i64 %tid.ext diff --git a/llvm/test/CodeGen/AMDGPU/slp-int-to-fp.ll b/llvm/test/CodeGen/AMDGPU/slp-int-to-fp.ll index f56b30d227fc7e..d0e2bb112a931a 100644 --- a/llvm/test/CodeGen/AMDGPU/slp-int-to-fp.ll +++ b/llvm/test/CodeGen/AMDGPU/slp-int-to-fp.ll @@ -1147,34 +1147,33 @@ define void @sitofp_i48_to_half(ptr addrspace(1) noalias %out, ptr addrspace(1) ; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX1250-NEXT: v_lshlrev_b64_e32 v[4:5], v12, v[4:5] ; GFX1250-NEXT: v_lshlrev_b64_e32 v[8:9], v13, v[8:9] -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX1250-NEXT: v_lshlrev_b64_e32 v[10:11], v14, v[10:11] +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) ; GFX1250-NEXT: v_lshlrev_b64_e32 v[6:7], v16, v[6:7] -; GFX1250-NEXT: v_dual_sub_nc_u32 v12, 32, v12 :: v_dual_sub_nc_u32 v14, 32, v14 +; GFX1250-NEXT: v_lshlrev_b64_e32 v[10:11], v14, v[10:11] ; GFX1250-NEXT: v_min_u32_e32 v4, 1, v4 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX1250-NEXT: v_min_u32_e32 v8, 1, v8 -; GFX1250-NEXT: v_min_u32_e32 v10, 1, v10 ; GFX1250-NEXT: v_min_u32_e32 v6, 1, v6 -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) -; GFX1250-NEXT: v_dual_sub_nc_u32 v13, 32, v13 :: v_dual_bitop2_b32 v4, v5, v4 bitop3:0x54 +; GFX1250-NEXT: v_sub_nc_u32_e32 v13, 32, v13 +; GFX1250-NEXT: v_min_u32_e32 v10, 1, v10 +; GFX1250-NEXT: v_or_b32_e32 v4, v5, v4 ; GFX1250-NEXT: v_or_b32_e32 v5, v9, v8 -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) ; GFX1250-NEXT: v_dual_sub_nc_u32 v8, 32, v16 :: v_dual_bitop2_b32 v6, v7, v6 bitop3:0x54 -; GFX1250-NEXT: v_or_b32_e32 v7, v11, v10 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX1250-NEXT: v_dual_sub_nc_u32 v12, 32, v12 :: v_dual_bitop2_b32 v7, v11, v10 bitop3:0x54 +; GFX1250-NEXT: v_sub_nc_u32_e32 v14, 32, v14 ; GFX1250-NEXT: v_cvt_f32_i32_e32 v4, v4 -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX1250-NEXT: v_cvt_f32_i32_e32 v5, v5 ; GFX1250-NEXT: v_cvt_f32_i32_e32 v6, v6 -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX1250-NEXT: v_cvt_f32_i32_e32 v7, v7 -; GFX1250-NEXT: v_ldexp_f32 v4, v4, v12 ; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX1250-NEXT: v_ldexp_f32 v4, v4, v12 ; GFX1250-NEXT: v_ldexp_f32 v5, v5, v13 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX1250-NEXT: v_ldexp_f32 v6, v6, v8 -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX1250-NEXT: v_ldexp_f32 v7, v7, v14 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX1250-NEXT: v_cvt_pk_f16_f32 v5, v5, v6 -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) ; GFX1250-NEXT: v_cvt_pk_f16_f32 v4, v4, v7 ; GFX1250-NEXT: global_store_b64 v[0:1], v[4:5], off ; GFX1250-NEXT: s_set_pc_i64 s[30:31] @@ -1458,34 +1457,34 @@ define void @uitofp_i56_to_half(ptr addrspace(1) noalias %out, ptr addrspace(1) ; GFX1250-NEXT: v_lshlrev_b64_e32 v[10:11], v14, v[10:11] ; GFX1250-NEXT: v_min_u32_e32 v15, 32, v15 ; GFX1250-NEXT: v_lshlrev_b64_e32 v[8:9], v12, v[8:9] -; GFX1250-NEXT: v_sub_nc_u32_e32 v12, 32, v12 ; GFX1250-NEXT: v_lshlrev_b64_e32 v[6:7], v13, v[6:7] -; GFX1250-NEXT: v_min_u32_e32 v10, 1, v10 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) ; GFX1250-NEXT: v_lshlrev_b64_e32 v[4:5], v15, v[4:5] ; GFX1250-NEXT: v_min_u32_e32 v8, 1, v8 -; GFX1250-NEXT: v_sub_nc_u32_e32 v13, 32, v13 +; GFX1250-NEXT: v_min_u32_e32 v10, 1, v10 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) ; GFX1250-NEXT: v_min_u32_e32 v6, 1, v6 -; GFX1250-NEXT: v_sub_nc_u32_e32 v15, 32, v15 +; GFX1250-NEXT: v_sub_nc_u32_e32 v13, 32, v13 ; GFX1250-NEXT: v_min_u32_e32 v4, 1, v4 ; GFX1250-NEXT: v_or_b32_e32 v8, v9, v8 -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX1250-NEXT: v_dual_sub_nc_u32 v12, 32, v12 :: v_dual_sub_nc_u32 v15, 32, v15 ; GFX1250-NEXT: v_or_b32_e32 v6, v7, v6 ; GFX1250-NEXT: v_or_b32_e32 v7, v11, v10 ; GFX1250-NEXT: v_dual_sub_nc_u32 v5, 32, v14 :: v_dual_bitop2_b32 v4, v5, v4 bitop3:0x54 -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) ; GFX1250-NEXT: v_cvt_f32_u32_e32 v8, v8 -; GFX1250-NEXT: v_cvt_f32_u32_e32 v6, v6 ; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX1250-NEXT: v_cvt_f32_u32_e32 v6, v6 ; GFX1250-NEXT: v_cvt_f32_u32_e32 v7, v7 -; GFX1250-NEXT: v_cvt_f32_u32_e32 v4, v4 ; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX1250-NEXT: v_cvt_f32_u32_e32 v4, v4 ; GFX1250-NEXT: v_ldexp_f32 v8, v8, v12 -; GFX1250-NEXT: v_ldexp_f32 v6, v6, v13 ; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX1250-NEXT: v_ldexp_f32 v6, v6, v13 ; GFX1250-NEXT: v_ldexp_f32 v5, v7, v5 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX1250-NEXT: v_ldexp_f32 v4, v4, v15 -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX1250-NEXT: v_cvt_pk_f16_f32 v5, v6, v5 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) ; GFX1250-NEXT: v_cvt_pk_f16_f32 v4, v8, v4 ; GFX1250-NEXT: global_store_b64 v[0:1], v[4:5], off ; GFX1250-NEXT: s_set_pc_i64 s[30:31] diff --git a/llvm/test/CodeGen/AMDGPU/strict_fptrunc_bf16.ll b/llvm/test/CodeGen/AMDGPU/strict_fptrunc_bf16.ll index 0eaa01d2685620..cba21eb769a8ac 100644 --- a/llvm/test/CodeGen/AMDGPU/strict_fptrunc_bf16.ll +++ b/llvm/test/CodeGen/AMDGPU/strict_fptrunc_bf16.ll @@ -128,25 +128,25 @@ define amdgpu_ps void @strict_fptrunc_v2f64_to_v2bf16(<2 x double> %a, ptr %out) ; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) ; GFX1250-NEXT: v_cvt_f64_f32_e32 v[6:7], v10 ; GFX1250-NEXT: v_cvt_f64_f32_e32 v[8:9], v11 -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) ; GFX1250-NEXT: v_cmp_gt_f64_e64 s1, |v[2:3]|, |v[6:7]| ; GFX1250-NEXT: v_cmp_nlg_f64_e32 vcc_lo, v[2:3], v[6:7] -; GFX1250-NEXT: v_cmp_nlg_f64_e64 s0, v[0:1], v[8:9] -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) ; GFX1250-NEXT: v_cndmask_b32_e64 v2, -1, 1, s1 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2) ; GFX1250-NEXT: v_cmp_gt_f64_e64 s1, |v[0:1]|, |v[8:9]| -; GFX1250-NEXT: v_dual_add_nc_u32 v1, v10, v2 :: v_dual_bitop2_b32 v13, 1, v11 bitop3:0x40 -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3) -; GFX1250-NEXT: v_cmp_ne_u32_e64 s2, 0, v13 +; GFX1250-NEXT: v_cmp_nlg_f64_e64 s0, v[0:1], v[8:9] ; GFX1250-NEXT: v_cndmask_b32_e64 v0, -1, 1, s1 -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1250-NEXT: v_dual_add_nc_u32 v0, v11, v0 :: v_dual_bitop2_b32 v12, 1, v10 bitop3:0x40 +; GFX1250-NEXT: v_and_b32_e32 v12, 1, v10 +; GFX1250-NEXT: v_dual_add_nc_u32 v1, v10, v2 :: v_dual_bitop2_b32 v13, 1, v11 bitop3:0x40 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX1250-NEXT: v_add_nc_u32_e32 v0, v11, v0 ; GFX1250-NEXT: v_cmp_ne_u32_e64 s1, 0, v12 +; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1) +; GFX1250-NEXT: v_cmp_ne_u32_e64 s2, 0, v13 ; GFX1250-NEXT: s_or_b32 vcc_lo, s1, vcc_lo ; GFX1250-NEXT: v_cndmask_b32_e32 v1, v1, v10, vcc_lo ; GFX1250-NEXT: s_or_b32 vcc_lo, s2, s0 ; GFX1250-NEXT: v_cndmask_b32_e32 v0, v0, v11, vcc_lo -; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, v1 ; GFX1250-NEXT: flat_store_b32 v[4:5], v0 ; GFX1250-NEXT: s_endpgm >From 0f14737f4efa2910f7fae109fe9c9cdd37790500 Mon Sep 17 00:00:00 2001 From: Jeffrey Byrnes <[email protected]> Date: Fri, 6 Nov 2026 16:54:42 -0800 Subject: [PATCH 2/2] Formatting Change-Id: I18ed65c34f9afe8c590d321858efd87ea73a11df --- llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp index ccc34018fdc52b..8887a2f3fc668f 100644 --- a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp +++ b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp @@ -279,7 +279,8 @@ void GCNHazardRecognizer::updateTRANSState(const MachineInstr &MI) { void GCNHazardRecognizer::updateMultiCycleVALUState(const MachineInstr &MI) { if (!hasCoExecWindowModel()) return; - // Multi-cycle VALU (CVT, etc.) blocks subsequent VALU for BlockingCycles cycles. + // Multi-cycle VALU (CVT, etc.) blocks subsequent VALU for BlockingCycles + // cycles. if (!SIInstrInfo::isVALU(MI, /*AllowLDSDMA=*/true)) return; @@ -294,7 +295,8 @@ void GCNHazardRecognizer::updateMultiCycleVALUState(const MachineInstr &MI) { // hazard check (same convention as CyclesUntilTRANS), so to expose // BlockingCycles-1 cycles of shadow we must seed with BlockingCycles. CyclesUntilVALU = BlockingCycles; - LLVM_DEBUG(dbgs() << " Multi-cycle VALU: BlockingCycles=" << BlockingCycles + LLVM_DEBUG(dbgs() << " Multi-cycle VALU: BlockingCycles=" + << BlockingCycles << ", CyclesUntilVALU=" << CyclesUntilVALU << "\n"); } } _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
