https://github.com/adelejjeh updated https://github.com/llvm/llvm-project/pull/230183
>From 97037679411481aa82e031639d8f0b5b854b05dd Mon Sep 17 00:00:00 2001 From: Adel Ejjeh <[email protected]> Date: Thu, 8 Oct 2026 11:04:36 -0500 Subject: [PATCH 1/2] [AMDGPU] Form VOPD dot2 pairs with a literal in src1 A V_DOT2 with a register in src0 and a literal in src1 is matched as if commuted, and commuted when the VOPD pair is built. This recovers the pairs lost once MachineCSE stopped leaving those literals in src0. Co-Authored-By: Claude <[email protected]> --- llvm/lib/Target/AMDGPU/GCNCreateVOPD.cpp | 9 ++ llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp | 73 ++++++--- llvm/lib/Target/AMDGPU/GCNVOPDUtils.h | 3 + llvm/lib/Target/AMDGPU/SIInstrInfo.h | 4 +- .../AMDGPU/llvm.amdgcn.fdot2.f32.bf16.ll | 9 +- llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fdot2.ll | 47 ++---- .../AMDGPU/vopd-dot2-commute-imm-src1.mir | 149 ++++++++++++++++++ 7 files changed, 232 insertions(+), 62 deletions(-) create mode 100644 llvm/test/CodeGen/AMDGPU/vopd-dot2-commute-imm-src1.mir diff --git a/llvm/lib/Target/AMDGPU/GCNCreateVOPD.cpp b/llvm/lib/Target/AMDGPU/GCNCreateVOPD.cpp index 2ca4fa50f7f29..0179a6422e98b 100644 --- a/llvm/lib/Target/AMDGPU/GCNCreateVOPD.cpp +++ b/llvm/lib/Target/AMDGPU/GCNCreateVOPD.cpp @@ -197,6 +197,14 @@ class GCNCreateVOPD { return Selected; } + // Commute the components the match was checked with commuted. + void applyCommutes(const SIInstrInfo &TII, VOPDMatchInfo &Match) { + for (unsigned I : {0u, 1u}) { + if (Match.Commute[I] && !TII.commuteInstruction(*Match.InOrder[I])) + llvm_unreachable("failed to commute VOPD component"); + } + } + void materializeLiteral(const SIInstrInfo &TII, VOPDCandidate &Candidate) { if (!Candidate.needsMaterialization()) return; @@ -326,6 +334,7 @@ class GCNCreateVOPD { assignMaterializationRegisters(MBB, Candidates); SmallVector<VOPDCandidate *, 8> Selected = selectCandidates(Candidates); for (VOPDCandidate *Candidate : Selected) { + applyCommutes(*SII, Candidate->Match); materializeLiteral(*SII, *Candidate); Changed |= doReplace(SII, Candidate->Match); } diff --git a/llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp b/llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp index 5241e01387b3d..ae0881ae8d819 100644 --- a/llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp +++ b/llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp @@ -82,7 +82,11 @@ static const MachineOperand &getNamedOp(const MachineInstr &MI, // Check if MI is a VOP3P instruction with operands that satisfy the constraints // for mapping it to a VOP2/VOPD opcode: no modifiers, no clamp, src1 and src2 // are registers (src0 can be register or literal), and src2 is same as dst. -static bool canMapVOP3PToVOPD(const MachineInstr &MI) { +// An immediate in src1 is accepted if src0 is a register and the operands can +// be commuted. \p Commute is set if that is needed. +static bool canMapVOP3PToVOPD(const SIInstrInfo &TII, const MachineInstr &MI, + bool &Commute) { + Commute = false; unsigned Opc = MI.getOpcode(); if (Opc != AMDGPU::V_DOT2_F32_F16 && Opc != AMDGPU::V_DOT2_F32_BF16) return false; @@ -93,8 +97,14 @@ static bool canMapVOP3PToVOPD(const MachineInstr &MI) { if (getNamedOp(MI, AMDGPU::OpName::src1_modifiers).getImm() != SISrcMods::OP_SEL_1) return false; - if (!getNamedOp(MI, AMDGPU::OpName::src1).isReg()) - return false; + if (!getNamedOp(MI, AMDGPU::OpName::src1).isReg()) { + if (!(getNamedOp(MI, AMDGPU::OpName::src0).isReg() && + getNamedOp(MI, AMDGPU::OpName::src1).isImm() && + TII.isLegalToSwap(MI, getNamedOperandIdx(Opc, AMDGPU::OpName::src0), + getNamedOperandIdx(Opc, AMDGPU::OpName::src1)))) + return false; + Commute = true; + } if (getNamedOp(MI, AMDGPU::OpName::src2_modifiers).getImm() != SISrcMods::OP_SEL_1) return false; @@ -156,11 +166,10 @@ static bool canMaterializeVOPDLiterals(const MachineFunction &MF) { !MF.getFunction().hasOptSize(); } -static bool -checkVOPDRegConstraints(const SIInstrInfo &TII, const MachineInstr &MIX, - const MachineInstr &MIY, bool IsVOPD3, - bool AllowSameVGPR, - SmallVectorImpl<VOPDLiteralFixup> &LiteralFixups) { +static bool checkVOPDRegConstraints( + const SIInstrInfo &TII, const MachineInstr &MIX, const MachineInstr &MIY, + bool IsVOPD3, bool AllowSameVGPR, + SmallVectorImpl<VOPDLiteralFixup> &LiteralFixups, bool (&CommuteXY)[2]) { namespace VOPD = AMDGPU::VOPD; const MachineFunction *MF = MIX.getMF(); @@ -170,8 +179,12 @@ checkVOPDRegConstraints(const SIInstrInfo &TII, const MachineInstr &MIX, return false; if (IsVOPD3 && isVOPD3F64OPYSrcHazard(TII, MIX, MIY)) return false; - if (!IsVOPD3 && ((TII.isVOP3(MIX) && !canMapVOP3PToVOPD(MIX)) || - (TII.isVOP3(MIY) && !canMapVOP3PToVOPD(MIY)))) + // Components which only map once src0 and src1 are commuted. The checks + // below look at their operands as they will be after the commute. + bool Commute[2] = {false, false}; + if (!IsVOPD3 && + ((TII.isVOP3(MIX) && !canMapVOP3PToVOPD(TII, MIX, Commute[VOPD::X])) || + (TII.isVOP3(MIY) && !canMapVOP3PToVOPD(TII, MIY, Commute[VOPD::Y])))) return false; if (TII.isDPP(MIX) || TII.isDPP(MIY)) return false; @@ -206,8 +219,12 @@ checkVOPDRegConstraints(const SIInstrInfo &TII, const MachineInstr &MIX, for (auto CompIdx : VOPD::COMPONENTS) { const MachineInstr &MI = (CompIdx == VOPD::X) ? MIX : MIY; + const AMDGPU::OpName Src0Name = + Commute[CompIdx] ? AMDGPU::OpName::src1 : AMDGPU::OpName::src0; + const AMDGPU::OpName Src1Name = + Commute[CompIdx] ? AMDGPU::OpName::src0 : AMDGPU::OpName::src1; - const MachineOperand &Src0 = *TII.getNamedOperand(MI, AMDGPU::OpName::src0); + const MachineOperand &Src0 = *TII.getNamedOperand(MI, Src0Name); if (Src0.isReg()) { if (!isValidVOPDSrc(TII, VOPDOpc, CompIdx, 0, Src0.getReg())) return false; @@ -224,7 +241,7 @@ checkVOPDRegConstraints(const SIInstrInfo &TII, const MachineInstr &MIX, return false; // Only a 32-bit slot is handled, because the caller produces the value // with a single S_MOV_B32. - int OpIdx = getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::src0); + int OpIdx = getNamedOperandIdx(MI.getOpcode(), Src0Name); if (TII.getOpSize(MI, OpIdx) != 4) return false; const TargetRegisterClass *SlotRC = @@ -251,8 +268,7 @@ checkVOPDRegConstraints(const SIInstrInfo &TII, const MachineInstr &MIX, if (MI.getDesc().hasImplicitUseOfPhysReg(AMDGPU::VCC)) UniqueScalarRegs.insert(AMDGPU::VCC_LO); - if (const MachineOperand *Src1 = - TII.getNamedOperand(MI, AMDGPU::OpName::src1)) { + if (const MachineOperand *Src1 = TII.getNamedOperand(MI, Src1Name)) { if (Src1->isReg()) { if (!isValidVOPDSrc(TII, VOPDOpc, CompIdx, 1, Src1->getReg())) return false; @@ -310,6 +326,14 @@ checkVOPDRegConstraints(const SIInstrInfo &TII, const MachineInstr &MIX, auto GetVRegIdx = [&](unsigned OpcodeIdx, unsigned OperandIdx) { const MachineInstr &MI = (OpcodeIdx == VOPD::X) ? MIX : MIY; + if (Commute[OpcodeIdx]) { + int Src0Idx = getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::src0); + int Src1Idx = getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::src1); + if ((int)OperandIdx == Src0Idx) + OperandIdx = Src1Idx; + else if ((int)OperandIdx == Src1Idx) + OperandIdx = Src0Idx; + } const MachineOperand &Operand = MI.getOperand(OperandIdx); if (Operand.isReg() && TRI->isVectorRegister(MRI, Operand.getReg())) return Operand.getReg(); @@ -330,6 +354,8 @@ checkVOPDRegConstraints(const SIInstrInfo &TII, const MachineInstr &MIX, LLVM_DEBUG(dbgs() << "VOPD Reg Constraints Passed\n\tX: " << MIX << "\n\tY: " << MIY << "\n"); LiteralFixups.assign(Fixups); + CommuteXY[VOPD::X] = Commute[VOPD::X]; + CommuteXY[VOPD::Y] = Commute[VOPD::Y]; return true; } @@ -358,14 +384,19 @@ tryMatchVOPDPairVariant(const SIInstrInfo &TII, unsigned EncodingFamily, bool AllowSameVGPR = ST.hasGFX12Insts(); // Only a VOPD3 component can need a fixup; a plain one may hold a literal. - // checkVOPDRegConstraints() only writes this when it succeeds. + // checkVOPDRegConstraints() only writes these when it succeeds. SmallVector<VOPDLiteralFixup, 2> Fixups; + bool CommuteXY[2] = {false, false}; if (FirstCanBeVOPD.X && SecondCanBeVOPD.Y) { if (checkVOPDRegConstraints(TII, FirstMI, SecondMI, IsVOPD3, AllowSameVGPR, - Fixups)) + Fixups, CommuteXY)) return VOPDMatchInfo{ - {&FirstMI, &SecondMI}, 0, IsVOPD3, std::move(Fixups)}; + {&FirstMI, &SecondMI}, + 0, + IsVOPD3, + std::move(Fixups), + {CommuteXY[AMDGPU::VOPD::X], CommuteXY[AMDGPU::VOPD::Y]}}; } if (FirstCanBeVOPD.Y && SecondCanBeVOPD.X) { @@ -376,9 +407,13 @@ tryMatchVOPDPairVariant(const SIInstrInfo &TII, unsigned EncodingFamily, if (IsAntiDep && !TII.isVOPDAntidependencyAllowed(SecondMI)) return std::nullopt; if (checkVOPDRegConstraints(TII, SecondMI, FirstMI, IsVOPD3, AllowSameVGPR, - Fixups)) + Fixups, CommuteXY)) return VOPDMatchInfo{ - {&FirstMI, &SecondMI}, 1, IsVOPD3, std::move(Fixups)}; + {&FirstMI, &SecondMI}, + 1, + IsVOPD3, + std::move(Fixups), + {CommuteXY[AMDGPU::VOPD::Y], CommuteXY[AMDGPU::VOPD::X]}}; } return std::nullopt; diff --git a/llvm/lib/Target/AMDGPU/GCNVOPDUtils.h b/llvm/lib/Target/AMDGPU/GCNVOPDUtils.h index dbb4d074bf6e9..773516fb832bc 100644 --- a/llvm/lib/Target/AMDGPU/GCNVOPDUtils.h +++ b/llvm/lib/Target/AMDGPU/GCNVOPDUtils.h @@ -51,6 +51,9 @@ struct VOPDMatchInfo { /// can be built. They all have the same 32-bit value, so one register serves /// the whole pair. Only a VOPD3 pair can need this. SmallVector<VOPDLiteralFixup, 2> LiteralFixups; + /// Entries in \p InOrder which only pair once commuted. The match was checked + /// with them commuted, but they are left unchanged until the pair is built. + bool Commute[2] = {false, false}; MachineInstr *getMIX() const { return InOrder[XIdx]; } MachineInstr *getMIY() const { return InOrder[1 - XIdx]; } diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h index fbb4da4c1c21a..1457721224ccb 100644 --- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h +++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h @@ -231,14 +231,14 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo { bool swapSourceModifiers(MachineInstr &MI, MachineOperand &Src0, AMDGPU::OpName Src0OpName, MachineOperand &Src1, AMDGPU::OpName Src1OpName) const; - bool isLegalToSwap(const MachineInstr &MI, unsigned fromIdx, - unsigned toIdx) const; bool isNonCommutableDPP(const MachineInstr &MI) const; MachineInstr *commuteInstructionImpl(MachineInstr &MI, bool NewMI, unsigned OpIdx0, unsigned OpIdx1) const override; public: + bool isLegalToSwap(const MachineInstr &MI, unsigned fromIdx, + unsigned toIdx) const; enum TargetOperandFlags { MO_MASK = 0xf, diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fdot2.f32.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fdot2.f32.bf16.ll index 1595e6e8a730d..9c92af928f018 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fdot2.f32.bf16.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fdot2.f32.bf16.ll @@ -1140,8 +1140,7 @@ define float @v_fdot2_f32_bf16_inline_literal_b_x(<2 x bfloat> %a, <2 x bfloat> ; ; GFX11PLUS-LABEL: v_fdot2_f32_bf16_inline_literal_b_x: ; GFX11PLUS: ; %bb.0: -; GFX11PLUS: v_dot2_f32_bf16 v2, v0, 0x40004000, v2 -; GFX11PLUS: v_dot2_f32_bf16 v5, v4, v3, v5 +; GFX11PLUS: v_dual_dot2acc_f32_bf16 v2, 0x40004000, v0 :: v_dual_dot2acc_f32_bf16 v5, v4, v3 ; GFX11PLUS: v_add_f32_e32 v0, v2, v5 %r0 = call float @llvm.amdgcn.fdot2.f32.bf16(<2 x bfloat> %a, <2 x bfloat> <bfloat 2.0, bfloat 2.0>, float %c, i1 false) %r1 = call float @llvm.amdgcn.fdot2.f32.bf16(<2 x bfloat> %e, <2 x bfloat> %d, float %f, i1 false) @@ -1158,8 +1157,7 @@ define float @v_fdot2_f32_bf16_inline_literal_b_y(<2 x bfloat> %a, <2 x bfloat> ; ; GFX11PLUS-LABEL: v_fdot2_f32_bf16_inline_literal_b_y: ; GFX11PLUS: ; %bb.0: -; GFX11PLUS: v_dot2_f32_bf16 v2, v1, v0, v2 -; GFX11PLUS: v_dot2_f32_bf16 v5, v3, 0x40004000, v5 +; GFX11PLUS: v_dual_dot2acc_f32_bf16 v2, v1, v0 :: v_dual_dot2acc_f32_bf16 v5, 0x40004000, v3 ; GFX11PLUS: v_add_f32_e32 v0, v2, v5 %r0 = call float @llvm.amdgcn.fdot2.f32.bf16(<2 x bfloat> %b, <2 x bfloat> %a, float %c, i1 false) %r1 = call float @llvm.amdgcn.fdot2.f32.bf16(<2 x bfloat> %d, <2 x bfloat> <bfloat 2.0, bfloat 2.0>, float %f, i1 false) @@ -1176,8 +1174,7 @@ define float @v_fdot2_f32_bf16_inline_literal_b_xy(<2 x bfloat> %a, <2 x bfloat> ; ; GFX11PLUS-LABEL: v_fdot2_f32_bf16_inline_literal_b_xy: ; GFX11PLUS: ; %bb.0: -; GFX11PLUS: v_dot2_f32_bf16 v2, v0, 0x40004000, v2 -; GFX11PLUS: v_dot2_f32_bf16 v5, v3, 0x40004000, v5 +; GFX11PLUS: v_dual_dot2acc_f32_bf16 v2, 0x40004000, v0 :: v_dual_dot2acc_f32_bf16 v5, 0x40004000, v3 ; GFX11PLUS: v_add_f32_e32 v0, v2, v5 %r0 = call float @llvm.amdgcn.fdot2.f32.bf16(<2 x bfloat> %a, <2 x bfloat> <bfloat 2.0, bfloat 2.0>, float %c, i1 false) %r1 = call float @llvm.amdgcn.fdot2.f32.bf16(<2 x bfloat> %d, <2 x bfloat> <bfloat 2.0, bfloat 2.0>, float %f, i1 false) diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fdot2.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fdot2.ll index f601b5d6ba3d2..8d9730dcb547f 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fdot2.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fdot2.ll @@ -826,11 +826,6 @@ define float @v_fdot2_dual_sgpr_src0_y(<2 x half> %a, <2 x half> %b, float %c, < ; GFX11: v_dot2acc_f32_f16 v2, v0, v1 ; GFX11: v_dot2acc_f32_f16 v5, s0, v3 ; GFX11: v_add_f32_e32 v0, v2, v5 -; -; GFX1170-GFX12-LABEL: v_fdot2_dual_sgpr_src0_y: -; GFX1170-GFX12: ; %bb.0: -; GFX1170-GFX12: v_dual_dot2acc_f32_f16 v2, v0, v1 :: v_dual_dot2acc_f32_f16 v5, s0, v3 -; GFX1170-GFX12: v_add_f32_e32 v0, v2, v5 %r0 = call float @llvm.amdgcn.fdot2(<2 x half> %a, <2 x half> %b, float %c, i1 false) %r1 = call float @llvm.amdgcn.fdot2(<2 x half> %d, <2 x half> %e, float %f, i1 false) %r = fadd float %r0, %r1 @@ -1534,16 +1529,10 @@ define float @v_fdot2_inline_literal_b_x(<2 x half> %a, <2 x half> %b, float %c, ; GFX10: v_dot2c_f32_f16 v5, v4, v3 ; GFX10: v_add_f32_e32 v0, v2, v5 ; -; GFX11-LABEL: v_fdot2_inline_literal_b_x: -; GFX11: ; %bb.0: -; GFX11: v_dual_dot2acc_f32_f16 v2, 0x40004000, v0 :: v_dual_dot2acc_f32_f16 v5, v4, v3 -; GFX11: v_add_f32_e32 v0, v2, v5 -; -; GFX1170-GFX12-LABEL: v_fdot2_inline_literal_b_x: -; GFX1170-GFX12: ; %bb.0: -; GFX1170-GFX12: v_dot2_f32_f16 v2, v0, 0x40004000, v2 -; GFX1170-GFX12: v_dot2_f32_f16 v5, v4, v3, v5 -; GFX1170-GFX12: v_add_f32_e32 v0, v2, v5 +; GFX11PLUS-LABEL: v_fdot2_inline_literal_b_x: +; GFX11PLUS: ; %bb.0: +; GFX11PLUS: v_dual_dot2acc_f32_f16 v2, 0x40004000, v0 :: v_dual_dot2acc_f32_f16 v5, v4, v3 +; GFX11PLUS: v_add_f32_e32 v0, v2, v5 %r0 = call float @llvm.amdgcn.fdot2(<2 x half> %a, <2 x half> <half 2.0, half 2.0>, float %c, i1 false) %r1 = call float @llvm.amdgcn.fdot2(<2 x half> %e, <2 x half> %d, float %f, i1 false) %r = fadd float %r0, %r1 @@ -1569,16 +1558,10 @@ define float @v_fdot2_inline_literal_b_y(<2 x half> %a, <2 x half> %b, float %c, ; GFX10: v_dot2c_f32_f16 v5, 0x40004000, v3 ; GFX10: v_add_f32_e32 v0, v2, v5 ; -; GFX11-LABEL: v_fdot2_inline_literal_b_y: -; GFX11: ; %bb.0: -; GFX11: v_dual_dot2acc_f32_f16 v2, v1, v0 :: v_dual_dot2acc_f32_f16 v5, 0x40004000, v3 -; GFX11: v_add_f32_e32 v0, v2, v5 -; -; GFX1170-GFX12-LABEL: v_fdot2_inline_literal_b_y: -; GFX1170-GFX12: ; %bb.0: -; GFX1170-GFX12: v_dot2_f32_f16 v2, v1, v0, v2 -; GFX1170-GFX12: v_dot2_f32_f16 v5, v3, 0x40004000, v5 -; GFX1170-GFX12: v_add_f32_e32 v0, v2, v5 +; GFX11PLUS-LABEL: v_fdot2_inline_literal_b_y: +; GFX11PLUS: ; %bb.0: +; GFX11PLUS: v_dual_dot2acc_f32_f16 v2, v1, v0 :: v_dual_dot2acc_f32_f16 v5, 0x40004000, v3 +; GFX11PLUS: v_add_f32_e32 v0, v2, v5 %r0 = call float @llvm.amdgcn.fdot2(<2 x half> %b, <2 x half> %a, float %c, i1 false) %r1 = call float @llvm.amdgcn.fdot2(<2 x half> %d, <2 x half> <half 2.0, half 2.0>, float %f, i1 false) %r = fadd float %r0, %r1 @@ -1604,16 +1587,10 @@ define float @v_fdot2_inline_literal_b_xy(<2 x half> %a, <2 x half> %b, float %c ; GFX10: v_dot2c_f32_f16 v5, 0x40004000, v3 ; GFX10: v_add_f32_e32 v0, v2, v5 ; -; GFX11-LABEL: v_fdot2_inline_literal_b_xy: -; GFX11: ; %bb.0: -; GFX11: v_dual_dot2acc_f32_f16 v2, 0x40004000, v0 :: v_dual_dot2acc_f32_f16 v5, 0x40004000, v3 -; GFX11: v_add_f32_e32 v0, v2, v5 -; -; GFX1170-GFX12-LABEL: v_fdot2_inline_literal_b_xy: -; GFX1170-GFX12: ; %bb.0: -; GFX1170-GFX12: v_dot2_f32_f16 v2, v0, 0x40004000, v2 -; GFX1170-GFX12: v_dot2_f32_f16 v5, v3, 0x40004000, v5 -; GFX1170-GFX12: v_add_f32_e32 v0, v2, v5 +; GFX11PLUS-LABEL: v_fdot2_inline_literal_b_xy: +; GFX11PLUS: ; %bb.0: +; GFX11PLUS: v_dual_dot2acc_f32_f16 v2, 0x40004000, v0 :: v_dual_dot2acc_f32_f16 v5, 0x40004000, v3 +; GFX11PLUS: v_add_f32_e32 v0, v2, v5 %r0 = call float @llvm.amdgcn.fdot2(<2 x half> %a, <2 x half> <half 2.0, half 2.0>, float %c, i1 false) %r1 = call float @llvm.amdgcn.fdot2(<2 x half> %d, <2 x half> <half 2.0, half 2.0>, float %f, i1 false) %r = fadd float %r0, %r1 diff --git a/llvm/test/CodeGen/AMDGPU/vopd-dot2-commute-imm-src1.mir b/llvm/test/CodeGen/AMDGPU/vopd-dot2-commute-imm-src1.mir new file mode 100644 index 0000000000000..e0402c3fbb20b --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/vopd-dot2-commute-imm-src1.mir @@ -0,0 +1,149 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6 +# RUN: llc -mtriple=amdgpu12.00 -run-pass=gcn-create-vopd -verify-machineinstrs %s -o - | FileCheck -check-prefix=GFX12 %s + +# A V_DOT2_F32_* can only be a VOPD component with a register in src1. With an +# immediate in src1 and a register in src0 the pair is checked as if the dot2 +# were commuted, and it is commuted when the pair is built. + +--- +# dot2 is X, the first instruction. +name: dot2_imm_src1_x +tracksRegLiveness: true +body: | + bb.0: + liveins: $vgpr0, $vgpr2, $vgpr3, $vgpr4 + ; GFX12-LABEL: name: dot2_imm_src1_x + ; GFX12: liveins: $vgpr0, $vgpr2, $vgpr3, $vgpr4 + ; GFX12-NEXT: {{ $}} + ; GFX12-NEXT: $vgpr2, $vgpr5 = V_DUAL_DOT2C_F32_F16_e32_X_ADD_F32_e32_gfx12 1073758208, $vgpr0, $vgpr2, $vgpr4, $vgpr3, implicit $mode, implicit $exec, implicit $mode, implicit $exec, implicit $mode, implicit $exec + ; GFX12-NEXT: S_ENDPGM 0, implicit $vgpr2, implicit $vgpr5 + $vgpr2 = V_DOT2_F32_F16 8, $vgpr0, 8, 1073758208, 8, $vgpr2, 0, 0, 0, 0, 0, implicit $mode, implicit $exec + $vgpr5 = V_ADD_F32_e32 $vgpr4, $vgpr3, implicit $mode, implicit $exec + S_ENDPGM 0, implicit $vgpr2, implicit $vgpr5 +... + +--- +# dot2 is Y, the second instruction. +name: dot2_imm_src1_y +tracksRegLiveness: true +body: | + bb.0: + liveins: $vgpr0, $vgpr2, $vgpr3, $vgpr4 + ; GFX12-LABEL: name: dot2_imm_src1_y + ; GFX12: liveins: $vgpr0, $vgpr2, $vgpr3, $vgpr4 + ; GFX12-NEXT: {{ $}} + ; GFX12-NEXT: $vgpr5, $vgpr2 = V_DUAL_ADD_F32_e32_X_DOT2C_F32_F16_e32_gfx12 $vgpr4, $vgpr3, 1073758208, $vgpr0, $vgpr2, implicit $mode, implicit $exec, implicit $mode, implicit $exec, implicit $mode, implicit $exec + ; GFX12-NEXT: S_ENDPGM 0, implicit $vgpr2, implicit $vgpr5 + $vgpr5 = V_ADD_F32_e32 $vgpr4, $vgpr3, implicit $mode, implicit $exec + $vgpr2 = V_DOT2_F32_F16 8, $vgpr0, 8, 1073758208, 8, $vgpr2, 0, 0, 0, 0, 0, implicit $mode, implicit $exec + S_ENDPGM 0, implicit $vgpr2, implicit $vgpr5 +... + +--- +# V_ADD_U32 can only be Y, so the second instruction (dot2) becomes X. +name: dot2_imm_src1_second_as_x +tracksRegLiveness: true +body: | + bb.0: + liveins: $vgpr0, $vgpr2, $vgpr3, $vgpr4 + ; GFX12-LABEL: name: dot2_imm_src1_second_as_x + ; GFX12: liveins: $vgpr0, $vgpr2, $vgpr3, $vgpr4 + ; GFX12-NEXT: {{ $}} + ; GFX12-NEXT: $vgpr2, $vgpr5 = V_DUAL_DOT2C_F32_F16_e32_X_ADD_U32_e32_gfx12 1073758208, $vgpr0, $vgpr2, $vgpr4, $vgpr3, implicit $mode, implicit $exec, implicit $mode, implicit $exec, implicit $exec + ; GFX12-NEXT: S_ENDPGM 0, implicit $vgpr2, implicit $vgpr5 + $vgpr5 = V_ADD_U32_e32 $vgpr4, $vgpr3, implicit $exec + $vgpr2 = V_DOT2_F32_F16 8, $vgpr0, 8, 1073758208, 8, $vgpr2, 0, 0, 0, 0, 0, implicit $mode, implicit $exec + S_ENDPGM 0, implicit $vgpr2, implicit $vgpr5 +... + +--- +# Both components have the same immediate in src1, so they share one literal. +name: dot2_imm_src1_both_same_literal +tracksRegLiveness: true +body: | + bb.0: + liveins: $vgpr0, $vgpr2, $vgpr3, $vgpr5 + ; GFX12-LABEL: name: dot2_imm_src1_both_same_literal + ; GFX12: liveins: $vgpr0, $vgpr2, $vgpr3, $vgpr5 + ; GFX12-NEXT: {{ $}} + ; GFX12-NEXT: $vgpr2, $vgpr5 = V_DUAL_DOT2C_F32_F16_e32_X_DOT2C_F32_F16_e32_gfx12 1073758208, $vgpr0, $vgpr2, 1073758208, $vgpr3, $vgpr5, implicit $mode, implicit $exec, implicit $mode, implicit $exec, implicit $mode, implicit $exec + ; GFX12-NEXT: S_ENDPGM 0, implicit $vgpr2, implicit $vgpr5 + $vgpr2 = V_DOT2_F32_F16 8, $vgpr0, 8, 1073758208, 8, $vgpr2, 0, 0, 0, 0, 0, implicit $mode, implicit $exec + $vgpr5 = V_DOT2_F32_F16 8, $vgpr3, 8, 1073758208, 8, $vgpr5, 0, 0, 0, 0, 0, implicit $mode, implicit $exec + S_ENDPGM 0, implicit $vgpr2, implicit $vgpr5 +... + +--- +# Two different literals cannot be encoded in one VOPD. +name: dot2_imm_src1_both_different_literals +tracksRegLiveness: true +body: | + bb.0: + liveins: $vgpr0, $vgpr2, $vgpr3, $vgpr5 + ; GFX12-LABEL: name: dot2_imm_src1_both_different_literals + ; GFX12: liveins: $vgpr0, $vgpr2, $vgpr3, $vgpr5 + ; GFX12-NEXT: {{ $}} + ; GFX12-NEXT: $vgpr2 = V_DOT2_F32_F16 8, $vgpr0, 8, 1073758208, 8, $vgpr2, 0, 0, 0, 0, 0, implicit $mode, implicit $exec + ; GFX12-NEXT: $vgpr5 = V_DOT2_F32_F16 8, $vgpr3, 8, 1073758209, 8, $vgpr5, 0, 0, 0, 0, 0, implicit $mode, implicit $exec + ; GFX12-NEXT: S_ENDPGM 0, implicit $vgpr2, implicit $vgpr5 + $vgpr2 = V_DOT2_F32_F16 8, $vgpr0, 8, 1073758208, 8, $vgpr2, 0, 0, 0, 0, 0, implicit $mode, implicit $exec + $vgpr5 = V_DOT2_F32_F16 8, $vgpr3, 8, 1073758209, 8, $vgpr5, 0, 0, 0, 0, 0, implicit $mode, implicit $exec + S_ENDPGM 0, implicit $vgpr2, implicit $vgpr5 +... + +--- +# Immediates in both src0 and src1: commuting would not put a register in src1. +name: dot2_imm_src0_and_src1 +tracksRegLiveness: true +body: | + bb.0: + liveins: $vgpr2, $vgpr3, $vgpr4 + ; GFX12-LABEL: name: dot2_imm_src0_and_src1 + ; GFX12: liveins: $vgpr2, $vgpr3, $vgpr4 + ; GFX12-NEXT: {{ $}} + ; GFX12-NEXT: $vgpr2 = V_DOT2_F32_F16 8, 1073758208, 8, 1073758208, 8, $vgpr2, 0, 0, 0, 0, 0, implicit $mode, implicit $exec + ; GFX12-NEXT: $vgpr5 = V_ADD_F32_e32 $vgpr4, $vgpr3, implicit $mode, implicit $exec + ; GFX12-NEXT: S_ENDPGM 0, implicit $vgpr2, implicit $vgpr5 + $vgpr2 = V_DOT2_F32_F16 8, 1073758208, 8, 1073758208, 8, $vgpr2, 0, 0, 0, 0, 0, implicit $mode, implicit $exec + $vgpr5 = V_ADD_F32_e32 $vgpr4, $vgpr3, implicit $mode, implicit $exec + S_ENDPGM 0, implicit $vgpr2, implicit $vgpr5 +... + +--- +# Once commuted, src1 of the dot2 is v0, which is in the same bank as src1 of +# the other component (v8), so the pair is rejected. Without the commute only +# src0 would be compared (v0 and v1), which do not conflict. +name: dot2_imm_src1_commuted_bank_conflict +tracksRegLiveness: true +body: | + bb.0: + liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr8 + ; GFX12-LABEL: name: dot2_imm_src1_commuted_bank_conflict + ; GFX12: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr8 + ; GFX12-NEXT: {{ $}} + ; GFX12-NEXT: $vgpr2 = V_DOT2_F32_F16 8, $vgpr0, 8, 1073758208, 8, $vgpr2, 0, 0, 0, 0, 0, implicit $mode, implicit $exec + ; GFX12-NEXT: $vgpr5 = V_ADD_F32_e32 $vgpr1, $vgpr8, implicit $mode, implicit $exec + ; GFX12-NEXT: S_ENDPGM 0, implicit $vgpr2, implicit $vgpr5 + $vgpr2 = V_DOT2_F32_F16 8, $vgpr0, 8, 1073758208, 8, $vgpr2, 0, 0, 0, 0, 0, implicit $mode, implicit $exec + $vgpr5 = V_ADD_F32_e32 $vgpr1, $vgpr8, implicit $mode, implicit $exec + S_ENDPGM 0, implicit $vgpr2, implicit $vgpr5 +... + +--- +# Once commuted, src0 of the dot2 is the literal and src1 is v0, which does not +# conflict with v4 and v6. Without the commute src0 would be v0, which is in the +# same bank as v4. +name: dot2_imm_src1_commuted_no_bank_conflict +tracksRegLiveness: true +body: | + bb.0: + liveins: $vgpr0, $vgpr2, $vgpr4, $vgpr6 + ; GFX12-LABEL: name: dot2_imm_src1_commuted_no_bank_conflict + ; GFX12: liveins: $vgpr0, $vgpr2, $vgpr4, $vgpr6 + ; GFX12-NEXT: {{ $}} + ; GFX12-NEXT: $vgpr2, $vgpr5 = V_DUAL_DOT2C_F32_F16_e32_X_ADD_F32_e32_gfx12 1073758208, $vgpr0, $vgpr2, $vgpr4, $vgpr6, implicit $mode, implicit $exec, implicit $mode, implicit $exec, implicit $mode, implicit $exec + ; GFX12-NEXT: S_ENDPGM 0, implicit $vgpr2, implicit $vgpr5 + $vgpr2 = V_DOT2_F32_F16 8, $vgpr0, 8, 1073758208, 8, $vgpr2, 0, 0, 0, 0, 0, implicit $mode, implicit $exec + $vgpr5 = V_ADD_F32_e32 $vgpr4, $vgpr6, implicit $mode, implicit $exec + S_ENDPGM 0, implicit $vgpr2, implicit $vgpr5 +... >From c048ebc1cb238fae03ac237fbcb6a21396b0137c Mon Sep 17 00:00:00 2001 From: Adel Ejjeh <[email protected]> Date: Thu, 8 Oct 2026 16:00:38 -0500 Subject: [PATCH 2/2] [AMDGPU] Address review comments Co-Authored-By: Claude <[email protected]> --- llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp | 14 +++++++++----- .../CodeGen/AMDGPU/vopd-dot2-commute-imm-src1.mir | 2 +- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp b/llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp index ae0881ae8d819..bc3dafc88e53c 100644 --- a/llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp +++ b/llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp @@ -97,11 +97,15 @@ static bool canMapVOP3PToVOPD(const SIInstrInfo &TII, const MachineInstr &MI, if (getNamedOp(MI, AMDGPU::OpName::src1_modifiers).getImm() != SISrcMods::OP_SEL_1) return false; - if (!getNamedOp(MI, AMDGPU::OpName::src1).isReg()) { - if (!(getNamedOp(MI, AMDGPU::OpName::src0).isReg() && - getNamedOp(MI, AMDGPU::OpName::src1).isImm() && - TII.isLegalToSwap(MI, getNamedOperandIdx(Opc, AMDGPU::OpName::src0), - getNamedOperandIdx(Opc, AMDGPU::OpName::src1)))) + int Src0Idx = getNamedOperandIdx(Opc, AMDGPU::OpName::src0); + int Src1Idx = getNamedOperandIdx(Opc, AMDGPU::OpName::src1); + const MachineOperand &Src0 = MI.getOperand(Src0Idx); + const MachineOperand &Src1 = MI.getOperand(Src1Idx); + // VOPD needs a register in src1. If src1 is an immediate, the dot2 can still + // be used once src0 and src1 are swapped. + if (!Src1.isReg()) { + if (!Src0.isReg() || !Src1.isImm() || + !TII.isLegalToSwap(MI, Src0Idx, Src1Idx)) return false; Commute = true; } diff --git a/llvm/test/CodeGen/AMDGPU/vopd-dot2-commute-imm-src1.mir b/llvm/test/CodeGen/AMDGPU/vopd-dot2-commute-imm-src1.mir index e0402c3fbb20b..d40e564932618 100644 --- a/llvm/test/CodeGen/AMDGPU/vopd-dot2-commute-imm-src1.mir +++ b/llvm/test/CodeGen/AMDGPU/vopd-dot2-commute-imm-src1.mir @@ -1,5 +1,5 @@ # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6 -# RUN: llc -mtriple=amdgpu12.00 -run-pass=gcn-create-vopd -verify-machineinstrs %s -o - | FileCheck -check-prefix=GFX12 %s +# RUN: llc -mtriple=amdgpu12.00 -run-pass=gcn-create-vopd %s -o - | FileCheck -check-prefix=GFX12 %s # A V_DOT2_F32_* can only be a VOPD component with a register in src1. With an # immediate in src1 and a register in src0 the pair is checked as if the dot2 _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
