https://github.com/petar-avramovic updated https://github.com/llvm/llvm-project/pull/179226
>From 81ecd00f7cdf7539ddea8748a3e63259e52294d3 Mon Sep 17 00:00:00 2001 From: Petar Avramovic <[email protected]> Date: Wed, 18 Mar 2026 12:49:49 +0100 Subject: [PATCH] AMDGPU: Codegen for v_dual_dot2acc_f32_f16/bf16 from VOP3 Codegen for v_dual_dot2acc_f32_f16/bf16 for targets that only have VOP3 version of the instruction. Since there is no VOP2 version, instroduce temporary mir DOT2ACC pseudo that is selected when there are no src_modifiers. This DOT2ACC pseudo has src2 tied to dst (like the VOP2 version), PostRA pseudo expansion will restore pseudo to VOP3 version of the instruction. CreateVOPD will recoginize such VOP3 pseudo and generate v_dual_dot2acc. --- llvm/lib/Target/AMDGPU/AMDGPU.td | 3 + llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp | 17 +- llvm/lib/Target/AMDGPU/SIFoldOperands.cpp | 27 + llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 10 + .../Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp | 4 + llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h | 29 +- llvm/lib/Target/AMDGPU/VOP3PInstructions.td | 35 +- llvm/lib/Target/AMDGPU/VOPInstructions.td | 4 +- .../AMDGPU/llvm.amdgcn.fdot2.f32.bf16.ll | 191 +++---- llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fdot2.ll | 482 ++++++------------ 10 files changed, 380 insertions(+), 422 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.td b/llvm/lib/Target/AMDGPU/AMDGPU.td index 616effeb5b9f2..0b0a72cae901d 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPU.td +++ b/llvm/lib/Target/AMDGPU/AMDGPU.td @@ -2706,6 +2706,9 @@ def isWave32Strict : Predicate<"Subtarget->isWave32()">, def isWave64Strict : Predicate<"Subtarget->isWave64()">, AssemblerPredicate <(all_of FeatureWavefrontSize64)>; +def HasOnlyDualDot2AccF32F16 : Predicate<"Subtarget->hasVOPDInsts() && Subtarget->hasDot10Insts() && !Subtarget->hasDot5Insts()">; +def HasOnlyDualDot2AccF32BF16 : Predicate<"Subtarget->hasVOPDInsts() && Subtarget->hasDot12Insts() && !Subtarget->hasDot13Insts()">; + //===----------------------------------------------------------------------===// // HwModes //===----------------------------------------------------------------------===// diff --git a/llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp b/llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp index b17cabf37d53f..e2b7c23803bcb 100644 --- a/llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp +++ b/llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp @@ -34,6 +34,20 @@ using namespace llvm; #define DEBUG_TYPE "gcn-vopd-utils" +static bool isVOPDDot(const MachineInstr &MI) { + unsigned Opc = MI.getOpcode(); + if (Opc != AMDGPU::V_DOT2_F32_F16 && Opc != AMDGPU::V_DOT2_F32_BF16) + return false; + // src0 can be register or literal + return MI.getOperand(1).getImm() == SISrcMods::OP_SEL_1 && // default src0mods + MI.getOperand(3).getImm() == SISrcMods::OP_SEL_1 && // default src1mods + MI.getOperand(4).isReg() && // register src1 + MI.getOperand(5).getImm() == SISrcMods::OP_SEL_1 && // default src2mods + MI.getOperand(6).isReg() && // register src2 + MI.getOperand(7).getImm() == 0 && // no clamp + MI.getOperand(0).getReg() == MI.getOperand(6).getReg(); // dst == src2 +} + bool llvm::checkVOPDRegConstraints(const SIInstrInfo &TII, const MachineInstr &MIX, const MachineInstr &MIY, bool IsVOPD3) { @@ -44,7 +58,8 @@ bool llvm::checkVOPDRegConstraints(const SIInstrInfo &TII, if (IsVOPD3 && !ST.hasVOPD3()) return false; - if (!IsVOPD3 && (TII.isVOP3(MIX) || TII.isVOP3(MIY))) + if (!IsVOPD3 && ((TII.isVOP3(MIX) && !isVOPDDot(MIX)) || + (TII.isVOP3(MIY) && !isVOPDDot(MIY)))) return false; if (TII.isDPP(MIX) || TII.isDPP(MIY)) return false; diff --git a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp index a2fe31bd849c3..0af4921040aad 100644 --- a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp +++ b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp @@ -827,6 +827,19 @@ bool SIFoldOperandsImpl::tryAddToFoldList( return false; }; + // For V_DOT2ACC pseudos, prefer folding literals into src0 (operand 2) for + // VOPD compatibility. If the literal would go into src1 (operand 4), commute + // src0 and src1 (including their modifiers) so the literal ends up in src0. + if (OpToFold.isImm() && OpNo == 4 && + (Opc == AMDGPU::V_DOT2ACC_F32_F16_PSEUDO || + Opc == AMDGPU::V_DOT2ACC_F32_BF16_PSEUDO)) { + int64_t SavedMod = MI->getOperand(1).getImm(); + MI->getOperand(1).setImm(MI->getOperand(3).getImm()); + MI->getOperand(3).setImm(SavedMod); + MI->getOperand(4).setReg(MI->getOperand(2).getReg()); + OpNo = 2; + } + bool IsLegal = OpToFold.isOperandLegal(*TII, *MI, OpNo); if (!IsLegal && OpToFold.isImm()) { if (std::optional<int64_t> ImmVal = OpToFold.getEffectiveImmVal()) @@ -1239,6 +1252,20 @@ void SIFoldOperandsImpl::foldOperand( return; } + // If a constant is folded into src2 of a V_DOT2ACC pseudo, convert it to + // V_DOT2_F32_F16/BF16. Folding the constant (ruling out VOPD) is preferable + // to leaving it unfolded to enable a potential VOPD pairing. + if (OpToFold.isImm() && UseOpIdx == 6) { + unsigned Opc = UseMI->getOpcode(); + if (Opc == AMDGPU::V_DOT2ACC_F32_F16_PSEUDO || + Opc == AMDGPU::V_DOT2ACC_F32_BF16_PSEUDO) { + UseMI->setDesc(TII->get(Opc == AMDGPU::V_DOT2ACC_F32_F16_PSEUDO + ? AMDGPU::V_DOT2_F32_F16 + : AMDGPU::V_DOT2_F32_BF16)); + UseMI->untieRegOperand(UseOpIdx); + } + } + if (tryToFoldACImm(OpToFold, UseMI, UseOpIdx, FoldList)) return; diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp index 45b8a49392e09..1bfdd7fbd7d85 100644 --- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp +++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp @@ -2069,6 +2069,16 @@ bool SIInstrInfo::expandPostRAPseudo(MachineInstr &MI) const { const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST); switch (MI.getOpcode()) { default: return TargetInstrInfo::expandPostRAPseudo(MI); + case AMDGPU::V_DOT2ACC_F32_F16_PSEUDO: + MI.setDesc(get(AMDGPU::V_DOT2_F32_F16)); + MI.untieRegOperand(6); + break; + + case AMDGPU::V_DOT2ACC_F32_BF16_PSEUDO: + MI.setDesc(get(AMDGPU::V_DOT2_F32_BF16)); + MI.untieRegOperand(6); + break; + case AMDGPU::S_MOV_B64_term: // This is only a terminator to get the correct spill code placement during // register allocation. diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp index dc56d746e1a8e..3a9e9dd462b56 100644 --- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp +++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp @@ -686,6 +686,10 @@ bool isVOPD(unsigned Opc) { return AMDGPU::hasNamedOperand(Opc, AMDGPU::OpName::src0X); } +bool isVOP3PDot2(unsigned Opc) { + return Opc == AMDGPU::V_DOT2_F32_F16 || Opc == AMDGPU::V_DOT2_F32_BF16; +} + bool isMAC(unsigned Opc) { return Opc == AMDGPU::V_MAC_F32_e64_gfx6_gfx7 || Opc == AMDGPU::V_MAC_F32_e64_gfx10 || diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h index 2032c3dbb3a86..4a0f247a088f9 100644 --- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h +++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h @@ -672,6 +672,9 @@ int getVOPDFull(unsigned OpX, unsigned OpY, unsigned EncodingFamily, LLVM_READONLY bool isVOPD(unsigned Opc); +LLVM_READONLY +bool isVOP3PDot2(unsigned Opc); + LLVM_READNONE bool isMAC(unsigned Opc); @@ -837,20 +840,24 @@ class ComponentLayout { const ComponentKind Kind; const ComponentProps PrevComp; const unsigned VOPD3ModsNum; - const int BitOp3Idx; // Index of bitop3 operand or -1 + const int BitOp3Idx; // Index of bitop3 operand or -1 + const bool IsVOP3PDot2; // True for V_DOT2_F32_F16 / V_DOT2_F32_BF16 public: // Create layout for COMPONENT_X or SINGLE component. - ComponentLayout(ComponentKind Kind, unsigned VOPD3ModsNum, int BitOp3Idx) - : Kind(Kind), VOPD3ModsNum(VOPD3ModsNum), BitOp3Idx(BitOp3Idx) { + ComponentLayout(ComponentKind Kind, unsigned VOPD3ModsNum, int BitOp3Idx, + bool IsVOP3PDot2 = false) + : Kind(Kind), VOPD3ModsNum(VOPD3ModsNum), BitOp3Idx(BitOp3Idx), + IsVOP3PDot2(IsVOP3PDot2) { assert(Kind == ComponentKind::SINGLE || Kind == ComponentKind::COMPONENT_X); } // Create layout for COMPONENT_Y which depends on COMPONENT_X layout. ComponentLayout(const ComponentProps &OpXProps, unsigned VOPD3ModsNum, - int BitOp3Idx) + int BitOp3Idx, bool IsVOP3PDot2 = false) : Kind(ComponentKind::COMPONENT_Y), PrevComp(OpXProps), - VOPD3ModsNum(VOPD3ModsNum), BitOp3Idx(BitOp3Idx) {} + VOPD3ModsNum(VOPD3ModsNum), BitOp3Idx(BitOp3Idx), + IsVOP3PDot2(IsVOP3PDot2) {} public: // Return the index of dst operand in MCInst operands. @@ -863,6 +870,11 @@ class ComponentLayout { if (Kind == SINGLE && CompSrcIdx == 2 && BitOp3Idx != -1) return BitOp3Idx; + if (IsVOP3PDot2) { + return SINGLE_MC_SRC_IDX[3][CompSrcIdx] + getPrevCompSrcNum() + + (Kind != SINGLE ? 1 : 0); + } + if (VOPD3) { return SINGLE_MC_SRC_IDX[VOPD3ModsNum][CompSrcIdx] + getPrevCompSrcNum() + getPrevCompVOPD3ModsNum() + (Kind != SINGLE ? 1 : 0); @@ -903,14 +915,15 @@ class ComponentInfo : public ComponentProps, public ComponentLayout { ComponentKind Kind = ComponentKind::SINGLE, bool VOP3Layout = false) : ComponentProps(OpDesc, VOP3Layout), - ComponentLayout(Kind, getCompVOPD3ModsNum(), getBitOp3OperandIdx()) {} + ComponentLayout(Kind, getCompVOPD3ModsNum(), getBitOp3OperandIdx(), + isVOP3PDot2(OpDesc.Opcode)) {} // Create ComponentInfo for COMPONENT_Y which depends on COMPONENT_X layout. ComponentInfo(const MCInstrDesc &OpDesc, const ComponentProps &OpXProps, bool VOP3Layout = false) : ComponentProps(OpDesc, VOP3Layout), - ComponentLayout(OpXProps, getCompVOPD3ModsNum(), - getBitOp3OperandIdx()) {} + ComponentLayout(OpXProps, getCompVOPD3ModsNum(), getBitOp3OperandIdx(), + isVOP3PDot2(OpDesc.Opcode)) {} // Map component operand index to parsed operand index. // Return 0 if the specified operand does not exist. diff --git a/llvm/lib/Target/AMDGPU/VOP3PInstructions.td b/llvm/lib/Target/AMDGPU/VOP3PInstructions.td index cc08710a299a3..1ff9870e24e7c 100644 --- a/llvm/lib/Target/AMDGPU/VOP3PInstructions.td +++ b/llvm/lib/Target/AMDGPU/VOP3PInstructions.td @@ -87,11 +87,13 @@ multiclass VOP3PInst<string OpName, VOPProfile P, } multiclass VOP3PInstDotWithDual<string OpName, VOPProfile P, - SDPatternOperator node = null_frag> { + SDPatternOperator node = null_frag, + bits<6> VOPDOp, string VOPDName> { def NAME : VOP3P_Pseudo<OpName, P, getVOP3PModPat<P, node, 1 /*HasExplicitClamp*/, 1/*IsDOT*/, - VOP3PModsDOT, VOP3PModsF32>.ret>; + VOP3PModsDOT, VOP3PModsF32>.ret>, + VOPD_Component<VOPDOp, VOPDName>; let SubtargetPredicate = isGFX11Plus in { if P.HasExtVOP3DPP then def _dpp : VOP3_DPP_Pseudo<OpName, P> { @@ -617,7 +619,7 @@ defm V_DOT2_F32_F16 : VOP3PInstDotWithDual<"v_dot2_f32_f16", VOP3P_Profile<VOP_F32_V2F16_V2F16_F32, VOP3_REGULAR, /*HasDPP*/ 1>, - AMDGPUfdot2>; + AMDGPUfdot2, 0xC, "v_dot2acc_f32_f16">; let OtherPredicates = [HasDot7Insts] in { defm V_DOT4_U32_U8 : VOP3PInst<"v_dot4_u32_u8", @@ -642,12 +644,37 @@ let SubtargetPredicate = HasDot12Insts in { defm V_DOT2_F32_BF16 : VOP3PInstDotWithDual<"v_dot2_f32_bf16", DOT2_BF16_Profile, - int_amdgcn_fdot2_f32_bf16>; + int_amdgcn_fdot2_f32_bf16, 0xD, "v_dot2acc_f32_bf16">; } // End SubtargetPredicate = HasDot12Insts } // End let IsDOT = 1 +let IsDOT = 1, isCommutable = 1, Constraints = "$vdst = $src2" in { +let OtherPredicates = [HasOnlyDualDot2AccF32F16] in +def V_DOT2ACC_F32_F16_PSEUDO + : VOP3P_Pseudo<"", VOP3P_Profile<VOP_F32_V2F16_V2F16_F32, VOP3_REGULAR>>; + +let OtherPredicates = [HasOnlyDualDot2AccF32BF16] in +def V_DOT2ACC_F32_BF16_PSEUDO : + VOP3P_Pseudo<"", VOP3P_Profile<VOP_F32_V2BF16_V2BF16_F32, VOP3_REGULAR>>; +} + +class Dot2AccPseudo_Pat <SDPatternOperator node, Instruction inst, ValueType ty> + : GCNPat < + (f32 (node (ty (VOP3PNoModsDOT ty:$src0)), (ty (VOP3PNoModsDOT ty:$src1)), + (f32 (VOP3PNoModsF32 f32:$src2)), (i1 DSTCLAMP.NONE))), + (f32 (inst (i32 SRCMODS.OP_SEL_1), $src0, (i32 SRCMODS.OP_SEL_1), $src1, + (i32 SRCMODS.OP_SEL_1), $src2)) +>; + +let SubtargetPredicate = HasOnlyDualDot2AccF32F16 in +def : Dot2AccPseudo_Pat<AMDGPUfdot2, V_DOT2ACC_F32_F16_PSEUDO, v2f16>; + +let SubtargetPredicate = HasOnlyDualDot2AccF32BF16 in +def : Dot2AccPseudo_Pat<int_amdgcn_fdot2_f32_bf16, V_DOT2ACC_F32_BF16_PSEUDO, + v2bf16>; + multiclass VOP3PDOTIUInst <string OpName, SDPatternOperator intrinsic_node> { let IsDOT = 1 in defm NAME : VOP3PInst<OpName, VOP3P_Profile<VOP_I32_I32_I32_I32, VOP3_PACKED_NO_OPSEL>, diff --git a/llvm/lib/Target/AMDGPU/VOPInstructions.td b/llvm/lib/Target/AMDGPU/VOPInstructions.td index 82545a472cf17..40b0476a84d25 100644 --- a/llvm/lib/Target/AMDGPU/VOPInstructions.td +++ b/llvm/lib/Target/AMDGPU/VOPInstructions.td @@ -34,8 +34,8 @@ class VOP <string opName> { string OpName = opName; } -// First 13 insts from VOPDY are also VOPDX. DOT2ACC_F32_BF16 is omitted -defvar VOPDX_Max_Index = 12; +// First 13 insts from VOPDY are also VOPDX. +defvar VOPDX_Max_Index = 13; defvar VOPD3X_Max_Index = 36; class VOPD_Component<bits<6> OpIn, string vOPDName> { diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fdot2.f32.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fdot2.f32.bf16.ll index e03b57cac6ab2..163874e447719 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fdot2.f32.bf16.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fdot2.f32.bf16.ll @@ -13,7 +13,8 @@ define float @v_fdot2_f32_bf16(<2 x bfloat> %a, <2 x bfloat> %b, float %c) { ; ; GFX11PLUS-LABEL: v_fdot2_f32_bf16: ; GFX11PLUS: ; %bb.0: -; GFX11PLUS: v_dot2_f32_bf16 v0, v0, v1, v2 +; GFX11PLUS: v_dot2_f32_bf16 v2, v0, v1, v2 +; GFX11PLUS: v_mov_b32_e32 v0, v2 %r = call float @llvm.amdgcn.fdot2.f32.bf16(<2 x bfloat> %a, <2 x bfloat> %b, float %c, i1 false) ret float %r } @@ -39,13 +40,15 @@ define float @v_fdot2_f32_bf16_neg_a_lo(<2 x bfloat> %a, <2 x bfloat> %b, float ; GFX11-LABEL: v_fdot2_f32_bf16_neg_a_lo: ; GFX11: ; %bb.0: ; GFX11: v_xor_b16 v0.l, 0x8000, v0.l -; GFX11: v_dot2_f32_bf16 v0, v0, v1, v2 +; GFX11: v_dot2_f32_bf16 v2, v0, v1, v2 +; GFX11: v_mov_b32_e32 v0, v2 ; ; GFX12-LABEL: v_fdot2_f32_bf16_neg_a_lo: ; GFX12: ; %bb.0: ; GFX12: v_xor_b32_e32 v3, 0x8000, v0 ; GFX12: v_bfi_b32 v0, 0xffff, v3, v0 -; GFX12: v_dot2_f32_bf16 v0, v0, v1, v2 +; GFX12: v_dot2_f32_bf16 v2, v0, v1, v2 +; GFX12: v_mov_b32_e32 v0, v2 %a_lo = extractelement <2 x bfloat> %a, i32 0 %neg.a_lo = fneg bfloat %a_lo %neg_lo.a = insertelement <2 x bfloat> %a, bfloat %neg.a_lo, i32 0 @@ -66,14 +69,16 @@ define float @v_fdot2_f32_bf16_neg_a_hi(<2 x bfloat> %a, <2 x bfloat> %b, float ; GFX11-LABEL: v_fdot2_f32_bf16_neg_a_hi: ; GFX11: ; %bb.0: ; GFX11: v_xor_b16 v0.h, 0x8000, v0.h -; GFX11: v_dot2_f32_bf16 v0, v0, v1, v2 +; GFX11: v_dot2_f32_bf16 v2, v0, v1, v2 +; GFX11: v_mov_b32_e32 v0, v2 ; ; GFX12-LABEL: v_fdot2_f32_bf16_neg_a_hi: ; GFX12: ; %bb.0: ; GFX12: v_lshrrev_b32_e32 v3, 16, v0 ; GFX12: v_xor_b32_e32 v3, 0x8000, v3 ; GFX12: v_perm_b32 v0, v3, v0, 0x5040100 -; GFX12: v_dot2_f32_bf16 v0, v0, v1, v2 +; GFX12: v_dot2_f32_bf16 v2, v0, v1, v2 +; GFX12: v_mov_b32_e32 v0, v2 %a_hi = extractelement <2 x bfloat> %a, i32 1 %neg.a_hi = fneg bfloat %a_hi %neg_hi.a = insertelement <2 x bfloat> %a, bfloat %neg.a_hi, i32 1 @@ -102,13 +107,15 @@ define float @v_fdot2_f32_bf16_neg_b_lo(<2 x bfloat> %a, <2 x bfloat> %b, float ; GFX11-LABEL: v_fdot2_f32_bf16_neg_b_lo: ; GFX11: ; %bb.0: ; GFX11: v_xor_b16 v1.l, 0x8000, v1.l -; GFX11: v_dot2_f32_bf16 v0, v0, v1, v2 +; GFX11: v_dot2_f32_bf16 v2, v0, v1, v2 +; GFX11: v_mov_b32_e32 v0, v2 ; ; GFX12-LABEL: v_fdot2_f32_bf16_neg_b_lo: ; GFX12: ; %bb.0: ; GFX12: v_xor_b32_e32 v3, 0x8000, v1 ; GFX12: v_bfi_b32 v1, 0xffff, v3, v1 -; GFX12: v_dot2_f32_bf16 v0, v0, v1, v2 +; GFX12: v_dot2_f32_bf16 v2, v0, v1, v2 +; GFX12: v_mov_b32_e32 v0, v2 %b_lo = extractelement <2 x bfloat> %b, i32 0 %neg.b_lo = fneg bfloat %b_lo %neg_lo.b = insertelement <2 x bfloat> %b, bfloat %neg.b_lo, i32 0 @@ -129,14 +136,16 @@ define float @v_fdot2_f32_bf16_neg_b_hi(<2 x bfloat> %a, <2 x bfloat> %b, float ; GFX11-LABEL: v_fdot2_f32_bf16_neg_b_hi: ; GFX11: ; %bb.0: ; GFX11: v_xor_b16 v1.h, 0x8000, v1.h -; GFX11: v_dot2_f32_bf16 v0, v0, v1, v2 +; GFX11: v_dot2_f32_bf16 v2, v0, v1, v2 +; GFX11: v_mov_b32_e32 v0, v2 ; ; GFX12-LABEL: v_fdot2_f32_bf16_neg_b_hi: ; GFX12: ; %bb.0: ; GFX12: v_lshrrev_b32_e32 v3, 16, v1 ; GFX12: v_xor_b32_e32 v3, 0x8000, v3 ; GFX12: v_perm_b32 v1, v3, v1, 0x5040100 -; GFX12: v_dot2_f32_bf16 v0, v0, v1, v2 +; GFX12: v_dot2_f32_bf16 v2, v0, v1, v2 +; GFX12: v_mov_b32_e32 v0, v2 %b_hi = extractelement <2 x bfloat> %b, i32 1 %neg.b_hi = fneg bfloat %b_hi %neg_hi.b = insertelement <2 x bfloat> %b, bfloat %neg.b_hi, i32 1 @@ -173,12 +182,14 @@ define float @v_fdot2_f32_bf16_opsel_lo_a(<2 x bfloat> %a, <2 x bfloat> %b, floa ; GFX11-LABEL: v_fdot2_f32_bf16_opsel_lo_a: ; GFX11: ; %bb.0: ; GFX11: v_mov_b16_e32 v0.l, v0.h -; GFX11: v_dot2_f32_bf16 v0, v0, v1, v2 +; GFX11: v_dot2_f32_bf16 v2, v0, v1, v2 +; GFX11: v_mov_b32_e32 v0, v2 ; ; GFX12-LABEL: v_fdot2_f32_bf16_opsel_lo_a: ; GFX12: ; %bb.0: ; GFX12: v_perm_b32 v0, v0, v0, 0x7060302 -; GFX12: v_dot2_f32_bf16 v0, v0, v1, v2 +; GFX12: v_dot2_f32_bf16 v2, v0, v1, v2 +; GFX12: v_mov_b32_e32 v0, v2 %shuf = shufflevector <2 x bfloat> %a, <2 x bfloat> poison, <2 x i32> <i32 1, i32 1> %r = call float @llvm.amdgcn.fdot2.f32.bf16(<2 x bfloat> %shuf, <2 x bfloat> %b, float %c, i1 false) ret float %r @@ -195,12 +206,14 @@ define float @v_fdot2_f32_bf16_opsel_hi_a(<2 x bfloat> %a, <2 x bfloat> %b, floa ; GFX11-LABEL: v_fdot2_f32_bf16_opsel_hi_a: ; GFX11: ; %bb.0: ; GFX11: v_mov_b16_e32 v0.h, v0.l -; GFX11: v_dot2_f32_bf16 v0, v0, v1, v2 +; GFX11: v_dot2_f32_bf16 v2, v0, v1, v2 +; GFX11: v_mov_b32_e32 v0, v2 ; ; GFX12-LABEL: v_fdot2_f32_bf16_opsel_hi_a: ; GFX12: ; %bb.0: ; GFX12: v_perm_b32 v0, v0, v0, 0x5040100 -; GFX12: v_dot2_f32_bf16 v0, v0, v1, v2 +; GFX12: v_dot2_f32_bf16 v2, v0, v1, v2 +; GFX12: v_mov_b32_e32 v0, v2 %shuf = shufflevector <2 x bfloat> %a, <2 x bfloat> poison, <2 x i32> <i32 0, i32 0> %r = call float @llvm.amdgcn.fdot2.f32.bf16(<2 x bfloat> %shuf, <2 x bfloat> %b, float %c, i1 false) ret float %r @@ -217,12 +230,14 @@ define float @v_fdot2_f32_bf16_opsel_lo_b(<2 x bfloat> %a, <2 x bfloat> %b, floa ; GFX11-LABEL: v_fdot2_f32_bf16_opsel_lo_b: ; GFX11: ; %bb.0: ; GFX11: v_mov_b16_e32 v1.l, v1.h -; GFX11: v_dot2_f32_bf16 v0, v0, v1, v2 +; GFX11: v_dot2_f32_bf16 v2, v0, v1, v2 +; GFX11: v_mov_b32_e32 v0, v2 ; ; GFX12-LABEL: v_fdot2_f32_bf16_opsel_lo_b: ; GFX12: ; %bb.0: ; GFX12: v_perm_b32 v1, v1, v1, 0x7060302 -; GFX12: v_dot2_f32_bf16 v0, v0, v1, v2 +; GFX12: v_dot2_f32_bf16 v2, v0, v1, v2 +; GFX12: v_mov_b32_e32 v0, v2 %shuf = shufflevector <2 x bfloat> %b, <2 x bfloat> poison, <2 x i32> <i32 1, i32 1> %r = call float @llvm.amdgcn.fdot2.f32.bf16(<2 x bfloat> %a, <2 x bfloat> %shuf, float %c, i1 false) ret float %r @@ -239,12 +254,14 @@ define float @v_fdot2_f32_bf16_opsel_hi_b(<2 x bfloat> %a, <2 x bfloat> %b, floa ; GFX11-LABEL: v_fdot2_f32_bf16_opsel_hi_b: ; GFX11: ; %bb.0: ; GFX11: v_mov_b16_e32 v1.h, v1.l -; GFX11: v_dot2_f32_bf16 v0, v0, v1, v2 +; GFX11: v_dot2_f32_bf16 v2, v0, v1, v2 +; GFX11: v_mov_b32_e32 v0, v2 ; ; GFX12-LABEL: v_fdot2_f32_bf16_opsel_hi_b: ; GFX12: ; %bb.0: ; GFX12: v_perm_b32 v1, v1, v1, 0x5040100 -; GFX12: v_dot2_f32_bf16 v0, v0, v1, v2 +; GFX12: v_dot2_f32_bf16 v2, v0, v1, v2 +; GFX12: v_mov_b32_e32 v0, v2 %shuf = shufflevector <2 x bfloat> %b, <2 x bfloat> poison, <2 x i32> <i32 0, i32 0> %r = call float @llvm.amdgcn.fdot2.f32.bf16(<2 x bfloat> %a, <2 x bfloat> %shuf, float %c, i1 false) ret float %r @@ -258,7 +275,8 @@ define float @v_fdot2_f32_bf16_inline_literal_a(<2 x bfloat> %b, float %c) { ; ; GFX11PLUS-LABEL: v_fdot2_f32_bf16_inline_literal_a: ; GFX11PLUS: ; %bb.0: -; GFX11PLUS: v_dot2_f32_bf16 v0, 0x3f003f00, v0, v1 +; GFX11PLUS: v_dot2_f32_bf16 v1, 0x3f003f00, v0, v1 +; GFX11PLUS: v_mov_b32_e32 v0, v1 %ret = tail call float @llvm.amdgcn.fdot2.f32.bf16(<2 x bfloat> <bfloat 0.5, bfloat 0.5>, <2 x bfloat> %b, float %c, i1 false) ret float %ret } @@ -271,7 +289,8 @@ define float @v_fdot2_f32_bf16_inline_literal_b(<2 x bfloat> %a, float %c) { ; ; GFX11PLUS-LABEL: v_fdot2_f32_bf16_inline_literal_b: ; GFX11PLUS: ; %bb.0: -; GFX11PLUS: v_dot2_f32_bf16 v0, v0, 0x40004000, v1 +; GFX11PLUS: v_dot2_f32_bf16 v1, 0x40004000, v0, v1 +; GFX11PLUS: v_mov_b32_e32 v0, v1 %ret = tail call float @llvm.amdgcn.fdot2.f32.bf16(<2 x bfloat> %a, <2 x bfloat> <bfloat 2.0, bfloat 2.0>, float %c, i1 false) ret float %ret } @@ -461,9 +480,8 @@ define float @v_fdot2_f32_bf16_dual(<2 x bfloat> %a, <2 x bfloat> %b, float %c, ; ; GFX11PLUS-LABEL: v_fdot2_f32_bf16_dual: ; GFX11PLUS: ; %bb.0: -; GFX11PLUS: v_dot2_f32_bf16 v0, v0, v1, v2 -; GFX11PLUS: v_dot2_f32_bf16 v1, v3, v4, v5 -; GFX11PLUS: v_add_f32_e32 v0, v0, v1 +; GFX11PLUS: v_dual_dot2acc_f32_bf16 v2, v0, v1 :: v_dual_dot2acc_f32_bf16 v5, v3, v4 +; GFX11PLUS: v_add_f32_e32 v0, v2, v5 %r0 = call float @llvm.amdgcn.fdot2.f32.bf16(<2 x bfloat> %a, <2 x bfloat> %b, float %c, i1 false) %r1 = call float @llvm.amdgcn.fdot2.f32.bf16(<2 x bfloat> %d, <2 x bfloat> %e, float %f, i1 false) %r = fadd float %r0, %r1 @@ -480,8 +498,8 @@ define float @v_fdot2_f32_bf16_neg_a_dual(<2 x bfloat> %a, <2 x bfloat> %b, floa ; GFX11PLUS-LABEL: v_fdot2_f32_bf16_neg_a_dual: ; GFX11PLUS: ; %bb.0: ; GFX11PLUS: v_dot2_f32_bf16 v0, v0, v1, v2 neg_lo:[1,0,0] neg_hi:[1,0,0] -; GFX11PLUS: v_dot2_f32_bf16 v1, v3, v4, v5 -; GFX11PLUS: v_add_f32_e32 v0, v0, v1 +; GFX11PLUS: v_dot2_f32_bf16 v5, v3, v4, v5 +; GFX11PLUS: v_add_f32_e32 v0, v0, v5 %neg.a = fneg <2 x bfloat> %a %r0 = call float @llvm.amdgcn.fdot2.f32.bf16(<2 x bfloat> %neg.a, <2 x bfloat> %b, float %c, i1 false) %r1 = call float @llvm.amdgcn.fdot2.f32.bf16(<2 x bfloat> %d, <2 x bfloat> %e, float %f, i1 false) @@ -502,17 +520,16 @@ define float @v_fdot2_f32_bf16_neg_a_lo_dual(<2 x bfloat> %a, <2 x bfloat> %b, f ; GFX11-LABEL: v_fdot2_f32_bf16_neg_a_lo_dual: ; GFX11: ; %bb.0: ; GFX11: v_xor_b16 v0.l, 0x8000, v0.l -; GFX11: v_dot2_f32_bf16 v0, v0, v1, v2 -; GFX11: v_dot2_f32_bf16 v1, v3, v4, v5 -; GFX11: v_add_f32_e32 v0, v0, v1 +; GFX11: v_dual_dot2acc_f32_bf16 v5, v3, v4 :: v_dual_dot2acc_f32_bf16 v2, v0, v1 +; GFX11: v_add_f32_e32 v0, v2, v5 ; ; GFX12-LABEL: v_fdot2_f32_bf16_neg_a_lo_dual: ; GFX12: ; %bb.0: ; GFX12: v_xor_b32_e32 v6, 0x8000, v0 +; GFX12: v_dot2_f32_bf16 v5, v3, v4, v5 ; GFX12: v_bfi_b32 v0, 0xffff, v6, v0 -; GFX12: v_dot2_f32_bf16 v0, v0, v1, v2 -; GFX12: v_dot2_f32_bf16 v1, v3, v4, v5 -; GFX12: v_add_f32_e32 v0, v0, v1 +; GFX12: v_dot2_f32_bf16 v2, v0, v1, v2 +; GFX12: v_add_f32_e32 v0, v2, v5 %a_lo = extractelement <2 x bfloat> %a, i32 0 %neg.a_lo = fneg bfloat %a_lo %neg_lo.a = insertelement <2 x bfloat> %a, bfloat %neg.a_lo, i32 0 @@ -536,18 +553,17 @@ define float @v_fdot2_f32_bf16_neg_a_hi_dual(<2 x bfloat> %a, <2 x bfloat> %b, f ; GFX11-LABEL: v_fdot2_f32_bf16_neg_a_hi_dual: ; GFX11: ; %bb.0: ; GFX11: v_xor_b16 v0.h, 0x8000, v0.h -; GFX11: v_dot2_f32_bf16 v0, v0, v1, v2 -; GFX11: v_dot2_f32_bf16 v1, v3, v4, v5 -; GFX11: v_add_f32_e32 v0, v0, v1 +; GFX11: v_dual_dot2acc_f32_bf16 v5, v3, v4 :: v_dual_dot2acc_f32_bf16 v2, v0, v1 +; GFX11: v_add_f32_e32 v0, v2, v5 ; ; GFX12-LABEL: v_fdot2_f32_bf16_neg_a_hi_dual: ; GFX12: ; %bb.0: ; GFX12: v_lshrrev_b32_e32 v6, 16, v0 +; GFX12: v_dot2_f32_bf16 v5, v3, v4, v5 ; GFX12: v_xor_b32_e32 v6, 0x8000, v6 ; GFX12: v_perm_b32 v0, v6, v0, 0x5040100 -; GFX12: v_dot2_f32_bf16 v0, v0, v1, v2 -; GFX12: v_dot2_f32_bf16 v1, v3, v4, v5 -; GFX12: v_add_f32_e32 v0, v0, v1 +; GFX12: v_dot2_f32_bf16 v2, v0, v1, v2 +; GFX12: v_add_f32_e32 v0, v2, v5 %a_hi = extractelement <2 x bfloat> %a, i32 1 %neg.a_hi = fneg bfloat %a_hi %neg_hi.a = insertelement <2 x bfloat> %a, bfloat %neg.a_hi, i32 1 @@ -567,8 +583,8 @@ define float @v_fdot2_f32_bf16_neg_b_dual(<2 x bfloat> %a, <2 x bfloat> %b, floa ; GFX11PLUS-LABEL: v_fdot2_f32_bf16_neg_b_dual: ; GFX11PLUS: ; %bb.0: ; GFX11PLUS: v_dot2_f32_bf16 v0, v0, v1, v2 neg_lo:[0,1,0] neg_hi:[0,1,0] -; GFX11PLUS: v_dot2_f32_bf16 v1, v3, v4, v5 -; GFX11PLUS: v_add_f32_e32 v0, v0, v1 +; GFX11PLUS: v_dot2_f32_bf16 v5, v3, v4, v5 +; GFX11PLUS: v_add_f32_e32 v0, v0, v5 %neg.b = fneg <2 x bfloat> %b %r0 = call float @llvm.amdgcn.fdot2.f32.bf16(<2 x bfloat> %a, <2 x bfloat> %neg.b, float %c, i1 false) %r1 = call float @llvm.amdgcn.fdot2.f32.bf16(<2 x bfloat> %d, <2 x bfloat> %e, float %f, i1 false) @@ -589,17 +605,16 @@ define float @v_fdot2_f32_bf16_neg_b_lo_dual(<2 x bfloat> %a, <2 x bfloat> %b, f ; GFX11-LABEL: v_fdot2_f32_bf16_neg_b_lo_dual: ; GFX11: ; %bb.0: ; GFX11: v_xor_b16 v1.l, 0x8000, v1.l -; GFX11: v_dot2_f32_bf16 v0, v0, v1, v2 -; GFX11: v_dot2_f32_bf16 v1, v3, v4, v5 -; GFX11: v_add_f32_e32 v0, v0, v1 +; GFX11: v_dual_dot2acc_f32_bf16 v5, v3, v4 :: v_dual_dot2acc_f32_bf16 v2, v0, v1 +; GFX11: v_add_f32_e32 v0, v2, v5 ; ; GFX12-LABEL: v_fdot2_f32_bf16_neg_b_lo_dual: ; GFX12: ; %bb.0: ; GFX12: v_xor_b32_e32 v6, 0x8000, v1 +; GFX12: v_dot2_f32_bf16 v5, v3, v4, v5 ; GFX12: v_bfi_b32 v1, 0xffff, v6, v1 -; GFX12: v_dot2_f32_bf16 v0, v0, v1, v2 -; GFX12: v_dot2_f32_bf16 v1, v3, v4, v5 -; GFX12: v_add_f32_e32 v0, v0, v1 +; GFX12: v_dot2_f32_bf16 v2, v0, v1, v2 +; GFX12: v_add_f32_e32 v0, v2, v5 %b_lo = extractelement <2 x bfloat> %b, i32 0 %neg.b_lo = fneg bfloat %b_lo %neg_lo.b = insertelement <2 x bfloat> %b, bfloat %neg.b_lo, i32 0 @@ -623,18 +638,17 @@ define float @v_fdot2_f32_bf16_neg_b_hi_dual(<2 x bfloat> %a, <2 x bfloat> %b, f ; GFX11-LABEL: v_fdot2_f32_bf16_neg_b_hi_dual: ; GFX11: ; %bb.0: ; GFX11: v_xor_b16 v1.h, 0x8000, v1.h -; GFX11: v_dot2_f32_bf16 v0, v0, v1, v2 -; GFX11: v_dot2_f32_bf16 v1, v3, v4, v5 -; GFX11: v_add_f32_e32 v0, v0, v1 +; GFX11: v_dual_dot2acc_f32_bf16 v5, v3, v4 :: v_dual_dot2acc_f32_bf16 v2, v0, v1 +; GFX11: v_add_f32_e32 v0, v2, v5 ; ; GFX12-LABEL: v_fdot2_f32_bf16_neg_b_hi_dual: ; GFX12: ; %bb.0: ; GFX12: v_lshrrev_b32_e32 v6, 16, v1 +; GFX12: v_dot2_f32_bf16 v5, v3, v4, v5 ; GFX12: v_xor_b32_e32 v6, 0x8000, v6 ; GFX12: v_perm_b32 v1, v6, v1, 0x5040100 -; GFX12: v_dot2_f32_bf16 v0, v0, v1, v2 -; GFX12: v_dot2_f32_bf16 v1, v3, v4, v5 -; GFX12: v_add_f32_e32 v0, v0, v1 +; GFX12: v_dot2_f32_bf16 v2, v0, v1, v2 +; GFX12: v_add_f32_e32 v0, v2, v5 %b_hi = extractelement <2 x bfloat> %b, i32 1 %neg.b_hi = fneg bfloat %b_hi %neg_hi.b = insertelement <2 x bfloat> %b, bfloat %neg.b_hi, i32 1 @@ -654,8 +668,8 @@ define float @v_fdot2_f32_bf16_neg_c_dual(<2 x bfloat> %a, <2 x bfloat> %b, floa ; GFX11PLUS-LABEL: v_fdot2_f32_bf16_neg_c_dual: ; GFX11PLUS: ; %bb.0: ; GFX11PLUS: v_dot2_f32_bf16 v0, v0, v1, v2 neg_lo:[0,0,1] -; GFX11PLUS: v_dot2_f32_bf16 v1, v3, v4, v5 -; GFX11PLUS: v_add_f32_e32 v0, v0, v1 +; GFX11PLUS: v_dot2_f32_bf16 v5, v3, v4, v5 +; GFX11PLUS: v_add_f32_e32 v0, v0, v5 %neg.c = fneg float %c %r0 = call float @llvm.amdgcn.fdot2.f32.bf16(<2 x bfloat> %a, <2 x bfloat> %b, float %neg.c, i1 false) %r1 = call float @llvm.amdgcn.fdot2.f32.bf16(<2 x bfloat> %d, <2 x bfloat> %e, float %f, i1 false) @@ -673,8 +687,8 @@ define float @v_fdot2_f32_bf16_abs_c_dual(<2 x bfloat> %a, <2 x bfloat> %b, floa ; GFX11PLUS-LABEL: v_fdot2_f32_bf16_abs_c_dual: ; GFX11PLUS: ; %bb.0: ; GFX11PLUS: v_dot2_f32_bf16 v0, v0, v1, v2 neg_hi:[0,0,1] -; GFX11PLUS: v_dot2_f32_bf16 v1, v3, v4, v5 -; GFX11PLUS: v_add_f32_e32 v0, v0, v1 +; GFX11PLUS: v_dot2_f32_bf16 v5, v3, v4, v5 +; GFX11PLUS: v_add_f32_e32 v0, v0, v5 %abs.c = call float @llvm.fabs.f32(float %c) %r0 = call float @llvm.amdgcn.fdot2.f32.bf16(<2 x bfloat> %a, <2 x bfloat> %b, float %abs.c, i1 false) %r1 = call float @llvm.amdgcn.fdot2.f32.bf16(<2 x bfloat> %d, <2 x bfloat> %e, float %f, i1 false) @@ -694,16 +708,14 @@ define float @v_fdot2_f32_bf16_opsel_lo_a_dual(<2 x bfloat> %a, <2 x bfloat> %b, ; GFX11-LABEL: v_fdot2_f32_bf16_opsel_lo_a_dual: ; GFX11: ; %bb.0: ; GFX11: v_mov_b16_e32 v0.l, v0.h -; GFX11: v_dot2_f32_bf16 v0, v0, v1, v2 -; GFX11: v_dot2_f32_bf16 v1, v3, v4, v5 -; GFX11: v_add_f32_e32 v0, v0, v1 +; GFX11: v_dual_dot2acc_f32_bf16 v5, v3, v4 :: v_dual_dot2acc_f32_bf16 v2, v0, v1 +; GFX11: v_add_f32_e32 v0, v2, v5 ; ; GFX12-LABEL: v_fdot2_f32_bf16_opsel_lo_a_dual: ; GFX12: ; %bb.0: ; GFX12: v_perm_b32 v0, v0, v0, 0x7060302 -; GFX12: v_dot2_f32_bf16 v0, v0, v1, v2 -; GFX12: v_dot2_f32_bf16 v1, v3, v4, v5 -; GFX12: v_add_f32_e32 v0, v0, v1 +; GFX12: v_dual_dot2acc_f32_bf16 v5, v3, v4 :: v_dual_dot2acc_f32_bf16 v2, v0, v1 +; GFX12: v_add_f32_e32 v0, v2, v5 %shuf = shufflevector <2 x bfloat> %a, <2 x bfloat> poison, <2 x i32> <i32 1, i32 1> %r0 = call float @llvm.amdgcn.fdot2.f32.bf16(<2 x bfloat> %shuf, <2 x bfloat> %b, float %c, i1 false) %r1 = call float @llvm.amdgcn.fdot2.f32.bf16(<2 x bfloat> %d, <2 x bfloat> %e, float %f, i1 false) @@ -723,16 +735,14 @@ define float @v_fdot2_f32_bf16_opsel_hi_a_dual(<2 x bfloat> %a, <2 x bfloat> %b, ; GFX11-LABEL: v_fdot2_f32_bf16_opsel_hi_a_dual: ; GFX11: ; %bb.0: ; GFX11: v_mov_b16_e32 v0.h, v0.l -; GFX11: v_dot2_f32_bf16 v0, v0, v1, v2 -; GFX11: v_dot2_f32_bf16 v1, v3, v4, v5 -; GFX11: v_add_f32_e32 v0, v0, v1 +; GFX11: v_dual_dot2acc_f32_bf16 v5, v3, v4 :: v_dual_dot2acc_f32_bf16 v2, v0, v1 +; GFX11: v_add_f32_e32 v0, v2, v5 ; ; GFX12-LABEL: v_fdot2_f32_bf16_opsel_hi_a_dual: ; GFX12: ; %bb.0: ; GFX12: v_perm_b32 v0, v0, v0, 0x5040100 -; GFX12: v_dot2_f32_bf16 v0, v0, v1, v2 -; GFX12: v_dot2_f32_bf16 v1, v3, v4, v5 -; GFX12: v_add_f32_e32 v0, v0, v1 +; GFX12: v_dual_dot2acc_f32_bf16 v5, v3, v4 :: v_dual_dot2acc_f32_bf16 v2, v0, v1 +; GFX12: v_add_f32_e32 v0, v2, v5 %shuf = shufflevector <2 x bfloat> %a, <2 x bfloat> poison, <2 x i32> <i32 0, i32 0> %r0 = call float @llvm.amdgcn.fdot2.f32.bf16(<2 x bfloat> %shuf, <2 x bfloat> %b, float %c, i1 false) %r1 = call float @llvm.amdgcn.fdot2.f32.bf16(<2 x bfloat> %d, <2 x bfloat> %e, float %f, i1 false) @@ -752,16 +762,14 @@ define float @v_fdot2_f32_bf16_opsel_lo_b_dual(<2 x bfloat> %a, <2 x bfloat> %b, ; GFX11-LABEL: v_fdot2_f32_bf16_opsel_lo_b_dual: ; GFX11: ; %bb.0: ; GFX11: v_mov_b16_e32 v1.l, v1.h -; GFX11: v_dot2_f32_bf16 v0, v0, v1, v2 -; GFX11: v_dot2_f32_bf16 v1, v3, v4, v5 -; GFX11: v_add_f32_e32 v0, v0, v1 +; GFX11: v_dual_dot2acc_f32_bf16 v5, v3, v4 :: v_dual_dot2acc_f32_bf16 v2, v0, v1 +; GFX11: v_add_f32_e32 v0, v2, v5 ; ; GFX12-LABEL: v_fdot2_f32_bf16_opsel_lo_b_dual: ; GFX12: ; %bb.0: ; GFX12: v_perm_b32 v1, v1, v1, 0x7060302 -; GFX12: v_dot2_f32_bf16 v0, v0, v1, v2 -; GFX12: v_dot2_f32_bf16 v1, v3, v4, v5 -; GFX12: v_add_f32_e32 v0, v0, v1 +; GFX12: v_dual_dot2acc_f32_bf16 v5, v3, v4 :: v_dual_dot2acc_f32_bf16 v2, v0, v1 +; GFX12: v_add_f32_e32 v0, v2, v5 %shuf = shufflevector <2 x bfloat> %b, <2 x bfloat> poison, <2 x i32> <i32 1, i32 1> %r0 = call float @llvm.amdgcn.fdot2.f32.bf16(<2 x bfloat> %a, <2 x bfloat> %shuf, float %c, i1 false) %r1 = call float @llvm.amdgcn.fdot2.f32.bf16(<2 x bfloat> %d, <2 x bfloat> %e, float %f, i1 false) @@ -781,16 +789,14 @@ define float @v_fdot2_f32_bf16_opsel_hi_b_dual(<2 x bfloat> %a, <2 x bfloat> %b, ; GFX11-LABEL: v_fdot2_f32_bf16_opsel_hi_b_dual: ; GFX11: ; %bb.0: ; GFX11: v_mov_b16_e32 v1.h, v1.l -; GFX11: v_dot2_f32_bf16 v0, v0, v1, v2 -; GFX11: v_dot2_f32_bf16 v1, v3, v4, v5 -; GFX11: v_add_f32_e32 v0, v0, v1 +; GFX11: v_dual_dot2acc_f32_bf16 v5, v3, v4 :: v_dual_dot2acc_f32_bf16 v2, v0, v1 +; GFX11: v_add_f32_e32 v0, v2, v5 ; ; GFX12-LABEL: v_fdot2_f32_bf16_opsel_hi_b_dual: ; GFX12: ; %bb.0: ; GFX12: v_perm_b32 v1, v1, v1, 0x5040100 -; GFX12: v_dot2_f32_bf16 v0, v0, v1, v2 -; GFX12: v_dot2_f32_bf16 v1, v3, v4, v5 -; GFX12: v_add_f32_e32 v0, v0, v1 +; GFX12: v_dual_dot2acc_f32_bf16 v5, v3, v4 :: v_dual_dot2acc_f32_bf16 v2, v0, v1 +; GFX12: v_add_f32_e32 v0, v2, v5 %shuf = shufflevector <2 x bfloat> %b, <2 x bfloat> poison, <2 x i32> <i32 0, i32 0> %r0 = call float @llvm.amdgcn.fdot2.f32.bf16(<2 x bfloat> %a, <2 x bfloat> %shuf, float %c, i1 false) %r1 = call float @llvm.amdgcn.fdot2.f32.bf16(<2 x bfloat> %d, <2 x bfloat> %e, float %f, i1 false) @@ -810,9 +816,8 @@ define float @v_fdot2_f32_bf16_inline_literal_a_y(<2 x bfloat> %a, <2 x bfloat> ; ; GFX11PLUS-LABEL: v_fdot2_f32_bf16_inline_literal_a_y: ; GFX11PLUS: ; %bb.0: -; GFX11PLUS: v_dot2_f32_bf16 v0, v0, v1, v2 -; GFX11PLUS: v_dot2_f32_bf16 v1, 0x40004000, v4, v5 -; GFX11PLUS: v_add_f32_e32 v0, v0, v1 +; GFX11PLUS: v_dual_dot2acc_f32_bf16 v2, v0, v1 :: v_dual_dot2acc_f32_bf16 v5, 0x40004000, v4 +; GFX11PLUS: v_add_f32_e32 v0, v2, v5 %r0 = call float @llvm.amdgcn.fdot2.f32.bf16(<2 x bfloat> %a, <2 x bfloat> %b, float %c, i1 false) %r1 = call float @llvm.amdgcn.fdot2.f32.bf16(<2 x bfloat> <bfloat 2.0, bfloat 2.0>, <2 x bfloat> %e, float %f, i1 false) %r = fadd float %r0, %r1 @@ -828,9 +833,8 @@ define float @v_fdot2_f32_bf16_inline_literal_a_xy(<2 x bfloat> %a, <2 x bfloat> ; ; GFX11PLUS-LABEL: v_fdot2_f32_bf16_inline_literal_a_xy: ; GFX11PLUS: ; %bb.0: -; GFX11PLUS: v_dot2_f32_bf16 v0, 0x40004000, v1, v2 -; GFX11PLUS: v_dot2_f32_bf16 v1, 0x40004000, v4, v5 -; GFX11PLUS: v_add_f32_e32 v0, v0, v1 +; GFX11PLUS: v_dual_dot2acc_f32_bf16 v2, 0x40004000, v1 :: v_dual_dot2acc_f32_bf16 v5, 0x40004000, v4 +; GFX11PLUS: v_add_f32_e32 v0, v2, v5 %r0 = call float @llvm.amdgcn.fdot2.f32.bf16(<2 x bfloat> <bfloat 2.0, bfloat 2.0>, <2 x bfloat> %b, float %c, i1 false) %r1 = call float @llvm.amdgcn.fdot2.f32.bf16(<2 x bfloat> <bfloat 2.0, bfloat 2.0>, <2 x bfloat> %e, float %f, i1 false) %r = fadd float %r0, %r1 @@ -846,9 +850,8 @@ define float @v_fdot2_f32_bf16_inline_literal_b_x(<2 x bfloat> %a, <2 x bfloat> ; ; GFX11PLUS-LABEL: v_fdot2_f32_bf16_inline_literal_b_x: ; GFX11PLUS: ; %bb.0: -; GFX11PLUS: v_dot2_f32_bf16 v0, v0, 0x40004000, v2 -; GFX11PLUS: v_dot2_f32_bf16 v1, v4, v3, v5 -; GFX11PLUS: v_add_f32_e32 v0, v0, v1 +; GFX11PLUS: v_dual_dot2acc_f32_bf16 v2, 0x40004000, v0 :: v_dual_dot2acc_f32_bf16 v5, v4, v3 +; GFX11PLUS: v_add_f32_e32 v0, v2, v5 %r0 = call float @llvm.amdgcn.fdot2.f32.bf16(<2 x bfloat> %a, <2 x bfloat> <bfloat 2.0, bfloat 2.0>, float %c, i1 false) %r1 = call float @llvm.amdgcn.fdot2.f32.bf16(<2 x bfloat> %e, <2 x bfloat> %d, float %f, i1 false) %r = fadd float %r0, %r1 @@ -864,9 +867,8 @@ define float @v_fdot2_f32_bf16_inline_literal_b_y(<2 x bfloat> %a, <2 x bfloat> ; ; GFX11PLUS-LABEL: v_fdot2_f32_bf16_inline_literal_b_y: ; GFX11PLUS: ; %bb.0: -; GFX11PLUS: v_dot2_f32_bf16 v0, v1, v0, v2 -; GFX11PLUS: v_dot2_f32_bf16 v1, v3, 0x40004000, v5 -; GFX11PLUS: v_add_f32_e32 v0, v0, v1 +; GFX11PLUS: v_dual_dot2acc_f32_bf16 v2, v1, v0 :: v_dual_dot2acc_f32_bf16 v5, 0x40004000, v3 +; GFX11PLUS: v_add_f32_e32 v0, v2, v5 %r0 = call float @llvm.amdgcn.fdot2.f32.bf16(<2 x bfloat> %b, <2 x bfloat> %a, float %c, i1 false) %r1 = call float @llvm.amdgcn.fdot2.f32.bf16(<2 x bfloat> %d, <2 x bfloat> <bfloat 2.0, bfloat 2.0>, float %f, i1 false) %r = fadd float %r0, %r1 @@ -882,9 +884,8 @@ define float @v_fdot2_f32_bf16_inline_literal_b_xy(<2 x bfloat> %a, <2 x bfloat> ; ; GFX11PLUS-LABEL: v_fdot2_f32_bf16_inline_literal_b_xy: ; GFX11PLUS: ; %bb.0: -; GFX11PLUS: v_dot2_f32_bf16 v0, v0, 0x40004000, v2 -; GFX11PLUS: v_dot2_f32_bf16 v1, v3, 0x40004000, v5 -; GFX11PLUS: v_add_f32_e32 v0, v0, v1 +; GFX11PLUS: v_dual_dot2acc_f32_bf16 v2, 0x40004000, v0 :: v_dual_dot2acc_f32_bf16 v5, 0x40004000, v3 +; GFX11PLUS: v_add_f32_e32 v0, v2, v5 %r0 = call float @llvm.amdgcn.fdot2.f32.bf16(<2 x bfloat> %a, <2 x bfloat> <bfloat 2.0, bfloat 2.0>, float %c, i1 false) %r1 = call float @llvm.amdgcn.fdot2.f32.bf16(<2 x bfloat> %d, <2 x bfloat> <bfloat 2.0, bfloat 2.0>, float %f, i1 false) %r = fadd float %r0, %r1 @@ -902,8 +903,8 @@ define float @v_fdot2_f32_bf16_inline_literal_c_dual(<2 x bfloat> %a, <2 x bfloa ; GFX11PLUS-LABEL: v_fdot2_f32_bf16_inline_literal_c_dual: ; GFX11PLUS: ; %bb.0: ; GFX11PLUS: v_dot2_f32_bf16 v0, v0, v1, 2.0 -; GFX11PLUS: v_dot2_f32_bf16 v1, v2, v3, v4 -; GFX11PLUS: v_add_f32_e32 v0, v0, v1 +; GFX11PLUS: v_dot2_f32_bf16 v4, v2, v3, v4 +; GFX11PLUS: v_add_f32_e32 v0, v0, v4 %r0 = tail call float @llvm.amdgcn.fdot2.f32.bf16(<2 x bfloat> %a, <2 x bfloat> %b, float 2.0, i1 false) %r1 = call float @llvm.amdgcn.fdot2.f32.bf16(<2 x bfloat> %d, <2 x bfloat> %e, float %f, i1 false) %r = fadd float %r0, %r1 diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fdot2.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fdot2.ll index 319d5b3e22760..27afecb587bd8 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fdot2.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fdot2.ll @@ -2,8 +2,8 @@ ; RUN: llc -mtriple=amdgcn -mcpu=gfx906 < %s | FileCheck %s --check-prefixes=GCN,GFX906 ; RUN: llc -mtriple=amdgcn -mcpu=gfx950 < %s | FileCheck %s --check-prefixes=GCN,GFX950 ; RUN: llc -mtriple=amdgcn -mcpu=gfx1012 < %s | FileCheck %s --check-prefixes=GCN,GFX10 -; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 < %s | FileCheck %s --check-prefixes=GCN,GFX11PLUS,GFX11 -; RUN: llc -mtriple=amdgcn -mcpu=gfx1170 < %s | FileCheck %s --check-prefixes=GCN,GFX11PLUS,GFX1170 +; RUN: llc -mtriple=amdgcn -mcpu=gfx1100 < %s | FileCheck %s --check-prefixes=GCN,GFX11PLUS,GFX11PLUS-TRUE16,GFX11 +; RUN: llc -mtriple=amdgcn -mcpu=gfx1170 < %s | FileCheck %s --check-prefixes=GCN,GFX11PLUS,GFX11PLUS-TRUE16,GFX1170 ; RUN: llc -mtriple=amdgcn -mcpu=gfx1200 < %s | FileCheck %s --check-prefixes=GCN,GFX11PLUS,GFX12 declare float @llvm.amdgcn.fdot2(<2 x half> %a, <2 x half> %b, float %c, i1 %clamp) @@ -30,11 +30,13 @@ define float @v_fdot2(<2 x half> %a, <2 x half> %b, float %c) { ; ; GFX1170-LABEL: v_fdot2: ; GFX1170: ; %bb.0: -; GFX1170: v_dot2_f32_f16 v0, v0, v1, v2 +; GFX1170: v_dot2_f32_f16 v2, v0, v1, v2 +; GFX1170: v_mov_b32_e32 v0, v2 ; ; GFX12-LABEL: v_fdot2: ; GFX12: ; %bb.0: -; GFX12: v_dot2_f32_f16 v0, v0, v1, v2 +; GFX12: v_dot2_f32_f16 v2, v0, v1, v2 +; GFX12: v_mov_b32_e32 v0, v2 %r = call float @llvm.amdgcn.fdot2(<2 x half> %a, <2 x half> %b, float %c, i1 false) ret float %r } @@ -74,13 +76,15 @@ define float @v_fdot2_neg_a_lo(<2 x half> %a, <2 x half> %b, float %c) { ; GFX1170-LABEL: v_fdot2_neg_a_lo: ; GFX1170: ; %bb.0: ; GFX1170: v_xor_b16 v0.l, 0x8000, v0.l -; GFX1170: v_dot2_f32_f16 v0, v0, v1, v2 +; GFX1170: v_dot2_f32_f16 v2, v0, v1, v2 +; GFX1170: v_mov_b32_e32 v0, v2 ; ; GFX12-LABEL: v_fdot2_neg_a_lo: ; GFX12: ; %bb.0: ; GFX12: v_xor_b32_e32 v3, 0x8000, v0 ; GFX12: v_bfi_b32 v0, 0xffff, v3, v0 -; GFX12: v_dot2_f32_f16 v0, v0, v1, v2 +; GFX12: v_dot2_f32_f16 v2, v0, v1, v2 +; GFX12: v_mov_b32_e32 v0, v2 %a_lo = extractelement <2 x half> %a, i32 0 %neg.a_lo = fneg half %a_lo %neg_lo.a = insertelement <2 x half> %a, half %neg.a_lo, i32 0 @@ -115,14 +119,16 @@ define float @v_fdot2_neg_a_hi(<2 x half> %a, <2 x half> %b, float %c) { ; GFX1170-LABEL: v_fdot2_neg_a_hi: ; GFX1170: ; %bb.0: ; GFX1170: v_xor_b16 v0.h, 0x8000, v0.h -; GFX1170: v_dot2_f32_f16 v0, v0, v1, v2 +; GFX1170: v_dot2_f32_f16 v2, v0, v1, v2 +; GFX1170: v_mov_b32_e32 v0, v2 ; ; GFX12-LABEL: v_fdot2_neg_a_hi: ; GFX12: ; %bb.0: ; GFX12: v_lshrrev_b32_e32 v3, 16, v0 ; GFX12: v_xor_b32_e32 v3, 0x8000, v3 ; GFX12: v_perm_b32 v0, v3, v0, 0x5040100 -; GFX12: v_dot2_f32_f16 v0, v0, v1, v2 +; GFX12: v_dot2_f32_f16 v2, v0, v1, v2 +; GFX12: v_mov_b32_e32 v0, v2 %a_hi = extractelement <2 x half> %a, i32 1 %neg.a_hi = fneg half %a_hi %neg_hi.a = insertelement <2 x half> %a, half %neg.a_hi, i32 1 @@ -165,13 +171,15 @@ define float @v_fdot2_neg_b_lo(<2 x half> %a, <2 x half> %b, float %c) { ; GFX1170-LABEL: v_fdot2_neg_b_lo: ; GFX1170: ; %bb.0: ; GFX1170: v_xor_b16 v1.l, 0x8000, v1.l -; GFX1170: v_dot2_f32_f16 v0, v0, v1, v2 +; GFX1170: v_dot2_f32_f16 v2, v0, v1, v2 +; GFX1170: v_mov_b32_e32 v0, v2 ; ; GFX12-LABEL: v_fdot2_neg_b_lo: ; GFX12: ; %bb.0: ; GFX12: v_xor_b32_e32 v3, 0x8000, v1 ; GFX12: v_bfi_b32 v1, 0xffff, v3, v1 -; GFX12: v_dot2_f32_f16 v0, v0, v1, v2 +; GFX12: v_dot2_f32_f16 v2, v0, v1, v2 +; GFX12: v_mov_b32_e32 v0, v2 %b_lo = extractelement <2 x half> %b, i32 0 %neg.b_lo = fneg half %b_lo %neg_lo.b = insertelement <2 x half> %b, half %neg.b_lo, i32 0 @@ -206,14 +214,16 @@ define float @v_fdot2_neg_b_hi(<2 x half> %a, <2 x half> %b, float %c) { ; GFX1170-LABEL: v_fdot2_neg_b_hi: ; GFX1170: ; %bb.0: ; GFX1170: v_xor_b16 v1.h, 0x8000, v1.h -; GFX1170: v_dot2_f32_f16 v0, v0, v1, v2 +; GFX1170: v_dot2_f32_f16 v2, v0, v1, v2 +; GFX1170: v_mov_b32_e32 v0, v2 ; ; GFX12-LABEL: v_fdot2_neg_b_hi: ; GFX12: ; %bb.0: ; GFX12: v_lshrrev_b32_e32 v3, 16, v1 ; GFX12: v_xor_b32_e32 v3, 0x8000, v3 ; GFX12: v_perm_b32 v1, v3, v1, 0x5040100 -; GFX12: v_dot2_f32_f16 v0, v0, v1, v2 +; GFX12: v_dot2_f32_f16 v2, v0, v1, v2 +; GFX12: v_mov_b32_e32 v0, v2 %b_hi = extractelement <2 x half> %b, i32 1 %neg.b_hi = fneg half %b_hi %neg_hi.b = insertelement <2 x half> %b, half %neg.b_hi, i32 1 @@ -264,12 +274,14 @@ define float @v_fdot2_opsel_lo_a(<2 x half> %a, <2 x half> %b, float %c) { ; GFX1170-LABEL: v_fdot2_opsel_lo_a: ; GFX1170: ; %bb.0: ; GFX1170: v_mov_b16_e32 v0.l, v0.h -; GFX1170: v_dot2_f32_f16 v0, v0, v1, v2 +; GFX1170: v_dot2_f32_f16 v2, v0, v1, v2 +; GFX1170: v_mov_b32_e32 v0, v2 ; ; GFX12-LABEL: v_fdot2_opsel_lo_a: ; GFX12: ; %bb.0: ; GFX12: v_perm_b32 v0, v0, v0, 0x7060302 -; GFX12: v_dot2_f32_f16 v0, v0, v1, v2 +; GFX12: v_dot2_f32_f16 v2, v0, v1, v2 +; GFX12: v_mov_b32_e32 v0, v2 %shuf = shufflevector <2 x half> %a, <2 x half> poison, <2 x i32> <i32 1, i32 1> %r = call float @llvm.amdgcn.fdot2(<2 x half> %shuf, <2 x half> %b, float %c, i1 false) ret float %r @@ -300,12 +312,14 @@ define float @v_fdot2_opsel_hi_a(<2 x half> %a, <2 x half> %b, float %c) { ; GFX1170-LABEL: v_fdot2_opsel_hi_a: ; GFX1170: ; %bb.0: ; GFX1170: v_mov_b16_e32 v0.h, v0.l -; GFX1170: v_dot2_f32_f16 v0, v0, v1, v2 +; GFX1170: v_dot2_f32_f16 v2, v0, v1, v2 +; GFX1170: v_mov_b32_e32 v0, v2 ; ; GFX12-LABEL: v_fdot2_opsel_hi_a: ; GFX12: ; %bb.0: ; GFX12: v_perm_b32 v0, v0, v0, 0x5040100 -; GFX12: v_dot2_f32_f16 v0, v0, v1, v2 +; GFX12: v_dot2_f32_f16 v2, v0, v1, v2 +; GFX12: v_mov_b32_e32 v0, v2 %shuf = shufflevector <2 x half> %a, <2 x half> poison, <2 x i32> <i32 0, i32 0> %r = call float @llvm.amdgcn.fdot2(<2 x half> %shuf, <2 x half> %b, float %c, i1 false) ret float %r @@ -336,12 +350,14 @@ define float @v_fdot2_opsel_lo_b(<2 x half> %a, <2 x half> %b, float %c) { ; GFX1170-LABEL: v_fdot2_opsel_lo_b: ; GFX1170: ; %bb.0: ; GFX1170: v_mov_b16_e32 v1.l, v1.h -; GFX1170: v_dot2_f32_f16 v0, v0, v1, v2 +; GFX1170: v_dot2_f32_f16 v2, v0, v1, v2 +; GFX1170: v_mov_b32_e32 v0, v2 ; ; GFX12-LABEL: v_fdot2_opsel_lo_b: ; GFX12: ; %bb.0: ; GFX12: v_perm_b32 v1, v1, v1, 0x7060302 -; GFX12: v_dot2_f32_f16 v0, v0, v1, v2 +; GFX12: v_dot2_f32_f16 v2, v0, v1, v2 +; GFX12: v_mov_b32_e32 v0, v2 %shuf = shufflevector <2 x half> %b, <2 x half> poison, <2 x i32> <i32 1, i32 1> %r = call float @llvm.amdgcn.fdot2(<2 x half> %a, <2 x half> %shuf, float %c, i1 false) ret float %r @@ -372,12 +388,14 @@ define float @v_fdot2_opsel_hi_b(<2 x half> %a, <2 x half> %b, float %c) { ; GFX1170-LABEL: v_fdot2_opsel_hi_b: ; GFX1170: ; %bb.0: ; GFX1170: v_mov_b16_e32 v1.h, v1.l -; GFX1170: v_dot2_f32_f16 v0, v0, v1, v2 +; GFX1170: v_dot2_f32_f16 v2, v0, v1, v2 +; GFX1170: v_mov_b32_e32 v0, v2 ; ; GFX12-LABEL: v_fdot2_opsel_hi_b: ; GFX12: ; %bb.0: ; GFX12: v_perm_b32 v1, v1, v1, 0x5040100 -; GFX12: v_dot2_f32_f16 v0, v0, v1, v2 +; GFX12: v_dot2_f32_f16 v2, v0, v1, v2 +; GFX12: v_mov_b32_e32 v0, v2 %shuf = shufflevector <2 x half> %b, <2 x half> poison, <2 x i32> <i32 0, i32 0> %r = call float @llvm.amdgcn.fdot2(<2 x half> %a, <2 x half> %shuf, float %c, i1 false) ret float %r @@ -405,11 +423,13 @@ define float @v_fdot2_inline_literal_a(<2 x half> %b, float %c) { ; ; GFX1170-LABEL: v_fdot2_inline_literal_a: ; GFX1170: ; %bb.0: -; GFX1170: v_dot2_f32_f16 v0, 0x40004000, v0, v1 +; GFX1170: v_dot2_f32_f16 v1, 0x40004000, v0, v1 +; GFX1170: v_mov_b32_e32 v0, v1 ; ; GFX12-LABEL: v_fdot2_inline_literal_a: ; GFX12: ; %bb.0: -; GFX12: v_dot2_f32_f16 v0, 0x40004000, v0, v1 +; GFX12: v_dot2_f32_f16 v1, 0x40004000, v0, v1 +; GFX12: v_mov_b32_e32 v0, v1 %ret = tail call float @llvm.amdgcn.fdot2(<2 x half> <half 2.0, half 2.0>, <2 x half> %b, float %c, i1 false) ret float %ret } @@ -436,11 +456,13 @@ define float @v_fdot2_inline_literal_b(<2 x half> %a, float %c) { ; ; GFX1170-LABEL: v_fdot2_inline_literal_b: ; GFX1170: ; %bb.0: -; GFX1170: v_dot2_f32_f16 v0, v0, 0x40004000, v1 +; GFX1170: v_dot2_f32_f16 v1, 0x40004000, v0, v1 +; GFX1170: v_mov_b32_e32 v0, v1 ; ; GFX12-LABEL: v_fdot2_inline_literal_b: ; GFX12: ; %bb.0: -; GFX12: v_dot2_f32_f16 v0, v0, 0x40004000, v1 +; GFX12: v_dot2_f32_f16 v1, 0x40004000, v0, v1 +; GFX12: v_mov_b32_e32 v0, v1 %ret = tail call float @llvm.amdgcn.fdot2(<2 x half> %a, <2 x half> <half 2.0, half 2.0>, float %c, i1 false) ret float %ret } @@ -538,15 +560,10 @@ define float @v_fdot2_opsel_lo_a_clamp(<2 x half> %a, <2 x half> %b, float %c) { ; GFX10: ; %bb.0: ; GFX10: v_dot2_f32_f16 v0, v0, v1, v2 op_sel:[1,0,0] clamp ; -; GFX11-LABEL: v_fdot2_opsel_lo_a_clamp: -; GFX11: ; %bb.0: -; GFX11: v_mov_b16_e32 v0.l, v0.h -; GFX11: v_dot2_f32_f16 v0, v0, v1, v2 clamp -; -; GFX1170-LABEL: v_fdot2_opsel_lo_a_clamp: -; GFX1170: ; %bb.0: -; GFX1170: v_mov_b16_e32 v0.l, v0.h -; GFX1170: v_dot2_f32_f16 v0, v0, v1, v2 clamp +; GFX11PLUS-TRUE16-LABEL: v_fdot2_opsel_lo_a_clamp: +; GFX11PLUS-TRUE16: ; %bb.0: +; GFX11PLUS-TRUE16: v_mov_b16_e32 v0.l, v0.h +; GFX11PLUS-TRUE16: v_dot2_f32_f16 v0, v0, v1, v2 clamp ; ; GFX12-LABEL: v_fdot2_opsel_lo_a_clamp: ; GFX12: ; %bb.0: @@ -572,15 +589,10 @@ define float @v_fdot2_opsel_hi_a_clamp(<2 x half> %a, <2 x half> %b, float %c) { ; GFX10: ; %bb.0: ; GFX10: v_dot2_f32_f16 v0, v0, v1, v2 op_sel_hi:[0,1,1] clamp ; -; GFX11-LABEL: v_fdot2_opsel_hi_a_clamp: -; GFX11: ; %bb.0: -; GFX11: v_mov_b16_e32 v0.h, v0.l -; GFX11: v_dot2_f32_f16 v0, v0, v1, v2 clamp -; -; GFX1170-LABEL: v_fdot2_opsel_hi_a_clamp: -; GFX1170: ; %bb.0: -; GFX1170: v_mov_b16_e32 v0.h, v0.l -; GFX1170: v_dot2_f32_f16 v0, v0, v1, v2 clamp +; GFX11PLUS-TRUE16-LABEL: v_fdot2_opsel_hi_a_clamp: +; GFX11PLUS-TRUE16: ; %bb.0: +; GFX11PLUS-TRUE16: v_mov_b16_e32 v0.h, v0.l +; GFX11PLUS-TRUE16: v_dot2_f32_f16 v0, v0, v1, v2 clamp ; ; GFX12-LABEL: v_fdot2_opsel_hi_a_clamp: ; GFX12: ; %bb.0: @@ -606,15 +618,10 @@ define float @v_fdot2_opsel_lo_b_clamp(<2 x half> %a, <2 x half> %b, float %c) { ; GFX10: ; %bb.0: ; GFX10: v_dot2_f32_f16 v0, v0, v1, v2 op_sel:[0,1,0] clamp ; -; GFX11-LABEL: v_fdot2_opsel_lo_b_clamp: -; GFX11: ; %bb.0: -; GFX11: v_mov_b16_e32 v1.l, v1.h -; GFX11: v_dot2_f32_f16 v0, v0, v1, v2 clamp -; -; GFX1170-LABEL: v_fdot2_opsel_lo_b_clamp: -; GFX1170: ; %bb.0: -; GFX1170: v_mov_b16_e32 v1.l, v1.h -; GFX1170: v_dot2_f32_f16 v0, v0, v1, v2 clamp +; GFX11PLUS-TRUE16-LABEL: v_fdot2_opsel_lo_b_clamp: +; GFX11PLUS-TRUE16: ; %bb.0: +; GFX11PLUS-TRUE16: v_mov_b16_e32 v1.l, v1.h +; GFX11PLUS-TRUE16: v_dot2_f32_f16 v0, v0, v1, v2 clamp ; ; GFX12-LABEL: v_fdot2_opsel_lo_b_clamp: ; GFX12: ; %bb.0: @@ -640,15 +647,10 @@ define float @v_fdot2_opsel_hi_b_clamp(<2 x half> %a, <2 x half> %b, float %c) { ; GFX10: ; %bb.0: ; GFX10: v_dot2_f32_f16 v0, v0, v1, v2 op_sel_hi:[1,0,1] clamp ; -; GFX11-LABEL: v_fdot2_opsel_hi_b_clamp: -; GFX11: ; %bb.0: -; GFX11: v_mov_b16_e32 v1.h, v1.l -; GFX11: v_dot2_f32_f16 v0, v0, v1, v2 clamp -; -; GFX1170-LABEL: v_fdot2_opsel_hi_b_clamp: -; GFX1170: ; %bb.0: -; GFX1170: v_mov_b16_e32 v1.h, v1.l -; GFX1170: v_dot2_f32_f16 v0, v0, v1, v2 clamp +; GFX11PLUS-TRUE16-LABEL: v_fdot2_opsel_hi_b_clamp: +; GFX11PLUS-TRUE16: ; %bb.0: +; GFX11PLUS-TRUE16: v_mov_b16_e32 v1.h, v1.l +; GFX11PLUS-TRUE16: v_dot2_f32_f16 v0, v0, v1, v2 clamp ; ; GFX12-LABEL: v_fdot2_opsel_hi_b_clamp: ; GFX12: ; %bb.0: @@ -728,22 +730,10 @@ define float @v_fdot2_dual(<2 x half> %a, <2 x half> %b, float %c, <2 x half> %d ; GFX10: v_dot2c_f32_f16 v5, v3, v4 ; GFX10: v_add_f32_e32 v0, v2, v5 ; -; GFX11-LABEL: v_fdot2_dual: -; GFX11: ; %bb.0: -; GFX11: v_dual_dot2acc_f32_f16 v2, v0, v1 :: v_dual_dot2acc_f32_f16 v5, v3, v4 -; GFX11: v_add_f32_e32 v0, v2, v5 -; -; GFX1170-LABEL: v_fdot2_dual: -; GFX1170: ; %bb.0: -; GFX1170: v_dot2_f32_f16 v0, v0, v1, v2 -; GFX1170: v_dot2_f32_f16 v1, v3, v4, v5 -; GFX1170: v_add_f32_e32 v0, v0, v1 -; -; GFX12-LABEL: v_fdot2_dual: -; GFX12: ; %bb.0: -; GFX12: v_dot2_f32_f16 v0, v0, v1, v2 -; GFX12: v_dot2_f32_f16 v1, v3, v4, v5 -; GFX12: v_add_f32_e32 v0, v0, v1 +; GFX11PLUS-LABEL: v_fdot2_dual: +; GFX11PLUS: ; %bb.0: +; GFX11PLUS: v_dual_dot2acc_f32_f16 v2, v0, v1 :: v_dual_dot2acc_f32_f16 v5, v3, v4 +; GFX11PLUS: v_add_f32_e32 v0, v2, v5 %r0 = call float @llvm.amdgcn.fdot2(<2 x half> %a, <2 x half> %b, float %c, i1 false) %r1 = call float @llvm.amdgcn.fdot2(<2 x half> %d, <2 x half> %e, float %f, i1 false) %r = fadd float %r0, %r1 @@ -778,14 +768,14 @@ define float @v_fdot2_neg_a_dual(<2 x half> %a, <2 x half> %b, float %c, <2 x ha ; GFX1170-LABEL: v_fdot2_neg_a_dual: ; GFX1170: ; %bb.0: ; GFX1170: v_dot2_f32_f16 v0, v0, v1, v2 neg_lo:[1,0,0] neg_hi:[1,0,0] -; GFX1170: v_dot2_f32_f16 v1, v3, v4, v5 -; GFX1170: v_add_f32_e32 v0, v0, v1 +; GFX1170: v_dot2_f32_f16 v5, v3, v4, v5 +; GFX1170: v_add_f32_e32 v0, v0, v5 ; ; GFX12-LABEL: v_fdot2_neg_a_dual: ; GFX12: ; %bb.0: ; GFX12: v_dot2_f32_f16 v0, v0, v1, v2 neg_lo:[1,0,0] neg_hi:[1,0,0] -; GFX12: v_dot2_f32_f16 v1, v3, v4, v5 -; GFX12: v_add_f32_e32 v0, v0, v1 +; GFX12: v_dot2_f32_f16 v5, v3, v4, v5 +; GFX12: v_add_f32_e32 v0, v0, v5 %neg.a = fneg <2 x half> %a %r0 = call float @llvm.amdgcn.fdot2(<2 x half> %neg.a, <2 x half> %b, float %c, i1 false) %r1 = call float @llvm.amdgcn.fdot2(<2 x half> %d, <2 x half> %e, float %f, i1 false) @@ -815,26 +805,19 @@ define float @v_fdot2_neg_a_lo_dual(<2 x half> %a, <2 x half> %b, float %c, <2 x ; GFX10: v_dot2c_f32_f16 v5, v3, v4 ; GFX10: v_add_f32_e32 v0, v0, v5 ; -; GFX11-LABEL: v_fdot2_neg_a_lo_dual: -; GFX11: ; %bb.0: -; GFX11: v_xor_b16 v0.l, 0x8000, v0.l -; GFX11: v_dual_dot2acc_f32_f16 v5, v3, v4 :: v_dual_dot2acc_f32_f16 v2, v0, v1 -; GFX11: v_add_f32_e32 v0, v2, v5 -; -; GFX1170-LABEL: v_fdot2_neg_a_lo_dual: -; GFX1170: ; %bb.0: -; GFX1170: v_xor_b16 v0.l, 0x8000, v0.l -; GFX1170: v_dot2_f32_f16 v0, v0, v1, v2 -; GFX1170: v_dot2_f32_f16 v1, v3, v4, v5 -; GFX1170: v_add_f32_e32 v0, v0, v1 +; GFX11PLUS-TRUE16-LABEL: v_fdot2_neg_a_lo_dual: +; GFX11PLUS-TRUE16: ; %bb.0: +; GFX11PLUS-TRUE16: v_xor_b16 v0.l, 0x8000, v0.l +; GFX11PLUS-TRUE16: v_dual_dot2acc_f32_f16 v5, v3, v4 :: v_dual_dot2acc_f32_f16 v2, v0, v1 +; GFX11PLUS-TRUE16: v_add_f32_e32 v0, v2, v5 ; ; GFX12-LABEL: v_fdot2_neg_a_lo_dual: ; GFX12: ; %bb.0: ; GFX12: v_xor_b32_e32 v6, 0x8000, v0 +; GFX12: v_dot2_f32_f16 v5, v3, v4, v5 ; GFX12: v_bfi_b32 v0, 0xffff, v6, v0 -; GFX12: v_dot2_f32_f16 v0, v0, v1, v2 -; GFX12: v_dot2_f32_f16 v1, v3, v4, v5 -; GFX12: v_add_f32_e32 v0, v0, v1 +; GFX12: v_dot2_f32_f16 v2, v0, v1, v2 +; GFX12: v_add_f32_e32 v0, v2, v5 %a_lo = extractelement <2 x half> %a, i32 0 %neg.a_lo = fneg half %a_lo %neg_lo.a = insertelement <2 x half> %a, half %neg.a_lo, i32 0 @@ -867,27 +850,20 @@ define float @v_fdot2_neg_a_hi_dual(<2 x half> %a, <2 x half> %b, float %c, <2 x ; GFX10: v_dot2c_f32_f16 v5, v3, v4 ; GFX10: v_add_f32_e32 v0, v0, v5 ; -; GFX11-LABEL: v_fdot2_neg_a_hi_dual: -; GFX11: ; %bb.0: -; GFX11: v_xor_b16 v0.h, 0x8000, v0.h -; GFX11: v_dual_dot2acc_f32_f16 v5, v3, v4 :: v_dual_dot2acc_f32_f16 v2, v0, v1 -; GFX11: v_add_f32_e32 v0, v2, v5 -; -; GFX1170-LABEL: v_fdot2_neg_a_hi_dual: -; GFX1170: ; %bb.0: -; GFX1170: v_xor_b16 v0.h, 0x8000, v0.h -; GFX1170: v_dot2_f32_f16 v0, v0, v1, v2 -; GFX1170: v_dot2_f32_f16 v1, v3, v4, v5 -; GFX1170: v_add_f32_e32 v0, v0, v1 +; GFX11PLUS-TRUE16-LABEL: v_fdot2_neg_a_hi_dual: +; GFX11PLUS-TRUE16: ; %bb.0: +; GFX11PLUS-TRUE16: v_xor_b16 v0.h, 0x8000, v0.h +; GFX11PLUS-TRUE16: v_dual_dot2acc_f32_f16 v5, v3, v4 :: v_dual_dot2acc_f32_f16 v2, v0, v1 +; GFX11PLUS-TRUE16: v_add_f32_e32 v0, v2, v5 ; ; GFX12-LABEL: v_fdot2_neg_a_hi_dual: ; GFX12: ; %bb.0: ; GFX12: v_lshrrev_b32_e32 v6, 16, v0 +; GFX12: v_dot2_f32_f16 v5, v3, v4, v5 ; GFX12: v_xor_b32_e32 v6, 0x8000, v6 ; GFX12: v_perm_b32 v0, v6, v0, 0x5040100 -; GFX12: v_dot2_f32_f16 v0, v0, v1, v2 -; GFX12: v_dot2_f32_f16 v1, v3, v4, v5 -; GFX12: v_add_f32_e32 v0, v0, v1 +; GFX12: v_dot2_f32_f16 v2, v0, v1, v2 +; GFX12: v_add_f32_e32 v0, v2, v5 %a_hi = extractelement <2 x half> %a, i32 1 %neg.a_hi = fneg half %a_hi %neg_hi.a = insertelement <2 x half> %a, half %neg.a_hi, i32 1 @@ -925,14 +901,14 @@ define float @v_fdot2_neg_b_dual(<2 x half> %a, <2 x half> %b, float %c, <2 x ha ; GFX1170-LABEL: v_fdot2_neg_b_dual: ; GFX1170: ; %bb.0: ; GFX1170: v_dot2_f32_f16 v0, v0, v1, v2 neg_lo:[0,1,0] neg_hi:[0,1,0] -; GFX1170: v_dot2_f32_f16 v1, v3, v4, v5 -; GFX1170: v_add_f32_e32 v0, v0, v1 +; GFX1170: v_dot2_f32_f16 v5, v3, v4, v5 +; GFX1170: v_add_f32_e32 v0, v0, v5 ; ; GFX12-LABEL: v_fdot2_neg_b_dual: ; GFX12: ; %bb.0: ; GFX12: v_dot2_f32_f16 v0, v0, v1, v2 neg_lo:[0,1,0] neg_hi:[0,1,0] -; GFX12: v_dot2_f32_f16 v1, v3, v4, v5 -; GFX12: v_add_f32_e32 v0, v0, v1 +; GFX12: v_dot2_f32_f16 v5, v3, v4, v5 +; GFX12: v_add_f32_e32 v0, v0, v5 %neg.b = fneg <2 x half> %b %r0 = call float @llvm.amdgcn.fdot2(<2 x half> %a, <2 x half> %neg.b, float %c, i1 false) %r1 = call float @llvm.amdgcn.fdot2(<2 x half> %d, <2 x half> %e, float %f, i1 false) @@ -962,26 +938,19 @@ define float @v_fdot2_neg_b_lo_dual(<2 x half> %a, <2 x half> %b, float %c, <2 x ; GFX10: v_dot2c_f32_f16 v5, v3, v4 ; GFX10: v_add_f32_e32 v0, v0, v5 ; -; GFX11-LABEL: v_fdot2_neg_b_lo_dual: -; GFX11: ; %bb.0: -; GFX11: v_xor_b16 v1.l, 0x8000, v1.l -; GFX11: v_dual_dot2acc_f32_f16 v5, v3, v4 :: v_dual_dot2acc_f32_f16 v2, v0, v1 -; GFX11: v_add_f32_e32 v0, v2, v5 -; -; GFX1170-LABEL: v_fdot2_neg_b_lo_dual: -; GFX1170: ; %bb.0: -; GFX1170: v_xor_b16 v1.l, 0x8000, v1.l -; GFX1170: v_dot2_f32_f16 v0, v0, v1, v2 -; GFX1170: v_dot2_f32_f16 v1, v3, v4, v5 -; GFX1170: v_add_f32_e32 v0, v0, v1 +; GFX11PLUS-TRUE16-LABEL: v_fdot2_neg_b_lo_dual: +; GFX11PLUS-TRUE16: ; %bb.0: +; GFX11PLUS-TRUE16: v_xor_b16 v1.l, 0x8000, v1.l +; GFX11PLUS-TRUE16: v_dual_dot2acc_f32_f16 v5, v3, v4 :: v_dual_dot2acc_f32_f16 v2, v0, v1 +; GFX11PLUS-TRUE16: v_add_f32_e32 v0, v2, v5 ; ; GFX12-LABEL: v_fdot2_neg_b_lo_dual: ; GFX12: ; %bb.0: ; GFX12: v_xor_b32_e32 v6, 0x8000, v1 +; GFX12: v_dot2_f32_f16 v5, v3, v4, v5 ; GFX12: v_bfi_b32 v1, 0xffff, v6, v1 -; GFX12: v_dot2_f32_f16 v0, v0, v1, v2 -; GFX12: v_dot2_f32_f16 v1, v3, v4, v5 -; GFX12: v_add_f32_e32 v0, v0, v1 +; GFX12: v_dot2_f32_f16 v2, v0, v1, v2 +; GFX12: v_add_f32_e32 v0, v2, v5 %b_lo = extractelement <2 x half> %b, i32 0 %neg.b_lo = fneg half %b_lo %neg_lo.b = insertelement <2 x half> %b, half %neg.b_lo, i32 0 @@ -1014,27 +983,20 @@ define float @v_fdot2_neg_b_hi_dual(<2 x half> %a, <2 x half> %b, float %c, <2 x ; GFX10: v_dot2c_f32_f16 v5, v3, v4 ; GFX10: v_add_f32_e32 v0, v0, v5 ; -; GFX11-LABEL: v_fdot2_neg_b_hi_dual: -; GFX11: ; %bb.0: -; GFX11: v_xor_b16 v1.h, 0x8000, v1.h -; GFX11: v_dual_dot2acc_f32_f16 v5, v3, v4 :: v_dual_dot2acc_f32_f16 v2, v0, v1 -; GFX11: v_add_f32_e32 v0, v2, v5 -; -; GFX1170-LABEL: v_fdot2_neg_b_hi_dual: -; GFX1170: ; %bb.0: -; GFX1170: v_xor_b16 v1.h, 0x8000, v1.h -; GFX1170: v_dot2_f32_f16 v0, v0, v1, v2 -; GFX1170: v_dot2_f32_f16 v1, v3, v4, v5 -; GFX1170: v_add_f32_e32 v0, v0, v1 +; GFX11PLUS-TRUE16-LABEL: v_fdot2_neg_b_hi_dual: +; GFX11PLUS-TRUE16: ; %bb.0: +; GFX11PLUS-TRUE16: v_xor_b16 v1.h, 0x8000, v1.h +; GFX11PLUS-TRUE16: v_dual_dot2acc_f32_f16 v5, v3, v4 :: v_dual_dot2acc_f32_f16 v2, v0, v1 +; GFX11PLUS-TRUE16: v_add_f32_e32 v0, v2, v5 ; ; GFX12-LABEL: v_fdot2_neg_b_hi_dual: ; GFX12: ; %bb.0: ; GFX12: v_lshrrev_b32_e32 v6, 16, v1 +; GFX12: v_dot2_f32_f16 v5, v3, v4, v5 ; GFX12: v_xor_b32_e32 v6, 0x8000, v6 ; GFX12: v_perm_b32 v1, v6, v1, 0x5040100 -; GFX12: v_dot2_f32_f16 v0, v0, v1, v2 -; GFX12: v_dot2_f32_f16 v1, v3, v4, v5 -; GFX12: v_add_f32_e32 v0, v0, v1 +; GFX12: v_dot2_f32_f16 v2, v0, v1, v2 +; GFX12: v_add_f32_e32 v0, v2, v5 %b_hi = extractelement <2 x half> %b, i32 1 %neg.b_hi = fneg half %b_hi %neg_hi.b = insertelement <2 x half> %b, half %neg.b_hi, i32 1 @@ -1072,14 +1034,14 @@ define float @v_fdot2_neg_c_dual(<2 x half> %a, <2 x half> %b, float %c, <2 x ha ; GFX1170-LABEL: v_fdot2_neg_c_dual: ; GFX1170: ; %bb.0: ; GFX1170: v_dot2_f32_f16 v0, v0, v1, v2 neg_lo:[0,0,1] -; GFX1170: v_dot2_f32_f16 v1, v3, v4, v5 -; GFX1170: v_add_f32_e32 v0, v0, v1 +; GFX1170: v_dot2_f32_f16 v5, v3, v4, v5 +; GFX1170: v_add_f32_e32 v0, v0, v5 ; ; GFX12-LABEL: v_fdot2_neg_c_dual: ; GFX12: ; %bb.0: ; GFX12: v_dot2_f32_f16 v0, v0, v1, v2 neg_lo:[0,0,1] -; GFX12: v_dot2_f32_f16 v1, v3, v4, v5 -; GFX12: v_add_f32_e32 v0, v0, v1 +; GFX12: v_dot2_f32_f16 v5, v3, v4, v5 +; GFX12: v_add_f32_e32 v0, v0, v5 %neg.c = fneg float %c %r0 = call float @llvm.amdgcn.fdot2(<2 x half> %a, <2 x half> %b, float %neg.c, i1 false) %r1 = call float @llvm.amdgcn.fdot2(<2 x half> %d, <2 x half> %e, float %f, i1 false) @@ -1115,14 +1077,14 @@ define float @v_fdot2_abs_c_dual(<2 x half> %a, <2 x half> %b, float %c, <2 x ha ; GFX1170-LABEL: v_fdot2_abs_c_dual: ; GFX1170: ; %bb.0: ; GFX1170: v_dot2_f32_f16 v0, v0, v1, v2 neg_hi:[0,0,1] -; GFX1170: v_dot2_f32_f16 v1, v3, v4, v5 -; GFX1170: v_add_f32_e32 v0, v0, v1 +; GFX1170: v_dot2_f32_f16 v5, v3, v4, v5 +; GFX1170: v_add_f32_e32 v0, v0, v5 ; ; GFX12-LABEL: v_fdot2_abs_c_dual: ; GFX12: ; %bb.0: ; GFX12: v_dot2_f32_f16 v0, v0, v1, v2 neg_hi:[0,0,1] -; GFX12: v_dot2_f32_f16 v1, v3, v4, v5 -; GFX12: v_add_f32_e32 v0, v0, v1 +; GFX12: v_dot2_f32_f16 v5, v3, v4, v5 +; GFX12: v_add_f32_e32 v0, v0, v5 %abs.c = call float @llvm.fabs.f32(float %c) %r0 = call float @llvm.amdgcn.fdot2(<2 x half> %a, <2 x half> %b, float %abs.c, i1 false) %r1 = call float @llvm.amdgcn.fdot2(<2 x half> %d, <2 x half> %e, float %f, i1 false) @@ -1151,25 +1113,17 @@ define float @v_fdot2_opsel_lo_a_dual(<2 x half> %a, <2 x half> %b, float %c, <2 ; GFX10: v_dot2c_f32_f16 v5, v3, v4 ; GFX10: v_add_f32_e32 v0, v0, v5 ; -; GFX11-LABEL: v_fdot2_opsel_lo_a_dual: -; GFX11: ; %bb.0: -; GFX11: v_mov_b16_e32 v0.l, v0.h -; GFX11: v_dual_dot2acc_f32_f16 v5, v3, v4 :: v_dual_dot2acc_f32_f16 v2, v0, v1 -; GFX11: v_add_f32_e32 v0, v2, v5 -; -; GFX1170-LABEL: v_fdot2_opsel_lo_a_dual: -; GFX1170: ; %bb.0: -; GFX1170: v_mov_b16_e32 v0.l, v0.h -; GFX1170: v_dot2_f32_f16 v0, v0, v1, v2 -; GFX1170: v_dot2_f32_f16 v1, v3, v4, v5 -; GFX1170: v_add_f32_e32 v0, v0, v1 +; GFX11PLUS-TRUE16-LABEL: v_fdot2_opsel_lo_a_dual: +; GFX11PLUS-TRUE16: ; %bb.0: +; GFX11PLUS-TRUE16: v_mov_b16_e32 v0.l, v0.h +; GFX11PLUS-TRUE16: v_dual_dot2acc_f32_f16 v5, v3, v4 :: v_dual_dot2acc_f32_f16 v2, v0, v1 +; GFX11PLUS-TRUE16: v_add_f32_e32 v0, v2, v5 ; ; GFX12-LABEL: v_fdot2_opsel_lo_a_dual: ; GFX12: ; %bb.0: ; GFX12: v_perm_b32 v0, v0, v0, 0x7060302 -; GFX12: v_dot2_f32_f16 v0, v0, v1, v2 -; GFX12: v_dot2_f32_f16 v1, v3, v4, v5 -; GFX12: v_add_f32_e32 v0, v0, v1 +; GFX12: v_dual_dot2acc_f32_f16 v5, v3, v4 :: v_dual_dot2acc_f32_f16 v2, v0, v1 +; GFX12: v_add_f32_e32 v0, v2, v5 %shuf = shufflevector <2 x half> %a, <2 x half> poison, <2 x i32> <i32 1, i32 1> %r0 = call float @llvm.amdgcn.fdot2(<2 x half> %shuf, <2 x half> %b, float %c, i1 false) %r1 = call float @llvm.amdgcn.fdot2(<2 x half> %d, <2 x half> %e, float %f, i1 false) @@ -1198,25 +1152,17 @@ define float @v_fdot2_opsel_hi_a_dual(<2 x half> %a, <2 x half> %b, float %c, <2 ; GFX10: v_dot2c_f32_f16 v5, v3, v4 ; GFX10: v_add_f32_e32 v0, v0, v5 ; -; GFX11-LABEL: v_fdot2_opsel_hi_a_dual: -; GFX11: ; %bb.0: -; GFX11: v_mov_b16_e32 v0.h, v0.l -; GFX11: v_dual_dot2acc_f32_f16 v5, v3, v4 :: v_dual_dot2acc_f32_f16 v2, v0, v1 -; GFX11: v_add_f32_e32 v0, v2, v5 -; -; GFX1170-LABEL: v_fdot2_opsel_hi_a_dual: -; GFX1170: ; %bb.0: -; GFX1170: v_mov_b16_e32 v0.h, v0.l -; GFX1170: v_dot2_f32_f16 v0, v0, v1, v2 -; GFX1170: v_dot2_f32_f16 v1, v3, v4, v5 -; GFX1170: v_add_f32_e32 v0, v0, v1 +; GFX11PLUS-TRUE16-LABEL: v_fdot2_opsel_hi_a_dual: +; GFX11PLUS-TRUE16: ; %bb.0: +; GFX11PLUS-TRUE16: v_mov_b16_e32 v0.h, v0.l +; GFX11PLUS-TRUE16: v_dual_dot2acc_f32_f16 v5, v3, v4 :: v_dual_dot2acc_f32_f16 v2, v0, v1 +; GFX11PLUS-TRUE16: v_add_f32_e32 v0, v2, v5 ; ; GFX12-LABEL: v_fdot2_opsel_hi_a_dual: ; GFX12: ; %bb.0: ; GFX12: v_perm_b32 v0, v0, v0, 0x5040100 -; GFX12: v_dot2_f32_f16 v0, v0, v1, v2 -; GFX12: v_dot2_f32_f16 v1, v3, v4, v5 -; GFX12: v_add_f32_e32 v0, v0, v1 +; GFX12: v_dual_dot2acc_f32_f16 v5, v3, v4 :: v_dual_dot2acc_f32_f16 v2, v0, v1 +; GFX12: v_add_f32_e32 v0, v2, v5 %shuf = shufflevector <2 x half> %a, <2 x half> poison, <2 x i32> <i32 0, i32 0> %r0 = call float @llvm.amdgcn.fdot2(<2 x half> %shuf, <2 x half> %b, float %c, i1 false) %r1 = call float @llvm.amdgcn.fdot2(<2 x half> %d, <2 x half> %e, float %f, i1 false) @@ -1245,25 +1191,17 @@ define float @v_fdot2_opsel_lo_b_dual(<2 x half> %a, <2 x half> %b, float %c, <2 ; GFX10: v_dot2c_f32_f16 v5, v3, v4 ; GFX10: v_add_f32_e32 v0, v0, v5 ; -; GFX11-LABEL: v_fdot2_opsel_lo_b_dual: -; GFX11: ; %bb.0: -; GFX11: v_mov_b16_e32 v1.l, v1.h -; GFX11: v_dual_dot2acc_f32_f16 v5, v3, v4 :: v_dual_dot2acc_f32_f16 v2, v0, v1 -; GFX11: v_add_f32_e32 v0, v2, v5 -; -; GFX1170-LABEL: v_fdot2_opsel_lo_b_dual: -; GFX1170: ; %bb.0: -; GFX1170: v_mov_b16_e32 v1.l, v1.h -; GFX1170: v_dot2_f32_f16 v0, v0, v1, v2 -; GFX1170: v_dot2_f32_f16 v1, v3, v4, v5 -; GFX1170: v_add_f32_e32 v0, v0, v1 +; GFX11PLUS-TRUE16-LABEL: v_fdot2_opsel_lo_b_dual: +; GFX11PLUS-TRUE16: ; %bb.0: +; GFX11PLUS-TRUE16: v_mov_b16_e32 v1.l, v1.h +; GFX11PLUS-TRUE16: v_dual_dot2acc_f32_f16 v5, v3, v4 :: v_dual_dot2acc_f32_f16 v2, v0, v1 +; GFX11PLUS-TRUE16: v_add_f32_e32 v0, v2, v5 ; ; GFX12-LABEL: v_fdot2_opsel_lo_b_dual: ; GFX12: ; %bb.0: ; GFX12: v_perm_b32 v1, v1, v1, 0x7060302 -; GFX12: v_dot2_f32_f16 v0, v0, v1, v2 -; GFX12: v_dot2_f32_f16 v1, v3, v4, v5 -; GFX12: v_add_f32_e32 v0, v0, v1 +; GFX12: v_dual_dot2acc_f32_f16 v5, v3, v4 :: v_dual_dot2acc_f32_f16 v2, v0, v1 +; GFX12: v_add_f32_e32 v0, v2, v5 %shuf = shufflevector <2 x half> %b, <2 x half> poison, <2 x i32> <i32 1, i32 1> %r0 = call float @llvm.amdgcn.fdot2(<2 x half> %a, <2 x half> %shuf, float %c, i1 false) %r1 = call float @llvm.amdgcn.fdot2(<2 x half> %d, <2 x half> %e, float %f, i1 false) @@ -1292,25 +1230,17 @@ define float @v_fdot2_opsel_hi_b_dual(<2 x half> %a, <2 x half> %b, float %c, <2 ; GFX10: v_dot2c_f32_f16 v5, v3, v4 ; GFX10: v_add_f32_e32 v0, v0, v5 ; -; GFX11-LABEL: v_fdot2_opsel_hi_b_dual: -; GFX11: ; %bb.0: -; GFX11: v_mov_b16_e32 v1.h, v1.l -; GFX11: v_dual_dot2acc_f32_f16 v5, v3, v4 :: v_dual_dot2acc_f32_f16 v2, v0, v1 -; GFX11: v_add_f32_e32 v0, v2, v5 -; -; GFX1170-LABEL: v_fdot2_opsel_hi_b_dual: -; GFX1170: ; %bb.0: -; GFX1170: v_mov_b16_e32 v1.h, v1.l -; GFX1170: v_dot2_f32_f16 v0, v0, v1, v2 -; GFX1170: v_dot2_f32_f16 v1, v3, v4, v5 -; GFX1170: v_add_f32_e32 v0, v0, v1 +; GFX11PLUS-TRUE16-LABEL: v_fdot2_opsel_hi_b_dual: +; GFX11PLUS-TRUE16: ; %bb.0: +; GFX11PLUS-TRUE16: v_mov_b16_e32 v1.h, v1.l +; GFX11PLUS-TRUE16: v_dual_dot2acc_f32_f16 v5, v3, v4 :: v_dual_dot2acc_f32_f16 v2, v0, v1 +; GFX11PLUS-TRUE16: v_add_f32_e32 v0, v2, v5 ; ; GFX12-LABEL: v_fdot2_opsel_hi_b_dual: ; GFX12: ; %bb.0: ; GFX12: v_perm_b32 v1, v1, v1, 0x5040100 -; GFX12: v_dot2_f32_f16 v0, v0, v1, v2 -; GFX12: v_dot2_f32_f16 v1, v3, v4, v5 -; GFX12: v_add_f32_e32 v0, v0, v1 +; GFX12: v_dual_dot2acc_f32_f16 v5, v3, v4 :: v_dual_dot2acc_f32_f16 v2, v0, v1 +; GFX12: v_add_f32_e32 v0, v2, v5 %shuf = shufflevector <2 x half> %b, <2 x half> poison, <2 x i32> <i32 0, i32 0> %r0 = call float @llvm.amdgcn.fdot2(<2 x half> %a, <2 x half> %shuf, float %c, i1 false) %r1 = call float @llvm.amdgcn.fdot2(<2 x half> %d, <2 x half> %e, float %f, i1 false) @@ -1340,22 +1270,10 @@ define float @v_fdot2_inline_literal_a_x(<2 x half> %a, <2 x half> %b, float %c, ; GFX10: v_dot2c_f32_f16 v5, v3, v4 ; GFX10: v_add_f32_e32 v0, v2, v5 ; -; GFX11-LABEL: v_fdot2_inline_literal_a_x: -; GFX11: ; %bb.0: -; GFX11: v_dual_dot2acc_f32_f16 v2, 0x40004000, v1 :: v_dual_dot2acc_f32_f16 v5, v3, v4 -; GFX11: v_add_f32_e32 v0, v2, v5 -; -; GFX1170-LABEL: v_fdot2_inline_literal_a_x: -; GFX1170: ; %bb.0: -; GFX1170: v_dot2_f32_f16 v0, 0x40004000, v1, v2 -; GFX1170: v_dot2_f32_f16 v1, v3, v4, v5 -; GFX1170: v_add_f32_e32 v0, v0, v1 -; -; GFX12-LABEL: v_fdot2_inline_literal_a_x: -; GFX12: ; %bb.0: -; GFX12: v_dot2_f32_f16 v0, 0x40004000, v1, v2 -; GFX12: v_dot2_f32_f16 v1, v3, v4, v5 -; GFX12: v_add_f32_e32 v0, v0, v1 +; GFX11PLUS-LABEL: v_fdot2_inline_literal_a_x: +; GFX11PLUS: ; %bb.0: +; GFX11PLUS: v_dual_dot2acc_f32_f16 v2, 0x40004000, v1 :: v_dual_dot2acc_f32_f16 v5, v3, v4 +; GFX11PLUS: v_add_f32_e32 v0, v2, v5 %r0 = call float @llvm.amdgcn.fdot2(<2 x half> <half 2.0, half 2.0>, <2 x half> %b, float %c, i1 false) %r1 = call float @llvm.amdgcn.fdot2(<2 x half> %d, <2 x half> %e, float %f, i1 false) %r = fadd float %r0, %r1 @@ -1381,22 +1299,10 @@ define float @v_fdot2_inline_literal_a_y(<2 x half> %a, <2 x half> %b, float %c, ; GFX10: v_dot2c_f32_f16 v5, 0x40004000, v4 ; GFX10: v_add_f32_e32 v0, v2, v5 ; -; GFX11-LABEL: v_fdot2_inline_literal_a_y: -; GFX11: ; %bb.0: -; GFX11: v_dual_dot2acc_f32_f16 v2, v0, v1 :: v_dual_dot2acc_f32_f16 v5, 0x40004000, v4 -; GFX11: v_add_f32_e32 v0, v2, v5 -; -; GFX1170-LABEL: v_fdot2_inline_literal_a_y: -; GFX1170: ; %bb.0: -; GFX1170: v_dot2_f32_f16 v0, v0, v1, v2 -; GFX1170: v_dot2_f32_f16 v1, 0x40004000, v4, v5 -; GFX1170: v_add_f32_e32 v0, v0, v1 -; -; GFX12-LABEL: v_fdot2_inline_literal_a_y: -; GFX12: ; %bb.0: -; GFX12: v_dot2_f32_f16 v0, v0, v1, v2 -; GFX12: v_dot2_f32_f16 v1, 0x40004000, v4, v5 -; GFX12: v_add_f32_e32 v0, v0, v1 +; GFX11PLUS-LABEL: v_fdot2_inline_literal_a_y: +; GFX11PLUS: ; %bb.0: +; GFX11PLUS: v_dual_dot2acc_f32_f16 v2, v0, v1 :: v_dual_dot2acc_f32_f16 v5, 0x40004000, v4 +; GFX11PLUS: v_add_f32_e32 v0, v2, v5 %r0 = call float @llvm.amdgcn.fdot2(<2 x half> %a, <2 x half> %b, float %c, i1 false) %r1 = call float @llvm.amdgcn.fdot2(<2 x half> <half 2.0, half 2.0>, <2 x half> %e, float %f, i1 false) %r = fadd float %r0, %r1 @@ -1422,22 +1328,10 @@ define float @v_fdot2_inline_literal_a_xy(<2 x half> %a, <2 x half> %b, float %c ; GFX10: v_dot2c_f32_f16 v5, 0x40004000, v4 ; GFX10: v_add_f32_e32 v0, v2, v5 ; -; GFX11-LABEL: v_fdot2_inline_literal_a_xy: -; GFX11: ; %bb.0: -; GFX11: v_dual_dot2acc_f32_f16 v2, 0x40004000, v1 :: v_dual_dot2acc_f32_f16 v5, 0x40004000, v4 -; GFX11: v_add_f32_e32 v0, v2, v5 -; -; GFX1170-LABEL: v_fdot2_inline_literal_a_xy: -; GFX1170: ; %bb.0: -; GFX1170: v_dot2_f32_f16 v0, 0x40004000, v1, v2 -; GFX1170: v_dot2_f32_f16 v1, 0x40004000, v4, v5 -; GFX1170: v_add_f32_e32 v0, v0, v1 -; -; GFX12-LABEL: v_fdot2_inline_literal_a_xy: -; GFX12: ; %bb.0: -; GFX12: v_dot2_f32_f16 v0, 0x40004000, v1, v2 -; GFX12: v_dot2_f32_f16 v1, 0x40004000, v4, v5 -; GFX12: v_add_f32_e32 v0, v0, v1 +; GFX11PLUS-LABEL: v_fdot2_inline_literal_a_xy: +; GFX11PLUS: ; %bb.0: +; GFX11PLUS: v_dual_dot2acc_f32_f16 v2, 0x40004000, v1 :: v_dual_dot2acc_f32_f16 v5, 0x40004000, v4 +; GFX11PLUS: v_add_f32_e32 v0, v2, v5 %r0 = call float @llvm.amdgcn.fdot2(<2 x half> <half 2.0, half 2.0>, <2 x half> %b, float %c, i1 false) %r1 = call float @llvm.amdgcn.fdot2(<2 x half> <half 2.0, half 2.0>, <2 x half> %e, float %f, i1 false) %r = fadd float %r0, %r1 @@ -1463,22 +1357,10 @@ define float @v_fdot2_inline_literal_b_x(<2 x half> %a, <2 x half> %b, float %c, ; GFX10: v_dot2c_f32_f16 v5, v4, v3 ; GFX10: v_add_f32_e32 v0, v2, v5 ; -; GFX11-LABEL: v_fdot2_inline_literal_b_x: -; GFX11: ; %bb.0: -; GFX11: v_dual_dot2acc_f32_f16 v2, 0x40004000, v0 :: v_dual_dot2acc_f32_f16 v5, v4, v3 -; GFX11: v_add_f32_e32 v0, v2, v5 -; -; GFX1170-LABEL: v_fdot2_inline_literal_b_x: -; GFX1170: ; %bb.0: -; GFX1170: v_dot2_f32_f16 v0, v0, 0x40004000, v2 -; GFX1170: v_dot2_f32_f16 v1, v4, v3, v5 -; GFX1170: v_add_f32_e32 v0, v0, v1 -; -; GFX12-LABEL: v_fdot2_inline_literal_b_x: -; GFX12: ; %bb.0: -; GFX12: v_dot2_f32_f16 v0, v0, 0x40004000, v2 -; GFX12: v_dot2_f32_f16 v1, v4, v3, v5 -; GFX12: v_add_f32_e32 v0, v0, v1 +; GFX11PLUS-LABEL: v_fdot2_inline_literal_b_x: +; GFX11PLUS: ; %bb.0: +; GFX11PLUS: v_dual_dot2acc_f32_f16 v2, 0x40004000, v0 :: v_dual_dot2acc_f32_f16 v5, v4, v3 +; GFX11PLUS: v_add_f32_e32 v0, v2, v5 %r0 = call float @llvm.amdgcn.fdot2(<2 x half> %a, <2 x half> <half 2.0, half 2.0>, float %c, i1 false) %r1 = call float @llvm.amdgcn.fdot2(<2 x half> %e, <2 x half> %d, float %f, i1 false) %r = fadd float %r0, %r1 @@ -1504,22 +1386,10 @@ define float @v_fdot2_inline_literal_b_y(<2 x half> %a, <2 x half> %b, float %c, ; GFX10: v_dot2c_f32_f16 v5, 0x40004000, v3 ; GFX10: v_add_f32_e32 v0, v2, v5 ; -; GFX11-LABEL: v_fdot2_inline_literal_b_y: -; GFX11: ; %bb.0: -; GFX11: v_dual_dot2acc_f32_f16 v2, v1, v0 :: v_dual_dot2acc_f32_f16 v5, 0x40004000, v3 -; GFX11: v_add_f32_e32 v0, v2, v5 -; -; GFX1170-LABEL: v_fdot2_inline_literal_b_y: -; GFX1170: ; %bb.0: -; GFX1170: v_dot2_f32_f16 v0, v1, v0, v2 -; GFX1170: v_dot2_f32_f16 v1, v3, 0x40004000, v5 -; GFX1170: v_add_f32_e32 v0, v0, v1 -; -; GFX12-LABEL: v_fdot2_inline_literal_b_y: -; GFX12: ; %bb.0: -; GFX12: v_dot2_f32_f16 v0, v1, v0, v2 -; GFX12: v_dot2_f32_f16 v1, v3, 0x40004000, v5 -; GFX12: v_add_f32_e32 v0, v0, v1 +; GFX11PLUS-LABEL: v_fdot2_inline_literal_b_y: +; GFX11PLUS: ; %bb.0: +; GFX11PLUS: v_dual_dot2acc_f32_f16 v2, v1, v0 :: v_dual_dot2acc_f32_f16 v5, 0x40004000, v3 +; GFX11PLUS: v_add_f32_e32 v0, v2, v5 %r0 = call float @llvm.amdgcn.fdot2(<2 x half> %b, <2 x half> %a, float %c, i1 false) %r1 = call float @llvm.amdgcn.fdot2(<2 x half> %d, <2 x half> <half 2.0, half 2.0>, float %f, i1 false) %r = fadd float %r0, %r1 @@ -1545,22 +1415,10 @@ define float @v_fdot2_inline_literal_b_xy(<2 x half> %a, <2 x half> %b, float %c ; GFX10: v_dot2c_f32_f16 v5, 0x40004000, v3 ; GFX10: v_add_f32_e32 v0, v2, v5 ; -; GFX11-LABEL: v_fdot2_inline_literal_b_xy: -; GFX11: ; %bb.0: -; GFX11: v_dual_dot2acc_f32_f16 v2, 0x40004000, v0 :: v_dual_dot2acc_f32_f16 v5, 0x40004000, v3 -; GFX11: v_add_f32_e32 v0, v2, v5 -; -; GFX1170-LABEL: v_fdot2_inline_literal_b_xy: -; GFX1170: ; %bb.0: -; GFX1170: v_dot2_f32_f16 v0, v0, 0x40004000, v2 -; GFX1170: v_dot2_f32_f16 v1, v3, 0x40004000, v5 -; GFX1170: v_add_f32_e32 v0, v0, v1 -; -; GFX12-LABEL: v_fdot2_inline_literal_b_xy: -; GFX12: ; %bb.0: -; GFX12: v_dot2_f32_f16 v0, v0, 0x40004000, v2 -; GFX12: v_dot2_f32_f16 v1, v3, 0x40004000, v5 -; GFX12: v_add_f32_e32 v0, v0, v1 +; GFX11PLUS-LABEL: v_fdot2_inline_literal_b_xy: +; GFX11PLUS: ; %bb.0: +; GFX11PLUS: v_dual_dot2acc_f32_f16 v2, 0x40004000, v0 :: v_dual_dot2acc_f32_f16 v5, 0x40004000, v3 +; GFX11PLUS: v_add_f32_e32 v0, v2, v5 %r0 = call float @llvm.amdgcn.fdot2(<2 x half> %a, <2 x half> <half 2.0, half 2.0>, float %c, i1 false) %r1 = call float @llvm.amdgcn.fdot2(<2 x half> %d, <2 x half> <half 2.0, half 2.0>, float %f, i1 false) %r = fadd float %r0, %r1 @@ -1597,14 +1455,14 @@ define float @v_fdot2_inline_literal_c_dual(<2 x half> %a, <2 x half> %b, <2 x h ; GFX1170-LABEL: v_fdot2_inline_literal_c_dual: ; GFX1170: ; %bb.0: ; GFX1170: v_dot2_f32_f16 v0, v0, v1, 2.0 -; GFX1170: v_dot2_f32_f16 v1, v2, v3, v4 -; GFX1170: v_add_f32_e32 v0, v0, v1 +; GFX1170: v_dot2_f32_f16 v4, v2, v3, v4 +; GFX1170: v_add_f32_e32 v0, v0, v4 ; ; GFX12-LABEL: v_fdot2_inline_literal_c_dual: ; GFX12: ; %bb.0: ; GFX12: v_dot2_f32_f16 v0, v0, v1, 2.0 -; GFX12: v_dot2_f32_f16 v1, v2, v3, v4 -; GFX12: v_add_f32_e32 v0, v0, v1 +; GFX12: v_dot2_f32_f16 v4, v2, v3, v4 +; GFX12: v_add_f32_e32 v0, v0, v4 %r0 = tail call float @llvm.amdgcn.fdot2(<2 x half> %a, <2 x half> %b, float 2.0, i1 false) %r1 = call float @llvm.amdgcn.fdot2(<2 x half> %d, <2 x half> %e, float %f, i1 false) %r = fadd float %r0, %r1 _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
