llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT--> @llvm/pr-subscribers-backend-amdgpu Author: Matt Arsenault (arsenm) <details> <summary>Changes</summary> Currently the AMDGPU implementation of foldImmediate handles a small set of special cases that require rewriting the instruction opcode (COPY -> mov, and fma-like to fmac-like). Most general immediate folding is handled in SIFoldOperands. Teach PeepholeOpt to fold all inline immediates into target instructions which should always be an improvement. This is to help alleviate some phase ordering problems in future changes. I've also never liked how SIFoldOperands is written and it could use a rewrite, and this will alleviate some of its responsibilties. It will always be necessary since some operand folds require additional context that foldImmediate won't have (namely, we have to make contextually aware tradeoffs for which operands are worth folding to respect constant bus restrictions, considering other users of the values). Co-authored-by: Claude (Opus 4.8) <noreply@<!-- -->anthropic.com> --- Full diff: https://github.com/llvm/llvm-project/pull/208423.diff 4 Files Affected: - (modified) llvm/lib/Target/AMDGPU/SIInstrInfo.cpp (+58-10) - (modified) llvm/test/CodeGen/AMDGPU/optimize-compare.mir (+4-4) - (modified) llvm/test/CodeGen/AMDGPU/peephole-fold-imm.mir (+93-8) - (modified) llvm/test/CodeGen/AMDGPU/sdiv64.ll (+4-6) ``````````diff diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp index 90ab5fd545c11..598c4a3801f97 100644 --- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp +++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp @@ -3569,6 +3569,28 @@ static unsigned getNewFMAMKInst(const GCNSubtarget &ST, unsigned Opc) { } } +// Return true if \p Opc is a mad/fma family which can be converted to to a +// madmk/madak form with a folded literal constant. +static bool isMADFMA(unsigned Opc) { + switch (Opc) { + case AMDGPU::V_MAD_F32_e64: + case AMDGPU::V_MAC_F32_e64: + case AMDGPU::V_MAD_F16_e64: + case AMDGPU::V_MAC_F16_e64: + case AMDGPU::V_FMA_F32_e64: + case AMDGPU::V_FMAC_F32_e64: + case AMDGPU::V_FMA_F16_e64: + case AMDGPU::V_FMAC_F16_e64: + case AMDGPU::V_FMAC_F16_t16_e64: + case AMDGPU::V_FMAC_F16_fake16_e64: + case AMDGPU::V_FMA_F64_e64: + case AMDGPU::V_FMAC_F64_e64: + return true; + default: + return false; + } +} + bool SIInstrInfo::foldImmediate(MachineInstr &UseMI, MachineInstr &DefMI, Register Reg, MachineRegisterInfo *MRI) const { int64_t Imm; @@ -3694,16 +3716,10 @@ bool SIInstrInfo::foldImmediate(MachineInstr &UseMI, MachineInstr &DefMI, return true; } - if (HasMultipleUses) - return false; + if (isMADFMA(Opc)) { + if (HasMultipleUses) + return false; - if (Opc == AMDGPU::V_MAD_F32_e64 || Opc == AMDGPU::V_MAC_F32_e64 || - Opc == AMDGPU::V_MAD_F16_e64 || Opc == AMDGPU::V_MAC_F16_e64 || - Opc == AMDGPU::V_FMA_F32_e64 || Opc == AMDGPU::V_FMAC_F32_e64 || - Opc == AMDGPU::V_FMA_F16_e64 || Opc == AMDGPU::V_FMAC_F16_e64 || - Opc == AMDGPU::V_FMAC_F16_t16_e64 || - Opc == AMDGPU::V_FMAC_F16_fake16_e64 || Opc == AMDGPU::V_FMA_F64_e64 || - Opc == AMDGPU::V_FMAC_F64_e64) { // Don't fold if we are using source or output modifiers. The new VOP2 // instructions don't have them. if (hasAnyModifiersSet(UseMI)) @@ -3899,9 +3915,41 @@ bool SIInstrInfo::foldImmediate(MachineInstr &UseMI, MachineInstr &DefMI, return true; } + + return false; } - return false; + // Early exit for generic instructions which will never fold an immediate. + if (!isTargetSpecificOpcode(UseMI.getOpcode())) + return false; + + // Directly fold inline immediates into the uses. These should be free-ish + // regardless of the uses. + bool FoldedInlineImm = false; + + for (MachineOperand &UseMO : UseMI.explicit_uses()) { + if (!UseMO.isReg() || UseMO.getReg() != Reg) + continue; + + unsigned UseOpIdx = UseMO.getOperandNo(); + + int64_t ImmVal = Imm; + if (unsigned UseSubReg = UseMO.getSubReg()) { + std::optional<int64_t> SubImm = extractSubregFromImm(Imm, UseSubReg); + if (!SubImm) + continue; + ImmVal = *SubImm; + } + + if (!isInlineConstant(UseMI, UseOpIdx, ImmVal)) + continue; + + UseMO.ChangeToImmediate(ImmVal); + FoldedInlineImm = true; + } + + // TODO: Consider applying tryConstantFoldOp here + return FoldedInlineImm; } static bool diff --git a/llvm/test/CodeGen/AMDGPU/optimize-compare.mir b/llvm/test/CodeGen/AMDGPU/optimize-compare.mir index ef9f4498d7b33..671a1d89711e5 100644 --- a/llvm/test/CodeGen/AMDGPU/optimize-compare.mir +++ b/llvm/test/CodeGen/AMDGPU/optimize-compare.mir @@ -2476,7 +2476,7 @@ body: | ; GCN-NEXT: {{ $}} ; GCN-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0 ; GCN-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 1 - ; GCN-NEXT: [[S_ADD_U32_:%[0-9]+]]:sreg_32 = S_ADD_U32 [[COPY]], [[S_MOV_B32_]], implicit-def $scc + ; GCN-NEXT: [[S_ADD_U32_:%[0-9]+]]:sreg_32 = S_ADD_U32 [[COPY]], 1, implicit-def $scc ; GCN-NEXT: S_ENDPGM 0 %0:sreg_32 = COPY $sgpr0 %1:sreg_32 = S_MOV_B32 1 @@ -2512,7 +2512,7 @@ body: | ; GCN-NEXT: {{ $}} ; GCN-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0 ; GCN-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 1 - ; GCN-NEXT: [[S_ADD_U32_:%[0-9]+]]:sreg_32 = S_ADD_U32 [[S_MOV_B32_]], [[COPY]], implicit-def $scc + ; GCN-NEXT: [[S_ADD_U32_:%[0-9]+]]:sreg_32 = S_ADD_U32 1, [[COPY]], implicit-def $scc ; GCN-NEXT: S_ENDPGM 0 %0:sreg_32 = COPY $sgpr0 %1:sreg_32 = S_MOV_B32 1 @@ -2549,7 +2549,7 @@ body: | ; GCN-NEXT: {{ $}} ; GCN-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0 ; GCN-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 3 - ; GCN-NEXT: [[S_ADD_U32_:%[0-9]+]]:sreg_32 = S_ADD_U32 [[COPY]], [[S_MOV_B32_]], implicit-def $scc + ; GCN-NEXT: [[S_ADD_U32_:%[0-9]+]]:sreg_32 = S_ADD_U32 [[COPY]], 3, implicit-def $scc ; GCN-NEXT: S_CMP_EQ_U32 killed [[S_ADD_U32_]], 0, implicit-def $scc ; GCN-NEXT: S_ENDPGM 0 %0:sreg_32 = COPY $sgpr0 @@ -2587,7 +2587,7 @@ body: | ; GCN-NEXT: {{ $}} ; GCN-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0 ; GCN-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4 - ; GCN-NEXT: [[S_ADD_U32_:%[0-9]+]]:sreg_32 = S_ADD_U32 [[S_MOV_B32_]], [[COPY]], implicit-def $scc + ; GCN-NEXT: [[S_ADD_U32_:%[0-9]+]]:sreg_32 = S_ADD_U32 4, [[COPY]], implicit-def $scc ; GCN-NEXT: S_CMP_EQ_U32 killed [[S_ADD_U32_]], 0, implicit-def $scc ; GCN-NEXT: S_ENDPGM 0 %0:sreg_32 = COPY $sgpr0 diff --git a/llvm/test/CodeGen/AMDGPU/peephole-fold-imm.mir b/llvm/test/CodeGen/AMDGPU/peephole-fold-imm.mir index e4fe3b951cf98..273f532a59359 100644 --- a/llvm/test/CodeGen/AMDGPU/peephole-fold-imm.mir +++ b/llvm/test/CodeGen/AMDGPU/peephole-fold-imm.mir @@ -437,9 +437,9 @@ name: fold_v_mov_b64_pseudo_64_to_unaligned body: | bb.0: ; GCN-LABEL: name: fold_v_mov_b64_pseudo_64_to_unaligned - ; GCN: [[V_MOV_B64_PSEUDO:%[0-9]+]]:vreg_64_align2 = V_MOV_B64_PSEUDO 1311768467750121200, implicit $exec - ; GCN-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO 1311768467750121200, implicit $exec - ; GCN-NEXT: SI_RETURN_TO_EPILOG implicit [[V_MOV_B]] + ; GCN: [[V_MOV_B:%[0-9]+]]:vreg_64_align2 = V_MOV_B64_PSEUDO 1311768467750121200, implicit $exec + ; GCN-NEXT: [[V_MOV_B1:%[0-9]+]]:vreg_64 = V_MOV_B64_PSEUDO 1311768467750121200, implicit $exec + ; GCN-NEXT: SI_RETURN_TO_EPILOG implicit [[V_MOV_B1]] %0:vreg_64_align2 = V_MOV_B64_PSEUDO 1311768467750121200, implicit $exec %1:vreg_64 = COPY killed %0 SI_RETURN_TO_EPILOG implicit %1 @@ -872,13 +872,13 @@ name: fold_imm_copy_kill_bug tracksRegLiveness: true body: | bb.0: + ; GCN-LABEL: name: fold_imm_copy_kill_bug - ; GCN: %0:vgpr_32 = V_MOV_B32_e32 0, implicit $exec - ; GCN-NEXT: %1:vgpr_32 = V_MOV_B32_e32 1, implicit $exec - ; GCN-NEXT: %3:vreg_64 = REG_SEQUENCE %1, %subreg.sub0, %0, %subreg.sub1 - ; GCN-NEXT: %4:vreg_64 = REG_SEQUENCE %1, %subreg.sub0, %0, %subreg.sub1 + ; GCN: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec + ; GCN-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec + ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_MOV_B32_e32_1]], %subreg.sub0, [[V_MOV_B32_e32_]], %subreg.sub1 + ; GCN-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_MOV_B32_e32_1]], %subreg.sub0, [[V_MOV_B32_e32_]], %subreg.sub1 ; GCN-NEXT: S_ENDPGM 0 - %0:vgpr_32 = V_MOV_B32_e32 0, implicit $exec %1:vgpr_32 = V_MOV_B32_e32 1, implicit $exec @@ -890,3 +890,88 @@ body: | S_ENDPGM 0 ... + +--- +name: fold_s_mov_b32_inline_imm_valu_inst_0 +body: | + bb.0: + liveins: $vgpr0 + + ; GCN-LABEL: name: fold_s_mov_b32_inline_imm_valu_inst_0 + ; GCN: liveins: $vgpr0 + ; GCN-NEXT: {{ $}} + ; GCN-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 + ; GCN-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 1 + ; GCN-NEXT: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY]], 1, 0, implicit $exec + ; GCN-NEXT: SI_RETURN_TO_EPILOG [[V_ADD_U32_e64_]] + %0:vgpr_32 = COPY $vgpr0 + %1:sreg_32 = S_MOV_B32 1 + %2:vgpr_32 = V_ADD_U32_e64 %0, %1, 0, implicit $exec + SI_RETURN_TO_EPILOG %2 + +... + +--- +name: fold_s_mov_b32_inline_imm_valu_inst_1 +body: | + bb.0: + liveins: $vgpr0 + + ; GCN-LABEL: name: fold_s_mov_b32_inline_imm_valu_inst_1 + ; GCN: liveins: $vgpr0 + ; GCN-NEXT: {{ $}} + ; GCN-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0 + ; GCN-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 1 + ; GCN-NEXT: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 1, [[COPY]], 0, implicit $exec + ; GCN-NEXT: SI_RETURN_TO_EPILOG [[V_ADD_U32_e64_]] + %0:vgpr_32 = COPY $vgpr0 + %1:sreg_32 = S_MOV_B32 1 + %2:vgpr_32 = V_ADD_U32_e64 %1, %0, 0, implicit $exec + SI_RETURN_TO_EPILOG %2 + +... + +--- +name: fold_s_mov_b32_inline_imm_valu_inst_with_multi_uses +body: | + bb.0: + + ; GCN-LABEL: name: fold_s_mov_b32_inline_imm_valu_inst_with_multi_uses + ; GCN: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 1 + ; GCN-NEXT: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 1, 1, 0, implicit $exec + ; GCN-NEXT: SI_RETURN_TO_EPILOG [[V_ADD_U32_e64_]] + %0:sreg_32 = S_MOV_B32 1 + %1:vgpr_32 = V_ADD_U32_e64 %0, %0, 0, implicit $exec + SI_RETURN_TO_EPILOG %1 + +... + +--- +name: fold_s_mov_b64_inline_imm_valu_inst_with_multi_uses +body: | + bb.0: + + ; GCN-LABEL: name: fold_s_mov_b64_inline_imm_valu_inst_with_multi_uses + ; GCN: [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 1 + ; GCN-NEXT: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 1, 0, 0, implicit $exec + ; GCN-NEXT: SI_RETURN_TO_EPILOG [[V_ADD_U32_e64_]] + %0:sreg_64 = S_MOV_B64 1 + %1:vgpr_32 = V_ADD_U32_e64 %0.sub0, %0.sub1, 0, implicit $exec + SI_RETURN_TO_EPILOG %1 + +... + +--- +name: fold_s_mov_b64_imm_subregs_partial_inline_imm_one_inst +body: | + bb.0: + + ; GCN-LABEL: name: fold_s_mov_b64_imm_subregs_partial_inline_imm_one_inst + ; GCN: [[S_MOV_B:%[0-9]+]]:sreg_64 = S_MOV_B64_IMM_PSEUDO 4096 + ; GCN-NEXT: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[S_MOV_B]].sub0, 0, 0, implicit $exec + ; GCN-NEXT: SI_RETURN_TO_EPILOG [[V_ADD_U32_e64_]] + %0:sreg_64 = S_MOV_B64_IMM_PSEUDO 4096 + %1:vgpr_32 = V_ADD_U32_e64 %0.sub0, %0.sub1, 0, implicit $exec + SI_RETURN_TO_EPILOG %1 + +... diff --git a/llvm/test/CodeGen/AMDGPU/sdiv64.ll b/llvm/test/CodeGen/AMDGPU/sdiv64.ll index 9554a03942579..f92ea3f2519fc 100644 --- a/llvm/test/CodeGen/AMDGPU/sdiv64.ll +++ b/llvm/test/CodeGen/AMDGPU/sdiv64.ll @@ -2403,9 +2403,8 @@ define i64 @v_test_sdiv24_pow2_k_num_i64(i64 %x) { ; GCN-NEXT: v_max_i32_e32 v0, v2, v0 ; GCN-NEXT: v_cvt_f32_u32_e32 v2, v0 ; GCN-NEXT: v_sub_i32_e32 v3, vcc, 0, v0 -; GCN-NEXT: s_mov_b32 s4, 0x8000 -; GCN-NEXT: v_rcp_f32_e32 v2, v2 ; GCN-NEXT: v_ashrrev_i32_e32 v1, 31, v1 +; GCN-NEXT: v_rcp_f32_e32 v2, v2 ; GCN-NEXT: v_mul_f32_e32 v2, 0x4f7ffffe, v2 ; GCN-NEXT: v_cvt_u32_f32_e32 v2, v2 ; GCN-NEXT: v_mul_lo_u32 v3, v3, v2 @@ -2414,7 +2413,7 @@ define i64 @v_test_sdiv24_pow2_k_num_i64(i64 %x) { ; GCN-NEXT: v_lshrrev_b32_e32 v2, 17, v2 ; GCN-NEXT: v_mul_u32_u24_e32 v3, v2, v0 ; GCN-NEXT: v_add_i32_e32 v4, vcc, 1, v2 -; GCN-NEXT: v_sub_i32_e32 v3, vcc, s4, v3 +; GCN-NEXT: v_sub_i32_e32 v3, vcc, 0x8000, v3 ; GCN-NEXT: v_cmp_ge_u32_e32 vcc, v3, v0 ; GCN-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc ; GCN-NEXT: v_sub_i32_e64 v4, s[4:5], v3, v0 @@ -2435,9 +2434,8 @@ define i64 @v_test_sdiv24_pow2_k_num_i64(i64 %x) { ; GCN-IR-NEXT: v_max_i32_e32 v0, v2, v0 ; GCN-IR-NEXT: v_cvt_f32_u32_e32 v2, v0 ; GCN-IR-NEXT: v_sub_i32_e32 v3, vcc, 0, v0 -; GCN-IR-NEXT: s_mov_b32 s4, 0x8000 -; GCN-IR-NEXT: v_rcp_f32_e32 v2, v2 ; GCN-IR-NEXT: v_ashrrev_i32_e32 v1, 31, v1 +; GCN-IR-NEXT: v_rcp_f32_e32 v2, v2 ; GCN-IR-NEXT: v_mul_f32_e32 v2, 0x4f7ffffe, v2 ; GCN-IR-NEXT: v_cvt_u32_f32_e32 v2, v2 ; GCN-IR-NEXT: v_mul_lo_u32 v3, v3, v2 @@ -2446,7 +2444,7 @@ define i64 @v_test_sdiv24_pow2_k_num_i64(i64 %x) { ; GCN-IR-NEXT: v_lshrrev_b32_e32 v2, 17, v2 ; GCN-IR-NEXT: v_mul_u32_u24_e32 v3, v2, v0 ; GCN-IR-NEXT: v_add_i32_e32 v4, vcc, 1, v2 -; GCN-IR-NEXT: v_sub_i32_e32 v3, vcc, s4, v3 +; GCN-IR-NEXT: v_sub_i32_e32 v3, vcc, 0x8000, v3 ; GCN-IR-NEXT: v_cmp_ge_u32_e32 vcc, v3, v0 ; GCN-IR-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc ; GCN-IR-NEXT: v_sub_i32_e64 v4, s[4:5], v3, v0 `````````` </details> https://github.com/llvm/llvm-project/pull/208423 _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
