llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT--> @llvm/pr-subscribers-backend-amdgpu Author: lijinpei-amd <details> <summary>Changes</summary> Depends on #<!-- -->239094. getMFMACoExecInfo allowed DS, VMEM, SALU or VALU in the wrong cycles after these gfx950 MFMAs: - SMFMAC 16x16 holds VALU until cycle 3, not 2. Split them from the XDL 16x16 entry, which allows VALU from cycle 2. - SMFMAC 32x32 allows DS, VMEM and VALU from cycle 3, not 4. - DGEMM 16x16x4 allows DS, VMEM and SALU from cycle 1, not 0. MI350X issues VALU at cycle 3 after both SMFMAC sizes. The first commit adds DS, VMEM and SALU expectations to GFX950IssueStages with the current values; the second updates them with the fix. Assisted-by: Claude Opus 5.5 --- Full diff: https://github.com/llvm/llvm-project/pull/239095.diff 2 Files Affected: - (modified) llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp (+12-6) - (modified) llvm/unittests/Target/AMDGPU/MFMACoExecRules.cpp (+15-10) ``````````diff diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp index 0e4441eeca2e3..3beab829f99c5 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUCoExecSchedStrategy.cpp @@ -136,9 +136,14 @@ llvm::AMDGPU::CoExecInfo llvm::AMDGPU::getMFMACoExecInfo(unsigned Opcode) { case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f6_vgprcd_e64: case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f6_gfx940_acd: case V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f6_gfx940_vcd: - // GFX9 Shader Programming Guide lists those SMFMAC separately, but for - // intended purposes here all those instructions are the same. This comment - // is to simplify reverse mapping to the SPG. + Res.TotalWindow = 8; + allowCoExec(Res, CoExecMask::SALU, 1); + allowCoExec(Res, CoExecMask::DS | CoExecMask::VALU | CoExecMask::VMEM, 2); + allowCoExec(Res, CoExecMask::WMMA, 4); + allowCoExec(Res, CoExecMask::PK_F32, 7); + return Res; + + // SMFMAC, 4-cycle occupancy, 8-cycle window. case V_SMFMAC_F32_16X16X64_BF16_e64: case V_SMFMAC_F32_16X16X64_BF16_gfx940: case V_SMFMAC_I32_16X16X128_I8_e64: @@ -155,7 +160,8 @@ llvm::AMDGPU::CoExecInfo llvm::AMDGPU::getMFMACoExecInfo(unsigned Opcode) { case V_SMFMAC_F32_16X16X64_F16_gfx940: Res.TotalWindow = 8; allowCoExec(Res, CoExecMask::SALU, 1); - allowCoExec(Res, CoExecMask::DS | CoExecMask::VALU | CoExecMask::VMEM, 2); + allowCoExec(Res, CoExecMask::DS | CoExecMask::VMEM, 2); + allowCoExec(Res, CoExecMask::VALU, 3); allowCoExec(Res, CoExecMask::WMMA, 4); allowCoExec(Res, CoExecMask::PK_F32, 7); return Res; @@ -371,7 +377,7 @@ llvm::AMDGPU::CoExecInfo llvm::AMDGPU::getMFMACoExecInfo(unsigned Opcode) { case V_SMFMAC_F32_32X32X32_F16_gfx940: Res.TotalWindow = 12; allowCoExec(Res, CoExecMask::SALU, 1); - allowCoExec(Res, CoExecMask::DS | CoExecMask::VALU | CoExecMask::VMEM, 4); + allowCoExec(Res, CoExecMask::DS | CoExecMask::VALU | CoExecMask::VMEM, 3); allowCoExec(Res, CoExecMask::WMMA, 9); allowCoExec(Res, CoExecMask::PK_F32, 11); return Res; @@ -382,7 +388,7 @@ llvm::AMDGPU::CoExecInfo llvm::AMDGPU::getMFMACoExecInfo(unsigned Opcode) { case V_MFMA_F64_16X16X4F64_mac_vgprcd_e64: case V_MFMA_F64_16X16X4F64_vgprcd_e64: Res.TotalWindow = 19; - allowCoExec(Res, CoExecMask::DS | CoExecMask::SALU | CoExecMask::VMEM, 0); + allowCoExec(Res, CoExecMask::DS | CoExecMask::SALU | CoExecMask::VMEM, 1); allowCoExec(Res, CoExecMask::WMMA | CoExecMask::VALU | CoExecMask::PK_F32, 18); return Res; diff --git a/llvm/unittests/Target/AMDGPU/MFMACoExecRules.cpp b/llvm/unittests/Target/AMDGPU/MFMACoExecRules.cpp index 48a9a359d392a..d67d5e0a2ecae 100644 --- a/llvm/unittests/Target/AMDGPU/MFMACoExecRules.cpp +++ b/llvm/unittests/Target/AMDGPU/MFMACoExecRules.cpp @@ -121,7 +121,7 @@ TEST_F(AMDGPUMFMACoExecRules, GFX950IssueStages) { // Opcode names without the _e64, _vgprcd_e64 or _mac*_e64 suffix. std::vector<StringRef> MFMAs; struct { - unsigned VALU, PackedF32; + unsigned VALU, PackedF32, DS, VMEM, SALU; } Expected; } Groups[] = { {{"V_MFMA_F32_16X16X32_BF16", "V_MFMA_F32_16X16X32_F16", @@ -133,7 +133,7 @@ TEST_F(AMDGPUMFMACoExecRules, GFX950IssueStages) { "V_MFMA_SCALE_F32_16X16X128_F8F6F4_f4_f6", "V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f4", "V_MFMA_SCALE_F32_16X16X128_F8F6F4_f6_f6"}, - {2, 7}}, + {2, 7, 2, 2, 1}}, {{"V_MFMA_F32_16X16X128_F8F6F4_f4_f8", "V_MFMA_F32_16X16X128_F8F6F4_f6_f8", "V_MFMA_F32_16X16X128_F8F6F4_f8_f4", @@ -144,7 +144,7 @@ TEST_F(AMDGPUMFMACoExecRules, GFX950IssueStages) { "V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f4", "V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f6", "V_MFMA_SCALE_F32_16X16X128_F8F6F4_f8_f8"}, - {3, 11}}, + {3, 11, 2, 2, 1}}, {{"V_MFMA_F32_32X32X16_BF16", "V_MFMA_F32_32X32X16_F16", "V_MFMA_I32_32X32X32_I8", "V_MFMA_F32_32X32X64_F8F6F4_f4_f4", "V_MFMA_F32_32X32X64_F8F6F4_f4_f6", "V_MFMA_F32_32X32X64_F8F6F4_f6_f4", @@ -153,7 +153,7 @@ TEST_F(AMDGPUMFMACoExecRules, GFX950IssueStages) { "V_MFMA_SCALE_F32_32X32X64_F8F6F4_f4_f6", "V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f4", "V_MFMA_SCALE_F32_32X32X64_F8F6F4_f6_f6"}, - {2, 11}}, + {2, 11, 2, 2, 1}}, {{"V_MFMA_F32_32X32X64_F8F6F4_f4_f8", "V_MFMA_F32_32X32X64_F8F6F4_f6_f8", "V_MFMA_F32_32X32X64_F8F6F4_f8_f4", "V_MFMA_F32_32X32X64_F8F6F4_f8_f6", "V_MFMA_F32_32X32X64_F8F6F4_f8_f8", @@ -162,20 +162,18 @@ TEST_F(AMDGPUMFMACoExecRules, GFX950IssueStages) { "V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f4", "V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f6", "V_MFMA_SCALE_F32_32X32X64_F8F6F4_f8_f8"}, - {3, 19}}, - // The table holds VALU until 2 (16x16) or 4 (32x32); MI350X issues it - // at 3. + {3, 19, 2, 2, 1}}, {{"V_SMFMAC_F32_16X16X64_BF16", "V_SMFMAC_F32_16X16X64_F16", "V_SMFMAC_I32_16X16X128_I8", "V_SMFMAC_F32_16X16X128_BF8_BF8", "V_SMFMAC_F32_16X16X128_BF8_FP8", "V_SMFMAC_F32_16X16X128_FP8_BF8", "V_SMFMAC_F32_16X16X128_FP8_FP8"}, - {2, 7}}, + {3, 7, 2, 2, 1}}, {{"V_SMFMAC_F32_32X32X32_BF16", "V_SMFMAC_F32_32X32X32_F16", "V_SMFMAC_I32_32X32X64_I8", "V_SMFMAC_F32_32X32X64_BF8_BF8", "V_SMFMAC_F32_32X32X64_BF8_FP8", "V_SMFMAC_F32_32X32X64_FP8_BF8", "V_SMFMAC_F32_32X32X64_FP8_FP8"}, - {4, 11}}, - {{"V_MFMA_F64_16X16X4F64"}, {18, 18}}, + {3, 11, 3, 3, 1}}, + {{"V_MFMA_F64_16X16X4F64"}, {18, 18, 1, 1, 1}}, }; for (const auto &G : Groups) { for (StringRef Base : G.MFMAs) { @@ -195,6 +193,13 @@ TEST_F(AMDGPUMFMACoExecRules, GFX950IssueStages) { EXPECT_EQ(Test.getIssueStage(MFMAOpc, V_PK_FMA_F32), G.Expected.PackedF32) << Name; + CoExecInfo Info = getMFMACoExecInfo(MFMAOpc); + EXPECT_EQ(Info.getFirstAllowedStage(CoExecMask::DS), G.Expected.DS) + << Name; + EXPECT_EQ(Info.getFirstAllowedStage(CoExecMask::VMEM), G.Expected.VMEM) + << Name; + EXPECT_EQ(Info.getFirstAllowedStage(CoExecMask::SALU), G.Expected.SALU) + << Name; } EXPECT_GT(NumForms, 0u) << Base; } `````````` </details> https://github.com/llvm/llvm-project/pull/239095 _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
