llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT--> @llvm/pr-subscribers-backend-risc-v Author: TelGome <details> <summary>Changes</summary> Add LLVM intrinsics and codegen support for packed multiplication with widening horizontal addition and accumulate defined in the RISC-V P extension specification: https://github.com/riscv/riscv-p-spec/blob/master/P-ext-intrinsics.adoc#packed-multiplication-with-widening-horizontal-addition-and-accumulate --- Patch is 24.30 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/234450.diff 9 Files Affected: - (modified) clang/include/clang/Basic/BuiltinsRISCV.td (+8) - (modified) clang/lib/CodeGen/TargetBuiltins/RISCV.cpp (+34) - (modified) clang/lib/Headers/riscv_packed_simd.h (+8) - (modified) clang/test/CodeGen/RISCV/rvp-intrinsics.c (+72) - (modified) cross-project-tests/intrinsic-header-tests/riscv_packed_simd.c (+52) - (modified) llvm/include/llvm/IR/IntrinsicsRISCV.td (+13) - (modified) llvm/lib/Target/RISCV/RISCVISelLowering.cpp (+89) - (modified) llvm/lib/Target/RISCV/RISCVInstrInfoP.td (+34) - (modified) llvm/test/CodeGen/RISCV/rvp-simd-64.ll (+94) ``````````diff diff --git a/clang/include/clang/Basic/BuiltinsRISCV.td b/clang/include/clang/Basic/BuiltinsRISCV.td index b7305e46cad6dc..f2e16692876ce9 100644 --- a/clang/include/clang/Basic/BuiltinsRISCV.td +++ b/clang/include/clang/Basic/BuiltinsRISCV.td @@ -404,6 +404,14 @@ def pm2wsub_i64 : RISCVBuiltin<"int64_t(_Vector<2, short>, _Vector<2, short>)">; def pm2wsub_x_i64 : RISCVBuiltin<"int64_t(_Vector<2, short>, _Vector<2, short>)">; def pm2waddsu_i64 : RISCVBuiltin<"int64_t(_Vector<2, short>, _Vector<2, unsigned short>)">; +// Packed Multiplication with Widening Horizontal Addition and Accumulate (32-bit) +def pm2wadda_i64 : RISCVBuiltin<"int64_t(int64_t, _Vector<2, short>, _Vector<2, short>)">; +def pm2wadda_x_i64 : RISCVBuiltin<"int64_t(int64_t, _Vector<2, short>, _Vector<2, short>)">; +def pm2waddau_u64 : RISCVBuiltin<"uint64_t(uint64_t, _Vector<2, unsigned short>, _Vector<2, unsigned short>)">; +def pm2wsuba_i64 : RISCVBuiltin<"int64_t(int64_t, _Vector<2, short>, _Vector<2, short>)">; +def pm2wsuba_x_i64 : RISCVBuiltin<"int64_t(int64_t, _Vector<2, short>, _Vector<2, short>)">; +def pm2waddasu_i64 : RISCVBuiltin<"int64_t(int64_t, _Vector<2, short>, _Vector<2, unsigned short>)">; + // Packed Absolute Difference Sum (32-bit) def pabdsumu_u8x4_u32 : RISCVBuiltin<"unsigned int(_Vector<4, unsigned char>, _Vector<4, unsigned char>)">; def pabdsumau_u8x4_u32 : RISCVBuiltin<"unsigned int(unsigned int, _Vector<4, unsigned char>, _Vector<4, unsigned char>)">; diff --git a/clang/lib/CodeGen/TargetBuiltins/RISCV.cpp b/clang/lib/CodeGen/TargetBuiltins/RISCV.cpp index 879693b9811c39..ebbba76bcee6a3 100644 --- a/clang/lib/CodeGen/TargetBuiltins/RISCV.cpp +++ b/clang/lib/CodeGen/TargetBuiltins/RISCV.cpp @@ -1891,6 +1891,40 @@ Value *CodeGenFunction::EmitRISCVBuiltinExpr(unsigned BuiltinID, break; } + // Packed Multiplication with Widening Horizontal Addition and Accumulate. + case RISCV::BI__builtin_riscv_pm2wadda_i64: + case RISCV::BI__builtin_riscv_pm2wadda_x_i64: + case RISCV::BI__builtin_riscv_pm2waddau_u64: + case RISCV::BI__builtin_riscv_pm2wsuba_i64: + case RISCV::BI__builtin_riscv_pm2wsuba_x_i64: + case RISCV::BI__builtin_riscv_pm2waddasu_i64: { + switch (BuiltinID) { + default: + llvm_unreachable("unexpected builtin ID"); + case RISCV::BI__builtin_riscv_pm2wadda_i64: + ID = Intrinsic::riscv_pm2wadda; + break; + case RISCV::BI__builtin_riscv_pm2wadda_x_i64: + ID = Intrinsic::riscv_pm2wadda_x; + break; + case RISCV::BI__builtin_riscv_pm2waddau_u64: + ID = Intrinsic::riscv_pm2waddau; + break; + case RISCV::BI__builtin_riscv_pm2wsuba_i64: + ID = Intrinsic::riscv_pm2wsuba; + break; + case RISCV::BI__builtin_riscv_pm2wsuba_x_i64: + ID = Intrinsic::riscv_pm2wsuba_x; + break; + case RISCV::BI__builtin_riscv_pm2waddasu_i64: + ID = Intrinsic::riscv_pm2waddasu; + break; + } + + IntrinsicTypes = {ResultType, Ops[1]->getType()}; + break; + } + // Packed Reduction Sum case RISCV::BI__builtin_riscv_predsum_i8x4_i32: case RISCV::BI__builtin_riscv_predsum_i16x2_i32: diff --git a/clang/lib/Headers/riscv_packed_simd.h b/clang/lib/Headers/riscv_packed_simd.h index a4b27eb2e62190..13c8ae8c3fdba8 100644 --- a/clang/lib/Headers/riscv_packed_simd.h +++ b/clang/lib/Headers/riscv_packed_simd.h @@ -1193,6 +1193,14 @@ __packed_binary_builtin_mixed(pm2wsub_i64, int64_t, int16x2_t, int16x2_t, __buil __packed_binary_builtin_mixed(pm2wsub_x_i64, int64_t, int16x2_t, int16x2_t, __builtin_riscv_pm2wsub_x_i64) __packed_binary_builtin_mixed(pm2waddsu_i64, int64_t, int16x2_t, uint16x2_t, __builtin_riscv_pm2waddsu_i64) +/* Packed Multiplication with Widening Horizontal Addition and Accumulate */ +__packed_ternary_builtin_mixed(pm2wadda_i64, int64_t, int16x2_t, int16x2_t, __builtin_riscv_pm2wadda_i64) +__packed_ternary_builtin_mixed(pm2wadda_x_i64, int64_t, int16x2_t, int16x2_t, __builtin_riscv_pm2wadda_x_i64) +__packed_ternary_builtin_mixed(pm2waddau_u64, uint64_t, uint16x2_t, uint16x2_t, __builtin_riscv_pm2waddau_u64) +__packed_ternary_builtin_mixed(pm2wsuba_i64, int64_t, int16x2_t, int16x2_t, __builtin_riscv_pm2wsuba_i64) +__packed_ternary_builtin_mixed(pm2wsuba_x_i64, int64_t, int16x2_t, int16x2_t, __builtin_riscv_pm2wsuba_x_i64) +__packed_ternary_builtin_mixed(pm2waddasu_i64, int64_t, int16x2_t, uint16x2_t, __builtin_riscv_pm2waddasu_i64) + /* Packed Absolute Difference Sum (32-bit) */ __packed_abdsum(pabdsumu_u8x4_u32, uint32_t, uint8x4_t, __builtin_riscv_pabdsumu_u8x4_u32) __packed_ternary_builtin_cast(pabdsumau_u8x4_u32, uint32_t, uint8x4_t, __builtin_riscv_pabdsumau_u8x4_u32) diff --git a/clang/test/CodeGen/RISCV/rvp-intrinsics.c b/clang/test/CodeGen/RISCV/rvp-intrinsics.c index bc9d12a8f59f80..56007cf747581c 100644 --- a/clang/test/CodeGen/RISCV/rvp-intrinsics.c +++ b/clang/test/CodeGen/RISCV/rvp-intrinsics.c @@ -10096,3 +10096,75 @@ int64_t test_pm2wsub_x_i64(int16x2_t rs1, int16x2_t rs2) { int64_t test_pm2waddsu_i64(int16x2_t rs1, uint16x2_t rs2) { return __riscv_pm2waddsu_i64(rs1, rs2); } + +// CHECK-LABEL: define dso_local i64 @test_pm2wadda_i64( +// CHECK-SAME: i64 noundef [[RD:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] { +// CHECK-NEXT: [[ENTRY:.*:]] +// CHECK-NEXT: [[TMP0:%.*]] = bitcast i32 [[RS1_COERCE]] to <2 x i16> +// CHECK-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS2_COERCE]] to <2 x i16> +// CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.riscv.pm2wadda.i64.v2i16(i64 [[RD]], <2 x i16> [[TMP0]], <2 x i16> [[TMP1]]) +// CHECK-NEXT: ret i64 [[TMP2]] +// +int64_t test_pm2wadda_i64(int64_t rd, int16x2_t rs1, int16x2_t rs2) { + return __riscv_pm2wadda_i64(rd, rs1, rs2); +} + +// CHECK-LABEL: define dso_local i64 @test_pm2wadda_x_i64( +// CHECK-SAME: i64 noundef [[RD:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] { +// CHECK-NEXT: [[ENTRY:.*:]] +// CHECK-NEXT: [[TMP0:%.*]] = bitcast i32 [[RS1_COERCE]] to <2 x i16> +// CHECK-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS2_COERCE]] to <2 x i16> +// CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.riscv.pm2wadda.x.i64.v2i16(i64 [[RD]], <2 x i16> [[TMP0]], <2 x i16> [[TMP1]]) +// CHECK-NEXT: ret i64 [[TMP2]] +// +int64_t test_pm2wadda_x_i64(int64_t rd, int16x2_t rs1, int16x2_t rs2) { + return __riscv_pm2wadda_x_i64(rd, rs1, rs2); +} + +// CHECK-LABEL: define dso_local i64 @test_pm2waddau_u64( +// CHECK-SAME: i64 noundef [[RD:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] { +// CHECK-NEXT: [[ENTRY:.*:]] +// CHECK-NEXT: [[TMP0:%.*]] = bitcast i32 [[RS1_COERCE]] to <2 x i16> +// CHECK-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS2_COERCE]] to <2 x i16> +// CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.riscv.pm2waddau.i64.v2i16(i64 [[RD]], <2 x i16> [[TMP0]], <2 x i16> [[TMP1]]) +// CHECK-NEXT: ret i64 [[TMP2]] +// +uint64_t test_pm2waddau_u64(uint64_t rd, uint16x2_t rs1, uint16x2_t rs2) { + return __riscv_pm2waddau_u64(rd, rs1, rs2); +} + +// CHECK-LABEL: define dso_local i64 @test_pm2wsuba_i64( +// CHECK-SAME: i64 noundef [[RD:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] { +// CHECK-NEXT: [[ENTRY:.*:]] +// CHECK-NEXT: [[TMP0:%.*]] = bitcast i32 [[RS1_COERCE]] to <2 x i16> +// CHECK-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS2_COERCE]] to <2 x i16> +// CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.riscv.pm2wsuba.i64.v2i16(i64 [[RD]], <2 x i16> [[TMP0]], <2 x i16> [[TMP1]]) +// CHECK-NEXT: ret i64 [[TMP2]] +// +int64_t test_pm2wsuba_i64(int64_t rd, int16x2_t rs1, int16x2_t rs2) { + return __riscv_pm2wsuba_i64(rd, rs1, rs2); +} + +// CHECK-LABEL: define dso_local i64 @test_pm2wsuba_x_i64( +// CHECK-SAME: i64 noundef [[RD:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] { +// CHECK-NEXT: [[ENTRY:.*:]] +// CHECK-NEXT: [[TMP0:%.*]] = bitcast i32 [[RS1_COERCE]] to <2 x i16> +// CHECK-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS2_COERCE]] to <2 x i16> +// CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.riscv.pm2wsuba.x.i64.v2i16(i64 [[RD]], <2 x i16> [[TMP0]], <2 x i16> [[TMP1]]) +// CHECK-NEXT: ret i64 [[TMP2]] +// +int64_t test_pm2wsuba_x_i64(int64_t rd, int16x2_t rs1, int16x2_t rs2) { + return __riscv_pm2wsuba_x_i64(rd, rs1, rs2); +} + +// CHECK-LABEL: define dso_local i64 @test_pm2waddasu_i64( +// CHECK-SAME: i64 noundef [[RD:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] { +// CHECK-NEXT: [[ENTRY:.*:]] +// CHECK-NEXT: [[TMP0:%.*]] = bitcast i32 [[RS1_COERCE]] to <2 x i16> +// CHECK-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS2_COERCE]] to <2 x i16> +// CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.riscv.pm2waddasu.i64.v2i16(i64 [[RD]], <2 x i16> [[TMP0]], <2 x i16> [[TMP1]]) +// CHECK-NEXT: ret i64 [[TMP2]] +// +int64_t test_pm2waddasu_i64(int64_t rd, int16x2_t rs1, uint16x2_t rs2) { + return __riscv_pm2waddasu_i64(rd, rs1, rs2); +} diff --git a/cross-project-tests/intrinsic-header-tests/riscv_packed_simd.c b/cross-project-tests/intrinsic-header-tests/riscv_packed_simd.c index a54807804b9836..9f784bae2c8035 100644 --- a/cross-project-tests/intrinsic-header-tests/riscv_packed_simd.c +++ b/cross-project-tests/intrinsic-header-tests/riscv_packed_simd.c @@ -4560,6 +4560,58 @@ int64_t test_pm2waddsu_i64(int16x2_t rs1, uint16x2_t rs2) { return __riscv_pm2waddsu_i64(rs1, rs2); } +// Packed Multiplication with Widening Horizontal Addition and Accumulate +// CHECK-LABEL: test_pm2wadda_i64: +// RV32: pm2wadda.h +// RV64: pack{{[[:space:]]}} +// RV64: pm4adda.h +int64_t test_pm2wadda_i64(int64_t rd, int16x2_t rs1, int16x2_t rs2) { + return __riscv_pm2wadda_i64(rd, rs1, rs2); +} + +// CHECK-LABEL: test_pm2wadda_x_i64: +// RV32: pm2wadda.hx +// RV64: pack{{[[:space:]]}} +// RV64: ppairoe.h +// RV64: pm4adda.h +int64_t test_pm2wadda_x_i64(int64_t rd, int16x2_t rs1, int16x2_t rs2) { + return __riscv_pm2wadda_x_i64(rd, rs1, rs2); +} + +// CHECK-LABEL: test_pm2waddau_u64: +// RV32: pm2waddau.h +// RV64: pack{{[[:space:]]}} +// RV64: pm4addau.h +uint64_t test_pm2waddau_u64(uint64_t rd, uint16x2_t rs1, uint16x2_t rs2) { + return __riscv_pm2waddau_u64(rd, rs1, rs2); +} + +// CHECK-LABEL: test_pm2wsuba_i64: +// RV32: pm2wsuba.h +// RV64: pm2sub.h +// RV64: zext.w +// RV64: predsum.ws +int64_t test_pm2wsuba_i64(int64_t rd, int16x2_t rs1, int16x2_t rs2) { + return __riscv_pm2wsuba_i64(rd, rs1, rs2); +} + +// CHECK-LABEL: test_pm2wsuba_x_i64: +// RV32: pm2wsuba.hx +// RV64: pm2sub.hx +// RV64: zext.w +// RV64: predsum.ws +int64_t test_pm2wsuba_x_i64(int64_t rd, int16x2_t rs1, int16x2_t rs2) { + return __riscv_pm2wsuba_x_i64(rd, rs1, rs2); +} + +// CHECK-LABEL: test_pm2waddasu_i64: +// RV32: pm2waddasu.h +// RV64: pack{{[[:space:]]}} +// RV64: pm4addasu.h +int64_t test_pm2waddasu_i64(int64_t rd, int16x2_t rs1, uint16x2_t rs2) { + return __riscv_pm2waddasu_i64(rd, rs1, rs2); +} + // Packed Multiply Parts. // CHECK-LABEL: test_pmul_b00_i16x2: // RV32: pmul.h.b00 diff --git a/llvm/include/llvm/IR/IntrinsicsRISCV.td b/llvm/include/llvm/IR/IntrinsicsRISCV.td index 321d6ac5e93af0..b5fb800309ea4b 100644 --- a/llvm/include/llvm/IR/IntrinsicsRISCV.td +++ b/llvm/include/llvm/IR/IntrinsicsRISCV.td @@ -2319,6 +2319,19 @@ class RVPBinaryIntrinsic def int_riscv_pm2wsub_x : RVPHorizontalMulIntrinsic; def int_riscv_pm2waddsu : RVPHorizontalMulIntrinsic; + // Packed Multiplication with Widening Horizontal Addition and Accumulate. + class RVPHorizontalMulAccIntrinsic + : DefaultAttrsIntrinsic<[llvm_any_ty], + [LLVMMatchType<0>, llvm_anyvector_ty, + LLVMMatchType<1>], + [IntrNoMem, IntrSpeculatable]>; + def int_riscv_pm2wadda : RVPHorizontalMulAccIntrinsic; + def int_riscv_pm2wadda_x : RVPHorizontalMulAccIntrinsic; + def int_riscv_pm2waddau : RVPHorizontalMulAccIntrinsic; + def int_riscv_pm2wsuba : RVPHorizontalMulAccIntrinsic; + def int_riscv_pm2wsuba_x : RVPHorizontalMulAccIntrinsic; + def int_riscv_pm2waddasu : RVPHorizontalMulAccIntrinsic; + // Packed Multiply Parts. The packed forms multiply the selected element of // every pair, so the operands are the result with each element split in two. class RVPPackedMulPartsIntrinsic diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index 0cc05e5f2c374d..e6f01e3c3d636a 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -12647,6 +12647,26 @@ static unsigned getRVPHorizontalMulOpcode(unsigned IntNo) { } } +static unsigned getRVPHorizontalMulAccOpcode(unsigned IntNo) { + switch (IntNo) { + default: + llvm_unreachable( + "Unexpected RISC-V packed horizontal multiply-accumulate intrinsic"); + case Intrinsic::riscv_pm2wadda: + return RISCVISD::PM2WADDA; + case Intrinsic::riscv_pm2wadda_x: + return RISCVISD::PM2WADDA_X; + case Intrinsic::riscv_pm2waddau: + return RISCVISD::PM2WADDAU; + case Intrinsic::riscv_pm2wsuba: + return RISCVISD::PM2WSUBA; + case Intrinsic::riscv_pm2wsuba_x: + return RISCVISD::PM2WSUBA_X; + case Intrinsic::riscv_pm2waddasu: + return RISCVISD::PM2WADDASU; + } +} + static SDValue lowerRV32HorizontalMul64(unsigned IntNo, SDValue Rs1, SDValue Rs2, const SDLoc &DL, SelectionDAG &DAG) { @@ -13706,6 +13726,55 @@ SDValue RISCVTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, : RISCVISD::PM4ADD; return DAG.getNode(MulOpc, DL, MVT::i64, A, B); } + case Intrinsic::riscv_pm2wadda: + case Intrinsic::riscv_pm2wadda_x: + case Intrinsic::riscv_pm2waddau: + case Intrinsic::riscv_pm2wsuba: + case Intrinsic::riscv_pm2wsuba_x: + case Intrinsic::riscv_pm2waddasu: { + + assert(Subtarget.is64Bit() && Op.getValueType() == MVT::i64 && + "Unexpected widening horizontal multiply-accumulate legalization"); + SDValue Rd = Op.getOperand(1); + SDValue Rs1 = Op.getOperand(2); + SDValue Rs2 = Op.getOperand(3); + + if (IntNo == Intrinsic::riscv_pm2wsuba || + IntNo == Intrinsic::riscv_pm2wsuba_x) { + + unsigned SubIntNo = IntNo == Intrinsic::riscv_pm2wsuba + ? Intrinsic::riscv_pm2sub + : Intrinsic::riscv_pm2sub_x; + SDValue SubId = DAG.getTargetConstant(SubIntNo, DL, MVT::i64); + SDValue Diff = + DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::i32, SubId, Rs1, Rs2); + SDValue DiffWide = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, Diff); + SDValue Packed = DAG.getBitcast(MVT::v2i32, DiffWide); + SDValue SumId = + DAG.getTargetConstant(Intrinsic::riscv_predsum, DL, MVT::i64); + return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::i64, SumId, Packed, + Rd); + } + + auto WidenAnyAcc = [&](SDValue V) { + return DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4i16, V, + DAG.getUNDEF(MVT::v2i16)); + }; + SDValue A = WidenAnyAcc(Rs1); + SDValue B; + if (IntNo == Intrinsic::riscv_pm2wadda_x) { + SDValue Rs2Wide = widenPackedVectorWithZeros(DAG, DL, Rs2, MVT::v4i16); + B = DAG.getNode(RISCVISD::PPAIROE, DL, MVT::v4i16, Rs2Wide, Rs2Wide); + } else { + B = widenPackedVectorWithZeros(DAG, DL, Rs2, MVT::v4i16); + } + + unsigned AccOpc = IntNo == Intrinsic::riscv_pm2waddau ? RISCVISD::PM4ADDAU + : IntNo == Intrinsic::riscv_pm2waddasu + ? RISCVISD::PM4ADDASU + : RISCVISD::PM4ADDA; + return DAG.getNode(AccOpc, DL, MVT::i64, Rd, A, B); + } case Intrinsic::riscv_pmerge: { EVT VT = Op.getValueType(); auto buildMerge = [&](SDValue Rs1, SDValue Rs2, SDValue Mask, @@ -17926,6 +17995,26 @@ void RISCVTargetLowering::ReplaceNodeResults(SDNode *N, return; } + case Intrinsic::riscv_pm2wadda: + case Intrinsic::riscv_pm2wadda_x: + case Intrinsic::riscv_pm2waddau: + case Intrinsic::riscv_pm2wsuba: + case Intrinsic::riscv_pm2wsuba_x: + case Intrinsic::riscv_pm2waddasu: { + + assert(!Subtarget.is64Bit() && N->getSimpleValueType(0) == MVT::i64 && + "Unexpected widening horizontal multiply-accumulate legalization"); + SDValue Rs1 = N->getOperand(2); + SDValue Rs2 = N->getOperand(3); + auto [RdLo, RdHi] = + DAG.SplitScalar(N->getOperand(1), DL, MVT::i32, MVT::i32); + SDValue Rd = DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v2i32, RdLo, RdHi); + SDValue Res = DAG.getNode(getRVPHorizontalMulAccOpcode(IntNo), DL, + MVT::v2i32, Rd, Rs1, Rs2); + Results.push_back(DAG.getBitcast(MVT::i64, Res)); + return; + } + case Intrinsic::riscv_pmul_00: case Intrinsic::riscv_pmul_01: case Intrinsic::riscv_pmul_11: diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoP.td b/llvm/lib/Target/RISCV/RISCVInstrInfoP.td index b344638a754c88..d1e5782f0cb6eb 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoP.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoP.td @@ -1942,8 +1942,14 @@ def riscv_pmqrwacc_h : RVSDNode<"PMQRWACC_H", SDT_RISCVPackedWideningAcc>; def riscv_pm2wadda : RVSDNode<"PM2WADDA", SDT_RISCVPackedWideningAcc>; +def riscv_pm2wadda_x + : RVSDNode<"PM2WADDA_X", SDT_RISCVPackedWideningAcc>; def riscv_pm2waddau : RVSDNode<"PM2WADDAU", SDT_RISCVPackedWideningAcc>; +def riscv_pm2wsuba + : RVSDNode<"PM2WSUBA", SDT_RISCVPackedWideningAcc>; +def riscv_pm2wsuba_x + : RVSDNode<"PM2WSUBA_X", SDT_RISCVPackedWideningAcc>; def riscv_pm2waddasu : RVSDNode<"PM2WADDASU", SDT_RISCVPackedWideningAcc>; @@ -2064,6 +2070,16 @@ def riscv_pm2sub_x : RVSDNode<"PM2SUB_X", SDT_RISCVHorizontalMul>; def riscv_pm4addsu : RVSDNode<"PM4ADDSU", SDT_RISCVHorizontalMul>; def riscv_pm2addsu : RVSDNode<"PM2ADDSU", SDT_RISCVHorizontalMul>; +// Horizontal-multiply-and-accumulate internal helpers, used only to +// synthesize the public pm2wadda(_x)/au/asu intrinsics on RV64 (never +// exposed as IR intrinsics themselves). +def SDT_RISCVHorizontalMulAcc + : SDTypeProfile<1, 3, [SDTCisSameAs<0, 1>, SDTCisVec<2>, + SDTCisSameAs<2, 3>]>; +def riscv_pm4adda : RVSDNode<"PM4ADDA", SDT_RISCVHorizontalMulAcc>; +def riscv_pm4addau : RVSDNode<"PM4ADDAU", SDT_RISCVHorizontalMulAcc>; +def riscv_pm4addasu : RVSDNode<"PM4ADDASU", SDT_RISCVHorizontalMulAcc>; + // "Q-format" multiplication def riscv_mulq : RVSDNode<"MULQ", SDTIntBinOp, [SDNPCommutative]>; def riscv_mulqr : RVSDNode<"MULQR", SDTIntBinOp, [SDNPCommutative]>; @@ -2208,6 +2224,12 @@ class PatHorizontalMul<SDPatternOperator OpNode, RVInst Inst, : Pat<(ResultVT (OpNode (SrcVT GPR:$rs1), (SrcVT GPR:$rs2))), (Inst GPR:$rs1, GPR:$rs2)>; +class PatHorizontalMulAcc<SDPatternOperator OpNode, RVInst Inst, + ValueType ResultVT, ValueType SrcVT> + : Pat<(ResultVT (OpNode (ResultVT GPR:$rd), (SrcVT GPR:$rs1), + (SrcVT GPR:$rs2))), + (Inst GPR:$rd, GPR:$rs1, GPR:$rs2)>; + // (rs1 << 1) + rs2 class PatPSh1Add<RVInst Inst, ValueType vt> : Pat<(vt (add (riscv_psll (vt GPR:$rs1), (XLenVT 1)), (vt GPR:$rs2))), @@ -2754,9 +2776,18 @@ let append Predicates = [IsRV32] in { def : Pat<(v2i32 (riscv_pm2wadda (v2i32 GPRPair:$rd), (v2i16 GPR:$rs1), (v2i16 GPR:$rs2))), (PM2WADDA_H GPRPair:$rd, GPR:$rs1, GPR:$rs2)>; + def : Pat<(v2i32 (riscv_pm2wadda_x (v2i32 GPRPair:$rd), + (v2i16 GPR:$rs1), (v2i16 GPR:$rs2))), + (PM2WADDA_HX GPRPair:$rd, GPR:$rs1, GPR:$rs2)>; def : Pat<(v2i32 (riscv_pm2waddau (v2i32 GPRPair:$rd), (v2i16 GPR:$rs1), (v2i16 GPR:$rs2))), (PM2WADDAU_H GPRPair:$rd, GPR:$rs1, GPR:$rs2)>; + def : Pat<(v2i32 (riscv_pm2wsuba (v2i32 GPRPair:$rd), + (v2i16 GPR:$rs1), (v2i16 GPR:$rs2))), + (PM2WSUBA_H GPRPair:$rd, GPR:$rs1, GPR:$rs2)>; + def : Pat<(v2i32 (riscv_pm2wsuba_x (v2i32 GPRPair:$rd), + ... [truncated] `````````` </details> https://github.com/llvm/llvm-project/pull/234450 _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
