Author: TelGome Date: 2026-10-22T18:59:25+08:00 New Revision: 4e28cc392c753f184e55a14f54ca1cc950bd525f
URL: https://github.com/llvm/llvm-project/commit/4e28cc392c753f184e55a14f54ca1cc950bd525f DIFF: https://github.com/llvm/llvm-project/commit/4e28cc392c753f184e55a14f54ca1cc950bd525f.diff LOG: [RISCV][P-ext] Support Packed Multiplication with Widening Horizontal Addition and Accumulate intrinsics (#234450) Added: Modified: clang/include/clang/Basic/BuiltinsRISCV.td clang/lib/CodeGen/TargetBuiltins/RISCV.cpp clang/lib/Headers/riscv_packed_simd.h clang/test/CodeGen/RISCV/rvp-intrinsics.c cross-project-tests/intrinsic-header-tests/riscv_packed_simd.c llvm/include/llvm/IR/IntrinsicsRISCV.td llvm/lib/Target/RISCV/RISCVISelLowering.cpp llvm/lib/Target/RISCV/RISCVInstrInfoP.td llvm/test/CodeGen/RISCV/rvp-simd-64.ll Removed: ################################################################################ diff --git a/clang/include/clang/Basic/BuiltinsRISCV.td b/clang/include/clang/Basic/BuiltinsRISCV.td index 4454b14bc7519..08986145b288f 100644 --- a/clang/include/clang/Basic/BuiltinsRISCV.td +++ b/clang/include/clang/Basic/BuiltinsRISCV.td @@ -412,6 +412,14 @@ def pm2wsub_i64 : RISCVBuiltin<"int64_t(_Vector<2, short>, _Vector<2, short>)">; def pm2wsub_x_i64 : RISCVBuiltin<"int64_t(_Vector<2, short>, _Vector<2, short>)">; def pm2waddsu_i64 : RISCVBuiltin<"int64_t(_Vector<2, short>, _Vector<2, unsigned short>)">; +// Packed Multiplication with Widening Horizontal Addition and Accumulate (32-bit) +def pm2wadda_i64 : RISCVBuiltin<"int64_t(int64_t, _Vector<2, short>, _Vector<2, short>)">; +def pm2wadda_x_i64 : RISCVBuiltin<"int64_t(int64_t, _Vector<2, short>, _Vector<2, short>)">; +def pm2waddau_u64 : RISCVBuiltin<"uint64_t(uint64_t, _Vector<2, unsigned short>, _Vector<2, unsigned short>)">; +def pm2wsuba_i64 : RISCVBuiltin<"int64_t(int64_t, _Vector<2, short>, _Vector<2, short>)">; +def pm2wsuba_x_i64 : RISCVBuiltin<"int64_t(int64_t, _Vector<2, short>, _Vector<2, short>)">; +def pm2waddasu_i64 : RISCVBuiltin<"int64_t(int64_t, _Vector<2, short>, _Vector<2, unsigned short>)">; + // Packed Absolute Difference Sum (32-bit) def pabdsumu_u8x4_u32 : RISCVBuiltin<"unsigned int(_Vector<4, unsigned char>, _Vector<4, unsigned char>)">; def pabdsumau_u8x4_u32 : RISCVBuiltin<"unsigned int(unsigned int, _Vector<4, unsigned char>, _Vector<4, unsigned char>)">; diff --git a/clang/lib/CodeGen/TargetBuiltins/RISCV.cpp b/clang/lib/CodeGen/TargetBuiltins/RISCV.cpp index 5a9b0e0b4a191..7c49a78f8c6e8 100644 --- a/clang/lib/CodeGen/TargetBuiltins/RISCV.cpp +++ b/clang/lib/CodeGen/TargetBuiltins/RISCV.cpp @@ -1904,6 +1904,40 @@ Value *CodeGenFunction::EmitRISCVBuiltinExpr(unsigned BuiltinID, break; } + // Packed Multiplication with Widening Horizontal Addition and Accumulate. + case RISCV::BI__builtin_riscv_pm2wadda_i64: + case RISCV::BI__builtin_riscv_pm2wadda_x_i64: + case RISCV::BI__builtin_riscv_pm2waddau_u64: + case RISCV::BI__builtin_riscv_pm2wsuba_i64: + case RISCV::BI__builtin_riscv_pm2wsuba_x_i64: + case RISCV::BI__builtin_riscv_pm2waddasu_i64: { + switch (BuiltinID) { + default: + llvm_unreachable("unexpected builtin ID"); + case RISCV::BI__builtin_riscv_pm2wadda_i64: + ID = Intrinsic::riscv_pm2wadda; + break; + case RISCV::BI__builtin_riscv_pm2wadda_x_i64: + ID = Intrinsic::riscv_pm2wadda_x; + break; + case RISCV::BI__builtin_riscv_pm2waddau_u64: + ID = Intrinsic::riscv_pm2waddau; + break; + case RISCV::BI__builtin_riscv_pm2wsuba_i64: + ID = Intrinsic::riscv_pm2wsuba; + break; + case RISCV::BI__builtin_riscv_pm2wsuba_x_i64: + ID = Intrinsic::riscv_pm2wsuba_x; + break; + case RISCV::BI__builtin_riscv_pm2waddasu_i64: + ID = Intrinsic::riscv_pm2waddasu; + break; + } + + IntrinsicTypes = {ResultType, Ops[1]->getType()}; + break; + } + // Packed Reduction Sum case RISCV::BI__builtin_riscv_predsum_i8x4_i32: case RISCV::BI__builtin_riscv_predsum_i16x2_i32: diff --git a/clang/lib/Headers/riscv_packed_simd.h b/clang/lib/Headers/riscv_packed_simd.h index ef5708689331c..c91cf66d0b31a 100644 --- a/clang/lib/Headers/riscv_packed_simd.h +++ b/clang/lib/Headers/riscv_packed_simd.h @@ -1201,6 +1201,14 @@ __packed_binary_builtin_mixed(pm2wsub_i64, int64_t, int16x2_t, int16x2_t, __buil __packed_binary_builtin_mixed(pm2wsub_x_i64, int64_t, int16x2_t, int16x2_t, __builtin_riscv_pm2wsub_x_i64) __packed_binary_builtin_mixed(pm2waddsu_i64, int64_t, int16x2_t, uint16x2_t, __builtin_riscv_pm2waddsu_i64) +/* Packed Multiplication with Widening Horizontal Addition and Accumulate */ +__packed_ternary_builtin_mixed(pm2wadda_i64, int64_t, int16x2_t, int16x2_t, __builtin_riscv_pm2wadda_i64) +__packed_ternary_builtin_mixed(pm2wadda_x_i64, int64_t, int16x2_t, int16x2_t, __builtin_riscv_pm2wadda_x_i64) +__packed_ternary_builtin_mixed(pm2waddau_u64, uint64_t, uint16x2_t, uint16x2_t, __builtin_riscv_pm2waddau_u64) +__packed_ternary_builtin_mixed(pm2wsuba_i64, int64_t, int16x2_t, int16x2_t, __builtin_riscv_pm2wsuba_i64) +__packed_ternary_builtin_mixed(pm2wsuba_x_i64, int64_t, int16x2_t, int16x2_t, __builtin_riscv_pm2wsuba_x_i64) +__packed_ternary_builtin_mixed(pm2waddasu_i64, int64_t, int16x2_t, uint16x2_t, __builtin_riscv_pm2waddasu_i64) + /* Packed Absolute Difference Sum (32-bit) */ __packed_abdsum(pabdsumu_u8x4_u32, uint32_t, uint8x4_t, __builtin_riscv_pabdsumu_u8x4_u32) __packed_ternary_builtin_cast(pabdsumau_u8x4_u32, uint32_t, uint8x4_t, __builtin_riscv_pabdsumau_u8x4_u32) diff --git a/clang/test/CodeGen/RISCV/rvp-intrinsics.c b/clang/test/CodeGen/RISCV/rvp-intrinsics.c index 74500ed9d79be..2233e5835ea1e 100644 --- a/clang/test/CodeGen/RISCV/rvp-intrinsics.c +++ b/clang/test/CodeGen/RISCV/rvp-intrinsics.c @@ -10152,3 +10152,75 @@ int64_t test_pm2wsub_x_i64(int16x2_t rs1, int16x2_t rs2) { int64_t test_pm2waddsu_i64(int16x2_t rs1, uint16x2_t rs2) { return __riscv_pm2waddsu_i64(rs1, rs2); } + +// CHECK-LABEL: define dso_local i64 @test_pm2wadda_i64( +// CHECK-SAME: i64 noundef [[RD:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] { +// CHECK-NEXT: [[ENTRY:.*:]] +// CHECK-NEXT: [[TMP0:%.*]] = bitcast i32 [[RS1_COERCE]] to <2 x i16> +// CHECK-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS2_COERCE]] to <2 x i16> +// CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.riscv.pm2wadda.i64.v2i16(i64 [[RD]], <2 x i16> [[TMP0]], <2 x i16> [[TMP1]]) +// CHECK-NEXT: ret i64 [[TMP2]] +// +int64_t test_pm2wadda_i64(int64_t rd, int16x2_t rs1, int16x2_t rs2) { + return __riscv_pm2wadda_i64(rd, rs1, rs2); +} + +// CHECK-LABEL: define dso_local i64 @test_pm2wadda_x_i64( +// CHECK-SAME: i64 noundef [[RD:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] { +// CHECK-NEXT: [[ENTRY:.*:]] +// CHECK-NEXT: [[TMP0:%.*]] = bitcast i32 [[RS1_COERCE]] to <2 x i16> +// CHECK-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS2_COERCE]] to <2 x i16> +// CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.riscv.pm2wadda.x.i64.v2i16(i64 [[RD]], <2 x i16> [[TMP0]], <2 x i16> [[TMP1]]) +// CHECK-NEXT: ret i64 [[TMP2]] +// +int64_t test_pm2wadda_x_i64(int64_t rd, int16x2_t rs1, int16x2_t rs2) { + return __riscv_pm2wadda_x_i64(rd, rs1, rs2); +} + +// CHECK-LABEL: define dso_local i64 @test_pm2waddau_u64( +// CHECK-SAME: i64 noundef [[RD:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] { +// CHECK-NEXT: [[ENTRY:.*:]] +// CHECK-NEXT: [[TMP0:%.*]] = bitcast i32 [[RS1_COERCE]] to <2 x i16> +// CHECK-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS2_COERCE]] to <2 x i16> +// CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.riscv.pm2waddau.i64.v2i16(i64 [[RD]], <2 x i16> [[TMP0]], <2 x i16> [[TMP1]]) +// CHECK-NEXT: ret i64 [[TMP2]] +// +uint64_t test_pm2waddau_u64(uint64_t rd, uint16x2_t rs1, uint16x2_t rs2) { + return __riscv_pm2waddau_u64(rd, rs1, rs2); +} + +// CHECK-LABEL: define dso_local i64 @test_pm2wsuba_i64( +// CHECK-SAME: i64 noundef [[RD:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] { +// CHECK-NEXT: [[ENTRY:.*:]] +// CHECK-NEXT: [[TMP0:%.*]] = bitcast i32 [[RS1_COERCE]] to <2 x i16> +// CHECK-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS2_COERCE]] to <2 x i16> +// CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.riscv.pm2wsuba.i64.v2i16(i64 [[RD]], <2 x i16> [[TMP0]], <2 x i16> [[TMP1]]) +// CHECK-NEXT: ret i64 [[TMP2]] +// +int64_t test_pm2wsuba_i64(int64_t rd, int16x2_t rs1, int16x2_t rs2) { + return __riscv_pm2wsuba_i64(rd, rs1, rs2); +} + +// CHECK-LABEL: define dso_local i64 @test_pm2wsuba_x_i64( +// CHECK-SAME: i64 noundef [[RD:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] { +// CHECK-NEXT: [[ENTRY:.*:]] +// CHECK-NEXT: [[TMP0:%.*]] = bitcast i32 [[RS1_COERCE]] to <2 x i16> +// CHECK-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS2_COERCE]] to <2 x i16> +// CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.riscv.pm2wsuba.x.i64.v2i16(i64 [[RD]], <2 x i16> [[TMP0]], <2 x i16> [[TMP1]]) +// CHECK-NEXT: ret i64 [[TMP2]] +// +int64_t test_pm2wsuba_x_i64(int64_t rd, int16x2_t rs1, int16x2_t rs2) { + return __riscv_pm2wsuba_x_i64(rd, rs1, rs2); +} + +// CHECK-LABEL: define dso_local i64 @test_pm2waddasu_i64( +// CHECK-SAME: i64 noundef [[RD:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] { +// CHECK-NEXT: [[ENTRY:.*:]] +// CHECK-NEXT: [[TMP0:%.*]] = bitcast i32 [[RS1_COERCE]] to <2 x i16> +// CHECK-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS2_COERCE]] to <2 x i16> +// CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.riscv.pm2waddasu.i64.v2i16(i64 [[RD]], <2 x i16> [[TMP0]], <2 x i16> [[TMP1]]) +// CHECK-NEXT: ret i64 [[TMP2]] +// +int64_t test_pm2waddasu_i64(int64_t rd, int16x2_t rs1, uint16x2_t rs2) { + return __riscv_pm2waddasu_i64(rd, rs1, rs2); +} diff --git a/cross-project-tests/intrinsic-header-tests/riscv_packed_simd.c b/cross-project-tests/intrinsic-header-tests/riscv_packed_simd.c index 875d2de46a0b3..526ba7bd3c630 100644 --- a/cross-project-tests/intrinsic-header-tests/riscv_packed_simd.c +++ b/cross-project-tests/intrinsic-header-tests/riscv_packed_simd.c @@ -4580,6 +4580,58 @@ int64_t test_pm2waddsu_i64(int16x2_t rs1, uint16x2_t rs2) { return __riscv_pm2waddsu_i64(rs1, rs2); } +// Packed Multiplication with Widening Horizontal Addition and Accumulate +// CHECK-LABEL: test_pm2wadda_i64: +// RV32: pm2wadda.h +// RV64: pack{{[[:space:]]}} +// RV64: pm4adda.h +int64_t test_pm2wadda_i64(int64_t rd, int16x2_t rs1, int16x2_t rs2) { + return __riscv_pm2wadda_i64(rd, rs1, rs2); +} + +// CHECK-LABEL: test_pm2wadda_x_i64: +// RV32: pm2wadda.hx +// RV64: pack{{[[:space:]]}} +// RV64: ppairoe.h +// RV64: pm4adda.h +int64_t test_pm2wadda_x_i64(int64_t rd, int16x2_t rs1, int16x2_t rs2) { + return __riscv_pm2wadda_x_i64(rd, rs1, rs2); +} + +// CHECK-LABEL: test_pm2waddau_u64: +// RV32: pm2waddau.h +// RV64: pack{{[[:space:]]}} +// RV64: pm4addau.h +uint64_t test_pm2waddau_u64(uint64_t rd, uint16x2_t rs1, uint16x2_t rs2) { + return __riscv_pm2waddau_u64(rd, rs1, rs2); +} + +// CHECK-LABEL: test_pm2wsuba_i64: +// RV32: pm2wsuba.h +// RV64: pack{{[[:space:]]}} +// RV64: pm2sub.h +// RV64: predsum.ws +int64_t test_pm2wsuba_i64(int64_t rd, int16x2_t rs1, int16x2_t rs2) { + return __riscv_pm2wsuba_i64(rd, rs1, rs2); +} + +// CHECK-LABEL: test_pm2wsuba_x_i64: +// RV32: pm2wsuba.hx +// RV64: pack{{[[:space:]]}} +// RV64: pm2sub.hx +// RV64: predsum.ws +int64_t test_pm2wsuba_x_i64(int64_t rd, int16x2_t rs1, int16x2_t rs2) { + return __riscv_pm2wsuba_x_i64(rd, rs1, rs2); +} + +// CHECK-LABEL: test_pm2waddasu_i64: +// RV32: pm2waddasu.h +// RV64: pack{{[[:space:]]}} +// RV64: pm4addasu.h +int64_t test_pm2waddasu_i64(int64_t rd, int16x2_t rs1, uint16x2_t rs2) { + return __riscv_pm2waddasu_i64(rd, rs1, rs2); +} + // Packed Multiply Parts. // CHECK-LABEL: test_pmul_b00_i16x2: // RV32: pmul.h.b00 diff --git a/llvm/include/llvm/IR/IntrinsicsRISCV.td b/llvm/include/llvm/IR/IntrinsicsRISCV.td index 171c2afe6d8aa..62240d869496e 100644 --- a/llvm/include/llvm/IR/IntrinsicsRISCV.td +++ b/llvm/include/llvm/IR/IntrinsicsRISCV.td @@ -2327,6 +2327,19 @@ class RVPBinaryIntrinsic def int_riscv_pm2wsub_x : RVPHorizontalMulIntrinsic; def int_riscv_pm2waddsu : RVPHorizontalMulIntrinsic; + // Packed Multiplication with Widening Horizontal Addition and Accumulate. + class RVPHorizontalMulAccIntrinsic + : DefaultAttrsIntrinsic<[llvm_any_ty], + [LLVMMatchType<0>, llvm_anyvector_ty, + LLVMMatchType<1>], + [IntrNoMem, IntrSpeculatable]>; + def int_riscv_pm2wadda : RVPHorizontalMulAccIntrinsic; + def int_riscv_pm2wadda_x : RVPHorizontalMulAccIntrinsic; + def int_riscv_pm2waddau : RVPHorizontalMulAccIntrinsic; + def int_riscv_pm2wsuba : RVPHorizontalMulAccIntrinsic; + def int_riscv_pm2wsuba_x : RVPHorizontalMulAccIntrinsic; + def int_riscv_pm2waddasu : RVPHorizontalMulAccIntrinsic; + // Packed Multiply Parts. The packed forms multiply the selected element of // every pair, so the operands are the result with each element split in two. class RVPPackedMulPartsIntrinsic diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index 5aaa11a9b0474..10d099c711aa8 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -12650,6 +12650,26 @@ static unsigned getRVPHorizontalMulOpcode(unsigned IntNo) { } } +static unsigned getRVPHorizontalMulAccOpcode(unsigned IntNo) { + switch (IntNo) { + default: + llvm_unreachable( + "Unexpected RISC-V packed horizontal multiply-accumulate intrinsic"); + case Intrinsic::riscv_pm2wadda: + return RISCVISD::PM2WADDA; + case Intrinsic::riscv_pm2wadda_x: + return RISCVISD::PM2WADDA_X; + case Intrinsic::riscv_pm2waddau: + return RISCVISD::PM2WADDAU; + case Intrinsic::riscv_pm2wsuba: + return RISCVISD::PM2WSUBA; + case Intrinsic::riscv_pm2wsuba_x: + return RISCVISD::PM2WSUBA_X; + case Intrinsic::riscv_pm2waddasu: + return RISCVISD::PM2WADDASU; + } +} + static SDValue lowerRV32HorizontalMul64(unsigned IntNo, SDValue Rs1, SDValue Rs2, const SDLoc &DL, SelectionDAG &DAG) { @@ -13712,6 +13732,52 @@ SDValue RISCVTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, : RISCVISD::PM4ADD; return DAG.getNode(MulOpc, DL, MVT::i64, A, B); } + case Intrinsic::riscv_pm2wadda: + case Intrinsic::riscv_pm2wadda_x: + case Intrinsic::riscv_pm2waddau: + case Intrinsic::riscv_pm2wsuba: + case Intrinsic::riscv_pm2wsuba_x: + case Intrinsic::riscv_pm2waddasu: { + + assert(Subtarget.is64Bit() && Op.getValueType() == MVT::i64 && + "Unexpected widening horizontal multiply-accumulate legalization"); + SDValue Rd = Op.getOperand(1); + SDValue Rs1 = Op.getOperand(2); + SDValue Rs2 = Op.getOperand(3); + + auto WidenAnyAcc = [&](SDValue V) { + return DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4i16, V, + DAG.getUNDEF(MVT::v2i16)); + }; + SDValue A = WidenAnyAcc(Rs1); + + if (IntNo == Intrinsic::riscv_pm2wsuba || + IntNo == Intrinsic::riscv_pm2wsuba_x) { + + SDValue B = widenPackedVectorWithZeros(DAG, DL, Rs2, MVT::v4i16); + unsigned SubOpc = IntNo == Intrinsic::riscv_pm2wsuba ? RISCVISD::PM2SUB + : RISCVISD::PM2SUB_X; + SDValue Packed = DAG.getNode(SubOpc, DL, MVT::v2i32, A, B); + SDValue SumId = + DAG.getTargetConstant(Intrinsic::riscv_predsum, DL, MVT::i64); + return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::i64, SumId, Packed, + Rd); + } + + SDValue B; + if (IntNo == Intrinsic::riscv_pm2wadda_x) { + SDValue Rs2Wide = widenPackedVectorWithZeros(DAG, DL, Rs2, MVT::v4i16); + B = DAG.getNode(RISCVISD::PPAIROE, DL, MVT::v4i16, Rs2Wide, Rs2Wide); + } else { + B = widenPackedVectorWithZeros(DAG, DL, Rs2, MVT::v4i16); + } + + unsigned AccOpc = IntNo == Intrinsic::riscv_pm2waddau ? RISCVISD::PM4ADDAU + : IntNo == Intrinsic::riscv_pm2waddasu + ? RISCVISD::PM4ADDASU + : RISCVISD::PM4ADDA; + return DAG.getNode(AccOpc, DL, MVT::i64, Rd, A, B); + } case Intrinsic::riscv_pmerge: { EVT VT = Op.getValueType(); auto buildMerge = [&](SDValue Rs1, SDValue Rs2, SDValue Mask, @@ -17932,6 +17998,26 @@ void RISCVTargetLowering::ReplaceNodeResults(SDNode *N, return; } + case Intrinsic::riscv_pm2wadda: + case Intrinsic::riscv_pm2wadda_x: + case Intrinsic::riscv_pm2waddau: + case Intrinsic::riscv_pm2wsuba: + case Intrinsic::riscv_pm2wsuba_x: + case Intrinsic::riscv_pm2waddasu: { + + assert(!Subtarget.is64Bit() && N->getSimpleValueType(0) == MVT::i64 && + "Unexpected widening horizontal multiply-accumulate legalization"); + SDValue Rs1 = N->getOperand(2); + SDValue Rs2 = N->getOperand(3); + auto [RdLo, RdHi] = + DAG.SplitScalar(N->getOperand(1), DL, MVT::i32, MVT::i32); + SDValue Rd = DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v2i32, RdLo, RdHi); + SDValue Res = DAG.getNode(getRVPHorizontalMulAccOpcode(IntNo), DL, + MVT::v2i32, Rd, Rs1, Rs2); + Results.push_back(DAG.getBitcast(MVT::i64, Res)); + return; + } + case Intrinsic::riscv_pmul_00: case Intrinsic::riscv_pmul_01: case Intrinsic::riscv_pmul_11: diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoP.td b/llvm/lib/Target/RISCV/RISCVInstrInfoP.td index 99439fe03dc47..ed7bbd41b60d3 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoP.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoP.td @@ -1942,8 +1942,14 @@ def riscv_pmqrwacc_h : RVSDNode<"PMQRWACC_H", SDT_RISCVPackedWideningAcc>; def riscv_pm2wadda : RVSDNode<"PM2WADDA", SDT_RISCVPackedWideningAcc>; +def riscv_pm2wadda_x + : RVSDNode<"PM2WADDA_X", SDT_RISCVPackedWideningAcc>; def riscv_pm2waddau : RVSDNode<"PM2WADDAU", SDT_RISCVPackedWideningAcc>; +def riscv_pm2wsuba + : RVSDNode<"PM2WSUBA", SDT_RISCVPackedWideningAcc>; +def riscv_pm2wsuba_x + : RVSDNode<"PM2WSUBA_X", SDT_RISCVPackedWideningAcc>; def riscv_pm2waddasu : RVSDNode<"PM2WADDASU", SDT_RISCVPackedWideningAcc>; @@ -2064,6 +2070,16 @@ def riscv_pm2sub_x : RVSDNode<"PM2SUB_X", SDT_RISCVHorizontalMul>; def riscv_pm4addsu : RVSDNode<"PM4ADDSU", SDT_RISCVHorizontalMul>; def riscv_pm2addsu : RVSDNode<"PM2ADDSU", SDT_RISCVHorizontalMul>; +// Horizontal-multiply-and-accumulate internal helpers, used only to +// synthesize the public pm2wadda(_x)/au/asu intrinsics on RV64 (never +// exposed as IR intrinsics themselves). +def SDT_RISCVHorizontalMulAcc + : SDTypeProfile<1, 3, [SDTCisSameAs<0, 1>, SDTCisVec<2>, + SDTCisSameAs<2, 3>]>; +def riscv_pm4adda : RVSDNode<"PM4ADDA", SDT_RISCVHorizontalMulAcc>; +def riscv_pm4addau : RVSDNode<"PM4ADDAU", SDT_RISCVHorizontalMulAcc>; +def riscv_pm4addasu : RVSDNode<"PM4ADDASU", SDT_RISCVHorizontalMulAcc>; + // "Q-format" multiplication def riscv_mulq : RVSDNode<"MULQ", SDTIntBinOp, [SDNPCommutative]>; def riscv_mulqr : RVSDNode<"MULQR", SDTIntBinOp, [SDNPCommutative]>; @@ -2210,6 +2226,12 @@ class PatHorizontalMul<SDPatternOperator OpNode, RVInst Inst, : Pat<(ResultVT (OpNode (SrcVT GPR:$rs1), (SrcVT GPR:$rs2))), (Inst GPR:$rs1, GPR:$rs2)>; +class PatHorizontalMulAcc<SDPatternOperator OpNode, RVInst Inst, + ValueType ResultVT, ValueType SrcVT> + : Pat<(ResultVT (OpNode (ResultVT GPR:$rd), (SrcVT GPR:$rs1), + (SrcVT GPR:$rs2))), + (Inst GPR:$rd, GPR:$rs1, GPR:$rs2)>; + // (rs1 << 1) + rs2 class PatPSh1Add<RVInst Inst, ValueType vt> : Pat<(vt (add (riscv_psll (vt GPR:$rs1), (XLenVT 1)), (vt GPR:$rs2))), @@ -2756,9 +2778,18 @@ let append Predicates = [IsRV32] in { def : Pat<(v2i32 (riscv_pm2wadda (v2i32 GPRPair:$rd), (v2i16 GPR:$rs1), (v2i16 GPR:$rs2))), (PM2WADDA_H GPRPair:$rd, GPR:$rs1, GPR:$rs2)>; + def : Pat<(v2i32 (riscv_pm2wadda_x (v2i32 GPRPair:$rd), + (v2i16 GPR:$rs1), (v2i16 GPR:$rs2))), + (PM2WADDA_HX GPRPair:$rd, GPR:$rs1, GPR:$rs2)>; def : Pat<(v2i32 (riscv_pm2waddau (v2i32 GPRPair:$rd), (v2i16 GPR:$rs1), (v2i16 GPR:$rs2))), (PM2WADDAU_H GPRPair:$rd, GPR:$rs1, GPR:$rs2)>; + def : Pat<(v2i32 (riscv_pm2wsuba (v2i32 GPRPair:$rd), + (v2i16 GPR:$rs1), (v2i16 GPR:$rs2))), + (PM2WSUBA_H GPRPair:$rd, GPR:$rs1, GPR:$rs2)>; + def : Pat<(v2i32 (riscv_pm2wsuba_x (v2i32 GPRPair:$rd), + (v2i16 GPR:$rs1), (v2i16 GPR:$rs2))), + (PM2WSUBA_HX GPRPair:$rd, GPR:$rs1, GPR:$rs2)>; def : Pat<(v2i32 (riscv_pm2waddasu (v2i32 GPRPair:$rd), (v2i16 GPR:$rs1), (v2i16 GPR:$rs2))), (PM2WADDASU_H GPRPair:$rd, GPR:$rs1, GPR:$rs2)>; @@ -3454,6 +3485,9 @@ let append Predicates = [IsRV64] in { def : PatHorizontalMul<riscv_pm4add, PM4ADD_H, i64, v4i16>; def : PatHorizontalMul<riscv_pm4addu, PM4ADDU_H, i64, v4i16>; def : PatHorizontalMul<riscv_pm4addsu, PM4ADDSU_H, i64, v4i16>; + def : PatHorizontalMulAcc<riscv_pm4adda, PM4ADDA_H, i64, v4i16>; + def : PatHorizontalMulAcc<riscv_pm4addau, PM4ADDAU_H, i64, v4i16>; + def : PatHorizontalMulAcc<riscv_pm4addasu, PM4ADDASU_H, i64, v4i16>; // 8/16/32-bit multiply low patterns // FIXME custom lower diff --git a/llvm/test/CodeGen/RISCV/rvp-simd-64.ll b/llvm/test/CodeGen/RISCV/rvp-simd-64.ll index 320b3600e4061..e79a69131e7b8 100644 --- a/llvm/test/CodeGen/RISCV/rvp-simd-64.ll +++ b/llvm/test/CodeGen/RISCV/rvp-simd-64.ll @@ -8177,6 +8177,100 @@ define i64 @test_pm2waddsu_v2i16_i64(<2 x i16> %a, <2 x i16> %b) { ret i64 %r } +; Packed Multiplication with Widening Horizontal Addition and Accumulate. +define i64 @test_pm2wadda_v2i16_i64(i64 %rd, <2 x i16> %a, <2 x i16> %b) { +; RV32-LABEL: test_pm2wadda_v2i16_i64: +; RV32: # %bb.0: +; RV32-NEXT: pm2wadda.h a0, a2, a3 +; RV32-NEXT: ret +; +; RV64-LABEL: test_pm2wadda_v2i16_i64: +; RV64: # %bb.0: +; RV64-NEXT: pack a2, a2, zero +; RV64-NEXT: pm4adda.h a0, a1, a2 +; RV64-NEXT: ret + %r = call i64 @llvm.riscv.pm2wadda.i64.v2i16(i64 %rd, <2 x i16> %a, <2 x i16> %b) + ret i64 %r +} + +define i64 @test_pm2wadda_x_v2i16_i64(i64 %rd, <2 x i16> %a, <2 x i16> %b) { +; RV32-LABEL: test_pm2wadda_x_v2i16_i64: +; RV32: # %bb.0: +; RV32-NEXT: pm2wadda.hx a0, a2, a3 +; RV32-NEXT: ret +; +; RV64-LABEL: test_pm2wadda_x_v2i16_i64: +; RV64: # %bb.0: +; RV64-NEXT: pack a2, a2, zero +; RV64-NEXT: ppairoe.h a2, a2, a2 +; RV64-NEXT: pm4adda.h a0, a1, a2 +; RV64-NEXT: ret + %r = call i64 @llvm.riscv.pm2wadda.x.i64.v2i16(i64 %rd, <2 x i16> %a, <2 x i16> %b) + ret i64 %r +} + +define i64 @test_pm2waddau_v2i16_i64(i64 %rd, <2 x i16> %a, <2 x i16> %b) { +; RV32-LABEL: test_pm2waddau_v2i16_i64: +; RV32: # %bb.0: +; RV32-NEXT: pm2waddau.h a0, a2, a3 +; RV32-NEXT: ret +; +; RV64-LABEL: test_pm2waddau_v2i16_i64: +; RV64: # %bb.0: +; RV64-NEXT: pack a2, a2, zero +; RV64-NEXT: pm4addau.h a0, a1, a2 +; RV64-NEXT: ret + %r = call i64 @llvm.riscv.pm2waddau.i64.v2i16(i64 %rd, <2 x i16> %a, <2 x i16> %b) + ret i64 %r +} + +define i64 @test_pm2wsuba_v2i16_i64(i64 %rd, <2 x i16> %a, <2 x i16> %b) { +; RV32-LABEL: test_pm2wsuba_v2i16_i64: +; RV32: # %bb.0: +; RV32-NEXT: pm2wsuba.h a0, a2, a3 +; RV32-NEXT: ret +; +; RV64-LABEL: test_pm2wsuba_v2i16_i64: +; RV64: # %bb.0: +; RV64-NEXT: pack a2, a2, zero +; RV64-NEXT: pm2sub.h a1, a1, a2 +; RV64-NEXT: predsum.ws a0, a1, a0 +; RV64-NEXT: ret + %r = call i64 @llvm.riscv.pm2wsuba.i64.v2i16(i64 %rd, <2 x i16> %a, <2 x i16> %b) + ret i64 %r +} + +define i64 @test_pm2wsuba_x_v2i16_i64(i64 %rd, <2 x i16> %a, <2 x i16> %b) { +; RV32-LABEL: test_pm2wsuba_x_v2i16_i64: +; RV32: # %bb.0: +; RV32-NEXT: pm2wsuba.hx a0, a2, a3 +; RV32-NEXT: ret +; +; RV64-LABEL: test_pm2wsuba_x_v2i16_i64: +; RV64: # %bb.0: +; RV64-NEXT: pack a2, a2, zero +; RV64-NEXT: pm2sub.hx a1, a1, a2 +; RV64-NEXT: predsum.ws a0, a1, a0 +; RV64-NEXT: ret + %r = call i64 @llvm.riscv.pm2wsuba.x.i64.v2i16(i64 %rd, <2 x i16> %a, <2 x i16> %b) + ret i64 %r +} + +define i64 @test_pm2waddasu_v2i16_i64(i64 %rd, <2 x i16> %a, <2 x i16> %b) { +; RV32-LABEL: test_pm2waddasu_v2i16_i64: +; RV32: # %bb.0: +; RV32-NEXT: pm2waddasu.h a0, a2, a3 +; RV32-NEXT: ret +; +; RV64-LABEL: test_pm2waddasu_v2i16_i64: +; RV64: # %bb.0: +; RV64-NEXT: pack a2, a2, zero +; RV64-NEXT: pm4addasu.h a0, a1, a2 +; RV64-NEXT: ret + %r = call i64 @llvm.riscv.pm2waddasu.i64.v2i16(i64 %rd, <2 x i16> %a, <2 x i16> %b) + ret i64 %r +} + ; Packed Multiply Parts Accumulate. declare <2 x i32> @llvm.riscv.pmacc.00.v2i32(<2 x i32>, <4 x i16>, <4 x i16>) declare <2 x i32> @llvm.riscv.pmacc.01.v2i32(<2 x i32>, <4 x i16>, <4 x i16>) _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
