https://github.com/TelGome updated https://github.com/llvm/llvm-project/pull/217918
>From d1440f24bdc679ca3e90182e90845ee20cd49788 Mon Sep 17 00:00:00 2001 From: Dongyan Chen <[email protected]> Date: Fri, 21 Aug 2026 13:05:05 +0000 Subject: [PATCH 1/2] [RISCV][P-ext] Support Packed "Q-format" Multiply Parts Accumulate --- clang/include/clang/Basic/BuiltinsRISCV.td | 22 + clang/lib/CodeGen/TargetBuiltins/RISCV.cpp | 82 ++++ clang/lib/Headers/riscv_packed_simd.h | 32 +- clang/test/CodeGen/RISCV/rvp-intrinsics.c | 388 ++++++++++++++++++ .../riscv_packed_simd.c | 126 ++++++ llvm/include/llvm/IR/IntrinsicsRISCV.td | 25 ++ llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp | 16 + llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 211 ++++++++++ llvm/lib/Target/RISCV/RISCVInstrInfoP.td | 122 ++++++ llvm/test/CodeGen/RISCV/rvp-simd-32.ll | 96 +++++ llvm/test/CodeGen/RISCV/rvp-simd-64.ll | 177 ++++++++ 11 files changed, 1292 insertions(+), 5 deletions(-) diff --git a/clang/include/clang/Basic/BuiltinsRISCV.td b/clang/include/clang/Basic/BuiltinsRISCV.td index 89dbf736f4435..b292ffc0140e8 100644 --- a/clang/include/clang/Basic/BuiltinsRISCV.td +++ b/clang/include/clang/Basic/BuiltinsRISCV.td @@ -319,6 +319,28 @@ def pmulqr_i16x4 : RISCVBuiltin<"_Vector<4, short>(_Vector<4, short>, _Vector<4, def pmulq_i32x2 : RISCVBuiltin<"_Vector<2, int>(_Vector<2, int>, _Vector<2, int>)">; def pmulqr_i32x2 : RISCVBuiltin<"_Vector<2, int>(_Vector<2, int>, _Vector<2, int>)">; +// Packed "Q-format" Multiply Parts Accumulate (32-bit) +def mqacc_h00_i32 : RISCVBuiltin<"int(int, _Vector<2, short>, _Vector<2, short>)">; +def mqacc_h01_i32 : RISCVBuiltin<"int(int, _Vector<2, short>, _Vector<2, short>)">; +def mqacc_h11_i32 : RISCVBuiltin<"int(int, _Vector<2, short>, _Vector<2, short>)">; +def mqracc_h00_i32 : RISCVBuiltin<"int(int, _Vector<2, short>, _Vector<2, short>)">; +def mqracc_h01_i32 : RISCVBuiltin<"int(int, _Vector<2, short>, _Vector<2, short>)">; +def mqracc_h11_i32 : RISCVBuiltin<"int(int, _Vector<2, short>, _Vector<2, short>)">; + +// Packed "Q-format" Multiply Parts Accumulate (64-bit) +def pmqacc_h00_i32x2 : RISCVBuiltin<"_Vector<2, int>(_Vector<2, int>, _Vector<4, short>, _Vector<4, short>)">; +def pmqacc_h01_i32x2 : RISCVBuiltin<"_Vector<2, int>(_Vector<2, int>, _Vector<4, short>, _Vector<4, short>)">; +def pmqacc_h11_i32x2 : RISCVBuiltin<"_Vector<2, int>(_Vector<2, int>, _Vector<4, short>, _Vector<4, short>)">; +def pmqracc_h00_i32x2 : RISCVBuiltin<"_Vector<2, int>(_Vector<2, int>, _Vector<4, short>, _Vector<4, short>)">; +def pmqracc_h01_i32x2 : RISCVBuiltin<"_Vector<2, int>(_Vector<2, int>, _Vector<4, short>, _Vector<4, short>)">; +def pmqracc_h11_i32x2 : RISCVBuiltin<"_Vector<2, int>(_Vector<2, int>, _Vector<4, short>, _Vector<4, short>)">; +def mqacc_w00_i64 : RISCVBuiltin<"int64_t(int64_t, _Vector<2, int>, _Vector<2, int>)">; +def mqacc_w01_i64 : RISCVBuiltin<"int64_t(int64_t, _Vector<2, int>, _Vector<2, int>)">; +def mqacc_w11_i64 : RISCVBuiltin<"int64_t(int64_t, _Vector<2, int>, _Vector<2, int>)">; +def mqracc_w00_i64 : RISCVBuiltin<"int64_t(int64_t, _Vector<2, int>, _Vector<2, int>)">; +def mqracc_w01_i64 : RISCVBuiltin<"int64_t(int64_t, _Vector<2, int>, _Vector<2, int>)">; +def mqracc_w11_i64 : RISCVBuiltin<"int64_t(int64_t, _Vector<2, int>, _Vector<2, int>)">; + // Packed Sign and Zero Extend (32-bit) def psext_b_i16x2 : RISCVBuiltin<"_Vector<2, short>(_Vector<2, short>)">; def pzext_b_u16x2 : RISCVBuiltin<"_Vector<2, unsigned short>(_Vector<2, unsigned short>)">; diff --git a/clang/lib/CodeGen/TargetBuiltins/RISCV.cpp b/clang/lib/CodeGen/TargetBuiltins/RISCV.cpp index 88fedecbee504..91f451e34c2c9 100644 --- a/clang/lib/CodeGen/TargetBuiltins/RISCV.cpp +++ b/clang/lib/CodeGen/TargetBuiltins/RISCV.cpp @@ -1626,6 +1626,88 @@ Value *CodeGenFunction::EmitRISCVBuiltinExpr(unsigned BuiltinID, break; } + // Packed "Q-format" Multiply Parts Accumulate + case RISCV::BI__builtin_riscv_mqacc_h00_i32: + case RISCV::BI__builtin_riscv_mqacc_h01_i32: + case RISCV::BI__builtin_riscv_mqacc_h11_i32: + case RISCV::BI__builtin_riscv_mqracc_h00_i32: + case RISCV::BI__builtin_riscv_mqracc_h01_i32: + case RISCV::BI__builtin_riscv_mqracc_h11_i32: + case RISCV::BI__builtin_riscv_pmqacc_h00_i32x2: + case RISCV::BI__builtin_riscv_pmqacc_h01_i32x2: + case RISCV::BI__builtin_riscv_pmqacc_h11_i32x2: + case RISCV::BI__builtin_riscv_pmqracc_h00_i32x2: + case RISCV::BI__builtin_riscv_pmqracc_h01_i32x2: + case RISCV::BI__builtin_riscv_pmqracc_h11_i32x2: + case RISCV::BI__builtin_riscv_mqacc_w00_i64: + case RISCV::BI__builtin_riscv_mqacc_w01_i64: + case RISCV::BI__builtin_riscv_mqacc_w11_i64: + case RISCV::BI__builtin_riscv_mqracc_w00_i64: + case RISCV::BI__builtin_riscv_mqracc_w01_i64: + case RISCV::BI__builtin_riscv_mqracc_w11_i64: { + switch (BuiltinID) { + default: + llvm_unreachable("unexpected builtin ID"); + case RISCV::BI__builtin_riscv_mqacc_h00_i32: + ID = Intrinsic::riscv_mqacc_h00; + break; + case RISCV::BI__builtin_riscv_mqacc_h01_i32: + ID = Intrinsic::riscv_mqacc_h01; + break; + case RISCV::BI__builtin_riscv_mqacc_h11_i32: + ID = Intrinsic::riscv_mqacc_h11; + break; + case RISCV::BI__builtin_riscv_mqracc_h00_i32: + ID = Intrinsic::riscv_mqracc_h00; + break; + case RISCV::BI__builtin_riscv_mqracc_h01_i32: + ID = Intrinsic::riscv_mqracc_h01; + break; + case RISCV::BI__builtin_riscv_mqracc_h11_i32: + ID = Intrinsic::riscv_mqracc_h11; + break; + case RISCV::BI__builtin_riscv_pmqacc_h00_i32x2: + ID = Intrinsic::riscv_pmqacc_h00; + break; + case RISCV::BI__builtin_riscv_pmqacc_h01_i32x2: + ID = Intrinsic::riscv_pmqacc_h01; + break; + case RISCV::BI__builtin_riscv_pmqacc_h11_i32x2: + ID = Intrinsic::riscv_pmqacc_h11; + break; + case RISCV::BI__builtin_riscv_pmqracc_h00_i32x2: + ID = Intrinsic::riscv_pmqracc_h00; + break; + case RISCV::BI__builtin_riscv_pmqracc_h01_i32x2: + ID = Intrinsic::riscv_pmqracc_h01; + break; + case RISCV::BI__builtin_riscv_pmqracc_h11_i32x2: + ID = Intrinsic::riscv_pmqracc_h11; + break; + case RISCV::BI__builtin_riscv_mqacc_w00_i64: + ID = Intrinsic::riscv_mqacc_w00; + break; + case RISCV::BI__builtin_riscv_mqacc_w01_i64: + ID = Intrinsic::riscv_mqacc_w01; + break; + case RISCV::BI__builtin_riscv_mqacc_w11_i64: + ID = Intrinsic::riscv_mqacc_w11; + break; + case RISCV::BI__builtin_riscv_mqracc_w00_i64: + ID = Intrinsic::riscv_mqracc_w00; + break; + case RISCV::BI__builtin_riscv_mqracc_w01_i64: + ID = Intrinsic::riscv_mqracc_w01; + break; + case RISCV::BI__builtin_riscv_mqracc_w11_i64: + ID = Intrinsic::riscv_mqracc_w11; + break; + } + + IntrinsicTypes = {ResultType, Ops[1]->getType()}; + break; + } + // Zk builtins // Zknh diff --git a/clang/lib/Headers/riscv_packed_simd.h b/clang/lib/Headers/riscv_packed_simd.h index 318900805043c..aaf7112d583f4 100644 --- a/clang/lib/Headers/riscv_packed_simd.h +++ b/clang/lib/Headers/riscv_packed_simd.h @@ -285,7 +285,7 @@ typedef uint32_t uint32x2_t __attribute__((__vector_size__(8))); return builtin(__rs1, __rs2); \ } -#define __packed_abdsum_acc(name, rty, ty, builtin) \ +#define __packed_ternary_builtin(name, rty, ty, builtin) \ static __inline__ rty __DEFAULT_FN_ATTRS __riscv_##name(rty __rd, ty __rs1, \ ty __rs2) { \ return builtin(__rd, __rs1, __rs2); \ @@ -815,13 +815,13 @@ __packed_ternary_builtin_mixed(pmhraccsu_i32x2, int32x2_t, int32x2_t, uint32x2_t /* Packed Absolute Difference Sum (32-bit) */ __packed_abdsum(pabdsumu_u8x4_u32, uint32_t, uint8x4_t, __builtin_riscv_pabdsumu_u8x4_u32) -__packed_abdsum_acc(pabdsumau_u8x4_u32, uint32_t, uint8x4_t, __builtin_riscv_pabdsumau_u8x4_u32) +__packed_ternary_builtin(pabdsumau_u8x4_u32, uint32_t, uint8x4_t, __builtin_riscv_pabdsumau_u8x4_u32) /* Packed Absolute Difference Sum (64-bit) */ __packed_abdsum(pabdsumu_u8x8_u32, uint32_t, uint8x8_t, __builtin_riscv_pabdsumu_u8x8_u32) __packed_abdsum(pabdsumu_u8x8_u64, uint64_t, uint8x8_t, __builtin_riscv_pabdsumu_u8x8_u64) -__packed_abdsum_acc(pabdsumau_u8x8_u32, uint32_t, uint8x8_t, __builtin_riscv_pabdsumau_u8x8_u32) -__packed_abdsum_acc(pabdsumau_u8x8_u64, uint64_t, uint8x8_t, __builtin_riscv_pabdsumau_u8x8_u64) +__packed_ternary_builtin(pabdsumau_u8x8_u32, uint32_t, uint8x8_t, __builtin_riscv_pabdsumau_u8x8_u32) +__packed_ternary_builtin(pabdsumau_u8x8_u64, uint64_t, uint8x8_t, __builtin_riscv_pabdsumau_u8x8_u64) /* Packed Saturating Absolute Value (32-bit) */ __packed_unary_builtin(psabs_i8x4, int8x4_t, __builtin_riscv_psabs_i8x4) @@ -852,6 +852,28 @@ __packed_binary_builtin(pmulqr_i16x4, int16x4_t, __builtin_riscv_pmulqr_i16x4) __packed_binary_builtin(pmulq_i32x2, int32x2_t, __builtin_riscv_pmulq_i32x2) __packed_binary_builtin(pmulqr_i32x2, int32x2_t, __builtin_riscv_pmulqr_i32x2) +/* Packed "Q-format" Multiply Parts Accumulate (32-bit) */ +__packed_ternary_builtin(mqacc_h00_i32, int, int16x2_t, __builtin_riscv_mqacc_h00_i32) +__packed_ternary_builtin(mqacc_h01_i32, int, int16x2_t, __builtin_riscv_mqacc_h01_i32) +__packed_ternary_builtin(mqacc_h11_i32, int, int16x2_t, __builtin_riscv_mqacc_h11_i32) +__packed_ternary_builtin(mqracc_h00_i32, int, int16x2_t, __builtin_riscv_mqracc_h00_i32) +__packed_ternary_builtin(mqracc_h01_i32, int, int16x2_t, __builtin_riscv_mqracc_h01_i32) +__packed_ternary_builtin(mqracc_h11_i32, int, int16x2_t, __builtin_riscv_mqracc_h11_i32) + +/* Packed "Q-format" Multiply Parts Accumulate (64-bit) */ +__packed_ternary_builtin(pmqacc_h00_i32x2, int32x2_t, int16x4_t, __builtin_riscv_pmqacc_h00_i32x2) +__packed_ternary_builtin(pmqacc_h01_i32x2, int32x2_t, int16x4_t, __builtin_riscv_pmqacc_h01_i32x2) +__packed_ternary_builtin(pmqacc_h11_i32x2, int32x2_t, int16x4_t, __builtin_riscv_pmqacc_h11_i32x2) +__packed_ternary_builtin(pmqracc_h00_i32x2, int32x2_t, int16x4_t, __builtin_riscv_pmqracc_h00_i32x2) +__packed_ternary_builtin(pmqracc_h01_i32x2, int32x2_t, int16x4_t, __builtin_riscv_pmqracc_h01_i32x2) +__packed_ternary_builtin(pmqracc_h11_i32x2, int32x2_t, int16x4_t, __builtin_riscv_pmqracc_h11_i32x2) +__packed_ternary_builtin(mqacc_w00_i64, int64_t, int32x2_t, __builtin_riscv_mqacc_w00_i64) +__packed_ternary_builtin(mqacc_w01_i64, int64_t, int32x2_t, __builtin_riscv_mqacc_w01_i64) +__packed_ternary_builtin(mqacc_w11_i64, int64_t, int32x2_t, __builtin_riscv_mqacc_w11_i64) +__packed_ternary_builtin(mqracc_w00_i64, int64_t, int32x2_t, __builtin_riscv_mqracc_w00_i64) +__packed_ternary_builtin(mqracc_w01_i64, int64_t, int32x2_t, __builtin_riscv_mqracc_w01_i64) +__packed_ternary_builtin(mqracc_w11_i64, int64_t, int32x2_t, __builtin_riscv_mqracc_w11_i64) + /* Packed Narrowing Clip Pair (32-bit) */ __packed_binary_builtin_cast(pnclipp_i8x4, int16x2_t, int8x4_t, __builtin_riscv_pnclipp_i8x4) __packed_binary_builtin_cast(pnclipup_u8x4, uint16x2_t, uint8x4_t, __builtin_riscv_pnclipup_u8x4) @@ -1012,7 +1034,7 @@ __packed_reinterpret(u32x2_i32x2, int32x2_t, uint32x2_t) #undef __packed_nziph2 #undef __packed_nziph4 #undef __packed_abdsum -#undef __packed_abdsum_acc +#undef __packed_ternary_builtin #undef __packed_reinterpret #undef __DEFAULT_FN_ATTRS diff --git a/clang/test/CodeGen/RISCV/rvp-intrinsics.c b/clang/test/CodeGen/RISCV/rvp-intrinsics.c index 87bc81ed7a645..74a7cb60a6afa 100644 --- a/clang/test/CodeGen/RISCV/rvp-intrinsics.c +++ b/clang/test/CodeGen/RISCV/rvp-intrinsics.c @@ -10105,3 +10105,391 @@ int32x2_t test_pnclipp_i32x2(int64_t rs1, int64_t rs2) { uint32x2_t test_pnclipup_u32x2(uint64_t rs1, uint64_t rs2) { return __riscv_pnclipup_u32x2(rs1, rs2); } + +/* Packed "Q-format" Multiply Parts Accumulate (32-bit) */ + +// RV32-LABEL: define dso_local i32 @test_mqacc_h00_i32( +// RV32-SAME: i32 noundef [[RD:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] { +// RV32-NEXT: [[ENTRY:.*:]] +// RV32-NEXT: [[TMP0:%.*]] = bitcast i32 [[RS1_COERCE]] to <2 x i16> +// RV32-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS2_COERCE]] to <2 x i16> +// RV32-NEXT: [[TMP2:%.*]] = call i32 @llvm.riscv.mqacc.h00.i32.v2i16(i32 [[RD]], <2 x i16> [[TMP0]], <2 x i16> [[TMP1]]) +// RV32-NEXT: ret i32 [[TMP2]] +// +// RV64-LABEL: define dso_local signext i32 @test_mqacc_h00_i32( +// RV64-SAME: i32 noundef signext [[RD:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] { +// RV64-NEXT: [[ENTRY:.*:]] +// RV64-NEXT: [[TMP0:%.*]] = bitcast i32 [[RS1_COERCE]] to <2 x i16> +// RV64-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS2_COERCE]] to <2 x i16> +// RV64-NEXT: [[TMP2:%.*]] = call i32 @llvm.riscv.mqacc.h00.i32.v2i16(i32 [[RD]], <2 x i16> [[TMP0]], <2 x i16> [[TMP1]]) +// RV64-NEXT: ret i32 [[TMP2]] +// + +int32_t test_mqacc_h00_i32(int32_t rd, int16x2_t rs1, int16x2_t rs2) { + return __riscv_mqacc_h00_i32(rd, rs1, rs2); +} +// RV32-LABEL: define dso_local i32 @test_mqacc_h01_i32( +// RV32-SAME: i32 noundef [[RD:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] { +// RV32-NEXT: [[ENTRY:.*:]] +// RV32-NEXT: [[TMP0:%.*]] = bitcast i32 [[RS1_COERCE]] to <2 x i16> +// RV32-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS2_COERCE]] to <2 x i16> +// RV32-NEXT: [[TMP2:%.*]] = call i32 @llvm.riscv.mqacc.h01.i32.v2i16(i32 [[RD]], <2 x i16> [[TMP0]], <2 x i16> [[TMP1]]) +// RV32-NEXT: ret i32 [[TMP2]] +// +// RV64-LABEL: define dso_local signext i32 @test_mqacc_h01_i32( +// RV64-SAME: i32 noundef signext [[RD:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] { +// RV64-NEXT: [[ENTRY:.*:]] +// RV64-NEXT: [[TMP0:%.*]] = bitcast i32 [[RS1_COERCE]] to <2 x i16> +// RV64-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS2_COERCE]] to <2 x i16> +// RV64-NEXT: [[TMP2:%.*]] = call i32 @llvm.riscv.mqacc.h01.i32.v2i16(i32 [[RD]], <2 x i16> [[TMP0]], <2 x i16> [[TMP1]]) +// RV64-NEXT: ret i32 [[TMP2]] +// +int32_t test_mqacc_h01_i32(int32_t rd, int16x2_t rs1, int16x2_t rs2) { + return __riscv_mqacc_h01_i32(rd, rs1, rs2); +} + +// RV32-LABEL: define dso_local i32 @test_mqacc_h11_i32( +// RV32-SAME: i32 noundef [[RD:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] { +// RV32-NEXT: [[ENTRY:.*:]] +// RV32-NEXT: [[TMP0:%.*]] = bitcast i32 [[RS1_COERCE]] to <2 x i16> +// RV32-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS2_COERCE]] to <2 x i16> +// RV32-NEXT: [[TMP2:%.*]] = call i32 @llvm.riscv.mqacc.h11.i32.v2i16(i32 [[RD]], <2 x i16> [[TMP0]], <2 x i16> [[TMP1]]) +// RV32-NEXT: ret i32 [[TMP2]] +// +// RV64-LABEL: define dso_local signext i32 @test_mqacc_h11_i32( +// RV64-SAME: i32 noundef signext [[RD:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] { +// RV64-NEXT: [[ENTRY:.*:]] +// RV64-NEXT: [[TMP0:%.*]] = bitcast i32 [[RS1_COERCE]] to <2 x i16> +// RV64-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS2_COERCE]] to <2 x i16> +// RV64-NEXT: [[TMP2:%.*]] = call i32 @llvm.riscv.mqacc.h11.i32.v2i16(i32 [[RD]], <2 x i16> [[TMP0]], <2 x i16> [[TMP1]]) +// RV64-NEXT: ret i32 [[TMP2]] +// +int32_t test_mqacc_h11_i32(int32_t rd, int16x2_t rs1, int16x2_t rs2) { + return __riscv_mqacc_h11_i32(rd, rs1, rs2); +} + +// RV32-LABEL: define dso_local i32 @test_mqracc_h00_i32( +// RV32-SAME: i32 noundef [[RD:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] { +// RV32-NEXT: [[ENTRY:.*:]] +// RV32-NEXT: [[TMP0:%.*]] = bitcast i32 [[RS1_COERCE]] to <2 x i16> +// RV32-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS2_COERCE]] to <2 x i16> +// RV32-NEXT: [[TMP2:%.*]] = call i32 @llvm.riscv.mqracc.h00.i32.v2i16(i32 [[RD]], <2 x i16> [[TMP0]], <2 x i16> [[TMP1]]) +// RV32-NEXT: ret i32 [[TMP2]] +// +// RV64-LABEL: define dso_local signext i32 @test_mqracc_h00_i32( +// RV64-SAME: i32 noundef signext [[RD:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] { +// RV64-NEXT: [[ENTRY:.*:]] +// RV64-NEXT: [[TMP0:%.*]] = bitcast i32 [[RS1_COERCE]] to <2 x i16> +// RV64-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS2_COERCE]] to <2 x i16> +// RV64-NEXT: [[TMP2:%.*]] = call i32 @llvm.riscv.mqracc.h00.i32.v2i16(i32 [[RD]], <2 x i16> [[TMP0]], <2 x i16> [[TMP1]]) +// RV64-NEXT: ret i32 [[TMP2]] +// +int32_t test_mqracc_h00_i32(int32_t rd, int16x2_t rs1, int16x2_t rs2) { + return __riscv_mqracc_h00_i32(rd, rs1, rs2); +} + +// RV32-LABEL: define dso_local i32 @test_mqracc_h01_i32( +// RV32-SAME: i32 noundef [[RD:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] { +// RV32-NEXT: [[ENTRY:.*:]] +// RV32-NEXT: [[TMP0:%.*]] = bitcast i32 [[RS1_COERCE]] to <2 x i16> +// RV32-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS2_COERCE]] to <2 x i16> +// RV32-NEXT: [[TMP2:%.*]] = call i32 @llvm.riscv.mqracc.h01.i32.v2i16(i32 [[RD]], <2 x i16> [[TMP0]], <2 x i16> [[TMP1]]) +// RV32-NEXT: ret i32 [[TMP2]] +// +// RV64-LABEL: define dso_local signext i32 @test_mqracc_h01_i32( +// RV64-SAME: i32 noundef signext [[RD:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] { +// RV64-NEXT: [[ENTRY:.*:]] +// RV64-NEXT: [[TMP0:%.*]] = bitcast i32 [[RS1_COERCE]] to <2 x i16> +// RV64-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS2_COERCE]] to <2 x i16> +// RV64-NEXT: [[TMP2:%.*]] = call i32 @llvm.riscv.mqracc.h01.i32.v2i16(i32 [[RD]], <2 x i16> [[TMP0]], <2 x i16> [[TMP1]]) +// RV64-NEXT: ret i32 [[TMP2]] +// +int32_t test_mqracc_h01_i32(int32_t rd, int16x2_t rs1, int16x2_t rs2) { + return __riscv_mqracc_h01_i32(rd, rs1, rs2); +} + +// RV32-LABEL: define dso_local i32 @test_mqracc_h11_i32( +// RV32-SAME: i32 noundef [[RD:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] { +// RV32-NEXT: [[ENTRY:.*:]] +// RV32-NEXT: [[TMP0:%.*]] = bitcast i32 [[RS1_COERCE]] to <2 x i16> +// RV32-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS2_COERCE]] to <2 x i16> +// RV32-NEXT: [[TMP2:%.*]] = call i32 @llvm.riscv.mqracc.h11.i32.v2i16(i32 [[RD]], <2 x i16> [[TMP0]], <2 x i16> [[TMP1]]) +// RV32-NEXT: ret i32 [[TMP2]] +// +// RV64-LABEL: define dso_local signext i32 @test_mqracc_h11_i32( +// RV64-SAME: i32 noundef signext [[RD:%.*]], i32 noundef [[RS1_COERCE:%.*]], i32 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] { +// RV64-NEXT: [[ENTRY:.*:]] +// RV64-NEXT: [[TMP0:%.*]] = bitcast i32 [[RS1_COERCE]] to <2 x i16> +// RV64-NEXT: [[TMP1:%.*]] = bitcast i32 [[RS2_COERCE]] to <2 x i16> +// RV64-NEXT: [[TMP2:%.*]] = call i32 @llvm.riscv.mqracc.h11.i32.v2i16(i32 [[RD]], <2 x i16> [[TMP0]], <2 x i16> [[TMP1]]) +// RV64-NEXT: ret i32 [[TMP2]] +// +int32_t test_mqracc_h11_i32(int32_t rd, int16x2_t rs1, int16x2_t rs2) { + return __riscv_mqracc_h11_i32(rd, rs1, rs2); +} + +/* Packed "Q-format" Multiply Parts Accumulate (64-bit) */ + +// RV32-LABEL: define dso_local i64 @test_pmqacc_h00_i32x2( +// RV32-SAME: i64 noundef [[RD_COERCE:%.*]], i64 noundef [[RS1_COERCE:%.*]], i64 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] { +// RV32-NEXT: [[ENTRY:.*:]] +// RV32-NEXT: [[TMP0:%.*]] = bitcast i64 [[RD_COERCE]] to <2 x i32> +// RV32-NEXT: [[TMP1:%.*]] = bitcast i64 [[RS1_COERCE]] to <4 x i16> +// RV32-NEXT: [[TMP2:%.*]] = bitcast i64 [[RS2_COERCE]] to <4 x i16> +// RV32-NEXT: [[TMP3:%.*]] = call <2 x i32> @llvm.riscv.pmqacc.h00.v2i32.v4i16(<2 x i32> [[TMP0]], <4 x i16> [[TMP1]], <4 x i16> [[TMP2]]) +// RV32-NEXT: [[TMP4:%.*]] = bitcast <2 x i32> [[TMP3]] to i64 +// RV32-NEXT: ret i64 [[TMP4]] +// +// RV64-LABEL: define dso_local i64 @test_pmqacc_h00_i32x2( +// RV64-SAME: i64 noundef [[RD_COERCE:%.*]], i64 noundef [[RS1_COERCE:%.*]], i64 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] { +// RV64-NEXT: [[ENTRY:.*:]] +// RV64-NEXT: [[TMP0:%.*]] = bitcast i64 [[RD_COERCE]] to <2 x i32> +// RV64-NEXT: [[TMP1:%.*]] = bitcast i64 [[RS1_COERCE]] to <4 x i16> +// RV64-NEXT: [[TMP2:%.*]] = bitcast i64 [[RS2_COERCE]] to <4 x i16> +// RV64-NEXT: [[TMP3:%.*]] = call <2 x i32> @llvm.riscv.pmqacc.h00.v2i32.v4i16(<2 x i32> [[TMP0]], <4 x i16> [[TMP1]], <4 x i16> [[TMP2]]) +// RV64-NEXT: [[TMP4:%.*]] = bitcast <2 x i32> [[TMP3]] to i64 +// RV64-NEXT: ret i64 [[TMP4]] +// +int32x2_t test_pmqacc_h00_i32x2(int32x2_t rd, int16x4_t rs1, int16x4_t rs2) { + return __riscv_pmqacc_h00_i32x2(rd, rs1, rs2); +} + +// RV32-LABEL: define dso_local i64 @test_pmqacc_h01_i32x2( +// RV32-SAME: i64 noundef [[RD_COERCE:%.*]], i64 noundef [[RS1_COERCE:%.*]], i64 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] { +// RV32-NEXT: [[ENTRY:.*:]] +// RV32-NEXT: [[TMP0:%.*]] = bitcast i64 [[RD_COERCE]] to <2 x i32> +// RV32-NEXT: [[TMP1:%.*]] = bitcast i64 [[RS1_COERCE]] to <4 x i16> +// RV32-NEXT: [[TMP2:%.*]] = bitcast i64 [[RS2_COERCE]] to <4 x i16> +// RV32-NEXT: [[TMP3:%.*]] = call <2 x i32> @llvm.riscv.pmqacc.h01.v2i32.v4i16(<2 x i32> [[TMP0]], <4 x i16> [[TMP1]], <4 x i16> [[TMP2]]) +// RV32-NEXT: [[TMP4:%.*]] = bitcast <2 x i32> [[TMP3]] to i64 +// RV32-NEXT: ret i64 [[TMP4]] +// +// RV64-LABEL: define dso_local i64 @test_pmqacc_h01_i32x2( +// RV64-SAME: i64 noundef [[RD_COERCE:%.*]], i64 noundef [[RS1_COERCE:%.*]], i64 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] { +// RV64-NEXT: [[ENTRY:.*:]] +// RV64-NEXT: [[TMP0:%.*]] = bitcast i64 [[RD_COERCE]] to <2 x i32> +// RV64-NEXT: [[TMP1:%.*]] = bitcast i64 [[RS1_COERCE]] to <4 x i16> +// RV64-NEXT: [[TMP2:%.*]] = bitcast i64 [[RS2_COERCE]] to <4 x i16> +// RV64-NEXT: [[TMP3:%.*]] = call <2 x i32> @llvm.riscv.pmqacc.h01.v2i32.v4i16(<2 x i32> [[TMP0]], <4 x i16> [[TMP1]], <4 x i16> [[TMP2]]) +// RV64-NEXT: [[TMP4:%.*]] = bitcast <2 x i32> [[TMP3]] to i64 +// RV64-NEXT: ret i64 [[TMP4]] +// +int32x2_t test_pmqacc_h01_i32x2(int32x2_t rd, int16x4_t rs1, int16x4_t rs2) { + return __riscv_pmqacc_h01_i32x2(rd, rs1, rs2); +} + +// RV32-LABEL: define dso_local i64 @test_pmqacc_h11_i32x2( +// RV32-SAME: i64 noundef [[RD_COERCE:%.*]], i64 noundef [[RS1_COERCE:%.*]], i64 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] { +// RV32-NEXT: [[ENTRY:.*:]] +// RV32-NEXT: [[TMP0:%.*]] = bitcast i64 [[RD_COERCE]] to <2 x i32> +// RV32-NEXT: [[TMP1:%.*]] = bitcast i64 [[RS1_COERCE]] to <4 x i16> +// RV32-NEXT: [[TMP2:%.*]] = bitcast i64 [[RS2_COERCE]] to <4 x i16> +// RV32-NEXT: [[TMP3:%.*]] = call <2 x i32> @llvm.riscv.pmqacc.h11.v2i32.v4i16(<2 x i32> [[TMP0]], <4 x i16> [[TMP1]], <4 x i16> [[TMP2]]) +// RV32-NEXT: [[TMP4:%.*]] = bitcast <2 x i32> [[TMP3]] to i64 +// RV32-NEXT: ret i64 [[TMP4]] +// +// RV64-LABEL: define dso_local i64 @test_pmqacc_h11_i32x2( +// RV64-SAME: i64 noundef [[RD_COERCE:%.*]], i64 noundef [[RS1_COERCE:%.*]], i64 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] { +// RV64-NEXT: [[ENTRY:.*:]] +// RV64-NEXT: [[TMP0:%.*]] = bitcast i64 [[RD_COERCE]] to <2 x i32> +// RV64-NEXT: [[TMP1:%.*]] = bitcast i64 [[RS1_COERCE]] to <4 x i16> +// RV64-NEXT: [[TMP2:%.*]] = bitcast i64 [[RS2_COERCE]] to <4 x i16> +// RV64-NEXT: [[TMP3:%.*]] = call <2 x i32> @llvm.riscv.pmqacc.h11.v2i32.v4i16(<2 x i32> [[TMP0]], <4 x i16> [[TMP1]], <4 x i16> [[TMP2]]) +// RV64-NEXT: [[TMP4:%.*]] = bitcast <2 x i32> [[TMP3]] to i64 +// RV64-NEXT: ret i64 [[TMP4]] +// +int32x2_t test_pmqacc_h11_i32x2(int32x2_t rd, int16x4_t rs1, int16x4_t rs2) { + return __riscv_pmqacc_h11_i32x2(rd, rs1, rs2); +} + +// RV32-LABEL: define dso_local i64 @test_pmqracc_h00_i32x2( +// RV32-SAME: i64 noundef [[RD_COERCE:%.*]], i64 noundef [[RS1_COERCE:%.*]], i64 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] { +// RV32-NEXT: [[ENTRY:.*:]] +// RV32-NEXT: [[TMP0:%.*]] = bitcast i64 [[RD_COERCE]] to <2 x i32> +// RV32-NEXT: [[TMP1:%.*]] = bitcast i64 [[RS1_COERCE]] to <4 x i16> +// RV32-NEXT: [[TMP2:%.*]] = bitcast i64 [[RS2_COERCE]] to <4 x i16> +// RV32-NEXT: [[TMP3:%.*]] = call <2 x i32> @llvm.riscv.pmqracc.h00.v2i32.v4i16(<2 x i32> [[TMP0]], <4 x i16> [[TMP1]], <4 x i16> [[TMP2]]) +// RV32-NEXT: [[TMP4:%.*]] = bitcast <2 x i32> [[TMP3]] to i64 +// RV32-NEXT: ret i64 [[TMP4]] +// +// RV64-LABEL: define dso_local i64 @test_pmqracc_h00_i32x2( +// RV64-SAME: i64 noundef [[RD_COERCE:%.*]], i64 noundef [[RS1_COERCE:%.*]], i64 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] { +// RV64-NEXT: [[ENTRY:.*:]] +// RV64-NEXT: [[TMP0:%.*]] = bitcast i64 [[RD_COERCE]] to <2 x i32> +// RV64-NEXT: [[TMP1:%.*]] = bitcast i64 [[RS1_COERCE]] to <4 x i16> +// RV64-NEXT: [[TMP2:%.*]] = bitcast i64 [[RS2_COERCE]] to <4 x i16> +// RV64-NEXT: [[TMP3:%.*]] = call <2 x i32> @llvm.riscv.pmqracc.h00.v2i32.v4i16(<2 x i32> [[TMP0]], <4 x i16> [[TMP1]], <4 x i16> [[TMP2]]) +// RV64-NEXT: [[TMP4:%.*]] = bitcast <2 x i32> [[TMP3]] to i64 +// RV64-NEXT: ret i64 [[TMP4]] +// +int32x2_t test_pmqracc_h00_i32x2(int32x2_t rd, int16x4_t rs1, int16x4_t rs2) { + return __riscv_pmqracc_h00_i32x2(rd, rs1, rs2); +} + +// RV32-LABEL: define dso_local i64 @test_pmqracc_h01_i32x2( +// RV32-SAME: i64 noundef [[RD_COERCE:%.*]], i64 noundef [[RS1_COERCE:%.*]], i64 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] { +// RV32-NEXT: [[ENTRY:.*:]] +// RV32-NEXT: [[TMP0:%.*]] = bitcast i64 [[RD_COERCE]] to <2 x i32> +// RV32-NEXT: [[TMP1:%.*]] = bitcast i64 [[RS1_COERCE]] to <4 x i16> +// RV32-NEXT: [[TMP2:%.*]] = bitcast i64 [[RS2_COERCE]] to <4 x i16> +// RV32-NEXT: [[TMP3:%.*]] = call <2 x i32> @llvm.riscv.pmqracc.h01.v2i32.v4i16(<2 x i32> [[TMP0]], <4 x i16> [[TMP1]], <4 x i16> [[TMP2]]) +// RV32-NEXT: [[TMP4:%.*]] = bitcast <2 x i32> [[TMP3]] to i64 +// RV32-NEXT: ret i64 [[TMP4]] +// +// RV64-LABEL: define dso_local i64 @test_pmqracc_h01_i32x2( +// RV64-SAME: i64 noundef [[RD_COERCE:%.*]], i64 noundef [[RS1_COERCE:%.*]], i64 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] { +// RV64-NEXT: [[ENTRY:.*:]] +// RV64-NEXT: [[TMP0:%.*]] = bitcast i64 [[RD_COERCE]] to <2 x i32> +// RV64-NEXT: [[TMP1:%.*]] = bitcast i64 [[RS1_COERCE]] to <4 x i16> +// RV64-NEXT: [[TMP2:%.*]] = bitcast i64 [[RS2_COERCE]] to <4 x i16> +// RV64-NEXT: [[TMP3:%.*]] = call <2 x i32> @llvm.riscv.pmqracc.h01.v2i32.v4i16(<2 x i32> [[TMP0]], <4 x i16> [[TMP1]], <4 x i16> [[TMP2]]) +// RV64-NEXT: [[TMP4:%.*]] = bitcast <2 x i32> [[TMP3]] to i64 +// RV64-NEXT: ret i64 [[TMP4]] +// +int32x2_t test_pmqracc_h01_i32x2(int32x2_t rd, int16x4_t rs1, int16x4_t rs2) { + return __riscv_pmqracc_h01_i32x2(rd, rs1, rs2); +} + +// RV32-LABEL: define dso_local i64 @test_pmqracc_h11_i32x2( +// RV32-SAME: i64 noundef [[RD_COERCE:%.*]], i64 noundef [[RS1_COERCE:%.*]], i64 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] { +// RV32-NEXT: [[ENTRY:.*:]] +// RV32-NEXT: [[TMP0:%.*]] = bitcast i64 [[RD_COERCE]] to <2 x i32> +// RV32-NEXT: [[TMP1:%.*]] = bitcast i64 [[RS1_COERCE]] to <4 x i16> +// RV32-NEXT: [[TMP2:%.*]] = bitcast i64 [[RS2_COERCE]] to <4 x i16> +// RV32-NEXT: [[TMP3:%.*]] = call <2 x i32> @llvm.riscv.pmqracc.h11.v2i32.v4i16(<2 x i32> [[TMP0]], <4 x i16> [[TMP1]], <4 x i16> [[TMP2]]) +// RV32-NEXT: [[TMP4:%.*]] = bitcast <2 x i32> [[TMP3]] to i64 +// RV32-NEXT: ret i64 [[TMP4]] +// +// RV64-LABEL: define dso_local i64 @test_pmqracc_h11_i32x2( +// RV64-SAME: i64 noundef [[RD_COERCE:%.*]], i64 noundef [[RS1_COERCE:%.*]], i64 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] { +// RV64-NEXT: [[ENTRY:.*:]] +// RV64-NEXT: [[TMP0:%.*]] = bitcast i64 [[RD_COERCE]] to <2 x i32> +// RV64-NEXT: [[TMP1:%.*]] = bitcast i64 [[RS1_COERCE]] to <4 x i16> +// RV64-NEXT: [[TMP2:%.*]] = bitcast i64 [[RS2_COERCE]] to <4 x i16> +// RV64-NEXT: [[TMP3:%.*]] = call <2 x i32> @llvm.riscv.pmqracc.h11.v2i32.v4i16(<2 x i32> [[TMP0]], <4 x i16> [[TMP1]], <4 x i16> [[TMP2]]) +// RV64-NEXT: [[TMP4:%.*]] = bitcast <2 x i32> [[TMP3]] to i64 +// RV64-NEXT: ret i64 [[TMP4]] +// +int32x2_t test_pmqracc_h11_i32x2(int32x2_t rd, int16x4_t rs1, int16x4_t rs2) { + return __riscv_pmqracc_h11_i32x2(rd, rs1, rs2); +} + +// RV32-LABEL: define dso_local i64 @test_mqacc_w00_i64( +// RV32-SAME: i64 noundef [[RD:%.*]], i64 noundef [[RS1_COERCE:%.*]], i64 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] { +// RV32-NEXT: [[ENTRY:.*:]] +// RV32-NEXT: [[TMP0:%.*]] = bitcast i64 [[RS1_COERCE]] to <2 x i32> +// RV32-NEXT: [[TMP1:%.*]] = bitcast i64 [[RS2_COERCE]] to <2 x i32> +// RV32-NEXT: [[TMP2:%.*]] = call i64 @llvm.riscv.mqacc.w00.i64.v2i32(i64 [[RD]], <2 x i32> [[TMP0]], <2 x i32> [[TMP1]]) +// RV32-NEXT: ret i64 [[TMP2]] +// +// RV64-LABEL: define dso_local i64 @test_mqacc_w00_i64( +// RV64-SAME: i64 noundef [[RD:%.*]], i64 noundef [[RS1_COERCE:%.*]], i64 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] { +// RV64-NEXT: [[ENTRY:.*:]] +// RV64-NEXT: [[TMP0:%.*]] = bitcast i64 [[RS1_COERCE]] to <2 x i32> +// RV64-NEXT: [[TMP1:%.*]] = bitcast i64 [[RS2_COERCE]] to <2 x i32> +// RV64-NEXT: [[TMP2:%.*]] = call i64 @llvm.riscv.mqacc.w00.i64.v2i32(i64 [[RD]], <2 x i32> [[TMP0]], <2 x i32> [[TMP1]]) +// RV64-NEXT: ret i64 [[TMP2]] +// +int64_t test_mqacc_w00_i64(int64_t rd, int32x2_t rs1, int32x2_t rs2) { + return __riscv_mqacc_w00_i64(rd, rs1, rs2); +} + +// RV32-LABEL: define dso_local i64 @test_mqacc_w01_i64( +// RV32-SAME: i64 noundef [[RD:%.*]], i64 noundef [[RS1_COERCE:%.*]], i64 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] { +// RV32-NEXT: [[ENTRY:.*:]] +// RV32-NEXT: [[TMP0:%.*]] = bitcast i64 [[RS1_COERCE]] to <2 x i32> +// RV32-NEXT: [[TMP1:%.*]] = bitcast i64 [[RS2_COERCE]] to <2 x i32> +// RV32-NEXT: [[TMP2:%.*]] = call i64 @llvm.riscv.mqacc.w01.i64.v2i32(i64 [[RD]], <2 x i32> [[TMP0]], <2 x i32> [[TMP1]]) +// RV32-NEXT: ret i64 [[TMP2]] +// +// RV64-LABEL: define dso_local i64 @test_mqacc_w01_i64( +// RV64-SAME: i64 noundef [[RD:%.*]], i64 noundef [[RS1_COERCE:%.*]], i64 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] { +// RV64-NEXT: [[ENTRY:.*:]] +// RV64-NEXT: [[TMP0:%.*]] = bitcast i64 [[RS1_COERCE]] to <2 x i32> +// RV64-NEXT: [[TMP1:%.*]] = bitcast i64 [[RS2_COERCE]] to <2 x i32> +// RV64-NEXT: [[TMP2:%.*]] = call i64 @llvm.riscv.mqacc.w01.i64.v2i32(i64 [[RD]], <2 x i32> [[TMP0]], <2 x i32> [[TMP1]]) +// RV64-NEXT: ret i64 [[TMP2]] +// +int64_t test_mqacc_w01_i64(int64_t rd, int32x2_t rs1, int32x2_t rs2) { + return __riscv_mqacc_w01_i64(rd, rs1, rs2); +} + +// RV32-LABEL: define dso_local i64 @test_mqacc_w11_i64( +// RV32-SAME: i64 noundef [[RD:%.*]], i64 noundef [[RS1_COERCE:%.*]], i64 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] { +// RV32-NEXT: [[ENTRY:.*:]] +// RV32-NEXT: [[TMP0:%.*]] = bitcast i64 [[RS1_COERCE]] to <2 x i32> +// RV32-NEXT: [[TMP1:%.*]] = bitcast i64 [[RS2_COERCE]] to <2 x i32> +// RV32-NEXT: [[TMP2:%.*]] = call i64 @llvm.riscv.mqacc.w11.i64.v2i32(i64 [[RD]], <2 x i32> [[TMP0]], <2 x i32> [[TMP1]]) +// RV32-NEXT: ret i64 [[TMP2]] +// +// RV64-LABEL: define dso_local i64 @test_mqacc_w11_i64( +// RV64-SAME: i64 noundef [[RD:%.*]], i64 noundef [[RS1_COERCE:%.*]], i64 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] { +// RV64-NEXT: [[ENTRY:.*:]] +// RV64-NEXT: [[TMP0:%.*]] = bitcast i64 [[RS1_COERCE]] to <2 x i32> +// RV64-NEXT: [[TMP1:%.*]] = bitcast i64 [[RS2_COERCE]] to <2 x i32> +// RV64-NEXT: [[TMP2:%.*]] = call i64 @llvm.riscv.mqacc.w11.i64.v2i32(i64 [[RD]], <2 x i32> [[TMP0]], <2 x i32> [[TMP1]]) +// RV64-NEXT: ret i64 [[TMP2]] +// +int64_t test_mqacc_w11_i64(int64_t rd, int32x2_t rs1, int32x2_t rs2) { + return __riscv_mqacc_w11_i64(rd, rs1, rs2); +} + +// RV32-LABEL: define dso_local i64 @test_mqracc_w00_i64( +// RV32-SAME: i64 noundef [[RD:%.*]], i64 noundef [[RS1_COERCE:%.*]], i64 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] { +// RV32-NEXT: [[ENTRY:.*:]] +// RV32-NEXT: [[TMP0:%.*]] = bitcast i64 [[RS1_COERCE]] to <2 x i32> +// RV32-NEXT: [[TMP1:%.*]] = bitcast i64 [[RS2_COERCE]] to <2 x i32> +// RV32-NEXT: [[TMP2:%.*]] = call i64 @llvm.riscv.mqracc.w00.i64.v2i32(i64 [[RD]], <2 x i32> [[TMP0]], <2 x i32> [[TMP1]]) +// RV32-NEXT: ret i64 [[TMP2]] +// +// RV64-LABEL: define dso_local i64 @test_mqracc_w00_i64( +// RV64-SAME: i64 noundef [[RD:%.*]], i64 noundef [[RS1_COERCE:%.*]], i64 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] { +// RV64-NEXT: [[ENTRY:.*:]] +// RV64-NEXT: [[TMP0:%.*]] = bitcast i64 [[RS1_COERCE]] to <2 x i32> +// RV64-NEXT: [[TMP1:%.*]] = bitcast i64 [[RS2_COERCE]] to <2 x i32> +// RV64-NEXT: [[TMP2:%.*]] = call i64 @llvm.riscv.mqracc.w00.i64.v2i32(i64 [[RD]], <2 x i32> [[TMP0]], <2 x i32> [[TMP1]]) +// RV64-NEXT: ret i64 [[TMP2]] +// +int64_t test_mqracc_w00_i64(int64_t rd, int32x2_t rs1, int32x2_t rs2) { + return __riscv_mqracc_w00_i64(rd, rs1, rs2); +} + +// RV32-LABEL: define dso_local i64 @test_mqracc_w01_i64( +// RV32-SAME: i64 noundef [[RD:%.*]], i64 noundef [[RS1_COERCE:%.*]], i64 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] { +// RV32-NEXT: [[ENTRY:.*:]] +// RV32-NEXT: [[TMP0:%.*]] = bitcast i64 [[RS1_COERCE]] to <2 x i32> +// RV32-NEXT: [[TMP1:%.*]] = bitcast i64 [[RS2_COERCE]] to <2 x i32> +// RV32-NEXT: [[TMP2:%.*]] = call i64 @llvm.riscv.mqracc.w01.i64.v2i32(i64 [[RD]], <2 x i32> [[TMP0]], <2 x i32> [[TMP1]]) +// RV32-NEXT: ret i64 [[TMP2]] +// +// RV64-LABEL: define dso_local i64 @test_mqracc_w01_i64( +// RV64-SAME: i64 noundef [[RD:%.*]], i64 noundef [[RS1_COERCE:%.*]], i64 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] { +// RV64-NEXT: [[ENTRY:.*:]] +// RV64-NEXT: [[TMP0:%.*]] = bitcast i64 [[RS1_COERCE]] to <2 x i32> +// RV64-NEXT: [[TMP1:%.*]] = bitcast i64 [[RS2_COERCE]] to <2 x i32> +// RV64-NEXT: [[TMP2:%.*]] = call i64 @llvm.riscv.mqracc.w01.i64.v2i32(i64 [[RD]], <2 x i32> [[TMP0]], <2 x i32> [[TMP1]]) +// RV64-NEXT: ret i64 [[TMP2]] +// +int64_t test_mqracc_w01_i64(int64_t rd, int32x2_t rs1, int32x2_t rs2) { + return __riscv_mqracc_w01_i64(rd, rs1, rs2); +} + +// RV32-LABEL: define dso_local i64 @test_mqracc_w11_i64( +// RV32-SAME: i64 noundef [[RD:%.*]], i64 noundef [[RS1_COERCE:%.*]], i64 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] { +// RV32-NEXT: [[ENTRY:.*:]] +// RV32-NEXT: [[TMP0:%.*]] = bitcast i64 [[RS1_COERCE]] to <2 x i32> +// RV32-NEXT: [[TMP1:%.*]] = bitcast i64 [[RS2_COERCE]] to <2 x i32> +// RV32-NEXT: [[TMP2:%.*]] = call i64 @llvm.riscv.mqracc.w11.i64.v2i32(i64 [[RD]], <2 x i32> [[TMP0]], <2 x i32> [[TMP1]]) +// RV32-NEXT: ret i64 [[TMP2]] +// +// RV64-LABEL: define dso_local i64 @test_mqracc_w11_i64( +// RV64-SAME: i64 noundef [[RD:%.*]], i64 noundef [[RS1_COERCE:%.*]], i64 noundef [[RS2_COERCE:%.*]]) #[[ATTR0]] { +// RV64-NEXT: [[ENTRY:.*:]] +// RV64-NEXT: [[TMP0:%.*]] = bitcast i64 [[RS1_COERCE]] to <2 x i32> +// RV64-NEXT: [[TMP1:%.*]] = bitcast i64 [[RS2_COERCE]] to <2 x i32> +// RV64-NEXT: [[TMP2:%.*]] = call i64 @llvm.riscv.mqracc.w11.i64.v2i32(i64 [[RD]], <2 x i32> [[TMP0]], <2 x i32> [[TMP1]]) +// RV64-NEXT: ret i64 [[TMP2]] +// +int64_t test_mqracc_w11_i64(int64_t rd, int32x2_t rs1, int32x2_t rs2) { + return __riscv_mqracc_w11_i64(rd, rs1, rs2); +} diff --git a/cross-project-tests/intrinsic-header-tests/riscv_packed_simd.c b/cross-project-tests/intrinsic-header-tests/riscv_packed_simd.c index 75dcb796b66cd..28e3a8911739d 100644 --- a/cross-project-tests/intrinsic-header-tests/riscv_packed_simd.c +++ b/cross-project-tests/intrinsic-header-tests/riscv_packed_simd.c @@ -3406,3 +3406,129 @@ int32x2_t test_pnclipp_i32x2(int64_t a, int64_t b) { uint32x2_t test_pnclipup_u32x2(uint64_t a, uint64_t b) { return __riscv_pnclipup_u32x2(a, b); } + +// CHECK-LABEL: test_mqacc_h00_i32: +// RV32: mqacc.h00 +// RV64: pmqacc.w.h00 +int32_t test_mqacc_h00_i32(int32_t rd, int16x2_t a, int16x2_t b) { + return __riscv_mqacc_h00_i32(rd, a, b); +} + +// CHECK-LABEL: test_mqacc_h01_i32: +// RV32: mqacc.h01 +// RV64: pmqacc.w.h01 +int32_t test_mqacc_h01_i32(int32_t rd, int16x2_t a, int16x2_t b) { + return __riscv_mqacc_h01_i32(rd, a, b); +} + +// CHECK-LABEL: test_mqacc_h11_i32: +// RV32: mqacc.h11 +// RV64: pmqacc.w.h11 +int32_t test_mqacc_h11_i32(int32_t rd, int16x2_t a, int16x2_t b) { + return __riscv_mqacc_h11_i32(rd, a, b); +} + +// CHECK-LABEL: test_mqracc_h00_i32: +// RV32: mqracc.h00 +// RV64: pmqracc.w.h00 +int32_t test_mqracc_h00_i32(int32_t rd, int16x2_t a, int16x2_t b) { + return __riscv_mqracc_h00_i32(rd, a, b); +} + +// CHECK-LABEL: test_mqracc_h01_i32: +// RV32: mqracc.h01 +// RV64: pmqracc.w.h01 +int32_t test_mqracc_h01_i32(int32_t rd, int16x2_t a, int16x2_t b) { + return __riscv_mqracc_h01_i32(rd, a, b); +} + +// CHECK-LABEL: test_mqracc_h11_i32: +// RV32: mqracc.h11 +// RV64: pmqracc.w.h11 +int32_t test_mqracc_h11_i32(int32_t rd, int16x2_t a, int16x2_t b) { + return __riscv_mqracc_h11_i32(rd, a, b); +} + +// CHECK-LABEL: test_pmqacc_h00_i32x2: +// RV32-COUNT-2: mqacc.h00 +// RV64: pmqacc.w.h00 +int32x2_t test_pmqacc_h00_i32x2(int32x2_t rd, int16x4_t a, int16x4_t b) { + return __riscv_pmqacc_h00_i32x2(rd, a, b); +} + +// CHECK-LABEL: test_pmqacc_h01_i32x2: +// RV32-COUNT-2: mqacc.h01 +// RV64: pmqacc.w.h01 +int32x2_t test_pmqacc_h01_i32x2(int32x2_t rd, int16x4_t a, int16x4_t b) { + return __riscv_pmqacc_h01_i32x2(rd, a, b); +} + +// CHECK-LABEL: test_pmqacc_h11_i32x2: +// RV32-COUNT-2: mqacc.h11 +// RV64: pmqacc.w.h11 +int32x2_t test_pmqacc_h11_i32x2(int32x2_t rd, int16x4_t a, int16x4_t b) { + return __riscv_pmqacc_h11_i32x2(rd, a, b); +} + +// CHECK-LABEL: test_pmqracc_h00_i32x2: +// RV32-COUNT-2: mqracc.h00 +// RV64: pmqracc.w.h00 +int32x2_t test_pmqracc_h00_i32x2(int32x2_t rd, int16x4_t a, int16x4_t b) { + return __riscv_pmqracc_h00_i32x2(rd, a, b); +} + +// CHECK-LABEL: test_pmqracc_h01_i32x2: +// RV32-COUNT-2: mqracc.h01 +// RV64: pmqracc.w.h01 +int32x2_t test_pmqracc_h01_i32x2(int32x2_t rd, int16x4_t a, int16x4_t b) { + return __riscv_pmqracc_h01_i32x2(rd, a, b); +} + +// CHECK-LABEL: test_pmqracc_h11_i32x2: +// RV32-COUNT-2: mqracc.h11 +// RV64: pmqracc.w.h11 +int32x2_t test_pmqracc_h11_i32x2(int32x2_t rd, int16x4_t a, int16x4_t b) { + return __riscv_pmqracc_h11_i32x2(rd, a, b); +} + +// CHECK-LABEL: test_mqacc_w00_i64: +// RV32: mqwacc +// RV64: mqacc.w00 +int64_t test_mqacc_w00_i64(int64_t rd, int32x2_t a, int32x2_t b) { + return __riscv_mqacc_w00_i64(rd, a, b); +} + +// CHECK-LABEL: test_mqacc_w01_i64: +// RV32: mqwacc +// RV64: mqacc.w01 +int64_t test_mqacc_w01_i64(int64_t rd, int32x2_t a, int32x2_t b) { + return __riscv_mqacc_w01_i64(rd, a, b); +} + +// CHECK-LABEL: test_mqacc_w11_i64: +// RV32: mqwacc +// RV64: mqacc.w11 +int64_t test_mqacc_w11_i64(int64_t rd, int32x2_t a, int32x2_t b) { + return __riscv_mqacc_w11_i64(rd, a, b); +} + +// CHECK-LABEL: test_mqracc_w00_i64: +// RV32: mqrwacc +// RV64: mqracc.w00 +int64_t test_mqracc_w00_i64(int64_t rd, int32x2_t a, int32x2_t b) { + return __riscv_mqracc_w00_i64(rd, a, b); +} + +// CHECK-LABEL: test_mqracc_w01_i64: +// RV32: mqrwacc +// RV64: mqracc.w01 +int64_t test_mqracc_w01_i64(int64_t rd, int32x2_t a, int32x2_t b) { + return __riscv_mqracc_w01_i64(rd, a, b); +} + +// CHECK-LABEL: test_mqracc_w11_i64: +// RV32: mqrwacc +// RV64: mqracc.w11 +int64_t test_mqracc_w11_i64(int64_t rd, int32x2_t a, int32x2_t b) { + return __riscv_mqracc_w11_i64(rd, a, b); +} diff --git a/llvm/include/llvm/IR/IntrinsicsRISCV.td b/llvm/include/llvm/IR/IntrinsicsRISCV.td index 8fa0ee78ca2a5..9c810ecc2a624 100644 --- a/llvm/include/llvm/IR/IntrinsicsRISCV.td +++ b/llvm/include/llvm/IR/IntrinsicsRISCV.td @@ -2150,6 +2150,31 @@ class RVPBinaryIntrinsic [IntrNoMem, IntrSpeculatable]>; def int_riscv_pnclipp : RVPNarrowingClipIntrinsic; def int_riscv_pnclipup : RVPNarrowingClipIntrinsic; + + // Packed "Q-format" Multiply Parts Accumulate + class RVPQFormatAccIntrinsic + : DefaultAttrsIntrinsic<[llvm_anyint_ty], + [LLVMMatchType<0>, llvm_any_ty, + LLVMMatchType<1>], + [IntrNoMem, IntrSpeculatable]>; + def int_riscv_mqacc_h00 : RVPQFormatAccIntrinsic; + def int_riscv_mqacc_h01 : RVPQFormatAccIntrinsic; + def int_riscv_mqacc_h11 : RVPQFormatAccIntrinsic; + def int_riscv_mqracc_h00 : RVPQFormatAccIntrinsic; + def int_riscv_mqracc_h01 : RVPQFormatAccIntrinsic; + def int_riscv_mqracc_h11 : RVPQFormatAccIntrinsic; + def int_riscv_pmqacc_h00 : RVPQFormatAccIntrinsic; + def int_riscv_pmqacc_h01 : RVPQFormatAccIntrinsic; + def int_riscv_pmqacc_h11 : RVPQFormatAccIntrinsic; + def int_riscv_pmqracc_h00 : RVPQFormatAccIntrinsic; + def int_riscv_pmqracc_h01 : RVPQFormatAccIntrinsic; + def int_riscv_pmqracc_h11 : RVPQFormatAccIntrinsic; + def int_riscv_mqacc_w00 : RVPQFormatAccIntrinsic; + def int_riscv_mqacc_w01 : RVPQFormatAccIntrinsic; + def int_riscv_mqacc_w11 : RVPQFormatAccIntrinsic; + def int_riscv_mqracc_w00 : RVPQFormatAccIntrinsic; + def int_riscv_mqracc_w01 : RVPQFormatAccIntrinsic; + def int_riscv_mqracc_w11 : RVPQFormatAccIntrinsic; } // TargetPrefix = "riscv" //===----------------------------------------------------------------------===// diff --git a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp index 017c9aac37230..29d133b5adba9 100644 --- a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp @@ -2073,6 +2073,22 @@ void RISCVDAGToDAGISel::Select(SDNode *Node) { CurDAG->RemoveDeadNode(Node); return; } + case RISCVISD::MQWACC: + case RISCVISD::MQRWACC: { + assert(!Subtarget->is64Bit() && Subtarget->hasStdExtP() && + "Unexpected opcode"); + + SDValue Op0 = buildGPRPair(CurDAG, DL, MVT::Untyped, Node->getOperand(0), + Node->getOperand(1)); + unsigned Opc = Opcode == RISCVISD::MQRWACC ? RISCV::MQRWACC : RISCV::MQWACC; + MachineSDNode *New = CurDAG->getMachineNode( + Opc, DL, MVT::Untyped, Op0, Node->getOperand(2), Node->getOperand(3)); + auto [Lo, Hi] = extractGPRPair(CurDAG, DL, SDValue(New, 0)); + ReplaceUses(SDValue(Node, 0), Lo); + ReplaceUses(SDValue(Node, 1), Hi); + CurDAG->RemoveDeadNode(Node); + return; + } case RISCVISD::ADDD: // Try to match WMACC pattern: ADDD where one operand pair comes from a // widening multiply. diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index 194aad22775cf..fe37fa4f3dfba 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -12254,6 +12254,94 @@ static unsigned getRVPMulHighAccumulateOpcode(unsigned IntNo) { } } +static unsigned getRVPQFormatAccScalarOpcode(Intrinsic::ID IntNo) { + switch (IntNo) { + default: + llvm_unreachable("Unexpected RISC-V packed Q-format accumulate intrinsic"); + case Intrinsic::riscv_mqacc_h00: + return RISCVISD::MQACC_H00; + case Intrinsic::riscv_mqacc_h01: + return RISCVISD::MQACC_H01; + case Intrinsic::riscv_mqacc_h11: + return RISCVISD::MQACC_H11; + case Intrinsic::riscv_mqracc_h00: + return RISCVISD::MQRACC_H00; + case Intrinsic::riscv_mqracc_h01: + return RISCVISD::MQRACC_H01; + case Intrinsic::riscv_mqracc_h11: + return RISCVISD::MQRACC_H11; + case Intrinsic::riscv_pmqacc_h00: + return RISCVISD::MQACC_H00; + case Intrinsic::riscv_pmqacc_h01: + return RISCVISD::MQACC_H01; + case Intrinsic::riscv_pmqacc_h11: + return RISCVISD::MQACC_H11; + case Intrinsic::riscv_pmqracc_h00: + return RISCVISD::MQRACC_H00; + case Intrinsic::riscv_pmqracc_h01: + return RISCVISD::MQRACC_H01; + case Intrinsic::riscv_pmqracc_h11: + return RISCVISD::MQRACC_H11; + } +} + +static unsigned getRVPQFormatAccOpcode(Intrinsic::ID IntNo) { + switch (IntNo) { + default: + llvm_unreachable("Unexpected RISC-V packed Q-format accumulate intrinsic"); + case Intrinsic::riscv_pmqacc_h00: + return RISCVISD::PMQACC_W_H00; + case Intrinsic::riscv_pmqacc_h01: + return RISCVISD::PMQACC_W_H01; + case Intrinsic::riscv_pmqacc_h11: + return RISCVISD::PMQACC_W_H11; + case Intrinsic::riscv_pmqracc_h00: + return RISCVISD::PMQRACC_W_H00; + case Intrinsic::riscv_pmqracc_h01: + return RISCVISD::PMQRACC_W_H01; + case Intrinsic::riscv_pmqracc_h11: + return RISCVISD::PMQRACC_W_H11; + } +} + +static unsigned getRVPQFormatAccWordOpcode(Intrinsic::ID IntNo) { + switch (IntNo) { + default: + llvm_unreachable("Unexpected RISC-V packed Q-format accumulate intrinsic"); + case Intrinsic::riscv_mqacc_w00: + return RISCVISD::MQACC_W00; + case Intrinsic::riscv_mqacc_w01: + return RISCVISD::MQACC_W01; + case Intrinsic::riscv_mqacc_w11: + return RISCVISD::MQACC_W11; + case Intrinsic::riscv_mqracc_w00: + return RISCVISD::MQRACC_W00; + case Intrinsic::riscv_mqracc_w01: + return RISCVISD::MQRACC_W01; + case Intrinsic::riscv_mqracc_w11: + return RISCVISD::MQRACC_W11; + } +} + +static unsigned getRVPQFormatAccWideOpcode(Intrinsic::ID IntNo) { + switch (IntNo) { + default: + llvm_unreachable("Unexpected RISC-V packed Q-format accumulate intrinsic"); + case Intrinsic::riscv_mqacc_h00: + return RISCVISD::PMQACC_W_H00_W; + case Intrinsic::riscv_mqacc_h01: + return RISCVISD::PMQACC_W_H01_W; + case Intrinsic::riscv_mqacc_h11: + return RISCVISD::PMQACC_W_H11_W; + case Intrinsic::riscv_mqracc_h00: + return RISCVISD::PMQRACC_W_H00_W; + case Intrinsic::riscv_mqracc_h01: + return RISCVISD::PMQRACC_W_H01_W; + case Intrinsic::riscv_mqracc_h11: + return RISCVISD::PMQRACC_W_H11_W; + } +} + SDValue RISCVTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const { unsigned IntNo = Op.getConstantOperandVal(0); @@ -12467,6 +12555,72 @@ SDValue RISCVTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, llvm_unreachable("unexpected VT for pnclipp/pnclipup on RV32"); } + case Intrinsic::riscv_mqacc_h00: + case Intrinsic::riscv_mqacc_h01: + case Intrinsic::riscv_mqacc_h11: + case Intrinsic::riscv_mqracc_h00: + case Intrinsic::riscv_mqracc_h01: + case Intrinsic::riscv_mqracc_h11: + case Intrinsic::riscv_pmqacc_h00: + case Intrinsic::riscv_pmqacc_h01: + case Intrinsic::riscv_pmqacc_h11: + case Intrinsic::riscv_pmqracc_h00: + case Intrinsic::riscv_pmqracc_h01: + case Intrinsic::riscv_pmqracc_h11: + case Intrinsic::riscv_mqacc_w00: + case Intrinsic::riscv_mqacc_w01: + case Intrinsic::riscv_mqacc_w11: + case Intrinsic::riscv_mqracc_w00: + case Intrinsic::riscv_mqracc_w01: + case Intrinsic::riscv_mqracc_w11: { + EVT VT = Op.getValueType(); + SDValue Rd = Op.getOperand(1); + SDValue Rs1 = Op.getOperand(2); + SDValue Rs2 = Op.getOperand(3); + MVT XLenVT = Subtarget.getXLenVT(); + + bool IsScalarHalfword = IntNo == Intrinsic::riscv_mqacc_h00 || + IntNo == Intrinsic::riscv_mqacc_h01 || + IntNo == Intrinsic::riscv_mqacc_h11 || + IntNo == Intrinsic::riscv_mqracc_h00 || + IntNo == Intrinsic::riscv_mqracc_h01 || + IntNo == Intrinsic::riscv_mqracc_h11; + if (Subtarget.is64Bit() && IsScalarHalfword) + return SDValue(); + + if (VT == MVT::v2i32 && Rs1.getSimpleValueType() == MVT::v4i16) { + if (Subtarget.is64Bit()) { + unsigned Opc = getRVPQFormatAccOpcode(IntNo); + return DAG.getNode(Opc, DL, VT, Rd, Rs1, Rs2); + } + + auto [Rs1Lo, Rs1Hi] = DAG.SplitVector(Rs1, DL); + auto [Rs2Lo, Rs2Hi] = DAG.SplitVector(Rs2, DL); + SDValue RdLo = DAG.getExtractVectorElt(DL, XLenVT, Rd, 0); + SDValue RdHi = DAG.getExtractVectorElt(DL, XLenVT, Rd, 1); + unsigned ScalarOpc = getRVPQFormatAccScalarOpcode(IntNo); + SDValue Lo = DAG.getNode(ScalarOpc, DL, XLenVT, RdLo, + DAG.getBitcast(XLenVT, Rs1Lo), + DAG.getBitcast(XLenVT, Rs2Lo)); + SDValue Hi = DAG.getNode(ScalarOpc, DL, XLenVT, RdHi, + DAG.getBitcast(XLenVT, Rs1Hi), + DAG.getBitcast(XLenVT, Rs2Hi)); + return DAG.getNode(ISD::BUILD_VECTOR, DL, VT, Lo, Hi); + } + + if (VT == MVT::i32 && !Subtarget.is64Bit()) { + unsigned Opc = getRVPQFormatAccScalarOpcode(IntNo); + return DAG.getNode(Opc, DL, XLenVT, Rd, DAG.getBitcast(XLenVT, Rs1), + DAG.getBitcast(XLenVT, Rs2)); + } + + if (VT == MVT::i64 && Subtarget.is64Bit()) { + unsigned Opc = getRVPQFormatAccWordOpcode(IntNo); + return DAG.getNode(Opc, DL, XLenVT, Rd, Rs1, Rs2); + } + + return SDValue(); + } case Intrinsic::riscv_pmulq: case Intrinsic::riscv_pmulqr: { unsigned Opc; @@ -16963,6 +17117,63 @@ void RISCVTargetLowering::ReplaceNodeResults(SDNode *N, Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res)); return; } + case Intrinsic::riscv_mqacc_h00: + case Intrinsic::riscv_mqacc_h01: + case Intrinsic::riscv_mqacc_h11: + case Intrinsic::riscv_mqracc_h00: + case Intrinsic::riscv_mqracc_h01: + case Intrinsic::riscv_mqracc_h11: { + EVT VT = N->getValueType(0); + if (!Subtarget.is64Bit() || VT != MVT::i32) + return; + MVT XLenVT = Subtarget.getXLenVT(); + // Widen a v2i16 source to XLenVT: bitcast to the 32-bit scalar it shares + // bits with, then zero-extend into the upper half of the GPR. + auto WidenSrc = [&](SDValue V) { + if (V.getValueType() == XLenVT) + return V; + assert(V.getValueType() == MVT::v2i16); + return DAG.getNode(ISD::ZERO_EXTEND, DL, XLenVT, + DAG.getBitcast(MVT::i32, V)); + }; + SDValue Rd = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, N->getOperand(1)); + SDValue Rs1 = WidenSrc(N->getOperand(2)); + SDValue Rs2 = WidenSrc(N->getOperand(3)); + unsigned Opc = getRVPQFormatAccWideOpcode(IntNo); + SDValue Res = DAG.getNode(Opc, DL, XLenVT, Rd, Rs1, Rs2); + Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res)); + return; + } + case Intrinsic::riscv_mqacc_w00: + case Intrinsic::riscv_mqacc_w01: + case Intrinsic::riscv_mqacc_w11: + case Intrinsic::riscv_mqracc_w00: + case Intrinsic::riscv_mqracc_w01: + case Intrinsic::riscv_mqracc_w11: { + if (Subtarget.is64Bit() || N->getValueType(0) != MVT::i64) + return; + bool IsRound = IntNo == Intrinsic::riscv_mqracc_w00 || + IntNo == Intrinsic::riscv_mqracc_w01 || + IntNo == Intrinsic::riscv_mqracc_w11; + bool Hi1 = IntNo == Intrinsic::riscv_mqacc_w11 || + IntNo == Intrinsic::riscv_mqracc_w11; + bool Hi2 = IntNo == Intrinsic::riscv_mqacc_w01 || + IntNo == Intrinsic::riscv_mqacc_w11 || + IntNo == Intrinsic::riscv_mqracc_w01 || + IntNo == Intrinsic::riscv_mqracc_w11; + MVT XLenVT = Subtarget.getXLenVT(); + SDValue Rs1 = N->getOperand(2); + SDValue Rs2 = N->getOperand(3); + SDValue A = DAG.getExtractVectorElt(DL, XLenVT, Rs1, Hi1 ? 1 : 0); + SDValue B = DAG.getExtractVectorElt(DL, XLenVT, Rs2, Hi2 ? 1 : 0); + auto [RdLo, RdHi] = DAG.SplitScalar(N->getOperand(1), DL, XLenVT, XLenVT); + unsigned Opc = IsRound ? RISCVISD::MQRWACC : RISCVISD::MQWACC; + SDVTList VTs = DAG.getVTList(XLenVT, XLenVT); + SDValue Acc = DAG.getNode(Opc, DL, VTs, {RdLo, RdHi, A, B}); + Results.push_back(DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, + Acc.getValue(0), Acc.getValue(1))); + return; + } case Intrinsic::riscv_orc_b: case Intrinsic::riscv_brev8: case Intrinsic::riscv_sha256sig0: diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoP.td b/llvm/lib/Target/RISCV/RISCVInstrInfoP.td index 9b72141ef9bcd..799949da7ce2b 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoP.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoP.td @@ -1966,6 +1966,50 @@ def SDT_RISCVPackedNarrowingClipW def riscv_pnclipp_w : RVSDNode<"PNCLIPP_W", SDT_RISCVPackedNarrowingClipW>; def riscv_pnclipup_w : RVSDNode<"PNCLIPUP_W", SDT_RISCVPackedNarrowingClipW>; +// Packed "Q-format" multiply parts accumulate. +def SDT_RISCVQFormatAccS + : SDTypeProfile<1, 3, [SDTCisVT<0, XLenVT>, SDTCisVT<1, XLenVT>, + SDTCisVT<2, XLenVT>, + SDTCisSameAs<2, 3>]>; +def SDT_RISCVQFormatAccP + : SDTypeProfile<1, 3, [SDTCisVT<0, v2i32>, SDTCisVT<1, v2i32>, + SDTCisVT<2, v4i16>, SDTCisSameAs<2, 3>]>; +def SDT_RISCVQFormatAccW + : SDTypeProfile<1, 3, [SDTCisVT<0, XLenVT>, SDTCisVT<1, XLenVT>, + SDTCisVT<2, v2i32>, + SDTCisSameAs<2, 3>]>; + +def riscv_mqacc_h00 : RVSDNode<"MQACC_H00", SDT_RISCVQFormatAccS>; +def riscv_mqacc_h01 : RVSDNode<"MQACC_H01", SDT_RISCVQFormatAccS>; +def riscv_mqacc_h11 : RVSDNode<"MQACC_H11", SDT_RISCVQFormatAccS>; +def riscv_mqracc_h00 : RVSDNode<"MQRACC_H00", SDT_RISCVQFormatAccS>; +def riscv_mqracc_h01 : RVSDNode<"MQRACC_H01", SDT_RISCVQFormatAccS>; +def riscv_mqracc_h11 : RVSDNode<"MQRACC_H11", SDT_RISCVQFormatAccS>; + +def riscv_pmqacc_h00 : RVSDNode<"PMQACC_W_H00", SDT_RISCVQFormatAccP>; +def riscv_pmqacc_h01 : RVSDNode<"PMQACC_W_H01", SDT_RISCVQFormatAccP>; +def riscv_pmqacc_h11 : RVSDNode<"PMQACC_W_H11", SDT_RISCVQFormatAccP>; +def riscv_pmqracc_h00 : RVSDNode<"PMQRACC_W_H00", SDT_RISCVQFormatAccP>; +def riscv_pmqracc_h01 : RVSDNode<"PMQRACC_W_H01", SDT_RISCVQFormatAccP>; +def riscv_pmqracc_h11 : RVSDNode<"PMQRACC_W_H11", SDT_RISCVQFormatAccP>; + +def riscv_pmqacc_w_h00 : RVSDNode<"PMQACC_W_H00_W", SDT_RISCVQFormatAccS>; +def riscv_pmqacc_w_h01 : RVSDNode<"PMQACC_W_H01_W", SDT_RISCVQFormatAccS>; +def riscv_pmqacc_w_h11 : RVSDNode<"PMQACC_W_H11_W", SDT_RISCVQFormatAccS>; +def riscv_pmqracc_w_h00 : RVSDNode<"PMQRACC_W_H00_W", SDT_RISCVQFormatAccS>; +def riscv_pmqracc_w_h01 : RVSDNode<"PMQRACC_W_H01_W", SDT_RISCVQFormatAccS>; +def riscv_pmqracc_w_h11 : RVSDNode<"PMQRACC_W_H11_W", SDT_RISCVQFormatAccS>; + +def riscv_mqacc_w00 : RVSDNode<"MQACC_W00", SDT_RISCVQFormatAccW>; +def riscv_mqacc_w01 : RVSDNode<"MQACC_W01", SDT_RISCVQFormatAccW>; +def riscv_mqacc_w11 : RVSDNode<"MQACC_W11", SDT_RISCVQFormatAccW>; +def riscv_mqracc_w00 : RVSDNode<"MQRACC_W00", SDT_RISCVQFormatAccW>; +def riscv_mqracc_w01 : RVSDNode<"MQRACC_W01", SDT_RISCVQFormatAccW>; +def riscv_mqracc_w11 : RVSDNode<"MQRACC_W11", SDT_RISCVQFormatAccW>; + +def riscv_mqwacc : RVSDNode<"MQWACC", SDT_RISCVWideningAddSubAccumulate>; +def riscv_mqrwacc : RVSDNode<"MQRWACC", SDT_RISCVWideningAddSubAccumulate>; + // The immediate for these is the number of trailing ones in the max value. def riscv_sati : RVSDNode<"SATI", SDTIntBinOp>; def riscv_usati : RVSDNode<"USATI", SDTIntBinOp>; @@ -2729,6 +2773,26 @@ let append Predicates = [IsRV32] in { (NCLIPI (BuildGPRPair GPR:$lo, GPR:$hi), timm:$imm)>; def : Pat<(XLenVT (riscv_nclipu XLenVT:$lo, XLenVT:$hi, timm:$imm)), (NCLIPIU (BuildGPRPair GPR:$lo, GPR:$hi), timm:$imm)>; + + // Packed "Q-format" multiply parts accumulate, scalar halfword (RV32). + def : Pat<(XLenVT (riscv_mqacc_h00 (XLenVT GPR:$rd), + (XLenVT GPR:$rs1), (XLenVT GPR:$rs2))), + (MQACC_H00 GPR:$rd, GPR:$rs1, GPR:$rs2)>; + def : Pat<(XLenVT (riscv_mqacc_h01 (XLenVT GPR:$rd), + (XLenVT GPR:$rs1), (XLenVT GPR:$rs2))), + (MQACC_H01 GPR:$rd, GPR:$rs1, GPR:$rs2)>; + def : Pat<(XLenVT (riscv_mqacc_h11 (XLenVT GPR:$rd), + (XLenVT GPR:$rs1), (XLenVT GPR:$rs2))), + (MQACC_H11 GPR:$rd, GPR:$rs1, GPR:$rs2)>; + def : Pat<(XLenVT (riscv_mqracc_h00 (XLenVT GPR:$rd), + (XLenVT GPR:$rs1), (XLenVT GPR:$rs2))), + (MQRACC_H00 GPR:$rd, GPR:$rs1, GPR:$rs2)>; + def : Pat<(XLenVT (riscv_mqracc_h01 (XLenVT GPR:$rd), + (XLenVT GPR:$rs1), (XLenVT GPR:$rs2))), + (MQRACC_H01 GPR:$rd, GPR:$rs1, GPR:$rs2)>; + def : Pat<(XLenVT (riscv_mqracc_h11 (XLenVT GPR:$rd), + (XLenVT GPR:$rs1), (XLenVT GPR:$rs2))), + (MQRACC_H11 GPR:$rd, GPR:$rs1, GPR:$rs2)>; } // append Predicates = [IsRV32] let append Predicates = [IsRV64] in { @@ -2751,6 +2815,64 @@ let append Predicates = [IsRV64] in { def : Pat<(v2i32 (riscv_pnclipup_w GPR:$rs1, GPR:$rs2)), (PNCLIPUP_W GPR:$rs1, GPR:$rs2)>; + // Packed "Q-format" multiply parts accumulate + def : Pat<(v2i32 (riscv_pmqacc_h00 (v2i32 GPR:$rd), + (v4i16 GPR:$rs1), (v4i16 GPR:$rs2))), + (PMQACC_W_H00 GPR:$rd, GPR:$rs1, GPR:$rs2)>; + def : Pat<(v2i32 (riscv_pmqacc_h01 (v2i32 GPR:$rd), + (v4i16 GPR:$rs1), (v4i16 GPR:$rs2))), + (PMQACC_W_H01 GPR:$rd, GPR:$rs1, GPR:$rs2)>; + def : Pat<(v2i32 (riscv_pmqacc_h11 (v2i32 GPR:$rd), + (v4i16 GPR:$rs1), (v4i16 GPR:$rs2))), + (PMQACC_W_H11 GPR:$rd, GPR:$rs1, GPR:$rs2)>; + def : Pat<(v2i32 (riscv_pmqracc_h00 (v2i32 GPR:$rd), + (v4i16 GPR:$rs1), (v4i16 GPR:$rs2))), + (PMQRACC_W_H00 GPR:$rd, GPR:$rs1, GPR:$rs2)>; + def : Pat<(v2i32 (riscv_pmqracc_h01 (v2i32 GPR:$rd), + (v4i16 GPR:$rs1), (v4i16 GPR:$rs2))), + (PMQRACC_W_H01 GPR:$rd, GPR:$rs1, GPR:$rs2)>; + def : Pat<(v2i32 (riscv_pmqracc_h11 (v2i32 GPR:$rd), + (v4i16 GPR:$rs1), (v4i16 GPR:$rs2))), + (PMQRACC_W_H11 GPR:$rd, GPR:$rs1, GPR:$rs2)>; + + def : Pat<(XLenVT (riscv_mqacc_w00 (XLenVT GPR:$rd), + (v2i32 GPR:$rs1), (v2i32 GPR:$rs2))), + (MQACC_W00 GPR:$rd, GPR:$rs1, GPR:$rs2)>; + def : Pat<(XLenVT (riscv_mqacc_w01 (XLenVT GPR:$rd), + (v2i32 GPR:$rs1), (v2i32 GPR:$rs2))), + (MQACC_W01 GPR:$rd, GPR:$rs1, GPR:$rs2)>; + def : Pat<(XLenVT (riscv_mqacc_w11 (XLenVT GPR:$rd), + (v2i32 GPR:$rs1), (v2i32 GPR:$rs2))), + (MQACC_W11 GPR:$rd, GPR:$rs1, GPR:$rs2)>; + def : Pat<(XLenVT (riscv_mqracc_w00 (XLenVT GPR:$rd), + (v2i32 GPR:$rs1), (v2i32 GPR:$rs2))), + (MQRACC_W00 GPR:$rd, GPR:$rs1, GPR:$rs2)>; + def : Pat<(XLenVT (riscv_mqracc_w01 (XLenVT GPR:$rd), + (v2i32 GPR:$rs1), (v2i32 GPR:$rs2))), + (MQRACC_W01 GPR:$rd, GPR:$rs1, GPR:$rs2)>; + def : Pat<(XLenVT (riscv_mqracc_w11 (XLenVT GPR:$rd), + (v2i32 GPR:$rs1), (v2i32 GPR:$rs2))), + (MQRACC_W11 GPR:$rd, GPR:$rs1, GPR:$rs2)>; + + def : Pat<(XLenVT (riscv_pmqacc_w_h00 (XLenVT GPR:$rd), + (XLenVT GPR:$rs1), (XLenVT GPR:$rs2))), + (PMQACC_W_H00 GPR:$rd, GPR:$rs1, GPR:$rs2)>; + def : Pat<(XLenVT (riscv_pmqacc_w_h01 (XLenVT GPR:$rd), + (XLenVT GPR:$rs1), (XLenVT GPR:$rs2))), + (PMQACC_W_H01 GPR:$rd, GPR:$rs1, GPR:$rs2)>; + def : Pat<(XLenVT (riscv_pmqacc_w_h11 (XLenVT GPR:$rd), + (XLenVT GPR:$rs1), (XLenVT GPR:$rs2))), + (PMQACC_W_H11 GPR:$rd, GPR:$rs1, GPR:$rs2)>; + def : Pat<(XLenVT (riscv_pmqracc_w_h00 (XLenVT GPR:$rd), + (XLenVT GPR:$rs1), (XLenVT GPR:$rs2))), + (PMQRACC_W_H00 GPR:$rd, GPR:$rs1, GPR:$rs2)>; + def : Pat<(XLenVT (riscv_pmqracc_w_h01 (XLenVT GPR:$rd), + (XLenVT GPR:$rs1), (XLenVT GPR:$rs2))), + (PMQRACC_W_H01 GPR:$rd, GPR:$rs1, GPR:$rs2)>; + def : Pat<(XLenVT (riscv_pmqracc_w_h11 (XLenVT GPR:$rd), + (XLenVT GPR:$rs1), (XLenVT GPR:$rs2))), + (PMQRACC_W_H11 GPR:$rd, GPR:$rs1, GPR:$rs2)>; + def : Pat<(XLenVT (riscv_sati GPR:$rs1, timm:$imm)), (SATI_RV64 GPR:$rs1, (IncImm timm:$imm))>; def : Pat<(XLenVT (riscv_usati GPR:$rs1, timm:$imm)), diff --git a/llvm/test/CodeGen/RISCV/rvp-simd-32.ll b/llvm/test/CodeGen/RISCV/rvp-simd-32.ll index d38012912872d..cebe60ef19249 100644 --- a/llvm/test/CodeGen/RISCV/rvp-simd-32.ll +++ b/llvm/test/CodeGen/RISCV/rvp-simd-32.ll @@ -3156,3 +3156,99 @@ define <2 x i16> @test_pnclipup_v2i16(i32 %a, i32 %b) { %r = call <2 x i16> @llvm.riscv.pnclipup.v2i16.i32(i32 %a, i32 %b) ret <2 x i16> %r } + +define i32 @test_mqacc_h00_i32(i32 %rd, <2 x i16> %a, <2 x i16> %b) { +; RV32-LABEL: test_mqacc_h00_i32: +; RV32: # %bb.0: +; RV32-NEXT: mqacc.h00 a0, a1, a2 +; RV32-NEXT: ret +; +; RV64-LABEL: test_mqacc_h00_i32: +; RV64: # %bb.0: +; RV64-NEXT: zext.w a2, a2 +; RV64-NEXT: zext.w a1, a1 +; RV64-NEXT: pmqacc.w.h00 a0, a1, a2 +; RV64-NEXT: ret + %r = call i32 @llvm.riscv.mqacc.h00.i32.v2i16(i32 %rd, <2 x i16> %a, <2 x i16> %b) + ret i32 %r +} + +define i32 @test_mqacc_h01_i32(i32 %rd, <2 x i16> %a, <2 x i16> %b) { +; RV32-LABEL: test_mqacc_h01_i32: +; RV32: # %bb.0: +; RV32-NEXT: mqacc.h01 a0, a1, a2 +; RV32-NEXT: ret +; +; RV64-LABEL: test_mqacc_h01_i32: +; RV64: # %bb.0: +; RV64-NEXT: zext.w a2, a2 +; RV64-NEXT: zext.w a1, a1 +; RV64-NEXT: pmqacc.w.h01 a0, a1, a2 +; RV64-NEXT: ret + %r = call i32 @llvm.riscv.mqacc.h01.i32.v2i16(i32 %rd, <2 x i16> %a, <2 x i16> %b) + ret i32 %r +} + +define i32 @test_mqacc_h11_i32(i32 %rd, <2 x i16> %a, <2 x i16> %b) { +; RV32-LABEL: test_mqacc_h11_i32: +; RV32: # %bb.0: +; RV32-NEXT: mqacc.h11 a0, a1, a2 +; RV32-NEXT: ret +; +; RV64-LABEL: test_mqacc_h11_i32: +; RV64: # %bb.0: +; RV64-NEXT: zext.w a2, a2 +; RV64-NEXT: zext.w a1, a1 +; RV64-NEXT: pmqacc.w.h11 a0, a1, a2 +; RV64-NEXT: ret + %r = call i32 @llvm.riscv.mqacc.h11.i32.v2i16(i32 %rd, <2 x i16> %a, <2 x i16> %b) + ret i32 %r +} + +define i32 @test_mqracc_h00_i32(i32 %rd, <2 x i16> %a, <2 x i16> %b) { +; RV32-LABEL: test_mqracc_h00_i32: +; RV32: # %bb.0: +; RV32-NEXT: mqracc.h00 a0, a1, a2 +; RV32-NEXT: ret +; +; RV64-LABEL: test_mqracc_h00_i32: +; RV64: # %bb.0: +; RV64-NEXT: zext.w a2, a2 +; RV64-NEXT: zext.w a1, a1 +; RV64-NEXT: pmqracc.w.h00 a0, a1, a2 +; RV64-NEXT: ret + %r = call i32 @llvm.riscv.mqracc.h00.i32.v2i16(i32 %rd, <2 x i16> %a, <2 x i16> %b) + ret i32 %r +} + +define i32 @test_mqracc_h01_i32(i32 %rd, <2 x i16> %a, <2 x i16> %b) { +; RV32-LABEL: test_mqracc_h01_i32: +; RV32: # %bb.0: +; RV32-NEXT: mqracc.h01 a0, a1, a2 +; RV32-NEXT: ret +; +; RV64-LABEL: test_mqracc_h01_i32: +; RV64: # %bb.0: +; RV64-NEXT: zext.w a2, a2 +; RV64-NEXT: zext.w a1, a1 +; RV64-NEXT: pmqracc.w.h01 a0, a1, a2 +; RV64-NEXT: ret + %r = call i32 @llvm.riscv.mqracc.h01.i32.v2i16(i32 %rd, <2 x i16> %a, <2 x i16> %b) + ret i32 %r +} + +define i32 @test_mqracc_h11_i32(i32 %rd, <2 x i16> %a, <2 x i16> %b) { +; RV32-LABEL: test_mqracc_h11_i32: +; RV32: # %bb.0: +; RV32-NEXT: mqracc.h11 a0, a1, a2 +; RV32-NEXT: ret +; +; RV64-LABEL: test_mqracc_h11_i32: +; RV64: # %bb.0: +; RV64-NEXT: zext.w a2, a2 +; RV64-NEXT: zext.w a1, a1 +; RV64-NEXT: pmqracc.w.h11 a0, a1, a2 +; RV64-NEXT: ret + %r = call i32 @llvm.riscv.mqracc.h11.i32.v2i16(i32 %rd, <2 x i16> %a, <2 x i16> %b) + ret i32 %r +} diff --git a/llvm/test/CodeGen/RISCV/rvp-simd-64.ll b/llvm/test/CodeGen/RISCV/rvp-simd-64.ll index 96f16c3d52103..51b86057ccdbe 100644 --- a/llvm/test/CodeGen/RISCV/rvp-simd-64.ll +++ b/llvm/test/CodeGen/RISCV/rvp-simd-64.ll @@ -6936,3 +6936,180 @@ define i64 @test_pnclipup_v2i32(i64 %a, i64 %b) { %s = bitcast <2 x i32> %r to i64 ret i64 %s } + +define <2 x i32> @test_pmqacc_h00_i32x2(<2 x i32> %rd, <4 x i16> %a, <4 x i16> %b) { +; RV32-LABEL: test_pmqacc_h00_i32x2: +; RV32: # %bb.0: +; RV32-NEXT: mqacc.h00 a1, a3, a5 +; RV32-NEXT: mqacc.h00 a0, a2, a4 +; RV32-NEXT: ret +; +; RV64-LABEL: test_pmqacc_h00_i32x2: +; RV64: # %bb.0: +; RV64-NEXT: pmqacc.w.h00 a0, a1, a2 +; RV64-NEXT: ret + %r = call <2 x i32> @llvm.riscv.pmqacc.h00.v2i32.v4i16(<2 x i32> %rd, <4 x i16> %a, <4 x i16> %b) + ret <2 x i32> %r +} + +define <2 x i32> @test_pmqacc_h01_i32x2(<2 x i32> %rd, <4 x i16> %a, <4 x i16> %b) { +; RV32-LABEL: test_pmqacc_h01_i32x2: +; RV32: # %bb.0: +; RV32-NEXT: mqacc.h01 a1, a3, a5 +; RV32-NEXT: mqacc.h01 a0, a2, a4 +; RV32-NEXT: ret +; +; RV64-LABEL: test_pmqacc_h01_i32x2: +; RV64: # %bb.0: +; RV64-NEXT: pmqacc.w.h01 a0, a1, a2 +; RV64-NEXT: ret + %r = call <2 x i32> @llvm.riscv.pmqacc.h01.v2i32.v4i16(<2 x i32> %rd, <4 x i16> %a, <4 x i16> %b) + ret <2 x i32> %r +} + +define <2 x i32> @test_pmqacc_h11_i32x2(<2 x i32> %rd, <4 x i16> %a, <4 x i16> %b) { +; RV32-LABEL: test_pmqacc_h11_i32x2: +; RV32: # %bb.0: +; RV32-NEXT: mqacc.h11 a1, a3, a5 +; RV32-NEXT: mqacc.h11 a0, a2, a4 +; RV32-NEXT: ret +; +; RV64-LABEL: test_pmqacc_h11_i32x2: +; RV64: # %bb.0: +; RV64-NEXT: pmqacc.w.h11 a0, a1, a2 +; RV64-NEXT: ret + %r = call <2 x i32> @llvm.riscv.pmqacc.h11.v2i32.v4i16(<2 x i32> %rd, <4 x i16> %a, <4 x i16> %b) + ret <2 x i32> %r +} + +define <2 x i32> @test_pmqracc_h00_i32x2(<2 x i32> %rd, <4 x i16> %a, <4 x i16> %b) { +; RV32-LABEL: test_pmqracc_h00_i32x2: +; RV32: # %bb.0: +; RV32-NEXT: mqracc.h00 a1, a3, a5 +; RV32-NEXT: mqracc.h00 a0, a2, a4 +; RV32-NEXT: ret +; +; RV64-LABEL: test_pmqracc_h00_i32x2: +; RV64: # %bb.0: +; RV64-NEXT: pmqracc.w.h00 a0, a1, a2 +; RV64-NEXT: ret + %r = call <2 x i32> @llvm.riscv.pmqracc.h00.v2i32.v4i16(<2 x i32> %rd, <4 x i16> %a, <4 x i16> %b) + ret <2 x i32> %r +} + +define <2 x i32> @test_pmqracc_h01_i32x2(<2 x i32> %rd, <4 x i16> %a, <4 x i16> %b) { +; RV32-LABEL: test_pmqracc_h01_i32x2: +; RV32: # %bb.0: +; RV32-NEXT: mqracc.h01 a1, a3, a5 +; RV32-NEXT: mqracc.h01 a0, a2, a4 +; RV32-NEXT: ret +; +; RV64-LABEL: test_pmqracc_h01_i32x2: +; RV64: # %bb.0: +; RV64-NEXT: pmqracc.w.h01 a0, a1, a2 +; RV64-NEXT: ret + %r = call <2 x i32> @llvm.riscv.pmqracc.h01.v2i32.v4i16(<2 x i32> %rd, <4 x i16> %a, <4 x i16> %b) + ret <2 x i32> %r +} + +define <2 x i32> @test_pmqracc_h11_i32x2(<2 x i32> %rd, <4 x i16> %a, <4 x i16> %b) { +; RV32-LABEL: test_pmqracc_h11_i32x2: +; RV32: # %bb.0: +; RV32-NEXT: mqracc.h11 a1, a3, a5 +; RV32-NEXT: mqracc.h11 a0, a2, a4 +; RV32-NEXT: ret +; +; RV64-LABEL: test_pmqracc_h11_i32x2: +; RV64: # %bb.0: +; RV64-NEXT: pmqracc.w.h11 a0, a1, a2 +; RV64-NEXT: ret + %r = call <2 x i32> @llvm.riscv.pmqracc.h11.v2i32.v4i16(<2 x i32> %rd, <4 x i16> %a, <4 x i16> %b) + ret <2 x i32> %r +} + +; === Packed "Q-format" Multiply Parts Accumulate: word (i64 result) === +; RV64 uses mqacc.wXX; RV32 uses mqwacc/mqrwacc (register pair). + +define i64 @test_mqacc_w00_i64(i64 %rd, <2 x i32> %a, <2 x i32> %b) { +; RV32-LABEL: test_mqacc_w00_i64: +; RV32: # %bb.0: +; RV32-NEXT: mqwacc a0, a2, a4 +; RV32-NEXT: ret +; +; RV64-LABEL: test_mqacc_w00_i64: +; RV64: # %bb.0: +; RV64-NEXT: mqacc.w00 a0, a1, a2 +; RV64-NEXT: ret + %r = call i64 @llvm.riscv.mqacc.w00.i64.v2i32(i64 %rd, <2 x i32> %a, <2 x i32> %b) + ret i64 %r +} + +define i64 @test_mqacc_w01_i64(i64 %rd, <2 x i32> %a, <2 x i32> %b) { +; RV32-LABEL: test_mqacc_w01_i64: +; RV32: # %bb.0: +; RV32-NEXT: mqwacc a0, a2, a5 +; RV32-NEXT: ret +; +; RV64-LABEL: test_mqacc_w01_i64: +; RV64: # %bb.0: +; RV64-NEXT: mqacc.w01 a0, a1, a2 +; RV64-NEXT: ret + %r = call i64 @llvm.riscv.mqacc.w01.i64.v2i32(i64 %rd, <2 x i32> %a, <2 x i32> %b) + ret i64 %r +} + +define i64 @test_mqacc_w11_i64(i64 %rd, <2 x i32> %a, <2 x i32> %b) { +; RV32-LABEL: test_mqacc_w11_i64: +; RV32: # %bb.0: +; RV32-NEXT: mqwacc a0, a3, a5 +; RV32-NEXT: ret +; +; RV64-LABEL: test_mqacc_w11_i64: +; RV64: # %bb.0: +; RV64-NEXT: mqacc.w11 a0, a1, a2 +; RV64-NEXT: ret + %r = call i64 @llvm.riscv.mqacc.w11.i64.v2i32(i64 %rd, <2 x i32> %a, <2 x i32> %b) + ret i64 %r +} + +define i64 @test_mqracc_w00_i64(i64 %rd, <2 x i32> %a, <2 x i32> %b) { +; RV32-LABEL: test_mqracc_w00_i64: +; RV32: # %bb.0: +; RV32-NEXT: mqrwacc a0, a2, a4 +; RV32-NEXT: ret +; +; RV64-LABEL: test_mqracc_w00_i64: +; RV64: # %bb.0: +; RV64-NEXT: mqracc.w00 a0, a1, a2 +; RV64-NEXT: ret + %r = call i64 @llvm.riscv.mqracc.w00.i64.v2i32(i64 %rd, <2 x i32> %a, <2 x i32> %b) + ret i64 %r +} + +define i64 @test_mqracc_w01_i64(i64 %rd, <2 x i32> %a, <2 x i32> %b) { +; RV32-LABEL: test_mqracc_w01_i64: +; RV32: # %bb.0: +; RV32-NEXT: mqrwacc a0, a2, a5 +; RV32-NEXT: ret +; +; RV64-LABEL: test_mqracc_w01_i64: +; RV64: # %bb.0: +; RV64-NEXT: mqracc.w01 a0, a1, a2 +; RV64-NEXT: ret + %r = call i64 @llvm.riscv.mqracc.w01.i64.v2i32(i64 %rd, <2 x i32> %a, <2 x i32> %b) + ret i64 %r +} + +define i64 @test_mqracc_w11_i64(i64 %rd, <2 x i32> %a, <2 x i32> %b) { +; RV32-LABEL: test_mqracc_w11_i64: +; RV32: # %bb.0: +; RV32-NEXT: mqrwacc a0, a3, a5 +; RV32-NEXT: ret +; +; RV64-LABEL: test_mqracc_w11_i64: +; RV64: # %bb.0: +; RV64-NEXT: mqracc.w11 a0, a1, a2 +; RV64-NEXT: ret + %r = call i64 @llvm.riscv.mqracc.w11.i64.v2i32(i64 %rd, <2 x i32> %a, <2 x i32> %b) + ret i64 %r +} >From 7737d85830fa00394a1e5e94ed2f62552e2fecd5 Mon Sep 17 00:00:00 2001 From: Dongyan Chen <[email protected]> Date: Sun, 23 Aug 2026 12:08:31 +0000 Subject: [PATCH 2/2] [RISCV][P-ext] resolve comment --- clang/lib/Headers/riscv_packed_simd.h | 46 ++++++++--------- llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 55 ++++++--------------- llvm/lib/Target/RISCV/RISCVInstrInfoP.td | 40 +++------------ llvm/test/CodeGen/RISCV/rvp-simd-32.ll | 26 ++++------ 4 files changed, 56 insertions(+), 111 deletions(-) diff --git a/clang/lib/Headers/riscv_packed_simd.h b/clang/lib/Headers/riscv_packed_simd.h index aaf7112d583f4..b5db8422a3381 100644 --- a/clang/lib/Headers/riscv_packed_simd.h +++ b/clang/lib/Headers/riscv_packed_simd.h @@ -285,7 +285,7 @@ typedef uint32_t uint32x2_t __attribute__((__vector_size__(8))); return builtin(__rs1, __rs2); \ } -#define __packed_ternary_builtin(name, rty, ty, builtin) \ +#define __packed_ternary_builtin_cast(name, rty, ty, builtin) \ static __inline__ rty __DEFAULT_FN_ATTRS __riscv_##name(rty __rd, ty __rs1, \ ty __rs2) { \ return builtin(__rd, __rs1, __rs2); \ @@ -815,13 +815,13 @@ __packed_ternary_builtin_mixed(pmhraccsu_i32x2, int32x2_t, int32x2_t, uint32x2_t /* Packed Absolute Difference Sum (32-bit) */ __packed_abdsum(pabdsumu_u8x4_u32, uint32_t, uint8x4_t, __builtin_riscv_pabdsumu_u8x4_u32) -__packed_ternary_builtin(pabdsumau_u8x4_u32, uint32_t, uint8x4_t, __builtin_riscv_pabdsumau_u8x4_u32) +__packed_ternary_builtin_cast(pabdsumau_u8x4_u32, uint32_t, uint8x4_t, __builtin_riscv_pabdsumau_u8x4_u32) /* Packed Absolute Difference Sum (64-bit) */ __packed_abdsum(pabdsumu_u8x8_u32, uint32_t, uint8x8_t, __builtin_riscv_pabdsumu_u8x8_u32) __packed_abdsum(pabdsumu_u8x8_u64, uint64_t, uint8x8_t, __builtin_riscv_pabdsumu_u8x8_u64) -__packed_ternary_builtin(pabdsumau_u8x8_u32, uint32_t, uint8x8_t, __builtin_riscv_pabdsumau_u8x8_u32) -__packed_ternary_builtin(pabdsumau_u8x8_u64, uint64_t, uint8x8_t, __builtin_riscv_pabdsumau_u8x8_u64) +__packed_ternary_builtin_cast(pabdsumau_u8x8_u32, uint32_t, uint8x8_t, __builtin_riscv_pabdsumau_u8x8_u32) +__packed_ternary_builtin_cast(pabdsumau_u8x8_u64, uint64_t, uint8x8_t, __builtin_riscv_pabdsumau_u8x8_u64) /* Packed Saturating Absolute Value (32-bit) */ __packed_unary_builtin(psabs_i8x4, int8x4_t, __builtin_riscv_psabs_i8x4) @@ -853,26 +853,26 @@ __packed_binary_builtin(pmulq_i32x2, int32x2_t, __builtin_riscv_pmulq_i32x2) __packed_binary_builtin(pmulqr_i32x2, int32x2_t, __builtin_riscv_pmulqr_i32x2) /* Packed "Q-format" Multiply Parts Accumulate (32-bit) */ -__packed_ternary_builtin(mqacc_h00_i32, int, int16x2_t, __builtin_riscv_mqacc_h00_i32) -__packed_ternary_builtin(mqacc_h01_i32, int, int16x2_t, __builtin_riscv_mqacc_h01_i32) -__packed_ternary_builtin(mqacc_h11_i32, int, int16x2_t, __builtin_riscv_mqacc_h11_i32) -__packed_ternary_builtin(mqracc_h00_i32, int, int16x2_t, __builtin_riscv_mqracc_h00_i32) -__packed_ternary_builtin(mqracc_h01_i32, int, int16x2_t, __builtin_riscv_mqracc_h01_i32) -__packed_ternary_builtin(mqracc_h11_i32, int, int16x2_t, __builtin_riscv_mqracc_h11_i32) +__packed_ternary_builtin_cast(mqacc_h00_i32, int, int16x2_t, __builtin_riscv_mqacc_h00_i32) +__packed_ternary_builtin_cast(mqacc_h01_i32, int, int16x2_t, __builtin_riscv_mqacc_h01_i32) +__packed_ternary_builtin_cast(mqacc_h11_i32, int, int16x2_t, __builtin_riscv_mqacc_h11_i32) +__packed_ternary_builtin_cast(mqracc_h00_i32, int, int16x2_t, __builtin_riscv_mqracc_h00_i32) +__packed_ternary_builtin_cast(mqracc_h01_i32, int, int16x2_t, __builtin_riscv_mqracc_h01_i32) +__packed_ternary_builtin_cast(mqracc_h11_i32, int, int16x2_t, __builtin_riscv_mqracc_h11_i32) /* Packed "Q-format" Multiply Parts Accumulate (64-bit) */ -__packed_ternary_builtin(pmqacc_h00_i32x2, int32x2_t, int16x4_t, __builtin_riscv_pmqacc_h00_i32x2) -__packed_ternary_builtin(pmqacc_h01_i32x2, int32x2_t, int16x4_t, __builtin_riscv_pmqacc_h01_i32x2) -__packed_ternary_builtin(pmqacc_h11_i32x2, int32x2_t, int16x4_t, __builtin_riscv_pmqacc_h11_i32x2) -__packed_ternary_builtin(pmqracc_h00_i32x2, int32x2_t, int16x4_t, __builtin_riscv_pmqracc_h00_i32x2) -__packed_ternary_builtin(pmqracc_h01_i32x2, int32x2_t, int16x4_t, __builtin_riscv_pmqracc_h01_i32x2) -__packed_ternary_builtin(pmqracc_h11_i32x2, int32x2_t, int16x4_t, __builtin_riscv_pmqracc_h11_i32x2) -__packed_ternary_builtin(mqacc_w00_i64, int64_t, int32x2_t, __builtin_riscv_mqacc_w00_i64) -__packed_ternary_builtin(mqacc_w01_i64, int64_t, int32x2_t, __builtin_riscv_mqacc_w01_i64) -__packed_ternary_builtin(mqacc_w11_i64, int64_t, int32x2_t, __builtin_riscv_mqacc_w11_i64) -__packed_ternary_builtin(mqracc_w00_i64, int64_t, int32x2_t, __builtin_riscv_mqracc_w00_i64) -__packed_ternary_builtin(mqracc_w01_i64, int64_t, int32x2_t, __builtin_riscv_mqracc_w01_i64) -__packed_ternary_builtin(mqracc_w11_i64, int64_t, int32x2_t, __builtin_riscv_mqracc_w11_i64) +__packed_ternary_builtin_cast(pmqacc_h00_i32x2, int32x2_t, int16x4_t, __builtin_riscv_pmqacc_h00_i32x2) +__packed_ternary_builtin_cast(pmqacc_h01_i32x2, int32x2_t, int16x4_t, __builtin_riscv_pmqacc_h01_i32x2) +__packed_ternary_builtin_cast(pmqacc_h11_i32x2, int32x2_t, int16x4_t, __builtin_riscv_pmqacc_h11_i32x2) +__packed_ternary_builtin_cast(pmqracc_h00_i32x2, int32x2_t, int16x4_t, __builtin_riscv_pmqracc_h00_i32x2) +__packed_ternary_builtin_cast(pmqracc_h01_i32x2, int32x2_t, int16x4_t, __builtin_riscv_pmqracc_h01_i32x2) +__packed_ternary_builtin_cast(pmqracc_h11_i32x2, int32x2_t, int16x4_t, __builtin_riscv_pmqracc_h11_i32x2) +__packed_ternary_builtin_cast(mqacc_w00_i64, int64_t, int32x2_t, __builtin_riscv_mqacc_w00_i64) +__packed_ternary_builtin_cast(mqacc_w01_i64, int64_t, int32x2_t, __builtin_riscv_mqacc_w01_i64) +__packed_ternary_builtin_cast(mqacc_w11_i64, int64_t, int32x2_t, __builtin_riscv_mqacc_w11_i64) +__packed_ternary_builtin_cast(mqracc_w00_i64, int64_t, int32x2_t, __builtin_riscv_mqracc_w00_i64) +__packed_ternary_builtin_cast(mqracc_w01_i64, int64_t, int32x2_t, __builtin_riscv_mqracc_w01_i64) +__packed_ternary_builtin_cast(mqracc_w11_i64, int64_t, int32x2_t, __builtin_riscv_mqracc_w11_i64) /* Packed Narrowing Clip Pair (32-bit) */ __packed_binary_builtin_cast(pnclipp_i8x4, int16x2_t, int8x4_t, __builtin_riscv_pnclipp_i8x4) @@ -1034,7 +1034,7 @@ __packed_reinterpret(u32x2_i32x2, int32x2_t, uint32x2_t) #undef __packed_nziph2 #undef __packed_nziph4 #undef __packed_abdsum -#undef __packed_ternary_builtin +#undef __packed_ternary_builtin_cast #undef __packed_reinterpret #undef __DEFAULT_FN_ATTRS diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index fe37fa4f3dfba..34b5d23aed088 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -12289,16 +12289,22 @@ static unsigned getRVPQFormatAccOpcode(Intrinsic::ID IntNo) { switch (IntNo) { default: llvm_unreachable("Unexpected RISC-V packed Q-format accumulate intrinsic"); + case Intrinsic::riscv_mqacc_h00: case Intrinsic::riscv_pmqacc_h00: return RISCVISD::PMQACC_W_H00; + case Intrinsic::riscv_mqacc_h01: case Intrinsic::riscv_pmqacc_h01: return RISCVISD::PMQACC_W_H01; + case Intrinsic::riscv_mqacc_h11: case Intrinsic::riscv_pmqacc_h11: return RISCVISD::PMQACC_W_H11; + case Intrinsic::riscv_mqracc_h00: case Intrinsic::riscv_pmqracc_h00: return RISCVISD::PMQRACC_W_H00; + case Intrinsic::riscv_mqracc_h01: case Intrinsic::riscv_pmqracc_h01: return RISCVISD::PMQRACC_W_H01; + case Intrinsic::riscv_mqracc_h11: case Intrinsic::riscv_pmqracc_h11: return RISCVISD::PMQRACC_W_H11; } @@ -12323,25 +12329,6 @@ static unsigned getRVPQFormatAccWordOpcode(Intrinsic::ID IntNo) { } } -static unsigned getRVPQFormatAccWideOpcode(Intrinsic::ID IntNo) { - switch (IntNo) { - default: - llvm_unreachable("Unexpected RISC-V packed Q-format accumulate intrinsic"); - case Intrinsic::riscv_mqacc_h00: - return RISCVISD::PMQACC_W_H00_W; - case Intrinsic::riscv_mqacc_h01: - return RISCVISD::PMQACC_W_H01_W; - case Intrinsic::riscv_mqacc_h11: - return RISCVISD::PMQACC_W_H11_W; - case Intrinsic::riscv_mqracc_h00: - return RISCVISD::PMQRACC_W_H00_W; - case Intrinsic::riscv_mqracc_h01: - return RISCVISD::PMQRACC_W_H01_W; - case Intrinsic::riscv_mqracc_h11: - return RISCVISD::PMQRACC_W_H11_W; - } -} - SDValue RISCVTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const { unsigned IntNo = Op.getConstantOperandVal(0); @@ -12599,19 +12586,14 @@ SDValue RISCVTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, SDValue RdLo = DAG.getExtractVectorElt(DL, XLenVT, Rd, 0); SDValue RdHi = DAG.getExtractVectorElt(DL, XLenVT, Rd, 1); unsigned ScalarOpc = getRVPQFormatAccScalarOpcode(IntNo); - SDValue Lo = DAG.getNode(ScalarOpc, DL, XLenVT, RdLo, - DAG.getBitcast(XLenVT, Rs1Lo), - DAG.getBitcast(XLenVT, Rs2Lo)); - SDValue Hi = DAG.getNode(ScalarOpc, DL, XLenVT, RdHi, - DAG.getBitcast(XLenVT, Rs1Hi), - DAG.getBitcast(XLenVT, Rs2Hi)); + SDValue Lo = DAG.getNode(ScalarOpc, DL, XLenVT, RdLo, Rs1Lo, Rs2Lo); + SDValue Hi = DAG.getNode(ScalarOpc, DL, XLenVT, RdHi, Rs1Hi, Rs2Hi); return DAG.getNode(ISD::BUILD_VECTOR, DL, VT, Lo, Hi); } if (VT == MVT::i32 && !Subtarget.is64Bit()) { unsigned Opc = getRVPQFormatAccScalarOpcode(IntNo); - return DAG.getNode(Opc, DL, XLenVT, Rd, DAG.getBitcast(XLenVT, Rs1), - DAG.getBitcast(XLenVT, Rs2)); + return DAG.getNode(Opc, DL, XLenVT, Rd, Rs1, Rs2); } if (VT == MVT::i64 && Subtarget.is64Bit()) { @@ -17126,22 +17108,17 @@ void RISCVTargetLowering::ReplaceNodeResults(SDNode *N, EVT VT = N->getValueType(0); if (!Subtarget.is64Bit() || VT != MVT::i32) return; - MVT XLenVT = Subtarget.getXLenVT(); - // Widen a v2i16 source to XLenVT: bitcast to the 32-bit scalar it shares - // bits with, then zero-extend into the upper half of the GPR. + SDValue Rd = DAG.getBuildVector(MVT::v2i32, DL, + {N->getOperand(1), DAG.getUNDEF(MVT::i32)}); auto WidenSrc = [&](SDValue V) { - if (V.getValueType() == XLenVT) - return V; - assert(V.getValueType() == MVT::v2i16); - return DAG.getNode(ISD::ZERO_EXTEND, DL, XLenVT, - DAG.getBitcast(MVT::i32, V)); + return DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4i16, + {V, DAG.getUNDEF(MVT::v2i16)}); }; - SDValue Rd = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, N->getOperand(1)); SDValue Rs1 = WidenSrc(N->getOperand(2)); SDValue Rs2 = WidenSrc(N->getOperand(3)); - unsigned Opc = getRVPQFormatAccWideOpcode(IntNo); - SDValue Res = DAG.getNode(Opc, DL, XLenVT, Rd, Rs1, Rs2); - Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res)); + unsigned Opc = getRVPQFormatAccOpcode(IntNo); + SDValue Res = DAG.getNode(Opc, DL, MVT::v2i32, Rd, Rs1, Rs2); + Results.push_back(DAG.getExtractVectorElt(DL, MVT::i32, Res, 0)); return; } case Intrinsic::riscv_mqacc_w00: diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoP.td b/llvm/lib/Target/RISCV/RISCVInstrInfoP.td index 799949da7ce2b..ef008ef0745da 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoP.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoP.td @@ -1969,7 +1969,7 @@ def riscv_pnclipup_w : RVSDNode<"PNCLIPUP_W", SDT_RISCVPackedNarrowingClipW>; // Packed "Q-format" multiply parts accumulate. def SDT_RISCVQFormatAccS : SDTypeProfile<1, 3, [SDTCisVT<0, XLenVT>, SDTCisVT<1, XLenVT>, - SDTCisVT<2, XLenVT>, + SDTCisVT<2, v2i16>, SDTCisSameAs<2, 3>]>; def SDT_RISCVQFormatAccP : SDTypeProfile<1, 3, [SDTCisVT<0, v2i32>, SDTCisVT<1, v2i32>, @@ -1993,13 +1993,6 @@ def riscv_pmqracc_h00 : RVSDNode<"PMQRACC_W_H00", SDT_RISCVQFormatAccP>; def riscv_pmqracc_h01 : RVSDNode<"PMQRACC_W_H01", SDT_RISCVQFormatAccP>; def riscv_pmqracc_h11 : RVSDNode<"PMQRACC_W_H11", SDT_RISCVQFormatAccP>; -def riscv_pmqacc_w_h00 : RVSDNode<"PMQACC_W_H00_W", SDT_RISCVQFormatAccS>; -def riscv_pmqacc_w_h01 : RVSDNode<"PMQACC_W_H01_W", SDT_RISCVQFormatAccS>; -def riscv_pmqacc_w_h11 : RVSDNode<"PMQACC_W_H11_W", SDT_RISCVQFormatAccS>; -def riscv_pmqracc_w_h00 : RVSDNode<"PMQRACC_W_H00_W", SDT_RISCVQFormatAccS>; -def riscv_pmqracc_w_h01 : RVSDNode<"PMQRACC_W_H01_W", SDT_RISCVQFormatAccS>; -def riscv_pmqracc_w_h11 : RVSDNode<"PMQRACC_W_H11_W", SDT_RISCVQFormatAccS>; - def riscv_mqacc_w00 : RVSDNode<"MQACC_W00", SDT_RISCVQFormatAccW>; def riscv_mqacc_w01 : RVSDNode<"MQACC_W01", SDT_RISCVQFormatAccW>; def riscv_mqacc_w11 : RVSDNode<"MQACC_W11", SDT_RISCVQFormatAccW>; @@ -2776,22 +2769,22 @@ let append Predicates = [IsRV32] in { // Packed "Q-format" multiply parts accumulate, scalar halfword (RV32). def : Pat<(XLenVT (riscv_mqacc_h00 (XLenVT GPR:$rd), - (XLenVT GPR:$rs1), (XLenVT GPR:$rs2))), + (v2i16 GPR:$rs1), (v2i16 GPR:$rs2))), (MQACC_H00 GPR:$rd, GPR:$rs1, GPR:$rs2)>; def : Pat<(XLenVT (riscv_mqacc_h01 (XLenVT GPR:$rd), - (XLenVT GPR:$rs1), (XLenVT GPR:$rs2))), + (v2i16 GPR:$rs1), (v2i16 GPR:$rs2))), (MQACC_H01 GPR:$rd, GPR:$rs1, GPR:$rs2)>; def : Pat<(XLenVT (riscv_mqacc_h11 (XLenVT GPR:$rd), - (XLenVT GPR:$rs1), (XLenVT GPR:$rs2))), + (v2i16 GPR:$rs1), (v2i16 GPR:$rs2))), (MQACC_H11 GPR:$rd, GPR:$rs1, GPR:$rs2)>; def : Pat<(XLenVT (riscv_mqracc_h00 (XLenVT GPR:$rd), - (XLenVT GPR:$rs1), (XLenVT GPR:$rs2))), + (v2i16 GPR:$rs1), (v2i16 GPR:$rs2))), (MQRACC_H00 GPR:$rd, GPR:$rs1, GPR:$rs2)>; def : Pat<(XLenVT (riscv_mqracc_h01 (XLenVT GPR:$rd), - (XLenVT GPR:$rs1), (XLenVT GPR:$rs2))), + (v2i16 GPR:$rs1), (v2i16 GPR:$rs2))), (MQRACC_H01 GPR:$rd, GPR:$rs1, GPR:$rs2)>; def : Pat<(XLenVT (riscv_mqracc_h11 (XLenVT GPR:$rd), - (XLenVT GPR:$rs1), (XLenVT GPR:$rs2))), + (v2i16 GPR:$rs1), (v2i16 GPR:$rs2))), (MQRACC_H11 GPR:$rd, GPR:$rs1, GPR:$rs2)>; } // append Predicates = [IsRV32] @@ -2854,25 +2847,6 @@ let append Predicates = [IsRV64] in { (v2i32 GPR:$rs1), (v2i32 GPR:$rs2))), (MQRACC_W11 GPR:$rd, GPR:$rs1, GPR:$rs2)>; - def : Pat<(XLenVT (riscv_pmqacc_w_h00 (XLenVT GPR:$rd), - (XLenVT GPR:$rs1), (XLenVT GPR:$rs2))), - (PMQACC_W_H00 GPR:$rd, GPR:$rs1, GPR:$rs2)>; - def : Pat<(XLenVT (riscv_pmqacc_w_h01 (XLenVT GPR:$rd), - (XLenVT GPR:$rs1), (XLenVT GPR:$rs2))), - (PMQACC_W_H01 GPR:$rd, GPR:$rs1, GPR:$rs2)>; - def : Pat<(XLenVT (riscv_pmqacc_w_h11 (XLenVT GPR:$rd), - (XLenVT GPR:$rs1), (XLenVT GPR:$rs2))), - (PMQACC_W_H11 GPR:$rd, GPR:$rs1, GPR:$rs2)>; - def : Pat<(XLenVT (riscv_pmqracc_w_h00 (XLenVT GPR:$rd), - (XLenVT GPR:$rs1), (XLenVT GPR:$rs2))), - (PMQRACC_W_H00 GPR:$rd, GPR:$rs1, GPR:$rs2)>; - def : Pat<(XLenVT (riscv_pmqracc_w_h01 (XLenVT GPR:$rd), - (XLenVT GPR:$rs1), (XLenVT GPR:$rs2))), - (PMQRACC_W_H01 GPR:$rd, GPR:$rs1, GPR:$rs2)>; - def : Pat<(XLenVT (riscv_pmqracc_w_h11 (XLenVT GPR:$rd), - (XLenVT GPR:$rs1), (XLenVT GPR:$rs2))), - (PMQRACC_W_H11 GPR:$rd, GPR:$rs1, GPR:$rs2)>; - def : Pat<(XLenVT (riscv_sati GPR:$rs1, timm:$imm)), (SATI_RV64 GPR:$rs1, (IncImm timm:$imm))>; def : Pat<(XLenVT (riscv_usati GPR:$rs1, timm:$imm)), diff --git a/llvm/test/CodeGen/RISCV/rvp-simd-32.ll b/llvm/test/CodeGen/RISCV/rvp-simd-32.ll index cebe60ef19249..07d2319f81e59 100644 --- a/llvm/test/CodeGen/RISCV/rvp-simd-32.ll +++ b/llvm/test/CodeGen/RISCV/rvp-simd-32.ll @@ -2408,10 +2408,10 @@ define <2 x i16> @test_select_v2i16(i1 %cond, <2 x i16> %a, <2 x i16> %b) { ; CHECK: # %bb.0: ; CHECK-NEXT: andi a3, a0, 1 ; CHECK-NEXT: mv a0, a1 -; CHECK-NEXT: bnez a3, [[SELECT_BB:.LBB[0-9]+_2]] +; CHECK-NEXT: bnez a3, .LBB170_2 ; CHECK-NEXT: # %bb.1: ; CHECK-NEXT: mv a0, a2 -; CHECK-NEXT: [[SELECT_BB]]: +; CHECK-NEXT: .LBB170_2: ; CHECK-NEXT: ret %res = select i1 %cond, <2 x i16> %a, <2 x i16> %b ret <2 x i16> %res @@ -2422,10 +2422,10 @@ define <4 x i8> @test_select_v4i8(i1 %cond, <4 x i8> %a, <4 x i8> %b) { ; CHECK: # %bb.0: ; CHECK-NEXT: andi a3, a0, 1 ; CHECK-NEXT: mv a0, a1 -; CHECK-NEXT: bnez a3, [[SELECT_BB:.LBB[0-9]+_2]] +; CHECK-NEXT: bnez a3, .LBB171_2 ; CHECK-NEXT: # %bb.1: ; CHECK-NEXT: mv a0, a2 -; CHECK-NEXT: [[SELECT_BB]]: +; CHECK-NEXT: .LBB171_2: ; CHECK-NEXT: ret %res = select i1 %cond, <4 x i8> %a, <4 x i8> %b ret <4 x i8> %res @@ -3165,8 +3165,7 @@ define i32 @test_mqacc_h00_i32(i32 %rd, <2 x i16> %a, <2 x i16> %b) { ; ; RV64-LABEL: test_mqacc_h00_i32: ; RV64: # %bb.0: -; RV64-NEXT: zext.w a2, a2 -; RV64-NEXT: zext.w a1, a1 +; RV64-NEXT: pmv.ws a0, a0 ; RV64-NEXT: pmqacc.w.h00 a0, a1, a2 ; RV64-NEXT: ret %r = call i32 @llvm.riscv.mqacc.h00.i32.v2i16(i32 %rd, <2 x i16> %a, <2 x i16> %b) @@ -3181,8 +3180,7 @@ define i32 @test_mqacc_h01_i32(i32 %rd, <2 x i16> %a, <2 x i16> %b) { ; ; RV64-LABEL: test_mqacc_h01_i32: ; RV64: # %bb.0: -; RV64-NEXT: zext.w a2, a2 -; RV64-NEXT: zext.w a1, a1 +; RV64-NEXT: pmv.ws a0, a0 ; RV64-NEXT: pmqacc.w.h01 a0, a1, a2 ; RV64-NEXT: ret %r = call i32 @llvm.riscv.mqacc.h01.i32.v2i16(i32 %rd, <2 x i16> %a, <2 x i16> %b) @@ -3197,8 +3195,7 @@ define i32 @test_mqacc_h11_i32(i32 %rd, <2 x i16> %a, <2 x i16> %b) { ; ; RV64-LABEL: test_mqacc_h11_i32: ; RV64: # %bb.0: -; RV64-NEXT: zext.w a2, a2 -; RV64-NEXT: zext.w a1, a1 +; RV64-NEXT: pmv.ws a0, a0 ; RV64-NEXT: pmqacc.w.h11 a0, a1, a2 ; RV64-NEXT: ret %r = call i32 @llvm.riscv.mqacc.h11.i32.v2i16(i32 %rd, <2 x i16> %a, <2 x i16> %b) @@ -3213,8 +3210,7 @@ define i32 @test_mqracc_h00_i32(i32 %rd, <2 x i16> %a, <2 x i16> %b) { ; ; RV64-LABEL: test_mqracc_h00_i32: ; RV64: # %bb.0: -; RV64-NEXT: zext.w a2, a2 -; RV64-NEXT: zext.w a1, a1 +; RV64-NEXT: pmv.ws a0, a0 ; RV64-NEXT: pmqracc.w.h00 a0, a1, a2 ; RV64-NEXT: ret %r = call i32 @llvm.riscv.mqracc.h00.i32.v2i16(i32 %rd, <2 x i16> %a, <2 x i16> %b) @@ -3229,8 +3225,7 @@ define i32 @test_mqracc_h01_i32(i32 %rd, <2 x i16> %a, <2 x i16> %b) { ; ; RV64-LABEL: test_mqracc_h01_i32: ; RV64: # %bb.0: -; RV64-NEXT: zext.w a2, a2 -; RV64-NEXT: zext.w a1, a1 +; RV64-NEXT: pmv.ws a0, a0 ; RV64-NEXT: pmqracc.w.h01 a0, a1, a2 ; RV64-NEXT: ret %r = call i32 @llvm.riscv.mqracc.h01.i32.v2i16(i32 %rd, <2 x i16> %a, <2 x i16> %b) @@ -3245,8 +3240,7 @@ define i32 @test_mqracc_h11_i32(i32 %rd, <2 x i16> %a, <2 x i16> %b) { ; ; RV64-LABEL: test_mqracc_h11_i32: ; RV64: # %bb.0: -; RV64-NEXT: zext.w a2, a2 -; RV64-NEXT: zext.w a1, a1 +; RV64-NEXT: pmv.ws a0, a0 ; RV64-NEXT: pmqracc.w.h11 a0, a1, a2 ; RV64-NEXT: ret %r = call i32 @llvm.riscv.mqracc.h11.i32.v2i16(i32 %rd, <2 x i16> %a, <2 x i16> %b) _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
