Implement packed multiply-high accumulate, cross-element multiply-accumulate, and mixed-width multiply-high accumulate operations. Cover signed, unsigned, mixed-sign, rounded, and source-element selection variants.Add the associated decode entries, translators and helpers.
Note that same-width multiply-high accumulate operations add the upper half of each product to the corresponding destination element, while their rounding variants round the product before extracting it. Mixed-width variants discard the low bits corresponding to the narrower source element before accumulation. Cross-element variants select source subelements and accumulate their full products into wider destination elements. Accumulation wraps to the destination element width, and these operations do not affect vxsat. Signed-off-by: Molly Chen <[email protected]> --- target/riscv/helper.h | 56 ++++++ target/riscv/insn32.decode | 92 +++++++++ target/riscv/tcg/insn_trans/trans_rvp.c.inc | 56 ++++++ target/riscv/tcg/psimd_helper.c | 195 ++++++++++++++++++++ 4 files changed, 399 insertions(+) diff --git a/target/riscv/helper.h b/target/riscv/helper.h index c6e69163262..e90e01373ed 100644 --- a/target/riscv/helper.h +++ b/target/riscv/helper.h @@ -1621,3 +1621,59 @@ DEF_HELPER_3(mulsu_w11, i64, env, i64, i64) DEF_HELPER_3(mulu_w00, i64, env, i64, i64) DEF_HELPER_3(mulu_w01, i64, env, i64, i64) DEF_HELPER_3(mulu_w11, i64, env, i64, i64) + +/* Packed SIMD - Multiply-Accumulate Operations */ +DEF_HELPER_4(pmhacc_h, tl, env, tl, tl, tl) +DEF_HELPER_4(pmhaccsu_h, tl, env, tl, tl, tl) +DEF_HELPER_4(pmhaccu_h, tl, env, tl, tl, tl) +DEF_HELPER_4(pmhracc_h, tl, env, tl, tl, tl) +DEF_HELPER_4(pmhraccsu_h, tl, env, tl, tl, tl) +DEF_HELPER_4(pmhraccu_h, tl, env, tl, tl, tl) +DEF_HELPER_4(pmhacc_w, i64, env, i64, i64, i64) +DEF_HELPER_4(pmhracc_w, i64, env, i64, i64, i64) +DEF_HELPER_4(pmhaccsu_w, i64, env, i64, i64, i64) +DEF_HELPER_4(pmhraccsu_w, i64, env, i64, i64, i64) +DEF_HELPER_4(pmhaccu_w, i64, env, i64, i64, i64) +DEF_HELPER_4(pmhraccu_w, i64, env, i64, i64, i64) +DEF_HELPER_4(mhacc, i32, env, i32, i32, i32) +DEF_HELPER_4(mhracc, i32, env, i32, i32, i32) +DEF_HELPER_4(mhaccsu, i32, env, i32, i32, i32) +DEF_HELPER_4(mhraccsu, i32, env, i32, i32, i32) +DEF_HELPER_4(mhaccu, i32, env, i32, i32, i32) +DEF_HELPER_4(mhraccu, i32, env, i32, i32, i32) +DEF_HELPER_4(pmhacc_h_b0, tl, env, tl, tl, tl) +DEF_HELPER_4(pmhacc_h_b1, tl, env, tl, tl, tl) +DEF_HELPER_4(pmhaccsu_h_b0, tl, env, tl, tl, tl) +DEF_HELPER_4(pmhaccsu_h_b1, tl, env, tl, tl, tl) +DEF_HELPER_4(mhacc_h0, i32, env, i32, i32, i32) +DEF_HELPER_4(mhacc_h1, i32, env, i32, i32, i32) +DEF_HELPER_4(mhaccsu_h0, i32, env, i32, i32, i32) +DEF_HELPER_4(mhaccsu_h1, i32, env, i32, i32, i32) +DEF_HELPER_4(pmhacc_w_h0, i64, env, i64, i64, i64) +DEF_HELPER_4(pmhacc_w_h1, i64, env, i64, i64, i64) +DEF_HELPER_4(pmhaccsu_w_h0, i64, env, i64, i64, i64) +DEF_HELPER_4(pmhaccsu_w_h1, i64, env, i64, i64, i64) +DEF_HELPER_4(pmacc_w_h00, i64, env, i64, i64, i64) +DEF_HELPER_4(pmacc_w_h01, i64, env, i64, i64, i64) +DEF_HELPER_4(pmacc_w_h11, i64, env, i64, i64, i64) +DEF_HELPER_4(pmaccsu_w_h00, i64, env, i64, i64, i64) +DEF_HELPER_4(pmaccsu_w_h11, i64, env, i64, i64, i64) +DEF_HELPER_4(pmaccu_w_h00, i64, env, i64, i64, i64) +DEF_HELPER_4(pmaccu_w_h01, i64, env, i64, i64, i64) +DEF_HELPER_4(pmaccu_w_h11, i64, env, i64, i64, i64) +DEF_HELPER_4(macc_h00, i32, env, i32, i32, i32) +DEF_HELPER_4(macc_h01, i32, env, i32, i32, i32) +DEF_HELPER_4(macc_h11, i32, env, i32, i32, i32) +DEF_HELPER_4(maccsu_h00, i32, env, i32, i32, i32) +DEF_HELPER_4(maccsu_h11, i32, env, i32, i32, i32) +DEF_HELPER_4(maccu_h00, i32, env, i32, i32, i32) +DEF_HELPER_4(maccu_h01, i32, env, i32, i32, i32) +DEF_HELPER_4(maccu_h11, i32, env, i32, i32, i32) +DEF_HELPER_4(macc_w00, i64, env, i64, i64, i64) +DEF_HELPER_4(macc_w01, i64, env, i64, i64, i64) +DEF_HELPER_4(macc_w11, i64, env, i64, i64, i64) +DEF_HELPER_4(maccsu_w00, i64, env, i64, i64, i64) +DEF_HELPER_4(maccsu_w11, i64, env, i64, i64, i64) +DEF_HELPER_4(maccu_w00, i64, env, i64, i64, i64) +DEF_HELPER_4(maccu_w01, i64, env, i64, i64, i64) +DEF_HELPER_4(maccu_w11, i64, env, i64, i64, i64) diff --git a/target/riscv/insn32.decode b/target/riscv/insn32.decode index 94736a24038..2bf26ec1563 100644 --- a/target/riscv/insn32.decode +++ b/target/riscv/insn32.decode @@ -1431,3 +1431,95 @@ mulsu_w11 11110 11 ..... ..... 011 ..... 0111011 @r mulu_w00 10100 11 ..... ..... 011 ..... 0111011 @r mulu_w01 10110 11 ..... ..... 001 ..... 0111011 @r mulu_w11 10110 11 ..... ..... 011 ..... 0111011 @r + +# Packed SIMD - Multiply-Accumulate Operations +pmhacc_h 10001 00 ..... ..... 111 ..... 0111011 @r +pmhaccsu_h 11001 00 ..... ..... 111 ..... 0111011 @r +pmhaccu_h 10011 00 ..... ..... 111 ..... 0111011 @r +pmhracc_h 10001 10 ..... ..... 111 ..... 0111011 @r +pmhraccsu_h 11001 10 ..... ..... 111 ..... 0111011 @r +pmhraccu_h 10011 10 ..... ..... 111 ..... 0111011 @r +{ + mhacc 10001 01 ..... ..... 111 ..... 0111011 @r + pmhacc_w 10001 01 ..... ..... 111 ..... 0111011 @r +} +{ + mhracc 10001 11 ..... ..... 111 ..... 0111011 @r + pmhracc_w 10001 11 ..... ..... 111 ..... 0111011 @r +} +{ + mhaccsu 11001 01 ..... ..... 111 ..... 0111011 @r + pmhaccsu_w 11001 01 ..... ..... 111 ..... 0111011 @r +} +{ + mhraccsu 11001 11 ..... ..... 111 ..... 0111011 @r + pmhraccsu_w 11001 11 ..... ..... 111 ..... 0111011 @r +} +{ + mhaccu 10011 01 ..... ..... 111 ..... 0111011 @r + pmhaccu_w 10011 01 ..... ..... 111 ..... 0111011 @r +} +{ + mhraccu 10011 11 ..... ..... 111 ..... 0111011 @r + pmhraccu_w 10011 11 ..... ..... 111 ..... 0111011 @r +} +pmhacc_h_b0 10101 00 ..... ..... 111 ..... 0111011 @r +pmhacc_h_b1 10111 00 ..... ..... 111 ..... 0111011 @r +pmhaccsu_h_b0 10101 10 ..... ..... 111 ..... 0111011 @r +pmhaccsu_h_b1 10111 10 ..... ..... 111 ..... 0111011 @r +{ + mhacc_h0 10101 01 ..... ..... 111 ..... 0111011 @r + pmhacc_w_h0 10101 01 ..... ..... 111 ..... 0111011 @r +} +{ + mhacc_h1 10111 01 ..... ..... 111 ..... 0111011 @r + pmhacc_w_h1 10111 01 ..... ..... 111 ..... 0111011 @r +} +{ + mhaccsu_h0 10101 11 ..... ..... 111 ..... 0111011 @r + pmhaccsu_w_h0 10101 11 ..... ..... 111 ..... 0111011 @r +} +{ + mhaccsu_h1 10111 11 ..... ..... 111 ..... 0111011 @r + pmhaccsu_w_h1 10111 11 ..... ..... 111 ..... 0111011 @r +} +{ + macc_h00 10001 01 ..... ..... 011 ..... 0111011 @r + pmacc_w_h00 10001 01 ..... ..... 011 ..... 0111011 @r +} +{ + macc_h01 10011 01 ..... ..... 001 ..... 0111011 @r + pmacc_w_h01 10011 01 ..... ..... 001 ..... 0111011 @r +} +{ + macc_h11 10011 01 ..... ..... 011 ..... 0111011 @r + pmacc_w_h11 10011 01 ..... ..... 011 ..... 0111011 @r +} +{ + maccsu_h00 11101 01 ..... ..... 011 ..... 0111011 @r + pmaccsu_w_h00 11101 01 ..... ..... 011 ..... 0111011 @r +} +{ + maccsu_h11 11111 01 ..... ..... 011 ..... 0111011 @r + pmaccsu_w_h11 11111 01 ..... ..... 011 ..... 0111011 @r +} +{ + maccu_h00 10101 01 ..... ..... 011 ..... 0111011 @r + pmaccu_w_h00 10101 01 ..... ..... 011 ..... 0111011 @r +} +{ + maccu_h01 10111 01 ..... ..... 001 ..... 0111011 @r + pmaccu_w_h01 10111 01 ..... ..... 001 ..... 0111011 @r +} +{ + maccu_h11 10111 01 ..... ..... 011 ..... 0111011 @r + pmaccu_w_h11 10111 01 ..... ..... 011 ..... 0111011 @r +} +macc_w00 10001 11 ..... ..... 011 ..... 0111011 @r +macc_w01 10011 11 ..... ..... 001 ..... 0111011 @r +macc_w11 10011 11 ..... ..... 011 ..... 0111011 @r +maccsu_w00 11101 11 ..... ..... 011 ..... 0111011 @r +maccsu_w11 11111 11 ..... ..... 011 ..... 0111011 @r +maccu_w00 10101 11 ..... ..... 011 ..... 0111011 @r +maccu_w01 10111 11 ..... ..... 001 ..... 0111011 @r +maccu_w11 10111 11 ..... ..... 011 ..... 0111011 @r diff --git a/target/riscv/tcg/insn_trans/trans_rvp.c.inc b/target/riscv/tcg/insn_trans/trans_rvp.c.inc index 39bec55bd25..5aaa3c6ffa5 100644 --- a/target/riscv/tcg/insn_trans/trans_rvp.c.inc +++ b/target/riscv/tcg/insn_trans/trans_rvp.c.inc @@ -790,3 +790,59 @@ GEN_SIMD_TRANS_64(mulsu_w11) GEN_SIMD_TRANS_64(mulu_w00) GEN_SIMD_TRANS_64(mulu_w01) GEN_SIMD_TRANS_64(mulu_w11) + +/* Packed SIMD - Multiply-Accumulate Operations */ +GEN_SIMD_TRANS_ACC(pmhacc_h) +GEN_SIMD_TRANS_ACC(pmhaccsu_h) +GEN_SIMD_TRANS_ACC(pmhaccu_h) +GEN_SIMD_TRANS_ACC(pmhracc_h) +GEN_SIMD_TRANS_ACC(pmhraccsu_h) +GEN_SIMD_TRANS_ACC(pmhraccu_h) +GEN_SIMD_TRANS_ACC_64(pmhacc_w) +GEN_SIMD_TRANS_ACC_64(pmhracc_w) +GEN_SIMD_TRANS_ACC_64(pmhaccsu_w) +GEN_SIMD_TRANS_ACC_64(pmhraccsu_w) +GEN_SIMD_TRANS_ACC_64(pmhaccu_w) +GEN_SIMD_TRANS_ACC_64(pmhraccu_w) +GEN_SIMD_TRANS_ACC_32(mhacc) +GEN_SIMD_TRANS_ACC_32(mhracc) +GEN_SIMD_TRANS_ACC_32(mhaccsu) +GEN_SIMD_TRANS_ACC_32(mhraccsu) +GEN_SIMD_TRANS_ACC_32(mhaccu) +GEN_SIMD_TRANS_ACC_32(mhraccu) +GEN_SIMD_TRANS_ACC(pmhacc_h_b0) +GEN_SIMD_TRANS_ACC(pmhacc_h_b1) +GEN_SIMD_TRANS_ACC(pmhaccsu_h_b0) +GEN_SIMD_TRANS_ACC(pmhaccsu_h_b1) +GEN_SIMD_TRANS_ACC_32(mhacc_h0) +GEN_SIMD_TRANS_ACC_32(mhacc_h1) +GEN_SIMD_TRANS_ACC_32(mhaccsu_h0) +GEN_SIMD_TRANS_ACC_32(mhaccsu_h1) +GEN_SIMD_TRANS_ACC_64(pmhacc_w_h0) +GEN_SIMD_TRANS_ACC_64(pmhacc_w_h1) +GEN_SIMD_TRANS_ACC_64(pmhaccsu_w_h0) +GEN_SIMD_TRANS_ACC_64(pmhaccsu_w_h1) +GEN_SIMD_TRANS_ACC_64(pmacc_w_h00) +GEN_SIMD_TRANS_ACC_64(pmacc_w_h01) +GEN_SIMD_TRANS_ACC_64(pmacc_w_h11) +GEN_SIMD_TRANS_ACC_64(pmaccsu_w_h00) +GEN_SIMD_TRANS_ACC_64(pmaccsu_w_h11) +GEN_SIMD_TRANS_ACC_64(pmaccu_w_h00) +GEN_SIMD_TRANS_ACC_64(pmaccu_w_h01) +GEN_SIMD_TRANS_ACC_64(pmaccu_w_h11) +GEN_SIMD_TRANS_ACC_32(macc_h00) +GEN_SIMD_TRANS_ACC_32(macc_h01) +GEN_SIMD_TRANS_ACC_32(macc_h11) +GEN_SIMD_TRANS_ACC_32(maccsu_h00) +GEN_SIMD_TRANS_ACC_32(maccsu_h11) +GEN_SIMD_TRANS_ACC_32(maccu_h00) +GEN_SIMD_TRANS_ACC_32(maccu_h01) +GEN_SIMD_TRANS_ACC_32(maccu_h11) +GEN_SIMD_TRANS_ACC_64(macc_w00) +GEN_SIMD_TRANS_ACC_64(macc_w01) +GEN_SIMD_TRANS_ACC_64(macc_w11) +GEN_SIMD_TRANS_ACC_64(maccsu_w00) +GEN_SIMD_TRANS_ACC_64(maccsu_w11) +GEN_SIMD_TRANS_ACC_64(maccu_w00) +GEN_SIMD_TRANS_ACC_64(maccu_w01) +GEN_SIMD_TRANS_ACC_64(maccu_w11) diff --git a/target/riscv/tcg/psimd_helper.c b/target/riscv/tcg/psimd_helper.c index f84b43bfaa7..b9bb7d497fc 100644 --- a/target/riscv/tcg/psimd_helper.c +++ b/target/riscv/tcg/psimd_helper.c @@ -2810,3 +2810,198 @@ GEN_PSIMD_MUL_ELEM_SCALAR(mulu_w01, uint64_t, uint32_t, uint32_t, uint64_t, EXTRACT32, 0, 1, PSIMD_MUL_U64) GEN_PSIMD_MUL_ELEM_SCALAR(mulu_w11, uint64_t, uint32_t, uint32_t, uint64_t, EXTRACT32, 1, 1, PSIMD_MUL_U64) + +/* Multiply-Accumulate Operations */ + +GEN_PSIMD_MUL_HIGH_ACC(pmhacc_h, target_ulong, int16_t, int16_t, int16_t, + int32_t, int16_t, uint16_t, EXTRACT16, INSERT16, + ELEMS_H, 16, 0, PSIMD_MUL_S32) +GEN_PSIMD_MUL_HIGH_ACC(pmhaccsu_h, target_ulong, int16_t, uint16_t, int16_t, + int32_t, int16_t, uint16_t, EXTRACT16, INSERT16, + ELEMS_H, 16, 0, PSIMD_MUL_SU32) +GEN_PSIMD_MUL_HIGH_ACC(pmhaccu_h, target_ulong, uint16_t, uint16_t, uint16_t, + uint32_t, uint16_t, uint16_t, EXTRACT16, INSERT16, + ELEMS_H, 16, 0, PSIMD_MUL_U32) +GEN_PSIMD_MUL_HIGH_ACC(pmhracc_h, target_ulong, int16_t, int16_t, int16_t, + int32_t, int16_t, uint16_t, EXTRACT16, INSERT16, + ELEMS_H, 16, 1 << 15, PSIMD_MUL_S32) +GEN_PSIMD_MUL_HIGH_ACC(pmhraccsu_h, target_ulong, int16_t, uint16_t, int16_t, + int32_t, int16_t, uint16_t, EXTRACT16, INSERT16, + ELEMS_H, 16, 1 << 15, PSIMD_MUL_SU32) +GEN_PSIMD_MUL_HIGH_ACC(pmhraccu_h, target_ulong, uint16_t, uint16_t, + uint16_t, uint32_t, uint16_t, uint16_t, + EXTRACT16, INSERT16, ELEMS_H, 16, 1 << 15, + PSIMD_MUL_U32) + +GEN_PSIMD_MUL_HIGH_ACC(pmhacc_w, uint64_t, int32_t, int32_t, int32_t, + int64_t, int32_t, uint32_t, EXTRACT32, INSERT32, + ELEMS_W, 32, 0, PSIMD_MUL_S64) +GEN_PSIMD_MUL_HIGH_ACC(pmhracc_w, uint64_t, int32_t, int32_t, int32_t, + int64_t, int32_t, uint32_t, EXTRACT32, INSERT32, + ELEMS_W, 32, 1LL << 31, PSIMD_MUL_S64) +GEN_PSIMD_MUL_HIGH_ACC(pmhaccsu_w, uint64_t, int32_t, uint32_t, int32_t, + int64_t, int32_t, uint32_t, EXTRACT32, INSERT32, + ELEMS_W, 32, 0, PSIMD_MUL_SU64) +GEN_PSIMD_MUL_HIGH_ACC(pmhraccsu_w, uint64_t, int32_t, uint32_t, int32_t, + int64_t, int32_t, uint32_t, EXTRACT32, INSERT32, + ELEMS_W, 32, 1LL << 31, PSIMD_MUL_SU64) +GEN_PSIMD_MUL_HIGH_ACC(pmhaccu_w, uint64_t, uint32_t, uint32_t, uint32_t, + uint64_t, uint32_t, uint32_t, EXTRACT32, INSERT32, + ELEMS_W, 32, 0, PSIMD_MUL_U64) +GEN_PSIMD_MUL_HIGH_ACC(pmhraccu_w, uint64_t, uint32_t, uint32_t, uint32_t, + uint64_t, uint32_t, uint32_t, EXTRACT32, INSERT32, + ELEMS_W, 32, 1LL << 31, PSIMD_MUL_U64) + +GEN_PSIMD_MUL_HIGH_ACC(mhacc, uint32_t, int32_t, int32_t, int32_t, + int64_t, int32_t, uint32_t, EXTRACT32, INSERT32, + ELEMS_W, 32, 0, PSIMD_MUL_S64) +GEN_PSIMD_MUL_HIGH_ACC(mhracc, uint32_t, int32_t, int32_t, int32_t, + int64_t, int32_t, uint32_t, EXTRACT32, INSERT32, + ELEMS_W, 32, 1LL << 31, PSIMD_MUL_S64) +GEN_PSIMD_MUL_HIGH_ACC(mhaccsu, uint32_t, int32_t, uint32_t, int32_t, + int64_t, int32_t, uint32_t, EXTRACT32, INSERT32, + ELEMS_W, 32, 0, PSIMD_MUL_SU64) +GEN_PSIMD_MUL_HIGH_ACC(mhraccsu, uint32_t, int32_t, uint32_t, int32_t, + int64_t, int32_t, uint32_t, EXTRACT32, INSERT32, + ELEMS_W, 32, 1LL << 31, PSIMD_MUL_SU64) +GEN_PSIMD_MUL_HIGH_ACC(mhaccu, uint32_t, uint32_t, uint32_t, uint32_t, + uint64_t, uint32_t, uint32_t, EXTRACT32, INSERT32, + ELEMS_W, 32, 0, PSIMD_MUL_U64) +GEN_PSIMD_MUL_HIGH_ACC(mhraccu, uint32_t, uint32_t, uint32_t, uint32_t, + uint64_t, uint32_t, uint32_t, EXTRACT32, INSERT32, + ELEMS_W, 32, 1LL << 31, PSIMD_MUL_U64) + +GEN_PSIMD_MUL_HIGH_ACC_SEL(pmhacc_h_b0, target_ulong, int16_t, int8_t, + int16_t, int32_t, int16_t, uint16_t, EXTRACT16, + EXTRACT8, INSERT16, ELEMS_H, 2, 0, 8, + PSIMD_MUL_S32) +GEN_PSIMD_MUL_HIGH_ACC_SEL(pmhacc_h_b1, target_ulong, int16_t, int8_t, + int16_t, int32_t, int16_t, uint16_t, EXTRACT16, + EXTRACT8, INSERT16, ELEMS_H, 2, 1, 8, + PSIMD_MUL_S32) +GEN_PSIMD_MUL_HIGH_ACC_SEL(pmhaccsu_h_b0, target_ulong, int16_t, uint8_t, + int16_t, int32_t, int16_t, uint16_t, EXTRACT16, + EXTRACT8, INSERT16, ELEMS_H, 2, 0, 8, + PSIMD_MUL_SU32) +GEN_PSIMD_MUL_HIGH_ACC_SEL(pmhaccsu_h_b1, target_ulong, int16_t, uint8_t, + int16_t, int32_t, int16_t, uint16_t, EXTRACT16, + EXTRACT8, INSERT16, ELEMS_H, 2, 1, 8, + PSIMD_MUL_SU32) + +GEN_PSIMD_MUL_HIGH_ACC_SEL(mhacc_h0, uint32_t, int32_t, int16_t, int32_t, + int64_t, int32_t, uint32_t, EXTRACT32, + EXTRACT16, INSERT32, ELEMS_W, 0, 0, 16, + PSIMD_MUL_S64) +GEN_PSIMD_MUL_HIGH_ACC_SEL(mhacc_h1, uint32_t, int32_t, int16_t, int32_t, + int64_t, int32_t, uint32_t, EXTRACT32, + EXTRACT16, INSERT32, ELEMS_W, 0, 1, 16, + PSIMD_MUL_S64) +GEN_PSIMD_MUL_HIGH_ACC_SEL(mhaccsu_h0, uint32_t, int32_t, uint16_t, int32_t, + int64_t, int32_t, uint32_t, EXTRACT32, + EXTRACT16, INSERT32, ELEMS_W, 0, 0, 16, + PSIMD_MUL_SU64) +GEN_PSIMD_MUL_HIGH_ACC_SEL(mhaccsu_h1, uint32_t, int32_t, uint16_t, int32_t, + int64_t, int32_t, uint32_t, EXTRACT32, + EXTRACT16, INSERT32, ELEMS_W, 0, 1, 16, + PSIMD_MUL_SU64) + +GEN_PSIMD_MUL_HIGH_ACC_SEL(pmhacc_w_h0, uint64_t, int32_t, int16_t, + int32_t, int64_t, int32_t, uint32_t, EXTRACT32, + EXTRACT16, INSERT32, ELEMS_W, 2, 0, 16, + PSIMD_MUL_S64) +GEN_PSIMD_MUL_HIGH_ACC_SEL(pmhacc_w_h1, uint64_t, int32_t, int16_t, + int32_t, int64_t, int32_t, uint32_t, EXTRACT32, + EXTRACT16, INSERT32, ELEMS_W, 2, 1, 16, + PSIMD_MUL_S64) +GEN_PSIMD_MUL_HIGH_ACC_SEL(pmhaccsu_w_h0, uint64_t, int32_t, uint16_t, + int32_t, int64_t, int32_t, uint32_t, EXTRACT32, + EXTRACT16, INSERT32, ELEMS_W, 2, 0, 16, + PSIMD_MUL_SU64) +GEN_PSIMD_MUL_HIGH_ACC_SEL(pmhaccsu_w_h1, uint64_t, int32_t, uint16_t, + int32_t, int64_t, int32_t, uint32_t, EXTRACT32, + EXTRACT16, INSERT32, ELEMS_W, 2, 1, 16, + PSIMD_MUL_SU64) + +GEN_PSIMD_MUL_ACC_INDEXED(pmacc_w_h00, uint64_t, int16_t, int16_t, + int32_t, int32_t, uint32_t, EXTRACT16, EXTRACT32, + INSERT32, ELEMS_W, 2, 0, 0, PSIMD_MUL_S32) +GEN_PSIMD_MUL_ACC_INDEXED(pmacc_w_h01, uint64_t, int16_t, int16_t, + int32_t, int32_t, uint32_t, EXTRACT16, EXTRACT32, + INSERT32, ELEMS_W, 2, 0, 1, PSIMD_MUL_S32) +GEN_PSIMD_MUL_ACC_INDEXED(pmacc_w_h11, uint64_t, int16_t, int16_t, + int32_t, int32_t, uint32_t, EXTRACT16, EXTRACT32, + INSERT32, ELEMS_W, 2, 1, 1, PSIMD_MUL_S32) +GEN_PSIMD_MUL_ACC_INDEXED(pmaccsu_w_h00, uint64_t, int16_t, uint16_t, + int32_t, int32_t, uint32_t, EXTRACT16, EXTRACT32, + INSERT32, ELEMS_W, 2, 0, 0, PSIMD_MUL_SU32) +GEN_PSIMD_MUL_ACC_INDEXED(pmaccsu_w_h11, uint64_t, int16_t, uint16_t, + int32_t, int32_t, uint32_t, EXTRACT16, EXTRACT32, + INSERT32, ELEMS_W, 2, 1, 1, PSIMD_MUL_SU32) +GEN_PSIMD_MUL_ACC_INDEXED(pmaccu_w_h00, uint64_t, uint16_t, uint16_t, + uint32_t, uint32_t, uint32_t, EXTRACT16, + EXTRACT32, INSERT32, ELEMS_W, 2, 0, 0, + PSIMD_MUL_U32) +GEN_PSIMD_MUL_ACC_INDEXED(pmaccu_w_h01, uint64_t, uint16_t, uint16_t, + uint32_t, uint32_t, uint32_t, EXTRACT16, + EXTRACT32, INSERT32, ELEMS_W, 2, 0, 1, + PSIMD_MUL_U32) +GEN_PSIMD_MUL_ACC_INDEXED(pmaccu_w_h11, uint64_t, uint16_t, uint16_t, + uint32_t, uint32_t, uint32_t, EXTRACT16, + EXTRACT32, INSERT32, ELEMS_W, 2, 1, 1, + PSIMD_MUL_U32) + +GEN_PSIMD_MUL_ACC_INDEXED(macc_h00, uint32_t, int16_t, int16_t, + int32_t, int32_t, uint32_t, EXTRACT16, EXTRACT32, + INSERT32, ELEMS_W, 0, 0, 0, PSIMD_MUL_S32) +GEN_PSIMD_MUL_ACC_INDEXED(macc_h01, uint32_t, int16_t, int16_t, + int32_t, int32_t, uint32_t, EXTRACT16, EXTRACT32, + INSERT32, ELEMS_W, 0, 0, 1, PSIMD_MUL_S32) +GEN_PSIMD_MUL_ACC_INDEXED(macc_h11, uint32_t, int16_t, int16_t, + int32_t, int32_t, uint32_t, EXTRACT16, EXTRACT32, + INSERT32, ELEMS_W, 0, 1, 1, PSIMD_MUL_S32) +GEN_PSIMD_MUL_ACC_INDEXED(maccsu_h00, uint32_t, int16_t, uint16_t, + int32_t, int32_t, uint32_t, EXTRACT16, EXTRACT32, + INSERT32, ELEMS_W, 0, 0, 0, PSIMD_MUL_SU32) +GEN_PSIMD_MUL_ACC_INDEXED(maccsu_h11, uint32_t, int16_t, uint16_t, + int32_t, int32_t, uint32_t, EXTRACT16, EXTRACT32, + INSERT32, ELEMS_W, 0, 1, 1, PSIMD_MUL_SU32) +GEN_PSIMD_MUL_ACC_INDEXED(maccu_h00, uint32_t, uint16_t, uint16_t, + uint32_t, uint32_t, uint32_t, EXTRACT16, + EXTRACT32, INSERT32, ELEMS_W, 0, 0, 0, + PSIMD_MUL_U32) +GEN_PSIMD_MUL_ACC_INDEXED(maccu_h01, uint32_t, uint16_t, uint16_t, + uint32_t, uint32_t, uint32_t, EXTRACT16, + EXTRACT32, INSERT32, ELEMS_W, 0, 0, 1, + PSIMD_MUL_U32) +GEN_PSIMD_MUL_ACC_INDEXED(maccu_h11, uint32_t, uint16_t, uint16_t, + uint32_t, uint32_t, uint32_t, EXTRACT16, + EXTRACT32, INSERT32, ELEMS_W, 0, 1, 1, + PSIMD_MUL_U32) + +GEN_PSIMD_MUL_ACC_INDEXED(macc_w00, uint64_t, int32_t, int32_t, + int64_t, int64_t, uint64_t, EXTRACT32, EXTRACT64, + INSERT64, ELEMS_D, 0, 0, 0, PSIMD_MUL_S64) +GEN_PSIMD_MUL_ACC_INDEXED(macc_w01, uint64_t, int32_t, int32_t, + int64_t, int64_t, uint64_t, EXTRACT32, EXTRACT64, + INSERT64, ELEMS_D, 0, 0, 1, PSIMD_MUL_S64) +GEN_PSIMD_MUL_ACC_INDEXED(macc_w11, uint64_t, int32_t, int32_t, + int64_t, int64_t, uint64_t, EXTRACT32, EXTRACT64, + INSERT64, ELEMS_D, 0, 1, 1, PSIMD_MUL_S64) +GEN_PSIMD_MUL_ACC_INDEXED(maccsu_w00, uint64_t, int32_t, uint32_t, + int64_t, int64_t, uint64_t, EXTRACT32, EXTRACT64, + INSERT64, ELEMS_D, 0, 0, 0, PSIMD_MUL_SU64) +GEN_PSIMD_MUL_ACC_INDEXED(maccsu_w11, uint64_t, int32_t, uint32_t, + int64_t, int64_t, uint64_t, EXTRACT32, EXTRACT64, + INSERT64, ELEMS_D, 0, 1, 1, PSIMD_MUL_SU64) +GEN_PSIMD_MUL_ACC_INDEXED(maccu_w00, uint64_t, uint32_t, uint32_t, + uint64_t, uint64_t, uint64_t, EXTRACT32, + EXTRACT64, INSERT64, ELEMS_D, 0, 0, 0, + PSIMD_MUL_U64) +GEN_PSIMD_MUL_ACC_INDEXED(maccu_w01, uint64_t, uint32_t, uint32_t, + uint64_t, uint64_t, uint64_t, EXTRACT32, + EXTRACT64, INSERT64, ELEMS_D, 0, 0, 1, + PSIMD_MUL_U64) +GEN_PSIMD_MUL_ACC_INDEXED(maccu_w11, uint64_t, uint32_t, uint32_t, + uint64_t, uint64_t, uint64_t, EXTRACT32, + EXTRACT64, INSERT64, ELEMS_D, 0, 1, 1, + PSIMD_MUL_U64) -- 2.34.1
