Implement the vfwdota.vv instruction from the Zvfwdota16bf extension. This instruction computes a dot product between two BF16 vectors, accumulating into an FP32 scalar in vd[0].
Signed-off-by: Max Chou <[email protected]> --- target/riscv/helper.h | 2 + target/riscv/insn32.decode | 3 ++ target/riscv/tcg/insn_trans/trans_rvv.c.inc | 42 ++++++++++++++++++ target/riscv/tcg/vector_helper.c | 48 +++++++++++++++++++++ 4 files changed, 95 insertions(+) diff --git a/target/riscv/helper.h b/target/riscv/helper.h index 23d3e9b3c43..918ff21b3e2 100644 --- a/target/riscv/helper.h +++ b/target/riscv/helper.h @@ -1393,6 +1393,8 @@ DEF_HELPER_6(vqwdotas_vv_b, void, ptr, ptr, ptr, ptr, env, i32) DEF_HELPER_6(vqwdotau_vv_h, void, ptr, ptr, ptr, ptr, env, i32) DEF_HELPER_6(vqwdotas_vv_h, void, ptr, ptr, ptr, ptr, env, i32) +/* Zvfwdota16bf - BF16 Dot Product */ +DEF_HELPER_6(vfwdota_vv, void, ptr, ptr, ptr, ptr, env, i32) /* SiFive Custom int8 Matrix-Multiply */ DEF_HELPER_5(sf_vqmaccu_4x8x4, void, ptr, ptr, ptr, env, i32) DEF_HELPER_5(sf_vqmacc_4x8x4, void, ptr, ptr, ptr, env, i32) diff --git a/target/riscv/insn32.decode b/target/riscv/insn32.decode index b8a5d0567b1..d800ad99451 100644 --- a/target/riscv/insn32.decode +++ b/target/riscv/insn32.decode @@ -1101,3 +1101,6 @@ vwabdau_vx 111110 . ..... ..... 100 ..... 1010111 @r_vm # *** Zvqwdota8i/Zvqwdota16i Integer Dot-Product Extension *** vqwdotau_vv 100110 . ..... ..... 000 ..... 1110111 @r_vm vqwdotas_vv 100111 . ..... ..... 000 ..... 1110111 @r_vm + +# *** Zvfwdota16bf BF16 Dot-Product Extension *** +vfwdota_vv 100100 . ..... ..... 001 ..... 1110111 @r_vm diff --git a/target/riscv/tcg/insn_trans/trans_rvv.c.inc b/target/riscv/tcg/insn_trans/trans_rvv.c.inc index fcff472a2ea..c50f863ea0b 100644 --- a/target/riscv/tcg/insn_trans/trans_rvv.c.inc +++ b/target/riscv/tcg/insn_trans/trans_rvv.c.inc @@ -4200,3 +4200,45 @@ GEN_ABD_OPIVX_WIDEN_TRANS(vwabdau_vx, vwabd_vx_check) GEN_ZVDOTA_INT_TRANS(vqwdotau_vv, vext_vqwdota_check) GEN_ZVDOTA_INT_TRANS(vqwdotas_vv, vext_vqwdota_check) + +static bool vext_vfwdota_check(DisasContext *s, arg_rmrr * a) +{ + return s->mstatus_fs != EXT_STATUS_DISABLED && + vext_zvdota_base_check(s, a) && + (s->cfg_ptr->ext_zvfwdota16bf && s->sew == MO_16 && s->altfmt == 1); +} + +static bool trans_vfwdota_vv(DisasContext *s, arg_rmrr * a) +{ + if (vext_vfwdota_check(s, a)) { + TCGv_ptr dest, src1, src2, mask; + TCGv_i32 desc; + uint32_t data = 0; + + gen_set_rm(s, RISCV_FRM_DYN); + + data = FIELD_DP32(data, VDATA, VM, a->vm); + data = FIELD_DP32(data, VDATA, LMUL, s->lmul); + data = FIELD_DP32(data, VDATA, VTA, s->vta); + data = FIELD_DP32(data, VDATA, VTA_ALL_1S, s->cfg_vta_all_1s); + + dest = tcg_temp_new_ptr(); + mask = tcg_temp_new_ptr(); + src1 = tcg_temp_new_ptr(); + src2 = tcg_temp_new_ptr(); + desc = tcg_constant_i32(simd_desc(s->cfg_ptr->vlenb, s->cfg_ptr->vlenb, + data)); + + tcg_gen_addi_ptr(dest, tcg_env, vreg_ofs(s, a->rd)); + tcg_gen_addi_ptr(src1, tcg_env, vreg_ofs(s, a->rs1)); + tcg_gen_addi_ptr(src2, tcg_env, vreg_ofs(s, a->rs2)); + tcg_gen_addi_ptr(mask, tcg_env, vreg_ofs(s, 0)); + + gen_helper_vfwdota_vv(dest, mask, src1, src2, tcg_env, desc); + + finalize_rvv_inst(s); + + return true; + } + return false; +} diff --git a/target/riscv/tcg/vector_helper.c b/target/riscv/tcg/vector_helper.c index 41b526d25ef..5947427b6b4 100644 --- a/target/riscv/tcg/vector_helper.c +++ b/target/riscv/tcg/vector_helper.c @@ -6025,3 +6025,51 @@ GEN_VEXT_ZVDOTA_INT(vqwdotau_vv_b, 32, 8, uint8_t, H4, H1) GEN_VEXT_ZVDOTA_INT(vqwdotas_vv_b, 32, 8, int8_t, H4, H1) GEN_VEXT_ZVDOTA_INT(vqwdotau_vv_h, 64, 16, uint16_t, H8, H2) GEN_VEXT_ZVDOTA_INT(vqwdotas_vv_h, 64, 16, int16_t, H8, H2) + +/* + * Zvfwdota16bf: BF16 Dot Product with Accumulation + */ +void HELPER(vfwdota_vv)(void *vd, void *v0, void *vs1, void *vs2, + CPURISCVState *env, uint32_t desc) +{ + uint32_t vlmax = vext_max_elems(desc, 1); + uint32_t vm = vext_vm(desc); + uint32_t vl = env->vl; + uint32_t vta = vext_vta(desc); + uint32_t vta_all_1s = vext_vta_all_1s(desc); + uint32_t total_elems = riscv_cpu_cfg(env)->vlenb >> 2; + uint32_t i; + bfloat16 *a_vec, *b_vec; + uint32_t dot_result, acc; + + VSTART_CHECK_EARLY_EXIT(env, vl); + + a_vec = g_new0(bfloat16, vlmax); + b_vec = g_new0(bfloat16, vlmax); + + for (i = 0; i < vl; i++) { + if (!vm && !vext_elem_mask(v0, i)) { + continue; + } + a_vec[i] = *((uint16_t *)vs1 + H2(i)); + b_vec[i] = *((uint16_t *)vs2 + H2(i)); + } + + FloatExceptionFlags pre_fflag = get_float_exception_flags(&env->fp_status); + dot_result = rvbna_dot_bfloat16_bfloat16(a_vec, b_vec, vlmax, + &env->fp_status); + + acc = *((uint32_t *)vd + H4(0)); + *((uint32_t *)vd + H4(0)) = rvbna_final_f32_add(dot_result, acc, + &env->fp_status); + riscv_cpu_check_fflags(env, pre_fflag); + + g_free(a_vec); + g_free(b_vec); + + env->vstart = 0; + + if (vta_all_1s) { + vext_set_elems_1s(vd, vta, 1 * 4, total_elems * 4); + } +} -- 2.56.0
