Add the E4M3FNUZ FP8 add, sub, mpy, convert, neg, abs, min and max instructions, and the USR.FPSAT bit that controls saturation.
Reviewed-by: Marco Liebel <[email protected]> Signed-off-by: Brian Cain <[email protected]> --- target/hexagon/mmvec/hvx_ieee_fp.h | 27 ++ target/hexagon/mmvec/mmvec.h | 1 + target/hexagon/attribs_def.h.inc | 1 + target/hexagon/reg_fields_def.h.inc | 1 + target/hexagon/genptr.c | 7 +- target/hexagon/mmvec/hvx_ieee_fp.c | 125 ++++++- tests/tcg/hexagon/check_rev_gating.c | 18 + tests/tcg/hexagon/fp_hvx_v79.c | 361 ++++++++++++++++++- target/hexagon/tag_rev_info.c.inc | 12 + target/hexagon/imported/mmvec/encode_ext.def | 12 + target/hexagon/imported/mmvec/ext.idef | 78 ++++ 11 files changed, 623 insertions(+), 20 deletions(-) diff --git a/target/hexagon/mmvec/hvx_ieee_fp.h b/target/hexagon/mmvec/hvx_ieee_fp.h index 6efc39eb203..229ee0ee516 100644 --- a/target/hexagon/mmvec/hvx_ieee_fp.h +++ b/target/hexagon/mmvec/hvx_ieee_fp.h @@ -35,6 +35,33 @@ uint16_t cmpgt_hf(float16 a1, float16 a2, float_status *fp_status); uint32_t cmpeq_sf(float32 a1, float32 a2, float_status *fp_status); uint16_t cmpeq_hf(float16 a1, float16 a2, float_status *fp_status); +/* + * FP8 instructions. The FP8 format is E4M3FNUZ: one sign bit, four exponent + * bits (bias 8) and three fraction bits, with no infinities and no negative + * zero. The encoding that would be negative zero, 0x80, is the only NaN. + * softfloat has no such format, so values are widened to float16 (which holds + * every FP8 value exactly) to operate on them. + */ +#define F8_NAN 0x80 +#define F8_MAX_NORM 0x7f + +float16 f8_to_f16(uint8_t a, float_status *fp_status); +/* + * Narrow with round-to-nearest-even. Infinity and results too large for FP8 + * become the largest normal number if sat is set and NaN otherwise. + */ +uint8_t f16_to_f8(float16 a, bool sat, float_status *fp_status); + +float16 fp_add_hf_f8(uint8_t a1, uint8_t a2, float_status *fp_status); +float16 fp_sub_hf_f8(uint8_t a1, uint8_t a2, float_status *fp_status); +float16 fp_mult_hf_f8(uint8_t a1, uint8_t a2, float_status *fp_status); +float16 fp_mult_hf_f8_acc(uint8_t a1, uint8_t a2, float16 acc, + float_status *fp_status); +uint8_t fp_neg_f8(uint8_t a); +uint8_t fp_abs_f8(uint8_t a); +uint8_t fp_min_f8(uint8_t a1, uint8_t a2, float_status *fp_status); +uint8_t fp_max_f8(uint8_t a1, uint8_t a2, float_status *fp_status); + /* IEEE BFloat instructions */ #define fp_mult_sf_bf(A, B) \ diff --git a/target/hexagon/mmvec/mmvec.h b/target/hexagon/mmvec/mmvec.h index de36e12e30a..7b8cce3a37b 100644 --- a/target/hexagon/mmvec/mmvec.h +++ b/target/hexagon/mmvec/mmvec.h @@ -55,6 +55,7 @@ typedef struct { int8_t b[MAX_VEC_SIZE_BYTES / 1]; float32 sf[MAX_VEC_SIZE_BYTES / 4]; float16 hf[MAX_VEC_SIZE_BYTES / 2]; + uint8_t f8[MAX_VEC_SIZE_BYTES / 1]; bfloat16 bf[MAX_VEC_SIZE_BYTES / 2]; /* qfloat "visible" mantissa/exponent, sans the extended bits below */ int32_t qf32[MAX_VEC_SIZE_BYTES / 4]; diff --git a/target/hexagon/attribs_def.h.inc b/target/hexagon/attribs_def.h.inc index 96ad1fd2f8e..ac086cb099b 100644 --- a/target/hexagon/attribs_def.h.inc +++ b/target/hexagon/attribs_def.h.inc @@ -205,6 +205,7 @@ DEF_ATTRIB(RESTRICT_TARGET, "Restricted branch target instruction", "", "") /* HVX IEEE FP extension attributes */ DEF_ATTRIB(HVX_IEEE_FP, "HVX IEEE FP extension instruction", "", "") DEF_ATTRIB(HVX_IEEE_FP_ACC, "HVX IEEE FP accumulate instruction", "", "") +DEF_ATTRIB(HVX_IEEE_FP_OUT_8, "HVX IEEE FP 8-bit output", "", "") DEF_ATTRIB(HVX_IEEE_FP_OUT_16, "HVX IEEE FP 16-bit output", "", "") DEF_ATTRIB(HVX_IEEE_FP_OUT_32, "HVX IEEE FP 32-bit output", "", "") DEF_ATTRIB(CVI_VX_NO_TMP_LD, "HVX multiply without tmp load", "", "") diff --git a/target/hexagon/reg_fields_def.h.inc b/target/hexagon/reg_fields_def.h.inc index 21babbbb982..2ae5d7ffe8e 100644 --- a/target/hexagon/reg_fields_def.h.inc +++ b/target/hexagon/reg_fields_def.h.inc @@ -35,6 +35,7 @@ DEF_REG_FIELD(USR_LPCFG, 8, 2) DEF_REG_FIELD(USR_FPDAZ, 17, 1) DEF_REG_FIELD(USR_FPRND, 22, 2) +DEF_REG_FIELD(USR_FPSAT, 24, 1) DEF_REG_FIELD(USR_FPINVE, 25, 1) DEF_REG_FIELD(USR_FPDBZE, 26, 1) diff --git a/target/hexagon/genptr.c b/target/hexagon/genptr.c index 7ef36e336fa..255db6d35c9 100644 --- a/target/hexagon/genptr.c +++ b/target/hexagon/genptr.c @@ -552,7 +552,12 @@ static inline void gen_write_ctrl_reg(DisasContext *ctx, int reg_num, if (reg_num == HEX_REG_P3_0_ALIASED) { gen_write_p3_0(ctx, val); } else { - const target_ulong reg_mask = reg_immut_masks[reg_num]; + target_ulong reg_mask = reg_immut_masks[reg_num]; + if (reg_num == HEX_REG_USR && + ctx->hex_def->hex_version >= HEX_VER_V79) { + /* USR.FPSAT is reserved before v79 */ + reg_mask &= ~fGET_REG_FIELD_MASK(USR_FPSAT); + } gen_masked_reg_write(val, hex_gpr[reg_num], reg_mask); tcg_gen_mov_tl(get_result_gpr(ctx, reg_num), val); if (reg_num == HEX_REG_QEMU_PKT_CNT) { diff --git a/target/hexagon/mmvec/hvx_ieee_fp.c b/target/hexagon/mmvec/hvx_ieee_fp.c index e0971147a3d..6394c175a13 100644 --- a/target/hexagon/mmvec/hvx_ieee_fp.c +++ b/target/hexagon/mmvec/hvx_ieee_fp.c @@ -6,6 +6,8 @@ #include "qemu/osdep.h" #include "hvx_ieee_fp.h" +#include "fpu/softfloat-parts.h" +#include "fpu/softfloat-macros.h" float32 fp_mult_sf_hf(float16 a1, float16 a2, float_status *fp_status) { @@ -145,11 +147,7 @@ uint16_t cmpgt_hf(float16 a1, float16 a2, float_status *fp_status) return float16_compare(a1, a2, fp_status) == float_relation_greater; } -/* - * Unlike cmpgt_sf/cmpgt_hf, equality has no NaN-ordering convention to - * apply: per IEEE-754, a compare-equal predicate is quiet and always - * false when either operand is NaN. - */ +/* IEEE equality is quiet and false when either operand is NaN. */ uint32_t cmpeq_sf(float32 a1, float32 a2, float_status *fp_status) { return float32_eq_quiet(a1, a2, fp_status); @@ -159,3 +157,120 @@ uint16_t cmpeq_hf(float16 a1, float16 a2, float_status *fp_status) { return float16_eq_quiet(a1, a2, fp_status); } + +float16 f8_to_f16(uint8_t a, float_status *fp_status) +{ + int exp = extract32(a, 3, 4); + int frac = extract32(a, 0, 3); + float16 ret; + + if (a == F8_NAN) { + return float16_default_nan(fp_status); + } + if (a == 0) { + return float16_zero; + } + + /* FP8 subnormals share the exponent of the smallest normal. */ + ret = uint64_to_float16_scalbn(exp ? frac | 8 : frac, (exp ? exp : 1) - 11, + fp_status); + return a & 0x80 ? float16_chs(ret) : ret; +} + +uint8_t f16_to_f8(float16 a, bool sat, float_status *fp_status) +{ + FloatParts64 p = float16_unpack_canonical(a, fp_status); + uint8_t sign = p.sign ? 0x80 : 0; + uint64_t frac; + int shift, code; + + switch (p.cls) { + case float_class_qnan: + case float_class_snan: + return F8_NAN; + case float_class_inf: + return sat ? sign | F8_MAX_NORM : F8_NAN; + case float_class_zero: + return 0; + default: + break; + } + + /* Retain four significant bits and three rounding bits. */ + shift = 60 + (p.exp < -7 ? -7 - p.exp : 0); + shift64RightJamming(p.frac, shift - 3, &frac); + code = frac >> 3; + if ((frac & 7) > 4 || ((frac & 7) == 4 && (code & 1))) { + code++; + } + + if (p.exp >= -7) { + /* Rounding up to 2^(exp+1) carries into the exponent field */ + code += (p.exp + 8) * 8 - 8; + } + if (p.exp > 7 || code > F8_MAX_NORM) { + return sat ? sign | F8_MAX_NORM : F8_NAN; + } + /* There's no negative zero */ + return code ? sign | code : 0; +} + +float16 fp_add_hf_f8(uint8_t a1, uint8_t a2, float_status *fp_status) +{ + return float16_add(f8_to_f16(a1, fp_status), f8_to_f16(a2, fp_status), + fp_status); +} + +float16 fp_sub_hf_f8(uint8_t a1, uint8_t a2, float_status *fp_status) +{ + return float16_sub(f8_to_f16(a1, fp_status), f8_to_f16(a2, fp_status), + fp_status); +} + +float16 fp_mult_hf_f8(uint8_t a1, uint8_t a2, float_status *fp_status) +{ + return float16_mul(f8_to_f16(a1, fp_status), f8_to_f16(a2, fp_status), + fp_status); +} + +float16 fp_mult_hf_f8_acc(uint8_t a1, uint8_t a2, float16 acc, + float_status *fp_status) +{ + return float16_muladd(f8_to_f16(a1, fp_status), f8_to_f16(a2, fp_status), + acc, 0, fp_status); +} + +uint8_t fp_neg_f8(uint8_t a) +{ + /* Zero has no negative counterpart, and NaN stays NaN */ + return a == 0 || a == F8_NAN ? a : a ^ 0x80; +} + +uint8_t fp_abs_f8(uint8_t a) +{ + return a == F8_NAN ? a : a & 0x7f; +} + +/* + * These return NaN if either input is, unlike the IEEE min/max which return + * the number. + */ +uint8_t fp_min_f8(uint8_t a1, uint8_t a2, float_status *fp_status) +{ + if (a1 == F8_NAN || a2 == F8_NAN) { + return F8_NAN; + } + return f16_to_f8(float16_min(f8_to_f16(a1, fp_status), + f8_to_f16(a2, fp_status), fp_status), + true, fp_status); +} + +uint8_t fp_max_f8(uint8_t a1, uint8_t a2, float_status *fp_status) +{ + if (a1 == F8_NAN || a2 == F8_NAN) { + return F8_NAN; + } + return f16_to_f8(float16_max(f8_to_f16(a1, fp_status), + f8_to_f16(a2, fp_status), fp_status), + true, fp_status); +} diff --git a/tests/tcg/hexagon/check_rev_gating.c b/tests/tcg/hexagon/check_rev_gating.c index 529cae70d85..df6cb426c62 100644 --- a/tests/tcg/hexagon/check_rev_gating.c +++ b/tests/tcg/hexagon/check_rev_gating.c @@ -135,6 +135,18 @@ TRY_FUNC(v79hvx_vcvt2_hf_ub, ".word 0x1ed5e0e2 /* v3:2.hf = vcvt2(v0.ub) */\n") TRY_FUNC(v79hvx_vcvt2_b_hf, ".word 0x1ac1e0c2 /* v2.b = vcvt2(v0.hf, v1.hf) */\n") +TRY_FUNC(v79hvx_vcvt_f8_hf, + ".word 0x1fe1e040 /* v0.f8 = vcvt(v0.hf, v1.hf) */\n") +TRY_FUNC(v79hvx_vcvt_hf_f8, + ".word 0x1e05e0a2 /* v3:2.hf = vcvt(v0.f8) */\n") +TRY_FUNC(v79hvx_vmpy_hf_f8, + ".word 0x1f80c0c0 /* v1:0.hf = vmpy(v0.f8, v0.f8) */\n") +TRY_FUNC(v79hvx_vmpy_hf_f8_acc, + ".word 0x1f80c0e0 /* v1:0.hf += vmpy(v0.f8, v0.f8) */\n") +TRY_FUNC(v79hvx_vfneg_f8, + ".word 0x1c66e0e2 /* v2.f8 = vfneg(v0.f8) */\n") +TRY_FUNC(v79hvx_vfmin_f8, + ".word 0x1c61e082 /* v2.f8 = vfmin(v0.f8, v1.f8) */\n") TRY_FUNC(v81hvx_veqhf, ".word 0x1f82c11c /* q0 = vcmp.eq(v1.hf, v2.hf) */\n") @@ -183,6 +195,12 @@ int main(void) assert(try_v79hvx_vmerge_qf() == SIGILL); assert(try_v79hvx_vcvt2_hf_ub() == SIGILL); assert(try_v79hvx_vcvt2_b_hf() == SIGILL); + assert(try_v79hvx_vcvt_f8_hf() == SIGILL); + assert(try_v79hvx_vcvt_hf_f8() == SIGILL); + assert(try_v79hvx_vmpy_hf_f8() == SIGILL); + assert(try_v79hvx_vmpy_hf_f8_acc() == SIGILL); + assert(try_v79hvx_vfneg_f8() == SIGILL); + assert(try_v79hvx_vfmin_f8() == SIGILL); assert(try_v81hvx_veqhf() == SIGILL); diff --git a/tests/tcg/hexagon/fp_hvx_v79.c b/tests/tcg/hexagon/fp_hvx_v79.c index 2f6bbd7cf40..6f65b4396dd 100644 --- a/tests/tcg/hexagon/fp_hvx_v79.c +++ b/tests/tcg/hexagon/fp_hvx_v79.c @@ -20,20 +20,7 @@ int err; #include "hvx_misc.h" #include "hex_test.h" -/* - * The vcvt2 instructions differ from vcvt in how elements are interleaved - * between the vectors of a pair or the sources: - * - * Vdd.v[0].hf[2*i] = OP(Vu.FROM[4*i]); - * Vdd.v[0].hf[2*i+1] = OP(Vu.FROM[4*i+2]); - * Vdd.v[1].hf[2*i] = OP(Vu.FROM[4*i+1]); - * Vdd.v[1].hf[2*i+1] = OP(Vu.FROM[4*i+3]); - * - * Vd.TO[4*i] = OP(Vu.hf[2*i]); - * Vd.TO[4*i+1] = OP(Vv.hf[2*i]); - * Vd.TO[4*i+2] = OP(Vu.hf[2*i+1]); - * Vd.TO[4*i+3] = OP(Vv.hf[2*i+1]); - */ +/* vcvt2 interleaves elements between result vectors and source vectors. */ #define DEF_TEST_VCVT2_WIDEN(FROM, MUL, ADD) \ static void test_vcvt2_hf_##FROM(void) \ { \ @@ -98,6 +85,340 @@ static const struct { DEF_TEST_VCVT2_NARROW(ub) DEF_TEST_VCVT2_NARROW(b) +/* FP8 tests cover edge cases and hash exhaustive input sweeps. */ +#define F8_NAN 0x80 +#define HF_NAN 0x7fff + +/* Widening FP8 instructions use vcvt2-style result interleaving. */ +#define WIDEN_VEC(I) ((I) % 2) +#define WIDEN_IDX(I) (2 * ((I) / 4) + (((I) % 4) / 2)) + +#define HASH_INIT 0xcbf29ce484222325ULL + +static void hash_bytes(uint64_t *hash, const void *buf, size_t len) +{ + const uint8_t *p = buf; + + for (size_t i = 0; i < len; i++) { + *hash = (*hash ^ p[i]) * 0x100000001b3ULL; + } +} + +static void check_hash(int line, uint64_t hash, uint64_t golden) +{ + if (hash != golden) { + printf("ERROR at line %d: hash 0x%016" PRIx64 " != 0x%016" PRIx64 "\n", + line, hash, golden); + err++; + } +} + +static const struct { + uint8_t a, b; + uint16_t add, sub, mpy; + uint8_t min, max; +} f8_cases[] = { + /* NaN inputs */ + { 0x80, 0x40, HF_NAN, HF_NAN, HF_NAN, F8_NAN, F8_NAN }, + { 0x40, 0x80, HF_NAN, HF_NAN, HF_NAN, F8_NAN, F8_NAN }, + { 0x80, 0x80, HF_NAN, HF_NAN, HF_NAN, F8_NAN, F8_NAN }, + /* Zero */ + { 0x00, 0x00, 0x0000, 0x0000, 0x0000, 0x00, 0x00 }, + { 0x00, 0x40, 0x3c00, 0xbc00, 0x0000, 0x00, 0x40 }, + /* Exact cancellation */ + { 0x40, 0x40, 0x4000, 0x0000, 0x3c00, 0x40, 0x40 }, + { 0x40, 0xc0, 0x0000, 0x4000, 0xbc00, 0xc0, 0x40 }, + /* Largest magnitude */ + { 0x7f, 0x7f, 0x5f80, 0x0000, 0x7b08, 0x7f, 0x7f }, + { 0x7f, 0xff, 0x0000, 0x5f80, 0xfb08, 0xff, 0x7f }, + { 0xff, 0xff, 0xdf80, 0x0000, 0x7b08, 0xff, 0xff }, + /* Subnormal boundaries */ + { 0x01, 0x01, 0x1800, 0x0000, 0x0010, 0x01, 0x01 }, + { 0x01, 0x40, 0x3c01, 0xbbfe, 0x1400, 0x01, 0x40 }, + { 0x07, 0x08, 0x2380, 0x9400, 0x0380, 0x07, 0x08 }, + { 0x44, 0x01, 0x3e01, 0x3dff, 0x1600, 0x01, 0x44 }, +}; +#define NUM_F8_CASES ARRAY_SIZE(f8_cases) + +#define DEF_TEST_F8_BINARY_HF(NAME, INTRIN, FIELD, GOLDEN) \ + static void test_##NAME(void) \ + { \ + HVX_VectorPair *hvx_output = (HVX_VectorPair *)&output[0]; \ + uint64_t hash = HASH_INIT; \ + HVX_Vector a, b; \ + for (int i = 0; i < MAX_VEC_SIZE_BYTES; i++) { \ + int c = i % NUM_F8_CASES; \ + ((MMVector *)&a)->ub[i] = f8_cases[c].a; \ + ((MMVector *)&b)->ub[i] = f8_cases[c].b; \ + expect[WIDEN_VEC(i)].hf[WIDEN_IDX(i)] = f8_cases[c].FIELD; \ + } \ + *hvx_output = INTRIN(a, b); \ + check_output_hf(__LINE__, 2); \ + /* All FP8 operand pairs. */ \ + for (int blk = 0; blk < 512; blk++) { \ + for (int i = 0; i < MAX_VEC_SIZE_BYTES; i++) { \ + int pair = blk * MAX_VEC_SIZE_BYTES + i; \ + ((MMVector *)&a)->ub[i] = pair & 0xff; \ + ((MMVector *)&b)->ub[i] = pair >> 8; \ + } \ + *hvx_output = INTRIN(a, b); \ + hash_bytes(&hash, hvx_output, sizeof(*hvx_output)); \ + } \ + check_hash(__LINE__, hash, GOLDEN); \ + } + +#define DEF_TEST_F8_BINARY_UB(NAME, INTRIN, FIELD, GOLDEN) \ + static void test_##NAME(void) \ + { \ + HVX_Vector *hvx_output = (HVX_Vector *)&output[0]; \ + uint64_t hash = HASH_INIT; \ + HVX_Vector a, b; \ + for (int i = 0; i < MAX_VEC_SIZE_BYTES; i++) { \ + int c = i % NUM_F8_CASES; \ + ((MMVector *)&a)->ub[i] = f8_cases[c].a; \ + ((MMVector *)&b)->ub[i] = f8_cases[c].b; \ + expect[0].ub[i] = f8_cases[c].FIELD; \ + } \ + *hvx_output = INTRIN(a, b); \ + check_output_ub(__LINE__, 1); \ + for (int blk = 0; blk < 512; blk++) { \ + for (int i = 0; i < MAX_VEC_SIZE_BYTES; i++) { \ + int pair = blk * MAX_VEC_SIZE_BYTES + i; \ + ((MMVector *)&a)->ub[i] = pair & 0xff; \ + ((MMVector *)&b)->ub[i] = pair >> 8; \ + } \ + *hvx_output = INTRIN(a, b); \ + hash_bytes(&hash, hvx_output, sizeof(*hvx_output)); \ + } \ + check_hash(__LINE__, hash, GOLDEN); \ + } + +DEF_TEST_F8_BINARY_HF(vadd_hf_f8, Q6_Whf_vadd_VV, add, 0x2e57ea6fa0a5770bULL) +DEF_TEST_F8_BINARY_HF(vsub_hf_f8, Q6_Whf_vsub_VV, sub, 0xe5fcfee5b66a9333ULL) +DEF_TEST_F8_BINARY_HF(vmpy_hf_f8, Q6_Whf_vmpy_VV, mpy, 0x9edf5d41219d76dfULL) +DEF_TEST_F8_BINARY_UB(vfmin_f8, Q6_V_vfmin_VV, min, 0x9d394d7796c56725ULL) +DEF_TEST_F8_BINARY_UB(vfmax_f8, Q6_V_vfmax_VV, max, 0xd4e8b1560e8feea5ULL) + +static const struct { + uint8_t a, b; + uint16_t acc, result; +} f8_acc_cases[] = { + { 0x40, 0x40, 0x0000, 0x3c00 }, + { 0x40, 0x40, 0x3c00, 0x4000 }, + { 0x40, 0x40, 0xbc00, 0x0000 }, + { 0x40, 0xc0, 0x3c00, 0x0000 }, + { 0x7f, 0x7f, 0x7bff, 0x7c00 }, + { 0x7f, 0xff, 0xfbff, 0xfc00 }, + { 0x01, 0x01, 0x0001, 0x0011 }, + /* NaN and infinity */ + { 0x80, 0x40, 0x3c00, HF_NAN }, + { 0x40, 0x40, 0x7e00, HF_NAN }, + { 0x40, 0x40, 0x7c00, 0x7c00 }, +}; +#define NUM_F8_ACC_CASES ARRAY_SIZE(f8_acc_cases) + +static const uint16_t acc_values[] = { + 0x0000, 0x8000, 0x3c00, 0xbc00, 0x6800, 0x0001, 0x7bff, 0xfbff, 0x7c00, + 0xfc00, 0x7e00, 0x7d00, 0x1234, 0xa5c3, +}; + +static void test_vmpy_hf_f8_acc(void) +{ + HVX_VectorPair *hvx_output = (HVX_VectorPair *)&output[0]; + uint64_t hash = HASH_INIT; + HVX_VectorPair acc; + HVX_Vector a, b; + + for (int i = 0; i < MAX_VEC_SIZE_BYTES; i++) { + int c = i % NUM_F8_ACC_CASES; + + ((MMVector *)&a)->ub[i] = f8_acc_cases[c].a; + ((MMVector *)&b)->ub[i] = f8_acc_cases[c].b; + ((MMVector *)&acc)->hf[WIDEN_VEC(i) * 64 + WIDEN_IDX(i)] = + f8_acc_cases[c].acc; + expect[WIDEN_VEC(i)].hf[WIDEN_IDX(i)] = f8_acc_cases[c].result; + } + *hvx_output = Q6_Whf_vmpyacc_WhfVV(acc, a, b); + check_output_hf(__LINE__, 2); + + /* Test fixed and near-canceling accumulators. */ + for (int n = 0; n <= ARRAY_SIZE(acc_values); n++) { + for (int blk = 0; blk < 512; blk++) { + for (int i = 0; i < MAX_VEC_SIZE_BYTES; i++) { + int pair = blk * MAX_VEC_SIZE_BYTES + i; + + ((MMVector *)&a)->ub[i] = pair & 0xff; + ((MMVector *)&b)->ub[i] = pair >> 8; + } + if (n == ARRAY_SIZE(acc_values)) { + acc = Q6_Whf_vmpy_VV(a, b); + for (int k = 0; k < MAX_VEC_SIZE_BYTES; k++) { + ((MMVector *)&acc)->hf[k] ^= 0x8000 ^ (k & 3); + } + } else { + for (int k = 0; k < MAX_VEC_SIZE_BYTES; k++) { + ((MMVector *)&acc)->hf[k] = acc_values[n]; + } + } + *hvx_output = Q6_Whf_vmpyacc_WhfVV(acc, a, b); + hash_bytes(&hash, hvx_output, sizeof(*hvx_output)); + } + } + check_hash(__LINE__, hash, 0x85def62d6d02aea8ULL); +} + +static const struct { + uint8_t f8; + uint16_t hf; +} cvt_hf_f8_cases[] = { + { 0x00, 0x0000 }, + { 0x01, 0x1400 }, + { 0x07, 0x1f00 }, + { 0x08, 0x2000 }, + { 0x40, 0x3c00 }, + { 0x7f, 0x5b80 }, + { 0x80, HF_NAN }, + { 0xff, 0xdb80 }, +}; +#define NUM_CVT_HF_F8_CASES ARRAY_SIZE(cvt_hf_f8_cases) + +static void test_vcvt_hf_f8(void) +{ + HVX_VectorPair *hvx_output = (HVX_VectorPair *)&output[0]; + uint64_t hash = HASH_INIT; + HVX_Vector a; + + for (int i = 0; i < MAX_VEC_SIZE_BYTES; i++) { + int c = i % NUM_CVT_HF_F8_CASES; + + ((MMVector *)&a)->ub[i] = cvt_hf_f8_cases[c].f8; + expect[WIDEN_VEC(i)].hf[WIDEN_IDX(i)] = cvt_hf_f8_cases[c].hf; + } + *hvx_output = Q6_Whf_vcvt_V(a); + check_output_hf(__LINE__, 2); + + /* All FP8 values. */ + for (int blk = 0; blk < 2; blk++) { + for (int i = 0; i < MAX_VEC_SIZE_BYTES; i++) { + ((MMVector *)&a)->ub[i] = blk * MAX_VEC_SIZE_BYTES + i; + } + *hvx_output = Q6_Whf_vcvt_V(a); + hash_bytes(&hash, hvx_output, sizeof(*hvx_output)); + } + check_hash(__LINE__, hash, 0xf09689ef20d70fafULL); +} + +#define DEF_TEST_F8_UNARY(NAME, INTRIN, REF) \ + static void test_##NAME(void) \ + { \ + HVX_Vector *hvx_output = (HVX_Vector *)&output[0]; \ + HVX_Vector a; \ + for (int blk = 0; blk < 2; blk++) { \ + for (int i = 0; i < MAX_VEC_SIZE_BYTES; i++) { \ + int f8 = blk * MAX_VEC_SIZE_BYTES + i; \ + ((MMVector *)&a)->ub[i] = f8; \ + expect[0].ub[i] = REF(f8); \ + } \ + *hvx_output = INTRIN(a); \ + check_output_ub(__LINE__, 1); \ + } \ + } + +static uint8_t ref_neg(uint8_t f8) +{ + return f8 == 0 || f8 == F8_NAN ? f8 : f8 ^ 0x80; +} + +static uint8_t ref_abs(uint8_t f8) +{ + return f8 == F8_NAN ? f8 : f8 & 0x7f; +} + +DEF_TEST_F8_UNARY(vfneg_f8, Q6_V_vfneg_V, ref_neg) +DEF_TEST_F8_UNARY(vabs_f8, Q6_V_vabs_V, ref_abs) + +/* Keep the conversion after the USR.FPSAT update. */ +static void set_fpsat(bool sat) +{ + uint32_t usr; + + asm volatile("%[usr] = usr" : [usr] "=r"(usr)); + usr = sat ? usr | (1 << 24) : usr & ~(1 << 24); + asm volatile("usr = %[usr]" : : [usr] "r"(usr)); +} + +static const struct { + uint16_t hf; + uint8_t f8, f8_sat; +} cvt_f8_hf_cases[] = { + /* NaN, infinity, and zero */ + { 0x7e00, F8_NAN, F8_NAN }, + { 0x7d00, F8_NAN, F8_NAN }, + { 0xfe00, F8_NAN, F8_NAN }, + { 0x7c00, F8_NAN, 0x7f }, + { 0xfc00, F8_NAN, 0xff }, + { 0x0000, 0x00, 0x00 }, + { 0x8000, 0x00, 0x00 }, + { 0x3c00, 0x40, 0x40 }, + { 0xbc00, 0xc0, 0xc0 }, + /* Largest finite values and overflow */ + { 0x5b80, 0x7f, 0x7f }, + { 0xdb80, 0xff, 0xff }, + { 0x5ba0, 0x7f, 0x7f }, + { 0x5b40, 0x7e, 0x7e }, + { 0x5bc0, F8_NAN, 0x7f }, + { 0x5c00, F8_NAN, 0x7f }, + /* Subnormal rounding */ + { 0x1400, 0x01, 0x01 }, + { 0x1000, 0x00, 0x00 }, + { 0x1600, 0x02, 0x02 }, + { 0x1800, 0x02, 0x02 }, + { 0x1200, 0x01, 0x01 }, + { 0x0001, 0x00, 0x00 }, + { 0x8001, 0x00, 0x00 }, +}; +#define NUM_CVT_F8_HF_CASES ARRAY_SIZE(cvt_f8_hf_cases) + +static void test_vcvt_f8_hf(bool sat) +{ + HVX_Vector *hvx_output = (HVX_Vector *)&output[0]; + HVX_Vector buffers[2]; + /* Prevent source loads from moving before set_fpsat(). */ + volatile HVX_Vector *vbuffers = buffers; + uint64_t hash = HASH_INIT; + + for (int j = 0; j < MAX_VEC_SIZE_BYTES / 2; j++) { + for (int s = 0; s < 2; s++) { + int c = (2 * j + s) % NUM_CVT_F8_HF_CASES; + + ((MMVector *)&buffers[s])->hf[j] = cvt_f8_hf_cases[c].hf; + expect[0].ub[4 * (j / 2) + 2 * (j % 2) + s] = + sat ? cvt_f8_hf_cases[c].f8_sat : cvt_f8_hf_cases[c].f8; + } + } + set_fpsat(sat); + *hvx_output = Q6_V_vcvt_VhfVhf(vbuffers[0], vbuffers[1]); + set_fpsat(false); + check_output_ub(__LINE__, 1); + + /* All half-precision values in alternating sources. */ + for (int blk = 0; blk < 1024; blk++) { + for (int j = 0; j < MAX_VEC_SIZE_BYTES / 2; j++) { + for (int s = 0; s < 2; s++) { + uint16_t h = (blk * 64 + j) * (s ? 40503 : 1) + (s ? 12345 : 0); + + ((MMVector *)&buffers[s])->hf[j] = h; + } + } + set_fpsat(sat); + *hvx_output = Q6_V_vcvt_VhfVhf(vbuffers[0], vbuffers[1]); + set_fpsat(false); + hash_bytes(&hash, hvx_output, sizeof(*hvx_output)); + } + check_hash(__LINE__, hash, + sat ? 0x711e375027f85c31ULL : 0xc90f8f22ad443a9dULL); +} + int main(void) { test_vcvt2_hf_ub(); @@ -105,6 +426,18 @@ int main(void) test_vcvt2_ub_hf(); test_vcvt2_b_hf(); + test_vcvt_hf_f8(); + test_vcvt_f8_hf(false); + test_vcvt_f8_hf(true); + test_vadd_hf_f8(); + test_vsub_hf_f8(); + test_vmpy_hf_f8(); + test_vmpy_hf_f8_acc(); + test_vfneg_f8(); + test_vabs_f8(); + test_vfmin_f8(); + test_vfmax_f8(); + puts(err ? "FAIL" : "PASS"); return err ? 1 : 0; } diff --git a/target/hexagon/tag_rev_info.c.inc b/target/hexagon/tag_rev_info.c.inc index ea586e1613e..92559052516 100644 --- a/target/hexagon/tag_rev_info.c.inc +++ b/target/hexagon/tag_rev_info.c.inc @@ -648,6 +648,7 @@ static const struct tag_rev_info tag_rev_info[XX_LAST_OPCODE] = { .introduced = HEX_VER_V79, .removed = HEX_VER_NONE }, [V6_set_qfext] = { .introduced = HEX_VER_V79, .removed = HEX_VER_NONE }, + [V6_vabs_f8] = { .introduced = HEX_VER_V79, .removed = HEX_VER_NONE }, [V6_vabs_qf16_hf] = { .introduced = HEX_VER_V79, .removed = HEX_VER_NONE }, [V6_vabs_qf16_qf16] = { .introduced = HEX_VER_V79, .removed = HEX_VER_NONE @@ -656,6 +657,7 @@ static const struct tag_rev_info tag_rev_info[XX_LAST_OPCODE] = { .introduced = HEX_VER_V79, .removed = HEX_VER_NONE }, [V6_vabs_qf32_sf] = { .introduced = HEX_VER_V79, .removed = HEX_VER_NONE }, + [V6_vadd_hf_f8] = { .introduced = HEX_VER_V79, .removed = HEX_VER_NONE }, [V6_vconv_qf16_hf] = { .introduced = HEX_VER_V79, .removed = HEX_VER_NONE }, [V6_vconv_qf16_qf16] = { .introduced = HEX_VER_V79, .removed = HEX_VER_NONE @@ -668,11 +670,20 @@ static const struct tag_rev_info tag_rev_info[XX_LAST_OPCODE] = { [V6_vcvt2_hf_b] = { .introduced = HEX_VER_V79, .removed = HEX_VER_NONE }, [V6_vcvt2_hf_ub] = { .introduced = HEX_VER_V79, .removed = HEX_VER_NONE }, [V6_vcvt2_ub_hf] = { .introduced = HEX_VER_V79, .removed = HEX_VER_NONE }, + [V6_vcvt_f8_hf] = { .introduced = HEX_VER_V79, .removed = HEX_VER_NONE }, + [V6_vcvt_hf_f8] = { .introduced = HEX_VER_V79, .removed = HEX_VER_NONE }, + [V6_vfmax_f8] = { .introduced = HEX_VER_V79, .removed = HEX_VER_NONE }, + [V6_vfmin_f8] = { .introduced = HEX_VER_V79, .removed = HEX_VER_NONE }, + [V6_vfneg_f8] = { .introduced = HEX_VER_V79, .removed = HEX_VER_NONE }, [V6_vilog2_hf] = { .introduced = HEX_VER_V79, .removed = HEX_VER_NONE }, [V6_vilog2_qf16] = { .introduced = HEX_VER_V79, .removed = HEX_VER_NONE }, [V6_vilog2_qf32] = { .introduced = HEX_VER_V79, .removed = HEX_VER_NONE }, [V6_vilog2_sf] = { .introduced = HEX_VER_V79, .removed = HEX_VER_NONE }, [V6_vmerge_qf] = { .introduced = HEX_VER_V79, .removed = HEX_VER_NONE }, + [V6_vmpy_hf_f8] = { .introduced = HEX_VER_V79, .removed = HEX_VER_NONE }, + [V6_vmpy_hf_f8_acc] = { + .introduced = HEX_VER_V79, .removed = HEX_VER_NONE + }, [V6_vmpy_rt_hf] = { .introduced = HEX_VER_V79, .removed = HEX_VER_NONE }, [V6_vmpy_rt_qf16] = { .introduced = HEX_VER_V79, .removed = HEX_VER_NONE }, [V6_vmpy_rt_sf] = { .introduced = HEX_VER_V79, .removed = HEX_VER_NONE }, @@ -684,6 +695,7 @@ static const struct tag_rev_info tag_rev_info[XX_LAST_OPCODE] = { .introduced = HEX_VER_V79, .removed = HEX_VER_NONE }, [V6_vneg_qf32_sf] = { .introduced = HEX_VER_V79, .removed = HEX_VER_NONE }, + [V6_vsub_hf_f8] = { .introduced = HEX_VER_V79, .removed = HEX_VER_NONE }, [V6_vsub_hf_mix] = { .introduced = HEX_VER_V79, .removed = HEX_VER_NONE }, [V6_vsub_sf_mix] = { .introduced = HEX_VER_V79, .removed = HEX_VER_NONE }, diff --git a/target/hexagon/imported/mmvec/encode_ext.def b/target/hexagon/imported/mmvec/encode_ext.def index 9605e870a1e..5941e09df93 100644 --- a/target/hexagon/imported/mmvec/encode_ext.def +++ b/target/hexagon/imported/mmvec/encode_ext.def @@ -882,6 +882,18 @@ DEF_ENC(V6_veqhf_or,"00011100100vvvvvPP1uuuuu010111xx") DEF_ENC(V6_veqsf_xor,"00011100100vvvvvPP1uuuuu100011xx") DEF_ENC(V6_veqhf_xor,"00011100100vvvvvPP1uuuuu100111xx") +/* FP8 instructions */ +DEF_ENC(V6_vadd_hf_f8,"00011111100vvvvvPP0uuuuu100ddddd") +DEF_ENC(V6_vsub_hf_f8,"00011111100vvvvvPP0uuuuu101ddddd") +DEF_ENC(V6_vmpy_hf_f8,"00011111100vvvvvPP0uuuuu110ddddd") +DEF_ENC(V6_vmpy_hf_f8_acc,"00011111100vvvvvPP0uuuuu111xxxxx") +DEF_ENC(V6_vfmin_f8,"00011100011vvvvvPP1uuuuu100ddddd") +DEF_ENC(V6_vfmax_f8,"00011100011vvvvvPP1uuuuu101ddddd") +DEF_ENC(V6_vabs_f8,"00011100011--110PP1uuuuu110ddddd") +DEF_ENC(V6_vfneg_f8,"00011100011--110PP1uuuuu111ddddd") +DEF_ENC(V6_vcvt_f8_hf,"00011111111vvvvvPP1uuuuu010ddddd") +DEF_ENC(V6_vcvt_hf_f8,"00011110--0--101PP1uuuuu101ddddd") + /* BFLOAT instructions */ DEF_ENC(V6_vmpy_sf_bf,"00011101010vvvvvPP1uuuuu100ddddd") DEF_ENC(V6_vmpy_sf_bf_acc,"00011101000vvvvvPP1uuuuu000xxxxx") diff --git a/target/hexagon/imported/mmvec/ext.idef b/target/hexagon/imported/mmvec/ext.idef index b9a4858d0c1..1b8afee8b6f 100644 --- a/target/hexagon/imported/mmvec/ext.idef +++ b/target/hexagon/imported/mmvec/ext.idef @@ -2920,6 +2920,12 @@ EXTINSN(V6_##TAG, SYNTAX, \ ATTRIBS(A_EXTENSION,A_HVX_IEEE_FP,A_CVI,A_CVI_VX,A_HVX_IEEE_FP_OUT_32), \ DESCR, DO_FOR_EACH_CODE(WIDTH, CODE)) +/* Single pipe, 8-bit output */ +#define ITERATOR_INSN_IEEE_FP_8(WIDTH,TAG,SYNTAX,DESCR,CODE) \ +EXTINSN(V6_##TAG, SYNTAX, \ + ATTRIBS(A_EXTENSION,A_HVX_IEEE_FP,A_CVI,A_CVI_VX,A_HVX_IEEE_FP_OUT_8), \ + DESCR, DO_FOR_EACH_CODE(WIDTH, CODE)) + /* Single pipe, 16-bit output */ #define ITERATOR_INSN_IEEE_FP_16(WIDTH,TAG,SYNTAX,DESCR,CODE) \ EXTINSN(V6_##TAG, SYNTAX, \ @@ -3373,6 +3379,78 @@ ITERATOR_INSN_IEEE_FP_16_32_LATE(16, vmin_bf, "Vd32.bf=vmin(Vu32.bf,Vv32.bf)", fBFLOAT()) +/****************************************************************************** + FP8 instructions + ******************************************************************************/ + +/* + * The widening FP8 instructions split the source elements between the two + * result vectors by position: the even-numbered bytes of each word go to the + * first vector and the odd-numbered ones to the second. The narrowing + * conversion, vcvt_f8_hf, takes its elements alternately from its two sources. + */ +ITERATOR_INSN_IEEE_FP_DOUBLE_16(32, vadd_hf_f8, "Vdd32.hf=vadd(Vu32.f8,Vv32.f8)", + "Vector FP8 add: f8 widen to hf", + VddV.v[0].hf[2*i] = fp_add_hf_f8(VuV.f8[4*i], VvV.f8[4*i], &env->hvx_fp_status); + VddV.v[0].hf[2*i+1] = fp_add_hf_f8(VuV.f8[4*i+2], VvV.f8[4*i+2], &env->hvx_fp_status); + VddV.v[1].hf[2*i] = fp_add_hf_f8(VuV.f8[4*i+1], VvV.f8[4*i+1], &env->hvx_fp_status); + VddV.v[1].hf[2*i+1] = fp_add_hf_f8(VuV.f8[4*i+3], VvV.f8[4*i+3], &env->hvx_fp_status)) + +ITERATOR_INSN_IEEE_FP_DOUBLE_16(32, vsub_hf_f8, "Vdd32.hf=vsub(Vu32.f8,Vv32.f8)", + "Vector FP8 sub: f8 widen to hf", + VddV.v[0].hf[2*i] = fp_sub_hf_f8(VuV.f8[4*i], VvV.f8[4*i], &env->hvx_fp_status); + VddV.v[0].hf[2*i+1] = fp_sub_hf_f8(VuV.f8[4*i+2], VvV.f8[4*i+2], &env->hvx_fp_status); + VddV.v[1].hf[2*i] = fp_sub_hf_f8(VuV.f8[4*i+1], VvV.f8[4*i+1], &env->hvx_fp_status); + VddV.v[1].hf[2*i+1] = fp_sub_hf_f8(VuV.f8[4*i+3], VvV.f8[4*i+3], &env->hvx_fp_status)) + +ITERATOR_INSN_IEEE_FP_DOUBLE_16(32, vmpy_hf_f8, "Vdd32.hf=vmpy(Vu32.f8,Vv32.f8)", + "Vector FP8 mul: f8 widen to hf", + VddV.v[0].hf[2*i] = fp_mult_hf_f8(VuV.f8[4*i], VvV.f8[4*i], &env->hvx_fp_status); + VddV.v[0].hf[2*i+1] = fp_mult_hf_f8(VuV.f8[4*i+2], VvV.f8[4*i+2], &env->hvx_fp_status); + VddV.v[1].hf[2*i] = fp_mult_hf_f8(VuV.f8[4*i+1], VvV.f8[4*i+1], &env->hvx_fp_status); + VddV.v[1].hf[2*i+1] = fp_mult_hf_f8(VuV.f8[4*i+3], VvV.f8[4*i+3], &env->hvx_fp_status)) + +ITERATOR_INSN_IEEE_FP_DOUBLE_16(32, vmpy_hf_f8_acc, "Vxx32.hf+=vmpy(Vu32.f8,Vv32.f8)", + "Vector FP8 fma: f8 widen to hf", + VxxV.v[0].hf[2*i] = fp_mult_hf_f8_acc(VuV.f8[4*i], VvV.f8[4*i], + VxxV.v[0].hf[2*i], &env->hvx_fp_status); + VxxV.v[0].hf[2*i+1] = fp_mult_hf_f8_acc(VuV.f8[4*i+2], VvV.f8[4*i+2], + VxxV.v[0].hf[2*i+1], &env->hvx_fp_status); + VxxV.v[1].hf[2*i] = fp_mult_hf_f8_acc(VuV.f8[4*i+1], VvV.f8[4*i+1], + VxxV.v[1].hf[2*i], &env->hvx_fp_status); + VxxV.v[1].hf[2*i+1] = fp_mult_hf_f8_acc(VuV.f8[4*i+3], VvV.f8[4*i+3], + VxxV.v[1].hf[2*i+1], &env->hvx_fp_status)) + +ITERATOR_INSN_IEEE_FP_DOUBLE_16(32, vcvt_hf_f8, "Vdd32.hf=vcvt(Vu32.f8)", + "Vector cvt to fp16: f8 widen to hf", + VddV.v[0].hf[2*i] = f8_to_f16(VuV.f8[4*i], &env->hvx_fp_status); + VddV.v[0].hf[2*i+1] = f8_to_f16(VuV.f8[4*i+2], &env->hvx_fp_status); + VddV.v[1].hf[2*i] = f8_to_f16(VuV.f8[4*i+1], &env->hvx_fp_status); + VddV.v[1].hf[2*i+1] = f8_to_f16(VuV.f8[4*i+3], &env->hvx_fp_status)) + +/* + * With USR.FPSAT set, overflow saturates to the largest FP8 normal number + * instead of producing NaN. + */ +ITERATOR_INSN_IEEE_FP_8(32, vcvt_f8_hf, "Vd32.f8=vcvt(Vu32.hf,Vv32.hf)", + "Vector cvt to fp8: hf narrow to f8", + VdV.f8[4*i] = f16_to_f8(VuV.hf[2*i], GET_USR_FIELD(USR_FPSAT), &env->hvx_fp_status); + VdV.f8[4*i+1] = f16_to_f8(VvV.hf[2*i], GET_USR_FIELD(USR_FPSAT), &env->hvx_fp_status); + VdV.f8[4*i+2] = f16_to_f8(VuV.hf[2*i+1], GET_USR_FIELD(USR_FPSAT), &env->hvx_fp_status); + VdV.f8[4*i+3] = f16_to_f8(VvV.hf[2*i+1], GET_USR_FIELD(USR_FPSAT), &env->hvx_fp_status)) + +ITERATOR_INSN_IEEE_FP_8(8, vfneg_f8, "Vd32.f8=vfneg(Vu32.f8)", + "Vector FP8 neg: f8", VdV.f8[i] = fp_neg_f8(VuV.f8[i])) +ITERATOR_INSN_IEEE_FP_8(8, vabs_f8, "Vd32.f8=vabs(Vu32.f8)", + "Vector FP8 abs: f8", VdV.f8[i] = fp_abs_f8(VuV.f8[i])) +ITERATOR_INSN_IEEE_FP_8(8, vfmin_f8, "Vd32.f8=vfmin(Vu32.f8,Vv32.f8)", + "Vector FP8 min: f8", + VdV.f8[i] = fp_min_f8(VuV.f8[i], VvV.f8[i], &env->hvx_fp_status)) +ITERATOR_INSN_IEEE_FP_8(8, vfmax_f8, "Vd32.f8=vfmax(Vu32.f8,Vv32.f8)", + "Vector FP8 max: f8", + VdV.f8[i] = fp_max_f8(VuV.f8[i], VvV.f8[i], &env->hvx_fp_status)) + + /****************************************************************************** QFloat extended-precision instructions ******************************************************************************/ -- 2.34.1
