Signed-off-by: Richard Henderson <[email protected]>
---
target/arm/cpu-features.h | 5 ++++
target/arm/tcg/helper-fp8-defs.h | 1 +
target/arm/tcg/vec_internal.h | 7 +++++
target/arm/tcg/fp8_helper.c | 7 +++++
target/arm/tcg/sme_helper.c | 44 ++++++++++++++++++++++++++++++++
target/arm/tcg/translate-sme.c | 2 ++
target/arm/tcg/sme.decode | 1 +
7 files changed, 67 insertions(+)
diff --git a/target/arm/cpu-features.h b/target/arm/cpu-features.h
index 1ab6c90b8c..280ead6f9c 100644
--- a/target/arm/cpu-features.h
+++ b/target/arm/cpu-features.h
@@ -1857,6 +1857,11 @@ static inline bool
isar_feature_aa64_sme_tmop_f8f16(const ARMISARegisters *id)
return isar_feature_aa64_sme_tmop(id) && isar_feature_aa64_sme_f8f16(id);
}
+static inline bool isar_feature_aa64_sme_tmop_f8f32(const ARMISARegisters *id)
+{
+ return isar_feature_aa64_sme_tmop(id) && isar_feature_aa64_sme_f8f32(id);
+}
+
/*
* Feature tests for "does this exist in either 32-bit or 64-bit?"
*/
diff --git a/target/arm/tcg/helper-fp8-defs.h b/target/arm/tcg/helper-fp8-defs.h
index 13be62aa15..e2cf2ba8c7 100644
--- a/target/arm/tcg/helper-fp8-defs.h
+++ b/target/arm/tcg/helper-fp8-defs.h
@@ -49,3 +49,4 @@ DEF_HELPER_FLAGS_5(sme_fmop4a_sb, TCG_CALL_NO_RWG, void, ptr,
ptr, ptr, env, i32
DEF_HELPER_FLAGS_5(sme_fmop4a_hb, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, env,
i32)
DEF_HELPER_FLAGS_6(sme_ftmopa_hb, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr,
env, i32)
+DEF_HELPER_FLAGS_6(sme_ftmopa_sb, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr,
env, i32)
diff --git a/target/arm/tcg/vec_internal.h b/target/arm/tcg/vec_internal.h
index 038a2a3439..8f6d8a984b 100644
--- a/target/arm/tcg/vec_internal.h
+++ b/target/arm/tcg/vec_internal.h
@@ -555,4 +555,11 @@ void sme_mop4(void *vza, void *vzn, void *vzm, void
*fn_opaque,
uint32_t desc, size_t esize,
void (*fn)(void *, void *, void *, void *));
+/*
+ * Perform SME sparse outer product, 4-way, 8 to 32-bit.
+ */
+void sme_tmop_4way_sb(uint32_t *za, uint8_t *zn0, uint32_t *zm,
+ uint64_t *zk, void *fn_opaque, uint32_t desc,
+ void (*fn)(void *, void *, void *, void *));
+
#endif /* TARGET_ARM_VEC_INTERNAL_H */
diff --git a/target/arm/tcg/fp8_helper.c b/target/arm/tcg/fp8_helper.c
index 305cafdc9b..0be811e71b 100644
--- a/target/arm/tcg/fp8_helper.c
+++ b/target/arm/tcg/fp8_helper.c
@@ -1048,3 +1048,10 @@ void HELPER(sme_ftmopa_hb)(void *vza, void *vzn, void
*vzm, void *vzk,
}
}
}
+
+void HELPER(sme_ftmopa_sb)(void *vza, void *vzn, void *vzm, void *vzk,
+ CPUArchState *env, uint32_t desc)
+{
+ FP8MulContext ctx = fp8_mul_start(env, 0xf);
+ sme_tmop_4way_sb(vza, vzn, vzm, vzk, &ctx, desc, inner_fmop4a_sb);
+}
diff --git a/target/arm/tcg/sme_helper.c b/target/arm/tcg/sme_helper.c
index c2b7c23d6c..3d7aefac5b 100644
--- a/target/arm/tcg/sme_helper.c
+++ b/target/arm/tcg/sme_helper.c
@@ -2723,6 +2723,50 @@ static void sme_tmop_2way_sh(uint32_t *za, uint16_t
*zn0, uint32_t *zm,
}
}
+void sme_tmop_4way_sb(uint32_t *za, uint8_t *zn0, uint32_t *zm,
+ uint64_t *zk, void *fn_opaque, uint32_t desc,
+ void (*fn)(void *, void *, void *, void *))
+{
+ intptr_t oprsz = simd_maxsz(desc);
+ intptr_t dim = oprsz >> MO_32;
+ intptr_t index = simd_data(desc);
+ intptr_t ctrl_base = (index * oprsz) >> 1;
+ uint8_t *zn1 = zn0 + sizeof(ARMVectorReg);
+
+ for (intptr_t row = 0; row < dim; row++) {
+ uint32_t *za_row = za + tile_vslice_offset(row);
+
+ for (intptr_t col = 0; col < dim; col++) {
+ uint32_t *e2 = zm + H4(col);
+ uint32_t *e3 = za_row + H4(col);
+ uint16_t e1l = 0, e1h = 0;
+ uint32_t e1;
+
+ /*
+ * Eight control bits select two elements from each row.
+ * The elements may be non-contiguous, so assemble them
+ * locally into e1.
+ * Pseudo-code has a triple loop running forward, with a
+ * test for (i < 2) to limit construction to 2 elements.
+ * Easier to run a single loop backward, shifting extra
+ * elements off the top.
+ */
+ uint64_t this_ctrl = extractn(zk, ctrl_base + col * 8, 8);
+ for (int e = 3; e >= 0; e--) {
+ if (this_ctrl & (0x01 << e)) {
+ e1l = (e1l << 8) | zn0[H1(4 * row + e)];
+ }
+ if (this_ctrl & (0x10 << e)) {
+ e1h = (e1h << 8) | zn1[H1(4 * row + e)];
+ }
+ }
+ e1 = (e1h << 16) | e1l;
+
+ fn(e3, &e1, e2, fn_opaque);
+ }
+ }
+}
+
static void inner_fmop4a_hh(void *vd, void *vn, void *vm, void *vinfo)
{
float16 *d = vd, *n = vn, *m = vm;
diff --git a/target/arm/tcg/translate-sme.c b/target/arm/tcg/translate-sme.c
index 395652f477..0fda2d9ac5 100644
--- a/target/arm/tcg/translate-sme.c
+++ b/target/arm/tcg/translate-sme.c
@@ -2179,3 +2179,5 @@ static bool do_tmop_fp8(DisasContext *s, arg_tmop *a,
MemOp esz,
TRANS_FEAT(FTMOPA_hb, aa64_sme_tmop_f8f16, do_tmop_fp8,
a, MO_16, gen_helper_sme_ftmopa_hb)
+TRANS_FEAT(FTMOPA_sb, aa64_sme_tmop_f8f32, do_tmop_fp8,
+ a, MO_32, gen_helper_sme_ftmopa_sb)
diff --git a/target/arm/tcg/sme.decode b/target/arm/tcg/sme.decode
index da6f4686d3..eab1f7e8fb 100644
--- a/target/arm/tcg/sme.decode
+++ b/target/arm/tcg/sme.decode
@@ -1150,3 +1150,4 @@ BFTMOPA_sh 1000 0001 010 ..... 000 ... .... .. 00 ..
@tmop_o2
FTMOPA_sh 1000 0001 011 ..... 000 ... .... .. 00 .. @tmop_o2
FTMOPA_hb 1000 0000 011 ..... 000 ... .... .. 100 . @tmop_o1
+FTMOPA_sb 1000 0000 011 ..... 000 ... .... .. 00 .. @tmop_o2
--
2.43.0
