Signed-off-by: Richard Henderson <[email protected]>
---
target/arm/cpu-features.h | 5 ++++
target/arm/tcg/helper-sme-defs.h | 2 ++
target/arm/tcg/sme_helper.c | 49 ++++++++++++++++++++++++++++++++
target/arm/tcg/translate-sme.c | 26 +++++++++++++++++
target/arm/tcg/sme.decode | 10 +++++++
5 files changed, 92 insertions(+)
diff --git a/target/arm/cpu-features.h b/target/arm/cpu-features.h
index fb5ed25ad0..6370c03e2a 100644
--- a/target/arm/cpu-features.h
+++ b/target/arm/cpu-features.h
@@ -1590,6 +1590,11 @@ static inline bool isar_feature_aa64_sme_mop4(const
ARMISARegisters *id)
return FIELD_EX64_IDREG(id, ID_AA64SMFR0, SMOP4);
}
+static inline bool isar_feature_aa64_sme_tmop(const ARMISARegisters *id)
+{
+ return FIELD_EX64_IDREG(id, ID_AA64SMFR0, STMOP);
+}
+
static inline bool isar_feature_aa64_ssve_aes(const ARMISARegisters *id)
{
return FIELD_EX64_IDREG(id, ID_AA64SMFR0, AES);
diff --git a/target/arm/tcg/helper-sme-defs.h b/target/arm/tcg/helper-sme-defs.h
index 7fc7129e54..8107eb40af 100644
--- a/target/arm/tcg/helper-sme-defs.h
+++ b/target/arm/tcg/helper-sme-defs.h
@@ -404,3 +404,5 @@ DEF_HELPER_FLAGS_4(sme_usmop4a_sb, TCG_CALL_NO_RWG, void,
ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sme_usmop4s_sb, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sme_usmop4a_dh, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_4(sme_usmop4s_dh, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+
+DEF_HELPER_FLAGS_6(sme_ftmopa_ss, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr,
fpst, i32)
diff --git a/target/arm/tcg/sme_helper.c b/target/arm/tcg/sme_helper.c
index 23bb816b6a..571b0add34 100644
--- a/target/arm/tcg/sme_helper.c
+++ b/target/arm/tcg/sme_helper.c
@@ -2636,6 +2636,49 @@ void sme_mop4(void *vza, void *vzn, void *vzm, void
*fn_opaque,
}
}
+/*
+ * Sparse outer product, non-widening. ESZ in {16, 32}.
+ */
+static void sme_tmop(void *vza, void *vzn, void *vzm, uint64_t *zk,
+ void *fn_opaque, uint32_t desc, MemOp esz,
+ void (*fn)(void *, void *, void *, void *))
+{
+ intptr_t oprsz = simd_maxsz(desc);
+ intptr_t index = simd_data(desc);
+ intptr_t esize = 1 << esz;
+ intptr_t host_adj = HOST_BIG_ENDIAN ? 8 - esize : 0;
+ /* Base in bits for op3[index*:csize], csize = (VL * 2) / esize. */
+ intptr_t ctrl_base = index * oprsz * 2;
+ /* Create a zero for use with the largest esz. */
+ uint32_t zero = 0;
+
+ for (intptr_t row = 0; row < oprsz; row += esize) {
+ void *vza_row = vza + tile_vslice_offset(row);
+
+ for (intptr_t col = 0; col < oprsz; col += esize) {
+ void *e2 = vzm + (col ^ host_adj);
+ void *e3 = vza_row + (col ^ host_adj);
+
+ /*
+ * Two control bits select one element:
+ * Zn[row], if [0] is set,
+ * Zn+1[row], if [1] is set,
+ * 0, otherwise.
+ * Compute the address of that element.
+ */
+ void *e1 = &zero;
+ uint64_t this_ctrl = extractn(zk, (ctrl_base + 2 * col) >> esz, 2);
+ if (this_ctrl) {
+ e1 = vzn + (row ^ host_adj);
+ if (!(this_ctrl & 1)) {
+ e1 += sizeof(ARMVectorReg);
+ }
+ }
+ fn(e3, e1, e2, fn_opaque);
+ }
+ }
+}
+
static void inner_fmop4a_hh(void *vd, void *vn, void *vm, void *vinfo)
{
float16 *d = vd, *n = vn, *m = vm;
@@ -2692,6 +2735,12 @@ void HELPER(sme_fmop4a_ss)(void *vza, void *vzn, void
*vzm,
sme_mop4(vza, vzn, vzm, fpst, desc, sizeof(float32), inner_fmop4a_ss);
}
+void HELPER(sme_ftmopa_ss)(void *vza, void *vzn, void *vzm, void *vzk,
+ float_status *fpst, uint32_t desc)
+{
+ sme_tmop(vza, vzn, vzm, vzk, fpst, desc, MO_32, inner_fmop4a_ss);
+}
+
static void inner_fmop4s_ss(void *vd, void *vn, void *vm, void *vinfo)
{
float32 *d = vd, *n = vn, *m = vm;
diff --git a/target/arm/tcg/translate-sme.c b/target/arm/tcg/translate-sme.c
index dcc4690fba..36a0b00089 100644
--- a/target/arm/tcg/translate-sme.c
+++ b/target/arm/tcg/translate-sme.c
@@ -31,6 +31,12 @@
* Include the generated decoder.
*/
+static int expand_tmop_zk(DisasContext *s, int x)
+{
+ /* Pseudocode for 1:K:1:zk. */
+ return 0b10100 | ((x & 4) << 1) | (x & 3);
+}
+
#include "decode-sme.c.inc"
static bool sme2_zt0_enabled_check(DisasContext *s)
@@ -2132,3 +2138,23 @@ TRANS_FEAT(USMOP4_sb, aa64_sme_mop4, do_mop4_int, a,
MO_32,
a->s ? gen_helper_sme_usmop4s_sb : gen_helper_sme_usmop4a_sb)
TRANS_FEAT(USMOP4_dh, aa64_sme_mop4_i16i64, do_mop4_int, a, MO_64,
a->s ? gen_helper_sme_usmop4s_dh : gen_helper_sme_usmop4a_dh)
+
+static bool do_tmop_fp(DisasContext *s, arg_tmop *a, MemOp esz,
+ int e_fpst, gen_helper_gvec_4_ptr *fn)
+{
+ if (sme_smza_enabled_check(s)) {
+ int svl = streaming_vec_reg_size(s);
+ uint32_t desc = simd_desc(svl, svl, a->idx);
+ TCGv_ptr za = get_tile(s, esz, a->zad);
+ TCGv_ptr zn = vec_full_reg_ptr(s, a->zn);
+ TCGv_ptr zm = vec_full_reg_ptr(s, a->zm);
+ TCGv_ptr zk = vec_full_reg_ptr(s, a->zm);
+ TCGv_ptr fpst = (e_fpst >= 0 ? fpstatus_ptr(e_fpst) : tcg_env);
+
+ fn(za, zn, zm, zk, fpst, tcg_constant_i32(desc));
+ }
+ return true;
+}
+
+TRANS_FEAT(FTMOPA_ss, aa64_sme_tmop, do_tmop_fp,
+ a, MO_32, FPST_ZA, gen_helper_sme_ftmopa_ss)
diff --git a/target/arm/tcg/sme.decode b/target/arm/tcg/sme.decode
index 53e7e67feb..0250177d0d 100644
--- a/target/arm/tcg/sme.decode
+++ b/target/arm/tcg/sme.decode
@@ -1131,3 +1131,13 @@ UMOP4_dh 1010 0001 111. ...0 0000 00.. ..0. 1...
@mop4_o3
USMOP4_sb 1000 0001 000. ...0 1000 00.. ..0. 00.. @mop4_o2
USMOP4_dh 1010 0001 110. ...0 0000 00.. ..0. 1... @mop4_o3
+
+# SME TMOP Sparse outer products
+
+&tmop zad zn zm zk idx
+%tmop_zk 10:3 !function=expand_tmop_zk
+
+@tmop_o2 .... .... ... zm:5 ... ... .... idx:2 .. zad:2 \
+ &tmop zn=%zn_ax2 zk=%tmop_zk
+
+FTMOPA_ss 1000 0000 010 ..... 000 ... .... .. 00 .. @tmop_o2
--
2.43.0
