From: Richard Henderson <[email protected]>
Reviewed-by: Peter Maydell <[email protected]>
Signed-off-by: Richard Henderson <[email protected]>
Message-id: [email protected]
Signed-off-by: Peter Maydell <[email protected]>
---
target/arm/cpu-features.h | 5 +++
target/arm/tcg/helper-sme-defs.h | 4 ++
target/arm/tcg/sme.decode | 12 +++++
target/arm/tcg/sme_helper.c | 76 ++++++++++++++++++++++++++++++++
target/arm/tcg/translate-sme.c | 29 ++++++++++++
target/arm/tcg/translate.h | 5 +++
target/arm/tcg/vec_internal.h | 8 ++++
7 files changed, 139 insertions(+)
diff --git a/target/arm/cpu-features.h b/target/arm/cpu-features.h
index de81c4f103..e480995cc6 100644
--- a/target/arm/cpu-features.h
+++ b/target/arm/cpu-features.h
@@ -1585,6 +1585,11 @@ static inline bool isar_feature_aa64_ssve_fexpa(const
ARMISARegisters *id)
return FIELD_EX64_IDREG(id, ID_AA64SMFR0, SFEXPA);
}
+static inline bool isar_feature_aa64_sme_mop4(const ARMISARegisters *id)
+{
+ return FIELD_EX64_IDREG(id, ID_AA64SMFR0, SMOP4);
+}
+
static inline bool isar_feature_aa64_ssve_aes(const ARMISARegisters *id)
{
return FIELD_EX64_IDREG(id, ID_AA64SMFR0, AES);
diff --git a/target/arm/tcg/helper-sme-defs.h b/target/arm/tcg/helper-sme-defs.h
index 01aad4c231..9e1f09ce5b 100644
--- a/target/arm/tcg/helper-sme-defs.h
+++ b/target/arm/tcg/helper-sme-defs.h
@@ -355,3 +355,7 @@ DEF_HELPER_FLAGS_5(sme2_sel_b, TCG_CALL_NO_RWG, void, ptr,
ptr, ptr, i32, i32)
DEF_HELPER_FLAGS_5(sme2_sel_h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32, i32)
DEF_HELPER_FLAGS_5(sme2_sel_s, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32, i32)
DEF_HELPER_FLAGS_5(sme2_sel_d, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32, i32)
+
+DEF_HELPER_FLAGS_5(sme_fmop4a_ss, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, fpst,
i32)
+DEF_HELPER_FLAGS_5(sme_fmop4s_ss, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, fpst,
i32)
+DEF_HELPER_FLAGS_5(sme_ah_fmop4s_ss, TCG_CALL_NO_RWG, void, ptr, ptr, ptr,
fpst, i32)
diff --git a/target/arm/tcg/sme.decode b/target/arm/tcg/sme.decode
index 3a65e1ad4b..5a3d47cdfe 100644
--- a/target/arm/tcg/sme.decode
+++ b/target/arm/tcg/sme.decode
@@ -1086,3 +1086,15 @@ LUTI4_s_4h 1100 0000 1001 101 idx:1 10 01 00 zn:5
zd:5 &lut
LUTI4_s_4b 1100 0000 1001 101 1 00 00 00 ....0 zd:5 \
&lut zn=%zn_ax2 idx=0
+
+# SME MOP4 Quarter-tile outer products
+
+&mop4 zad zn zm s:bool n:bool m:bool
+
+%mop4_zm 17:3 !function=times_2_plus_16
+%mop4_zn 6:3 !function=times_2
+
+@mop4_o2 .... .... ... m:1 .... .... .. n:1 ... . s:1 .. zad:2 \
+ &mop4 zm=%mop4_zm zn=%mop4_zn
+
+FMOP4_ss 1000 0000 000. ...0 0000 00.. ..0. 00.. @mop4_o2
diff --git a/target/arm/tcg/sme_helper.c b/target/arm/tcg/sme_helper.c
index 685b6b46be..b08d91e6de 100644
--- a/target/arm/tcg/sme_helper.c
+++ b/target/arm/tcg/sme_helper.c
@@ -2601,3 +2601,79 @@ void HELPER(sme2_sel_d)(void *vd, void *vn, void *vm,
}
}
}
+
+void sme_mop4(void *vza, void *vzn, void *vzm, void *fn_opaque,
+ uint32_t desc, size_t esize,
+ void (*fn)(void *, void *, void *, void *))
+{
+ intptr_t oprsz = simd_maxsz(desc);
+ intptr_t dim = oprsz / 2; /* in bytes */
+ bool nreg_m1 = extract32(desc, SIMD_DATA_SHIFT + 0, 1);
+ bool mreg_m1 = extract32(desc, SIMD_DATA_SHIFT + 1, 1);
+ intptr_t host_adj = HOST_BIG_ENDIAN ? 8 - esize : 0;
+
+ for (int outprod = 0; outprod < 4; outprod++) {
+ bool row_hv = outprod & 2;
+ bool col_hv = outprod & 1;
+ intptr_t row_base = row_hv ? dim : 0;
+ intptr_t col_base = col_hv ? dim : 0;
+ void *op1 = vzn + (col_hv && nreg_m1 ? sizeof(ARMVectorReg) : 0);
+ void *op2 = vzm + (row_hv && mreg_m1 ? sizeof(ARMVectorReg) : 0);
+
+ for (intptr_t row = 0; row < dim; row += esize) {
+ intptr_t row_idx = row_base + row;
+ void *vza_row = vza + tile_vslice_offset(row_idx);
+ void *e1 = op1 + (row_idx ^ host_adj);
+
+ for (intptr_t col = 0; col < dim; col += esize) {
+ intptr_t col_idx = col_base + col;
+ void *e2 = op2 + (col_idx ^ host_adj);
+ void *e3 = vza_row + (col_idx ^ host_adj);
+
+ fn(e3, e1, e2, fn_opaque);
+ }
+ }
+ }
+}
+
+static void inner_fmop4a_ss(void *vd, void *vn, void *vm, void *vinfo)
+{
+ float32 *d = vd, *n = vn, *m = vm;
+ float_status *fpst = vinfo;
+
+ *d = float32_muladd(*n, *m, *d, 0, fpst);
+}
+
+void HELPER(sme_fmop4a_ss)(void *vza, void *vzn, void *vzm,
+ float_status *fpst, uint32_t desc)
+{
+ sme_mop4(vza, vzn, vzm, fpst, desc, sizeof(float32), inner_fmop4a_ss);
+}
+
+static void inner_fmop4s_ss(void *vd, void *vn, void *vm, void *vinfo)
+{
+ float32 *d = vd, *n = vn, *m = vm;
+ float_status *fpst = vinfo;
+
+ *d = float32_muladd(float32_chs(*n), *m, *d, 0, fpst);
+}
+
+void HELPER(sme_fmop4s_ss)(void *vza, void *vzn, void *vzm,
+ float_status *fpst, uint32_t desc)
+{
+ sme_mop4(vza, vzn, vzm, fpst, desc, sizeof(float32), inner_fmop4s_ss);
+}
+
+static void inner_ah_fmop4s_ss(void *vd, void *vn, void *vm, void *vinfo)
+{
+ float32 *d = vd, *n = vn, *m = vm;
+ float_status *fpst = vinfo;
+
+ *d = float32_muladd(*n, *m, *d, float_muladd_negate_product, fpst);
+}
+
+void HELPER(sme_ah_fmop4s_ss)(void *vza, void *vzn, void *vzm,
+ float_status *fpst, uint32_t desc)
+{
+ sme_mop4(vza, vzn, vzm, fpst, desc, sizeof(float32), inner_ah_fmop4s_ss);
+}
diff --git a/target/arm/tcg/translate-sme.c b/target/arm/tcg/translate-sme.c
index ff5554eefb..aab5eab9df 100644
--- a/target/arm/tcg/translate-sme.c
+++ b/target/arm/tcg/translate-sme.c
@@ -2002,3 +2002,32 @@ TRANS_FEAT(LUTI4_s_4h, aa64_sme2p1, do_lut_s4, a,
gen_helper_sme2_luti4_4h)
TRANS_FEAT(LUTI4_s_4b, aa64_sme2p1_lutv2, do_lut_s4, a,
gen_helper_sme2_luti4_4b)
+
+static bool do_mop4_fp(DisasContext *s, arg_mop4 *a, MemOp esz,
+ ARMFPStatusFlavour e_fpst,
+ gen_helper_gvec_3_ptr * const fns[3])
+{
+ int svl = streaming_vec_reg_size(s);
+ uint32_t desc = simd_desc(svl, svl, (a->m << 1) | a->n);
+ int fns_idx = (a->s ? 1 + s->fpcr_ah : 0);
+ TCGv_ptr za, zn, zm, fpst;
+
+ if (!sme_smza_enabled_check(s)) {
+ return true;
+ }
+
+ za = get_tile(s, esz, a->zad);
+ zn = vec_full_reg_ptr(s, a->zn);
+ zm = vec_full_reg_ptr(s, a->zm);
+ fpst = fpstatus_ptr(e_fpst);
+
+ fns[fns_idx](za, zn, zm, fpst, tcg_constant_i32(desc));
+ return true;
+}
+
+static gen_helper_gvec_3_ptr * const fmop4_ss[3] = {
+ gen_helper_sme_fmop4a_ss,
+ gen_helper_sme_fmop4s_ss,
+ gen_helper_sme_ah_fmop4s_ss
+};
+TRANS_FEAT(FMOP4_ss, aa64_sme_mop4, do_mop4_fp, a, MO_32, FPST_ZA, fmop4_ss)
diff --git a/target/arm/tcg/translate.h b/target/arm/tcg/translate.h
index 83b413ee36..a3d03159ad 100644
--- a/target/arm/tcg/translate.h
+++ b/target/arm/tcg/translate.h
@@ -268,6 +268,11 @@ static inline int times_2_plus_1(DisasContext *s, int x)
return x * 2 + 1;
}
+static inline int times_2_plus_16(DisasContext *s, int x)
+{
+ return x * 2 + 16;
+}
+
static inline int rsub_64(DisasContext *s, int x)
{
return 64 - x;
diff --git a/target/arm/tcg/vec_internal.h b/target/arm/tcg/vec_internal.h
index 77e5b01550..038a2a3439 100644
--- a/target/arm/tcg/vec_internal.h
+++ b/target/arm/tcg/vec_internal.h
@@ -547,4 +547,12 @@ void HELPER(NAME)(void *vd, void *vn, void *vm,
\
clear_tail(d, oprsz, simd_maxsz(desc)); \
}
+/*
+ * Perform SME quarter-tile outer product.
+ * Iterate over ZAtile[] for esize, calling fn for each element.
+ */
+void sme_mop4(void *vza, void *vzn, void *vzm, void *fn_opaque,
+ uint32_t desc, size_t esize,
+ void (*fn)(void *, void *, void *, void *));
+
#endif /* TARGET_ARM_VEC_INTERNAL_H */
--
2.43.0
