Signed-off-by: Richard Henderson <[email protected]>
---
target/arm/tcg/helper-sme-defs.h | 2 ++
target/arm/tcg/sme_helper.c | 54 ++++++++++++++++++++++++++++++++
target/arm/tcg/translate-sme.c | 3 ++
target/arm/tcg/sme.decode | 2 ++
4 files changed, 61 insertions(+)
diff --git a/target/arm/tcg/helper-sme-defs.h b/target/arm/tcg/helper-sme-defs.h
index 224de45294..f05b53f412 100644
--- a/target/arm/tcg/helper-sme-defs.h
+++ b/target/arm/tcg/helper-sme-defs.h
@@ -408,3 +408,5 @@ DEF_HELPER_FLAGS_4(sme_usmop4s_dh, TCG_CALL_NO_RWG, void,
ptr, ptr, ptr, i32)
DEF_HELPER_FLAGS_6(sme_bftmopa_hh, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr,
fpst, i32)
DEF_HELPER_FLAGS_6(sme_ftmopa_hh, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr,
fpst, i32)
DEF_HELPER_FLAGS_6(sme_ftmopa_ss, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr,
fpst, i32)
+
+DEF_HELPER_FLAGS_6(sme_bftmopa_sh, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, ptr,
env, i32)
diff --git a/target/arm/tcg/sme_helper.c b/target/arm/tcg/sme_helper.c
index e57c56d5f2..91447d9c0e 100644
--- a/target/arm/tcg/sme_helper.c
+++ b/target/arm/tcg/sme_helper.c
@@ -2679,6 +2679,50 @@ static void sme_tmop(void *vza, void *vzn, void *vzm,
uint64_t *zk,
}
}
+/*
+ * Sparse outer product, widening 2-way, 16 to 32-bit.
+ */
+static void sme_tmop_2way_sh(uint32_t *za, uint16_t *zn0, uint32_t *zm,
+ uint64_t *zk, void *fn_opaque, uint32_t desc,
+ void (*fn)(void *, void *, void *, void *))
+{
+ intptr_t oprsz = simd_maxsz(desc);
+ intptr_t dim = oprsz >> MO_32;
+ intptr_t index = simd_data(desc);
+ intptr_t ctrl_base = (index * oprsz) >> 1;
+ uint16_t *zn1 = zn0 + sizeof(ARMVectorReg) / 2;
+
+ for (intptr_t row = 0; row < dim; row++) {
+ uint32_t *za_row = za + tile_vslice_offset(row);
+
+ for (intptr_t col = 0; col < dim; col++) {
+ uint32_t *e2 = zm + H4(col);
+ uint32_t *e3 = za_row + H4(col);
+ uint32_t e1 = 0;
+
+ /*
+ * Four control bits select two elements. The two elements
+ * may be non-contiguous, so assemble them locally into e1.
+ * Pseudo-code has a double loop running forward, with a
+ * test for (i < 2) to limit construction to 2 elements.
+ * Easier to run a single loop backward, shifting extra
+ * elements off the top of our uint32_t.
+ */
+ uint64_t this_ctrl = extractn(zk, ctrl_base + col * 4, 4);
+ for (int i = 3; i >= 0; i--) {
+ if (this_ctrl & (1 << i)) {
+ bool e = i & 1;
+ bool r = i & 2;
+ uint16_t *p = (r ? zn1 : zn0) + H2(2 * row + e);
+ e1 = (e1 << 16) | *p;
+ }
+ }
+
+ fn(e3, &e1, e2, fn_opaque);
+ }
+ }
+}
+
static void inner_fmop4a_hh(void *vd, void *vn, void *vm, void *vinfo)
{
float16 *d = vd, *n = vn, *m = vm;
@@ -2893,6 +2937,16 @@ void HELPER(sme_bfmop4a_sh)(void *vza, void *vzn, void
*vzm,
: inner_bfmop4a_sh);
}
+void HELPER(sme_bftmopa_sh)(void *vza, void *vzn, void *vzm, void *vzk,
+ CPUArchState *env, uint32_t desc)
+{
+ float_status fpst;
+
+ sme_tmop_2way_sh(vza, vzn, vzm, vzk, &fpst, desc,
+ is_ebf(env, &fpst) ? inner_ebf_bfmop4a_sh
+ : inner_bfmop4a_sh);
+}
+
static void inner_bfmop4s_sh(void *vd, void *vn, void *vm, void *vinfo)
{
float32 *d = vd;
diff --git a/target/arm/tcg/translate-sme.c b/target/arm/tcg/translate-sme.c
index 7d21ebabc3..8548bfd5c9 100644
--- a/target/arm/tcg/translate-sme.c
+++ b/target/arm/tcg/translate-sme.c
@@ -2162,3 +2162,6 @@ TRANS_FEAT(FTMOPA_hh, aa64_sme_tmop_f16f16, do_tmop_fp,
a, MO_16, FPST_ZA_F16, gen_helper_sme_ftmopa_hh)
TRANS_FEAT(FTMOPA_ss, aa64_sme_tmop, do_tmop_fp,
a, MO_32, FPST_ZA, gen_helper_sme_ftmopa_ss)
+
+TRANS_FEAT(BFTMOPA_sh, aa64_sme_tmop, do_tmop_fp,
+ a, MO_32, FPST_ENV, gen_helper_sme_bftmopa_sh)
diff --git a/target/arm/tcg/sme.decode b/target/arm/tcg/sme.decode
index f0f9b3f61d..49af8fd19e 100644
--- a/target/arm/tcg/sme.decode
+++ b/target/arm/tcg/sme.decode
@@ -1145,3 +1145,5 @@ USMOP4_dh 1010 0001 110. ...0 0000 00.. ..0. 1...
@mop4_o3
BFTMOPA_hh 1000 0001 011 ..... 000 ... .... .. 100 . @tmop_o1
FTMOPA_hh 1000 0001 010 ..... 000 ... .... .. 100 . @tmop_o1
FTMOPA_ss 1000 0000 010 ..... 000 ... .... .. 00 .. @tmop_o2
+
+BFTMOPA_sh 1000 0001 010 ..... 000 ... .... .. 00 .. @tmop_o2
--
2.43.0
