Split out the tail of vext_page_ldst_us, after probing. Merge vext_continuous_ldst_host into vext_page_ldst_us.
Signed-off-by: Richard Henderson <[email protected]> --- target/riscv/tcg/vector_helper.c | 101 ++++++++++++++++--------------- 1 file changed, 51 insertions(+), 50 deletions(-) diff --git a/target/riscv/tcg/vector_helper.c b/target/riscv/tcg/vector_helper.c index 18840336cb..505f0331f2 100644 --- a/target/riscv/tcg/vector_helper.c +++ b/target/riscv/tcg/vector_helper.c @@ -275,33 +275,6 @@ GEN_VEXT_HOST_ST_ELEM(ste_h, uint16_t, H2, stw_le) GEN_VEXT_HOST_ST_ELEM(ste_w, uint32_t, H4, stl_le) GEN_VEXT_HOST_ST_ELEM(ste_d, uint64_t, H8, stq_le) -static inline QEMU_ALWAYS_INLINE void -vext_continuous_ldst_host(CPURISCVState *env, vext_ldst_elem_fn_host *ldst_host, - void *vd, uint32_t evl, uint32_t reg_start, void *host, - uint32_t esz, bool is_load) -{ - if (HOST_BIG_ENDIAN) { - for (; reg_start < evl; reg_start++, host += esz) { - ldst_host(vd, reg_start, host); - } - } else { - if (esz == 1) { - uint32_t byte_offset = reg_start * esz; - uint32_t size = (evl - reg_start) * esz; - - if (is_load) { - memcpy(vd + byte_offset, host, size); - } else { - memcpy(host, vd + byte_offset, size); - } - } else { - for (; reg_start < evl; reg_start++, host += esz) { - ldst_host(vd, reg_start, host); - } - } - } -} - static void vext_set_tail_elems_1s(uint32_t vl, void *vd, uint32_t desc, uint32_t nf, uint32_t esz, uint32_t max_elems) @@ -411,6 +384,52 @@ GEN_VEXT_ST_STRIDE(vsse64_v, int64_t, ste_d_tlb) */ /* unmasked unit-stride load and store operation */ +static inline QEMU_ALWAYS_INLINE void +vext_page_ldst_us_host(void *vd, void *host, uint32_t i, uint32_t evl, + uint32_t nf, uint32_t log2_esz, uint32_t max_elems, + vext_ldst_elem_fn_host *ldst_host, bool is_load) +{ + uint32_t esz = 1 << log2_esz; + + if (nf == 1) { + if (!HOST_BIG_ENDIAN && esz == 1) { + size_t size = evl - i; + if (is_load) { + memcpy(vd + i, host, size); + } else { + memcpy(host, vd + i, size); + } + } else { + do { + ldst_host(vd, i, host); + host += esz; + } while (++i < evl); + } + } else { + uint32_t msize = nf << log2_esz; + do { + vext_ldst_nf_host(vd, host, i, nf, esz, max_elems, ldst_host); + host += msize; + } while (++i < evl); + } +} + +static inline QEMU_ALWAYS_INLINE void +vext_page_ldst_us_tlb(CPURISCVState *env, void *vd, target_ulong addr, + uint32_t i, uint32_t evl, uint32_t nf, + uint32_t log2_esz, uint32_t max_elems, + vext_ldst_elem_fn_tlb *ldst_tlb, + int mmu_index, uintptr_t ra) +{ + uint32_t esz = 1 << log2_esz; + uint32_t msize = nf << log2_esz; + do { + vext_ldst_nf_tlb(env, vd, addr, i, nf, esz, max_elems, ldst_tlb, ra); + addr += msize; + env->vstart = ++i; + } while (i < evl); +} + static inline QEMU_ALWAYS_INLINE void vext_page_ldst_us(CPURISCVState *env, void *vd, target_ulong addr, uint32_t elems, uint32_t nf, uint32_t max_elems, @@ -420,39 +439,21 @@ vext_page_ldst_us(CPURISCVState *env, void *vd, target_ulong addr, { void *host; int flags; - uint32_t esz = 1 << log2_esz; uint32_t size = (elems * nf) << log2_esz; - uint32_t msize = nf * esz; - uint32_t evl = env->vstart + elems; + uint32_t i = env->vstart; MMUAccessType access_type = is_load ? MMU_DATA_LOAD : MMU_DATA_STORE; - /* - * Maximum vector length is VLMAX == 2^16 == LMUL * VL / SEW, and - * occurs for LMUL == 8, SEW == 8, VL == 2^16. - */ - g_assert(env->vstart < UINT16_MAX && UINT16_MAX - env->vstart >= elems); - /* Check page permission/pmp/watchpoint/etc. */ probe_pages(env, addr, size, ra, access_type, mmu_index, &host, &flags, true); if (flags == 0) { - if (nf == 1) { - vext_continuous_ldst_host(env, ldst_host, vd, evl, env->vstart, - host, esz, is_load); - } else { - for (uint32_t i = env->vstart; i < evl; ++i) { - vext_ldst_nf_host(vd, host, i, nf, esz, max_elems, ldst_host); - host += msize; - } - } + vext_page_ldst_us_host(vd, host, i, i + elems, nf, log2_esz, + max_elems, ldst_host, is_load); env->vstart += elems; } else { - for (uint32_t i = env->vstart; i < evl; env->vstart = ++i) { - vext_ldst_nf_tlb(env, vd, addr, i, nf, esz, - max_elems, ldst_tlb, ra); - addr += msize; - } + vext_page_ldst_us_tlb(env, vd, addr, i, i + elems, nf, log2_esz, + max_elems, ldst_tlb, mmu_index, ra); } } -- 2.43.0
