On Thu, 2026-08-20 at 21:03 -0700, Richard Henderson wrote: > Avoids continually looking up mmu_index with every operation. > > Signed-off-by: Richard Henderson <[email protected]>
Reviewed-by: Alistair Francis <[email protected]> Alistair > --- > target/riscv/tcg/vector_helper.c | 91 +++++++++++++++--------------- > -- > 1 file changed, 43 insertions(+), 48 deletions(-) > > diff --git a/target/riscv/tcg/vector_helper.c > b/target/riscv/tcg/vector_helper.c > index 98a0a5d025..f65da46144 100644 > --- a/target/riscv/tcg/vector_helper.c > +++ b/target/riscv/tcg/vector_helper.c > @@ -171,32 +171,20 @@ static inline MemOpIdx vext_oi(uint32_t desc, > MemOp memop) > return make_memop_idx(memop, FIELD_EX32(data, VDATA, MEM_IDX)); > } > > -static inline MemOpIdx vext_make_memop_idx(CPURISCVState *env, > size_t size) > -{ > - int mmu_idx = riscv_env_mmu_index(env, false); > - MemOp memop = size_memop(size) | mo_endian_env(env); > - > - if (!riscv_cpu_cfg(env)->ext_zicclsm) { > - memop |= MO_ALIGN; > - } > - > - return make_memop_idx(memop, mmu_idx); > -} > - > /* elements operations for load and store */ > typedef void vext_ldst_elem_fn_tlb(CPURISCVState *env, abi_ptr addr, > - uint32_t idx, void *vd, uintptr_t > retaddr); > + uint32_t idx, void *vd, > + MemOpIdx oi, uintptr_t retaddr); > typedef void vext_ldst_elem_fn_host(void *vd, uint32_t idx, void > *host); > > -#define GEN_VEXT_TLB_LD_ELEM(NAME, ETYPE, H, LDSUF) \ > -static inline QEMU_ALWAYS_INLINE \ > -void NAME##_tlb(CPURISCVState *env, abi_ptr addr, \ > - uint32_t idx, void *vd, uintptr_t retaddr) \ > -{ \ > - ETYPE *cur = ((ETYPE *)vd + H(idx)); \ > - MemOpIdx oi = vext_make_memop_idx(env, sizeof(ETYPE)); \ > - *cur = cpu_##LDSUF##_mmu(env, addr, oi, retaddr); \ > -} \ > +#define GEN_VEXT_TLB_LD_ELEM(NAME, ETYPE, H, > LDSUF) \ > +static inline > QEMU_ALWAYS_INLINE \ > +void NAME##_tlb(CPURISCVState *env, abi_ptr addr, uint32_t > idx, \ > + void *vd, MemOpIdx oi, uintptr_t > retaddr) \ > +{ > \ > + ETYPE *cur = ((ETYPE *)vd + > H(idx)); \ > + *cur = cpu_##LDSUF##_mmu(env, addr, oi, > retaddr); \ > +} > > #define GEN_VEXT_HOST_LD_ELEM(NAME, ETYPE, H, LDSUF) \ > static inline QEMU_ALWAYS_INLINE \ > @@ -216,15 +204,14 @@ GEN_VEXT_HOST_LD_ELEM(lde_h, uint16_t, H2, > lduw_le) > GEN_VEXT_HOST_LD_ELEM(lde_w, uint32_t, H4, ldl_le) > GEN_VEXT_HOST_LD_ELEM(lde_d, uint64_t, H8, ldq_le) > > -#define GEN_VEXT_TLB_ST_ELEM(NAME, ETYPE, H, STSUF) \ > -static inline QEMU_ALWAYS_INLINE \ > -void NAME##_tlb(CPURISCVState *env, abi_ptr addr, \ > - uint32_t idx, void *vd, uintptr_t retaddr) \ > -{ \ > - ETYPE data = *((ETYPE *)vd + H(idx)); \ > - MemOpIdx oi = vext_make_memop_idx(env, sizeof(ETYPE)); \ > - cpu_##STSUF##_mmu(env, addr, data, oi, retaddr); \ > -} \ > +#define GEN_VEXT_TLB_ST_ELEM(NAME, ETYPE, H, > STSUF) \ > +static inline > QEMU_ALWAYS_INLINE \ > +void NAME##_tlb(CPURISCVState *env, abi_ptr addr, uint32_t > idx, \ > + void *vd, MemOpIdx oi, uintptr_t > retaddr) \ > +{ > \ > + ETYPE data = *((ETYPE *)vd + > H(idx)); \ > + cpu_##STSUF##_mmu(env, addr, data, oi, > retaddr); \ > +} > > #define GEN_VEXT_HOST_ST_ELEM(NAME, ETYPE, H, STSUF) \ > static inline QEMU_ALWAYS_INLINE \ > @@ -282,10 +269,11 @@ static void vext_ldst_nf_host(void *vd, void > *host, uint32_t i, uint32_t nf, > static void vext_ldst_nf_tlb(CPURISCVState *env, void *vd, > target_ulong addr, > uint32_t i, uint32_t nf, > uint32_t esz, uint32_t max_elems, > - vext_ldst_elem_fn_tlb *ldst_tlb, > uintptr_t ra) > + vext_ldst_elem_fn_tlb *ldst_tlb, > + MemOpIdx oi, uintptr_t ra) > { > for (uint32_t k = 0; k < nf; k++, addr += esz) { > - ldst_tlb(env, adjust_addr(env, addr), i + k * max_elems, vd, > ra); > + ldst_tlb(env, adjust_addr(env, addr), i + k * max_elems, vd, > oi, ra); > } > } > > @@ -302,13 +290,14 @@ vext_ldst_stride(void *vd, void *v0, > target_ulong base, target_ulong stride, > uint32_t max_elems = vext_max_elems(desc, log2_esz); > uint32_t esz = 1 << log2_esz; > uint32_t vma = vext_vma(desc); > + MemOpIdx oi = vext_oi(desc, log2_esz); > > VSTART_CHECK_EARLY_EXIT(env, env->vl); > > for (uint32_t i = env->vstart; i < env->vl; env->vstart = ++i) { > if (vm || vext_elem_mask(v0, i)) { > vext_ldst_nf_tlb(env, vd, base + stride * i, i, nf, > - esz, max_elems, ldst_elem, ra); > + esz, max_elems, ldst_elem, oi, ra); > } else if (vma) { > vext_set_nf_elems_1s(vd, i, nf, esz, max_elems); > } > @@ -388,12 +377,13 @@ vext_page_ldst_us_tlb(CPURISCVState *env, void > *vd, target_ulong addr, > uint32_t i, uint32_t evl, uint32_t nf, > uint32_t log2_esz, uint32_t max_elems, > vext_ldst_elem_fn_tlb *ldst_tlb, > - int mmu_index, uintptr_t ra) > + MemOpIdx oi, uintptr_t ra) > { > uint32_t esz = 1 << log2_esz; > uint32_t msize = nf << log2_esz; > do { > - vext_ldst_nf_tlb(env, vd, addr, i, nf, esz, max_elems, > ldst_tlb, ra); > + vext_ldst_nf_tlb(env, vd, addr, i, nf, esz, > + max_elems, ldst_tlb, oi, ra); > addr += msize; > env->vstart = ++i; > } while (i < evl); > @@ -443,7 +433,7 @@ vext_ldst_us_notail(void *vd, target_ulong base, > CPURISCVState *env, > */ > if (nf == 1 && (evl << log2_esz) <= 6) { > for (; i < evl; env->vstart = ++i, addr += esz) { > - ldst_tlb(env, adjust_addr(env, addr), i, vd, ra); > + ldst_tlb(env, adjust_addr(env, addr), i, vd, oi, ra); > } > return; > } > @@ -472,7 +462,7 @@ vext_ldst_us_notail(void *vd, target_ulong base, > CPURISCVState *env, > } else { > vext_page_ldst_us_tlb(env, vd, addr, i, page_evl, nf, > log2_esz, max_elems, > - ldst_tlb, mmu_index, ra); > + ldst_tlb, oi, ra); > } > > /* Usually the first page contains the entire vector. */ > @@ -485,7 +475,8 @@ vext_ldst_us_notail(void *vd, target_ulong base, > CPURISCVState *env, > > /* Cross page element */ > if (unlikely(page_split % msize)) { > - vext_ldst_nf_tlb(env, vd, addr, i++, nf, esz, max_elems, > ldst_tlb, ra); > + vext_ldst_nf_tlb(env, vd, addr, i++, nf, esz, > + max_elems, ldst_tlb, oi, ra); > if (i == evl) { > return; > } > @@ -504,7 +495,7 @@ vext_ldst_us_notail(void *vd, target_ulong base, > CPURISCVState *env, > max_elems, ldst_host, is_load); > } else { > vext_page_ldst_us_tlb(env, vd, addr, i, evl, nf, log2_esz, > max_elems, > - ldst_tlb, mmu_index, ra); > + ldst_tlb, oi, ra); > } > } > > @@ -623,6 +614,7 @@ vext_ldst_index(void *vd, void *v0, target_ulong > base, > uint32_t max_elems = vext_max_elems(desc, log2_esz); > uint32_t esz = 1 << log2_esz; > uint32_t vma = vext_vma(desc); > + MemOpIdx oi = vext_oi(desc, log2_esz); > > VSTART_CHECK_EARLY_EXIT(env, env->vl); > > @@ -636,7 +628,8 @@ vext_ldst_index(void *vd, void *v0, target_ulong > base, > > for (uint32_t k = 0; k < nf; ++k) { > abi_ptr addr = get_index_addr(base, i, vs2) + (k << > log2_esz); > - ldst_elem(env, adjust_addr(env, addr), i + k * > max_elems, vd, ra); > + ldst_elem(env, adjust_addr(env, addr), > + i + k * max_elems, vd, oi, ra); > } > } > > @@ -753,7 +746,8 @@ vext_ldff(void *vd, void *v0, target_ulong base, > CPURISCVState *env, > * If it also crosses the page, just go ahead and load it. > */ > if (unlikely(page_split < msize) && first_active) { > - vext_ldst_nf_tlb(env, vd, addr, 0, nf, esz, max_elems, > ldst_tlb, ra); > + vext_ldst_nf_tlb(env, vd, addr, 0, nf, esz, > + max_elems, ldst_tlb, oi, ra); > env->vstart = i = 1; > addr += msize; > page_split = -(addr | TARGET_PAGE_MASK); > @@ -798,7 +792,7 @@ vext_ldff(void *vd, void *v0, target_ulong base, > CPURISCVState *env, > */ > if (first_active) { > vext_ldst_nf_tlb(env, vd, addr, 0, nf, esz, > - max_elems, ldst_tlb, ra); > + max_elems, ldst_tlb, oi, ra); > i = 1; > addr += msize; > } > @@ -813,12 +807,12 @@ vext_ldff(void *vd, void *v0, target_ulong > base, CPURISCVState *env, > if (vm) { > vext_page_ldst_us_tlb(env, vd, addr, i, page_evl, > nf, > log2_esz, max_elems, > - ldst_tlb, mmu_index, ra); > + ldst_tlb, oi, ra); > } else { > do { > if (vext_elem_mask(v0, i)) { > vext_ldst_nf_tlb(env, vd, base + i * msize, > i, nf, > - esz, max_elems, ldst_tlb, > ra); > + esz, max_elems, ldst_tlb, > oi, ra); > } else if (vma) { > vext_set_nf_elems_1s(vd, i, nf, esz, > max_elems); > } > @@ -868,7 +862,8 @@ vext_ldff(void *vd, void *v0, target_ulong base, > CPURISCVState *env, > goto tail; > } > > - vext_ldst_nf_tlb(env, vd, addr, i, nf, esz, max_elems, > ldst_tlb, ra); > + vext_ldst_nf_tlb(env, vd, addr, i, nf, esz, > + max_elems, ldst_tlb, oi, ra); > if (++i == vl) { > goto tail; > } > @@ -909,12 +904,12 @@ vext_ldff(void *vd, void *v0, target_ulong > base, CPURISCVState *env, > if (vm) { > vext_page_ldst_us_tlb(env, vd, addr, i, vl, nf, > log2_esz, max_elems, > - ldst_tlb, mmu_index, ra); > + ldst_tlb, oi, ra); > } else { > do { > if (vext_elem_mask(v0, i)) { > vext_ldst_nf_tlb(env, vd, base + i * msize, i, > nf, > - esz, max_elems, ldst_tlb, ra); > + esz, max_elems, ldst_tlb, oi, > ra); > } else if (vma) { > vext_set_nf_elems_1s(vd, i, nf, esz, max_elems); > }
