On Thu, 2026-08-20 at 21:03 -0700, Richard Henderson wrote:
> We already computed the page split here, which means we
> were quite ready to call probe_access directly. Do so,
> then use vext_ldst_nf_{host,tlb} to load the elements.
>
> Signed-off-by: Richard Henderson <[email protected]>
Reviewed-by: Alistair Francis <[email protected]>
Alistair
> ---
> target/riscv/tcg/vector_helper.c | 88 ++++++++++++++++++++----------
> --
> 1 file changed, 56 insertions(+), 32 deletions(-)
>
> diff --git a/target/riscv/tcg/vector_helper.c
> b/target/riscv/tcg/vector_helper.c
> index 505f0331f2..36e64465f2 100644
> --- a/target/riscv/tcg/vector_helper.c
> +++ b/target/riscv/tcg/vector_helper.c
> @@ -477,17 +477,19 @@ vext_ldst_us(void *vd, target_ulong base,
> CPURISCVState *env, uint32_t desc,
> vext_ldst_elem_fn_host *ldst_host, uint32_t log2_esz,
> uint32_t evl, uintptr_t ra, bool is_load)
> {
> - target_ulong page_split, elems, addr;
> + target_ulong elems, addr, last, last_in_page, page_split;
> uint32_t nf = vext_nf(desc);
> uint32_t max_elems = vext_max_elems(desc, log2_esz);
> uint32_t esz = 1 << log2_esz;
> uint32_t msize = nf * esz;
> int mmu_index = riscv_env_mmu_index(env, false);
> MMUAccessType access_type = is_load ? MMU_DATA_LOAD :
> MMU_DATA_STORE;
> + uint32_t i = env->vstart;
> + void *host;
>
> VSTART_CHECK_EARLY_EXIT(env, evl);
>
> - addr = base + env->vstart * msize;
> + addr = base + i * msize;
>
> /* Recognize alignment fault before memory protection fault. */
> vext_test_alignment(env, addr, esz, access_type, mmu_index, ra);
> @@ -498,50 +500,72 @@ vext_ldst_us(void *vd, target_ulong base,
> CPURISCVState *env, uint32_t desc,
> * by simply calling ldst_tlb.
> */
> if (nf == 1 && (evl << log2_esz) <= 6) {
> - for (uint32_t i = env->vstart; i < evl;
> - env->vstart = ++i, addr += esz) {
> + for (; i < evl; env->vstart = ++i, addr += esz) {
> ldst_tlb(env, adjust_addr(env, addr), i, vd, ra);
> }
> - env->vstart = 0;
> - vext_set_tail_elems_1s(evl, vd, desc, nf, esz, max_elems);
> - return;
> + goto tail;
> }
> #endif
>
> - /* Calculate the page range of first page */
> + /* Calculate the page range of first page. */
> + last = base + evl * msize - 1;
> + last_in_page = addr | ~TARGET_PAGE_MASK;
> page_split = -(addr | TARGET_PAGE_MASK);
> - /* Get number of elements */
> - elems = page_split / msize;
> - if (unlikely(env->vstart + elems >= evl)) {
> - elems = evl - env->vstart;
> - }
> +
> + /* Validate the first page is accessible. */
> + host = probe_access(env, adjust_addr(env, addr),
> + MIN(last, last_in_page) - addr + 1,
> + access_type, mmu_index, ra);
> +
> + /* Get number of complete elements in the first page. */
> + elems = MIN(page_split / msize, evl - i);
>
> /* Load/store elements in the first page */
> if (likely(elems)) {
> - vext_page_ldst_us(env, vd, addr, elems, nf, max_elems,
> log2_esz,
> - is_load, mmu_index, ldst_tlb, ldst_host,
> ra);
> - }
> + uint32_t page_evl = i + elems;
>
> - /* Load/store elements in the second page */
> - if (unlikely(env->vstart < evl)) {
> - addr = base + env->vstart * msize;
> -
> - /* Cross page element */
> - if (unlikely(page_split % msize)) {
> - vext_ldst_nf_tlb(env, vd, addr, env->vstart,
> - nf, esz, max_elems, ldst_tlb, ra);
> - env->vstart++;
> - addr += msize;
> + if (host) {
> + vext_page_ldst_us_host(vd, host, i, page_evl, nf,
> log2_esz,
> + max_elems, ldst_host, is_load);
> + } else {
> + vext_page_ldst_us_tlb(env, vd, addr, i, page_evl, nf,
> + log2_esz, max_elems,
> + ldst_tlb, mmu_index, ra);
> }
>
> - /* Get number of elements of second page */
> - elems = evl - env->vstart;
> -
> - /* Load/store elements in the second page */
> - vext_page_ldst_us(env, vd, addr, elems, nf, max_elems,
> log2_esz,
> - is_load, mmu_index, ldst_tlb, ldst_host,
> ra);
> + /* Usually the first page contains the entire vector. */
> + if (likely(page_evl == evl)) {
> + goto tail;
> + }
> + env->vstart = i = page_evl;
> + addr += elems * msize;
> }
>
> + /* Cross page element */
> + if (unlikely(page_split % msize)) {
> + vext_ldst_nf_tlb(env, vd, addr, i++, nf, esz, max_elems,
> ldst_tlb, ra);
> + if (i == evl) {
> + goto tail;
> + }
> + env->vstart = i;
> + addr += msize;
> + }
> +
> + /* Validate the second page is accessible. */
> + assert(i < evl);
> + elems = evl - i;
> + host = probe_access(env, adjust_addr(env, addr), elems * msize,
> + access_type, mmu_index, ra);
> +
> + if (host) {
> + vext_page_ldst_us_host(vd, host, i, evl, nf, log2_esz,
> + max_elems, ldst_host, is_load);
> + } else {
> + vext_page_ldst_us_tlb(env, vd, addr, i, evl, nf, log2_esz,
> max_elems,
> + ldst_tlb, mmu_index, ra);
> + }
> +
> + tail:
> env->vstart = 0;
> vext_set_tail_elems_1s(evl, vd, desc, nf, esz, max_elems);
> }
