Probe only the first access of the range with probe_access_full and inspect the resulting lg_page_size: when the page is subdivided, fall back to the per-element TLB path. Pages with uniform permissions keep the direct host fast path.
Signed-off-by: Max Chou <[email protected]> --- target/riscv/tcg/vector_helper.c | 45 ++++++++++++++++++++++++++++---- 1 file changed, 40 insertions(+), 5 deletions(-) diff --git a/target/riscv/tcg/vector_helper.c b/target/riscv/tcg/vector_helper.c index efe10156daa..569da995482 100644 --- a/target/riscv/tcg/vector_helper.c +++ b/target/riscv/tcg/vector_helper.c @@ -406,6 +406,41 @@ static void vext_test_alignment(CPURISCVState *env, vaddr addr, uint32_t esz, } } +static void *vext_probe_host_page(CPURISCVState *env, target_ulong addr, + target_ulong probe_bytes, uint32_t msize, + MMUAccessType access_type, int mmu_index, + uintptr_t ra) +{ +#ifdef CONFIG_USER_ONLY + return probe_access(env, addr, probe_bytes, access_type, mmu_index, ra); +#else + CPUTLBEntryFull *full; + void *host; + int flags; + + flags = probe_access_full(env, addr, MIN(msize, probe_bytes), + access_type, mmu_index, false, &host, &full, ra); + if (flags || full->lg_page_size < TARGET_PAGE_BITS) { + return NULL; + } + + if (access_type == MMU_DATA_STORE) { + /* + * The permissions are uniform across the page, so probing the + * first access has validated the whole range. It has only + * marked MIN(msize, probe_bytes) bytes as dirty, though, while + * the caller writes probe_bytes through the returned host + * pointer. Probe the whole range as well, so that + * notdirty_write invalidates every translation block that it + * overlaps and the migration dirty bitmap covers all of it. + */ + return probe_access(env, addr, probe_bytes, access_type, + mmu_index, ra); + } + return host; +#endif +} + static void vext_ldst_us_notail(void *vd, target_ulong base, CPURISCVState *env, uint32_t log2_esz, uint32_t nf, uint32_t evl, @@ -448,9 +483,9 @@ vext_ldst_us_notail(void *vd, target_ulong base, CPURISCVState *env, page_split = -(addr | TARGET_PAGE_MASK); /* Validate the first page is accessible. */ - host = probe_access(env, adjust_addr(env, addr), - MIN(last, last_in_page) - addr + 1, - access_type, mmu_index, ra); + host = vext_probe_host_page(env, adjust_addr(env, addr), + MIN(last, last_in_page) - addr + 1, + msize, access_type, mmu_index, ra); /* Get number of complete elements in the first page. */ elems = MIN(page_split / msize, evl - i); @@ -490,8 +525,8 @@ vext_ldst_us_notail(void *vd, target_ulong base, CPURISCVState *env, /* Validate the second page is accessible. */ assert(i < evl); elems = evl - i; - host = probe_access(env, adjust_addr(env, addr), elems * msize, - access_type, mmu_index, ra); + host = vext_probe_host_page(env, adjust_addr(env, addr), elems * msize, + msize, access_type, mmu_index, ra); if (host) { vext_page_ldst_us_host(vd, host, i, evl, nf, log2_esz, -- 2.43.0
