Do not call probe_pages for every active element. We can make do with no more than 2 such calls for the two pages the insn might reference.
Signed-off-by: Richard Henderson <[email protected]> --- target/riscv/tcg/vector_helper.c | 256 +++++++++++++++++++++---------- 1 file changed, 171 insertions(+), 85 deletions(-) diff --git a/target/riscv/tcg/vector_helper.c b/target/riscv/tcg/vector_helper.c index 36e64465f2..abfd2ec45d 100644 --- a/target/riscv/tcg/vector_helper.c +++ b/target/riscv/tcg/vector_helper.c @@ -748,135 +748,221 @@ vext_ldff(void *vd, void *v0, target_ulong base, CPURISCVState *env, uint32_t desc, vext_ldst_elem_fn_tlb *ldst_tlb, vext_ldst_elem_fn_host *ldst_host, uint32_t log2_esz, uintptr_t ra) { - uint32_t i, vl = 0; + uint32_t i = env->vstart, vl = env->vl; uint32_t nf = vext_nf(desc); uint32_t vm = vext_vm(desc); uint32_t max_elems = vext_max_elems(desc, log2_esz); uint32_t esz = 1 << log2_esz; uint32_t msize = nf * esz; uint32_t vma = vext_vma(desc); - target_ulong addr, addr_i, offset, remain, page_split, elems; + target_ulong addr, last, last_in_page, page_split, elems; int mmu_index = riscv_env_mmu_index(env, false); + bool first_active; int flags; void *host; VSTART_CHECK_EARLY_EXIT(env, env->vl); - /* Search for the first active element. */ - if (!vm) { - for (i = env->vstart; i < env->vl; ++i) { + /* Skip forward to the first active element. */ + if (vm) { + first_active = i == 0; + } else if (i == 0 && vext_elem_mask(v0, 0)) { + first_active = true; + } else { + first_active = false; + while (1) { if (vext_elem_mask(v0, i)) { break; } if (vma) { vext_set_nf_elems_1s(vd, i, nf, esz, max_elems); } + if (++i == vl) { + goto tail; + } } - if (i == env->vl) { - goto tail; - } - env->vstart = i; } - - addr = base + ((env->vstart * nf) << log2_esz); + addr = base + i * msize; /* Recognize alignment fault before memory protection fault. */ vext_test_alignment(env, addr, esz, MMU_DATA_LOAD, mmu_index, ra); + /* Calculate the page range of first page. */ + last = base + vl * msize - 1; + last_in_page = addr | ~TARGET_PAGE_MASK; page_split = -(addr | TARGET_PAGE_MASK); - /* Get number of elements */ - elems = page_split / msize; - if (unlikely(env->vstart + elems >= env->vl)) { - elems = env->vl - env->vstart; + + /* + * If the first element is active, it must succeed. + * If it also crosses the page, just go ahead and load it. + */ + if (unlikely(page_split < msize) && first_active) { + vext_ldst_nf_tlb(env, vd, addr, 0, nf, esz, max_elems, ldst_tlb, ra); + env->vstart = i = 1; + addr += msize; + page_split = -(addr | TARGET_PAGE_MASK); + first_active = false; } - /* Check page permission/pmp/watchpoint/etc. */ - probe_pages(env, addr, (env->vl - env->vstart) * msize, ra, MMU_DATA_LOAD, - mmu_index, &host, &flags, true); + /* + * Test whether the first page is accessible. + * If the first element is active, it must succeed. + */ + flags = probe_access_flags(env, adjust_addr(env, addr), + MIN(last, last_in_page) - addr + 1, + MMU_DATA_LOAD, mmu_index, !first_active, + &host, ra); - if (flags & ~TLB_WATCHPOINT) { - /* probe every access */ - for (i = env->vstart; i < env->vl; i++) { - if (!vm && !vext_elem_mask(v0, i)) { - continue; - } - addr_i = adjust_addr(env, base + i * (nf << log2_esz)); - if (i == 0) { - /* Allow fault on first element. */ - probe_pages(env, addr_i, nf << log2_esz, ra, MMU_DATA_LOAD, - mmu_index, &host, NULL, false); + /* Get number of complete elements in the first page. */ + elems = MIN(page_split / msize, vl - i); + + /* Load complete elements from the first page. */ + if (likely(elems)) { + uint32_t page_evl = i + elems; + + if (flags == 0) { + if (vm) { + vext_page_ldst_us_host(vd, host, i, page_evl, nf, log2_esz, + max_elems, ldst_host, true); } else { - remain = nf << log2_esz; - while (remain > 0) { - offset = -(addr_i | TARGET_PAGE_MASK); - - /* Probe nonfault on subsequent elements. */ - probe_pages(env, addr_i, offset, 0, MMU_DATA_LOAD, - mmu_index, &host, &flags, true); - - /* - * Stop if invalid (unmapped) or mmio (transaction may - * fail). Do not stop if watchpoint, as the spec says that - * first-fault should continue to access the same - * elements regardless of any watchpoint. - */ - if (flags & ~TLB_WATCHPOINT) { - vl = i; - goto ProbeSuccess; + host -= addr - base; + do { + if (vext_elem_mask(v0, i)) { + vext_ldst_nf_host(vd, host + i * msize, i, nf, esz, + max_elems, ldst_host); + } else if (vma) { + vext_set_nf_elems_1s(vd, i, nf, esz, max_elems); } - if (remain <= offset) { - break; + } while (++i < page_evl); + } + } else { + /* + * If the first element is active, it must succeed. + * This will load from MMIO or fault from INVALID. + */ + if (first_active) { + vext_ldst_nf_tlb(env, vd, addr, 0, nf, esz, + max_elems, ldst_tlb, ra); + i = 1; + addr += msize; + } + + /* Stop if invalid (unmapped) or mmio (transaction may fail). */ + if (flags & (TLB_INVALID_MASK | TLB_MMIO)) { + env->vl = i; + goto tail; + } + + /* None of these ldst_tlb calls may fault. */ + if (vm) { + vext_page_ldst_us_tlb(env, vd, addr, i, page_evl, nf, + log2_esz, max_elems, + ldst_tlb, mmu_index, ra); + } else { + do { + if (vext_elem_mask(v0, i)) { + vext_ldst_nf_tlb(env, vd, base + i * msize, i, nf, + esz, max_elems, ldst_tlb, ra); + } else if (vma) { + vext_set_nf_elems_1s(vd, i, nf, esz, max_elems); } - remain -= offset; - addr_i = adjust_addr(env, addr_i + offset); - } + } while (++i < page_evl); + } + } + + /* Usually the first page contains the entire vector. */ + if (likely(page_evl == vl)) { + goto tail; + } + i = page_evl; + } + + /* Skip forward to the next active element. */ + if (!vm) { + while (1) { + if (vext_elem_mask(v0, i)) { + break; + } + if (vma) { + vext_set_nf_elems_1s(vd, i, nf, esz, max_elems); + } + if (++i == vl) { + goto tail; } } } -ProbeSuccess: - /* load bytes from guest memory */ - if (vl != 0) { - env->vl = vl; + + addr = base + i * msize; + page_split = -(addr | TARGET_PAGE_MASK); + + /* Validate the second page is accessible. */ + if (unlikely(page_split < msize)) { + /* + * Cross page element which isn't first. + * We have not yet advanced addr to the next page. + */ + target_ulong next_page = addr + page_split; + flags |= probe_access_flags(env, adjust_addr(env, next_page), + last - next_page + 1, MMU_DATA_LOAD, + mmu_index, true, &host, ra); + + /* Stop if invalid (unmapped) or mmio (transaction may fail). */ + if (flags & (TLB_INVALID_MASK | TLB_MMIO)) { + env->vl = i; + goto tail; + } + + vext_ldst_nf_tlb(env, vd, addr, i, nf, esz, max_elems, ldst_tlb, ra); + if (++i == vl) { + goto tail; + } + addr += msize; + if (host) { + host += addr - next_page; + } + } else { + flags = probe_access_flags(env, adjust_addr(env, addr), + last - addr + 1, MMU_DATA_LOAD, + mmu_index, true, &host, ra); + + /* Stop if invalid (unmapped) or mmio (transaction may fail). */ + if (flags & (TLB_INVALID_MASK | TLB_MMIO)) { + env->vl = i; + goto tail; + } } - if (env->vstart < env->vl) { + /* Load complete elements from the second page. */ + if (flags == 0) { if (vm) { - /* Load/store elements in the first page */ - if (likely(elems)) { - vext_page_ldst_us(env, vd, addr, elems, nf, max_elems, - log2_esz, true, mmu_index, ldst_tlb, - ldst_host, ra); - } - - /* Load/store elements in the second page */ - if (unlikely(env->vstart < env->vl)) { - addr = base + env->vstart * msize; - - /* Cross page element */ - if (unlikely(page_split % msize)) { - vext_ldst_nf_tlb(env, vd, addr, env->vstart, nf, - esz, max_elems, ldst_tlb, ra); - env->vstart++; - addr += msize; - } - - /* Get number of elements of second page */ - elems = env->vl - env->vstart; - - /* Load/store elements in the second page */ - vext_page_ldst_us(env, vd, addr, elems, nf, max_elems, - log2_esz, true, mmu_index, ldst_tlb, - ldst_host, ra); - } + vext_page_ldst_us_host(vd, host, i, vl, nf, log2_esz, + max_elems, ldst_host, true); } else { - for (i = env->vstart; i < env->vl; i++) { + host -= addr - base; + do { + if (vext_elem_mask(v0, i)) { + vext_ldst_nf_host(vd, host + i * msize, i, nf, esz, + max_elems, ldst_host); + } else if (vma) { + vext_set_nf_elems_1s(vd, i, nf, esz, max_elems); + } + } while (++i < vl); + } + } else { + /* None of these ldst_tlb calls may fault. */ + if (vm) { + vext_page_ldst_us_tlb(env, vd, addr, i, vl, nf, + log2_esz, max_elems, + ldst_tlb, mmu_index, ra); + } else { + do { if (vext_elem_mask(v0, i)) { vext_ldst_nf_tlb(env, vd, base + i * msize, i, nf, esz, max_elems, ldst_tlb, ra); } else if (vma) { vext_set_nf_elems_1s(vd, i, nf, esz, max_elems); } - } + } while (++i < vl); } } -- 2.43.0
