We already computed the page split here, which means we
were quite ready to call probe_access directly. Do so,
then use vext_ldst_nf_{host,tlb} to load the elements.
Signed-off-by: Richard Henderson <[email protected]>
---
target/riscv/tcg/vector_helper.c | 88 ++++++++++++++++++++------------
1 file changed, 56 insertions(+), 32 deletions(-)
diff --git a/target/riscv/tcg/vector_helper.c b/target/riscv/tcg/vector_helper.c
index 505f0331f2..36e64465f2 100644
--- a/target/riscv/tcg/vector_helper.c
+++ b/target/riscv/tcg/vector_helper.c
@@ -477,17 +477,19 @@ vext_ldst_us(void *vd, target_ulong base, CPURISCVState
*env, uint32_t desc,
vext_ldst_elem_fn_host *ldst_host, uint32_t log2_esz,
uint32_t evl, uintptr_t ra, bool is_load)
{
- target_ulong page_split, elems, addr;
+ target_ulong elems, addr, last, last_in_page, page_split;
uint32_t nf = vext_nf(desc);
uint32_t max_elems = vext_max_elems(desc, log2_esz);
uint32_t esz = 1 << log2_esz;
uint32_t msize = nf * esz;
int mmu_index = riscv_env_mmu_index(env, false);
MMUAccessType access_type = is_load ? MMU_DATA_LOAD : MMU_DATA_STORE;
+ uint32_t i = env->vstart;
+ void *host;
VSTART_CHECK_EARLY_EXIT(env, evl);
- addr = base + env->vstart * msize;
+ addr = base + i * msize;
/* Recognize alignment fault before memory protection fault. */
vext_test_alignment(env, addr, esz, access_type, mmu_index, ra);
@@ -498,50 +500,72 @@ vext_ldst_us(void *vd, target_ulong base, CPURISCVState
*env, uint32_t desc,
* by simply calling ldst_tlb.
*/
if (nf == 1 && (evl << log2_esz) <= 6) {
- for (uint32_t i = env->vstart; i < evl;
- env->vstart = ++i, addr += esz) {
+ for (; i < evl; env->vstart = ++i, addr += esz) {
ldst_tlb(env, adjust_addr(env, addr), i, vd, ra);
}
- env->vstart = 0;
- vext_set_tail_elems_1s(evl, vd, desc, nf, esz, max_elems);
- return;
+ goto tail;
}
#endif
- /* Calculate the page range of first page */
+ /* Calculate the page range of first page. */
+ last = base + evl * msize - 1;
+ last_in_page = addr | ~TARGET_PAGE_MASK;
page_split = -(addr | TARGET_PAGE_MASK);
- /* Get number of elements */
- elems = page_split / msize;
- if (unlikely(env->vstart + elems >= evl)) {
- elems = evl - env->vstart;
- }
+
+ /* Validate the first page is accessible. */
+ host = probe_access(env, adjust_addr(env, addr),
+ MIN(last, last_in_page) - addr + 1,
+ access_type, mmu_index, ra);
+
+ /* Get number of complete elements in the first page. */
+ elems = MIN(page_split / msize, evl - i);
/* Load/store elements in the first page */
if (likely(elems)) {
- vext_page_ldst_us(env, vd, addr, elems, nf, max_elems, log2_esz,
- is_load, mmu_index, ldst_tlb, ldst_host, ra);
- }
+ uint32_t page_evl = i + elems;
- /* Load/store elements in the second page */
- if (unlikely(env->vstart < evl)) {
- addr = base + env->vstart * msize;
-
- /* Cross page element */
- if (unlikely(page_split % msize)) {
- vext_ldst_nf_tlb(env, vd, addr, env->vstart,
- nf, esz, max_elems, ldst_tlb, ra);
- env->vstart++;
- addr += msize;
+ if (host) {
+ vext_page_ldst_us_host(vd, host, i, page_evl, nf, log2_esz,
+ max_elems, ldst_host, is_load);
+ } else {
+ vext_page_ldst_us_tlb(env, vd, addr, i, page_evl, nf,
+ log2_esz, max_elems,
+ ldst_tlb, mmu_index, ra);
}
- /* Get number of elements of second page */
- elems = evl - env->vstart;
-
- /* Load/store elements in the second page */
- vext_page_ldst_us(env, vd, addr, elems, nf, max_elems, log2_esz,
- is_load, mmu_index, ldst_tlb, ldst_host, ra);
+ /* Usually the first page contains the entire vector. */
+ if (likely(page_evl == evl)) {
+ goto tail;
+ }
+ env->vstart = i = page_evl;
+ addr += elems * msize;
}
+ /* Cross page element */
+ if (unlikely(page_split % msize)) {
+ vext_ldst_nf_tlb(env, vd, addr, i++, nf, esz, max_elems, ldst_tlb, ra);
+ if (i == evl) {
+ goto tail;
+ }
+ env->vstart = i;
+ addr += msize;
+ }
+
+ /* Validate the second page is accessible. */
+ assert(i < evl);
+ elems = evl - i;
+ host = probe_access(env, adjust_addr(env, addr), elems * msize,
+ access_type, mmu_index, ra);
+
+ if (host) {
+ vext_page_ldst_us_host(vd, host, i, evl, nf, log2_esz,
+ max_elems, ldst_host, is_load);
+ } else {
+ vext_page_ldst_us_tlb(env, vd, addr, i, evl, nf, log2_esz, max_elems,
+ ldst_tlb, mmu_index, ra);
+ }
+
+ tail:
env->vstart = 0;
vext_set_tail_elems_1s(evl, vd, desc, nf, esz, max_elems);
}
--
2.43.0
