Avoids continually looking up mmu_index with every operation.
Signed-off-by: Richard Henderson <[email protected]>
---
target/riscv/tcg/vector_helper.c | 91 +++++++++++++++-----------------
1 file changed, 43 insertions(+), 48 deletions(-)
diff --git a/target/riscv/tcg/vector_helper.c b/target/riscv/tcg/vector_helper.c
index 98a0a5d025..f65da46144 100644
--- a/target/riscv/tcg/vector_helper.c
+++ b/target/riscv/tcg/vector_helper.c
@@ -171,32 +171,20 @@ static inline MemOpIdx vext_oi(uint32_t desc, MemOp memop)
return make_memop_idx(memop, FIELD_EX32(data, VDATA, MEM_IDX));
}
-static inline MemOpIdx vext_make_memop_idx(CPURISCVState *env, size_t size)
-{
- int mmu_idx = riscv_env_mmu_index(env, false);
- MemOp memop = size_memop(size) | mo_endian_env(env);
-
- if (!riscv_cpu_cfg(env)->ext_zicclsm) {
- memop |= MO_ALIGN;
- }
-
- return make_memop_idx(memop, mmu_idx);
-}
-
/* elements operations for load and store */
typedef void vext_ldst_elem_fn_tlb(CPURISCVState *env, abi_ptr addr,
- uint32_t idx, void *vd, uintptr_t retaddr);
+ uint32_t idx, void *vd,
+ MemOpIdx oi, uintptr_t retaddr);
typedef void vext_ldst_elem_fn_host(void *vd, uint32_t idx, void *host);
-#define GEN_VEXT_TLB_LD_ELEM(NAME, ETYPE, H, LDSUF) \
-static inline QEMU_ALWAYS_INLINE \
-void NAME##_tlb(CPURISCVState *env, abi_ptr addr, \
- uint32_t idx, void *vd, uintptr_t retaddr) \
-{ \
- ETYPE *cur = ((ETYPE *)vd + H(idx)); \
- MemOpIdx oi = vext_make_memop_idx(env, sizeof(ETYPE)); \
- *cur = cpu_##LDSUF##_mmu(env, addr, oi, retaddr); \
-} \
+#define GEN_VEXT_TLB_LD_ELEM(NAME, ETYPE, H, LDSUF) \
+static inline QEMU_ALWAYS_INLINE \
+void NAME##_tlb(CPURISCVState *env, abi_ptr addr, uint32_t idx, \
+ void *vd, MemOpIdx oi, uintptr_t retaddr) \
+{ \
+ ETYPE *cur = ((ETYPE *)vd + H(idx)); \
+ *cur = cpu_##LDSUF##_mmu(env, addr, oi, retaddr); \
+}
#define GEN_VEXT_HOST_LD_ELEM(NAME, ETYPE, H, LDSUF) \
static inline QEMU_ALWAYS_INLINE \
@@ -216,15 +204,14 @@ GEN_VEXT_HOST_LD_ELEM(lde_h, uint16_t, H2, lduw_le)
GEN_VEXT_HOST_LD_ELEM(lde_w, uint32_t, H4, ldl_le)
GEN_VEXT_HOST_LD_ELEM(lde_d, uint64_t, H8, ldq_le)
-#define GEN_VEXT_TLB_ST_ELEM(NAME, ETYPE, H, STSUF) \
-static inline QEMU_ALWAYS_INLINE \
-void NAME##_tlb(CPURISCVState *env, abi_ptr addr, \
- uint32_t idx, void *vd, uintptr_t retaddr) \
-{ \
- ETYPE data = *((ETYPE *)vd + H(idx)); \
- MemOpIdx oi = vext_make_memop_idx(env, sizeof(ETYPE)); \
- cpu_##STSUF##_mmu(env, addr, data, oi, retaddr); \
-} \
+#define GEN_VEXT_TLB_ST_ELEM(NAME, ETYPE, H, STSUF) \
+static inline QEMU_ALWAYS_INLINE \
+void NAME##_tlb(CPURISCVState *env, abi_ptr addr, uint32_t idx, \
+ void *vd, MemOpIdx oi, uintptr_t retaddr) \
+{ \
+ ETYPE data = *((ETYPE *)vd + H(idx)); \
+ cpu_##STSUF##_mmu(env, addr, data, oi, retaddr); \
+}
#define GEN_VEXT_HOST_ST_ELEM(NAME, ETYPE, H, STSUF) \
static inline QEMU_ALWAYS_INLINE \
@@ -282,10 +269,11 @@ static void vext_ldst_nf_host(void *vd, void *host,
uint32_t i, uint32_t nf,
static void vext_ldst_nf_tlb(CPURISCVState *env, void *vd, target_ulong addr,
uint32_t i, uint32_t nf,
uint32_t esz, uint32_t max_elems,
- vext_ldst_elem_fn_tlb *ldst_tlb, uintptr_t ra)
+ vext_ldst_elem_fn_tlb *ldst_tlb,
+ MemOpIdx oi, uintptr_t ra)
{
for (uint32_t k = 0; k < nf; k++, addr += esz) {
- ldst_tlb(env, adjust_addr(env, addr), i + k * max_elems, vd, ra);
+ ldst_tlb(env, adjust_addr(env, addr), i + k * max_elems, vd, oi, ra);
}
}
@@ -302,13 +290,14 @@ vext_ldst_stride(void *vd, void *v0, target_ulong base,
target_ulong stride,
uint32_t max_elems = vext_max_elems(desc, log2_esz);
uint32_t esz = 1 << log2_esz;
uint32_t vma = vext_vma(desc);
+ MemOpIdx oi = vext_oi(desc, log2_esz);
VSTART_CHECK_EARLY_EXIT(env, env->vl);
for (uint32_t i = env->vstart; i < env->vl; env->vstart = ++i) {
if (vm || vext_elem_mask(v0, i)) {
vext_ldst_nf_tlb(env, vd, base + stride * i, i, nf,
- esz, max_elems, ldst_elem, ra);
+ esz, max_elems, ldst_elem, oi, ra);
} else if (vma) {
vext_set_nf_elems_1s(vd, i, nf, esz, max_elems);
}
@@ -388,12 +377,13 @@ vext_page_ldst_us_tlb(CPURISCVState *env, void *vd,
target_ulong addr,
uint32_t i, uint32_t evl, uint32_t nf,
uint32_t log2_esz, uint32_t max_elems,
vext_ldst_elem_fn_tlb *ldst_tlb,
- int mmu_index, uintptr_t ra)
+ MemOpIdx oi, uintptr_t ra)
{
uint32_t esz = 1 << log2_esz;
uint32_t msize = nf << log2_esz;
do {
- vext_ldst_nf_tlb(env, vd, addr, i, nf, esz, max_elems, ldst_tlb, ra);
+ vext_ldst_nf_tlb(env, vd, addr, i, nf, esz,
+ max_elems, ldst_tlb, oi, ra);
addr += msize;
env->vstart = ++i;
} while (i < evl);
@@ -443,7 +433,7 @@ vext_ldst_us_notail(void *vd, target_ulong base,
CPURISCVState *env,
*/
if (nf == 1 && (evl << log2_esz) <= 6) {
for (; i < evl; env->vstart = ++i, addr += esz) {
- ldst_tlb(env, adjust_addr(env, addr), i, vd, ra);
+ ldst_tlb(env, adjust_addr(env, addr), i, vd, oi, ra);
}
return;
}
@@ -472,7 +462,7 @@ vext_ldst_us_notail(void *vd, target_ulong base,
CPURISCVState *env,
} else {
vext_page_ldst_us_tlb(env, vd, addr, i, page_evl, nf,
log2_esz, max_elems,
- ldst_tlb, mmu_index, ra);
+ ldst_tlb, oi, ra);
}
/* Usually the first page contains the entire vector. */
@@ -485,7 +475,8 @@ vext_ldst_us_notail(void *vd, target_ulong base,
CPURISCVState *env,
/* Cross page element */
if (unlikely(page_split % msize)) {
- vext_ldst_nf_tlb(env, vd, addr, i++, nf, esz, max_elems, ldst_tlb, ra);
+ vext_ldst_nf_tlb(env, vd, addr, i++, nf, esz,
+ max_elems, ldst_tlb, oi, ra);
if (i == evl) {
return;
}
@@ -504,7 +495,7 @@ vext_ldst_us_notail(void *vd, target_ulong base,
CPURISCVState *env,
max_elems, ldst_host, is_load);
} else {
vext_page_ldst_us_tlb(env, vd, addr, i, evl, nf, log2_esz, max_elems,
- ldst_tlb, mmu_index, ra);
+ ldst_tlb, oi, ra);
}
}
@@ -623,6 +614,7 @@ vext_ldst_index(void *vd, void *v0, target_ulong base,
uint32_t max_elems = vext_max_elems(desc, log2_esz);
uint32_t esz = 1 << log2_esz;
uint32_t vma = vext_vma(desc);
+ MemOpIdx oi = vext_oi(desc, log2_esz);
VSTART_CHECK_EARLY_EXIT(env, env->vl);
@@ -636,7 +628,8 @@ vext_ldst_index(void *vd, void *v0, target_ulong base,
for (uint32_t k = 0; k < nf; ++k) {
abi_ptr addr = get_index_addr(base, i, vs2) + (k << log2_esz);
- ldst_elem(env, adjust_addr(env, addr), i + k * max_elems, vd, ra);
+ ldst_elem(env, adjust_addr(env, addr),
+ i + k * max_elems, vd, oi, ra);
}
}
@@ -753,7 +746,8 @@ vext_ldff(void *vd, void *v0, target_ulong base,
CPURISCVState *env,
* If it also crosses the page, just go ahead and load it.
*/
if (unlikely(page_split < msize) && first_active) {
- vext_ldst_nf_tlb(env, vd, addr, 0, nf, esz, max_elems, ldst_tlb, ra);
+ vext_ldst_nf_tlb(env, vd, addr, 0, nf, esz,
+ max_elems, ldst_tlb, oi, ra);
env->vstart = i = 1;
addr += msize;
page_split = -(addr | TARGET_PAGE_MASK);
@@ -798,7 +792,7 @@ vext_ldff(void *vd, void *v0, target_ulong base,
CPURISCVState *env,
*/
if (first_active) {
vext_ldst_nf_tlb(env, vd, addr, 0, nf, esz,
- max_elems, ldst_tlb, ra);
+ max_elems, ldst_tlb, oi, ra);
i = 1;
addr += msize;
}
@@ -813,12 +807,12 @@ vext_ldff(void *vd, void *v0, target_ulong base,
CPURISCVState *env,
if (vm) {
vext_page_ldst_us_tlb(env, vd, addr, i, page_evl, nf,
log2_esz, max_elems,
- ldst_tlb, mmu_index, ra);
+ ldst_tlb, oi, ra);
} else {
do {
if (vext_elem_mask(v0, i)) {
vext_ldst_nf_tlb(env, vd, base + i * msize, i, nf,
- esz, max_elems, ldst_tlb, ra);
+ esz, max_elems, ldst_tlb, oi, ra);
} else if (vma) {
vext_set_nf_elems_1s(vd, i, nf, esz, max_elems);
}
@@ -868,7 +862,8 @@ vext_ldff(void *vd, void *v0, target_ulong base,
CPURISCVState *env,
goto tail;
}
- vext_ldst_nf_tlb(env, vd, addr, i, nf, esz, max_elems, ldst_tlb, ra);
+ vext_ldst_nf_tlb(env, vd, addr, i, nf, esz,
+ max_elems, ldst_tlb, oi, ra);
if (++i == vl) {
goto tail;
}
@@ -909,12 +904,12 @@ vext_ldff(void *vd, void *v0, target_ulong base,
CPURISCVState *env,
if (vm) {
vext_page_ldst_us_tlb(env, vd, addr, i, vl, nf,
log2_esz, max_elems,
- ldst_tlb, mmu_index, ra);
+ ldst_tlb, oi, ra);
} else {
do {
if (vext_elem_mask(v0, i)) {
vext_ldst_nf_tlb(env, vd, base + i * msize, i, nf,
- esz, max_elems, ldst_tlb, ra);
+ esz, max_elems, ldst_tlb, oi, ra);
} else if (vma) {
vext_set_nf_elems_1s(vd, i, nf, esz, max_elems);
}
--
2.43.0
