It turns out there are fewer places that need annotation if we just flatten from the outer helper macros instead of everything along the call stack.
Signed-off-by: Richard Henderson <[email protected]> --- target/riscv/tcg/vector_helper.c | 147 +++++++++++++++---------------- 1 file changed, 70 insertions(+), 77 deletions(-) diff --git a/target/riscv/tcg/vector_helper.c b/target/riscv/tcg/vector_helper.c index f65da46144..c5b4085f9c 100644 --- a/target/riscv/tcg/vector_helper.c +++ b/target/riscv/tcg/vector_helper.c @@ -158,7 +158,7 @@ static inline void vext_set_elem_mask(void *v0, int index, ((uint64_t *)v0)[idx] = deposit64(old, pos, 1, value); } -static inline MemOpIdx vext_oi(uint32_t desc, MemOp memop) +static MemOpIdx vext_oi(uint32_t desc, MemOp memop) { uint32_t data = simd_data(desc); @@ -178,20 +178,18 @@ typedef void vext_ldst_elem_fn_tlb(CPURISCVState *env, abi_ptr addr, typedef void vext_ldst_elem_fn_host(void *vd, uint32_t idx, void *host); #define GEN_VEXT_TLB_LD_ELEM(NAME, ETYPE, H, LDSUF) \ -static inline QEMU_ALWAYS_INLINE \ -void NAME##_tlb(CPURISCVState *env, abi_ptr addr, uint32_t idx, \ - void *vd, MemOpIdx oi, uintptr_t retaddr) \ +static void NAME##_tlb(CPURISCVState *env, abi_ptr addr, uint32_t idx, \ + void *vd, MemOpIdx oi, uintptr_t retaddr) \ { \ ETYPE *cur = ((ETYPE *)vd + H(idx)); \ *cur = cpu_##LDSUF##_mmu(env, addr, oi, retaddr); \ } -#define GEN_VEXT_HOST_LD_ELEM(NAME, ETYPE, H, LDSUF) \ -static inline QEMU_ALWAYS_INLINE \ -void NAME##_host(void *vd, uint32_t idx, void *host) \ -{ \ - ETYPE *cur = ((ETYPE *)vd + H(idx)); \ - *cur = (ETYPE)LDSUF##_p(host); \ +#define GEN_VEXT_HOST_LD_ELEM(NAME, ETYPE, H, LDSUF) \ +static void NAME##_host(void *vd, uint32_t idx, void *host) \ +{ \ + ETYPE *cur = ((ETYPE *)vd + H(idx)); \ + *cur = (ETYPE)LDSUF##_p(host); \ } GEN_VEXT_TLB_LD_ELEM(lde_b, uint8_t, H1, ldb) @@ -205,20 +203,18 @@ GEN_VEXT_HOST_LD_ELEM(lde_w, uint32_t, H4, ldl_le) GEN_VEXT_HOST_LD_ELEM(lde_d, uint64_t, H8, ldq_le) #define GEN_VEXT_TLB_ST_ELEM(NAME, ETYPE, H, STSUF) \ -static inline QEMU_ALWAYS_INLINE \ -void NAME##_tlb(CPURISCVState *env, abi_ptr addr, uint32_t idx, \ +static void NAME##_tlb(CPURISCVState *env, abi_ptr addr, uint32_t idx, \ void *vd, MemOpIdx oi, uintptr_t retaddr) \ { \ ETYPE data = *((ETYPE *)vd + H(idx)); \ cpu_##STSUF##_mmu(env, addr, data, oi, retaddr); \ } -#define GEN_VEXT_HOST_ST_ELEM(NAME, ETYPE, H, STSUF) \ -static inline QEMU_ALWAYS_INLINE \ -void NAME##_host(void *vd, uint32_t idx, void *host) \ -{ \ - ETYPE data = *((ETYPE *)vd + H(idx)); \ - STSUF##_p(host, data); \ +#define GEN_VEXT_HOST_ST_ELEM(NAME, ETYPE, H, STSUF) \ +static void NAME##_host(void *vd, uint32_t idx, void *host) \ +{ \ + ETYPE data = *((ETYPE *)vd + H(idx)); \ + STSUF##_p(host, data); \ } GEN_VEXT_TLB_ST_ELEM(ste_b, uint8_t, H1, stb) @@ -308,9 +304,9 @@ vext_ldst_stride(void *vd, void *v0, target_ulong base, target_ulong stride, } #define GEN_VEXT_LD_STRIDE(NAME, ETYPE, LOAD_FN) \ -void HELPER(NAME)(void *vd, void * v0, target_ulong base, \ - target_ulong stride, CPURISCVState *env, \ - uint32_t desc) \ +QEMU_FLATTEN void HELPER(NAME)(void *vd, void * v0, target_ulong base, \ + target_ulong stride, CPURISCVState *env, \ + uint32_t desc) \ { \ uint32_t vm = vext_vm(desc); \ vext_ldst_stride(vd, v0, base, stride, env, desc, vm, LOAD_FN, \ @@ -323,9 +319,9 @@ GEN_VEXT_LD_STRIDE(vlse32_v, int32_t, lde_w_tlb) GEN_VEXT_LD_STRIDE(vlse64_v, int64_t, lde_d_tlb) #define GEN_VEXT_ST_STRIDE(NAME, ETYPE, STORE_FN) \ -void HELPER(NAME)(void *vd, void *v0, target_ulong base, \ - target_ulong stride, CPURISCVState *env, \ - uint32_t desc) \ +QEMU_FLATTEN void HELPER(NAME)(void *vd, void *v0, target_ulong base, \ + target_ulong stride, CPURISCVState *env, \ + uint32_t desc) \ { \ uint32_t vm = vext_vm(desc); \ vext_ldst_stride(vd, v0, base, stride, env, desc, vm, STORE_FN, \ @@ -342,7 +338,7 @@ GEN_VEXT_ST_STRIDE(vsse64_v, int64_t, ste_d_tlb) */ /* unmasked unit-stride load and store operation */ -static inline QEMU_ALWAYS_INLINE void +static void vext_page_ldst_us_host(void *vd, void *host, uint32_t i, uint32_t evl, uint32_t nf, uint32_t log2_esz, uint32_t max_elems, vext_ldst_elem_fn_host *ldst_host, bool is_load) @@ -372,7 +368,7 @@ vext_page_ldst_us_host(void *vd, void *host, uint32_t i, uint32_t evl, } } -static inline QEMU_ALWAYS_INLINE void +static void vext_page_ldst_us_tlb(CPURISCVState *env, void *vd, target_ulong addr, uint32_t i, uint32_t evl, uint32_t nf, uint32_t log2_esz, uint32_t max_elems, @@ -403,7 +399,7 @@ static void vext_test_alignment(CPURISCVState *env, vaddr addr, uint32_t esz, } } -static inline QEMU_ALWAYS_INLINE void +static void vext_ldst_us_notail(void *vd, target_ulong base, CPURISCVState *env, uint32_t log2_esz, uint32_t nf, uint32_t evl, uint32_t max_elems, bool vma, @@ -499,7 +495,7 @@ vext_ldst_us_notail(void *vd, target_ulong base, CPURISCVState *env, } } -static inline QEMU_ALWAYS_INLINE void +static void vext_ldst_us(void *vd, target_ulong base, CPURISCVState *env, uint32_t desc, vext_ldst_elem_fn_tlb *ldst_tlb, vext_ldst_elem_fn_host *ldst_host, uint32_t log2_esz, @@ -521,20 +517,19 @@ vext_ldst_us(void *vd, target_ulong base, CPURISCVState *env, uint32_t desc, * stride, stride = NF * sizeof (ETYPE) */ -#define GEN_VEXT_LD_US(NAME, ETYPE, LOAD_FN_TLB, LOAD_FN_HOST) \ -void HELPER(NAME##_mask)(void *vd, void *v0, target_ulong base, \ - CPURISCVState *env, uint32_t desc) \ -{ \ - uint32_t stride = vext_nf(desc) << ctzl(sizeof(ETYPE)); \ - vext_ldst_stride(vd, v0, base, stride, env, desc, false, \ - LOAD_FN_TLB, ctzl(sizeof(ETYPE)), GETPC()); \ -} \ - \ -void HELPER(NAME)(void *vd, void *v0, target_ulong base, \ - CPURISCVState *env, uint32_t desc) \ -{ \ - vext_ldst_us(vd, base, env, desc, LOAD_FN_TLB, LOAD_FN_HOST, \ - ctzl(sizeof(ETYPE)), env->vl, GETPC(), true); \ +#define GEN_VEXT_LD_US(NAME, ETYPE, LOAD_FN_TLB, LOAD_FN_HOST) \ +QEMU_FLATTEN void HELPER(NAME##_mask)(void *vd, void *v0, \ + target_ulong base, CPURISCVState *env, uint32_t desc) \ +{ \ + uint32_t stride = vext_nf(desc) << ctzl(sizeof(ETYPE)); \ + vext_ldst_stride(vd, v0, base, stride, env, desc, false, \ + LOAD_FN_TLB, ctzl(sizeof(ETYPE)), GETPC()); \ +} \ +QEMU_FLATTEN void HELPER(NAME)(void *vd, void *v0, target_ulong base, \ + CPURISCVState *env, uint32_t desc) \ +{ \ + vext_ldst_us(vd, base, env, desc, LOAD_FN_TLB, LOAD_FN_HOST, \ + ctzl(sizeof(ETYPE)), env->vl, GETPC(), true); \ } GEN_VEXT_LD_US(vle8_v, int8_t, lde_b_tlb, lde_b_host) @@ -543,16 +538,15 @@ GEN_VEXT_LD_US(vle32_v, int32_t, lde_w_tlb, lde_w_host) GEN_VEXT_LD_US(vle64_v, int64_t, lde_d_tlb, lde_d_host) #define GEN_VEXT_ST_US(NAME, ETYPE, STORE_FN_TLB, STORE_FN_HOST) \ -void HELPER(NAME##_mask)(void *vd, void *v0, target_ulong base, \ - CPURISCVState *env, uint32_t desc) \ +QEMU_FLATTEN void HELPER(NAME##_mask)(void *vd, void *v0, \ + target_ulong base, CPURISCVState *env, uint32_t desc) \ { \ uint32_t stride = vext_nf(desc) << ctzl(sizeof(ETYPE)); \ vext_ldst_stride(vd, v0, base, stride, env, desc, false, \ STORE_FN_TLB, ctzl(sizeof(ETYPE)), GETPC()); \ } \ - \ -void HELPER(NAME)(void *vd, void *v0, target_ulong base, \ - CPURISCVState *env, uint32_t desc) \ +QEMU_FLATTEN void HELPER(NAME)(void *vd, void *v0, target_ulong base, \ + CPURISCVState *env, uint32_t desc) \ { \ vext_ldst_us(vd, base, env, desc, STORE_FN_TLB, STORE_FN_HOST, \ ctzl(sizeof(ETYPE)), env->vl, GETPC(), false); \ @@ -566,8 +560,8 @@ GEN_VEXT_ST_US(vse64_v, int64_t, ste_d_tlb, ste_d_host) /* * unit stride mask load and store, EEW = 1 */ -void HELPER(vlm_v)(void *vd, void *v0, target_ulong base, - CPURISCVState *env, uint32_t desc) +QEMU_FLATTEN void HELPER(vlm_v)(void *vd, void *v0, target_ulong base, + CPURISCVState *env, uint32_t desc) { /* evl = ceil(vl/8) */ uint8_t evl = (env->vl + 7) >> 3; @@ -575,8 +569,8 @@ void HELPER(vlm_v)(void *vd, void *v0, target_ulong base, 0, evl, GETPC(), true); } -void HELPER(vsm_v)(void *vd, void *v0, target_ulong base, - CPURISCVState *env, uint32_t desc) +QEMU_FLATTEN void HELPER(vsm_v)(void *vd, void *v0, target_ulong base, + CPURISCVState *env, uint32_t desc) { /* evl = ceil(vl/8) */ uint8_t evl = (env->vl + 7) >> 3; @@ -602,7 +596,7 @@ GEN_VEXT_GET_INDEX_ADDR(idx_h, uint16_t, H2) GEN_VEXT_GET_INDEX_ADDR(idx_w, uint32_t, H4) GEN_VEXT_GET_INDEX_ADDR(idx_d, uint64_t, H8) -static inline void +static void vext_ldst_index(void *vd, void *v0, target_ulong base, void *vs2, CPURISCVState *env, uint32_t desc, vext_get_index_addr get_index_addr, @@ -637,12 +631,12 @@ vext_ldst_index(void *vd, void *v0, target_ulong base, vext_set_tail_elems_1s(env->vl, vd, desc, nf, esz, max_elems); } -#define GEN_VEXT_LD_INDEX(NAME, ETYPE, INDEX_FN, LOAD_FN) \ -void HELPER(NAME)(void *vd, void *v0, target_ulong base, \ - void *vs2, CPURISCVState *env, uint32_t desc) \ -{ \ - vext_ldst_index(vd, v0, base, vs2, env, desc, INDEX_FN, \ - LOAD_FN, ctzl(sizeof(ETYPE)), GETPC()); \ +#define GEN_VEXT_LD_INDEX(NAME, ETYPE, INDEX_FN, LOAD_FN) \ +QEMU_FLATTEN void HELPER(NAME)(void *vd, void *v0, target_ulong base, \ + void *vs2, CPURISCVState *env, uint32_t desc) \ +{ \ + vext_ldst_index(vd, v0, base, vs2, env, desc, INDEX_FN, \ + LOAD_FN, ctzl(sizeof(ETYPE)), GETPC()); \ } GEN_VEXT_LD_INDEX(vlxei8_8_v, int8_t, idx_b, lde_b_tlb) @@ -662,13 +656,12 @@ GEN_VEXT_LD_INDEX(vlxei64_16_v, int16_t, idx_d, lde_h_tlb) GEN_VEXT_LD_INDEX(vlxei64_32_v, int32_t, idx_d, lde_w_tlb) GEN_VEXT_LD_INDEX(vlxei64_64_v, int64_t, idx_d, lde_d_tlb) -#define GEN_VEXT_ST_INDEX(NAME, ETYPE, INDEX_FN, STORE_FN) \ -void HELPER(NAME)(void *vd, void *v0, target_ulong base, \ - void *vs2, CPURISCVState *env, uint32_t desc) \ -{ \ - vext_ldst_index(vd, v0, base, vs2, env, desc, INDEX_FN, \ - STORE_FN, ctzl(sizeof(ETYPE)), \ - GETPC()); \ +#define GEN_VEXT_ST_INDEX(NAME, ETYPE, INDEX_FN, STORE_FN) \ +QEMU_FLATTEN void HELPER(NAME)(void *vd, void *v0, target_ulong base, \ + void *vs2, CPURISCVState *env, uint32_t desc) \ +{ \ + vext_ldst_index(vd, v0, base, vs2, env, desc, INDEX_FN, \ + STORE_FN, ctzl(sizeof(ETYPE)), GETPC()); \ } GEN_VEXT_ST_INDEX(vsxei8_8_v, int8_t, idx_b, ste_b_tlb) @@ -691,7 +684,7 @@ GEN_VEXT_ST_INDEX(vsxei64_64_v, int64_t, idx_d, ste_d_tlb) /* * unit-stride fault-only-fisrt load instructions */ -static inline void +static void vext_ldff(void *vd, void *v0, target_ulong base, CPURISCVState *env, uint32_t desc, vext_ldst_elem_fn_tlb *ldst_tlb, vext_ldst_elem_fn_host *ldst_host, uint32_t log2_esz, uintptr_t ra) @@ -922,12 +915,12 @@ vext_ldff(void *vd, void *v0, target_ulong base, CPURISCVState *env, vext_set_tail_elems_1s(env->vl, vd, desc, nf, esz, max_elems); } -#define GEN_VEXT_LDFF(NAME, ETYPE, LOAD_FN_TLB, LOAD_FN_HOST) \ -void HELPER(NAME)(void *vd, void *v0, target_ulong base, \ - CPURISCVState *env, uint32_t desc) \ -{ \ - vext_ldff(vd, v0, base, env, desc, LOAD_FN_TLB, \ - LOAD_FN_HOST, ctzl(sizeof(ETYPE)), GETPC()); \ +#define GEN_VEXT_LDFF(NAME, ETYPE, LOAD_FN_TLB, LOAD_FN_HOST) \ +QEMU_FLATTEN void HELPER(NAME)(void *vd, void *v0, target_ulong base, \ + CPURISCVState *env, uint32_t desc) \ +{ \ + vext_ldff(vd, v0, base, env, desc, LOAD_FN_TLB, \ + LOAD_FN_HOST, ctzl(sizeof(ETYPE)), GETPC()); \ } GEN_VEXT_LDFF(vle8ff_v, int8_t, lde_b_tlb, lde_b_host) @@ -948,7 +941,7 @@ GEN_VEXT_LDFF(vle64ff_v, int64_t, lde_d_tlb, lde_d_host) /* * load and store whole register instructions */ -static inline QEMU_ALWAYS_INLINE void +static void vext_ldst_whole(void *vd, target_ulong base, CPURISCVState *env, uint32_t desc, vext_ldst_elem_fn_tlb *ldst_tlb, vext_ldst_elem_fn_host *ldst_host, uint32_t log2_esz, @@ -963,8 +956,8 @@ vext_ldst_whole(void *vd, target_ulong base, CPURISCVState *env, uint32_t desc, } #define GEN_VEXT_LD_WHOLE(NAME, ETYPE, LOAD_FN_TLB, LOAD_FN_HOST) \ -void HELPER(NAME)(void *vd, target_ulong base, CPURISCVState *env, \ - uint32_t desc) \ +QEMU_FLATTEN void HELPER(NAME)(void *vd, target_ulong base, \ + CPURISCVState *env, uint32_t desc) \ { \ vext_ldst_whole(vd, base, env, desc, LOAD_FN_TLB, LOAD_FN_HOST, \ ctzl(sizeof(ETYPE)), GETPC(), true); \ @@ -988,8 +981,8 @@ GEN_VEXT_LD_WHOLE(vl8re32_v, int32_t, lde_w_tlb, lde_w_host) GEN_VEXT_LD_WHOLE(vl8re64_v, int64_t, lde_d_tlb, lde_d_host) #define GEN_VEXT_ST_WHOLE(NAME, ETYPE, STORE_FN_TLB, STORE_FN_HOST) \ -void HELPER(NAME)(void *vd, target_ulong base, CPURISCVState *env, \ - uint32_t desc) \ +QEMU_FLATTEN void HELPER(NAME)(void *vd, target_ulong base, \ + CPURISCVState *env, uint32_t desc) \ { \ vext_ldst_whole(vd, base, env, desc, STORE_FN_TLB, STORE_FN_HOST, \ ctzl(sizeof(ETYPE)), GETPC(), false); \ -- 2.43.0
