Perform alignment trap on device memory. Recognize watchpoints. Perform both in the correct order wrt MTX faults.
Signed-off-by: Richard Henderson <[email protected]> --- target/arm/tcg/helper-a64-defs.h | 5 +- target/arm/tcg/mte_helper.c | 127 ++++++++++++++++++++++--------- target/arm/tcg/translate-a64.c | 49 +++++------- 3 files changed, 115 insertions(+), 66 deletions(-) diff --git a/target/arm/tcg/helper-a64-defs.h b/target/arm/tcg/helper-a64-defs.h index 518e74c8aa..0d0830c837 100644 --- a/target/arm/tcg/helper-a64-defs.h +++ b/target/arm/tcg/helper-a64-defs.h @@ -82,6 +82,9 @@ DEF_HELPER_3(vfp_ah_maxd, f64, f64, f64, fpst) DEF_HELPER_FLAGS_3(dc_zva, TCG_CALL_NO_WG, void, env, i64, i32) DEF_HELPER_FLAGS_3(dc_zva_mte, TCG_CALL_NO_WG, void, env, i64, i32) +DEF_HELPER_FLAGS_3(dc_gva, TCG_CALL_NO_WG, void, env, i64, i32) +DEF_HELPER_FLAGS_3(dc_gva_stub, TCG_CALL_NO_WG, void, env, i64, i32) +DEF_HELPER_FLAGS_3(dc_gzva, TCG_CALL_NO_WG, void, env, i64, i32) DEF_HELPER_FLAGS_3(pacia, TCG_CALL_NO_WG, i64, env, i64, i64) DEF_HELPER_FLAGS_3(pacib, TCG_CALL_NO_WG, i64, env, i64, i64) @@ -111,7 +114,7 @@ DEF_HELPER_FLAGS_4(st2g_parallel, TCG_CALL_NO_WG, void, env, i64, i64, i32) DEF_HELPER_FLAGS_2(st2g_stub, TCG_CALL_NO_WG, void, env, i64) DEF_HELPER_FLAGS_3(ldgm, TCG_CALL_NO_WG, i64, env, i64, i32) DEF_HELPER_FLAGS_4(stgm, TCG_CALL_NO_WG, void, env, i64, i64, i32) -DEF_HELPER_FLAGS_4(stzgm_tags, TCG_CALL_NO_WG, void, env, i64, i64, i32) +DEF_HELPER_FLAGS_4(stzgm, TCG_CALL_NO_WG, void, env, i64, i64, i32) DEF_HELPER_FLAGS_4(arm_unaligned_access, TCG_CALL_NO_WG, noreturn, env, i64, i32, i32) diff --git a/target/arm/tcg/mte_helper.c b/target/arm/tcg/mte_helper.c index 4161f4eb32..25adc8c714 100644 --- a/target/arm/tcg/mte_helper.c +++ b/target/arm/tcg/mte_helper.c @@ -671,36 +671,6 @@ void HELPER(stgm)(CPUARMState *env, uint64_t ptr, uint64_t val, uint32_t mtx) } } -void HELPER(stzgm_tags)(CPUARMState *env, uint64_t ptr, uint64_t val, - uint32_t mtx) -{ - uintptr_t ra = GETPC(); - int mmu_idx = arm_env_mmu_index(env); - int log2_dcz_bytes, log2_tag_bytes; - intptr_t dcz_bytes, tag_bytes; - uint8_t *mem; - - /* - * In arm_cpu_realizefn, we assert that dcz > LOG2_TAG_GRANULE+1, - * i.e. 32 bytes, which is an unreasonably small dcz anyway, - * to make sure that we can access one complete tag byte here. - */ - log2_dcz_bytes = get_dczid_bs(env_archcpu(env)) + 2; - log2_tag_bytes = log2_dcz_bytes - (LOG2_TAG_GRANULE + 1); - dcz_bytes = (intptr_t)1 << log2_dcz_bytes; - tag_bytes = (intptr_t)1 << log2_tag_bytes; - ptr &= -dcz_bytes; - - mem = allocation_tag_mem(env, mmu_idx, ptr, MMU_DATA_STORE, dcz_bytes, - MMU_DATA_STORE, ra); - if (mem) { - int tag_pair = (val & 0xf) * 0x11; - memset(mem, tag_pair, tag_bytes); - } else if (raw_mtx_check(mtx, extract64(ptr, 55, 1))) { - canonical_tag_write_fail(env, ptr, ra); - } -} - static void mte_sync_check_fail(CPUARMState *env, uint32_t desc, uint64_t dirty_ptr, uintptr_t ra) { @@ -1092,8 +1062,9 @@ bool mte_probe(CPUARMState *env, uint32_t desc, uint64_t ptr) } /* Traps for DC_ZVA, DC_GVA, and friends, after successful page lookup. */ -static int do_dcxva_traps(CPUARMState *env, vaddr addr, size_t len, int mmu_idx, - int flags, MemTxAttrs attrs, uintptr_t ra) +static int do_dcxva_traps(CPUARMState *env, vaddr addr, size_t len, + int mmu_idx, int flags, MemTxAttrs attrs, + uint32_t mtx_write, uintptr_t ra) { #ifndef CONFIG_USER_ONLY /* @@ -1106,7 +1077,17 @@ static int do_dcxva_traps(CPUARMState *env, vaddr addr, size_t len, int mmu_idx, arm_cpu_do_unaligned_access(env_cpu(env), addr, MMU_DATA_STORE, mmu_idx, ra); } +#endif + /* + * MTX write fault is via AArch64_S1CheckPermissions, which happens + * after AArch64_S1HasAlignmentFaultDueToMemType. + */ + if (mtx_check(mtx_write, extract64(addr, 55, 1))) { + canonical_tag_write_fail(env, addr, ra); + } + +#ifndef CONFIG_USER_ONLY /* Watchpoints have lower priority than alignment faults. */ if (unlikely(flags & TLB_WATCHPOINT)) { cpu_check_watchpoint(env_cpu(env), addr & -len, len, @@ -1158,7 +1139,7 @@ void HELPER(dc_zva)(CPUARMState *env, uint64_t addr, uint32_t desc) attrs = full->attrs; #endif - flags = do_dcxva_traps(env, addr, len, mmu_idx, flags, attrs, ra); + flags = do_dcxva_traps(env, addr, len, mmu_idx, flags, attrs, 0, ra); /* After traps, treat as aligned blocks. */ addr = addr & -len; @@ -1180,7 +1161,7 @@ void HELPER(dc_zva_mte)(CPUARMState *env, uintptr_t ptr_orig, uint32_t desc) dcz_bytes, MMU_DATA_LOAD, ra, ATM_ZVA); r.flags = do_dcxva_traps(env, ptr, dcz_bytes, mmu_idx, - r.flags, r.attrs, ra); + r.flags, r.attrs, 0, ra); bit55 = extract64(ptr, 55, 1); ptr_tag = allocation_tag_from_addr(ptr); @@ -1266,6 +1247,84 @@ void HELPER(dc_zva_mte)(CPUARMState *env, uintptr_t ptr_orig, uint32_t desc) do_dczva_0(env, ptr, dcz_bytes, r.ptr_mem, mmu_idx, r.flags, ra); } +static void do_stzgm_tags(void *mem, int dcz_bytes, int tag) +{ + /* + * In arm_cpu_realizefn, we asserted that dcz > LOG2_TAG_GRANULE+1, + * i.e. 32 bytes, which is an unreasonably small dcz anyway, to make + * sure that we can access one complete tag byte here. + */ + int tag_bytes = dcz_bytes / (TAG_GRANULE * 2); + int tag_pair = (tag & 0xf) * 0x11; + memset(mem, tag_pair, tag_bytes); +} + +void HELPER(dc_gva)(CPUARMState *env, uint64_t ptr, uint32_t desc) +{ + int dcz_bytes = FIELD_EX32(desc, MTEDESC, SIZEM1) + 1; + int mmu_idx = FIELD_EX32(desc, MTEDESC, MIDX); + uintptr_t ra = GETPC(); + + AllocationTagMem r = + allocation_tag_mem_internal(env, mmu_idx, ptr, MMU_DATA_STORE, + dcz_bytes, MMU_DATA_STORE, ra, ATM_ZVA); + + do_dcxva_traps(env, ptr, dcz_bytes, mmu_idx, r.flags, r.attrs, + r.tag_mem ? 0 : desc, ra); + + if (r.tag_mem) { + do_stzgm_tags(r.tag_mem, dcz_bytes, allocation_tag_from_addr(ptr)); + } +} + +/* DC GVA when tag access is disabled -- we still want all of the traps. */ +void HELPER(dc_gva_stub)(CPUARMState *env, uint64_t ptr, uint32_t desc) +{ + uintptr_t ra = GETPC(); + int mmu_idx = FIELD_EX32(desc, MTEDESC, MIDX); + +#ifdef CONFIG_USER_ONLY + probe_write(env, ptr, 0, mmu_idx, ra); +#else + int len = FIELD_EX32(desc, MTEDESC, SIZEM1) + 1; + void *mem; + CPUTLBEntryFull *full; + int flags = probe_access_full(env, ptr, 0, MMU_DATA_STORE, mmu_idx, + false, &mem, &full, ra); + do_dcxva_traps(env, ptr, len, mmu_idx, flags, full->attrs, 0, ra); +#endif +} + +static void do_gzva_stzgm(CPUARMState *env, uint64_t ptr, uint64_t tag, + uint32_t desc, uintptr_t ra) +{ + int dcz_bytes = FIELD_EX32(desc, MTEDESC, SIZEM1) + 1; + int mmu_idx = FIELD_EX32(desc, MTEDESC, MIDX); + + AllocationTagMem r = + allocation_tag_mem_internal(env, mmu_idx, ptr, MMU_DATA_STORE, + dcz_bytes, MMU_DATA_STORE, ra, ATM_ZVA); + + r.flags = do_dcxva_traps(env, ptr, dcz_bytes, mmu_idx, r.flags, r.attrs, + r.tag_mem ? 0 : desc, ra); + + if (r.tag_mem) { + do_stzgm_tags(r.tag_mem, dcz_bytes, tag); + } + do_dczva_0(env, ptr, dcz_bytes, r.ptr_mem, mmu_idx, r.flags, ra); +} + +void HELPER(dc_gzva)(CPUARMState *env, uint64_t ptr, uint32_t desc) +{ + do_gzva_stzgm(env, ptr, allocation_tag_from_addr(ptr), desc, GETPC()); +} + +void HELPER(stzgm)(CPUARMState *env, uint64_t ptr, uint64_t tag, uint32_t desc) +{ + int dcz_bytes = FIELD_EX32(desc, MTEDESC, SIZEM1) + 1; + do_gzva_stzgm(env, ptr & -dcz_bytes, tag, desc, GETPC()); +} + uint64_t mte_mops_probe(CPUARMState *env, uint64_t ptr, uint64_t size, uint32_t desc) { diff --git a/target/arm/tcg/translate-a64.c b/target/arm/tcg/translate-a64.c index cae8d6826f..6ce123eafc 100644 --- a/target/arm/tcg/translate-a64.c +++ b/target/arm/tcg/translate-a64.c @@ -3150,41 +3150,25 @@ static void handle_sys(DisasContext *s, bool isread, return; case ARM_CP_DC_GVA: { - TCGv_i64 clean_addr, tag; + TCGv_i32 desc = gen_mtedesc_zva(s); - /* - * DC_GVA, like DC_ZVA, requires that we supply the original - * pointer for an invalid page. Probe that address first. - */ tcg_rt = cpu_reg(s, rt); - clean_addr = clean_data_tbi(s, tcg_rt); - gen_probe_access(s, clean_addr, MMU_DATA_STORE, MO_8); - if (s->ata[0]) { - /* Extract the tag from the register to match STZGM. */ - tag = tcg_temp_new_i64(); - tcg_gen_shri_i64(tag, tcg_rt, 56); - gen_helper_stzgm_tags(tcg_env, clean_addr, tag, - tcg_constant_i32(s->mtx)); + gen_helper_dc_gva(tcg_env, tcg_rt, desc); + } else { + gen_helper_dc_gva_stub(tcg_env, tcg_rt, desc); } } return; case ARM_CP_DC_GZVA: { - TCGv_i64 clean_addr, tag; TCGv_i32 desc = gen_mtedesc_zva(s); - /* For DC_GZVA, we can rely on DC_ZVA for the proper fault. */ tcg_rt = cpu_reg(s, rt); - clean_addr = clean_data_tbi(s, tcg_rt); - gen_helper_dc_zva(tcg_env, clean_addr, desc); - if (s->ata[0]) { - /* Extract the tag from the register to match STZGM. */ - tag = tcg_temp_new_i64(); - tcg_gen_shri_i64(tag, tcg_rt, 56); - gen_helper_stzgm_tags(tcg_env, clean_addr, tag, - tcg_constant_i32(s->mtx)); + gen_helper_dc_gzva(tcg_env, tcg_rt, desc); + } else { + gen_helper_dc_zva(tcg_env, tcg_rt, desc); } } return; @@ -4767,6 +4751,7 @@ static bool trans_LD_single_repl(DisasContext *s, arg_LD_single_repl *a) static bool trans_STZGM(DisasContext *s, arg_ldst_tag *a) { TCGv_i64 addr, tcg_rt; + TCGv_i32 desc; if (!dc_isar_feature(aa64_mte, s)) { return false; @@ -4781,17 +4766,19 @@ static bool trans_STZGM(DisasContext *s, arg_ldst_tag *a) addr = read_cpu_reg_sp(s, a->rn, true); tcg_gen_addi_i64(addr, addr, a->imm); - tcg_rt = cpu_reg(s, a->rt); + desc = gen_mtedesc_zva(s); if (s->ata[0]) { - gen_helper_stzgm_tags(tcg_env, addr, tcg_rt, tcg_constant_i32(s->mtx)); + tcg_rt = cpu_reg(s, a->rt); + gen_helper_stzgm(tcg_env, addr, tcg_rt, desc); + } else { + /* + * The non-tags portion of STZGM is mostly like DC_ZVA, + * except the alignment happens before the access. + */ + tcg_gen_andi_i64(addr, addr, -s->dcz_blocksize); + gen_helper_dc_zva(tcg_env, addr, desc); } - /* - * The non-tags portion of STZGM is mostly like DC_ZVA, - * except the alignment happens before the access. - */ - tcg_gen_andi_i64(addr, addr, -s->dcz_blocksize); - gen_helper_dc_zva(tcg_env, addr, gen_mtedesc_zva(s)); return true; } -- 2.43.0
