The branch main has been updated by andrew: URL: https://cgit.FreeBSD.org/src/commit/?id=80b4129bef8b908eb19fe47853cb6e45e4513d76
commit 80b4129bef8b908eb19fe47853cb6e45e4513d76 Author: Andrew Turner <[email protected]> AuthorDate: 2026-03-05 14:28:01 +0000 Commit: Andrew Turner <[email protected]> CommitDate: 2026-03-05 15:13:30 +0000 arm64: Optimise the repeated TLBI workaround It has been reported that the overhead of repeating all TLBI instructions is too large [1]. The Software Developer Errata Notices (SDEN) for the relevant Arm CPUs have been updated so a single "tlbi vale1is, xzr" followed by "dsb ish" is sufficient to work around the issues. Replace the places we repeat TLBI instructions with the new sequence. [1] https://lore.kernel.org/linux-arm-kernel/[email protected]/ Reviewed by: kib Sponsored by: Arm Ltd Differential Revision: https://reviews.freebsd.org/D55646 --- sys/arm64/arm64/pmap.c | 44 ++++++++++++++------------------------------ 1 file changed, 14 insertions(+), 30 deletions(-) diff --git a/sys/arm64/arm64/pmap.c b/sys/arm64/arm64/pmap.c index 25192bfef653..86ef7359bbe9 100644 --- a/sys/arm64/arm64/pmap.c +++ b/sys/arm64/arm64/pmap.c @@ -1926,17 +1926,13 @@ pmap_s1_invalidate_page(pmap_t pmap, vm_offset_t va, bool final_only) r = TLBI_VA(va); if (pmap == kernel_pmap) { pmap_s1_invalidate_kernel(r, final_only); - if (pmap_multiple_tlbi) { - dsb(ish); - pmap_s1_invalidate_kernel(r, final_only); - } } else { r |= ASID_TO_OPERAND(COOKIE_TO_ASID(pmap->pm_cookie)); pmap_s1_invalidate_user(r, final_only); - if (pmap_multiple_tlbi) { - dsb(ish); - pmap_s1_invalidate_user(r, final_only); - } + } + if (pmap_multiple_tlbi) { + dsb(ish); + __asm __volatile("tlbi vale1is, xzr" ::: "memory"); } dsb(ish); isb(); @@ -1978,24 +1974,16 @@ pmap_s1_invalidate_strided(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, end = TLBI_VA(eva); for (r = start; r < end; r += TLBI_VA(stride)) pmap_s1_invalidate_kernel(r, final_only); - - if (pmap_multiple_tlbi) { - dsb(ish); - for (r = start; r < end; r += TLBI_VA(stride)) - pmap_s1_invalidate_kernel(r, final_only); - } } else { start = end = ASID_TO_OPERAND(COOKIE_TO_ASID(pmap->pm_cookie)); start |= TLBI_VA(sva); end |= TLBI_VA(eva); for (r = start; r < end; r += TLBI_VA(stride)) pmap_s1_invalidate_user(r, final_only); - - if (pmap_multiple_tlbi) { - dsb(ish); - for (r = start; r < end; r += TLBI_VA(stride)) - pmap_s1_invalidate_user(r, final_only); - } + } + if (pmap_multiple_tlbi) { + dsb(ish); + __asm __volatile("tlbi vale1is, xzr" ::: "memory"); } dsb(ish); isb(); @@ -2036,11 +2024,11 @@ pmap_s1_invalidate_all_kernel(void) { dsb(ishst); __asm __volatile("tlbi vmalle1is"); - dsb(ish); if (pmap_multiple_tlbi) { - __asm __volatile("tlbi vmalle1is"); dsb(ish); + __asm __volatile("tlbi vale1is, xzr" ::: "memory"); } + dsb(ish); isb(); } @@ -2058,17 +2046,13 @@ pmap_s1_invalidate_all(pmap_t pmap) dsb(ishst); if (pmap == kernel_pmap) { __asm __volatile("tlbi vmalle1is"); - if (pmap_multiple_tlbi) { - dsb(ish); - __asm __volatile("tlbi vmalle1is"); - } } else { r = ASID_TO_OPERAND(COOKIE_TO_ASID(pmap->pm_cookie)); __asm __volatile("tlbi aside1is, %0" : : "r" (r)); - if (pmap_multiple_tlbi) { - dsb(ish); - __asm __volatile("tlbi aside1is, %0" : : "r" (r)); - } + } + if (pmap_multiple_tlbi) { + dsb(ish); + __asm __volatile("tlbi vale1is, xzr" ::: "memory"); } dsb(ish); isb();
