Module Name: src
Committed By: skrll
Date: Sun Oct 10 07:15:25 UTC 2021
Modified Files:
src/sys/arch/aarch64/aarch64: pmap.c
src/sys/arch/aarch64/conf: files.aarch64
src/sys/arch/aarch64/include: cpu.h pmap.h pte.h types.h
Added Files:
src/sys/arch/aarch64/aarch64: aarch64_tlb.c
Log Message:
Use sys/uvm/pmap/pmap_tlb.c on Aarch64 in the same way that some Arm, MIPS,
and some PPC kernels do. This removes the limitation of 256 processes on
CPUs with 8bit ASID field, e.g. Apple M1.
Additionally the following changes have been made
- removed a couple of unnecessary aarch64_tlbi_all calls
- removed any invalidation after freeing page tables due to
_pmap_sweep_pdp. This was never necessary afaict.
- all kernel mappings are marked global and userland mapping not-global.
Performance testing hasn't show a significant difference. The data here
is from building a kernel on an lx2k system with nvme.
before
1489.6u 400.4s 2:40.65 1176.5% 228+224k 0+32289io 57pf+0w
1482.6u 403.2s 2:38.49 1189.9% 228+222k 0+32274io 46pf+0w
1485.4u 402.2s 2:37.27 1200.2% 228+222k 0+32275io 12pf+0w
after
1493.9u 404.6s 2:37.50 1205.4% 227+221k 0+32265io 48pf+0w
1485.0u 408.0s 2:38.54 1194.0% 227+222k 0+32272io 36pf+0w
1484.3u 407.0s 2:35.88 1213.3% 228+224k 0+32268io 14pf+0w
>>> stats.ttest_ind([160.65,158.49,157.27], [157.5,158.54,155.88])
Ttest_indResult(statistic=1.1923622711296888, pvalue=0.2990182944606766)
>>>
To generate a diff of this commit:
cvs rdiff -u -r0 -r1.1 src/sys/arch/aarch64/aarch64/aarch64_tlb.c
cvs rdiff -u -r1.116 -r1.117 src/sys/arch/aarch64/aarch64/pmap.c
cvs rdiff -u -r1.33 -r1.34 src/sys/arch/aarch64/conf/files.aarch64
cvs rdiff -u -r1.39 -r1.40 src/sys/arch/aarch64/include/cpu.h
cvs rdiff -u -r1.48 -r1.49 src/sys/arch/aarch64/include/pmap.h
cvs rdiff -u -r1.12 -r1.13 src/sys/arch/aarch64/include/pte.h
cvs rdiff -u -r1.19 -r1.20 src/sys/arch/aarch64/include/types.h
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/aarch64/aarch64/pmap.c
diff -u src/sys/arch/aarch64/aarch64/pmap.c:1.116 src/sys/arch/aarch64/aarch64/pmap.c:1.117
--- src/sys/arch/aarch64/aarch64/pmap.c:1.116 Thu Sep 30 21:19:16 2021
+++ src/sys/arch/aarch64/aarch64/pmap.c Sun Oct 10 07:15:25 2021
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap.c,v 1.116 2021/09/30 21:19:16 skrll Exp $ */
+/* $NetBSD: pmap.c,v 1.117 2021/10/10 07:15:25 skrll Exp $ */
/*
* Copyright (c) 2017 Ryo Shimizu <r...@nerv.org>
@@ -27,7 +27,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.116 2021/09/30 21:19:16 skrll Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.117 2021/10/10 07:15:25 skrll Exp $");
#include "opt_arm_debug.h"
#include "opt_ddb.h"
@@ -41,6 +41,7 @@ __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.1
#include <sys/asan.h>
#include <sys/atomic.h>
+#include <sys/cpu.h>
#include <sys/kmem.h>
#include <sys/vmem.h>
@@ -181,10 +182,10 @@ PMAP_COUNTER(unwire_failure, "pmap_unwir
* change the pte to accessible temporarly before cpu_icache_sync_range().
* this macro modifies PTE (*ptep). need to update PTE after this.
*/
-#define PTE_ICACHE_SYNC_PAGE(pte, ptep, pm, va, ll) \
+#define PTE_ICACHE_SYNC_PAGE(pte, ptep, asid, va, ll) \
do { \
atomic_swap_64((ptep), (pte) | LX_BLKPAG_AF); \
- AARCH64_TLBI_BY_ASID_VA((pm)->pm_asid, (va), (ll)); \
+ AARCH64_TLBI_BY_ASID_VA((asid), (va), (ll)); \
cpu_icache_sync_range((va), PAGE_SIZE); \
} while (0/*CONSTCOND*/)
@@ -336,7 +337,6 @@ pmap_map_chunk(vaddr_t va, paddr_t pa, v
attr = _pmap_pte_adjust_prot(0, prot, VM_PROT_ALL, false);
attr = _pmap_pte_adjust_cacheflags(attr, flags);
pmapboot_enter_range(va, pa, resid, attr, printf);
- aarch64_tlbi_all();
return resid;
}
@@ -472,18 +472,23 @@ pmap_bootstrap(vaddr_t vstart, vaddr_t v
virtual_end = vend;
pmap_maxkvaddr = vstart;
- aarch64_tlbi_all();
-
l0pa = reg_ttbr1_el1_read();
l0 = (void *)AARCH64_PA_TO_KVA(l0pa);
+ pmap_tlb_info_init(&pmap_tlb0_info);
+
memset(&kernel_pmap, 0, sizeof(kernel_pmap));
+
kpm = pmap_kernel();
- kpm->pm_asid = 0;
+ struct pmap_asid_info * const pai = PMAP_PAI(kpm, cpu_tlb_info(ci));
+
+ pai->pai_asid = KERNEL_PID;
kpm->pm_refcnt = 1;
kpm->pm_idlepdp = 0;
kpm->pm_l0table = l0;
kpm->pm_l0table_pa = l0pa;
+ kpm->pm_onproc = kcpuset_running;
+ kpm->pm_active = kcpuset_running;
kpm->pm_activated = true;
LIST_INIT(&kpm->pm_vmlist);
LIST_INIT(&kpm->pm_pvlist); /* not used for kernel pmap */
@@ -493,6 +498,12 @@ pmap_bootstrap(vaddr_t vstart, vaddr_t v
CTASSERT(sizeof(kpm->pm_stats.resident_count) == sizeof(long));
}
+void
+pmap_md_tlb_info_attach(struct pmap_tlb_info *ti, struct cpu_info *ci)
+{
+ /* nothing */
+}
+
static inline void
_pmap_adj_wired_count(struct pmap *pm, int adj)
{
@@ -556,9 +567,7 @@ pmap_init(void)
32, 0, PR_LARGECACHE, "pvpl", NULL, IPL_NONE, _pmap_pv_ctor,
NULL, NULL);
- int nmaxproc = cpu_maxproc();
- if (maxproc > nmaxproc)
- maxproc = nmaxproc;
+ pmap_tlb_info_evcnt_attach(&pmap_tlb0_info);
}
void
@@ -674,18 +683,16 @@ pmap_free_pdp(struct pmap *pm, struct vm
}
/* free empty page table pages */
-static int
+static void
_pmap_sweep_pdp(struct pmap *pm)
{
struct vm_page *pg, *tmp;
pd_entry_t *ptep_in_parent, opte __diagused;
paddr_t pa, pdppa;
- int nsweep;
uint16_t wirecount __diagused;
KASSERT(mutex_owned(&pm->pm_lock) || pm->pm_refcnt == 0);
- nsweep = 0;
LIST_FOREACH_SAFE(pg, &pm->pm_vmlist, pageq.list, tmp) {
if (pg->wire_count != 1)
continue;
@@ -698,7 +705,6 @@ _pmap_sweep_pdp(struct pmap *pm)
if (ptep_in_parent == NULL) {
/* no parent */
pmap_free_pdp(pm, pg);
- nsweep++;
continue;
}
@@ -708,7 +714,6 @@ _pmap_sweep_pdp(struct pmap *pm)
wirecount = --pg->wire_count; /* 1 -> 0 */
KASSERT(wirecount == 0);
pmap_free_pdp(pm, pg);
- nsweep++;
/* L3->L2->L1. no need for L0 */
pdppa = AARCH64_KVA_TO_PA(trunc_page((vaddr_t)ptep_in_parent));
@@ -722,12 +727,10 @@ _pmap_sweep_pdp(struct pmap *pm)
/* decrement wire_count of parent */
wirecount = --pg->wire_count;
KASSERTMSG(pg->wire_count <= (Ln_ENTRIES + 1),
- "pm=%p[%d], pg=%p, wire_count=%d",
- pm, pm->pm_asid, pg, pg->wire_count);
+ "pm=%p, pg=%p, wire_count=%d",
+ pm, pg, pg->wire_count);
}
pm->pm_idlepdp = 0;
-
- return nsweep;
}
static void
@@ -977,11 +980,14 @@ pmap_icache_sync_range(pmap_t pm, vaddr_
* change to accessible temporally
* to do cpu_icache_sync_range()
*/
+ struct pmap_asid_info * const pai = PMAP_PAI(pm,
+ cpu_tlb_info(ci));
+
atomic_swap_64(ptep, pte | LX_BLKPAG_AF);
- AARCH64_TLBI_BY_ASID_VA(pm->pm_asid, va, true);
+ AARCH64_TLBI_BY_ASID_VA(pai->pai_asid, va, true);
cpu_icache_sync_range(va, len);
atomic_swap_64(ptep, pte);
- AARCH64_TLBI_BY_ASID_VA(pm->pm_asid, va, true);
+ AARCH64_TLBI_BY_ASID_VA(pai->pai_asid, va, true);
}
}
@@ -1179,14 +1185,15 @@ pmap_db_mdpg_print(struct vm_page *pg, v
KASSERT(pv == &pp->pp_pv);
continue;
}
- pr(" pv[%d] pv=%p\n",
- i, pv);
- pr(" pv[%d].pv_pmap = %p (asid=%d)\n",
- i, pv->pv_pmap, pv->pv_pmap->pm_asid);
- pr(" pv[%d].pv_va = %016lx (color=%d)\n",
- i, trunc_page(pv->pv_va), _pmap_color(pv->pv_va));
- pr(" pv[%d].pv_ptep = %p\n",
- i, pv->pv_ptep);
+ struct pmap * const pm = pv->pv_pmap;
+ struct pmap_asid_info * const pai = PMAP_PAI(pm,
+ cpu_tlb_info(ci));
+
+ pr(" pv[%d] pv=%p\n", i, pv);
+ pr(" pv[%d].pv_pmap = %p (asid=%d)\n", i, pm, pai->pai_asid);
+ pr(" pv[%d].pv_va = %016lx (color=%d)\n", i,
+ trunc_page(pv->pv_va), _pmap_color(pv->pv_va));
+ pr(" pv[%d].pv_ptep = %p\n", i, pv->pv_ptep);
i++;
}
}
@@ -1298,7 +1305,11 @@ _pmap_protect_pv(struct pmap_page *pp, s
/* new prot = prot & pteprot & mdattr */
pte = _pmap_pte_adjust_prot(pte, prot & pteprot, mdattr, user);
atomic_swap_64(ptep, pte);
- AARCH64_TLBI_BY_ASID_VA(pv->pv_pmap->pm_asid, trunc_page(pv->pv_va),
+
+ struct pmap * const pm = pv->pv_pmap;
+ struct pmap_asid_info * const pai = PMAP_PAI(pm, cpu_tlb_info(ci));
+
+ AARCH64_TLBI_BY_ASID_VA(pai->pai_asid, trunc_page(pv->pv_va),
true);
}
@@ -1392,23 +1403,27 @@ pmap_protect(struct pmap *pm, vaddr_t sv
executable = l3pte_executable(pte, user);
pte = _pmap_pte_adjust_prot(pte, prot, mdattr, user);
+ struct pmap_asid_info * const pai = PMAP_PAI(pm,
+ cpu_tlb_info(ci));
if (!executable && (prot & VM_PROT_EXECUTE)) {
/* non-exec -> exec */
UVMHIST_LOG(pmaphist, "icache_sync: "
"pm=%p, va=%016lx, pte: %016lx -> %016lx",
pm, va, opte, pte);
+
if (!l3pte_readable(pte)) {
- PTE_ICACHE_SYNC_PAGE(pte, ptep, pm, va, true);
+ PTE_ICACHE_SYNC_PAGE(pte, ptep, pai->pai_asid,
+ va, true);
atomic_swap_64(ptep, pte);
- AARCH64_TLBI_BY_ASID_VA(pm->pm_asid, va, true);
+ AARCH64_TLBI_BY_ASID_VA(pai->pai_asid, va, true);
} else {
atomic_swap_64(ptep, pte);
- AARCH64_TLBI_BY_ASID_VA(pm->pm_asid, va, true);
+ AARCH64_TLBI_BY_ASID_VA(pai->pai_asid, va, true);
cpu_icache_sync_range(va, PAGE_SIZE);
}
} else {
atomic_swap_64(ptep, pte);
- AARCH64_TLBI_BY_ASID_VA(pm->pm_asid, va, true);
+ AARCH64_TLBI_BY_ASID_VA(pai->pai_asid, va, true);
}
}
@@ -1420,11 +1435,11 @@ void
pmap_activate(struct lwp *l)
{
struct pmap *pm = l->l_proc->p_vmspace->vm_map.pmap;
- uint64_t ttbr0, tcr;
+ uint64_t tcr;
UVMHIST_FUNC(__func__);
- UVMHIST_CALLARGS(pmaphist, "lwp=%p asid=%d (pid=%d)", l, pm->pm_asid,
- l->l_proc->p_pid, 0);
+ UVMHIST_CALLARGS(pmaphist, "lwp=%p (pid=%d, kernel=%u)", l,
+ l->l_proc->p_pid, pm == pmap_kernel() ? 1 : 0, 0);
KASSERT((reg_tcr_el1_read() & TCR_EPD0) != 0);
@@ -1435,15 +1450,13 @@ pmap_activate(struct lwp *l)
KASSERT(pm->pm_l0table != NULL);
+ /* this calls tlb_set_asid which calls cpu_set_ttbr0 */
+ pmap_tlb_asid_acquire(pm, l);
- /* XXX: allocate asid, and regenerate if needed */
- if (pm->pm_asid == -1)
- pm->pm_asid = l->l_proc->p_pid;
-
- ttbr0 =
- __SHIFTIN(pm->pm_asid, TTBR_ASID) |
- __SHIFTIN(pm->pm_l0table_pa, TTBR_BADDR);
- cpu_set_ttbr0(ttbr0);
+ struct pmap_asid_info * const pai __debugused = PMAP_PAI(pm,
+ cpu_tlb_info(ci));
+
+ UVMHIST_LOG(pmaphist, "lwp=%p, asid=%d", l, pai->pai_asid, 0, 0);
/* Re-enable translation table walks using TTBR0 */
tcr = reg_tcr_el1_read();
@@ -1462,17 +1475,21 @@ pmap_deactivate(struct lwp *l)
uint64_t tcr;
UVMHIST_FUNC(__func__);
- UVMHIST_CALLARGS(pmaphist, "lwp=%p, asid=%d", l, pm->pm_asid, 0, 0);
-
- if (pm == pmap_kernel())
- return;
+ UVMHIST_CALLARGS(pmaphist, "lwp=%p (pid=%d, (kernel=%u))", l,
+ l->l_proc->p_pid, pm == pmap_kernel() ? 1 : 0, 0);
/* Disable translation table walks using TTBR0 */
tcr = reg_tcr_el1_read();
reg_tcr_el1_write(tcr | TCR_EPD0);
isb();
- /* XXX */
+ struct pmap_asid_info * const pai __debugused = PMAP_PAI(pm,
+ cpu_tlb_info(ci));
+
+ UVMHIST_LOG(pmaphist, "lwp=%p, asid=%d", l, pai->pai_asid, 0, 0);
+
+ pmap_tlb_asid_deactivate(pm);
+
pm->pm_activated = false;
PMAP_COUNT(deactivate);
@@ -1490,11 +1507,14 @@ pmap_create(void)
memset(pm, 0, sizeof(*pm));
pm->pm_refcnt = 1;
pm->pm_idlepdp = 0;
- pm->pm_asid = -1;
LIST_INIT(&pm->pm_vmlist);
LIST_INIT(&pm->pm_pvlist);
mutex_init(&pm->pm_lock, MUTEX_DEFAULT, IPL_NONE);
+ kcpuset_create(&pm->pm_active, true);
+ kcpuset_create(&pm->pm_onproc, true);
+ pm->pm_remove_all = false;
+
pm->pm_l0table_pa = pmap_alloc_pdp(pm, NULL, 0, true);
KASSERT(pm->pm_l0table_pa != POOL_PADDR_INVALID);
pm->pm_l0table = (pd_entry_t *)AARCH64_PA_TO_KVA(pm->pm_l0table_pa);
@@ -1514,8 +1534,8 @@ pmap_destroy(struct pmap *pm)
UVMHIST_FUNC(__func__);
UVMHIST_CALLARGS(pmaphist,
- "pm=%p, pm_l0table=%016lx, pm_l0table_pa=%016lx, refcnt=%d",
- pm, pm->pm_l0table, pm->pm_l0table_pa, pm->pm_refcnt);
+ "pm=%p, pm_l0table=%016lx, pm_remove_all=%jd, refcnt=%jd",
+ pm, pm->pm_l0table, pm->pm_remove_all, pm->pm_refcnt);
if (pm == NULL)
return;
@@ -1523,19 +1543,23 @@ pmap_destroy(struct pmap *pm)
if (pm == pmap_kernel())
panic("cannot destroy kernel pmap");
+ if (pm->pm_remove_all) {
+ pmap_tlb_asid_release_all(pm);
+ pm->pm_remove_all = false;
+ }
+
refcnt = atomic_dec_uint_nv(&pm->pm_refcnt);
if (refcnt > 0)
return;
KASSERT(LIST_EMPTY(&pm->pm_pvlist));
- /*
- * no need to call aarch64_tlbi_by_asid(pm->pm_asid).
- * TLB should already be invalidated in pmap_remove_all()
- */
_pmap_free_pdp_all(pm, true);
mutex_destroy(&pm->pm_lock);
+ kcpuset_destroy(pm->pm_active);
+ kcpuset_destroy(pm->pm_onproc);
+
pool_cache_put(&_pmap_cache, pm);
PMAP_COUNT(destroy);
@@ -1653,8 +1677,8 @@ _pmap_pdp_delref(struct pmap *pm, paddr_
/* decrement wire_count of parent */
wirecount = atomic_add_32_nv(&pg->wire_count, -1);
KASSERTMSG(pg->wire_count <= (Ln_ENTRIES + 1),
- "pm=%p[%d], pg=%p, wire_count=%d",
- pm, pm->pm_asid, pg, pg->wire_count);
+ "pm=%p, pg=%p, wire_count=%d",
+ pm, pg, pg->wire_count);
}
return removed;
@@ -1822,10 +1846,8 @@ _pmap_enter(struct pmap *pm, vaddr_t va,
}
pm_lock(pm);
- if (pm->pm_idlepdp >= PDPSWEEP_TRIGGER &&
- _pmap_sweep_pdp(pm) != 0) {
- /* several L1-L3 page table pages have been freed */
- aarch64_tlbi_by_asid(pm->pm_asid);
+ if (pm->pm_idlepdp >= PDPSWEEP_TRIGGER) {
+ _pmap_sweep_pdp(pm);
}
}
@@ -1937,7 +1959,10 @@ _pmap_enter(struct pmap *pm, vaddr_t va,
bool pdpremoved = _pmap_pdp_delref(pm,
AARCH64_KVA_TO_PA(trunc_page(
(vaddr_t)ptep)), true);
- AARCH64_TLBI_BY_ASID_VA(pm->pm_asid,
+ struct pmap_asid_info * const pai = PMAP_PAI(pm,
+ cpu_tlb_info(ci));
+
+ AARCH64_TLBI_BY_ASID_VA(pai->pai_asid,
va, !pdpremoved);
}
PMAP_COUNT(pv_entry_cannotalloc);
@@ -1963,7 +1988,8 @@ _pmap_enter(struct pmap *pm, vaddr_t va,
}
#endif
- attr = _pmap_pte_adjust_prot(L3_PAGE, prot, mdattr, user);
+ attr = L3_PAGE | (kenter ? 0 : LX_BLKPAG_NG);
+ attr = _pmap_pte_adjust_prot(attr, prot, mdattr, user);
attr = _pmap_pte_adjust_cacheflags(attr, flags);
if (VM_MAXUSER_ADDRESS > va)
attr |= LX_BLKPAG_APUSER;
@@ -1975,23 +2001,27 @@ _pmap_enter(struct pmap *pm, vaddr_t va,
pte = pa | attr;
+ struct pmap_asid_info * const pai = PMAP_PAI(pm, cpu_tlb_info(ci));
+ const tlb_asid_t asid = pai->pai_asid;
+
if (need_sync_icache) {
/* non-exec -> exec */
UVMHIST_LOG(pmaphist,
"icache_sync: pm=%p, va=%016lx, pte: %016lx -> %016lx",
pm, va, opte, pte);
+
if (!l3pte_readable(pte)) {
- PTE_ICACHE_SYNC_PAGE(pte, ptep, pm, va, l3only);
+ PTE_ICACHE_SYNC_PAGE(pte, ptep, asid, va, l3only);
atomic_swap_64(ptep, pte);
- AARCH64_TLBI_BY_ASID_VA(pm->pm_asid, va ,true);
+ AARCH64_TLBI_BY_ASID_VA(asid, va ,true);
} else {
atomic_swap_64(ptep, pte);
- AARCH64_TLBI_BY_ASID_VA(pm->pm_asid, va, l3only);
+ AARCH64_TLBI_BY_ASID_VA(asid, va, l3only);
cpu_icache_sync_range(va, PAGE_SIZE);
}
} else {
atomic_swap_64(ptep, pte);
- AARCH64_TLBI_BY_ASID_VA(pm->pm_asid, va, l3only);
+ AARCH64_TLBI_BY_ASID_VA(asid, va, l3only);
}
if (pte & LX_BLKPAG_OS_WIRED) {
@@ -2025,6 +2055,42 @@ pmap_enter(struct pmap *pm, vaddr_t va,
return _pmap_enter(pm, va, pa, prot, flags, false);
}
+
+
+void
+pmap_update(pmap_t pm)
+{
+
+ UVMHIST_FUNC(__func__);
+ UVMHIST_CALLARGS(maphist, "pm=%#jx remove_all %jd", (uintptr_t)pm,
+ pm->pm_remove_all, 0, 0);
+
+ kpreempt_disable();
+ /*
+ * If pmap_remove_all was called, we deactivated ourselves and released
+ * our ASID. Now we have to reactivate ourselves.
+ */
+ if (__predict_false(pm->pm_remove_all)) {
+ pm->pm_remove_all = false;
+
+ KASSERT(pm != pmap_kernel());
+
+ /* this calls tlb_set_asid which calls cpu_set_ttbr0 */
+ pmap_tlb_asid_acquire(pm, curlwp);
+
+ /* Enable translation table walks using TTBR0 */
+ uint64_t tcr = reg_tcr_el1_read();
+ reg_tcr_el1_write(tcr & ~TCR_EPD0);
+ isb();
+
+ pm->pm_activated = true;
+ }
+
+ kpreempt_enable();
+
+ UVMHIST_LOG(maphist, " <-- done", 0, 0, 0, 0);
+}
+
bool
pmap_remove_all(struct pmap *pm)
{
@@ -2036,8 +2102,28 @@ pmap_remove_all(struct pmap *pm)
UVMHIST_FUNC(__func__);
UVMHIST_CALLARGS(pmaphist, "pm=%p", pm, 0, 0, 0);
- if (pm == pmap_kernel())
- return false;
+ KASSERT(pm != pmap_kernel());
+
+ struct cpu_info * const ci = curcpu();
+ // This should be the last CPU with this pmap onproc
+ KASSERT(!kcpuset_isotherset(pm->pm_onproc, cpu_index(ci)));
+ if (kcpuset_isset(pm->pm_onproc, cpu_index(ci))) {
+ /* Disable translation table walks using TTBR0 */
+ uint64_t tcr = reg_tcr_el1_read();
+ reg_tcr_el1_write(tcr | TCR_EPD0);
+ isb();
+
+ pmap_tlb_asid_deactivate(pm);
+ }
+
+ KASSERT(kcpuset_iszero(pm->pm_onproc));
+
+ pmap_tlb_asid_release_all(pm);
+ pm->pm_remove_all = true;
+
+ struct pmap_asid_info * const pai __debugused = PMAP_PAI(pm,
+ cpu_tlb_info(ci));
+ UVMHIST_LOG(pmaphist, "pm=%p, asid=%d", pm, pai->pai_asid, 0, 0);
pm_lock(pm);
@@ -2046,16 +2132,16 @@ pmap_remove_all(struct pmap *pm)
pte = *ptep;
KASSERTMSG(lxpde_valid(pte),
- "pte is not valid: pmap=%p, asid=%d, va=%016lx",
- pm, pm->pm_asid, pv->pv_va);
+ "pte is not valid: pmap=%p, va=%016lx",
+ pm, pv->pv_va);
pa = lxpde_pa(pte);
pp = phys_to_pp(pa);
KASSERTMSG(pp != NULL,
"no pmap_page of physical address:%016lx, "
- "pmap=%p, asid=%d, va=%016lx",
- pa, pm, pm->pm_asid, pv->pv_va);
+ "pmap=%p, va=%016lx",
+ pa, pm, pv->pv_va);
pmap_pv_lock(pp);
opv = _pmap_remove_pv(pp, pm, trunc_page(pv->pv_va), pte);
@@ -2071,7 +2157,6 @@ pmap_remove_all(struct pmap *pm)
/* clear L0 page table page */
pmap_zero_page(pm->pm_l0table_pa);
- aarch64_tlbi_by_asid(pm->pm_asid);
/* free L1-L3 page table pages, but not L0 */
_pmap_free_pdp_all(pm, false);
@@ -2135,10 +2220,12 @@ _pmap_remove(struct pmap *pm, vaddr_t sv
pte = atomic_swap_64(ptep, 0);
if (!lxpde_valid(pte))
continue;
+ struct pmap_asid_info * const pai = PMAP_PAI(pm,
+ cpu_tlb_info(ci));
pdpremoved = _pmap_pdp_delref(pm,
AARCH64_KVA_TO_PA(trunc_page((vaddr_t)ptep)), true);
- AARCH64_TLBI_BY_ASID_VA(pm->pm_asid, va, !pdpremoved);
+ AARCH64_TLBI_BY_ASID_VA(pai->pai_asid, va, !pdpremoved);
if (pdpremoved) {
/*
@@ -2196,12 +2283,14 @@ pmap_page_remove(struct pmap_page *pp, v
continue;
}
opte = atomic_swap_64(pv->pv_ptep, 0);
+ struct pmap_asid_info * const pai = PMAP_PAI(pm, cpu_tlb_info(ci));
const vaddr_t va = trunc_page(pv->pv_va);
+
if (lxpde_valid(opte)) {
_pmap_pdp_delref(pm,
AARCH64_KVA_TO_PA(trunc_page(
(vaddr_t)pv->pv_ptep)), false);
- AARCH64_TLBI_BY_ASID_VA(pm->pm_asid, va, true);
+ AARCH64_TLBI_BY_ASID_VA(pai->pai_asid, va, true);
if ((opte & LX_BLKPAG_OS_WIRED) != 0) {
_pmap_adj_wired_count(pm, -1);
@@ -2441,7 +2530,8 @@ pmap_fault_fixup(struct pmap *pm, vaddr_
pmap_pv_unlock(pp);
atomic_swap_64(ptep, pte);
- AARCH64_TLBI_BY_ASID_VA(pm->pm_asid, va, true);
+ struct pmap_asid_info * const pai = PMAP_PAI(pm, cpu_tlb_info(ci));
+ AARCH64_TLBI_BY_ASID_VA(pai->pai_asid, va, true);
fixed = true;
@@ -2510,7 +2600,9 @@ pmap_clear_modify(struct vm_page *pg)
goto tryagain;
}
- AARCH64_TLBI_BY_ASID_VA(pv->pv_pmap->pm_asid, va, true);
+ struct pmap * const pm = pv->pv_pmap;
+ struct pmap_asid_info * const pai = PMAP_PAI(pm, cpu_tlb_info(ci));
+ AARCH64_TLBI_BY_ASID_VA(pai->pai_asid, va, true);
UVMHIST_LOG(pmaphist,
"va=%016llx, ptep=%p, pa=%016lx, RW -> RO",
@@ -2568,7 +2660,9 @@ pmap_clear_reference(struct vm_page *pg)
goto tryagain;
}
- AARCH64_TLBI_BY_ASID_VA(pv->pv_pmap->pm_asid, va, true);
+ struct pmap * const pm = pv->pv_pmap;
+ struct pmap_asid_info * const pai = PMAP_PAI(pm, cpu_tlb_info(ci));
+ AARCH64_TLBI_BY_ASID_VA(pai->pai_asid, va, true);
UVMHIST_LOG(pmaphist, "va=%016llx, ptep=%p, pa=%016lx, unse AF",
va, ptep, l3pte_pa(pte), 0);
@@ -2610,8 +2704,9 @@ void
pmap_db_pmap_print(struct pmap *pm,
void (*pr)(const char *, ...) __printflike(1, 2))
{
+ struct pmap_asid_info * const pai = PMAP_PAI(pm, cpu_tlb_info(ci));
- pr(" pm_asid = %d\n", pm->pm_asid);
+ pr(" pm_asid = %d\n", pai->pai_asid);
pr(" pm_l0table = %p\n", pm->pm_l0table);
pr(" pm_l0table_pa = %lx\n", pm->pm_l0table_pa);
pr(" pm_activated = %d\n\n", pm->pm_activated);
Index: src/sys/arch/aarch64/conf/files.aarch64
diff -u src/sys/arch/aarch64/conf/files.aarch64:1.33 src/sys/arch/aarch64/conf/files.aarch64:1.34
--- src/sys/arch/aarch64/conf/files.aarch64:1.33 Thu Sep 23 06:56:26 2021
+++ src/sys/arch/aarch64/conf/files.aarch64 Sun Oct 10 07:15:25 2021
@@ -1,4 +1,4 @@
-# $NetBSD: files.aarch64,v 1.33 2021/09/23 06:56:26 ryo Exp $
+# $NetBSD: files.aarch64,v 1.34 2021/10/10 07:15:25 skrll Exp $
defflag opt_cpuoptions.h AARCH64_ALIGNMENT_CHECK
defflag opt_cpuoptions.h AARCH64_EL0_STACK_ALIGNMENT_CHECK
@@ -112,9 +112,11 @@ file arch/aarch64/aarch64/vectors.S
file arch/aarch64/aarch64/vm_machdep.c
# pmap
+file arch/aarch64/aarch64/aarch64_tlb.c
file arch/aarch64/aarch64/pmap.c
file arch/aarch64/aarch64/pmapboot.c
file arch/aarch64/aarch64/pmap_page.S
+file uvm/pmap/pmap_tlb.c
file uvm/pmap/pmap_pvt.c
# EFI runtime (machdep)
Index: src/sys/arch/aarch64/include/cpu.h
diff -u src/sys/arch/aarch64/include/cpu.h:1.39 src/sys/arch/aarch64/include/cpu.h:1.40
--- src/sys/arch/aarch64/include/cpu.h:1.39 Sat Sep 18 12:25:06 2021
+++ src/sys/arch/aarch64/include/cpu.h Sun Oct 10 07:15:25 2021
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu.h,v 1.39 2021/09/18 12:25:06 jmcneill Exp $ */
+/* $NetBSD: cpu.h,v 1.40 2021/10/10 07:15:25 skrll Exp $ */
/*-
* Copyright (c) 2014, 2020 The NetBSD Foundation, Inc.
@@ -113,6 +113,8 @@ struct cpu_info {
int ci_kfpu_spl;
+ tlb_asid_t ci_pmap_asid_cur;
+
/* event counters */
struct evcnt ci_vfp_use;
struct evcnt ci_vfp_reuse;
@@ -158,22 +160,6 @@ static __inline struct cpu_info *lwp_get
#undef curlwp
#define curlwp (aarch64_curlwp())
-static inline int
-cpu_maxproc(void)
-{
- /*
- * the pmap uses PID for ASID.
- */
- switch (__SHIFTOUT(reg_id_aa64mmfr0_el1_read(), ID_AA64MMFR0_EL1_ASIDBITS)) {
- case ID_AA64MMFR0_EL1_ASIDBITS_8BIT:
- return (1U << 8) - 1;
- case ID_AA64MMFR0_EL1_ASIDBITS_16BIT:
- return (1U << 16) - 1;
- default:
- return 0;
- }
-}
-
void cpu_signotify(struct lwp *l);
void cpu_need_proftick(struct lwp *l);
Index: src/sys/arch/aarch64/include/pmap.h
diff -u src/sys/arch/aarch64/include/pmap.h:1.48 src/sys/arch/aarch64/include/pmap.h:1.49
--- src/sys/arch/aarch64/include/pmap.h:1.48 Wed May 19 12:16:01 2021
+++ src/sys/arch/aarch64/include/pmap.h Sun Oct 10 07:15:25 2021
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap.h,v 1.48 2021/05/19 12:16:01 skrll Exp $ */
+/* $NetBSD: pmap.h,v 1.49 2021/10/10 07:15:25 skrll Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -42,6 +42,7 @@
#include <sys/types.h>
#include <sys/pool.h>
#include <sys/queue.h>
+
#include <uvm/uvm_pglist.h>
#include <aarch64/armreg.h>
@@ -54,6 +55,43 @@
#define __HAVE_VM_PAGE_MD
#define __HAVE_PMAP_PV_TRACK 1
+#define PMAP_HWPAGEWALKER 1
+
+#define PMAP_TLB_MAX 1
+#if PMAP_TLB_MAX > 1
+#define PMAP_TLB_NEED_SHOOTDOWN 1
+#endif
+
+#define PMAP_TLB_FLUSH_ASID_ON_RESET (true)
+
+/* Maximum number of ASIDs. Some CPUs have less.*/
+#define PMAP_TLB_NUM_PIDS 65536
+#define PMAP_TLB_BITMAP_LENGTH PMAP_TLB_NUM_PIDS
+#define cpu_set_tlb_info(ci, ti) ((void)((ci)->ci_tlb_info = (ti)))
+#if PMAP_TLB_MAX > 1
+#define cpu_tlb_info(ci) ((ci)->ci_tlb_info)
+#else
+#define cpu_tlb_info(ci) (&pmap_tlb0_info)
+#endif
+
+static inline tlb_asid_t
+pmap_md_tlb_asid_max(void)
+{
+ switch (__SHIFTOUT(reg_id_aa64mmfr0_el1_read(), ID_AA64MMFR0_EL1_ASIDBITS)) {
+ case ID_AA64MMFR0_EL1_ASIDBITS_8BIT:
+ return (1U << 8) - 1;
+ case ID_AA64MMFR0_EL1_ASIDBITS_16BIT:
+ return (1U << 16) - 1;
+ default:
+ return 0;
+ }
+}
+
+#include <uvm/pmap/tlb.h>
+#include <uvm/pmap/pmap_tlb.h>
+
+#define KERNEL_PID 0 /* The kernel uses ASID 0 */
+
#ifndef KASAN
#define PMAP_MAP_POOLPAGE(pa) AARCH64_PA_TO_KVA(pa)
#define PMAP_UNMAP_POOLPAGE(va) AARCH64_KVA_TO_PA(va)
@@ -81,10 +119,21 @@ struct pmap {
struct pmap_statistics pm_stats;
unsigned int pm_refcnt;
unsigned int pm_idlepdp;
- int pm_asid;
+
+ kcpuset_t *pm_onproc;
+ kcpuset_t *pm_active;
+
+ struct pmap_asid_info pm_pai[PMAP_TLB_MAX];
bool pm_activated;
+ bool pm_remove_all;
};
+static inline paddr_t
+pmap_l0pa(struct pmap *pm)
+{
+ return pm->pm_l0table_pa;
+}
+
/*
* should be kept <=32 bytes sized to reduce memory consumption & cache misses,
* but it doesn't...
@@ -173,6 +222,19 @@ struct vm_page_md {
#define l3pte_is_page(pde) (((pde) & LX_TYPE) == L3_TYPE_PAG)
/* l3pte contains always page entries */
+
+static inline uint64_t
+pte_value(pt_entry_t pte)
+{
+ return pte;
+}
+
+static inline bool
+pte_valid_p(pt_entry_t pte)
+{
+ return l3pte_valid(pte);
+}
+
void pmap_bootstrap(vaddr_t, vaddr_t);
bool pmap_fault_fixup(struct pmap *, vaddr_t, vm_prot_t, bool user);
@@ -322,7 +384,6 @@ aarch64_mmap_flags(paddr_t mdpgno)
#define pmap_phys_address(pa) aarch64_ptob((pa))
#define pmap_mmap_flags(ppn) aarch64_mmap_flags((ppn))
-#define pmap_update(pmap) ((void)0)
#define pmap_copy(dp,sp,d,l,s) ((void)0)
#define pmap_wired_count(pmap) ((pmap)->pm_stats.wired_count)
#define pmap_resident_count(pmap) ((pmap)->pm_stats.resident_count)
Index: src/sys/arch/aarch64/include/pte.h
diff -u src/sys/arch/aarch64/include/pte.h:1.12 src/sys/arch/aarch64/include/pte.h:1.13
--- src/sys/arch/aarch64/include/pte.h:1.12 Sat Feb 29 21:34:37 2020
+++ src/sys/arch/aarch64/include/pte.h Sun Oct 10 07:15:25 2021
@@ -1,4 +1,4 @@
-/* $NetBSD: pte.h,v 1.12 2020/02/29 21:34:37 ryo Exp $ */
+/* $NetBSD: pte.h,v 1.13 2021/10/10 07:15:25 skrll Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -110,7 +110,7 @@ typedef uint64_t pt_entry_t; /* L3(4k) t
#define L1_SIZE (1UL << L1_SHIFT)
#define L1_OFFSET (L1_SIZE - 1UL)
#define L1_FRAME (~L1_OFFSET)
-#define L1_BLOCK (LX_BLKPAG_NG | LX_TYPE_BLK | LX_VALID)
+#define L1_BLOCK (LX_TYPE_BLK | LX_VALID)
#define L1_TABLE (LX_TYPE_TBL | LX_VALID)
/* L2 table, 2MB/entry * 512 */
@@ -119,7 +119,7 @@ typedef uint64_t pt_entry_t; /* L3(4k) t
#define L2_SIZE (1UL << L2_SHIFT)
#define L2_OFFSET (L2_SIZE - 1UL)
#define L2_FRAME (~L2_OFFSET)
-#define L2_BLOCK (LX_BLKPAG_NG | LX_TYPE_BLK | LX_VALID)
+#define L2_BLOCK (LX_TYPE_BLK | LX_VALID)
#define L2_TABLE (LX_TYPE_TBL | LX_VALID)
#define L2_BLOCK_MASK __BITS(47,21)
@@ -129,7 +129,7 @@ typedef uint64_t pt_entry_t; /* L3(4k) t
#define L3_SIZE (1UL << L3_SHIFT)
#define L3_OFFSET (L3_SIZE - 1UL)
#define L3_FRAME (~L3_OFFSET)
-#define L3_PAGE (LX_BLKPAG_NG | L3_TYPE_PAG | LX_VALID)
+#define L3_PAGE (L3_TYPE_PAG | LX_VALID)
#define Ln_ENTRIES_SHIFT 9
#define Ln_ENTRIES (1 << Ln_ENTRIES_SHIFT)
Index: src/sys/arch/aarch64/include/types.h
diff -u src/sys/arch/aarch64/include/types.h:1.19 src/sys/arch/aarch64/include/types.h:1.20
--- src/sys/arch/aarch64/include/types.h:1.19 Thu Sep 30 18:33:28 2021
+++ src/sys/arch/aarch64/include/types.h Sun Oct 10 07:15:25 2021
@@ -1,4 +1,4 @@
-/* $NetBSD: types.h,v 1.19 2021/09/30 18:33:28 skrll Exp $ */
+/* $NetBSD: types.h,v 1.20 2021/10/10 07:15:25 skrll Exp $ */
/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
@@ -97,7 +97,6 @@ typedef __uint64_t __register_t;
#define __HAVE_COMMON___TLS_GET_ADDR
#define __HAVE_CPU_COUNTER
#define __HAVE_CPU_DATA_FIRST
-#define __HAVE_CPU_MAXPROC
#define __HAVE_FAST_SOFTINTS
#define __HAVE_MINIMAL_EMUL
#define __HAVE_MM_MD_DIRECT_MAPPED_PHYS
Added files:
Index: src/sys/arch/aarch64/aarch64/aarch64_tlb.c
diff -u /dev/null src/sys/arch/aarch64/aarch64/aarch64_tlb.c:1.1
--- /dev/null Sun Oct 10 07:15:25 2021
+++ src/sys/arch/aarch64/aarch64/aarch64_tlb.c Sun Oct 10 07:15:25 2021
@@ -0,0 +1,118 @@
+/*-
+ * Copyright (c) 2021 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The NetBSD Foundation
+ * by Nick Hudson
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "opt_cputypes.h"
+#include "opt_multiprocessor.h"
+
+#include <sys/cdefs.h>
+__KERNEL_RCSID(1, "$NetBSD: aarch64_tlb.c,v 1.1 2021/10/10 07:15:25 skrll Exp $");
+
+#include <sys/param.h>
+#include <sys/types.h>
+
+#include <uvm/uvm.h>
+
+#include <arm/cpufunc.h>
+
+#include <aarch64/armreg.h>
+
+tlb_asid_t
+tlb_get_asid(void)
+{
+
+ return __SHIFTOUT(reg_ttbr0_el1_read(), TTBR_ASID);
+}
+
+void
+tlb_set_asid(tlb_asid_t asid, pmap_t pm)
+{
+ const uint64_t ttbr =
+ __SHIFTIN(asid, TTBR_ASID) |
+ __SHIFTIN(pmap_l0pa(pm), TTBR_BADDR);
+
+ cpu_set_ttbr0(ttbr);
+}
+
+void
+tlb_invalidate_all(void)
+{
+
+ aarch64_tlbi_all();
+}
+
+void
+tlb_invalidate_globals(void)
+{
+ tlb_invalidate_all();
+}
+
+void
+tlb_invalidate_asids(tlb_asid_t lo, tlb_asid_t hi)
+{
+ for (; lo <= hi; lo++) {
+ aarch64_tlbi_by_asid(lo);
+ }
+}
+
+void
+tlb_invalidate_addr(vaddr_t va, tlb_asid_t asid)
+{
+ KASSERT((va & PAGE_MASK) == 0);
+
+ aarch64_tlbi_by_asid_va(asid, va);
+}
+
+bool
+tlb_update_addr(vaddr_t va, tlb_asid_t asid, pt_entry_t pte, bool insert_p)
+{
+ KASSERT((va & PAGE_MASK) == 0);
+
+ tlb_invalidate_addr(va, asid);
+
+ return true;
+}
+
+u_int
+tlb_record_asids(u_long *mapp, tlb_asid_t asid_max)
+{
+ KASSERT(asid_max == pmap_md_tlb_asid_max());
+
+#if DIAGNOSTIC
+ memset(mapp, 0xff, (asid_max + 1) / (NBBY * sizeof(u_long)));
+ mapp[0] ^= __BITS(0, KERNEL_PID);
+#endif
+ return asid_max;
+}
+
+void
+tlb_walk(void *ctx, bool (*func)(void *, vaddr_t, tlb_asid_t, pt_entry_t))
+{
+
+ /* no way to view the TLB */
+}
