Module Name: src
Committed By: ad
Date: Sat Mar 14 18:24:10 UTC 2020
Modified Files:
src/sys/arch/x86/include: pmap.h pmap_pv.h
src/sys/arch/x86/x86: pmap.c
Log Message:
PR kern/55071 (Panic shortly after running X11 due to kernel diagnostic
assertion "mutex_owned(&pp->pp_lock)")
- Fix a locking bug in pmap_pp_clear_attrs() and in pmap_pp_remove() do the
TLB shootdown while still holding the target pmap's lock.
Also:
- Finish PV list locking for x86 & update comments around same.
- Keep track of the min/max index of PTEs inserted into each PTP, and use
that to clip ranges of VAs passed to pmap_remove_ptes().
- Based on the above, implement a pmap_remove_all() for x86 that clears out
the pmap in a single pass. Makes exit() / fork() much cheaper.
To generate a diff of this commit:
cvs rdiff -u -r1.112 -r1.113 src/sys/arch/x86/include/pmap.h
cvs rdiff -u -r1.13 -r1.14 src/sys/arch/x86/include/pmap_pv.h
cvs rdiff -u -r1.365 -r1.366 src/sys/arch/x86/x86/pmap.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/x86/include/pmap.h
diff -u src/sys/arch/x86/include/pmap.h:1.112 src/sys/arch/x86/include/pmap.h:1.113
--- src/sys/arch/x86/include/pmap.h:1.112 Sat Mar 14 14:05:44 2020
+++ src/sys/arch/x86/include/pmap.h Sat Mar 14 18:24:10 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap.h,v 1.112 2020/03/14 14:05:44 ad Exp $ */
+/* $NetBSD: pmap.h,v 1.113 2020/03/14 18:24:10 ad Exp $ */
/*
* Copyright (c) 1997 Charles D. Cranor and Washington University.
@@ -248,6 +248,8 @@ extern struct pool_cache pmap_cache;
* (the other object locks are only used when uvm_pagealloc is called)
*/
+struct pv_page;
+
struct pmap {
struct uvm_object pm_obj[PTP_LEVELS-1];/* objects for lvl >= 1) */
LIST_ENTRY(pmap) pm_list; /* list of all pmaps */
@@ -256,11 +258,11 @@ struct pmap {
struct vm_page *pm_ptphint[PTP_LEVELS-1];
/* pointer to a PTP in our pmap */
struct pmap_statistics pm_stats; /* pmap stats */
+ struct pv_entry *pm_pve; /* spare pv_entry */
#if !defined(__x86_64__)
vaddr_t pm_hiexec; /* highest executable mapping */
#endif /* !defined(__x86_64__) */
- struct lwp *pm_remove_all; /* who's emptying the pmap */
union descriptor *pm_ldt; /* user-set LDT */
size_t pm_ldt_len; /* size of LDT in bytes */
Index: src/sys/arch/x86/include/pmap_pv.h
diff -u src/sys/arch/x86/include/pmap_pv.h:1.13 src/sys/arch/x86/include/pmap_pv.h:1.14
--- src/sys/arch/x86/include/pmap_pv.h:1.13 Tue Mar 10 22:38:41 2020
+++ src/sys/arch/x86/include/pmap_pv.h Sat Mar 14 18:24:10 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap_pv.h,v 1.13 2020/03/10 22:38:41 ad Exp $ */
+/* $NetBSD: pmap_pv.h,v 1.14 2020/03/14 18:24:10 ad Exp $ */
/*-
* Copyright (c)2008 YAMAMOTO Takashi,
@@ -34,6 +34,7 @@
#include <sys/rbtree.h>
struct vm_page;
+struct pmap_page;
/*
* structures to track P->V mapping
@@ -51,14 +52,14 @@ struct pv_pte {
};
/*
- * pv_entry: plug pv_pte into lists.
+ * pv_entry: plug pv_pte into lists. 32 bytes on i386, 64 on amd64.
*/
struct pv_entry {
struct pv_pte pve_pte; /* should be the first member */
LIST_ENTRY(pv_entry) pve_list; /* on pmap_page::pp_pvlist */
rb_node_t pve_rb; /* red-black tree node */
- uintptr_t pve_padding; /* unused */
+ struct pmap_page *pve_pp; /* backpointer to mapped page */
};
#define pve_next pve_list.le_next
@@ -71,16 +72,14 @@ struct pmap_page {
/* PTPs */
rb_tree_t rb;
- /* PTPs */
+ /* PTPs, when being freed */
LIST_ENTRY(vm_page) link;
- /* Non-PTPs */
+ /* Non-PTPs (i.e. normal pages) */
struct {
- /* PP_EMBEDDED */
struct pv_pte pte;
-
LIST_HEAD(, pv_entry) pvlist;
- uint8_t flags;
+ uint8_t embedded;
uint8_t attrs;
} s;
} pp_u;
@@ -89,7 +88,7 @@ struct pmap_page {
#define pp_link pp_u.link
#define pp_pte pp_u.s.pte
#define pp_pvlist pp_u.s.pvlist
-#define pp_pflags pp_u.s.flags
+#define pp_embedded pp_u.s.embedded
#define pp_attrs pp_u.s.attrs
};
@@ -97,10 +96,6 @@ struct pmap_page {
#define PP_ATTRS_A 0x02 /* Accessed */
#define PP_ATTRS_W 0x04 /* Writable */
-/* pp_flags */
-#define PP_EMBEDDED 1
-#define PP_FREEING 2
-
#define PMAP_PAGE_INIT(pp) \
do { \
LIST_INIT(&(pp)->pp_pvlist); \
Index: src/sys/arch/x86/x86/pmap.c
diff -u src/sys/arch/x86/x86/pmap.c:1.365 src/sys/arch/x86/x86/pmap.c:1.366
--- src/sys/arch/x86/x86/pmap.c:1.365 Sat Mar 14 14:05:44 2020
+++ src/sys/arch/x86/x86/pmap.c Sat Mar 14 18:24:10 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap.c,v 1.365 2020/03/14 14:05:44 ad Exp $ */
+/* $NetBSD: pmap.c,v 1.366 2020/03/14 18:24:10 ad Exp $ */
/*
* Copyright (c) 2008, 2010, 2016, 2017, 2019, 2020 The NetBSD Foundation, Inc.
@@ -130,7 +130,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.365 2020/03/14 14:05:44 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.366 2020/03/14 18:24:10 ad Exp $");
#include "opt_user_ldt.h"
#include "opt_lockdebug.h"
@@ -224,23 +224,39 @@ __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.3
/*
* Locking
*
- * We have the following locks that we must contend with, listed in the
- * order that they must be acquired:
+ * We have the following locks that we must deal with, listed in the order
+ * that they are acquired:
*
- * - pg->uobject->vmobjlock, pg->uanon->an_lock
- * These per-object locks are taken by the VM system before calling into
- * the pmap module. Holding them prevents concurrent operations on the
- * given page or set of pages.
- *
- * - pmap->pm_lock (per pmap)
- * This lock protects the fields in the pmap structure including the
- * non-kernel PDEs in the PDP, the PTEs, and the PVE radix tree. For
- * modifying kernel PTEs it is not required as kernel PDEs are never
- * freed, and the kernel is expected to be self consistent.
- *
- * - pmaps_lock
- * This lock protects the list of active pmaps (headed by "pmaps"). We
- * lock it when adding or removing pmaps from this list.
+ * pg->uobject->vmobjlock, pg->uanon->an_lock
+ *
+ * For managed pages, these per-object locks are taken by the VM system
+ * before calling into the pmap module - either a read or write hold.
+ * The lock hold prevent pages from changing identity while the pmap is
+ * operating on them. For example, the same lock is held across a call
+ * to pmap_remove() and the following call to pmap_update(), so that a
+ * page does not gain a new identity while its TLB visibility is stale.
+ *
+ * pmap->pm_lock
+ *
+ * This lock protects the fields in the pmap structure including the
+ * non-kernel PDEs in the PDP, the PTEs, and PTPs and connected data
+ * structures. For modifying unmanaged kernel PTEs it is not needed as
+ * kernel PDEs are never freed, and the kernel is expected to be self
+ * consistent (and the lock can't be taken for unmanaged kernel PTEs,
+ * because they can be modified from interrupt context).
+ *
+ * pmaps_lock
+ *
+ * This lock protects the list of active pmaps (headed by "pmaps").
+ * It's acqired when adding or removing pmaps or adjusting kernel PDEs.
+ *
+ * pp_lock
+ *
+ * This per-page lock protects PV entry lists and the embedded PV entry
+ * in each vm_page, allowing for concurrent operation on pages by
+ * different pmaps. This is a spin mutex at IPL_VM, because at the
+ * points it is taken context switching is usually not tolerable, and
+ * spin mutexes must block out interrupts that could take kernel_lock.
*/
/* uvm_object is abused here to index pmap_pages; make assertions happy. */
@@ -530,7 +546,7 @@ pv_pte_first(struct pmap_page *pp)
{
KASSERT(mutex_owned(&pp->pp_lock));
- if ((pp->pp_pflags & PP_EMBEDDED) != 0) {
+ if (pp->pp_embedded) {
return &pp->pp_pte;
}
return pve_to_pvpte(LIST_FIRST(&pp->pp_pvlist));
@@ -543,7 +559,7 @@ pv_pte_next(struct pmap_page *pp, struct
KASSERT(mutex_owned(&pp->pp_lock));
KASSERT(pvpte != NULL);
if (pvpte == &pp->pp_pte) {
- KASSERT((pp->pp_pflags & PP_EMBEDDED) != 0);
+ KASSERT(pp->pp_embedded);
return pve_to_pvpte(LIST_FIRST(&pp->pp_pvlist));
}
return pve_to_pvpte(LIST_NEXT(pvpte_to_pve(pvpte), pve_list));
@@ -605,6 +621,82 @@ pmap_compare_key(void *context, const vo
}
/*
+ * pmap_ptp_init: initialize new page table page
+ */
+static inline void
+pmap_ptp_init(struct vm_page *ptp)
+{
+ uint16_t *minidx, *maxidx;
+
+ minidx = (uint16_t *)&ptp->uanon;
+ maxidx = (uint16_t *)&ptp->uanon + 1;
+
+ *minidx = UINT16_MAX;
+ *maxidx = 0;
+ rb_tree_init(&VM_PAGE_TO_PP(ptp)->pp_rb, &pmap_rbtree_ops);
+}
+
+/*
+ * pmap_ptp_fini: finalize a page table page
+ */
+static inline void
+pmap_ptp_fini(struct vm_page *ptp)
+{
+
+ KASSERT(RB_TREE_MIN(&VM_PAGE_TO_PP(ptp)->pp_rb) == NULL);
+ ptp->uanon = NULL;
+}
+
+/*
+ * pmap_ptp_range_set: abuse ptp->uanon to record min/max idx of PTE
+ */
+static inline void
+pmap_ptp_range_set(struct vm_page *ptp, vaddr_t va)
+{
+ uint16_t *minidx, *maxidx;
+ u_int idx;
+
+ idx = pl1_pi(va);
+ KASSERT(idx < UINT16_MAX);
+ minidx = (uint16_t *)&ptp->uanon;
+ maxidx = (uint16_t *)&ptp->uanon + 1;
+
+ if (idx < *minidx) {
+ *minidx = idx;
+ }
+ if (idx > *maxidx) {
+ *maxidx = idx;
+ }
+}
+
+/*
+ * pmap_ptp_range_clip: abuse ptp->uanon to clip range of PTEs to remove
+ */
+static inline void
+pmap_ptp_range_clip(struct vm_page *ptp, vaddr_t *startva, vaddr_t *endva,
+ pt_entry_t **pte)
+{
+ vaddr_t sclip, eclip;
+
+ if (ptp == NULL) {
+ return;
+ }
+
+ sclip = (*startva & L2_FRAME) +
+ x86_ptob(*(uint16_t *)&ptp->uanon);
+ eclip = (*startva & L2_FRAME) +
+ x86_ptob(*((uint16_t *)&ptp->uanon + 1) + 1);
+
+ KASSERT(sclip < eclip);
+
+ sclip = (*startva < sclip ? sclip : *startva);
+ eclip = (*endva > eclip ? eclip : *endva);
+ *pte += (sclip - *startva) / PAGE_SIZE;
+ *startva = sclip;
+ *endva = eclip;
+}
+
+/*
* pmap_map_ptes: map a pmap's PTEs into KVM and lock them in
*
* there are several pmaps involved. some or all of them might be same.
@@ -656,7 +748,9 @@ pmap_map_ptes(struct pmap *pmap, struct
* often the case during exit(), when we have switched
* to the kernel pmap in order to destroy a user pmap.
*/
- pmap_reactivate(pmap);
+ if (__predict_false(ci->ci_tlbstate != TLBSTATE_VALID)) {
+ pmap_reactivate(pmap);
+ }
*pmap2 = NULL;
} else {
/*
@@ -1771,7 +1865,7 @@ pmap_init(void)
* The kernel doesn't keep track of PTPs, so there's nowhere handy
* to hang a tree of pv_entry records. Dynamically allocated
* pv_entry lists are not heavily used in the kernel's pmap (the
- * usual case is PP_EMBEDDED), so cop out and use a single RB tree
+ * usual case is embedded), so cop out and use a single RB tree
* to cover them.
*/
rb_tree_init(&pmap_kernel_rb, &pmap_rbtree_ops);
@@ -1857,28 +1951,6 @@ pmap_vpage_cpu_init(struct cpu_info *ci)
* p v _ e n t r y f u n c t i o n s
*/
-
-/*
- * pmap_pp_needs_pve: return true if we need to allocate a pv entry.
- */
-static bool
-pmap_pp_needs_pve(struct pmap_page *pp, struct vm_page *ptp, vaddr_t va)
-{
-
- /*
- * Adding a pv entry for this page only needs to allocate a pv_entry
- * structure if the page already has at least one pv entry, since
- * the first pv entry is stored in the pmap_page. However, because
- * of subsequent removal(s), PP_EMBEDDED can be false and there can
- * still be pv entries on the list.
- */
-
- if (pp == NULL || (pp->pp_pflags & PP_EMBEDDED) == 0) {
- return false;
- }
- return pp->pp_pte.pte_ptp != ptp || pp->pp_pte.pte_va != va;
-}
-
/*
* pmap_free_pvs: free a linked list of pv entries. the pv entries have
* been removed from their respective pages, but are still entered into the
@@ -1900,140 +1972,214 @@ pmap_free_pvs(struct pmap *pmap, struct
}
/*
- * pmap_lookup_pv: look up a non-PP_EMBEDDED pv entry for the given pmap
+ * pmap_treelookup_pv: search the PV tree for a dynamic entry
*
* => pmap must be locked
*/
-
static struct pv_entry *
-pmap_lookup_pv(struct pmap *pmap, struct vm_page *ptp,
- struct pmap_page *pp, vaddr_t va)
+pmap_treelookup_pv(const struct pmap *pmap, const struct vm_page *ptp,
+ const rb_tree_t *tree, const vaddr_t va)
{
- struct rb_node *node;
struct pv_entry *pve;
-
- KASSERT(mutex_owned(&pmap->pm_lock));
-
- /*
- * Do an unlocked check on the page: if tracked with PP_EMBEDDED we
- * can avoid touching the tree.
- */
- if ((pp->pp_pflags & PP_EMBEDDED) != 0 &&
- pp->pp_pte.pte_ptp == ptp &&
- pp->pp_pte.pte_va == va) {
- return NULL;
- }
-
- if (ptp != NULL) {
- node = VM_PAGE_TO_PP(ptp)->pp_rb.rbt_root;
- } else {
- KASSERT(pmap == pmap_kernel());
- node = pmap_kernel_rb.rbt_root;
- }
+ rb_node_t *node;
/*
- * Search the RB tree for the key. This is an inlined lookup
- * tailored for exactly what's needed here that is quite a bit
- * faster than using rb_tree_find_node().
+ * Inlined lookup tailored for exactly what's needed here that is
+ * quite a bit faster than using rb_tree_find_node().
*/
- for (;;) {
+ for (node = tree->rbt_root;;) {
if (__predict_false(RB_SENTINEL_P(node))) {
return NULL;
}
pve = (struct pv_entry *)
((uintptr_t)node - offsetof(struct pv_entry, pve_rb));
if (pve->pve_pte.pte_va == va) {
+ KASSERT(pve->pve_pte.pte_ptp == ptp);
return pve;
}
node = node->rb_nodes[pve->pve_pte.pte_va < va];
}
}
+/*
+ * pmap_lookup_pv: look up a non-embedded pv entry for the given pmap
+ *
+ * => a PV entry must be known present (doesn't check for existence)
+ * => pmap must be locked
+ */
+static struct pv_entry *
+pmap_lookup_pv(const struct pmap *pmap, const struct vm_page *ptp,
+ const struct pmap_page * const old_pp, const vaddr_t va)
+{
+ struct pv_entry *pve;
+ const rb_tree_t *tree;
+
+ KASSERT(mutex_owned(&pmap->pm_lock));
+ KASSERT(ptp != NULL || pmap == pmap_kernel());
+
+ /*
+ * [This mostly deals with the case of process-private pages, i.e.
+ * anonymous memory allocations or COW.]
+ *
+ * If the page is tracked with an embedded entry then the tree
+ * lookup can be avoided. It's safe to check for this specific
+ * set of values without pp_lock because all 3 will only ever be
+ * set together for this pmap.
+ *
+ */
+ if (atomic_load_relaxed(&old_pp->pp_embedded) &&
+ atomic_load_relaxed(&old_pp->pp_pte.pte_ptp) == ptp &&
+ atomic_load_relaxed(&old_pp->pp_pte.pte_va) == va) {
+ return NULL;
+ }
+
+ /*
+ * [This mostly deals with shared mappings, for example shared libs
+ * and executables.]
+ *
+ * Optimise for pmap_remove_all() which works by ascending scan:
+ * look at the lowest numbered node in the tree first. The tree is
+ * known non-empty because of the check above. For short lived
+ * processes where pmap_remove() isn't used much this gets close to
+ * a 100% hit rate.
+ */
+ tree = (ptp != NULL ? &VM_PAGE_TO_PP(ptp)->pp_rb : &pmap_kernel_rb);
+ KASSERT(!RB_SENTINEL_P(tree->rbt_root));
+ pve = (struct pv_entry *)
+ ((uintptr_t)tree->rbt_minmax[RB_DIR_LEFT] -
+ offsetof(struct pv_entry, pve_rb));
+ if (__predict_true(pve->pve_pte.pte_va == va)) {
+ KASSERT(pve->pve_pte.pte_ptp == ptp);
+ return pve;
+ }
+
+ /* Search the RB tree for the key (uncommon). */
+ return pmap_treelookup_pv(pmap, ptp, tree, va);
+}
/*
* pmap_enter_pv: enter a mapping onto a pmap_page lst
*
- * => caller should adjust ptp's wire_count before calling
- * => caller has preallocated pve for us
- * => if not embedded, tree node must be in place beforehand
+ * => pmap must be locked
+ * => does NOT insert dynamic entries to tree (pmap_enter() does later)
*/
-static struct pv_entry *
-pmap_enter_pv(struct pmap *pmap, struct pmap_page *pp, struct pv_entry *pve,
- struct vm_page *ptp, vaddr_t va)
+static int
+pmap_enter_pv(struct pmap *pmap, struct pmap_page *pp, struct vm_page *ptp,
+ vaddr_t va, struct pv_entry **new_pve, struct pv_entry **old_pve,
+ bool *samepage, bool *new_embedded, rb_tree_t *tree)
{
+ struct pv_entry *pve;
+ int error;
KASSERT(mutex_owned(&pmap->pm_lock));
KASSERT(ptp_to_pmap(ptp) == pmap);
- KASSERT(ptp == NULL || ptp->wire_count >= 2);
KASSERT(ptp == NULL || ptp->uobject != NULL);
KASSERT(ptp == NULL || ptp_va2o(va, 1) == ptp->offset);
+ /*
+ * If entering the same page and it's already tracked with an
+ * embedded entry, we can avoid the expense below. It's safe
+ * to check for this very specific set of values without a lock
+ * because all 3 will only ever be set together for this pmap.
+ */
+ if (atomic_load_relaxed(&pp->pp_embedded) &&
+ atomic_load_relaxed(&pp->pp_pte.pte_ptp) == ptp &&
+ atomic_load_relaxed(&pp->pp_pte.pte_va) == va) {
+ *samepage = true;
+ return 0;
+ }
+
+ /*
+ * Check for an existing dynamic mapping at this address. If it's
+ * for the same page, then it will be reused and nothing needs to be
+ * changed.
+ */
+ *old_pve = pmap_treelookup_pv(pmap, ptp, tree, va);
+ if (*old_pve != NULL && (*old_pve)->pve_pp == pp) {
+ *samepage = true;
+ return 0;
+ }
+
+ /*
+ * Need to put a new mapping in place. Grab a spare pv_entry in
+ * case it's needed; won't know for sure until the lock is taken.
+ */
+ if (pmap->pm_pve == NULL) {
+ pmap->pm_pve = pool_cache_get(&pmap_pv_cache, PR_NOWAIT);
+ }
+
+ error = 0;
mutex_spin_enter(&pp->pp_lock);
- if ((pp->pp_pflags & PP_EMBEDDED) == 0) {
- pp->pp_pflags |= PP_EMBEDDED;
+ if (!pp->pp_embedded) {
+ /*
+ * Embedded PV tracking available - easy.
+ */
pp->pp_pte.pte_ptp = ptp;
pp->pp_pte.pte_va = va;
- mutex_spin_exit(&pp->pp_lock);
- return pve;
+ pp->pp_embedded = true;
+ *new_embedded = true;
+ } else if (__predict_false(pmap->pm_pve == NULL)) {
+ /*
+ * No memory.
+ */
+ error = ENOMEM;
+ } else {
+ /*
+ * Install new pv_entry on the page.
+ */
+ pve = pmap->pm_pve;
+ pmap->pm_pve = NULL;
+ *new_pve = pve;
+ pve->pve_pte.pte_ptp = ptp;
+ pve->pve_pte.pte_va = va;
+ pve->pve_pp = pp;
+ LIST_INSERT_HEAD(&pp->pp_pvlist, pve, pve_list);
}
-
- KASSERT(pve != NULL);
- pve->pve_pte.pte_ptp = ptp;
- pve->pve_pte.pte_va = va;
- KASSERT(pmap_lookup_pv(pmap, ptp, pp, va) == NULL);
- LIST_INSERT_HEAD(&pp->pp_pvlist, pve, pve_list);
mutex_spin_exit(&pp->pp_lock);
- if (ptp != NULL) {
- rb_tree_insert_node(&VM_PAGE_TO_PP(ptp)->pp_rb, pve);
- } else {
- KASSERT(pmap == pmap_kernel());
- rb_tree_insert_node(&pmap_kernel_rb, pve);
- }
- return NULL;
+ return error;
}
/*
* pmap_remove_pv: try to remove a mapping from a pv_list
*
+ * => pmap must be locked
+ * => removes dynamic entries from tree
* => caller should adjust ptp's wire_count and free PTP if needed
- * => we don't remove radix tree entry; defer till later (it could block)
- * => we return the removed pve
- * => caller can optionally supply pve, if looked up already
*/
static void
pmap_remove_pv(struct pmap *pmap, struct pmap_page *pp, struct vm_page *ptp,
vaddr_t va, struct pv_entry *pve, uint8_t oattrs)
{
+ rb_tree_t *tree;
KASSERT(mutex_owned(&pmap->pm_lock));
KASSERT(ptp_to_pmap(ptp) == pmap);
KASSERT(ptp == NULL || ptp->uobject != NULL);
KASSERT(ptp == NULL || ptp_va2o(va, 1) == ptp->offset);
+ KASSERT(ptp != NULL || pmap == pmap_kernel());
mutex_spin_enter(&pp->pp_lock);
pp->pp_attrs |= oattrs;
- if ((pp->pp_pflags & PP_EMBEDDED) != 0 &&
- pp->pp_pte.pte_ptp == ptp &&
- pp->pp_pte.pte_va == va) {
- KASSERT(pve == NULL);
- pp->pp_pflags &= ~PP_EMBEDDED;
+ if (pve == NULL) {
+ KASSERT(pp->pp_embedded);
+ KASSERT(pp->pp_pte.pte_ptp == ptp);
+ KASSERT(pp->pp_pte.pte_va == va);
+ pp->pp_embedded = false;
+#ifdef DIAGNOSTIC
pp->pp_pte.pte_ptp = NULL;
pp->pp_pte.pte_va = 0;
+#endif
mutex_spin_exit(&pp->pp_lock);
} else {
- KASSERT(pve != NULL);
KASSERT(pve == pmap_lookup_pv(pmap, ptp, pp, va));
KASSERT(pve->pve_pte.pte_ptp == ptp);
KASSERT(pve->pve_pte.pte_va == va);
LIST_REMOVE(pve, pve_list);
mutex_spin_exit(&pp->pp_lock);
- if (ptp != NULL) {
- rb_tree_remove_node(&VM_PAGE_TO_PP(ptp)->pp_rb, pve);
- } else {
- KASSERT(pmap == pmap_kernel());
- rb_tree_remove_node(&pmap_kernel_rb, pve);
- }
+ tree = (ptp != NULL ? &VM_PAGE_TO_PP(ptp)->pp_rb :
+ &pmap_kernel_rb);
+ rb_tree_remove_node(tree, pve);
}
}
@@ -2077,6 +2223,7 @@ pmap_freepage(struct pmap *pmap, struct
if (pmap->pm_ptphint[lidx] == ptp)
pmap->pm_ptphint[lidx] = NULL;
ptp->wire_count = 0;
+ pmap_ptp_fini(ptp);
/*
* Enqueue the PTP to be freed by pmap_update(). We can't remove
@@ -2085,7 +2232,6 @@ pmap_freepage(struct pmap *pmap, struct
* Instead mark the PTP as free and if we bump into it again, we'll
* either ignore or reuse (depending on what's useful at the time).
*/
- KASSERT(RB_TREE_MIN(&VM_PAGE_TO_PP(ptp)->pp_rb) == NULL);
LIST_INSERT_HEAD(&pmap->pm_gc_ptp, ptp, mdpage.mp_pp.pp_link);
}
@@ -2178,14 +2324,12 @@ pmap_get_ptp(struct pmap *pmap, struct p
pt->pg[i] = uvm_pagealloc(obj, off, NULL, aflags);
pt->alloced[i] = true;
if (pt->pg[i] != NULL) {
- rb_tree_init(&VM_PAGE_TO_PP(pt->pg[i])->pp_rb,
- &pmap_rbtree_ops);
+ pmap_ptp_init(pt->pg[i]);
}
} else if (pt->pg[i]->wire_count == 0) {
/* This page was queued to be freed; dequeue it. */
LIST_REMOVE(pt->pg[i], mdpage.mp_pp.pp_link);
- rb_tree_init(&VM_PAGE_TO_PP(pt->pg[i])->pp_rb,
- &pmap_rbtree_ops);
+ pmap_ptp_init(pt->pg[i]);
}
PMAP_DUMMY_UNLOCK(pmap);
if (pt->pg[i] == NULL) {
@@ -2294,6 +2438,7 @@ pmap_unget_ptp(struct pmap *pmap, struct
KASSERT(pt->pg[i]->wire_count == 0);
/* pmap zeros all pages before freeing. */
pt->pg[i]->flags |= PG_ZERO;
+ pmap_ptp_fini(pt->pg[i]);
PMAP_DUMMY_LOCK(pmap);
uvm_pagefree(pt->pg[i]);
PMAP_DUMMY_UNLOCK(pmap);
@@ -2488,7 +2633,7 @@ pmap_ctor(void *arg, void *obj, int flag
kcpuset_create(&pmap->pm_xen_ptp_cpus, true);
#endif
LIST_INIT(&pmap->pm_gc_ptp);
- pmap->pm_remove_all = NULL;
+ pmap->pm_pve = NULL;
/* allocate and init PDP */
pmap->pm_pdir = pool_get(&pmap_pdp_pool, PR_WAITOK);
@@ -2521,6 +2666,10 @@ pmap_dtor(void *arg, void *obj)
{
struct pmap *pmap = obj;
+ if (pmap->pm_pve != NULL) {
+ pool_cache_put(&pmap_pv_cache, pmap->pm_pve);
+ }
+
mutex_enter(&pmaps_lock);
LIST_REMOVE(pmap, pm_list);
mutex_exit(&pmaps_lock);
@@ -2637,11 +2786,6 @@ pmap_destroy(struct pmap *pmap)
{
int i;
- /* Undo pmap_remove_all(). */
- if (pmap->pm_remove_all == curlwp) {
- pmap_update(pmap);
- }
-
/*
* drop reference count
*/
@@ -2656,7 +2800,6 @@ pmap_destroy(struct pmap *pmap)
* Reference count is zero, free pmap resources and then free pmap.
*/
- KASSERT(pmap->pm_remove_all == NULL);
pmap_check_ptps(pmap);
KASSERT(LIST_EMPTY(&pmap->pm_gc_ptp));
@@ -2697,20 +2840,82 @@ pmap_destroy(struct pmap *pmap)
}
/*
- * pmap_remove_all: pmap is being torn down by the current thread.
- * avoid unnecessary invalidations.
+ * pmap_remove_all: remove all mappings from pmap in bulk.
+ *
+ * Ordinarily when removing mappings it's important to hold the UVM object's
+ * lock, so that pages do not gain a new identity while retaining stale TLB
+ * entries (the same lock hold covers both pmap_remove() and pmap_update()).
+ * Here it's known that the address space is no longer visible to any user
+ * process, so we don't need to worry about that.
*/
bool
pmap_remove_all(struct pmap *pmap)
{
+ struct vm_page *ptps[32];
+ vaddr_t va, blkendva;
+ struct pmap *pmap2;
+ pt_entry_t *ptes;
+ pd_entry_t pde __diagused;
+ pd_entry_t * const *pdes;
+ struct pv_entry *pv_tofree;
+ int lvl __diagused, i, n;
- /*
- * No locking needed; at this point it should only ever be checked
- * by curlwp.
- */
- KASSERT(pmap->pm_remove_all == NULL);
- pmap->pm_remove_all = curlwp;
- return false;
+ /* XXX Can't handle EPT just yet. */
+ if (pmap->pm_remove != NULL) {
+ return false;
+ }
+
+ for (;;) {
+ /* Fetch a block of PTPs from tree. */
+ mutex_enter(&pmap->pm_lock);
+ n = radix_tree_gang_lookup_node(&pmap->pm_obj[0].uo_pages, 0,
+ (void **)ptps, __arraycount(ptps), false);
+ if (n == 0) {
+ mutex_exit(&pmap->pm_lock);
+ break;
+ }
+
+ /* Remove all mappings in the set of PTPs. */
+ pmap_map_ptes(pmap, &pmap2, &ptes, &pdes);
+ pv_tofree = NULL;
+ for (i = 0; i < n; i++) {
+ if (ptps[i]->wire_count == 0) {
+ /* It's dead: pmap_update() will expunge. */
+ continue;
+ }
+
+ /* Determine range of block. */
+ va = ptps[i]->offset * PAGE_SIZE / sizeof(pt_entry_t);
+ blkendva = x86_round_pdr(va + 1);
+
+ /* Make sure everything squares up... */
+ KASSERT(pmap_pdes_valid(va, pdes, &pde, &lvl));
+ KASSERT(lvl == 1);
+ KASSERT(pmap_find_ptp(pmap, va, 1) == ptps[i]);
+
+ /* Zap! */
+ pmap_remove_ptes(pmap, ptps[i],
+ (vaddr_t)&ptes[pl1_i(va)], va,
+ blkendva, &pv_tofree);
+
+ /* PTP should now be unused - free it. */
+ KASSERT(ptps[i]->wire_count <= 1);
+ pmap_free_ptp(pmap, ptps[i], va, ptes, pdes);
+ }
+ pmap_unmap_ptes(pmap, pmap2);
+ pmap_free_pvs(pmap, pv_tofree);
+ mutex_exit(&pmap->pm_lock);
+
+ /* Process deferred frees. */
+ pmap_update(pmap);
+
+ /* A breathing point. */
+ preempt_point();
+ }
+
+ /* Verify that the pmap is now completely empty. */
+ pmap_check_ptps(pmap);
+ return true;
}
#if defined(PMAP_FORK)
@@ -2952,7 +3157,7 @@ pmap_reactivate(struct pmap *pmap)
ci->ci_tlbstate = TLBSTATE_VALID;
KASSERT(kcpuset_isset(pmap->pm_kernel_cpus, cid));
- if (kcpuset_isset(pmap->pm_cpus, cid)) {
+ if (__predict_true(kcpuset_isset(pmap->pm_cpus, cid))) {
/* We have the reference, state is valid. */
} else {
/*
@@ -3542,6 +3747,12 @@ pmap_remove_ptes(struct pmap *pmap, stru
KASSERT(kpreempt_disabled());
/*
+ * mappings are very often sparse, so clip the given range to the
+ * range of PTEs that are known present in the PTP.
+ */
+ pmap_ptp_range_clip(ptp, &startva, &endva, &pte);
+
+ /*
* note that ptpva points to the PTE that maps startva. this may
* or may not be the first PTE in the PTP.
*
@@ -3753,8 +3964,7 @@ pmap_remove(struct pmap *pmap, vaddr_t s
pmap_unmap_ptes(pmap, pmap2);
/*
* Now safe to free, as we no longer have the PTEs mapped and can
- * block again. Radix tree nodes are removed here, so we need to
- * continue holding the pmap locked until complete.
+ * block again.
*/
if (pv_tofree != NULL) {
pmap_free_pvs(pmap, pv_tofree);
@@ -3893,6 +4103,16 @@ pmap_pp_remove(struct pmap_page *pp, pad
bool locked;
int count;
+ /*
+ * Do an unlocked check to see if the page has no mappings, eg when
+ * pmap_remove_all() was called before amap_wipeout() for a process
+ * private amap - common.
+ */
+ if (atomic_load_relaxed(&pp->pp_embedded) == 0 &&
+ atomic_load_relaxed(&pp->pp_pvlist.lh_first) == NULL) {
+ return;
+ }
+
count = SPINLOCK_BACKOFF_MIN;
kpreempt_disable();
startover:
@@ -3932,9 +4152,13 @@ startover:
}
goto startover;
}
-
+
error = pmap_sync_pv(pvpte, pa, ~0, &oattrs, &opte);
if (error == EAGAIN) {
+ /*
+ * XXXAD Shouldn't need to loop here, as the mapping
+ * can't disappear, seen as the pmap's lock is held.
+ */
int hold_count;
KERNEL_UNLOCK_ALL(curlwp, &hold_count);
mutex_exit(&pmap->pm_lock);
@@ -3964,13 +4188,10 @@ startover:
pmap_stats_update_bypte(pmap, 0, opte);
}
if (pve != NULL) {
- /*
- * Must free pve, and remove from PV tree with the
- * pmap's lock still held.
- */
pve->pve_next = NULL;
pmap_free_pvs(pmap, pve);
}
+ pmap_tlb_shootnow();
mutex_exit(&pmap->pm_lock);
if (ptp != NULL) {
pmap_destroy(pmap);
@@ -3978,7 +4199,6 @@ startover:
mutex_spin_enter(&pp->pp_lock);
}
mutex_spin_exit(&pp->pp_lock);
- pmap_tlb_shootnow();
kpreempt_enable();
}
@@ -4069,8 +4289,8 @@ pmap_pp_clear_attrs(struct pmap_page *pp
int count;
count = SPINLOCK_BACKOFF_MIN;
- mutex_spin_enter(&pp->pp_lock);
startover:
+ mutex_spin_enter(&pp->pp_lock);
for (pvpte = pv_pte_first(pp); pvpte; pvpte = pv_pte_next(pp, pvpte)) {
int error;
@@ -4175,8 +4395,6 @@ pmap_write_protect(struct pmap *pmap, va
vaddr_t blockend, va;
int lvl, i;
- KASSERT(pmap->pm_remove_all == NULL);
-
if (__predict_false(pmap->pm_write_protect != NULL)) {
(*pmap->pm_write_protect)(pmap, sva, eva, prot);
return;
@@ -4339,10 +4557,10 @@ pmap_enter_ma(struct pmap *pmap, vaddr_t
bool wired = (flags & PMAP_WIRED) != 0;
struct pmap *pmap2;
struct pmap_ptparray pt;
- bool getptp;
+ bool getptp, samepage, new_embedded;
+ rb_tree_t *tree;
KASSERT(pmap_initialized);
- KASSERT(pmap->pm_remove_all == NULL);
KASSERT(va < VM_MAX_KERNEL_ADDRESS);
KASSERTMSG(va != (vaddr_t)PDP_BASE, "%s: trying to map va=%#"
PRIxVADDR " over PDP!", __func__, va);
@@ -4408,18 +4626,36 @@ pmap_enter_ma(struct pmap *pmap, vaddr_t
error);
}
}
+ /* Remember min/max index of PTE in PTP. */
+ pmap_ptp_range_set(ptp, va);
+ tree = &VM_PAGE_TO_PP(ptp)->pp_rb;
+ } else {
+ tree = &pmap_kernel_rb;
}
/*
- * Now check to see if we need a pv entry for this VA. If we do,
- * allocate and install in the PV tree. In any case look up the
- * pv entry in case the old mapping used it.
+ * Look up the old PV entry at this VA (if any), and insert a new PV
+ * entry if required for the new mapping. Temporarily track the old
+ * and new mappings concurrently. Only after the old mapping is
+ * evicted from the pmap will we remove its PV entry. Otherwise,
+ * our picture of modified/accessed state for either page could get
+ * out of sync (we need any P->V operation for either page to stall
+ * on pmap->pm_lock until done here).
*/
- old_pve = NULL;
new_pve = NULL;
- if (pmap_pp_needs_pve(new_pp, ptp, va)) {
- new_pve = pool_cache_get(&pmap_pv_cache, PR_NOWAIT);
- if (new_pve == NULL) {
+ old_pve = NULL;
+ samepage = false;
+ new_embedded = false;
+
+ if (new_pp != NULL) {
+ error = pmap_enter_pv(pmap, new_pp, ptp, va, &new_pve,
+ &old_pve, &samepage, &new_embedded, tree);
+
+ /*
+ * If a new pv_entry was needed and none was available, we
+ * can go no further.
+ */
+ if (error != 0) {
if (flags & PMAP_CANFAIL) {
if (getptp) {
pmap_unget_ptp(pmap, &pt);
@@ -4429,6 +4665,8 @@ pmap_enter_ma(struct pmap *pmap, vaddr_t
}
panic("%s: alloc pve failed", __func__);
}
+ } else {
+ old_pve = pmap_lookup_pv(pmap, ptp, NULL, va);
}
/* Map PTEs into address space. */
@@ -4469,11 +4707,26 @@ pmap_enter_ma(struct pmap *pmap, vaddr_t
vtomach((vaddr_t)ptep), npte, domid);
splx(s);
if (error) {
+ /* Undo pv_entry tracking - oof. */
+ if (new_pp != NULL) {
+ mutex_spin_enter(&new_pp->pp_lock);
+ if (new_pve != NULL) {
+ LIST_REMOVE(new_pve, pve_list);
+ KASSERT(pmap->pm_pve == NULL);
+ pmap->pm_pve = new_pve;
+ } else if (new_embedded) {
+ new_pp->pp_embedded = false;
+ }
+ mutex_spin_exit(&new_pp->pp_lock);
+ pmap_unmap_ptes(pmap, pmap2);
+ mutex_exit(&pmap->pm_lock);
+ }
+ /* Free new PTP. */
if (ptp != NULL && ptp->wire_count <= 1) {
pmap_free_ptp(pmap, ptp, va, ptes,
pdes);
}
- goto out;
+ return error;
}
break;
}
@@ -4481,6 +4734,11 @@ pmap_enter_ma(struct pmap *pmap, vaddr_t
} while (pmap_pte_cas(ptep, opte, npte) != opte);
/*
+ * Done with the PTEs: they can now be unmapped.
+ */
+ pmap_unmap_ptes(pmap, pmap2);
+
+ /*
* Update statistics and PTP's reference count.
*/
pmap_stats_update_bypte(pmap, npte, opte);
@@ -4510,16 +4768,24 @@ pmap_enter_ma(struct pmap *pmap, vaddr_t
__func__, va, oldpa, atop(pa));
}
- old_pve = pmap_lookup_pv(pmap, ptp, old_pp, va);
pmap_remove_pv(pmap, old_pp, ptp, va, old_pve,
pmap_pte_to_pp_attrs(opte));
+ if (old_pve != NULL) {
+ if (pmap->pm_pve == NULL) {
+ pmap->pm_pve = old_pve;
+ } else {
+ pool_cache_put(&pmap_pv_cache, old_pve);
+ }
+ }
+ } else {
+ KASSERT(old_pve == NULL);
}
/*
- * If new page is pv-tracked, insert pv_entry into its list.
+ * If new page is dynamically PV tracked, insert to tree.
*/
- if (new_pp) {
- new_pve = pmap_enter_pv(pmap, new_pp, new_pve, ptp, va);
+ if (new_pve != NULL) {
+ rb_tree_insert_node(tree, new_pve);
}
same_pa:
@@ -4531,20 +4797,8 @@ same_pa:
((opte ^ npte) & (PTE_FRAME | PTE_W)) != 0) {
pmap_tlb_shootdown(pmap, va, opte, TLBSHOOT_ENTER);
}
-
- error = 0;
-#if defined(XENPV)
-out:
-#endif
- pmap_unmap_ptes(pmap, pmap2);
- if (old_pve != NULL) {
- pool_cache_put(&pmap_pv_cache, old_pve);
- }
- if (new_pve != NULL) {
- pool_cache_put(&pmap_pv_cache, new_pve);
- }
mutex_exit(&pmap->pm_lock);
- return error;
+ return 0;
}
paddr_t
@@ -4863,20 +5117,10 @@ pmap_update(struct pmap *pmap)
struct vm_page *ptp;
/*
- * If pmap_remove_all() was in effect, re-enable invalidations from
- * this point on; issue a shootdown for all the mappings just
- * removed.
- */
- kpreempt_disable();
- if (pmap->pm_remove_all == curlwp) {
- pmap->pm_remove_all = NULL;
- pmap_tlb_shootdown(pmap, (vaddr_t)-1LL, 0, TLBSHOOT_UPDATE);
- }
-
- /*
* Initiate any pending TLB shootdowns. Wait for them to
* complete before returning control to the caller.
*/
+ kpreempt_disable();
pmap_tlb_shootnow();
kpreempt_enable();
@@ -4885,7 +5129,7 @@ pmap_update(struct pmap *pmap)
* is an unlocked check, but is safe as we're only interested in
* work done in this LWP - we won't get a false negative.
*/
- if (!LIST_EMPTY(&pmap->pm_gc_ptp)) {
+ if (__predict_false(!LIST_EMPTY(&pmap->pm_gc_ptp))) {
mutex_enter(&pmap->pm_lock);
while ((ptp = LIST_FIRST(&pmap->pm_gc_ptp)) != NULL) {
KASSERT(ptp->wire_count == 0);
@@ -4893,7 +5137,7 @@ pmap_update(struct pmap *pmap)
pp = VM_PAGE_TO_PP(ptp);
LIST_INIT(&pp->pp_pvlist);
pp->pp_attrs = 0;
- pp->pp_pflags = 0;
+ pp->pp_embedded = false;
/*
* XXX Hack to avoid extra locking, and lock
@@ -5248,10 +5492,10 @@ pmap_ept_enter(struct pmap *pmap, vaddr_
bool accessed;
struct pmap_ptparray pt;
int error;
- bool getptp;
+ bool getptp, samepage, new_embedded;
+ rb_tree_t *tree;
KASSERT(pmap_initialized);
- KASSERT(pmap->pm_remove_all == NULL);
KASSERT(va < VM_MAXUSER_ADDRESS);
npte = pa | pmap_ept_prot(prot) | pmap_ept_type(flags);
@@ -5298,18 +5542,36 @@ pmap_ept_enter(struct pmap *pmap, vaddr_
error);
}
}
+ /* Remember min/max index of PTE in PTP. */
+ pmap_ptp_range_set(ptp, va);
+ tree = &VM_PAGE_TO_PP(ptp)->pp_rb;
+ } else {
+ tree = &pmap_kernel_rb;
}
/*
- * Now check to see if we need a pv entry for this VA. If we do,
- * allocate and install in the radix tree. In any case look up the
- * pv entry in case the old mapping used it.
+ * Look up the old PV entry at this VA (if any), and insert a new PV
+ * entry if required for the new mapping. Temporarily track the old
+ * and new mappings concurrently. Only after the old mapping is
+ * evicted from the pmap will we remove its PV entry. Otherwise,
+ * our picture of modified/accessed state for either page could get
+ * out of sync (we need any P->V operation for either page to stall
+ * on pmap->pm_lock until done here).
*/
- old_pve = NULL;
new_pve = NULL;
- if (pmap_pp_needs_pve(new_pp, ptp, va)) {
- new_pve = pool_cache_get(&pmap_pv_cache, PR_NOWAIT);
- if (new_pve == NULL) {
+ old_pve = NULL;
+ samepage = false;
+ new_embedded = false;
+
+ if (new_pp != NULL) {
+ error = pmap_enter_pv(pmap, new_pp, ptp, va, &new_pve,
+ &old_pve, &samepage, &new_embedded, tree);
+
+ /*
+ * If a new pv_entry was needed and none was available, we
+ * can go no further.
+ */
+ if (error != 0) {
if (flags & PMAP_CANFAIL) {
if (getptp) {
pmap_unget_ptp(pmap, &pt);
@@ -5318,7 +5580,9 @@ pmap_ept_enter(struct pmap *pmap, vaddr_
return error;
}
panic("%s: alloc pve failed", __func__);
- }
+ }
+ } else {
+ old_pve = pmap_lookup_pv(pmap, ptp, NULL, va);
}
/* Map PTEs into address space. */
@@ -5373,7 +5637,7 @@ pmap_ept_enter(struct pmap *pmap, vaddr_
}
/*
- * If old page is pv-tracked, replace pv_entry from its list.
+ * If old page is pv-tracked, remove pv_entry from its list.
*/
if ((~opte & (EPT_R | EPT_PVLIST)) == 0) {
if ((old_pg = PHYS_TO_VM_PAGE(oldpa)) != NULL) {
@@ -5385,16 +5649,24 @@ pmap_ept_enter(struct pmap *pmap, vaddr_
__func__, va, oldpa, atop(pa));
}
- old_pve = pmap_lookup_pv(pmap, ptp, old_pp, va);
pmap_remove_pv(pmap, old_pp, ptp, va, old_pve,
pmap_ept_to_pp_attrs(opte));
+ if (old_pve != NULL) {
+ if (pmap->pm_pve == NULL) {
+ pmap->pm_pve = old_pve;
+ } else {
+ pool_cache_put(&pmap_pv_cache, old_pve);
+ }
+ }
+ } else {
+ KASSERT(old_pve == NULL);
}
/*
- * If new page is pv-tracked, insert pv_entry into its list.
+ * If new page is dynamically PV tracked, insert to tree.
*/
- if (new_pp) {
- new_pve = pmap_enter_pv(pmap, new_pp, new_pve, ptp, va);
+ if (new_pve != NULL) {
+ rb_tree_insert_node(tree, new_pve);
}
same_pa:
@@ -5575,6 +5847,12 @@ pmap_ept_remove_ptes(struct pmap *pmap,
KASSERT(kpreempt_disabled());
/*
+ * mappings are very often sparse, so clip the given range to the
+ * range of PTEs that are known present in the PTP.
+ */
+ pmap_ptp_range_clip(ptp, &startva, &endva, &pte);
+
+ /*
* note that ptpva points to the PTE that maps startva. this may
* or may not be the first PTE in the PTP.
*
@@ -5636,10 +5914,6 @@ pmap_ept_remove(struct pmap *pmap, vaddr
}
kpreempt_enable();
- /*
- * Radix tree nodes are removed here, so we need to continue holding
- * the pmap locked until complete.
- */
if (pv_tofree != NULL) {
pmap_free_pvs(pmap, pv_tofree);
}
