Module Name: src
Committed By: maxv
Date: Wed Feb 13 08:38:25 UTC 2019
Modified Files:
src/sys/arch/x86/include: pmap.h
src/sys/arch/x86/x86: pmap.c
Log Message:
Add the EPT pmap code, used by Intel-VMX.
The idea is that under NVMM, we don't want to implement the hypervisor page
tables manually in NVMM directly, because we want pageable guests; that is,
we want to allow UVM to unmap guest pages when the host comes under
pressure.
Contrary to AMD-SVM, Intel-VMX uses a different set of PTE bits from
native, and this has three important consequences:
- We can't use the native PTE bits, so each time we want to modify the
page tables, we need to know whether we're dealing with a native pmap
or an EPT pmap. This is accomplished with callbacks, that handle
everything PTE-related.
- There is no recursive slot possible, so we can't use pmap_map_ptes().
Rather, we walk down the EPT trees via the direct map, and that's
actually a lot simpler (and probably faster too...).
- The kernel is never mapped in an EPT pmap. An EPT pmap cannot be loaded
on the host. This has two sub-consequences: at creation time we must
zero out all of the top-level PTEs, and at destruction time we force
the page out of the pool cache and into the pool, to ensure that a next
allocation will invoke pmap_pdp_ctor() to create a native pmap and not
recycle some stale EPT entries.
To create an EPT pmap, the caller must invoke pmap_ept_transform() on a
newly-allocated native pmap. And that's about it, from then on the EPT
callbacks will be invoked, and the pmap can be destroyed via the usual
pmap_destroy(). The TLB shootdown callback is not initialized however,
it is the responsibility of the hypervisor (NVMM) to set it.
There are some twisted cases that we need to handle. For example if
pmap_is_referenced() is called on a physical page that is entered both by
a native pmap and by an EPT pmap, we take the Accessed bits from the
two pmaps using different PTE sets in each case, and combine them into a
generic PP_ATTRS_U flag (that does not depend on the pmap type).
Given that the EPT layout is a 4-Level tree with the same address space as
native x86_64, we allow ourselves to use a few native macros in EPT, such
as pmap_pa2pte(), rather than re-defining them with "ept" in the name.
Even though this EPT code is rather complex, it is not too intrusive: just
a few callbacks in a few pmap functions, predicted-false to give priority
to native. So this comes with no messy #ifdef or performance cost.
To generate a diff of this commit:
cvs rdiff -u -r1.96 -r1.97 src/sys/arch/x86/include/pmap.h
cvs rdiff -u -r1.321 -r1.322 src/sys/arch/x86/x86/pmap.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/x86/include/pmap.h
diff -u src/sys/arch/x86/include/pmap.h:1.96 src/sys/arch/x86/include/pmap.h:1.97
--- src/sys/arch/x86/include/pmap.h:1.96 Mon Feb 11 14:59:32 2019
+++ src/sys/arch/x86/include/pmap.h Wed Feb 13 08:38:25 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap.h,v 1.96 2019/02/11 14:59:32 cherry Exp $ */
+/* $NetBSD: pmap.h,v 1.97 2019/02/13 08:38:25 maxv Exp $ */
/*
* Copyright (c) 1997 Charles D. Cranor and Washington University.
@@ -370,6 +370,8 @@ u_int x86_mmap_flags(paddr_t);
bool pmap_is_curpmap(struct pmap *);
+void pmap_ept_transform(struct pmap *);
+
#ifndef __HAVE_DIRECT_MAP
void pmap_vpage_cpu_init(struct cpu_info *);
#endif
Index: src/sys/arch/x86/x86/pmap.c
diff -u src/sys/arch/x86/x86/pmap.c:1.321 src/sys/arch/x86/x86/pmap.c:1.322
--- src/sys/arch/x86/x86/pmap.c:1.321 Mon Feb 11 14:59:33 2019
+++ src/sys/arch/x86/x86/pmap.c Wed Feb 13 08:38:25 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap.c,v 1.321 2019/02/11 14:59:33 cherry Exp $ */
+/* $NetBSD: pmap.c,v 1.322 2019/02/13 08:38:25 maxv Exp $ */
/*
* Copyright (c) 2008, 2010, 2016, 2017 The NetBSD Foundation, Inc.
@@ -130,7 +130,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.321 2019/02/11 14:59:33 cherry Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.322 2019/02/13 08:38:25 maxv Exp $");
#include "opt_user_ldt.h"
#include "opt_lockdebug.h"
@@ -4797,3 +4797,873 @@ x86_mmap_flags(paddr_t mdpgno)
return pflag;
}
+
+#if defined(__HAVE_DIRECT_MAP) && defined(__x86_64__) && !defined(XEN)
+
+/*
+ * -----------------------------------------------------------------------------
+ * *****************************************************************************
+ * *****************************************************************************
+ * *****************************************************************************
+ * *****************************************************************************
+ * **************** HERE BEGINS THE EPT CODE, USED BY INTEL-VMX ****************
+ * *****************************************************************************
+ * *****************************************************************************
+ * *****************************************************************************
+ * *****************************************************************************
+ * -----------------------------------------------------------------------------
+ *
+ * These functions are invoked as callbacks from the code above. Contrary to
+ * native, EPT does not have a recursive slot; therefore, it is not possible
+ * to call pmap_map_ptes(). Instead, we use the direct map and walk down the
+ * tree manually.
+ *
+ * Apart from that, the logic is mostly the same as native. Once a pmap has
+ * been created, NVMM calls pmap_ept_transform() to make it an EPT pmap.
+ * After that we're good, and the callbacks will handle the translations
+ * for us.
+ *
+ * -----------------------------------------------------------------------------
+ */
+
+/* Hardware bits. */
+#define EPT_R __BIT(0) /* read */
+#define EPT_W __BIT(1) /* write */
+#define EPT_X __BIT(2) /* execute */
+#define EPT_T __BITS(5,3) /* type */
+#define TYPE_UC 0
+#define TYPE_WC 1
+#define TYPE_WT 4
+#define TYPE_WP 5
+#define TYPE_WB 6
+#define EPT_NOPAT __BIT(6)
+#define EPT_L __BIT(7) /* large */
+#define EPT_A __BIT(8) /* accessed */
+#define EPT_D __BIT(9) /* dirty */
+/* Software bits. */
+#define EPT_PVLIST __BIT(60)
+#define EPT_WIRED __BIT(61)
+
+#define pmap_ept_valid_entry(pte) (pte & EPT_R)
+
+static inline void
+pmap_ept_stats_update_bypte(struct pmap *pmap, pt_entry_t npte, pt_entry_t opte)
+{
+ int resid_diff = ((npte & EPT_R) ? 1 : 0) - ((opte & EPT_R) ? 1 : 0);
+ int wired_diff = ((npte & EPT_WIRED) ? 1 : 0) - ((opte & EPT_WIRED) ? 1 : 0);
+
+ KASSERT((npte & (EPT_R | EPT_WIRED)) != EPT_WIRED);
+ KASSERT((opte & (EPT_R | EPT_WIRED)) != EPT_WIRED);
+
+ pmap_stats_update(pmap, resid_diff, wired_diff);
+}
+
+static pt_entry_t
+pmap_ept_type(u_int flags)
+{
+ u_int cacheflags = (flags & PMAP_CACHE_MASK);
+ pt_entry_t ret;
+
+ switch (cacheflags) {
+ case PMAP_NOCACHE:
+ case PMAP_NOCACHE_OVR:
+ ret = __SHIFTIN(TYPE_UC, EPT_T);
+ break;
+ case PMAP_WRITE_COMBINE:
+ ret = __SHIFTIN(TYPE_WC, EPT_T);
+ break;
+ case PMAP_WRITE_BACK:
+ default:
+ ret = __SHIFTIN(TYPE_WB, EPT_T);
+ break;
+ }
+
+ ret |= EPT_NOPAT;
+ return ret;
+}
+
+static inline pt_entry_t
+pmap_ept_prot(vm_prot_t prot)
+{
+ pt_entry_t res = 0;
+
+ if (prot & VM_PROT_READ)
+ res |= EPT_R;
+ if (prot & VM_PROT_WRITE)
+ res |= EPT_W;
+ if (prot & VM_PROT_EXECUTE)
+ res |= EPT_X;
+
+ return res;
+}
+
+static inline uint8_t
+pmap_ept_to_pp_attrs(pt_entry_t ept)
+{
+ uint8_t ret = 0;
+ if (ept & EPT_D)
+ ret |= PP_ATTRS_M;
+ if (ept & EPT_A)
+ ret |= PP_ATTRS_U;
+ if (ept & EPT_W)
+ ret |= PP_ATTRS_W;
+ return ret;
+}
+
+static inline pt_entry_t
+pmap_pp_attrs_to_ept(uint8_t attrs)
+{
+ pt_entry_t ept = 0;
+ if (attrs & PP_ATTRS_M)
+ ept |= EPT_D;
+ if (attrs & PP_ATTRS_U)
+ ept |= EPT_A;
+ if (attrs & PP_ATTRS_W)
+ ept |= EPT_W;
+ return ept;
+}
+
+/*
+ * Helper for pmap_ept_free_ptp.
+ * tree[0] = &L2[L2idx]
+ * tree[1] = &L3[L3idx]
+ * tree[2] = &L4[L4idx]
+ */
+static void
+pmap_ept_get_tree(struct pmap *pmap, vaddr_t va, pd_entry_t **tree)
+{
+ pt_entry_t *pteva;
+ paddr_t ptepa;
+ int i, index;
+
+ ptepa = pmap->pm_pdirpa[0];
+ for (i = PTP_LEVELS; i > 1; i--) {
+ index = pl_pi(va, i);
+ pteva = (pt_entry_t *)PMAP_DIRECT_MAP(ptepa);
+ KASSERT(pmap_ept_valid_entry(pteva[index]));
+ tree[i - 2] = &pteva[index];
+ ptepa = pmap_pte2pa(pteva[index]);
+ }
+}
+
+static void
+pmap_ept_free_ptp(struct pmap *pmap, struct vm_page *ptp, vaddr_t va)
+{
+ pd_entry_t *tree[3];
+ int level;
+
+ KASSERT(pmap != pmap_kernel());
+ KASSERT(mutex_owned(pmap->pm_lock));
+ KASSERT(kpreempt_disabled());
+
+ pmap_ept_get_tree(pmap, va, tree);
+
+ level = 1;
+ do {
+ (void)pmap_pte_testset(tree[level - 1], 0);
+
+ pmap_freepage(pmap, ptp, level);
+ if (level < PTP_LEVELS - 1) {
+ ptp = pmap_find_ptp(pmap, va, (paddr_t)-1, level + 1);
+ ptp->wire_count--;
+ if (ptp->wire_count > 1)
+ break;
+ }
+ } while (++level < PTP_LEVELS);
+ pmap_pte_flush();
+}
+
+/* Allocate L4->L3->L2. Return L2. */
+static struct vm_page *
+pmap_ept_get_ptp(struct pmap *pmap, vaddr_t va, int flags)
+{
+ struct vm_page *ptp;
+ struct {
+ struct vm_page *pg;
+ bool new;
+ } pt[PTP_LEVELS + 1];
+ int i, aflags;
+ unsigned long index;
+ pd_entry_t *pteva;
+ paddr_t ptepa;
+ struct uvm_object *obj;
+ voff_t off;
+
+ KASSERT(pmap != pmap_kernel());
+ KASSERT(mutex_owned(pmap->pm_lock));
+ KASSERT(kpreempt_disabled());
+
+ memset(pt, 0, sizeof(pt));
+ aflags = ((flags & PMAP_CANFAIL) ? 0 : UVM_PGA_USERESERVE) |
+ UVM_PGA_ZERO;
+
+ /*
+ * Loop through all page table levels allocating a page
+ * for any level where we don't already have one.
+ */
+ for (i = PTP_LEVELS; i > 1; i--) {
+ obj = &pmap->pm_obj[i - 2];
+ off = ptp_va2o(va, i - 1);
+
+ PMAP_SUBOBJ_LOCK(pmap, i - 2);
+ pt[i].pg = uvm_pagelookup(obj, off);
+ if (pt[i].pg == NULL) {
+ pt[i].pg = uvm_pagealloc(obj, off, NULL, aflags);
+ pt[i].new = true;
+ }
+ PMAP_SUBOBJ_UNLOCK(pmap, i - 2);
+
+ if (pt[i].pg == NULL)
+ goto fail;
+ }
+
+ /*
+ * Now that we have all the pages looked up or allocated,
+ * loop through again installing any new ones into the tree.
+ */
+ ptepa = pmap->pm_pdirpa[0];
+ for (i = PTP_LEVELS; i > 1; i--) {
+ index = pl_pi(va, i);
+ pteva = (pt_entry_t *)PMAP_DIRECT_MAP(ptepa);
+
+ if (pmap_ept_valid_entry(pteva[index])) {
+ KASSERT(!pt[i].new);
+ ptepa = pmap_pte2pa(pteva[index]);
+ continue;
+ }
+
+ ptp = pt[i].pg;
+ ptp->flags &= ~PG_BUSY; /* never busy */
+ ptp->wire_count = 1;
+ pmap->pm_ptphint[i - 2] = ptp;
+ ptepa = VM_PAGE_TO_PHYS(ptp);
+ pmap_pte_set(&pteva[index], ptepa | EPT_R | EPT_W | EPT_X);
+
+ pmap_pte_flush();
+ pmap_stats_update(pmap, 1, 0);
+
+ /*
+ * If we're not in the top level, increase the
+ * wire count of the parent page.
+ */
+ if (i < PTP_LEVELS) {
+ pt[i + 1].pg->wire_count++;
+ }
+ }
+ ptp = pt[2].pg;
+ KASSERT(ptp != NULL);
+ pmap->pm_ptphint[0] = ptp;
+ return ptp;
+
+ /*
+ * Allocation of a PTP failed, free any others that we just allocated.
+ */
+fail:
+ for (i = PTP_LEVELS; i > 1; i--) {
+ if (pt[i].pg == NULL) {
+ break;
+ }
+ if (!pt[i].new) {
+ continue;
+ }
+ obj = &pmap->pm_obj[i - 2];
+ PMAP_SUBOBJ_LOCK(pmap, i - 2);
+ uvm_pagefree(pt[i].pg);
+ PMAP_SUBOBJ_UNLOCK(pmap, i - 2);
+ }
+ return NULL;
+}
+
+static int
+pmap_ept_enter(struct pmap *pmap, vaddr_t va, paddr_t pa, vm_prot_t prot,
+ u_int flags)
+{
+ pt_entry_t *ptes, opte, npte;
+ pt_entry_t *ptep;
+ struct vm_page *ptp;
+ struct vm_page *new_pg, *old_pg;
+ struct pmap_page *new_pp, *old_pp;
+ struct pv_entry *old_pve = NULL;
+ struct pv_entry *new_pve;
+ struct pv_entry *new_sparepve;
+ bool wired = (flags & PMAP_WIRED) != 0;
+ int error;
+
+ KASSERT(pmap_initialized);
+ KASSERT(curlwp->l_md.md_gc_pmap != pmap);
+ KASSERT(va < VM_MAXUSER_ADDRESS);
+
+ npte = pa | pmap_ept_prot(prot) | pmap_ept_type(flags);
+
+ if (wired)
+ npte |= EPT_WIRED;
+ if (flags & VM_PROT_ALL) {
+ npte |= EPT_A;
+ if (flags & VM_PROT_WRITE) {
+ KASSERT((npte & EPT_W) != 0);
+ npte |= EPT_D;
+ }
+ }
+
+ new_pg = PHYS_TO_VM_PAGE(pa);
+ if (new_pg != NULL) {
+ /* This is a managed page */
+ npte |= EPT_PVLIST;
+ new_pp = VM_PAGE_TO_PP(new_pg);
+ } else if ((new_pp = pmap_pv_tracked(pa)) != NULL) {
+ /* This is an unmanaged pv-tracked page */
+ npte |= EPT_PVLIST;
+ } else {
+ new_pp = NULL;
+ }
+
+ /*
+ * Try to get pves now if we might need them.
+ * Keep going even if we fail, since we will not actually need them
+ * if we are just changing the permissions on an existing mapping,
+ * but we won't know if that's the case until later.
+ */
+
+ bool needpves = pmap_pp_needs_pve(new_pp);
+ if (needpves) {
+ new_pve = pool_cache_get(&pmap_pv_cache, PR_NOWAIT);
+ new_sparepve = pool_cache_get(&pmap_pv_cache, PR_NOWAIT);
+ } else {
+ new_pve = NULL;
+ new_sparepve = NULL;
+ }
+
+ kpreempt_disable();
+ mutex_enter(pmap->pm_lock);
+
+ ptp = pmap_ept_get_ptp(pmap, va, flags);
+ if (ptp == NULL) {
+ mutex_exit(pmap->pm_lock);
+ if (flags & PMAP_CANFAIL) {
+ error = ENOMEM;
+ goto out;
+ }
+ panic("%s: get ptp failed", __func__);
+ }
+
+ /*
+ * Check if there is an existing mapping. If we are now sure that
+ * we need pves and we failed to allocate them earlier, handle that.
+ * Caching the value of oldpa here is safe because only the mod/ref
+ * bits can change while the pmap is locked.
+ */
+ ptes = (pt_entry_t *)PMAP_DIRECT_MAP(VM_PAGE_TO_PHYS(ptp));
+ ptep = &ptes[pl1_pi(va)];
+ opte = *ptep;
+ bool have_oldpa = pmap_ept_valid_entry(opte);
+ paddr_t oldpa = pmap_pte2pa(opte);
+
+ if (needpves && (!have_oldpa || oldpa != pa) &&
+ (new_pve == NULL || new_sparepve == NULL)) {
+ mutex_exit(pmap->pm_lock);
+ if (flags & PMAP_CANFAIL) {
+ error = ENOMEM;
+ goto out;
+ }
+ panic("%s: pve allocation failed", __func__);
+ }
+
+ /*
+ * Update the pte.
+ */
+ do {
+ opte = *ptep;
+
+ /*
+ * if the same page, inherit PG_U and PG_M.
+ */
+ if (((opte ^ npte) & (PG_FRAME | PG_V)) == 0) {
+ npte |= opte & (EPT_A | EPT_D);
+ }
+ } while (pmap_pte_cas(ptep, opte, npte) != opte);
+
+ /*
+ * Update statistics and PTP's reference count.
+ */
+ pmap_ept_stats_update_bypte(pmap, npte, opte);
+ if (ptp != NULL && !have_oldpa) {
+ ptp->wire_count++;
+ }
+ KASSERT(ptp == NULL || ptp->wire_count > 1);
+
+ /*
+ * If the same page, we can skip pv_entry handling.
+ */
+ if (((opte ^ npte) & (PG_FRAME | EPT_R)) == 0) {
+ KASSERT(((opte ^ npte) & EPT_PVLIST) == 0);
+ goto same_pa;
+ }
+
+ /*
+ * If old page is pv-tracked, remove pv_entry from its list.
+ */
+ if ((~opte & (EPT_R | EPT_PVLIST)) == 0) {
+ if ((old_pg = PHYS_TO_VM_PAGE(oldpa)) != NULL) {
+ KASSERT(uvm_page_locked_p(old_pg));
+ old_pp = VM_PAGE_TO_PP(old_pg);
+ } else if ((old_pp = pmap_pv_tracked(oldpa)) == NULL) {
+ panic("%s: EPT_PVLIST with pv-untracked page"
+ " va = %#"PRIxVADDR
+ " pa = %#" PRIxPADDR " (%#" PRIxPADDR ")",
+ __func__, va, oldpa, atop(pa));
+ }
+
+ old_pve = pmap_remove_pv(old_pp, ptp, va);
+ old_pp->pp_attrs |= pmap_ept_to_pp_attrs(opte);
+ }
+
+ /*
+ * If new page is pv-tracked, insert pv_entry into its list.
+ */
+ if (new_pp) {
+ new_pve = pmap_enter_pv(new_pp, new_pve, &new_sparepve, ptp, va);
+ }
+
+same_pa:
+ mutex_exit(pmap->pm_lock);
+
+ if ((~opte & (EPT_R | EPT_A)) == 0 &&
+ ((opte ^ npte) & (PG_FRAME | EPT_W)) != 0) {
+ pmap_tlb_shootdown(pmap, va, 0, TLBSHOOT_ENTER);
+ }
+
+ error = 0;
+out:
+ kpreempt_enable();
+ if (old_pve != NULL) {
+ pool_cache_put(&pmap_pv_cache, old_pve);
+ }
+ if (new_pve != NULL) {
+ pool_cache_put(&pmap_pv_cache, new_pve);
+ }
+ if (new_sparepve != NULL) {
+ pool_cache_put(&pmap_pv_cache, new_sparepve);
+ }
+
+ return error;
+}
+
+/* Pay close attention, this returns L2. */
+static int
+pmap_ept_pdes_invalid(struct pmap *pmap, vaddr_t va, pd_entry_t *lastpde)
+{
+ pt_entry_t *pteva;
+ paddr_t ptepa;
+ int i, index;
+
+ KASSERT(mutex_owned(pmap->pm_lock));
+
+ ptepa = pmap->pm_pdirpa[0];
+ for (i = PTP_LEVELS; i > 1; i--) {
+ pteva = (pt_entry_t *)PMAP_DIRECT_MAP(ptepa);
+ index = pl_pi(va, i);
+ if (!pmap_ept_valid_entry(pteva[index]))
+ return i;
+ ptepa = pmap_pte2pa(pteva[index]);
+ }
+ if (lastpde != NULL) {
+ *lastpde = pteva[index];
+ }
+
+ return 0;
+}
+
+static bool
+pmap_ept_extract(struct pmap *pmap, vaddr_t va, paddr_t *pap)
+{
+ pt_entry_t *ptes, pte;
+ pd_entry_t pde;
+ paddr_t ptppa, pa;
+ bool rv;
+
+#ifdef __HAVE_DIRECT_MAP
+ if (va >= PMAP_DIRECT_BASE && va < PMAP_DIRECT_END) {
+ if (pap != NULL) {
+ *pap = PMAP_DIRECT_UNMAP(va);
+ }
+ return true;
+ }
+#endif
+
+ rv = false;
+ pa = 0;
+
+ kpreempt_disable();
+ mutex_enter(pmap->pm_lock);
+
+ if (!pmap_ept_pdes_invalid(pmap, va, &pde)) {
+ ptppa = pmap_pte2pa(pde);
+ ptes = (pt_entry_t *)PMAP_DIRECT_MAP(ptppa);
+ pte = ptes[pl1_pi(va)];
+ if (__predict_true((pte & EPT_R) != 0)) {
+ pa = pmap_pte2pa(pte) | (va & (NBPD_L1 - 1));
+ rv = true;
+ }
+ }
+
+ mutex_exit(pmap->pm_lock);
+ kpreempt_enable();
+
+ if (pap != NULL) {
+ *pap = pa;
+ }
+ return rv;
+}
+
+static bool
+pmap_ept_remove_pte(struct pmap *pmap, struct vm_page *ptp, pt_entry_t *pte,
+ vaddr_t va, struct pv_entry **pv_tofree)
+{
+ struct pv_entry *pve;
+ struct vm_page *pg;
+ struct pmap_page *pp;
+ pt_entry_t opte;
+
+ KASSERT(pmap != pmap_kernel());
+ KASSERT(mutex_owned(pmap->pm_lock));
+ KASSERT(kpreempt_disabled());
+
+ if (!pmap_ept_valid_entry(*pte)) {
+ /* VA not mapped. */
+ return false;
+ }
+
+ /* Atomically save the old PTE and zap it. */
+ opte = pmap_pte_testset(pte, 0);
+ if (!pmap_ept_valid_entry(opte)) {
+ return false;
+ }
+
+ pmap_ept_stats_update_bypte(pmap, 0, opte);
+
+ if (ptp) {
+ /*
+ * Dropping a PTE. Make sure that the PDE is flushed.
+ */
+ ptp->wire_count--;
+ if (ptp->wire_count <= 1) {
+ opte |= EPT_A;
+ }
+ }
+
+ pmap_tlb_shootdown(pmap, va, 0, TLBSHOOT_REMOVE_PTE);
+
+ /*
+ * If we are not on a pv_head list - we are done.
+ */
+ if ((opte & EPT_PVLIST) == 0) {
+ KASSERTMSG((PHYS_TO_VM_PAGE(pmap_pte2pa(opte)) == NULL),
+ "managed page without EPT_PVLIST for %#"PRIxVADDR, va);
+ KASSERTMSG((pmap_pv_tracked(pmap_pte2pa(opte)) == NULL),
+ "pv-tracked page without EPT_PVLIST for %#"PRIxVADDR, va);
+ return true;
+ }
+
+ if ((pg = PHYS_TO_VM_PAGE(pmap_pte2pa(opte))) != NULL) {
+ KASSERT(uvm_page_locked_p(pg));
+ pp = VM_PAGE_TO_PP(pg);
+ } else if ((pp = pmap_pv_tracked(pmap_pte2pa(opte))) == NULL) {
+ paddr_t pa = pmap_pte2pa(opte);
+ panic("%s: EPT_PVLIST with pv-untracked page"
+ " va = %#"PRIxVADDR"pa = %#"PRIxPADDR" (%#"PRIxPADDR")",
+ __func__, va, pa, atop(pa));
+ }
+
+ /* Sync R/M bits. */
+ pp->pp_attrs |= pmap_ept_to_pp_attrs(opte);
+ pve = pmap_remove_pv(pp, ptp, va);
+
+ if (pve) {
+ pve->pve_next = *pv_tofree;
+ *pv_tofree = pve;
+ }
+ return true;
+}
+
+static void
+pmap_ept_remove_ptes(struct pmap *pmap, struct vm_page *ptp, vaddr_t ptpva,
+ vaddr_t startva, vaddr_t endva, struct pv_entry **pv_tofree)
+{
+ pt_entry_t *pte = (pt_entry_t *)ptpva;
+
+ KASSERT(pmap != pmap_kernel());
+ KASSERT(mutex_owned(pmap->pm_lock));
+ KASSERT(kpreempt_disabled());
+
+ /*
+ * note that ptpva points to the PTE that maps startva. this may
+ * or may not be the first PTE in the PTP.
+ *
+ * we loop through the PTP while there are still PTEs to look at
+ * and the wire_count is greater than 1 (because we use the wire_count
+ * to keep track of the number of real PTEs in the PTP).
+ */
+ while (startva < endva && (ptp == NULL || ptp->wire_count > 1)) {
+ (void)pmap_ept_remove_pte(pmap, ptp, pte, startva, pv_tofree);
+ startva += PAGE_SIZE;
+ pte++;
+ }
+}
+
+static void
+pmap_ept_remove(struct pmap *pmap, vaddr_t sva, vaddr_t eva)
+{
+ struct pv_entry *pv_tofree = NULL;
+ pt_entry_t *ptes;
+ pd_entry_t pde;
+ paddr_t ptppa;
+ vaddr_t blkendva, va = sva;
+ struct vm_page *ptp;
+
+ kpreempt_disable();
+ mutex_enter(pmap->pm_lock);
+
+ for (/* null */ ; va < eva ; va = blkendva) {
+ int lvl;
+
+ /* determine range of block */
+ blkendva = x86_round_pdr(va+1);
+ if (blkendva > eva)
+ blkendva = eva;
+
+ lvl = pmap_ept_pdes_invalid(pmap, va, &pde);
+ if (lvl != 0) {
+ /* Skip a range corresponding to an invalid pde. */
+ blkendva = (va & ptp_frames[lvl - 1]) + nbpd[lvl - 1];
+ continue;
+ }
+
+ /* PA of the PTP */
+ ptppa = pmap_pte2pa(pde);
+
+ ptp = pmap_find_ptp(pmap, va, ptppa, 1);
+ KASSERTMSG(ptp != NULL, "%s: unmanaged PTP detected",
+ __func__);
+
+ ptes = (pt_entry_t *)PMAP_DIRECT_MAP(ptppa);
+
+ pmap_ept_remove_ptes(pmap, ptp, (vaddr_t)&ptes[pl1_pi(va)], va,
+ blkendva, &pv_tofree);
+
+ /* If PTP is no longer being used, free it. */
+ if (ptp && ptp->wire_count <= 1) {
+ pmap_ept_free_ptp(pmap, ptp, va);
+ }
+ }
+
+ mutex_exit(pmap->pm_lock);
+ kpreempt_enable();
+
+ /* Now we free unused PVs */
+ if (pv_tofree)
+ pmap_free_pvs(pv_tofree);
+}
+
+static int
+pmap_ept_sync_pv(struct vm_page *ptp, vaddr_t va, paddr_t pa, int clearbits,
+ uint8_t *oattrs, pt_entry_t *optep)
+{
+ struct pmap *pmap;
+ pt_entry_t *ptep;
+ pt_entry_t opte;
+ pt_entry_t npte;
+ pt_entry_t expect;
+ bool need_shootdown;
+
+ expect = pmap_pa2pte(pa) | EPT_R;
+ KASSERT(ptp == NULL || ptp->uobject != NULL);
+ KASSERT(ptp == NULL || ptp_va2o(va, 1) == ptp->offset);
+ pmap = ptp_to_pmap(ptp);
+
+ KASSERT(clearbits == ~0 || (clearbits & ~(EPT_D | EPT_A | EPT_W)) == 0);
+ KASSERT(kpreempt_disabled());
+
+ if (clearbits != ~0) {
+ KASSERT((clearbits & ~(PP_ATTRS_M|PP_ATTRS_U|PP_ATTRS_W)) == 0);
+ clearbits = pmap_pp_attrs_to_ept(clearbits);
+ }
+
+ ptep = pmap_map_pte(pmap, ptp, va);
+ do {
+ opte = *ptep;
+ KASSERT((opte & (EPT_D | EPT_A)) != EPT_D);
+ KASSERT((opte & (EPT_A | EPT_R)) != EPT_A);
+ KASSERT(opte == 0 || (opte & EPT_R) != 0);
+ if ((opte & (PG_FRAME | EPT_R)) != expect) {
+ /*
+ * We lost a race with a V->P operation like
+ * pmap_remove(). Wait for the competitor
+ * reflecting pte bits into mp_attrs.
+ *
+ * Issue a redundant TLB shootdown so that
+ * we can wait for its completion.
+ */
+ pmap_unmap_pte();
+ if (clearbits != 0) {
+ pmap_tlb_shootdown(pmap, va, 0,
+ TLBSHOOT_SYNC_PV1);
+ }
+ return EAGAIN;
+ }
+
+ /*
+ * Check if there's anything to do on this PTE.
+ */
+ if ((opte & clearbits) == 0) {
+ need_shootdown = false;
+ break;
+ }
+
+ /*
+ * We need a shootdown if the PTE is cached (PG_U) ...
+ * ... Unless we are clearing only the PG_RW bit and
+ * it isn't cached as RW (PG_M).
+ */
+ need_shootdown = (opte & EPT_A) != 0 &&
+ !(clearbits == EPT_W && (opte & EPT_D) == 0);
+
+ npte = opte & ~clearbits;
+
+ /*
+ * If we need a shootdown anyway, clear PG_U and PG_M.
+ */
+ if (need_shootdown) {
+ npte &= ~(EPT_A | EPT_D);
+ }
+ KASSERT((npte & (EPT_D | EPT_A)) != EPT_D);
+ KASSERT((npte & (EPT_A | EPT_R)) != EPT_A);
+ KASSERT(npte == 0 || (opte & EPT_R) != 0);
+ } while (pmap_pte_cas(ptep, opte, npte) != opte);
+
+ if (need_shootdown) {
+ pmap_tlb_shootdown(pmap, va, 0, TLBSHOOT_SYNC_PV2);
+ }
+ pmap_unmap_pte();
+
+ *oattrs = pmap_ept_to_pp_attrs(opte);
+ *optep = opte;
+ return 0;
+}
+
+static void
+pmap_ept_pp_remove_ent(struct pmap *pmap, struct vm_page *ptp, pt_entry_t opte,
+ vaddr_t va)
+{
+ mutex_enter(pmap->pm_lock);
+ pmap_ept_stats_update_bypte(pmap, 0, opte);
+ ptp->wire_count--;
+ if (ptp->wire_count <= 1) {
+ pmap_ept_free_ptp(pmap, ptp, va);
+ }
+ mutex_exit(pmap->pm_lock);
+}
+
+static void
+pmap_ept_write_protect(struct pmap *pmap, vaddr_t sva, vaddr_t eva, vm_prot_t prot)
+{
+ pt_entry_t bit_rem;
+ pt_entry_t *ptes, *spte;
+ pt_entry_t opte, npte;
+ pd_entry_t pde;
+ paddr_t ptppa;
+ vaddr_t va;
+
+ bit_rem = 0;
+ if (!(prot & VM_PROT_WRITE))
+ bit_rem = EPT_W;
+
+ sva &= PG_FRAME;
+ eva &= PG_FRAME;
+
+ /* Acquire pmap. */
+ kpreempt_disable();
+
+ mutex_enter(pmap->pm_lock);
+
+ for (va = sva; va < eva; va += PAGE_SIZE) {
+ if (pmap_ept_pdes_invalid(pmap, va, &pde)) {
+ continue;
+ }
+
+ ptppa = pmap_pte2pa(pde);
+ ptes = (pt_entry_t *)PMAP_DIRECT_MAP(ptppa);
+ spte = &ptes[pl1_pi(va)];
+
+ do {
+ opte = *spte;
+ if (!pmap_ept_valid_entry(opte)) {
+ goto next;
+ }
+ npte = (opte & ~bit_rem);
+ } while (pmap_pte_cas(spte, opte, npte) != opte);
+
+ if ((opte & EPT_D) != 0) {
+ vaddr_t tva = x86_ptob(spte - ptes);
+ pmap_tlb_shootdown(pmap, tva, 0,
+ TLBSHOOT_WRITE_PROTECT);
+ }
+next:;
+ }
+
+ mutex_exit(pmap->pm_lock);
+ kpreempt_enable();
+}
+
+static void
+pmap_ept_unwire(struct pmap *pmap, vaddr_t va)
+{
+ pt_entry_t *ptes, *ptep, opte;
+ pd_entry_t pde;
+ paddr_t ptppa;
+
+ /* Acquire pmap. */
+ kpreempt_disable();
+ mutex_enter(pmap->pm_lock);
+
+ if (pmap_ept_pdes_invalid(pmap, va, &pde)) {
+ panic("%s: invalid PDE va=%#" PRIxVADDR, __func__, va);
+ }
+
+ ptppa = pmap_pte2pa(pde);
+ ptes = (pt_entry_t *)PMAP_DIRECT_MAP(ptppa);
+ ptep = &ptes[pl1_pi(va)];
+ opte = *ptep;
+ KASSERT(pmap_ept_valid_entry(opte));
+
+ if (opte & EPT_WIRED) {
+ pt_entry_t npte = opte & ~EPT_WIRED;
+
+ opte = pmap_pte_testset(ptep, npte);
+ pmap_ept_stats_update_bypte(pmap, npte, opte);
+ } else {
+ printf("%s: wiring for pmap %p va %#" PRIxVADDR
+ "did not change!\n", __func__, pmap, va);
+ }
+
+ /* Release pmap. */
+ mutex_exit(pmap->pm_lock);
+ kpreempt_enable();
+}
+
+/* -------------------------------------------------------------------------- */
+
+void
+pmap_ept_transform(struct pmap *pmap)
+{
+ pmap->pm_enter = pmap_ept_enter;
+ pmap->pm_extract = pmap_ept_extract;
+ pmap->pm_remove = pmap_ept_remove;
+ pmap->pm_sync_pv = pmap_ept_sync_pv;
+ pmap->pm_pp_remove_ent = pmap_ept_pp_remove_ent;
+ pmap->pm_write_protect = pmap_ept_write_protect;
+ pmap->pm_unwire = pmap_ept_unwire;
+
+ memset(pmap->pm_pdir, 0, PAGE_SIZE);
+}
+
+#endif /* __HAVE_DIRECT_MAP && __x86_64__ && !XEN */
