Optimize pmap on amd64 based on a patch for i386 by Art from 2008 that removes the APTE stuff. Some additional bits were taken from a patch by Art for amd64 from 2005.
Instead of mapping the ptes of an inactive pmap in the APTE range, and then doing an unconditional remote TLB-flush on all CPUs, we just switch to the other pmap locally. This only causes a local TLB-flush. In cases where we only need to access a single PTE, walk the page tables manually in the direct mapping. This saves some more TLB flushes. I have also reordered some stuff so that we will usually send the TLB-shootdown-IPIs first, then do some local stuff, and then wait for the remote TLB-shootdown to finish. diff --git a/sys/arch/amd64/amd64/pmap.c b/sys/arch/amd64/amd64/pmap.c index 05cbd3e..c137412 100644 --- a/sys/arch/amd64/amd64/pmap.c +++ b/sys/arch/amd64/amd64/pmap.c @@ -203,7 +203,6 @@ long nkptp[] = NKPTP_INITIALIZER; long nkptpmax[] = NKPTPMAX_INITIALIZER; long nbpd[] = NBPD_INITIALIZER; pd_entry_t *normal_pdes[] = PDES_INITIALIZER; -pd_entry_t *alternate_pdes[] = APDES_INITIALIZER; /* int nkpde = NKPTP; */ @@ -285,11 +284,12 @@ void pmap_enter_pv(struct vm_page *, struct pv_entry *, struct pmap *, vaddr_t, struct vm_page *); struct vm_page *pmap_get_ptp(struct pmap *, vaddr_t, pd_entry_t **); struct vm_page *pmap_find_ptp(struct pmap *, vaddr_t, paddr_t, int); +int pmap_find_pte_direct(struct pmap *pm, vaddr_t va, pt_entry_t **pd, int *offs); void pmap_free_ptp(struct pmap *, struct vm_page *, vaddr_t, pt_entry_t *, pd_entry_t **, struct pg_to_free *); void pmap_freepage(struct pmap *, struct vm_page *, int, struct pg_to_free *); static boolean_t pmap_is_active(struct pmap *, int); -void pmap_map_ptes(struct pmap *, pt_entry_t **, pd_entry_t ***); +void pmap_map_ptes(struct pmap *, pt_entry_t **, pd_entry_t ***, paddr_t *); struct pv_entry *pmap_remove_pv(struct vm_page *, struct pmap *, vaddr_t); void pmap_do_remove(struct pmap *, vaddr_t, vaddr_t, int); boolean_t pmap_remove_pte(struct pmap *, struct vm_page *, pt_entry_t *, @@ -299,14 +299,23 @@ void pmap_remove_ptes(struct pmap *, struct vm_page *, vaddr_t, #define PMAP_REMOVE_ALL 0 /* remove all mappings */ #define PMAP_REMOVE_SKIPWIRED 1 /* skip wired mappings */ -void pmap_unmap_ptes(struct pmap *); +void pmap_unmap_ptes(struct pmap *, paddr_t); boolean_t pmap_get_physpage(vaddr_t, int, paddr_t *); boolean_t pmap_pdes_valid(vaddr_t, pd_entry_t **, pd_entry_t *); void pmap_alloc_level(pd_entry_t **, vaddr_t, int, long *); -void pmap_apte_flush(struct pmap *pmap); void pmap_sync_flags_pte(struct vm_page *, u_long); +void pmap_tlb_shootpage(struct pmap *, vaddr_t, int); +void pmap_tlb_shootrange(struct pmap *, vaddr_t, vaddr_t, int); +void pmap_tlb_shoottlb(struct pmap *, int); +#ifdef MULTIPROCESSOR +void pmap_tlb_shootwait(void); +#else +#define pmap_tlb_shootwait() +#endif + + /* * p m a p i n l i n e h e l p e r f u n c t i o n s */ @@ -349,57 +358,75 @@ pmap_sync_flags_pte(struct vm_page *pg, u_long pte) } } -void -pmap_apte_flush(struct pmap *pmap) -{ - pmap_tlb_shoottlb(); - pmap_tlb_shootwait(); -} - /* * pmap_map_ptes: map a pmap's PTEs into KVM - * - * => we lock enough pmaps to keep things locked in - * => must be undone with pmap_unmap_ptes before returning */ void -pmap_map_ptes(struct pmap *pmap, pt_entry_t **ptepp, pd_entry_t ***pdeppp) +pmap_map_ptes(struct pmap *pmap, pt_entry_t **ptepp, pd_entry_t ***pdeppp, paddr_t *save_cr3) { - pd_entry_t opde, npde; + paddr_t cr3 = rcr3(); - /* if curpmap then we are always mapped */ - if (pmap_is_curpmap(pmap)) { - *ptepp = PTE_BASE; - *pdeppp = normal_pdes; - return; - } + /* the kernel's pmap is always accessible */ + if (pmap == pmap_kernel() || pmap->pm_pdirpa == cr3) { + *save_cr3 = 0; + } else { + *save_cr3 = cr3; - /* need to load a new alternate pt space into curpmap? */ - opde = *APDP_PDE; - if (!pmap_valid_entry(opde) || (opde & PG_FRAME) != pmap->pm_pdirpa) { - npde = (pd_entry_t) (pmap->pm_pdirpa | PG_RW | PG_V); - *APDP_PDE = npde; - if (pmap_valid_entry(opde)) - pmap_apte_flush(curpcb->pcb_pmap); + /* + * Not sure if we need this, but better be safe. + * We don't have the current pmap in order to unset its + * active bit, but this just means that we may receive + * an unneccessary cross-CPU TLB flush now and then. + */ + x86_atomic_setbits_u64(&pmap->pm_cpus, (1ULL << cpu_number())); + + lcr3(pmap->pm_pdirpa); } - *ptepp = APTE_BASE; - *pdeppp = alternate_pdes; + + *ptepp = PTE_BASE; + *pdeppp = normal_pdes; + return; } void -pmap_unmap_ptes(struct pmap *pmap) +pmap_unmap_ptes(struct pmap *pmap, paddr_t save_cr3) { - if (pmap_is_curpmap(pmap)) - return; + if (save_cr3 != 0) { + x86_atomic_clearbits_u64(&pmap->pm_cpus, (1ULL << cpu_number())); + lcr3(save_cr3); + } +} -#if defined(MULTIPROCESSOR) - *APDP_PDE = 0; - pmap_apte_flush(curpcb->pcb_pmap); -#endif - COUNT(apdp_pde_unmap); +int +pmap_find_pte_direct(struct pmap *pm, vaddr_t va, pt_entry_t **pd, int *offs) +{ + u_long mask, shift; + pd_entry_t pde; + paddr_t pdpa; + int lev; + + pdpa = pm->pm_pdirpa; + shift = L4_SHIFT; + mask = L4_MASK; + for (lev = PTP_LEVELS; lev > 0; lev--) { + *pd = (pd_entry_t *)PMAP_DIRECT_MAP(pdpa); + *offs = (VA_SIGN_POS(va) & mask) >> shift; + pde = (*pd)[*offs]; + + /* Large pages are different, break early if we run into one. */ + if ((pde & (PG_PS|PG_V)) != PG_V) + return (lev - 1); + + pdpa = ((*pd)[*offs] & PG_FRAME); + shift -= 9; + mask >>= 9; + } + + return (0); } + /* * p m a p k e n t e r f u n c t i o n s * @@ -442,7 +469,7 @@ pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot) if (pa & PMAP_NOCACHE && (opte & PG_N) == 0) wbinvd(); /* This shouldn't happen */ - pmap_tlb_shootpage(pmap_kernel(), va); + pmap_tlb_shootpage(pmap_kernel(), va, 1); pmap_tlb_shootwait(); } } @@ -476,7 +503,7 @@ pmap_kremove(vaddr_t sva, vsize_t len) KASSERT((opte & PG_PVLIST) == 0); } - pmap_tlb_shootrange(pmap_kernel(), sva, eva); + pmap_tlb_shootrange(pmap_kernel(), sva, eva, 1); pmap_tlb_shootwait(); } @@ -822,11 +849,13 @@ pmap_free_ptp(struct pmap *pmap, struct vm_page *ptp, vaddr_t va, invaladdr = level == 1 ? (vaddr_t)ptes : (vaddr_t)pdes[level - 2]; pmap_tlb_shootpage(curpcb->pcb_pmap, - invaladdr + index * PAGE_SIZE); + invaladdr + index * PAGE_SIZE, + pmap_is_curpmap(curpcb->pcb_pmap)); #if defined(MULTIPROCESSOR) invaladdr = level == 1 ? (vaddr_t)PTE_BASE : (vaddr_t)normal_pdes[level - 2]; - pmap_tlb_shootpage(pmap, invaladdr + index * PAGE_SIZE); + pmap_tlb_shootpage(pmap, invaladdr + index * PAGE_SIZE, + pmap_is_curpmap(curpcb->pcb_pmap)); #endif if (level < PTP_LEVELS - 1) { ptp = pmap_find_ptp(pmap, va, (paddr_t)-1, level + 1); @@ -1054,11 +1083,7 @@ pmap_destroy(struct pmap *pmap) } } - /* - * MULTIPROCESSOR -- no need to flush out of other processors' - * APTE space because we do that in pmap_unmap_ptes(). - */ - /* XXX: need to flush it out of other processor's APTE space? */ + /* XXX: need to flush it out of other processor's space? */ pool_put(&pmap_pdp_pool, pmap->pm_pdir); pool_put(&pmap_pmap_pool, pmap); @@ -1147,8 +1172,8 @@ pmap_pdes_valid(vaddr_t va, pd_entry_t **pdes, pd_entry_t *lastpde) boolean_t pmap_extract(struct pmap *pmap, vaddr_t va, paddr_t *pap) { - pt_entry_t *ptes, pte; - pd_entry_t pde, **pdes; + pt_entry_t *ptes; + int level, offs; if (pmap == pmap_kernel() && va >= PMAP_DIRECT_BASE && va < PMAP_DIRECT_END) { @@ -1156,24 +1181,17 @@ pmap_extract(struct pmap *pmap, vaddr_t va, paddr_t *pap) return (TRUE); } - pmap_map_ptes(pmap, &ptes, &pdes); - if (pmap_pdes_valid(va, pdes, &pde) == FALSE) { - return FALSE; - } + level = pmap_find_pte_direct(pmap, va, &ptes, &offs); - if (pde & PG_PS) { + if (__predict_true(level == 0 && pmap_valid_entry(ptes[offs]))) { if (pap != NULL) - *pap = (pde & PG_LGFRAME) | (va & 0x1fffff); - pmap_unmap_ptes(pmap); + *pap = (ptes[offs] & PG_FRAME) | (va & 0xfff); return (TRUE); } - - pte = ptes[pl1_i(va)]; - pmap_unmap_ptes(pmap); - - if (__predict_true(pmap_valid_entry(pte))) { + if (level == 1 && + (ptes[offs] & (PG_PS|PG_V)) == (PG_PS|PG_V)) { if (pap != NULL) - *pap = (pte & PG_FRAME) | (va & 0xfff); + *pap = (ptes[offs] & PG_LGFRAME) | (va & 0x1fffff); return (TRUE); } @@ -1438,11 +1456,12 @@ pmap_do_remove(struct pmap *pmap, vaddr_t sva, vaddr_t eva, int flags) vaddr_t va; int shootall = 0; struct pg_to_free empty_ptps; + paddr_t scr3; TAILQ_INIT(&empty_ptps); PMAP_MAP_TO_HEAD_LOCK(); - pmap_map_ptes(pmap, &ptes, &pdes); + pmap_map_ptes(pmap, &ptes, &pdes, &scr3); /* * removing one page? take shortcut function. @@ -1480,11 +1499,12 @@ pmap_do_remove(struct pmap *pmap, vaddr_t sva, vaddr_t eva, int flags) if (result && ptp && ptp->wire_count <= 1) pmap_free_ptp(pmap, ptp, sva, ptes, pdes, &empty_ptps); - pmap_tlb_shootpage(pmap, sva); + pmap_tlb_shootpage(pmap, sva, scr3 == 0); + pmap_unmap_ptes(pmap, scr3); + pmap_tlb_shootwait(); + } else { + pmap_unmap_ptes(pmap, scr3); } - - pmap_tlb_shootwait(); - pmap_unmap_ptes(pmap); PMAP_MAP_TO_HEAD_UNLOCK(); while ((ptp = TAILQ_FIRST(&empty_ptps)) != NULL) { @@ -1549,13 +1569,12 @@ pmap_do_remove(struct pmap *pmap, vaddr_t sva, vaddr_t eva, int flags) } if (shootall) - pmap_tlb_shoottlb(); + pmap_tlb_shoottlb(pmap, scr3 == 0); else - pmap_tlb_shootrange(pmap, sva, eva); + pmap_tlb_shootrange(pmap, sva, eva, scr3 == 0); + pmap_unmap_ptes(pmap, scr3); pmap_tlb_shootwait(); - - pmap_unmap_ptes(pmap); PMAP_MAP_TO_HEAD_UNLOCK(); while ((ptp = TAILQ_FIRST(&empty_ptps)) != NULL) { @@ -1581,6 +1600,7 @@ pmap_page_remove(struct vm_page *pg) #endif struct pg_to_free empty_ptps; struct vm_page *ptp; + paddr_t scr3; TAILQ_INIT(&empty_ptps); @@ -1589,7 +1609,8 @@ pmap_page_remove(struct vm_page *pg) while ((pve = pg->mdpage.pv_list) != NULL) { pg->mdpage.pv_list = pve->pv_next; - pmap_map_ptes(pve->pv_pmap, &ptes, &pdes); + /* XXX use direct map? */ + pmap_map_ptes(pve->pv_pmap, &ptes, &pdes, &scr3); #ifdef DIAGNOSTIC if (pve->pv_ptp && pmap_pdes_valid(pve->pv_va, pdes, &pde) && @@ -1612,7 +1633,7 @@ pmap_page_remove(struct vm_page *pg) pve->pv_pmap->pm_stats.wired_count--; pve->pv_pmap->pm_stats.resident_count--; - pmap_tlb_shootpage(pve->pv_pmap, pve->pv_va); + pmap_tlb_shootpage(pve->pv_pmap, pve->pv_va, scr3 == 0); pmap_sync_flags_pte(pg, opte); @@ -1624,7 +1645,7 @@ pmap_page_remove(struct vm_page *pg) pve->pv_va, ptes, pdes, &empty_ptps); } } - pmap_unmap_ptes(pve->pv_pmap); + pmap_unmap_ptes(pve->pv_pmap, scr3); pool_put(&pmap_pv_pool, pve); } @@ -1652,8 +1673,8 @@ boolean_t pmap_test_attrs(struct vm_page *pg, unsigned int testbits) { struct pv_entry *pve; - pt_entry_t *ptes, pte; - pd_entry_t **pdes; + pt_entry_t *ptes; + int level, offs; u_long mybits, testflags; testflags = pmap_pte2flags(testbits); @@ -1665,10 +1686,10 @@ pmap_test_attrs(struct vm_page *pg, unsigned int testbits) mybits = 0; for (pve = pg->mdpage.pv_list; pve != NULL && mybits == 0; pve = pve->pv_next) { - pmap_map_ptes(pve->pv_pmap, &ptes, &pdes); - pte = ptes[pl1_i(pve->pv_va)]; - pmap_unmap_ptes(pve->pv_pmap); - mybits |= (pte & testbits); + level = pmap_find_pte_direct(pve->pv_pmap, pve->pv_va, &ptes, + &offs); + KASSERT(level == 0); + mybits |= (ptes[offs] & testbits); } PMAP_HEAD_TO_MAP_UNLOCK(); @@ -1691,9 +1712,8 @@ pmap_clear_attrs(struct vm_page *pg, unsigned long clearbits) { struct pv_entry *pve; pt_entry_t *ptes, opte; - pd_entry_t **pdes; u_long clearflags; - int result; + int result, level, offs; clearflags = pmap_pte2flags(clearbits); @@ -1704,20 +1724,19 @@ pmap_clear_attrs(struct vm_page *pg, unsigned long clearbits) atomic_clearbits_int(&pg->pg_flags, clearflags); for (pve = pg->mdpage.pv_list; pve != NULL; pve = pve->pv_next) { - pmap_map_ptes(pve->pv_pmap, &ptes, &pdes); + level = pmap_find_pte_direct(pve->pv_pmap, pve->pv_va, &ptes, + &offs); #ifdef DIAGNOSTIC - if (!pmap_pdes_valid(pve->pv_va, pdes, NULL)) - panic("%s: mapping without PTP detected", __func__); + KASSERT(level == 0); #endif - - opte = ptes[pl1_i(pve->pv_va)]; + opte = ptes[offs]; if (opte & clearbits) { result = 1; - pmap_pte_clearbits(&ptes[pl1_i(pve->pv_va)], + pmap_pte_clearbits(&ptes[offs], (opte & clearbits)); - pmap_tlb_shootpage(pve->pv_pmap, pve->pv_va); + pmap_tlb_shootpage(pve->pv_pmap, pve->pv_va, + pmap_is_curpmap(pve->pv_pmap)); } - pmap_unmap_ptes(pve->pv_pmap); } PMAP_HEAD_TO_MAP_UNLOCK(); @@ -1760,8 +1779,9 @@ pmap_write_protect(struct pmap *pmap, vaddr_t sva, vaddr_t eva, vm_prot_t prot) vaddr_t blockend; int shootall = 0; vaddr_t va; + paddr_t scr3; - pmap_map_ptes(pmap, &ptes, &pdes); + pmap_map_ptes(pmap, &ptes, &pdes, &scr3); /* should be ok, but just in case ... */ sva &= PG_FRAME; @@ -1813,13 +1833,13 @@ pmap_write_protect(struct pmap *pmap, vaddr_t sva, vaddr_t eva, vm_prot_t prot) } if (shootall) - pmap_tlb_shoottlb(); + pmap_tlb_shoottlb(pmap, scr3 == 0); else - pmap_tlb_shootrange(pmap, sva, eva); + pmap_tlb_shootrange(pmap, sva, eva, scr3 == 0); - pmap_tlb_shootwait(); + pmap_unmap_ptes(pmap, scr3); - pmap_unmap_ptes(pmap); + pmap_tlb_shootwait(); } /* @@ -1836,18 +1856,18 @@ void pmap_unwire(struct pmap *pmap, vaddr_t va) { pt_entry_t *ptes; - pd_entry_t **pdes; + int level, offs; - pmap_map_ptes(pmap, &ptes, &pdes); + level = pmap_find_pte_direct(pmap, va, &ptes, &offs); - if (pmap_pdes_valid(va, pdes, NULL)) { + if (level == 0) { #ifdef DIAGNOSTIC - if (!pmap_valid_entry(ptes[pl1_i(va)])) + if (!pmap_valid_entry(ptes[offs])) panic("%s: invalid (unmapped) va 0x%lx", __func__, va); #endif - if (__predict_true((ptes[pl1_i(va)] & PG_W) != 0)) { - pmap_pte_clearbits(&ptes[pl1_i(va)], PG_W); + if (__predict_true((ptes[offs] & PG_W) != 0)) { + pmap_pte_clearbits(&ptes[offs], PG_W); pmap->pm_stats.wired_count--; } #ifdef DIAGNOSTIC @@ -1856,7 +1876,6 @@ pmap_unwire(struct pmap *pmap, vaddr_t va) "didn't change!\n", __func__, pmap, va); } #endif - pmap_unmap_ptes(pmap); } #ifdef DIAGNOSTIC else { @@ -1914,12 +1933,13 @@ pmap_enter(struct pmap *pmap, vaddr_t va, paddr_t pa, vm_prot_t prot, int flags) boolean_t nocache = (pa & PMAP_NOCACHE) != 0; boolean_t wc = (pa & PMAP_WC) != 0; int error; + paddr_t scr3; KASSERT(!(wc && nocache)); pa &= PMAP_PA_MASK; #ifdef DIAGNOSTIC - if (va == (vaddr_t) PDP_BASE || va == (vaddr_t) APDP_BASE) + if (va == (vaddr_t) PDP_BASE) panic("%s: trying to map over PDP/APDP!", __func__); /* sanity check: kernel PTPs should already have been pre-allocated */ @@ -1936,7 +1956,7 @@ pmap_enter(struct pmap *pmap, vaddr_t va, paddr_t pa, vm_prot_t prot, int flags) * map in ptes and get a pointer to our PTP (unless we are the kernel) */ - pmap_map_ptes(pmap, &ptes, &pdes); + pmap_map_ptes(pmap, &ptes, &pdes, &scr3); if (pmap == pmap_kernel()) { ptp = NULL; } else { @@ -2109,14 +2129,14 @@ enter_now: if (pmap_valid_entry(opte)) { if (nocache && (opte & PG_N) == 0) wbinvd(); - pmap_tlb_shootpage(pmap, va); + pmap_tlb_shootpage(pmap, va, scr3 == 0); pmap_tlb_shootwait(); } error = 0; out: - pmap_unmap_ptes(pmap); + pmap_unmap_ptes(pmap, scr3); PMAP_MAP_TO_HEAD_UNLOCK(); return error; @@ -2337,6 +2357,7 @@ pmap_dump(struct pmap *pmap, vaddr_t sva, vaddr_t eva) pt_entry_t *ptes, *pte; pd_entry_t **pdes; vaddr_t blkendva; + paddr_t scr3; /* * if end is out of range truncate. @@ -2348,7 +2369,7 @@ pmap_dump(struct pmap *pmap, vaddr_t sva, vaddr_t eva) PMAP_MAP_TO_HEAD_LOCK(); - pmap_map_ptes(pmap, &ptes, &pdes); + pmap_map_ptes(pmap, &ptes, &pdes, &scr3); /* * dumping a range of pages: we dump in PTP sized blocks (4MB) @@ -2373,7 +2394,7 @@ pmap_dump(struct pmap *pmap, vaddr_t sva, vaddr_t eva) sva, *pte, *pte & PG_FRAME); } } - pmap_unmap_ptes(pmap); + pmap_unmap_ptes(pmap, scr3); PMAP_MAP_TO_HEAD_UNLOCK(); } #endif @@ -2414,7 +2435,7 @@ volatile vaddr_t tlb_shoot_addr1; volatile vaddr_t tlb_shoot_addr2; void -pmap_tlb_shootpage(struct pmap *pm, vaddr_t va) +pmap_tlb_shootpage(struct pmap *pm, vaddr_t va, int shootself) { struct cpu_info *ci, *self = curcpu(); CPU_INFO_ITERATOR cii; @@ -2446,12 +2467,12 @@ pmap_tlb_shootpage(struct pmap *pm, vaddr_t va) splx(s); } - if (pmap_is_curpmap(pm)) + if (shootself) pmap_update_pg(va); } void -pmap_tlb_shootrange(struct pmap *pm, vaddr_t sva, vaddr_t eva) +pmap_tlb_shootrange(struct pmap *pm, vaddr_t sva, vaddr_t eva, int shootself) { struct cpu_info *ci, *self = curcpu(); CPU_INFO_ITERATOR cii; @@ -2485,13 +2506,13 @@ pmap_tlb_shootrange(struct pmap *pm, vaddr_t sva, vaddr_t eva) splx(s); } - if (pmap_is_curpmap(pm)) + if (shootself) for (va = sva; va < eva; va += PAGE_SIZE) pmap_update_pg(va); } void -pmap_tlb_shoottlb(void) +pmap_tlb_shoottlb(struct pmap *pm, int shootself) { struct cpu_info *ci, *self = curcpu(); CPU_INFO_ITERATOR cii; @@ -2499,7 +2520,7 @@ pmap_tlb_shoottlb(void) u_int64_t mask = 0; CPU_INFO_FOREACH(cii, ci) { - if (ci == self || !(ci->ci_flags & CPUF_RUNNING)) + if (ci == self || !pmap_is_active(pm, ci->ci_cpuid) || !(ci->ci_flags & CPUF_RUNNING)) continue; mask |= (1ULL << ci->ci_cpuid); wait++; @@ -2522,7 +2543,8 @@ pmap_tlb_shoottlb(void) splx(s); } - tlbflush(); + if (shootself) + tlbflush(); } void @@ -2535,26 +2557,30 @@ pmap_tlb_shootwait(void) #else void -pmap_tlb_shootpage(struct pmap *pm, vaddr_t va) +pmap_tlb_shootpage(struct pmap *pm, vaddr_t va, int shootself) { - if (pmap_is_curpmap(pm)) + if (shootself) pmap_update_pg(va); } void -pmap_tlb_shootrange(struct pmap *pm, vaddr_t sva, vaddr_t eva) +pmap_tlb_shootrange(struct pmap *pm, vaddr_t sva, vaddr_t eva, int shootself) { vaddr_t va; + if (!shootself) + return; + for (va = sva; va < eva; va += PAGE_SIZE) pmap_update_pg(va); } void -pmap_tlb_shoottlb(void) +pmap_tlb_shoottlb(struct pmap *pm, int shootself) { - tlbflush(); + if (shootself) + tlbflush(); } #endif /* MULTIPROCESSOR */ diff --git a/sys/arch/amd64/include/pmap.h b/sys/arch/amd64/include/pmap.h index 3796fd3..0342128 100644 --- a/sys/arch/amd64/include/pmap.h +++ b/sys/arch/amd64/include/pmap.h @@ -79,35 +79,32 @@ /* * The x86_64 pmap module closely resembles the i386 one. It uses - * the same recursive entry scheme, and the same alternate area - * trick for accessing non-current pmaps. See the i386 pmap.h - * for a description. The first obvious difference is that 2 extra - * levels of page table need to be dealt with. The level 1 page - * table pages are at: + * the same recursive entry scheme. See the i386 pmap.h for a + * description. The alternate area trick for accessing non-current + * pmaps has been removed, though, because it performs badly on SMP + * systems. + * The most obvious difference to i386 is that 2 extra levels of page + * table need to be dealt with. The level 1 page table pages are at: * * l1: 0x00007f8000000000 - 0x00007fffffffffff (39 bits, needs PML4 entry) * - * The alternate space is at: - * - * l1: 0xffffff0000000000 - 0xffffff7fffffffff (39 bits, needs PML4 entry) - * * The other levels are kept as physical pages in 3 UVM objects and are * temporarily mapped for virtual access when needed. * * The other obvious difference from i386 is that it has a direct map of all * physical memory in the VA range: * - * 0xfffffe8000000000 - 0xfffffeffffffffff + * 0xffffff0000000000 - 0xffffff7fffffffff + * + * The direct map is used in some cases to access PTEs of non-current pmaps. * * Note that address space is signed, so the layout for 48 bits is: * * +---------------------------------+ 0xffffffffffffffff * | Kernel Image | * +---------------------------------+ 0xffffff8000000000 - * | alt.L1 table (PTE pages) | - * +---------------------------------+ 0xffffff0000000000 * | Direct Map | - * +---------------------------------+ 0xfffffe8000000000 + * +---------------------------------+ 0xffffff0000000000 * ~ ~ * | | * | Kernel Space | @@ -155,44 +152,34 @@ #define L4_SLOT_PTE 255 #define L4_SLOT_KERN 256 #define L4_SLOT_KERNBASE 511 -#define L4_SLOT_APTE 510 -#define L4_SLOT_DIRECT 509 +#define L4_SLOT_DIRECT 510 #define PDIR_SLOT_KERN L4_SLOT_KERN #define PDIR_SLOT_PTE L4_SLOT_PTE -#define PDIR_SLOT_APTE L4_SLOT_APTE #define PDIR_SLOT_DIRECT L4_SLOT_DIRECT /* * the following defines give the virtual addresses of various MMU * data structures: - * PTE_BASE and APTE_BASE: the base VA of the linear PTE mappings - * PTD_BASE and APTD_BASE: the base VA of the recursive mapping of the PTD - * PDP_PDE and APDP_PDE: the VA of the PDE that points back to the PDP/APDP + * PTE_BASE: the base VA of the linear PTE mappings + * PTD_BASE: the base VA of the recursive mapping of the PTD + * PDP_PDE: the VA of the PDE that points back to the PDP * */ #define PTE_BASE ((pt_entry_t *) (L4_SLOT_PTE * NBPD_L4)) -#define APTE_BASE ((pt_entry_t *) (VA_SIGN_NEG((L4_SLOT_APTE * NBPD_L4)))) #define PMAP_DIRECT_BASE (VA_SIGN_NEG((L4_SLOT_DIRECT * NBPD_L4))) #define PMAP_DIRECT_END (VA_SIGN_NEG(((L4_SLOT_DIRECT + 1) * NBPD_L4))) #define L1_BASE PTE_BASE -#define AL1_BASE APTE_BASE #define L2_BASE ((pd_entry_t *)((char *)L1_BASE + L4_SLOT_PTE * NBPD_L3)) #define L3_BASE ((pd_entry_t *)((char *)L2_BASE + L4_SLOT_PTE * NBPD_L2)) #define L4_BASE ((pd_entry_t *)((char *)L3_BASE + L4_SLOT_PTE * NBPD_L1)) -#define AL2_BASE ((pd_entry_t *)((char *)AL1_BASE + L4_SLOT_PTE * NBPD_L3)) -#define AL3_BASE ((pd_entry_t *)((char *)AL2_BASE + L4_SLOT_PTE * NBPD_L2)) -#define AL4_BASE ((pd_entry_t *)((char *)AL3_BASE + L4_SLOT_PTE * NBPD_L1)) - #define PDP_PDE (L4_BASE + PDIR_SLOT_PTE) -#define APDP_PDE (L4_BASE + PDIR_SLOT_APTE) #define PDP_BASE L4_BASE -#define APDP_BASE AL4_BASE #define NKL4_MAX_ENTRIES (unsigned long)1 #define NKL3_MAX_ENTRIES (unsigned long)(NKL4_MAX_ENTRIES * 512) @@ -247,7 +234,6 @@ NKL3_MAX_ENTRIES, NKL4_MAX_ENTRIES } #define NBPD_INITIALIZER { NBPD_L1, NBPD_L2, NBPD_L3, NBPD_L4 } #define PDES_INITIALIZER { L2_BASE, L3_BASE, L4_BASE } -#define APDES_INITIALIZER { AL2_BASE, AL3_BASE, AL4_BASE } /* * PTP macros: @@ -410,15 +396,6 @@ void pmap_write_protect(struct pmap *, vaddr_t, vaddr_t reserve_dumppages(vaddr_t); /* XXX: not a pmap fn */ -void pmap_tlb_shootpage(struct pmap *, vaddr_t); -void pmap_tlb_shootrange(struct pmap *, vaddr_t, vaddr_t); -void pmap_tlb_shoottlb(void); -#ifdef MULTIPROCESSOR -void pmap_tlb_shootwait(void); -#else -#define pmap_tlb_shootwait() -#endif - paddr_t pmap_prealloc_lowmem_ptps(paddr_t); void pagezero(vaddr_t);