Author: kib
Date: Fri Mar 28 16:07:27 2014
New Revision: 263878
URL: http://svnweb.freebsd.org/changeset/base/263878
Log:
Several fixes for the PCID implementation:
- When clearing a bit for a cpuid in pmap->pm_save, ensure that the
cpuid is not set in pm_active. The pm_save indicates which CPUs may
have cached translations for given PCID, which implies that a CPU
executing with the given pmap active have the translations
cached. [1]
- In smp_masked_invltlb(), pass pmap to smp_targeted_tlb_shootdown(). [1]
- In invlrng_handler(), check for the special values of pcid (0 and
-1) and do corresponding global or total invalidations before
checking for performing PCID-specific range invalidation with
INVPCID_ADDR. [2]
- In invltlb_pcid_handler(), do not read %cr3 unless needed. [2]
- Do minor style tweaks. [2]
Submitted by: Henrik Gulbrandsen <hen...@gulbra.net> [1]
Other parts sponsored by: The FreeBSD Foundation [2]
Tested by: Henrik Gulbrandsen, pho
MFC after: 1 week
Modified:
head/sys/amd64/amd64/mp_machdep.c
head/sys/amd64/amd64/pmap.c
Modified: head/sys/amd64/amd64/mp_machdep.c
==============================================================================
--- head/sys/amd64/amd64/mp_machdep.c Fri Mar 28 15:41:04 2014
(r263877)
+++ head/sys/amd64/amd64/mp_machdep.c Fri Mar 28 16:07:27 2014
(r263878)
@@ -1257,7 +1257,7 @@ smp_masked_invltlb(cpuset_t mask, pmap_t
{
if (smp_started) {
- smp_targeted_tlb_shootdown(mask, IPI_INVLTLB, NULL, 0, 0);
+ smp_targeted_tlb_shootdown(mask, IPI_INVLTLB, pmap, 0, 0);
#ifdef COUNT_XINVLTLB_HITS
ipi_masked_global++;
#endif
@@ -1517,6 +1517,7 @@ void
invltlb_pcid_handler(void)
{
uint64_t cr3;
+ u_int cpuid;
#ifdef COUNT_XINVLTLB_HITS
xhits_gbl[PCPU_GET(cpuid)]++;
#endif /* COUNT_XINVLTLB_HITS */
@@ -1524,14 +1525,13 @@ invltlb_pcid_handler(void)
(*ipi_invltlb_counts[PCPU_GET(cpuid)])++;
#endif /* COUNT_IPIS */
- cr3 = rcr3();
if (smp_tlb_invpcid.pcid != (uint64_t)-1 &&
smp_tlb_invpcid.pcid != 0) {
-
if (invpcid_works) {
invpcid(&smp_tlb_invpcid, INVPCID_CTX);
} else {
/* Otherwise reload %cr3 twice. */
+ cr3 = rcr3();
if (cr3 != pcid_cr3) {
load_cr3(pcid_cr3);
cr3 |= CR3_PCID_SAVE;
@@ -1541,8 +1541,11 @@ invltlb_pcid_handler(void)
} else {
invltlb_globpcid();
}
- if (smp_tlb_pmap != NULL)
- CPU_CLR_ATOMIC(PCPU_GET(cpuid), &smp_tlb_pmap->pm_save);
+ if (smp_tlb_pmap != NULL) {
+ cpuid = PCPU_GET(cpuid);
+ if (!CPU_ISSET(cpuid, &smp_tlb_pmap->pm_active))
+ CPU_CLR_ATOMIC(cpuid, &smp_tlb_pmap->pm_save);
+ }
atomic_add_int(&smp_tlb_wait, 1);
}
@@ -1608,7 +1611,10 @@ invlpg_range(vm_offset_t start, vm_offse
void
invlrng_handler(void)
{
+ struct invpcid_descr d;
vm_offset_t addr;
+ uint64_t cr3;
+ u_int cpuid;
#ifdef COUNT_XINVLTLB_HITS
xhits_rng[PCPU_GET(cpuid)]++;
#endif /* COUNT_XINVLTLB_HITS */
@@ -1618,15 +1624,7 @@ invlrng_handler(void)
addr = smp_tlb_invpcid.addr;
if (pmap_pcid_enabled) {
- if (invpcid_works) {
- struct invpcid_descr d;
-
- d = smp_tlb_invpcid;
- do {
- invpcid(&d, INVPCID_ADDR);
- d.addr += PAGE_SIZE;
- } while (d.addr < smp_tlb_addr2);
- } else if (smp_tlb_invpcid.pcid == 0) {
+ if (smp_tlb_invpcid.pcid == 0) {
/*
* kernel pmap - use invlpg to invalidate
* global mapping.
@@ -1635,12 +1633,18 @@ invlrng_handler(void)
} else if (smp_tlb_invpcid.pcid == (uint64_t)-1) {
invltlb_globpcid();
if (smp_tlb_pmap != NULL) {
- CPU_CLR_ATOMIC(PCPU_GET(cpuid),
- &smp_tlb_pmap->pm_save);
+ cpuid = PCPU_GET(cpuid);
+ if (!CPU_ISSET(cpuid, &smp_tlb_pmap->pm_active))
+ CPU_CLR_ATOMIC(cpuid,
+ &smp_tlb_pmap->pm_save);
}
+ } else if (invpcid_works) {
+ d = smp_tlb_invpcid;
+ do {
+ invpcid(&d, INVPCID_ADDR);
+ d.addr += PAGE_SIZE;
+ } while (d.addr <= smp_tlb_addr2);
} else {
- uint64_t cr3;
-
cr3 = rcr3();
if (cr3 != pcid_cr3)
load_cr3(pcid_cr3 | CR3_PCID_SAVE);
Modified: head/sys/amd64/amd64/pmap.c
==============================================================================
--- head/sys/amd64/amd64/pmap.c Fri Mar 28 15:41:04 2014 (r263877)
+++ head/sys/amd64/amd64/pmap.c Fri Mar 28 16:07:27 2014 (r263878)
@@ -838,7 +838,7 @@ pmap_bootstrap(vm_paddr_t *firstaddr)
kernel_pmap->pm_pml4 = (pdp_entry_t *)PHYS_TO_DMAP(KPML4phys);
kernel_pmap->pm_cr3 = KPML4phys;
CPU_FILL(&kernel_pmap->pm_active); /* don't allow deactivation */
- CPU_ZERO(&kernel_pmap->pm_save);
+ CPU_FILL(&kernel_pmap->pm_save); /* always superset of pm_active
*/
TAILQ_INIT(&kernel_pmap->pm_pvchunk);
kernel_pmap->pm_flags = pmap_flags;
@@ -1494,7 +1494,8 @@ pmap_invalidate_all(pmap_t pmap)
} else {
invltlb_globpcid();
}
- CPU_CLR_ATOMIC(cpuid, &pmap->pm_save);
+ if (!CPU_ISSET(cpuid, &pmap->pm_active))
+ CPU_CLR_ATOMIC(cpuid, &pmap->pm_save);
smp_invltlb(pmap);
} else {
other_cpus = all_cpus;
@@ -1528,7 +1529,8 @@ pmap_invalidate_all(pmap_t pmap)
}
} else if (CPU_ISSET(cpuid, &pmap->pm_active))
invltlb();
- CPU_CLR_ATOMIC(cpuid, &pmap->pm_save);
+ if (!CPU_ISSET(cpuid, &pmap->pm_active))
+ CPU_CLR_ATOMIC(cpuid, &pmap->pm_save);
if (pmap_pcid_enabled)
CPU_AND(&other_cpus, &pmap->pm_save);
else
_______________________________________________
svn-src-head@freebsd.org mailing list
http://lists.freebsd.org/mailman/listinfo/svn-src-head
To unsubscribe, send any mail to "svn-src-head-unsubscr...@freebsd.org"
