Author: kib Date: Fri Mar 28 16:07:27 2014 New Revision: 263878 URL: http://svnweb.freebsd.org/changeset/base/263878
Log: Several fixes for the PCID implementation: - When clearing a bit for a cpuid in pmap->pm_save, ensure that the cpuid is not set in pm_active. The pm_save indicates which CPUs may have cached translations for given PCID, which implies that a CPU executing with the given pmap active have the translations cached. [1] - In smp_masked_invltlb(), pass pmap to smp_targeted_tlb_shootdown(). [1] - In invlrng_handler(), check for the special values of pcid (0 and -1) and do corresponding global or total invalidations before checking for performing PCID-specific range invalidation with INVPCID_ADDR. [2] - In invltlb_pcid_handler(), do not read %cr3 unless needed. [2] - Do minor style tweaks. [2] Submitted by: Henrik Gulbrandsen <hen...@gulbra.net> [1] Other parts sponsored by: The FreeBSD Foundation [2] Tested by: Henrik Gulbrandsen, pho MFC after: 1 week Modified: head/sys/amd64/amd64/mp_machdep.c head/sys/amd64/amd64/pmap.c Modified: head/sys/amd64/amd64/mp_machdep.c ============================================================================== --- head/sys/amd64/amd64/mp_machdep.c Fri Mar 28 15:41:04 2014 (r263877) +++ head/sys/amd64/amd64/mp_machdep.c Fri Mar 28 16:07:27 2014 (r263878) @@ -1257,7 +1257,7 @@ smp_masked_invltlb(cpuset_t mask, pmap_t { if (smp_started) { - smp_targeted_tlb_shootdown(mask, IPI_INVLTLB, NULL, 0, 0); + smp_targeted_tlb_shootdown(mask, IPI_INVLTLB, pmap, 0, 0); #ifdef COUNT_XINVLTLB_HITS ipi_masked_global++; #endif @@ -1517,6 +1517,7 @@ void invltlb_pcid_handler(void) { uint64_t cr3; + u_int cpuid; #ifdef COUNT_XINVLTLB_HITS xhits_gbl[PCPU_GET(cpuid)]++; #endif /* COUNT_XINVLTLB_HITS */ @@ -1524,14 +1525,13 @@ invltlb_pcid_handler(void) (*ipi_invltlb_counts[PCPU_GET(cpuid)])++; #endif /* COUNT_IPIS */ - cr3 = rcr3(); if (smp_tlb_invpcid.pcid != (uint64_t)-1 && smp_tlb_invpcid.pcid != 0) { - if (invpcid_works) { invpcid(&smp_tlb_invpcid, INVPCID_CTX); } else { /* Otherwise reload %cr3 twice. */ + cr3 = rcr3(); if (cr3 != pcid_cr3) { load_cr3(pcid_cr3); cr3 |= CR3_PCID_SAVE; @@ -1541,8 +1541,11 @@ invltlb_pcid_handler(void) } else { invltlb_globpcid(); } - if (smp_tlb_pmap != NULL) - CPU_CLR_ATOMIC(PCPU_GET(cpuid), &smp_tlb_pmap->pm_save); + if (smp_tlb_pmap != NULL) { + cpuid = PCPU_GET(cpuid); + if (!CPU_ISSET(cpuid, &smp_tlb_pmap->pm_active)) + CPU_CLR_ATOMIC(cpuid, &smp_tlb_pmap->pm_save); + } atomic_add_int(&smp_tlb_wait, 1); } @@ -1608,7 +1611,10 @@ invlpg_range(vm_offset_t start, vm_offse void invlrng_handler(void) { + struct invpcid_descr d; vm_offset_t addr; + uint64_t cr3; + u_int cpuid; #ifdef COUNT_XINVLTLB_HITS xhits_rng[PCPU_GET(cpuid)]++; #endif /* COUNT_XINVLTLB_HITS */ @@ -1618,15 +1624,7 @@ invlrng_handler(void) addr = smp_tlb_invpcid.addr; if (pmap_pcid_enabled) { - if (invpcid_works) { - struct invpcid_descr d; - - d = smp_tlb_invpcid; - do { - invpcid(&d, INVPCID_ADDR); - d.addr += PAGE_SIZE; - } while (d.addr < smp_tlb_addr2); - } else if (smp_tlb_invpcid.pcid == 0) { + if (smp_tlb_invpcid.pcid == 0) { /* * kernel pmap - use invlpg to invalidate * global mapping. @@ -1635,12 +1633,18 @@ invlrng_handler(void) } else if (smp_tlb_invpcid.pcid == (uint64_t)-1) { invltlb_globpcid(); if (smp_tlb_pmap != NULL) { - CPU_CLR_ATOMIC(PCPU_GET(cpuid), - &smp_tlb_pmap->pm_save); + cpuid = PCPU_GET(cpuid); + if (!CPU_ISSET(cpuid, &smp_tlb_pmap->pm_active)) + CPU_CLR_ATOMIC(cpuid, + &smp_tlb_pmap->pm_save); } + } else if (invpcid_works) { + d = smp_tlb_invpcid; + do { + invpcid(&d, INVPCID_ADDR); + d.addr += PAGE_SIZE; + } while (d.addr <= smp_tlb_addr2); } else { - uint64_t cr3; - cr3 = rcr3(); if (cr3 != pcid_cr3) load_cr3(pcid_cr3 | CR3_PCID_SAVE); Modified: head/sys/amd64/amd64/pmap.c ============================================================================== --- head/sys/amd64/amd64/pmap.c Fri Mar 28 15:41:04 2014 (r263877) +++ head/sys/amd64/amd64/pmap.c Fri Mar 28 16:07:27 2014 (r263878) @@ -838,7 +838,7 @@ pmap_bootstrap(vm_paddr_t *firstaddr) kernel_pmap->pm_pml4 = (pdp_entry_t *)PHYS_TO_DMAP(KPML4phys); kernel_pmap->pm_cr3 = KPML4phys; CPU_FILL(&kernel_pmap->pm_active); /* don't allow deactivation */ - CPU_ZERO(&kernel_pmap->pm_save); + CPU_FILL(&kernel_pmap->pm_save); /* always superset of pm_active */ TAILQ_INIT(&kernel_pmap->pm_pvchunk); kernel_pmap->pm_flags = pmap_flags; @@ -1494,7 +1494,8 @@ pmap_invalidate_all(pmap_t pmap) } else { invltlb_globpcid(); } - CPU_CLR_ATOMIC(cpuid, &pmap->pm_save); + if (!CPU_ISSET(cpuid, &pmap->pm_active)) + CPU_CLR_ATOMIC(cpuid, &pmap->pm_save); smp_invltlb(pmap); } else { other_cpus = all_cpus; @@ -1528,7 +1529,8 @@ pmap_invalidate_all(pmap_t pmap) } } else if (CPU_ISSET(cpuid, &pmap->pm_active)) invltlb(); - CPU_CLR_ATOMIC(cpuid, &pmap->pm_save); + if (!CPU_ISSET(cpuid, &pmap->pm_active)) + CPU_CLR_ATOMIC(cpuid, &pmap->pm_save); if (pmap_pcid_enabled) CPU_AND(&other_cpus, &pmap->pm_save); else _______________________________________________ svn-src-head@freebsd.org mailing list http://lists.freebsd.org/mailman/listinfo/svn-src-head To unsubscribe, send any mail to "svn-src-head-unsubscr...@freebsd.org"