Module Name: src
Committed By: matt
Date: Wed Jul 17 23:15:20 UTC 2013
Modified Files:
src/sys/uvm/pmap: pmap.c pmap.h pmap_tlb.c pmap_tlb.h
Log Message:
Make this kcpuset_t instead of the private __cpuset_t
Add improvements for single TLB implementation (PPC, ARM).
To generate a diff of this commit:
cvs rdiff -u -r1.1 -r1.2 src/sys/uvm/pmap/pmap.c src/sys/uvm/pmap/pmap_tlb.h
cvs rdiff -u -r1.2 -r1.3 src/sys/uvm/pmap/pmap.h src/sys/uvm/pmap/pmap_tlb.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/uvm/pmap/pmap.c
diff -u src/sys/uvm/pmap/pmap.c:1.1 src/sys/uvm/pmap/pmap.c:1.2
--- src/sys/uvm/pmap/pmap.c:1.1 Wed Oct 3 00:51:45 2012
+++ src/sys/uvm/pmap/pmap.c Wed Jul 17 23:15:20 2013
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap.c,v 1.1 2012/10/03 00:51:45 christos Exp $ */
+/* $NetBSD: pmap.c,v 1.2 2013/07/17 23:15:20 matt Exp $ */
/*-
* Copyright (c) 1998, 2001 The NetBSD Foundation, Inc.
@@ -67,7 +67,7 @@
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.1 2012/10/03 00:51:45 christos Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.2 2013/07/17 23:15:20 matt Exp $");
/*
* Manages physical address maps.
@@ -209,10 +209,6 @@ struct pmap_kernel kernel_pmap_store = {
.pm_segtab = PMAP_INVALID_SEGTAB_ADDRESS,
.pm_minaddr = VM_MIN_KERNEL_ADDRESS,
.pm_maxaddr = VM_MAX_KERNEL_ADDRESS,
-#ifdef MULTIPROCESSOR
- .pm_active = 1,
- .pm_onproc = 1,
-#endif
},
};
@@ -309,18 +305,22 @@ pmap_page_syncicache(struct vm_page *pg)
#endif
struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg);
pv_entry_t pv = &mdpg->mdpg_first;
- __cpuset_t onproc = CPUSET_NULLSET;
+ kcpuset_t *onproc;
+#ifdef MULTIPROCESSOR
+ kcpuset_create(&onproc, true);
+#endif
(void)VM_PAGEMD_PVLIST_LOCK(mdpg, false);
+
if (pv->pv_pmap != NULL) {
for (; pv != NULL; pv = pv->pv_next) {
#ifdef MULTIPROCESSOR
- CPUSET_MERGE(onproc, pv->pv_pmap->pm_onproc);
- if (CPUSET_EQUAL_P(onproc, cpuset_info.cpus_running)) {
+ kcpuset_merge(onproc, pv->pv_pmap->pm_onproc);
+ if (kcpuset_match(onproc, kcpuset_running)) {
break;
}
#else
if (pv->pv_pmap == curpmap) {
- onproc = CPUSET_SINGLE(0);
+ onproc = curcpu()->ci_data.cpu_kcpuset;
break;
}
#endif
@@ -329,6 +329,9 @@ pmap_page_syncicache(struct vm_page *pg)
VM_PAGEMD_PVLIST_UNLOCK(mdpg);
kpreempt_disable();
pmap_md_page_syncicache(pg, onproc);
+#ifdef MULTIPROCESSOR
+ kcpuset_destroy(onproc);
+#endif
kpreempt_enable();
}
Index: src/sys/uvm/pmap/pmap_tlb.h
diff -u src/sys/uvm/pmap/pmap_tlb.h:1.1 src/sys/uvm/pmap/pmap_tlb.h:1.2
--- src/sys/uvm/pmap/pmap_tlb.h:1.1 Tue Jul 2 09:35:48 2013
+++ src/sys/uvm/pmap/pmap_tlb.h Wed Jul 17 23:15:20 2013
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap_tlb.h,v 1.1 2013/07/02 09:35:48 matt Exp $ */
+/* $NetBSD: pmap_tlb.h,v 1.2 2013/07/17 23:15:20 matt Exp $ */
/*
* Copyright (c) 1992, 1993
@@ -74,6 +74,12 @@
#ifndef _COMMON_PMAP_TLB_H_
#define _COMMON_PMAP_TLB_H_
+#include <sys/kcpuset.h>
+
+#if defined(MULTIPROCESSOR) && !defined(PMAP_TLB_MAX)
+#define PMAP_TLB_MAX MAXCPUS
+#endif
+
/*
* Per TLB (normally same as CPU) asid info
*/
@@ -110,10 +116,14 @@ struct pmap_tlb_info {
#ifdef MULTIPROCESSOR
pmap_t ti_victim;
uint32_t ti_synci_page_bitmap; /* page indices needing a syncicache */
- __cpuset_t ti_cpu_mask; /* bitmask of CPUs sharing this TLB */
- enum tlb_invalidate_op ti_tlbinvop;
+#if PMAP_TLB_MAX > 1
+ kcpuset_t *ti_kcpuset; /* bitmask of CPUs sharing this TLB */
u_int ti_index;
+ enum tlb_invalidate_op ti_tlbinvop;
#define tlbinfo_index(ti) ((ti)->ti_index)
+#else
+#define tlbinfo_index(ti) (0)
+#endif
struct evcnt ti_evcnt_synci_asts;
struct evcnt ti_evcnt_synci_all;
struct evcnt ti_evcnt_synci_pages;
Index: src/sys/uvm/pmap/pmap.h
diff -u src/sys/uvm/pmap/pmap.h:1.2 src/sys/uvm/pmap/pmap.h:1.3
--- src/sys/uvm/pmap/pmap.h:1.2 Tue Jul 2 09:35:48 2013
+++ src/sys/uvm/pmap/pmap.h Wed Jul 17 23:15:20 2013
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap.h,v 1.2 2013/07/02 09:35:48 matt Exp $ */
+/* $NetBSD: pmap.h,v 1.3 2013/07/17 23:15:20 matt Exp $ */
/*
* Copyright (c) 1992, 1993
@@ -113,6 +113,9 @@ void pmap_segtab_destroy(struct pmap *,
extern kmutex_t pmap_segtab_lock;
#endif /* _KERNEL */
+#ifdef MULTIPROCESSOR
+#include <sys/kcpuset.h>
+#endif
#include <uvm/pmap/pmap_tlb.h>
/*
@@ -120,8 +123,8 @@ extern kmutex_t pmap_segtab_lock;
*/
struct pmap {
#ifdef MULTIPROCESSOR
- __cpuset_t pm_active; /* pmap was active on ... */
- __cpuset_t pm_onproc; /* pmap is active on ... */
+ kcpuset_t *pm_active; /* pmap was active on ... */
+ kcpuset_t *pm_onproc; /* pmap is active on ... */
volatile u_int pm_shootdown_pending;
#endif
pmap_segtab_t * pm_segtab; /* pointers to pages of PTEs */
@@ -137,8 +140,8 @@ struct pmap {
#ifdef _KERNEL
struct pmap_kernel {
struct pmap kernel_pmap;
-#ifdef MULTIPROCESSOR
- struct pmap_asid_info kernel_pai[MAXCPUS-1];
+#if defined(MULTIPROCESSOR) && PMAP_TLB_MAX > 1
+ struct pmap_asid_info kernel_pai[PMAP_TLB_MAX-1];
#endif
};
Index: src/sys/uvm/pmap/pmap_tlb.c
diff -u src/sys/uvm/pmap/pmap_tlb.c:1.2 src/sys/uvm/pmap/pmap_tlb.c:1.3
--- src/sys/uvm/pmap/pmap_tlb.c:1.2 Tue Jul 2 09:35:48 2013
+++ src/sys/uvm/pmap/pmap_tlb.c Wed Jul 17 23:15:20 2013
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap_tlb.c,v 1.2 2013/07/02 09:35:48 matt Exp $ */
+/* $NetBSD: pmap_tlb.c,v 1.3 2013/07/17 23:15:20 matt Exp $ */
/*-
* Copyright (c) 2010 The NetBSD Foundation, Inc.
@@ -31,7 +31,7 @@
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: pmap_tlb.c,v 1.2 2013/07/02 09:35:48 matt Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pmap_tlb.c,v 1.3 2013/07/17 23:15:20 matt Exp $");
/*
* Manages address spaces in a TLB.
@@ -153,16 +153,15 @@ struct pmap_tlb_info pmap_tlb0_info = {
#endif
.ti_lock = &pmap_tlb0_mutex,
.ti_pais = LIST_HEAD_INITIALIZER(pmap_tlb0_info.ti_pais),
-#if defined(MULTIPROCESSOR)
- .ti_cpu_mask = 1,
+#if defined(MULTIPROCESSOR) && PMAP_TLB_MAX > 1
.ti_tlbinvop = TLBINV_NOBODY,
#endif
};
#undef IFCONSTANT
-#if defined(MULTIPROCESSOR)
-struct pmap_tlb_info *pmap_tlbs[MAXCPUS] = {
+#if defined(MULTIPROCESSOR) && PMAP_TLB_MAX > 1
+struct pmap_tlb_info *pmap_tlbs[PMAP_TLB_MAX] = {
[0] = &pmap_tlb0_info,
};
u_int pmap_ntlbs = 1;
@@ -194,6 +193,28 @@ pmap_pai_check(struct pmap_tlb_info *ti)
#endif
}
+#ifdef MULTIPROCESSOR
+static inline bool
+pmap_tlb_intersecting_active_p(pmap_t pm, struct pmap_tlb_info *ti)
+{
+#if PMAP_TLB_MAX == 1
+ return !kcpuset_iszero(pm->pm_active);
+#else
+ return kcpuset_intersecting_p(pm->pm_active, ti->ti_kcpuset);
+#endif
+}
+
+static inline bool
+pmap_tlb_intersecting_onproc_p(pmap_t pm, struct pmap_tlb_info *ti)
+{
+#if PMAP_TLB_MAX == 1
+ return !kcpuset_iszero(pm->pm_onproc);
+#else
+ return kcpuset_intersecting_p(pm->pm_onproc, ti->ti_kcpuset);
+#endif
+}
+#endif
+
static inline void
pmap_pai_reset(struct pmap_tlb_info *ti, struct pmap_asid_info *pai,
struct pmap *pm)
@@ -203,7 +224,7 @@ pmap_pai_reset(struct pmap_tlb_info *ti,
*/
KASSERT(pai->pai_asid > KERNEL_PID);
#if defined(MULTIPROCESSOR)
- KASSERT((pm->pm_onproc & ti->ti_cpu_mask) == 0);
+ KASSERT(!pmap_tlb_intersecting_onproc_p(pm, ti));
#endif
LIST_REMOVE(pai, pai_link);
#ifdef DIAGNOSTIC
@@ -224,7 +245,11 @@ pmap_pai_reset(struct pmap_tlb_info *ti,
* The bits in pm_active belonging to this TLB can only be changed
* while this TLB's lock is held.
*/
- CPUSET_DELSET(pm->pm_active, ti->ti_cpu_mask);
+#if PMAP_TLB_MAX == 1
+ kcpuset_zero(pm->pm_active);
+#else
+ kcpuset_atomicly_remove(pm->pm_active, ti->ti_kcpuset);
+#endif
#endif /* MULTIPROCESSOR */
}
@@ -260,7 +285,11 @@ void
pmap_tlb_info_init(struct pmap_tlb_info *ti)
{
#if defined(MULTIPROCESSOR)
+#if PMAP_TLB_MAX == 1
+ KASSERT(ti == &pmap_tlb0_info);
+#else
if (ti != &pmap_tlb0_info) {
+ KASSERT(pmap_ntlbs < PMAP_TLB_MAX);
KASSERT(pmap_tlbs[pmap_ntlbs] == NULL);
@@ -271,7 +300,7 @@ pmap_tlb_info_init(struct pmap_tlb_info
ti->ti_asids_free = ti->ti_asid_max - KERNEL_PID;
ti->ti_tlbinvop = TLBINV_NOBODY,
ti->ti_victim = NULL;
- ti->ti_cpu_mask = 0;
+ kcpuset_create(&ti->ti_kcpuset, true);
ti->ti_index = pmap_ntlbs++;
ti->ti_wired = 0;
pmap_tlbs[ti->ti_index] = ti;
@@ -280,9 +309,14 @@ pmap_tlb_info_init(struct pmap_tlb_info
pmap_tlb_info_evcnt_attach(ti);
return;
}
+#endif
#endif /* MULTIPROCESSOR */
KASSERT(ti == &pmap_tlb0_info);
mutex_init(ti->ti_lock, MUTEX_DEFAULT, IPL_SCHED);
+#if defined(MULTIPROCESSOR) && PMAP_TLB_MAX > 1
+ kcpuset_create(&ti->ti_kcpuset, true);
+ kcpuset_set(&ti->ti_kcpuset, cpu_index(curcpu()));
+#endif
if (ti->ti_asid_max == 0) {
ti->ti_asid_max = pmap_md_tlb_asid_max();
ti->ti_asids_free = ti->ti_asid_max - (KERNEL_PID + 1);
@@ -300,8 +334,9 @@ pmap_tlb_info_attach(struct pmap_tlb_inf
KASSERT(cold);
TLBINFO_LOCK(ti);
- const __cpuset_t cpu_mask = CPUSET_SINGLE(cpu_index(ci));
- CPUSET_ADDSET(ti->ti_cpu_mask, cpu_mask);
+#if PMAP_TLB_MAX > 1
+ kcpuset_set(ti->ti_kcpuset, cpu_index(ci));
+#endif
cpu_set_tlb_info(ci, ti);
/*
@@ -310,11 +345,9 @@ pmap_tlb_info_attach(struct pmap_tlb_inf
pmap_md_tlb_info_attach(ti, ci);
/*
- * Mark the kernel as active and "onproc" for this cpu. We assume
- * we are the only CPU running so atomic ops are not needed.
+ * The kernel pmap uses the kcpuset_running set so it's always
+ * up-to-date.
*/
- CPUSET_ADDSET(pmap_kernel()->pm_active, cpu_mask);
- CPUSET_ADDSET(pmap_kernel()->pm_onproc, cpu_mask);
TLBINFO_UNLOCK(ti);
}
#endif /* MULTIPROCESSOR */
@@ -417,7 +450,7 @@ pmap_tlb_asid_reinitialize(struct pmap_t
KASSERT(pm != pmap_kernel());
KASSERT(pai->pai_asid > KERNEL_PID);
#if defined(MULTIPROCESSOR)
- if (!CPUSET_EMPTY_P(CPUSET_SUBSET(pm->pm_onproc, ti->ti_cpu_mask))) {
+ if (pmap_tlb_intersecting_onproc_p(pm, ti)) {
if (!TLBINFO_ASID_INUSE_P(ti, pai->pai_asid)) {
TLBINFO_ASID_MARK_USED(ti, pai->pai_asid);
ti->ti_asids_free--;
@@ -440,6 +473,9 @@ pmap_tlb_asid_reinitialize(struct pmap_t
}
#if defined(MULTIPROCESSOR) && defined(PMAP_NEED_TLB_SHOOTDOWN)
+#if PMAP_MAX_TLB == 1
+#error shootdown not required for single TLB systems
+#endif
void
pmap_tlb_shootdown_process(void)
{
@@ -463,7 +499,7 @@ pmap_tlb_shootdown_process(void)
*/
struct pmap_asid_info * const pai = PMAP_PAI(ti->ti_victim, ti);
KASSERT(ti->ti_victim != pmap_kernel());
- if (!CPUSET_EMPTY_P(CPUSET_SUBSET(ti->ti_victim->pm_onproc, ti->ti_cpu_mask))) {
+ if (!pmap_tlb_intersecting_onproc_p(ti_victim->pm_onproc, ti)) {
/*
* The victim is an active pmap so we will just
* invalidate its TLB entries.
@@ -479,7 +515,7 @@ pmap_tlb_shootdown_process(void)
* next called for this pmap, it will allocate a new
* ASID.
*/
- KASSERT(!CPUSET_EMPTY_P(CPUSET_SUBSET(pm->pm_onproc, ti->ti_cpu_mask)));
+ KASSERT(!pmap_tlb_intersecting_onproc_p(pm, ti));
pmap_pai_reset(ti, pai, PAI_PMAP(pai, ti));
}
break;
@@ -546,8 +582,11 @@ pmap_tlb_shootdown_bystanders(pmap_t pm)
/*
* We don't need to deal our own TLB.
*/
- __cpuset_t pm_active =
- CPUSET_EXCLUDE(pm->pm_active, cpu_tlb_info(curcpu())->ti_cpu_mask);
+ kcpuset_t *pm_active;
+
+ kcpuset_clone(&pm_active, pm->pm_active);
+ kcpuset_atomicly_remove(pm->pm_active,
+ cpu_tlb_info(curcpu())->ti_kcpuset);
const bool kernel_p = (pm == pmap_kernel());
bool ipi_sent = false;
@@ -556,21 +595,21 @@ pmap_tlb_shootdown_bystanders(pmap_t pm)
* have been made so they will already be cognizant of them.
*/
- for (size_t i = 0; !CPUSET_EMPTY_P(pm_active); i++) {
+ for (size_t i = 0; !kcpuset_iszero(pm_active); i++) {
KASSERT(i < pmap_ntlbs);
struct pmap_tlb_info * const ti = pmap_tlbs[i];
KASSERT(tlbinfo_index(ti) == i);
/*
* Skip this TLB if there are no active mappings for it.
*/
- if (CPUSET_EMPTY_P(CPUSET_SUBSET(pm_active, ti->ti_cpu_mask)))
+ if (!kcpuset_intersecting_p(pm_active, ti->ti_kcpuset))
continue;
struct pmap_asid_info * const pai = PMAP_PAI(pm, ti);
- CPUSET_DELSET(pm_active, ti->ti_cpu_mask);
+ kcpuset_remove(pm_active, ti->ti_kcpuset);
TLBINFO_LOCK(ti);
- const __cpuset onproc = CPUSET_SUBSET(pm->pm_onproc,
- ti->ti_cpu_mask);
- if (onproc != 0) {
+ if (pmap_tlb_intersecting_onproc_p(pm, ti)) {
+ cpuid_t j = kcpuset_ffs_intersecting(pm->pm_onproc,
+ ti->ti_kcpuset);
if (kernel_p) {
ti->ti_tlbinvop =
TLBINV_KERNEL_MAP(ti->ti_tlbinvop);
@@ -602,13 +641,11 @@ pmap_tlb_shootdown_bystanders(pmap_t pm)
* change now that we have released the lock but we
* can tolerate spurious shootdowns.
*/
- KASSERT(!CPUSET_EMPTY_P(onproc));
- u_int j = CPUSET_NEXT(onproc);
cpu_send_ipi(cpu_lookup(j), IPI_SHOOTDOWN);
ipi_sent = true;
continue;
}
- if (!CPUSET_EMPTY_P(CPUSET_SUBSET(pm->pm_active, ti->ti_cpu_mask) {
+ if (!pmap_tlb_intersecting_active_p(pm, ti)) {
/*
* If this pmap has an ASID assigned but it's not
* currently running, nuke its ASID. Next time the
@@ -622,6 +659,8 @@ pmap_tlb_shootdown_bystanders(pmap_t pm)
TLBINFO_UNLOCK(ti);
}
+ kcpuset_destroy(pm_active);
+
return ipi_sent;
}
#endif /* MULTIPROCESSOR && PMAP_NEED_TLB_SHOOTDOWN */
@@ -684,8 +723,8 @@ pmap_tlb_asid_alloc(struct pmap_tlb_info
KASSERT(pai->pai_asid == 0);
KASSERT(pai->pai_link.le_prev == NULL);
#if defined(MULTIPROCESSOR)
- KASSERT(CPUSET_EMPTY_P(CPUSET_SUBSET(pm->pm_onproc, ti->ti_cpu_mask)));
- KASSERT(CPUSET_EMPTY_P(CPUSET_SUBSET(pm->pm_active, ti->ti_cpu_mask)));
+ KASSERT(pmap_tlb_intersecting_onproc_p(pm, ti));
+ KASSERT(pmap_tlb_intersecting_active_p(pm, ti));
#endif
KASSERT(ti->ti_asids_free > 0);
KASSERT(ti->ti_asid_hint <= ti->ti_asid_max);
@@ -743,7 +782,11 @@ pmap_tlb_asid_alloc(struct pmap_tlb_info
* The bits in pm_active belonging to this TLB can only be changed
* while this TLBs lock is held.
*/
- atomic_or_32(&pm->pm_active, ti->ti_cpu_mask);
+#if PMAP_TLB_MAX == 1
+ kcpuset_copy(pm->pm_active, kcpuset_running);
+#else
+ kcpuset_atomicly_merge(pm->pm_active, ti->ti_kcpuset);
+#endif
#endif
}
@@ -792,7 +835,7 @@ pmap_tlb_asid_acquire(pmap_t pm, struct
* be changed while this TLBs lock is held unless atomic
* operations are used.
*/
- CPUSET_ADD(pm->pm_onproc, cpu_index(ci));
+ kcpuset_atomic_set(pm->pm_onproc, cpu_index(ci));
#endif
ci->ci_pmap_asid_cur = pai->pai_asid;
tlb_set_asid(pai->pai_asid);
@@ -817,15 +860,15 @@ pmap_tlb_asid_deactivate(pmap_t pm)
if (pm != pmap_kernel() && pm->pm_onproc != 0) {
struct cpu_info * const ci = curcpu();
KASSERT(!cpu_intr_p());
- KASSERTMSG(pm->pm_onproc & CPUSET_SINGLE(cpu_index(ci)),
- "%s: pmap %p onproc %#x doesn't include cpu %d (%p)",
+ KASSERTMSG(kcpuset_isset(pm->pm_onproc, cpu_index(ci)),
+ "%s: pmap %p onproc %p doesn't include cpu %d (%p)",
__func__, pm, pm->pm_onproc, cpu_index(ci), ci);
/*
* The bits in pm_onproc that belong to this TLB can
* be changed while this TLBs lock is not held as long
* as we use atomic ops.
*/
- CPUSET_DEL(pm->pm_onproc, cpu_index(ci));
+ kcpuset_atomic_clear(pm->pm_onproc, cpu_index(ci));
}
#elif defined(DEBUG)
curcpu()->ci_pmap_asid_cur = 0;
@@ -839,18 +882,24 @@ pmap_tlb_asid_release_all(struct pmap *p
{
KASSERT(pm != pmap_kernel());
#if defined(MULTIPROCESSOR)
- KASSERT(CPUSET_EMPTY_P(pm->pm_onproc));
- for (u_int i = 0; !CPUSET_EMPTY_P(pm->pm_active); i++) {
+ KASSERT(kcpuset_iszero(pm->pm_onproc));
+#if PMAP_TLB_MAX > 1
+ for (u_int i = 0; !kcpuset_iszero(pm->pm_active); i++) {
KASSERT(i < pmap_ntlbs);
struct pmap_tlb_info * const ti = pmap_tlbs[i];
- if (!CPUSET_EMPTY_P(CPUSET_SUBSET(pm->pm_active, ti->ti_cpu_mask))) {
+#else
+ struct pmap_tlb_info * const ti = &pmap_tlb0_info;
+#endif
+ if (!pmap_tlb_intersecting_onproc_p(pm, ti)) {
struct pmap_asid_info * const pai = PMAP_PAI(pm, ti);
TLBINFO_LOCK(ti);
KASSERT(ti->ti_victim != pm);
pmap_pai_reset(ti, pai, pm);
TLBINFO_UNLOCK(ti);
}
+#if PMAP_TLB_MAX > 1
}
+#endif
#else
/*
* Handle the case of an UP kernel which only has, at most, one ASID.
