Module Name: src Committed By: rin Date: Fri Dec 15 09:33:30 UTC 2023
Modified Files: src/sys/arch/powerpc/oea: pmap.c Log Message: powerpc/oea: pmap: Rework pmap_pte_spill() It was broken in many ways... Now, it gets working stable both for OEA and OEA64_BRIDGE, as far as I can see. Part of PR kern/57621 To generate a diff of this commit: cvs rdiff -u -r1.117 -r1.118 src/sys/arch/powerpc/oea/pmap.c Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files.
Modified files: Index: src/sys/arch/powerpc/oea/pmap.c diff -u src/sys/arch/powerpc/oea/pmap.c:1.117 src/sys/arch/powerpc/oea/pmap.c:1.118 --- src/sys/arch/powerpc/oea/pmap.c:1.117 Fri Dec 15 09:32:05 2023 +++ src/sys/arch/powerpc/oea/pmap.c Fri Dec 15 09:33:29 2023 @@ -1,4 +1,4 @@ -/* $NetBSD: pmap.c,v 1.117 2023/12/15 09:32:05 rin Exp $ */ +/* $NetBSD: pmap.c,v 1.118 2023/12/15 09:33:29 rin Exp $ */ /*- * Copyright (c) 2001 The NetBSD Foundation, Inc. * All rights reserved. @@ -63,7 +63,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.117 2023/12/15 09:32:05 rin Exp $"); +__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.118 2023/12/15 09:33:29 rin Exp $"); #define PMAP_NOOPNAMES @@ -879,48 +879,35 @@ pmap_pte_insert(int ptegidx, struct pte * Tries to spill a page table entry from the overflow area. * This runs in either real mode (if dealing with a exception spill) * or virtual mode when dealing with manually spilling one of the - * kernel's pte entries. In either case, interrupts are already - * disabled. + * kernel's pte entries. */ int -pmap_pte_spill(struct pmap *pm, vaddr_t addr, bool exec) +pmap_pte_spill(struct pmap *pm, vaddr_t addr, bool isi_p) { - struct pvo_entry *source_pvo, *victim_pvo, *next_pvo; - struct pvo_entry *pvo; - /* XXX: gcc -- vpvoh is always set at either *1* or *2* */ - struct pvo_tqhead *pvoh, *vpvoh = NULL; - int ptegidx, i, j; + struct pvo_tqhead *spvoh, *vpvoh; + struct pvo_entry *pvo, *source_pvo, *victim_pvo; volatile struct pteg *pteg; volatile struct pte *pt; + register_t msr, vsid, hash; + int ptegidx, hid, i, j; + int done = 0; PMAP_LOCK(); + msr = pmap_interrupts_off(); + + /* XXXRO paranoid? */ + if (pm->pm_evictions == 0) + goto out; ptegidx = va_to_pteg(pm, addr); /* - * Have to substitute some entry. Use the primary hash for this. - * Use low bits of timebase as random generator. Make sure we are - * not picking a kernel pte for replacement. + * Find source pvo. */ - pteg = &pmap_pteg_table[ptegidx]; - i = MFTB() & 7; - for (j = 0; j < 8; j++) { - pt = &pteg->pt[i]; - if ((pt->pte_hi & PTE_VALID) == 0) - break; - if (VSID_TO_HASH((pt->pte_hi & PTE_VSID) >> PTE_VSID_SHFT) - < PHYSMAP_VSIDBITS) - break; - i = (i + 1) & 7; - } - KASSERT(j < 8); - + spvoh = &pmap_pvo_table[ptegidx]; source_pvo = NULL; - victim_pvo = NULL; - pvoh = &pmap_pvo_table[ptegidx]; - TAILQ_FOREACH(pvo, pvoh, pvo_olink) { - + TAILQ_FOREACH(pvo, spvoh, pvo_olink) { /* * We need to find pvo entry for this address... */ @@ -931,101 +918,105 @@ pmap_pte_spill(struct pmap *pm, vaddr_t * a valid PTE, then we know we can't find it because all * evicted PVOs always are first in the list. */ - if (source_pvo == NULL && (pvo->pvo_pte.pte_hi & PTE_VALID)) + if ((pvo->pvo_pte.pte_hi & PTE_VALID) != 0) break; - if (source_pvo == NULL && pm == pvo->pvo_pmap && - addr == PVO_VADDR(pvo)) { - /* - * Now we have found the entry to be spilled into the - * pteg. Attempt to insert it into the page table. - */ - j = pmap_pte_insert(ptegidx, &pvo->pvo_pte); - if (j >= 0) { - PVO_PTEGIDX_SET(pvo, j); - PMAP_PVO_CHECK(pvo); /* sanity check */ - PVO_WHERE(pvo, SPILL_INSERT); - pvo->pvo_pmap->pm_evictions--; - PMAPCOUNT(ptes_spilled); - PMAPCOUNT2(((pvo->pvo_pte.pte_hi & PTE_HID) - ? pmap_evcnt_ptes_secondary - : pmap_evcnt_ptes_primary)[j]); - - /* - * Since we keep the evicted entries at the - * from of the PVO list, we need move this - * (now resident) PVO after the evicted - * entries. - */ - next_pvo = TAILQ_NEXT(pvo, pvo_olink); - - /* - * If we don't have to move (either we were the - * last entry or the next entry was valid), - * don't change our position. Otherwise - * move ourselves to the tail of the queue. - */ - if (next_pvo != NULL && - !(next_pvo->pvo_pte.pte_hi & PTE_VALID)) { - TAILQ_REMOVE(pvoh, pvo, pvo_olink); - TAILQ_INSERT_TAIL(pvoh, pvo, pvo_olink); - } - PMAP_UNLOCK(); - return 1; + if (pm == pvo->pvo_pmap && addr == PVO_VADDR(pvo)) { + if (isi_p) { + if (!PVO_EXECUTABLE_P(pvo)) + goto out; +#if defined(PMAP_OEA) || defined(PMAP_OEA64_BRIDGE) + int sr __diagused = + PVO_VADDR(pvo) >> ADDR_SR_SHFT; + KASSERT((pm->pm_sr[sr] & SR_NOEXEC) == 0); +#endif } + KASSERT(!PVO_PTEGIDX_ISSET(pvo)); + /* XXXRO where check */ source_pvo = pvo; - if (exec && !PVO_EXECUTABLE_P(source_pvo)) { - PMAP_UNLOCK(); - return 0; - } - if (victim_pvo != NULL) - break; - } - - /* - * We also need the pvo entry of the victim we are replacing - * so save the R & C bits of the PTE. - */ - if ((pt->pte_hi & PTE_HID) == 0 && victim_pvo == NULL && - pmap_pte_compare(pt, &pvo->pvo_pte)) { - vpvoh = pvoh; /* *1* */ - victim_pvo = pvo; - if (source_pvo != NULL) - break; + break; } } - if (source_pvo == NULL) { PMAPCOUNT(ptes_unspilled); - PMAP_UNLOCK(); - return 0; + goto out; } - if (victim_pvo == NULL) { - if ((pt->pte_hi & PTE_HID) == 0) - panic("pmap_pte_spill: victim p-pte (%p) has " - "no pvo entry!", pt); + /* + * Now we have found the entry to be spilled into the + * pteg. Attempt to insert it into the page table. + */ + i = pmap_pte_insert(ptegidx, &pvo->pvo_pte); + if (i >= 0) { + PVO_PTEGIDX_SET(pvo, i); + PMAP_PVO_CHECK(pvo); /* sanity check */ + PVO_WHERE(pvo, SPILL_INSERT); + pvo->pvo_pmap->pm_evictions--; + PMAPCOUNT(ptes_spilled); + PMAPCOUNT2(((pvo->pvo_pte.pte_hi & PTE_HID) != 0 + ? pmap_evcnt_ptes_secondary + : pmap_evcnt_ptes_primary)[i]); - /* - * If this is a secondary PTE, we need to search - * its primary pvo bucket for the matching PVO. - */ - vpvoh = &pmap_pvo_table[ptegidx ^ pmap_pteg_mask]; /* *2* */ - TAILQ_FOREACH(pvo, vpvoh, pvo_olink) { - PMAP_PVO_CHECK(pvo); /* sanity check */ + TAILQ_REMOVE(spvoh, pvo, pvo_olink); + TAILQ_INSERT_TAIL(spvoh, pvo, pvo_olink); - /* - * We also need the pvo entry of the victim we are - * replacing so save the R & C bits of the PTE. - */ - if (pmap_pte_compare(pt, &pvo->pvo_pte)) { - victim_pvo = pvo; - break; - } + done = 1; + goto out; + } + + /* + * Have to substitute some entry. Use the primary hash for this. + * Use low bits of timebase as random generator. + * + * XXX: + * Make sure we are not picking a kernel pte for replacement. + */ + hid = 0; + i = MFTB() & 7; + pteg = &pmap_pteg_table[ptegidx]; + retry: + for (j = 0; j < 8; j++, i = (i + 1) & 7) { + pt = &pteg->pt[i]; + + if ((pt->pte_hi & PTE_VALID) == 0) + break; + + vsid = (pt->pte_hi & PTE_VSID) >> PTE_VSID_SHFT; + hash = VSID_TO_HASH(vsid); + if (hash < PHYSMAP_VSIDBITS) + break; + } + if (j == 8) { + if (hid != 0) + panic("%s: no victim\n", __func__); + hid = PTE_HID; + pteg = &pmap_pteg_table[ptegidx ^ pmap_pteg_mask]; + goto retry; + } + + /* + * We also need the pvo entry of the victim we are replacing + * so save the R & C bits of the PTE. + */ + if ((pt->pte_hi & PTE_HID) == hid) + vpvoh = spvoh; + else + vpvoh = &pmap_pvo_table[ptegidx ^ pmap_pteg_mask]; + victim_pvo = NULL; + TAILQ_FOREACH(pvo, vpvoh, pvo_olink) { + PMAP_PVO_CHECK(pvo); /* sanity check */ + + if ((pvo->pvo_pte.pte_hi & PTE_VALID) == 0) + continue; + + if (pmap_pte_compare(pt, &pvo->pvo_pte)) { + victim_pvo = pvo; + break; } - if (victim_pvo == NULL) - panic("pmap_pte_spill: victim s-pte (%p) has " - "no pvo entry!", pt); + } + if (victim_pvo == NULL) { + panic("%s: victim p-pte (%p) has no pvo entry!", + __func__, pt); } /* @@ -1041,7 +1032,10 @@ pmap_pte_spill(struct pmap *pm, vaddr_t * we lose any ref/chg bit changes contained in the TLB * entry. */ - source_pvo->pvo_pte.pte_hi &= ~PTE_HID; + if (hid == 0) + source_pvo->pvo_pte.pte_hi &= ~PTE_HID; + else + source_pvo->pvo_pte.pte_hi |= PTE_HID; /* * To enforce the PVO list ordering constraint that all @@ -1050,8 +1044,8 @@ pmap_pte_spill(struct pmap *pm, vaddr_t * victim PVO to the head of its list (which might not be * the same list, if the victim was using the secondary hash). */ - TAILQ_REMOVE(pvoh, source_pvo, pvo_olink); - TAILQ_INSERT_TAIL(pvoh, source_pvo, pvo_olink); + TAILQ_REMOVE(spvoh, source_pvo, pvo_olink); + TAILQ_INSERT_TAIL(spvoh, source_pvo, pvo_olink); TAILQ_REMOVE(vpvoh, victim_pvo, pvo_olink); TAILQ_INSERT_HEAD(vpvoh, victim_pvo, pvo_olink); pmap_pte_unset(pt, &victim_pvo->pvo_pte, victim_pvo->pvo_vaddr); @@ -1071,8 +1065,12 @@ pmap_pte_spill(struct pmap *pm, vaddr_t PMAP_PVO_CHECK(victim_pvo); PMAP_PVO_CHECK(source_pvo); + done = 1; + + out: + pmap_interrupts_restore(msr); PMAP_UNLOCK(); - return 1; + return done; } /*