Module Name: src
Committed By: matt
Date: Wed Jul 3 21:37:35 UTC 2013
Modified Files:
src/sys/arch/arm/arm32: pmap.c
src/sys/arch/arm/include/arm32: pmap.h
Log Message:
Add l2pte_set and l2pte_reset inlines to set/reset a pte. These will be
used to support > 4KB pages sizes.
Don't use >> L1_S_SHIFT, use L1_IDX() instead.
To generate a diff of this commit:
cvs rdiff -u -r1.261 -r1.262 src/sys/arch/arm/arm32/pmap.c
cvs rdiff -u -r1.120 -r1.121 src/sys/arch/arm/include/arm32/pmap.h
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/arm/arm32/pmap.c
diff -u src/sys/arch/arm/arm32/pmap.c:1.261 src/sys/arch/arm/arm32/pmap.c:1.262
--- src/sys/arch/arm/arm32/pmap.c:1.261 Wed Jul 3 15:30:24 2013
+++ src/sys/arch/arm/arm32/pmap.c Wed Jul 3 21:37:35 2013
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap.c,v 1.261 2013/07/03 15:30:24 matt Exp $ */
+/* $NetBSD: pmap.c,v 1.262 2013/07/03 21:37:35 matt Exp $ */
/*
* Copyright 2003 Wasabi Systems, Inc.
@@ -212,7 +212,7 @@
#include <arm/cpuconf.h>
#include <arm/arm32/katelib.h>
-__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.261 2013/07/03 15:30:24 matt Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.262 2013/07/03 21:37:35 matt Exp $");
#ifdef PMAP_DEBUG
@@ -1496,8 +1496,6 @@ static int
pmap_l2ptp_ctor(void *arg, void *v, int flags)
{
#ifndef PMAP_INCLUDE_PTE_SYNC
- struct l2_bucket *l2b;
- pt_entry_t *ptep, pte;
vaddr_t va = (vaddr_t)v & ~PGOFSET;
/*
@@ -1508,16 +1506,18 @@ pmap_l2ptp_ctor(void *arg, void *v, int
* page tables, we simply fix up the cache-mode here if it's not
* correct.
*/
- l2b = pmap_get_l2_bucket(pmap_kernel(), va);
+ struct l2_bucket * const l2b = pmap_get_l2_bucket(pmap_kernel(), va);
KDASSERT(l2b != NULL);
- ptep = &l2b->l2b_kva[l2pte_index(va)];
- pte = *ptep;
+ pt_entry_t *ptep = &l2b->l2b_kva[l2pte_index(va)];
+ pt_entry_t opte = *ptep;
- if ((pte & L2_S_CACHE_MASK) != pte_l2_s_cache_mode_pt) {
+ if ((opte & L2_S_CACHE_MASK) != pte_l2_s_cache_mode_pt) {
/*
* Page tables must have the cache-mode set to Write-Thru.
*/
- *ptep = (pte & ~L2_S_CACHE_MASK) | pte_l2_s_cache_mode_pt;
+ const pt_entry_t npte = (pte & ~L2_S_CACHE_MASK)
+ | pte_l2_s_cache_mode_pt;
+ l2pte_set(ptep, npte, opte);
PTE_SYNC(ptep);
cpu_tlb_flushD_SE(va);
cpu_cpwait();
@@ -1556,7 +1556,8 @@ pmap_pinit(pmap_t pm)
* Map the vector page.
*/
pmap_enter(pm, vector_page, systempage.pv_pa,
- VM_PROT_READ, VM_PROT_READ | PMAP_WIRED);
+ VM_PROT_READ | VM_PROT_EXECUTE,
+ VM_PROT_READ | VM_PROT_EXECUTE | PMAP_WIRED);
pmap_update(pm);
pm->pm_pl1vec = pmap_l1_kva(pm) + L1_IDX(vector_page);
@@ -1730,8 +1731,6 @@ pmap_vac_me_user(struct vm_page_md *md,
{
pmap_t kpmap = pmap_kernel();
struct pv_entry *pv, *npv = NULL;
- struct l2_bucket *l2b;
- pt_entry_t *ptep, pte;
u_int entries = 0;
u_int writable = 0;
u_int cacheable_entries = 0;
@@ -1783,13 +1782,16 @@ pmap_vac_me_user(struct vm_page_md *md,
pv->pv_flags |= PVF_NC;
- l2b = pmap_get_l2_bucket(pv->pv_pmap, pv->pv_va);
+ struct l2_bucket * const l2b
+ = pmap_get_l2_bucket(pv->pv_pmap, pv->pv_va);
KDASSERT(l2b != NULL);
- ptep = &l2b->l2b_kva[l2pte_index(pv->pv_va)];
- pte = *ptep & ~L2_S_CACHE_MASK;
+ pt_entry_t * const ptep
+ = &l2b->l2b_kva[l2pte_index(pv->pv_va)];
+ const pt_entry_t opte = *ptep;
+ pt_entry_t npte = opte & ~L2_S_CACHE_MASK;
if ((va != pv->pv_va || pm != pv->pv_pmap)
- && l2pte_valid(pte)) {
+ && l2pte_valid(npte)) {
#ifdef PMAP_CACHE_VIVT
pmap_cache_wbinv_page(pv->pv_pmap, pv->pv_va,
true, pv->pv_flags);
@@ -1798,7 +1800,7 @@ pmap_vac_me_user(struct vm_page_md *md,
pv->pv_flags);
}
- *ptep = pte;
+ l2pte_set(ptep, npte, opte);
PTE_SYNC_CURRENT(pv->pv_pmap, ptep);
}
cpu_cpwait();
@@ -1815,17 +1817,21 @@ pmap_vac_me_user(struct vm_page_md *md,
pv->pv_flags &= ~PVF_NC;
- l2b = pmap_get_l2_bucket(pv->pv_pmap, pv->pv_va);
+ struct l2_bucket * const l2b
+ = pmap_get_l2_bucket(pv->pv_pmap, pv->pv_va);
KDASSERT(l2b != NULL);
- ptep = &l2b->l2b_kva[l2pte_index(pv->pv_va)];
- pte = (*ptep & ~L2_S_CACHE_MASK) | pte_l2_s_cache_mode;
+ pt_entry_t * const ptep
+ = &l2b->l2b_kva[l2pte_index(pv->pv_va)];
+ const pt_entry_t opte = *ptep;
+ pt_entry_t npte = (opte & ~L2_S_CACHE_MASK)
+ | pte_l2_s_cache_mode;
- if (l2pte_valid(pte)) {
+ if (l2pte_valid(opte)) {
pmap_tlb_flush_SE(pv->pv_pmap, pv->pv_va,
pv->pv_flags);
}
- *ptep = pte;
+ l2pte_set(ptep, npte, opte);
PTE_SYNC_CURRENT(pv->pv_pmap, ptep);
}
}
@@ -1839,8 +1845,6 @@ pmap_vac_me_harder(struct vm_page_md *md
struct pv_entry *pv;
vaddr_t tst_mask;
bool bad_alias;
- struct l2_bucket *l2b;
- pt_entry_t *ptep, pte, opte;
const u_int
rw_mappings = md->urw_mappings + md->krw_mappings,
ro_mappings = md->uro_mappings + md->kro_mappings;
@@ -2098,27 +2102,27 @@ pmap_vac_me_harder(struct vm_page_md *md
* Turn cacheing on/off for all pages.
*/
SLIST_FOREACH(pv, &md->pvh_list, pv_link) {
- l2b = pmap_get_l2_bucket(pv->pv_pmap, pv->pv_va);
+ struct l2_bucket * const l2b = pmap_get_l2_bucket(pv->pv_pmap,
+ pv->pv_va);
KDASSERT(l2b != NULL);
- ptep = &l2b->l2b_kva[l2pte_index(pv->pv_va)];
- opte = *ptep;
- pte = opte & ~L2_S_CACHE_MASK;
+ pt_entry_t * const ptep = &l2b->l2b_kva[l2pte_index(pv->pv_va)];
+ const pt_entry_t opte = *ptep;
+ pt_entry_t npte = opte & ~L2_S_CACHE_MASK;
if (bad_alias) {
pv->pv_flags |= PVF_NC;
} else {
pv->pv_flags &= ~PVF_NC;
- pte |= pte_l2_s_cache_mode;
+ npte |= pte_l2_s_cache_mode;
}
- if (opte == pte) /* only update is there's a change */
+ if (opte == npte) /* only update is there's a change */
continue;
- if (l2pte_valid(pte)) {
- pmap_tlb_flush_SE(pv->pv_pmap, pv->pv_va,
- pv->pv_flags);
+ if (l2pte_valid(npte)) {
+ pmap_tlb_flush_SE(pv->pv_pmap, pv->pv_va, pv->pv_flags);
}
- *ptep = pte;
+ l2pte_set(ptep, npte, opte);
PTE_SYNC_CURRENT(pv->pv_pmap, ptep);
}
}
@@ -2133,17 +2137,14 @@ pmap_vac_me_harder(struct vm_page_md *md
static void
pmap_clearbit(struct vm_page_md *md, paddr_t pa, u_int maskbits)
{
- struct l2_bucket *l2b;
struct pv_entry *pv;
- pt_entry_t *ptep, npte, opte;
pmap_t pm;
vaddr_t va;
u_int oflags;
#ifdef PMAP_CACHE_VIPT
const bool want_syncicache = PV_IS_EXEC_P(md->pvh_attrs);
- bool need_syncicache = false;
- bool did_syncicache = false;
bool need_vac_me_harder = false;
+ bool need_syncicache = false;
#endif
NPDEBUG(PDB_BITS,
@@ -2155,8 +2156,9 @@ pmap_clearbit(struct vm_page_md *md, pad
* If we might want to sync the I-cache and we've modified it,
* then we know we definitely need to sync or discard it.
*/
- if (want_syncicache)
+ if (want_syncicache) {
need_syncicache = md->pvh_attrs & PVF_MOD;
+ }
#endif
/*
* Clear saved attributes (modify, reference)
@@ -2164,7 +2166,7 @@ pmap_clearbit(struct vm_page_md *md, pad
md->pvh_attrs &= ~(maskbits & (PVF_MOD | PVF_REF));
if (SLIST_EMPTY(&md->pvh_list)) {
-#ifdef PMAP_CACHE_VIPT
+#if defined(PMAP_CACHE_VIPT)
if (need_syncicache) {
/*
* No one has it mapped, so just discard it. The next
@@ -2193,11 +2195,12 @@ pmap_clearbit(struct vm_page_md *md, pad
pmap_acquire_pmap_lock(pm);
- l2b = pmap_get_l2_bucket(pm, va);
+ struct l2_bucket * const l2b = pmap_get_l2_bucket(pm, va);
KDASSERT(l2b != NULL);
- ptep = &l2b->l2b_kva[l2pte_index(va)];
- npte = opte = *ptep;
+ pt_entry_t * const ptep = &l2b->l2b_kva[l2pte_index(va)];
+ const pt_entry_t opte = *ptep;
+ pt_entry_t npte = opte;
NPDEBUG(PDB_BITS,
printf(
@@ -2292,8 +2295,9 @@ pmap_clearbit(struct vm_page_md *md, pad
}
if (npte != opte) {
- *ptep = npte;
+ l2pte_set(ptep, npte, opte);
PTE_SYNC(ptep);
+
/* Flush the TLB entry if a current pmap. */
pmap_tlb_flush_SE(pm, pv->pv_va, oflags);
}
@@ -2309,10 +2313,11 @@ pmap_clearbit(struct vm_page_md *md, pad
/*
* If we need to sync the I-cache and we haven't done it yet, do it.
*/
- if (need_syncicache && !did_syncicache) {
+ if (need_syncicache) {
pmap_syncicache_page(md, pa);
PMAPCOUNT(exec_synced_clearbit);
}
+
/*
* If we are changing this to read-only, we need to call vac_me_harder
* so we can change all the read-only pages to cacheable. We pretend
@@ -2430,10 +2435,11 @@ pmap_syncicache_page(struct vm_page_md *
/*
* Set up a PTE with the right coloring to flush existing cache lines.
*/
- *ptep = L2_S_PROTO |
+ const pt_entry_t npte = L2_S_PROTO |
pa
| L2_S_PROT(PTE_KERNEL, VM_PROT_READ|VM_PROT_WRITE)
| pte_l2_s_cache_mode;
+ l2pte_set(ptep, npte, 0);
PTE_SYNC(ptep);
/*
@@ -2443,7 +2449,7 @@ pmap_syncicache_page(struct vm_page_md *
/*
* Unmap the page.
*/
- *ptep = 0;
+ l2pte_reset(ptep);
PTE_SYNC(ptep);
pmap_tlb_flush_SE(pmap_kernel(), cdstp + va_offset, PVF_REF | PVF_EXEC);
@@ -2510,7 +2516,7 @@ pmap_flush_page(struct vm_page_md *md, p
for (; va_offset <= end_va; va_offset += PAGE_SIZE) {
const size_t pte_offset = va_offset >> PGSHIFT;
pt_entry_t * const ptep = &cdst_pte[pte_offset];
- const pt_entry_t oldpte = *ptep;
+ const pt_entry_t opte = *ptep;
if (flush == PMAP_FLUSH_SECONDARY
&& va_offset == (md->pvh_attrs & arm_cache_prefer_mask))
@@ -2522,14 +2528,16 @@ pmap_flush_page(struct vm_page_md *md, p
* Set up a PTE with the right coloring to flush
* existing cache entries.
*/
- *ptep = L2_S_PROTO
+ const pt_entry_t npte = L2_S_PROTO
| pa
| L2_S_PROT(PTE_KERNEL, VM_PROT_READ|VM_PROT_WRITE)
| pte_l2_s_cache_mode;
+ l2pte_set(ptep, npte, opte);
PTE_SYNC(ptep);
/*
- * Flush it.
+ * Flush it. Make sure to flush secondary cache too since
+ * bus_dma will ignore uncached pages.
*/
vaddr_t va = cdstp + va_offset;
if (scache_line_size != 0) {
@@ -2553,7 +2561,7 @@ pmap_flush_page(struct vm_page_md *md, p
* pmap_zero_page or pmap_copy_page which was already using
* this pte.
*/
- *ptep = oldpte;
+ l2pte_set(ptep, opte, npte);
PTE_SYNC(ptep);
pmap_tlb_flush_SE(pmap_kernel(), cdstp + va_offset,
PVF_REF | PVF_EXEC);
@@ -2665,7 +2673,7 @@ pmap_page_remove(struct vm_page_md *md,
/*
* Invalidate the PTEs.
*/
- *ptep = 0;
+ l2pte_reset(ptep);
PTE_SYNC_CURRENT(pm, ptep);
pmap_free_l2_bucket(pm, l2b, 1);
@@ -2775,7 +2783,6 @@ pmap_enter(pmap_t pm, vaddr_t va, paddr_
struct l2_bucket *l2b;
struct vm_page *pg, *opg;
struct pv_entry *pv;
- pt_entry_t *ptep, npte, opte;
u_int nflags;
u_int oflags;
#ifdef ARM_HAS_VBAR
@@ -2820,9 +2827,9 @@ pmap_enter(pmap_t pm, vaddr_t va, paddr_
}
panic("pmap_enter: failed to allocate L2 bucket");
}
- ptep = &l2b->l2b_kva[l2pte_index(va)];
- opte = *ptep;
- npte = pa;
+ pt_entry_t *ptep = &l2b->l2b_kva[l2pte_index(va)];
+ const pt_entry_t opte = *ptep;
+ pt_entry_t npte = pa;
oflags = 0;
if (opte) {
@@ -3021,7 +3028,7 @@ pmap_enter(pmap_t pm, vaddr_t va, paddr_
if (npte != opte) {
bool is_cached = pmap_is_cached(pm);
- *ptep = npte;
+ l2pte_set(ptep, npte, opte);
PTE_SYNC(ptep);
if (is_cached) {
/*
@@ -3104,9 +3111,7 @@ pmap_enter(pmap_t pm, vaddr_t va, paddr_
void
pmap_remove(pmap_t pm, vaddr_t sva, vaddr_t eva)
{
- struct l2_bucket *l2b;
vaddr_t next_bucket;
- pt_entry_t *ptep;
u_int cleanlist_idx, total, cnt;
struct {
vaddr_t va;
@@ -3139,35 +3144,34 @@ pmap_remove(pmap_t pm, vaddr_t sva, vadd
if (next_bucket > eva)
next_bucket = eva;
- l2b = pmap_get_l2_bucket(pm, sva);
+ struct l2_bucket * const l2b = pmap_get_l2_bucket(pm, sva);
if (l2b == NULL) {
sva = next_bucket;
continue;
}
- ptep = &l2b->l2b_kva[l2pte_index(sva)];
-
- for (mappings = 0; sva < next_bucket; sva += PAGE_SIZE, ptep++){
- struct vm_page *pg;
- pt_entry_t pte;
- paddr_t pa;
+ pt_entry_t *ptep = &l2b->l2b_kva[l2pte_index(sva)];
- pte = *ptep;
+ for (mappings = 0;
+ sva < next_bucket;
+ sva += PAGE_SIZE, ptep += PAGE_SIZE / L2_S_SIZE) {
+ pt_entry_t opte = *ptep;
- if (pte == 0) {
+ if (opte == 0) {
/* Nothing here, move along */
continue;
}
- pa = l2pte_pa(pte);
u_int flags = PVF_REF;
+ paddr_t pa = l2pte_pa(opte);
+ struct vm_page * const pg = PHYS_TO_VM_PAGE(pa);
/*
* Update flags. In a number of circumstances,
* we could cluster a lot of these and do a
* number of sequential pages in one go.
*/
- if ((pg = PHYS_TO_VM_PAGE(pa)) != NULL) {
+ if (pg != NULL) {
struct vm_page_md *md = VM_PAGE_TO_MD(pg);
struct pv_entry *pv;
@@ -3185,13 +3189,13 @@ pmap_remove(pmap_t pm, vaddr_t sva, vadd
}
mappings++;
- if (!l2pte_valid(pte)) {
+ if (!l2pte_valid(opte)) {
/*
* Ref/Mod emulation is still active for this
* mapping, therefore it is has not yet been
* accessed. No need to frob the cache/tlb.
*/
- *ptep = 0;
+ l2pte_reset(ptep);
PTE_SYNC_CURRENT(pm, ptep);
continue;
}
@@ -3216,15 +3220,15 @@ pmap_remove(pmap_t pm, vaddr_t sva, vadd
*/
for (cnt = 0;
cnt < PMAP_REMOVE_CLEAN_LIST_SIZE; cnt++) {
- *cleanlist[cnt].ptep = 0;
+ l2pte_reset(cleanlist[cnt].ptep);
PTE_SYNC(cleanlist[cnt].ptep);
}
- *ptep = 0;
+ l2pte_reset(ptep);
PTE_SYNC(ptep);
cleanlist_idx++;
pm->pm_remove_all = true;
} else {
- *ptep = 0;
+ l2pte_reset(ptep);
PTE_SYNC(ptep);
if (pm->pm_remove_all == false) {
pmap_tlb_flush_SE(pm, sva, flags);
@@ -3249,7 +3253,7 @@ pmap_remove(pmap_t pm, vaddr_t sva, vadd
pmap_tlb_flush_SE(pm, clva,
PVF_REF | flags);
}
- *cleanlist[cnt].ptep = 0;
+ l2pte_reset(cleanlist[cnt].ptep);
PTE_SYNC_CURRENT(pm, cleanlist[cnt].ptep);
}
@@ -3323,8 +3327,6 @@ pmap_kremove_pg(struct vm_page *pg, vadd
void
pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags)
{
- struct l2_bucket *l2b;
- pt_entry_t *ptep, opte;
#ifdef PMAP_CACHE_VIVT
struct vm_page *pg = (flags & PMAP_KMPAGE) ? PHYS_TO_VM_PAGE(pa) : NULL;
#endif
@@ -3339,11 +3341,11 @@ pmap_kenter_pa(vaddr_t va, paddr_t pa, v
printf("pmap_kenter_pa: va 0x%08lx, pa 0x%08lx, prot 0x%x\n",
va, pa, prot));
- l2b = pmap_get_l2_bucket(pmap_kernel(), va);
+ struct l2_bucket * const l2b = pmap_get_l2_bucket(pmap_kernel(), va);
KDASSERT(l2b != NULL);
- ptep = &l2b->l2b_kva[l2pte_index(va)];
- opte = *ptep;
+ pt_entry_t * const ptep = &l2b->l2b_kva[l2pte_index(va)];
+ const pt_entry_t opte = *ptep;
if (opte == 0) {
PMAPCOUNT(kenter_mappings);
@@ -3371,8 +3373,9 @@ pmap_kenter_pa(vaddr_t va, paddr_t pa, v
}
}
- *ptep = L2_S_PROTO | pa | L2_S_PROT(PTE_KERNEL, prot)
+ const pt_entry_t npte = L2_S_PROTO | pa | L2_S_PROT(PTE_KERNEL, prot)
| ((flags & PMAP_NOCACHE) ? 0 : pte_l2_s_cache_mode);
+ l2pte_set(ptep, npte, opte);
PTE_SYNC(ptep);
if (pg) {
@@ -3437,11 +3440,8 @@ pmap_kenter_pa(vaddr_t va, paddr_t pa, v
void
pmap_kremove(vaddr_t va, vsize_t len)
{
- struct l2_bucket *l2b;
- pt_entry_t *ptep, *sptep, opte;
vaddr_t next_bucket, eva;
u_int mappings;
- struct vm_page *opg;
PMAPCOUNT(kenter_unmappings);
@@ -3455,16 +3455,17 @@ pmap_kremove(vaddr_t va, vsize_t len)
if (next_bucket > eva)
next_bucket = eva;
- l2b = pmap_get_l2_bucket(pmap_kernel(), va);
+ struct l2_bucket * const l2b = pmap_get_l2_bucket(pmap_kernel(), va);
KDASSERT(l2b != NULL);
- sptep = ptep = &l2b->l2b_kva[l2pte_index(va)];
+ pt_entry_t * const sptep = &l2b->l2b_kva[l2pte_index(va)];
+ pt_entry_t *ptep = sptep;
mappings = 0;
while (va < next_bucket) {
- opte = *ptep;
- opg = PHYS_TO_VM_PAGE(l2pte_pa(opte));
- if (opg) {
+ const pt_entry_t opte = *ptep;
+ struct vm_page *opg = PHYS_TO_VM_PAGE(l2pte_pa(opte));
+ if (opg != NULL) {
struct vm_page_md *omd = VM_PAGE_TO_MD(opg);
if (omd->pvh_attrs & PVF_KMPAGE) {
@@ -3493,11 +3494,11 @@ pmap_kremove(vaddr_t va, vsize_t len)
cpu_tlb_flushD_SE(va);
}
if (opte) {
- *ptep = 0;
+ l2pte_reset(ptep);
mappings++;
}
va += PAGE_SIZE;
- ptep++;
+ ptep += PAGE_SIZE / L2_S_SIZE;
}
KDASSERT(mappings <= l2b->l2b_occupancy);
l2b->l2b_occupancy -= mappings;
@@ -3577,9 +3578,6 @@ pmap_protect(pmap_t pm, vaddr_t sva, vad
struct l2_bucket *l2b;
pt_entry_t *ptep, pte;
vaddr_t next_bucket;
- u_int flags;
- u_int clr_mask;
- int flush;
NPDEBUG(PDB_PROTECT,
printf("pmap_protect: pm %p sva 0x%lx eva 0x%lx prot 0x%x\n",
@@ -3600,9 +3598,9 @@ pmap_protect(pmap_t pm, vaddr_t sva, vad
pmap_acquire_pmap_lock(pm);
- flush = ((eva - sva) >= (PAGE_SIZE * 4)) ? 0 : -1;
- flags = 0;
- clr_mask = PVF_WRITE | ((prot & VM_PROT_EXECUTE) ? 0 : PVF_EXEC);
+ const bool flush = eva - sva >= PAGE_SIZE * 4;
+ u_int clr_mask = PVF_WRITE | ((prot & VM_PROT_EXECUTE) ? 0 : PVF_EXEC);
+ u_int flags = 0;
while (sva < eva) {
next_bucket = L2_NEXT_BUCKET(sva);
@@ -3651,8 +3649,7 @@ pmap_protect(pmap_t pm, vaddr_t sva, vad
f = PVF_REF | PVF_EXEC;
}
- if (flush >= 0) {
- flush++;
+ if (flush) {
flags |= f;
} else {
pmap_tlb_flush_SE(pm, sva, f);
@@ -3664,15 +3661,15 @@ pmap_protect(pmap_t pm, vaddr_t sva, vad
}
}
- pmap_release_pmap_lock(pm);
-
if (flush) {
- if (PV_BEEN_EXECD(flags))
+ if (PV_BEEN_EXECD(flags)) {
pmap_tlb_flushID(pm);
- else
- if (PV_BEEN_REFD(flags))
+ } else if (PV_BEEN_REFD(flags)) {
pmap_tlb_flushD(pm);
+ }
}
+
+ pmap_release_pmap_lock(pm);
}
void
@@ -5699,9 +5696,9 @@ pmap_map_section(vaddr_t l1pt, vaddr_t v
break;
}
- pde[va >> L1_S_SHIFT] = L1_S_PROTO | pa |
+ pde[L1_IDX(va)] = L1_S_PROTO | pa |
L1_S_PROT(PTE_KERNEL, prot) | fl | L1_S_DOM(PMAP_DOMAIN_KERNEL);
- PTE_SYNC(&pde[va >> L1_S_SHIFT]);
+ PTE_SYNC(&pde[L1_IDX(va)]);
}
/*
@@ -5713,47 +5710,46 @@ void
pmap_map_entry(vaddr_t l1pt, vaddr_t va, paddr_t pa, int prot, int cache)
{
pd_entry_t *pde = (pd_entry_t *) l1pt;
- pt_entry_t fl;
- pt_entry_t *pte;
+ pt_entry_t npte;
+ pt_entry_t *ptep;
KASSERT(((va | pa) & PGOFSET) == 0);
switch (cache) {
case PTE_NOCACHE:
default:
- fl = 0;
+ npte = 0;
break;
case PTE_CACHE:
- fl = pte_l2_s_cache_mode;
+ npte = pte_l2_s_cache_mode;
break;
case PTE_PAGETABLE:
- fl = pte_l2_s_cache_mode_pt;
+ npte = pte_l2_s_cache_mode_pt;
break;
}
- if ((pde[va >> L1_S_SHIFT] & L1_TYPE_MASK) != L1_TYPE_C)
+ if ((pde[L1_IDX(va)] & L1_TYPE_MASK) != L1_TYPE_C)
panic("pmap_map_entry: no L2 table for VA 0x%08lx", va);
#ifndef ARM32_NEW_VM_LAYOUT
- pte = (pt_entry_t *)
- kernel_pt_lookup(pde[va >> L1_S_SHIFT] & L2_S_FRAME);
+ ptep = (pt_entry_t *)
+ kernel_pt_lookup(pde[L1_IDX(va)] & L2_S_FRAME);
#else
- pte = (pt_entry_t *) kernel_pt_lookup(pde[L1_IDX(va)] & L1_C_ADDR_MASK);
+ ptep = (pt_entry_t *) kernel_pt_lookup(pde[L1_IDX(va)] & L1_C_ADDR_MASK);
#endif
- if (pte == NULL)
+ if (ptep == NULL)
panic("pmap_map_entry: can't find L2 table for VA 0x%08lx", va);
- fl |= L2_S_PROTO | pa | L2_S_PROT(PTE_KERNEL, prot);
+ npte |= L2_S_PROTO | pa | L2_S_PROT(PTE_KERNEL, prot);
#ifndef ARM32_NEW_VM_LAYOUT
- pte += (va >> PGSHIFT) & 0x3ff;
+ ptep += (va >> PGSHIFT) & 0x3ff;
#else
- pte += l2pte_index(va);
- L2_S_PROTO | pa | L2_S_PROT(PTE_KERNEL, prot) | fl;
+ ptep += l2pte_index(va);
#endif
- *pte = fl;
- PTE_SYNC(pte);
+ l2pte_set(ptep, npte, 0);
+ PTE_SYNC(ptep);
}
/*
@@ -5766,7 +5762,7 @@ void
pmap_link_l2pt(vaddr_t l1pt, vaddr_t va, pv_addr_t *l2pv)
{
pd_entry_t *pde = (pd_entry_t *) l1pt, proto;
- u_int slot = va >> L1_S_SHIFT;
+ u_int slot = L1_IDX(va);
#ifndef ARM32_NEW_VM_LAYOUT
KASSERT((va & ((L1_S_SIZE * 4) - 1)) == 0);
@@ -5779,10 +5775,10 @@ pmap_link_l2pt(vaddr_t l1pt, vaddr_t va,
#ifdef ARM32_NEW_VM_LAYOUT
PTE_SYNC(&pde[slot]);
#else
- pde[slot + 1] = proto | (l2pv->pv_pa + 0x400);
- pde[slot + 2] = proto | (l2pv->pv_pa + 0x800);
- pde[slot + 3] = proto | (l2pv->pv_pa + 0xc00);
- PTE_SYNC_RANGE(&pde[slot + 0], 4);
+ for (u_int off = 0, i = 0; off < PAGE_SIZE; off += L2_T_SIZE, i++) {
+ pde[slot + i] = proto | (l2pv->pv_pa + off);
+ }
+ PTE_SYNC_RANGE(&pde[slot + 0], PAGE_SIZE / L2_T_SIZE);
#endif
SLIST_INSERT_HEAD(&kernel_pt_list, l2pv, pv_list);
@@ -5838,6 +5834,7 @@ pmap_map_chunk(vaddr_t l1pt, vaddr_t va,
size = resid;
while (resid > 0) {
+ size_t l1idx = L1_IDX(va);
#if (ARM_MMU_V6 + ARM_MMU_V7) > 0
/* See if we can use a supersection mapping. */
if (L1_SS_PROTO && L1_SS_MAPPABLE_P(va, pa, resid)) {
@@ -5847,7 +5844,7 @@ pmap_map_chunk(vaddr_t l1pt, vaddr_t va,
#ifdef VERBOSE_INIT_ARM
printf("sS");
#endif
- for (size_t s = va >> L1_S_SHIFT,
+ for (size_t s = l1idx,
e = s + L1_SS_SIZE / L1_S_SIZE;
s < e;
s++) {
@@ -5865,10 +5862,10 @@ pmap_map_chunk(vaddr_t l1pt, vaddr_t va,
#ifdef VERBOSE_INIT_ARM
printf("S");
#endif
- pdep[va >> L1_S_SHIFT] = L1_S_PROTO | pa |
+ pdep[l1idx] = L1_S_PROTO | pa |
L1_S_PROT(PTE_KERNEL, prot) | f1 |
L1_S_DOM(PMAP_DOMAIN_KERNEL);
- PTE_SYNC(&pdep[va >> L1_S_SHIFT]);
+ PTE_SYNC(&pdep[l1idx]);
va += L1_S_SIZE;
pa += L1_S_SIZE;
resid -= L1_S_SIZE;
@@ -5880,15 +5877,15 @@ pmap_map_chunk(vaddr_t l1pt, vaddr_t va,
* one is actually in the corresponding L1 slot
* for the current VA.
*/
- if ((pdep[va >> L1_S_SHIFT] & L1_TYPE_MASK) != L1_TYPE_C)
+ if ((pdep[l1idx] & L1_TYPE_MASK) != L1_TYPE_C)
panic("pmap_map_chunk: no L2 table for VA 0x%08lx", va);
#ifndef ARM32_NEW_VM_LAYOUT
pte = (pt_entry_t *)
- kernel_pt_lookup(pdep[va >> L1_S_SHIFT] & L2_S_FRAME);
+ kernel_pt_lookup(pdep[l1idx] & L2_S_FRAME);
#else
pte = (pt_entry_t *) kernel_pt_lookup(
- pdep[L1_IDX(va)] & L1_C_ADDR_MASK);
+ pdep[l1idx] & L1_C_ADDR_MASK);
#endif
if (pte == NULL)
panic("pmap_map_chunk: can't find L2 table for VA"
@@ -5922,15 +5919,15 @@ pmap_map_chunk(vaddr_t l1pt, vaddr_t va,
#ifdef VERBOSE_INIT_ARM
printf("P");
#endif
-#ifndef ARM32_NEW_VM_LAYOUT
- pte[(va >> PGSHIFT) & 0x3ff] =
+ pt_entry_t npte =
L2_S_PROTO | pa | L2_S_PROT(PTE_KERNEL, prot) | f2s;
- PTE_SYNC(&pte[(va >> PGSHIFT) & 0x3ff]);
+#ifndef ARM32_NEW_VM_LAYOUT
+ pt_entry_t *ptep = &pte[(va >> PGSHIFT) & 0x3ff];
#else
- pte[l2pte_index(va)] =
- L2_S_PROTO | pa | L2_S_PROT(PTE_KERNEL, prot) | f2s;
- PTE_SYNC(&pte[l2pte_index(va)]);
+ pt_entry_t *ptep = &pte[l2pte_index(va)];
#endif
+ l2pte_set(ptep, npte, 0);
+ PTE_SYNC(ptep);
va += PAGE_SIZE;
pa += PAGE_SIZE;
resid -= PAGE_SIZE;
@@ -6381,7 +6378,6 @@ xscale_setup_minidata(vaddr_t l1pt, vadd
extern vaddr_t xscale_minidata_clean_addr;
extern vsize_t xscale_minidata_clean_size; /* already initialized */
pd_entry_t *pde = (pd_entry_t *) l1pt;
- pt_entry_t *pte;
vsize_t size;
uint32_t auxctl;
@@ -6392,23 +6388,27 @@ xscale_setup_minidata(vaddr_t l1pt, vadd
for (; size != 0;
va += L2_S_SIZE, pa += L2_S_SIZE, size -= L2_S_SIZE) {
+ const size_t l1idx = L1_IDX(va);
#ifndef ARM32_NEW_VM_LAYOUT
- pte = (pt_entry_t *)
- kernel_pt_lookup(pde[va >> L1_S_SHIFT] & L2_S_FRAME);
+ pt_entry_t *ptep = (pt_entry_t *)
+ kernel_pt_lookup(pde[l1idx] & L2_S_FRAME);
#else
- pte = (pt_entry_t *) kernel_pt_lookup(
- pde[L1_IDX(va)] & L1_C_ADDR_MASK);
+ pt_entry_t *ptep = (pt_entry_t *) kernel_pt_lookup(
+ pde[l1idx] & L1_C_ADDR_MASK);
#endif
- if (pte == NULL)
+ if (ptep == NULL)
panic("xscale_setup_minidata: can't find L2 table for "
"VA 0x%08lx", va);
+
#ifndef ARM32_NEW_VM_LAYOUT
- pte[(va >> PGSHIFT) & 0x3ff] =
+ ptep += (va >> PGSHIFT) & 0x3ff;
#else
- pte[l2pte_index(va)] =
+ ptep += l2pte_index(va);
#endif
- L2_S_PROTO | pa | L2_S_PROT(PTE_KERNEL, VM_PROT_READ) |
- L2_C | L2_XS_T_TEX(TEX_XSCALE_X);
+ pt_entry_t opte = *ptep;
+ l2pte_set(ptep,
+ L2_S_PROTO | pa | L2_S_PROT(PTE_KERNEL, VM_PROT_READ)
+ | L2_C | L2_XS_T_TEX(TEX_XSCALE_X), opte);
}
/*
@@ -6437,8 +6437,6 @@ xscale_setup_minidata(vaddr_t l1pt, vadd
void
pmap_uarea(vaddr_t va)
{
- struct l2_bucket *l2b;
- pt_entry_t *ptep, *sptep, pte;
vaddr_t next_bucket, eva;
#if (ARM_NMMUS > 1)
@@ -6453,19 +6451,20 @@ pmap_uarea(vaddr_t va)
if (next_bucket > eva)
next_bucket = eva;
- l2b = pmap_get_l2_bucket(pmap_kernel(), va);
+ struct l2_bucket *l2b = pmap_get_l2_bucket(pmap_kernel(), va);
KDASSERT(l2b != NULL);
- sptep = ptep = &l2b->l2b_kva[l2pte_index(va)];
+ pt_entry_t * const sptep = &l2b->l2b_kva[l2pte_index(va)];
+ pt_entry_t *ptep = sptep;
while (va < next_bucket) {
- pte = *ptep;
- if (!l2pte_minidata(pte)) {
+ const pt_entry_t opte = *ptep;
+ if (!l2pte_minidata(opte)) {
cpu_dcache_wbinv_range(va, PAGE_SIZE);
cpu_tlb_flushD_SE(va);
- *ptep = pte & ~L2_B;
+ l2pte_set(ptep, opte & ~L2_B, opte);
}
- ptep++;
+ ptep += PAGE_SIZE / L2_S_SIZE;
va += PAGE_SIZE;
}
PTE_SYNC_RANGE(sptep, (u_int)(ptep - sptep));
Index: src/sys/arch/arm/include/arm32/pmap.h
diff -u src/sys/arch/arm/include/arm32/pmap.h:1.120 src/sys/arch/arm/include/arm32/pmap.h:1.121
--- src/sys/arch/arm/include/arm32/pmap.h:1.120 Wed Jun 12 21:34:12 2013
+++ src/sys/arch/arm/include/arm32/pmap.h Wed Jul 3 21:37:35 2013
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap.h,v 1.120 2013/06/12 21:34:12 matt Exp $ */
+/* $NetBSD: pmap.h,v 1.121 2013/07/03 21:37:35 matt Exp $ */
/*
* Copyright (c) 2002, 2003 Wasabi Systems, Inc.
@@ -451,6 +451,27 @@ pmap_ptesync(pt_entry_t *ptep, size_t cn
(L2_B | L2_C | L2_XS_T_TEX(TEX_XSCALE_X)))\
== (L2_C | L2_XS_T_TEX(TEX_XSCALE_X)))
+static inline void
+l2pte_set(pt_entry_t *ptep, pt_entry_t pte, pt_entry_t opte)
+{
+ KASSERT(*ptep == opte);
+ *ptep = pte;
+ for (vsize_t k = 1; k < PAGE_SIZE / L2_S_SIZE; k++) {
+ KASSERT(ptep[k] == opte ? opte + k * L2_S_SIZE : 0);
+ pte += L2_S_SIZE;
+ ptep[k] = pte;
+ }
+}
+
+static inline void
+l2pte_reset(pt_entry_t *ptep)
+{
+ *ptep = 0;
+ for (vsize_t k = 1; k < PAGE_SIZE / L2_S_SIZE; k++) {
+ ptep[k] = 0;
+ }
+}
+
/* L1 and L2 page table macros */
#define pmap_pde_v(pde) l1pte_valid(*(pde))
#define pmap_pde_section(pde) l1pte_section_p(*(pde))
