Module Name: src
Committed By: matt
Date: Thu Feb 7 06:52:54 UTC 2013
Modified Files:
src/sys/arch/arm/arm32 [matt-nb6-plus]: pmap.c
src/sys/arch/arm/include/arm32 [matt-nb6-plus]: pmap.h
Log Message:
pullup pmap changes from HEAD
To generate a diff of this commit:
cvs rdiff -u -r1.228.2.1.2.1 -r1.228.2.1.2.2 src/sys/arch/arm/arm32/pmap.c
cvs rdiff -u -r1.101.4.1 -r1.101.4.2 src/sys/arch/arm/include/arm32/pmap.h
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/arm/arm32/pmap.c
diff -u src/sys/arch/arm/arm32/pmap.c:1.228.2.1.2.1 src/sys/arch/arm/arm32/pmap.c:1.228.2.1.2.2
--- src/sys/arch/arm/arm32/pmap.c:1.228.2.1.2.1 Wed Nov 28 22:40:19 2012
+++ src/sys/arch/arm/arm32/pmap.c Thu Feb 7 06:52:53 2013
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap.c,v 1.228.2.1.2.1 2012/11/28 22:40:19 matt Exp $ */
+/* $NetBSD: pmap.c,v 1.228.2.1.2.2 2013/02/07 06:52:53 matt Exp $ */
/*
* Copyright 2003 Wasabi Systems, Inc.
@@ -212,7 +212,7 @@
#include <arm/cpuconf.h>
#include <arm/arm32/katelib.h>
-__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.228.2.1.2.1 2012/11/28 22:40:19 matt Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.228.2.1.2.2 2013/02/07 06:52:53 matt Exp $");
#ifdef PMAP_DEBUG
@@ -668,12 +668,6 @@ static vaddr_t kernel_pt_lookup(paddr_t
/*
- * External function prototypes
- */
-extern void bzero_page(vaddr_t);
-extern void bcopy_page(vaddr_t, vaddr_t);
-
-/*
* Misc variables
*/
vaddr_t virtual_avail;
@@ -699,6 +693,12 @@ pmap_debug(int level)
}
#endif /* PMAP_DEBUG */
+#ifdef PMAP_CACHE_VIPT
+#define PMAP_VALIDATE_MD_PAGE(md) \
+ KASSERTMSG(arm_cache_prefer_mask == 0 || (((md)->pvh_attrs & PVF_WRITE) == 0) == ((md)->urw_mappings + (md)->krw_mappings == 0), \
+ "(md) %p: attrs=%#x urw=%u krw=%u", (md), \
+ (md)->pvh_attrs, (md)->urw_mappings, (md)->krw_mappings);
+#endif /* PMAP_CACHE_VIPT */
/*
* A bunch of routines to conditionally flush the caches/TLB depending
* on whether the specified pmap actually needs to be flushed at any
@@ -829,10 +829,10 @@ do { \
/*
* main pv_entry manipulation functions:
* pmap_enter_pv: enter a mapping onto a vm_page list
- * pmap_remove_pv: remove a mappiing from a vm_page list
+ * pmap_remove_pv: remove a mapping from a vm_page list
*
* NOTE: pmap_enter_pv expects to lock the pvh itself
- * pmap_remove_pv expects te caller to lock the pvh before calling
+ * pmap_remove_pv expects the caller to lock the pvh before calling
*/
/*
@@ -896,6 +896,13 @@ pmap_enter_pv(struct vm_page_md *md, pad
#ifdef PMAP_CACHE_VIPT
/*
+ * Even though pmap_vac_me_harder will set PVF_WRITE for us,
+ * do it here as well to keep the mappings & KVF_WRITE consistent.
+ */
+ if (arm_cache_prefer_mask != 0 && (flags & PVF_WRITE) != 0) {
+ md->pvh_attrs |= PVF_WRITE;
+ }
+ /*
* If this is an exec mapping and its the first exec mapping
* for this page, make sure to sync the I-cache.
*/
@@ -1014,8 +1021,11 @@ pmap_remove_pv(struct vm_page_md *md, pa
* mappings (ignoring KMPAGE), clear the WRITE flag and writeback
* the contents to memory.
*/
- if (md->krw_mappings + md->urw_mappings == 0)
- md->pvh_attrs &= ~PVF_WRITE;
+ if (arm_cache_prefer_mask != 0) {
+ if (md->krw_mappings + md->urw_mappings == 0)
+ md->pvh_attrs &= ~PVF_WRITE;
+ PMAP_VALIDATE_MD_PAGE(md);
+ }
KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC)));
#endif /* PMAP_CACHE_VIPT */
@@ -1093,8 +1103,13 @@ pmap_modify_pv(struct vm_page_md *md, pa
}
}
#ifdef PMAP_CACHE_VIPT
- if (md->urw_mappings + md->krw_mappings == 0)
- md->pvh_attrs &= ~PVF_WRITE;
+ if (arm_cache_prefer_mask != 0) {
+ if (md->urw_mappings + md->krw_mappings == 0) {
+ md->pvh_attrs &= ~PVF_WRITE;
+ } else {
+ md->pvh_attrs |= PVF_WRITE;
+ }
+ }
/*
* We have two cases here: the first is from enter_pv (new exec
* page), the second is a combined pmap_remove_pv/pmap_enter_pv.
@@ -1850,7 +1865,7 @@ pmap_vac_me_harder(struct vm_page_md *md
* Only check for a bad alias if we have writable mappings.
*/
tst_mask &= arm_cache_prefer_mask;
- if (rw_mappings > 0 && arm_cache_prefer_mask) {
+ if (rw_mappings > 0) {
for (; pv && !bad_alias; pv = SLIST_NEXT(pv, pv_link)) {
/* if there's a bad alias, stop checking. */
if (tst_mask != (pv->pv_va & arm_cache_prefer_mask))
@@ -1906,7 +1921,7 @@ pmap_vac_me_harder(struct vm_page_md *md
KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC)));
KASSERT((rw_mappings == 0) == !(md->pvh_attrs & PVF_WRITE));
} else if (!va) {
- KASSERT(arm_cache_prefer_mask == 0 || pmap_is_page_colored_p(md));
+ KASSERT(pmap_is_page_colored_p(md));
KASSERT(!(md->pvh_attrs & PVF_WRITE)
|| (md->pvh_attrs & PVF_DIRTY));
if (rw_mappings == 0) {
@@ -2230,8 +2245,13 @@ pmap_clearbit(struct vm_page_md *md, pad
md->uro_mappings++;
}
#ifdef PMAP_CACHE_VIPT
- if (md->urw_mappings + md->krw_mappings == 0)
- md->pvh_attrs &= ~PVF_WRITE;
+ if (arm_cache_prefer_mask != 0) {
+ if (md->urw_mappings + md->krw_mappings == 0) {
+ md->pvh_attrs &= ~PVF_WRITE;
+ } else {
+ PMAP_VALIDATE_MD_PAGE(md);
+ }
+ }
if (want_syncicache)
need_syncicache = true;
need_vac_me_harder = true;
@@ -2564,7 +2584,7 @@ pmap_page_remove(struct vm_page_md *md,
if (PV_IS_EXEC_P(md->pvh_attrs))
PMAPCOUNT(exec_discarded_page_protect);
md->pvh_attrs &= ~PVF_EXEC;
- KASSERT((md->urw_mappings + md->krw_mappings == 0) == !(md->pvh_attrs & PVF_WRITE));
+ PMAP_VALIDATE_MD_PAGE(md);
#endif
return;
}
@@ -2663,9 +2683,11 @@ pmap_page_remove(struct vm_page_md *md,
md->pvh_attrs &= ~PVF_EXEC;
KASSERT(md->urw_mappings == 0);
KASSERT(md->uro_mappings == 0);
- if (md->krw_mappings == 0)
- md->pvh_attrs &= ~PVF_WRITE;
- KASSERT((md->urw_mappings + md->krw_mappings == 0) == !(md->pvh_attrs & PVF_WRITE));
+ if (arm_cache_prefer_mask != 0) {
+ if (md->krw_mappings == 0)
+ md->pvh_attrs &= ~PVF_WRITE;
+ PMAP_VALIDATE_MD_PAGE(md);
+ }
#endif
if (flush) {
@@ -2809,8 +2831,7 @@ pmap_enter(pmap_t pm, vaddr_t va, paddr_
/*
* This is to be a managed mapping.
*/
- if ((flags & VM_PROT_ALL) ||
- (md->pvh_attrs & PVF_REF)) {
+ if ((flags & VM_PROT_ALL) || (md->pvh_attrs & PVF_REF)) {
/*
* - The access type indicates that we don't need
* to do referenced emulation.
@@ -2841,7 +2862,10 @@ pmap_enter(pmap_t pm, vaddr_t va, paddr_
npte |= L2_TYPE_INV;
}
- npte |= pte_l2_s_cache_mode;
+ if (flags & ARM32_MMAP_WRITECOMBINE) {
+ npte |= pte_l2_s_wc_mode;
+ } else
+ npte |= pte_l2_s_cache_mode;
if (pg == opg) {
/*
@@ -3050,7 +3074,7 @@ pmap_enter(pmap_t pm, vaddr_t va, paddr_
KASSERT(uvm_page_locked_p(pg));
#endif
KASSERT((md->pvh_attrs & PVF_DMOD) == 0 || (md->pvh_attrs & (PVF_DIRTY|PVF_NC)));
- KASSERT(arm_cache_prefer_mask == 0 || ((md->pvh_attrs & PVF_WRITE) == 0) == (md->urw_mappings + md->krw_mappings == 0));
+ PMAP_VALIDATE_MD_PAGE(md);
}
#endif
@@ -3468,7 +3492,9 @@ pmap_kremove(vaddr_t va, vsize_t len)
KASSERT(omd->kro_mappings == 0);
omd->pvh_attrs &= ~PVF_KMPAGE;
#ifdef PMAP_CACHE_VIPT
- omd->pvh_attrs &= ~PVF_WRITE;
+ if (arm_cache_prefer_mask != 0) {
+ omd->pvh_attrs &= ~PVF_WRITE;
+ }
#endif
pmap_kmpages--;
#ifdef PMAP_CACHE_VIPT
@@ -4047,6 +4073,7 @@ pmap_fault_fixup(pmap_t pm, vaddr_t va,
*/
if (rv == 0 && pm->pm_l1->l1_domain_use_count == 1) {
extern int last_fault_code;
+ extern int kernel_debug;
printf("fixup: pm %p, va 0x%lx, ftype %d - nothing to do!\n",
pm, va, ftype);
printf("fixup: l2 %p, l2b %p, ptep %p, pl1pd %p\n",
@@ -4054,7 +4081,8 @@ pmap_fault_fixup(pmap_t pm, vaddr_t va,
printf("fixup: pte 0x%x, l1pd 0x%x, last code 0x%x\n",
pte, l1pd, last_fault_code);
#ifdef DDB
- Debugger();
+ if (kernel_debug & 2)
+ Debugger();
#endif
}
#endif
@@ -4440,14 +4468,26 @@ pmap_zero_page_generic(paddr_t phys)
struct vm_page *pg = PHYS_TO_VM_PAGE(phys);
struct vm_page_md *md = VM_PAGE_TO_MD(pg);
#endif
-#ifdef PMAP_CACHE_VIPT
+#if defined(PMAP_CACHE_VIPT)
/* Choose the last page color it had, if any */
const vsize_t va_offset = md->pvh_attrs & arm_cache_prefer_mask;
#else
const vsize_t va_offset = 0;
#endif
+#if defined(__HAVE_MM_MD_DIRECT_MAPPED_PHYS)
+ /*
+ * Is this page mapped at its natural color?
+ * If we have all of memory mapped, then just convert PA to VA.
+ */
+ const bool okcolor = va_offset == (phys & arm_cache_prefer_mask);
+ const vaddr_t vdstp = KERNEL_BASE + (phys - physical_start);
+#else
+ const bool okcolor = false;
+ const vaddr_t vdstp = cdstp + va_offset;
+#endif
pt_entry_t * const ptep = &cdst_pte[va_offset >> PGSHIFT];
+
#ifdef DEBUG
if (!SLIST_EMPTY(&md->pvh_list))
panic("pmap_zero_page: page has mappings");
@@ -4455,25 +4495,39 @@ pmap_zero_page_generic(paddr_t phys)
KDASSERT((phys & PGOFSET) == 0);
- /*
- * Hook in the page, zero it, and purge the cache for that
- * zeroed page. Invalidate the TLB as needed.
- */
- *ptep = L2_S_PROTO | phys |
- L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | pte_l2_s_cache_mode;
- PTE_SYNC(ptep);
- cpu_tlb_flushD_SE(cdstp + va_offset);
- cpu_cpwait();
- bzero_page(cdstp + va_offset);
- /*
- * Unmap the page.
- */
- *ptep = 0;
- PTE_SYNC(ptep);
- cpu_tlb_flushD_SE(cdstp + va_offset);
+ if (!okcolor) {
+ /*
+ * Hook in the page, zero it, and purge the cache for that
+ * zeroed page. Invalidate the TLB as needed.
+ */
+ *ptep = L2_S_PROTO | phys |
+ L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | pte_l2_s_cache_mode;
+ PTE_SYNC(ptep);
+ cpu_tlb_flushD_SE(cdstp + va_offset);
+ cpu_cpwait();
+#if defined(__HAVE_MM_MD_DIRECT_MAPPED_PHYS) && defined(PMAP_CACHE_VIPT)
+ /*
+ * If we are direct-mapped and our color isn't ok, then before
+ * we bzero the page invalidate its contents from the cache and
+ * reset the color to its natural color.
+ */
+ cpu_dcache_inv_range(cdstp + va_offset, PAGE_SIZE);
+ md->pvh_attrs &= ~arm_cache_prefer_mask;
+ md->pvh_attrs |= (phys & arm_cache_prefer_mask);
+#endif
+ }
+ bzero_page(vdstp);
+ if (!okcolor) {
+ /*
+ * Unmap the page.
+ */
+ *ptep = 0;
+ PTE_SYNC(ptep);
+ cpu_tlb_flushD_SE(cdstp + va_offset);
#ifdef PMAP_CACHE_VIVT
- cpu_dcache_wbinv_range(cdstp + va_offset, PAGE_SIZE);
+ cpu_dcache_wbinv_range(cdstp + va_offset, PAGE_SIZE);
#endif
+ }
#ifdef PMAP_CACHE_VIPT
/*
* This page is now cache resident so it now has a page color.
@@ -4633,6 +4687,23 @@ pmap_copy_page_generic(paddr_t src, padd
const vsize_t src_va_offset = 0;
const vsize_t dst_va_offset = 0;
#endif
+#if defined(__HAVE_MM_MD_DIRECT_MAPPED_PHYS)
+ /*
+ * Is this page mapped at its natural color?
+ * If we have all of memory mapped, then just convert PA to VA.
+ */
+ const bool src_okcolor = src_va_offset == (src & arm_cache_prefer_mask);
+ const bool dst_okcolor = dst_va_offset == (dst & arm_cache_prefer_mask);
+ const vaddr_t vsrcp = src_okcolor
+ ? KERNEL_BASE + (src - physical_start)
+ : csrcp + src_va_offset;
+ const vaddr_t vdstp = KERNEL_BASE + (dst - physical_start);
+#else
+ const bool src_okcolor = false;
+ const bool dst_okcolor = false;
+ const vaddr_t vsrcp = csrcp + src_va_offset;
+ const vaddr_t vdstp = cdstp + dst_va_offset;
+#endif
pt_entry_t * const src_ptep = &csrc_pte[src_va_offset >> PGSHIFT];
pt_entry_t * const dst_ptep = &cdst_pte[dst_va_offset >> PGSHIFT];
@@ -4664,38 +4735,57 @@ pmap_copy_page_generic(paddr_t src, padd
* the cache for the appropriate page. Invalidate the TLB
* as required.
*/
- *src_ptep = L2_S_PROTO
- | src
+ if (!src_okcolor) {
+ *src_ptep = L2_S_PROTO
+ | src
#ifdef PMAP_CACHE_VIPT
- | ((src_md->pvh_attrs & PVF_NC) ? 0 : pte_l2_s_cache_mode)
+ | ((src_md->pvh_attrs & PVF_NC) ? 0 : pte_l2_s_cache_mode)
#endif
#ifdef PMAP_CACHE_VIVT
- | pte_l2_s_cache_mode
+ | pte_l2_s_cache_mode
#endif
- | L2_S_PROT(PTE_KERNEL, VM_PROT_READ);
- *dst_ptep = L2_S_PROTO | dst |
- L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | pte_l2_s_cache_mode;
- PTE_SYNC(src_ptep);
- PTE_SYNC(dst_ptep);
- cpu_tlb_flushD_SE(csrcp + src_va_offset);
- cpu_tlb_flushD_SE(cdstp + dst_va_offset);
- cpu_cpwait();
- bcopy_page(csrcp + src_va_offset, cdstp + dst_va_offset);
-#ifdef PMAP_CACHE_VIVT
- cpu_dcache_inv_range(csrcp + src_va_offset, PAGE_SIZE);
+ | L2_S_PROT(PTE_KERNEL, VM_PROT_READ);
+ PTE_SYNC(src_ptep);
+ cpu_tlb_flushD_SE(csrcp + src_va_offset);
+ cpu_cpwait();
+ }
+ if (!dst_okcolor) {
+ *dst_ptep = L2_S_PROTO | dst |
+ L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | pte_l2_s_cache_mode;
+ PTE_SYNC(dst_ptep);
+ cpu_tlb_flushD_SE(cdstp + dst_va_offset);
+ cpu_cpwait();
+#if defined(__HAVE_MM_MD_DIRECT_MAPPED_PHYS) && defined(PMAP_CACHE_VIPT)
+ /*
+ * If we are direct-mapped and our color isn't ok, then before
+ * we bcopy to the new page invalidate its contents from the
+ * cache and reset its color to its natural color.
+ */
+ cpu_dcache_inv_range(cdstp + dst_va_offset, PAGE_SIZE);
+ dst_md->pvh_attrs &= ~arm_cache_prefer_mask;
+ dst_md->pvh_attrs |= (dst & arm_cache_prefer_mask);
#endif
+ }
+ bcopy_page(vsrcp, vdstp);
#ifdef PMAP_CACHE_VIVT
- cpu_dcache_wbinv_range(cdstp + dst_va_offset, PAGE_SIZE);
+ cpu_dcache_inv_range(vsrcp, PAGE_SIZE);
+ cpu_dcache_wbinv_range(vdstp, PAGE_SIZE);
#endif
/*
* Unmap the pages.
*/
- *src_ptep = 0;
- *dst_ptep = 0;
- PTE_SYNC(src_ptep);
- PTE_SYNC(dst_ptep);
- cpu_tlb_flushD_SE(csrcp + src_va_offset);
- cpu_tlb_flushD_SE(cdstp + dst_va_offset);
+ if (!src_okcolor) {
+ *src_ptep = 0;
+ PTE_SYNC(src_ptep);
+ cpu_tlb_flushD_SE(csrcp + src_va_offset);
+ cpu_cpwait();
+ }
+ if (!dst_okcolor) {
+ *dst_ptep = 0;
+ PTE_SYNC(dst_ptep);
+ cpu_tlb_flushD_SE(cdstp + dst_va_offset);
+ cpu_cpwait();
+ }
#ifdef PMAP_CACHE_VIPT
/*
* Now that the destination page is in the cache, mark it as colored.
@@ -6863,11 +6953,13 @@ arm_pmap_alloc_poolpage(int flags)
{
/*
* On some systems, only some pages may be "coherent" for dma and we
- * want to use those for pool pages (think mbufs).
+ * want to prefer those for pool pages (think mbufs) but fallback to
+ * any page if none is available.
*/
- if (arm_poolpage_vmfreelist != VM_FREELIST_DEFAULT)
+ if (arm_poolpage_vmfreelist != VM_FREELIST_DEFAULT) {
return uvm_pagealloc_strat(NULL, 0, NULL, flags,
- UVM_PGA_STRAT_ONLY, arm_poolpage_vmfreelist);
+ UVM_PGA_STRAT_FALLBACK, arm_poolpage_vmfreelist);
+ }
return uvm_pagealloc(NULL, 0, NULL, flags);
}
Index: src/sys/arch/arm/include/arm32/pmap.h
diff -u src/sys/arch/arm/include/arm32/pmap.h:1.101.4.1 src/sys/arch/arm/include/arm32/pmap.h:1.101.4.2
--- src/sys/arch/arm/include/arm32/pmap.h:1.101.4.1 Wed Nov 28 22:40:32 2012
+++ src/sys/arch/arm/include/arm32/pmap.h Thu Feb 7 06:52:54 2013
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap.h,v 1.101.4.1 2012/11/28 22:40:32 matt Exp $ */
+/* $NetBSD: pmap.h,v 1.101.4.2 2013/02/07 06:52:54 matt Exp $ */
/*
* Copyright (c) 2002, 2003 Wasabi Systems, Inc.
@@ -463,6 +463,9 @@ pmap_ptesync(pt_entry_t *ptep, size_t cn
#define KERNEL_PD_SIZE \
(L1_TABLE_SIZE - (KERNEL_BASE >> L1_S_SHIFT) * sizeof(pd_entry_t))
+void bzero_page(vaddr_t);
+void bcopy_page(vaddr_t, vaddr_t);
+
/************************* ARM MMU configuration *****************************/
#if (ARM_MMU_GENERIC + ARM_MMU_SA1 + ARM_MMU_V6 + ARM_MMU_V7) != 0
@@ -882,22 +885,23 @@ extern void (*pmap_zero_page_func)(paddr
#define L2_L_MAPPABLE_P(va, pa, size) \
((((va) | (pa)) & L2_L_OFFSET) == 0 && (size) >= L2_L_SIZE)
+#ifndef _LOCORE
/*
* Hooks for the pool allocator.
*/
#define POOL_VTOPHYS(va) vtophys((vaddr_t) (va))
+extern paddr_t physical_start, physical_end;
#ifdef PMAP_NEED_ALLOC_POOLPAGE
-extern paddr_t physical_start;
struct vm_page *arm_pmap_alloc_poolpage(int);
#define PMAP_ALLOC_POOLPAGE arm_pmap_alloc_poolpage
+#endif
+#if defined(PMAP_NEED_ALLOC_POOLPAGE) || defined(__HAVE_MM_MD_DIRECT_MAPPED_PHYS)
#define PMAP_MAP_POOLPAGE(pa) \
((vaddr_t)((paddr_t)(pa) - physical_start + KERNEL_BASE))
#define PMAP_UNMAP_POOLPAGE(va) \
((paddr_t)((vaddr_t)(va) - KERNEL_BASE + physical_start))
#endif
-#ifndef _LOCORE
-
/*
* pmap-specific data store in the vm_page structure.
*/
