Module Name: src Committed By: matt Date: Tue Dec 11 01:16:10 UTC 2012
Modified Files: src/sys/arch/arm/arm32: pmap.c Log Message: Optimize pmap_{copy,zerp}_page_generic to understand that when all of memory is always mapped that you don't have create mappings dynamically. To generate a diff of this commit: cvs rdiff -u -r1.243 -r1.244 src/sys/arch/arm/arm32/pmap.c Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files.
Modified files: Index: src/sys/arch/arm/arm32/pmap.c diff -u src/sys/arch/arm/arm32/pmap.c:1.243 src/sys/arch/arm/arm32/pmap.c:1.244 --- src/sys/arch/arm/arm32/pmap.c:1.243 Mon Dec 10 06:54:23 2012 +++ src/sys/arch/arm/arm32/pmap.c Tue Dec 11 01:16:10 2012 @@ -1,4 +1,4 @@ -/* $NetBSD: pmap.c,v 1.243 2012/12/10 06:54:23 matt Exp $ */ +/* $NetBSD: pmap.c,v 1.244 2012/12/11 01:16:10 matt Exp $ */ /* * Copyright 2003 Wasabi Systems, Inc. @@ -212,7 +212,7 @@ #include <arm/cpuconf.h> #include <arm/arm32/katelib.h> -__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.243 2012/12/10 06:54:23 matt Exp $"); +__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.244 2012/12/11 01:16:10 matt Exp $"); #ifdef PMAP_DEBUG @@ -668,12 +668,6 @@ static vaddr_t kernel_pt_lookup(paddr_t /* - * External function prototypes - */ -extern void bzero_page(vaddr_t); -extern void bcopy_page(vaddr_t, vaddr_t); - -/* * Misc variables */ vaddr_t virtual_avail; @@ -4442,14 +4436,26 @@ pmap_zero_page_generic(paddr_t phys) struct vm_page *pg = PHYS_TO_VM_PAGE(phys); struct vm_page_md *md = VM_PAGE_TO_MD(pg); #endif -#ifdef PMAP_CACHE_VIPT +#if defined(PMAP_CACHE_VIPT) /* Choose the last page color it had, if any */ const vsize_t va_offset = md->pvh_attrs & arm_cache_prefer_mask; #else const vsize_t va_offset = 0; #endif +#if defined(__HAVE_MM_MD_DIRECT_MAPPED_PHYS) + /* + * Is this page mapped at its natural color? + * If we have all of memory mapped, then just convert PA to VA. + */ + const bool okcolor = va_offset == (phys & arm_cache_prefer_mask); + const vaddr_t vdstp = KERNEL_BASE + (phys - physical_start); +#else + const bool okcolor = false; + const vaddr_t vdstp = cdstp + va_offset; +#endif pt_entry_t * const ptep = &cdst_pte[va_offset >> PGSHIFT]; + #ifdef DEBUG if (!SLIST_EMPTY(&md->pvh_list)) panic("pmap_zero_page: page has mappings"); @@ -4457,25 +4463,39 @@ pmap_zero_page_generic(paddr_t phys) KDASSERT((phys & PGOFSET) == 0); - /* - * Hook in the page, zero it, and purge the cache for that - * zeroed page. Invalidate the TLB as needed. - */ - *ptep = L2_S_PROTO | phys | - L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | pte_l2_s_cache_mode; - PTE_SYNC(ptep); - cpu_tlb_flushD_SE(cdstp + va_offset); - cpu_cpwait(); - bzero_page(cdstp + va_offset); - /* - * Unmap the page. - */ - *ptep = 0; - PTE_SYNC(ptep); - cpu_tlb_flushD_SE(cdstp + va_offset); + if (!okcolor) { + /* + * Hook in the page, zero it, and purge the cache for that + * zeroed page. Invalidate the TLB as needed. + */ + *ptep = L2_S_PROTO | phys | + L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | pte_l2_s_cache_mode; + PTE_SYNC(ptep); + cpu_tlb_flushD_SE(cdstp + va_offset); + cpu_cpwait(); +#if defined(__HAVE_MM_MD_DIRECT_MAPPED_PHYS) && defined(PMAP_CACHE_VIPT) + /* + * If we are direct-mapped and our color isn't ok, then before + * we bzero the page invalidate its contents from the cache and + * reset the color to its natural color. + */ + cpu_dcache_inv_range(cdstp + va_offset, PAGE_SIZE); + md->pvh_attrs &= ~arm_cache_prefer_mask; + md->pvh_attrs |= (phys & arm_cache_prefer_mask); +#endif + } + bzero_page(vdstp); + if (!okcolor) { + /* + * Unmap the page. + */ + *ptep = 0; + PTE_SYNC(ptep); + cpu_tlb_flushD_SE(cdstp + va_offset); #ifdef PMAP_CACHE_VIVT - cpu_dcache_wbinv_range(cdstp + va_offset, PAGE_SIZE); + cpu_dcache_wbinv_range(cdstp + va_offset, PAGE_SIZE); #endif + } #ifdef PMAP_CACHE_VIPT /* * This page is now cache resident so it now has a page color. @@ -4635,6 +4655,23 @@ pmap_copy_page_generic(paddr_t src, padd const vsize_t src_va_offset = 0; const vsize_t dst_va_offset = 0; #endif +#if defined(__HAVE_MM_MD_DIRECT_MAPPED_PHYS) + /* + * Is this page mapped at its natural color? + * If we have all of memory mapped, then just convert PA to VA. + */ + const bool src_okcolor = src_va_offset == (src & arm_cache_prefer_mask); + const bool dst_okcolor = dst_va_offset == (dst & arm_cache_prefer_mask); + const vaddr_t vsrcp = src_okcolor + ? KERNEL_BASE + (src - physical_start) + : csrcp + src_va_offset; + const vaddr_t vdstp = KERNEL_BASE + (dst - physical_start); +#else + const bool src_okcolor = false; + const bool dst_okcolor = false; + const vaddr_t vsrcp = csrcp + va_offset; + const vaddr_t vdstp = cdstp + va_offset; +#endif pt_entry_t * const src_ptep = &csrc_pte[src_va_offset >> PGSHIFT]; pt_entry_t * const dst_ptep = &cdst_pte[dst_va_offset >> PGSHIFT]; @@ -4666,38 +4703,57 @@ pmap_copy_page_generic(paddr_t src, padd * the cache for the appropriate page. Invalidate the TLB * as required. */ - *src_ptep = L2_S_PROTO - | src + if (!src_okcolor) { + *src_ptep = L2_S_PROTO + | src #ifdef PMAP_CACHE_VIPT - | ((src_md->pvh_attrs & PVF_NC) ? 0 : pte_l2_s_cache_mode) + | ((src_md->pvh_attrs & PVF_NC) ? 0 : pte_l2_s_cache_mode) #endif #ifdef PMAP_CACHE_VIVT - | pte_l2_s_cache_mode + | pte_l2_s_cache_mode #endif - | L2_S_PROT(PTE_KERNEL, VM_PROT_READ); - *dst_ptep = L2_S_PROTO | dst | - L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | pte_l2_s_cache_mode; - PTE_SYNC(src_ptep); - PTE_SYNC(dst_ptep); - cpu_tlb_flushD_SE(csrcp + src_va_offset); - cpu_tlb_flushD_SE(cdstp + dst_va_offset); - cpu_cpwait(); - bcopy_page(csrcp + src_va_offset, cdstp + dst_va_offset); -#ifdef PMAP_CACHE_VIVT - cpu_dcache_inv_range(csrcp + src_va_offset, PAGE_SIZE); + | L2_S_PROT(PTE_KERNEL, VM_PROT_READ); + PTE_SYNC(src_ptep); + cpu_tlb_flushD_SE(csrcp + src_va_offset); + cpu_cpwait(); + } + if (!dst_okcolor) { + *dst_ptep = L2_S_PROTO | dst | + L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | pte_l2_s_cache_mode; + PTE_SYNC(dst_ptep); + cpu_tlb_flushD_SE(cdstp + dst_va_offset); + cpu_cpwait(); +#if defined(__HAVE_MM_MD_DIRECT_MAPPED_PHYS) && defined(PMAP_CACHE_VIPT) + /* + * If we are direct-mapped and our color isn't ok, then before + * we bcopy to the new page invalidate its contents from the + * cache and reset its color to its natural color. + */ + cpu_dcache_inv_range(cdstp + dst_va_offset, PAGE_SIZE); + dst_md->pvh_attrs &= ~arm_cache_prefer_mask; + dst_md->pvh_attrs |= (dst & arm_cache_prefer_mask); #endif + } + bcopy_page(vsrcp, vdstp); #ifdef PMAP_CACHE_VIVT - cpu_dcache_wbinv_range(cdstp + dst_va_offset, PAGE_SIZE); + cpu_dcache_inv_range(vsrcp, PAGE_SIZE); + cpu_dcache_wbinv_range(vdstp, PAGE_SIZE); #endif /* * Unmap the pages. */ - *src_ptep = 0; - *dst_ptep = 0; - PTE_SYNC(src_ptep); - PTE_SYNC(dst_ptep); - cpu_tlb_flushD_SE(csrcp + src_va_offset); - cpu_tlb_flushD_SE(cdstp + dst_va_offset); + if (!src_okcolor) { + *src_ptep = 0; + PTE_SYNC(src_ptep); + cpu_tlb_flushD_SE(csrcp + src_va_offset); + cpu_cpwait(); + } + if (!dst_okcolor) { + *dst_ptep = 0; + PTE_SYNC(dst_ptep); + cpu_tlb_flushD_SE(cdstp + dst_va_offset); + cpu_cpwait(); + } #ifdef PMAP_CACHE_VIPT /* * Now that the destination page is in the cache, mark it as colored.