Module Name: src Committed By: matt Date: Tue Jan 4 08:26:33 UTC 2011
Modified Files: src/sys/uvm: uvm_extern.h uvm_fault.c uvm_km.c uvm_page.c Log Message: Add better color matching selecting free pages. KM pages will now allocated so that VA and PA have the same color. On a page fault, choose a physical page that has the same color as the virtual address. When allocating kernel memory pages, allow the MD to specify a preferred VM_FREELIST from which to choose pages. For machines with large amounts of memory (> 4GB), all kernel memory to come from <4GB to reduce the amount of bounce buffering needed with 32bit DMA devices. To generate a diff of this commit: cvs rdiff -u -r1.167 -r1.168 src/sys/uvm/uvm_extern.h cvs rdiff -u -r1.178 -r1.179 src/sys/uvm/uvm_fault.c cvs rdiff -u -r1.106 -r1.107 src/sys/uvm/uvm_km.c cvs rdiff -u -r1.168 -r1.169 src/sys/uvm/uvm_page.c Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files.
Modified files: Index: src/sys/uvm/uvm_extern.h diff -u src/sys/uvm/uvm_extern.h:1.167 src/sys/uvm/uvm_extern.h:1.168 --- src/sys/uvm/uvm_extern.h:1.167 Mon Dec 20 00:25:47 2010 +++ src/sys/uvm/uvm_extern.h Tue Jan 4 08:26:33 2011 @@ -1,4 +1,4 @@ -/* $NetBSD: uvm_extern.h,v 1.167 2010/12/20 00:25:47 matt Exp $ */ +/* $NetBSD: uvm_extern.h,v 1.168 2011/01/04 08:26:33 matt Exp $ */ /* * @@ -148,6 +148,7 @@ #define UVM_FLAG_QUANTUM 0x800000 /* entry can never be split later */ #define UVM_FLAG_WAITVA 0x1000000 /* wait for va */ #define UVM_FLAG_VAONLY 0x2000000 /* unmap: no pages are mapped */ +#define UVM_FLAG_COLORMATCH 0x4000000 /* match color given in off */ /* macros to extract info */ #define UVM_PROTECTION(X) ((X) & UVM_PROT_MASK) @@ -466,6 +467,8 @@ #ifdef _KERNEL /* we need this before including uvm_page.h on some platforms */ extern struct uvmexp uvmexp; +/* MD code needs this without including <uvm/uvm.h> */ +extern bool vm_page_zero_enable; #endif /* Index: src/sys/uvm/uvm_fault.c diff -u src/sys/uvm/uvm_fault.c:1.178 src/sys/uvm/uvm_fault.c:1.179 --- src/sys/uvm/uvm_fault.c:1.178 Mon Dec 20 00:25:47 2010 +++ src/sys/uvm/uvm_fault.c Tue Jan 4 08:26:33 2011 @@ -1,4 +1,4 @@ -/* $NetBSD: uvm_fault.c,v 1.178 2010/12/20 00:25:47 matt Exp $ */ +/* $NetBSD: uvm_fault.c,v 1.179 2011/01/04 08:26:33 matt Exp $ */ /* * @@ -39,7 +39,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: uvm_fault.c,v 1.178 2010/12/20 00:25:47 matt Exp $"); +__KERNEL_RCSID(0, "$NetBSD: uvm_fault.c,v 1.179 2011/01/04 08:26:33 matt Exp $"); #include "opt_uvmhist.h" @@ -361,7 +361,8 @@ * no page, we must try and bring it in. */ - pg = uvm_pagealloc(NULL, 0, anon, 0); + pg = uvm_pagealloc(NULL, ufi->orig_rvaddr, + NULL, UVM_FLAG_COLORMATCH); if (pg == NULL) { /* out of RAM. */ uvmfault_unlockall(ufi, amap, NULL, anon); uvmexp.fltnoram++; @@ -619,8 +620,8 @@ * so have uvm_pagealloc() do that for us. */ - pg = uvm_pagealloc(NULL, 0, anon, - (opg == NULL) ? UVM_PGA_ZERO : 0); + pg = uvm_pagealloc(NULL, ufi->orig_rvaddr, anon, + UVM_FLAG_COLORMATCH | (opg == NULL ? UVM_PGA_ZERO : 0)); } else { pg = NULL; } Index: src/sys/uvm/uvm_km.c diff -u src/sys/uvm/uvm_km.c:1.106 src/sys/uvm/uvm_km.c:1.107 --- src/sys/uvm/uvm_km.c:1.106 Fri May 14 05:02:06 2010 +++ src/sys/uvm/uvm_km.c Tue Jan 4 08:26:33 2011 @@ -1,4 +1,4 @@ -/* $NetBSD: uvm_km.c,v 1.106 2010/05/14 05:02:06 cegger Exp $ */ +/* $NetBSD: uvm_km.c,v 1.107 2011/01/04 08:26:33 matt Exp $ */ /* * Copyright (c) 1997 Charles D. Cranor and Washington University. @@ -127,7 +127,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: uvm_km.c,v 1.106 2010/05/14 05:02:06 cegger Exp $"); +__KERNEL_RCSID(0, "$NetBSD: uvm_km.c,v 1.107 2011/01/04 08:26:33 matt Exp $"); #include "opt_uvmhist.h" @@ -588,7 +588,7 @@ loopva = kva; loopsize = size; - pgaflags = 0; + pgaflags = UVM_FLAG_COLORMATCH; if (flags & UVM_KMF_NOWAIT) pgaflags |= UVM_PGA_USERESERVE; if (flags & UVM_KMF_ZERO) @@ -599,7 +599,13 @@ while (loopsize) { KASSERT(!pmap_extract(pmap_kernel(), loopva, NULL)); - pg = uvm_pagealloc(NULL, offset, NULL, pgaflags); + pg = uvm_pagealloc_strat(NULL, offset, NULL, pgaflags, +#ifdef UVM_KM_VMFREELIST + UVM_PGA_STRAT_ONLY, UVM_KM_VMFREELIST +#else + UVM_PGA_STRAT_NORMAL, 0 +#endif + ); /* * out of memory? @@ -725,8 +731,13 @@ struct vm_page *pg; vaddr_t va; + again: +#ifdef PMAP_ALLOC_POOLPAGE + pg = PMAP_ALLOC_POOLPAGE(waitok ? 0 : UVM_PGA_USERESERVE); +#else pg = uvm_pagealloc(NULL, 0, NULL, waitok ? 0 : UVM_PGA_USERESERVE); +#endif if (__predict_false(pg == NULL)) { if (waitok) { uvm_wait("plpg"); Index: src/sys/uvm/uvm_page.c diff -u src/sys/uvm/uvm_page.c:1.168 src/sys/uvm/uvm_page.c:1.169 --- src/sys/uvm/uvm_page.c:1.168 Sat Dec 11 22:34:03 2010 +++ src/sys/uvm/uvm_page.c Tue Jan 4 08:26:33 2011 @@ -1,4 +1,4 @@ -/* $NetBSD: uvm_page.c,v 1.168 2010/12/11 22:34:03 matt Exp $ */ +/* $NetBSD: uvm_page.c,v 1.169 2011/01/04 08:26:33 matt Exp $ */ /* * Copyright (c) 2010 The NetBSD Foundation, Inc. @@ -97,7 +97,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: uvm_page.c,v 1.168 2010/12/11 22:34:03 matt Exp $"); +__KERNEL_RCSID(0, "$NetBSD: uvm_page.c,v 1.169 2011/01/04 08:26:33 matt Exp $"); #include "opt_ddb.h" #include "opt_uvmhist.h" @@ -1220,7 +1220,7 @@ lwp_t *l; KASSERT(obj == NULL || anon == NULL); - KASSERT(anon == NULL || off == 0); + KASSERT(anon == NULL || (flags & UVM_FLAG_COLORMATCH) || off == 0); KASSERT(off == trunc_page(off)); KASSERT(obj == NULL || mutex_owned(&obj->vmobjlock)); KASSERT(anon == NULL || mutex_owned(&anon->an_lock)); @@ -1230,12 +1230,14 @@ /* * This implements a global round-robin page coloring * algorithm. - * - * XXXJRT: What about virtually-indexed caches? */ ucpu = curcpu()->ci_data.cpu_uvm; - color = ucpu->page_free_nextcolor; + if (flags & UVM_FLAG_COLORMATCH) { + color = atop(off) & uvmexp.colormask; + } else { + color = ucpu->page_free_nextcolor; + } /* * check to see if we need to generate some free pages waking