On Mon, Feb 15, 2021 at 01:15:33PM +0100, Martin Pieuchot wrote:
> On 15/02/21(Mon) 11:47, Martin Pieuchot wrote:
> > Diff below includes non-functional changes:
> >
> > - Sync comments with NetBSD including locking details.
> > - Remove superfluous parenthesis and spaces.
> > - Add brackets, even if questionable, to reduce diff with NetBSD
> > - Use for (;;) instead of while(1)
> > - Rename a variable from 'result' into 'error'.
> > - Move uvm_fault() and uvm_fault_upper_lookup()
> > - Add an locking assert in uvm_fault_upper_lookup()
>
> Updated diff on top of recent fix, still ok?
>
I reviewed the diff and agree it introduces no functional changes. If you are
still looking for oks and it helps you with the locking work, ok mlarkin.
-ml
> Index: uvm/uvm_fault.c
> ===================================================================
> RCS file: /cvs/src/sys/uvm/uvm_fault.c,v
> retrieving revision 1.114
> diff -u -p -r1.114 uvm_fault.c
> --- uvm/uvm_fault.c 15 Feb 2021 12:12:54 -0000 1.114
> +++ uvm/uvm_fault.c 15 Feb 2021 12:14:08 -0000
> @@ -55,11 +55,11 @@
> * read/write1 write>1 read/write +-cow_write/zero
> * | | | |
> * +--|--+ +--|--+ +-----+ + | + | +-----+
> - * amap | V | | ----------->new| | | | ^ |
> + * amap | V | | ---------> new | | | | ^ |
> * +-----+ +-----+ +-----+ + | + | +--|--+
> * | | |
> * +-----+ +-----+ +--|--+ | +--|--+
> - * uobj | d/c | | d/c | | V | +----| |
> + * uobj | d/c | | d/c | | V | +----+ |
> * +-----+ +-----+ +-----+ +-----+
> *
> * d/c = don't care
> @@ -69,7 +69,7 @@
> *
> * case [1]: upper layer fault [anon active]
> * 1A: [read] or [write with anon->an_ref == 1]
> - * I/O takes place in top level anon and uobj is not touched.
> + * I/O takes place in upper level anon and uobj is not touched.
> * 1B: [write with anon->an_ref > 1]
> * new anon is alloc'd and data is copied off ["COW"]
> *
> @@ -89,7 +89,7 @@
> * the code is structured as follows:
> *
> * - init the "IN" params in the ufi structure
> - * ReFault:
> + * ReFault: (ERESTART returned to the loop in uvm_fault)
> * - do lookups [locks maps], check protection, handle needs_copy
> * - check for case 0 fault (error)
> * - establish "range" of fault
> @@ -136,8 +136,8 @@
> * by multiple map entries, and figuring out what should wait could be
> * complex as well...).
> *
> - * we use alternative 2 currently. maybe alternative 3 would be useful
> - * in the future. XXX keep in mind for future consideration//rechecking.
> + * we use alternative 2. given that we are multi-threaded now we may want
> + * to reconsider the choice.
> */
>
> /*
> @@ -177,7 +177,7 @@ uvmfault_anonflush(struct vm_anon **anon
> int lcv;
> struct vm_page *pg;
>
> - for (lcv = 0 ; lcv < n ; lcv++) {
> + for (lcv = 0; lcv < n; lcv++) {
> if (anons[lcv] == NULL)
> continue;
> KASSERT(rw_lock_held(anons[lcv]->an_lock));
> @@ -222,14 +222,14 @@ uvmfault_init(void)
> /*
> * uvmfault_amapcopy: clear "needs_copy" in a map.
> *
> + * => called with VM data structures unlocked (usually, see below)
> + * => we get a write lock on the maps and clear needs_copy for a VA
> * => if we are out of RAM we sleep (waiting for more)
> */
> static void
> uvmfault_amapcopy(struct uvm_faultinfo *ufi)
> {
> -
> - /* while we haven't done the job */
> - while (1) {
> + for (;;) {
> /* no mapping? give up. */
> if (uvmfault_lookup(ufi, TRUE) == FALSE)
> return;
> @@ -258,36 +258,46 @@ uvmfault_amapcopy(struct uvm_faultinfo *
> * uvmfault_anonget: get data in an anon into a non-busy, non-released
> * page in that anon.
> *
> - * => we don't move the page on the queues [gets moved later]
> - * => if we allocate a new page [we_own], it gets put on the queues.
> - * either way, the result is that the page is on the queues at return time
> + * => Map, amap and thus anon should be locked by caller.
> + * => If we fail, we unlock everything and error is returned.
> + * => If we are successful, return with everything still locked.
> + * => We do not move the page on the queues [gets moved later]. If we
> + * allocate a new page [we_own], it gets put on the queues. Either way,
> + * the result is that the page is on the queues at return time
> */
> int
> uvmfault_anonget(struct uvm_faultinfo *ufi, struct vm_amap *amap,
> struct vm_anon *anon)
> {
> - boolean_t we_own; /* we own anon's page? */
> - boolean_t locked; /* did we relock? */
> struct vm_page *pg;
> - int result;
> + int error;
>
> KASSERT(rw_lock_held(anon->an_lock));
> KASSERT(anon->an_lock == amap->am_lock);
>
> - result = 0; /* XXX shut up gcc */
> + /* Increment the counters.*/
> counters_inc(uvmexp_counters, flt_anget);
> - /* bump rusage counters */
> - if (anon->an_page)
> + if (anon->an_page) {
> curproc->p_ru.ru_minflt++;
> - else
> + } else {
> curproc->p_ru.ru_majflt++;
> + }
> + error = 0;
>
> - /* loop until we get it, or fail. */
> - while (1) {
> - we_own = FALSE; /* TRUE if we set PG_BUSY on a page */
> + /*
> + * Loop until we get the anon data, or fail.
> + */
> + for (;;) {
> + boolean_t we_own, locked;
> + /*
> + * Note: 'we_own' will become true if we set PG_BUSY on a page.
> + */
> + we_own = FALSE;
> pg = anon->an_page;
>
> - /* page there? make sure it is not busy/released. */
> + /*
> + * Is page resident? Make sure it is not busy/released.
> + */
> if (pg) {
> KASSERT(pg->pg_flags & PQ_ANON);
> KASSERT(pg->uanon == anon);
> @@ -302,29 +312,31 @@ uvmfault_anonget(struct uvm_faultinfo *u
> counters_inc(uvmexp_counters, flt_pgwait);
>
> /*
> - * the last unlock must be an atomic unlock+wait on
> - * the owner of page
> + * The last unlock must be an atomic unlock and wait
> + * on the owner of page.
> */
> if (pg->uobject) {
> + /* Owner of page is UVM object. */
> uvmfault_unlockall(ufi, amap, NULL);
> tsleep_nsec(pg, PVM, "anonget1", INFSLP);
> } else {
> + /* Owner of page is anon. */
> uvmfault_unlockall(ufi, NULL, NULL);
> rwsleep_nsec(pg, anon->an_lock, PVM | PNORELOCK,
> "anonget2", INFSLP);
> }
> - /* ready to relock and try again */
> } else {
> - /* no page, we must try and bring it in. */
> + /*
> + * No page, therefore allocate one.
> + */
> pg = uvm_pagealloc(NULL, 0, anon, 0);
> -
> - if (pg == NULL) { /* out of RAM. */
> + if (pg == NULL) {
> + /* Out of memory. Wait a little. */
> uvmfault_unlockall(ufi, amap, NULL);
> counters_inc(uvmexp_counters, flt_noram);
> uvm_wait("flt_noram1");
> - /* ready to relock and try again */
> } else {
> - /* we set the PG_BUSY bit */
> + /* PG_BUSY bit is set. */
> we_own = TRUE;
> uvmfault_unlockall(ufi, amap, NULL);
>
> @@ -336,32 +348,33 @@ uvmfault_anonget(struct uvm_faultinfo *u
> * we hold PG_BUSY on the page.
> */
> counters_inc(uvmexp_counters, pageins);
> - result = uvm_swap_get(pg, anon->an_swslot,
> + error = uvm_swap_get(pg, anon->an_swslot,
> PGO_SYNCIO);
>
> /*
> - * we clean up after the i/o below in the
> - * "we_own" case
> + * We clean up after the I/O below in the
> + * 'we_own' case.
> */
> - /* ready to relock and try again */
> }
> }
>
> - /* now relock and try again */
> + /*
> + * Re-lock the map and anon.
> + */
> locked = uvmfault_relock(ufi);
> if (locked || we_own) {
> rw_enter(anon->an_lock, RW_WRITE);
> }
>
> /*
> - * if we own the page (i.e. we set PG_BUSY), then we need
> - * to clean up after the I/O. there are three cases to
> + * If we own the page (i.e. we set PG_BUSY), then we need
> + * to clean up after the I/O. There are three cases to
> * consider:
> - * [1] page released during I/O: free anon and ReFault.
> - * [2] I/O not OK. free the page and cause the fault
> - * to fail.
> - * [3] I/O OK! activate the page and sync with the
> - * non-we_own case (i.e. drop anon lock if not locked).
> + *
> + * 1) Page was released during I/O: free anon and ReFault.
> + * 2) I/O not OK. Free the page and cause the fault to fail.
> + * 3) I/O OK! Activate the page and sync with the non-we_own
> + * case (i.e. drop anon lock if not locked).
> */
> if (we_own) {
> if (pg->pg_flags & PG_WANTED) {
> @@ -387,32 +400,33 @@ uvmfault_anonget(struct uvm_faultinfo *u
> return (VM_PAGER_REFAULT); /* refault! */
> }
>
> - if (result != VM_PAGER_OK) {
> - KASSERT(result != VM_PAGER_PEND);
> + if (error != VM_PAGER_OK) {
> + KASSERT(error != VM_PAGER_PEND);
>
> /* remove page from anon */
> anon->an_page = NULL;
>
> /*
> - * remove the swap slot from the anon
> - * and mark the anon as having no real slot.
> - * don't free the swap slot, thus preventing
> + * Remove the swap slot from the anon and
> + * mark the anon as having no real slot.
> + * Do not free the swap slot, thus preventing
> * it from being used again.
> */
> uvm_swap_markbad(anon->an_swslot, 1);
> anon->an_swslot = SWSLOT_BAD;
>
> /*
> - * note: page was never !PG_BUSY, so it
> - * can't be mapped and thus no need to
> + * Note: page was never !PG_BUSY, so it
> + * cannot be mapped and thus no need to
> * pmap_page_protect it...
> */
> uvm_lock_pageq();
> uvm_pagefree(pg);
> uvm_unlock_pageq();
>
> - if (locked)
> + if (locked) {
> uvmfault_unlockall(ufi, NULL, NULL);
> + }
> rw_exit(anon->an_lock);
> return (VM_PAGER_ERROR);
> }
> @@ -427,7 +441,9 @@ uvmfault_anonget(struct uvm_faultinfo *u
> uvm_unlock_pageq();
> }
>
> - /* we were not able to relock. restart fault. */
> + /*
> + * We were not able to re-lock the map - restart the fault.
> + */
> if (!locked) {
> if (we_own) {
> rw_exit(anon->an_lock);
> @@ -435,20 +451,24 @@ uvmfault_anonget(struct uvm_faultinfo *u
> return (VM_PAGER_REFAULT);
> }
>
> - /* verify no one touched the amap and moved the anon on us. */
> - if (ufi != NULL &&
> - amap_lookup(&ufi->entry->aref,
> + /*
> + * Verify that no one has touched the amap and moved
> + * the anon on us.
> + */
> + if (ufi != NULL && amap_lookup(&ufi->entry->aref,
> ufi->orig_rvaddr - ufi->entry->start) != anon) {
>
> uvmfault_unlockall(ufi, amap, NULL);
> return (VM_PAGER_REFAULT);
> }
>
> - /* try it again! */
> + /*
> + * Retry..
> + */
> counters_inc(uvmexp_counters, flt_anretry);
> continue;
>
> - } /* while (1) */
> + }
> /*NOTREACHED*/
> }
>
> @@ -489,6 +509,23 @@ uvmfault_update_stats(struct uvm_faultin
> }
> }
>
> +/*
> + * F A U L T - m a i n e n t r y p o i n t
> + */
> +
> +/*
> + * uvm_fault: page fault handler
> + *
> + * => called from MD code to resolve a page fault
> + * => VM data structures usually should be unlocked. however, it is
> + * possible to call here with the main map locked if the caller
> + * gets a write lock, sets it recursive, and then calls us (c.f.
> + * uvm_map_pageable). this should be avoided because it keeps
> + * the map locked off during I/O.
> + * => MUST NEVER BE CALLED IN INTERRUPT CONTEXT
> + */
> +#define MASK(entry) (UVM_ET_ISCOPYONWRITE(entry) ? \
> + ~PROT_WRITE : PROT_MASK)
> struct uvm_faultctx {
> /*
> * the following members are set up by uvm_fault_check() and
> @@ -504,8 +541,72 @@ struct uvm_faultctx {
> paddr_t pa_flags;
> };
>
> -int uvm_fault_lower(struct uvm_faultinfo *, struct uvm_faultctx *,
> - struct vm_page **, vm_fault_t);
> +int uvm_fault_check(
> + struct uvm_faultinfo *, struct uvm_faultctx *,
> + struct vm_anon ***);
> +
> +int uvm_fault_upper(
> + struct uvm_faultinfo *, struct uvm_faultctx *,
> + struct vm_anon **, vm_fault_t);
> +boolean_t uvm_fault_upper_lookup(
> + struct uvm_faultinfo *, const struct uvm_faultctx *,
> + struct vm_anon **, struct vm_page **);
> +
> +int uvm_fault_lower(
> + struct uvm_faultinfo *, struct uvm_faultctx *,
> + struct vm_page **, vm_fault_t);
> +
> +int
> +uvm_fault(vm_map_t orig_map, vaddr_t vaddr, vm_fault_t fault_type,
> + vm_prot_t access_type)
> +{
> + struct uvm_faultinfo ufi;
> + struct uvm_faultctx flt;
> + boolean_t shadowed;
> + struct vm_anon *anons_store[UVM_MAXRANGE], **anons;
> + struct vm_page *pages[UVM_MAXRANGE];
> + int error;
> +
> + counters_inc(uvmexp_counters, faults);
> + TRACEPOINT(uvm, fault, vaddr, fault_type, access_type, NULL);
> +
> + /*
> + * init the IN parameters in the ufi
> + */
> + ufi.orig_map = orig_map;
> + ufi.orig_rvaddr = trunc_page(vaddr);
> + ufi.orig_size = PAGE_SIZE; /* can't get any smaller than this */
> + if (fault_type == VM_FAULT_WIRE)
> + flt.narrow = TRUE; /* don't look for neighborhood
> + * pages on wire */
> + else
> + flt.narrow = FALSE; /* normal fault */
> + flt.access_type = access_type;
> +
> +
> + error = ERESTART;
> + while (error == ERESTART) { /* ReFault: */
> + anons = anons_store;
> +
> + error = uvm_fault_check(&ufi, &flt, &anons);
> + if (error != 0)
> + continue;
> +
> + /* True if there is an anon at the faulting address */
> + shadowed = uvm_fault_upper_lookup(&ufi, &flt, anons, pages);
> + if (shadowed == TRUE) {
> + /* case 1: fault on an anon in our amap */
> + error = uvm_fault_upper(&ufi, &flt, anons, fault_type);
> + } else {
> + /* case 2: fault on backing object or zero fill */
> + KERNEL_LOCK();
> + error = uvm_fault_lower(&ufi, &flt, pages, fault_type);
> + KERNEL_UNLOCK();
> + }
> + }
> +
> + return error;
> +}
>
> /*
> * uvm_fault_check: check prot, handle needs-copy, etc.
> @@ -530,10 +631,13 @@ uvm_fault_check(struct uvm_faultinfo *uf
> struct uvm_object *uobj;
> int nback, nforw;
>
> - /* lookup and lock the maps */
> + /*
> + * lookup and lock the maps
> + */
> if (uvmfault_lookup(ufi, FALSE) == FALSE) {
> - return (EFAULT);
> + return EFAULT;
> }
> + /* locked: maps(read) */
>
> #ifdef DIAGNOSTIC
> if ((ufi->map->flags & VM_MAP_PAGEABLE) == 0)
> @@ -541,10 +645,12 @@ uvm_fault_check(struct uvm_faultinfo *uf
> ufi->map, ufi->orig_rvaddr);
> #endif
>
> - /* check protection */
> + /*
> + * check protection
> + */
> if ((ufi->entry->protection & flt->access_type) != flt->access_type) {
> uvmfault_unlockmaps(ufi, FALSE);
> - return (EACCES);
> + return EACCES;
> }
>
> /*
> @@ -568,7 +674,7 @@ uvm_fault_check(struct uvm_faultinfo *uf
> uvmfault_unlockmaps(ufi, FALSE);
> uvmfault_amapcopy(ufi);
> counters_inc(uvmexp_counters, flt_amcopy);
> - return (ERESTART);
> + return ERESTART;
> } else {
> /*
> * ensure that we pmap_enter page R/O since
> @@ -578,9 +684,11 @@ uvm_fault_check(struct uvm_faultinfo *uf
> }
> }
>
> - /* identify the players */
> - amap = ufi->entry->aref.ar_amap; /* top layer */
> - uobj = ufi->entry->object.uvm_obj; /* bottom layer */
> + /*
> + * identify the players
> + */
> + amap = ufi->entry->aref.ar_amap; /* upper layer */
> + uobj = ufi->entry->object.uvm_obj; /* lower layer */
>
> /*
> * check for a case 0 fault. if nothing backing the entry then
> @@ -588,7 +696,7 @@ uvm_fault_check(struct uvm_faultinfo *uf
> */
> if (amap == NULL && uobj == NULL) {
> uvmfault_unlockmaps(ufi, FALSE);
> - return (EFAULT);
> + return EFAULT;
> }
>
> /*
> @@ -621,7 +729,9 @@ uvm_fault_check(struct uvm_faultinfo *uf
> flt->centeridx = 0;
> }
>
> - /* if we've got an amap, extract current anons. */
> + /*
> + * if we've got an amap then lock it and extract current anons.
> + */
> if (amap) {
> amap_lock(amap);
> amap_lookups(&ufi->entry->aref,
> @@ -662,11 +772,100 @@ uvm_fault_check(struct uvm_faultinfo *uf
> }
>
> /*
> - * uvm_fault_upper: handle upper fault (case 1A & 1B)
> + * uvm_fault_upper_lookup: look up existing h/w mapping and amap.
> + *
> + * iterate range of interest:
> + * 1. check if h/w mapping exists. if yes, we don't care
> + * 2. check if anon exists. if not, page is lower.
> + * 3. if anon exists, enter h/w mapping for neighbors.
> *
> - * 1. get anon. let uvmfault_anonget do the dirty work.
> - * 2. if COW, promote data to new anon
> - * 3. enter h/w mapping
> + * => called with amap locked (if exists).
> + */
> +boolean_t
> +uvm_fault_upper_lookup(struct uvm_faultinfo *ufi,
> + const struct uvm_faultctx *flt, struct vm_anon **anons,
> + struct vm_page **pages)
> +{
> + struct vm_amap *amap = ufi->entry->aref.ar_amap;
> + struct vm_anon *anon;
> + boolean_t shadowed;
> + vaddr_t currva;
> + paddr_t pa;
> + int lcv;
> +
> + /* locked: maps(read), amap(if there) */
> + KASSERT(amap == NULL ||
> + rw_write_held(amap->am_lock));
> +
> + /*
> + * map in the backpages and frontpages we found in the amap in hopes
> + * of preventing future faults. we also init the pages[] array as
> + * we go.
> + */
> + currva = flt->startva;
> + shadowed = FALSE;
> + for (lcv = 0; lcv < flt->npages; lcv++, currva += PAGE_SIZE) {
> + /*
> + * dont play with VAs that are already mapped
> + * except for center)
> + */
> + if (lcv != flt->centeridx &&
> + pmap_extract(ufi->orig_map->pmap, currva, &pa)) {
> + pages[lcv] = PGO_DONTCARE;
> + continue;
> + }
> +
> + /*
> + * unmapped or center page. check if any anon at this level.
> + */
> + if (amap == NULL || anons[lcv] == NULL) {
> + pages[lcv] = NULL;
> + continue;
> + }
> +
> + /*
> + * check for present page and map if possible.
> + */
> + pages[lcv] = PGO_DONTCARE;
> + if (lcv == flt->centeridx) { /* save center for later! */
> + shadowed = TRUE;
> + continue;
> + }
> + anon = anons[lcv];
> + KASSERT(anon->an_lock == amap->am_lock);
> + if (anon->an_page &&
> + (anon->an_page->pg_flags & (PG_RELEASED|PG_BUSY)) == 0) {
> + uvm_lock_pageq();
> + uvm_pageactivate(anon->an_page); /* reactivate */
> + uvm_unlock_pageq();
> + counters_inc(uvmexp_counters, flt_namap);
> +
> + /*
> + * Since this isn't the page that's actually faulting,
> + * ignore pmap_enter() failures; it's not critical
> + * that we enter these right now.
> + */
> + (void) pmap_enter(ufi->orig_map->pmap, currva,
> + VM_PAGE_TO_PHYS(anon->an_page) | flt->pa_flags,
> + (anon->an_ref > 1) ?
> + (flt->enter_prot & ~PROT_WRITE) : flt->enter_prot,
> + PMAP_CANFAIL |
> + (VM_MAPENT_ISWIRED(ufi->entry) ? PMAP_WIRED : 0));
> + }
> + }
> + if (flt->npages > 1)
> + pmap_update(ufi->orig_map->pmap);
> +
> + return shadowed;
> +}
> +
> +/*
> + * uvm_fault_upper: handle upper fault.
> + *
> + * 1. acquire anon lock.
> + * 2. get anon. let uvmfault_anonget do the dirty work.
> + * 3. if COW, promote data to new anon
> + * 4. enter h/w mapping
> */
> int
> uvm_fault_upper(struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
> @@ -686,7 +885,11 @@ uvm_fault_upper(struct uvm_faultinfo *uf
> */
> /*
> * let uvmfault_anonget do the dirty work.
> + * if it fails (!OK) it will unlock everything for us.
> + * if it succeeds, locks are still valid and locked.
> * also, if it is OK, then the anon's page is on the queues.
> + * if the page is on loan from a uvm_object, then anonget will
> + * lock that object for us if it does not fail.
> */
> error = uvmfault_anonget(ufi, amap, anon);
> switch (error) {
> @@ -722,7 +925,7 @@ uvm_fault_upper(struct uvm_faultinfo *uf
> * it is > 1 and we are only dropping one ref.
> *
> * in the (hopefully very rare) case that we are out of RAM we
> - * will wait for more RAM, and refault.
> + * will unlock, wait for more RAM, and refault.
> *
> * if we are out of anon VM we wait for RAM to become available.
> */
> @@ -785,10 +988,7 @@ uvm_fault_upper(struct uvm_faultinfo *uf
> }
>
> /*
> - * now map the page in ...
> - * XXX: old fault unlocks object before pmap_enter. this seems
> - * suspect since some other thread could blast the page out from
> - * under us between the unlock and the pmap_enter.
> + * now map the page in .
> */
> if (pmap_enter(ufi->orig_map->pmap, ufi->orig_rvaddr,
> VM_PAGE_TO_PHYS(pg) | flt->pa_flags, flt->enter_prot,
> @@ -830,158 +1030,18 @@ uvm_fault_upper(struct uvm_faultinfo *uf
>
> uvm_unlock_pageq();
>
> - /* done case 1! finish up by unlocking everything and returning
> success */
> + /*
> + * done case 1! finish up by unlocking everything and returning success
> + */
> uvmfault_unlockall(ufi, amap, NULL);
> pmap_update(ufi->orig_map->pmap);
> return 0;
> }
>
> /*
> - * uvm_fault_upper_lookup: look up existing h/w mapping and amap.
> + * uvm_fault_lower: handle lower fault.
> *
> - * iterate range of interest:
> - * 1. check if h/w mapping exists. if yes, we don't care
> - * 2. check if anon exists. if not, page is lower.
> - * 3. if anon exists, enter h/w mapping for neighbors.
> */
> -boolean_t
> -uvm_fault_upper_lookup(struct uvm_faultinfo *ufi,
> - const struct uvm_faultctx *flt, struct vm_anon **anons,
> - struct vm_page **pages)
> -{
> - struct vm_amap *amap = ufi->entry->aref.ar_amap;
> - struct vm_anon *anon;
> - boolean_t shadowed;
> - vaddr_t currva;
> - paddr_t pa;
> - int lcv;
> -
> - /*
> - * map in the backpages and frontpages we found in the amap in hopes
> - * of preventing future faults. we also init the pages[] array as
> - * we go.
> - */
> - currva = flt->startva;
> - shadowed = FALSE;
> - for (lcv = 0 ; lcv < flt->npages ; lcv++, currva += PAGE_SIZE) {
> - /*
> - * dont play with VAs that are already mapped
> - * except for center)
> - */
> - if (lcv != flt->centeridx &&
> - pmap_extract(ufi->orig_map->pmap, currva, &pa)) {
> - pages[lcv] = PGO_DONTCARE;
> - continue;
> - }
> -
> - /* unmapped or center page. check if any anon at this level. */
> - if (amap == NULL || anons[lcv] == NULL) {
> - pages[lcv] = NULL;
> - continue;
> - }
> -
> - /* check for present page and map if possible. re-activate it */
> - pages[lcv] = PGO_DONTCARE;
> - if (lcv == flt->centeridx) { /* save center for later! */
> - shadowed = TRUE;
> - continue;
> - }
> - anon = anons[lcv];
> - KASSERT(anon->an_lock == amap->am_lock);
> - if (anon->an_page &&
> - (anon->an_page->pg_flags & (PG_RELEASED|PG_BUSY)) == 0) {
> - uvm_lock_pageq();
> - uvm_pageactivate(anon->an_page); /* reactivate */
> - uvm_unlock_pageq();
> - counters_inc(uvmexp_counters, flt_namap);
> -
> - /*
> - * Since this isn't the page that's actually faulting,
> - * ignore pmap_enter() failures; it's not critical
> - * that we enter these right now.
> - */
> - (void) pmap_enter(ufi->orig_map->pmap, currva,
> - VM_PAGE_TO_PHYS(anon->an_page) | flt->pa_flags,
> - (anon->an_ref > 1) ?
> - (flt->enter_prot & ~PROT_WRITE) : flt->enter_prot,
> - PMAP_CANFAIL |
> - (VM_MAPENT_ISWIRED(ufi->entry) ? PMAP_WIRED : 0));
> - }
> - }
> - if (flt->npages > 1)
> - pmap_update(ufi->orig_map->pmap);
> -
> - return shadowed;
> -}
> -
> -/*
> - * F A U L T - m a i n e n t r y p o i n t
> - */
> -
> -/*
> - * uvm_fault: page fault handler
> - *
> - * => called from MD code to resolve a page fault
> - * => VM data structures usually should be unlocked. however, it is
> - * possible to call here with the main map locked if the caller
> - * gets a write lock, sets it recursive, and then calls us (c.f.
> - * uvm_map_pageable). this should be avoided because it keeps
> - * the map locked off during I/O.
> - */
> -#define MASK(entry) (UVM_ET_ISCOPYONWRITE(entry) ? \
> - ~PROT_WRITE : PROT_MASK)
> -int
> -uvm_fault(vm_map_t orig_map, vaddr_t vaddr, vm_fault_t fault_type,
> - vm_prot_t access_type)
> -{
> - struct uvm_faultinfo ufi;
> - struct uvm_faultctx flt;
> - boolean_t shadowed;
> - struct vm_anon *anons_store[UVM_MAXRANGE], **anons;
> - struct vm_page *pages[UVM_MAXRANGE];
> - int error = ERESTART;
> -
> - counters_inc(uvmexp_counters, faults);
> - TRACEPOINT(uvm, fault, vaddr, fault_type, access_type, NULL);
> -
> - /* init the IN parameters in the ufi */
> - ufi.orig_map = orig_map;
> - ufi.orig_rvaddr = trunc_page(vaddr);
> - ufi.orig_size = PAGE_SIZE; /* can't get any smaller than this */
> - if (fault_type == VM_FAULT_WIRE)
> - flt.narrow = TRUE; /* don't look for neighborhood
> - * pages on wire */
> - else
> - flt.narrow = FALSE; /* normal fault */
> - flt.access_type = access_type;
> -
> -
> - /*
> - * ReFault
> - */
> - while (error == ERESTART) {
> - anons = anons_store;
> -
> - error = uvm_fault_check(&ufi, &flt, &anons);
> - if (error != 0)
> - continue;
> -
> - /* True if there is an anon at the faulting address */
> - shadowed = uvm_fault_upper_lookup(&ufi, &flt, anons, pages);
> - if (shadowed == TRUE) {
> - /* case 1: fault on an anon in our amap */
> - error = uvm_fault_upper(&ufi, &flt, anons, fault_type);
> - } else {
> - /* case 2: fault on backing object or zero fill */
> - KERNEL_LOCK();
> - error = uvm_fault_lower(&ufi, &flt, pages, fault_type);
> - KERNEL_UNLOCK();
> - }
> - }
> -
> - return error;
> -}
> -
> int
> uvm_fault_lower(struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
> struct vm_page **pages, vm_fault_t fault_type)
>
