Home | History | Annotate | Line # | Download | only in uvm
uvm_fault.c revision 1.214.2.1
      1 /*	$NetBSD: uvm_fault.c,v 1.214.2.1 2020/01/17 21:47:38 ad Exp $	*/
      2 
      3 /*
      4  * Copyright (c) 1997 Charles D. Cranor and Washington University.
      5  * All rights reserved.
      6  *
      7  * Redistribution and use in source and binary forms, with or without
      8  * modification, are permitted provided that the following conditions
      9  * are met:
     10  * 1. Redistributions of source code must retain the above copyright
     11  *    notice, this list of conditions and the following disclaimer.
     12  * 2. Redistributions in binary form must reproduce the above copyright
     13  *    notice, this list of conditions and the following disclaimer in the
     14  *    documentation and/or other materials provided with the distribution.
     15  *
     16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
     17  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
     18  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
     19  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
     20  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
     21  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
     22  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
     23  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
     24  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
     25  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     26  *
     27  * from: Id: uvm_fault.c,v 1.1.2.23 1998/02/06 05:29:05 chs Exp
     28  */
     29 
     30 /*
     31  * uvm_fault.c: fault handler
     32  */
     33 
     34 #include <sys/cdefs.h>
     35 __KERNEL_RCSID(0, "$NetBSD: uvm_fault.c,v 1.214.2.1 2020/01/17 21:47:38 ad Exp $");
     36 
     37 #include "opt_uvmhist.h"
     38 
     39 #include <sys/param.h>
     40 #include <sys/systm.h>
     41 #include <sys/atomic.h>
     42 #include <sys/kernel.h>
     43 #include <sys/mman.h>
     44 
     45 #include <uvm/uvm.h>
     46 
     47 /*
     48  *
     49  * a word on page faults:
     50  *
     51  * types of page faults we handle:
     52  *
     53  * CASE 1: upper layer faults                   CASE 2: lower layer faults
     54  *
     55  *    CASE 1A         CASE 1B                  CASE 2A        CASE 2B
     56  *    read/write1     write>1                  read/write   +-cow_write/zero
     57  *         |             |                         |        |
     58  *      +--|--+       +--|--+     +-----+       +  |  +     | +-----+
     59  * amap |  V  |       |  ---------> new |          |        | |  ^  |
     60  *      +-----+       +-----+     +-----+       +  |  +     | +--|--+
     61  *                                                 |        |    |
     62  *      +-----+       +-----+                   +--|--+     | +--|--+
     63  * uobj | d/c |       | d/c |                   |  V  |     +----+  |
     64  *      +-----+       +-----+                   +-----+       +-----+
     65  *
     66  * d/c = don't care
     67  *
     68  *   case [0]: layerless fault
     69  *	no amap or uobj is present.   this is an error.
     70  *
     71  *   case [1]: upper layer fault [anon active]
     72  *     1A: [read] or [write with anon->an_ref == 1]
     73  *		I/O takes place in upper level anon and uobj is not touched.
     74  *     1B: [write with anon->an_ref > 1]
     75  *		new anon is alloc'd and data is copied off ["COW"]
     76  *
     77  *   case [2]: lower layer fault [uobj]
     78  *     2A: [read on non-NULL uobj] or [write to non-copy_on_write area]
     79  *		I/O takes place directly in object.
     80  *     2B: [write to copy_on_write] or [read on NULL uobj]
     81  *		data is "promoted" from uobj to a new anon.
     82  *		if uobj is null, then we zero fill.
     83  *
     84  * we follow the standard UVM locking protocol ordering:
     85  *
     86  * MAPS => AMAP => UOBJ => ANON => PAGE QUEUES (PQ)
     87  * we hold a PG_BUSY page if we unlock for I/O
     88  *
     89  *
     90  * the code is structured as follows:
     91  *
     92  *     - init the "IN" params in the ufi structure
     93  *   ReFault: (ERESTART returned to the loop in uvm_fault_internal)
     94  *     - do lookups [locks maps], check protection, handle needs_copy
     95  *     - check for case 0 fault (error)
     96  *     - establish "range" of fault
     97  *     - if we have an amap lock it and extract the anons
     98  *     - if sequential advice deactivate pages behind us
     99  *     - at the same time check pmap for unmapped areas and anon for pages
    100  *	 that we could map in (and do map it if found)
    101  *     - check object for resident pages that we could map in
    102  *     - if (case 2) goto Case2
    103  *     - >>> handle case 1
    104  *           - ensure source anon is resident in RAM
    105  *           - if case 1B alloc new anon and copy from source
    106  *           - map the correct page in
    107  *   Case2:
    108  *     - >>> handle case 2
    109  *           - ensure source page is resident (if uobj)
    110  *           - if case 2B alloc new anon and copy from source (could be zero
    111  *		fill if uobj == NULL)
    112  *           - map the correct page in
    113  *     - done!
    114  *
    115  * note on paging:
    116  *   if we have to do I/O we place a PG_BUSY page in the correct object,
    117  * unlock everything, and do the I/O.   when I/O is done we must reverify
    118  * the state of the world before assuming that our data structures are
    119  * valid.   [because mappings could change while the map is unlocked]
    120  *
    121  *  alternative 1: unbusy the page in question and restart the page fault
    122  *    from the top (ReFault).   this is easy but does not take advantage
    123  *    of the information that we already have from our previous lookup,
    124  *    although it is possible that the "hints" in the vm_map will help here.
    125  *
    126  * alternative 2: the system already keeps track of a "version" number of
    127  *    a map.   [i.e. every time you write-lock a map (e.g. to change a
    128  *    mapping) you bump the version number up by one...]   so, we can save
    129  *    the version number of the map before we release the lock and start I/O.
    130  *    then when I/O is done we can relock and check the version numbers
    131  *    to see if anything changed.    this might save us some over 1 because
    132  *    we don't have to unbusy the page and may be less compares(?).
    133  *
    134  * alternative 3: put in backpointers or a way to "hold" part of a map
    135  *    in place while I/O is in progress.   this could be complex to
    136  *    implement (especially with structures like amap that can be referenced
    137  *    by multiple map entries, and figuring out what should wait could be
    138  *    complex as well...).
    139  *
    140  * we use alternative 2.  given that we are multi-threaded now we may want
    141  * to reconsider the choice.
    142  */
    143 
    144 /*
    145  * local data structures
    146  */
    147 
    148 struct uvm_advice {
    149 	int advice;
    150 	int nback;
    151 	int nforw;
    152 };
    153 
    154 /*
    155  * page range array:
    156  * note: index in array must match "advice" value
    157  * XXX: borrowed numbers from freebsd.   do they work well for us?
    158  */
    159 
    160 static const struct uvm_advice uvmadvice[] = {
    161 	{ UVM_ADV_NORMAL, 3, 4 },
    162 	{ UVM_ADV_RANDOM, 0, 0 },
    163 	{ UVM_ADV_SEQUENTIAL, 8, 7},
    164 };
    165 
    166 #define UVM_MAXRANGE 16	/* must be MAX() of nback+nforw+1 */
    167 
    168 /*
    169  * private prototypes
    170  */
    171 
    172 /*
    173  * externs from other modules
    174  */
    175 
    176 extern int start_init_exec;	/* Is init_main() done / init running? */
    177 
    178 /*
    179  * inline functions
    180  */
    181 
    182 /*
    183  * uvmfault_anonflush: try and deactivate pages in specified anons
    184  *
    185  * => does not have to deactivate page if it is busy
    186  */
    187 
    188 static inline void
    189 uvmfault_anonflush(struct vm_anon **anons, int n)
    190 {
    191 	int lcv;
    192 	struct vm_page *pg;
    193 
    194 	for (lcv = 0; lcv < n; lcv++) {
    195 		if (anons[lcv] == NULL)
    196 			continue;
    197 		KASSERT(mutex_owned(anons[lcv]->an_lock));
    198 		pg = anons[lcv]->an_page;
    199 		if (pg && (pg->flags & PG_BUSY) == 0) {
    200 			uvm_pagelock(pg);
    201 			uvm_pagedeactivate(pg);
    202 			uvm_pageunlock(pg);
    203 		}
    204 	}
    205 }
    206 
    207 /*
    208  * normal functions
    209  */
    210 
    211 /*
    212  * uvmfault_amapcopy: clear "needs_copy" in a map.
    213  *
    214  * => called with VM data structures unlocked (usually, see below)
    215  * => we get a write lock on the maps and clear needs_copy for a VA
    216  * => if we are out of RAM we sleep (waiting for more)
    217  */
    218 
    219 static void
    220 uvmfault_amapcopy(struct uvm_faultinfo *ufi)
    221 {
    222 	for (;;) {
    223 
    224 		/*
    225 		 * no mapping?  give up.
    226 		 */
    227 
    228 		if (uvmfault_lookup(ufi, true) == false)
    229 			return;
    230 
    231 		/*
    232 		 * copy if needed.
    233 		 */
    234 
    235 		if (UVM_ET_ISNEEDSCOPY(ufi->entry))
    236 			amap_copy(ufi->map, ufi->entry, AMAP_COPY_NOWAIT,
    237 				ufi->orig_rvaddr, ufi->orig_rvaddr + 1);
    238 
    239 		/*
    240 		 * didn't work?  must be out of RAM.   unlock and sleep.
    241 		 */
    242 
    243 		if (UVM_ET_ISNEEDSCOPY(ufi->entry)) {
    244 			uvmfault_unlockmaps(ufi, true);
    245 			uvm_wait("fltamapcopy");
    246 			continue;
    247 		}
    248 
    249 		/*
    250 		 * got it!   unlock and return.
    251 		 */
    252 
    253 		uvmfault_unlockmaps(ufi, true);
    254 		return;
    255 	}
    256 	/*NOTREACHED*/
    257 }
    258 
    259 /*
    260  * uvmfault_anonget: get data in an anon into a non-busy, non-released
    261  * page in that anon.
    262  *
    263  * => Map, amap and thus anon should be locked by caller.
    264  * => If we fail, we unlock everything and error is returned.
    265  * => If we are successful, return with everything still locked.
    266  * => We do not move the page on the queues [gets moved later].  If we
    267  *    allocate a new page [we_own], it gets put on the queues.  Either way,
    268  *    the result is that the page is on the queues at return time
    269  * => For pages which are on loan from a uvm_object (and thus are not owned
    270  *    by the anon): if successful, return with the owning object locked.
    271  *    The caller must unlock this object when it unlocks everything else.
    272  */
    273 
    274 int
    275 uvmfault_anonget(struct uvm_faultinfo *ufi, struct vm_amap *amap,
    276     struct vm_anon *anon)
    277 {
    278 	struct vm_page *pg;
    279 	int error;
    280 
    281 	UVMHIST_FUNC("uvmfault_anonget"); UVMHIST_CALLED(maphist);
    282 	KASSERT(mutex_owned(anon->an_lock));
    283 	KASSERT(anon->an_lock == amap->am_lock);
    284 
    285 	/* Increment the counters.*/
    286 	cpu_count(CPU_COUNT_FLTANGET, 1);
    287 	if (anon->an_page) {
    288 		curlwp->l_ru.ru_minflt++;
    289 	} else {
    290 		curlwp->l_ru.ru_majflt++;
    291 	}
    292 	error = 0;
    293 
    294 	/*
    295 	 * Loop until we get the anon data, or fail.
    296 	 */
    297 
    298 	for (;;) {
    299 		bool we_own, locked;
    300 		/*
    301 		 * Note: 'we_own' will become true if we set PG_BUSY on a page.
    302 		 */
    303 		we_own = false;
    304 		pg = anon->an_page;
    305 
    306 		/*
    307 		 * If there is a resident page and it is loaned, then anon
    308 		 * may not own it.  Call out to uvm_anon_lockloanpg() to
    309 		 * identify and lock the real owner of the page.
    310 		 */
    311 
    312 		if (pg && pg->loan_count)
    313 			pg = uvm_anon_lockloanpg(anon);
    314 
    315 		/*
    316 		 * Is page resident?  Make sure it is not busy/released.
    317 		 */
    318 
    319 		if (pg) {
    320 
    321 			/*
    322 			 * at this point, if the page has a uobject [meaning
    323 			 * we have it on loan], then that uobject is locked
    324 			 * by us!   if the page is busy, we drop all the
    325 			 * locks (including uobject) and try again.
    326 			 */
    327 
    328 			if ((pg->flags & PG_BUSY) == 0) {
    329 				UVMHIST_LOG(maphist, "<- OK",0,0,0,0);
    330 				return 0;
    331 			}
    332 			pg->flags |= PG_WANTED;
    333 			cpu_count(CPU_COUNT_FLTPGWAIT, 1);
    334 
    335 			/*
    336 			 * The last unlock must be an atomic unlock and wait
    337 			 * on the owner of page.
    338 			 */
    339 
    340 			if (pg->uobject) {
    341 				/* Owner of page is UVM object. */
    342 				uvmfault_unlockall(ufi, amap, NULL);
    343 				UVMHIST_LOG(maphist, " unlock+wait on uobj",0,
    344 				    0,0,0);
    345 				UVM_UNLOCK_AND_WAIT(pg,
    346 				    pg->uobject->vmobjlock,
    347 				    false, "anonget1", 0);
    348 			} else {
    349 				/* Owner of page is anon. */
    350 				uvmfault_unlockall(ufi, NULL, NULL);
    351 				UVMHIST_LOG(maphist, " unlock+wait on anon",0,
    352 				    0,0,0);
    353 				UVM_UNLOCK_AND_WAIT(pg, anon->an_lock,
    354 				    false, "anonget2", 0);
    355 			}
    356 		} else {
    357 #if defined(VMSWAP)
    358 			/*
    359 			 * No page, therefore allocate one.
    360 			 */
    361 
    362 			pg = uvm_pagealloc(NULL,
    363 			    ufi != NULL ? ufi->orig_rvaddr : 0,
    364 			    anon, ufi != NULL ? UVM_FLAG_COLORMATCH : 0);
    365 			if (pg == NULL) {
    366 				/* Out of memory.  Wait a little. */
    367 				uvmfault_unlockall(ufi, amap, NULL);
    368 				cpu_count(CPU_COUNT_FLTNORAM, 1);
    369 				UVMHIST_LOG(maphist, "  noram -- UVM_WAIT",0,
    370 				    0,0,0);
    371 				if (!uvm_reclaimable()) {
    372 					return ENOMEM;
    373 				}
    374 				uvm_wait("flt_noram1");
    375 			} else {
    376 				/* PG_BUSY bit is set. */
    377 				we_own = true;
    378 				uvmfault_unlockall(ufi, amap, NULL);
    379 
    380 				/*
    381 				 * Pass a PG_BUSY+PG_FAKE clean page into
    382 				 * the uvm_swap_get() function with all data
    383 				 * structures unlocked.  Note that it is OK
    384 				 * to read an_swslot here, because we hold
    385 				 * PG_BUSY on the page.
    386 				 */
    387 				cpu_count(CPU_COUNT_PAGEINS, 1);
    388 				error = uvm_swap_get(pg, anon->an_swslot,
    389 				    PGO_SYNCIO);
    390 
    391 				/*
    392 				 * We clean up after the I/O below in the
    393 				 * 'we_own' case.
    394 				 */
    395 			}
    396 #else
    397 			panic("%s: no page", __func__);
    398 #endif /* defined(VMSWAP) */
    399 		}
    400 
    401 		/*
    402 		 * Re-lock the map and anon.
    403 		 */
    404 
    405 		locked = uvmfault_relock(ufi);
    406 		if (locked || we_own) {
    407 			mutex_enter(anon->an_lock);
    408 		}
    409 
    410 		/*
    411 		 * If we own the page (i.e. we set PG_BUSY), then we need
    412 		 * to clean up after the I/O.  There are three cases to
    413 		 * consider:
    414 		 *
    415 		 * 1) Page was released during I/O: free anon and ReFault.
    416 		 * 2) I/O not OK.  Free the page and cause the fault to fail.
    417 		 * 3) I/O OK!  Activate the page and sync with the non-we_own
    418 		 *    case (i.e. drop anon lock if not locked).
    419 		 */
    420 
    421 		if (we_own) {
    422 #if defined(VMSWAP)
    423 			if (pg->flags & PG_WANTED) {
    424 				wakeup(pg);
    425 			}
    426 			if (error) {
    427 
    428 				/*
    429 				 * Remove the swap slot from the anon and
    430 				 * mark the anon as having no real slot.
    431 				 * Do not free the swap slot, thus preventing
    432 				 * it from being used again.
    433 				 */
    434 
    435 				if (anon->an_swslot > 0) {
    436 					uvm_swap_markbad(anon->an_swslot, 1);
    437 				}
    438 				anon->an_swslot = SWSLOT_BAD;
    439 
    440 				if ((pg->flags & PG_RELEASED) != 0) {
    441 					goto released;
    442 				}
    443 
    444 				/*
    445 				 * Note: page was never !PG_BUSY, so it
    446 				 * cannot be mapped and thus no need to
    447 				 * pmap_page_protect() it.
    448 				 */
    449 
    450 				uvm_pagefree(pg);
    451 
    452 				if (locked) {
    453 					uvmfault_unlockall(ufi, NULL, NULL);
    454 				}
    455 				mutex_exit(anon->an_lock);
    456 				UVMHIST_LOG(maphist, "<- ERROR", 0,0,0,0);
    457 				return error;
    458 			}
    459 
    460 			if ((pg->flags & PG_RELEASED) != 0) {
    461 released:
    462 				KASSERT(anon->an_ref == 0);
    463 
    464 				/*
    465 				 * Released while we had unlocked amap.
    466 				 */
    467 
    468 				if (locked) {
    469 					uvmfault_unlockall(ufi, NULL, NULL);
    470 				}
    471 				uvm_anon_release(anon);
    472 
    473 				if (error) {
    474 					UVMHIST_LOG(maphist,
    475 					    "<- ERROR/RELEASED", 0,0,0,0);
    476 					return error;
    477 				}
    478 
    479 				UVMHIST_LOG(maphist, "<- RELEASED", 0,0,0,0);
    480 				return ERESTART;
    481 			}
    482 
    483 			/*
    484 			 * We have successfully read the page, activate it.
    485 			 */
    486 
    487 			uvm_pagelock(pg);
    488 			uvm_pageactivate(pg);
    489 			uvm_pageunlock(pg);
    490 			pg->flags &= ~(PG_WANTED|PG_BUSY|PG_FAKE);
    491 			uvm_pagemarkdirty(pg, UVM_PAGE_STATUS_UNKNOWN);
    492 			UVM_PAGE_OWN(pg, NULL);
    493 #else
    494 			panic("%s: we_own", __func__);
    495 #endif /* defined(VMSWAP) */
    496 		}
    497 
    498 		/*
    499 		 * We were not able to re-lock the map - restart the fault.
    500 		 */
    501 
    502 		if (!locked) {
    503 			if (we_own) {
    504 				mutex_exit(anon->an_lock);
    505 			}
    506 			UVMHIST_LOG(maphist, "<- REFAULT", 0,0,0,0);
    507 			return ERESTART;
    508 		}
    509 
    510 		/*
    511 		 * Verify that no one has touched the amap and moved
    512 		 * the anon on us.
    513 		 */
    514 
    515 		if (ufi != NULL && amap_lookup(&ufi->entry->aref,
    516 		    ufi->orig_rvaddr - ufi->entry->start) != anon) {
    517 
    518 			uvmfault_unlockall(ufi, amap, NULL);
    519 			UVMHIST_LOG(maphist, "<- REFAULT", 0,0,0,0);
    520 			return ERESTART;
    521 		}
    522 
    523 		/*
    524 		 * Retry..
    525 		 */
    526 
    527 		cpu_count(CPU_COUNT_FLTANRETRY, 1);
    528 		continue;
    529 	}
    530 	/*NOTREACHED*/
    531 }
    532 
    533 /*
    534  * uvmfault_promote: promote data to a new anon.  used for 1B and 2B.
    535  *
    536  *	1. allocate an anon and a page.
    537  *	2. fill its contents.
    538  *	3. put it into amap.
    539  *
    540  * => if we fail (result != 0) we unlock everything.
    541  * => on success, return a new locked anon via 'nanon'.
    542  *    (*nanon)->an_page will be a resident, locked, dirty page.
    543  * => it's caller's responsibility to put the promoted nanon->an_page to the
    544  *    page queue.
    545  */
    546 
    547 static int
    548 uvmfault_promote(struct uvm_faultinfo *ufi,
    549     struct vm_anon *oanon,
    550     struct vm_page *uobjpage,
    551     struct vm_anon **nanon, /* OUT: allocated anon */
    552     struct vm_anon **spare)
    553 {
    554 	struct vm_amap *amap = ufi->entry->aref.ar_amap;
    555 	struct uvm_object *uobj;
    556 	struct vm_anon *anon;
    557 	struct vm_page *pg;
    558 	struct vm_page *opg;
    559 	int error;
    560 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
    561 
    562 	if (oanon) {
    563 		/* anon COW */
    564 		opg = oanon->an_page;
    565 		KASSERT(opg != NULL);
    566 		KASSERT(opg->uobject == NULL || opg->loan_count > 0);
    567 	} else if (uobjpage != PGO_DONTCARE) {
    568 		/* object-backed COW */
    569 		opg = uobjpage;
    570 	} else {
    571 		/* ZFOD */
    572 		opg = NULL;
    573 	}
    574 	if (opg != NULL) {
    575 		uobj = opg->uobject;
    576 	} else {
    577 		uobj = NULL;
    578 	}
    579 
    580 	KASSERT(amap != NULL);
    581 	KASSERT(uobjpage != NULL);
    582 	KASSERT(uobjpage == PGO_DONTCARE || (uobjpage->flags & PG_BUSY) != 0);
    583 	KASSERT(mutex_owned(amap->am_lock));
    584 	KASSERT(oanon == NULL || amap->am_lock == oanon->an_lock);
    585 	KASSERT(uobj == NULL || mutex_owned(uobj->vmobjlock));
    586 
    587 	if (*spare != NULL) {
    588 		anon = *spare;
    589 		*spare = NULL;
    590 	} else {
    591 		anon = uvm_analloc();
    592 	}
    593 	if (anon) {
    594 
    595 		/*
    596 		 * The new anon is locked.
    597 		 *
    598 		 * if opg == NULL, we want a zero'd, dirty page,
    599 		 * so have uvm_pagealloc() do that for us.
    600 		 */
    601 
    602 		KASSERT(anon->an_lock == NULL);
    603 		anon->an_lock = amap->am_lock;
    604 		pg = uvm_pagealloc(NULL, ufi->orig_rvaddr, anon,
    605 		    UVM_FLAG_COLORMATCH | (opg == NULL ? UVM_PGA_ZERO : 0));
    606 		if (pg == NULL) {
    607 			anon->an_lock = NULL;
    608 		}
    609 	} else {
    610 		pg = NULL;
    611 	}
    612 
    613 	/*
    614 	 * out of memory resources?
    615 	 */
    616 
    617 	if (pg == NULL) {
    618 		/* save anon for the next try. */
    619 		if (anon != NULL) {
    620 			*spare = anon;
    621 		}
    622 
    623 		/* unlock and fail ... */
    624 		uvm_page_unbusy(&uobjpage, 1);
    625 		uvmfault_unlockall(ufi, amap, uobj);
    626 		if (!uvm_reclaimable()) {
    627 			UVMHIST_LOG(maphist, "out of VM", 0,0,0,0);
    628 			cpu_count(CPU_COUNT_FLTNOANON, 1);
    629 			error = ENOMEM;
    630 			goto done;
    631 		}
    632 
    633 		UVMHIST_LOG(maphist, "out of RAM, waiting for more", 0,0,0,0);
    634 		cpu_count(CPU_COUNT_FLTNORAM, 1);
    635 		uvm_wait("flt_noram5");
    636 		error = ERESTART;
    637 		goto done;
    638 	}
    639 
    640 	/* copy page [pg now dirty] */
    641 	if (opg) {
    642 		uvm_pagecopy(opg, pg);
    643 	}
    644 	KASSERT(uvm_pagegetdirty(pg) == UVM_PAGE_STATUS_DIRTY);
    645 
    646 	amap_add(&ufi->entry->aref, ufi->orig_rvaddr - ufi->entry->start, anon,
    647 	    oanon != NULL);
    648 
    649 	*nanon = anon;
    650 	error = 0;
    651 done:
    652 	return error;
    653 }
    654 
    655 /*
    656  * Update statistics after fault resolution.
    657  * - maxrss
    658  */
    659 void
    660 uvmfault_update_stats(struct uvm_faultinfo *ufi)
    661 {
    662 	struct vm_map		*map;
    663 	struct vmspace 		*vm;
    664 	struct proc		*p;
    665 	vsize_t			 res;
    666 
    667 	map = ufi->orig_map;
    668 
    669 	p = curproc;
    670 	KASSERT(p != NULL);
    671 	vm = p->p_vmspace;
    672 
    673 	if (&vm->vm_map != map)
    674 		return;
    675 
    676 	res = pmap_resident_count(map->pmap);
    677 	if (vm->vm_rssmax < res)
    678 		vm->vm_rssmax = res;
    679 }
    680 
    681 /*
    682  *   F A U L T   -   m a i n   e n t r y   p o i n t
    683  */
    684 
    685 /*
    686  * uvm_fault: page fault handler
    687  *
    688  * => called from MD code to resolve a page fault
    689  * => VM data structures usually should be unlocked.   however, it is
    690  *	possible to call here with the main map locked if the caller
    691  *	gets a write lock, sets it recusive, and then calls us (c.f.
    692  *	uvm_map_pageable).   this should be avoided because it keeps
    693  *	the map locked off during I/O.
    694  * => MUST NEVER BE CALLED IN INTERRUPT CONTEXT
    695  */
    696 
    697 #define MASK(entry)     (UVM_ET_ISCOPYONWRITE(entry) ? \
    698 			 ~VM_PROT_WRITE : VM_PROT_ALL)
    699 
    700 /* fault_flag values passed from uvm_fault_wire to uvm_fault_internal */
    701 #define UVM_FAULT_WIRE		(1 << 0)
    702 #define UVM_FAULT_MAXPROT	(1 << 1)
    703 
    704 struct uvm_faultctx {
    705 
    706 	/*
    707 	 * the following members are set up by uvm_fault_check() and
    708 	 * read-only after that.
    709 	 *
    710 	 * note that narrow is used by uvm_fault_check() to change
    711 	 * the behaviour after ERESTART.
    712 	 *
    713 	 * most of them might change after RESTART if the underlying
    714 	 * map entry has been changed behind us.  an exception is
    715 	 * wire_paging, which does never change.
    716 	 */
    717 	vm_prot_t access_type;
    718 	vaddr_t startva;
    719 	int npages;
    720 	int centeridx;
    721 	bool narrow;		/* work on a single requested page only */
    722 	bool wire_mapping;	/* request a PMAP_WIRED mapping
    723 				   (UVM_FAULT_WIRE or VM_MAPENT_ISWIRED) */
    724 	bool wire_paging;	/* request uvm_pagewire
    725 				   (true for UVM_FAULT_WIRE) */
    726 	bool cow_now;		/* VM_PROT_WRITE is actually requested
    727 				   (ie. should break COW and page loaning) */
    728 
    729 	/*
    730 	 * enter_prot is set up by uvm_fault_check() and clamped
    731 	 * (ie. drop the VM_PROT_WRITE bit) in various places in case
    732 	 * of !cow_now.
    733 	 */
    734 	vm_prot_t enter_prot;	/* prot at which we want to enter pages in */
    735 
    736 	/*
    737 	 * the following member is for uvmfault_promote() and ERESTART.
    738 	 */
    739 	struct vm_anon *anon_spare;
    740 
    741 	/*
    742 	 * the folloing is actually a uvm_fault_lower() internal.
    743 	 * it's here merely for debugging.
    744 	 * (or due to the mechanical separation of the function?)
    745 	 */
    746 	bool promote;
    747 };
    748 
    749 static inline int	uvm_fault_check(
    750 			    struct uvm_faultinfo *, struct uvm_faultctx *,
    751 			    struct vm_anon ***, bool);
    752 
    753 static int		uvm_fault_upper(
    754 			    struct uvm_faultinfo *, struct uvm_faultctx *,
    755 			    struct vm_anon **);
    756 static inline int	uvm_fault_upper_lookup(
    757 			    struct uvm_faultinfo *, const struct uvm_faultctx *,
    758 			    struct vm_anon **, struct vm_page **);
    759 static inline void	uvm_fault_upper_neighbor(
    760 			    struct uvm_faultinfo *, const struct uvm_faultctx *,
    761 			    vaddr_t, struct vm_page *, bool);
    762 static inline int	uvm_fault_upper_loan(
    763 			    struct uvm_faultinfo *, struct uvm_faultctx *,
    764 			    struct vm_anon *, struct uvm_object **);
    765 static inline int	uvm_fault_upper_promote(
    766 			    struct uvm_faultinfo *, struct uvm_faultctx *,
    767 			    struct uvm_object *, struct vm_anon *);
    768 static inline int	uvm_fault_upper_direct(
    769 			    struct uvm_faultinfo *, struct uvm_faultctx *,
    770 			    struct uvm_object *, struct vm_anon *);
    771 static int		uvm_fault_upper_enter(
    772 			    struct uvm_faultinfo *, const struct uvm_faultctx *,
    773 			    struct uvm_object *, struct vm_anon *,
    774 			    struct vm_page *, struct vm_anon *);
    775 static inline void	uvm_fault_upper_done(
    776 			    struct uvm_faultinfo *, const struct uvm_faultctx *,
    777 			    struct vm_anon *, struct vm_page *);
    778 
    779 static int		uvm_fault_lower(
    780 			    struct uvm_faultinfo *, struct uvm_faultctx *,
    781 			    struct vm_page **);
    782 static inline void	uvm_fault_lower_lookup(
    783 			    struct uvm_faultinfo *, const struct uvm_faultctx *,
    784 			    struct vm_page **);
    785 static inline void	uvm_fault_lower_neighbor(
    786 			    struct uvm_faultinfo *, const struct uvm_faultctx *,
    787 			    vaddr_t, struct vm_page *);
    788 static inline int	uvm_fault_lower_io(
    789 			    struct uvm_faultinfo *, const struct uvm_faultctx *,
    790 			    struct uvm_object **, struct vm_page **);
    791 static inline int	uvm_fault_lower_direct(
    792 			    struct uvm_faultinfo *, struct uvm_faultctx *,
    793 			    struct uvm_object *, struct vm_page *);
    794 static inline int	uvm_fault_lower_direct_loan(
    795 			    struct uvm_faultinfo *, struct uvm_faultctx *,
    796 			    struct uvm_object *, struct vm_page **,
    797 			    struct vm_page **);
    798 static inline int	uvm_fault_lower_promote(
    799 			    struct uvm_faultinfo *, struct uvm_faultctx *,
    800 			    struct uvm_object *, struct vm_page *);
    801 static int		uvm_fault_lower_enter(
    802 			    struct uvm_faultinfo *, const struct uvm_faultctx *,
    803 			    struct uvm_object *,
    804 			    struct vm_anon *, struct vm_page *);
    805 static inline void	uvm_fault_lower_done(
    806 			    struct uvm_faultinfo *, const struct uvm_faultctx *,
    807 			    struct uvm_object *, struct vm_page *);
    808 
    809 int
    810 uvm_fault_internal(struct vm_map *orig_map, vaddr_t vaddr,
    811     vm_prot_t access_type, int fault_flag)
    812 {
    813 	struct uvm_faultinfo ufi;
    814 	struct uvm_faultctx flt = {
    815 		.access_type = access_type,
    816 
    817 		/* don't look for neighborhood * pages on "wire" fault */
    818 		.narrow = (fault_flag & UVM_FAULT_WIRE) != 0,
    819 
    820 		/* "wire" fault causes wiring of both mapping and paging */
    821 		.wire_mapping = (fault_flag & UVM_FAULT_WIRE) != 0,
    822 		.wire_paging = (fault_flag & UVM_FAULT_WIRE) != 0,
    823 	};
    824 	const bool maxprot = (fault_flag & UVM_FAULT_MAXPROT) != 0;
    825 	struct vm_anon *anons_store[UVM_MAXRANGE], **anons;
    826 	struct vm_page *pages_store[UVM_MAXRANGE], **pages;
    827 	int error;
    828 
    829 	UVMHIST_FUNC("uvm_fault"); UVMHIST_CALLED(maphist);
    830 
    831 	UVMHIST_LOG(maphist, "(map=%#jx, vaddr=%#jx, at=%jd, ff=%jd)",
    832 	      (uintptr_t)orig_map, vaddr, access_type, fault_flag);
    833 
    834 	/* Don't count anything until user interaction is possible */
    835 	kpreempt_disable();
    836 	if (__predict_true(start_init_exec)) {
    837 		struct cpu_info *ci = curcpu();
    838 		CPU_COUNT(CPU_COUNT_NFAULT, 1);
    839 		/* Don't flood RNG subsystem with samples. */
    840 		if (++(ci->ci_faultrng) == 503) {
    841 			ci->ci_faultrng = 0;
    842 			rnd_add_uint32(&curcpu()->ci_data.cpu_uvm->rs,
    843 			    sizeof(vaddr_t) == sizeof(uint32_t) ?
    844 			    (uint32_t)vaddr : sizeof(vaddr_t) ==
    845 			    sizeof(uint64_t) ?
    846 			    (uint32_t)vaddr :
    847 			    (uint32_t)ci->ci_counts[CPU_COUNT_NFAULT]);
    848 		}
    849 	}
    850 	kpreempt_enable();
    851 
    852 	/*
    853 	 * init the IN parameters in the ufi
    854 	 */
    855 
    856 	ufi.orig_map = orig_map;
    857 	ufi.orig_rvaddr = trunc_page(vaddr);
    858 	ufi.orig_size = PAGE_SIZE;	/* can't get any smaller than this */
    859 
    860 	error = ERESTART;
    861 	while (error == ERESTART) { /* ReFault: */
    862 		anons = anons_store;
    863 		pages = pages_store;
    864 
    865 		error = uvm_fault_check(&ufi, &flt, &anons, maxprot);
    866 		if (error != 0)
    867 			continue;
    868 
    869 		error = uvm_fault_upper_lookup(&ufi, &flt, anons, pages);
    870 		if (error != 0)
    871 			continue;
    872 
    873 		if (pages[flt.centeridx] == PGO_DONTCARE)
    874 			error = uvm_fault_upper(&ufi, &flt, anons);
    875 		else {
    876 			struct uvm_object * const uobj =
    877 			    ufi.entry->object.uvm_obj;
    878 
    879 			if (uobj && uobj->pgops->pgo_fault != NULL) {
    880 				/*
    881 				 * invoke "special" fault routine.
    882 				 */
    883 				mutex_enter(uobj->vmobjlock);
    884 				/* locked: maps(read), amap(if there), uobj */
    885 				error = uobj->pgops->pgo_fault(&ufi,
    886 				    flt.startva, pages, flt.npages,
    887 				    flt.centeridx, flt.access_type,
    888 				    PGO_LOCKED|PGO_SYNCIO);
    889 
    890 				/*
    891 				 * locked: nothing, pgo_fault has unlocked
    892 				 * everything
    893 				 */
    894 
    895 				/*
    896 				 * object fault routine responsible for
    897 				 * pmap_update().
    898 				 */
    899 
    900 				/*
    901 				 * Wake up the pagedaemon if the fault method
    902 				 * failed for lack of memory but some can be
    903 				 * reclaimed.
    904 				 */
    905 				if (error == ENOMEM && uvm_reclaimable()) {
    906 					uvm_wait("pgo_fault");
    907 					error = ERESTART;
    908 				}
    909 			} else {
    910 				error = uvm_fault_lower(&ufi, &flt, pages);
    911 			}
    912 		}
    913 	}
    914 
    915 	if (flt.anon_spare != NULL) {
    916 		flt.anon_spare->an_ref--;
    917 		KASSERT(flt.anon_spare->an_ref == 0);
    918 		KASSERT(flt.anon_spare->an_lock == NULL);
    919 		uvm_anon_free(flt.anon_spare);
    920 	}
    921 	return error;
    922 }
    923 
    924 /*
    925  * uvm_fault_check: check prot, handle needs-copy, etc.
    926  *
    927  *	1. lookup entry.
    928  *	2. check protection.
    929  *	3. adjust fault condition (mainly for simulated fault).
    930  *	4. handle needs-copy (lazy amap copy).
    931  *	5. establish range of interest for neighbor fault (aka pre-fault).
    932  *	6. look up anons (if amap exists).
    933  *	7. flush pages (if MADV_SEQUENTIAL)
    934  *
    935  * => called with nothing locked.
    936  * => if we fail (result != 0) we unlock everything.
    937  * => initialize/adjust many members of flt.
    938  */
    939 
    940 static int
    941 uvm_fault_check(
    942 	struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
    943 	struct vm_anon ***ranons, bool maxprot)
    944 {
    945 	struct vm_amap *amap;
    946 	struct uvm_object *uobj;
    947 	vm_prot_t check_prot;
    948 	int nback, nforw;
    949 	UVMHIST_FUNC("uvm_fault_check"); UVMHIST_CALLED(maphist);
    950 
    951 	/*
    952 	 * lookup and lock the maps
    953 	 */
    954 
    955 	if (uvmfault_lookup(ufi, false) == false) {
    956 		UVMHIST_LOG(maphist, "<- no mapping @ 0x%#jx", ufi->orig_rvaddr,
    957 		    0,0,0);
    958 		return EFAULT;
    959 	}
    960 	/* locked: maps(read) */
    961 
    962 #ifdef DIAGNOSTIC
    963 	if ((ufi->map->flags & VM_MAP_PAGEABLE) == 0) {
    964 		printf("Page fault on non-pageable map:\n");
    965 		printf("ufi->map = %p\n", ufi->map);
    966 		printf("ufi->orig_map = %p\n", ufi->orig_map);
    967 		printf("ufi->orig_rvaddr = 0x%lx\n", (u_long) ufi->orig_rvaddr);
    968 		panic("uvm_fault: (ufi->map->flags & VM_MAP_PAGEABLE) == 0");
    969 	}
    970 #endif
    971 
    972 	/*
    973 	 * check protection
    974 	 */
    975 
    976 	check_prot = maxprot ?
    977 	    ufi->entry->max_protection : ufi->entry->protection;
    978 	if ((check_prot & flt->access_type) != flt->access_type) {
    979 		UVMHIST_LOG(maphist,
    980 		    "<- protection failure (prot=%#jx, access=%#jx)",
    981 		    ufi->entry->protection, flt->access_type, 0, 0);
    982 		uvmfault_unlockmaps(ufi, false);
    983 		return EFAULT;
    984 	}
    985 
    986 	/*
    987 	 * "enter_prot" is the protection we want to enter the page in at.
    988 	 * for certain pages (e.g. copy-on-write pages) this protection can
    989 	 * be more strict than ufi->entry->protection.  "wired" means either
    990 	 * the entry is wired or we are fault-wiring the pg.
    991 	 */
    992 
    993 	flt->enter_prot = ufi->entry->protection;
    994 	if (VM_MAPENT_ISWIRED(ufi->entry)) {
    995 		flt->wire_mapping = true;
    996 		flt->wire_paging = true;
    997 		flt->narrow = true;
    998 	}
    999 
   1000 	if (flt->wire_mapping) {
   1001 		flt->access_type = flt->enter_prot; /* full access for wired */
   1002 		flt->cow_now = (check_prot & VM_PROT_WRITE) != 0;
   1003 	} else {
   1004 		flt->cow_now = (flt->access_type & VM_PROT_WRITE) != 0;
   1005 	}
   1006 
   1007 	flt->promote = false;
   1008 
   1009 	/*
   1010 	 * handle "needs_copy" case.   if we need to copy the amap we will
   1011 	 * have to drop our readlock and relock it with a write lock.  (we
   1012 	 * need a write lock to change anything in a map entry [e.g.
   1013 	 * needs_copy]).
   1014 	 */
   1015 
   1016 	if (UVM_ET_ISNEEDSCOPY(ufi->entry)) {
   1017 		if (flt->cow_now || (ufi->entry->object.uvm_obj == NULL)) {
   1018 			KASSERT(!maxprot);
   1019 			/* need to clear */
   1020 			UVMHIST_LOG(maphist,
   1021 			    "  need to clear needs_copy and refault",0,0,0,0);
   1022 			uvmfault_unlockmaps(ufi, false);
   1023 			uvmfault_amapcopy(ufi);
   1024 			cpu_count(CPU_COUNT_FLTAMCOPY, 1);
   1025 			return ERESTART;
   1026 
   1027 		} else {
   1028 
   1029 			/*
   1030 			 * ensure that we pmap_enter page R/O since
   1031 			 * needs_copy is still true
   1032 			 */
   1033 
   1034 			flt->enter_prot &= ~VM_PROT_WRITE;
   1035 		}
   1036 	}
   1037 
   1038 	/*
   1039 	 * identify the players
   1040 	 */
   1041 
   1042 	amap = ufi->entry->aref.ar_amap;	/* upper layer */
   1043 	uobj = ufi->entry->object.uvm_obj;	/* lower layer */
   1044 
   1045 	/*
   1046 	 * check for a case 0 fault.  if nothing backing the entry then
   1047 	 * error now.
   1048 	 */
   1049 
   1050 	if (amap == NULL && uobj == NULL) {
   1051 		uvmfault_unlockmaps(ufi, false);
   1052 		UVMHIST_LOG(maphist,"<- no backing store, no overlay",0,0,0,0);
   1053 		return EFAULT;
   1054 	}
   1055 
   1056 	/*
   1057 	 * establish range of interest based on advice from mapper
   1058 	 * and then clip to fit map entry.   note that we only want
   1059 	 * to do this the first time through the fault.   if we
   1060 	 * ReFault we will disable this by setting "narrow" to true.
   1061 	 */
   1062 
   1063 	if (flt->narrow == false) {
   1064 
   1065 		/* wide fault (!narrow) */
   1066 		KASSERT(uvmadvice[ufi->entry->advice].advice ==
   1067 			 ufi->entry->advice);
   1068 		nback = MIN(uvmadvice[ufi->entry->advice].nback,
   1069 		    (ufi->orig_rvaddr - ufi->entry->start) >> PAGE_SHIFT);
   1070 		flt->startva = ufi->orig_rvaddr - (nback << PAGE_SHIFT);
   1071 		/*
   1072 		 * note: "-1" because we don't want to count the
   1073 		 * faulting page as forw
   1074 		 */
   1075 		nforw = MIN(uvmadvice[ufi->entry->advice].nforw,
   1076 			    ((ufi->entry->end - ufi->orig_rvaddr) >>
   1077 			     PAGE_SHIFT) - 1);
   1078 		flt->npages = nback + nforw + 1;
   1079 		flt->centeridx = nback;
   1080 
   1081 		flt->narrow = true;	/* ensure only once per-fault */
   1082 
   1083 	} else {
   1084 
   1085 		/* narrow fault! */
   1086 		nback = nforw = 0;
   1087 		flt->startva = ufi->orig_rvaddr;
   1088 		flt->npages = 1;
   1089 		flt->centeridx = 0;
   1090 
   1091 	}
   1092 	/* offset from entry's start to pgs' start */
   1093 	const voff_t eoff = flt->startva - ufi->entry->start;
   1094 
   1095 	/* locked: maps(read) */
   1096 	UVMHIST_LOG(maphist, "  narrow=%jd, back=%jd, forw=%jd, startva=%#jx",
   1097 		    flt->narrow, nback, nforw, flt->startva);
   1098 	UVMHIST_LOG(maphist, "  entry=%#jx, amap=%#jx, obj=%#jx",
   1099 	    (uintptr_t)ufi->entry, (uintptr_t)amap, (uintptr_t)uobj, 0);
   1100 
   1101 	/*
   1102 	 * if we've got an amap, lock it and extract current anons.
   1103 	 */
   1104 
   1105 	if (amap) {
   1106 		amap_lock(amap);
   1107 		amap_lookups(&ufi->entry->aref, eoff, *ranons, flt->npages);
   1108 	} else {
   1109 		*ranons = NULL;	/* to be safe */
   1110 	}
   1111 
   1112 	/* locked: maps(read), amap(if there) */
   1113 	KASSERT(amap == NULL || mutex_owned(amap->am_lock));
   1114 
   1115 	/*
   1116 	 * for MADV_SEQUENTIAL mappings we want to deactivate the back pages
   1117 	 * now and then forget about them (for the rest of the fault).
   1118 	 */
   1119 
   1120 	if (ufi->entry->advice == MADV_SEQUENTIAL && nback != 0) {
   1121 
   1122 		UVMHIST_LOG(maphist, "  MADV_SEQUENTIAL: flushing backpages",
   1123 		    0,0,0,0);
   1124 		/* flush back-page anons? */
   1125 		if (amap)
   1126 			uvmfault_anonflush(*ranons, nback);
   1127 
   1128 		/* flush object? */
   1129 		if (uobj) {
   1130 			voff_t uoff;
   1131 
   1132 			uoff = ufi->entry->offset + eoff;
   1133 			mutex_enter(uobj->vmobjlock);
   1134 			(void) (uobj->pgops->pgo_put)(uobj, uoff, uoff +
   1135 				    (nback << PAGE_SHIFT), PGO_DEACTIVATE);
   1136 		}
   1137 
   1138 		/* now forget about the backpages */
   1139 		if (amap)
   1140 			*ranons += nback;
   1141 		flt->startva += (nback << PAGE_SHIFT);
   1142 		flt->npages -= nback;
   1143 		flt->centeridx = 0;
   1144 	}
   1145 	/*
   1146 	 * => startva is fixed
   1147 	 * => npages is fixed
   1148 	 */
   1149 	KASSERT(flt->startva <= ufi->orig_rvaddr);
   1150 	KASSERT(ufi->orig_rvaddr + ufi->orig_size <=
   1151 	    flt->startva + (flt->npages << PAGE_SHIFT));
   1152 	return 0;
   1153 }
   1154 
   1155 /*
   1156  * uvm_fault_upper_lookup: look up existing h/w mapping and amap.
   1157  *
   1158  * iterate range of interest:
   1159  *	1. check if h/w mapping exists.  if yes, we don't care
   1160  *	2. check if anon exists.  if not, page is lower.
   1161  *	3. if anon exists, enter h/w mapping for neighbors.
   1162  *
   1163  * => called with amap locked (if exists).
   1164  */
   1165 
   1166 static int
   1167 uvm_fault_upper_lookup(
   1168 	struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt,
   1169 	struct vm_anon **anons, struct vm_page **pages)
   1170 {
   1171 	struct vm_amap *amap = ufi->entry->aref.ar_amap;
   1172 	int lcv;
   1173 	vaddr_t currva;
   1174 	bool shadowed __unused;
   1175 	UVMHIST_FUNC("uvm_fault_upper_lookup"); UVMHIST_CALLED(maphist);
   1176 
   1177 	/* locked: maps(read), amap(if there) */
   1178 	KASSERT(amap == NULL || mutex_owned(amap->am_lock));
   1179 
   1180 	/*
   1181 	 * map in the backpages and frontpages we found in the amap in hopes
   1182 	 * of preventing future faults.    we also init the pages[] array as
   1183 	 * we go.
   1184 	 */
   1185 
   1186 	currva = flt->startva;
   1187 	shadowed = false;
   1188 	for (lcv = 0; lcv < flt->npages; lcv++, currva += PAGE_SIZE) {
   1189 		/*
   1190 		 * don't play with VAs that are already mapped
   1191 		 * (except for center)
   1192 		 */
   1193 		if (lcv != flt->centeridx &&
   1194 		    pmap_extract(ufi->orig_map->pmap, currva, NULL)) {
   1195 			pages[lcv] = PGO_DONTCARE;
   1196 			continue;
   1197 		}
   1198 
   1199 		/*
   1200 		 * unmapped or center page.   check if any anon at this level.
   1201 		 */
   1202 		if (amap == NULL || anons[lcv] == NULL) {
   1203 			pages[lcv] = NULL;
   1204 			continue;
   1205 		}
   1206 
   1207 		/*
   1208 		 * check for present page and map if possible.   re-activate it.
   1209 		 */
   1210 
   1211 		pages[lcv] = PGO_DONTCARE;
   1212 		if (lcv == flt->centeridx) {	/* save center for later! */
   1213 			shadowed = true;
   1214 			continue;
   1215 		}
   1216 
   1217 		struct vm_anon *anon = anons[lcv];
   1218 		struct vm_page *pg = anon->an_page;
   1219 
   1220 		KASSERT(anon->an_lock == amap->am_lock);
   1221 
   1222 		/* Ignore loaned and busy pages. */
   1223 		if (pg && pg->loan_count == 0 && (pg->flags & PG_BUSY) == 0) {
   1224 			uvm_fault_upper_neighbor(ufi, flt, currva,
   1225 			    pg, anon->an_ref > 1);
   1226 		}
   1227 	}
   1228 
   1229 	/* locked: maps(read), amap(if there) */
   1230 	KASSERT(amap == NULL || mutex_owned(amap->am_lock));
   1231 	/* (shadowed == true) if there is an anon at the faulting address */
   1232 	UVMHIST_LOG(maphist, "  shadowed=%jd, will_get=%jd", shadowed,
   1233 	    (ufi->entry->object.uvm_obj && shadowed != false),0,0);
   1234 
   1235 	/*
   1236 	 * note that if we are really short of RAM we could sleep in the above
   1237 	 * call to pmap_enter with everything locked.   bad?
   1238 	 *
   1239 	 * XXX Actually, that is bad; pmap_enter() should just fail in that
   1240 	 * XXX case.  --thorpej
   1241 	 */
   1242 
   1243 	return 0;
   1244 }
   1245 
   1246 /*
   1247  * uvm_fault_upper_neighbor: enter single upper neighbor page.
   1248  *
   1249  * => called with amap and anon locked.
   1250  */
   1251 
   1252 static void
   1253 uvm_fault_upper_neighbor(
   1254 	struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt,
   1255 	vaddr_t currva, struct vm_page *pg, bool readonly)
   1256 {
   1257 	UVMHIST_FUNC("uvm_fault_upper_neighbor"); UVMHIST_CALLED(maphist);
   1258 
   1259 	/* locked: amap, anon */
   1260 
   1261 	KASSERT(pg->uobject == NULL);
   1262 	KASSERT(pg->uanon != NULL);
   1263 	KASSERT(mutex_owned(pg->uanon->an_lock));
   1264 	KASSERT(uvm_pagegetdirty(pg) != UVM_PAGE_STATUS_CLEAN);
   1265 
   1266 	uvm_pagelock(pg);
   1267 	uvm_pageenqueue(pg);
   1268 	uvm_pageunlock(pg);
   1269 	UVMHIST_LOG(maphist,
   1270 	    "  MAPPING: n anon: pm=%#jx, va=%#jx, pg=%#jx",
   1271 	    (uintptr_t)ufi->orig_map->pmap, currva, (uintptr_t)pg, 0);
   1272 	cpu_count(CPU_COUNT_FLTNAMAP, 1);
   1273 
   1274 	/*
   1275 	 * Since this page isn't the page that's actually faulting,
   1276 	 * ignore pmap_enter() failures; it's not critical that we
   1277 	 * enter these right now.
   1278 	 */
   1279 
   1280 	(void) pmap_enter(ufi->orig_map->pmap, currva,
   1281 	    VM_PAGE_TO_PHYS(pg),
   1282 	    readonly ? (flt->enter_prot & ~VM_PROT_WRITE) :
   1283 	    flt->enter_prot,
   1284 	    PMAP_CANFAIL | (flt->wire_mapping ? PMAP_WIRED : 0));
   1285 
   1286 	pmap_update(ufi->orig_map->pmap);
   1287 }
   1288 
   1289 /*
   1290  * uvm_fault_upper: handle upper fault.
   1291  *
   1292  *	1. acquire anon lock.
   1293  *	2. get anon.  let uvmfault_anonget do the dirty work.
   1294  *	3. handle loan.
   1295  *	4. dispatch direct or promote handlers.
   1296  */
   1297 
   1298 static int
   1299 uvm_fault_upper(
   1300 	struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
   1301 	struct vm_anon **anons)
   1302 {
   1303 	struct vm_amap * const amap = ufi->entry->aref.ar_amap;
   1304 	struct vm_anon * const anon = anons[flt->centeridx];
   1305 	struct uvm_object *uobj;
   1306 	int error;
   1307 	UVMHIST_FUNC("uvm_fault_upper"); UVMHIST_CALLED(maphist);
   1308 
   1309 	/* locked: maps(read), amap, anon */
   1310 	KASSERT(mutex_owned(amap->am_lock));
   1311 	KASSERT(anon->an_lock == amap->am_lock);
   1312 
   1313 	/*
   1314 	 * handle case 1: fault on an anon in our amap
   1315 	 */
   1316 
   1317 	UVMHIST_LOG(maphist, "  case 1 fault: anon=%#jx",
   1318 	    (uintptr_t)anon, 0, 0, 0);
   1319 
   1320 	/*
   1321 	 * no matter if we have case 1A or case 1B we are going to need to
   1322 	 * have the anon's memory resident.   ensure that now.
   1323 	 */
   1324 
   1325 	/*
   1326 	 * let uvmfault_anonget do the dirty work.
   1327 	 * if it fails (!OK) it will unlock everything for us.
   1328 	 * if it succeeds, locks are still valid and locked.
   1329 	 * also, if it is OK, then the anon's page is on the queues.
   1330 	 * if the page is on loan from a uvm_object, then anonget will
   1331 	 * lock that object for us if it does not fail.
   1332 	 */
   1333 
   1334 	error = uvmfault_anonget(ufi, amap, anon);
   1335 	switch (error) {
   1336 	case 0:
   1337 		break;
   1338 
   1339 	case ERESTART:
   1340 		return ERESTART;
   1341 
   1342 	case EAGAIN:
   1343 		kpause("fltagain1", false, hz/2, NULL);
   1344 		return ERESTART;
   1345 
   1346 	default:
   1347 		return error;
   1348 	}
   1349 
   1350 	/*
   1351 	 * uobj is non null if the page is on loan from an object (i.e. uobj)
   1352 	 */
   1353 
   1354 	uobj = anon->an_page->uobject;	/* locked by anonget if !NULL */
   1355 
   1356 	/* locked: maps(read), amap, anon, uobj(if one) */
   1357 	KASSERT(mutex_owned(amap->am_lock));
   1358 	KASSERT(anon->an_lock == amap->am_lock);
   1359 	KASSERT(uobj == NULL || mutex_owned(uobj->vmobjlock));
   1360 
   1361 	/*
   1362 	 * special handling for loaned pages
   1363 	 */
   1364 
   1365 	if (anon->an_page->loan_count) {
   1366 		error = uvm_fault_upper_loan(ufi, flt, anon, &uobj);
   1367 		if (error != 0)
   1368 			return error;
   1369 	}
   1370 
   1371 	/*
   1372 	 * if we are case 1B then we will need to allocate a new blank
   1373 	 * anon to transfer the data into.   note that we have a lock
   1374 	 * on anon, so no one can busy or release the page until we are done.
   1375 	 * also note that the ref count can't drop to zero here because
   1376 	 * it is > 1 and we are only dropping one ref.
   1377 	 *
   1378 	 * in the (hopefully very rare) case that we are out of RAM we
   1379 	 * will unlock, wait for more RAM, and refault.
   1380 	 *
   1381 	 * if we are out of anon VM we kill the process (XXX: could wait?).
   1382 	 */
   1383 
   1384 	if (flt->cow_now && anon->an_ref > 1) {
   1385 		flt->promote = true;
   1386 		error = uvm_fault_upper_promote(ufi, flt, uobj, anon);
   1387 	} else {
   1388 		error = uvm_fault_upper_direct(ufi, flt, uobj, anon);
   1389 	}
   1390 	return error;
   1391 }
   1392 
   1393 /*
   1394  * uvm_fault_upper_loan: handle loaned upper page.
   1395  *
   1396  *	1. if not cow'ing now, simply adjust flt->enter_prot.
   1397  *	2. if cow'ing now, and if ref count is 1, break loan.
   1398  */
   1399 
   1400 static int
   1401 uvm_fault_upper_loan(
   1402 	struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
   1403 	struct vm_anon *anon, struct uvm_object **ruobj)
   1404 {
   1405 	struct vm_amap * const amap = ufi->entry->aref.ar_amap;
   1406 	int error = 0;
   1407 	UVMHIST_FUNC("uvm_fault_upper_loan"); UVMHIST_CALLED(maphist);
   1408 
   1409 	if (!flt->cow_now) {
   1410 
   1411 		/*
   1412 		 * for read faults on loaned pages we just cap the
   1413 		 * protection at read-only.
   1414 		 */
   1415 
   1416 		flt->enter_prot = flt->enter_prot & ~VM_PROT_WRITE;
   1417 
   1418 	} else {
   1419 		/*
   1420 		 * note that we can't allow writes into a loaned page!
   1421 		 *
   1422 		 * if we have a write fault on a loaned page in an
   1423 		 * anon then we need to look at the anon's ref count.
   1424 		 * if it is greater than one then we are going to do
   1425 		 * a normal copy-on-write fault into a new anon (this
   1426 		 * is not a problem).  however, if the reference count
   1427 		 * is one (a case where we would normally allow a
   1428 		 * write directly to the page) then we need to kill
   1429 		 * the loan before we continue.
   1430 		 */
   1431 
   1432 		/* >1 case is already ok */
   1433 		if (anon->an_ref == 1) {
   1434 			error = uvm_loanbreak_anon(anon, *ruobj);
   1435 			if (error != 0) {
   1436 				uvmfault_unlockall(ufi, amap, *ruobj);
   1437 				uvm_wait("flt_noram2");
   1438 				return ERESTART;
   1439 			}
   1440 			/* if we were a loan receiver uobj is gone */
   1441 			if (*ruobj)
   1442 				*ruobj = NULL;
   1443 		}
   1444 	}
   1445 	return error;
   1446 }
   1447 
   1448 /*
   1449  * uvm_fault_upper_promote: promote upper page.
   1450  *
   1451  *	1. call uvmfault_promote.
   1452  *	2. enqueue page.
   1453  *	3. deref.
   1454  *	4. pass page to uvm_fault_upper_enter.
   1455  */
   1456 
   1457 static int
   1458 uvm_fault_upper_promote(
   1459 	struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
   1460 	struct uvm_object *uobj, struct vm_anon *anon)
   1461 {
   1462 	struct vm_anon * const oanon = anon;
   1463 	struct vm_page *pg;
   1464 	int error;
   1465 	UVMHIST_FUNC("uvm_fault_upper_promote"); UVMHIST_CALLED(maphist);
   1466 
   1467 	UVMHIST_LOG(maphist, "  case 1B: COW fault",0,0,0,0);
   1468 	cpu_count(CPU_COUNT_FLT_ACOW, 1);
   1469 
   1470 	error = uvmfault_promote(ufi, oanon, PGO_DONTCARE, &anon,
   1471 	    &flt->anon_spare);
   1472 	switch (error) {
   1473 	case 0:
   1474 		break;
   1475 	case ERESTART:
   1476 		return ERESTART;
   1477 	default:
   1478 		return error;
   1479 	}
   1480 
   1481 	KASSERT(anon == NULL || anon->an_lock == oanon->an_lock);
   1482 
   1483 	pg = anon->an_page;
   1484 	/* uvm_fault_upper_done will activate the page */
   1485 	uvm_pagelock(pg);
   1486 	uvm_pageenqueue(pg);
   1487 	uvm_pageunlock(pg);
   1488 	pg->flags &= ~(PG_BUSY|PG_FAKE);
   1489 	UVM_PAGE_OWN(pg, NULL);
   1490 
   1491 	/* deref: can not drop to zero here by defn! */
   1492 	KASSERT(oanon->an_ref > 1);
   1493 	oanon->an_ref--;
   1494 
   1495 	/*
   1496 	 * note: oanon is still locked, as is the new anon.  we
   1497 	 * need to check for this later when we unlock oanon; if
   1498 	 * oanon != anon, we'll have to unlock anon, too.
   1499 	 */
   1500 
   1501 	return uvm_fault_upper_enter(ufi, flt, uobj, anon, pg, oanon);
   1502 }
   1503 
   1504 /*
   1505  * uvm_fault_upper_direct: handle direct fault.
   1506  */
   1507 
   1508 static int
   1509 uvm_fault_upper_direct(
   1510 	struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
   1511 	struct uvm_object *uobj, struct vm_anon *anon)
   1512 {
   1513 	struct vm_anon * const oanon = anon;
   1514 	struct vm_page *pg;
   1515 	UVMHIST_FUNC("uvm_fault_upper_direct"); UVMHIST_CALLED(maphist);
   1516 
   1517 	cpu_count(CPU_COUNT_FLT_ANON, 1);
   1518 	pg = anon->an_page;
   1519 	if (anon->an_ref > 1)     /* disallow writes to ref > 1 anons */
   1520 		flt->enter_prot = flt->enter_prot & ~VM_PROT_WRITE;
   1521 
   1522 	return uvm_fault_upper_enter(ufi, flt, uobj, anon, pg, oanon);
   1523 }
   1524 
   1525 /*
   1526  * uvm_fault_upper_enter: enter h/w mapping of upper page.
   1527  */
   1528 
   1529 static int
   1530 uvm_fault_upper_enter(
   1531 	struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt,
   1532 	struct uvm_object *uobj, struct vm_anon *anon, struct vm_page *pg,
   1533 	struct vm_anon *oanon)
   1534 {
   1535 	struct pmap *pmap = ufi->orig_map->pmap;
   1536 	vaddr_t va = ufi->orig_rvaddr;
   1537 	struct vm_amap * const amap = ufi->entry->aref.ar_amap;
   1538 	UVMHIST_FUNC("uvm_fault_upper_enter"); UVMHIST_CALLED(maphist);
   1539 
   1540 	/* locked: maps(read), amap, oanon, anon(if different from oanon) */
   1541 	KASSERT(mutex_owned(amap->am_lock));
   1542 	KASSERT(anon->an_lock == amap->am_lock);
   1543 	KASSERT(oanon->an_lock == amap->am_lock);
   1544 	KASSERT(uobj == NULL || mutex_owned(uobj->vmobjlock));
   1545 	KASSERT(uvm_pagegetdirty(pg) != UVM_PAGE_STATUS_CLEAN);
   1546 
   1547 	/*
   1548 	 * now map the page in.
   1549 	 */
   1550 
   1551 	UVMHIST_LOG(maphist,
   1552 	    "  MAPPING: anon: pm=%#jx, va=%#jx, pg=%#jx, promote=%jd",
   1553 	    (uintptr_t)pmap, va, (uintptr_t)pg, flt->promote);
   1554 	if (pmap_enter(pmap, va, VM_PAGE_TO_PHYS(pg),
   1555 	    flt->enter_prot, flt->access_type | PMAP_CANFAIL |
   1556 	    (flt->wire_mapping ? PMAP_WIRED : 0)) != 0) {
   1557 
   1558 		/*
   1559 		 * If pmap_enter() fails, it must not leave behind an existing
   1560 		 * pmap entry.  In particular, a now-stale entry for a different
   1561 		 * page would leave the pmap inconsistent with the vm_map.
   1562 		 * This is not to imply that pmap_enter() should remove an
   1563 		 * existing mapping in such a situation (since that could create
   1564 		 * different problems, eg. if the existing mapping is wired),
   1565 		 * but rather that the pmap should be designed such that it
   1566 		 * never needs to fail when the new mapping is replacing an
   1567 		 * existing mapping and the new page has no existing mappings.
   1568 		 */
   1569 
   1570 		KASSERT(!pmap_extract(pmap, va, NULL));
   1571 
   1572 		/*
   1573 		 * No need to undo what we did; we can simply think of
   1574 		 * this as the pmap throwing away the mapping information.
   1575 		 *
   1576 		 * We do, however, have to go through the ReFault path,
   1577 		 * as the map may change while we're asleep.
   1578 		 */
   1579 
   1580 		uvmfault_unlockall(ufi, amap, uobj);
   1581 		if (!uvm_reclaimable()) {
   1582 			UVMHIST_LOG(maphist,
   1583 			    "<- failed.  out of VM",0,0,0,0);
   1584 			/* XXX instrumentation */
   1585 			return ENOMEM;
   1586 		}
   1587 		/* XXX instrumentation */
   1588 		uvm_wait("flt_pmfail1");
   1589 		return ERESTART;
   1590 	}
   1591 
   1592 	uvm_fault_upper_done(ufi, flt, anon, pg);
   1593 
   1594 	/*
   1595 	 * done case 1!  finish up by unlocking everything and returning success
   1596 	 */
   1597 
   1598 	pmap_update(pmap);
   1599 	uvmfault_unlockall(ufi, amap, uobj);
   1600 	return 0;
   1601 }
   1602 
   1603 /*
   1604  * uvm_fault_upper_done: queue upper center page.
   1605  */
   1606 
   1607 static void
   1608 uvm_fault_upper_done(
   1609 	struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt,
   1610 	struct vm_anon *anon, struct vm_page *pg)
   1611 {
   1612 	const bool wire_paging = flt->wire_paging;
   1613 
   1614 	UVMHIST_FUNC("uvm_fault_upper_done"); UVMHIST_CALLED(maphist);
   1615 
   1616 	/*
   1617 	 * ... update the page queues.
   1618 	 */
   1619 
   1620 	uvm_pagelock(pg);
   1621 	if (wire_paging) {
   1622 		uvm_pagewire(pg);
   1623 	} else {
   1624 		uvm_pageactivate(pg);
   1625 	}
   1626 	uvm_pageunlock(pg);
   1627 
   1628 	if (wire_paging) {
   1629 		/*
   1630 		 * since the now-wired page cannot be paged out,
   1631 		 * release its swap resources for others to use.
   1632 		 * and since an anon with no swap cannot be clean,
   1633 		 * mark it dirty now.
   1634 		 */
   1635 
   1636 		uvm_pagemarkdirty(pg, UVM_PAGE_STATUS_DIRTY);
   1637 		uvm_anon_dropswap(anon);
   1638 	}
   1639 }
   1640 
   1641 /*
   1642  * uvm_fault_lower: handle lower fault.
   1643  *
   1644  *	1. check uobj
   1645  *	1.1. if null, ZFOD.
   1646  *	1.2. if not null, look up unnmapped neighbor pages.
   1647  *	2. for center page, check if promote.
   1648  *	2.1. ZFOD always needs promotion.
   1649  *	2.2. other uobjs, when entry is marked COW (usually MAP_PRIVATE vnode).
   1650  *	3. if uobj is not ZFOD and page is not found, do i/o.
   1651  *	4. dispatch either direct / promote fault.
   1652  */
   1653 
   1654 static int
   1655 uvm_fault_lower(
   1656 	struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
   1657 	struct vm_page **pages)
   1658 {
   1659 	struct vm_amap *amap __diagused = ufi->entry->aref.ar_amap;
   1660 	struct uvm_object *uobj = ufi->entry->object.uvm_obj;
   1661 	struct vm_page *uobjpage;
   1662 	int error;
   1663 	UVMHIST_FUNC("uvm_fault_lower"); UVMHIST_CALLED(maphist);
   1664 
   1665 	/*
   1666 	 * now, if the desired page is not shadowed by the amap and we have
   1667 	 * a backing object that does not have a special fault routine, then
   1668 	 * we ask (with pgo_get) the object for resident pages that we care
   1669 	 * about and attempt to map them in.  we do not let pgo_get block
   1670 	 * (PGO_LOCKED).
   1671 	 */
   1672 
   1673 	if (uobj == NULL) {
   1674 		/* zero fill; don't care neighbor pages */
   1675 		uobjpage = NULL;
   1676 	} else {
   1677 		uvm_fault_lower_lookup(ufi, flt, pages);
   1678 		uobjpage = pages[flt->centeridx];
   1679 	}
   1680 
   1681 	/*
   1682 	 * note that at this point we are done with any front or back pages.
   1683 	 * we are now going to focus on the center page (i.e. the one we've
   1684 	 * faulted on).  if we have faulted on the upper (anon) layer
   1685 	 * [i.e. case 1], then the anon we want is anons[centeridx] (we have
   1686 	 * not touched it yet).  if we have faulted on the bottom (uobj)
   1687 	 * layer [i.e. case 2] and the page was both present and available,
   1688 	 * then we've got a pointer to it as "uobjpage" and we've already
   1689 	 * made it BUSY.
   1690 	 */
   1691 
   1692 	/*
   1693 	 * locked:
   1694 	 * maps(read), amap(if there), uobj(if !null), uobjpage(if !null)
   1695 	 */
   1696 	KASSERT(amap == NULL || mutex_owned(amap->am_lock));
   1697 	KASSERT(uobj == NULL || mutex_owned(uobj->vmobjlock));
   1698 	KASSERT(uobjpage == NULL || (uobjpage->flags & PG_BUSY) != 0);
   1699 
   1700 	/*
   1701 	 * note that uobjpage can not be PGO_DONTCARE at this point.  we now
   1702 	 * set uobjpage to PGO_DONTCARE if we are doing a zero fill.  if we
   1703 	 * have a backing object, check and see if we are going to promote
   1704 	 * the data up to an anon during the fault.
   1705 	 */
   1706 
   1707 	if (uobj == NULL) {
   1708 		uobjpage = PGO_DONTCARE;
   1709 		flt->promote = true;		/* always need anon here */
   1710 	} else {
   1711 		KASSERT(uobjpage != PGO_DONTCARE);
   1712 		flt->promote = flt->cow_now && UVM_ET_ISCOPYONWRITE(ufi->entry);
   1713 	}
   1714 	UVMHIST_LOG(maphist, "  case 2 fault: promote=%jd, zfill=%jd",
   1715 	    flt->promote, (uobj == NULL), 0,0);
   1716 
   1717 	/*
   1718 	 * if uobjpage is not null then we do not need to do I/O to get the
   1719 	 * uobjpage.
   1720 	 *
   1721 	 * if uobjpage is null, then we need to unlock and ask the pager to
   1722 	 * get the data for us.   once we have the data, we need to reverify
   1723 	 * the state the world.   we are currently not holding any resources.
   1724 	 */
   1725 
   1726 	if (uobjpage) {
   1727 		/* update rusage counters */
   1728 		curlwp->l_ru.ru_minflt++;
   1729 	} else {
   1730 		error = uvm_fault_lower_io(ufi, flt, &uobj, &uobjpage);
   1731 		if (error != 0)
   1732 			return error;
   1733 	}
   1734 
   1735 	/*
   1736 	 * locked:
   1737 	 * maps(read), amap(if !null), uobj(if !null), uobjpage(if uobj)
   1738 	 */
   1739 	KASSERT(amap == NULL || mutex_owned(amap->am_lock));
   1740 	KASSERT(uobj == NULL || mutex_owned(uobj->vmobjlock));
   1741 	KASSERT(uobj == NULL || (uobjpage->flags & PG_BUSY) != 0);
   1742 
   1743 	/*
   1744 	 * notes:
   1745 	 *  - at this point uobjpage can not be NULL
   1746 	 *  - at this point uobjpage can not be PG_RELEASED (since we checked
   1747 	 *  for it above)
   1748 	 *  - at this point uobjpage could be PG_WANTED (handle later)
   1749 	 */
   1750 
   1751 	KASSERT(uobjpage != NULL);
   1752 	KASSERT(uobj == NULL || uobj == uobjpage->uobject);
   1753 	KASSERT(uobj == NULL || !UVM_OBJ_IS_CLEAN(uobjpage->uobject) ||
   1754 	    uvm_pagegetdirty(uobjpage) == UVM_PAGE_STATUS_CLEAN);
   1755 
   1756 	if (!flt->promote) {
   1757 		error = uvm_fault_lower_direct(ufi, flt, uobj, uobjpage);
   1758 	} else {
   1759 		error = uvm_fault_lower_promote(ufi, flt, uobj, uobjpage);
   1760 	}
   1761 	return error;
   1762 }
   1763 
   1764 /*
   1765  * uvm_fault_lower_lookup: look up on-memory uobj pages.
   1766  *
   1767  *	1. get on-memory pages.
   1768  *	2. if failed, give up (get only center page later).
   1769  *	3. if succeeded, enter h/w mapping of neighbor pages.
   1770  */
   1771 
   1772 static void
   1773 uvm_fault_lower_lookup(
   1774 	struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt,
   1775 	struct vm_page **pages)
   1776 {
   1777 	struct uvm_object *uobj = ufi->entry->object.uvm_obj;
   1778 	int lcv, gotpages;
   1779 	vaddr_t currva;
   1780 	UVMHIST_FUNC("uvm_fault_lower_lookup"); UVMHIST_CALLED(maphist);
   1781 
   1782 	mutex_enter(uobj->vmobjlock);
   1783 	/* Locked: maps(read), amap(if there), uobj */
   1784 
   1785 	cpu_count(CPU_COUNT_FLTLGET, 1);
   1786 	gotpages = flt->npages;
   1787 	(void) uobj->pgops->pgo_get(uobj,
   1788 	    ufi->entry->offset + flt->startva - ufi->entry->start,
   1789 	    pages, &gotpages, flt->centeridx,
   1790 	    flt->access_type & MASK(ufi->entry), ufi->entry->advice, PGO_LOCKED);
   1791 
   1792 	KASSERT(mutex_owned(uobj->vmobjlock));
   1793 
   1794 	/*
   1795 	 * check for pages to map, if we got any
   1796 	 */
   1797 
   1798 	if (gotpages == 0) {
   1799 		pages[flt->centeridx] = NULL;
   1800 		return;
   1801 	}
   1802 
   1803 	currva = flt->startva;
   1804 	for (lcv = 0; lcv < flt->npages; lcv++, currva += PAGE_SIZE) {
   1805 		struct vm_page *curpg;
   1806 
   1807 		curpg = pages[lcv];
   1808 		if (curpg == NULL || curpg == PGO_DONTCARE) {
   1809 			continue;
   1810 		}
   1811 		KASSERT(curpg->uobject == uobj);
   1812 
   1813 		/*
   1814 		 * if center page is resident and not PG_BUSY|PG_RELEASED
   1815 		 * then pgo_get made it PG_BUSY for us and gave us a handle
   1816 		 * to it.
   1817 		 */
   1818 
   1819 		if (lcv == flt->centeridx) {
   1820 			UVMHIST_LOG(maphist, "  got uobjpage (0x%#jx) "
   1821 			    "with locked get", (uintptr_t)curpg, 0, 0, 0);
   1822 		} else {
   1823 			uvm_fault_lower_neighbor(ufi, flt, currva, curpg);
   1824 		}
   1825 	}
   1826 	pmap_update(ufi->orig_map->pmap);
   1827 }
   1828 
   1829 /*
   1830  * uvm_fault_lower_neighbor: enter h/w mapping of lower neighbor page.
   1831  */
   1832 
   1833 static void
   1834 uvm_fault_lower_neighbor(
   1835 	struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt,
   1836 	vaddr_t currva, struct vm_page *pg)
   1837 {
   1838 	const bool readonly = uvm_pagereadonly_p(pg) || pg->loan_count > 0;
   1839 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
   1840 
   1841 	/* locked: maps(read), amap(if there), uobj */
   1842 
   1843 	/*
   1844 	 * calling pgo_get with PGO_LOCKED returns us pages which
   1845 	 * are neither busy nor released, so we don't need to check
   1846 	 * for this.  we can just directly enter the pages.
   1847 	 */
   1848 
   1849 	uvm_pagelock(pg);
   1850 	uvm_pageenqueue(pg);
   1851 	uvm_pageunlock(pg);
   1852 	UVMHIST_LOG(maphist,
   1853 	    "  MAPPING: n obj: pm=%#jx, va=%#jx, pg=%#jx",
   1854 	    (uintptr_t)ufi->orig_map->pmap, currva, (uintptr_t)pg, 0);
   1855 	cpu_count(CPU_COUNT_FLTNOMAP, 1);
   1856 
   1857 	/*
   1858 	 * Since this page isn't the page that's actually faulting,
   1859 	 * ignore pmap_enter() failures; it's not critical that we
   1860 	 * enter these right now.
   1861 	 * NOTE: page can't be PG_WANTED or PG_RELEASED because we've
   1862 	 * held the lock the whole time we've had the handle.
   1863 	 */
   1864 	KASSERT((pg->flags & PG_PAGEOUT) == 0);
   1865 	KASSERT((pg->flags & PG_RELEASED) == 0);
   1866 	KASSERT((pg->flags & PG_WANTED) == 0);
   1867 	KASSERT(!UVM_OBJ_IS_CLEAN(pg->uobject) ||
   1868 	    uvm_pagegetdirty(pg) == UVM_PAGE_STATUS_CLEAN);
   1869 	pg->flags &= ~(PG_BUSY);
   1870 	UVM_PAGE_OWN(pg, NULL);
   1871 
   1872 	KASSERT(mutex_owned(pg->uobject->vmobjlock));
   1873 
   1874 	const vm_prot_t mapprot =
   1875 	    readonly ? (flt->enter_prot & ~VM_PROT_WRITE) :
   1876 	    flt->enter_prot & MASK(ufi->entry);
   1877 	const u_int mapflags =
   1878 	    PMAP_CANFAIL | (flt->wire_mapping ? (mapprot | PMAP_WIRED) : 0);
   1879 	(void) pmap_enter(ufi->orig_map->pmap, currva,
   1880 	    VM_PAGE_TO_PHYS(pg), mapprot, mapflags);
   1881 }
   1882 
   1883 /*
   1884  * uvm_fault_lower_io: get lower page from backing store.
   1885  *
   1886  *	1. unlock everything, because i/o will block.
   1887  *	2. call pgo_get.
   1888  *	3. if failed, recover.
   1889  *	4. if succeeded, relock everything and verify things.
   1890  */
   1891 
   1892 static int
   1893 uvm_fault_lower_io(
   1894 	struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt,
   1895 	struct uvm_object **ruobj, struct vm_page **ruobjpage)
   1896 {
   1897 	struct vm_amap * const amap = ufi->entry->aref.ar_amap;
   1898 	struct uvm_object *uobj = *ruobj;
   1899 	struct vm_page *pg;
   1900 	bool locked;
   1901 	int gotpages;
   1902 	int error;
   1903 	voff_t uoff;
   1904 	vm_prot_t access_type;
   1905 	int advice;
   1906 	UVMHIST_FUNC("uvm_fault_lower_io"); UVMHIST_CALLED(maphist);
   1907 
   1908 	/* update rusage counters */
   1909 	curlwp->l_ru.ru_majflt++;
   1910 
   1911 	/* grab everything we need from the entry before we unlock */
   1912 	uoff = (ufi->orig_rvaddr - ufi->entry->start) + ufi->entry->offset;
   1913 	access_type = flt->access_type & MASK(ufi->entry);
   1914 	advice = ufi->entry->advice;
   1915 
   1916 	/* Locked: maps(read), amap(if there), uobj */
   1917 	uvmfault_unlockall(ufi, amap, NULL);
   1918 
   1919 	/* Locked: uobj */
   1920 	KASSERT(uobj == NULL || mutex_owned(uobj->vmobjlock));
   1921 
   1922 	cpu_count(CPU_COUNT_FLTGET, 1);
   1923 	gotpages = 1;
   1924 	pg = NULL;
   1925 	error = uobj->pgops->pgo_get(uobj, uoff, &pg, &gotpages,
   1926 	    0, access_type, advice, PGO_SYNCIO);
   1927 	/* locked: pg(if no error) */
   1928 
   1929 	/*
   1930 	 * recover from I/O
   1931 	 */
   1932 
   1933 	if (error) {
   1934 		if (error == EAGAIN) {
   1935 			UVMHIST_LOG(maphist,
   1936 			    "  pgo_get says TRY AGAIN!",0,0,0,0);
   1937 			kpause("fltagain2", false, hz/2, NULL);
   1938 			return ERESTART;
   1939 		}
   1940 
   1941 #if 0
   1942 		KASSERT(error != ERESTART);
   1943 #else
   1944 		/* XXXUEBS don't re-fault? */
   1945 		if (error == ERESTART)
   1946 			error = EIO;
   1947 #endif
   1948 
   1949 		UVMHIST_LOG(maphist, "<- pgo_get failed (code %jd)",
   1950 		    error, 0,0,0);
   1951 		return error;
   1952 	}
   1953 
   1954 	/*
   1955 	 * re-verify the state of the world by first trying to relock
   1956 	 * the maps.  always relock the object.
   1957 	 */
   1958 
   1959 	locked = uvmfault_relock(ufi);
   1960 	if (locked && amap)
   1961 		amap_lock(amap);
   1962 
   1963 	/* might be changed */
   1964 	uobj = pg->uobject;
   1965 
   1966 	mutex_enter(uobj->vmobjlock);
   1967 	KASSERT((pg->flags & PG_BUSY) != 0);
   1968 
   1969 	uvm_pagelock(pg);
   1970 	uvm_pageactivate(pg);
   1971 	uvm_pageunlock(pg);
   1972 
   1973 	/* locked(locked): maps(read), amap(if !null), uobj, pg */
   1974 	/* locked(!locked): uobj, pg */
   1975 
   1976 	/*
   1977 	 * verify that the page has not be released and re-verify
   1978 	 * that amap slot is still free.   if there is a problem,
   1979 	 * we unlock and clean up.
   1980 	 */
   1981 
   1982 	if ((pg->flags & PG_RELEASED) != 0 ||
   1983 	    (locked && amap && amap_lookup(&ufi->entry->aref,
   1984 	      ufi->orig_rvaddr - ufi->entry->start))) {
   1985 		if (locked)
   1986 			uvmfault_unlockall(ufi, amap, NULL);
   1987 		locked = false;
   1988 	}
   1989 
   1990 	/*
   1991 	 * didn't get the lock?   release the page and retry.
   1992 	 */
   1993 
   1994 	if (locked == false) {
   1995 		UVMHIST_LOG(maphist,
   1996 		    "  wasn't able to relock after fault: retry",
   1997 		    0,0,0,0);
   1998 		if (pg->flags & PG_WANTED) {
   1999 			wakeup(pg);
   2000 		}
   2001 		if ((pg->flags & PG_RELEASED) == 0) {
   2002 			pg->flags &= ~(PG_BUSY | PG_WANTED);
   2003 			UVM_PAGE_OWN(pg, NULL);
   2004 		} else {
   2005 			cpu_count(CPU_COUNT_FLTPGRELE, 1);
   2006 			uvm_pagefree(pg);
   2007 		}
   2008 		mutex_exit(uobj->vmobjlock);
   2009 		return ERESTART;
   2010 	}
   2011 
   2012 	/*
   2013 	 * we have the data in pg which is busy and
   2014 	 * not released.  we are holding object lock (so the page
   2015 	 * can't be released on us).
   2016 	 */
   2017 
   2018 	/* locked: maps(read), amap(if !null), uobj, pg */
   2019 
   2020 	*ruobj = uobj;
   2021 	*ruobjpage = pg;
   2022 	return 0;
   2023 }
   2024 
   2025 /*
   2026  * uvm_fault_lower_direct: fault lower center page
   2027  *
   2028  *	1. adjust flt->enter_prot.
   2029  *	2. if page is loaned, resolve.
   2030  */
   2031 
   2032 int
   2033 uvm_fault_lower_direct(
   2034 	struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
   2035 	struct uvm_object *uobj, struct vm_page *uobjpage)
   2036 {
   2037 	struct vm_page *pg;
   2038 	UVMHIST_FUNC("uvm_fault_lower_direct"); UVMHIST_CALLED(maphist);
   2039 
   2040 	/*
   2041 	 * we are not promoting.   if the mapping is COW ensure that we
   2042 	 * don't give more access than we should (e.g. when doing a read
   2043 	 * fault on a COPYONWRITE mapping we want to map the COW page in
   2044 	 * R/O even though the entry protection could be R/W).
   2045 	 *
   2046 	 * set "pg" to the page we want to map in (uobjpage, usually)
   2047 	 */
   2048 
   2049 	cpu_count(CPU_COUNT_FLT_OBJ, 1);
   2050 	if (UVM_ET_ISCOPYONWRITE(ufi->entry) ||
   2051 	    UVM_OBJ_NEEDS_WRITEFAULT(uobjpage->uobject))
   2052 		flt->enter_prot &= ~VM_PROT_WRITE;
   2053 	pg = uobjpage;		/* map in the actual object */
   2054 
   2055 	KASSERT(uobjpage != PGO_DONTCARE);
   2056 
   2057 	/*
   2058 	 * we are faulting directly on the page.   be careful
   2059 	 * about writing to loaned pages...
   2060 	 */
   2061 
   2062 	if (uobjpage->loan_count) {
   2063 		uvm_fault_lower_direct_loan(ufi, flt, uobj, &pg, &uobjpage);
   2064 	}
   2065 	KASSERT(pg == uobjpage);
   2066 
   2067 	KASSERT(uobj == NULL || (uobjpage->flags & PG_BUSY) != 0);
   2068 	return uvm_fault_lower_enter(ufi, flt, uobj, NULL, pg);
   2069 }
   2070 
   2071 /*
   2072  * uvm_fault_lower_direct_loan: resolve loaned page.
   2073  *
   2074  *	1. if not cow'ing, adjust flt->enter_prot.
   2075  *	2. if cow'ing, break loan.
   2076  */
   2077 
   2078 static int
   2079 uvm_fault_lower_direct_loan(
   2080 	struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
   2081 	struct uvm_object *uobj, struct vm_page **rpg,
   2082 	struct vm_page **ruobjpage)
   2083 {
   2084 	struct vm_amap * const amap = ufi->entry->aref.ar_amap;
   2085 	struct vm_page *pg;
   2086 	struct vm_page *uobjpage = *ruobjpage;
   2087 	UVMHIST_FUNC("uvm_fault_lower_direct_loan"); UVMHIST_CALLED(maphist);
   2088 
   2089 	if (!flt->cow_now) {
   2090 		/* read fault: cap the protection at readonly */
   2091 		/* cap! */
   2092 		flt->enter_prot = flt->enter_prot & ~VM_PROT_WRITE;
   2093 	} else {
   2094 		/* write fault: must break the loan here */
   2095 
   2096 		pg = uvm_loanbreak(uobjpage);
   2097 		if (pg == NULL) {
   2098 
   2099 			/*
   2100 			 * drop ownership of page, it can't be released
   2101 			 */
   2102 
   2103 			if (uobjpage->flags & PG_WANTED)
   2104 				wakeup(uobjpage);
   2105 			uobjpage->flags &= ~(PG_BUSY|PG_WANTED);
   2106 			UVM_PAGE_OWN(uobjpage, NULL);
   2107 
   2108 			uvmfault_unlockall(ufi, amap, uobj);
   2109 			UVMHIST_LOG(maphist,
   2110 			  "  out of RAM breaking loan, waiting",
   2111 			  0,0,0,0);
   2112 			cpu_count(CPU_COUNT_FLTNORAM, 1);
   2113 			uvm_wait("flt_noram4");
   2114 			return ERESTART;
   2115 		}
   2116 		*rpg = pg;
   2117 		*ruobjpage = pg;
   2118 	}
   2119 	return 0;
   2120 }
   2121 
   2122 /*
   2123  * uvm_fault_lower_promote: promote lower page.
   2124  *
   2125  *	1. call uvmfault_promote.
   2126  *	2. fill in data.
   2127  *	3. if not ZFOD, dispose old page.
   2128  */
   2129 
   2130 int
   2131 uvm_fault_lower_promote(
   2132 	struct uvm_faultinfo *ufi, struct uvm_faultctx *flt,
   2133 	struct uvm_object *uobj, struct vm_page *uobjpage)
   2134 {
   2135 	struct vm_amap * const amap = ufi->entry->aref.ar_amap;
   2136 	struct vm_anon *anon;
   2137 	struct vm_page *pg;
   2138 	int error;
   2139 	UVMHIST_FUNC("uvm_fault_lower_promote"); UVMHIST_CALLED(maphist);
   2140 
   2141 	KASSERT(amap != NULL);
   2142 
   2143 	/*
   2144 	 * If we are going to promote the data to an anon we
   2145 	 * allocate a blank anon here and plug it into our amap.
   2146 	 */
   2147 	error = uvmfault_promote(ufi, NULL, uobjpage,
   2148 	    &anon, &flt->anon_spare);
   2149 	switch (error) {
   2150 	case 0:
   2151 		break;
   2152 	case ERESTART:
   2153 		return ERESTART;
   2154 	default:
   2155 		return error;
   2156 	}
   2157 
   2158 	pg = anon->an_page;
   2159 
   2160 	/*
   2161 	 * Fill in the data.
   2162 	 */
   2163 	KASSERT(uobj == NULL || (uobjpage->flags & PG_BUSY) != 0);
   2164 
   2165 	if (uobjpage != PGO_DONTCARE) {
   2166 		cpu_count(CPU_COUNT_FLT_PRCOPY, 1);
   2167 
   2168 		/*
   2169 		 * promote to shared amap?  make sure all sharing
   2170 		 * procs see it
   2171 		 */
   2172 
   2173 		if ((amap_flags(amap) & AMAP_SHARED) != 0) {
   2174 			pmap_page_protect(uobjpage, VM_PROT_NONE);
   2175 			/*
   2176 			 * XXX: PAGE MIGHT BE WIRED!
   2177 			 */
   2178 		}
   2179 
   2180 		/*
   2181 		 * dispose of uobjpage.  it can't be PG_RELEASED
   2182 		 * since we still hold the object lock.
   2183 		 */
   2184 
   2185 		if (uobjpage->flags & PG_WANTED) {
   2186 			/* still have the obj lock */
   2187 			wakeup(uobjpage);
   2188 		}
   2189 		uobjpage->flags &= ~(PG_BUSY|PG_WANTED);
   2190 		UVM_PAGE_OWN(uobjpage, NULL);
   2191 
   2192 		UVMHIST_LOG(maphist,
   2193 		    "  promote uobjpage 0x%#jx to anon/page 0x%#jx/0x%#jx",
   2194 		    (uintptr_t)uobjpage, (uintptr_t)anon, (uintptr_t)pg, 0);
   2195 
   2196 	} else {
   2197 		cpu_count(CPU_COUNT_FLT_PRZERO, 1);
   2198 
   2199 		/*
   2200 		 * Page is zero'd and marked dirty by
   2201 		 * uvmfault_promote().
   2202 		 */
   2203 
   2204 		UVMHIST_LOG(maphist,"  zero fill anon/page 0x%#jx/0%#jx",
   2205 		    (uintptr_t)anon, (uintptr_t)pg, 0, 0);
   2206 	}
   2207 
   2208 	return uvm_fault_lower_enter(ufi, flt, uobj, anon, pg);
   2209 }
   2210 
   2211 /*
   2212  * uvm_fault_lower_enter: enter h/w mapping of lower page or anon page promoted
   2213  * from the lower page.
   2214  */
   2215 
   2216 int
   2217 uvm_fault_lower_enter(
   2218 	struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt,
   2219 	struct uvm_object *uobj,
   2220 	struct vm_anon *anon, struct vm_page *pg)
   2221 {
   2222 	struct vm_amap * const amap = ufi->entry->aref.ar_amap;
   2223 	const bool readonly = uvm_pagereadonly_p(pg);
   2224 	int error;
   2225 	UVMHIST_FUNC("uvm_fault_lower_enter"); UVMHIST_CALLED(maphist);
   2226 
   2227 	/*
   2228 	 * Locked:
   2229 	 *
   2230 	 *	maps(read), amap(if !null), uobj(if !null),
   2231 	 *	anon(if !null), pg(if anon), unlock_uobj(if !null)
   2232 	 *
   2233 	 * Note: pg is either the uobjpage or the new page in the new anon.
   2234 	 */
   2235 	KASSERT(amap == NULL || mutex_owned(amap->am_lock));
   2236 	KASSERT(uobj == NULL || mutex_owned(uobj->vmobjlock));
   2237 	KASSERT(anon == NULL || anon->an_lock == amap->am_lock);
   2238 	KASSERT((pg->flags & PG_BUSY) != 0);
   2239 
   2240 	/*
   2241 	 * all resources are present.   we can now map it in and free our
   2242 	 * resources.
   2243 	 */
   2244 
   2245 	UVMHIST_LOG(maphist,
   2246 	    "  MAPPING: case2: pm=%#jx, va=%#jx, pg=%#jx, promote=%jd",
   2247 	    (uintptr_t)ufi->orig_map->pmap, ufi->orig_rvaddr,
   2248 	    (uintptr_t)pg, flt->promote);
   2249 	KASSERTMSG((flt->access_type & VM_PROT_WRITE) == 0 || !readonly,
   2250 	    "promote=%u cow_now=%u access_type=%x enter_prot=%x cow=%u "
   2251 	    "entry=%p map=%p orig_rvaddr=%p pg=%p",
   2252 	    flt->promote, flt->cow_now, flt->access_type, flt->enter_prot,
   2253 	    UVM_ET_ISCOPYONWRITE(ufi->entry), ufi->entry, ufi->orig_map,
   2254 	    (void *)ufi->orig_rvaddr, pg);
   2255 	KASSERT((flt->access_type & VM_PROT_WRITE) == 0 || !readonly);
   2256 	if (pmap_enter(ufi->orig_map->pmap, ufi->orig_rvaddr,
   2257 	    VM_PAGE_TO_PHYS(pg),
   2258 	    readonly ? flt->enter_prot & ~VM_PROT_WRITE : flt->enter_prot,
   2259 	    flt->access_type | PMAP_CANFAIL |
   2260 	    (flt->wire_mapping ? PMAP_WIRED : 0)) != 0) {
   2261 
   2262 		/*
   2263 		 * No need to undo what we did; we can simply think of
   2264 		 * this as the pmap throwing away the mapping information.
   2265 		 *
   2266 		 * We do, however, have to go through the ReFault path,
   2267 		 * as the map may change while we're asleep.
   2268 		 */
   2269 
   2270 		/*
   2271 		 * ensure that the page is queued in the case that
   2272 		 * we just promoted the page.
   2273 		 */
   2274 
   2275 		uvm_pagelock(pg);
   2276 		uvm_pageenqueue(pg);
   2277 		uvm_pageunlock(pg);
   2278 
   2279 		if (pg->flags & PG_WANTED)
   2280 			wakeup(pg);
   2281 
   2282 		/*
   2283 		 * note that pg can't be PG_RELEASED since we did not drop
   2284 		 * the object lock since the last time we checked.
   2285 		 */
   2286 		KASSERT((pg->flags & PG_RELEASED) == 0);
   2287 
   2288 		pg->flags &= ~(PG_BUSY|PG_FAKE|PG_WANTED);
   2289 		UVM_PAGE_OWN(pg, NULL);
   2290 
   2291 		uvmfault_unlockall(ufi, amap, uobj);
   2292 		if (!uvm_reclaimable()) {
   2293 			UVMHIST_LOG(maphist,
   2294 			    "<- failed.  out of VM",0,0,0,0);
   2295 			/* XXX instrumentation */
   2296 			error = ENOMEM;
   2297 			return error;
   2298 		}
   2299 		/* XXX instrumentation */
   2300 		uvm_wait("flt_pmfail2");
   2301 		return ERESTART;
   2302 	}
   2303 
   2304 	uvm_fault_lower_done(ufi, flt, uobj, pg);
   2305 
   2306 	/*
   2307 	 * note that pg can't be PG_RELEASED since we did not drop the object
   2308 	 * lock since the last time we checked.
   2309 	 */
   2310 	KASSERT((pg->flags & PG_RELEASED) == 0);
   2311 	if (pg->flags & PG_WANTED)
   2312 		wakeup(pg);
   2313 	pg->flags &= ~(PG_BUSY|PG_FAKE|PG_WANTED);
   2314 	UVM_PAGE_OWN(pg, NULL);
   2315 
   2316 	pmap_update(ufi->orig_map->pmap);
   2317 	uvmfault_unlockall(ufi, amap, uobj);
   2318 
   2319 	UVMHIST_LOG(maphist, "<- done (SUCCESS!)",0,0,0,0);
   2320 	return 0;
   2321 }
   2322 
   2323 /*
   2324  * uvm_fault_lower_done: queue lower center page.
   2325  */
   2326 
   2327 void
   2328 uvm_fault_lower_done(
   2329 	struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt,
   2330 	struct uvm_object *uobj, struct vm_page *pg)
   2331 {
   2332 	bool dropswap = false;
   2333 
   2334 	UVMHIST_FUNC("uvm_fault_lower_done"); UVMHIST_CALLED(maphist);
   2335 
   2336 	uvm_pagelock(pg);
   2337 	if (flt->wire_paging) {
   2338 		uvm_pagewire(pg);
   2339 		if (pg->flags & PG_AOBJ) {
   2340 
   2341 			/*
   2342 			 * since the now-wired page cannot be paged out,
   2343 			 * release its swap resources for others to use.
   2344 			 * since an aobj page with no swap cannot be clean,
   2345 			 * mark it dirty now.
   2346 			 */
   2347 
   2348 			KASSERT(uobj != NULL);
   2349 			uvm_pagemarkdirty(pg, UVM_PAGE_STATUS_DIRTY);
   2350 			dropswap = true;
   2351 		}
   2352 	} else {
   2353 		uvm_pageactivate(pg);
   2354 	}
   2355 	uvm_pageunlock(pg);
   2356 
   2357 	if (dropswap) {
   2358 		uao_dropswap(uobj, pg->offset >> PAGE_SHIFT);
   2359 	}
   2360 }
   2361 
   2362 
   2363 /*
   2364  * uvm_fault_wire: wire down a range of virtual addresses in a map.
   2365  *
   2366  * => map may be read-locked by caller, but MUST NOT be write-locked.
   2367  * => if map is read-locked, any operations which may cause map to
   2368  *	be write-locked in uvm_fault() must be taken care of by
   2369  *	the caller.  See uvm_map_pageable().
   2370  */
   2371 
   2372 int
   2373 uvm_fault_wire(struct vm_map *map, vaddr_t start, vaddr_t end,
   2374     vm_prot_t access_type, int maxprot)
   2375 {
   2376 	vaddr_t va;
   2377 	int error;
   2378 
   2379 	/*
   2380 	 * now fault it in a page at a time.   if the fault fails then we have
   2381 	 * to undo what we have done.   note that in uvm_fault VM_PROT_NONE
   2382 	 * is replaced with the max protection if fault_type is VM_FAULT_WIRE.
   2383 	 */
   2384 
   2385 	/*
   2386 	 * XXX work around overflowing a vaddr_t.  this prevents us from
   2387 	 * wiring the last page in the address space, though.
   2388 	 */
   2389 	if (start > end) {
   2390 		return EFAULT;
   2391 	}
   2392 
   2393 	for (va = start; va < end; va += PAGE_SIZE) {
   2394 		error = uvm_fault_internal(map, va, access_type,
   2395 		    (maxprot ? UVM_FAULT_MAXPROT : 0) | UVM_FAULT_WIRE);
   2396 		if (error) {
   2397 			if (va != start) {
   2398 				uvm_fault_unwire(map, start, va);
   2399 			}
   2400 			return error;
   2401 		}
   2402 	}
   2403 	return 0;
   2404 }
   2405 
   2406 /*
   2407  * uvm_fault_unwire(): unwire range of virtual space.
   2408  */
   2409 
   2410 void
   2411 uvm_fault_unwire(struct vm_map *map, vaddr_t start, vaddr_t end)
   2412 {
   2413 	vm_map_lock_read(map);
   2414 	uvm_fault_unwire_locked(map, start, end);
   2415 	vm_map_unlock_read(map);
   2416 }
   2417 
   2418 /*
   2419  * uvm_fault_unwire_locked(): the guts of uvm_fault_unwire().
   2420  *
   2421  * => map must be at least read-locked.
   2422  */
   2423 
   2424 void
   2425 uvm_fault_unwire_locked(struct vm_map *map, vaddr_t start, vaddr_t end)
   2426 {
   2427 	struct vm_map_entry *entry, *oentry;
   2428 	pmap_t pmap = vm_map_pmap(map);
   2429 	vaddr_t va;
   2430 	paddr_t pa;
   2431 	struct vm_page *pg;
   2432 
   2433 	/*
   2434 	 * we assume that the area we are unwiring has actually been wired
   2435 	 * in the first place.   this means that we should be able to extract
   2436 	 * the PAs from the pmap.   we also lock out the page daemon so that
   2437 	 * we can call uvm_pageunwire.
   2438 	 */
   2439 
   2440 	/*
   2441 	 * find the beginning map entry for the region.
   2442 	 */
   2443 
   2444 	KASSERT(start >= vm_map_min(map) && end <= vm_map_max(map));
   2445 	if (uvm_map_lookup_entry(map, start, &entry) == false)
   2446 		panic("uvm_fault_unwire_locked: address not in map");
   2447 
   2448 	oentry = NULL;
   2449 	for (va = start; va < end; va += PAGE_SIZE) {
   2450 
   2451 		/*
   2452 		 * find the map entry for the current address.
   2453 		 */
   2454 
   2455 		KASSERT(va >= entry->start);
   2456 		while (va >= entry->end) {
   2457 			KASSERT(entry->next != &map->header &&
   2458 				entry->next->start <= entry->end);
   2459 			entry = entry->next;
   2460 		}
   2461 
   2462 		/*
   2463 		 * lock it.
   2464 		 */
   2465 
   2466 		if (entry != oentry) {
   2467 			if (oentry != NULL) {
   2468 				uvm_map_unlock_entry(oentry);
   2469 			}
   2470 			uvm_map_lock_entry(entry);
   2471 			oentry = entry;
   2472 		}
   2473 
   2474 		/*
   2475 		 * if the entry is no longer wired, tell the pmap.
   2476 		 */
   2477 
   2478 		if (!pmap_extract(pmap, va, &pa))
   2479 			continue;
   2480 
   2481 		if (VM_MAPENT_ISWIRED(entry) == 0)
   2482 			pmap_unwire(pmap, va);
   2483 
   2484 		pg = PHYS_TO_VM_PAGE(pa);
   2485 		if (pg) {
   2486 			uvm_pagelock(pg);
   2487 			uvm_pageunwire(pg);
   2488 			uvm_pageunlock(pg);
   2489 		}
   2490 	}
   2491 
   2492 	if (oentry != NULL) {
   2493 		uvm_map_unlock_entry(entry);
   2494 	}
   2495 }
   2496