Home | History | Annotate | Line # | Download | only in uvm
uvm_fault.c revision 1.119.12.1
      1  1.119.12.1     skrll /*	$NetBSD: uvm_fault.c,v 1.119.12.1 2007/08/15 13:51:21 skrll Exp $	*/
      2         1.1       mrg 
      3         1.1       mrg /*
      4         1.1       mrg  *
      5         1.1       mrg  * Copyright (c) 1997 Charles D. Cranor and Washington University.
      6         1.1       mrg  * All rights reserved.
      7         1.1       mrg  *
      8         1.1       mrg  * Redistribution and use in source and binary forms, with or without
      9         1.1       mrg  * modification, are permitted provided that the following conditions
     10         1.1       mrg  * are met:
     11         1.1       mrg  * 1. Redistributions of source code must retain the above copyright
     12         1.1       mrg  *    notice, this list of conditions and the following disclaimer.
     13         1.1       mrg  * 2. Redistributions in binary form must reproduce the above copyright
     14         1.1       mrg  *    notice, this list of conditions and the following disclaimer in the
     15         1.1       mrg  *    documentation and/or other materials provided with the distribution.
     16         1.1       mrg  * 3. All advertising materials mentioning features or use of this software
     17         1.1       mrg  *    must display the following acknowledgement:
     18         1.1       mrg  *      This product includes software developed by Charles D. Cranor and
     19         1.1       mrg  *      Washington University.
     20         1.1       mrg  * 4. The name of the author may not be used to endorse or promote products
     21         1.1       mrg  *    derived from this software without specific prior written permission.
     22         1.1       mrg  *
     23         1.1       mrg  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
     24         1.1       mrg  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
     25         1.1       mrg  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
     26         1.1       mrg  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
     27         1.1       mrg  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
     28         1.1       mrg  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
     29         1.1       mrg  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
     30         1.1       mrg  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
     31         1.1       mrg  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
     32         1.1       mrg  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     33         1.4       mrg  *
     34         1.4       mrg  * from: Id: uvm_fault.c,v 1.1.2.23 1998/02/06 05:29:05 chs Exp
     35         1.1       mrg  */
     36         1.1       mrg 
     37         1.1       mrg /*
     38         1.1       mrg  * uvm_fault.c: fault handler
     39         1.1       mrg  */
     40        1.71     lukem 
     41        1.71     lukem #include <sys/cdefs.h>
     42  1.119.12.1     skrll __KERNEL_RCSID(0, "$NetBSD: uvm_fault.c,v 1.119.12.1 2007/08/15 13:51:21 skrll Exp $");
     43        1.71     lukem 
     44        1.71     lukem #include "opt_uvmhist.h"
     45         1.1       mrg 
     46         1.1       mrg #include <sys/param.h>
     47         1.1       mrg #include <sys/systm.h>
     48         1.1       mrg #include <sys/kernel.h>
     49         1.1       mrg #include <sys/proc.h>
     50         1.1       mrg #include <sys/malloc.h>
     51         1.1       mrg #include <sys/mman.h>
     52         1.9     chuck #include <sys/user.h>
     53         1.1       mrg 
     54         1.1       mrg #include <uvm/uvm.h>
     55         1.1       mrg 
     56         1.1       mrg /*
     57         1.1       mrg  *
     58         1.1       mrg  * a word on page faults:
     59         1.1       mrg  *
     60         1.1       mrg  * types of page faults we handle:
     61         1.1       mrg  *
     62         1.1       mrg  * CASE 1: upper layer faults                   CASE 2: lower layer faults
     63         1.1       mrg  *
     64         1.1       mrg  *    CASE 1A         CASE 1B                  CASE 2A        CASE 2B
     65         1.1       mrg  *    read/write1     write>1                  read/write   +-cow_write/zero
     66        1.63       chs  *         |             |                         |        |
     67         1.1       mrg  *      +--|--+       +--|--+     +-----+       +  |  +     | +-----+
     68         1.1       mrg  * amap |  V  |       |  ----------->new|          |        | |  ^  |
     69         1.1       mrg  *      +-----+       +-----+     +-----+       +  |  +     | +--|--+
     70         1.1       mrg  *                                                 |        |    |
     71         1.1       mrg  *      +-----+       +-----+                   +--|--+     | +--|--+
     72         1.1       mrg  * uobj | d/c |       | d/c |                   |  V  |     +----|  |
     73         1.1       mrg  *      +-----+       +-----+                   +-----+       +-----+
     74         1.1       mrg  *
     75         1.1       mrg  * d/c = don't care
     76        1.63       chs  *
     77         1.1       mrg  *   case [0]: layerless fault
     78         1.1       mrg  *	no amap or uobj is present.   this is an error.
     79         1.1       mrg  *
     80         1.1       mrg  *   case [1]: upper layer fault [anon active]
     81         1.1       mrg  *     1A: [read] or [write with anon->an_ref == 1]
     82         1.1       mrg  *		I/O takes place in top level anon and uobj is not touched.
     83         1.1       mrg  *     1B: [write with anon->an_ref > 1]
     84         1.1       mrg  *		new anon is alloc'd and data is copied off ["COW"]
     85         1.1       mrg  *
     86         1.1       mrg  *   case [2]: lower layer fault [uobj]
     87         1.1       mrg  *     2A: [read on non-NULL uobj] or [write to non-copy_on_write area]
     88         1.1       mrg  *		I/O takes place directly in object.
     89         1.1       mrg  *     2B: [write to copy_on_write] or [read on NULL uobj]
     90        1.63       chs  *		data is "promoted" from uobj to a new anon.
     91         1.1       mrg  *		if uobj is null, then we zero fill.
     92         1.1       mrg  *
     93         1.1       mrg  * we follow the standard UVM locking protocol ordering:
     94         1.1       mrg  *
     95        1.63       chs  * MAPS => AMAP => UOBJ => ANON => PAGE QUEUES (PQ)
     96         1.1       mrg  * we hold a PG_BUSY page if we unlock for I/O
     97         1.1       mrg  *
     98         1.1       mrg  *
     99         1.1       mrg  * the code is structured as follows:
    100        1.63       chs  *
    101         1.1       mrg  *     - init the "IN" params in the ufi structure
    102         1.1       mrg  *   ReFault:
    103         1.1       mrg  *     - do lookups [locks maps], check protection, handle needs_copy
    104         1.1       mrg  *     - check for case 0 fault (error)
    105         1.1       mrg  *     - establish "range" of fault
    106         1.1       mrg  *     - if we have an amap lock it and extract the anons
    107         1.1       mrg  *     - if sequential advice deactivate pages behind us
    108         1.1       mrg  *     - at the same time check pmap for unmapped areas and anon for pages
    109         1.1       mrg  *	 that we could map in (and do map it if found)
    110         1.1       mrg  *     - check object for resident pages that we could map in
    111         1.1       mrg  *     - if (case 2) goto Case2
    112         1.1       mrg  *     - >>> handle case 1
    113         1.1       mrg  *           - ensure source anon is resident in RAM
    114         1.1       mrg  *           - if case 1B alloc new anon and copy from source
    115         1.1       mrg  *           - map the correct page in
    116         1.1       mrg  *   Case2:
    117         1.1       mrg  *     - >>> handle case 2
    118         1.1       mrg  *           - ensure source page is resident (if uobj)
    119         1.1       mrg  *           - if case 2B alloc new anon and copy from source (could be zero
    120         1.1       mrg  *		fill if uobj == NULL)
    121         1.1       mrg  *           - map the correct page in
    122         1.1       mrg  *     - done!
    123         1.1       mrg  *
    124         1.1       mrg  * note on paging:
    125         1.1       mrg  *   if we have to do I/O we place a PG_BUSY page in the correct object,
    126         1.1       mrg  * unlock everything, and do the I/O.   when I/O is done we must reverify
    127         1.1       mrg  * the state of the world before assuming that our data structures are
    128         1.1       mrg  * valid.   [because mappings could change while the map is unlocked]
    129         1.1       mrg  *
    130         1.1       mrg  *  alternative 1: unbusy the page in question and restart the page fault
    131         1.1       mrg  *    from the top (ReFault).   this is easy but does not take advantage
    132        1.63       chs  *    of the information that we already have from our previous lookup,
    133         1.1       mrg  *    although it is possible that the "hints" in the vm_map will help here.
    134         1.1       mrg  *
    135         1.1       mrg  * alternative 2: the system already keeps track of a "version" number of
    136         1.1       mrg  *    a map.   [i.e. every time you write-lock a map (e.g. to change a
    137         1.1       mrg  *    mapping) you bump the version number up by one...]   so, we can save
    138         1.1       mrg  *    the version number of the map before we release the lock and start I/O.
    139         1.1       mrg  *    then when I/O is done we can relock and check the version numbers
    140         1.1       mrg  *    to see if anything changed.    this might save us some over 1 because
    141         1.1       mrg  *    we don't have to unbusy the page and may be less compares(?).
    142         1.1       mrg  *
    143         1.1       mrg  * alternative 3: put in backpointers or a way to "hold" part of a map
    144         1.1       mrg  *    in place while I/O is in progress.   this could be complex to
    145         1.1       mrg  *    implement (especially with structures like amap that can be referenced
    146         1.1       mrg  *    by multiple map entries, and figuring out what should wait could be
    147         1.1       mrg  *    complex as well...).
    148         1.1       mrg  *
    149         1.1       mrg  * given that we are not currently multiprocessor or multithreaded we might
    150         1.1       mrg  * as well choose alternative 2 now.   maybe alternative 3 would be useful
    151         1.1       mrg  * in the future.    XXX keep in mind for future consideration//rechecking.
    152         1.1       mrg  */
    153         1.1       mrg 
    154         1.1       mrg /*
    155         1.1       mrg  * local data structures
    156         1.1       mrg  */
    157         1.1       mrg 
    158         1.1       mrg struct uvm_advice {
    159         1.7       mrg 	int advice;
    160         1.7       mrg 	int nback;
    161         1.7       mrg 	int nforw;
    162         1.1       mrg };
    163         1.1       mrg 
    164         1.1       mrg /*
    165         1.1       mrg  * page range array:
    166        1.63       chs  * note: index in array must match "advice" value
    167         1.1       mrg  * XXX: borrowed numbers from freebsd.   do they work well for us?
    168         1.1       mrg  */
    169         1.1       mrg 
    170        1.95   thorpej static const struct uvm_advice uvmadvice[] = {
    171         1.7       mrg 	{ MADV_NORMAL, 3, 4 },
    172         1.7       mrg 	{ MADV_RANDOM, 0, 0 },
    173         1.7       mrg 	{ MADV_SEQUENTIAL, 8, 7},
    174         1.1       mrg };
    175         1.1       mrg 
    176        1.69       chs #define UVM_MAXRANGE 16	/* must be MAX() of nback+nforw+1 */
    177         1.1       mrg 
    178         1.1       mrg /*
    179         1.1       mrg  * private prototypes
    180         1.1       mrg  */
    181         1.1       mrg 
    182         1.1       mrg /*
    183         1.1       mrg  * inline functions
    184         1.1       mrg  */
    185         1.1       mrg 
    186         1.1       mrg /*
    187         1.1       mrg  * uvmfault_anonflush: try and deactivate pages in specified anons
    188         1.1       mrg  *
    189         1.1       mrg  * => does not have to deactivate page if it is busy
    190         1.1       mrg  */
    191         1.1       mrg 
    192       1.103     perry static inline void
    193        1.95   thorpej uvmfault_anonflush(struct vm_anon **anons, int n)
    194         1.1       mrg {
    195         1.7       mrg 	int lcv;
    196         1.7       mrg 	struct vm_page *pg;
    197        1.63       chs 
    198         1.7       mrg 	for (lcv = 0 ; lcv < n ; lcv++) {
    199         1.7       mrg 		if (anons[lcv] == NULL)
    200         1.7       mrg 			continue;
    201         1.7       mrg 		simple_lock(&anons[lcv]->an_lock);
    202        1.94      yamt 		pg = anons[lcv]->an_page;
    203       1.117      yamt 		if (pg && (pg->flags & PG_BUSY) == 0) {
    204         1.7       mrg 			uvm_lock_pageq();
    205         1.7       mrg 			if (pg->wire_count == 0) {
    206        1.55   thorpej 				pmap_clear_reference(pg);
    207         1.7       mrg 				uvm_pagedeactivate(pg);
    208         1.7       mrg 			}
    209         1.7       mrg 			uvm_unlock_pageq();
    210         1.7       mrg 		}
    211         1.7       mrg 		simple_unlock(&anons[lcv]->an_lock);
    212         1.7       mrg 	}
    213         1.1       mrg }
    214         1.1       mrg 
    215         1.1       mrg /*
    216         1.1       mrg  * normal functions
    217         1.1       mrg  */
    218         1.1       mrg 
    219         1.1       mrg /*
    220         1.1       mrg  * uvmfault_amapcopy: clear "needs_copy" in a map.
    221         1.1       mrg  *
    222         1.1       mrg  * => called with VM data structures unlocked (usually, see below)
    223         1.1       mrg  * => we get a write lock on the maps and clear needs_copy for a VA
    224         1.1       mrg  * => if we are out of RAM we sleep (waiting for more)
    225         1.1       mrg  */
    226         1.1       mrg 
    227         1.7       mrg static void
    228        1.95   thorpej uvmfault_amapcopy(struct uvm_faultinfo *ufi)
    229         1.1       mrg {
    230        1.69       chs 	for (;;) {
    231         1.1       mrg 
    232         1.7       mrg 		/*
    233         1.7       mrg 		 * no mapping?  give up.
    234         1.7       mrg 		 */
    235         1.1       mrg 
    236       1.119   thorpej 		if (uvmfault_lookup(ufi, true) == false)
    237         1.7       mrg 			return;
    238         1.1       mrg 
    239         1.7       mrg 		/*
    240         1.7       mrg 		 * copy if needed.
    241         1.7       mrg 		 */
    242         1.1       mrg 
    243         1.7       mrg 		if (UVM_ET_ISNEEDSCOPY(ufi->entry))
    244       1.108      yamt 			amap_copy(ufi->map, ufi->entry, AMAP_COPY_NOWAIT,
    245        1.13     chuck 				ufi->orig_rvaddr, ufi->orig_rvaddr + 1);
    246         1.1       mrg 
    247         1.7       mrg 		/*
    248         1.7       mrg 		 * didn't work?  must be out of RAM.   unlock and sleep.
    249         1.7       mrg 		 */
    250         1.7       mrg 
    251         1.7       mrg 		if (UVM_ET_ISNEEDSCOPY(ufi->entry)) {
    252       1.119   thorpej 			uvmfault_unlockmaps(ufi, true);
    253         1.7       mrg 			uvm_wait("fltamapcopy");
    254         1.7       mrg 			continue;
    255         1.7       mrg 		}
    256         1.7       mrg 
    257         1.7       mrg 		/*
    258         1.7       mrg 		 * got it!   unlock and return.
    259         1.7       mrg 		 */
    260        1.63       chs 
    261       1.119   thorpej 		uvmfault_unlockmaps(ufi, true);
    262         1.7       mrg 		return;
    263         1.7       mrg 	}
    264         1.7       mrg 	/*NOTREACHED*/
    265         1.1       mrg }
    266         1.1       mrg 
    267         1.1       mrg /*
    268         1.1       mrg  * uvmfault_anonget: get data in an anon into a non-busy, non-released
    269         1.1       mrg  * page in that anon.
    270         1.1       mrg  *
    271         1.1       mrg  * => maps, amap, and anon locked by caller.
    272        1.57       chs  * => if we fail (result != 0) we unlock everything.
    273         1.1       mrg  * => if we are successful, we return with everything still locked.
    274         1.1       mrg  * => we don't move the page on the queues [gets moved later]
    275         1.1       mrg  * => if we allocate a new page [we_own], it gets put on the queues.
    276         1.1       mrg  *    either way, the result is that the page is on the queues at return time
    277         1.1       mrg  * => for pages which are on loan from a uvm_object (and thus are not
    278         1.1       mrg  *    owned by the anon): if successful, we return with the owning object
    279         1.1       mrg  *    locked.   the caller must unlock this object when it unlocks everything
    280         1.1       mrg  *    else.
    281         1.1       mrg  */
    282         1.1       mrg 
    283        1.47       chs int
    284        1.95   thorpej uvmfault_anonget(struct uvm_faultinfo *ufi, struct vm_amap *amap,
    285        1.95   thorpej     struct vm_anon *anon)
    286         1.7       mrg {
    287       1.118   thorpej 	bool we_own;	/* we own anon's page? */
    288       1.118   thorpej 	bool locked;	/* did we relock? */
    289         1.7       mrg 	struct vm_page *pg;
    290        1.58       chs 	int error;
    291         1.7       mrg 	UVMHIST_FUNC("uvmfault_anonget"); UVMHIST_CALLED(maphist);
    292         1.7       mrg 
    293        1.53   thorpej 	LOCK_ASSERT(simple_lock_held(&anon->an_lock));
    294        1.53   thorpej 
    295        1.58       chs 	error = 0;
    296         1.9     chuck 	uvmexp.fltanget++;
    297         1.9     chuck         /* bump rusage counters */
    298        1.94      yamt 	if (anon->an_page)
    299        1.80   thorpej 		curproc->p_stats->p_ru.ru_minflt++;
    300         1.9     chuck 	else
    301        1.80   thorpej 		curproc->p_stats->p_ru.ru_majflt++;
    302         1.7       mrg 
    303        1.63       chs 	/*
    304         1.7       mrg 	 * loop until we get it, or fail.
    305         1.7       mrg 	 */
    306         1.7       mrg 
    307        1.69       chs 	for (;;) {
    308       1.119   thorpej 		we_own = false;		/* true if we set PG_BUSY on a page */
    309        1.94      yamt 		pg = anon->an_page;
    310         1.1       mrg 
    311         1.7       mrg 		/*
    312         1.7       mrg 		 * if there is a resident page and it is loaned, then anon
    313         1.7       mrg 		 * may not own it.   call out to uvm_anon_lockpage() to ensure
    314         1.7       mrg 		 * the real owner of the page has been identified and locked.
    315         1.7       mrg 		 */
    316         1.7       mrg 
    317         1.7       mrg 		if (pg && pg->loan_count)
    318        1.13     chuck 			pg = uvm_anon_lockloanpg(anon);
    319         1.7       mrg 
    320         1.7       mrg 		/*
    321         1.7       mrg 		 * page there?   make sure it is not busy/released.
    322         1.7       mrg 		 */
    323         1.7       mrg 
    324         1.7       mrg 		if (pg) {
    325         1.7       mrg 
    326         1.7       mrg 			/*
    327         1.7       mrg 			 * at this point, if the page has a uobject [meaning
    328         1.7       mrg 			 * we have it on loan], then that uobject is locked
    329         1.7       mrg 			 * by us!   if the page is busy, we drop all the
    330         1.7       mrg 			 * locks (including uobject) and try again.
    331         1.7       mrg 			 */
    332         1.7       mrg 
    333        1.69       chs 			if ((pg->flags & PG_BUSY) == 0) {
    334         1.7       mrg 				UVMHIST_LOG(maphist, "<- OK",0,0,0,0);
    335        1.57       chs 				return (0);
    336         1.7       mrg 			}
    337         1.7       mrg 			pg->flags |= PG_WANTED;
    338         1.7       mrg 			uvmexp.fltpgwait++;
    339         1.7       mrg 
    340         1.7       mrg 			/*
    341         1.7       mrg 			 * the last unlock must be an atomic unlock+wait on
    342         1.7       mrg 			 * the owner of page
    343         1.7       mrg 			 */
    344        1.69       chs 
    345         1.7       mrg 			if (pg->uobject) {	/* owner is uobject ? */
    346         1.7       mrg 				uvmfault_unlockall(ufi, amap, NULL, anon);
    347         1.7       mrg 				UVMHIST_LOG(maphist, " unlock+wait on uobj",0,
    348         1.7       mrg 				    0,0,0);
    349         1.7       mrg 				UVM_UNLOCK_AND_WAIT(pg,
    350         1.7       mrg 				    &pg->uobject->vmobjlock,
    351       1.119   thorpej 				    false, "anonget1",0);
    352         1.7       mrg 			} else {
    353         1.7       mrg 				/* anon owns page */
    354         1.7       mrg 				uvmfault_unlockall(ufi, amap, NULL, NULL);
    355         1.7       mrg 				UVMHIST_LOG(maphist, " unlock+wait on anon",0,
    356         1.7       mrg 				    0,0,0);
    357         1.7       mrg 				UVM_UNLOCK_AND_WAIT(pg,&anon->an_lock,0,
    358         1.7       mrg 				    "anonget2",0);
    359         1.7       mrg 			}
    360         1.7       mrg 		} else {
    361       1.101      yamt #if defined(VMSWAP)
    362        1.63       chs 
    363         1.7       mrg 			/*
    364         1.7       mrg 			 * no page, we must try and bring it in.
    365         1.7       mrg 			 */
    366        1.69       chs 
    367        1.28       chs 			pg = uvm_pagealloc(NULL, 0, anon, 0);
    368         1.7       mrg 			if (pg == NULL) {		/* out of RAM.  */
    369         1.7       mrg 				uvmfault_unlockall(ufi, amap, NULL, anon);
    370         1.7       mrg 				uvmexp.fltnoram++;
    371         1.7       mrg 				UVMHIST_LOG(maphist, "  noram -- UVM_WAIT",0,
    372         1.7       mrg 				    0,0,0);
    373        1.93      yamt 				if (!uvm_reclaimable()) {
    374        1.93      yamt 					return ENOMEM;
    375        1.93      yamt 				}
    376         1.7       mrg 				uvm_wait("flt_noram1");
    377         1.7       mrg 			} else {
    378         1.7       mrg 				/* we set the PG_BUSY bit */
    379       1.119   thorpej 				we_own = true;
    380         1.7       mrg 				uvmfault_unlockall(ufi, amap, NULL, anon);
    381         1.7       mrg 
    382         1.7       mrg 				/*
    383         1.7       mrg 				 * we are passing a PG_BUSY+PG_FAKE+PG_CLEAN
    384         1.7       mrg 				 * page into the uvm_swap_get function with
    385        1.18     chuck 				 * all data structures unlocked.  note that
    386        1.18     chuck 				 * it is ok to read an_swslot here because
    387        1.18     chuck 				 * we hold PG_BUSY on the page.
    388         1.7       mrg 				 */
    389         1.7       mrg 				uvmexp.pageins++;
    390        1.58       chs 				error = uvm_swap_get(pg, anon->an_swslot,
    391         1.7       mrg 				    PGO_SYNCIO);
    392         1.7       mrg 
    393         1.7       mrg 				/*
    394         1.7       mrg 				 * we clean up after the i/o below in the
    395         1.7       mrg 				 * "we_own" case
    396         1.7       mrg 				 */
    397         1.7       mrg 			}
    398       1.101      yamt #else /* defined(VMSWAP) */
    399       1.101      yamt 			panic("%s: no page", __func__);
    400       1.101      yamt #endif /* defined(VMSWAP) */
    401         1.7       mrg 		}
    402         1.7       mrg 
    403         1.7       mrg 		/*
    404         1.7       mrg 		 * now relock and try again
    405         1.7       mrg 		 */
    406         1.7       mrg 
    407         1.7       mrg 		locked = uvmfault_relock(ufi);
    408        1.47       chs 		if (locked && amap != NULL) {
    409        1.19     chuck 			amap_lock(amap);
    410         1.7       mrg 		}
    411         1.7       mrg 		if (locked || we_own)
    412         1.7       mrg 			simple_lock(&anon->an_lock);
    413         1.7       mrg 
    414         1.7       mrg 		/*
    415         1.7       mrg 		 * if we own the page (i.e. we set PG_BUSY), then we need
    416         1.7       mrg 		 * to clean up after the I/O. there are three cases to
    417         1.7       mrg 		 * consider:
    418         1.7       mrg 		 *   [1] page released during I/O: free anon and ReFault.
    419        1.63       chs 		 *   [2] I/O not OK.   free the page and cause the fault
    420         1.7       mrg 		 *       to fail.
    421         1.7       mrg 		 *   [3] I/O OK!   activate the page and sync with the
    422         1.7       mrg 		 *       non-we_own case (i.e. drop anon lock if not locked).
    423         1.7       mrg 		 */
    424        1.63       chs 
    425         1.7       mrg 		if (we_own) {
    426       1.101      yamt #if defined(VMSWAP)
    427         1.7       mrg 			if (pg->flags & PG_WANTED) {
    428        1.63       chs 				wakeup(pg);
    429         1.7       mrg 			}
    430        1.58       chs 			if (error) {
    431         1.1       mrg 
    432        1.47       chs 				/*
    433        1.47       chs 				 * remove the swap slot from the anon
    434        1.47       chs 				 * and mark the anon as having no real slot.
    435        1.47       chs 				 * don't free the swap slot, thus preventing
    436        1.47       chs 				 * it from being used again.
    437        1.47       chs 				 */
    438        1.69       chs 
    439        1.84        pk 				if (anon->an_swslot > 0)
    440        1.84        pk 					uvm_swap_markbad(anon->an_swslot, 1);
    441        1.47       chs 				anon->an_swslot = SWSLOT_BAD;
    442        1.47       chs 
    443        1.88      yamt 				if ((pg->flags & PG_RELEASED) != 0)
    444        1.88      yamt 					goto released;
    445        1.88      yamt 
    446        1.47       chs 				/*
    447         1.7       mrg 				 * note: page was never !PG_BUSY, so it
    448         1.7       mrg 				 * can't be mapped and thus no need to
    449         1.7       mrg 				 * pmap_page_protect it...
    450         1.7       mrg 				 */
    451        1.69       chs 
    452         1.7       mrg 				uvm_lock_pageq();
    453         1.7       mrg 				uvm_pagefree(pg);
    454         1.7       mrg 				uvm_unlock_pageq();
    455         1.7       mrg 
    456         1.7       mrg 				if (locked)
    457         1.7       mrg 					uvmfault_unlockall(ufi, amap, NULL,
    458         1.7       mrg 					    anon);
    459         1.7       mrg 				else
    460         1.7       mrg 					simple_unlock(&anon->an_lock);
    461         1.7       mrg 				UVMHIST_LOG(maphist, "<- ERROR", 0,0,0,0);
    462        1.58       chs 				return error;
    463         1.7       mrg 			}
    464        1.63       chs 
    465        1.88      yamt 			if ((pg->flags & PG_RELEASED) != 0) {
    466        1.88      yamt released:
    467        1.88      yamt 				KASSERT(anon->an_ref == 0);
    468        1.88      yamt 
    469        1.88      yamt 				/*
    470        1.88      yamt 				 * released while we unlocked amap.
    471        1.88      yamt 				 */
    472        1.88      yamt 
    473        1.88      yamt 				if (locked)
    474        1.88      yamt 					uvmfault_unlockall(ufi, amap, NULL,
    475        1.88      yamt 					    NULL);
    476        1.88      yamt 
    477        1.88      yamt 				uvm_anon_release(anon);
    478        1.88      yamt 
    479        1.88      yamt 				if (error) {
    480        1.88      yamt 					UVMHIST_LOG(maphist,
    481        1.88      yamt 					    "<- ERROR/RELEASED", 0,0,0,0);
    482        1.88      yamt 					return error;
    483        1.88      yamt 				}
    484        1.88      yamt 
    485        1.88      yamt 				UVMHIST_LOG(maphist, "<- RELEASED", 0,0,0,0);
    486        1.88      yamt 				return ERESTART;
    487        1.88      yamt 			}
    488        1.88      yamt 
    489         1.7       mrg 			/*
    490        1.69       chs 			 * we've successfully read the page, activate it.
    491         1.7       mrg 			 */
    492        1.69       chs 
    493         1.7       mrg 			uvm_lock_pageq();
    494         1.7       mrg 			uvm_pageactivate(pg);
    495         1.7       mrg 			uvm_unlock_pageq();
    496        1.69       chs 			pg->flags &= ~(PG_WANTED|PG_BUSY|PG_FAKE);
    497        1.69       chs 			UVM_PAGE_OWN(pg, NULL);
    498         1.7       mrg 			if (!locked)
    499         1.7       mrg 				simple_unlock(&anon->an_lock);
    500       1.101      yamt #else /* defined(VMSWAP) */
    501       1.101      yamt 			panic("%s: we_own", __func__);
    502       1.101      yamt #endif /* defined(VMSWAP) */
    503         1.7       mrg 		}
    504         1.7       mrg 
    505         1.7       mrg 		/*
    506         1.7       mrg 		 * we were not able to relock.   restart fault.
    507         1.7       mrg 		 */
    508         1.7       mrg 
    509         1.7       mrg 		if (!locked) {
    510         1.7       mrg 			UVMHIST_LOG(maphist, "<- REFAULT", 0,0,0,0);
    511        1.57       chs 			return (ERESTART);
    512         1.7       mrg 		}
    513         1.7       mrg 
    514         1.7       mrg 		/*
    515         1.7       mrg 		 * verify no one has touched the amap and moved the anon on us.
    516         1.7       mrg 		 */
    517         1.1       mrg 
    518        1.47       chs 		if (ufi != NULL &&
    519        1.63       chs 		    amap_lookup(&ufi->entry->aref,
    520        1.47       chs 				ufi->orig_rvaddr - ufi->entry->start) != anon) {
    521        1.63       chs 
    522         1.7       mrg 			uvmfault_unlockall(ufi, amap, NULL, anon);
    523         1.7       mrg 			UVMHIST_LOG(maphist, "<- REFAULT", 0,0,0,0);
    524        1.57       chs 			return (ERESTART);
    525         1.7       mrg 		}
    526        1.63       chs 
    527         1.7       mrg 		/*
    528        1.63       chs 		 * try it again!
    529         1.7       mrg 		 */
    530         1.1       mrg 
    531         1.7       mrg 		uvmexp.fltanretry++;
    532         1.7       mrg 		continue;
    533        1.69       chs 	}
    534         1.7       mrg 	/*NOTREACHED*/
    535         1.1       mrg }
    536         1.1       mrg 
    537         1.1       mrg /*
    538       1.106      yamt  * uvmfault_promote: promote data to a new anon.  used for 1B and 2B.
    539       1.106      yamt  *
    540       1.106      yamt  *	1. allocate an anon and a page.
    541       1.106      yamt  *	2. fill its contents.
    542       1.106      yamt  *	3. put it into amap.
    543       1.106      yamt  *
    544       1.106      yamt  * => if we fail (result != 0) we unlock everything.
    545       1.106      yamt  * => on success, return a new locked anon via 'nanon'.
    546       1.106      yamt  *    (*nanon)->an_page will be a resident, locked, dirty page.
    547       1.106      yamt  */
    548       1.106      yamt 
    549       1.106      yamt static int
    550       1.106      yamt uvmfault_promote(struct uvm_faultinfo *ufi,
    551       1.106      yamt     struct vm_anon *oanon,
    552       1.106      yamt     struct vm_page *uobjpage,
    553       1.106      yamt     struct vm_anon **nanon, /* OUT: allocated anon */
    554       1.106      yamt     struct vm_anon **spare)
    555       1.106      yamt {
    556       1.106      yamt 	struct vm_amap *amap = ufi->entry->aref.ar_amap;
    557       1.106      yamt 	struct uvm_object *uobj;
    558       1.106      yamt 	struct vm_anon *anon;
    559       1.106      yamt 	struct vm_page *pg;
    560       1.106      yamt 	struct vm_page *opg;
    561       1.106      yamt 	int error;
    562       1.106      yamt 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
    563       1.106      yamt 
    564       1.106      yamt 	if (oanon) {
    565       1.106      yamt 		/* anon COW */
    566       1.106      yamt 		opg = oanon->an_page;
    567       1.106      yamt 		KASSERT(opg != NULL);
    568       1.106      yamt 		KASSERT(opg->uobject == NULL || opg->loan_count > 0);
    569       1.106      yamt 	} else if (uobjpage != PGO_DONTCARE) {
    570       1.106      yamt 		/* object-backed COW */
    571       1.106      yamt 		opg = uobjpage;
    572       1.106      yamt 	} else {
    573       1.106      yamt 		/* ZFOD */
    574       1.106      yamt 		opg = NULL;
    575       1.106      yamt 	}
    576       1.106      yamt 	if (opg != NULL) {
    577       1.106      yamt 		uobj = opg->uobject;
    578       1.106      yamt 	} else {
    579       1.106      yamt 		uobj = NULL;
    580       1.106      yamt 	}
    581       1.106      yamt 
    582       1.106      yamt 	KASSERT(amap != NULL);
    583       1.106      yamt 	KASSERT(uobjpage != NULL);
    584       1.106      yamt 	KASSERT(uobjpage == PGO_DONTCARE || (uobjpage->flags & PG_BUSY) != 0);
    585  1.119.12.1     skrll 	KASSERT(mutex_owned(&amap->am_l));
    586       1.106      yamt 	LOCK_ASSERT(oanon == NULL || simple_lock_held(&oanon->an_lock));
    587       1.106      yamt 	LOCK_ASSERT(uobj == NULL || simple_lock_held(&uobj->vmobjlock));
    588       1.106      yamt 	LOCK_ASSERT(*spare == NULL || !simple_lock_held(&(*spare)->an_lock));
    589       1.106      yamt 
    590       1.106      yamt 	if (*spare != NULL) {
    591       1.106      yamt 		anon = *spare;
    592       1.106      yamt 		*spare = NULL;
    593       1.106      yamt 		simple_lock(&anon->an_lock);
    594       1.106      yamt 	} else if (ufi->map != kernel_map) {
    595       1.106      yamt 		anon = uvm_analloc();
    596       1.106      yamt 	} else {
    597       1.106      yamt 		UVMHIST_LOG(maphist, "kernel_map, unlock and retry", 0,0,0,0);
    598       1.106      yamt 
    599       1.106      yamt 		/*
    600       1.106      yamt 		 * we can't allocate anons with kernel_map locked.
    601       1.106      yamt 		 */
    602       1.106      yamt 
    603       1.106      yamt 		uvm_page_unbusy(&uobjpage, 1);
    604       1.106      yamt 		uvmfault_unlockall(ufi, amap, uobj, oanon);
    605       1.106      yamt 
    606       1.106      yamt 		*spare = uvm_analloc();
    607       1.106      yamt 		if (*spare == NULL) {
    608       1.106      yamt 			goto nomem;
    609       1.106      yamt 		}
    610       1.106      yamt 		simple_unlock(&(*spare)->an_lock);
    611       1.106      yamt 		error = ERESTART;
    612       1.106      yamt 		goto done;
    613       1.106      yamt 	}
    614       1.106      yamt 	if (anon) {
    615       1.106      yamt 
    616       1.106      yamt 		/*
    617       1.106      yamt 		 * The new anon is locked.
    618       1.106      yamt 		 *
    619       1.106      yamt 		 * if opg == NULL, we want a zero'd, dirty page,
    620       1.106      yamt 		 * so have uvm_pagealloc() do that for us.
    621       1.106      yamt 		 */
    622       1.106      yamt 
    623       1.106      yamt 		pg = uvm_pagealloc(NULL, 0, anon,
    624       1.106      yamt 		    (opg == NULL) ? UVM_PGA_ZERO : 0);
    625       1.106      yamt 	} else {
    626       1.106      yamt 		pg = NULL;
    627       1.106      yamt 	}
    628       1.106      yamt 
    629       1.106      yamt 	/*
    630       1.106      yamt 	 * out of memory resources?
    631       1.106      yamt 	 */
    632       1.106      yamt 
    633       1.106      yamt 	if (pg == NULL) {
    634       1.106      yamt 		/* save anon for the next try. */
    635       1.106      yamt 		if (anon != NULL) {
    636       1.106      yamt 			simple_unlock(&anon->an_lock);
    637       1.106      yamt 			*spare = anon;
    638       1.106      yamt 		}
    639       1.106      yamt 
    640       1.106      yamt 		/* unlock and fail ... */
    641       1.106      yamt 		uvm_page_unbusy(&uobjpage, 1);
    642       1.106      yamt 		uvmfault_unlockall(ufi, amap, uobj, oanon);
    643       1.106      yamt nomem:
    644       1.106      yamt 		if (!uvm_reclaimable()) {
    645       1.106      yamt 			UVMHIST_LOG(maphist, "out of VM", 0,0,0,0);
    646       1.106      yamt 			uvmexp.fltnoanon++;
    647       1.106      yamt 			error = ENOMEM;
    648       1.106      yamt 			goto done;
    649       1.106      yamt 		}
    650       1.106      yamt 
    651       1.106      yamt 		UVMHIST_LOG(maphist, "out of RAM, waiting for more", 0,0,0,0);
    652       1.106      yamt 		uvmexp.fltnoram++;
    653       1.106      yamt 		uvm_wait("flt_noram5");
    654       1.106      yamt 		error = ERESTART;
    655       1.106      yamt 		goto done;
    656       1.106      yamt 	}
    657       1.106      yamt 
    658       1.106      yamt 	/* copy page [pg now dirty] */
    659       1.106      yamt 	if (opg) {
    660       1.106      yamt 		uvm_pagecopy(opg, pg);
    661       1.106      yamt 	}
    662       1.106      yamt 
    663       1.106      yamt 	amap_add(&ufi->entry->aref, ufi->orig_rvaddr - ufi->entry->start, anon,
    664       1.106      yamt 	    oanon != NULL);
    665       1.106      yamt 
    666       1.106      yamt 	*nanon = anon;
    667       1.106      yamt 	error = 0;
    668       1.106      yamt done:
    669       1.106      yamt 	return error;
    670       1.106      yamt }
    671       1.106      yamt 
    672       1.106      yamt 
    673       1.106      yamt /*
    674         1.1       mrg  *   F A U L T   -   m a i n   e n t r y   p o i n t
    675         1.1       mrg  */
    676         1.1       mrg 
    677         1.1       mrg /*
    678         1.1       mrg  * uvm_fault: page fault handler
    679         1.1       mrg  *
    680         1.1       mrg  * => called from MD code to resolve a page fault
    681        1.63       chs  * => VM data structures usually should be unlocked.   however, it is
    682         1.1       mrg  *	possible to call here with the main map locked if the caller
    683         1.1       mrg  *	gets a write lock, sets it recusive, and then calls us (c.f.
    684         1.1       mrg  *	uvm_map_pageable).   this should be avoided because it keeps
    685         1.1       mrg  *	the map locked off during I/O.
    686        1.66   thorpej  * => MUST NEVER BE CALLED IN INTERRUPT CONTEXT
    687         1.1       mrg  */
    688         1.1       mrg 
    689        1.24   mycroft #define MASK(entry)     (UVM_ET_ISCOPYONWRITE(entry) ? \
    690        1.24   mycroft 			 ~VM_PROT_WRITE : VM_PROT_ALL)
    691        1.24   mycroft 
    692       1.110  drochner /* fault_flag values passed from uvm_fault_wire to uvm_fault_internal */
    693       1.110  drochner #define UVM_FAULT_WIRE 1
    694       1.110  drochner #define UVM_FAULT_WIREMAX 2
    695       1.110  drochner 
    696         1.7       mrg int
    697       1.110  drochner uvm_fault_internal(struct vm_map *orig_map, vaddr_t vaddr,
    698       1.110  drochner     vm_prot_t access_type, int fault_flag)
    699         1.1       mrg {
    700         1.7       mrg 	struct uvm_faultinfo ufi;
    701        1.72       chs 	vm_prot_t enter_prot, check_prot;
    702       1.118   thorpej 	bool wired, narrow, promote, locked, shadowed, wire_fault, cow_now;
    703        1.58       chs 	int npages, nback, nforw, centeridx, error, lcv, gotpages;
    704        1.90      yamt 	vaddr_t startva, currva;
    705        1.79      yamt 	voff_t uoff;
    706         1.7       mrg 	struct vm_amap *amap;
    707         1.7       mrg 	struct uvm_object *uobj;
    708         1.7       mrg 	struct vm_anon *anons_store[UVM_MAXRANGE], **anons, *anon, *oanon;
    709       1.106      yamt 	struct vm_anon *anon_spare;
    710         1.7       mrg 	struct vm_page *pages[UVM_MAXRANGE], *pg, *uobjpage;
    711         1.7       mrg 	UVMHIST_FUNC("uvm_fault"); UVMHIST_CALLED(maphist);
    712         1.1       mrg 
    713       1.110  drochner 	UVMHIST_LOG(maphist, "(map=0x%x, vaddr=0x%x, at=%d, ff=%d)",
    714       1.110  drochner 	      orig_map, vaddr, access_type, fault_flag);
    715         1.1       mrg 
    716       1.106      yamt 	anon = anon_spare = NULL;
    717        1.52       chs 	pg = NULL;
    718         1.1       mrg 
    719         1.7       mrg 	uvmexp.faults++;	/* XXX: locking? */
    720         1.7       mrg 
    721         1.7       mrg 	/*
    722         1.7       mrg 	 * init the IN parameters in the ufi
    723         1.7       mrg 	 */
    724         1.1       mrg 
    725         1.7       mrg 	ufi.orig_map = orig_map;
    726         1.7       mrg 	ufi.orig_rvaddr = trunc_page(vaddr);
    727         1.7       mrg 	ufi.orig_size = PAGE_SIZE;	/* can't get any smaller than this */
    728       1.110  drochner 	wire_fault = (fault_flag > 0);
    729        1.72       chs 	if (wire_fault)
    730       1.119   thorpej 		narrow = true;		/* don't look for neighborhood
    731         1.7       mrg 					 * pages on wire */
    732         1.7       mrg 	else
    733       1.119   thorpej 		narrow = false;		/* normal fault */
    734         1.7       mrg 
    735         1.7       mrg 	/*
    736         1.7       mrg 	 * "goto ReFault" means restart the page fault from ground zero.
    737         1.7       mrg 	 */
    738         1.1       mrg ReFault:
    739         1.1       mrg 
    740         1.7       mrg 	/*
    741         1.7       mrg 	 * lookup and lock the maps
    742         1.7       mrg 	 */
    743         1.7       mrg 
    744       1.119   thorpej 	if (uvmfault_lookup(&ufi, false) == false) {
    745         1.7       mrg 		UVMHIST_LOG(maphist, "<- no mapping @ 0x%x", vaddr, 0,0,0);
    746       1.106      yamt 		error = EFAULT;
    747       1.106      yamt 		goto done;
    748         1.7       mrg 	}
    749         1.7       mrg 	/* locked: maps(read) */
    750         1.7       mrg 
    751        1.61   thorpej #ifdef DIAGNOSTIC
    752        1.61   thorpej 	if ((ufi.map->flags & VM_MAP_PAGEABLE) == 0) {
    753        1.61   thorpej 		printf("Page fault on non-pageable map:\n");
    754        1.61   thorpej 		printf("ufi.map = %p\n", ufi.map);
    755        1.61   thorpej 		printf("ufi.orig_map = %p\n", ufi.orig_map);
    756        1.61   thorpej 		printf("ufi.orig_rvaddr = 0x%lx\n", (u_long) ufi.orig_rvaddr);
    757        1.61   thorpej 		panic("uvm_fault: (ufi.map->flags & VM_MAP_PAGEABLE) == 0");
    758        1.61   thorpej 	}
    759        1.61   thorpej #endif
    760        1.58       chs 
    761         1.7       mrg 	/*
    762         1.7       mrg 	 * check protection
    763         1.7       mrg 	 */
    764         1.7       mrg 
    765       1.110  drochner 	check_prot = fault_flag == UVM_FAULT_WIREMAX ?
    766        1.72       chs 	    ufi.entry->max_protection : ufi.entry->protection;
    767        1.72       chs 	if ((check_prot & access_type) != access_type) {
    768         1.7       mrg 		UVMHIST_LOG(maphist,
    769         1.7       mrg 		    "<- protection failure (prot=0x%x, access=0x%x)",
    770         1.7       mrg 		    ufi.entry->protection, access_type, 0, 0);
    771       1.119   thorpej 		uvmfault_unlockmaps(&ufi, false);
    772       1.106      yamt 		error = EACCES;
    773       1.106      yamt 		goto done;
    774         1.7       mrg 	}
    775         1.7       mrg 
    776         1.7       mrg 	/*
    777         1.7       mrg 	 * "enter_prot" is the protection we want to enter the page in at.
    778         1.7       mrg 	 * for certain pages (e.g. copy-on-write pages) this protection can
    779         1.7       mrg 	 * be more strict than ufi.entry->protection.  "wired" means either
    780         1.7       mrg 	 * the entry is wired or we are fault-wiring the pg.
    781         1.7       mrg 	 */
    782         1.7       mrg 
    783         1.7       mrg 	enter_prot = ufi.entry->protection;
    784        1.72       chs 	wired = VM_MAPENT_ISWIRED(ufi.entry) || wire_fault;
    785        1.73       chs 	if (wired) {
    786         1.7       mrg 		access_type = enter_prot; /* full access for wired */
    787        1.73       chs 		cow_now = (check_prot & VM_PROT_WRITE) != 0;
    788        1.73       chs 	} else {
    789        1.73       chs 		cow_now = (access_type & VM_PROT_WRITE) != 0;
    790        1.73       chs 	}
    791         1.7       mrg 
    792         1.7       mrg 	/*
    793         1.7       mrg 	 * handle "needs_copy" case.   if we need to copy the amap we will
    794         1.7       mrg 	 * have to drop our readlock and relock it with a write lock.  (we
    795         1.7       mrg 	 * need a write lock to change anything in a map entry [e.g.
    796         1.7       mrg 	 * needs_copy]).
    797         1.7       mrg 	 */
    798         1.7       mrg 
    799         1.7       mrg 	if (UVM_ET_ISNEEDSCOPY(ufi.entry)) {
    800        1.73       chs 		if (cow_now || (ufi.entry->object.uvm_obj == NULL)) {
    801       1.116      yamt 			KASSERT(fault_flag != UVM_FAULT_WIREMAX);
    802         1.7       mrg 			/* need to clear */
    803         1.7       mrg 			UVMHIST_LOG(maphist,
    804         1.7       mrg 			    "  need to clear needs_copy and refault",0,0,0,0);
    805       1.119   thorpej 			uvmfault_unlockmaps(&ufi, false);
    806         1.7       mrg 			uvmfault_amapcopy(&ufi);
    807         1.7       mrg 			uvmexp.fltamcopy++;
    808         1.7       mrg 			goto ReFault;
    809         1.7       mrg 
    810         1.7       mrg 		} else {
    811         1.7       mrg 
    812         1.7       mrg 			/*
    813         1.7       mrg 			 * ensure that we pmap_enter page R/O since
    814         1.7       mrg 			 * needs_copy is still true
    815         1.7       mrg 			 */
    816        1.72       chs 
    817        1.63       chs 			enter_prot &= ~VM_PROT_WRITE;
    818         1.7       mrg 		}
    819         1.7       mrg 	}
    820         1.7       mrg 
    821         1.7       mrg 	/*
    822         1.7       mrg 	 * identify the players
    823         1.7       mrg 	 */
    824         1.7       mrg 
    825        1.46   thorpej 	amap = ufi.entry->aref.ar_amap;		/* top layer */
    826         1.7       mrg 	uobj = ufi.entry->object.uvm_obj;	/* bottom layer */
    827         1.7       mrg 
    828         1.7       mrg 	/*
    829         1.7       mrg 	 * check for a case 0 fault.  if nothing backing the entry then
    830         1.7       mrg 	 * error now.
    831         1.7       mrg 	 */
    832         1.7       mrg 
    833         1.7       mrg 	if (amap == NULL && uobj == NULL) {
    834       1.119   thorpej 		uvmfault_unlockmaps(&ufi, false);
    835         1.7       mrg 		UVMHIST_LOG(maphist,"<- no backing store, no overlay",0,0,0,0);
    836       1.106      yamt 		error = EFAULT;
    837       1.106      yamt 		goto done;
    838         1.7       mrg 	}
    839         1.1       mrg 
    840         1.7       mrg 	/*
    841         1.7       mrg 	 * establish range of interest based on advice from mapper
    842         1.7       mrg 	 * and then clip to fit map entry.   note that we only want
    843        1.63       chs 	 * to do this the first time through the fault.   if we
    844         1.7       mrg 	 * ReFault we will disable this by setting "narrow" to true.
    845         1.7       mrg 	 */
    846         1.1       mrg 
    847       1.119   thorpej 	if (narrow == false) {
    848         1.7       mrg 
    849         1.7       mrg 		/* wide fault (!narrow) */
    850        1.52       chs 		KASSERT(uvmadvice[ufi.entry->advice].advice ==
    851        1.52       chs 			 ufi.entry->advice);
    852        1.69       chs 		nback = MIN(uvmadvice[ufi.entry->advice].nback,
    853        1.15       chs 			    (ufi.orig_rvaddr - ufi.entry->start) >> PAGE_SHIFT);
    854        1.15       chs 		startva = ufi.orig_rvaddr - (nback << PAGE_SHIFT);
    855        1.69       chs 		nforw = MIN(uvmadvice[ufi.entry->advice].nforw,
    856        1.15       chs 			    ((ufi.entry->end - ufi.orig_rvaddr) >>
    857        1.15       chs 			     PAGE_SHIFT) - 1);
    858         1.7       mrg 		/*
    859         1.7       mrg 		 * note: "-1" because we don't want to count the
    860         1.7       mrg 		 * faulting page as forw
    861         1.7       mrg 		 */
    862         1.7       mrg 		npages = nback + nforw + 1;
    863         1.7       mrg 		centeridx = nback;
    864         1.7       mrg 
    865       1.119   thorpej 		narrow = true;	/* ensure only once per-fault */
    866         1.7       mrg 
    867         1.7       mrg 	} else {
    868        1.63       chs 
    869         1.7       mrg 		/* narrow fault! */
    870         1.7       mrg 		nback = nforw = 0;
    871        1.13     chuck 		startva = ufi.orig_rvaddr;
    872         1.7       mrg 		npages = 1;
    873         1.7       mrg 		centeridx = 0;
    874         1.1       mrg 
    875         1.7       mrg 	}
    876         1.1       mrg 
    877         1.7       mrg 	/* locked: maps(read) */
    878        1.13     chuck 	UVMHIST_LOG(maphist, "  narrow=%d, back=%d, forw=%d, startva=0x%x",
    879        1.16       chs 		    narrow, nback, nforw, startva);
    880         1.7       mrg 	UVMHIST_LOG(maphist, "  entry=0x%x, amap=0x%x, obj=0x%x", ufi.entry,
    881        1.16       chs 		    amap, uobj, 0);
    882         1.1       mrg 
    883         1.7       mrg 	/*
    884         1.7       mrg 	 * if we've got an amap, lock it and extract current anons.
    885         1.7       mrg 	 */
    886         1.7       mrg 
    887         1.7       mrg 	if (amap) {
    888        1.19     chuck 		amap_lock(amap);
    889         1.7       mrg 		anons = anons_store;
    890         1.7       mrg 		amap_lookups(&ufi.entry->aref, startva - ufi.entry->start,
    891         1.7       mrg 		    anons, npages);
    892         1.7       mrg 	} else {
    893         1.7       mrg 		anons = NULL;	/* to be safe */
    894         1.7       mrg 	}
    895         1.7       mrg 
    896         1.7       mrg 	/* locked: maps(read), amap(if there) */
    897  1.119.12.1     skrll 	KASSERT(amap == NULL || mutex_owned(&amap->am_l));
    898         1.7       mrg 
    899         1.7       mrg 	/*
    900         1.7       mrg 	 * for MADV_SEQUENTIAL mappings we want to deactivate the back pages
    901         1.7       mrg 	 * now and then forget about them (for the rest of the fault).
    902         1.7       mrg 	 */
    903         1.7       mrg 
    904        1.70       chs 	if (ufi.entry->advice == MADV_SEQUENTIAL && nback != 0) {
    905         1.7       mrg 
    906         1.7       mrg 		UVMHIST_LOG(maphist, "  MADV_SEQUENTIAL: flushing backpages",
    907         1.7       mrg 		    0,0,0,0);
    908         1.7       mrg 		/* flush back-page anons? */
    909        1.63       chs 		if (amap)
    910         1.7       mrg 			uvmfault_anonflush(anons, nback);
    911         1.7       mrg 
    912         1.7       mrg 		/* flush object? */
    913         1.7       mrg 		if (uobj) {
    914        1.90      yamt 			uoff = (startva - ufi.entry->start) + ufi.entry->offset;
    915         1.7       mrg 			simple_lock(&uobj->vmobjlock);
    916        1.90      yamt 			(void) (uobj->pgops->pgo_put)(uobj, uoff, uoff +
    917        1.15       chs 				    (nback << PAGE_SHIFT), PGO_DEACTIVATE);
    918         1.7       mrg 		}
    919         1.7       mrg 
    920         1.7       mrg 		/* now forget about the backpages */
    921         1.7       mrg 		if (amap)
    922         1.7       mrg 			anons += nback;
    923        1.52       chs 		startva += (nback << PAGE_SHIFT);
    924         1.7       mrg 		npages -= nback;
    925         1.7       mrg 		nback = centeridx = 0;
    926         1.7       mrg 	}
    927         1.7       mrg 
    928         1.7       mrg 	/* locked: maps(read), amap(if there) */
    929  1.119.12.1     skrll 	KASSERT(amap == NULL || mutex_owned(&amap->am_l));
    930         1.1       mrg 
    931         1.7       mrg 	/*
    932         1.7       mrg 	 * map in the backpages and frontpages we found in the amap in hopes
    933         1.7       mrg 	 * of preventing future faults.    we also init the pages[] array as
    934         1.7       mrg 	 * we go.
    935         1.7       mrg 	 */
    936         1.7       mrg 
    937        1.13     chuck 	currva = startva;
    938       1.119   thorpej 	shadowed = false;
    939         1.7       mrg 	for (lcv = 0 ; lcv < npages ; lcv++, currva += PAGE_SIZE) {
    940         1.7       mrg 
    941         1.7       mrg 		/*
    942         1.7       mrg 		 * dont play with VAs that are already mapped
    943        1.13     chuck 		 * except for center)
    944         1.7       mrg 		 */
    945        1.52       chs 		if (lcv != centeridx &&
    946        1.89      yamt 		    pmap_extract(ufi.orig_map->pmap, currva, NULL)) {
    947        1.52       chs 			pages[lcv] = PGO_DONTCARE;
    948        1.52       chs 			continue;
    949         1.7       mrg 		}
    950         1.7       mrg 
    951         1.7       mrg 		/*
    952         1.7       mrg 		 * unmapped or center page.   check if any anon at this level.
    953         1.7       mrg 		 */
    954         1.7       mrg 		if (amap == NULL || anons[lcv] == NULL) {
    955         1.7       mrg 			pages[lcv] = NULL;
    956         1.7       mrg 			continue;
    957         1.7       mrg 		}
    958         1.7       mrg 
    959         1.7       mrg 		/*
    960         1.7       mrg 		 * check for present page and map if possible.   re-activate it.
    961         1.7       mrg 		 */
    962         1.7       mrg 
    963         1.7       mrg 		pages[lcv] = PGO_DONTCARE;
    964         1.7       mrg 		if (lcv == centeridx) {		/* save center for later! */
    965       1.119   thorpej 			shadowed = true;
    966         1.7       mrg 			continue;
    967         1.7       mrg 		}
    968         1.7       mrg 		anon = anons[lcv];
    969         1.7       mrg 		simple_lock(&anon->an_lock);
    970         1.7       mrg 		/* ignore loaned pages */
    971        1.94      yamt 		if (anon->an_page && anon->an_page->loan_count == 0 &&
    972        1.94      yamt 		    (anon->an_page->flags & PG_BUSY) == 0) {
    973         1.7       mrg 			uvm_lock_pageq();
    974       1.112      yamt 			uvm_pageenqueue(anon->an_page);
    975         1.7       mrg 			uvm_unlock_pageq();
    976         1.7       mrg 			UVMHIST_LOG(maphist,
    977         1.7       mrg 			    "  MAPPING: n anon: pm=0x%x, va=0x%x, pg=0x%x",
    978        1.94      yamt 			    ufi.orig_map->pmap, currva, anon->an_page, 0);
    979         1.7       mrg 			uvmexp.fltnamap++;
    980        1.52       chs 
    981        1.46   thorpej 			/*
    982        1.46   thorpej 			 * Since this isn't the page that's actually faulting,
    983        1.46   thorpej 			 * ignore pmap_enter() failures; it's not critical
    984        1.46   thorpej 			 * that we enter these right now.
    985        1.46   thorpej 			 */
    986        1.52       chs 
    987        1.46   thorpej 			(void) pmap_enter(ufi.orig_map->pmap, currva,
    988        1.94      yamt 			    VM_PAGE_TO_PHYS(anon->an_page),
    989        1.25   mycroft 			    (anon->an_ref > 1) ? (enter_prot & ~VM_PROT_WRITE) :
    990        1.46   thorpej 			    enter_prot,
    991        1.46   thorpej 			    PMAP_CANFAIL |
    992        1.46   thorpej 			     (VM_MAPENT_ISWIRED(ufi.entry) ? PMAP_WIRED : 0));
    993         1.7       mrg 		}
    994         1.7       mrg 		simple_unlock(&anon->an_lock);
    995        1.68     chris 		pmap_update(ufi.orig_map->pmap);
    996         1.7       mrg 	}
    997         1.7       mrg 
    998         1.7       mrg 	/* locked: maps(read), amap(if there) */
    999  1.119.12.1     skrll 	KASSERT(amap == NULL || mutex_owned(&amap->am_l));
   1000       1.119   thorpej 	/* (shadowed == true) if there is an anon at the faulting address */
   1001        1.63       chs 	UVMHIST_LOG(maphist, "  shadowed=%d, will_get=%d", shadowed,
   1002       1.119   thorpej 	    (uobj && shadowed == false),0,0);
   1003         1.1       mrg 
   1004         1.7       mrg 	/*
   1005         1.7       mrg 	 * note that if we are really short of RAM we could sleep in the above
   1006         1.7       mrg 	 * call to pmap_enter with everything locked.   bad?
   1007        1.46   thorpej 	 *
   1008        1.46   thorpej 	 * XXX Actually, that is bad; pmap_enter() should just fail in that
   1009        1.46   thorpej 	 * XXX case.  --thorpej
   1010         1.7       mrg 	 */
   1011        1.63       chs 
   1012         1.7       mrg 	/*
   1013         1.7       mrg 	 * if the desired page is not shadowed by the amap and we have a
   1014         1.7       mrg 	 * backing object, then we check to see if the backing object would
   1015         1.7       mrg 	 * prefer to handle the fault itself (rather than letting us do it
   1016         1.7       mrg 	 * with the usual pgo_get hook).  the backing object signals this by
   1017         1.7       mrg 	 * providing a pgo_fault routine.
   1018         1.7       mrg 	 */
   1019         1.1       mrg 
   1020       1.119   thorpej 	if (uobj && shadowed == false && uobj->pgops->pgo_fault != NULL) {
   1021         1.7       mrg 		simple_lock(&uobj->vmobjlock);
   1022         1.1       mrg 
   1023         1.7       mrg 		/* locked: maps(read), amap (if there), uobj */
   1024        1.58       chs 		error = uobj->pgops->pgo_fault(&ufi, startva, pages, npages,
   1025       1.109  drochner 		    centeridx, access_type, PGO_LOCKED|PGO_SYNCIO);
   1026        1.52       chs 
   1027         1.7       mrg 		/* locked: nothing, pgo_fault has unlocked everything */
   1028         1.7       mrg 
   1029        1.59       chs 		if (error == ERESTART)
   1030         1.7       mrg 			goto ReFault;		/* try again! */
   1031        1.61   thorpej 		/*
   1032        1.61   thorpej 		 * object fault routine responsible for pmap_update().
   1033        1.61   thorpej 		 */
   1034       1.106      yamt 		goto done;
   1035         1.7       mrg 	}
   1036         1.7       mrg 
   1037         1.7       mrg 	/*
   1038         1.7       mrg 	 * now, if the desired page is not shadowed by the amap and we have
   1039         1.7       mrg 	 * a backing object that does not have a special fault routine, then
   1040         1.7       mrg 	 * we ask (with pgo_get) the object for resident pages that we care
   1041         1.7       mrg 	 * about and attempt to map them in.  we do not let pgo_get block
   1042         1.7       mrg 	 * (PGO_LOCKED).
   1043         1.7       mrg 	 */
   1044         1.7       mrg 
   1045       1.119   thorpej 	if (uobj && shadowed == false) {
   1046         1.7       mrg 		simple_lock(&uobj->vmobjlock);
   1047         1.1       mrg 
   1048         1.7       mrg 		/* locked (!shadowed): maps(read), amap (if there), uobj */
   1049         1.7       mrg 		/*
   1050         1.7       mrg 		 * the following call to pgo_get does _not_ change locking state
   1051         1.7       mrg 		 */
   1052         1.7       mrg 
   1053         1.7       mrg 		uvmexp.fltlget++;
   1054         1.7       mrg 		gotpages = npages;
   1055        1.52       chs 		(void) uobj->pgops->pgo_get(uobj, ufi.entry->offset +
   1056         1.1       mrg 				(startva - ufi.entry->start),
   1057         1.1       mrg 				pages, &gotpages, centeridx,
   1058        1.24   mycroft 				access_type & MASK(ufi.entry),
   1059         1.1       mrg 				ufi.entry->advice, PGO_LOCKED);
   1060         1.1       mrg 
   1061         1.7       mrg 		/*
   1062         1.7       mrg 		 * check for pages to map, if we got any
   1063         1.7       mrg 		 */
   1064         1.7       mrg 
   1065         1.7       mrg 		uobjpage = NULL;
   1066         1.7       mrg 
   1067         1.7       mrg 		if (gotpages) {
   1068        1.13     chuck 			currva = startva;
   1069        1.69       chs 			for (lcv = 0; lcv < npages;
   1070        1.69       chs 			     lcv++, currva += PAGE_SIZE) {
   1071        1.86      yamt 				struct vm_page *curpg;
   1072       1.118   thorpej 				bool readonly;
   1073        1.86      yamt 
   1074        1.86      yamt 				curpg = pages[lcv];
   1075        1.86      yamt 				if (curpg == NULL || curpg == PGO_DONTCARE) {
   1076         1.7       mrg 					continue;
   1077        1.69       chs 				}
   1078       1.107      yamt 				KASSERT(curpg->uobject == uobj);
   1079         1.1       mrg 
   1080        1.52       chs 				/*
   1081        1.52       chs 				 * if center page is resident and not
   1082        1.52       chs 				 * PG_BUSY|PG_RELEASED then pgo_get
   1083        1.52       chs 				 * made it PG_BUSY for us and gave
   1084        1.52       chs 				 * us a handle to it.   remember this
   1085        1.52       chs 				 * page as "uobjpage." (for later use).
   1086        1.52       chs 				 */
   1087        1.63       chs 
   1088        1.52       chs 				if (lcv == centeridx) {
   1089        1.86      yamt 					uobjpage = curpg;
   1090        1.52       chs 					UVMHIST_LOG(maphist, "  got uobjpage "
   1091        1.63       chs 					    "(0x%x) with locked get",
   1092         1.7       mrg 					    uobjpage, 0,0,0);
   1093        1.52       chs 					continue;
   1094         1.7       mrg 				}
   1095        1.63       chs 
   1096        1.63       chs 				/*
   1097        1.69       chs 				 * calling pgo_get with PGO_LOCKED returns us
   1098        1.69       chs 				 * pages which are neither busy nor released,
   1099        1.69       chs 				 * so we don't need to check for this.
   1100        1.69       chs 				 * we can just directly enter the pages.
   1101         1.7       mrg 				 */
   1102         1.7       mrg 
   1103         1.7       mrg 				uvm_lock_pageq();
   1104       1.112      yamt 				uvm_pageenqueue(curpg);
   1105         1.7       mrg 				uvm_unlock_pageq();
   1106         1.7       mrg 				UVMHIST_LOG(maphist,
   1107         1.7       mrg 				  "  MAPPING: n obj: pm=0x%x, va=0x%x, pg=0x%x",
   1108        1.86      yamt 				  ufi.orig_map->pmap, currva, curpg, 0);
   1109         1.7       mrg 				uvmexp.fltnomap++;
   1110        1.52       chs 
   1111        1.46   thorpej 				/*
   1112        1.46   thorpej 				 * Since this page isn't the page that's
   1113        1.85       dbj 				 * actually faulting, ignore pmap_enter()
   1114        1.46   thorpej 				 * failures; it's not critical that we
   1115        1.46   thorpej 				 * enter these right now.
   1116        1.46   thorpej 				 */
   1117        1.86      yamt 				KASSERT((curpg->flags & PG_PAGEOUT) == 0);
   1118        1.86      yamt 				KASSERT((curpg->flags & PG_RELEASED) == 0);
   1119        1.97      yamt 				KASSERT(!UVM_OBJ_IS_CLEAN(curpg->uobject) ||
   1120        1.96      yamt 				    (curpg->flags & PG_CLEAN) != 0);
   1121        1.86      yamt 				readonly = (curpg->flags & PG_RDONLY)
   1122        1.96      yamt 				    || (curpg->loan_count > 0)
   1123        1.98      yamt 				    || UVM_OBJ_NEEDS_WRITEFAULT(curpg->uobject);
   1124        1.52       chs 
   1125        1.46   thorpej 				(void) pmap_enter(ufi.orig_map->pmap, currva,
   1126        1.86      yamt 				    VM_PAGE_TO_PHYS(curpg),
   1127        1.86      yamt 				    readonly ?
   1128        1.76       chs 				    enter_prot & ~VM_PROT_WRITE :
   1129        1.76       chs 				    enter_prot & MASK(ufi.entry),
   1130        1.46   thorpej 				    PMAP_CANFAIL |
   1131        1.46   thorpej 				     (wired ? PMAP_WIRED : 0));
   1132         1.7       mrg 
   1133        1.63       chs 				/*
   1134         1.7       mrg 				 * NOTE: page can't be PG_WANTED or PG_RELEASED
   1135         1.7       mrg 				 * because we've held the lock the whole time
   1136         1.7       mrg 				 * we've had the handle.
   1137         1.7       mrg 				 */
   1138       1.111      yamt 				KASSERT((curpg->flags & PG_WANTED) == 0);
   1139       1.111      yamt 				KASSERT((curpg->flags & PG_RELEASED) == 0);
   1140        1.52       chs 
   1141        1.86      yamt 				curpg->flags &= ~(PG_BUSY);
   1142        1.86      yamt 				UVM_PAGE_OWN(curpg, NULL);
   1143        1.69       chs 			}
   1144        1.68     chris 			pmap_update(ufi.orig_map->pmap);
   1145        1.69       chs 		}
   1146         1.7       mrg 	} else {
   1147         1.7       mrg 		uobjpage = NULL;
   1148         1.7       mrg 	}
   1149         1.7       mrg 
   1150         1.7       mrg 	/* locked (shadowed): maps(read), amap */
   1151        1.63       chs 	/* locked (!shadowed): maps(read), amap(if there),
   1152         1.7       mrg 		 uobj(if !null), uobjpage(if !null) */
   1153       1.111      yamt 	if (shadowed) {
   1154  1.119.12.1     skrll 		KASSERT(mutex_owned(&amap->am_l));
   1155       1.111      yamt 	} else {
   1156  1.119.12.1     skrll 		KASSERT(amap == NULL || mutex_owned(&amap->am_l));
   1157       1.111      yamt 		LOCK_ASSERT(uobj == NULL || simple_lock_held(&uobj->vmobjlock));
   1158       1.111      yamt 		KASSERT(uobjpage == NULL || (uobjpage->flags & PG_BUSY) != 0);
   1159       1.111      yamt 	}
   1160         1.7       mrg 
   1161         1.7       mrg 	/*
   1162         1.7       mrg 	 * note that at this point we are done with any front or back pages.
   1163         1.7       mrg 	 * we are now going to focus on the center page (i.e. the one we've
   1164         1.7       mrg 	 * faulted on).  if we have faulted on the top (anon) layer
   1165         1.7       mrg 	 * [i.e. case 1], then the anon we want is anons[centeridx] (we have
   1166         1.7       mrg 	 * not touched it yet).  if we have faulted on the bottom (uobj)
   1167         1.7       mrg 	 * layer [i.e. case 2] and the page was both present and available,
   1168         1.7       mrg 	 * then we've got a pointer to it as "uobjpage" and we've already
   1169         1.8     chuck 	 * made it BUSY.
   1170         1.7       mrg 	 */
   1171         1.7       mrg 
   1172         1.7       mrg 	/*
   1173         1.7       mrg 	 * there are four possible cases we must address: 1A, 1B, 2A, and 2B
   1174         1.7       mrg 	 */
   1175         1.7       mrg 
   1176         1.7       mrg 	/*
   1177         1.7       mrg 	 * redirect case 2: if we are not shadowed, go to case 2.
   1178         1.7       mrg 	 */
   1179         1.7       mrg 
   1180       1.119   thorpej 	if (shadowed == false)
   1181         1.7       mrg 		goto Case2;
   1182         1.7       mrg 
   1183         1.7       mrg 	/* locked: maps(read), amap */
   1184         1.7       mrg 
   1185         1.7       mrg 	/*
   1186         1.7       mrg 	 * handle case 1: fault on an anon in our amap
   1187         1.7       mrg 	 */
   1188         1.7       mrg 
   1189         1.7       mrg 	anon = anons[centeridx];
   1190         1.7       mrg 	UVMHIST_LOG(maphist, "  case 1 fault: anon=0x%x", anon, 0,0,0);
   1191         1.7       mrg 	simple_lock(&anon->an_lock);
   1192         1.7       mrg 
   1193         1.7       mrg 	/* locked: maps(read), amap, anon */
   1194  1.119.12.1     skrll 	KASSERT(mutex_owned(&amap->am_l));
   1195       1.111      yamt 	LOCK_ASSERT(simple_lock_held(&anon->an_lock));
   1196         1.7       mrg 
   1197         1.7       mrg 	/*
   1198         1.7       mrg 	 * no matter if we have case 1A or case 1B we are going to need to
   1199         1.7       mrg 	 * have the anon's memory resident.   ensure that now.
   1200         1.7       mrg 	 */
   1201         1.7       mrg 
   1202         1.7       mrg 	/*
   1203        1.47       chs 	 * let uvmfault_anonget do the dirty work.
   1204        1.51   thorpej 	 * if it fails (!OK) it will unlock everything for us.
   1205        1.47       chs 	 * if it succeeds, locks are still valid and locked.
   1206         1.7       mrg 	 * also, if it is OK, then the anon's page is on the queues.
   1207         1.7       mrg 	 * if the page is on loan from a uvm_object, then anonget will
   1208         1.7       mrg 	 * lock that object for us if it does not fail.
   1209         1.7       mrg 	 */
   1210         1.7       mrg 
   1211        1.58       chs 	error = uvmfault_anonget(&ufi, amap, anon);
   1212        1.58       chs 	switch (error) {
   1213        1.57       chs 	case 0:
   1214        1.63       chs 		break;
   1215         1.7       mrg 
   1216        1.57       chs 	case ERESTART:
   1217         1.7       mrg 		goto ReFault;
   1218         1.7       mrg 
   1219        1.57       chs 	case EAGAIN:
   1220        1.52       chs 		tsleep(&lbolt, PVM, "fltagain1", 0);
   1221        1.52       chs 		goto ReFault;
   1222        1.51   thorpej 
   1223        1.51   thorpej 	default:
   1224       1.106      yamt 		goto done;
   1225         1.1       mrg 	}
   1226         1.7       mrg 
   1227         1.7       mrg 	/*
   1228         1.7       mrg 	 * uobj is non null if the page is on loan from an object (i.e. uobj)
   1229         1.7       mrg 	 */
   1230         1.7       mrg 
   1231        1.94      yamt 	uobj = anon->an_page->uobject;	/* locked by anonget if !NULL */
   1232         1.7       mrg 
   1233         1.7       mrg 	/* locked: maps(read), amap, anon, uobj(if one) */
   1234  1.119.12.1     skrll 	KASSERT(mutex_owned(&amap->am_l));
   1235       1.111      yamt 	LOCK_ASSERT(simple_lock_held(&anon->an_lock));
   1236       1.111      yamt 	LOCK_ASSERT(uobj == NULL || simple_lock_held(&uobj->vmobjlock));
   1237         1.7       mrg 
   1238         1.7       mrg 	/*
   1239        1.63       chs 	 * special handling for loaned pages
   1240         1.7       mrg 	 */
   1241        1.52       chs 
   1242        1.94      yamt 	if (anon->an_page->loan_count) {
   1243         1.7       mrg 
   1244        1.73       chs 		if (!cow_now) {
   1245        1.63       chs 
   1246         1.7       mrg 			/*
   1247         1.7       mrg 			 * for read faults on loaned pages we just cap the
   1248         1.7       mrg 			 * protection at read-only.
   1249         1.7       mrg 			 */
   1250         1.7       mrg 
   1251         1.7       mrg 			enter_prot = enter_prot & ~VM_PROT_WRITE;
   1252         1.7       mrg 
   1253         1.7       mrg 		} else {
   1254         1.7       mrg 			/*
   1255         1.7       mrg 			 * note that we can't allow writes into a loaned page!
   1256         1.7       mrg 			 *
   1257         1.7       mrg 			 * if we have a write fault on a loaned page in an
   1258         1.7       mrg 			 * anon then we need to look at the anon's ref count.
   1259         1.7       mrg 			 * if it is greater than one then we are going to do
   1260         1.7       mrg 			 * a normal copy-on-write fault into a new anon (this
   1261         1.7       mrg 			 * is not a problem).  however, if the reference count
   1262         1.7       mrg 			 * is one (a case where we would normally allow a
   1263         1.7       mrg 			 * write directly to the page) then we need to kill
   1264         1.7       mrg 			 * the loan before we continue.
   1265         1.7       mrg 			 */
   1266         1.7       mrg 
   1267         1.7       mrg 			/* >1 case is already ok */
   1268         1.7       mrg 			if (anon->an_ref == 1) {
   1269         1.7       mrg 
   1270         1.7       mrg 				/* get new un-owned replacement page */
   1271        1.28       chs 				pg = uvm_pagealloc(NULL, 0, NULL, 0);
   1272         1.7       mrg 				if (pg == NULL) {
   1273         1.7       mrg 					uvmfault_unlockall(&ufi, amap, uobj,
   1274         1.7       mrg 					    anon);
   1275         1.7       mrg 					uvm_wait("flt_noram2");
   1276         1.7       mrg 					goto ReFault;
   1277         1.7       mrg 				}
   1278         1.7       mrg 
   1279         1.7       mrg 				/*
   1280         1.7       mrg 				 * copy data, kill loan, and drop uobj lock
   1281         1.7       mrg 				 * (if any)
   1282         1.7       mrg 				 */
   1283         1.7       mrg 				/* copy old -> new */
   1284        1.94      yamt 				uvm_pagecopy(anon->an_page, pg);
   1285         1.7       mrg 
   1286         1.7       mrg 				/* force reload */
   1287       1.115      yamt 				pmap_page_protect(anon->an_page, VM_PROT_NONE);
   1288         1.7       mrg 				uvm_lock_pageq();	  /* KILL loan */
   1289        1.78   thorpej 
   1290        1.94      yamt 				anon->an_page->uanon = NULL;
   1291         1.7       mrg 				/* in case we owned */
   1292        1.94      yamt 				anon->an_page->pqflags &= ~PQ_ANON;
   1293        1.78   thorpej 
   1294        1.78   thorpej 				if (uobj) {
   1295        1.78   thorpej 					/* if we were receiver of loan */
   1296        1.94      yamt 					anon->an_page->loan_count--;
   1297        1.78   thorpej 				} else {
   1298        1.78   thorpej 					/*
   1299        1.78   thorpej 					 * we were the lender (A->K); need
   1300        1.78   thorpej 					 * to remove the page from pageq's.
   1301        1.78   thorpej 					 */
   1302        1.94      yamt 					uvm_pagedequeue(anon->an_page);
   1303        1.78   thorpej 				}
   1304        1.78   thorpej 
   1305         1.7       mrg 				if (uobj) {
   1306         1.7       mrg 					simple_unlock(&uobj->vmobjlock);
   1307         1.7       mrg 					uobj = NULL;
   1308         1.7       mrg 				}
   1309         1.7       mrg 
   1310         1.7       mrg 				/* install new page in anon */
   1311        1.94      yamt 				anon->an_page = pg;
   1312         1.7       mrg 				pg->uanon = anon;
   1313         1.7       mrg 				pg->pqflags |= PQ_ANON;
   1314       1.112      yamt 
   1315       1.112      yamt 				uvm_pageactivate(pg);
   1316       1.112      yamt 				uvm_unlock_pageq();
   1317       1.112      yamt 
   1318         1.7       mrg 				pg->flags &= ~(PG_BUSY|PG_FAKE);
   1319         1.7       mrg 				UVM_PAGE_OWN(pg, NULL);
   1320         1.7       mrg 
   1321         1.7       mrg 				/* done! */
   1322         1.7       mrg 			}     /* ref == 1 */
   1323         1.7       mrg 		}       /* write fault */
   1324         1.7       mrg 	}         /* loan count */
   1325         1.7       mrg 
   1326         1.7       mrg 	/*
   1327         1.7       mrg 	 * if we are case 1B then we will need to allocate a new blank
   1328         1.7       mrg 	 * anon to transfer the data into.   note that we have a lock
   1329         1.7       mrg 	 * on anon, so no one can busy or release the page until we are done.
   1330         1.7       mrg 	 * also note that the ref count can't drop to zero here because
   1331         1.7       mrg 	 * it is > 1 and we are only dropping one ref.
   1332         1.7       mrg 	 *
   1333        1.63       chs 	 * in the (hopefully very rare) case that we are out of RAM we
   1334        1.63       chs 	 * will unlock, wait for more RAM, and refault.
   1335         1.7       mrg 	 *
   1336         1.7       mrg 	 * if we are out of anon VM we kill the process (XXX: could wait?).
   1337         1.7       mrg 	 */
   1338         1.7       mrg 
   1339        1.73       chs 	if (cow_now && anon->an_ref > 1) {
   1340         1.7       mrg 
   1341         1.7       mrg 		UVMHIST_LOG(maphist, "  case 1B: COW fault",0,0,0,0);
   1342         1.7       mrg 		uvmexp.flt_acow++;
   1343         1.7       mrg 		oanon = anon;		/* oanon = old, locked anon */
   1344        1.22       chs 
   1345       1.106      yamt 		error = uvmfault_promote(&ufi, oanon, PGO_DONTCARE,
   1346       1.106      yamt 		    &anon, &anon_spare);
   1347       1.106      yamt 		switch (error) {
   1348       1.106      yamt 		case 0:
   1349       1.106      yamt 			break;
   1350       1.106      yamt 		case ERESTART:
   1351         1.7       mrg 			goto ReFault;
   1352       1.106      yamt 		default:
   1353       1.106      yamt 			goto done;
   1354         1.7       mrg 		}
   1355         1.7       mrg 
   1356       1.106      yamt 		pg = anon->an_page;
   1357        1.81        pk 		uvm_lock_pageq();
   1358        1.69       chs 		uvm_pageactivate(pg);
   1359       1.106      yamt 		uvm_unlock_pageq();
   1360       1.105      yamt 		pg->flags &= ~(PG_BUSY|PG_FAKE);
   1361         1.7       mrg 		UVM_PAGE_OWN(pg, NULL);
   1362         1.7       mrg 
   1363         1.7       mrg 		/* deref: can not drop to zero here by defn! */
   1364         1.7       mrg 		oanon->an_ref--;
   1365        1.53   thorpej 
   1366         1.7       mrg 		/*
   1367        1.53   thorpej 		 * note: oanon is still locked, as is the new anon.  we
   1368        1.53   thorpej 		 * need to check for this later when we unlock oanon; if
   1369        1.53   thorpej 		 * oanon != anon, we'll have to unlock anon, too.
   1370         1.7       mrg 		 */
   1371         1.7       mrg 
   1372         1.7       mrg 	} else {
   1373        1.52       chs 
   1374         1.7       mrg 		uvmexp.flt_anon++;
   1375         1.7       mrg 		oanon = anon;		/* old, locked anon is same as anon */
   1376        1.94      yamt 		pg = anon->an_page;
   1377         1.7       mrg 		if (anon->an_ref > 1)     /* disallow writes to ref > 1 anons */
   1378         1.7       mrg 			enter_prot = enter_prot & ~VM_PROT_WRITE;
   1379         1.7       mrg 
   1380         1.7       mrg 	}
   1381         1.7       mrg 
   1382        1.53   thorpej 	/* locked: maps(read), amap, oanon, anon (if different from oanon) */
   1383  1.119.12.1     skrll 	KASSERT(mutex_owned(&amap->am_l));
   1384       1.111      yamt 	LOCK_ASSERT(simple_lock_held(&anon->an_lock));
   1385       1.111      yamt 	LOCK_ASSERT(simple_lock_held(&oanon->an_lock));
   1386         1.7       mrg 
   1387         1.7       mrg 	/*
   1388        1.69       chs 	 * now map the page in.
   1389         1.7       mrg 	 */
   1390         1.7       mrg 
   1391         1.7       mrg 	UVMHIST_LOG(maphist, "  MAPPING: anon: pm=0x%x, va=0x%x, pg=0x%x",
   1392         1.7       mrg 	    ufi.orig_map->pmap, ufi.orig_rvaddr, pg, 0);
   1393        1.46   thorpej 	if (pmap_enter(ufi.orig_map->pmap, ufi.orig_rvaddr, VM_PAGE_TO_PHYS(pg),
   1394        1.46   thorpej 	    enter_prot, access_type | PMAP_CANFAIL | (wired ? PMAP_WIRED : 0))
   1395        1.58       chs 	    != 0) {
   1396        1.69       chs 
   1397        1.46   thorpej 		/*
   1398        1.46   thorpej 		 * No need to undo what we did; we can simply think of
   1399        1.46   thorpej 		 * this as the pmap throwing away the mapping information.
   1400        1.46   thorpej 		 *
   1401        1.46   thorpej 		 * We do, however, have to go through the ReFault path,
   1402        1.46   thorpej 		 * as the map may change while we're asleep.
   1403        1.46   thorpej 		 */
   1404        1.69       chs 
   1405        1.53   thorpej 		if (anon != oanon)
   1406        1.53   thorpej 			simple_unlock(&anon->an_lock);
   1407        1.46   thorpej 		uvmfault_unlockall(&ufi, amap, uobj, oanon);
   1408        1.92      yamt 		if (!uvm_reclaimable()) {
   1409        1.46   thorpej 			UVMHIST_LOG(maphist,
   1410        1.46   thorpej 			    "<- failed.  out of VM",0,0,0,0);
   1411        1.46   thorpej 			/* XXX instrumentation */
   1412       1.106      yamt 			error = ENOMEM;
   1413       1.106      yamt 			goto done;
   1414        1.46   thorpej 		}
   1415        1.46   thorpej 		/* XXX instrumentation */
   1416        1.46   thorpej 		uvm_wait("flt_pmfail1");
   1417        1.46   thorpej 		goto ReFault;
   1418        1.46   thorpej 	}
   1419         1.7       mrg 
   1420         1.7       mrg 	/*
   1421        1.46   thorpej 	 * ... update the page queues.
   1422         1.7       mrg 	 */
   1423         1.7       mrg 
   1424         1.7       mrg 	uvm_lock_pageq();
   1425        1.72       chs 	if (wire_fault) {
   1426         1.8     chuck 		uvm_pagewire(pg);
   1427        1.29       chs 
   1428        1.29       chs 		/*
   1429        1.29       chs 		 * since the now-wired page cannot be paged out,
   1430        1.29       chs 		 * release its swap resources for others to use.
   1431        1.29       chs 		 * since an anon with no swap cannot be PG_CLEAN,
   1432        1.29       chs 		 * clear its clean flag now.
   1433        1.29       chs 		 */
   1434        1.29       chs 
   1435        1.29       chs 		pg->flags &= ~(PG_CLEAN);
   1436        1.22       chs 		uvm_anon_dropswap(anon);
   1437         1.7       mrg 	} else {
   1438         1.7       mrg 		uvm_pageactivate(pg);
   1439         1.7       mrg 	}
   1440         1.7       mrg 	uvm_unlock_pageq();
   1441         1.7       mrg 
   1442         1.7       mrg 	/*
   1443         1.7       mrg 	 * done case 1!  finish up by unlocking everything and returning success
   1444         1.7       mrg 	 */
   1445         1.1       mrg 
   1446        1.53   thorpej 	if (anon != oanon)
   1447        1.53   thorpej 		simple_unlock(&anon->an_lock);
   1448         1.7       mrg 	uvmfault_unlockall(&ufi, amap, uobj, oanon);
   1449        1.68     chris 	pmap_update(ufi.orig_map->pmap);
   1450       1.106      yamt 	error = 0;
   1451       1.106      yamt 	goto done;
   1452         1.1       mrg 
   1453         1.1       mrg Case2:
   1454         1.7       mrg 	/*
   1455         1.7       mrg 	 * handle case 2: faulting on backing object or zero fill
   1456         1.7       mrg 	 */
   1457         1.7       mrg 
   1458         1.7       mrg 	/*
   1459         1.7       mrg 	 * locked:
   1460         1.7       mrg 	 * maps(read), amap(if there), uobj(if !null), uobjpage(if !null)
   1461         1.7       mrg 	 */
   1462  1.119.12.1     skrll 	KASSERT(amap == NULL || mutex_owned(&amap->am_l));
   1463       1.111      yamt 	LOCK_ASSERT(uobj == NULL || simple_lock_held(&uobj->vmobjlock));
   1464  1.119.12.1     skrll 	KASSERT(uobjpage == NULL || (uobjpage->flags & PG_BUSY) != 0);
   1465         1.7       mrg 
   1466         1.7       mrg 	/*
   1467         1.7       mrg 	 * note that uobjpage can not be PGO_DONTCARE at this point.  we now
   1468         1.7       mrg 	 * set uobjpage to PGO_DONTCARE if we are doing a zero fill.  if we
   1469         1.7       mrg 	 * have a backing object, check and see if we are going to promote
   1470         1.7       mrg 	 * the data up to an anon during the fault.
   1471         1.7       mrg 	 */
   1472         1.7       mrg 
   1473         1.7       mrg 	if (uobj == NULL) {
   1474        1.63       chs 		uobjpage = PGO_DONTCARE;
   1475       1.119   thorpej 		promote = true;		/* always need anon here */
   1476         1.7       mrg 	} else {
   1477        1.52       chs 		KASSERT(uobjpage != PGO_DONTCARE);
   1478        1.73       chs 		promote = cow_now && UVM_ET_ISCOPYONWRITE(ufi.entry);
   1479         1.7       mrg 	}
   1480         1.7       mrg 	UVMHIST_LOG(maphist, "  case 2 fault: promote=%d, zfill=%d",
   1481        1.46   thorpej 	    promote, (uobj == NULL), 0,0);
   1482         1.1       mrg 
   1483         1.7       mrg 	/*
   1484         1.9     chuck 	 * if uobjpage is not null then we do not need to do I/O to get the
   1485         1.9     chuck 	 * uobjpage.
   1486         1.9     chuck 	 *
   1487        1.63       chs 	 * if uobjpage is null, then we need to unlock and ask the pager to
   1488         1.7       mrg 	 * get the data for us.   once we have the data, we need to reverify
   1489         1.7       mrg 	 * the state the world.   we are currently not holding any resources.
   1490         1.7       mrg 	 */
   1491         1.1       mrg 
   1492         1.9     chuck 	if (uobjpage) {
   1493         1.9     chuck 		/* update rusage counters */
   1494        1.80   thorpej 		curproc->p_stats->p_ru.ru_minflt++;
   1495         1.9     chuck 	} else {
   1496         1.9     chuck 		/* update rusage counters */
   1497        1.80   thorpej 		curproc->p_stats->p_ru.ru_majflt++;
   1498        1.63       chs 
   1499         1.7       mrg 		/* locked: maps(read), amap(if there), uobj */
   1500         1.7       mrg 		uvmfault_unlockall(&ufi, amap, NULL, NULL);
   1501         1.7       mrg 		/* locked: uobj */
   1502         1.7       mrg 
   1503         1.7       mrg 		uvmexp.fltget++;
   1504         1.7       mrg 		gotpages = 1;
   1505        1.52       chs 		uoff = (ufi.orig_rvaddr - ufi.entry->start) + ufi.entry->offset;
   1506        1.58       chs 		error = uobj->pgops->pgo_get(uobj, uoff, &uobjpage, &gotpages,
   1507        1.52       chs 		    0, access_type & MASK(ufi.entry), ufi.entry->advice,
   1508        1.52       chs 		    PGO_SYNCIO);
   1509        1.58       chs 		/* locked: uobjpage(if no error) */
   1510  1.119.12.1     skrll 		KASSERT(error != 0 || (uobjpage->flags & PG_BUSY) != 0);
   1511        1.52       chs 
   1512         1.7       mrg 		/*
   1513         1.7       mrg 		 * recover from I/O
   1514         1.7       mrg 		 */
   1515         1.1       mrg 
   1516        1.58       chs 		if (error) {
   1517        1.58       chs 			if (error == EAGAIN) {
   1518        1.46   thorpej 				UVMHIST_LOG(maphist,
   1519        1.46   thorpej 				    "  pgo_get says TRY AGAIN!",0,0,0,0);
   1520        1.57       chs 				tsleep(&lbolt, PVM, "fltagain2", 0);
   1521        1.46   thorpej 				goto ReFault;
   1522         1.7       mrg 			}
   1523         1.1       mrg 
   1524         1.7       mrg 			UVMHIST_LOG(maphist, "<- pgo_get failed (code %d)",
   1525        1.58       chs 			    error, 0,0,0);
   1526       1.106      yamt 			goto done;
   1527         1.7       mrg 		}
   1528         1.7       mrg 
   1529         1.7       mrg 		/* locked: uobjpage */
   1530         1.7       mrg 
   1531        1.69       chs 		uvm_lock_pageq();
   1532        1.69       chs 		uvm_pageactivate(uobjpage);
   1533        1.69       chs 		uvm_unlock_pageq();
   1534        1.69       chs 
   1535         1.7       mrg 		/*
   1536         1.7       mrg 		 * re-verify the state of the world by first trying to relock
   1537         1.7       mrg 		 * the maps.  always relock the object.
   1538         1.7       mrg 		 */
   1539         1.7       mrg 
   1540         1.7       mrg 		locked = uvmfault_relock(&ufi);
   1541         1.7       mrg 		if (locked && amap)
   1542        1.19     chuck 			amap_lock(amap);
   1543       1.107      yamt 		uobj = uobjpage->uobject;
   1544         1.7       mrg 		simple_lock(&uobj->vmobjlock);
   1545        1.63       chs 
   1546         1.7       mrg 		/* locked(locked): maps(read), amap(if !null), uobj, uobjpage */
   1547         1.7       mrg 		/* locked(!locked): uobj, uobjpage */
   1548         1.7       mrg 
   1549         1.7       mrg 		/*
   1550         1.7       mrg 		 * verify that the page has not be released and re-verify
   1551         1.7       mrg 		 * that amap slot is still free.   if there is a problem,
   1552         1.7       mrg 		 * we unlock and clean up.
   1553         1.7       mrg 		 */
   1554         1.7       mrg 
   1555         1.7       mrg 		if ((uobjpage->flags & PG_RELEASED) != 0 ||
   1556        1.63       chs 		    (locked && amap &&
   1557         1.7       mrg 		    amap_lookup(&ufi.entry->aref,
   1558        1.13     chuck 		      ufi.orig_rvaddr - ufi.entry->start))) {
   1559        1.63       chs 			if (locked)
   1560         1.7       mrg 				uvmfault_unlockall(&ufi, amap, NULL, NULL);
   1561       1.119   thorpej 			locked = false;
   1562         1.7       mrg 		}
   1563         1.7       mrg 
   1564         1.7       mrg 		/*
   1565         1.7       mrg 		 * didn't get the lock?   release the page and retry.
   1566         1.7       mrg 		 */
   1567         1.7       mrg 
   1568       1.119   thorpej 		if (locked == false) {
   1569         1.7       mrg 			UVMHIST_LOG(maphist,
   1570        1.63       chs 			    "  wasn't able to relock after fault: retry",
   1571         1.7       mrg 			    0,0,0,0);
   1572         1.7       mrg 			if (uobjpage->flags & PG_WANTED)
   1573        1.44   thorpej 				wakeup(uobjpage);
   1574         1.7       mrg 			if (uobjpage->flags & PG_RELEASED) {
   1575         1.7       mrg 				uvmexp.fltpgrele++;
   1576        1.69       chs 				uvm_pagefree(uobjpage);
   1577         1.7       mrg 				goto ReFault;
   1578         1.7       mrg 			}
   1579         1.7       mrg 			uobjpage->flags &= ~(PG_BUSY|PG_WANTED);
   1580         1.7       mrg 			UVM_PAGE_OWN(uobjpage, NULL);
   1581         1.7       mrg 			simple_unlock(&uobj->vmobjlock);
   1582         1.7       mrg 			goto ReFault;
   1583         1.7       mrg 		}
   1584         1.7       mrg 
   1585         1.7       mrg 		/*
   1586        1.69       chs 		 * we have the data in uobjpage which is busy and
   1587        1.69       chs 		 * not released.  we are holding object lock (so the page
   1588         1.7       mrg 		 * can't be released on us).
   1589         1.7       mrg 		 */
   1590         1.7       mrg 
   1591         1.7       mrg 		/* locked: maps(read), amap(if !null), uobj, uobjpage */
   1592         1.7       mrg 	}
   1593         1.1       mrg 
   1594         1.1       mrg 	/*
   1595         1.7       mrg 	 * locked:
   1596         1.7       mrg 	 * maps(read), amap(if !null), uobj(if !null), uobjpage(if uobj)
   1597         1.1       mrg 	 */
   1598  1.119.12.1     skrll 	KASSERT(amap == NULL || mutex_owned(&amap->am_l));
   1599       1.111      yamt 	LOCK_ASSERT(uobj == NULL || simple_lock_held(&uobj->vmobjlock));
   1600  1.119.12.1     skrll 	KASSERT(uobj == NULL || (uobjpage->flags & PG_BUSY) != 0);
   1601         1.1       mrg 
   1602         1.7       mrg 	/*
   1603         1.7       mrg 	 * notes:
   1604         1.7       mrg 	 *  - at this point uobjpage can not be NULL
   1605         1.7       mrg 	 *  - at this point uobjpage can not be PG_RELEASED (since we checked
   1606         1.7       mrg 	 *  for it above)
   1607         1.7       mrg 	 *  - at this point uobjpage could be PG_WANTED (handle later)
   1608         1.7       mrg 	 */
   1609        1.63       chs 
   1610       1.107      yamt 	KASSERT(uobj == NULL || uobj == uobjpage->uobject);
   1611        1.97      yamt 	KASSERT(uobj == NULL || !UVM_OBJ_IS_CLEAN(uobjpage->uobject) ||
   1612        1.96      yamt 	    (uobjpage->flags & PG_CLEAN) != 0);
   1613       1.119   thorpej 	if (promote == false) {
   1614         1.1       mrg 
   1615         1.7       mrg 		/*
   1616         1.7       mrg 		 * we are not promoting.   if the mapping is COW ensure that we
   1617         1.7       mrg 		 * don't give more access than we should (e.g. when doing a read
   1618         1.7       mrg 		 * fault on a COPYONWRITE mapping we want to map the COW page in
   1619         1.7       mrg 		 * R/O even though the entry protection could be R/W).
   1620         1.7       mrg 		 *
   1621         1.7       mrg 		 * set "pg" to the page we want to map in (uobjpage, usually)
   1622         1.7       mrg 		 */
   1623         1.7       mrg 
   1624        1.53   thorpej 		/* no anon in this case. */
   1625        1.53   thorpej 		anon = NULL;
   1626        1.53   thorpej 
   1627         1.7       mrg 		uvmexp.flt_obj++;
   1628        1.96      yamt 		if (UVM_ET_ISCOPYONWRITE(ufi.entry) ||
   1629        1.98      yamt 		    UVM_OBJ_NEEDS_WRITEFAULT(uobjpage->uobject))
   1630        1.24   mycroft 			enter_prot &= ~VM_PROT_WRITE;
   1631         1.7       mrg 		pg = uobjpage;		/* map in the actual object */
   1632         1.7       mrg 
   1633       1.111      yamt 		KASSERT(uobjpage != PGO_DONTCARE);
   1634         1.7       mrg 
   1635         1.7       mrg 		/*
   1636         1.7       mrg 		 * we are faulting directly on the page.   be careful
   1637         1.7       mrg 		 * about writing to loaned pages...
   1638         1.7       mrg 		 */
   1639        1.69       chs 
   1640         1.7       mrg 		if (uobjpage->loan_count) {
   1641        1.73       chs 			if (!cow_now) {
   1642         1.7       mrg 				/* read fault: cap the protection at readonly */
   1643         1.7       mrg 				/* cap! */
   1644         1.7       mrg 				enter_prot = enter_prot & ~VM_PROT_WRITE;
   1645         1.7       mrg 			} else {
   1646         1.7       mrg 				/* write fault: must break the loan here */
   1647         1.7       mrg 
   1648        1.82      yamt 				pg = uvm_loanbreak(uobjpage);
   1649         1.7       mrg 				if (pg == NULL) {
   1650        1.69       chs 
   1651         1.7       mrg 					/*
   1652         1.7       mrg 					 * drop ownership of page, it can't
   1653         1.7       mrg 					 * be released
   1654        1.46   thorpej 					 */
   1655        1.69       chs 
   1656         1.7       mrg 					if (uobjpage->flags & PG_WANTED)
   1657        1.44   thorpej 						wakeup(uobjpage);
   1658         1.7       mrg 					uobjpage->flags &= ~(PG_BUSY|PG_WANTED);
   1659         1.7       mrg 					UVM_PAGE_OWN(uobjpage, NULL);
   1660         1.7       mrg 
   1661         1.7       mrg 					uvmfault_unlockall(&ufi, amap, uobj,
   1662         1.7       mrg 					  NULL);
   1663         1.7       mrg 					UVMHIST_LOG(maphist,
   1664        1.20       mrg 					  "  out of RAM breaking loan, waiting",
   1665        1.20       mrg 					  0,0,0,0);
   1666         1.7       mrg 					uvmexp.fltnoram++;
   1667         1.7       mrg 					uvm_wait("flt_noram4");
   1668         1.7       mrg 					goto ReFault;
   1669         1.7       mrg 				}
   1670         1.7       mrg 				uobjpage = pg;
   1671        1.69       chs 			}
   1672        1.69       chs 		}
   1673         1.7       mrg 	} else {
   1674        1.63       chs 
   1675         1.7       mrg 		/*
   1676         1.7       mrg 		 * if we are going to promote the data to an anon we
   1677         1.7       mrg 		 * allocate a blank anon here and plug it into our amap.
   1678         1.7       mrg 		 */
   1679         1.1       mrg #if DIAGNOSTIC
   1680         1.7       mrg 		if (amap == NULL)
   1681         1.7       mrg 			panic("uvm_fault: want to promote data, but no anon");
   1682         1.1       mrg #endif
   1683       1.106      yamt 		error = uvmfault_promote(&ufi, NULL, uobjpage,
   1684       1.106      yamt 		    &anon, &anon_spare);
   1685       1.106      yamt 		switch (error) {
   1686       1.106      yamt 		case 0:
   1687       1.106      yamt 			break;
   1688       1.106      yamt 		case ERESTART:
   1689       1.106      yamt 			goto ReFault;
   1690       1.106      yamt 		default:
   1691       1.106      yamt 			goto done;
   1692       1.105      yamt 		}
   1693       1.105      yamt 
   1694       1.106      yamt 		pg = anon->an_page;
   1695         1.1       mrg 
   1696         1.7       mrg 		/*
   1697         1.7       mrg 		 * fill in the data
   1698         1.7       mrg 		 */
   1699         1.7       mrg 
   1700         1.7       mrg 		if (uobjpage != PGO_DONTCARE) {
   1701         1.7       mrg 			uvmexp.flt_prcopy++;
   1702         1.7       mrg 
   1703         1.7       mrg 			/*
   1704         1.7       mrg 			 * promote to shared amap?  make sure all sharing
   1705         1.7       mrg 			 * procs see it
   1706         1.7       mrg 			 */
   1707        1.69       chs 
   1708        1.19     chuck 			if ((amap_flags(amap) & AMAP_SHARED) != 0) {
   1709        1.45       chs 				pmap_page_protect(uobjpage, VM_PROT_NONE);
   1710        1.62   thorpej 				/*
   1711        1.62   thorpej 				 * XXX: PAGE MIGHT BE WIRED!
   1712        1.62   thorpej 				 */
   1713         1.7       mrg 			}
   1714        1.63       chs 
   1715         1.7       mrg 			/*
   1716         1.7       mrg 			 * dispose of uobjpage.  it can't be PG_RELEASED
   1717        1.52       chs 			 * since we still hold the object lock.
   1718        1.52       chs 			 * drop handle to uobj as well.
   1719         1.7       mrg 			 */
   1720         1.7       mrg 
   1721         1.7       mrg 			if (uobjpage->flags & PG_WANTED)
   1722         1.7       mrg 				/* still have the obj lock */
   1723        1.44   thorpej 				wakeup(uobjpage);
   1724         1.7       mrg 			uobjpage->flags &= ~(PG_BUSY|PG_WANTED);
   1725         1.7       mrg 			UVM_PAGE_OWN(uobjpage, NULL);
   1726         1.7       mrg 			simple_unlock(&uobj->vmobjlock);
   1727         1.7       mrg 			uobj = NULL;
   1728        1.52       chs 
   1729         1.7       mrg 			UVMHIST_LOG(maphist,
   1730         1.7       mrg 			    "  promote uobjpage 0x%x to anon/page 0x%x/0x%x",
   1731         1.7       mrg 			    uobjpage, anon, pg, 0);
   1732         1.7       mrg 
   1733         1.7       mrg 		} else {
   1734         1.7       mrg 			uvmexp.flt_przero++;
   1735        1.69       chs 
   1736        1.48   thorpej 			/*
   1737       1.106      yamt 			 * Page is zero'd and marked dirty by
   1738       1.106      yamt 			 * uvmfault_promote().
   1739        1.48   thorpej 			 */
   1740        1.69       chs 
   1741         1.7       mrg 			UVMHIST_LOG(maphist,"  zero fill anon/page 0x%x/0%x",
   1742         1.7       mrg 			    anon, pg, 0, 0);
   1743         1.7       mrg 		}
   1744         1.7       mrg 	}
   1745         1.7       mrg 
   1746         1.7       mrg 	/*
   1747         1.7       mrg 	 * locked:
   1748        1.53   thorpej 	 * maps(read), amap(if !null), uobj(if !null), uobjpage(if uobj),
   1749        1.53   thorpej 	 *   anon(if !null), pg(if anon)
   1750         1.7       mrg 	 *
   1751         1.7       mrg 	 * note: pg is either the uobjpage or the new page in the new anon
   1752         1.7       mrg 	 */
   1753  1.119.12.1     skrll 	KASSERT(amap == NULL || mutex_owned(&amap->am_l));
   1754       1.111      yamt 	LOCK_ASSERT(uobj == NULL || simple_lock_held(&uobj->vmobjlock));
   1755  1.119.12.1     skrll 	KASSERT(uobj == NULL || (uobjpage->flags & PG_BUSY) != 0);
   1756       1.111      yamt 	LOCK_ASSERT(anon == NULL || simple_lock_held(&anon->an_lock));
   1757  1.119.12.1     skrll 	KASSERT((pg->flags & PG_BUSY) != 0);
   1758         1.7       mrg 
   1759         1.7       mrg 	/*
   1760         1.7       mrg 	 * all resources are present.   we can now map it in and free our
   1761         1.7       mrg 	 * resources.
   1762         1.7       mrg 	 */
   1763         1.7       mrg 
   1764         1.7       mrg 	UVMHIST_LOG(maphist,
   1765         1.7       mrg 	    "  MAPPING: case2: pm=0x%x, va=0x%x, pg=0x%x, promote=%d",
   1766         1.7       mrg 	    ufi.orig_map->pmap, ufi.orig_rvaddr, pg, promote);
   1767        1.75       chs 	KASSERT((access_type & VM_PROT_WRITE) == 0 ||
   1768        1.75       chs 		(pg->flags & PG_RDONLY) == 0);
   1769        1.46   thorpej 	if (pmap_enter(ufi.orig_map->pmap, ufi.orig_rvaddr, VM_PAGE_TO_PHYS(pg),
   1770        1.76       chs 	    pg->flags & PG_RDONLY ? enter_prot & ~VM_PROT_WRITE : enter_prot,
   1771        1.58       chs 	    access_type | PMAP_CANFAIL | (wired ? PMAP_WIRED : 0)) != 0) {
   1772        1.52       chs 
   1773        1.46   thorpej 		/*
   1774        1.46   thorpej 		 * No need to undo what we did; we can simply think of
   1775        1.46   thorpej 		 * this as the pmap throwing away the mapping information.
   1776        1.46   thorpej 		 *
   1777        1.46   thorpej 		 * We do, however, have to go through the ReFault path,
   1778        1.46   thorpej 		 * as the map may change while we're asleep.
   1779        1.46   thorpej 		 */
   1780        1.52       chs 
   1781        1.46   thorpej 		if (pg->flags & PG_WANTED)
   1782        1.69       chs 			wakeup(pg);
   1783        1.46   thorpej 
   1784        1.63       chs 		/*
   1785        1.46   thorpej 		 * note that pg can't be PG_RELEASED since we did not drop
   1786        1.46   thorpej 		 * the object lock since the last time we checked.
   1787        1.46   thorpej 		 */
   1788       1.111      yamt 		KASSERT((pg->flags & PG_RELEASED) == 0);
   1789        1.63       chs 
   1790        1.46   thorpej 		pg->flags &= ~(PG_BUSY|PG_FAKE|PG_WANTED);
   1791        1.46   thorpej 		UVM_PAGE_OWN(pg, NULL);
   1792        1.53   thorpej 		uvmfault_unlockall(&ufi, amap, uobj, anon);
   1793        1.92      yamt 		if (!uvm_reclaimable()) {
   1794        1.46   thorpej 			UVMHIST_LOG(maphist,
   1795        1.46   thorpej 			    "<- failed.  out of VM",0,0,0,0);
   1796        1.46   thorpej 			/* XXX instrumentation */
   1797       1.106      yamt 			error = ENOMEM;
   1798       1.106      yamt 			goto done;
   1799        1.46   thorpej 		}
   1800        1.46   thorpej 		/* XXX instrumentation */
   1801        1.46   thorpej 		uvm_wait("flt_pmfail2");
   1802        1.46   thorpej 		goto ReFault;
   1803        1.46   thorpej 	}
   1804         1.1       mrg 
   1805         1.1       mrg 	uvm_lock_pageq();
   1806        1.72       chs 	if (wire_fault) {
   1807         1.8     chuck 		uvm_pagewire(pg);
   1808        1.22       chs 		if (pg->pqflags & PQ_AOBJ) {
   1809        1.29       chs 
   1810        1.29       chs 			/*
   1811        1.29       chs 			 * since the now-wired page cannot be paged out,
   1812        1.29       chs 			 * release its swap resources for others to use.
   1813        1.29       chs 			 * since an aobj page with no swap cannot be PG_CLEAN,
   1814        1.29       chs 			 * clear its clean flag now.
   1815        1.29       chs 			 */
   1816        1.29       chs 
   1817       1.113  christos 			KASSERT(uobj != NULL);
   1818        1.29       chs 			pg->flags &= ~(PG_CLEAN);
   1819        1.22       chs 			uao_dropswap(uobj, pg->offset >> PAGE_SHIFT);
   1820        1.22       chs 		}
   1821         1.7       mrg 	} else {
   1822         1.7       mrg 		uvm_pageactivate(pg);
   1823         1.7       mrg 	}
   1824         1.1       mrg 	uvm_unlock_pageq();
   1825         1.7       mrg 	if (pg->flags & PG_WANTED)
   1826        1.69       chs 		wakeup(pg);
   1827         1.7       mrg 
   1828        1.63       chs 	/*
   1829        1.63       chs 	 * note that pg can't be PG_RELEASED since we did not drop the object
   1830         1.7       mrg 	 * lock since the last time we checked.
   1831         1.7       mrg 	 */
   1832       1.111      yamt 	KASSERT((pg->flags & PG_RELEASED) == 0);
   1833        1.63       chs 
   1834         1.7       mrg 	pg->flags &= ~(PG_BUSY|PG_FAKE|PG_WANTED);
   1835         1.7       mrg 	UVM_PAGE_OWN(pg, NULL);
   1836        1.53   thorpej 	uvmfault_unlockall(&ufi, amap, uobj, anon);
   1837        1.68     chris 	pmap_update(ufi.orig_map->pmap);
   1838         1.7       mrg 	UVMHIST_LOG(maphist, "<- done (SUCCESS!)",0,0,0,0);
   1839       1.106      yamt 	error = 0;
   1840       1.106      yamt done:
   1841       1.106      yamt 	if (anon_spare != NULL) {
   1842       1.106      yamt 		anon_spare->an_ref--;
   1843       1.106      yamt 		uvm_anfree(anon_spare);
   1844       1.106      yamt 	}
   1845       1.106      yamt 	return error;
   1846         1.1       mrg }
   1847         1.1       mrg 
   1848       1.110  drochner 
   1849         1.1       mrg /*
   1850         1.1       mrg  * uvm_fault_wire: wire down a range of virtual addresses in a map.
   1851         1.1       mrg  *
   1852        1.36   thorpej  * => map may be read-locked by caller, but MUST NOT be write-locked.
   1853        1.36   thorpej  * => if map is read-locked, any operations which may cause map to
   1854        1.36   thorpej  *	be write-locked in uvm_fault() must be taken care of by
   1855        1.36   thorpej  *	the caller.  See uvm_map_pageable().
   1856         1.1       mrg  */
   1857         1.1       mrg 
   1858         1.7       mrg int
   1859        1.95   thorpej uvm_fault_wire(struct vm_map *map, vaddr_t start, vaddr_t end,
   1860       1.110  drochner     vm_prot_t access_type, int wiremax)
   1861         1.7       mrg {
   1862        1.12       eeh 	vaddr_t va;
   1863        1.58       chs 	int error;
   1864         1.7       mrg 
   1865         1.7       mrg 	/*
   1866        1.47       chs 	 * now fault it in a page at a time.   if the fault fails then we have
   1867        1.63       chs 	 * to undo what we have done.   note that in uvm_fault VM_PROT_NONE
   1868        1.47       chs 	 * is replaced with the max protection if fault_type is VM_FAULT_WIRE.
   1869         1.7       mrg 	 */
   1870         1.1       mrg 
   1871        1.65       chs 	/*
   1872        1.65       chs 	 * XXX work around overflowing a vaddr_t.  this prevents us from
   1873        1.65       chs 	 * wiring the last page in the address space, though.
   1874        1.65       chs 	 */
   1875        1.65       chs 	if (start > end) {
   1876        1.65       chs 		return EFAULT;
   1877        1.65       chs 	}
   1878        1.65       chs 
   1879         1.7       mrg 	for (va = start ; va < end ; va += PAGE_SIZE) {
   1880       1.110  drochner 		error = uvm_fault_internal(map, va, access_type,
   1881       1.110  drochner 				wiremax ? UVM_FAULT_WIREMAX : UVM_FAULT_WIRE);
   1882        1.58       chs 		if (error) {
   1883         1.7       mrg 			if (va != start) {
   1884        1.31   thorpej 				uvm_fault_unwire(map, start, va);
   1885         1.7       mrg 			}
   1886        1.58       chs 			return error;
   1887         1.7       mrg 		}
   1888         1.7       mrg 	}
   1889        1.58       chs 	return 0;
   1890         1.1       mrg }
   1891         1.1       mrg 
   1892         1.1       mrg /*
   1893         1.1       mrg  * uvm_fault_unwire(): unwire range of virtual space.
   1894         1.1       mrg  */
   1895         1.1       mrg 
   1896         1.7       mrg void
   1897        1.95   thorpej uvm_fault_unwire(struct vm_map *map, vaddr_t start, vaddr_t end)
   1898        1.36   thorpej {
   1899        1.36   thorpej 	vm_map_lock_read(map);
   1900        1.36   thorpej 	uvm_fault_unwire_locked(map, start, end);
   1901        1.36   thorpej 	vm_map_unlock_read(map);
   1902        1.36   thorpej }
   1903        1.36   thorpej 
   1904        1.36   thorpej /*
   1905        1.36   thorpej  * uvm_fault_unwire_locked(): the guts of uvm_fault_unwire().
   1906        1.36   thorpej  *
   1907        1.36   thorpej  * => map must be at least read-locked.
   1908        1.36   thorpej  */
   1909        1.36   thorpej 
   1910        1.36   thorpej void
   1911        1.95   thorpej uvm_fault_unwire_locked(struct vm_map *map, vaddr_t start, vaddr_t end)
   1912         1.7       mrg {
   1913        1.64       chs 	struct vm_map_entry *entry;
   1914        1.31   thorpej 	pmap_t pmap = vm_map_pmap(map);
   1915        1.42   thorpej 	vaddr_t va;
   1916        1.12       eeh 	paddr_t pa;
   1917        1.42   thorpej 	struct vm_page *pg;
   1918        1.31   thorpej 
   1919        1.52       chs 	KASSERT((map->flags & VM_MAP_INTRSAFE) == 0);
   1920         1.7       mrg 
   1921         1.7       mrg 	/*
   1922         1.7       mrg 	 * we assume that the area we are unwiring has actually been wired
   1923         1.7       mrg 	 * in the first place.   this means that we should be able to extract
   1924         1.7       mrg 	 * the PAs from the pmap.   we also lock out the page daemon so that
   1925         1.7       mrg 	 * we can call uvm_pageunwire.
   1926         1.7       mrg 	 */
   1927        1.37   thorpej 
   1928         1.7       mrg 	uvm_lock_pageq();
   1929         1.7       mrg 
   1930        1.37   thorpej 	/*
   1931        1.37   thorpej 	 * find the beginning map entry for the region.
   1932        1.37   thorpej 	 */
   1933        1.74       chs 
   1934        1.56       chs 	KASSERT(start >= vm_map_min(map) && end <= vm_map_max(map));
   1935       1.119   thorpej 	if (uvm_map_lookup_entry(map, start, &entry) == false)
   1936        1.37   thorpej 		panic("uvm_fault_unwire_locked: address not in map");
   1937        1.37   thorpej 
   1938        1.69       chs 	for (va = start; va < end; va += PAGE_SIZE) {
   1939       1.119   thorpej 		if (pmap_extract(pmap, va, &pa) == false)
   1940        1.74       chs 			continue;
   1941        1.42   thorpej 
   1942        1.42   thorpej 		/*
   1943        1.74       chs 		 * find the map entry for the current address.
   1944        1.42   thorpej 		 */
   1945        1.56       chs 
   1946        1.56       chs 		KASSERT(va >= entry->start);
   1947        1.74       chs 		while (va >= entry->end) {
   1948        1.56       chs 			KASSERT(entry->next != &map->header &&
   1949        1.56       chs 				entry->next->start <= entry->end);
   1950        1.42   thorpej 			entry = entry->next;
   1951        1.42   thorpej 		}
   1952        1.37   thorpej 
   1953        1.42   thorpej 		/*
   1954        1.42   thorpej 		 * if the entry is no longer wired, tell the pmap.
   1955        1.42   thorpej 		 */
   1956        1.74       chs 
   1957        1.42   thorpej 		if (VM_MAPENT_ISWIRED(entry) == 0)
   1958        1.42   thorpej 			pmap_unwire(pmap, va);
   1959        1.42   thorpej 
   1960        1.42   thorpej 		pg = PHYS_TO_VM_PAGE(pa);
   1961        1.42   thorpej 		if (pg)
   1962        1.42   thorpej 			uvm_pageunwire(pg);
   1963         1.7       mrg 	}
   1964         1.1       mrg 
   1965         1.7       mrg 	uvm_unlock_pageq();
   1966         1.1       mrg }
   1967