Home | History | Annotate | Line # | Download | only in uvm
uvm_anon.c revision 1.65
      1  1.65        ad /*	$NetBSD: uvm_anon.c,v 1.65 2019/12/01 14:40:31 ad Exp $	*/
      2   1.1     chuck 
      3   1.1     chuck /*
      4   1.1     chuck  * Copyright (c) 1997 Charles D. Cranor and Washington University.
      5   1.1     chuck  * All rights reserved.
      6   1.1     chuck  *
      7   1.1     chuck  * Redistribution and use in source and binary forms, with or without
      8   1.1     chuck  * modification, are permitted provided that the following conditions
      9   1.1     chuck  * are met:
     10   1.1     chuck  * 1. Redistributions of source code must retain the above copyright
     11   1.1     chuck  *    notice, this list of conditions and the following disclaimer.
     12   1.1     chuck  * 2. Redistributions in binary form must reproduce the above copyright
     13   1.1     chuck  *    notice, this list of conditions and the following disclaimer in the
     14   1.1     chuck  *    documentation and/or other materials provided with the distribution.
     15   1.1     chuck  *
     16   1.1     chuck  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
     17   1.1     chuck  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
     18   1.1     chuck  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
     19   1.1     chuck  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
     20   1.1     chuck  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
     21   1.1     chuck  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
     22   1.1     chuck  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
     23   1.1     chuck  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
     24   1.1     chuck  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
     25   1.1     chuck  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     26   1.1     chuck  */
     27   1.1     chuck 
     28   1.1     chuck /*
     29   1.1     chuck  * uvm_anon.c: uvm anon ops
     30   1.1     chuck  */
     31  1.21     lukem 
     32  1.21     lukem #include <sys/cdefs.h>
     33  1.65        ad __KERNEL_RCSID(0, "$NetBSD: uvm_anon.c,v 1.65 2019/12/01 14:40:31 ad Exp $");
     34   1.1     chuck 
     35   1.1     chuck #include "opt_uvmhist.h"
     36   1.1     chuck 
     37   1.1     chuck #include <sys/param.h>
     38   1.1     chuck #include <sys/systm.h>
     39   1.1     chuck #include <sys/pool.h>
     40   1.5       chs #include <sys/kernel.h>
     41   1.1     chuck 
     42   1.1     chuck #include <uvm/uvm.h>
     43   1.1     chuck #include <uvm/uvm_swap.h>
     44  1.39      yamt #include <uvm/uvm_pdpolicy.h>
     45   1.1     chuck 
     46  1.55     rmind static struct pool_cache	uvm_anon_cache;
     47   1.5       chs 
     48  1.55     rmind static int			uvm_anon_ctor(void *, void *, int);
     49   1.5       chs 
     50   1.1     chuck void
     51  1.34   thorpej uvm_anon_init(void)
     52   1.1     chuck {
     53   1.5       chs 
     54  1.49        ad 	pool_cache_bootstrap(&uvm_anon_cache, sizeof(struct vm_anon), 0, 0,
     55  1.49        ad 	    PR_LARGECACHE, "anonpl", NULL, IPL_NONE, uvm_anon_ctor,
     56  1.55     rmind 	    NULL, NULL);
     57   1.1     chuck }
     58   1.1     chuck 
     59  1.33      yamt static int
     60  1.41      yamt uvm_anon_ctor(void *arg, void *object, int flags)
     61   1.1     chuck {
     62  1.33      yamt 	struct vm_anon *anon = object;
     63   1.1     chuck 
     64  1.33      yamt 	anon->an_ref = 0;
     65  1.59     rmind 	anon->an_lock = NULL;
     66  1.33      yamt 	anon->an_page = NULL;
     67  1.37      yamt #if defined(VMSWAP)
     68  1.33      yamt 	anon->an_swslot = 0;
     69  1.55     rmind #endif
     70  1.11       chs 	return 0;
     71   1.1     chuck }
     72   1.1     chuck 
     73   1.1     chuck /*
     74  1.55     rmind  * uvm_analloc: allocate a new anon.
     75  1.13   thorpej  *
     76  1.55     rmind  * => anon will have no lock associated.
     77   1.1     chuck  */
     78   1.1     chuck struct vm_anon *
     79  1.34   thorpej uvm_analloc(void)
     80   1.1     chuck {
     81  1.33      yamt 	struct vm_anon *anon;
     82   1.1     chuck 
     83  1.47        ad 	anon = pool_cache_get(&uvm_anon_cache, PR_NOWAIT);
     84  1.33      yamt 	if (anon) {
     85  1.33      yamt 		KASSERT(anon->an_ref == 0);
     86  1.59     rmind 		KASSERT(anon->an_lock == NULL);
     87  1.33      yamt 		KASSERT(anon->an_page == NULL);
     88  1.37      yamt #if defined(VMSWAP)
     89  1.33      yamt 		KASSERT(anon->an_swslot == 0);
     90  1.55     rmind #endif
     91  1.33      yamt 		anon->an_ref = 1;
     92   1.1     chuck 	}
     93  1.33      yamt 	return anon;
     94   1.1     chuck }
     95   1.1     chuck 
     96   1.1     chuck /*
     97  1.59     rmind  * uvm_anon_dispose: free any resident page or swap resources of anon.
     98   1.1     chuck  *
     99  1.55     rmind  * => anon must be removed from the amap (if anon was in an amap).
    100  1.59     rmind  * => amap must be locked; we may drop and re-acquire the lock here.
    101   1.1     chuck  */
    102  1.60     rmind static bool
    103  1.59     rmind uvm_anon_dispose(struct vm_anon *anon)
    104   1.1     chuck {
    105  1.55     rmind 	struct vm_page *pg = anon->an_page;
    106  1.55     rmind 
    107  1.59     rmind 	UVMHIST_FUNC("uvm_anon_dispose"); UVMHIST_CALLED(maphist);
    108  1.64  pgoyette 	UVMHIST_LOG(maphist,"(anon=0x%#jx)", (uintptr_t)anon, 0,0,0);
    109   1.1     chuck 
    110  1.59     rmind 	KASSERT(mutex_owned(anon->an_lock));
    111  1.12   thorpej 
    112   1.1     chuck 	/*
    113  1.55     rmind 	 * If there is a resident page and it is loaned, then anon may not
    114  1.55     rmind 	 * own it.  Call out to uvm_anon_lockloanpg() to identify and lock
    115  1.55     rmind 	 * the real owner of the page.
    116   1.1     chuck 	 */
    117   1.1     chuck 
    118  1.22       chs 	if (pg && pg->loan_count) {
    119  1.54     rmind 		KASSERT(anon->an_lock != NULL);
    120   1.1     chuck 		pg = uvm_anon_lockloanpg(anon);
    121  1.22       chs 	}
    122   1.1     chuck 
    123   1.1     chuck 	/*
    124  1.55     rmind 	 * Dispose the page, if it is resident.
    125   1.1     chuck 	 */
    126   1.1     chuck 
    127   1.1     chuck 	if (pg) {
    128  1.54     rmind 		KASSERT(anon->an_lock != NULL);
    129   1.1     chuck 
    130   1.1     chuck 		/*
    131  1.55     rmind 		 * If the page is owned by a UVM object (now locked),
    132  1.55     rmind 		 * then kill the loan on the page rather than free it,
    133  1.55     rmind 		 * and release the object lock.
    134   1.1     chuck 		 */
    135   1.1     chuck 
    136   1.1     chuck 		if (pg->uobject) {
    137  1.50        ad 			mutex_enter(&uvm_pageqlock);
    138  1.10       chs 			KASSERT(pg->loan_count > 0);
    139   1.1     chuck 			pg->loan_count--;
    140   1.1     chuck 			pg->uanon = NULL;
    141  1.50        ad 			mutex_exit(&uvm_pageqlock);
    142  1.54     rmind 			mutex_exit(pg->uobject->vmobjlock);
    143   1.1     chuck 		} else {
    144   1.1     chuck 
    145   1.1     chuck 			/*
    146  1.55     rmind 			 * If page has no UVM object, then anon is the owner,
    147  1.55     rmind 			 * and it is already locked.
    148   1.1     chuck 			 */
    149   1.1     chuck 
    150  1.18       chs 			KASSERT((pg->flags & PG_RELEASED) == 0);
    151  1.18       chs 			pmap_page_protect(pg, VM_PROT_NONE);
    152  1.29      yamt 
    153  1.29      yamt 			/*
    154  1.55     rmind 			 * If the page is busy, mark it as PG_RELEASED, so
    155  1.55     rmind 			 * that uvm_anon_release(9) would release it later.
    156  1.29      yamt 			 */
    157  1.29      yamt 
    158  1.29      yamt 			if (pg->flags & PG_BUSY) {
    159  1.29      yamt 				pg->flags |= PG_RELEASED;
    160  1.58      yamt 				mutex_obj_hold(anon->an_lock);
    161  1.60     rmind 				return false;
    162  1.17       chs 			}
    163  1.50        ad 			mutex_enter(&uvm_pageqlock);
    164  1.29      yamt 			uvm_pagefree(pg);
    165  1.50        ad 			mutex_exit(&uvm_pageqlock);
    166  1.64  pgoyette 			UVMHIST_LOG(maphist, "anon 0x%#jx, page 0x%#jx: "
    167  1.64  pgoyette 			    "freed now!", (uintptr_t)anon, (uintptr_t)pg,
    168  1.64  pgoyette 			    0, 0);
    169   1.1     chuck 		}
    170   1.1     chuck 	}
    171  1.55     rmind 
    172  1.37      yamt #if defined(VMSWAP)
    173  1.26        pk 	if (pg == NULL && anon->an_swslot > 0) {
    174  1.55     rmind 		/* This page is no longer only in swap. */
    175  1.20       chs 		KASSERT(uvmexp.swpgonly > 0);
    176  1.65        ad 		atomic_dec_uint(&uvmexp.swpgonly);
    177  1.20       chs 	}
    178  1.55     rmind #endif
    179   1.1     chuck 
    180   1.1     chuck 	/*
    181  1.59     rmind 	 * Free any swap resources, leave a page replacement hint.
    182   1.1     chuck 	 */
    183  1.18       chs 
    184   1.2       chs 	uvm_anon_dropswap(anon);
    185  1.39      yamt 	uvmpdpol_anfree(anon);
    186  1.59     rmind 	UVMHIST_LOG(maphist,"<- done!",0,0,0,0);
    187  1.60     rmind 	return true;
    188  1.59     rmind }
    189  1.59     rmind 
    190  1.59     rmind /*
    191  1.59     rmind  * uvm_anon_free: free a single anon.
    192  1.59     rmind  *
    193  1.59     rmind  * => anon must be already disposed.
    194  1.59     rmind  */
    195  1.59     rmind void
    196  1.59     rmind uvm_anon_free(struct vm_anon *anon)
    197  1.59     rmind {
    198  1.39      yamt 
    199  1.59     rmind 	KASSERT(anon->an_ref == 0);
    200  1.59     rmind 	KASSERT(anon->an_lock == NULL);
    201  1.33      yamt 	KASSERT(anon->an_page == NULL);
    202  1.37      yamt #if defined(VMSWAP)
    203  1.31      yamt 	KASSERT(anon->an_swslot == 0);
    204  1.55     rmind #endif
    205  1.47        ad 	pool_cache_put(&uvm_anon_cache, anon);
    206   1.2       chs }
    207   1.2       chs 
    208  1.55     rmind /*
    209  1.59     rmind  * uvm_anon_freelst: free a linked list of anon structures.
    210  1.59     rmind  *
    211  1.59     rmind  * => anon must be locked, we will unlock it.
    212  1.55     rmind  */
    213  1.54     rmind void
    214  1.59     rmind uvm_anon_freelst(struct vm_amap *amap, struct vm_anon *anonlst)
    215  1.54     rmind {
    216  1.61      yamt 	struct vm_anon *anon;
    217  1.61      yamt 	struct vm_anon **anonp = &anonlst;
    218  1.59     rmind 
    219  1.59     rmind 	KASSERT(mutex_owned(amap->am_lock));
    220  1.61      yamt 	while ((anon = *anonp) != NULL) {
    221  1.60     rmind 		if (!uvm_anon_dispose(anon)) {
    222  1.60     rmind 			/* Do not free this anon. */
    223  1.61      yamt 			*anonp = anon->an_link;
    224  1.61      yamt 			/* Note: clears an_ref as well. */
    225  1.61      yamt 			anon->an_link = NULL;
    226  1.61      yamt 		} else {
    227  1.61      yamt 			anonp = &anon->an_link;
    228  1.60     rmind 		}
    229  1.59     rmind 	}
    230  1.59     rmind 	amap_unlock(amap);
    231  1.59     rmind 
    232  1.59     rmind 	while (anonlst) {
    233  1.59     rmind 		anon = anonlst->an_link;
    234  1.59     rmind 		/* Note: clears an_ref as well. */
    235  1.59     rmind 		anonlst->an_link = NULL;
    236  1.59     rmind 		anonlst->an_lock = NULL;
    237  1.59     rmind 		uvm_anon_free(anonlst);
    238  1.59     rmind 		anonlst = anon;
    239  1.54     rmind 	}
    240  1.54     rmind }
    241  1.54     rmind 
    242   1.2       chs /*
    243  1.55     rmind  * uvm_anon_lockloanpg: given a locked anon, lock its resident page owner.
    244   1.1     chuck  *
    245   1.1     chuck  * => anon is locked by caller
    246   1.1     chuck  * => on return: anon is locked
    247   1.1     chuck  *		 if there is a resident page:
    248   1.1     chuck  *			if it has a uobject, it is locked by us
    249   1.1     chuck  *			if it is ownerless, we take over as owner
    250   1.1     chuck  *		 we return the resident page (it can change during
    251   1.1     chuck  *		 this function)
    252   1.1     chuck  * => note that the only time an anon has an ownerless resident page
    253   1.1     chuck  *	is if the page was loaned from a uvm_object and the uvm_object
    254   1.1     chuck  *	disowned it
    255   1.1     chuck  * => this only needs to be called when you want to do an operation
    256   1.1     chuck  *	on an anon's resident page and that page has a non-zero loan
    257   1.1     chuck  *	count.
    258   1.1     chuck  */
    259   1.1     chuck struct vm_page *
    260  1.34   thorpej uvm_anon_lockloanpg(struct vm_anon *anon)
    261   1.1     chuck {
    262   1.1     chuck 	struct vm_page *pg;
    263  1.43   thorpej 	bool locked = false;
    264   1.1     chuck 
    265  1.54     rmind 	KASSERT(mutex_owned(anon->an_lock));
    266  1.12   thorpej 
    267   1.1     chuck 	/*
    268   1.1     chuck 	 * loop while we have a resident page that has a non-zero loan count.
    269   1.1     chuck 	 * if we successfully get our lock, we will "break" the loop.
    270   1.1     chuck 	 * note that the test for pg->loan_count is not protected -- this
    271   1.1     chuck 	 * may produce false positive results.   note that a false positive
    272   1.1     chuck 	 * result may cause us to do more work than we need to, but it will
    273   1.1     chuck 	 * not produce an incorrect result.
    274   1.1     chuck 	 */
    275   1.1     chuck 
    276  1.33      yamt 	while (((pg = anon->an_page) != NULL) && pg->loan_count != 0) {
    277   1.1     chuck 
    278   1.1     chuck 		/*
    279   1.1     chuck 		 * quickly check to see if the page has an object before
    280   1.1     chuck 		 * bothering to lock the page queues.   this may also produce
    281   1.1     chuck 		 * a false positive result, but that's ok because we do a real
    282   1.1     chuck 		 * check after that.
    283   1.1     chuck 		 */
    284   1.1     chuck 
    285   1.1     chuck 		if (pg->uobject) {
    286  1.50        ad 			mutex_enter(&uvm_pageqlock);
    287  1.18       chs 			if (pg->uobject) {
    288   1.1     chuck 				locked =
    289  1.54     rmind 				    mutex_tryenter(pg->uobject->vmobjlock);
    290   1.1     chuck 			} else {
    291   1.1     chuck 				/* object disowned before we got PQ lock */
    292  1.43   thorpej 				locked = true;
    293   1.1     chuck 			}
    294  1.50        ad 			mutex_exit(&uvm_pageqlock);
    295   1.1     chuck 
    296   1.1     chuck 			/*
    297   1.1     chuck 			 * if we didn't get a lock (try lock failed), then we
    298   1.1     chuck 			 * toggle our anon lock and try again
    299   1.1     chuck 			 */
    300   1.1     chuck 
    301   1.1     chuck 			if (!locked) {
    302   1.1     chuck 				/*
    303   1.1     chuck 				 * someone locking the object has a chance to
    304   1.1     chuck 				 * lock us right now
    305  1.54     rmind 				 *
    306  1.54     rmind 				 * XXX Better than yielding but inadequate.
    307   1.1     chuck 				 */
    308  1.54     rmind 				kpause("livelock", false, 1, anon->an_lock);
    309  1.10       chs 				continue;
    310   1.1     chuck 			}
    311   1.1     chuck 		}
    312   1.1     chuck 
    313   1.1     chuck 		/*
    314  1.55     rmind 		 * If page is un-owned i.e. the object dropped its ownership,
    315  1.55     rmind 		 * then we have to take the ownership.
    316   1.1     chuck 		 */
    317   1.1     chuck 
    318   1.1     chuck 		if (pg->uobject == NULL && (pg->pqflags & PQ_ANON) == 0) {
    319  1.50        ad 			mutex_enter(&uvm_pageqlock);
    320  1.18       chs 			pg->pqflags |= PQ_ANON;
    321  1.18       chs 			pg->loan_count--;
    322  1.50        ad 			mutex_exit(&uvm_pageqlock);
    323   1.1     chuck 		}
    324   1.1     chuck 		break;
    325   1.1     chuck 	}
    326  1.55     rmind 	return pg;
    327   1.5       chs }
    328   1.5       chs 
    329  1.37      yamt #if defined(VMSWAP)
    330  1.37      yamt 
    331   1.5       chs /*
    332  1.55     rmind  * uvm_anon_pagein: fetch an anon's page.
    333   1.5       chs  *
    334   1.5       chs  * => anon must be locked, and is unlocked upon return.
    335  1.43   thorpej  * => returns true if pagein was aborted due to lack of memory.
    336   1.5       chs  */
    337   1.5       chs 
    338  1.42   thorpej bool
    339  1.57     rmind uvm_anon_pagein(struct vm_amap *amap, struct vm_anon *anon)
    340   1.5       chs {
    341   1.5       chs 	struct vm_page *pg;
    342   1.5       chs 	struct uvm_object *uobj;
    343   1.8   thorpej 
    344  1.54     rmind 	KASSERT(mutex_owned(anon->an_lock));
    345  1.57     rmind 	KASSERT(anon->an_lock == amap->am_lock);
    346  1.12   thorpej 
    347   1.8   thorpej 	/*
    348  1.55     rmind 	 * Get the page of the anon.
    349   1.8   thorpej 	 */
    350   1.5       chs 
    351  1.57     rmind 	switch (uvmfault_anonget(NULL, amap, anon)) {
    352  1.16       chs 	case 0:
    353  1.55     rmind 		/* Success - we have the page. */
    354  1.55     rmind 		KASSERT(mutex_owned(anon->an_lock));
    355   1.5       chs 		break;
    356  1.16       chs 	case EIO:
    357  1.16       chs 	case ERESTART:
    358   1.5       chs 		/*
    359  1.55     rmind 		 * Nothing more to do on errors.  ERESTART means that the
    360  1.55     rmind 		 * anon was freed.
    361   1.5       chs 		 */
    362  1.43   thorpej 		return false;
    363  1.25        pk 	default:
    364  1.43   thorpej 		return true;
    365   1.5       chs 	}
    366   1.5       chs 
    367   1.5       chs 	/*
    368  1.55     rmind 	 * Mark the page as dirty, clear its swslot and un-busy it.
    369   1.5       chs 	 */
    370   1.5       chs 
    371  1.33      yamt 	pg = anon->an_page;
    372   1.5       chs 	uobj = pg->uobject;
    373  1.55     rmind 	if (anon->an_swslot > 0) {
    374  1.23        pk 		uvm_swap_free(anon->an_swslot, 1);
    375  1.55     rmind 	}
    376   1.5       chs 	anon->an_swslot = 0;
    377  1.55     rmind 	pg->flags &= ~PG_CLEAN;
    378   1.5       chs 
    379   1.5       chs 	/*
    380  1.55     rmind 	 * Deactivate the page (to put it on a page queue).
    381   1.5       chs 	 */
    382   1.5       chs 
    383  1.50        ad 	mutex_enter(&uvm_pageqlock);
    384  1.55     rmind 	if (pg->wire_count == 0) {
    385  1.24        pk 		uvm_pagedeactivate(pg);
    386  1.55     rmind 	}
    387  1.50        ad 	mutex_exit(&uvm_pageqlock);
    388  1.25        pk 
    389  1.25        pk 	if (pg->flags & PG_WANTED) {
    390  1.55     rmind 		pg->flags &= ~PG_WANTED;
    391  1.25        pk 		wakeup(pg);
    392  1.25        pk 	}
    393   1.5       chs 
    394  1.54     rmind 	mutex_exit(anon->an_lock);
    395   1.5       chs 	if (uobj) {
    396  1.54     rmind 		mutex_exit(uobj->vmobjlock);
    397   1.5       chs 	}
    398  1.43   thorpej 	return false;
    399   1.1     chuck }
    400  1.29      yamt 
    401  1.55     rmind /*
    402  1.55     rmind  * uvm_anon_dropswap: release any swap resources from this anon.
    403  1.55     rmind  *
    404  1.55     rmind  * => anon must be locked or have a reference count of 0.
    405  1.55     rmind  */
    406  1.55     rmind void
    407  1.55     rmind uvm_anon_dropswap(struct vm_anon *anon)
    408  1.55     rmind {
    409  1.55     rmind 	UVMHIST_FUNC("uvm_anon_dropswap"); UVMHIST_CALLED(maphist);
    410  1.55     rmind 
    411  1.55     rmind 	if (anon->an_swslot == 0)
    412  1.55     rmind 		return;
    413  1.55     rmind 
    414  1.64  pgoyette 	UVMHIST_LOG(maphist,"freeing swap for anon %#jx, paged to swslot 0x%jx",
    415  1.64  pgoyette 		    (uintptr_t)anon, anon->an_swslot, 0, 0);
    416  1.55     rmind 	uvm_swap_free(anon->an_swslot, 1);
    417  1.55     rmind 	anon->an_swslot = 0;
    418  1.55     rmind }
    419  1.55     rmind 
    420  1.55     rmind #endif
    421  1.37      yamt 
    422  1.29      yamt /*
    423  1.29      yamt  * uvm_anon_release: release an anon and its page.
    424  1.29      yamt  *
    425  1.55     rmind  * => anon should not have any references.
    426  1.55     rmind  * => anon must be locked.
    427  1.29      yamt  */
    428  1.29      yamt 
    429  1.29      yamt void
    430  1.34   thorpej uvm_anon_release(struct vm_anon *anon)
    431  1.29      yamt {
    432  1.33      yamt 	struct vm_page *pg = anon->an_page;
    433  1.63    martin 	bool success __diagused;
    434  1.29      yamt 
    435  1.59     rmind 	KASSERT(mutex_owned(anon->an_lock));
    436  1.29      yamt 	KASSERT(pg != NULL);
    437  1.29      yamt 	KASSERT((pg->flags & PG_RELEASED) != 0);
    438  1.29      yamt 	KASSERT((pg->flags & PG_BUSY) != 0);
    439  1.29      yamt 	KASSERT(pg->uobject == NULL);
    440  1.29      yamt 	KASSERT(pg->uanon == anon);
    441  1.29      yamt 	KASSERT(pg->loan_count == 0);
    442  1.29      yamt 	KASSERT(anon->an_ref == 0);
    443  1.29      yamt 
    444  1.50        ad 	mutex_enter(&uvm_pageqlock);
    445  1.29      yamt 	uvm_pagefree(pg);
    446  1.50        ad 	mutex_exit(&uvm_pageqlock);
    447  1.62      yamt 	KASSERT(anon->an_page == NULL);
    448  1.62      yamt 	/* dispose should succeed as no one can reach this anon anymore. */
    449  1.62      yamt 	success = uvm_anon_dispose(anon);
    450  1.62      yamt 	KASSERT(success);
    451  1.59     rmind 	mutex_exit(anon->an_lock);
    452  1.59     rmind 	/* Note: extra reference is held for PG_RELEASED case. */
    453  1.62      yamt 	mutex_obj_free(anon->an_lock);
    454  1.59     rmind 	anon->an_lock = NULL;
    455  1.59     rmind 	uvm_anon_free(anon);
    456  1.29      yamt }
    457