Home | History | Annotate | Line # | Download | only in uvm
uvm_anon.c revision 1.73
      1  1.73        ad /*	$NetBSD: uvm_anon.c,v 1.73 2020/02/23 21:36:29 ad Exp $	*/
      2   1.1     chuck 
      3   1.1     chuck /*
      4   1.1     chuck  * Copyright (c) 1997 Charles D. Cranor and Washington University.
      5   1.1     chuck  * All rights reserved.
      6   1.1     chuck  *
      7   1.1     chuck  * Redistribution and use in source and binary forms, with or without
      8   1.1     chuck  * modification, are permitted provided that the following conditions
      9   1.1     chuck  * are met:
     10   1.1     chuck  * 1. Redistributions of source code must retain the above copyright
     11   1.1     chuck  *    notice, this list of conditions and the following disclaimer.
     12   1.1     chuck  * 2. Redistributions in binary form must reproduce the above copyright
     13   1.1     chuck  *    notice, this list of conditions and the following disclaimer in the
     14   1.1     chuck  *    documentation and/or other materials provided with the distribution.
     15   1.1     chuck  *
     16   1.1     chuck  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
     17   1.1     chuck  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
     18   1.1     chuck  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
     19   1.1     chuck  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
     20   1.1     chuck  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
     21   1.1     chuck  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
     22   1.1     chuck  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
     23   1.1     chuck  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
     24   1.1     chuck  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
     25   1.1     chuck  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     26   1.1     chuck  */
     27   1.1     chuck 
     28   1.1     chuck /*
     29   1.1     chuck  * uvm_anon.c: uvm anon ops
     30   1.1     chuck  */
     31  1.21     lukem 
     32  1.21     lukem #include <sys/cdefs.h>
     33  1.73        ad __KERNEL_RCSID(0, "$NetBSD: uvm_anon.c,v 1.73 2020/02/23 21:36:29 ad Exp $");
     34   1.1     chuck 
     35   1.1     chuck #include "opt_uvmhist.h"
     36   1.1     chuck 
     37   1.1     chuck #include <sys/param.h>
     38   1.1     chuck #include <sys/systm.h>
     39   1.1     chuck #include <sys/pool.h>
     40   1.5       chs #include <sys/kernel.h>
     41  1.67       uwe #include <sys/atomic.h>
     42   1.1     chuck 
     43   1.1     chuck #include <uvm/uvm.h>
     44   1.1     chuck #include <uvm/uvm_swap.h>
     45  1.39      yamt #include <uvm/uvm_pdpolicy.h>
     46   1.1     chuck 
     47  1.55     rmind static struct pool_cache	uvm_anon_cache;
     48   1.5       chs 
     49  1.55     rmind static int			uvm_anon_ctor(void *, void *, int);
     50   1.5       chs 
     51   1.1     chuck void
     52  1.34   thorpej uvm_anon_init(void)
     53   1.1     chuck {
     54   1.5       chs 
     55  1.49        ad 	pool_cache_bootstrap(&uvm_anon_cache, sizeof(struct vm_anon), 0, 0,
     56  1.49        ad 	    PR_LARGECACHE, "anonpl", NULL, IPL_NONE, uvm_anon_ctor,
     57  1.55     rmind 	    NULL, NULL);
     58   1.1     chuck }
     59   1.1     chuck 
     60  1.33      yamt static int
     61  1.41      yamt uvm_anon_ctor(void *arg, void *object, int flags)
     62   1.1     chuck {
     63  1.33      yamt 	struct vm_anon *anon = object;
     64   1.1     chuck 
     65  1.33      yamt 	anon->an_ref = 0;
     66  1.59     rmind 	anon->an_lock = NULL;
     67  1.33      yamt 	anon->an_page = NULL;
     68  1.37      yamt #if defined(VMSWAP)
     69  1.33      yamt 	anon->an_swslot = 0;
     70  1.55     rmind #endif
     71  1.11       chs 	return 0;
     72   1.1     chuck }
     73   1.1     chuck 
     74   1.1     chuck /*
     75  1.55     rmind  * uvm_analloc: allocate a new anon.
     76  1.13   thorpej  *
     77  1.55     rmind  * => anon will have no lock associated.
     78   1.1     chuck  */
     79   1.1     chuck struct vm_anon *
     80  1.34   thorpej uvm_analloc(void)
     81   1.1     chuck {
     82  1.33      yamt 	struct vm_anon *anon;
     83   1.1     chuck 
     84  1.47        ad 	anon = pool_cache_get(&uvm_anon_cache, PR_NOWAIT);
     85  1.33      yamt 	if (anon) {
     86  1.33      yamt 		KASSERT(anon->an_ref == 0);
     87  1.59     rmind 		KASSERT(anon->an_lock == NULL);
     88  1.33      yamt 		KASSERT(anon->an_page == NULL);
     89  1.37      yamt #if defined(VMSWAP)
     90  1.33      yamt 		KASSERT(anon->an_swslot == 0);
     91  1.55     rmind #endif
     92  1.33      yamt 		anon->an_ref = 1;
     93   1.1     chuck 	}
     94  1.33      yamt 	return anon;
     95   1.1     chuck }
     96   1.1     chuck 
     97   1.1     chuck /*
     98  1.69        ad  * uvm_anon_dispose: free any resident page or swap resources of anon.
     99   1.1     chuck  *
    100  1.55     rmind  * => anon must be removed from the amap (if anon was in an amap).
    101  1.59     rmind  * => amap must be locked; we may drop and re-acquire the lock here.
    102   1.1     chuck  */
    103  1.60     rmind static bool
    104  1.59     rmind uvm_anon_dispose(struct vm_anon *anon)
    105   1.1     chuck {
    106  1.55     rmind 	struct vm_page *pg = anon->an_page;
    107  1.55     rmind 
    108  1.59     rmind 	UVMHIST_FUNC("uvm_anon_dispose"); UVMHIST_CALLED(maphist);
    109  1.64  pgoyette 	UVMHIST_LOG(maphist,"(anon=0x%#jx)", (uintptr_t)anon, 0,0,0);
    110   1.1     chuck 
    111  1.72        ad 	KASSERT(rw_write_held(anon->an_lock));
    112  1.12   thorpej 
    113   1.1     chuck 	/*
    114  1.66        ad 	 * Dispose the page, if it is resident.
    115   1.1     chuck 	 */
    116   1.1     chuck 
    117  1.66        ad 	if (pg) {
    118  1.54     rmind 		KASSERT(anon->an_lock != NULL);
    119   1.1     chuck 
    120  1.66        ad 		/*
    121  1.66        ad 		 * If there is a resident page and it is loaned, then anon
    122  1.66        ad 		 * may not own it.  Call out to uvm_anon_lockloanpg() to
    123  1.66        ad 		 * identify and lock the real owner of the page.
    124  1.66        ad 		 */
    125   1.1     chuck 
    126  1.66        ad 		if (pg->loan_count) {
    127  1.66        ad 			pg = uvm_anon_lockloanpg(anon);
    128  1.66        ad 		}
    129   1.1     chuck 
    130   1.1     chuck 		/*
    131  1.55     rmind 		 * If the page is owned by a UVM object (now locked),
    132  1.55     rmind 		 * then kill the loan on the page rather than free it,
    133  1.55     rmind 		 * and release the object lock.
    134   1.1     chuck 		 */
    135   1.1     chuck 
    136   1.1     chuck 		if (pg->uobject) {
    137  1.69        ad 			mutex_enter(&pg->interlock);
    138  1.10       chs 			KASSERT(pg->loan_count > 0);
    139   1.1     chuck 			pg->loan_count--;
    140   1.1     chuck 			pg->uanon = NULL;
    141  1.69        ad 			mutex_exit(&pg->interlock);
    142  1.72        ad 			rw_exit(pg->uobject->vmobjlock);
    143   1.1     chuck 		} else {
    144   1.1     chuck 
    145   1.1     chuck 			/*
    146  1.55     rmind 			 * If page has no UVM object, then anon is the owner,
    147  1.55     rmind 			 * and it is already locked.
    148   1.1     chuck 			 */
    149   1.1     chuck 
    150  1.18       chs 			KASSERT((pg->flags & PG_RELEASED) == 0);
    151  1.18       chs 			pmap_page_protect(pg, VM_PROT_NONE);
    152  1.29      yamt 
    153  1.29      yamt 			/*
    154  1.55     rmind 			 * If the page is busy, mark it as PG_RELEASED, so
    155  1.55     rmind 			 * that uvm_anon_release(9) would release it later.
    156  1.29      yamt 			 */
    157  1.29      yamt 
    158  1.29      yamt 			if (pg->flags & PG_BUSY) {
    159  1.29      yamt 				pg->flags |= PG_RELEASED;
    160  1.72        ad 				rw_obj_hold(anon->an_lock);
    161  1.60     rmind 				return false;
    162  1.17       chs 			}
    163  1.69        ad 			uvm_pagefree(pg);
    164  1.69        ad 			UVMHIST_LOG(maphist, "anon 0x%#jx, page 0x%#jx: "
    165  1.69        ad 			    "freed now!", (uintptr_t)anon, (uintptr_t)pg,
    166  1.69        ad 			    0, 0);
    167   1.1     chuck 		}
    168   1.1     chuck 	}
    169  1.55     rmind 
    170  1.37      yamt #if defined(VMSWAP)
    171  1.26        pk 	if (pg == NULL && anon->an_swslot > 0) {
    172  1.55     rmind 		/* This page is no longer only in swap. */
    173  1.20       chs 		KASSERT(uvmexp.swpgonly > 0);
    174  1.65        ad 		atomic_dec_uint(&uvmexp.swpgonly);
    175  1.20       chs 	}
    176  1.55     rmind #endif
    177   1.1     chuck 
    178  1.69        ad 	/*
    179  1.69        ad 	 * Free any swap resources, leave a page replacement hint.
    180  1.69        ad 	 */
    181  1.69        ad 
    182  1.69        ad 	uvm_anon_dropswap(anon);
    183  1.69        ad 	uvmpdpol_anfree(anon);
    184  1.59     rmind 	UVMHIST_LOG(maphist,"<- done!",0,0,0,0);
    185  1.60     rmind 	return true;
    186  1.59     rmind }
    187  1.59     rmind 
    188  1.59     rmind /*
    189  1.59     rmind  * uvm_anon_free: free a single anon.
    190  1.59     rmind  *
    191  1.59     rmind  * => anon must be already disposed.
    192  1.59     rmind  */
    193  1.59     rmind void
    194  1.59     rmind uvm_anon_free(struct vm_anon *anon)
    195  1.59     rmind {
    196  1.39      yamt 
    197  1.59     rmind 	KASSERT(anon->an_ref == 0);
    198  1.59     rmind 	KASSERT(anon->an_lock == NULL);
    199  1.33      yamt 	KASSERT(anon->an_page == NULL);
    200  1.37      yamt #if defined(VMSWAP)
    201  1.31      yamt 	KASSERT(anon->an_swslot == 0);
    202  1.55     rmind #endif
    203  1.47        ad 	pool_cache_put(&uvm_anon_cache, anon);
    204   1.2       chs }
    205   1.2       chs 
    206  1.55     rmind /*
    207  1.59     rmind  * uvm_anon_freelst: free a linked list of anon structures.
    208  1.59     rmind  *
    209  1.66        ad  * => amap must be locked, we will unlock it.
    210  1.55     rmind  */
    211  1.54     rmind void
    212  1.59     rmind uvm_anon_freelst(struct vm_amap *amap, struct vm_anon *anonlst)
    213  1.54     rmind {
    214  1.69        ad 	struct vm_anon *next;
    215  1.59     rmind 
    216  1.68       chs 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist);
    217  1.68       chs 
    218  1.72        ad 	KASSERT(rw_write_held(amap->am_lock));
    219  1.66        ad 
    220  1.69        ad 	for (; anonlst != NULL; anonlst = next) {
    221  1.69        ad 		next = anonlst->an_link;
    222  1.69        ad 		/* Note: clears an_ref as well. */
    223  1.69        ad 		anonlst->an_link = NULL;
    224  1.69        ad 		if (uvm_anon_dispose(anonlst)) {
    225  1.69        ad 			anonlst->an_lock = NULL;
    226  1.69        ad 			uvm_anon_free(anonlst);
    227  1.60     rmind 		}
    228  1.59     rmind 	}
    229  1.59     rmind 	amap_unlock(amap);
    230  1.54     rmind }
    231  1.54     rmind 
    232   1.2       chs /*
    233  1.55     rmind  * uvm_anon_lockloanpg: given a locked anon, lock its resident page owner.
    234   1.1     chuck  *
    235   1.1     chuck  * => anon is locked by caller
    236   1.1     chuck  * => on return: anon is locked
    237   1.1     chuck  *		 if there is a resident page:
    238   1.1     chuck  *			if it has a uobject, it is locked by us
    239   1.1     chuck  *			if it is ownerless, we take over as owner
    240   1.1     chuck  *		 we return the resident page (it can change during
    241   1.1     chuck  *		 this function)
    242   1.1     chuck  * => note that the only time an anon has an ownerless resident page
    243   1.1     chuck  *	is if the page was loaned from a uvm_object and the uvm_object
    244   1.1     chuck  *	disowned it
    245   1.1     chuck  * => this only needs to be called when you want to do an operation
    246   1.1     chuck  *	on an anon's resident page and that page has a non-zero loan
    247   1.1     chuck  *	count.
    248   1.1     chuck  */
    249   1.1     chuck struct vm_page *
    250  1.34   thorpej uvm_anon_lockloanpg(struct vm_anon *anon)
    251   1.1     chuck {
    252   1.1     chuck 	struct vm_page *pg;
    253  1.72        ad 	krw_t op;
    254   1.1     chuck 
    255  1.72        ad 	KASSERT(rw_lock_held(anon->an_lock));
    256  1.12   thorpej 
    257   1.1     chuck 	/*
    258   1.1     chuck 	 * loop while we have a resident page that has a non-zero loan count.
    259   1.1     chuck 	 * if we successfully get our lock, we will "break" the loop.
    260   1.1     chuck 	 * note that the test for pg->loan_count is not protected -- this
    261   1.1     chuck 	 * may produce false positive results.   note that a false positive
    262   1.1     chuck 	 * result may cause us to do more work than we need to, but it will
    263   1.1     chuck 	 * not produce an incorrect result.
    264   1.1     chuck 	 */
    265   1.1     chuck 
    266  1.33      yamt 	while (((pg = anon->an_page) != NULL) && pg->loan_count != 0) {
    267  1.69        ad 		mutex_enter(&pg->interlock);
    268   1.1     chuck 		if (pg->uobject) {
    269   1.1     chuck 			/*
    270   1.1     chuck 			 * if we didn't get a lock (try lock failed), then we
    271   1.1     chuck 			 * toggle our anon lock and try again
    272   1.1     chuck 			 */
    273   1.1     chuck 
    274  1.72        ad 			if (!rw_tryenter(pg->uobject->vmobjlock, RW_WRITER)) {
    275   1.1     chuck 				/*
    276   1.1     chuck 				 * someone locking the object has a chance to
    277   1.1     chuck 				 * lock us right now
    278  1.54     rmind 				 *
    279  1.54     rmind 				 * XXX Better than yielding but inadequate.
    280   1.1     chuck 				 */
    281  1.69        ad 				mutex_exit(&pg->interlock);
    282  1.73        ad 				op = rw_lock_op(anon->an_lock);
    283  1.72        ad 				rw_exit(anon->an_lock);
    284  1.72        ad 				kpause("lkloanpg", false, 1, NULL);
    285  1.72        ad 				rw_enter(anon->an_lock, op);
    286  1.10       chs 				continue;
    287   1.1     chuck 			}
    288   1.1     chuck 		}
    289   1.1     chuck 
    290   1.1     chuck 		/*
    291  1.55     rmind 		 * If page is un-owned i.e. the object dropped its ownership,
    292  1.55     rmind 		 * then we have to take the ownership.
    293   1.1     chuck 		 */
    294   1.1     chuck 
    295  1.69        ad 		if (pg->uobject == NULL && (pg->flags & PG_ANON) == 0) {
    296  1.69        ad 			pg->flags |= PG_ANON;
    297  1.18       chs 			pg->loan_count--;
    298   1.1     chuck 		}
    299  1.69        ad 		mutex_exit(&pg->interlock);
    300   1.1     chuck 		break;
    301   1.1     chuck 	}
    302  1.55     rmind 	return pg;
    303   1.5       chs }
    304   1.5       chs 
    305  1.37      yamt #if defined(VMSWAP)
    306  1.37      yamt 
    307   1.5       chs /*
    308  1.55     rmind  * uvm_anon_pagein: fetch an anon's page.
    309   1.5       chs  *
    310   1.5       chs  * => anon must be locked, and is unlocked upon return.
    311  1.43   thorpej  * => returns true if pagein was aborted due to lack of memory.
    312   1.5       chs  */
    313   1.5       chs 
    314  1.42   thorpej bool
    315  1.57     rmind uvm_anon_pagein(struct vm_amap *amap, struct vm_anon *anon)
    316   1.5       chs {
    317   1.5       chs 	struct vm_page *pg;
    318   1.5       chs 	struct uvm_object *uobj;
    319   1.8   thorpej 
    320  1.72        ad 	KASSERT(rw_write_held(anon->an_lock));
    321  1.57     rmind 	KASSERT(anon->an_lock == amap->am_lock);
    322  1.12   thorpej 
    323   1.8   thorpej 	/*
    324  1.55     rmind 	 * Get the page of the anon.
    325   1.8   thorpej 	 */
    326   1.5       chs 
    327  1.57     rmind 	switch (uvmfault_anonget(NULL, amap, anon)) {
    328  1.16       chs 	case 0:
    329  1.55     rmind 		/* Success - we have the page. */
    330  1.72        ad 		KASSERT(rw_write_held(anon->an_lock));
    331   1.5       chs 		break;
    332  1.16       chs 	case EIO:
    333  1.16       chs 	case ERESTART:
    334   1.5       chs 		/*
    335  1.55     rmind 		 * Nothing more to do on errors.  ERESTART means that the
    336  1.55     rmind 		 * anon was freed.
    337   1.5       chs 		 */
    338  1.43   thorpej 		return false;
    339  1.25        pk 	default:
    340  1.43   thorpej 		return true;
    341   1.5       chs 	}
    342   1.5       chs 
    343   1.5       chs 	/*
    344  1.55     rmind 	 * Mark the page as dirty, clear its swslot and un-busy it.
    345   1.5       chs 	 */
    346   1.5       chs 
    347  1.33      yamt 	pg = anon->an_page;
    348   1.5       chs 	uobj = pg->uobject;
    349  1.55     rmind 	if (anon->an_swslot > 0) {
    350  1.23        pk 		uvm_swap_free(anon->an_swslot, 1);
    351  1.55     rmind 	}
    352   1.5       chs 	anon->an_swslot = 0;
    353  1.71        ad 	uvm_pagemarkdirty(pg, UVM_PAGE_STATUS_DIRTY);
    354   1.5       chs 
    355   1.5       chs 	/*
    356  1.55     rmind 	 * Deactivate the page (to put it on a page queue).
    357   1.5       chs 	 */
    358   1.5       chs 
    359  1.70        ad 	uvm_pagelock(pg);
    360  1.69        ad 	uvm_pagedeactivate(pg);
    361  1.70        ad 	uvm_pageunlock(pg);
    362  1.25        pk 	if (pg->flags & PG_WANTED) {
    363  1.55     rmind 		pg->flags &= ~PG_WANTED;
    364  1.25        pk 		wakeup(pg);
    365  1.25        pk 	}
    366   1.5       chs 
    367  1.72        ad 	rw_exit(anon->an_lock);
    368   1.5       chs 	if (uobj) {
    369  1.72        ad 		rw_exit(uobj->vmobjlock);
    370   1.5       chs 	}
    371  1.43   thorpej 	return false;
    372   1.1     chuck }
    373  1.29      yamt 
    374  1.55     rmind /*
    375  1.55     rmind  * uvm_anon_dropswap: release any swap resources from this anon.
    376  1.55     rmind  *
    377  1.55     rmind  * => anon must be locked or have a reference count of 0.
    378  1.55     rmind  */
    379  1.55     rmind void
    380  1.55     rmind uvm_anon_dropswap(struct vm_anon *anon)
    381  1.55     rmind {
    382  1.55     rmind 	UVMHIST_FUNC("uvm_anon_dropswap"); UVMHIST_CALLED(maphist);
    383  1.55     rmind 
    384  1.55     rmind 	if (anon->an_swslot == 0)
    385  1.55     rmind 		return;
    386  1.55     rmind 
    387  1.64  pgoyette 	UVMHIST_LOG(maphist,"freeing swap for anon %#jx, paged to swslot 0x%jx",
    388  1.64  pgoyette 		    (uintptr_t)anon, anon->an_swslot, 0, 0);
    389  1.55     rmind 	uvm_swap_free(anon->an_swslot, 1);
    390  1.55     rmind 	anon->an_swslot = 0;
    391  1.55     rmind }
    392  1.55     rmind 
    393  1.55     rmind #endif
    394  1.37      yamt 
    395  1.29      yamt /*
    396  1.29      yamt  * uvm_anon_release: release an anon and its page.
    397  1.29      yamt  *
    398  1.55     rmind  * => anon should not have any references.
    399  1.55     rmind  * => anon must be locked.
    400  1.29      yamt  */
    401  1.29      yamt 
    402  1.29      yamt void
    403  1.34   thorpej uvm_anon_release(struct vm_anon *anon)
    404  1.29      yamt {
    405  1.33      yamt 	struct vm_page *pg = anon->an_page;
    406  1.63    martin 	bool success __diagused;
    407  1.29      yamt 
    408  1.72        ad 	KASSERT(rw_write_held(anon->an_lock));
    409  1.29      yamt 	KASSERT(pg != NULL);
    410  1.29      yamt 	KASSERT((pg->flags & PG_RELEASED) != 0);
    411  1.29      yamt 	KASSERT((pg->flags & PG_BUSY) != 0);
    412  1.29      yamt 	KASSERT(pg->uobject == NULL);
    413  1.29      yamt 	KASSERT(pg->uanon == anon);
    414  1.29      yamt 	KASSERT(pg->loan_count == 0);
    415  1.29      yamt 	KASSERT(anon->an_ref == 0);
    416  1.29      yamt 
    417  1.29      yamt 	uvm_pagefree(pg);
    418  1.62      yamt 	KASSERT(anon->an_page == NULL);
    419  1.62      yamt 	/* dispose should succeed as no one can reach this anon anymore. */
    420  1.62      yamt 	success = uvm_anon_dispose(anon);
    421  1.62      yamt 	KASSERT(success);
    422  1.72        ad 	rw_exit(anon->an_lock);
    423  1.59     rmind 	/* Note: extra reference is held for PG_RELEASED case. */
    424  1.72        ad 	rw_obj_free(anon->an_lock);
    425  1.59     rmind 	anon->an_lock = NULL;
    426  1.59     rmind 	uvm_anon_free(anon);
    427  1.29      yamt }
    428