Home | History | Annotate | Line # | Download | only in uvm
uvm_pager.c revision 1.120 
      1 /*	$NetBSD: uvm_pager.c,v 1.120 2020/01/15 17:55:45 ad Exp $	*/
      2 
      3 /*
      4  * Copyright (c) 1997 Charles D. Cranor and Washington University.
      5  * All rights reserved.
      6  *
      7  * Redistribution and use in source and binary forms, with or without
      8  * modification, are permitted provided that the following conditions
      9  * are met:
     10  * 1. Redistributions of source code must retain the above copyright
     11  *    notice, this list of conditions and the following disclaimer.
     12  * 2. Redistributions in binary form must reproduce the above copyright
     13  *    notice, this list of conditions and the following disclaimer in the
     14  *    documentation and/or other materials provided with the distribution.
     15  *
     16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
     17  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
     18  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
     19  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
     20  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
     21  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
     22  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
     23  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
     24  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
     25  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     26  *
     27  * from: Id: uvm_pager.c,v 1.1.2.23 1998/02/02 20:38:06 chuck Exp
     28  */
     29 
     30 /*
     31  * uvm_pager.c: generic functions used to assist the pagers.
     32  */
     33 
     34 #include <sys/cdefs.h>
     35 __KERNEL_RCSID(0, "$NetBSD: uvm_pager.c,v 1.120 2020/01/15 17:55:45 ad Exp $");
     36 
     37 #include "opt_uvmhist.h"
     38 #include "opt_readahead.h"
     39 #include "opt_pagermap.h"
     40 
     41 #include <sys/param.h>
     42 #include <sys/systm.h>
     43 #include <sys/atomic.h>
     44 #include <sys/vnode.h>
     45 #include <sys/buf.h>
     46 
     47 #include <uvm/uvm.h>
     48 
     49 /*
     50  * XXX
     51  * this is needed until the device strategy interface
     52  * is changed to do physically-addressed i/o.
     53  */
     54 
     55 #ifndef PAGER_MAP_DEFAULT_SIZE
     56 #define PAGER_MAP_DEFAULT_SIZE	(16 * 1024 * 1024)
     57 #endif
     58 
     59 #ifndef PAGER_MAP_SIZE
     60 #define PAGER_MAP_SIZE	PAGER_MAP_DEFAULT_SIZE
     61 #endif
     62 
     63 size_t pager_map_size = PAGER_MAP_SIZE;
     64 
     65 /*
     66  * list of uvm pagers in the system
     67  */
     68 
     69 const struct uvm_pagerops * const uvmpagerops[] = {
     70 	&aobj_pager,
     71 	&uvm_deviceops,
     72 	&uvm_vnodeops,
     73 	&ubc_pager,
     74 };
     75 
     76 /*
     77  * the pager map: provides KVA for I/O
     78  */
     79 
     80 struct vm_map *pager_map;		/* XXX */
     81 kmutex_t pager_map_wanted_lock __cacheline_aligned;
     82 bool pager_map_wanted;	/* locked by pager map */
     83 static vaddr_t emergva;
     84 static int emerg_ncolors;
     85 static bool emerginuse;
     86 
     87 void
     88 uvm_pager_realloc_emerg(void)
     89 {
     90 	vaddr_t new_emergva, old_emergva;
     91 	int old_emerg_ncolors;
     92 
     93 	if (__predict_true(emergva != 0 && emerg_ncolors >= uvmexp.ncolors))
     94 		return;
     95 
     96 	KASSERT(!emerginuse);
     97 
     98 	new_emergva = uvm_km_alloc(kernel_map,
     99 	    round_page(MAXPHYS) + ptoa(uvmexp.ncolors), ptoa(uvmexp.ncolors),
    100 	    UVM_KMF_VAONLY);
    101 
    102 	KASSERT(new_emergva != 0);
    103 
    104 	old_emergva = emergva;
    105 	old_emerg_ncolors = emerg_ncolors;
    106 
    107 	/*
    108 	 * don't support re-color in late boot anyway.
    109 	 */
    110 	if (0) /* XXX */
    111 		mutex_enter(&pager_map_wanted_lock);
    112 
    113 	emergva = new_emergva;
    114 	emerg_ncolors = uvmexp.ncolors;
    115 	wakeup(&old_emergva);
    116 
    117 	if (0) /* XXX */
    118 		mutex_exit(&pager_map_wanted_lock);
    119 
    120 	if (old_emergva)
    121 		uvm_km_free(kernel_map, old_emergva,
    122 		    round_page(MAXPHYS) + ptoa(old_emerg_ncolors),
    123 		    UVM_KMF_VAONLY);
    124 }
    125 
    126 /*
    127  * uvm_pager_init: init pagers (at boot time)
    128  */
    129 
    130 void
    131 uvm_pager_init(void)
    132 {
    133 	u_int lcv;
    134 	vaddr_t sva, eva;
    135 
    136 	/*
    137 	 * init pager map
    138 	 */
    139 
    140 	sva = 0;
    141 	pager_map = uvm_km_suballoc(kernel_map, &sva, &eva, pager_map_size, 0,
    142 	    false, NULL);
    143 	mutex_init(&pager_map_wanted_lock, MUTEX_DEFAULT, IPL_NONE);
    144 	pager_map_wanted = false;
    145 
    146 	uvm_pager_realloc_emerg();
    147 
    148 	/*
    149 	 * init ASYNC I/O queue
    150 	 */
    151 
    152 	TAILQ_INIT(&uvm.aio_done);
    153 
    154 	/*
    155 	 * call pager init functions
    156 	 */
    157 	for (lcv = 0 ; lcv < __arraycount(uvmpagerops); lcv++) {
    158 		if (uvmpagerops[lcv]->pgo_init)
    159 			uvmpagerops[lcv]->pgo_init();
    160 	}
    161 }
    162 
    163 /*
    164  * uvm_pagermapin: map pages into KVA (pager_map) for I/O that needs mappings
    165  *
    166  * we basically just map in a blank map entry to reserve the space in the
    167  * map and then use pmap_enter() to put the mappings in by hand.
    168  */
    169 
    170 vaddr_t
    171 uvm_pagermapin(struct vm_page **pps, int npages, int flags)
    172 {
    173 	vsize_t size;
    174 	vaddr_t kva;
    175 	vaddr_t cva;
    176 	struct vm_page *pp;
    177 	vm_prot_t prot;
    178 	const bool pdaemon = (curlwp == uvm.pagedaemon_lwp);
    179 	const u_int first_color = VM_PGCOLOR(*pps);
    180 	UVMHIST_FUNC("uvm_pagermapin"); UVMHIST_CALLED(maphist);
    181 
    182 	UVMHIST_LOG(maphist,"(pps=0x%#jx, npages=%jd, first_color=%ju)",
    183 		(uintptr_t)pps, npages, first_color, 0);
    184 
    185 	/*
    186 	 * compute protection.  outgoing I/O only needs read
    187 	 * access to the page, whereas incoming needs read/write.
    188 	 */
    189 
    190 	prot = VM_PROT_READ;
    191 	if (flags & UVMPAGER_MAPIN_READ)
    192 		prot |= VM_PROT_WRITE;
    193 
    194 ReStart:
    195 	size = ptoa(npages);
    196 	kva = 0;			/* let system choose VA */
    197 
    198 	if (uvm_map(pager_map, &kva, size, NULL, UVM_UNKNOWN_OFFSET,
    199 	    first_color, UVM_FLAG_COLORMATCH | UVM_FLAG_NOMERGE
    200 	    | (pdaemon ? UVM_FLAG_NOWAIT : 0)) != 0) {
    201 		if (pdaemon) {
    202 			mutex_enter(&pager_map_wanted_lock);
    203 			if (emerginuse) {
    204 				UVM_UNLOCK_AND_WAIT(&emergva,
    205 				    &pager_map_wanted_lock, false,
    206 				    "emergva", 0);
    207 				goto ReStart;
    208 			}
    209 			emerginuse = true;
    210 			mutex_exit(&pager_map_wanted_lock);
    211 			kva = emergva + ptoa(first_color);
    212 			/* The shift implicitly truncates to PAGE_SIZE */
    213 			KASSERT(npages <= (MAXPHYS >> PAGE_SHIFT));
    214 			goto enter;
    215 		}
    216 		if ((flags & UVMPAGER_MAPIN_WAITOK) == 0) {
    217 			UVMHIST_LOG(maphist,"<- NOWAIT failed", 0,0,0,0);
    218 			return(0);
    219 		}
    220 		mutex_enter(&pager_map_wanted_lock);
    221 		pager_map_wanted = true;
    222 		UVMHIST_LOG(maphist, "  SLEEPING on pager_map",0,0,0,0);
    223 		UVM_UNLOCK_AND_WAIT(pager_map, &pager_map_wanted_lock, false,
    224 		    "pager_map", 0);
    225 		goto ReStart;
    226 	}
    227 
    228 enter:
    229 	/* got it */
    230 	for (cva = kva; npages != 0; npages--, cva += PAGE_SIZE) {
    231 		pp = *pps++;
    232 		KASSERT(pp);
    233 		// KASSERT(!((VM_PAGE_TO_PHYS(pp) ^ cva) & uvmexp.colormask));
    234 		KASSERT(pp->flags & PG_BUSY);
    235 		pmap_kenter_pa(cva, VM_PAGE_TO_PHYS(pp), prot, 0);
    236 	}
    237 	pmap_update(vm_map_pmap(pager_map));
    238 
    239 	UVMHIST_LOG(maphist, "<- done (KVA=0x%jx)", kva,0,0,0);
    240 	return(kva);
    241 }
    242 
    243 /*
    244  * uvm_pagermapout: remove pager_map mapping
    245  *
    246  * we remove our mappings by hand and then remove the mapping (waking
    247  * up anyone wanting space).
    248  */
    249 
    250 void
    251 uvm_pagermapout(vaddr_t kva, int npages)
    252 {
    253 	vsize_t size = ptoa(npages);
    254 	struct vm_map_entry *entries;
    255 	UVMHIST_FUNC("uvm_pagermapout"); UVMHIST_CALLED(maphist);
    256 
    257 	UVMHIST_LOG(maphist, " (kva=0x%jx, npages=%jd)", kva, npages,0,0);
    258 
    259 	/*
    260 	 * duplicate uvm_unmap, but add in pager_map_wanted handling.
    261 	 */
    262 
    263 	pmap_kremove(kva, size);
    264 	pmap_update(pmap_kernel());
    265 
    266 	if ((kva & ~ptoa(uvmexp.colormask)) == emergva) {
    267 		mutex_enter(&pager_map_wanted_lock);
    268 		KASSERT(emerginuse);
    269 		emerginuse = false;
    270 		wakeup(&emergva);
    271 		mutex_exit(&pager_map_wanted_lock);
    272 		return;
    273 	}
    274 
    275 	vm_map_lock(pager_map);
    276 	uvm_unmap_remove(pager_map, kva, kva + size, &entries, 0);
    277 	mutex_enter(&pager_map_wanted_lock);
    278 	if (pager_map_wanted) {
    279 		pager_map_wanted = false;
    280 		wakeup(pager_map);
    281 	}
    282 	mutex_exit(&pager_map_wanted_lock);
    283 	vm_map_unlock(pager_map);
    284 	if (entries)
    285 		uvm_unmap_detach(entries, 0);
    286 	UVMHIST_LOG(maphist,"<- done",0,0,0,0);
    287 }
    288 
    289 /*
    290  * interrupt-context iodone handler for single-buf i/os
    291  * or the top-level buf of a nested-buf i/o.
    292  */
    293 
    294 void
    295 uvm_aio_biodone(struct buf *bp)
    296 {
    297 	/* reset b_iodone for when this is a single-buf i/o. */
    298 	bp->b_iodone = uvm_aio_aiodone;
    299 
    300 	workqueue_enqueue(uvm.aiodone_queue, &bp->b_work, NULL);
    301 }
    302 
    303 void
    304 uvm_aio_aiodone_pages(struct vm_page **pgs, int npages, bool write, int error)
    305 {
    306 	struct uvm_object *uobj;
    307 	struct vm_page *pg;
    308 	kmutex_t *slock;
    309 	int pageout_done;	/* number of PG_PAGEOUT pages processed */
    310 	int swslot;
    311 	int i;
    312 	bool swap;
    313 	UVMHIST_FUNC("uvm_aio_aiodone_pages"); UVMHIST_CALLED(ubchist);
    314 
    315 	swslot = 0;
    316 	pageout_done = 0;
    317 	slock = NULL;
    318 	uobj = NULL;
    319 	pg = pgs[0];
    320 	swap = (pg->uanon != NULL && pg->uobject == NULL) ||
    321 		(pg->flags & PG_AOBJ) != 0;
    322 	if (!swap) {
    323 		uobj = pg->uobject;
    324 		slock = uobj->vmobjlock;
    325 		mutex_enter(slock);
    326 	} else {
    327 #if defined(VMSWAP)
    328 		if (error) {
    329 			if (pg->uobject != NULL) {
    330 				swslot = uao_find_swslot(pg->uobject,
    331 				    pg->offset >> PAGE_SHIFT);
    332 			} else {
    333 				KASSERT(pg->uanon != NULL);
    334 				swslot = pg->uanon->an_swslot;
    335 			}
    336 			KASSERT(swslot);
    337 		}
    338 #else /* defined(VMSWAP) */
    339 		panic("%s: swap", __func__);
    340 #endif /* defined(VMSWAP) */
    341 	}
    342 	for (i = 0; i < npages; i++) {
    343 #if defined(VMSWAP)
    344 		bool anon_disposed = false; /* XXX gcc */
    345 #endif /* defined(VMSWAP) */
    346 
    347 		pg = pgs[i];
    348 		KASSERT(swap || pg->uobject == uobj);
    349 		UVMHIST_LOG(ubchist, "pg %#jx", (uintptr_t)pg, 0,0,0);
    350 
    351 #if defined(VMSWAP)
    352 		/*
    353 		 * for swap i/os, lock each page's object (or anon)
    354 		 * individually since each page may need a different lock.
    355 		 */
    356 
    357 		if (swap) {
    358 			if (pg->uobject != NULL) {
    359 				slock = pg->uobject->vmobjlock;
    360 			} else {
    361 				slock = pg->uanon->an_lock;
    362 			}
    363 			mutex_enter(slock);
    364 			anon_disposed = (pg->flags & PG_RELEASED) != 0;
    365 			KASSERT(!anon_disposed || pg->uobject != NULL ||
    366 			    pg->uanon->an_ref == 0);
    367 		}
    368 #endif /* defined(VMSWAP) */
    369 
    370 		if (write && uobj != NULL) {
    371 			KASSERT(radix_tree_get_tag(&uobj->uo_pages,
    372 			    pg->offset >> PAGE_SHIFT, UVM_PAGE_WRITEBACK_TAG));
    373 			radix_tree_clear_tag(&uobj->uo_pages,
    374 			    pg->offset >> PAGE_SHIFT, UVM_PAGE_WRITEBACK_TAG);
    375 		}
    376 
    377 		/*
    378 		 * process errors.  for reads, just mark the page to be freed.
    379 		 * for writes, if the error was ENOMEM, we assume this was
    380 		 * a transient failure so we mark the page dirty so that
    381 		 * we'll try to write it again later.  for all other write
    382 		 * errors, we assume the error is permanent, thus the data
    383 		 * in the page is lost.  bummer.
    384 		 */
    385 
    386 		if (error) {
    387 			int slot;
    388 			if (!write) {
    389 				pg->flags |= PG_RELEASED;
    390 				continue;
    391 			} else if (error == ENOMEM) {
    392 				if (pg->flags & PG_PAGEOUT) {
    393 					pg->flags &= ~PG_PAGEOUT;
    394 					pageout_done++;
    395 				}
    396 				uvm_pagemarkdirty(pg, UVM_PAGE_STATUS_DIRTY);
    397 				uvm_pagelock(pg);
    398 				uvm_pageactivate(pg);
    399 				uvm_pageunlock(pg);
    400 				slot = 0;
    401 			} else
    402 				slot = SWSLOT_BAD;
    403 
    404 #if defined(VMSWAP)
    405 			if (swap) {
    406 				if (pg->uobject != NULL) {
    407 					int oldslot __diagused;
    408 					oldslot = uao_set_swslot(pg->uobject,
    409 						pg->offset >> PAGE_SHIFT, slot);
    410 					KASSERT(oldslot == swslot + i);
    411 				} else {
    412 					KASSERT(pg->uanon->an_swslot ==
    413 						swslot + i);
    414 					pg->uanon->an_swslot = slot;
    415 				}
    416 			}
    417 #endif /* defined(VMSWAP) */
    418 		}
    419 
    420 		/*
    421 		 * if the page is PG_FAKE, this must have been a read to
    422 		 * initialize the page.  clear PG_FAKE and activate the page.
    423 		 */
    424 
    425 		if (pg->flags & PG_FAKE) {
    426 			KASSERT(!write);
    427 			pg->flags &= ~PG_FAKE;
    428 #if defined(READAHEAD_STATS)
    429 			pg->flags |= PG_READAHEAD;
    430 			uvm_ra_total.ev_count++;
    431 #endif /* defined(READAHEAD_STATS) */
    432 			KASSERT(uvm_pagegetdirty(pg) == UVM_PAGE_STATUS_CLEAN);
    433 			uvm_pagelock(pg);
    434 			uvm_pageenqueue(pg);
    435 			uvm_pageunlock(pg);
    436 		}
    437 
    438 		/*
    439 		 * do accounting for pagedaemon i/o and arrange to free
    440 		 * the pages instead of just unbusying them.
    441 		 */
    442 
    443 		if (pg->flags & PG_PAGEOUT) {
    444 			pg->flags &= ~PG_PAGEOUT;
    445 			pageout_done++;
    446 			atomic_inc_uint(&uvmexp.pdfreed);
    447 			pg->flags |= PG_RELEASED;
    448 		}
    449 
    450 #if defined(VMSWAP)
    451 		/*
    452 		 * for swap pages, unlock everything for this page now.
    453 		 */
    454 
    455 		if (swap) {
    456 			if (pg->uobject == NULL && anon_disposed) {
    457 				uvm_anon_release(pg->uanon);
    458 			} else {
    459 				uvm_page_unbusy(&pg, 1);
    460 				mutex_exit(slock);
    461 			}
    462 		}
    463 #endif /* defined(VMSWAP) */
    464 	}
    465 	uvm_pageout_done(pageout_done);
    466 	if (!swap) {
    467 		uvm_page_unbusy(pgs, npages);
    468 		mutex_exit(slock);
    469 	} else {
    470 #if defined(VMSWAP)
    471 		KASSERT(write);
    472 
    473 		/* these pages are now only in swap. */
    474 		if (error != ENOMEM) {
    475 			atomic_add_int(&uvmexp.swpgonly, npages);
    476 		}
    477 		if (error) {
    478 			if (error != ENOMEM)
    479 				uvm_swap_markbad(swslot, npages);
    480 			else
    481 				uvm_swap_free(swslot, npages);
    482 		}
    483 		atomic_dec_uint(&uvmexp.pdpending);
    484 #endif /* defined(VMSWAP) */
    485 	}
    486 }
    487 
    488 /*
    489  * uvm_aio_aiodone: do iodone processing for async i/os.
    490  * this should be called in thread context, not interrupt context.
    491  */
    492 
    493 void
    494 uvm_aio_aiodone(struct buf *bp)
    495 {
    496 	int npages = bp->b_bufsize >> PAGE_SHIFT;
    497 	struct vm_page *pgs[npages];
    498 	int i, error;
    499 	bool write;
    500 	UVMHIST_FUNC("uvm_aio_aiodone"); UVMHIST_CALLED(ubchist);
    501 	UVMHIST_LOG(ubchist, "bp %#jx", (uintptr_t)bp, 0,0,0);
    502 
    503 	error = bp->b_error;
    504 	write = (bp->b_flags & B_READ) == 0;
    505 
    506 	for (i = 0; i < npages; i++) {
    507 		pgs[i] = uvm_pageratop((vaddr_t)bp->b_data + (i << PAGE_SHIFT));
    508 		UVMHIST_LOG(ubchist, "pgs[%jd] = %#jx", i,
    509 		    (uintptr_t)pgs[i], 0, 0);
    510 	}
    511 	uvm_pagermapout((vaddr_t)bp->b_data, npages);
    512 
    513 	uvm_aio_aiodone_pages(pgs, npages, write, error);
    514 
    515 	if (write && (bp->b_cflags & BC_AGE) != 0) {
    516 		mutex_enter(bp->b_objlock);
    517 		vwakeup(bp);
    518 		mutex_exit(bp->b_objlock);
    519 	}
    520 	putiobuf(bp);
    521 }
    522 
    523 /*
    524  * uvm_pageratop: convert KVAs in the pager map back to their page
    525  * structures.
    526  */
    527 
    528 struct vm_page *
    529 uvm_pageratop(vaddr_t kva)
    530 {
    531 	struct vm_page *pg;
    532 	paddr_t pa;
    533 	bool rv __diagused;
    534 
    535 	rv = pmap_extract(pmap_kernel(), kva, &pa);
    536 	KASSERT(rv);
    537 	pg = PHYS_TO_VM_PAGE(pa);
    538 	KASSERT(pg != NULL);
    539 	return (pg);
    540 }
    541