uvm_pager.c revision 1.92.18.1
1 /* uvm_pager.c,v 1.92 2008/04/17 05:39:41 simonb Exp */ 2 3 /* 4 * 5 * Copyright (c) 1997 Charles D. Cranor and Washington University. 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 3. All advertising materials mentioning features or use of this software 17 * must display the following acknowledgement: 18 * This product includes software developed by Charles D. Cranor and 19 * Washington University. 20 * 4. The name of the author may not be used to endorse or promote products 21 * derived from this software without specific prior written permission. 22 * 23 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 24 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 25 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 26 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 27 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 28 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 29 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 30 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 31 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 32 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 * 34 * from: Id: uvm_pager.c,v 1.1.2.23 1998/02/02 20:38:06 chuck Exp 35 */ 36 37 /* 38 * uvm_pager.c: generic functions used to assist the pagers. 39 */ 40 41 #include <sys/cdefs.h> 42 __KERNEL_RCSID(0, "uvm_pager.c,v 1.92 2008/04/17 05:39:41 simonb Exp"); 43 44 #include "opt_uvmhist.h" 45 #include "opt_readahead.h" 46 #include "opt_pagermap.h" 47 48 #include <sys/param.h> 49 #include <sys/systm.h> 50 #include <sys/proc.h> 51 #include <sys/malloc.h> 52 #include <sys/pool.h> 53 #include <sys/vnode.h> 54 55 #include <uvm/uvm.h> 56 57 /* 58 * XXX 59 * this is needed until the device strategy interface 60 * is changed to do physically-addressed i/o. 61 */ 62 63 #ifndef PAGER_MAP_DEFAULT_SIZE 64 #define PAGER_MAP_DEFAULT_SIZE (16 * 1024 * 1024) 65 #endif 66 67 #ifndef PAGER_MAP_SIZE 68 #define PAGER_MAP_SIZE PAGER_MAP_DEFAULT_SIZE 69 #endif 70 71 size_t pager_map_size = PAGER_MAP_SIZE; 72 73 struct pool *uvm_aiobuf_pool; 74 75 /* 76 * list of uvm pagers in the system 77 */ 78 79 const struct uvm_pagerops * const uvmpagerops[] = { 80 &aobj_pager, 81 &uvm_deviceops, 82 &uvm_vnodeops, 83 &ubc_pager, 84 }; 85 86 /* 87 * the pager map: provides KVA for I/O 88 */ 89 90 struct vm_map *pager_map; /* XXX */ 91 kmutex_t pager_map_wanted_lock; 92 bool pager_map_wanted; /* locked by pager map */ 93 static vaddr_t emergva; 94 static bool emerginuse; 95 96 /* 97 * uvm_pager_init: init pagers (at boot time) 98 */ 99 100 void 101 uvm_pager_init(void) 102 { 103 u_int lcv; 104 vaddr_t sva, eva; 105 106 /* 107 * init pager map 108 */ 109 110 sva = 0; 111 pager_map = uvm_km_suballoc(kernel_map, &sva, &eva, pager_map_size, 0, 112 false, NULL); 113 mutex_init(&pager_map_wanted_lock, MUTEX_DEFAULT, IPL_NONE); 114 pager_map_wanted = false; 115 emergva = uvm_km_alloc(kernel_map, 116 round_page(MAXPHYS) + ptoa(uvmexp.ncolors), 0, 117 UVM_KMF_VAONLY); 118 #if defined(DEBUG) 119 if (emergva == 0) 120 panic("emergva"); 121 #endif 122 emerginuse = false; 123 124 /* 125 * init ASYNC I/O queue 126 */ 127 128 TAILQ_INIT(&uvm.aio_done); 129 130 /* 131 * call pager init functions 132 */ 133 for (lcv = 0 ; lcv < __arraycount(uvmpagerops); lcv++) { 134 if (uvmpagerops[lcv]->pgo_init) 135 uvmpagerops[lcv]->pgo_init(); 136 } 137 } 138 139 /* 140 * uvm_pagermapin: map pages into KVA (pager_map) for I/O that needs mappings 141 * 142 * we basically just map in a blank map entry to reserve the space in the 143 * map and then use pmap_enter() to put the mappings in by hand. 144 */ 145 146 vaddr_t 147 uvm_pagermapin(struct vm_page **pps, int npages, int flags) 148 { 149 vsize_t size; 150 vaddr_t kva; 151 vaddr_t cva; 152 struct vm_page *pp; 153 vm_prot_t prot; 154 const bool pdaemon = (curlwp == uvm.pagedaemon_lwp); 155 const u_int first_color = VM_PAGE_TO_COLOR(*pps); 156 UVMHIST_FUNC("uvm_pagermapin"); UVMHIST_CALLED(maphist); 157 158 UVMHIST_LOG(maphist,"(pps=0x%x, npages=%d, first_color=%u)", 159 pps, npages, first_color, 0); 160 161 /* 162 * compute protection. outgoing I/O only needs read 163 * access to the page, whereas incoming needs read/write. 164 */ 165 166 prot = VM_PROT_READ; 167 if (flags & UVMPAGER_MAPIN_READ) 168 prot |= VM_PROT_WRITE; 169 170 ReStart: 171 size = ptoa(npages); 172 kva = 0; /* let system choose VA */ 173 174 if (uvm_map(pager_map, &kva, size, NULL, UVM_UNKNOWN_OFFSET, 175 first_color, UVM_FLAG_COLORMATCH | UVM_FLAG_NOMERGE 176 | (pdaemon ? UVM_FLAG_NOWAIT : 0)) != 0) { 177 if (pdaemon) { 178 mutex_enter(&pager_map_wanted_lock); 179 if (emerginuse) { 180 UVM_UNLOCK_AND_WAIT(&emergva, 181 &pager_map_wanted_lock, false, 182 "emergva", 0); 183 goto ReStart; 184 } 185 emerginuse = true; 186 mutex_exit(&pager_map_wanted_lock); 187 kva = emergva + ptoa(first_color); 188 /* The shift implicitly truncates to PAGE_SIZE */ 189 KASSERT(npages <= (MAXPHYS >> PAGE_SHIFT)); 190 goto enter; 191 } 192 if ((flags & UVMPAGER_MAPIN_WAITOK) == 0) { 193 UVMHIST_LOG(maphist,"<- NOWAIT failed", 0,0,0,0); 194 return(0); 195 } 196 mutex_enter(&pager_map_wanted_lock); 197 pager_map_wanted = true; 198 UVMHIST_LOG(maphist, " SLEEPING on pager_map",0,0,0,0); 199 UVM_UNLOCK_AND_WAIT(pager_map, &pager_map_wanted_lock, false, 200 "pager_map", 0); 201 goto ReStart; 202 } 203 204 enter: 205 /* got it */ 206 for (cva = kva; npages != 0; npages--, cva += PAGE_SIZE) { 207 pp = *pps++; 208 KASSERT(pp); 209 KASSERT(((VM_PAGE_TO_PHYS(pp) ^ cva) & uvmexp.colormask) == 0); 210 KASSERT(pp->flags & PG_BUSY); 211 pmap_kenter_pa(cva, VM_PAGE_TO_PHYS(pp), prot); 212 } 213 pmap_update(vm_map_pmap(pager_map)); 214 215 UVMHIST_LOG(maphist, "<- done (KVA=0x%x)", kva,0,0,0); 216 return(kva); 217 } 218 219 /* 220 * uvm_pagermapout: remove pager_map mapping 221 * 222 * we remove our mappings by hand and then remove the mapping (waking 223 * up anyone wanting space). 224 */ 225 226 void 227 uvm_pagermapout(vaddr_t kva, int npages) 228 { 229 vsize_t size = ptoa(npages); 230 struct vm_map_entry *entries; 231 UVMHIST_FUNC("uvm_pagermapout"); UVMHIST_CALLED(maphist); 232 233 UVMHIST_LOG(maphist, " (kva=0x%x, npages=%d)", kva, npages,0,0); 234 235 /* 236 * duplicate uvm_unmap, but add in pager_map_wanted handling. 237 */ 238 239 pmap_kremove(kva, size); 240 241 if ((kva & ~ptoa(uvmexp.colormask)) == emergva) { 242 mutex_enter(&pager_map_wanted_lock); 243 emerginuse = false; 244 wakeup(&emergva); 245 mutex_exit(&pager_map_wanted_lock); 246 return; 247 } 248 249 vm_map_lock(pager_map); 250 uvm_unmap_remove(pager_map, kva, kva + size, &entries, NULL, 0); 251 mutex_enter(&pager_map_wanted_lock); 252 if (pager_map_wanted) { 253 pager_map_wanted = false; 254 wakeup(pager_map); 255 } 256 mutex_exit(&pager_map_wanted_lock); 257 vm_map_unlock(pager_map); 258 if (entries) 259 uvm_unmap_detach(entries, 0); 260 pmap_update(pmap_kernel()); 261 UVMHIST_LOG(maphist,"<- done",0,0,0,0); 262 } 263 264 /* 265 * interrupt-context iodone handler for nested i/o bufs. 266 * 267 * => the buffer is private so need not be locked here 268 */ 269 270 void 271 uvm_aio_biodone1(struct buf *bp) 272 { 273 struct buf *mbp = bp->b_private; 274 275 KASSERT(mbp != bp); 276 if (bp->b_error != 0) { 277 mbp->b_error = bp->b_error; 278 } 279 mbp->b_resid -= bp->b_bcount; 280 putiobuf(bp); 281 if (mbp->b_resid == 0) { 282 biodone(mbp); 283 } 284 } 285 286 /* 287 * interrupt-context iodone handler for single-buf i/os 288 * or the top-level buf of a nested-buf i/o. 289 */ 290 291 void 292 uvm_aio_biodone(struct buf *bp) 293 { 294 /* reset b_iodone for when this is a single-buf i/o. */ 295 bp->b_iodone = uvm_aio_aiodone; 296 297 workqueue_enqueue(uvm.aiodone_queue, &bp->b_work, NULL); 298 } 299 300 void 301 uvm_aio_aiodone_pages(struct vm_page **pgs, int npages, bool write, int error) 302 { 303 struct uvm_object *uobj; 304 struct vm_page *pg; 305 kmutex_t *slock; 306 int pageout_done; 307 int swslot; 308 int i; 309 bool swap; 310 UVMHIST_FUNC("uvm_aio_aiodone_pages"); UVMHIST_CALLED(ubchist); 311 312 swslot = 0; 313 pageout_done = 0; 314 slock = NULL; 315 uobj = NULL; 316 pg = pgs[0]; 317 swap = (pg->uanon != NULL && pg->uobject == NULL) || 318 (pg->pqflags & PQ_AOBJ) != 0; 319 if (!swap) { 320 uobj = pg->uobject; 321 slock = &uobj->vmobjlock; 322 mutex_enter(slock); 323 mutex_enter(&uvm_pageqlock); 324 } else { 325 #if defined(VMSWAP) 326 if (error) { 327 if (pg->uobject != NULL) { 328 swslot = uao_find_swslot(pg->uobject, 329 pg->offset >> PAGE_SHIFT); 330 } else { 331 KASSERT(pg->uanon != NULL); 332 swslot = pg->uanon->an_swslot; 333 } 334 KASSERT(swslot); 335 } 336 #else /* defined(VMSWAP) */ 337 panic("%s: swap", __func__); 338 #endif /* defined(VMSWAP) */ 339 } 340 for (i = 0; i < npages; i++) { 341 pg = pgs[i]; 342 KASSERT(swap || pg->uobject == uobj); 343 UVMHIST_LOG(ubchist, "pg %p", pg, 0,0,0); 344 345 #if defined(VMSWAP) 346 /* 347 * for swap i/os, lock each page's object (or anon) 348 * individually since each page may need a different lock. 349 */ 350 351 if (swap) { 352 if (pg->uobject != NULL) { 353 slock = &pg->uobject->vmobjlock; 354 } else { 355 slock = &pg->uanon->an_lock; 356 } 357 mutex_enter(slock); 358 mutex_enter(&uvm_pageqlock); 359 } 360 #endif /* defined(VMSWAP) */ 361 362 /* 363 * process errors. for reads, just mark the page to be freed. 364 * for writes, if the error was ENOMEM, we assume this was 365 * a transient failure so we mark the page dirty so that 366 * we'll try to write it again later. for all other write 367 * errors, we assume the error is permanent, thus the data 368 * in the page is lost. bummer. 369 */ 370 371 if (error) { 372 int slot; 373 if (!write) { 374 pg->flags |= PG_RELEASED; 375 continue; 376 } else if (error == ENOMEM) { 377 if (pg->flags & PG_PAGEOUT) { 378 pg->flags &= ~PG_PAGEOUT; 379 pageout_done++; 380 } 381 pg->flags &= ~PG_CLEAN; 382 uvm_pageactivate(pg); 383 slot = 0; 384 } else 385 slot = SWSLOT_BAD; 386 387 #if defined(VMSWAP) 388 if (swap) { 389 if (pg->uobject != NULL) { 390 int oldslot; 391 oldslot = uao_set_swslot(pg->uobject, 392 pg->offset >> PAGE_SHIFT, slot); 393 KASSERT(oldslot == swslot + i); 394 } else { 395 KASSERT(pg->uanon->an_swslot == 396 swslot + i); 397 pg->uanon->an_swslot = slot; 398 } 399 } 400 #endif /* defined(VMSWAP) */ 401 } 402 403 /* 404 * if the page is PG_FAKE, this must have been a read to 405 * initialize the page. clear PG_FAKE and activate the page. 406 * we must also clear the pmap "modified" flag since it may 407 * still be set from the page's previous identity. 408 */ 409 410 if (pg->flags & PG_FAKE) { 411 KASSERT(!write); 412 pg->flags &= ~PG_FAKE; 413 #if defined(READAHEAD_STATS) 414 pg->pqflags |= PQ_READAHEAD; 415 uvm_ra_total.ev_count++; 416 #endif /* defined(READAHEAD_STATS) */ 417 KASSERT((pg->flags & PG_CLEAN) != 0); 418 uvm_pageenqueue(pg); 419 pmap_clear_modify(pg); 420 } 421 422 /* 423 * do accounting for pagedaemon i/o and arrange to free 424 * the pages instead of just unbusying them. 425 */ 426 427 if (pg->flags & PG_PAGEOUT) { 428 pg->flags &= ~PG_PAGEOUT; 429 pageout_done++; 430 uvmexp.pdfreed++; 431 pg->flags |= PG_RELEASED; 432 } 433 434 #if defined(VMSWAP) 435 /* 436 * for swap pages, unlock everything for this page now. 437 */ 438 439 if (swap) { 440 if (pg->uobject == NULL && pg->uanon->an_ref == 0 && 441 (pg->flags & PG_RELEASED) != 0) { 442 mutex_exit(&uvm_pageqlock); 443 uvm_anon_release(pg->uanon); 444 } else { 445 uvm_page_unbusy(&pg, 1); 446 mutex_exit(&uvm_pageqlock); 447 mutex_exit(slock); 448 } 449 } 450 #endif /* defined(VMSWAP) */ 451 } 452 uvm_pageout_done(pageout_done); 453 if (!swap) { 454 uvm_page_unbusy(pgs, npages); 455 mutex_exit(&uvm_pageqlock); 456 mutex_exit(slock); 457 } else { 458 #if defined(VMSWAP) 459 KASSERT(write); 460 461 /* these pages are now only in swap. */ 462 mutex_enter(&uvm_swap_data_lock); 463 KASSERT(uvmexp.swpgonly + npages <= uvmexp.swpginuse); 464 if (error != ENOMEM) 465 uvmexp.swpgonly += npages; 466 mutex_exit(&uvm_swap_data_lock); 467 if (error) { 468 if (error != ENOMEM) 469 uvm_swap_markbad(swslot, npages); 470 else 471 uvm_swap_free(swslot, npages); 472 } 473 uvmexp.pdpending--; 474 #endif /* defined(VMSWAP) */ 475 } 476 } 477 478 /* 479 * uvm_aio_aiodone: do iodone processing for async i/os. 480 * this should be called in thread context, not interrupt context. 481 */ 482 483 void 484 uvm_aio_aiodone(struct buf *bp) 485 { 486 int npages = bp->b_bufsize >> PAGE_SHIFT; 487 struct vm_page *pgs[npages]; 488 int i, error; 489 bool write; 490 UVMHIST_FUNC("uvm_aio_aiodone"); UVMHIST_CALLED(ubchist); 491 UVMHIST_LOG(ubchist, "bp %p", bp, 0,0,0); 492 493 error = bp->b_error; 494 write = (bp->b_flags & B_READ) == 0; 495 /* XXXUBC BC_NOCACHE is for swap pager, should be done differently */ 496 if (write && !(bp->b_cflags & BC_NOCACHE) && bioopsp != NULL) 497 (*bioopsp->io_pageiodone)(bp); 498 499 for (i = 0; i < npages; i++) { 500 pgs[i] = uvm_pageratop((vaddr_t)bp->b_data + (i << PAGE_SHIFT)); 501 UVMHIST_LOG(ubchist, "pgs[%d] = %p", i, pgs[i],0,0); 502 } 503 uvm_pagermapout((vaddr_t)bp->b_data, npages); 504 505 uvm_aio_aiodone_pages(pgs, npages, write, error); 506 507 if (write && (bp->b_cflags & BC_AGE) != 0) { 508 mutex_enter(bp->b_objlock); 509 vwakeup(bp); 510 mutex_exit(bp->b_objlock); 511 } 512 putiobuf(bp); 513 } 514 515 /* 516 * uvm_pageratop: convert KVAs in the pager map back to their page 517 * structures. 518 */ 519 520 struct vm_page * 521 uvm_pageratop(vaddr_t kva) 522 { 523 struct vm_page *pg; 524 paddr_t pa; 525 bool rv; 526 527 rv = pmap_extract(pmap_kernel(), kva, &pa); 528 KASSERT(rv); 529 pg = PHYS_TO_VM_PAGE(pa); 530 KASSERT(pg != NULL); 531 return (pg); 532 } 533
Indexes created Wed Oct 15 16:09:53 GMT 2025