uvm_pager.c revision 1.107.2.3
1 /* $NetBSD: uvm_pager.c,v 1.107.2.3 2012/04/17 00:09:00 yamt Exp $ */ 2 3 /* 4 * Copyright (c) 1997 Charles D. Cranor and Washington University. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 * 27 * from: Id: uvm_pager.c,v 1.1.2.23 1998/02/02 20:38:06 chuck Exp 28 */ 29 30 /* 31 * uvm_pager.c: generic functions used to assist the pagers. 32 */ 33 34 #include <sys/cdefs.h> 35 __KERNEL_RCSID(0, "$NetBSD: uvm_pager.c,v 1.107.2.3 2012/04/17 00:09:00 yamt Exp $"); 36 37 #include "opt_uvmhist.h" 38 #include "opt_readahead.h" 39 #include "opt_pagermap.h" 40 41 #include <sys/param.h> 42 #include <sys/systm.h> 43 #include <sys/vnode.h> 44 #include <sys/buf.h> 45 46 #include <uvm/uvm.h> 47 48 /* 49 * XXX 50 * this is needed until the device strategy interface 51 * is changed to do physically-addressed i/o. 52 */ 53 54 #ifndef PAGER_MAP_DEFAULT_SIZE 55 #define PAGER_MAP_DEFAULT_SIZE (16 * 1024 * 1024) 56 #endif 57 58 #ifndef PAGER_MAP_SIZE 59 #define PAGER_MAP_SIZE PAGER_MAP_DEFAULT_SIZE 60 #endif 61 62 size_t pager_map_size = PAGER_MAP_SIZE; 63 64 /* 65 * list of uvm pagers in the system 66 */ 67 68 const struct uvm_pagerops * const uvmpagerops[] = { 69 &aobj_pager, 70 &uvm_deviceops, 71 &uvm_vnodeops, 72 &ubc_pager, 73 }; 74 75 /* 76 * the pager map: provides KVA for I/O 77 */ 78 79 struct vm_map *pager_map; /* XXX */ 80 kmutex_t pager_map_wanted_lock; 81 bool pager_map_wanted; /* locked by pager map */ 82 static vaddr_t emergva; 83 static int emerg_ncolors; 84 static bool emerginuse; 85 86 void 87 uvm_pager_realloc_emerg(void) 88 { 89 vaddr_t new_emergva, old_emergva; 90 int old_emerg_ncolors; 91 92 if (__predict_true(emergva != 0 && emerg_ncolors >= uvmexp.ncolors)) 93 return; 94 95 KASSERT(!emerginuse); 96 97 new_emergva = uvm_km_alloc(kernel_map, 98 round_page(MAXPHYS) + ptoa(uvmexp.ncolors), ptoa(uvmexp.ncolors), 99 UVM_KMF_VAONLY); 100 101 KASSERT(new_emergva != 0); 102 103 old_emergva = emergva; 104 old_emerg_ncolors = emerg_ncolors; 105 106 /* 107 * don't support re-color in late boot anyway. 108 */ 109 if (0) /* XXX */ 110 mutex_enter(&pager_map_wanted_lock); 111 112 emergva = new_emergva; 113 emerg_ncolors = uvmexp.ncolors; 114 wakeup(&old_emergva); 115 116 if (0) /* XXX */ 117 mutex_exit(&pager_map_wanted_lock); 118 119 if (old_emergva) 120 uvm_km_free(kernel_map, old_emergva, 121 round_page(MAXPHYS) + ptoa(old_emerg_ncolors), 122 UVM_KMF_VAONLY); 123 } 124 125 /* 126 * uvm_pager_init: init pagers (at boot time) 127 */ 128 129 void 130 uvm_pager_init(void) 131 { 132 u_int lcv; 133 vaddr_t sva, eva; 134 135 /* 136 * init pager map 137 */ 138 139 sva = 0; 140 pager_map = uvm_km_suballoc(kernel_map, &sva, &eva, pager_map_size, 0, 141 false, NULL); 142 mutex_init(&pager_map_wanted_lock, MUTEX_DEFAULT, IPL_NONE); 143 pager_map_wanted = false; 144 145 uvm_pager_realloc_emerg(); 146 147 /* 148 * init ASYNC I/O queue 149 */ 150 151 TAILQ_INIT(&uvm.aio_done); 152 153 /* 154 * call pager init functions 155 */ 156 for (lcv = 0 ; lcv < __arraycount(uvmpagerops); lcv++) { 157 if (uvmpagerops[lcv]->pgo_init) 158 uvmpagerops[lcv]->pgo_init(); 159 } 160 } 161 162 /* 163 * uvm_pagermapin: map pages into KVA (pager_map) for I/O that needs mappings 164 * 165 * we basically just map in a blank map entry to reserve the space in the 166 * map and then use pmap_enter() to put the mappings in by hand. 167 */ 168 169 vaddr_t 170 uvm_pagermapin(struct vm_page **pps, int npages, int flags) 171 { 172 vsize_t size; 173 vaddr_t kva; 174 vaddr_t cva; 175 struct vm_page *pp; 176 vm_prot_t prot; 177 const bool pdaemon = (curlwp == uvm.pagedaemon_lwp); 178 const u_int first_color = VM_PGCOLOR_BUCKET(*pps); 179 UVMHIST_FUNC("uvm_pagermapin"); UVMHIST_CALLED(maphist); 180 181 UVMHIST_LOG(maphist,"(pps=0x%x, npages=%d, first_color=%u)", 182 pps, npages, first_color, 0); 183 184 /* 185 * compute protection. outgoing I/O only needs read 186 * access to the page, whereas incoming needs read/write. 187 */ 188 189 prot = VM_PROT_READ; 190 if (flags & UVMPAGER_MAPIN_READ) 191 prot |= VM_PROT_WRITE; 192 193 ReStart: 194 size = ptoa(npages); 195 kva = 0; /* let system choose VA */ 196 197 if (uvm_map(pager_map, &kva, size, NULL, UVM_UNKNOWN_OFFSET, 198 first_color, UVM_FLAG_COLORMATCH | UVM_FLAG_NOMERGE 199 | (pdaemon ? UVM_FLAG_NOWAIT : 0)) != 0) { 200 if (pdaemon) { 201 mutex_enter(&pager_map_wanted_lock); 202 if (emerginuse) { 203 UVM_UNLOCK_AND_WAIT(&emergva, 204 &pager_map_wanted_lock, false, 205 "emergva", 0); 206 goto ReStart; 207 } 208 emerginuse = true; 209 mutex_exit(&pager_map_wanted_lock); 210 kva = emergva + ptoa(first_color); 211 /* The shift implicitly truncates to PAGE_SIZE */ 212 KASSERT(npages <= (MAXPHYS >> PAGE_SHIFT)); 213 goto enter; 214 } 215 if ((flags & UVMPAGER_MAPIN_WAITOK) == 0) { 216 UVMHIST_LOG(maphist,"<- NOWAIT failed", 0,0,0,0); 217 return(0); 218 } 219 mutex_enter(&pager_map_wanted_lock); 220 pager_map_wanted = true; 221 UVMHIST_LOG(maphist, " SLEEPING on pager_map",0,0,0,0); 222 UVM_UNLOCK_AND_WAIT(pager_map, &pager_map_wanted_lock, false, 223 "pager_map", 0); 224 goto ReStart; 225 } 226 227 enter: 228 /* got it */ 229 for (cva = kva; npages != 0; npages--, cva += PAGE_SIZE) { 230 pp = *pps++; 231 KASSERT(pp); 232 // KASSERT(!((VM_PAGE_TO_PHYS(pp) ^ cva) & uvmexp.colormask)); 233 KASSERT(pp->flags & PG_BUSY); 234 pmap_kenter_pa(cva, VM_PAGE_TO_PHYS(pp), prot, 0); 235 } 236 pmap_update(vm_map_pmap(pager_map)); 237 238 UVMHIST_LOG(maphist, "<- done (KVA=0x%x)", kva,0,0,0); 239 return(kva); 240 } 241 242 /* 243 * uvm_pagermapout: remove pager_map mapping 244 * 245 * we remove our mappings by hand and then remove the mapping (waking 246 * up anyone wanting space). 247 */ 248 249 void 250 uvm_pagermapout(vaddr_t kva, int npages) 251 { 252 vsize_t size = ptoa(npages); 253 struct vm_map_entry *entries; 254 UVMHIST_FUNC("uvm_pagermapout"); UVMHIST_CALLED(maphist); 255 256 UVMHIST_LOG(maphist, " (kva=0x%x, npages=%d)", kva, npages,0,0); 257 258 /* 259 * duplicate uvm_unmap, but add in pager_map_wanted handling. 260 */ 261 262 pmap_kremove(kva, size); 263 pmap_update(pmap_kernel()); 264 265 if ((kva & ~ptoa(uvmexp.colormask)) == emergva) { 266 mutex_enter(&pager_map_wanted_lock); 267 KASSERT(emerginuse); 268 emerginuse = false; 269 wakeup(&emergva); 270 mutex_exit(&pager_map_wanted_lock); 271 return; 272 } 273 274 vm_map_lock(pager_map); 275 uvm_unmap_remove(pager_map, kva, kva + size, &entries, 0); 276 mutex_enter(&pager_map_wanted_lock); 277 if (pager_map_wanted) { 278 pager_map_wanted = false; 279 wakeup(pager_map); 280 } 281 mutex_exit(&pager_map_wanted_lock); 282 vm_map_unlock(pager_map); 283 if (entries) 284 uvm_unmap_detach(entries, 0); 285 UVMHIST_LOG(maphist,"<- done",0,0,0,0); 286 } 287 288 /* 289 * interrupt-context iodone handler for single-buf i/os 290 * or the top-level buf of a nested-buf i/o. 291 */ 292 293 void 294 uvm_aio_biodone(struct buf *bp) 295 { 296 /* reset b_iodone for when this is a single-buf i/o. */ 297 bp->b_iodone = uvm_aio_aiodone; 298 299 workqueue_enqueue(uvm.aiodone_queue, &bp->b_work, NULL); 300 } 301 302 void 303 uvm_aio_aiodone_pages(struct vm_page **pgs, int npages, bool write, int error) 304 { 305 struct uvm_object *uobj; 306 struct vm_page *pg; 307 kmutex_t *slock; 308 int pageout_done; /* number of PG_PAGEOUT pages processed */ 309 int swslot; 310 int i; 311 bool swap; 312 UVMHIST_FUNC("uvm_aio_aiodone_pages"); UVMHIST_CALLED(ubchist); 313 314 swslot = 0; 315 pageout_done = 0; 316 slock = NULL; 317 uobj = NULL; 318 pg = pgs[0]; 319 swap = (pg->uanon != NULL && pg->uobject == NULL) || 320 (pg->pqflags & PQ_AOBJ) != 0; 321 if (!swap) { 322 uobj = pg->uobject; 323 slock = uobj->vmobjlock; 324 mutex_enter(slock); 325 mutex_enter(&uvm_pageqlock); 326 } else { 327 #if defined(VMSWAP) 328 if (error) { 329 if (pg->uobject != NULL) { 330 swslot = uao_find_swslot(pg->uobject, 331 pg->offset >> PAGE_SHIFT); 332 } else { 333 KASSERT(pg->uanon != NULL); 334 swslot = pg->uanon->an_swslot; 335 } 336 KASSERT(swslot); 337 } 338 #else /* defined(VMSWAP) */ 339 panic("%s: swap", __func__); 340 #endif /* defined(VMSWAP) */ 341 } 342 for (i = 0; i < npages; i++) { 343 #if defined(VMSWAP) 344 bool anon_disposed = false; /* XXX gcc */ 345 #endif /* defined(VMSWAP) */ 346 347 pg = pgs[i]; 348 KASSERT(swap || pg->uobject == uobj); 349 UVMHIST_LOG(ubchist, "pg %p", pg, 0,0,0); 350 351 #if defined(VMSWAP) 352 /* 353 * for swap i/os, lock each page's object (or anon) 354 * individually since each page may need a different lock. 355 */ 356 357 if (swap) { 358 if (pg->uobject != NULL) { 359 slock = pg->uobject->vmobjlock; 360 } else { 361 slock = pg->uanon->an_lock; 362 } 363 mutex_enter(slock); 364 mutex_enter(&uvm_pageqlock); 365 anon_disposed = (pg->flags & PG_RELEASED) != 0; 366 KASSERT(!anon_disposed || pg->uobject != NULL || 367 pg->uanon->an_ref == 0); 368 } 369 #endif /* defined(VMSWAP) */ 370 371 /* 372 * process errors. for reads, just mark the page to be freed. 373 * for writes, if the error was ENOMEM, we assume this was 374 * a transient failure so we mark the page dirty so that 375 * we'll try to write it again later. for all other write 376 * errors, we assume the error is permanent, thus the data 377 * in the page is lost. bummer. 378 */ 379 380 if (error) { 381 int slot; 382 if (!write) { 383 pg->flags |= PG_RELEASED; 384 continue; 385 } else if (error == ENOMEM) { 386 if (pg->flags & PG_PAGEOUT) { 387 pg->flags &= ~PG_PAGEOUT; 388 pageout_done++; 389 } 390 uvm_pagemarkdirty(pg, UVM_PAGE_STATUS_DIRTY); 391 uvm_pageactivate(pg); 392 slot = 0; 393 } else 394 slot = SWSLOT_BAD; 395 396 #if defined(VMSWAP) 397 if (swap) { 398 if (pg->uobject != NULL) { 399 int oldslot; 400 oldslot = uao_set_swslot(pg->uobject, 401 pg->offset >> PAGE_SHIFT, slot); 402 KASSERT(oldslot == swslot + i); 403 } else { 404 KASSERT(pg->uanon->an_swslot == 405 swslot + i); 406 pg->uanon->an_swslot = slot; 407 } 408 } 409 #endif /* defined(VMSWAP) */ 410 } 411 412 /* 413 * if the page is PG_FAKE, this must have been a read to 414 * initialize the page. clear PG_FAKE and activate the page. 415 */ 416 417 if (pg->flags & PG_FAKE) { 418 KASSERT(!write); 419 pg->flags &= ~PG_FAKE; 420 #if defined(READAHEAD_STATS) 421 pg->pqflags |= PQ_READAHEAD; 422 uvm_ra_total.ev_count++; 423 #endif /* defined(READAHEAD_STATS) */ 424 KASSERT(uvm_pagegetdirty(pg) == UVM_PAGE_STATUS_CLEAN); 425 uvm_pageenqueue(pg); 426 } 427 428 /* 429 * do accounting for pagedaemon i/o and arrange to free 430 * the pages instead of just unbusying them. 431 */ 432 433 if (pg->flags & PG_PAGEOUT) { 434 pg->flags &= ~PG_PAGEOUT; 435 pageout_done++; 436 uvmexp.pdfreed++; 437 pg->flags |= PG_RELEASED; 438 } 439 440 #if defined(VMSWAP) 441 /* 442 * for swap pages, unlock everything for this page now. 443 */ 444 445 if (swap) { 446 if (pg->uobject == NULL && anon_disposed) { 447 mutex_exit(&uvm_pageqlock); 448 uvm_anon_release(pg->uanon); 449 } else { 450 uvm_page_unbusy(&pg, 1); 451 mutex_exit(&uvm_pageqlock); 452 mutex_exit(slock); 453 } 454 } 455 #endif /* defined(VMSWAP) */ 456 } 457 uvm_pageout_done(pageout_done); 458 if (!swap) { 459 uvm_page_unbusy(pgs, npages); 460 mutex_exit(&uvm_pageqlock); 461 mutex_exit(slock); 462 } else { 463 #if defined(VMSWAP) 464 KASSERT(write); 465 466 /* these pages are now only in swap. */ 467 mutex_enter(&uvm_swap_data_lock); 468 KASSERT(uvmexp.swpgonly + npages <= uvmexp.swpginuse); 469 if (error != ENOMEM) 470 uvmexp.swpgonly += npages; 471 mutex_exit(&uvm_swap_data_lock); 472 if (error) { 473 if (error != ENOMEM) 474 uvm_swap_markbad(swslot, npages); 475 else 476 uvm_swap_free(swslot, npages); 477 } 478 uvmexp.pdpending--; 479 #endif /* defined(VMSWAP) */ 480 } 481 } 482 483 /* 484 * uvm_aio_aiodone: do iodone processing for async i/os. 485 * this should be called in thread context, not interrupt context. 486 */ 487 488 void 489 uvm_aio_aiodone(struct buf *bp) 490 { 491 int npages = bp->b_bufsize >> PAGE_SHIFT; 492 struct vm_page *pgs[npages]; 493 int i, error; 494 bool write; 495 UVMHIST_FUNC("uvm_aio_aiodone"); UVMHIST_CALLED(ubchist); 496 UVMHIST_LOG(ubchist, "bp %p", bp, 0,0,0); 497 498 error = bp->b_error; 499 write = (bp->b_flags & B_READ) == 0; 500 501 for (i = 0; i < npages; i++) { 502 pgs[i] = uvm_pageratop((vaddr_t)bp->b_data + (i << PAGE_SHIFT)); 503 UVMHIST_LOG(ubchist, "pgs[%d] = %p", i, pgs[i],0,0); 504 } 505 uvm_pagermapout((vaddr_t)bp->b_data, npages); 506 507 uvm_aio_aiodone_pages(pgs, npages, write, error); 508 509 if (write && (bp->b_cflags & BC_AGE) != 0) { 510 mutex_enter(bp->b_objlock); 511 vwakeup(bp); 512 mutex_exit(bp->b_objlock); 513 } 514 putiobuf(bp); 515 } 516 517 /* 518 * uvm_pageratop: convert KVAs in the pager map back to their page 519 * structures. 520 */ 521 522 struct vm_page * 523 uvm_pageratop(vaddr_t kva) 524 { 525 struct vm_page *pg; 526 paddr_t pa; 527 bool rv; 528 529 rv = pmap_extract(pmap_kernel(), kva, &pa); 530 KASSERT(rv); 531 pg = PHYS_TO_VM_PAGE(pa); 532 KASSERT(pg != NULL); 533 return (pg); 534 } 535
Indexes created Fri Oct 17 09:09:57 GMT 2025