uvm_pager.c revision 1.115
1 /* $NetBSD: uvm_pager.c,v 1.115 2019/12/14 15:04:47 ad Exp $ */ 2 3 /* 4 * Copyright (c) 1997 Charles D. Cranor and Washington University. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 * 27 * from: Id: uvm_pager.c,v 1.1.2.23 1998/02/02 20:38:06 chuck Exp 28 */ 29 30 /* 31 * uvm_pager.c: generic functions used to assist the pagers. 32 */ 33 34 #include <sys/cdefs.h> 35 __KERNEL_RCSID(0, "$NetBSD: uvm_pager.c,v 1.115 2019/12/14 15:04:47 ad Exp $"); 36 37 #include "opt_uvmhist.h" 38 #include "opt_readahead.h" 39 #include "opt_pagermap.h" 40 41 #include <sys/param.h> 42 #include <sys/systm.h> 43 #include <sys/atomic.h> 44 #include <sys/vnode.h> 45 #include <sys/buf.h> 46 47 #include <uvm/uvm.h> 48 49 /* 50 * XXX 51 * this is needed until the device strategy interface 52 * is changed to do physically-addressed i/o. 53 */ 54 55 #ifndef PAGER_MAP_DEFAULT_SIZE 56 #define PAGER_MAP_DEFAULT_SIZE (16 * 1024 * 1024) 57 #endif 58 59 #ifndef PAGER_MAP_SIZE 60 #define PAGER_MAP_SIZE PAGER_MAP_DEFAULT_SIZE 61 #endif 62 63 size_t pager_map_size = PAGER_MAP_SIZE; 64 65 /* 66 * list of uvm pagers in the system 67 */ 68 69 const struct uvm_pagerops * const uvmpagerops[] = { 70 &aobj_pager, 71 &uvm_deviceops, 72 &uvm_vnodeops, 73 &ubc_pager, 74 }; 75 76 /* 77 * the pager map: provides KVA for I/O 78 */ 79 80 struct vm_map *pager_map; /* XXX */ 81 kmutex_t pager_map_wanted_lock __cacheline_aligned; 82 bool pager_map_wanted; /* locked by pager map */ 83 static vaddr_t emergva; 84 static int emerg_ncolors; 85 static bool emerginuse; 86 87 void 88 uvm_pager_realloc_emerg(void) 89 { 90 vaddr_t new_emergva, old_emergva; 91 int old_emerg_ncolors; 92 93 if (__predict_true(emergva != 0 && emerg_ncolors >= uvmexp.ncolors)) 94 return; 95 96 KASSERT(!emerginuse); 97 98 new_emergva = uvm_km_alloc(kernel_map, 99 round_page(MAXPHYS) + ptoa(uvmexp.ncolors), ptoa(uvmexp.ncolors), 100 UVM_KMF_VAONLY); 101 102 KASSERT(new_emergva != 0); 103 104 old_emergva = emergva; 105 old_emerg_ncolors = emerg_ncolors; 106 107 /* 108 * don't support re-color in late boot anyway. 109 */ 110 if (0) /* XXX */ 111 mutex_enter(&pager_map_wanted_lock); 112 113 emergva = new_emergva; 114 emerg_ncolors = uvmexp.ncolors; 115 wakeup(&old_emergva); 116 117 if (0) /* XXX */ 118 mutex_exit(&pager_map_wanted_lock); 119 120 if (old_emergva) 121 uvm_km_free(kernel_map, old_emergva, 122 round_page(MAXPHYS) + ptoa(old_emerg_ncolors), 123 UVM_KMF_VAONLY); 124 } 125 126 /* 127 * uvm_pager_init: init pagers (at boot time) 128 */ 129 130 void 131 uvm_pager_init(void) 132 { 133 u_int lcv; 134 vaddr_t sva, eva; 135 136 /* 137 * init pager map 138 */ 139 140 sva = 0; 141 pager_map = uvm_km_suballoc(kernel_map, &sva, &eva, pager_map_size, 0, 142 false, NULL); 143 mutex_init(&pager_map_wanted_lock, MUTEX_DEFAULT, IPL_NONE); 144 pager_map_wanted = false; 145 146 uvm_pager_realloc_emerg(); 147 148 /* 149 * init ASYNC I/O queue 150 */ 151 152 TAILQ_INIT(&uvm.aio_done); 153 154 /* 155 * call pager init functions 156 */ 157 for (lcv = 0 ; lcv < __arraycount(uvmpagerops); lcv++) { 158 if (uvmpagerops[lcv]->pgo_init) 159 uvmpagerops[lcv]->pgo_init(); 160 } 161 } 162 163 /* 164 * uvm_pagermapin: map pages into KVA (pager_map) for I/O that needs mappings 165 * 166 * we basically just map in a blank map entry to reserve the space in the 167 * map and then use pmap_enter() to put the mappings in by hand. 168 */ 169 170 vaddr_t 171 uvm_pagermapin(struct vm_page **pps, int npages, int flags) 172 { 173 vsize_t size; 174 vaddr_t kva; 175 vaddr_t cva; 176 struct vm_page *pp; 177 vm_prot_t prot; 178 const bool pdaemon = (curlwp == uvm.pagedaemon_lwp); 179 const u_int first_color = VM_PGCOLOR_BUCKET(*pps); 180 UVMHIST_FUNC("uvm_pagermapin"); UVMHIST_CALLED(maphist); 181 182 UVMHIST_LOG(maphist,"(pps=0x%#jx, npages=%jd, first_color=%ju)", 183 (uintptr_t)pps, npages, first_color, 0); 184 185 /* 186 * compute protection. outgoing I/O only needs read 187 * access to the page, whereas incoming needs read/write. 188 */ 189 190 prot = VM_PROT_READ; 191 if (flags & UVMPAGER_MAPIN_READ) 192 prot |= VM_PROT_WRITE; 193 194 ReStart: 195 size = ptoa(npages); 196 kva = 0; /* let system choose VA */ 197 198 if (uvm_map(pager_map, &kva, size, NULL, UVM_UNKNOWN_OFFSET, 199 first_color, UVM_FLAG_COLORMATCH | UVM_FLAG_NOMERGE 200 | (pdaemon ? UVM_FLAG_NOWAIT : 0)) != 0) { 201 if (pdaemon) { 202 mutex_enter(&pager_map_wanted_lock); 203 if (emerginuse) { 204 UVM_UNLOCK_AND_WAIT(&emergva, 205 &pager_map_wanted_lock, false, 206 "emergva", 0); 207 goto ReStart; 208 } 209 emerginuse = true; 210 mutex_exit(&pager_map_wanted_lock); 211 kva = emergva + ptoa(first_color); 212 /* The shift implicitly truncates to PAGE_SIZE */ 213 KASSERT(npages <= (MAXPHYS >> PAGE_SHIFT)); 214 goto enter; 215 } 216 if ((flags & UVMPAGER_MAPIN_WAITOK) == 0) { 217 UVMHIST_LOG(maphist,"<- NOWAIT failed", 0,0,0,0); 218 return(0); 219 } 220 mutex_enter(&pager_map_wanted_lock); 221 pager_map_wanted = true; 222 UVMHIST_LOG(maphist, " SLEEPING on pager_map",0,0,0,0); 223 UVM_UNLOCK_AND_WAIT(pager_map, &pager_map_wanted_lock, false, 224 "pager_map", 0); 225 goto ReStart; 226 } 227 228 enter: 229 /* got it */ 230 for (cva = kva; npages != 0; npages--, cva += PAGE_SIZE) { 231 pp = *pps++; 232 KASSERT(pp); 233 // KASSERT(!((VM_PAGE_TO_PHYS(pp) ^ cva) & uvmexp.colormask)); 234 KASSERT(pp->flags & PG_BUSY); 235 pmap_kenter_pa(cva, VM_PAGE_TO_PHYS(pp), prot, 0); 236 } 237 pmap_update(vm_map_pmap(pager_map)); 238 239 UVMHIST_LOG(maphist, "<- done (KVA=0x%jx)", kva,0,0,0); 240 return(kva); 241 } 242 243 /* 244 * uvm_pagermapout: remove pager_map mapping 245 * 246 * we remove our mappings by hand and then remove the mapping (waking 247 * up anyone wanting space). 248 */ 249 250 void 251 uvm_pagermapout(vaddr_t kva, int npages) 252 { 253 vsize_t size = ptoa(npages); 254 struct vm_map_entry *entries; 255 UVMHIST_FUNC("uvm_pagermapout"); UVMHIST_CALLED(maphist); 256 257 UVMHIST_LOG(maphist, " (kva=0x%jx, npages=%jd)", kva, npages,0,0); 258 259 /* 260 * duplicate uvm_unmap, but add in pager_map_wanted handling. 261 */ 262 263 pmap_kremove(kva, size); 264 pmap_update(pmap_kernel()); 265 266 if ((kva & ~ptoa(uvmexp.colormask)) == emergva) { 267 mutex_enter(&pager_map_wanted_lock); 268 KASSERT(emerginuse); 269 emerginuse = false; 270 wakeup(&emergva); 271 mutex_exit(&pager_map_wanted_lock); 272 return; 273 } 274 275 vm_map_lock(pager_map); 276 uvm_unmap_remove(pager_map, kva, kva + size, &entries, 0); 277 mutex_enter(&pager_map_wanted_lock); 278 if (pager_map_wanted) { 279 pager_map_wanted = false; 280 wakeup(pager_map); 281 } 282 mutex_exit(&pager_map_wanted_lock); 283 vm_map_unlock(pager_map); 284 if (entries) 285 uvm_unmap_detach(entries, 0); 286 UVMHIST_LOG(maphist,"<- done",0,0,0,0); 287 } 288 289 /* 290 * interrupt-context iodone handler for single-buf i/os 291 * or the top-level buf of a nested-buf i/o. 292 */ 293 294 void 295 uvm_aio_biodone(struct buf *bp) 296 { 297 /* reset b_iodone for when this is a single-buf i/o. */ 298 bp->b_iodone = uvm_aio_aiodone; 299 300 workqueue_enqueue(uvm.aiodone_queue, &bp->b_work, NULL); 301 } 302 303 void 304 uvm_aio_aiodone_pages(struct vm_page **pgs, int npages, bool write, int error) 305 { 306 struct uvm_object *uobj; 307 struct vm_page *pg; 308 kmutex_t *slock; 309 int pageout_done; /* number of PG_PAGEOUT pages processed */ 310 int swslot; 311 int i; 312 bool swap; 313 UVMHIST_FUNC("uvm_aio_aiodone_pages"); UVMHIST_CALLED(ubchist); 314 315 swslot = 0; 316 pageout_done = 0; 317 slock = NULL; 318 uobj = NULL; 319 pg = pgs[0]; 320 swap = (pg->uanon != NULL && pg->uobject == NULL) || 321 (pg->flags & PG_AOBJ) != 0; 322 if (!swap) { 323 uobj = pg->uobject; 324 slock = uobj->vmobjlock; 325 mutex_enter(slock); 326 } else { 327 #if defined(VMSWAP) 328 if (error) { 329 if (pg->uobject != NULL) { 330 swslot = uao_find_swslot(pg->uobject, 331 pg->offset >> PAGE_SHIFT); 332 } else { 333 KASSERT(pg->uanon != NULL); 334 swslot = pg->uanon->an_swslot; 335 } 336 KASSERT(swslot); 337 } 338 #else /* defined(VMSWAP) */ 339 panic("%s: swap", __func__); 340 #endif /* defined(VMSWAP) */ 341 } 342 for (i = 0; i < npages; i++) { 343 #if defined(VMSWAP) 344 bool anon_disposed = false; /* XXX gcc */ 345 #endif /* defined(VMSWAP) */ 346 347 pg = pgs[i]; 348 KASSERT(swap || pg->uobject == uobj); 349 UVMHIST_LOG(ubchist, "pg %#jx", (uintptr_t)pg, 0,0,0); 350 351 #if defined(VMSWAP) 352 /* 353 * for swap i/os, lock each page's object (or anon) 354 * individually since each page may need a different lock. 355 */ 356 357 if (swap) { 358 if (pg->uobject != NULL) { 359 slock = pg->uobject->vmobjlock; 360 } else { 361 slock = pg->uanon->an_lock; 362 } 363 mutex_enter(slock); 364 anon_disposed = (pg->flags & PG_RELEASED) != 0; 365 KASSERT(!anon_disposed || pg->uobject != NULL || 366 pg->uanon->an_ref == 0); 367 } 368 #endif /* defined(VMSWAP) */ 369 370 /* 371 * process errors. for reads, just mark the page to be freed. 372 * for writes, if the error was ENOMEM, we assume this was 373 * a transient failure so we mark the page dirty so that 374 * we'll try to write it again later. for all other write 375 * errors, we assume the error is permanent, thus the data 376 * in the page is lost. bummer. 377 */ 378 379 if (error) { 380 int slot; 381 if (!write) { 382 pg->flags |= PG_RELEASED; 383 continue; 384 } else if (error == ENOMEM) { 385 if (pg->flags & PG_PAGEOUT) { 386 pg->flags &= ~PG_PAGEOUT; 387 pageout_done++; 388 } 389 pg->flags &= ~PG_CLEAN; 390 uvm_pageactivate(pg); 391 slot = 0; 392 } else 393 slot = SWSLOT_BAD; 394 395 #if defined(VMSWAP) 396 if (swap) { 397 if (pg->uobject != NULL) { 398 int oldslot __diagused; 399 oldslot = uao_set_swslot(pg->uobject, 400 pg->offset >> PAGE_SHIFT, slot); 401 KASSERT(oldslot == swslot + i); 402 } else { 403 KASSERT(pg->uanon->an_swslot == 404 swslot + i); 405 pg->uanon->an_swslot = slot; 406 } 407 } 408 #endif /* defined(VMSWAP) */ 409 } 410 411 /* 412 * if the page is PG_FAKE, this must have been a read to 413 * initialize the page. clear PG_FAKE and activate the page. 414 * we must also clear the pmap "modified" flag since it may 415 * still be set from the page's previous identity. 416 */ 417 418 if (pg->flags & PG_FAKE) { 419 KASSERT(!write); 420 pg->flags &= ~PG_FAKE; 421 #if defined(READAHEAD_STATS) 422 pg->flags |= PG_READAHEAD; 423 uvm_ra_total.ev_count++; 424 #endif /* defined(READAHEAD_STATS) */ 425 KASSERT((pg->flags & PG_CLEAN) != 0); 426 uvm_pageenqueue(pg); 427 pmap_clear_modify(pg); 428 } 429 430 /* 431 * do accounting for pagedaemon i/o and arrange to free 432 * the pages instead of just unbusying them. 433 */ 434 435 if (pg->flags & PG_PAGEOUT) { 436 pg->flags &= ~PG_PAGEOUT; 437 pageout_done++; 438 atomic_inc_uint(&uvmexp.pdfreed); 439 pg->flags |= PG_RELEASED; 440 } 441 442 #if defined(VMSWAP) 443 /* 444 * for swap pages, unlock everything for this page now. 445 */ 446 447 if (swap) { 448 if (pg->uobject == NULL && anon_disposed) { 449 uvm_anon_release(pg->uanon); 450 } else { 451 uvm_page_unbusy(&pg, 1); 452 mutex_exit(slock); 453 } 454 } 455 #endif /* defined(VMSWAP) */ 456 } 457 uvm_pageout_done(pageout_done); 458 if (!swap) { 459 uvm_page_unbusy(pgs, npages); 460 mutex_exit(slock); 461 } else { 462 #if defined(VMSWAP) 463 KASSERT(write); 464 465 /* these pages are now only in swap. */ 466 if (error != ENOMEM) { 467 KASSERT(uvmexp.swpgonly + npages <= uvmexp.swpginuse); 468 atomic_add_int(&uvmexp.swpgonly, npages); 469 } 470 if (error) { 471 if (error != ENOMEM) 472 uvm_swap_markbad(swslot, npages); 473 else 474 uvm_swap_free(swslot, npages); 475 } 476 #endif /* defined(VMSWAP) */ 477 } 478 } 479 480 /* 481 * uvm_aio_aiodone: do iodone processing for async i/os. 482 * this should be called in thread context, not interrupt context. 483 */ 484 485 void 486 uvm_aio_aiodone(struct buf *bp) 487 { 488 int npages = bp->b_bufsize >> PAGE_SHIFT; 489 struct vm_page *pgs[npages]; 490 int i, error; 491 bool write; 492 UVMHIST_FUNC("uvm_aio_aiodone"); UVMHIST_CALLED(ubchist); 493 UVMHIST_LOG(ubchist, "bp %#jx", (uintptr_t)bp, 0,0,0); 494 495 error = bp->b_error; 496 write = (bp->b_flags & B_READ) == 0; 497 498 for (i = 0; i < npages; i++) { 499 pgs[i] = uvm_pageratop((vaddr_t)bp->b_data + (i << PAGE_SHIFT)); 500 UVMHIST_LOG(ubchist, "pgs[%jd] = %#jx", i, 501 (uintptr_t)pgs[i], 0, 0); 502 } 503 uvm_pagermapout((vaddr_t)bp->b_data, npages); 504 505 uvm_aio_aiodone_pages(pgs, npages, write, error); 506 507 if (write && (bp->b_cflags & BC_AGE) != 0) { 508 mutex_enter(bp->b_objlock); 509 vwakeup(bp); 510 mutex_exit(bp->b_objlock); 511 } 512 putiobuf(bp); 513 } 514 515 /* 516 * uvm_pageratop: convert KVAs in the pager map back to their page 517 * structures. 518 */ 519 520 struct vm_page * 521 uvm_pageratop(vaddr_t kva) 522 { 523 struct vm_page *pg; 524 paddr_t pa; 525 bool rv __diagused; 526 527 rv = pmap_extract(pmap_kernel(), kva, &pa); 528 KASSERT(rv); 529 pg = PHYS_TO_VM_PAGE(pa); 530 KASSERT(pg != NULL); 531 return (pg); 532 } 533
Indexes created Fri Oct 17 00:09:41 GMT 2025