1 /* $NetBSD: uvm_vnode.c,v 1.122 2026/07/21 14:37:53 thorpej Exp $ */ 2 3 /* 4 * Copyright (c) 1997 Charles D. Cranor and Washington University. 5 * Copyright (c) 1991, 1993 6 * The Regents of the University of California. 7 * Copyright (c) 1990 University of Utah. 8 * 9 * All rights reserved. 10 * 11 * This code is derived from software contributed to Berkeley by 12 * the Systems Programming Group of the University of Utah Computer 13 * Science Department. 14 * 15 * Redistribution and use in source and binary forms, with or without 16 * modification, are permitted provided that the following conditions 17 * are met: 18 * 1. Redistributions of source code must retain the above copyright 19 * notice, this list of conditions and the following disclaimer. 20 * 2. Redistributions in binary form must reproduce the above copyright 21 * notice, this list of conditions and the following disclaimer in the 22 * documentation and/or other materials provided with the distribution. 23 * 3. Neither the name of the University nor the names of its contributors 24 * may be used to endorse or promote products derived from this software 25 * without specific prior written permission. 26 * 27 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 28 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 29 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 30 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 31 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 32 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 33 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 34 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 35 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 36 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 37 * SUCH DAMAGE. 38 * 39 * @(#)vnode_pager.c 8.8 (Berkeley) 2/13/94 40 * from: Id: uvm_vnode.c,v 1.1.2.26 1998/02/02 20:38:07 chuck Exp 41 */ 42 43 /* 44 * uvm_vnode.c: the vnode pager. 45 */ 46 47 #include <sys/cdefs.h> 48 __KERNEL_RCSID(0, "$NetBSD: uvm_vnode.c,v 1.122 2026/07/21 14:37:53 thorpej Exp $"); 49 50 #ifdef _KERNEL_OPT 51 #include "opt_uvmhist.h" 52 #endif 53 54 #include <sys/atomic.h> 55 #include <sys/param.h> 56 #include <sys/systm.h> 57 #include <sys/kernel.h> 58 #include <sys/vnode.h> 59 #include <sys/ioctl.h> 60 #include <sys/fcntl.h> 61 #include <sys/conf.h> 62 #include <sys/pool.h> 63 #include <sys/mount.h> 64 65 #include <miscfs/specfs/specdev.h> 66 67 #include <uvm/uvm.h> 68 #include <uvm/uvm_readahead.h> 69 #include <uvm/uvm_page_array.h> 70 71 #ifdef UVMHIST 72 UVMHIST_DEFINE(ubchist); 73 #endif 74 75 /* 76 * functions 77 */ 78 79 static void uvn_alloc_ractx(struct uvm_object *); 80 static void uvn_detach(struct uvm_object *); 81 static int uvn_get(struct uvm_object *, voff_t, struct vm_page **, int *, 82 int, vm_prot_t, int, int); 83 static void uvn_markdirty(struct uvm_object *); 84 static int uvn_put(struct uvm_object *, voff_t, voff_t, int); 85 static void uvn_reference(struct uvm_object *); 86 87 static int uvn_findpage(struct uvm_object *, voff_t, struct vm_page **, 88 unsigned int, struct uvm_page_array *a, 89 unsigned int); 90 91 /* 92 * master pager structure 93 */ 94 95 const struct uvm_pagerops uvm_vnodeops = { 96 .pgo_reference = uvn_reference, 97 .pgo_detach = uvn_detach, 98 .pgo_get = uvn_get, 99 .pgo_put = uvn_put, 100 .pgo_markdirty = uvn_markdirty, 101 }; 102 103 /* 104 * the ops! 105 */ 106 107 /* 108 * uvn_reference 109 * 110 * duplicate a reference to a VM object. Note that the reference 111 * count must already be at least one (the passed in reference) so 112 * there is no chance of the uvn being killed or locked out here. 113 * 114 * => caller must call with object unlocked. 115 * => caller must be using the same accessprot as was used at attach time 116 */ 117 118 static void 119 uvn_reference(struct uvm_object *uobj) 120 { 121 vref((struct vnode *)uobj); 122 } 123 124 125 /* 126 * uvn_detach 127 * 128 * remove a reference to a VM object. 129 * 130 * => caller must call with object unlocked and map locked. 131 */ 132 133 static void 134 uvn_detach(struct uvm_object *uobj) 135 { 136 vrele((struct vnode *)uobj); 137 } 138 139 /* 140 * uvn_put: flush page data to backing store. 141 * 142 * => object must be locked on entry! VOP_PUTPAGES must unlock it. 143 * => flags: PGO_SYNCIO -- use sync. I/O 144 */ 145 146 static int 147 uvn_put(struct uvm_object *uobj, voff_t offlo, voff_t offhi, int flags) 148 { 149 struct vnode *vp = (struct vnode *)uobj; 150 int error; 151 152 KASSERT(rw_write_held(uobj->vmobjlock)); 153 error = VOP_PUTPAGES(vp, offlo, offhi, flags); 154 155 return error; 156 } 157 158 /* 159 * uvn_get: get pages (synchronously) from backing store 160 * 161 * => prefer map unlocked (not required) 162 * => object must be locked! we will _unlock_ it before starting any I/O. 163 * => flags: PGO_LOCKED: fault data structures are locked 164 * => NOTE: offset is the offset of pps[0], _NOT_ pps[centeridx] 165 * => NOTE: caller must check for released pages!! 166 */ 167 168 static int 169 uvn_get(struct uvm_object *uobj, voff_t offset, 170 struct vm_page **pps /* IN/OUT */, 171 int *npagesp /* IN (OUT if PGO_LOCKED)*/, 172 int centeridx, vm_prot_t access_type, int advice, int flags) 173 { 174 struct vnode *vp = (struct vnode *)uobj; 175 int error; 176 177 UVMHIST_FUNC(__func__); 178 UVMHIST_CALLARGS(ubchist, "vp %#jx off %#jx", (uintptr_t)vp, offset, 179 0, 0); 180 181 if (vp->v_type == VREG && (access_type & VM_PROT_WRITE) == 0 182 && (flags & PGO_LOCKED) == 0 && vp->v_tag != VT_TMPFS) { 183 uvn_alloc_ractx(uobj); 184 uvm_ra_request(vp->v_ractx, advice, uobj, offset, 185 *npagesp << PAGE_SHIFT); 186 } 187 188 error = VOP_GETPAGES(vp, offset, pps, npagesp, centeridx, 189 access_type, advice, flags); 190 191 if (flags & PGO_LOCKED) 192 KASSERT(rw_lock_held(uobj->vmobjlock)); 193 return error; 194 } 195 196 /* 197 * uvn_markdirty: called when the object gains first dirty page 198 * 199 * => uobj must be write locked. 200 */ 201 202 static void 203 uvn_markdirty(struct uvm_object *uobj) 204 { 205 struct vnode *vp = (struct vnode *)uobj; 206 207 KASSERT(rw_write_held(uobj->vmobjlock)); 208 209 mutex_enter(vp->v_interlock); 210 if ((vp->v_iflag & VI_ONWORKLST) == 0) { 211 vn_syncer_add_to_worklist(vp, filedelay); 212 } 213 mutex_exit(vp->v_interlock); 214 } 215 216 /* 217 * uvn_findpages: 218 * return the page for the uobj and offset requested, allocating if needed. 219 * => uobj must be locked. 220 * => returned pages will be BUSY. 221 */ 222 223 int 224 uvn_findpages(struct uvm_object *uobj, voff_t offset, unsigned int *npagesp, 225 struct vm_page **pgs, struct uvm_page_array *a, unsigned int flags) 226 { 227 unsigned int count, found, npages; 228 int i, rv; 229 struct uvm_page_array a_store; 230 231 if (a == NULL) { 232 /* 233 * XXX fragile API 234 * note that the array can be the one supplied by the caller of 235 * uvn_findpages. in that case, fillflags used by the caller 236 * might not match strictly with ours. 237 * in particular, the caller might have filled the array 238 * without DENSE but passed us UFP_DIRTYONLY (thus DENSE). 239 */ 240 const unsigned int fillflags = 241 ((flags & UFP_BACKWARD) ? UVM_PAGE_ARRAY_FILL_BACKWARD : 0) | 242 ((flags & UFP_DIRTYONLY) ? 243 (UVM_PAGE_ARRAY_FILL_DIRTY|UVM_PAGE_ARRAY_FILL_DENSE) : 0); 244 a = &a_store; 245 uvm_page_array_init(a, uobj, fillflags); 246 } 247 count = found = 0; 248 npages = *npagesp; 249 if (flags & UFP_BACKWARD) { 250 for (i = npages - 1; i >= 0; i--, offset -= PAGE_SIZE) { 251 rv = uvn_findpage(uobj, offset, &pgs[i], flags, a, 252 i + 1); 253 if (rv == 0) { 254 if (flags & UFP_DIRTYONLY) 255 break; 256 } else 257 found++; 258 count++; 259 } 260 } else { 261 for (i = 0; i < npages; i++, offset += PAGE_SIZE) { 262 rv = uvn_findpage(uobj, offset, &pgs[i], flags, a, 263 npages - i); 264 if (rv == 0) { 265 if (flags & UFP_DIRTYONLY) 266 break; 267 } else 268 found++; 269 count++; 270 } 271 } 272 if (a == &a_store) { 273 uvm_page_array_fini(a); 274 } 275 *npagesp = count; 276 return (found); 277 } 278 279 /* 280 * uvn_findpage: find a single page 281 * 282 * if a suitable page was found, put it in *pgp and return 1. 283 * otherwise return 0. 284 */ 285 286 static int 287 uvn_findpage(struct uvm_object *uobj, voff_t offset, struct vm_page **pgp, 288 unsigned int flags, struct uvm_page_array *a, unsigned int nleft) 289 { 290 struct vm_page *pg; 291 UVMHIST_FUNC(__func__); 292 UVMHIST_CALLARGS(ubchist, "vp %#jx off %#jx", (uintptr_t)uobj, offset, 293 0, 0); 294 295 /* 296 * NOBUSY must come with NOWAIT and NOALLOC. if NOBUSY is 297 * specified, this may be called with a reader lock. 298 */ 299 300 KASSERT(rw_lock_held(uobj->vmobjlock)); 301 KASSERT((flags & UFP_NOBUSY) == 0 || (flags & UFP_NOWAIT) != 0); 302 KASSERT((flags & UFP_NOBUSY) == 0 || (flags & UFP_NOALLOC) != 0); 303 KASSERT((flags & UFP_NOBUSY) != 0 || rw_write_held(uobj->vmobjlock)); 304 305 if (*pgp != NULL) { 306 UVMHIST_LOG(ubchist, "dontcare", 0,0,0,0); 307 goto skip_offset; 308 } 309 for (;;) { 310 /* 311 * look for an existing page. 312 */ 313 pg = uvm_page_array_fill_and_peek(a, offset, nleft); 314 if (pg != NULL && pg->offset != offset) { 315 struct vm_page __diagused *tpg; 316 KASSERT( 317 ((a->ar_flags & UVM_PAGE_ARRAY_FILL_BACKWARD) != 0) 318 == (pg->offset < offset)); 319 KASSERT((tpg = uvm_pagelookup(uobj, offset)) == NULL || 320 ((a->ar_flags & UVM_PAGE_ARRAY_FILL_DIRTY) != 0 && 321 !uvm_obj_page_dirty_p(tpg))); 322 pg = NULL; 323 if ((a->ar_flags & UVM_PAGE_ARRAY_FILL_DENSE) != 0) { 324 UVMHIST_LOG(ubchist, "dense", 0,0,0,0); 325 return 0; 326 } 327 } 328 329 /* nope? allocate one now */ 330 if (pg == NULL) { 331 if (flags & UFP_NOALLOC) { 332 UVMHIST_LOG(ubchist, "noalloc", 0,0,0,0); 333 return 0; 334 } 335 pg = uvm_pagealloc(uobj, offset, NULL, 336 UVM_FLAG_COLORMATCH); 337 if (pg == NULL) { 338 if (flags & UFP_NOWAIT) { 339 UVMHIST_LOG(ubchist, "nowait",0,0,0,0); 340 return 0; 341 } 342 rw_exit(uobj->vmobjlock); 343 uvm_wait("uvnfp1"); 344 uvm_page_array_clear(a); 345 rw_enter(uobj->vmobjlock, RW_WRITER); 346 continue; 347 } 348 UVMHIST_LOG(ubchist, "alloced %#jx (color %ju)", 349 (uintptr_t)pg, VM_PGCOLOR(pg), 0, 0); 350 KASSERTMSG(uvm_pagegetdirty(pg) == 351 UVM_PAGE_STATUS_CLEAN, "page %p not clean", pg); 352 break; 353 } else if (flags & UFP_NOCACHE) { 354 UVMHIST_LOG(ubchist, "nocache",0,0,0,0); 355 goto skip; 356 } 357 358 /* page is there, see if we need to wait on it */ 359 if ((pg->flags & PG_BUSY) != 0) { 360 if (flags & UFP_NOWAIT) { 361 UVMHIST_LOG(ubchist, "nowait",0,0,0,0); 362 goto skip; 363 } 364 UVMHIST_LOG(ubchist, "wait %#jx (color %ju)", 365 (uintptr_t)pg, VM_PGCOLOR(pg), 0, 0); 366 uvm_pagewait(pg, uobj->vmobjlock, "uvnfp2"); 367 uvm_page_array_clear(a); 368 rw_enter(uobj->vmobjlock, RW_WRITER); 369 continue; 370 } 371 372 /* skip PG_RDONLY pages if requested */ 373 if ((flags & UFP_NORDONLY) && (pg->flags & PG_RDONLY)) { 374 UVMHIST_LOG(ubchist, "nordonly",0,0,0,0); 375 goto skip; 376 } 377 378 /* stop on clean pages if requested */ 379 if (flags & UFP_DIRTYONLY) { 380 const bool dirty = uvm_pagecheckdirty(pg, false); 381 if (!dirty) { 382 UVMHIST_LOG(ubchist, "dirtonly", 0,0,0,0); 383 return 0; 384 } 385 } 386 387 /* mark the page BUSY and we're done. */ 388 if ((flags & UFP_NOBUSY) == 0) { 389 pg->flags |= PG_BUSY; 390 UVM_PAGE_OWN(pg, "uvn_findpage"); 391 } 392 UVMHIST_LOG(ubchist, "found %#jx (color %ju)", 393 (uintptr_t)pg, VM_PGCOLOR(pg), 0, 0); 394 uvm_page_array_advance(a); 395 break; 396 } 397 *pgp = pg; 398 return 1; 399 400 skip_offset: 401 /* 402 * skip this offset 403 */ 404 pg = uvm_page_array_peek(a); 405 if (pg != NULL) { 406 if (pg->offset == offset) { 407 uvm_page_array_advance(a); 408 } else { 409 KASSERT((a->ar_flags & UVM_PAGE_ARRAY_FILL_DENSE) == 0); 410 } 411 } 412 return 0; 413 414 skip: 415 /* 416 * skip this page 417 */ 418 KASSERT(pg != NULL); 419 uvm_page_array_advance(a); 420 return 0; 421 } 422 423 /* 424 * uvm_vnp_setsize: grow or shrink a vnode uobj 425 * 426 * grow => just update size value 427 * shrink => toss un-needed pages 428 * 429 * => we assume that the caller has a reference of some sort to the 430 * vnode in question so that it will not be yanked out from under 431 * us. 432 */ 433 434 void 435 uvm_vnp_setsize(struct vnode *vp, voff_t newsize) 436 { 437 struct uvm_object *uobj = &vp->v_uobj; 438 voff_t pgend = round_page(newsize); 439 voff_t oldsize; 440 UVMHIST_FUNC(__func__); UVMHIST_CALLED(ubchist); 441 442 rw_enter(uobj->vmobjlock, RW_WRITER); 443 UVMHIST_LOG(ubchist, "vp %#jx old %#jx new %#jx", 444 (uintptr_t)vp, vp->v_size, newsize, 0); 445 446 /* 447 * now check if the size has changed: if we shrink we had better 448 * toss some pages... 449 */ 450 451 KASSERT(newsize != VSIZENOTSET); 452 KASSERT(newsize >= 0); 453 KASSERTMSG(vp->v_size <= vp->v_writesize, "vp=%p" 454 " v_size=0x%llx v_writesize=0x%llx", vp, 455 (unsigned long long)vp->v_size, 456 (unsigned long long)vp->v_writesize); 457 KASSERTMSG((vp->v_size == vp->v_writesize || 458 newsize == vp->v_writesize || newsize <= vp->v_size), 459 "vp=%p v_size=0x%llx v_writesize=0x%llx newsize=0x%llx", 460 vp, 461 (unsigned long long)vp->v_size, 462 (unsigned long long)vp->v_writesize, 463 (unsigned long long)newsize); 464 465 oldsize = vp->v_writesize; 466 467 /* 468 * check whether size shrinks 469 * if old size hasn't been set, there are no pages to drop 470 * if there was an integer overflow in pgend, then this is no shrink 471 */ 472 if (oldsize > pgend && oldsize != VSIZENOTSET && pgend >= 0) { 473 (void) uvn_put(uobj, pgend, 0, PGO_FREE | PGO_SYNCIO); 474 rw_enter(uobj->vmobjlock, RW_WRITER); 475 } 476 mutex_enter(vp->v_interlock); 477 vp->v_size = vp->v_writesize = newsize; 478 mutex_exit(vp->v_interlock); 479 rw_exit(uobj->vmobjlock); 480 } 481 482 void 483 uvm_vnp_setwritesize(struct vnode *vp, voff_t newsize) 484 { 485 486 rw_enter(vp->v_uobj.vmobjlock, RW_WRITER); 487 KASSERT(newsize != VSIZENOTSET); 488 KASSERT(newsize >= 0); 489 KASSERT(vp->v_size != VSIZENOTSET); 490 KASSERT(vp->v_writesize != VSIZENOTSET); 491 KASSERTMSG(vp->v_size <= vp->v_writesize, "vp=%p" 492 " v_size=0x%llx v_writesize=0x%llx newsize=0x%llx", vp, 493 (unsigned long long)vp->v_size, 494 (unsigned long long)vp->v_writesize, 495 (unsigned long long)newsize); 496 KASSERTMSG(vp->v_size <= newsize, "vp=%p" 497 " v_size=0x%llx v_writesize=0x%llx newsize=0x%llx", vp, 498 (unsigned long long)vp->v_size, 499 (unsigned long long)vp->v_writesize, 500 (unsigned long long)newsize); 501 mutex_enter(vp->v_interlock); 502 vp->v_writesize = newsize; 503 mutex_exit(vp->v_interlock); 504 rw_exit(vp->v_uobj.vmobjlock); 505 } 506 507 bool 508 uvn_text_p(struct uvm_object *uobj) 509 { 510 struct vnode *vp = (struct vnode *)uobj; 511 int iflag; 512 513 /* 514 * v_interlock is not held here, but VI_EXECMAP is only ever changed 515 * with the vmobjlock held too. 516 */ 517 iflag = atomic_load_relaxed(&vp->v_iflag); 518 return (iflag & VI_EXECMAP) != 0; 519 } 520 521 static void 522 uvn_alloc_ractx(struct uvm_object *uobj) 523 { 524 struct vnode *vp = (struct vnode *)uobj; 525 struct uvm_ractx *ra = NULL; 526 527 KASSERT(rw_write_held(uobj->vmobjlock)); 528 529 if (vp->v_type != VREG) { 530 return; 531 } 532 if (vp->v_ractx != NULL) { 533 return; 534 } 535 if (vp->v_ractx == NULL) { 536 rw_exit(uobj->vmobjlock); 537 ra = uvm_ra_allocctx(); 538 rw_enter(uobj->vmobjlock, RW_WRITER); 539 if (ra != NULL && vp->v_ractx == NULL) { 540 vp->v_ractx = ra; 541 ra = NULL; 542 } 543 } 544 if (ra != NULL) { 545 uvm_ra_freectx(ra); 546 } 547 } 548
Indexes created Sun Aug 09 00:24:28 UTC 2026