1 /* $NetBSD: lfs_pages.c,v 1.33 2026/09/09 15:23:11 perseant Exp $ */ 2 3 /*- 4 * Copyright (c) 1999, 2000, 2001, 2002, 2003, 2019 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Konrad E. Schroder <perseant (at) hhhh.org>. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 /* 32 * Copyright (c) 1986, 1989, 1991, 1993, 1995 33 * The Regents of the University of California. All rights reserved. 34 * 35 * Redistribution and use in source and binary forms, with or without 36 * modification, are permitted provided that the following conditions 37 * are met: 38 * 1. Redistributions of source code must retain the above copyright 39 * notice, this list of conditions and the following disclaimer. 40 * 2. Redistributions in binary form must reproduce the above copyright 41 * notice, this list of conditions and the following disclaimer in the 42 * documentation and/or other materials provided with the distribution. 43 * 3. Neither the name of the University nor the names of its contributors 44 * may be used to endorse or promote products derived from this software 45 * without specific prior written permission. 46 * 47 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 48 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 49 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 50 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 51 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 52 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 53 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 54 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 55 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 56 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 57 * SUCH DAMAGE. 58 * 59 * @(#)lfs_vnops.c 8.13 (Berkeley) 6/10/95 60 */ 61 62 #include <sys/cdefs.h> 63 __KERNEL_RCSID(0, "$NetBSD: lfs_pages.c,v 1.33 2026/09/09 15:23:11 perseant Exp $"); 64 65 #ifdef _KERNEL_OPT 66 #include "opt_compat_netbsd.h" 67 #include "opt_uvm_page_trkown.h" 68 #endif 69 70 #include <sys/param.h> 71 #include <sys/systm.h> 72 #include <sys/namei.h> 73 #include <sys/resourcevar.h> 74 #include <sys/kernel.h> 75 #include <sys/file.h> 76 #include <sys/stat.h> 77 #include <sys/buf.h> 78 #include <sys/proc.h> 79 #include <sys/mount.h> 80 #include <sys/vnode.h> 81 #include <sys/pool.h> 82 #include <sys/signalvar.h> 83 #include <sys/kauth.h> 84 #include <sys/syslog.h> 85 #include <sys/fstrans.h> 86 87 #include <miscfs/fifofs/fifo.h> 88 #include <miscfs/genfs/genfs.h> 89 #include <miscfs/specfs/specdev.h> 90 91 #include <ufs/lfs/ulfs_inode.h> 92 #include <ufs/lfs/ulfsmount.h> 93 #include <ufs/lfs/ulfs_bswap.h> 94 #include <ufs/lfs/ulfs_extern.h> 95 96 #include <uvm/uvm.h> 97 #include <uvm/uvm_page.h> 98 #include <uvm/uvm_pager.h> 99 #include <uvm/uvm_pmap.h> 100 #include <uvm/uvm_stat.h> 101 102 #include <ufs/lfs/lfs.h> 103 #include <ufs/lfs/lfs_accessors.h> 104 #include <ufs/lfs/lfs_kernel.h> 105 #include <ufs/lfs/lfs_extern.h> 106 107 extern kcondvar_t lfs_writerd_cv; 108 109 static int check_dirty(struct lfs *, struct vnode *, off_t, off_t, off_t, int, int, struct vm_page **); 110 111 int 112 lfs_getpages(void *v) 113 { 114 struct vop_getpages_args /* { 115 struct vnode *a_vp; 116 voff_t a_offset; 117 struct vm_page **a_m; 118 int *a_count; 119 int a_centeridx; 120 vm_prot_t a_access_type; 121 int a_advice; 122 int a_flags; 123 } */ *ap = v; 124 125 if (VTOI(ap->a_vp)->i_number == LFS_IFILE_INUM && 126 (ap->a_access_type & VM_PROT_WRITE) != 0) { 127 return EPERM; 128 } 129 if ((ap->a_access_type & VM_PROT_WRITE) != 0) { 130 mutex_enter(&lfs_lock); 131 LFS_SET_UINO(VTOI(ap->a_vp), IN_MODIFIED); 132 mutex_exit(&lfs_lock); 133 } 134 135 /* 136 * we're relying on the fact that genfs_getpages() always read in 137 * entire filesystem blocks. 138 */ 139 return genfs_getpages(v); 140 } 141 142 /* 143 * Wait for a page to become unbusy, possibly printing diagnostic messages 144 * as well. 145 * 146 * Called with vp->v_uobj.vmobjlock held; return with it held. 147 */ 148 static void 149 wait_for_page(struct vnode *vp, struct vm_page *pg, const char *label) 150 { 151 KASSERT(rw_write_held(vp->v_uobj.vmobjlock)); 152 if ((pg->flags & PG_BUSY) == 0) 153 return; /* Nothing to wait for! */ 154 155 #if defined(DEBUG) && defined(UVM_PAGE_TRKOWN) 156 static struct vm_page *lastpg; 157 158 if (label != NULL && pg != lastpg) { 159 if (pg->owner_tag) { 160 printf("lfs_putpages[%d.%d]: %s: page %p owner %d.%d [%s]\n", 161 curproc->p_pid, curlwp->l_lid, label, 162 pg, pg->owner, pg->lowner, pg->owner_tag); 163 } else { 164 printf("lfs_putpages[%d.%d]: %s: page %p unowned?!\n", 165 curproc->p_pid, curlwp->l_lid, label, pg); 166 } 167 } 168 lastpg = pg; 169 #endif 170 171 uvm_pagewait(pg, vp->v_uobj.vmobjlock, "lfsput"); 172 rw_enter(vp->v_uobj.vmobjlock, RW_WRITER); 173 } 174 175 /* 176 * This routine is called by lfs_putpages() when it can't complete the 177 * write because a page is busy. This means that either (1) someone, 178 * possibly the pagedaemon, is looking at this page, and will give it up 179 * presently; or (2) we ourselves are holding the page busy in the 180 * process of being written (either gathered or actually on its way to 181 * disk). We don't need to give up the segment lock, but we might need 182 * to call lfs_writeseg() to expedite the page's journey to disk. 183 * 184 * Called with vp->v_uobj.vmobjlock held; return with it held. 185 */ 186 /* #define BUSYWAIT */ 187 static void 188 write_and_wait(struct lfs *fs, struct vnode *vp, struct vm_page *pg, 189 int seglocked, const char *label) 190 { 191 KASSERT(rw_write_held(vp->v_uobj.vmobjlock)); 192 #ifndef BUSYWAIT 193 struct inode *ip = VTOI(vp); 194 struct segment *sp = fs->lfs_sp; 195 int count = 0; 196 197 if (pg == NULL) 198 return; 199 200 while (pg->flags & PG_BUSY && 201 pg->uobject == &vp->v_uobj) { 202 rw_exit(vp->v_uobj.vmobjlock); 203 if (sp->cbpp - sp->bpp > 1) { 204 /* Write gathered pages */ 205 lfs_updatemeta(sp); 206 lfs_release_finfo(fs); 207 lfs_writeinode(fs, sp, ip); 208 (void) lfs_writeseg(fs, sp); 209 210 /* 211 * Reinitialize FIP 212 */ 213 KASSERT(sp->vp == vp); 214 lfs_acquire_finfo(fs, ip->i_number, 215 ip->i_gen); 216 } 217 ++count; 218 rw_enter(vp->v_uobj.vmobjlock, RW_WRITER); 219 wait_for_page(vp, pg, label); 220 } 221 if (label != NULL && count > 1) { 222 DLOG((DLOG_PAGE, "lfs_putpages[%d]: %s: %sn = %d\n", 223 curproc->p_pid, label, (count > 0 ? "looping, " : ""), 224 count)); 225 } 226 #else 227 preempt(1); 228 #endif 229 KASSERT(rw_write_held(vp->v_uobj.vmobjlock)); 230 } 231 232 /* 233 * Make sure that for all pages in every block in the given range, 234 * either all are dirty or all are clean. If any of the pages 235 * we've seen so far are dirty, put the vnode on the paging chain, 236 * and mark it IN_PAGING. 237 * 238 * If checkfirst != 0, don't check all the pages but return at the 239 * first dirty page. 240 */ 241 static int 242 check_dirty(struct lfs *fs, struct vnode *vp, 243 off_t startoffset, off_t endoffset, off_t blkeof, 244 int flags, int checkfirst, struct vm_page **pgp) 245 { 246 struct vm_page *pgs[MAXBSIZE / MIN_PAGE_SIZE], *pg; 247 off_t soff = 0; /* XXX: gcc */ 248 voff_t off; 249 int i; 250 int nonexistent; 251 int any_dirty; /* number of dirty pages */ 252 int dirty; /* number of dirty pages in a block */ 253 int tdirty; 254 int pages_per_block = lfs_sb_getbsize(fs) >> PAGE_SHIFT; 255 int pagedaemon = uvm_lwp_is_pagedaemon(curlwp); 256 257 KASSERT(rw_write_held(vp->v_uobj.vmobjlock)); 258 ASSERT_MAYBE_SEGLOCK(fs); 259 top: 260 any_dirty = 0; 261 262 soff = startoffset; 263 KASSERT((soff & (lfs_sb_getbsize(fs) - 1)) == 0); 264 while (soff < MIN(blkeof, endoffset)) { 265 266 /* 267 * Mark all pages in extended range busy; find out if any 268 * of them are dirty. 269 */ 270 nonexistent = dirty = 0; 271 for (i = 0; i == 0 || i < pages_per_block; i++) { 272 KASSERT(rw_write_held(vp->v_uobj.vmobjlock)); 273 off = soff + (i << PAGE_SHIFT); 274 pgs[i] = pg = uvm_pagelookup(&vp->v_uobj, off); 275 if (pg == NULL) { 276 ++nonexistent; 277 continue; 278 } 279 KASSERT(pg != NULL); 280 281 /* 282 * If we're holding the segment lock, we can deadlock 283 * against a process that has our page and is waiting 284 * for the cleaner, while the cleaner waits for the 285 * segment lock. Just bail in that case. 286 */ 287 if ((pg->flags & PG_BUSY) && 288 (pagedaemon || LFS_SEGLOCK_HELD(fs))) { 289 if (i > 0) 290 uvm_page_unbusy(pgs, i); 291 DLOG((DLOG_PAGE, "lfs_putpages: avoiding 3-way or pagedaemon deadlock\n")); 292 if (pgp) 293 *pgp = pg; 294 KASSERT(rw_write_held(vp->v_uobj.vmobjlock)); 295 return -1; 296 } 297 298 while (pg->flags & PG_BUSY) { 299 wait_for_page(vp, pg, NULL); 300 KASSERT(rw_write_held(vp->v_uobj.vmobjlock)); 301 if (i > 0) 302 uvm_page_unbusy(pgs, i); 303 KASSERT(rw_write_held(vp->v_uobj.vmobjlock)); 304 goto top; 305 } 306 pg->flags |= PG_BUSY; 307 UVM_PAGE_OWN(pg, "lfs_putpages"); 308 309 pmap_page_protect(pg, VM_PROT_NONE); 310 tdirty = 311 uvm_pagegetdirty(pg) != UVM_PAGE_STATUS_CLEAN && 312 (uvm_pagegetdirty(pg) == UVM_PAGE_STATUS_DIRTY || 313 pmap_clear_modify(pg)); 314 dirty += tdirty; 315 } 316 if ((pages_per_block > 0 && nonexistent >= pages_per_block) || 317 (pages_per_block == 0 && nonexistent > 0)) { 318 soff += MAX(PAGE_SIZE, lfs_sb_getbsize(fs)); 319 continue; 320 } 321 322 any_dirty += dirty; 323 KASSERT(nonexistent == 0); 324 KASSERT(rw_write_held(vp->v_uobj.vmobjlock)); 325 326 /* 327 * If any are dirty make all dirty; unbusy them, 328 * but if we were asked to clean, wire them so that 329 * the pagedaemon doesn't bother us about them while 330 * they're on their way to disk. 331 */ 332 for (i = 0; i == 0 || i < pages_per_block; i++) { 333 KASSERT(rw_write_held(vp->v_uobj.vmobjlock)); 334 pg = pgs[i]; 335 KASSERT(!(uvm_pagegetdirty(pg) != UVM_PAGE_STATUS_DIRTY 336 && (pg->flags & PG_DELWRI))); 337 KASSERT(pg->flags & PG_BUSY); 338 if (dirty) { 339 uvm_pagemarkdirty(pg, UVM_PAGE_STATUS_DIRTY); 340 if (flags & PGO_FREE) { 341 /* 342 * Wire the page so that 343 * pdaemon doesn't see it again. 344 */ 345 uvm_pagelock(pg); 346 uvm_pagewire(pg); 347 uvm_pageunlock(pg); 348 349 /* Suspended write flag */ 350 pg->flags |= PG_DELWRI; 351 } 352 } 353 pg->flags &= ~PG_BUSY; 354 uvm_pagelock(pg); 355 uvm_pagewakeup(pg); 356 uvm_pageunlock(pg); 357 UVM_PAGE_OWN(pg, NULL); 358 } 359 360 if (checkfirst && any_dirty) 361 break; 362 363 soff += MAX(PAGE_SIZE, lfs_sb_getbsize(fs)); 364 } 365 366 KASSERT(rw_write_held(vp->v_uobj.vmobjlock)); 367 return any_dirty; 368 } 369 370 /* 371 * lfs_putpages functions like genfs_putpages except that 372 * 373 * (1) It needs to bounds-check the incoming requests to ensure that 374 * they are block-aligned; if they are not, expand the range and 375 * do the right thing in case, e.g., the requested range is clean 376 * but the expanded range is dirty. 377 * 378 * (2) It needs to explicitly send blocks to be written when it is done. 379 * If VOP_PUTPAGES is called without the seglock held, we simply take 380 * the seglock and let lfs_segunlock wait for us. 381 * XXX There might be a bad situation if we have to flush a vnode while 382 * XXX lfs_markv is in operation. As of this writing we panic in this 383 * XXX case. 384 * 385 * Assumptions: 386 * 387 * (1) The caller does not hold any pages in this vnode busy. If it does, 388 * there is a danger that when we expand the page range and busy the 389 * pages we will deadlock. 390 * 391 * (2) We are called with vp->v_uobj.vmobjlock held; we must return with it 392 * released. 393 * 394 * (3) We don't absolutely have to free pages right away, provided that 395 * the request does not have PGO_SYNCIO. When the pagedaemon gives 396 * us a request with PGO_FREE, we take the pages out of the paging 397 * queue and wake up the writer, which will handle freeing them for us. 398 * 399 * We ensure that for any filesystem block, all pages for that 400 * block are either resident or not, even if those pages are higher 401 * than EOF; that means that we will be getting requests to free 402 * "unused" pages above EOF all the time, and should ignore them. 403 * 404 * (4) If we are called with PGO_LOCKED, the finfo array we are to write 405 * into has been set up for us by lfs_writefile. If not, we will 406 * have to handle allocating and/or freeing an finfo entry. 407 * 408 * XXX note that we're (ab)using PGO_LOCKED as "seglock held". 409 */ 410 411 /* How many times to loop before we should start to worry */ 412 #define TOOMANY 4 413 414 int 415 lfs_putpages(void *v) 416 { 417 int error = 0; 418 struct vop_putpages_args /* { 419 struct vnode *a_vp; 420 voff_t a_offlo; 421 voff_t a_offhi; 422 int a_flags; 423 } */ *ap = v; 424 struct vnode *vp; 425 struct inode *ip; 426 struct lfs *fs; 427 struct segment *sp; 428 off_t origoffset, startoffset, endoffset, origendoffset, blkeof; 429 off_t off, max_endoffset; 430 bool seglocked, sync, pagedaemon, reclaim; 431 struct vm_page *pg, *busypg; 432 UVMHIST_FUNC("lfs_putpages"); UVMHIST_CALLED(ubchist); 433 struct mount *trans_mp; 434 int oreclaim = 0; 435 int donewriting = 0; 436 #ifdef DEBUG 437 int debug_n_again, debug_n_dirtyclean; 438 #endif 439 440 vp = ap->a_vp; 441 ip = VTOI(vp); 442 fs = ip->i_lfs; 443 sync = (ap->a_flags & PGO_SYNCIO) != 0; 444 reclaim = (ap->a_flags & PGO_RECLAIM) != 0; 445 pagedaemon = uvm_lwp_is_pagedaemon(curlwp); 446 trans_mp = NULL; 447 448 KASSERT(rw_write_held(vp->v_uobj.vmobjlock)); 449 450 /* Putpages does nothing for metadata. */ 451 if (vp == fs->lfs_ivnode || vp->v_type != VREG) { 452 rw_exit(vp->v_uobj.vmobjlock); 453 return 0; 454 } 455 456 retry: 457 /* 458 * If there are no pages, don't do anything. 459 */ 460 if (vp->v_uobj.uo_npages == 0) { 461 mutex_enter(vp->v_interlock); 462 if ((vp->v_iflag & VI_ONWORKLST) && 463 LIST_FIRST(&vp->v_dirtyblkhd) == NULL) { 464 vn_syncer_remove_from_worklist(vp); 465 } 466 mutex_exit(vp->v_interlock); 467 if (trans_mp) 468 fstrans_done(trans_mp); 469 rw_exit(vp->v_uobj.vmobjlock); 470 471 /* Remove us from paging queue, if we were on it */ 472 mutex_enter(&lfs_lock); 473 if (ip->i_state & IN_PAGING) { 474 ip->i_state &= ~IN_PAGING; 475 TAILQ_REMOVE(&fs->lfs_pchainhd, ip, i_lfs_pchain); 476 } 477 mutex_exit(&lfs_lock); 478 479 KASSERT(!rw_write_held(vp->v_uobj.vmobjlock)); 480 return 0; 481 } 482 483 blkeof = lfs_blkroundup(fs, ip->i_size); 484 485 /* 486 * Ignore requests to free pages past EOF but in the same block 487 * as EOF, unless the vnode is being reclaimed or the request 488 * is synchronous. (If the request is sync, it comes from 489 * lfs_truncate.) 490 * 491 * To avoid being flooded with this request, make these pages 492 * look "active". 493 */ 494 if (!sync && !reclaim && 495 ap->a_offlo >= ip->i_size && ap->a_offlo < blkeof) { 496 origoffset = ap->a_offlo; 497 for (off = origoffset; off < blkeof; off += lfs_sb_getbsize(fs)) { 498 pg = uvm_pagelookup(&vp->v_uobj, off); 499 KASSERT(pg != NULL); 500 while (pg->flags & PG_BUSY) { 501 uvm_pagewait(pg, vp->v_uobj.vmobjlock, "lfsput2"); 502 rw_enter(vp->v_uobj.vmobjlock, RW_WRITER); 503 /* XXX Page can't change identity here? */ 504 KDASSERT(pg == 505 uvm_pagelookup(&vp->v_uobj, off)); 506 } 507 uvm_pagelock(pg); 508 uvm_pageactivate(pg); 509 uvm_pageunlock(pg); 510 } 511 ap->a_offlo = blkeof; 512 if (ap->a_offhi > 0 && ap->a_offhi <= ap->a_offlo) { 513 rw_exit(vp->v_uobj.vmobjlock); 514 return 0; 515 } 516 } 517 518 /* 519 * Extend page range to start and end at block boundaries. 520 * (For the purposes of VOP_PUTPAGES, fragments don't exist.) 521 */ 522 origoffset = ap->a_offlo; 523 origendoffset = ap->a_offhi; 524 startoffset = origoffset & ~(lfs_sb_getbmask(fs)); 525 max_endoffset = (trunc_page(LLONG_MAX) >> lfs_sb_getbshift(fs)) 526 << lfs_sb_getbshift(fs); 527 528 if (origendoffset == 0 || ap->a_flags & PGO_ALLPAGES) { 529 endoffset = max_endoffset; 530 origendoffset = endoffset; 531 } else { 532 origendoffset = round_page(ap->a_offhi); 533 endoffset = round_page(lfs_blkroundup(fs, origendoffset)); 534 } 535 536 KASSERT(startoffset > 0 || endoffset >= startoffset); 537 if (startoffset == endoffset) { 538 /* Nothing to do, why were we called? */ 539 rw_exit(vp->v_uobj.vmobjlock); 540 DLOG((DLOG_PAGE, "lfs_putpages: startoffset = endoffset = %" 541 PRId64 "\n", startoffset)); 542 return 0; 543 } 544 545 ap->a_offlo = startoffset; 546 ap->a_offhi = endoffset; 547 548 /* 549 * If not cleaning, just send the pages through genfs_putpages 550 * to be returned to the pool. 551 */ 552 if (!(ap->a_flags & PGO_CLEANIT)) { 553 DLOG((DLOG_PAGE, "lfs_putpages: no cleanit vn %p ino %d (flags %x)\n", 554 vp, (int)ip->i_number, ap->a_flags)); 555 int r = genfs_putpages(v); 556 KASSERT(!rw_write_held(vp->v_uobj.vmobjlock)); 557 return r; 558 } 559 560 if (trans_mp /* && (ap->a_flags & PGO_CLEANIT) != 0 */) { 561 if (pagedaemon) { 562 /* Pagedaemon must not sleep here. */ 563 trans_mp = vp->v_mount; 564 error = fstrans_start_nowait(trans_mp); 565 if (error) { 566 rw_exit(vp->v_uobj.vmobjlock); 567 return error; 568 } 569 } else { 570 /* 571 * Cannot use vdeadcheck() here as this operation 572 * usually gets used from VOP_RECLAIM(). Test for 573 * change of v_mount instead and retry on change. 574 */ 575 rw_exit(vp->v_uobj.vmobjlock); 576 trans_mp = vp->v_mount; 577 fstrans_start(trans_mp); 578 if (vp->v_mount != trans_mp) { 579 fstrans_done(trans_mp); 580 trans_mp = NULL; 581 } 582 } 583 rw_enter(vp->v_uobj.vmobjlock, RW_WRITER); 584 goto retry; 585 } 586 587 /* Set PGO_BUSYFAIL to avoid deadlocks */ 588 ap->a_flags |= PGO_BUSYFAIL; 589 590 /* 591 * Likewise, if we are asked to clean but the pages are not 592 * dirty, we can just free them using genfs_putpages. 593 */ 594 #ifdef DEBUG 595 debug_n_dirtyclean = 0; 596 #endif 597 do { 598 int r; 599 KASSERT(rw_write_held(vp->v_uobj.vmobjlock)); 600 601 /* Count the number of dirty pages */ 602 r = check_dirty(fs, vp, startoffset, endoffset, blkeof, 603 ap->a_flags, 1, NULL); 604 if (r < 0) { 605 /* Pages are busy with another process */ 606 rw_exit(vp->v_uobj.vmobjlock); 607 error = EDEADLK; 608 goto out; 609 } 610 if (r > 0) { /* Some pages are dirty */ 611 LFS_SET_UINO(ip, IN_MODIFIED); 612 break; 613 } 614 615 /* 616 * Sometimes pages are dirtied between the time that 617 * we check and the time we try to clean them. 618 * Instruct lfs_gop_write to return EDEADLK in this case 619 * so we can write them properly. 620 */ 621 ip->i_lfs_iflags |= LFSI_NO_GOP_WRITE; 622 r = genfs_do_putpages(vp, startoffset, endoffset, 623 ap->a_flags & ~PGO_SYNCIO, &busypg); 624 ip->i_lfs_iflags &= ~LFSI_NO_GOP_WRITE; 625 if (r != EDEADLK) { 626 KASSERT(!rw_write_held(vp->v_uobj.vmobjlock)); 627 error = r; 628 goto out; 629 } 630 631 /* One of the pages was busy. Start over. */ 632 rw_enter(vp->v_uobj.vmobjlock, RW_WRITER); 633 wait_for_page(vp, busypg, "dirtyclean"); 634 #ifdef DEBUG 635 ++debug_n_dirtyclean; 636 #endif 637 } while(1); 638 639 #ifdef DEBUG 640 if (debug_n_dirtyclean > TOOMANY) 641 DLOG((DLOG_PAGE, "lfs_putpages: dirtyclean: looping, n = %d\n", 642 debug_n_dirtyclean)); 643 #endif 644 645 /* 646 * Dirty and asked to clean. 647 * 648 * Pagedaemon can't actually write LFS pages; wake up 649 * the writer to take care of that. The writer will 650 * notice the pager inode queue and act on that. 651 * 652 * XXX We must drop the vp->interlock before taking the lfs_lock or we 653 * get a nasty deadlock with lfs_flush_pchain(). 654 */ 655 if (pagedaemon) { 656 rw_exit(vp->v_uobj.vmobjlock); 657 mutex_enter(&lfs_lock); 658 if (!(ip->i_state & IN_PAGING)) { 659 ip->i_state |= IN_PAGING; 660 TAILQ_INSERT_TAIL(&fs->lfs_pchainhd, ip, i_lfs_pchain); 661 } 662 cv_broadcast(&lfs_writerd_cv); 663 mutex_exit(&lfs_lock); 664 preempt(); 665 KASSERT(!rw_write_held(vp->v_uobj.vmobjlock)); 666 error = EWOULDBLOCK; 667 goto out; 668 } 669 670 /* 671 * If this is a file created in a recent dirop, we can't flush its 672 * inode until the dirop is complete. Drain dirops, then flush the 673 * filesystem (taking care of any other pending dirops while we're 674 * at it). 675 */ 676 if ((ap->a_flags & (PGO_CLEANIT|PGO_LOCKED)) == PGO_CLEANIT && 677 (vp->v_uflag & VU_DIROP)) { 678 DLOG((DLOG_PAGE, "lfs_putpages: flushing VU_DIROP\n")); 679 680 /* 681 * NB: lfs_flush_fs can recursively call lfs_putpages, 682 * but it won't reach this branch because it passes 683 * PGO_LOCKED. 684 */ 685 686 rw_exit(vp->v_uobj.vmobjlock); 687 mutex_enter(&lfs_lock); 688 lfs_flush_fs(fs, 0); 689 mutex_exit(&lfs_lock); 690 691 /* 692 * The flush will have cleaned out this vnode as well, 693 * no need to do more to it. We may, though, need to wait. 694 */ 695 goto wait; 696 } 697 698 /* 699 * This is it. We are going to write some pages. From here on 700 * down it's all just mechanics. 701 * 702 * Don't let genfs_putpages wait; lfs_segunlock will wait for us. 703 */ 704 ap->a_flags &= ~PGO_SYNCIO; 705 706 /* 707 * If we've already got the seglock, flush the node and return. 708 * The FIP has already been set up for us by lfs_writefile, 709 * and FIP cleanup and lfs_updatemeta will also be done there, 710 * unless genfs_putpages returns EDEADLK; then we must flush 711 * what we have, and correct FIP and segment header accounting. 712 */ 713 get_seglock: 714 /* 715 * If we are not called with the segment locked, lock it. 716 * Account for a new FIP in the segment header, and set sp->vp. 717 * (This should duplicate the setup at the top of lfs_writefile().) 718 */ 719 seglocked = (ap->a_flags & PGO_LOCKED) != 0; 720 if (!seglocked) { 721 rw_exit(vp->v_uobj.vmobjlock); 722 error = lfs_seglock(fs, 0); 723 if (error != 0) { 724 KASSERT(!rw_write_held(vp->v_uobj.vmobjlock)); 725 goto out; 726 } 727 rw_enter(vp->v_uobj.vmobjlock, RW_WRITER); 728 lfs_acquire_finfo(fs, ip->i_number, ip->i_gen); 729 } 730 sp = fs->lfs_sp; 731 KASSERT(sp->vp == NULL); 732 sp->vp = vp; 733 734 /* Note segments written by reclaim; only for debugging */ 735 mutex_enter(vp->v_interlock); 736 if (vdead_check(vp, VDEAD_NOWAIT) != 0) { 737 sp->seg_flags |= SEGM_RECLAIM; 738 fs->lfs_reclino = ip->i_number; 739 } 740 mutex_exit(vp->v_interlock); 741 742 /* 743 * Ensure that the partial segment is marked SS_DIROP if this 744 * vnode is a DIROP. 745 */ 746 if (!seglocked && vp->v_uflag & VU_DIROP) { 747 SEGSUM *ssp = sp->segsum; 748 749 lfs_ss_setflags(fs, ssp, 750 lfs_ss_getflags(fs, ssp) | (SS_DIROP|SS_CONT)); 751 } 752 753 /* 754 * Loop over genfs_putpages until all pages are gathered. 755 * genfs_putpages() drops the interlock, so reacquire it if necessary. 756 * Whenever we lose the interlock we have to rerun check_dirty, as 757 * well, since more pages might have been dirtied in our absence. 758 */ 759 #ifdef DEBUG 760 debug_n_again = 0; 761 #endif 762 do { 763 busypg = NULL; 764 KASSERT(rw_write_held(vp->v_uobj.vmobjlock)); 765 if (check_dirty(fs, vp, startoffset, endoffset, blkeof, 766 ap->a_flags, 0, &busypg) < 0) { 767 write_and_wait(fs, vp, busypg, seglocked, NULL); 768 if (!seglocked) { 769 rw_exit(vp->v_uobj.vmobjlock); 770 lfs_release_finfo(fs); 771 lfs_segunlock(fs); 772 rw_enter(vp->v_uobj.vmobjlock, RW_WRITER); 773 } 774 sp->vp = NULL; 775 goto get_seglock; 776 } 777 778 busypg = NULL; 779 oreclaim = (ap->a_flags & PGO_RECLAIM); 780 ap->a_flags &= ~PGO_RECLAIM; 781 error = genfs_do_putpages(vp, startoffset, endoffset, 782 ap->a_flags, &busypg); 783 ap->a_flags |= oreclaim; 784 785 if (error == EDEADLK || error == EAGAIN) { 786 DLOG((DLOG_PAGE, "lfs_putpages: genfs_putpages returned" 787 " %d ino %d off %jx (seg %d)\n", error, 788 ip->i_number, (uintmax_t)lfs_sb_getoffset(fs), 789 lfs_dtosn(fs, lfs_sb_getoffset(fs)))); 790 791 if (oreclaim) { 792 rw_enter(vp->v_uobj.vmobjlock, RW_WRITER); 793 write_and_wait(fs, vp, busypg, seglocked, "again"); 794 rw_exit(vp->v_uobj.vmobjlock); 795 } else { 796 if ((sp->seg_flags & SEGM_SINGLE) && 797 lfs_sb_getcurseg(fs) != fs->lfs_startseg) 798 donewriting = 1; 799 } 800 } else if (error) { 801 DLOG((DLOG_PAGE, "lfs_putpages: genfs_putpages returned" 802 " %d ino %d off %jx (seg %d)\n", error, 803 (int)ip->i_number, (uintmax_t)lfs_sb_getoffset(fs), 804 lfs_dtosn(fs, lfs_sb_getoffset(fs)))); 805 } 806 /* genfs_do_putpages loses the interlock */ 807 #ifdef DEBUG 808 ++debug_n_again; 809 #endif 810 if (oreclaim && error == EAGAIN) { 811 DLOG((DLOG_PAGE, "vp %p ino %d vi_flags %x a_flags %x avoiding vclean panic\n", 812 vp, (int)ip->i_number, vp->v_iflag, ap->a_flags)); 813 rw_enter(vp->v_uobj.vmobjlock, RW_WRITER); 814 } 815 if (error == EDEADLK) 816 rw_enter(vp->v_uobj.vmobjlock, RW_WRITER); 817 } while (error == EDEADLK || (oreclaim && error == EAGAIN)); 818 #ifdef DEBUG 819 if (debug_n_again > TOOMANY) 820 DLOG((DLOG_PAGE, "lfs_putpages: again: looping, n = %d\n", debug_n_again)); 821 #endif 822 823 KASSERT(sp != NULL && sp->vp == vp); 824 if (!seglocked && !donewriting) { 825 sp->vp = NULL; 826 827 /* Write indirect blocks as well */ 828 lfs_gather(fs, fs->lfs_sp, vp, lfs_match_indir); 829 lfs_gather(fs, fs->lfs_sp, vp, lfs_match_dindir); 830 lfs_gather(fs, fs->lfs_sp, vp, lfs_match_tindir); 831 832 KASSERT(sp->vp == NULL); 833 sp->vp = vp; 834 } 835 836 /* 837 * Blocks are now gathered into a segment waiting to be written. 838 * All that's left to do is update metadata, and write them. 839 */ 840 lfs_updatemeta(sp); 841 KASSERT(sp->vp == vp); 842 sp->vp = NULL; 843 844 /* 845 * Remove us from paging queue if we wrote all our pages. 846 */ 847 if (origendoffset == 0 || ap->a_flags & PGO_ALLPAGES) { 848 mutex_enter(&lfs_lock); 849 if (ip->i_state & IN_PAGING) { 850 ip->i_state &= ~IN_PAGING; 851 TAILQ_REMOVE(&fs->lfs_pchainhd, ip, i_lfs_pchain); 852 } 853 mutex_exit(&lfs_lock); 854 } 855 856 /* 857 * If we were called from lfs_writefile, we don't need to clean up 858 * the FIP or unlock the segment lock. We're done. 859 */ 860 if (seglocked) { 861 KASSERT(!rw_write_held(vp->v_uobj.vmobjlock)); 862 goto out; 863 } 864 865 /* Clean up FIP and send it to disk. */ 866 lfs_release_finfo(fs); 867 lfs_writeinode(fs, sp, ip); 868 lfs_writeseg(fs, fs->lfs_sp); 869 870 /* 871 * XXX - with the malloc/copy writeseg, the pages are freed by now 872 * even if we don't wait (e.g. if we hold a nested lock). This 873 * will not be true if we stop using malloc/copy. 874 */ 875 lfs_segunlock(fs); 876 877 wait: 878 /* 879 * Wait for v_numoutput to drop to zero. The seglock should 880 * take care of this, but there is a slight possibility that 881 * aiodoned might not have got around to our buffers yet. 882 */ 883 if (sync) { 884 mutex_enter(vp->v_interlock); 885 while (vp->v_numoutput > 0) { 886 DLOG((DLOG_PAGE, "lfs_putpages: ino %d sleeping on" 887 " num %d\n", ip->i_number, vp->v_numoutput)); 888 cv_wait(&vp->v_cv, vp->v_interlock); 889 } 890 mutex_exit(vp->v_interlock); 891 } 892 893 out: 894 if (trans_mp) 895 fstrans_done(trans_mp); 896 KASSERT(!rw_write_held(vp->v_uobj.vmobjlock)); 897 return error; 898 } 899 900
Indexes created Fri Sep 18 00:27:12 UTC 2026