1 /* $NetBSD: vfs_vnode.c,v 1.159 2026/08/18 15:57:33 riastradh Exp $ */ 2 3 /*- 4 * Copyright (c) 1997-2011, 2019, 2020 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 9 * NASA Ames Research Center, by Charles M. Hannum, and by Andrew Doran. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 * POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 /* 34 * Copyright (c) 1989, 1993 35 * The Regents of the University of California. All rights reserved. 36 * (c) UNIX System Laboratories, Inc. 37 * All or some portions of this file are derived from material licensed 38 * to the University of California by American Telephone and Telegraph 39 * Co. or Unix System Laboratories, Inc. and are reproduced herein with 40 * the permission of UNIX System Laboratories, Inc. 41 * 42 * Redistribution and use in source and binary forms, with or without 43 * modification, are permitted provided that the following conditions 44 * are met: 45 * 1. Redistributions of source code must retain the above copyright 46 * notice, this list of conditions and the following disclaimer. 47 * 2. Redistributions in binary form must reproduce the above copyright 48 * notice, this list of conditions and the following disclaimer in the 49 * documentation and/or other materials provided with the distribution. 50 * 3. Neither the name of the University nor the names of its contributors 51 * may be used to endorse or promote products derived from this software 52 * without specific prior written permission. 53 * 54 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 55 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 56 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 57 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 58 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 59 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 60 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 61 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 62 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 63 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 64 * SUCH DAMAGE. 65 * 66 * @(#)vfs_subr.c 8.13 (Berkeley) 4/18/94 67 */ 68 69 /* 70 * The vnode cache subsystem. 71 * 72 * Life-cycle 73 * 74 * Normally, there are two points where new vnodes are created: 75 * VOP_CREATE(9) and VOP_LOOKUP(9). The life-cycle of a vnode 76 * starts in one of the following ways: 77 * 78 * - Allocation, via vcache_get(9) or vcache_new(9). 79 * - Reclamation of inactive vnode, via vcache_vget(9). 80 * 81 * Recycle from a free list, via getnewvnode(9) -> getcleanvnode(9) 82 * was another, traditional way. Currently, only the draining thread 83 * recycles the vnodes. This behaviour might be revisited. 84 * 85 * The life-cycle ends when the last reference is dropped, usually 86 * in VOP_REMOVE(9). In such case, VOP_INACTIVE(9) is called to inform 87 * the file system that vnode is inactive. Via this call, file system 88 * indicates whether vnode can be recycled (usually, it checks its own 89 * references, e.g. count of links, whether the file was removed). 90 * 91 * Depending on indication, vnode can be put into a free list (cache), 92 * or cleaned via vcache_reclaim, which calls VOP_RECLAIM(9) to 93 * disassociate underlying file system from the vnode, and finally 94 * destroyed. 95 * 96 * Vnode state 97 * 98 * Vnode is always in one of six states: 99 * - MARKER This is a marker vnode to help list traversal. It 100 * will never change its state. 101 * - LOADING Vnode is associating underlying file system and not 102 * yet ready to use. 103 * - LOADED Vnode has associated underlying file system and is 104 * ready to use. 105 * - BLOCKED Vnode is active but cannot get new references. 106 * - RECLAIMING Vnode is disassociating from the underlying file 107 * system. 108 * - RECLAIMED Vnode has disassociated from underlying file system 109 * and is dead. 110 * 111 * Valid state changes are: 112 * LOADING -> LOADED 113 * Vnode has been initialised in vcache_get() or 114 * vcache_new() and is ready to use. 115 * BLOCKED -> RECLAIMING 116 * Vnode starts disassociation from underlying file 117 * system in vcache_reclaim(). 118 * RECLAIMING -> RECLAIMED 119 * Vnode finished disassociation from underlying file 120 * system in vcache_reclaim(). 121 * LOADED -> BLOCKED 122 * Either vcache_rekey*() is changing the vnode key or 123 * vrelel() is about to call VOP_INACTIVE(). 124 * BLOCKED -> LOADED 125 * The block condition is over. 126 * LOADING -> RECLAIMED 127 * Either vcache_get() or vcache_new() failed to 128 * associate the underlying file system or vcache_rekey*() 129 * drops a vnode used as placeholder. 130 * 131 * Of these states LOADING, BLOCKED and RECLAIMING are intermediate 132 * and it is possible to wait for state change. 133 * 134 * State is protected with v_interlock with one exception: 135 * to change from LOADING both v_interlock and vcache_lock must be held 136 * so it is possible to check "state == LOADING" without holding 137 * v_interlock. See vcache_get() for details. 138 * 139 * Reference counting 140 * 141 * Vnode is considered active, if reference count (vnode_t::v_usecount) 142 * is non-zero. It is maintained using: vref(9) and vrele(9), as well 143 * as vput(9), routines. Common points holding references are e.g. 144 * file openings, current working directory, mount points, etc. 145 * 146 * v_usecount is adjusted with atomic operations, however to change 147 * from a non-zero value to zero the interlock must also be held. 148 */ 149 150 #include <sys/cdefs.h> 151 __KERNEL_RCSID(0, "$NetBSD: vfs_vnode.c,v 1.159 2026/08/18 15:57:33 riastradh Exp $"); 152 153 #ifdef _KERNEL_OPT 154 #include "opt_pax.h" 155 #endif 156 157 #include <sys/param.h> 158 #include <sys/types.h> 159 160 #include <sys/atomic.h> 161 #include <sys/buf.h> 162 #include <sys/conf.h> 163 #include <sys/device.h> 164 #include <sys/fstrans.h> 165 #include <sys/hash.h> 166 #include <sys/kauth.h> 167 #include <sys/kernel.h> 168 #include <sys/kmem.h> 169 #include <sys/module.h> 170 #include <sys/mount.h> 171 #include <sys/namei.h> 172 #include <sys/pax.h> 173 #include <sys/sdt.h> 174 #include <sys/syscallargs.h> 175 #include <sys/sysctl.h> 176 #include <sys/systm.h> 177 #include <sys/threadpool.h> 178 #include <sys/vnode_impl.h> 179 #include <sys/wapbl.h> 180 181 #include <miscfs/deadfs/deadfs.h> 182 #include <miscfs/specfs/specdev.h> 183 184 #include <uvm/uvm.h> 185 #include <uvm/uvm_readahead.h> 186 #include <uvm/uvm_stat.h> 187 188 /* Flags to vrelel. */ 189 #define VRELEL_ASYNC 0x0001 /* Always defer to vrele thread. */ 190 191 #define LRU_VRELE 0 192 #define LRU_FREE 1 193 #define LRU_HOLD 2 194 #define LRU_COUNT 3 195 196 /* 197 * There are three lru lists: one holds vnodes waiting for async release, 198 * one is for vnodes which have no buffer/page references and one for those 199 * which do (i.e. v_holdcnt is non-zero). We put the lists into a single, 200 * private cache line as vnodes migrate between them while under the same 201 * lock (vdrain_lock). 202 */ 203 204 typedef struct { 205 vnode_impl_t *li_marker; 206 } lru_iter_t; 207 208 u_int numvnodes __cacheline_aligned; 209 static vnodelst_t lru_list[LRU_COUNT] __cacheline_aligned; 210 static struct threadpool *threadpool; 211 static struct threadpool_job vdrain_job; 212 static struct threadpool_job vrele_job; 213 static kmutex_t vdrain_lock __cacheline_aligned; 214 SLIST_HEAD(hashhead, vnode_impl); 215 static kmutex_t vcache_lock __cacheline_aligned; 216 static kcondvar_t vcache_cv; 217 static u_int vcache_hashsize; 218 static u_long vcache_hashmask; 219 static struct hashhead *vcache_hashtab; 220 static pool_cache_t vcache_pool; 221 static void lru_requeue(vnode_t *, vnodelst_t *); 222 static vnodelst_t * lru_which(vnode_t *); 223 static vnode_impl_t * lru_iter_first(int, lru_iter_t *); 224 static vnode_impl_t * lru_iter_next(lru_iter_t *); 225 static void lru_iter_release(lru_iter_t *); 226 static vnode_impl_t * vcache_alloc(void); 227 static void vcache_dealloc(vnode_impl_t *); 228 static void vcache_free(vnode_impl_t *); 229 static void vcache_init(void); 230 static void vcache_reinit(void); 231 static void vcache_reclaim(vnode_t *); 232 static void vrele_deferred(vnode_impl_t *); 233 static void vrelel(vnode_t *, int, int); 234 static void vnpanic(vnode_t *, const char *, ...) 235 __printflike(2, 3); 236 static bool vdrain_one(u_int); 237 static void vdrain_task(struct threadpool_job *); 238 static void vrele_task(struct threadpool_job *); 239 240 /* Routines having to do with the management of the vnode table. */ 241 242 /* 243 * The high bit of v_usecount is a gate for vcache_tryvget(). It's set 244 * only when the vnode state is LOADED. 245 * The next bit of v_usecount is a flag for vrelel(). It's set 246 * from vcache_vget() and vcache_tryvget() whenever the operation succeeds. 247 */ 248 #define VUSECOUNT_MASK 0x3fffffff 249 #define VUSECOUNT_GATE 0x80000000 250 #define VUSECOUNT_VGET 0x40000000 251 252 /* 253 * Return the current usecount of a vnode. 254 */ 255 inline int 256 vrefcnt(struct vnode *vp) 257 { 258 259 return atomic_load_relaxed(&vp->v_usecount) & VUSECOUNT_MASK; 260 } 261 262 /* Vnode state operations and diagnostics. */ 263 264 #if defined(DIAGNOSTIC) 265 266 #define VSTATE_VALID(state) \ 267 ((state) != VS_ACTIVE && (state) != VS_MARKER) 268 #define VSTATE_GET(vp) \ 269 vstate_assert_get((vp), __func__, __LINE__) 270 #define VSTATE_CHANGE(vp, from, to) \ 271 vstate_assert_change((vp), (from), (to), __func__, __LINE__) 272 #define VSTATE_WAIT_STABLE(vp) \ 273 vstate_assert_wait_stable((vp), __func__, __LINE__) 274 275 void 276 _vstate_assert(vnode_t *vp, enum vnode_state state, const char *func, int line, 277 bool has_lock) 278 { 279 vnode_impl_t *vip = VNODE_TO_VIMPL(vp); 280 int refcnt = vrefcnt(vp); 281 282 if (!has_lock) { 283 enum vnode_state vstate = atomic_load_relaxed(&vip->vi_state); 284 285 if (state == VS_ACTIVE && refcnt > 0 && 286 (vstate == VS_LOADED || vstate == VS_BLOCKED)) 287 return; 288 if (vstate == state) 289 return; 290 mutex_enter((vp)->v_interlock); 291 } 292 293 KASSERTMSG(mutex_owned(vp->v_interlock), "at %s:%d", func, line); 294 295 if ((state == VS_ACTIVE && refcnt > 0 && 296 (vip->vi_state == VS_LOADED || vip->vi_state == VS_BLOCKED)) || 297 vip->vi_state == state) { 298 if (!has_lock) 299 mutex_exit((vp)->v_interlock); 300 return; 301 } 302 vnpanic(vp, "state is %s, usecount %d, expected %s at %s:%d", 303 vstate_name(vip->vi_state), refcnt, 304 vstate_name(state), func, line); 305 } 306 307 static enum vnode_state 308 vstate_assert_get(vnode_t *vp, const char *func, int line) 309 { 310 vnode_impl_t *vip = VNODE_TO_VIMPL(vp); 311 312 KASSERTMSG(mutex_owned(vp->v_interlock), "at %s:%d", func, line); 313 if (! VSTATE_VALID(vip->vi_state)) 314 vnpanic(vp, "state is %s at %s:%d", 315 vstate_name(vip->vi_state), func, line); 316 317 return vip->vi_state; 318 } 319 320 static void 321 vstate_assert_wait_stable(vnode_t *vp, const char *func, int line) 322 { 323 vnode_impl_t *vip = VNODE_TO_VIMPL(vp); 324 325 KASSERTMSG(mutex_owned(vp->v_interlock), "at %s:%d", func, line); 326 if (! VSTATE_VALID(vip->vi_state)) 327 vnpanic(vp, "state is %s at %s:%d", 328 vstate_name(vip->vi_state), func, line); 329 330 while (vip->vi_state != VS_LOADED && vip->vi_state != VS_RECLAIMED) 331 cv_wait(&vp->v_cv, vp->v_interlock); 332 333 if (! VSTATE_VALID(vip->vi_state)) 334 vnpanic(vp, "state is %s at %s:%d", 335 vstate_name(vip->vi_state), func, line); 336 } 337 338 static void 339 vstate_assert_change(vnode_t *vp, enum vnode_state from, enum vnode_state to, 340 const char *func, int line) 341 { 342 bool gated = (atomic_load_relaxed(&vp->v_usecount) & VUSECOUNT_GATE); 343 vnode_impl_t *vip = VNODE_TO_VIMPL(vp); 344 345 KASSERTMSG(mutex_owned(vp->v_interlock), "at %s:%d", func, line); 346 if (from == VS_LOADING) 347 KASSERTMSG(mutex_owned(&vcache_lock), "at %s:%d", func, line); 348 349 if (! VSTATE_VALID(from)) 350 vnpanic(vp, "from is %s at %s:%d", 351 vstate_name(from), func, line); 352 if (! VSTATE_VALID(to)) 353 vnpanic(vp, "to is %s at %s:%d", 354 vstate_name(to), func, line); 355 if (vip->vi_state != from) 356 vnpanic(vp, "from is %s, expected %s at %s:%d\n", 357 vstate_name(vip->vi_state), vstate_name(from), func, line); 358 if ((from == VS_LOADED) != gated) 359 vnpanic(vp, "state is %s, gate %d does not match at %s:%d\n", 360 vstate_name(vip->vi_state), gated, func, line); 361 362 /* Open/close the gate for vcache_tryvget(). */ 363 if (to == VS_LOADED) { 364 membar_release(); 365 atomic_or_uint(&vp->v_usecount, VUSECOUNT_GATE); 366 } else { 367 atomic_and_uint(&vp->v_usecount, ~VUSECOUNT_GATE); 368 } 369 370 atomic_store_relaxed(&vip->vi_state, to); 371 if (from == VS_LOADING) 372 cv_broadcast(&vcache_cv); 373 if (to == VS_LOADED || to == VS_RECLAIMED) 374 cv_broadcast(&vp->v_cv); 375 } 376 377 #else /* defined(DIAGNOSTIC) */ 378 379 #define VSTATE_GET(vp) \ 380 (VNODE_TO_VIMPL((vp))->vi_state) 381 #define VSTATE_CHANGE(vp, from, to) \ 382 vstate_change((vp), (from), (to)) 383 #define VSTATE_WAIT_STABLE(vp) \ 384 vstate_wait_stable((vp)) 385 void 386 _vstate_assert(vnode_t *vp, enum vnode_state state, const char *func, int line, 387 bool has_lock) 388 { 389 390 } 391 392 static void 393 vstate_wait_stable(vnode_t *vp) 394 { 395 vnode_impl_t *vip = VNODE_TO_VIMPL(vp); 396 397 while (vip->vi_state != VS_LOADED && vip->vi_state != VS_RECLAIMED) 398 cv_wait(&vp->v_cv, vp->v_interlock); 399 } 400 401 static void 402 vstate_change(vnode_t *vp, enum vnode_state from, enum vnode_state to) 403 { 404 vnode_impl_t *vip = VNODE_TO_VIMPL(vp); 405 406 /* Open/close the gate for vcache_tryvget(). */ 407 if (to == VS_LOADED) { 408 membar_release(); 409 atomic_or_uint(&vp->v_usecount, VUSECOUNT_GATE); 410 } else { 411 atomic_and_uint(&vp->v_usecount, ~VUSECOUNT_GATE); 412 } 413 414 atomic_store_relaxed(&vip->vi_state, to); 415 if (from == VS_LOADING) 416 cv_broadcast(&vcache_cv); 417 if (to == VS_LOADED || to == VS_RECLAIMED) 418 cv_broadcast(&vp->v_cv); 419 } 420 421 #endif /* defined(DIAGNOSTIC) */ 422 423 void 424 vfs_vnode_sysinit(void) 425 { 426 int error __diagused, i; 427 428 dead_rootmount = vfs_mountalloc(&dead_vfsops, NULL); 429 KASSERT(dead_rootmount != NULL); 430 dead_rootmount->mnt_iflag |= IMNT_MPSAFE; 431 432 mutex_init(&vdrain_lock, MUTEX_DEFAULT, IPL_NONE); 433 for (i = 0; i < LRU_COUNT; i++) { 434 TAILQ_INIT(&lru_list[i]); 435 } 436 vcache_init(); 437 438 error = threadpool_get(&threadpool, PRI_NONE); 439 KASSERTMSG((error == 0), "threadpool_get failed: %d", error); 440 threadpool_job_init(&vdrain_job, vdrain_task, &vdrain_lock, "vdrain"); 441 threadpool_job_init(&vrele_job, vrele_task, &vdrain_lock, "vrele"); 442 } 443 444 /* 445 * Allocate a new marker vnode. 446 */ 447 vnode_t * 448 vnalloc_marker(struct mount *mp) 449 { 450 vnode_impl_t *vip; 451 vnode_t *vp; 452 453 vip = pool_cache_get(vcache_pool, PR_WAITOK); 454 memset(vip, 0, sizeof(*vip)); 455 vp = VIMPL_TO_VNODE(vip); 456 uvm_obj_init(&vp->v_uobj, &uvm_vnodeops, true, 1); 457 vp->v_mount = mp; 458 vp->v_type = VBAD; 459 vp->v_interlock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE); 460 klist_init(&vip->vi_klist.vk_klist); 461 vp->v_klist = &vip->vi_klist; 462 vip->vi_state = VS_MARKER; 463 464 return vp; 465 } 466 467 /* 468 * Free a marker vnode. 469 */ 470 void 471 vnfree_marker(vnode_t *vp) 472 { 473 vnode_impl_t *vip; 474 475 vip = VNODE_TO_VIMPL(vp); 476 KASSERT(vip->vi_state == VS_MARKER); 477 mutex_obj_free(vp->v_interlock); 478 uvm_obj_destroy(&vp->v_uobj, true); 479 klist_fini(&vip->vi_klist.vk_klist); 480 pool_cache_put(vcache_pool, vip); 481 } 482 483 /* 484 * Test a vnode for being a marker vnode. 485 */ 486 bool 487 vnis_marker(vnode_t *vp) 488 { 489 490 return (VNODE_TO_VIMPL(vp)->vi_state == VS_MARKER); 491 } 492 493 /* 494 * Return the lru list this node should be on. 495 */ 496 static vnodelst_t * 497 lru_which(vnode_t *vp) 498 { 499 500 KASSERT(mutex_owned(vp->v_interlock)); 501 502 if (vp->v_holdcnt > 0) 503 return &lru_list[LRU_HOLD]; 504 else 505 return &lru_list[LRU_FREE]; 506 } 507 508 /* 509 * Put vnode to end of given list. 510 * Both the current and the new list may be NULL, used on vnode alloc/free. 511 * Adjust numvnodes and signal vdrain thread if there is work. 512 */ 513 static void 514 lru_requeue(vnode_t *vp, vnodelst_t *listhd) 515 { 516 vnode_impl_t *vip; 517 int d; 518 519 /* 520 * If the vnode is on the correct list, and was put there recently, 521 * then leave it be, thus avoiding huge cache and lock contention. 522 */ 523 vip = VNODE_TO_VIMPL(vp); 524 if (listhd == vip->vi_lrulisthd && 525 (getticks() - vip->vi_lrulisttm) < hz) { 526 return; 527 } 528 529 mutex_enter(&vdrain_lock); 530 d = 0; 531 if (vip->vi_lrulisthd != NULL) 532 TAILQ_REMOVE(vip->vi_lrulisthd, vip, vi_lrulist); 533 else 534 d++; 535 vip->vi_lrulisthd = listhd; 536 vip->vi_lrulisttm = getticks(); 537 if (vip->vi_lrulisthd != NULL) 538 TAILQ_INSERT_TAIL(vip->vi_lrulisthd, vip, vi_lrulist); 539 else 540 d--; 541 if (d != 0) { 542 /* 543 * Looks strange? This is not a bug. Don't store 544 * numvnodes unless there is a change - avoid false 545 * sharing on MP. 546 */ 547 numvnodes += d; 548 } 549 if (listhd == &lru_list[LRU_VRELE]) 550 threadpool_schedule_job(threadpool, &vrele_job); 551 if (d > 0 && numvnodes > desiredvnodes) 552 threadpool_schedule_job(threadpool, &vdrain_job); 553 if (d > 0 && numvnodes > desiredvnodes + desiredvnodes / 16) 554 kpause("vnfull", false, MAX(1, mstohz(10)), &vdrain_lock); 555 mutex_exit(&vdrain_lock); 556 } 557 558 /* 559 * LRU list iterator. 560 * Caller holds vdrain_lock. 561 */ 562 static vnode_impl_t * 563 lru_iter_first(int idx, lru_iter_t *iterp) 564 { 565 vnode_impl_t *marker; 566 567 KASSERT(mutex_owned(&vdrain_lock)); 568 569 mutex_exit(&vdrain_lock); 570 marker = VNODE_TO_VIMPL(vnalloc_marker(NULL)); 571 mutex_enter(&vdrain_lock); 572 marker->vi_lrulisthd = &lru_list[idx]; 573 iterp->li_marker = marker; 574 575 TAILQ_INSERT_HEAD(marker->vi_lrulisthd, marker, vi_lrulist); 576 577 return lru_iter_next(iterp); 578 } 579 580 static vnode_impl_t * 581 lru_iter_next(lru_iter_t *iter) 582 { 583 vnode_impl_t *vip, *marker; 584 vnodelst_t *listhd; 585 586 KASSERT(mutex_owned(&vdrain_lock)); 587 588 marker = iter->li_marker; 589 listhd = marker->vi_lrulisthd; 590 591 while ((vip = TAILQ_NEXT(marker, vi_lrulist))) { 592 TAILQ_REMOVE(listhd, marker, vi_lrulist); 593 TAILQ_INSERT_AFTER(listhd, vip, marker, vi_lrulist); 594 if (!vnis_marker(VIMPL_TO_VNODE(vip))) 595 break; 596 } 597 598 return vip; 599 } 600 601 static void 602 lru_iter_release(lru_iter_t *iter) 603 { 604 vnode_impl_t *marker; 605 606 KASSERT(mutex_owned(&vdrain_lock)); 607 608 marker = iter->li_marker; 609 TAILQ_REMOVE(marker->vi_lrulisthd, marker, vi_lrulist); 610 611 mutex_exit(&vdrain_lock); 612 vnfree_marker(VIMPL_TO_VNODE(marker)); 613 mutex_enter(&vdrain_lock); 614 } 615 616 /* 617 * Release deferred vrele vnodes for this mount. 618 * Called with file system suspended. 619 */ 620 void 621 vrele_flush(struct mount *mp) 622 { 623 lru_iter_t iter; 624 vnode_impl_t *vip; 625 626 KASSERT(fstrans_is_owner(mp)); 627 628 mutex_enter(&vdrain_lock); 629 for (vip = lru_iter_first(LRU_VRELE, &iter); vip != NULL; 630 vip = lru_iter_next(&iter)) { 631 if (VIMPL_TO_VNODE(vip)->v_mount != mp) 632 continue; 633 vrele_deferred(vip); 634 } 635 lru_iter_release(&iter); 636 mutex_exit(&vdrain_lock); 637 } 638 639 /* 640 * One pass through the LRU lists to keep the number of allocated 641 * vnodes below target. Returns true if target met. 642 */ 643 static bool 644 vdrain_one(u_int target) 645 { 646 int ix, lists[] = { LRU_FREE, LRU_HOLD }; 647 lru_iter_t iter; 648 vnode_impl_t *vip; 649 vnode_t *vp; 650 struct mount *mp; 651 652 KASSERT(mutex_owned(&vdrain_lock)); 653 654 for (ix = 0; ix < __arraycount(lists); ix++) { 655 for (vip = lru_iter_first(lists[ix], &iter); vip != NULL; 656 vip = lru_iter_next(&iter)) { 657 if (numvnodes < target) { 658 lru_iter_release(&iter); 659 return true; 660 } 661 662 vp = VIMPL_TO_VNODE(vip); 663 664 /* Probe usecount (unlocked). */ 665 if (vrefcnt(vp) > 0) 666 continue; 667 /* Try v_interlock -- we lock the wrong direction! */ 668 if (!mutex_tryenter(vp->v_interlock)) 669 continue; 670 /* Probe usecount and state. */ 671 if (vrefcnt(vp) > 0 || VSTATE_GET(vp) != VS_LOADED) { 672 mutex_exit(vp->v_interlock); 673 continue; 674 } 675 mutex_exit(&vdrain_lock); 676 677 mp = vp->v_mount; 678 if (fstrans_start_nowait(mp) != 0) { 679 mutex_exit(vp->v_interlock); 680 mutex_enter(&vdrain_lock); 681 continue; 682 } 683 684 if (vcache_vget(vp) == 0) { 685 if (!vrecycle(vp)) { 686 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 687 mutex_enter(vp->v_interlock); 688 vrelel(vp, 0, LK_EXCLUSIVE); 689 } 690 } 691 fstrans_done(mp); 692 693 mutex_enter(&vdrain_lock); 694 } 695 lru_iter_release(&iter); 696 } 697 698 return false; 699 } 700 701 /* 702 * threadpool task to keep the number of vnodes below desiredvnodes. 703 */ 704 static void 705 vdrain_task(struct threadpool_job *job) 706 { 707 708 mutex_enter(&vdrain_lock); 709 710 while (!vdrain_one(desiredvnodes - desiredvnodes / 16)) 711 kpause("vdrain", false, 1, &vdrain_lock); 712 713 threadpool_job_done(job); 714 mutex_exit(&vdrain_lock); 715 } 716 717 /* 718 * threadpool task to process asynchronous vrele. 719 */ 720 static void 721 vrele_task(struct threadpool_job *job) 722 { 723 int skipped; 724 lru_iter_t iter; 725 vnode_impl_t *vip; 726 struct mount *mp; 727 728 mutex_enter(&vdrain_lock); 729 while ((vip = lru_iter_first(LRU_VRELE, &iter)) != NULL) { 730 for (skipped = 0; vip != NULL; vip = lru_iter_next(&iter)) { 731 mp = VIMPL_TO_VNODE(vip)->v_mount; 732 if (fstrans_start_nowait(mp) == 0) { 733 vrele_deferred(vip); 734 fstrans_done(mp); 735 } else { 736 skipped++; 737 } 738 } 739 740 lru_iter_release(&iter); 741 if (skipped) { 742 kpause("vrele", false, MAX(1, mstohz(10)), 743 &vdrain_lock); 744 } 745 } 746 747 threadpool_job_done(job); 748 lru_iter_release(&iter); 749 mutex_exit(&vdrain_lock); 750 } 751 752 /* 753 * Try to drop reference on a vnode. Abort if we are releasing the 754 * last reference. Note: this _must_ succeed if not the last reference. 755 */ 756 static bool 757 vtryrele(vnode_t *vp) 758 { 759 u_int use, next; 760 761 membar_release(); 762 for (use = atomic_load_relaxed(&vp->v_usecount);; use = next) { 763 if (__predict_false((use & VUSECOUNT_MASK) == 1)) { 764 return false; 765 } 766 KASSERT((use & VUSECOUNT_MASK) > 1); 767 next = atomic_cas_uint(&vp->v_usecount, use, use - 1); 768 if (__predict_true(next == use)) { 769 return true; 770 } 771 } 772 } 773 774 /* 775 * vput: unlock and release the reference. 776 */ 777 void 778 vput(vnode_t *vp) 779 { 780 int lktype; 781 782 /* 783 * Do an unlocked check of the usecount. If it looks like we're not 784 * about to drop the last reference, then unlock the vnode and try 785 * to drop the reference. If it ends up being the last reference 786 * after all, vrelel() can fix it all up. Most of the time this 787 * will all go to plan. 788 */ 789 if (vrefcnt(vp) > 1) { 790 VOP_UNLOCK(vp); 791 if (vtryrele(vp)) { 792 return; 793 } 794 lktype = LK_NONE; 795 } else { 796 lktype = VOP_ISLOCKED(vp); 797 KASSERT(lktype != LK_NONE); 798 } 799 mutex_enter(vp->v_interlock); 800 vrelel(vp, 0, lktype); 801 } 802 803 /* 804 * Release a vnode from the deferred list. 805 */ 806 static void 807 vrele_deferred(vnode_impl_t *vip) 808 { 809 vnode_t *vp; 810 811 KASSERT(mutex_owned(&vdrain_lock)); 812 KASSERT(vip->vi_lrulisthd == &lru_list[LRU_VRELE]); 813 814 vp = VIMPL_TO_VNODE(vip); 815 816 /* 817 * First remove the vnode from the vrele list. 818 * Put it on the last lru list, the last vrele() 819 * will put it back onto the right list before 820 * its usecount reaches zero. 821 */ 822 TAILQ_REMOVE(vip->vi_lrulisthd, vip, vi_lrulist); 823 vip->vi_lrulisthd = &lru_list[LRU_HOLD]; 824 vip->vi_lrulisttm = getticks(); 825 TAILQ_INSERT_TAIL(vip->vi_lrulisthd, vip, vi_lrulist); 826 827 mutex_exit(&vdrain_lock); 828 829 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 830 mutex_enter(vp->v_interlock); 831 vrelel(vp, 0, LK_EXCLUSIVE); 832 833 mutex_enter(&vdrain_lock); 834 } 835 836 /* 837 * Vnode release. If reference count drops to zero, call inactive 838 * routine and either return to freelist or free to the pool. 839 */ 840 static void 841 vrelel(vnode_t *vp, int flags, int lktype) 842 { 843 const bool async = ((flags & VRELEL_ASYNC) != 0); 844 bool recycle, defer, objlock_held; 845 u_int use, next; 846 int error; 847 848 objlock_held = false; 849 850 retry: 851 KASSERT(mutex_owned(vp->v_interlock)); 852 853 if (__predict_false(vp->v_op == dead_vnodeop_p && 854 VSTATE_GET(vp) != VS_RECLAIMED)) { 855 vnpanic(vp, "dead but not clean"); 856 } 857 858 /* 859 * If not the last reference, just unlock and drop the reference count. 860 * 861 * Otherwise make sure we pass a point in time where we hold the 862 * last reference with VGET flag unset. 863 */ 864 for (use = atomic_load_relaxed(&vp->v_usecount);; use = next) { 865 if (__predict_false((use & VUSECOUNT_MASK) > 1)) { 866 if (objlock_held) { 867 objlock_held = false; 868 rw_exit(vp->v_uobj.vmobjlock); 869 } 870 if (lktype != LK_NONE) { 871 mutex_exit(vp->v_interlock); 872 lktype = LK_NONE; 873 VOP_UNLOCK(vp); 874 mutex_enter(vp->v_interlock); 875 } 876 if (vtryrele(vp)) { 877 mutex_exit(vp->v_interlock); 878 return; 879 } 880 next = atomic_load_relaxed(&vp->v_usecount); 881 continue; 882 } 883 KASSERT((use & VUSECOUNT_MASK) == 1); 884 next = use & ~VUSECOUNT_VGET; 885 if (next != use) { 886 next = atomic_cas_uint(&vp->v_usecount, use, next); 887 } 888 if (__predict_true(next == use)) { 889 break; 890 } 891 } 892 membar_acquire(); 893 if (vrefcnt(vp) <= 0 || vp->v_writecount != 0) { 894 vnpanic(vp, "%s: bad ref count", __func__); 895 } 896 897 #ifdef DIAGNOSTIC 898 if ((vp->v_type == VBLK || vp->v_type == VCHR) && 899 vp->v_specnode != NULL && vp->v_specnode->sn_opencnt != 0) { 900 vprint("vrelel: missing VOP_CLOSE()", vp); 901 } 902 #endif 903 904 /* 905 * If already clean there is no need to lock, defer or 906 * deactivate this node. 907 */ 908 if (VSTATE_GET(vp) == VS_RECLAIMED) { 909 if (objlock_held) { 910 objlock_held = false; 911 rw_exit(vp->v_uobj.vmobjlock); 912 } 913 if (lktype != LK_NONE) { 914 mutex_exit(vp->v_interlock); 915 lktype = LK_NONE; 916 VOP_UNLOCK(vp); 917 mutex_enter(vp->v_interlock); 918 } 919 goto out; 920 } 921 922 /* 923 * First try to get the vnode locked for VOP_INACTIVE(). 924 * Defer vnode release to vrele task if caller requests 925 * it explicitly, is the pagedaemon or the lock failed. 926 */ 927 defer = false; 928 if (uvm_lwp_is_pagedaemon(curlwp) || async) { 929 defer = true; 930 } else if (lktype == LK_SHARED) { 931 /* Excellent chance of getting, if the last ref. */ 932 error = vn_lock(vp, LK_UPGRADE | LK_RETRY | LK_NOWAIT); 933 if (error != 0) { 934 defer = true; 935 } else { 936 lktype = LK_EXCLUSIVE; 937 } 938 } else if (lktype == LK_NONE) { 939 /* Excellent chance of getting, if the last ref. */ 940 error = vn_lock(vp, LK_EXCLUSIVE | LK_RETRY | LK_NOWAIT); 941 if (error != 0) { 942 defer = true; 943 } else { 944 lktype = LK_EXCLUSIVE; 945 } 946 } 947 KASSERT(mutex_owned(vp->v_interlock)); 948 if (defer) { 949 /* 950 * Defer reclaim to the vrele task; it's not safe to 951 * clean it here. We donate it our last reference. 952 */ 953 if (lktype != LK_NONE) { 954 mutex_exit(vp->v_interlock); 955 VOP_UNLOCK(vp); 956 mutex_enter(vp->v_interlock); 957 } 958 lru_requeue(vp, &lru_list[LRU_VRELE]); 959 mutex_exit(vp->v_interlock); 960 return; 961 } 962 KASSERT(lktype == LK_EXCLUSIVE); 963 964 /* If the node gained another reference, retry. */ 965 use = atomic_load_relaxed(&vp->v_usecount); 966 if ((use & VUSECOUNT_VGET) != 0) { 967 goto retry; 968 } 969 KASSERT((use & VUSECOUNT_MASK) == 1); 970 971 if ((vp->v_iflag & (VI_TEXT|VI_EXECMAP|VI_WRMAP)) != 0 || 972 (vp->v_vflag & VV_MAPPED) != 0) { 973 /* Take care of space accounting. */ 974 if (!objlock_held) { 975 objlock_held = true; 976 if (!rw_tryenter(vp->v_uobj.vmobjlock, RW_WRITER)) { 977 mutex_exit(vp->v_interlock); 978 rw_enter(vp->v_uobj.vmobjlock, RW_WRITER); 979 mutex_enter(vp->v_interlock); 980 goto retry; 981 } 982 } 983 if ((vp->v_iflag & VI_EXECMAP) != 0) { 984 cpu_count(CPU_COUNT_EXECPAGES, -vp->v_uobj.uo_npages); 985 } 986 vp->v_iflag &= ~(VI_TEXT|VI_EXECMAP|VI_WRMAP); 987 vp->v_vflag &= ~VV_MAPPED; 988 } 989 if (objlock_held) { 990 objlock_held = false; 991 rw_exit(vp->v_uobj.vmobjlock); 992 } 993 994 /* 995 * Deactivate the vnode, but preserve our reference across 996 * the call to VOP_INACTIVE(). 997 * 998 * If VOP_INACTIVE() indicates that the file has been 999 * deleted, then recycle the vnode. 1000 * 1001 * Note that VOP_INACTIVE() will not drop the vnode lock. 1002 */ 1003 mutex_exit(vp->v_interlock); 1004 recycle = false; 1005 VOP_INACTIVE(vp, &recycle); 1006 if (!recycle) { 1007 lktype = LK_NONE; 1008 VOP_UNLOCK(vp); 1009 } 1010 mutex_enter(vp->v_interlock); 1011 1012 /* 1013 * Block new references then check again to see if a 1014 * new reference was acquired in the meantime. If 1015 * it was, restore the vnode state and try again. 1016 */ 1017 if (recycle) { 1018 VSTATE_CHANGE(vp, VS_LOADED, VS_BLOCKED); 1019 use = atomic_load_relaxed(&vp->v_usecount); 1020 if ((use & VUSECOUNT_VGET) != 0) { 1021 VSTATE_CHANGE(vp, VS_BLOCKED, VS_LOADED); 1022 goto retry; 1023 } 1024 KASSERT((use & VUSECOUNT_MASK) == 1); 1025 } 1026 1027 /* 1028 * Recycle the vnode if the file is now unused (unlinked). 1029 */ 1030 if (recycle) { 1031 VSTATE_ASSERT(vp, VS_BLOCKED); 1032 KASSERT(lktype == LK_EXCLUSIVE); 1033 /* vcache_reclaim drops the lock. */ 1034 lktype = LK_NONE; 1035 vcache_reclaim(vp); 1036 } 1037 KASSERT(vrefcnt(vp) > 0); 1038 KASSERT(lktype == LK_NONE); 1039 1040 out: 1041 for (use = atomic_load_relaxed(&vp->v_usecount);; use = next) { 1042 if (__predict_false((use & VUSECOUNT_VGET) != 0 && 1043 (use & VUSECOUNT_MASK) == 1)) { 1044 /* Gained and released another reference, retry. */ 1045 goto retry; 1046 } 1047 next = atomic_cas_uint(&vp->v_usecount, use, use - 1); 1048 if (__predict_true(next == use)) { 1049 if (__predict_false((use & VUSECOUNT_MASK) != 1)) { 1050 /* Gained another reference. */ 1051 mutex_exit(vp->v_interlock); 1052 return; 1053 } 1054 break; 1055 } 1056 } 1057 membar_acquire(); 1058 1059 if (VSTATE_GET(vp) == VS_RECLAIMED && vp->v_holdcnt == 0) { 1060 /* 1061 * It's clean so destroy it. It isn't referenced 1062 * anywhere since it has been reclaimed. 1063 */ 1064 vcache_free(VNODE_TO_VIMPL(vp)); 1065 } else { 1066 /* 1067 * Otherwise, put it back onto the freelist. It 1068 * can't be destroyed while still associated with 1069 * a file system. 1070 */ 1071 lru_requeue(vp, lru_which(vp)); 1072 mutex_exit(vp->v_interlock); 1073 } 1074 } 1075 1076 void 1077 vrele(vnode_t *vp) 1078 { 1079 1080 if (vtryrele(vp)) { 1081 return; 1082 } 1083 mutex_enter(vp->v_interlock); 1084 vrelel(vp, 0, LK_NONE); 1085 } 1086 1087 /* 1088 * Asynchronous vnode release, vnode is released in different context. 1089 */ 1090 void 1091 vrele_async(vnode_t *vp) 1092 { 1093 1094 if (vtryrele(vp)) { 1095 return; 1096 } 1097 mutex_enter(vp->v_interlock); 1098 vrelel(vp, VRELEL_ASYNC, LK_NONE); 1099 } 1100 1101 /* 1102 * Vnode reference, where a reference is already held by some other 1103 * object (for example, a file structure). 1104 * 1105 * NB: lockless code sequences may rely on this not blocking. 1106 */ 1107 void 1108 vref(vnode_t *vp) 1109 { 1110 1111 KASSERT(vrefcnt(vp) > 0); 1112 1113 atomic_inc_uint(&vp->v_usecount); 1114 } 1115 1116 /* 1117 * Page or buffer structure gets a reference. 1118 * Called with v_interlock held. 1119 */ 1120 void 1121 vholdl(vnode_t *vp) 1122 { 1123 1124 KASSERT(mutex_owned(vp->v_interlock)); 1125 1126 if (vp->v_holdcnt++ == 0 && vrefcnt(vp) == 0) 1127 lru_requeue(vp, lru_which(vp)); 1128 } 1129 1130 /* 1131 * Page or buffer structure gets a reference. 1132 */ 1133 void 1134 vhold(vnode_t *vp) 1135 { 1136 1137 mutex_enter(vp->v_interlock); 1138 vholdl(vp); 1139 mutex_exit(vp->v_interlock); 1140 } 1141 1142 /* 1143 * Page or buffer structure frees a reference. 1144 * Called with v_interlock held. 1145 */ 1146 void 1147 holdrelel(vnode_t *vp) 1148 { 1149 1150 KASSERT(mutex_owned(vp->v_interlock)); 1151 1152 if (vp->v_holdcnt <= 0) { 1153 vnpanic(vp, "%s: holdcnt vp %p", __func__, vp); 1154 } 1155 1156 vp->v_holdcnt--; 1157 if (vp->v_holdcnt == 0 && vrefcnt(vp) == 0) 1158 lru_requeue(vp, lru_which(vp)); 1159 } 1160 1161 /* 1162 * Page or buffer structure frees a reference. 1163 */ 1164 void 1165 holdrele(vnode_t *vp) 1166 { 1167 1168 mutex_enter(vp->v_interlock); 1169 holdrelel(vp); 1170 mutex_exit(vp->v_interlock); 1171 } 1172 1173 /* 1174 * Recycle an unused vnode if caller holds the last reference. 1175 */ 1176 bool 1177 vrecycle(vnode_t *vp) 1178 { 1179 int error __diagused; 1180 1181 mutex_enter(vp->v_interlock); 1182 1183 /* If the vnode is already clean we're done. */ 1184 VSTATE_WAIT_STABLE(vp); 1185 if (VSTATE_GET(vp) != VS_LOADED) { 1186 VSTATE_ASSERT(vp, VS_RECLAIMED); 1187 vrelel(vp, 0, LK_NONE); 1188 return true; 1189 } 1190 1191 /* Prevent further references until the vnode is locked. */ 1192 VSTATE_CHANGE(vp, VS_LOADED, VS_BLOCKED); 1193 1194 /* Make sure we hold the last reference. */ 1195 if (vrefcnt(vp) != 1) { 1196 VSTATE_CHANGE(vp, VS_BLOCKED, VS_LOADED); 1197 mutex_exit(vp->v_interlock); 1198 return false; 1199 } 1200 1201 mutex_exit(vp->v_interlock); 1202 1203 /* 1204 * On a leaf file system this lock will always succeed as we hold 1205 * the last reference and prevent further references. 1206 * On layered file systems waiting for the lock would open a can of 1207 * deadlocks as the lower vnodes may have other active references. 1208 */ 1209 error = vn_lock(vp, LK_EXCLUSIVE | LK_RETRY | LK_NOWAIT); 1210 1211 mutex_enter(vp->v_interlock); 1212 if (error) { 1213 VSTATE_CHANGE(vp, VS_BLOCKED, VS_LOADED); 1214 mutex_exit(vp->v_interlock); 1215 return false; 1216 } 1217 1218 KASSERT(vrefcnt(vp) == 1); 1219 vcache_reclaim(vp); 1220 vrelel(vp, 0, LK_NONE); 1221 1222 return true; 1223 } 1224 1225 /* 1226 * Helper for vrevoke() to propagate suspension from lastmp 1227 * to thismp. Both args may be NULL. 1228 * Returns the currently suspended file system or NULL. 1229 */ 1230 static struct mount * 1231 vrevoke_suspend_next(struct mount *lastmp, struct mount *thismp) 1232 { 1233 int error; 1234 1235 if (lastmp == thismp) 1236 return thismp; 1237 1238 if (lastmp != NULL) 1239 vfs_resume(lastmp); 1240 1241 if (thismp == NULL) 1242 return NULL; 1243 1244 do { 1245 error = vfs_suspend(thismp, 0); 1246 } while (error == EINTR || error == ERESTART); 1247 1248 if (error == 0) 1249 return thismp; 1250 1251 KASSERT(error == EOPNOTSUPP || error == ENOENT); 1252 return NULL; 1253 } 1254 1255 /* 1256 * Eliminate all activity associated with the requested vnode 1257 * and with all vnodes aliased to the requested vnode. 1258 */ 1259 void 1260 vrevoke(vnode_t *vp) 1261 { 1262 struct mount *mp; 1263 vnode_t *vq; 1264 enum vtype type; 1265 dev_t dev; 1266 1267 KASSERT(vrefcnt(vp) > 0); 1268 1269 mp = vrevoke_suspend_next(NULL, vp->v_mount); 1270 1271 mutex_enter(vp->v_interlock); 1272 VSTATE_WAIT_STABLE(vp); 1273 if (VSTATE_GET(vp) == VS_RECLAIMED) { 1274 mutex_exit(vp->v_interlock); 1275 } else if (vp->v_type != VBLK && vp->v_type != VCHR) { 1276 atomic_inc_uint(&vp->v_usecount); 1277 mutex_exit(vp->v_interlock); 1278 vgone(vp); 1279 } else { 1280 dev = vp->v_rdev; 1281 type = vp->v_type; 1282 mutex_exit(vp->v_interlock); 1283 1284 while (spec_node_lookup_by_dev(type, dev, VDEAD_NOWAIT, &vq) 1285 == 0) { 1286 mp = vrevoke_suspend_next(mp, vq->v_mount); 1287 vgone(vq); 1288 } 1289 } 1290 vrevoke_suspend_next(mp, NULL); 1291 } 1292 1293 /* 1294 * Eliminate all activity associated with a vnode in preparation for 1295 * reuse. Drops a reference from the vnode. 1296 */ 1297 void 1298 vgone(vnode_t *vp) 1299 { 1300 int lktype; 1301 1302 KASSERT(vp->v_mount == dead_rootmount || 1303 fstrans_is_owner(vp->v_mount)); 1304 1305 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 1306 lktype = LK_EXCLUSIVE; 1307 mutex_enter(vp->v_interlock); 1308 VSTATE_WAIT_STABLE(vp); 1309 if (VSTATE_GET(vp) == VS_LOADED) { 1310 VSTATE_CHANGE(vp, VS_LOADED, VS_BLOCKED); 1311 vcache_reclaim(vp); 1312 lktype = LK_NONE; 1313 } 1314 VSTATE_ASSERT(vp, VS_RECLAIMED); 1315 vrelel(vp, 0, lktype); 1316 } 1317 1318 static inline uint32_t 1319 vcache_hash(const struct vcache_key *key) 1320 { 1321 uint32_t hash = HASH32_BUF_INIT; 1322 1323 KASSERT(key->vk_key_len > 0); 1324 1325 hash = hash32_buf(&key->vk_mount, sizeof(struct mount *), hash); 1326 hash = hash32_buf(key->vk_key, key->vk_key_len, hash); 1327 return hash; 1328 } 1329 1330 static int 1331 vcache_stats(struct hashstat_sysctl *hs, bool fill) 1332 { 1333 vnode_impl_t *vip; 1334 uint64_t chain; 1335 1336 strlcpy(hs->hash_name, "vcache", sizeof(hs->hash_name)); 1337 strlcpy(hs->hash_desc, "vnode cache hash", sizeof(hs->hash_desc)); 1338 if (!fill) 1339 return 0; 1340 1341 hs->hash_size = vcache_hashmask + 1; 1342 1343 for (size_t i = 0; i < hs->hash_size; i++) { 1344 chain = 0; 1345 mutex_enter(&vcache_lock); 1346 SLIST_FOREACH(vip, &vcache_hashtab[i], vi_hash) { 1347 chain++; 1348 } 1349 mutex_exit(&vcache_lock); 1350 if (chain > 0) { 1351 hs->hash_used++; 1352 hs->hash_items += chain; 1353 if (chain > hs->hash_maxchain) 1354 hs->hash_maxchain = chain; 1355 } 1356 preempt_point(); 1357 } 1358 1359 return 0; 1360 } 1361 1362 static void 1363 vcache_init(void) 1364 { 1365 1366 vcache_pool = pool_cache_init(sizeof(vnode_impl_t), coherency_unit, 1367 0, 0, "vcachepl", NULL, IPL_NONE, NULL, NULL, NULL); 1368 KASSERT(vcache_pool != NULL); 1369 mutex_init(&vcache_lock, MUTEX_DEFAULT, IPL_NONE); 1370 cv_init(&vcache_cv, "vcache"); 1371 vcache_hashsize = desiredvnodes; 1372 vcache_hashtab = hashinit(desiredvnodes, HASH_SLIST, true, 1373 &vcache_hashmask); 1374 hashstat_register("vcache", vcache_stats); 1375 } 1376 1377 static void 1378 vcache_reinit(void) 1379 { 1380 int i; 1381 uint32_t hash; 1382 u_long oldmask, newmask; 1383 struct hashhead *oldtab, *newtab; 1384 vnode_impl_t *vip; 1385 1386 newtab = hashinit(desiredvnodes, HASH_SLIST, true, &newmask); 1387 mutex_enter(&vcache_lock); 1388 oldtab = vcache_hashtab; 1389 oldmask = vcache_hashmask; 1390 vcache_hashsize = desiredvnodes; 1391 vcache_hashtab = newtab; 1392 vcache_hashmask = newmask; 1393 for (i = 0; i <= oldmask; i++) { 1394 while ((vip = SLIST_FIRST(&oldtab[i])) != NULL) { 1395 SLIST_REMOVE(&oldtab[i], vip, vnode_impl, vi_hash); 1396 hash = vcache_hash(&vip->vi_key); 1397 SLIST_INSERT_HEAD(&newtab[hash & vcache_hashmask], 1398 vip, vi_hash); 1399 } 1400 } 1401 mutex_exit(&vcache_lock); 1402 hashdone(oldtab, HASH_SLIST, oldmask); 1403 } 1404 1405 static inline vnode_impl_t * 1406 vcache_hash_lookup(const struct vcache_key *key, uint32_t hash) 1407 { 1408 struct hashhead *hashp; 1409 vnode_impl_t *vip; 1410 1411 KASSERT(mutex_owned(&vcache_lock)); 1412 1413 hashp = &vcache_hashtab[hash & vcache_hashmask]; 1414 SLIST_FOREACH(vip, hashp, vi_hash) { 1415 if (key->vk_mount != vip->vi_key.vk_mount) 1416 continue; 1417 if (key->vk_key_len != vip->vi_key.vk_key_len) 1418 continue; 1419 if (memcmp(key->vk_key, vip->vi_key.vk_key, key->vk_key_len)) 1420 continue; 1421 return vip; 1422 } 1423 return NULL; 1424 } 1425 1426 /* 1427 * Allocate a new, uninitialized vcache node. 1428 */ 1429 static vnode_impl_t * 1430 vcache_alloc(void) 1431 { 1432 vnode_impl_t *vip; 1433 vnode_t *vp; 1434 1435 vip = pool_cache_get(vcache_pool, PR_WAITOK); 1436 vp = VIMPL_TO_VNODE(vip); 1437 memset(vip, 0, sizeof(*vip)); 1438 1439 rw_init(&vip->vi_lock); 1440 vp->v_interlock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE); 1441 1442 uvm_obj_init(&vp->v_uobj, &uvm_vnodeops, true, 1); 1443 klist_init(&vip->vi_klist.vk_klist); 1444 vp->v_klist = &vip->vi_klist; 1445 cv_init(&vp->v_cv, "vnode"); 1446 cache_vnode_init(vp); 1447 1448 vp->v_usecount = 1; 1449 vp->v_type = VNON; 1450 vp->v_size = vp->v_writesize = VSIZENOTSET; 1451 1452 vip->vi_state = VS_LOADING; 1453 1454 lru_requeue(vp, &lru_list[LRU_FREE]); 1455 1456 return vip; 1457 } 1458 1459 /* 1460 * Deallocate a vcache node in state VS_LOADING. 1461 * 1462 * vcache_lock held on entry and released on return. 1463 */ 1464 static void 1465 vcache_dealloc(vnode_impl_t *vip) 1466 { 1467 vnode_t *vp; 1468 1469 KASSERT(mutex_owned(&vcache_lock)); 1470 1471 vp = VIMPL_TO_VNODE(vip); 1472 vfs_ref(dead_rootmount); 1473 vfs_insmntque(vp, dead_rootmount); 1474 mutex_enter(vp->v_interlock); 1475 vp->v_op = dead_vnodeop_p; 1476 VSTATE_CHANGE(vp, VS_LOADING, VS_RECLAIMED); 1477 mutex_exit(&vcache_lock); 1478 vrelel(vp, 0, LK_NONE); 1479 } 1480 1481 /* 1482 * Free an unused, unreferenced vcache node. 1483 * v_interlock locked on entry. 1484 */ 1485 static void 1486 vcache_free(vnode_impl_t *vip) 1487 { 1488 vnode_t *vp; 1489 1490 vp = VIMPL_TO_VNODE(vip); 1491 KASSERT(mutex_owned(vp->v_interlock)); 1492 1493 KASSERT(vrefcnt(vp) == 0); 1494 KASSERT(vp->v_holdcnt == 0); 1495 KASSERT(vp->v_writecount == 0); 1496 lru_requeue(vp, NULL); 1497 mutex_exit(vp->v_interlock); 1498 1499 vfs_insmntque(vp, NULL); 1500 if (vp->v_type == VBLK || vp->v_type == VCHR) 1501 spec_node_destroy(vp); 1502 1503 mutex_obj_free(vp->v_interlock); 1504 rw_destroy(&vip->vi_lock); 1505 uvm_obj_destroy(&vp->v_uobj, true); 1506 KASSERT(vp->v_klist == &vip->vi_klist); 1507 klist_fini(&vip->vi_klist.vk_klist); 1508 cv_destroy(&vp->v_cv); 1509 cache_vnode_fini(vp); 1510 pool_cache_put(vcache_pool, vip); 1511 } 1512 1513 /* 1514 * Try to get an initial reference on this cached vnode. 1515 * Returns zero on success or EBUSY if the vnode state is not LOADED. 1516 * 1517 * NB: lockless code sequences may rely on this not blocking. 1518 */ 1519 int 1520 vcache_tryvget(vnode_t *vp) 1521 { 1522 u_int use, next; 1523 1524 for (use = atomic_load_relaxed(&vp->v_usecount);; use = next) { 1525 if (__predict_false((use & VUSECOUNT_GATE) == 0)) { 1526 return SET_ERROR(EBUSY); 1527 } 1528 next = atomic_cas_uint(&vp->v_usecount, 1529 use, (use + 1) | VUSECOUNT_VGET); 1530 if (__predict_true(next == use)) { 1531 membar_acquire(); 1532 return 0; 1533 } 1534 } 1535 } 1536 1537 /* 1538 * Try to get an initial reference on this cached vnode. 1539 * Returns zero on success and ENOENT if the vnode has been reclaimed. 1540 * Will wait for the vnode state to be stable. 1541 * 1542 * v_interlock locked on entry and unlocked on exit. 1543 */ 1544 int 1545 vcache_vget(vnode_t *vp) 1546 { 1547 int error; 1548 1549 KASSERT(mutex_owned(vp->v_interlock)); 1550 1551 /* Increment hold count to prevent vnode from disappearing. */ 1552 vp->v_holdcnt++; 1553 VSTATE_WAIT_STABLE(vp); 1554 vp->v_holdcnt--; 1555 1556 /* If this was the last reference to a reclaimed vnode free it now. */ 1557 if (__predict_false(VSTATE_GET(vp) == VS_RECLAIMED)) { 1558 if (vp->v_holdcnt == 0 && vrefcnt(vp) == 0) 1559 vcache_free(VNODE_TO_VIMPL(vp)); 1560 else 1561 mutex_exit(vp->v_interlock); 1562 return SET_ERROR(ENOENT); 1563 } 1564 VSTATE_ASSERT(vp, VS_LOADED); 1565 error = vcache_tryvget(vp); 1566 KASSERT(error == 0); 1567 mutex_exit(vp->v_interlock); 1568 1569 return 0; 1570 } 1571 1572 /* 1573 * Get a vnode / fs node pair by key and return it referenced through vpp. 1574 */ 1575 int 1576 vcache_get(struct mount *mp, const void *key, size_t key_len, 1577 struct vnode **vpp) 1578 { 1579 int error; 1580 uint32_t hash; 1581 const void *new_key; 1582 struct vnode *vp; 1583 struct vcache_key vcache_key; 1584 vnode_impl_t *vip, *new_vip; 1585 1586 new_key = NULL; 1587 *vpp = NULL; 1588 1589 vcache_key.vk_mount = mp; 1590 vcache_key.vk_key = key; 1591 vcache_key.vk_key_len = key_len; 1592 hash = vcache_hash(&vcache_key); 1593 1594 again: 1595 mutex_enter(&vcache_lock); 1596 vip = vcache_hash_lookup(&vcache_key, hash); 1597 1598 /* If found, take a reference or retry. */ 1599 if (__predict_true(vip != NULL)) { 1600 /* 1601 * If the vnode is loading we cannot take the v_interlock 1602 * here as it might change during load (see uvm_obj_setlock()). 1603 * As changing state from VS_LOADING requires both vcache_lock 1604 * and v_interlock it is safe to test with vcache_lock held. 1605 * 1606 * Wait for vnodes changing state from VS_LOADING and retry. 1607 */ 1608 if (__predict_false(vip->vi_state == VS_LOADING)) { 1609 cv_wait(&vcache_cv, &vcache_lock); 1610 mutex_exit(&vcache_lock); 1611 goto again; 1612 } 1613 vp = VIMPL_TO_VNODE(vip); 1614 mutex_enter(vp->v_interlock); 1615 mutex_exit(&vcache_lock); 1616 error = vcache_vget(vp); 1617 if (error == ENOENT) 1618 goto again; 1619 if (error == 0) 1620 *vpp = vp; 1621 KASSERT((error != 0) == (*vpp == NULL)); 1622 return error; 1623 } 1624 mutex_exit(&vcache_lock); 1625 1626 /* Allocate and initialize a new vcache / vnode pair. */ 1627 error = vfs_busy(mp); 1628 if (error) 1629 return error; 1630 new_vip = vcache_alloc(); 1631 new_vip->vi_key = vcache_key; 1632 vp = VIMPL_TO_VNODE(new_vip); 1633 mutex_enter(&vcache_lock); 1634 vip = vcache_hash_lookup(&vcache_key, hash); 1635 if (vip == NULL) { 1636 SLIST_INSERT_HEAD(&vcache_hashtab[hash & vcache_hashmask], 1637 new_vip, vi_hash); 1638 vip = new_vip; 1639 } 1640 1641 /* If another thread beat us inserting this node, retry. */ 1642 if (vip != new_vip) { 1643 vcache_dealloc(new_vip); 1644 vfs_unbusy(mp); 1645 goto again; 1646 } 1647 mutex_exit(&vcache_lock); 1648 1649 /* Load the fs node. Exclusive as new_node is VS_LOADING. */ 1650 error = VFS_LOADVNODE(mp, vp, key, key_len, &new_key); 1651 if (error) { 1652 mutex_enter(&vcache_lock); 1653 SLIST_REMOVE(&vcache_hashtab[hash & vcache_hashmask], 1654 new_vip, vnode_impl, vi_hash); 1655 vcache_dealloc(new_vip); 1656 vfs_unbusy(mp); 1657 KASSERT(*vpp == NULL); 1658 return error; 1659 } 1660 KASSERT(new_key != NULL); 1661 KASSERT(memcmp(key, new_key, key_len) == 0); 1662 KASSERT(vp->v_op != NULL); 1663 vfs_insmntque(vp, mp); 1664 if ((mp->mnt_iflag & IMNT_MPSAFE) != 0) 1665 vp->v_vflag |= VV_MPSAFE; 1666 vfs_ref(mp); 1667 vfs_unbusy(mp); 1668 1669 /* Finished loading, finalize node. */ 1670 mutex_enter(&vcache_lock); 1671 new_vip->vi_key.vk_key = new_key; 1672 mutex_enter(vp->v_interlock); 1673 VSTATE_CHANGE(vp, VS_LOADING, VS_LOADED); 1674 mutex_exit(vp->v_interlock); 1675 mutex_exit(&vcache_lock); 1676 *vpp = vp; 1677 return 0; 1678 } 1679 1680 /* 1681 * Create a new vnode / fs node pair and return it referenced through vpp. 1682 */ 1683 int 1684 vcache_new(struct mount *mp, struct vnode *dvp, struct vattr *vap, 1685 kauth_cred_t cred, void *extra, struct vnode **vpp) 1686 { 1687 int error; 1688 uint32_t hash; 1689 struct vnode *vp, *ovp; 1690 vnode_impl_t *vip, *ovip; 1691 1692 *vpp = NULL; 1693 1694 /* Allocate and initialize a new vcache / vnode pair. */ 1695 error = vfs_busy(mp); 1696 if (error) 1697 return error; 1698 vip = vcache_alloc(); 1699 vip->vi_key.vk_mount = mp; 1700 vp = VIMPL_TO_VNODE(vip); 1701 1702 /* Create and load the fs node. */ 1703 error = VFS_NEWVNODE(mp, dvp, vp, vap, cred, extra, 1704 &vip->vi_key.vk_key_len, &vip->vi_key.vk_key); 1705 if (error) { 1706 mutex_enter(&vcache_lock); 1707 vcache_dealloc(vip); 1708 vfs_unbusy(mp); 1709 KASSERT(*vpp == NULL); 1710 return error; 1711 } 1712 KASSERT(vp->v_op != NULL); 1713 KASSERT((vip->vi_key.vk_key_len == 0) == (mp == dead_rootmount)); 1714 if (vip->vi_key.vk_key_len > 0) { 1715 KASSERT(vip->vi_key.vk_key != NULL); 1716 hash = vcache_hash(&vip->vi_key); 1717 1718 /* 1719 * Wait for previous instance to be reclaimed, 1720 * then insert new node. 1721 */ 1722 mutex_enter(&vcache_lock); 1723 while ((ovip = vcache_hash_lookup(&vip->vi_key, hash))) { 1724 ovp = VIMPL_TO_VNODE(ovip); 1725 mutex_enter(ovp->v_interlock); 1726 mutex_exit(&vcache_lock); 1727 error = vcache_vget(ovp); 1728 KASSERT(error == ENOENT); 1729 mutex_enter(&vcache_lock); 1730 } 1731 SLIST_INSERT_HEAD(&vcache_hashtab[hash & vcache_hashmask], 1732 vip, vi_hash); 1733 mutex_exit(&vcache_lock); 1734 } 1735 vfs_insmntque(vp, mp); 1736 if ((mp->mnt_iflag & IMNT_MPSAFE) != 0) 1737 vp->v_vflag |= VV_MPSAFE; 1738 vfs_ref(mp); 1739 vfs_unbusy(mp); 1740 1741 /* Finished loading, finalize node. */ 1742 mutex_enter(&vcache_lock); 1743 mutex_enter(vp->v_interlock); 1744 VSTATE_CHANGE(vp, VS_LOADING, VS_LOADED); 1745 mutex_exit(&vcache_lock); 1746 mutex_exit(vp->v_interlock); 1747 *vpp = vp; 1748 return 0; 1749 } 1750 1751 /* 1752 * Prepare key change: update old cache nodes key and lock new cache node. 1753 * Return an error if the new node already exists. 1754 */ 1755 int 1756 vcache_rekey_enter(struct mount *mp, struct vnode *vp, 1757 const void *old_key, size_t old_key_len, 1758 const void *new_key, size_t new_key_len) 1759 { 1760 uint32_t old_hash, new_hash; 1761 struct vcache_key old_vcache_key, new_vcache_key; 1762 vnode_impl_t *vip, *new_vip; 1763 1764 old_vcache_key.vk_mount = mp; 1765 old_vcache_key.vk_key = old_key; 1766 old_vcache_key.vk_key_len = old_key_len; 1767 old_hash = vcache_hash(&old_vcache_key); 1768 1769 new_vcache_key.vk_mount = mp; 1770 new_vcache_key.vk_key = new_key; 1771 new_vcache_key.vk_key_len = new_key_len; 1772 new_hash = vcache_hash(&new_vcache_key); 1773 1774 new_vip = vcache_alloc(); 1775 new_vip->vi_key = new_vcache_key; 1776 1777 /* Insert locked new node used as placeholder. */ 1778 mutex_enter(&vcache_lock); 1779 vip = vcache_hash_lookup(&new_vcache_key, new_hash); 1780 if (vip != NULL) { 1781 vcache_dealloc(new_vip); 1782 return SET_ERROR(EEXIST); 1783 } 1784 SLIST_INSERT_HEAD(&vcache_hashtab[new_hash & vcache_hashmask], 1785 new_vip, vi_hash); 1786 1787 /* Replace old nodes key with the temporary copy. */ 1788 vip = vcache_hash_lookup(&old_vcache_key, old_hash); 1789 KASSERT(vip != NULL); 1790 KASSERT(VIMPL_TO_VNODE(vip) == vp); 1791 KASSERT(vip->vi_key.vk_key != old_vcache_key.vk_key); 1792 vip->vi_key = old_vcache_key; 1793 mutex_exit(&vcache_lock); 1794 return 0; 1795 } 1796 1797 /* 1798 * Key change complete: update old node and remove placeholder. 1799 */ 1800 void 1801 vcache_rekey_exit(struct mount *mp, struct vnode *vp, 1802 const void *old_key, size_t old_key_len, 1803 const void *new_key, size_t new_key_len) 1804 { 1805 uint32_t old_hash, new_hash; 1806 struct vcache_key old_vcache_key, new_vcache_key; 1807 vnode_impl_t *vip, *new_vip; 1808 struct vnode *new_vp; 1809 1810 old_vcache_key.vk_mount = mp; 1811 old_vcache_key.vk_key = old_key; 1812 old_vcache_key.vk_key_len = old_key_len; 1813 old_hash = vcache_hash(&old_vcache_key); 1814 1815 new_vcache_key.vk_mount = mp; 1816 new_vcache_key.vk_key = new_key; 1817 new_vcache_key.vk_key_len = new_key_len; 1818 new_hash = vcache_hash(&new_vcache_key); 1819 1820 mutex_enter(&vcache_lock); 1821 1822 /* Lookup old and new node. */ 1823 vip = vcache_hash_lookup(&old_vcache_key, old_hash); 1824 KASSERT(vip != NULL); 1825 KASSERT(VIMPL_TO_VNODE(vip) == vp); 1826 1827 new_vip = vcache_hash_lookup(&new_vcache_key, new_hash); 1828 KASSERT(new_vip != NULL); 1829 KASSERT(new_vip->vi_key.vk_key_len == new_key_len); 1830 new_vp = VIMPL_TO_VNODE(new_vip); 1831 mutex_enter(new_vp->v_interlock); 1832 VSTATE_ASSERT(VIMPL_TO_VNODE(new_vip), VS_LOADING); 1833 mutex_exit(new_vp->v_interlock); 1834 1835 /* Rekey old node and put it onto its new hashlist. */ 1836 vip->vi_key = new_vcache_key; 1837 if (old_hash != new_hash) { 1838 SLIST_REMOVE(&vcache_hashtab[old_hash & vcache_hashmask], 1839 vip, vnode_impl, vi_hash); 1840 SLIST_INSERT_HEAD(&vcache_hashtab[new_hash & vcache_hashmask], 1841 vip, vi_hash); 1842 } 1843 1844 /* Remove new node used as placeholder. */ 1845 SLIST_REMOVE(&vcache_hashtab[new_hash & vcache_hashmask], 1846 new_vip, vnode_impl, vi_hash); 1847 vcache_dealloc(new_vip); 1848 } 1849 1850 /* 1851 * Disassociate the underlying file system from a vnode. 1852 * 1853 * Must be called with vnode locked and will return unlocked. 1854 * Must be called with the interlock held, and will return with it held. 1855 */ 1856 static void 1857 vcache_reclaim(vnode_t *vp) 1858 { 1859 lwp_t *l = curlwp; 1860 vnode_impl_t *vip = VNODE_TO_VIMPL(vp); 1861 struct mount *mp = vp->v_mount; 1862 uint32_t hash; 1863 uint8_t temp_buf[64], *temp_key; 1864 size_t temp_key_len; 1865 bool recycle; 1866 int error; 1867 1868 KASSERT(VOP_ISLOCKED(vp) == LK_EXCLUSIVE); 1869 KASSERT(mutex_owned(vp->v_interlock)); 1870 KASSERT(vrefcnt(vp) != 0); 1871 1872 temp_key_len = vip->vi_key.vk_key_len; 1873 /* 1874 * Prevent the vnode from being recycled or brought into use 1875 * while we clean it out. 1876 */ 1877 VSTATE_CHANGE(vp, VS_BLOCKED, VS_RECLAIMING); 1878 1879 /* 1880 * Send NOTE_REVOKE now, before we call VOP_RECLAIM(), 1881 * because VOP_RECLAIM() could cause vp->v_klist to 1882 * become invalid. Don't check for interest in NOTE_REVOKE 1883 * here; it's always posted because it sets EV_EOF. 1884 * 1885 * Once it's been posted, reset vp->v_klist to point to 1886 * our own local storage, in case we were sharing with 1887 * someone else. 1888 */ 1889 KNOTE(&vp->v_klist->vk_klist, NOTE_REVOKE); 1890 vp->v_klist = &vip->vi_klist; 1891 mutex_exit(vp->v_interlock); 1892 1893 rw_enter(vp->v_uobj.vmobjlock, RW_WRITER); 1894 mutex_enter(vp->v_interlock); 1895 if ((vp->v_iflag & VI_EXECMAP) != 0) { 1896 cpu_count(CPU_COUNT_EXECPAGES, -vp->v_uobj.uo_npages); 1897 } 1898 vp->v_iflag &= ~(VI_TEXT|VI_EXECMAP); 1899 vp->v_iflag |= VI_DEADCHECK; /* for genfs_getpages() */ 1900 mutex_exit(vp->v_interlock); 1901 rw_exit(vp->v_uobj.vmobjlock); 1902 1903 /* 1904 * With vnode state set to reclaiming, purge name cache immediately 1905 * to prevent new handles on vnode, and wait for existing threads 1906 * trying to get a handle to notice VS_RECLAIMED status and abort. 1907 */ 1908 cache_purge(vp); 1909 1910 /* Replace the vnode key with a temporary copy. */ 1911 if (vip->vi_key.vk_key_len > sizeof(temp_buf)) { 1912 temp_key = kmem_alloc(temp_key_len, KM_SLEEP); 1913 } else { 1914 temp_key = temp_buf; 1915 } 1916 if (vip->vi_key.vk_key_len > 0) { 1917 mutex_enter(&vcache_lock); 1918 memcpy(temp_key, vip->vi_key.vk_key, temp_key_len); 1919 vip->vi_key.vk_key = temp_key; 1920 mutex_exit(&vcache_lock); 1921 } 1922 1923 fstrans_start(mp); 1924 1925 /* 1926 * Clean out any cached data associated with the vnode. 1927 */ 1928 error = vinvalbuf(vp, V_SAVE, NOCRED, l, 0, 0); 1929 if (error != 0) { 1930 if (wapbl_vphaswapbl(vp)) 1931 WAPBL_DISCARD(wapbl_vptomp(vp)); 1932 error = vinvalbuf(vp, 0, NOCRED, l, 0, 0); 1933 } 1934 KASSERTMSG((error == 0), "vinvalbuf failed: %d", error); 1935 KASSERT((vp->v_iflag & VI_ONWORKLST) == 0); 1936 if (vp->v_type == VBLK || vp->v_type == VCHR) { 1937 spec_node_revoke(vp); 1938 } 1939 1940 /* 1941 * Disassociate the underlying file system from the vnode. 1942 * VOP_INACTIVE leaves the vnode locked; VOP_RECLAIM unlocks 1943 * the vnode, and may destroy the vnode so that VOP_UNLOCK 1944 * would no longer function. 1945 */ 1946 VOP_INACTIVE(vp, &recycle); 1947 KASSERT(VOP_ISLOCKED(vp) == LK_EXCLUSIVE); 1948 if (VOP_RECLAIM(vp)) { 1949 vnpanic(vp, "%s: cannot reclaim", __func__); 1950 } 1951 1952 KASSERT(vp->v_data == NULL); 1953 KASSERT((vp->v_iflag & VI_PAGES) == 0); 1954 1955 if (vp->v_type == VREG && vp->v_ractx != NULL) { 1956 uvm_ra_freectx(vp->v_ractx); 1957 vp->v_ractx = NULL; 1958 } 1959 1960 if (vip->vi_key.vk_key_len > 0) { 1961 /* Remove from vnode cache. */ 1962 hash = vcache_hash(&vip->vi_key); 1963 mutex_enter(&vcache_lock); 1964 KASSERT(vip == vcache_hash_lookup(&vip->vi_key, hash)); 1965 SLIST_REMOVE(&vcache_hashtab[hash & vcache_hashmask], 1966 vip, vnode_impl, vi_hash); 1967 mutex_exit(&vcache_lock); 1968 } 1969 if (temp_key != temp_buf) 1970 kmem_free(temp_key, temp_key_len); 1971 1972 /* Done with purge, notify sleepers of the grim news. */ 1973 mutex_enter(vp->v_interlock); 1974 vp->v_op = dead_vnodeop_p; 1975 VSTATE_CHANGE(vp, VS_RECLAIMING, VS_RECLAIMED); 1976 vp->v_tag = VT_NON; 1977 mutex_exit(vp->v_interlock); 1978 1979 /* 1980 * Move to dead mount. Must be after changing the operations 1981 * vector as vnode operations enter the mount before using the 1982 * operations vector. See sys/kern/vnode_if.c. 1983 */ 1984 vp->v_vflag &= ~VV_ROOT; 1985 vfs_ref(dead_rootmount); 1986 vfs_insmntque(vp, dead_rootmount); 1987 1988 #ifdef PAX_SEGVGUARD 1989 pax_segvguard_cleanup(vp); 1990 #endif /* PAX_SEGVGUARD */ 1991 1992 mutex_enter(vp->v_interlock); 1993 fstrans_done(mp); 1994 KASSERT((vp->v_iflag & VI_ONWORKLST) == 0); 1995 } 1996 1997 /* 1998 * Disassociate the underlying file system from an open device vnode 1999 * and make it anonymous. 2000 * 2001 * Vnode unlocked on entry, drops a reference to the vnode. 2002 */ 2003 void 2004 vcache_make_anon(vnode_t *vp) 2005 { 2006 vnode_impl_t *vip = VNODE_TO_VIMPL(vp); 2007 uint32_t hash; 2008 bool recycle; 2009 2010 KASSERT(vp->v_type == VBLK || vp->v_type == VCHR); 2011 KASSERT(vp->v_mount == dead_rootmount || 2012 fstrans_is_owner(vp->v_mount)); 2013 VSTATE_ASSERT_UNLOCKED(vp, VS_ACTIVE); 2014 2015 /* Remove from vnode cache. */ 2016 hash = vcache_hash(&vip->vi_key); 2017 mutex_enter(&vcache_lock); 2018 KASSERT(vip == vcache_hash_lookup(&vip->vi_key, hash)); 2019 SLIST_REMOVE(&vcache_hashtab[hash & vcache_hashmask], 2020 vip, vnode_impl, vi_hash); 2021 vip->vi_key.vk_mount = dead_rootmount; 2022 vip->vi_key.vk_key_len = 0; 2023 vip->vi_key.vk_key = NULL; 2024 mutex_exit(&vcache_lock); 2025 2026 /* 2027 * Disassociate the underlying file system from the vnode. 2028 * VOP_INACTIVE leaves the vnode locked; VOP_RECLAIM unlocks 2029 * the vnode, and may destroy the vnode so that VOP_UNLOCK 2030 * would no longer function. 2031 */ 2032 if (vn_lock(vp, LK_EXCLUSIVE)) { 2033 vnpanic(vp, "%s: cannot lock", __func__); 2034 } 2035 VOP_INACTIVE(vp, &recycle); 2036 KASSERT(VOP_ISLOCKED(vp) == LK_EXCLUSIVE); 2037 if (VOP_RECLAIM(vp)) { 2038 vnpanic(vp, "%s: cannot reclaim", __func__); 2039 } 2040 2041 /* Purge name cache. */ 2042 cache_purge(vp); 2043 2044 /* Done with purge, change operations vector. */ 2045 mutex_enter(vp->v_interlock); 2046 vp->v_op = spec_vnodeop_p; 2047 vp->v_vflag |= VV_MPSAFE; 2048 mutex_exit(vp->v_interlock); 2049 2050 /* 2051 * Move to dead mount. Must be after changing the operations 2052 * vector as vnode operations enter the mount before using the 2053 * operations vector. See sys/kern/vnode_if.c. 2054 */ 2055 vfs_ref(dead_rootmount); 2056 vfs_insmntque(vp, dead_rootmount); 2057 2058 vrele(vp); 2059 } 2060 2061 /* 2062 * Update outstanding I/O count and do wakeup if requested. 2063 */ 2064 void 2065 vwakeup(struct buf *bp) 2066 { 2067 vnode_t *vp; 2068 2069 if ((vp = bp->b_vp) == NULL) 2070 return; 2071 2072 KASSERT(bp->b_objlock == vp->v_interlock); 2073 KASSERT(mutex_owned(bp->b_objlock)); 2074 2075 if (--vp->v_numoutput < 0) 2076 vnpanic(vp, "%s: neg numoutput, vp %p", __func__, vp); 2077 if (vp->v_numoutput == 0) 2078 cv_broadcast(&vp->v_cv); 2079 } 2080 2081 /* 2082 * Test a vnode for being or becoming dead. Returns one of: 2083 * EBUSY: vnode is becoming dead, with "flags == VDEAD_NOWAIT" only. 2084 * ENOENT: vnode is dead. 2085 * 0: otherwise. 2086 * 2087 * Whenever this function returns a non-zero value all future 2088 * calls will also return a non-zero value. 2089 */ 2090 int 2091 vdead_check(struct vnode *vp, int flags) 2092 { 2093 2094 KASSERT(mutex_owned(vp->v_interlock)); 2095 2096 if (! ISSET(flags, VDEAD_NOWAIT)) 2097 VSTATE_WAIT_STABLE(vp); 2098 2099 if (VSTATE_GET(vp) == VS_RECLAIMING) { 2100 KASSERT(ISSET(flags, VDEAD_NOWAIT)); 2101 return SET_ERROR(EBUSY); 2102 } else if (VSTATE_GET(vp) == VS_RECLAIMED) { 2103 return SET_ERROR(ENOENT); 2104 } 2105 2106 return 0; 2107 } 2108 2109 int 2110 vfs_drainvnodes(void) 2111 { 2112 2113 mutex_enter(&vdrain_lock); 2114 2115 if (!vdrain_one(desiredvnodes)) { 2116 mutex_exit(&vdrain_lock); 2117 return SET_ERROR(EBUSY); 2118 } 2119 2120 mutex_exit(&vdrain_lock); 2121 2122 if (vcache_hashsize != desiredvnodes) 2123 vcache_reinit(); 2124 2125 return 0; 2126 } 2127 2128 void 2129 vnpanic(vnode_t *vp, const char *fmt, ...) 2130 { 2131 va_list ap; 2132 2133 #ifdef DIAGNOSTIC 2134 vprint(NULL, vp); 2135 #endif 2136 va_start(ap, fmt); 2137 vpanic(fmt, ap); 2138 va_end(ap); 2139 } 2140 2141 void 2142 vshareilock(vnode_t *tvp, vnode_t *fvp) 2143 { 2144 kmutex_t *oldlock; 2145 2146 oldlock = tvp->v_interlock; 2147 mutex_obj_hold(fvp->v_interlock); 2148 tvp->v_interlock = fvp->v_interlock; 2149 mutex_obj_free(oldlock); 2150 } 2151 2152 void 2153 vshareklist(vnode_t *tvp, vnode_t *fvp) 2154 { 2155 /* 2156 * If two vnodes share klist state, they must also share 2157 * an interlock. 2158 */ 2159 KASSERT(tvp->v_interlock == fvp->v_interlock); 2160 2161 /* 2162 * We make the following assumptions: 2163 * 2164 * ==> Some other synchronization is happening outside of 2165 * our view to make this safe. 2166 * 2167 * ==> That the "to" vnode will have the necessary references 2168 * on the "from" vnode so that the storage for the klist 2169 * won't be yanked out from beneath us (the vnode_impl). 2170 * 2171 * ==> If "from" is also sharing, we then assume that "from" 2172 * has the necessary references, and so on. 2173 */ 2174 tvp->v_klist = fvp->v_klist; 2175 } 2176
Indexes created Fri Aug 21 00:24:51 UTC 2026