1 /* $NetBSD: vfs_vnode.c,v 1.160 2026/09/27 22:00:05 riastradh Exp $ */ 2 3 /*- 4 * Copyright (c) 1997-2011, 2019, 2020 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 9 * NASA Ames Research Center, by Charles M. Hannum, and by Andrew Doran. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 * POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 /* 34 * Copyright (c) 1989, 1993 35 * The Regents of the University of California. All rights reserved. 36 * (c) UNIX System Laboratories, Inc. 37 * All or some portions of this file are derived from material licensed 38 * to the University of California by American Telephone and Telegraph 39 * Co. or Unix System Laboratories, Inc. and are reproduced herein with 40 * the permission of UNIX System Laboratories, Inc. 41 * 42 * Redistribution and use in source and binary forms, with or without 43 * modification, are permitted provided that the following conditions 44 * are met: 45 * 1. Redistributions of source code must retain the above copyright 46 * notice, this list of conditions and the following disclaimer. 47 * 2. Redistributions in binary form must reproduce the above copyright 48 * notice, this list of conditions and the following disclaimer in the 49 * documentation and/or other materials provided with the distribution. 50 * 3. Neither the name of the University nor the names of its contributors 51 * may be used to endorse or promote products derived from this software 52 * without specific prior written permission. 53 * 54 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 55 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 56 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 57 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 58 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 59 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 60 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 61 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 62 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 63 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 64 * SUCH DAMAGE. 65 * 66 * @(#)vfs_subr.c 8.13 (Berkeley) 4/18/94 67 */ 68 69 /* 70 * The vnode cache subsystem. 71 * 72 * Life-cycle 73 * 74 * Normally, there are two points where new vnodes are created: 75 * VOP_CREATE(9) and VOP_LOOKUP(9). The life-cycle of a vnode 76 * starts in one of the following ways: 77 * 78 * - Allocation, via vcache_get(9) or vcache_new(9). 79 * - Reclamation of inactive vnode, via vcache_vget(9). 80 * 81 * Recycle from a free list, via getnewvnode(9) -> getcleanvnode(9) 82 * was another, traditional way. Currently, only the draining thread 83 * recycles the vnodes. This behaviour might be revisited. 84 * 85 * The life-cycle ends when the last reference is dropped, usually 86 * in VOP_REMOVE(9). In such case, VOP_INACTIVE(9) is called to inform 87 * the file system that vnode is inactive. Via this call, file system 88 * indicates whether vnode can be recycled (usually, it checks its own 89 * references, e.g. count of links, whether the file was removed). 90 * 91 * Depending on indication, vnode can be put into a free list (cache), 92 * or cleaned via vcache_reclaim, which calls VOP_RECLAIM(9) to 93 * disassociate underlying file system from the vnode, and finally 94 * destroyed. 95 * 96 * Vnode state 97 * 98 * Vnode is always in one of six states: 99 * - MARKER This is a marker vnode to help list traversal. It 100 * will never change its state. 101 * - LOADING Vnode is associating underlying file system and not 102 * yet ready to use. 103 * - LOADED Vnode has associated underlying file system and is 104 * ready to use. 105 * - BLOCKED Vnode is active but cannot get new references. 106 * - RECLAIMING Vnode is disassociating from the underlying file 107 * system. 108 * - RECLAIMED Vnode has disassociated from underlying file system 109 * and is dead. 110 * 111 * Valid state changes are: 112 * LOADING -> LOADED 113 * Vnode has been initialised in vcache_get() or 114 * vcache_new() and is ready to use. 115 * BLOCKED -> RECLAIMING 116 * Vnode starts disassociation from underlying file 117 * system in vcache_reclaim(). 118 * RECLAIMING -> RECLAIMED 119 * Vnode finished disassociation from underlying file 120 * system in vcache_reclaim(). 121 * LOADED -> BLOCKED 122 * Either vcache_rekey*() is changing the vnode key or 123 * vrelel() is about to call VOP_INACTIVE(). 124 * BLOCKED -> LOADED 125 * The block condition is over. 126 * LOADING -> RECLAIMED 127 * Either vcache_get() or vcache_new() failed to 128 * associate the underlying file system or vcache_rekey*() 129 * drops a vnode used as placeholder. 130 * 131 * Of these states LOADING, BLOCKED and RECLAIMING are intermediate 132 * and it is possible to wait for state change. 133 * 134 * State is protected with v_interlock with one exception: 135 * to change from LOADING both v_interlock and vcache_lock must be held 136 * so it is possible to check "state == LOADING" without holding 137 * v_interlock. See vcache_get() for details. 138 * 139 * Reference counting 140 * 141 * Vnode is considered active, if reference count (vnode_t::v_usecount) 142 * is non-zero. It is maintained using: vref(9) and vrele(9), as well 143 * as vput(9), routines. Common points holding references are e.g. 144 * file openings, current working directory, mount points, etc. 145 * 146 * v_usecount is adjusted with atomic operations, however to change 147 * from a non-zero value to zero the interlock must also be held. 148 */ 149 150 #include <sys/cdefs.h> 151 __KERNEL_RCSID(0, "$NetBSD: vfs_vnode.c,v 1.160 2026/09/27 22:00:05 riastradh Exp $"); 152 153 #ifdef _KERNEL_OPT 154 #include "opt_pax.h" 155 #endif 156 157 #include <sys/param.h> 158 #include <sys/types.h> 159 160 #include <sys/atomic.h> 161 #include <sys/buf.h> 162 #include <sys/conf.h> 163 #include <sys/device.h> 164 #include <sys/fstrans.h> 165 #include <sys/hash.h> 166 #include <sys/kauth.h> 167 #include <sys/kernel.h> 168 #include <sys/kmem.h> 169 #include <sys/module.h> 170 #include <sys/mount.h> 171 #include <sys/namei.h> 172 #include <sys/pax.h> 173 #include <sys/sdt.h> 174 #include <sys/stdalign.h> 175 #include <sys/syscallargs.h> 176 #include <sys/sysctl.h> 177 #include <sys/systm.h> 178 #include <sys/threadpool.h> 179 #include <sys/vnode_impl.h> 180 #include <sys/wapbl.h> 181 182 #include <miscfs/deadfs/deadfs.h> 183 #include <miscfs/specfs/specdev.h> 184 185 #include <uvm/uvm.h> 186 #include <uvm/uvm_readahead.h> 187 #include <uvm/uvm_stat.h> 188 189 /* Flags to vrelel. */ 190 #define VRELEL_ASYNC 0x0001 /* Always defer to vrele thread. */ 191 192 #define LRU_VRELE 0 193 #define LRU_FREE 1 194 #define LRU_HOLD 2 195 #define LRU_COUNT 3 196 197 /* 198 * There are three lru lists: one holds vnodes waiting for async release, 199 * one is for vnodes which have no buffer/page references and one for those 200 * which do (i.e. v_holdcnt is non-zero). We put the lists into a single, 201 * private cache line as vnodes migrate between them while under the same 202 * lock (vdrain_lock). 203 */ 204 205 typedef struct { 206 vnode_impl_t *li_marker; 207 } lru_iter_t; 208 209 u_int numvnodes __cacheline_aligned; 210 static vnodelst_t lru_list[LRU_COUNT] __cacheline_aligned; 211 static struct threadpool *threadpool; 212 static struct threadpool_job vdrain_job; 213 static struct threadpool_job vrele_job; 214 static kmutex_t vdrain_lock __cacheline_aligned; 215 SLIST_HEAD(hashhead, vnode_impl); 216 static kmutex_t vcache_lock __cacheline_aligned; 217 static kcondvar_t vcache_cv; 218 static u_int vcache_hashsize; 219 static u_long vcache_hashmask; 220 static struct hashhead *vcache_hashtab; 221 static pool_cache_t vcache_pool; 222 static void lru_requeue(vnode_t *, vnodelst_t *); 223 static vnodelst_t * lru_which(vnode_t *); 224 static vnode_impl_t * lru_iter_first(int, lru_iter_t *); 225 static vnode_impl_t * lru_iter_next(lru_iter_t *); 226 static void lru_iter_release(lru_iter_t *); 227 static vnode_impl_t * vcache_alloc(void); 228 static void vcache_dealloc(vnode_impl_t *); 229 static void vcache_free(vnode_impl_t *); 230 static void vcache_init(void); 231 static void vcache_reinit(void); 232 static void vcache_reclaim(vnode_t *); 233 static void vrele_deferred(vnode_impl_t *); 234 static void vrelel(vnode_t *, int, int); 235 static void vnpanic(vnode_t *, const char *, ...) 236 __printflike(2, 3); 237 static bool vdrain_one(u_int); 238 static void vdrain_task(struct threadpool_job *); 239 static void vrele_task(struct threadpool_job *); 240 241 /* Routines having to do with the management of the vnode table. */ 242 243 /* 244 * The high bit of v_usecount is a gate for vcache_tryvget(). It's set 245 * only when the vnode state is LOADED. 246 * The next bit of v_usecount is a flag for vrelel(). It's set 247 * from vcache_vget() and vcache_tryvget() whenever the operation succeeds. 248 */ 249 #define VUSECOUNT_MASK 0x3fffffff 250 #define VUSECOUNT_GATE 0x80000000 251 #define VUSECOUNT_VGET 0x40000000 252 253 /* 254 * Return the current usecount of a vnode. 255 */ 256 inline int 257 vrefcnt(struct vnode *vp) 258 { 259 260 return atomic_load_relaxed(&vp->v_usecount) & VUSECOUNT_MASK; 261 } 262 263 /* Vnode state operations and diagnostics. */ 264 265 #if defined(DIAGNOSTIC) 266 267 #define VSTATE_VALID(state) \ 268 ((state) != VS_ACTIVE && (state) != VS_MARKER) 269 #define VSTATE_GET(vp) \ 270 vstate_assert_get((vp), __func__, __LINE__) 271 #define VSTATE_CHANGE(vp, from, to) \ 272 vstate_assert_change((vp), (from), (to), __func__, __LINE__) 273 #define VSTATE_WAIT_STABLE(vp) \ 274 vstate_assert_wait_stable((vp), __func__, __LINE__) 275 276 void 277 _vstate_assert(vnode_t *vp, enum vnode_state state, const char *func, int line, 278 bool has_lock) 279 { 280 vnode_impl_t *vip = VNODE_TO_VIMPL(vp); 281 int refcnt = vrefcnt(vp); 282 283 if (!has_lock) { 284 enum vnode_state vstate = atomic_load_relaxed(&vip->vi_state); 285 286 if (state == VS_ACTIVE && refcnt > 0 && 287 (vstate == VS_LOADED || vstate == VS_BLOCKED)) 288 return; 289 if (vstate == state) 290 return; 291 mutex_enter((vp)->v_interlock); 292 } 293 294 KASSERTMSG(mutex_owned(vp->v_interlock), "at %s:%d", func, line); 295 296 if ((state == VS_ACTIVE && refcnt > 0 && 297 (vip->vi_state == VS_LOADED || vip->vi_state == VS_BLOCKED)) || 298 vip->vi_state == state) { 299 if (!has_lock) 300 mutex_exit((vp)->v_interlock); 301 return; 302 } 303 vnpanic(vp, "state is %s, usecount %d, expected %s at %s:%d", 304 vstate_name(vip->vi_state), refcnt, 305 vstate_name(state), func, line); 306 } 307 308 static enum vnode_state 309 vstate_assert_get(vnode_t *vp, const char *func, int line) 310 { 311 vnode_impl_t *vip = VNODE_TO_VIMPL(vp); 312 313 KASSERTMSG(mutex_owned(vp->v_interlock), "at %s:%d", func, line); 314 if (! VSTATE_VALID(vip->vi_state)) 315 vnpanic(vp, "state is %s at %s:%d", 316 vstate_name(vip->vi_state), func, line); 317 318 return vip->vi_state; 319 } 320 321 static void 322 vstate_assert_wait_stable(vnode_t *vp, const char *func, int line) 323 { 324 vnode_impl_t *vip = VNODE_TO_VIMPL(vp); 325 326 KASSERTMSG(mutex_owned(vp->v_interlock), "at %s:%d", func, line); 327 if (! VSTATE_VALID(vip->vi_state)) 328 vnpanic(vp, "state is %s at %s:%d", 329 vstate_name(vip->vi_state), func, line); 330 331 while (vip->vi_state != VS_LOADED && vip->vi_state != VS_RECLAIMED) 332 cv_wait(&vp->v_cv, vp->v_interlock); 333 334 if (! VSTATE_VALID(vip->vi_state)) 335 vnpanic(vp, "state is %s at %s:%d", 336 vstate_name(vip->vi_state), func, line); 337 } 338 339 static void 340 vstate_assert_change(vnode_t *vp, enum vnode_state from, enum vnode_state to, 341 const char *func, int line) 342 { 343 bool gated = (atomic_load_relaxed(&vp->v_usecount) & VUSECOUNT_GATE); 344 vnode_impl_t *vip = VNODE_TO_VIMPL(vp); 345 346 KASSERTMSG(mutex_owned(vp->v_interlock), "at %s:%d", func, line); 347 if (from == VS_LOADING) 348 KASSERTMSG(mutex_owned(&vcache_lock), "at %s:%d", func, line); 349 350 if (! VSTATE_VALID(from)) 351 vnpanic(vp, "from is %s at %s:%d", 352 vstate_name(from), func, line); 353 if (! VSTATE_VALID(to)) 354 vnpanic(vp, "to is %s at %s:%d", 355 vstate_name(to), func, line); 356 if (vip->vi_state != from) 357 vnpanic(vp, "from is %s, expected %s at %s:%d\n", 358 vstate_name(vip->vi_state), vstate_name(from), func, line); 359 if ((from == VS_LOADED) != gated) 360 vnpanic(vp, "state is %s, gate %d does not match at %s:%d\n", 361 vstate_name(vip->vi_state), gated, func, line); 362 363 /* Open/close the gate for vcache_tryvget(). */ 364 if (to == VS_LOADED) { 365 membar_release(); 366 atomic_or_uint(&vp->v_usecount, VUSECOUNT_GATE); 367 } else { 368 atomic_and_uint(&vp->v_usecount, ~VUSECOUNT_GATE); 369 } 370 371 atomic_store_relaxed(&vip->vi_state, to); 372 if (from == VS_LOADING) 373 cv_broadcast(&vcache_cv); 374 if (to == VS_LOADED || to == VS_RECLAIMED) 375 cv_broadcast(&vp->v_cv); 376 } 377 378 #else /* defined(DIAGNOSTIC) */ 379 380 #define VSTATE_GET(vp) \ 381 (VNODE_TO_VIMPL((vp))->vi_state) 382 #define VSTATE_CHANGE(vp, from, to) \ 383 vstate_change((vp), (from), (to)) 384 #define VSTATE_WAIT_STABLE(vp) \ 385 vstate_wait_stable((vp)) 386 void 387 _vstate_assert(vnode_t *vp, enum vnode_state state, const char *func, int line, 388 bool has_lock) 389 { 390 391 } 392 393 static void 394 vstate_wait_stable(vnode_t *vp) 395 { 396 vnode_impl_t *vip = VNODE_TO_VIMPL(vp); 397 398 while (vip->vi_state != VS_LOADED && vip->vi_state != VS_RECLAIMED) 399 cv_wait(&vp->v_cv, vp->v_interlock); 400 } 401 402 static void 403 vstate_change(vnode_t *vp, enum vnode_state from, enum vnode_state to) 404 { 405 vnode_impl_t *vip = VNODE_TO_VIMPL(vp); 406 407 /* Open/close the gate for vcache_tryvget(). */ 408 if (to == VS_LOADED) { 409 membar_release(); 410 atomic_or_uint(&vp->v_usecount, VUSECOUNT_GATE); 411 } else { 412 atomic_and_uint(&vp->v_usecount, ~VUSECOUNT_GATE); 413 } 414 415 atomic_store_relaxed(&vip->vi_state, to); 416 if (from == VS_LOADING) 417 cv_broadcast(&vcache_cv); 418 if (to == VS_LOADED || to == VS_RECLAIMED) 419 cv_broadcast(&vp->v_cv); 420 } 421 422 #endif /* defined(DIAGNOSTIC) */ 423 424 void 425 vfs_vnode_sysinit(void) 426 { 427 int error __diagused, i; 428 429 dead_rootmount = vfs_mountalloc(&dead_vfsops, NULL); 430 KASSERT(dead_rootmount != NULL); 431 dead_rootmount->mnt_iflag |= IMNT_MPSAFE; 432 433 mutex_init(&vdrain_lock, MUTEX_DEFAULT, IPL_NONE); 434 for (i = 0; i < LRU_COUNT; i++) { 435 TAILQ_INIT(&lru_list[i]); 436 } 437 vcache_init(); 438 439 error = threadpool_get(&threadpool, PRI_NONE); 440 KASSERTMSG((error == 0), "threadpool_get failed: %d", error); 441 threadpool_job_init(&vdrain_job, vdrain_task, &vdrain_lock, "vdrain"); 442 threadpool_job_init(&vrele_job, vrele_task, &vdrain_lock, "vrele"); 443 } 444 445 /* 446 * Allocate a new marker vnode. 447 */ 448 vnode_t * 449 vnalloc_marker(struct mount *mp) 450 { 451 vnode_impl_t *vip; 452 vnode_t *vp; 453 454 vip = pool_cache_get(vcache_pool, PR_WAITOK); 455 memset(vip, 0, sizeof(*vip)); 456 vp = VIMPL_TO_VNODE(vip); 457 uvm_obj_init(&vp->v_uobj, &uvm_vnodeops, true, 1); 458 vp->v_mount = mp; 459 vp->v_type = VBAD; 460 vp->v_interlock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE); 461 klist_init(&vip->vi_klist.vk_klist); 462 vp->v_klist = &vip->vi_klist; 463 vip->vi_state = VS_MARKER; 464 465 return vp; 466 } 467 468 /* 469 * Free a marker vnode. 470 */ 471 void 472 vnfree_marker(vnode_t *vp) 473 { 474 vnode_impl_t *vip; 475 476 vip = VNODE_TO_VIMPL(vp); 477 KASSERT(vip->vi_state == VS_MARKER); 478 mutex_obj_free(vp->v_interlock); 479 uvm_obj_destroy(&vp->v_uobj, true); 480 klist_fini(&vip->vi_klist.vk_klist); 481 pool_cache_put(vcache_pool, vip); 482 } 483 484 /* 485 * Test a vnode for being a marker vnode. 486 */ 487 bool 488 vnis_marker(vnode_t *vp) 489 { 490 491 return (VNODE_TO_VIMPL(vp)->vi_state == VS_MARKER); 492 } 493 494 /* 495 * Return the lru list this node should be on. 496 */ 497 static vnodelst_t * 498 lru_which(vnode_t *vp) 499 { 500 501 KASSERT(mutex_owned(vp->v_interlock)); 502 503 if (vp->v_holdcnt > 0) 504 return &lru_list[LRU_HOLD]; 505 else 506 return &lru_list[LRU_FREE]; 507 } 508 509 /* 510 * Put vnode to end of given list. 511 * Both the current and the new list may be NULL, used on vnode alloc/free. 512 * Adjust numvnodes and signal vdrain thread if there is work. 513 */ 514 static void 515 lru_requeue(vnode_t *vp, vnodelst_t *listhd) 516 { 517 vnode_impl_t *vip; 518 int d; 519 520 /* 521 * If the vnode is on the correct list, and was put there recently, 522 * then leave it be, thus avoiding huge cache and lock contention. 523 */ 524 vip = VNODE_TO_VIMPL(vp); 525 if (listhd == vip->vi_lrulisthd && 526 (getticks() - vip->vi_lrulisttm) < hz) { 527 return; 528 } 529 530 mutex_enter(&vdrain_lock); 531 d = 0; 532 if (vip->vi_lrulisthd != NULL) 533 TAILQ_REMOVE(vip->vi_lrulisthd, vip, vi_lrulist); 534 else 535 d++; 536 vip->vi_lrulisthd = listhd; 537 vip->vi_lrulisttm = getticks(); 538 if (vip->vi_lrulisthd != NULL) 539 TAILQ_INSERT_TAIL(vip->vi_lrulisthd, vip, vi_lrulist); 540 else 541 d--; 542 if (d != 0) { 543 /* 544 * Looks strange? This is not a bug. Don't store 545 * numvnodes unless there is a change - avoid false 546 * sharing on MP. 547 */ 548 numvnodes += d; 549 } 550 if (listhd == &lru_list[LRU_VRELE]) 551 threadpool_schedule_job(threadpool, &vrele_job); 552 if (d > 0 && numvnodes > desiredvnodes) 553 threadpool_schedule_job(threadpool, &vdrain_job); 554 if (d > 0 && numvnodes > desiredvnodes + desiredvnodes / 16) 555 kpause("vnfull", false, MAX(1, mstohz(10)), &vdrain_lock); 556 mutex_exit(&vdrain_lock); 557 } 558 559 /* 560 * LRU list iterator. 561 * Caller holds vdrain_lock. 562 */ 563 static vnode_impl_t * 564 lru_iter_first(int idx, lru_iter_t *iterp) 565 { 566 vnode_impl_t *marker; 567 568 KASSERT(mutex_owned(&vdrain_lock)); 569 570 mutex_exit(&vdrain_lock); 571 marker = VNODE_TO_VIMPL(vnalloc_marker(NULL)); 572 mutex_enter(&vdrain_lock); 573 marker->vi_lrulisthd = &lru_list[idx]; 574 iterp->li_marker = marker; 575 576 TAILQ_INSERT_HEAD(marker->vi_lrulisthd, marker, vi_lrulist); 577 578 return lru_iter_next(iterp); 579 } 580 581 static vnode_impl_t * 582 lru_iter_next(lru_iter_t *iter) 583 { 584 vnode_impl_t *vip, *marker; 585 vnodelst_t *listhd; 586 587 KASSERT(mutex_owned(&vdrain_lock)); 588 589 marker = iter->li_marker; 590 listhd = marker->vi_lrulisthd; 591 592 while ((vip = TAILQ_NEXT(marker, vi_lrulist))) { 593 TAILQ_REMOVE(listhd, marker, vi_lrulist); 594 TAILQ_INSERT_AFTER(listhd, vip, marker, vi_lrulist); 595 if (!vnis_marker(VIMPL_TO_VNODE(vip))) 596 break; 597 } 598 599 return vip; 600 } 601 602 static void 603 lru_iter_release(lru_iter_t *iter) 604 { 605 vnode_impl_t *marker; 606 607 KASSERT(mutex_owned(&vdrain_lock)); 608 609 marker = iter->li_marker; 610 TAILQ_REMOVE(marker->vi_lrulisthd, marker, vi_lrulist); 611 612 mutex_exit(&vdrain_lock); 613 vnfree_marker(VIMPL_TO_VNODE(marker)); 614 mutex_enter(&vdrain_lock); 615 } 616 617 /* 618 * Release deferred vrele vnodes for this mount. 619 * Called with file system suspended. 620 */ 621 void 622 vrele_flush(struct mount *mp) 623 { 624 lru_iter_t iter; 625 vnode_impl_t *vip; 626 627 KASSERT(fstrans_is_owner(mp)); 628 629 mutex_enter(&vdrain_lock); 630 for (vip = lru_iter_first(LRU_VRELE, &iter); vip != NULL; 631 vip = lru_iter_next(&iter)) { 632 if (VIMPL_TO_VNODE(vip)->v_mount != mp) 633 continue; 634 vrele_deferred(vip); 635 } 636 lru_iter_release(&iter); 637 mutex_exit(&vdrain_lock); 638 } 639 640 /* 641 * One pass through the LRU lists to keep the number of allocated 642 * vnodes below target. Returns true if target met. 643 */ 644 static bool 645 vdrain_one(u_int target) 646 { 647 int ix, lists[] = { LRU_FREE, LRU_HOLD }; 648 lru_iter_t iter; 649 vnode_impl_t *vip; 650 vnode_t *vp; 651 struct mount *mp; 652 653 KASSERT(mutex_owned(&vdrain_lock)); 654 655 for (ix = 0; ix < __arraycount(lists); ix++) { 656 for (vip = lru_iter_first(lists[ix], &iter); vip != NULL; 657 vip = lru_iter_next(&iter)) { 658 if (numvnodes < target) { 659 lru_iter_release(&iter); 660 return true; 661 } 662 663 vp = VIMPL_TO_VNODE(vip); 664 665 /* Probe usecount (unlocked). */ 666 if (vrefcnt(vp) > 0) 667 continue; 668 /* Try v_interlock -- we lock the wrong direction! */ 669 if (!mutex_tryenter(vp->v_interlock)) 670 continue; 671 /* Probe usecount and state. */ 672 if (vrefcnt(vp) > 0 || VSTATE_GET(vp) != VS_LOADED) { 673 mutex_exit(vp->v_interlock); 674 continue; 675 } 676 mutex_exit(&vdrain_lock); 677 678 mp = vp->v_mount; 679 if (fstrans_start_nowait(mp) != 0) { 680 mutex_exit(vp->v_interlock); 681 mutex_enter(&vdrain_lock); 682 continue; 683 } 684 685 if (vcache_vget(vp) == 0) { 686 if (!vrecycle(vp)) { 687 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 688 mutex_enter(vp->v_interlock); 689 vrelel(vp, 0, LK_EXCLUSIVE); 690 } 691 } 692 fstrans_done(mp); 693 694 mutex_enter(&vdrain_lock); 695 } 696 lru_iter_release(&iter); 697 } 698 699 return false; 700 } 701 702 /* 703 * threadpool task to keep the number of vnodes below desiredvnodes. 704 */ 705 static void 706 vdrain_task(struct threadpool_job *job) 707 { 708 709 mutex_enter(&vdrain_lock); 710 711 while (!vdrain_one(desiredvnodes - desiredvnodes / 16)) 712 kpause("vdrain", false, 1, &vdrain_lock); 713 714 threadpool_job_done(job); 715 mutex_exit(&vdrain_lock); 716 } 717 718 /* 719 * threadpool task to process asynchronous vrele. 720 */ 721 static void 722 vrele_task(struct threadpool_job *job) 723 { 724 int skipped; 725 lru_iter_t iter; 726 vnode_impl_t *vip; 727 struct mount *mp; 728 729 mutex_enter(&vdrain_lock); 730 while ((vip = lru_iter_first(LRU_VRELE, &iter)) != NULL) { 731 for (skipped = 0; vip != NULL; vip = lru_iter_next(&iter)) { 732 mp = VIMPL_TO_VNODE(vip)->v_mount; 733 if (fstrans_start_nowait(mp) == 0) { 734 vrele_deferred(vip); 735 fstrans_done(mp); 736 } else { 737 skipped++; 738 } 739 } 740 741 lru_iter_release(&iter); 742 if (skipped) { 743 kpause("vrele", false, MAX(1, mstohz(10)), 744 &vdrain_lock); 745 } 746 } 747 748 threadpool_job_done(job); 749 lru_iter_release(&iter); 750 mutex_exit(&vdrain_lock); 751 } 752 753 /* 754 * Try to drop reference on a vnode. Abort if we are releasing the 755 * last reference. Note: this _must_ succeed if not the last reference. 756 */ 757 static bool 758 vtryrele(vnode_t *vp) 759 { 760 u_int use, next; 761 762 membar_release(); 763 for (use = atomic_load_relaxed(&vp->v_usecount);; use = next) { 764 if (__predict_false((use & VUSECOUNT_MASK) == 1)) { 765 return false; 766 } 767 KASSERT((use & VUSECOUNT_MASK) > 1); 768 next = atomic_cas_uint(&vp->v_usecount, use, use - 1); 769 if (__predict_true(next == use)) { 770 return true; 771 } 772 } 773 } 774 775 /* 776 * vput: unlock and release the reference. 777 */ 778 void 779 vput(vnode_t *vp) 780 { 781 int lktype; 782 783 /* 784 * Do an unlocked check of the usecount. If it looks like we're not 785 * about to drop the last reference, then unlock the vnode and try 786 * to drop the reference. If it ends up being the last reference 787 * after all, vrelel() can fix it all up. Most of the time this 788 * will all go to plan. 789 */ 790 if (vrefcnt(vp) > 1) { 791 VOP_UNLOCK(vp); 792 if (vtryrele(vp)) { 793 return; 794 } 795 lktype = LK_NONE; 796 } else { 797 lktype = VOP_ISLOCKED(vp); 798 KASSERT(lktype != LK_NONE); 799 } 800 mutex_enter(vp->v_interlock); 801 vrelel(vp, 0, lktype); 802 } 803 804 /* 805 * Release a vnode from the deferred list. 806 */ 807 static void 808 vrele_deferred(vnode_impl_t *vip) 809 { 810 vnode_t *vp; 811 812 KASSERT(mutex_owned(&vdrain_lock)); 813 KASSERT(vip->vi_lrulisthd == &lru_list[LRU_VRELE]); 814 815 vp = VIMPL_TO_VNODE(vip); 816 817 /* 818 * First remove the vnode from the vrele list. 819 * Put it on the last lru list, the last vrele() 820 * will put it back onto the right list before 821 * its usecount reaches zero. 822 */ 823 TAILQ_REMOVE(vip->vi_lrulisthd, vip, vi_lrulist); 824 vip->vi_lrulisthd = &lru_list[LRU_HOLD]; 825 vip->vi_lrulisttm = getticks(); 826 TAILQ_INSERT_TAIL(vip->vi_lrulisthd, vip, vi_lrulist); 827 828 mutex_exit(&vdrain_lock); 829 830 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 831 mutex_enter(vp->v_interlock); 832 vrelel(vp, 0, LK_EXCLUSIVE); 833 834 mutex_enter(&vdrain_lock); 835 } 836 837 /* 838 * Vnode release. If reference count drops to zero, call inactive 839 * routine and either return to freelist or free to the pool. 840 */ 841 static void 842 vrelel(vnode_t *vp, int flags, int lktype) 843 { 844 const bool async = ((flags & VRELEL_ASYNC) != 0); 845 bool recycle, defer, objlock_held; 846 u_int use, next; 847 int error; 848 849 objlock_held = false; 850 851 retry: 852 KASSERT(mutex_owned(vp->v_interlock)); 853 854 if (__predict_false(vp->v_op == dead_vnodeop_p && 855 VSTATE_GET(vp) != VS_RECLAIMED)) { 856 vnpanic(vp, "dead but not clean"); 857 } 858 859 /* 860 * If not the last reference, just unlock and drop the reference count. 861 * 862 * Otherwise make sure we pass a point in time where we hold the 863 * last reference with VGET flag unset. 864 */ 865 for (use = atomic_load_relaxed(&vp->v_usecount);; use = next) { 866 if (__predict_false((use & VUSECOUNT_MASK) > 1)) { 867 if (objlock_held) { 868 objlock_held = false; 869 rw_exit(vp->v_uobj.vmobjlock); 870 } 871 if (lktype != LK_NONE) { 872 mutex_exit(vp->v_interlock); 873 lktype = LK_NONE; 874 VOP_UNLOCK(vp); 875 mutex_enter(vp->v_interlock); 876 } 877 if (vtryrele(vp)) { 878 mutex_exit(vp->v_interlock); 879 return; 880 } 881 next = atomic_load_relaxed(&vp->v_usecount); 882 continue; 883 } 884 KASSERT((use & VUSECOUNT_MASK) == 1); 885 next = use & ~VUSECOUNT_VGET; 886 if (next != use) { 887 next = atomic_cas_uint(&vp->v_usecount, use, next); 888 } 889 if (__predict_true(next == use)) { 890 break; 891 } 892 } 893 membar_acquire(); 894 if (vrefcnt(vp) <= 0 || vp->v_writecount != 0) { 895 vnpanic(vp, "%s: bad ref count", __func__); 896 } 897 898 #ifdef DIAGNOSTIC 899 if ((vp->v_type == VBLK || vp->v_type == VCHR) && 900 vp->v_specnode != NULL && vp->v_specnode->sn_opencnt != 0) { 901 vprint("vrelel: missing VOP_CLOSE()", vp); 902 } 903 #endif 904 905 /* 906 * If already clean there is no need to lock, defer or 907 * deactivate this node. 908 */ 909 if (VSTATE_GET(vp) == VS_RECLAIMED) { 910 if (objlock_held) { 911 objlock_held = false; 912 rw_exit(vp->v_uobj.vmobjlock); 913 } 914 if (lktype != LK_NONE) { 915 mutex_exit(vp->v_interlock); 916 lktype = LK_NONE; 917 VOP_UNLOCK(vp); 918 mutex_enter(vp->v_interlock); 919 } 920 goto out; 921 } 922 923 /* 924 * First try to get the vnode locked for VOP_INACTIVE(). 925 * Defer vnode release to vrele task if caller requests 926 * it explicitly, is the pagedaemon or the lock failed. 927 */ 928 defer = false; 929 if (uvm_lwp_is_pagedaemon(curlwp) || async) { 930 defer = true; 931 } else if (lktype == LK_SHARED) { 932 /* Excellent chance of getting, if the last ref. */ 933 error = vn_lock(vp, LK_UPGRADE | LK_RETRY | LK_NOWAIT); 934 if (error != 0) { 935 defer = true; 936 } else { 937 lktype = LK_EXCLUSIVE; 938 } 939 } else if (lktype == LK_NONE) { 940 /* Excellent chance of getting, if the last ref. */ 941 error = vn_lock(vp, LK_EXCLUSIVE | LK_RETRY | LK_NOWAIT); 942 if (error != 0) { 943 defer = true; 944 } else { 945 lktype = LK_EXCLUSIVE; 946 } 947 } 948 KASSERT(mutex_owned(vp->v_interlock)); 949 if (defer) { 950 /* 951 * Defer reclaim to the vrele task; it's not safe to 952 * clean it here. We donate it our last reference. 953 */ 954 if (lktype != LK_NONE) { 955 mutex_exit(vp->v_interlock); 956 VOP_UNLOCK(vp); 957 mutex_enter(vp->v_interlock); 958 } 959 lru_requeue(vp, &lru_list[LRU_VRELE]); 960 mutex_exit(vp->v_interlock); 961 return; 962 } 963 KASSERT(lktype == LK_EXCLUSIVE); 964 965 /* If the node gained another reference, retry. */ 966 use = atomic_load_relaxed(&vp->v_usecount); 967 if ((use & VUSECOUNT_VGET) != 0) { 968 goto retry; 969 } 970 KASSERT((use & VUSECOUNT_MASK) == 1); 971 972 if ((vp->v_iflag & (VI_TEXT|VI_EXECMAP|VI_WRMAP)) != 0 || 973 (vp->v_vflag & VV_MAPPED) != 0) { 974 /* Take care of space accounting. */ 975 if (!objlock_held) { 976 objlock_held = true; 977 if (!rw_tryenter(vp->v_uobj.vmobjlock, RW_WRITER)) { 978 mutex_exit(vp->v_interlock); 979 rw_enter(vp->v_uobj.vmobjlock, RW_WRITER); 980 mutex_enter(vp->v_interlock); 981 goto retry; 982 } 983 } 984 if ((vp->v_iflag & VI_EXECMAP) != 0) { 985 cpu_count(CPU_COUNT_EXECPAGES, -vp->v_uobj.uo_npages); 986 } 987 vp->v_iflag &= ~(VI_TEXT|VI_EXECMAP|VI_WRMAP); 988 vp->v_vflag &= ~VV_MAPPED; 989 } 990 if (objlock_held) { 991 objlock_held = false; 992 rw_exit(vp->v_uobj.vmobjlock); 993 } 994 995 /* 996 * Deactivate the vnode, but preserve our reference across 997 * the call to VOP_INACTIVE(). 998 * 999 * If VOP_INACTIVE() indicates that the file has been 1000 * deleted, then recycle the vnode. 1001 * 1002 * Note that VOP_INACTIVE() will not drop the vnode lock. 1003 */ 1004 mutex_exit(vp->v_interlock); 1005 recycle = false; 1006 VOP_INACTIVE(vp, &recycle); 1007 if (!recycle) { 1008 lktype = LK_NONE; 1009 VOP_UNLOCK(vp); 1010 } 1011 mutex_enter(vp->v_interlock); 1012 1013 /* 1014 * Block new references then check again to see if a 1015 * new reference was acquired in the meantime. If 1016 * it was, restore the vnode state and try again. 1017 */ 1018 if (recycle) { 1019 VSTATE_CHANGE(vp, VS_LOADED, VS_BLOCKED); 1020 use = atomic_load_relaxed(&vp->v_usecount); 1021 if ((use & VUSECOUNT_VGET) != 0) { 1022 VSTATE_CHANGE(vp, VS_BLOCKED, VS_LOADED); 1023 goto retry; 1024 } 1025 KASSERT((use & VUSECOUNT_MASK) == 1); 1026 } 1027 1028 /* 1029 * Recycle the vnode if the file is now unused (unlinked). 1030 */ 1031 if (recycle) { 1032 VSTATE_ASSERT(vp, VS_BLOCKED); 1033 KASSERT(lktype == LK_EXCLUSIVE); 1034 /* vcache_reclaim drops the lock. */ 1035 lktype = LK_NONE; 1036 vcache_reclaim(vp); 1037 } 1038 KASSERT(vrefcnt(vp) > 0); 1039 KASSERT(lktype == LK_NONE); 1040 1041 out: 1042 for (use = atomic_load_relaxed(&vp->v_usecount);; use = next) { 1043 if (__predict_false((use & VUSECOUNT_VGET) != 0 && 1044 (use & VUSECOUNT_MASK) == 1)) { 1045 /* Gained and released another reference, retry. */ 1046 goto retry; 1047 } 1048 next = atomic_cas_uint(&vp->v_usecount, use, use - 1); 1049 if (__predict_true(next == use)) { 1050 if (__predict_false((use & VUSECOUNT_MASK) != 1)) { 1051 /* Gained another reference. */ 1052 mutex_exit(vp->v_interlock); 1053 return; 1054 } 1055 break; 1056 } 1057 } 1058 membar_acquire(); 1059 1060 if (VSTATE_GET(vp) == VS_RECLAIMED && vp->v_holdcnt == 0) { 1061 /* 1062 * It's clean so destroy it. It isn't referenced 1063 * anywhere since it has been reclaimed. 1064 */ 1065 vcache_free(VNODE_TO_VIMPL(vp)); 1066 } else { 1067 /* 1068 * Otherwise, put it back onto the freelist. It 1069 * can't be destroyed while still associated with 1070 * a file system. 1071 */ 1072 lru_requeue(vp, lru_which(vp)); 1073 mutex_exit(vp->v_interlock); 1074 } 1075 } 1076 1077 void 1078 vrele(vnode_t *vp) 1079 { 1080 1081 if (vtryrele(vp)) { 1082 return; 1083 } 1084 mutex_enter(vp->v_interlock); 1085 vrelel(vp, 0, LK_NONE); 1086 } 1087 1088 /* 1089 * Asynchronous vnode release, vnode is released in different context. 1090 */ 1091 void 1092 vrele_async(vnode_t *vp) 1093 { 1094 1095 if (vtryrele(vp)) { 1096 return; 1097 } 1098 mutex_enter(vp->v_interlock); 1099 vrelel(vp, VRELEL_ASYNC, LK_NONE); 1100 } 1101 1102 /* 1103 * Vnode reference, where a reference is already held by some other 1104 * object (for example, a file structure). 1105 * 1106 * NB: lockless code sequences may rely on this not blocking. 1107 */ 1108 void 1109 vref(vnode_t *vp) 1110 { 1111 1112 KASSERT(vrefcnt(vp) > 0); 1113 1114 atomic_inc_uint(&vp->v_usecount); 1115 } 1116 1117 /* 1118 * Page or buffer structure gets a reference. 1119 * Called with v_interlock held. 1120 */ 1121 void 1122 vholdl(vnode_t *vp) 1123 { 1124 1125 KASSERT(mutex_owned(vp->v_interlock)); 1126 1127 if (vp->v_holdcnt++ == 0 && vrefcnt(vp) == 0) 1128 lru_requeue(vp, lru_which(vp)); 1129 } 1130 1131 /* 1132 * Page or buffer structure gets a reference. 1133 */ 1134 void 1135 vhold(vnode_t *vp) 1136 { 1137 1138 mutex_enter(vp->v_interlock); 1139 vholdl(vp); 1140 mutex_exit(vp->v_interlock); 1141 } 1142 1143 /* 1144 * Page or buffer structure frees a reference. 1145 * Called with v_interlock held. 1146 */ 1147 void 1148 holdrelel(vnode_t *vp) 1149 { 1150 1151 KASSERT(mutex_owned(vp->v_interlock)); 1152 1153 if (vp->v_holdcnt <= 0) { 1154 vnpanic(vp, "%s: holdcnt vp %p", __func__, vp); 1155 } 1156 1157 vp->v_holdcnt--; 1158 if (vp->v_holdcnt == 0 && vrefcnt(vp) == 0) 1159 lru_requeue(vp, lru_which(vp)); 1160 } 1161 1162 /* 1163 * Page or buffer structure frees a reference. 1164 */ 1165 void 1166 holdrele(vnode_t *vp) 1167 { 1168 1169 mutex_enter(vp->v_interlock); 1170 holdrelel(vp); 1171 mutex_exit(vp->v_interlock); 1172 } 1173 1174 /* 1175 * Recycle an unused vnode if caller holds the last reference. 1176 */ 1177 bool 1178 vrecycle(vnode_t *vp) 1179 { 1180 int error __diagused; 1181 1182 mutex_enter(vp->v_interlock); 1183 1184 /* If the vnode is already clean we're done. */ 1185 VSTATE_WAIT_STABLE(vp); 1186 if (VSTATE_GET(vp) != VS_LOADED) { 1187 VSTATE_ASSERT(vp, VS_RECLAIMED); 1188 vrelel(vp, 0, LK_NONE); 1189 return true; 1190 } 1191 1192 /* Prevent further references until the vnode is locked. */ 1193 VSTATE_CHANGE(vp, VS_LOADED, VS_BLOCKED); 1194 1195 /* Make sure we hold the last reference. */ 1196 if (vrefcnt(vp) != 1) { 1197 VSTATE_CHANGE(vp, VS_BLOCKED, VS_LOADED); 1198 mutex_exit(vp->v_interlock); 1199 return false; 1200 } 1201 1202 mutex_exit(vp->v_interlock); 1203 1204 /* 1205 * On a leaf file system this lock will always succeed as we hold 1206 * the last reference and prevent further references. 1207 * On layered file systems waiting for the lock would open a can of 1208 * deadlocks as the lower vnodes may have other active references. 1209 */ 1210 error = vn_lock(vp, LK_EXCLUSIVE | LK_RETRY | LK_NOWAIT); 1211 1212 mutex_enter(vp->v_interlock); 1213 if (error) { 1214 VSTATE_CHANGE(vp, VS_BLOCKED, VS_LOADED); 1215 mutex_exit(vp->v_interlock); 1216 return false; 1217 } 1218 1219 KASSERT(vrefcnt(vp) == 1); 1220 vcache_reclaim(vp); 1221 vrelel(vp, 0, LK_NONE); 1222 1223 return true; 1224 } 1225 1226 /* 1227 * Helper for vrevoke() to propagate suspension from lastmp 1228 * to thismp. Both args may be NULL. 1229 * Returns the currently suspended file system or NULL. 1230 */ 1231 static struct mount * 1232 vrevoke_suspend_next(struct mount *lastmp, struct mount *thismp) 1233 { 1234 int error; 1235 1236 if (lastmp == thismp) 1237 return thismp; 1238 1239 if (lastmp != NULL) 1240 vfs_resume(lastmp); 1241 1242 if (thismp == NULL) 1243 return NULL; 1244 1245 do { 1246 error = vfs_suspend(thismp, 0); 1247 } while (error == EINTR || error == ERESTART); 1248 1249 if (error == 0) 1250 return thismp; 1251 1252 KASSERT(error == EOPNOTSUPP || error == ENOENT); 1253 return NULL; 1254 } 1255 1256 /* 1257 * Eliminate all activity associated with the requested vnode 1258 * and with all vnodes aliased to the requested vnode. 1259 */ 1260 void 1261 vrevoke(vnode_t *vp) 1262 { 1263 struct mount *mp; 1264 vnode_t *vq; 1265 enum vtype type; 1266 dev_t dev; 1267 1268 KASSERT(vrefcnt(vp) > 0); 1269 1270 mp = vrevoke_suspend_next(NULL, vp->v_mount); 1271 1272 mutex_enter(vp->v_interlock); 1273 VSTATE_WAIT_STABLE(vp); 1274 if (VSTATE_GET(vp) == VS_RECLAIMED) { 1275 mutex_exit(vp->v_interlock); 1276 } else if (vp->v_type != VBLK && vp->v_type != VCHR) { 1277 atomic_inc_uint(&vp->v_usecount); 1278 mutex_exit(vp->v_interlock); 1279 vgone(vp); 1280 } else { 1281 dev = vp->v_rdev; 1282 type = vp->v_type; 1283 mutex_exit(vp->v_interlock); 1284 1285 while (spec_node_lookup_by_dev(type, dev, VDEAD_NOWAIT, &vq) 1286 == 0) { 1287 mp = vrevoke_suspend_next(mp, vq->v_mount); 1288 vgone(vq); 1289 } 1290 } 1291 vrevoke_suspend_next(mp, NULL); 1292 } 1293 1294 /* 1295 * Eliminate all activity associated with a vnode in preparation for 1296 * reuse. Drops a reference from the vnode. 1297 */ 1298 void 1299 vgone(vnode_t *vp) 1300 { 1301 int lktype; 1302 1303 KASSERT(vp->v_mount == dead_rootmount || 1304 fstrans_is_owner(vp->v_mount)); 1305 1306 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 1307 lktype = LK_EXCLUSIVE; 1308 mutex_enter(vp->v_interlock); 1309 VSTATE_WAIT_STABLE(vp); 1310 if (VSTATE_GET(vp) == VS_LOADED) { 1311 VSTATE_CHANGE(vp, VS_LOADED, VS_BLOCKED); 1312 vcache_reclaim(vp); 1313 lktype = LK_NONE; 1314 } 1315 VSTATE_ASSERT(vp, VS_RECLAIMED); 1316 vrelel(vp, 0, lktype); 1317 } 1318 1319 static inline uint32_t 1320 vcache_hash(const struct vcache_key *key) 1321 { 1322 uint32_t hash = HASH32_BUF_INIT; 1323 1324 KASSERT(key->vk_key_len > 0); 1325 1326 hash = hash32_buf(&key->vk_mount, sizeof(struct mount *), hash); 1327 hash = hash32_buf(key->vk_key, key->vk_key_len, hash); 1328 return hash; 1329 } 1330 1331 static int 1332 vcache_stats(struct hashstat_sysctl *hs, bool fill) 1333 { 1334 vnode_impl_t *vip; 1335 uint64_t chain; 1336 1337 strlcpy(hs->hash_name, "vcache", sizeof(hs->hash_name)); 1338 strlcpy(hs->hash_desc, "vnode cache hash", sizeof(hs->hash_desc)); 1339 if (!fill) 1340 return 0; 1341 1342 hs->hash_size = vcache_hashmask + 1; 1343 1344 for (size_t i = 0; i < hs->hash_size; i++) { 1345 chain = 0; 1346 mutex_enter(&vcache_lock); 1347 SLIST_FOREACH(vip, &vcache_hashtab[i], vi_hash) { 1348 chain++; 1349 } 1350 mutex_exit(&vcache_lock); 1351 if (chain > 0) { 1352 hs->hash_used++; 1353 hs->hash_items += chain; 1354 if (chain > hs->hash_maxchain) 1355 hs->hash_maxchain = chain; 1356 } 1357 preempt_point(); 1358 } 1359 1360 return 0; 1361 } 1362 1363 static void 1364 vcache_init(void) 1365 { 1366 1367 vcache_pool = pool_cache_init(sizeof(vnode_impl_t), MAX(coherency_unit, 1368 alignof(vnode_impl_t)), 1369 0, 0, "vcachepl", NULL, IPL_NONE, NULL, NULL, NULL); 1370 KASSERT(vcache_pool != NULL); 1371 mutex_init(&vcache_lock, MUTEX_DEFAULT, IPL_NONE); 1372 cv_init(&vcache_cv, "vcache"); 1373 vcache_hashsize = desiredvnodes; 1374 vcache_hashtab = hashinit(desiredvnodes, HASH_SLIST, true, 1375 &vcache_hashmask); 1376 hashstat_register("vcache", vcache_stats); 1377 } 1378 1379 static void 1380 vcache_reinit(void) 1381 { 1382 int i; 1383 uint32_t hash; 1384 u_long oldmask, newmask; 1385 struct hashhead *oldtab, *newtab; 1386 vnode_impl_t *vip; 1387 1388 newtab = hashinit(desiredvnodes, HASH_SLIST, true, &newmask); 1389 mutex_enter(&vcache_lock); 1390 oldtab = vcache_hashtab; 1391 oldmask = vcache_hashmask; 1392 vcache_hashsize = desiredvnodes; 1393 vcache_hashtab = newtab; 1394 vcache_hashmask = newmask; 1395 for (i = 0; i <= oldmask; i++) { 1396 while ((vip = SLIST_FIRST(&oldtab[i])) != NULL) { 1397 SLIST_REMOVE(&oldtab[i], vip, vnode_impl, vi_hash); 1398 hash = vcache_hash(&vip->vi_key); 1399 SLIST_INSERT_HEAD(&newtab[hash & vcache_hashmask], 1400 vip, vi_hash); 1401 } 1402 } 1403 mutex_exit(&vcache_lock); 1404 hashdone(oldtab, HASH_SLIST, oldmask); 1405 } 1406 1407 static inline vnode_impl_t * 1408 vcache_hash_lookup(const struct vcache_key *key, uint32_t hash) 1409 { 1410 struct hashhead *hashp; 1411 vnode_impl_t *vip; 1412 1413 KASSERT(mutex_owned(&vcache_lock)); 1414 1415 hashp = &vcache_hashtab[hash & vcache_hashmask]; 1416 SLIST_FOREACH(vip, hashp, vi_hash) { 1417 if (key->vk_mount != vip->vi_key.vk_mount) 1418 continue; 1419 if (key->vk_key_len != vip->vi_key.vk_key_len) 1420 continue; 1421 if (memcmp(key->vk_key, vip->vi_key.vk_key, key->vk_key_len)) 1422 continue; 1423 return vip; 1424 } 1425 return NULL; 1426 } 1427 1428 /* 1429 * Allocate a new, uninitialized vcache node. 1430 */ 1431 static vnode_impl_t * 1432 vcache_alloc(void) 1433 { 1434 vnode_impl_t *vip; 1435 vnode_t *vp; 1436 1437 vip = pool_cache_get(vcache_pool, PR_WAITOK); 1438 vp = VIMPL_TO_VNODE(vip); 1439 memset(vip, 0, sizeof(*vip)); 1440 1441 rw_init(&vip->vi_lock); 1442 vp->v_interlock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE); 1443 1444 uvm_obj_init(&vp->v_uobj, &uvm_vnodeops, true, 1); 1445 klist_init(&vip->vi_klist.vk_klist); 1446 vp->v_klist = &vip->vi_klist; 1447 cv_init(&vp->v_cv, "vnode"); 1448 cache_vnode_init(vp); 1449 1450 vp->v_usecount = 1; 1451 vp->v_type = VNON; 1452 vp->v_size = vp->v_writesize = VSIZENOTSET; 1453 1454 vip->vi_state = VS_LOADING; 1455 1456 lru_requeue(vp, &lru_list[LRU_FREE]); 1457 1458 return vip; 1459 } 1460 1461 /* 1462 * Deallocate a vcache node in state VS_LOADING. 1463 * 1464 * vcache_lock held on entry and released on return. 1465 */ 1466 static void 1467 vcache_dealloc(vnode_impl_t *vip) 1468 { 1469 vnode_t *vp; 1470 1471 KASSERT(mutex_owned(&vcache_lock)); 1472 1473 vp = VIMPL_TO_VNODE(vip); 1474 vfs_ref(dead_rootmount); 1475 vfs_insmntque(vp, dead_rootmount); 1476 mutex_enter(vp->v_interlock); 1477 vp->v_op = dead_vnodeop_p; 1478 VSTATE_CHANGE(vp, VS_LOADING, VS_RECLAIMED); 1479 mutex_exit(&vcache_lock); 1480 vrelel(vp, 0, LK_NONE); 1481 } 1482 1483 /* 1484 * Free an unused, unreferenced vcache node. 1485 * v_interlock locked on entry. 1486 */ 1487 static void 1488 vcache_free(vnode_impl_t *vip) 1489 { 1490 vnode_t *vp; 1491 1492 vp = VIMPL_TO_VNODE(vip); 1493 KASSERT(mutex_owned(vp->v_interlock)); 1494 1495 KASSERT(vrefcnt(vp) == 0); 1496 KASSERT(vp->v_holdcnt == 0); 1497 KASSERT(vp->v_writecount == 0); 1498 lru_requeue(vp, NULL); 1499 mutex_exit(vp->v_interlock); 1500 1501 vfs_insmntque(vp, NULL); 1502 if (vp->v_type == VBLK || vp->v_type == VCHR) 1503 spec_node_destroy(vp); 1504 1505 mutex_obj_free(vp->v_interlock); 1506 rw_destroy(&vip->vi_lock); 1507 uvm_obj_destroy(&vp->v_uobj, true); 1508 KASSERT(vp->v_klist == &vip->vi_klist); 1509 klist_fini(&vip->vi_klist.vk_klist); 1510 cv_destroy(&vp->v_cv); 1511 cache_vnode_fini(vp); 1512 pool_cache_put(vcache_pool, vip); 1513 } 1514 1515 /* 1516 * Try to get an initial reference on this cached vnode. 1517 * Returns zero on success or EBUSY if the vnode state is not LOADED. 1518 * 1519 * NB: lockless code sequences may rely on this not blocking. 1520 */ 1521 int 1522 vcache_tryvget(vnode_t *vp) 1523 { 1524 u_int use, next; 1525 1526 for (use = atomic_load_relaxed(&vp->v_usecount);; use = next) { 1527 if (__predict_false((use & VUSECOUNT_GATE) == 0)) { 1528 return SET_ERROR(EBUSY); 1529 } 1530 next = atomic_cas_uint(&vp->v_usecount, 1531 use, (use + 1) | VUSECOUNT_VGET); 1532 if (__predict_true(next == use)) { 1533 membar_acquire(); 1534 return 0; 1535 } 1536 } 1537 } 1538 1539 /* 1540 * Try to get an initial reference on this cached vnode. 1541 * Returns zero on success and ENOENT if the vnode has been reclaimed. 1542 * Will wait for the vnode state to be stable. 1543 * 1544 * v_interlock locked on entry and unlocked on exit. 1545 */ 1546 int 1547 vcache_vget(vnode_t *vp) 1548 { 1549 int error; 1550 1551 KASSERT(mutex_owned(vp->v_interlock)); 1552 1553 /* Increment hold count to prevent vnode from disappearing. */ 1554 vp->v_holdcnt++; 1555 VSTATE_WAIT_STABLE(vp); 1556 vp->v_holdcnt--; 1557 1558 /* If this was the last reference to a reclaimed vnode free it now. */ 1559 if (__predict_false(VSTATE_GET(vp) == VS_RECLAIMED)) { 1560 if (vp->v_holdcnt == 0 && vrefcnt(vp) == 0) 1561 vcache_free(VNODE_TO_VIMPL(vp)); 1562 else 1563 mutex_exit(vp->v_interlock); 1564 return SET_ERROR(ENOENT); 1565 } 1566 VSTATE_ASSERT(vp, VS_LOADED); 1567 error = vcache_tryvget(vp); 1568 KASSERT(error == 0); 1569 mutex_exit(vp->v_interlock); 1570 1571 return 0; 1572 } 1573 1574 /* 1575 * Get a vnode / fs node pair by key and return it referenced through vpp. 1576 */ 1577 int 1578 vcache_get(struct mount *mp, const void *key, size_t key_len, 1579 struct vnode **vpp) 1580 { 1581 int error; 1582 uint32_t hash; 1583 const void *new_key; 1584 struct vnode *vp; 1585 struct vcache_key vcache_key; 1586 vnode_impl_t *vip, *new_vip; 1587 1588 new_key = NULL; 1589 *vpp = NULL; 1590 1591 vcache_key.vk_mount = mp; 1592 vcache_key.vk_key = key; 1593 vcache_key.vk_key_len = key_len; 1594 hash = vcache_hash(&vcache_key); 1595 1596 again: 1597 mutex_enter(&vcache_lock); 1598 vip = vcache_hash_lookup(&vcache_key, hash); 1599 1600 /* If found, take a reference or retry. */ 1601 if (__predict_true(vip != NULL)) { 1602 /* 1603 * If the vnode is loading we cannot take the v_interlock 1604 * here as it might change during load (see uvm_obj_setlock()). 1605 * As changing state from VS_LOADING requires both vcache_lock 1606 * and v_interlock it is safe to test with vcache_lock held. 1607 * 1608 * Wait for vnodes changing state from VS_LOADING and retry. 1609 */ 1610 if (__predict_false(vip->vi_state == VS_LOADING)) { 1611 cv_wait(&vcache_cv, &vcache_lock); 1612 mutex_exit(&vcache_lock); 1613 goto again; 1614 } 1615 vp = VIMPL_TO_VNODE(vip); 1616 mutex_enter(vp->v_interlock); 1617 mutex_exit(&vcache_lock); 1618 error = vcache_vget(vp); 1619 if (error == ENOENT) 1620 goto again; 1621 if (error == 0) 1622 *vpp = vp; 1623 KASSERT((error != 0) == (*vpp == NULL)); 1624 return error; 1625 } 1626 mutex_exit(&vcache_lock); 1627 1628 /* Allocate and initialize a new vcache / vnode pair. */ 1629 error = vfs_busy(mp); 1630 if (error) 1631 return error; 1632 new_vip = vcache_alloc(); 1633 new_vip->vi_key = vcache_key; 1634 vp = VIMPL_TO_VNODE(new_vip); 1635 mutex_enter(&vcache_lock); 1636 vip = vcache_hash_lookup(&vcache_key, hash); 1637 if (vip == NULL) { 1638 SLIST_INSERT_HEAD(&vcache_hashtab[hash & vcache_hashmask], 1639 new_vip, vi_hash); 1640 vip = new_vip; 1641 } 1642 1643 /* If another thread beat us inserting this node, retry. */ 1644 if (vip != new_vip) { 1645 vcache_dealloc(new_vip); 1646 vfs_unbusy(mp); 1647 goto again; 1648 } 1649 mutex_exit(&vcache_lock); 1650 1651 /* Load the fs node. Exclusive as new_node is VS_LOADING. */ 1652 error = VFS_LOADVNODE(mp, vp, key, key_len, &new_key); 1653 if (error) { 1654 mutex_enter(&vcache_lock); 1655 SLIST_REMOVE(&vcache_hashtab[hash & vcache_hashmask], 1656 new_vip, vnode_impl, vi_hash); 1657 vcache_dealloc(new_vip); 1658 vfs_unbusy(mp); 1659 KASSERT(*vpp == NULL); 1660 return error; 1661 } 1662 KASSERT(new_key != NULL); 1663 KASSERT(memcmp(key, new_key, key_len) == 0); 1664 KASSERT(vp->v_op != NULL); 1665 vfs_insmntque(vp, mp); 1666 if ((mp->mnt_iflag & IMNT_MPSAFE) != 0) 1667 vp->v_vflag |= VV_MPSAFE; 1668 vfs_ref(mp); 1669 vfs_unbusy(mp); 1670 1671 /* Finished loading, finalize node. */ 1672 mutex_enter(&vcache_lock); 1673 new_vip->vi_key.vk_key = new_key; 1674 mutex_enter(vp->v_interlock); 1675 VSTATE_CHANGE(vp, VS_LOADING, VS_LOADED); 1676 mutex_exit(vp->v_interlock); 1677 mutex_exit(&vcache_lock); 1678 *vpp = vp; 1679 return 0; 1680 } 1681 1682 /* 1683 * Create a new vnode / fs node pair and return it referenced through vpp. 1684 */ 1685 int 1686 vcache_new(struct mount *mp, struct vnode *dvp, struct vattr *vap, 1687 kauth_cred_t cred, void *extra, struct vnode **vpp) 1688 { 1689 int error; 1690 uint32_t hash; 1691 struct vnode *vp, *ovp; 1692 vnode_impl_t *vip, *ovip; 1693 1694 *vpp = NULL; 1695 1696 /* Allocate and initialize a new vcache / vnode pair. */ 1697 error = vfs_busy(mp); 1698 if (error) 1699 return error; 1700 vip = vcache_alloc(); 1701 vip->vi_key.vk_mount = mp; 1702 vp = VIMPL_TO_VNODE(vip); 1703 1704 /* Create and load the fs node. */ 1705 error = VFS_NEWVNODE(mp, dvp, vp, vap, cred, extra, 1706 &vip->vi_key.vk_key_len, &vip->vi_key.vk_key); 1707 if (error) { 1708 mutex_enter(&vcache_lock); 1709 vcache_dealloc(vip); 1710 vfs_unbusy(mp); 1711 KASSERT(*vpp == NULL); 1712 return error; 1713 } 1714 KASSERT(vp->v_op != NULL); 1715 KASSERT((vip->vi_key.vk_key_len == 0) == (mp == dead_rootmount)); 1716 if (vip->vi_key.vk_key_len > 0) { 1717 KASSERT(vip->vi_key.vk_key != NULL); 1718 hash = vcache_hash(&vip->vi_key); 1719 1720 /* 1721 * Wait for previous instance to be reclaimed, 1722 * then insert new node. 1723 */ 1724 mutex_enter(&vcache_lock); 1725 while ((ovip = vcache_hash_lookup(&vip->vi_key, hash))) { 1726 ovp = VIMPL_TO_VNODE(ovip); 1727 mutex_enter(ovp->v_interlock); 1728 mutex_exit(&vcache_lock); 1729 error = vcache_vget(ovp); 1730 KASSERT(error == ENOENT); 1731 mutex_enter(&vcache_lock); 1732 } 1733 SLIST_INSERT_HEAD(&vcache_hashtab[hash & vcache_hashmask], 1734 vip, vi_hash); 1735 mutex_exit(&vcache_lock); 1736 } 1737 vfs_insmntque(vp, mp); 1738 if ((mp->mnt_iflag & IMNT_MPSAFE) != 0) 1739 vp->v_vflag |= VV_MPSAFE; 1740 vfs_ref(mp); 1741 vfs_unbusy(mp); 1742 1743 /* Finished loading, finalize node. */ 1744 mutex_enter(&vcache_lock); 1745 mutex_enter(vp->v_interlock); 1746 VSTATE_CHANGE(vp, VS_LOADING, VS_LOADED); 1747 mutex_exit(&vcache_lock); 1748 mutex_exit(vp->v_interlock); 1749 *vpp = vp; 1750 return 0; 1751 } 1752 1753 /* 1754 * Prepare key change: update old cache nodes key and lock new cache node. 1755 * Return an error if the new node already exists. 1756 */ 1757 int 1758 vcache_rekey_enter(struct mount *mp, struct vnode *vp, 1759 const void *old_key, size_t old_key_len, 1760 const void *new_key, size_t new_key_len) 1761 { 1762 uint32_t old_hash, new_hash; 1763 struct vcache_key old_vcache_key, new_vcache_key; 1764 vnode_impl_t *vip, *new_vip; 1765 1766 old_vcache_key.vk_mount = mp; 1767 old_vcache_key.vk_key = old_key; 1768 old_vcache_key.vk_key_len = old_key_len; 1769 old_hash = vcache_hash(&old_vcache_key); 1770 1771 new_vcache_key.vk_mount = mp; 1772 new_vcache_key.vk_key = new_key; 1773 new_vcache_key.vk_key_len = new_key_len; 1774 new_hash = vcache_hash(&new_vcache_key); 1775 1776 new_vip = vcache_alloc(); 1777 new_vip->vi_key = new_vcache_key; 1778 1779 /* Insert locked new node used as placeholder. */ 1780 mutex_enter(&vcache_lock); 1781 vip = vcache_hash_lookup(&new_vcache_key, new_hash); 1782 if (vip != NULL) { 1783 vcache_dealloc(new_vip); 1784 return SET_ERROR(EEXIST); 1785 } 1786 SLIST_INSERT_HEAD(&vcache_hashtab[new_hash & vcache_hashmask], 1787 new_vip, vi_hash); 1788 1789 /* Replace old nodes key with the temporary copy. */ 1790 vip = vcache_hash_lookup(&old_vcache_key, old_hash); 1791 KASSERT(vip != NULL); 1792 KASSERT(VIMPL_TO_VNODE(vip) == vp); 1793 KASSERT(vip->vi_key.vk_key != old_vcache_key.vk_key); 1794 vip->vi_key = old_vcache_key; 1795 mutex_exit(&vcache_lock); 1796 return 0; 1797 } 1798 1799 /* 1800 * Key change complete: update old node and remove placeholder. 1801 */ 1802 void 1803 vcache_rekey_exit(struct mount *mp, struct vnode *vp, 1804 const void *old_key, size_t old_key_len, 1805 const void *new_key, size_t new_key_len) 1806 { 1807 uint32_t old_hash, new_hash; 1808 struct vcache_key old_vcache_key, new_vcache_key; 1809 vnode_impl_t *vip, *new_vip; 1810 struct vnode *new_vp; 1811 1812 old_vcache_key.vk_mount = mp; 1813 old_vcache_key.vk_key = old_key; 1814 old_vcache_key.vk_key_len = old_key_len; 1815 old_hash = vcache_hash(&old_vcache_key); 1816 1817 new_vcache_key.vk_mount = mp; 1818 new_vcache_key.vk_key = new_key; 1819 new_vcache_key.vk_key_len = new_key_len; 1820 new_hash = vcache_hash(&new_vcache_key); 1821 1822 mutex_enter(&vcache_lock); 1823 1824 /* Lookup old and new node. */ 1825 vip = vcache_hash_lookup(&old_vcache_key, old_hash); 1826 KASSERT(vip != NULL); 1827 KASSERT(VIMPL_TO_VNODE(vip) == vp); 1828 1829 new_vip = vcache_hash_lookup(&new_vcache_key, new_hash); 1830 KASSERT(new_vip != NULL); 1831 KASSERT(new_vip->vi_key.vk_key_len == new_key_len); 1832 new_vp = VIMPL_TO_VNODE(new_vip); 1833 mutex_enter(new_vp->v_interlock); 1834 VSTATE_ASSERT(VIMPL_TO_VNODE(new_vip), VS_LOADING); 1835 mutex_exit(new_vp->v_interlock); 1836 1837 /* Rekey old node and put it onto its new hashlist. */ 1838 vip->vi_key = new_vcache_key; 1839 if (old_hash != new_hash) { 1840 SLIST_REMOVE(&vcache_hashtab[old_hash & vcache_hashmask], 1841 vip, vnode_impl, vi_hash); 1842 SLIST_INSERT_HEAD(&vcache_hashtab[new_hash & vcache_hashmask], 1843 vip, vi_hash); 1844 } 1845 1846 /* Remove new node used as placeholder. */ 1847 SLIST_REMOVE(&vcache_hashtab[new_hash & vcache_hashmask], 1848 new_vip, vnode_impl, vi_hash); 1849 vcache_dealloc(new_vip); 1850 } 1851 1852 /* 1853 * Disassociate the underlying file system from a vnode. 1854 * 1855 * Must be called with vnode locked and will return unlocked. 1856 * Must be called with the interlock held, and will return with it held. 1857 */ 1858 static void 1859 vcache_reclaim(vnode_t *vp) 1860 { 1861 lwp_t *l = curlwp; 1862 vnode_impl_t *vip = VNODE_TO_VIMPL(vp); 1863 struct mount *mp = vp->v_mount; 1864 uint32_t hash; 1865 uint8_t temp_buf[64], *temp_key; 1866 size_t temp_key_len; 1867 bool recycle; 1868 int error; 1869 1870 KASSERT(VOP_ISLOCKED(vp) == LK_EXCLUSIVE); 1871 KASSERT(mutex_owned(vp->v_interlock)); 1872 KASSERT(vrefcnt(vp) != 0); 1873 1874 temp_key_len = vip->vi_key.vk_key_len; 1875 /* 1876 * Prevent the vnode from being recycled or brought into use 1877 * while we clean it out. 1878 */ 1879 VSTATE_CHANGE(vp, VS_BLOCKED, VS_RECLAIMING); 1880 1881 /* 1882 * Send NOTE_REVOKE now, before we call VOP_RECLAIM(), 1883 * because VOP_RECLAIM() could cause vp->v_klist to 1884 * become invalid. Don't check for interest in NOTE_REVOKE 1885 * here; it's always posted because it sets EV_EOF. 1886 * 1887 * Once it's been posted, reset vp->v_klist to point to 1888 * our own local storage, in case we were sharing with 1889 * someone else. 1890 */ 1891 KNOTE(&vp->v_klist->vk_klist, NOTE_REVOKE); 1892 vp->v_klist = &vip->vi_klist; 1893 mutex_exit(vp->v_interlock); 1894 1895 rw_enter(vp->v_uobj.vmobjlock, RW_WRITER); 1896 mutex_enter(vp->v_interlock); 1897 if ((vp->v_iflag & VI_EXECMAP) != 0) { 1898 cpu_count(CPU_COUNT_EXECPAGES, -vp->v_uobj.uo_npages); 1899 } 1900 vp->v_iflag &= ~(VI_TEXT|VI_EXECMAP); 1901 vp->v_iflag |= VI_DEADCHECK; /* for genfs_getpages() */ 1902 mutex_exit(vp->v_interlock); 1903 rw_exit(vp->v_uobj.vmobjlock); 1904 1905 /* 1906 * With vnode state set to reclaiming, purge name cache immediately 1907 * to prevent new handles on vnode, and wait for existing threads 1908 * trying to get a handle to notice VS_RECLAIMED status and abort. 1909 */ 1910 cache_purge(vp); 1911 1912 /* Replace the vnode key with a temporary copy. */ 1913 if (vip->vi_key.vk_key_len > sizeof(temp_buf)) { 1914 temp_key = kmem_alloc(temp_key_len, KM_SLEEP); 1915 } else { 1916 temp_key = temp_buf; 1917 } 1918 if (vip->vi_key.vk_key_len > 0) { 1919 mutex_enter(&vcache_lock); 1920 memcpy(temp_key, vip->vi_key.vk_key, temp_key_len); 1921 vip->vi_key.vk_key = temp_key; 1922 mutex_exit(&vcache_lock); 1923 } 1924 1925 fstrans_start(mp); 1926 1927 /* 1928 * Clean out any cached data associated with the vnode. 1929 */ 1930 error = vinvalbuf(vp, V_SAVE, NOCRED, l, 0, 0); 1931 if (error != 0) { 1932 if (wapbl_vphaswapbl(vp)) 1933 WAPBL_DISCARD(wapbl_vptomp(vp)); 1934 error = vinvalbuf(vp, 0, NOCRED, l, 0, 0); 1935 } 1936 KASSERTMSG((error == 0), "vinvalbuf failed: %d", error); 1937 KASSERT((vp->v_iflag & VI_ONWORKLST) == 0); 1938 if (vp->v_type == VBLK || vp->v_type == VCHR) { 1939 spec_node_revoke(vp); 1940 } 1941 1942 /* 1943 * Disassociate the underlying file system from the vnode. 1944 * VOP_INACTIVE leaves the vnode locked; VOP_RECLAIM unlocks 1945 * the vnode, and may destroy the vnode so that VOP_UNLOCK 1946 * would no longer function. 1947 */ 1948 VOP_INACTIVE(vp, &recycle); 1949 KASSERT(VOP_ISLOCKED(vp) == LK_EXCLUSIVE); 1950 if (VOP_RECLAIM(vp)) { 1951 vnpanic(vp, "%s: cannot reclaim", __func__); 1952 } 1953 1954 KASSERT(vp->v_data == NULL); 1955 KASSERT((vp->v_iflag & VI_PAGES) == 0); 1956 1957 if (vp->v_type == VREG && vp->v_ractx != NULL) { 1958 uvm_ra_freectx(vp->v_ractx); 1959 vp->v_ractx = NULL; 1960 } 1961 1962 if (vip->vi_key.vk_key_len > 0) { 1963 /* Remove from vnode cache. */ 1964 hash = vcache_hash(&vip->vi_key); 1965 mutex_enter(&vcache_lock); 1966 KASSERT(vip == vcache_hash_lookup(&vip->vi_key, hash)); 1967 SLIST_REMOVE(&vcache_hashtab[hash & vcache_hashmask], 1968 vip, vnode_impl, vi_hash); 1969 mutex_exit(&vcache_lock); 1970 } 1971 if (temp_key != temp_buf) 1972 kmem_free(temp_key, temp_key_len); 1973 1974 /* Done with purge, notify sleepers of the grim news. */ 1975 mutex_enter(vp->v_interlock); 1976 vp->v_op = dead_vnodeop_p; 1977 VSTATE_CHANGE(vp, VS_RECLAIMING, VS_RECLAIMED); 1978 vp->v_tag = VT_NON; 1979 mutex_exit(vp->v_interlock); 1980 1981 /* 1982 * Move to dead mount. Must be after changing the operations 1983 * vector as vnode operations enter the mount before using the 1984 * operations vector. See sys/kern/vnode_if.c. 1985 */ 1986 vp->v_vflag &= ~VV_ROOT; 1987 vfs_ref(dead_rootmount); 1988 vfs_insmntque(vp, dead_rootmount); 1989 1990 #ifdef PAX_SEGVGUARD 1991 pax_segvguard_cleanup(vp); 1992 #endif /* PAX_SEGVGUARD */ 1993 1994 mutex_enter(vp->v_interlock); 1995 fstrans_done(mp); 1996 KASSERT((vp->v_iflag & VI_ONWORKLST) == 0); 1997 } 1998 1999 /* 2000 * Disassociate the underlying file system from an open device vnode 2001 * and make it anonymous. 2002 * 2003 * Vnode unlocked on entry, drops a reference to the vnode. 2004 */ 2005 void 2006 vcache_make_anon(vnode_t *vp) 2007 { 2008 vnode_impl_t *vip = VNODE_TO_VIMPL(vp); 2009 uint32_t hash; 2010 bool recycle; 2011 2012 KASSERT(vp->v_type == VBLK || vp->v_type == VCHR); 2013 KASSERT(vp->v_mount == dead_rootmount || 2014 fstrans_is_owner(vp->v_mount)); 2015 VSTATE_ASSERT_UNLOCKED(vp, VS_ACTIVE); 2016 2017 /* Remove from vnode cache. */ 2018 hash = vcache_hash(&vip->vi_key); 2019 mutex_enter(&vcache_lock); 2020 KASSERT(vip == vcache_hash_lookup(&vip->vi_key, hash)); 2021 SLIST_REMOVE(&vcache_hashtab[hash & vcache_hashmask], 2022 vip, vnode_impl, vi_hash); 2023 vip->vi_key.vk_mount = dead_rootmount; 2024 vip->vi_key.vk_key_len = 0; 2025 vip->vi_key.vk_key = NULL; 2026 mutex_exit(&vcache_lock); 2027 2028 /* 2029 * Disassociate the underlying file system from the vnode. 2030 * VOP_INACTIVE leaves the vnode locked; VOP_RECLAIM unlocks 2031 * the vnode, and may destroy the vnode so that VOP_UNLOCK 2032 * would no longer function. 2033 */ 2034 if (vn_lock(vp, LK_EXCLUSIVE)) { 2035 vnpanic(vp, "%s: cannot lock", __func__); 2036 } 2037 VOP_INACTIVE(vp, &recycle); 2038 KASSERT(VOP_ISLOCKED(vp) == LK_EXCLUSIVE); 2039 if (VOP_RECLAIM(vp)) { 2040 vnpanic(vp, "%s: cannot reclaim", __func__); 2041 } 2042 2043 /* Purge name cache. */ 2044 cache_purge(vp); 2045 2046 /* Done with purge, change operations vector. */ 2047 mutex_enter(vp->v_interlock); 2048 vp->v_op = spec_vnodeop_p; 2049 vp->v_vflag |= VV_MPSAFE; 2050 mutex_exit(vp->v_interlock); 2051 2052 /* 2053 * Move to dead mount. Must be after changing the operations 2054 * vector as vnode operations enter the mount before using the 2055 * operations vector. See sys/kern/vnode_if.c. 2056 */ 2057 vfs_ref(dead_rootmount); 2058 vfs_insmntque(vp, dead_rootmount); 2059 2060 vrele(vp); 2061 } 2062 2063 /* 2064 * Update outstanding I/O count and do wakeup if requested. 2065 */ 2066 void 2067 vwakeup(struct buf *bp) 2068 { 2069 vnode_t *vp; 2070 2071 if ((vp = bp->b_vp) == NULL) 2072 return; 2073 2074 KASSERT(bp->b_objlock == vp->v_interlock); 2075 KASSERT(mutex_owned(bp->b_objlock)); 2076 2077 if (--vp->v_numoutput < 0) 2078 vnpanic(vp, "%s: neg numoutput, vp %p", __func__, vp); 2079 if (vp->v_numoutput == 0) 2080 cv_broadcast(&vp->v_cv); 2081 } 2082 2083 /* 2084 * Test a vnode for being or becoming dead. Returns one of: 2085 * EBUSY: vnode is becoming dead, with "flags == VDEAD_NOWAIT" only. 2086 * ENOENT: vnode is dead. 2087 * 0: otherwise. 2088 * 2089 * Whenever this function returns a non-zero value all future 2090 * calls will also return a non-zero value. 2091 */ 2092 int 2093 vdead_check(struct vnode *vp, int flags) 2094 { 2095 2096 KASSERT(mutex_owned(vp->v_interlock)); 2097 2098 if (! ISSET(flags, VDEAD_NOWAIT)) 2099 VSTATE_WAIT_STABLE(vp); 2100 2101 if (VSTATE_GET(vp) == VS_RECLAIMING) { 2102 KASSERT(ISSET(flags, VDEAD_NOWAIT)); 2103 return SET_ERROR(EBUSY); 2104 } else if (VSTATE_GET(vp) == VS_RECLAIMED) { 2105 return SET_ERROR(ENOENT); 2106 } 2107 2108 return 0; 2109 } 2110 2111 int 2112 vfs_drainvnodes(void) 2113 { 2114 2115 mutex_enter(&vdrain_lock); 2116 2117 if (!vdrain_one(desiredvnodes)) { 2118 mutex_exit(&vdrain_lock); 2119 return SET_ERROR(EBUSY); 2120 } 2121 2122 mutex_exit(&vdrain_lock); 2123 2124 if (vcache_hashsize != desiredvnodes) 2125 vcache_reinit(); 2126 2127 return 0; 2128 } 2129 2130 void 2131 vnpanic(vnode_t *vp, const char *fmt, ...) 2132 { 2133 va_list ap; 2134 2135 #ifdef DIAGNOSTIC 2136 vprint(NULL, vp); 2137 #endif 2138 va_start(ap, fmt); 2139 vpanic(fmt, ap); 2140 va_end(ap); 2141 } 2142 2143 void 2144 vshareilock(vnode_t *tvp, vnode_t *fvp) 2145 { 2146 kmutex_t *oldlock; 2147 2148 oldlock = tvp->v_interlock; 2149 mutex_obj_hold(fvp->v_interlock); 2150 tvp->v_interlock = fvp->v_interlock; 2151 mutex_obj_free(oldlock); 2152 } 2153 2154 void 2155 vshareklist(vnode_t *tvp, vnode_t *fvp) 2156 { 2157 /* 2158 * If two vnodes share klist state, they must also share 2159 * an interlock. 2160 */ 2161 KASSERT(tvp->v_interlock == fvp->v_interlock); 2162 2163 /* 2164 * We make the following assumptions: 2165 * 2166 * ==> Some other synchronization is happening outside of 2167 * our view to make this safe. 2168 * 2169 * ==> That the "to" vnode will have the necessary references 2170 * on the "from" vnode so that the storage for the klist 2171 * won't be yanked out from beneath us (the vnode_impl). 2172 * 2173 * ==> If "from" is also sharing, we then assume that "from" 2174 * has the necessary references, and so on. 2175 */ 2176 tvp->v_klist = fvp->v_klist; 2177 } 2178
Indexes created Sun Oct 25 00:26:45 UTC 2026