Home | History | Annotate | Line # | Download | only in kern
vfs_vnode.c revision 1.105.2.3
      1 /*	$NetBSD: vfs_vnode.c,v 1.105.2.3 2020/01/17 21:55:13 ad Exp $	*/
      2 
      3 /*-
      4  * Copyright (c) 1997-2011, 2019 The NetBSD Foundation, Inc.
      5  * All rights reserved.
      6  *
      7  * This code is derived from software contributed to The NetBSD Foundation
      8  * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
      9  * NASA Ames Research Center, by Charles M. Hannum, and by Andrew Doran.
     10  *
     11  * Redistribution and use in source and binary forms, with or without
     12  * modification, are permitted provided that the following conditions
     13  * are met:
     14  * 1. Redistributions of source code must retain the above copyright
     15  *    notice, this list of conditions and the following disclaimer.
     16  * 2. Redistributions in binary form must reproduce the above copyright
     17  *    notice, this list of conditions and the following disclaimer in the
     18  *    documentation and/or other materials provided with the distribution.
     19  *
     20  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     21  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     22  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     23  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     24  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     25  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     26  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     27  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     28  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     29  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     30  * POSSIBILITY OF SUCH DAMAGE.
     31  */
     32 
     33 /*
     34  * Copyright (c) 1989, 1993
     35  *	The Regents of the University of California.  All rights reserved.
     36  * (c) UNIX System Laboratories, Inc.
     37  * All or some portions of this file are derived from material licensed
     38  * to the University of California by American Telephone and Telegraph
     39  * Co. or Unix System Laboratories, Inc. and are reproduced herein with
     40  * the permission of UNIX System Laboratories, Inc.
     41  *
     42  * Redistribution and use in source and binary forms, with or without
     43  * modification, are permitted provided that the following conditions
     44  * are met:
     45  * 1. Redistributions of source code must retain the above copyright
     46  *    notice, this list of conditions and the following disclaimer.
     47  * 2. Redistributions in binary form must reproduce the above copyright
     48  *    notice, this list of conditions and the following disclaimer in the
     49  *    documentation and/or other materials provided with the distribution.
     50  * 3. Neither the name of the University nor the names of its contributors
     51  *    may be used to endorse or promote products derived from this software
     52  *    without specific prior written permission.
     53  *
     54  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
     55  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
     56  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
     57  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
     58  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
     59  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
     60  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
     61  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
     62  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
     63  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
     64  * SUCH DAMAGE.
     65  *
     66  *	@(#)vfs_subr.c	8.13 (Berkeley) 4/18/94
     67  */
     68 
     69 /*
     70  * The vnode cache subsystem.
     71  *
     72  * Life-cycle
     73  *
     74  *	Normally, there are two points where new vnodes are created:
     75  *	VOP_CREATE(9) and VOP_LOOKUP(9).  The life-cycle of a vnode
     76  *	starts in one of the following ways:
     77  *
     78  *	- Allocation, via vcache_get(9) or vcache_new(9).
     79  *	- Reclamation of inactive vnode, via vcache_vget(9).
     80  *
     81  *	Recycle from a free list, via getnewvnode(9) -> getcleanvnode(9)
     82  *	was another, traditional way.  Currently, only the draining thread
     83  *	recycles the vnodes.  This behaviour might be revisited.
     84  *
     85  *	The life-cycle ends when the last reference is dropped, usually
     86  *	in VOP_REMOVE(9).  In such case, VOP_INACTIVE(9) is called to inform
     87  *	the file system that vnode is inactive.  Via this call, file system
     88  *	indicates whether vnode can be recycled (usually, it checks its own
     89  *	references, e.g. count of links, whether the file was removed).
     90  *
     91  *	Depending on indication, vnode can be put into a free list (cache),
     92  *	or cleaned via vcache_reclaim, which calls VOP_RECLAIM(9) to
     93  *	disassociate underlying file system from the vnode, and finally
     94  *	destroyed.
     95  *
     96  * Vnode state
     97  *
     98  *	Vnode is always in one of six states:
     99  *	- MARKER	This is a marker vnode to help list traversal.  It
    100  *			will never change its state.
    101  *	- LOADING	Vnode is associating underlying file system and not
    102  *			yet ready to use.
    103  *	- LOADED	Vnode has associated underlying file system and is
    104  *			ready to use.
    105  *	- BLOCKED	Vnode is active but cannot get new references.
    106  *	- RECLAIMING	Vnode is disassociating from the underlying file
    107  *			system.
    108  *	- RECLAIMED	Vnode has disassociated from underlying file system
    109  *			and is dead.
    110  *
    111  *	Valid state changes are:
    112  *	LOADING -> LOADED
    113  *			Vnode has been initialised in vcache_get() or
    114  *			vcache_new() and is ready to use.
    115  *	LOADED -> RECLAIMING
    116  *			Vnode starts disassociation from underlying file
    117  *			system in vcache_reclaim().
    118  *	RECLAIMING -> RECLAIMED
    119  *			Vnode finished disassociation from underlying file
    120  *			system in vcache_reclaim().
    121  *	LOADED -> BLOCKED
    122  *			vcache_rekey*() is changing the vnode key.
    123  *	BLOCKED -> LOADED
    124  *			The block condition is over.
    125  *	LOADING -> RECLAIMED
    126  *			Either vcache_get() or vcache_new() failed to
    127  *			associate the underlying file system or vcache_rekey*()
    128  *			drops a vnode used as placeholder.
    129  *
    130  *	Of these states LOADING, BLOCKED and RECLAIMING are intermediate
    131  *	and it is possible to wait for state change.
    132  *
    133  *	State is protected with v_interlock with one exception:
    134  *	to change from LOADING both v_interlock and vcache_lock must be held
    135  *	so it is possible to check "state == LOADING" without holding
    136  *	v_interlock.  See vcache_get() for details.
    137  *
    138  * Reference counting
    139  *
    140  *	Vnode is considered active, if reference count (vnode_t::v_usecount)
    141  *	is non-zero.  It is maintained using: vref(9) and vrele(9), as well
    142  *	as vput(9), routines.  Common points holding references are e.g.
    143  *	file openings, current working directory, mount points, etc.
    144  *
    145  */
    146 
    147 #include <sys/cdefs.h>
    148 __KERNEL_RCSID(0, "$NetBSD: vfs_vnode.c,v 1.105.2.3 2020/01/17 21:55:13 ad Exp $");
    149 
    150 #include <sys/param.h>
    151 #include <sys/kernel.h>
    152 
    153 #include <sys/atomic.h>
    154 #include <sys/buf.h>
    155 #include <sys/conf.h>
    156 #include <sys/device.h>
    157 #include <sys/hash.h>
    158 #include <sys/kauth.h>
    159 #include <sys/kmem.h>
    160 #include <sys/kthread.h>
    161 #include <sys/module.h>
    162 #include <sys/mount.h>
    163 #include <sys/namei.h>
    164 #include <sys/syscallargs.h>
    165 #include <sys/sysctl.h>
    166 #include <sys/systm.h>
    167 #include <sys/vnode_impl.h>
    168 #include <sys/wapbl.h>
    169 #include <sys/fstrans.h>
    170 
    171 #include <uvm/uvm.h>
    172 #include <uvm/uvm_readahead.h>
    173 #include <uvm/uvm_stat.h>
    174 
    175 /* Flags to vrelel. */
    176 #define	VRELEL_ASYNC	0x0001	/* Always defer to vrele thread. */
    177 
    178 #define	LRU_VRELE	0
    179 #define	LRU_FREE	1
    180 #define	LRU_HOLD	2
    181 #define	LRU_COUNT	3
    182 
    183 /*
    184  * There are three lru lists: one holds vnodes waiting for async release,
    185  * one is for vnodes which have no buffer/page references and one for those
    186  * which do (i.e.  v_holdcnt is non-zero).  We put the lists into a single,
    187  * private cache line as vnodes migrate between them while under the same
    188  * lock (vdrain_lock).
    189  */
    190 u_int			numvnodes		__cacheline_aligned;
    191 static vnodelst_t	lru_list[LRU_COUNT]	__cacheline_aligned;
    192 static kmutex_t		vdrain_lock		__cacheline_aligned;
    193 static kcondvar_t	vdrain_cv;
    194 static int		vdrain_gen;
    195 static kcondvar_t	vdrain_gen_cv;
    196 static bool		vdrain_retry;
    197 static lwp_t *		vdrain_lwp;
    198 SLIST_HEAD(hashhead, vnode_impl);
    199 static kmutex_t		vcache_lock		__cacheline_aligned;
    200 static kcondvar_t	vcache_cv;
    201 static u_int		vcache_hashsize;
    202 static u_long		vcache_hashmask;
    203 static struct hashhead	*vcache_hashtab;
    204 static pool_cache_t	vcache_pool;
    205 static void		lru_requeue(vnode_t *, vnodelst_t *);
    206 static vnodelst_t *	lru_which(vnode_t *);
    207 static vnode_impl_t *	vcache_alloc(void);
    208 static void		vcache_dealloc(vnode_impl_t *);
    209 static void		vcache_free(vnode_impl_t *);
    210 static void		vcache_init(void);
    211 static void		vcache_reinit(void);
    212 static void		vcache_reclaim(vnode_t *);
    213 static void		vrelel(vnode_t *, int, int);
    214 static void		vdrain_thread(void *);
    215 static void		vnpanic(vnode_t *, const char *, ...)
    216     __printflike(2, 3);
    217 
    218 /* Routines having to do with the management of the vnode table. */
    219 extern struct mount	*dead_rootmount;
    220 extern int		(**dead_vnodeop_p)(void *);
    221 extern int		(**spec_vnodeop_p)(void *);
    222 extern struct vfsops	dead_vfsops;
    223 
    224 /* Vnode state operations and diagnostics. */
    225 
    226 #if defined(DIAGNOSTIC)
    227 
    228 #define VSTATE_VALID(state) \
    229 	((state) != VS_ACTIVE && (state) != VS_MARKER)
    230 #define VSTATE_GET(vp) \
    231 	vstate_assert_get((vp), __func__, __LINE__)
    232 #define VSTATE_CHANGE(vp, from, to) \
    233 	vstate_assert_change((vp), (from), (to), __func__, __LINE__)
    234 #define VSTATE_WAIT_STABLE(vp) \
    235 	vstate_assert_wait_stable((vp), __func__, __LINE__)
    236 
    237 void
    238 _vstate_assert(vnode_t *vp, enum vnode_state state, const char *func, int line,
    239     bool has_lock)
    240 {
    241 	vnode_impl_t *vip = VNODE_TO_VIMPL(vp);
    242 
    243 	if (!has_lock) {
    244 		/*
    245 		 * Prevent predictive loads from the CPU, but check the state
    246 		 * without loooking first.
    247 		 */
    248 		membar_enter();
    249 		if (state == VS_ACTIVE && vp->v_usecount > 0 &&
    250 		    (vip->vi_state == VS_LOADED || vip->vi_state == VS_BLOCKED))
    251 			return;
    252 		if (vip->vi_state == state)
    253 			return;
    254 		mutex_enter((vp)->v_interlock);
    255 	}
    256 
    257 	KASSERTMSG(mutex_owned(vp->v_interlock), "at %s:%d", func, line);
    258 
    259 	if ((state == VS_ACTIVE && vp->v_usecount > 0 &&
    260 	    (vip->vi_state == VS_LOADED || vip->vi_state == VS_BLOCKED)) ||
    261 	    vip->vi_state == state) {
    262 		if (!has_lock)
    263 			mutex_exit((vp)->v_interlock);
    264 		return;
    265 	}
    266 	vnpanic(vp, "state is %s, usecount %d, expected %s at %s:%d",
    267 	    vstate_name(vip->vi_state), vp->v_usecount,
    268 	    vstate_name(state), func, line);
    269 }
    270 
    271 static enum vnode_state
    272 vstate_assert_get(vnode_t *vp, const char *func, int line)
    273 {
    274 	vnode_impl_t *vip = VNODE_TO_VIMPL(vp);
    275 
    276 	KASSERTMSG(mutex_owned(vp->v_interlock), "at %s:%d", func, line);
    277 	if (! VSTATE_VALID(vip->vi_state))
    278 		vnpanic(vp, "state is %s at %s:%d",
    279 		    vstate_name(vip->vi_state), func, line);
    280 
    281 	return vip->vi_state;
    282 }
    283 
    284 static void
    285 vstate_assert_wait_stable(vnode_t *vp, const char *func, int line)
    286 {
    287 	vnode_impl_t *vip = VNODE_TO_VIMPL(vp);
    288 
    289 	KASSERTMSG(mutex_owned(vp->v_interlock), "at %s:%d", func, line);
    290 	if (! VSTATE_VALID(vip->vi_state))
    291 		vnpanic(vp, "state is %s at %s:%d",
    292 		    vstate_name(vip->vi_state), func, line);
    293 
    294 	while (vip->vi_state != VS_LOADED && vip->vi_state != VS_RECLAIMED)
    295 		cv_wait(&vp->v_cv, vp->v_interlock);
    296 
    297 	if (! VSTATE_VALID(vip->vi_state))
    298 		vnpanic(vp, "state is %s at %s:%d",
    299 		    vstate_name(vip->vi_state), func, line);
    300 }
    301 
    302 static void
    303 vstate_assert_change(vnode_t *vp, enum vnode_state from, enum vnode_state to,
    304     const char *func, int line)
    305 {
    306 	vnode_impl_t *vip = VNODE_TO_VIMPL(vp);
    307 
    308 	KASSERTMSG(mutex_owned(vp->v_interlock), "at %s:%d", func, line);
    309 	if (from == VS_LOADING)
    310 		KASSERTMSG(mutex_owned(&vcache_lock), "at %s:%d", func, line);
    311 
    312 	if (! VSTATE_VALID(from))
    313 		vnpanic(vp, "from is %s at %s:%d",
    314 		    vstate_name(from), func, line);
    315 	if (! VSTATE_VALID(to))
    316 		vnpanic(vp, "to is %s at %s:%d",
    317 		    vstate_name(to), func, line);
    318 	if (vip->vi_state != from)
    319 		vnpanic(vp, "from is %s, expected %s at %s:%d\n",
    320 		    vstate_name(vip->vi_state), vstate_name(from), func, line);
    321 	if ((from == VS_BLOCKED || to == VS_BLOCKED) && vp->v_usecount != 1)
    322 		vnpanic(vp, "%s to %s with usecount %d at %s:%d",
    323 		    vstate_name(from), vstate_name(to), vp->v_usecount,
    324 		    func, line);
    325 
    326 	vip->vi_state = to;
    327 	if (from == VS_LOADING)
    328 		cv_broadcast(&vcache_cv);
    329 	if (to == VS_LOADED || to == VS_RECLAIMED)
    330 		cv_broadcast(&vp->v_cv);
    331 }
    332 
    333 #else /* defined(DIAGNOSTIC) */
    334 
    335 #define VSTATE_GET(vp) \
    336 	(VNODE_TO_VIMPL((vp))->vi_state)
    337 #define VSTATE_CHANGE(vp, from, to) \
    338 	vstate_change((vp), (from), (to))
    339 #define VSTATE_WAIT_STABLE(vp) \
    340 	vstate_wait_stable((vp))
    341 void
    342 _vstate_assert(vnode_t *vp, enum vnode_state state, const char *func, int line,
    343     bool has_lock)
    344 {
    345 
    346 }
    347 
    348 static void
    349 vstate_wait_stable(vnode_t *vp)
    350 {
    351 	vnode_impl_t *vip = VNODE_TO_VIMPL(vp);
    352 
    353 	while (vip->vi_state != VS_LOADED && vip->vi_state != VS_RECLAIMED)
    354 		cv_wait(&vp->v_cv, vp->v_interlock);
    355 }
    356 
    357 static void
    358 vstate_change(vnode_t *vp, enum vnode_state from, enum vnode_state to)
    359 {
    360 	vnode_impl_t *vip = VNODE_TO_VIMPL(vp);
    361 
    362 	vip->vi_state = to;
    363 	if (from == VS_LOADING)
    364 		cv_broadcast(&vcache_cv);
    365 	if (to == VS_LOADED || to == VS_RECLAIMED)
    366 		cv_broadcast(&vp->v_cv);
    367 }
    368 
    369 #endif /* defined(DIAGNOSTIC) */
    370 
    371 void
    372 vfs_vnode_sysinit(void)
    373 {
    374 	int error __diagused, i;
    375 
    376 	dead_rootmount = vfs_mountalloc(&dead_vfsops, NULL);
    377 	KASSERT(dead_rootmount != NULL);
    378 	dead_rootmount->mnt_iflag |= IMNT_MPSAFE;
    379 
    380 	mutex_init(&vdrain_lock, MUTEX_DEFAULT, IPL_NONE);
    381 	for (i = 0; i < LRU_COUNT; i++) {
    382 		TAILQ_INIT(&lru_list[i]);
    383 	}
    384 	vcache_init();
    385 
    386 	cv_init(&vdrain_cv, "vdrain");
    387 	cv_init(&vdrain_gen_cv, "vdrainwt");
    388 	error = kthread_create(PRI_VM, KTHREAD_MPSAFE, NULL, vdrain_thread,
    389 	    NULL, &vdrain_lwp, "vdrain");
    390 	KASSERTMSG((error == 0), "kthread_create(vdrain) failed: %d", error);
    391 }
    392 
    393 /*
    394  * Allocate a new marker vnode.
    395  */
    396 vnode_t *
    397 vnalloc_marker(struct mount *mp)
    398 {
    399 	vnode_impl_t *vip;
    400 	vnode_t *vp;
    401 
    402 	vip = pool_cache_get(vcache_pool, PR_WAITOK);
    403 	memset(vip, 0, sizeof(*vip));
    404 	vp = VIMPL_TO_VNODE(vip);
    405 	uvm_obj_init(&vp->v_uobj, &uvm_vnodeops, true, 0);
    406 	vp->v_mount = mp;
    407 	vp->v_type = VBAD;
    408 	vip->vi_state = VS_MARKER;
    409 
    410 	return vp;
    411 }
    412 
    413 /*
    414  * Free a marker vnode.
    415  */
    416 void
    417 vnfree_marker(vnode_t *vp)
    418 {
    419 	vnode_impl_t *vip;
    420 
    421 	vip = VNODE_TO_VIMPL(vp);
    422 	KASSERT(vip->vi_state == VS_MARKER);
    423 	uvm_obj_destroy(&vp->v_uobj, true);
    424 	pool_cache_put(vcache_pool, vip);
    425 }
    426 
    427 /*
    428  * Test a vnode for being a marker vnode.
    429  */
    430 bool
    431 vnis_marker(vnode_t *vp)
    432 {
    433 
    434 	return (VNODE_TO_VIMPL(vp)->vi_state == VS_MARKER);
    435 }
    436 
    437 /*
    438  * Return the lru list this node should be on.
    439  */
    440 static vnodelst_t *
    441 lru_which(vnode_t *vp)
    442 {
    443 
    444 	KASSERT(mutex_owned(vp->v_interlock));
    445 
    446 	if (vp->v_holdcnt > 0)
    447 		return &lru_list[LRU_HOLD];
    448 	else
    449 		return &lru_list[LRU_FREE];
    450 }
    451 
    452 /*
    453  * Put vnode to end of given list.
    454  * Both the current and the new list may be NULL, used on vnode alloc/free.
    455  * Adjust numvnodes and signal vdrain thread if there is work.
    456  */
    457 static void
    458 lru_requeue(vnode_t *vp, vnodelst_t *listhd)
    459 {
    460 	vnode_impl_t *vip;
    461 	int d;
    462 
    463 	/*
    464 	 * If the vnode is on the correct list, and was put there recently,
    465 	 * then leave it be, thus avoiding huge cache and lock contention.
    466 	 */
    467 	vip = VNODE_TO_VIMPL(vp);
    468 	if (listhd == vip->vi_lrulisthd &&
    469 	    (hardclock_ticks - vip->vi_lrulisttm) < hz) {
    470 	    	return;
    471 	}
    472 
    473 	mutex_enter(&vdrain_lock);
    474 	d = 0;
    475 	if (vip->vi_lrulisthd != NULL)
    476 		TAILQ_REMOVE(vip->vi_lrulisthd, vip, vi_lrulist);
    477 	else
    478 		d++;
    479 	vip->vi_lrulisthd = listhd;
    480 	vip->vi_lrulisttm = hardclock_ticks;
    481 	if (vip->vi_lrulisthd != NULL)
    482 		TAILQ_INSERT_TAIL(vip->vi_lrulisthd, vip, vi_lrulist);
    483 	else
    484 		d--;
    485 	if (d != 0) {
    486 		/*
    487 		 * Looks strange?  This is not a bug.  Don't store
    488 		 * numvnodes unless there is a change - avoid false
    489 		 * sharing on MP.
    490 		 */
    491 		numvnodes += d;
    492 	}
    493 	if (numvnodes > desiredvnodes || listhd == &lru_list[LRU_VRELE])
    494 		cv_broadcast(&vdrain_cv);
    495 	mutex_exit(&vdrain_lock);
    496 }
    497 
    498 /*
    499  * Release deferred vrele vnodes for this mount.
    500  * Called with file system suspended.
    501  */
    502 void
    503 vrele_flush(struct mount *mp)
    504 {
    505 	vnode_impl_t *vip, *marker;
    506 	vnode_t *vp;
    507 
    508 	KASSERT(fstrans_is_owner(mp));
    509 
    510 	marker = VNODE_TO_VIMPL(vnalloc_marker(NULL));
    511 
    512 	mutex_enter(&vdrain_lock);
    513 	TAILQ_INSERT_HEAD(&lru_list[LRU_VRELE], marker, vi_lrulist);
    514 
    515 	while ((vip = TAILQ_NEXT(marker, vi_lrulist))) {
    516 		TAILQ_REMOVE(&lru_list[LRU_VRELE], marker, vi_lrulist);
    517 		TAILQ_INSERT_AFTER(&lru_list[LRU_VRELE], vip, marker,
    518 		    vi_lrulist);
    519 		vp = VIMPL_TO_VNODE(vip);
    520 		if (vnis_marker(vp))
    521 			continue;
    522 
    523 		KASSERT(vip->vi_lrulisthd == &lru_list[LRU_VRELE]);
    524 		TAILQ_REMOVE(vip->vi_lrulisthd, vip, vi_lrulist);
    525 		vip->vi_lrulisthd = &lru_list[LRU_HOLD];
    526 		vip->vi_lrulisttm = hardclock_ticks;
    527 		TAILQ_INSERT_TAIL(vip->vi_lrulisthd, vip, vi_lrulist);
    528 		mutex_exit(&vdrain_lock);
    529 
    530 		vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
    531 		mutex_enter(vp->v_interlock);
    532 		vrelel(vp, 0, LK_EXCLUSIVE);
    533 
    534 		mutex_enter(&vdrain_lock);
    535 	}
    536 
    537 	TAILQ_REMOVE(&lru_list[LRU_VRELE], marker, vi_lrulist);
    538 	mutex_exit(&vdrain_lock);
    539 
    540 	vnfree_marker(VIMPL_TO_VNODE(marker));
    541 }
    542 
    543 /*
    544  * Reclaim a cached vnode.  Used from vdrain_thread only.
    545  */
    546 static __inline void
    547 vdrain_remove(vnode_t *vp)
    548 {
    549 	struct mount *mp;
    550 
    551 	KASSERT(mutex_owned(&vdrain_lock));
    552 
    553 	/* Probe usecount (unlocked). */
    554 	if (vp->v_usecount > 0)
    555 		return;
    556 	/* Try v_interlock -- we lock the wrong direction! */
    557 	if (!mutex_tryenter(vp->v_interlock))
    558 		return;
    559 	/* Probe usecount and state. */
    560 	if (vp->v_usecount > 0 || VSTATE_GET(vp) != VS_LOADED) {
    561 		mutex_exit(vp->v_interlock);
    562 		return;
    563 	}
    564 	mp = vp->v_mount;
    565 	if (fstrans_start_nowait(mp) != 0) {
    566 		mutex_exit(vp->v_interlock);
    567 		return;
    568 	}
    569 	vdrain_retry = true;
    570 	mutex_exit(&vdrain_lock);
    571 
    572 	if (vcache_vget(vp) == 0) {
    573 		if (!vrecycle(vp)) {
    574 			vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
    575 			mutex_enter(vp->v_interlock);
    576 			vrelel(vp, 0, LK_EXCLUSIVE);
    577 		}
    578 	}
    579 	fstrans_done(mp);
    580 
    581 	mutex_enter(&vdrain_lock);
    582 }
    583 
    584 /*
    585  * Release a cached vnode.  Used from vdrain_thread only.
    586  */
    587 static __inline void
    588 vdrain_vrele(vnode_t *vp)
    589 {
    590 	vnode_impl_t *vip = VNODE_TO_VIMPL(vp);
    591 	struct mount *mp;
    592 
    593 	KASSERT(mutex_owned(&vdrain_lock));
    594 
    595 	mp = vp->v_mount;
    596 	if (fstrans_start_nowait(mp) != 0)
    597 		return;
    598 
    599 	/*
    600 	 * First remove the vnode from the vrele list.
    601 	 * Put it on the last lru list, the last vrele()
    602 	 * will put it back onto the right list before
    603 	 * its v_usecount reaches zero.
    604 	 */
    605 	KASSERT(vip->vi_lrulisthd == &lru_list[LRU_VRELE]);
    606 	TAILQ_REMOVE(vip->vi_lrulisthd, vip, vi_lrulist);
    607 	vip->vi_lrulisthd = &lru_list[LRU_HOLD];
    608 	vip->vi_lrulisttm = hardclock_ticks;
    609 	TAILQ_INSERT_TAIL(vip->vi_lrulisthd, vip, vi_lrulist);
    610 
    611 	vdrain_retry = true;
    612 	mutex_exit(&vdrain_lock);
    613 
    614 	vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
    615 	mutex_enter(vp->v_interlock);
    616 	vrelel(vp, 0, LK_EXCLUSIVE);
    617 	fstrans_done(mp);
    618 
    619 	mutex_enter(&vdrain_lock);
    620 }
    621 
    622 /*
    623  * Helper thread to keep the number of vnodes below desiredvnodes
    624  * and release vnodes from asynchronous vrele.
    625  */
    626 static void
    627 vdrain_thread(void *cookie)
    628 {
    629 	int i;
    630 	u_int target;
    631 	vnode_impl_t *vip, *marker;
    632 
    633 	marker = VNODE_TO_VIMPL(vnalloc_marker(NULL));
    634 
    635 	mutex_enter(&vdrain_lock);
    636 
    637 	for (;;) {
    638 		vdrain_retry = false;
    639 		target = desiredvnodes - desiredvnodes/10;
    640 
    641 		for (i = 0; i < LRU_COUNT; i++) {
    642 			TAILQ_INSERT_HEAD(&lru_list[i], marker, vi_lrulist);
    643 			while ((vip = TAILQ_NEXT(marker, vi_lrulist))) {
    644 				TAILQ_REMOVE(&lru_list[i], marker, vi_lrulist);
    645 				TAILQ_INSERT_AFTER(&lru_list[i], vip, marker,
    646 				    vi_lrulist);
    647 				if (vnis_marker(VIMPL_TO_VNODE(vip)))
    648 					continue;
    649 				if (i == LRU_VRELE)
    650 					vdrain_vrele(VIMPL_TO_VNODE(vip));
    651 				else if (numvnodes < target)
    652 					break;
    653 				else
    654 					vdrain_remove(VIMPL_TO_VNODE(vip));
    655 			}
    656 			TAILQ_REMOVE(&lru_list[i], marker, vi_lrulist);
    657 		}
    658 
    659 		if (vdrain_retry) {
    660 			mutex_exit(&vdrain_lock);
    661 			yield();
    662 			mutex_enter(&vdrain_lock);
    663 		} else {
    664 			vdrain_gen++;
    665 			cv_broadcast(&vdrain_gen_cv);
    666 			cv_wait(&vdrain_cv, &vdrain_lock);
    667 		}
    668 	}
    669 }
    670 
    671 /*
    672  * vput: unlock and release the reference.
    673  */
    674 void
    675 vput(vnode_t *vp)
    676 {
    677 	int lktype;
    678 
    679 	if ((vp->v_vflag & VV_LOCKSWORK) == 0) {
    680 		lktype = LK_EXCLUSIVE;
    681 	} else {
    682 		lktype = VOP_ISLOCKED(vp);
    683 		KASSERT(lktype != LK_NONE);
    684 	}
    685 	mutex_enter(vp->v_interlock);
    686 	vrelel(vp, 0, lktype);
    687 }
    688 
    689 /*
    690  * Vnode release.  If reference count drops to zero, call inactive
    691  * routine and either return to freelist or free to the pool.
    692  */
    693 static void
    694 vrelel(vnode_t *vp, int flags, int lktype)
    695 {
    696 	const bool async = ((flags & VRELEL_ASYNC) != 0);
    697 	bool recycle, defer;
    698 	int error;
    699 
    700 	KASSERT(mutex_owned(vp->v_interlock));
    701 
    702 	if (__predict_false(vp->v_op == dead_vnodeop_p &&
    703 	    VSTATE_GET(vp) != VS_RECLAIMED)) {
    704 		vnpanic(vp, "dead but not clean");
    705 	}
    706 
    707 	/*
    708 	 * If not the last reference, just drop the reference count
    709 	 * and unlock.
    710 	 */
    711 	if (vp->v_usecount > 1) {
    712 		if (lktype != LK_NONE) {
    713 			VOP_UNLOCK(vp);
    714 		}
    715 		vp->v_usecount--;
    716 		mutex_exit(vp->v_interlock);
    717 		return;
    718 	}
    719 	if (vp->v_usecount <= 0 || vp->v_writecount != 0) {
    720 		vnpanic(vp, "%s: bad ref count", __func__);
    721 	}
    722 
    723 #ifdef DIAGNOSTIC
    724 	if ((vp->v_type == VBLK || vp->v_type == VCHR) &&
    725 	    vp->v_specnode != NULL && vp->v_specnode->sn_opencnt != 0) {
    726 		vprint("vrelel: missing VOP_CLOSE()", vp);
    727 	}
    728 #endif
    729 
    730 	/*
    731 	 * First try to get the vnode locked for VOP_INACTIVE().
    732 	 * Defer vnode release to vdrain_thread if caller requests
    733 	 * it explicitly, is the pagedaemon or the lock failed.
    734 	 */
    735 	defer = false;
    736 	if ((curlwp == uvm.pagedaemon_lwp) || async) {
    737 		defer = true;
    738 	} else if (lktype == LK_SHARED) {
    739 		/* Excellent chance of getting, if the last ref. */
    740 		error = vn_lock(vp, LK_UPGRADE | LK_RETRY |
    741 		    LK_NOWAIT);
    742 		if (error != 0) {
    743 			defer = true;
    744 		} else {
    745 			lktype = LK_EXCLUSIVE;
    746 		}
    747 	} else if (lktype == LK_NONE) {
    748 		/* Excellent chance of getting, if the last ref. */
    749 		error = vn_lock(vp, LK_EXCLUSIVE | LK_RETRY |
    750 		    LK_NOWAIT);
    751 		if (error != 0) {
    752 			defer = true;
    753 		} else {
    754 			lktype = LK_EXCLUSIVE;
    755 		}
    756 	}
    757 	KASSERT(mutex_owned(vp->v_interlock));
    758 	if (defer) {
    759 		/*
    760 		 * Defer reclaim to the kthread; it's not safe to
    761 		 * clean it here.  We donate it our last reference.
    762 		 */
    763 		if (lktype != LK_NONE) {
    764 			VOP_UNLOCK(vp);
    765 		}
    766 		lru_requeue(vp, &lru_list[LRU_VRELE]);
    767 		mutex_exit(vp->v_interlock);
    768 		return;
    769 	}
    770 	KASSERT(lktype == LK_EXCLUSIVE);
    771 
    772 	/*
    773 	 * If not clean, deactivate the vnode, but preserve
    774 	 * our reference across the call to VOP_INACTIVE().
    775 	 */
    776 	if (VSTATE_GET(vp) == VS_RECLAIMED) {
    777 		VOP_UNLOCK(vp);
    778 	} else {
    779 		/*
    780 		 * If VOP_INACTIVE() indicates that the described file has
    781 		 * been deleted, then recycle the vnode.  Note that
    782 		 * VOP_INACTIVE() will not drop the vnode lock.
    783 		 *
    784 		 * If the file has been deleted, this is a lingering
    785 		 * reference and there is no need to worry about new
    786 		 * references looking to do real work with the vnode (as it
    787 		 * will have been purged from directories, caches, etc).
    788 		 */
    789 		recycle = false;
    790 		mutex_exit(vp->v_interlock);
    791 		VOP_INACTIVE(vp, &recycle);
    792 		mutex_enter(vp->v_interlock);
    793 		if (!recycle) {
    794 			VOP_UNLOCK(vp);
    795 			if (vp->v_usecount > 1) {
    796 				vp->v_usecount--;
    797 				mutex_exit(vp->v_interlock);
    798 				return;
    799 			}
    800 		}
    801 
    802 		/* Take care of space accounting. */
    803 		if ((vp->v_iflag & VI_EXECMAP) != 0 &&
    804 		    vp->v_uobj.uo_npages != 0) {
    805 			cpu_count(CPU_COUNT_EXECPAGES, -vp->v_uobj.uo_npages);
    806 			cpu_count(CPU_COUNT_FILEPAGES, vp->v_uobj.uo_npages);
    807 		}
    808 		vp->v_iflag &= ~(VI_TEXT|VI_EXECMAP|VI_WRMAP);
    809 		vp->v_vflag &= ~VV_MAPPED;
    810 
    811 		/*
    812 		 * Recycle the vnode if the file is now unused (unlinked),
    813 		 * otherwise just free it.
    814 		 */
    815 		if (recycle) {
    816 			VSTATE_ASSERT(vp, VS_LOADED);
    817 			/* vcache_reclaim drops the lock. */
    818 			vcache_reclaim(vp);
    819 		}
    820 		KASSERT(vp->v_usecount > 0);
    821 	}
    822 
    823 	vp->v_usecount--;
    824 	if (vp->v_usecount != 0) {
    825 		/* Gained another reference while being reclaimed. */
    826 		mutex_exit(vp->v_interlock);
    827 		return;
    828 	}
    829 
    830 	if (VSTATE_GET(vp) == VS_RECLAIMED && vp->v_holdcnt == 0) {
    831 		/*
    832 		 * It's clean so destroy it.  It isn't referenced
    833 		 * anywhere since it has been reclaimed.
    834 		 */
    835 		vcache_free(VNODE_TO_VIMPL(vp));
    836 	} else {
    837 		/*
    838 		 * Otherwise, put it back onto the freelist.  It
    839 		 * can't be destroyed while still associated with
    840 		 * a file system.
    841 		 */
    842 		lru_requeue(vp, lru_which(vp));
    843 		mutex_exit(vp->v_interlock);
    844 	}
    845 }
    846 
    847 void
    848 vrele(vnode_t *vp)
    849 {
    850 
    851 	mutex_enter(vp->v_interlock);
    852 	vrelel(vp, 0, LK_NONE);
    853 }
    854 
    855 /*
    856  * Asynchronous vnode release, vnode is released in different context.
    857  */
    858 void
    859 vrele_async(vnode_t *vp)
    860 {
    861 
    862 	mutex_enter(vp->v_interlock);
    863 	vrelel(vp, VRELEL_ASYNC, LK_NONE);
    864 }
    865 
    866 /*
    867  * Vnode reference, where a reference is already held by some other
    868  * object (for example, a file structure).
    869  */
    870 void
    871 vref(vnode_t *vp)
    872 {
    873 
    874 	KASSERT(vp->v_usecount != 0);
    875 
    876 	mutex_enter(vp->v_interlock);
    877 	vp->v_usecount++;
    878 	mutex_exit(vp->v_interlock);
    879 }
    880 
    881 /*
    882  * Page or buffer structure gets a reference.
    883  * Called with v_interlock held.
    884  */
    885 void
    886 vholdl(vnode_t *vp)
    887 {
    888 
    889 	KASSERT(mutex_owned(vp->v_interlock));
    890 
    891 	if (vp->v_holdcnt++ == 0 && vp->v_usecount == 0)
    892 		lru_requeue(vp, lru_which(vp));
    893 }
    894 
    895 /*
    896  * Page or buffer structure frees a reference.
    897  * Called with v_interlock held.
    898  */
    899 void
    900 holdrelel(vnode_t *vp)
    901 {
    902 
    903 	KASSERT(mutex_owned(vp->v_interlock));
    904 
    905 	if (vp->v_holdcnt <= 0) {
    906 		vnpanic(vp, "%s: holdcnt vp %p", __func__, vp);
    907 	}
    908 
    909 	vp->v_holdcnt--;
    910 	if (vp->v_holdcnt == 0 && vp->v_usecount == 0)
    911 		lru_requeue(vp, lru_which(vp));
    912 }
    913 
    914 /*
    915  * Recycle an unused vnode if caller holds the last reference.
    916  */
    917 bool
    918 vrecycle(vnode_t *vp)
    919 {
    920 	int error __diagused;
    921 
    922 	mutex_enter(vp->v_interlock);
    923 
    924 	/* Make sure we hold the last reference. */
    925 	VSTATE_WAIT_STABLE(vp);
    926 	if (vp->v_usecount != 1) {
    927 		mutex_exit(vp->v_interlock);
    928 		return false;
    929 	}
    930 
    931 	/* If the vnode is already clean we're done. */
    932 	if (VSTATE_GET(vp) != VS_LOADED) {
    933 		VSTATE_ASSERT(vp, VS_RECLAIMED);
    934 		vrelel(vp, 0, LK_NONE);
    935 		return true;
    936 	}
    937 
    938 	/* Prevent further references until the vnode is locked. */
    939 	VSTATE_CHANGE(vp, VS_LOADED, VS_BLOCKED);
    940 	mutex_exit(vp->v_interlock);
    941 
    942 	/*
    943 	 * On a leaf file system this lock will always succeed as we hold
    944 	 * the last reference and prevent further references.
    945 	 * On layered file systems waiting for the lock would open a can of
    946 	 * deadlocks as the lower vnodes may have other active references.
    947 	 */
    948 	error = vn_lock(vp, LK_EXCLUSIVE | LK_RETRY | LK_NOWAIT);
    949 
    950 	mutex_enter(vp->v_interlock);
    951 	VSTATE_CHANGE(vp, VS_BLOCKED, VS_LOADED);
    952 
    953 	if (error) {
    954 		mutex_exit(vp->v_interlock);
    955 		return false;
    956 	}
    957 
    958 	KASSERT(vp->v_usecount == 1);
    959 	vcache_reclaim(vp);
    960 	vrelel(vp, 0, LK_NONE);
    961 
    962 	return true;
    963 }
    964 
    965 /*
    966  * Helper for vrevoke() to propagate suspension from lastmp
    967  * to thismp.  Both args may be NULL.
    968  * Returns the currently suspended file system or NULL.
    969  */
    970 static struct mount *
    971 vrevoke_suspend_next(struct mount *lastmp, struct mount *thismp)
    972 {
    973 	int error;
    974 
    975 	if (lastmp == thismp)
    976 		return thismp;
    977 
    978 	if (lastmp != NULL)
    979 		vfs_resume(lastmp);
    980 
    981 	if (thismp == NULL)
    982 		return NULL;
    983 
    984 	do {
    985 		error = vfs_suspend(thismp, 0);
    986 	} while (error == EINTR || error == ERESTART);
    987 
    988 	if (error == 0)
    989 		return thismp;
    990 
    991 	KASSERT(error == EOPNOTSUPP);
    992 	return NULL;
    993 }
    994 
    995 /*
    996  * Eliminate all activity associated with the requested vnode
    997  * and with all vnodes aliased to the requested vnode.
    998  */
    999 void
   1000 vrevoke(vnode_t *vp)
   1001 {
   1002 	struct mount *mp;
   1003 	vnode_t *vq;
   1004 	enum vtype type;
   1005 	dev_t dev;
   1006 
   1007 	KASSERT(vp->v_usecount > 0);
   1008 
   1009 	mp = vrevoke_suspend_next(NULL, vp->v_mount);
   1010 
   1011 	mutex_enter(vp->v_interlock);
   1012 	VSTATE_WAIT_STABLE(vp);
   1013 	if (VSTATE_GET(vp) == VS_RECLAIMED) {
   1014 		mutex_exit(vp->v_interlock);
   1015 	} else if (vp->v_type != VBLK && vp->v_type != VCHR) {
   1016 		vp->v_usecount++;
   1017 		mutex_exit(vp->v_interlock);
   1018 		vgone(vp);
   1019 	} else {
   1020 		dev = vp->v_rdev;
   1021 		type = vp->v_type;
   1022 		mutex_exit(vp->v_interlock);
   1023 
   1024 		while (spec_node_lookup_by_dev(type, dev, &vq) == 0) {
   1025 			mp = vrevoke_suspend_next(mp, vq->v_mount);
   1026 			vgone(vq);
   1027 		}
   1028 	}
   1029 	vrevoke_suspend_next(mp, NULL);
   1030 }
   1031 
   1032 /*
   1033  * Eliminate all activity associated with a vnode in preparation for
   1034  * reuse.  Drops a reference from the vnode.
   1035  */
   1036 void
   1037 vgone(vnode_t *vp)
   1038 {
   1039 	int lktype;
   1040 
   1041 	KASSERT(vp->v_mount == dead_rootmount || fstrans_is_owner(vp->v_mount));
   1042 
   1043 	vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
   1044 	lktype = LK_EXCLUSIVE;
   1045 	mutex_enter(vp->v_interlock);
   1046 	VSTATE_WAIT_STABLE(vp);
   1047 	if (VSTATE_GET(vp) == VS_LOADED) {
   1048 		vcache_reclaim(vp);
   1049 		lktype = LK_NONE;
   1050 	}
   1051 	VSTATE_ASSERT(vp, VS_RECLAIMED);
   1052 	vrelel(vp, 0, lktype);
   1053 }
   1054 
   1055 static inline uint32_t
   1056 vcache_hash(const struct vcache_key *key)
   1057 {
   1058 	uint32_t hash = HASH32_BUF_INIT;
   1059 
   1060 	KASSERT(key->vk_key_len > 0);
   1061 
   1062 	hash = hash32_buf(&key->vk_mount, sizeof(struct mount *), hash);
   1063 	hash = hash32_buf(key->vk_key, key->vk_key_len, hash);
   1064 	return hash;
   1065 }
   1066 
   1067 static void
   1068 vcache_init(void)
   1069 {
   1070 
   1071 	vcache_pool = pool_cache_init(sizeof(vnode_impl_t), 0, 0, 0,
   1072 	    "vcachepl", NULL, IPL_NONE, NULL, NULL, NULL);
   1073 	KASSERT(vcache_pool != NULL);
   1074 	mutex_init(&vcache_lock, MUTEX_DEFAULT, IPL_NONE);
   1075 	cv_init(&vcache_cv, "vcache");
   1076 	vcache_hashsize = desiredvnodes;
   1077 	vcache_hashtab = hashinit(desiredvnodes, HASH_SLIST, true,
   1078 	    &vcache_hashmask);
   1079 }
   1080 
   1081 static void
   1082 vcache_reinit(void)
   1083 {
   1084 	int i;
   1085 	uint32_t hash;
   1086 	u_long oldmask, newmask;
   1087 	struct hashhead *oldtab, *newtab;
   1088 	vnode_impl_t *vip;
   1089 
   1090 	newtab = hashinit(desiredvnodes, HASH_SLIST, true, &newmask);
   1091 	mutex_enter(&vcache_lock);
   1092 	oldtab = vcache_hashtab;
   1093 	oldmask = vcache_hashmask;
   1094 	vcache_hashsize = desiredvnodes;
   1095 	vcache_hashtab = newtab;
   1096 	vcache_hashmask = newmask;
   1097 	for (i = 0; i <= oldmask; i++) {
   1098 		while ((vip = SLIST_FIRST(&oldtab[i])) != NULL) {
   1099 			SLIST_REMOVE(&oldtab[i], vip, vnode_impl, vi_hash);
   1100 			hash = vcache_hash(&vip->vi_key);
   1101 			SLIST_INSERT_HEAD(&newtab[hash & vcache_hashmask],
   1102 			    vip, vi_hash);
   1103 		}
   1104 	}
   1105 	mutex_exit(&vcache_lock);
   1106 	hashdone(oldtab, HASH_SLIST, oldmask);
   1107 }
   1108 
   1109 static inline vnode_impl_t *
   1110 vcache_hash_lookup(const struct vcache_key *key, uint32_t hash)
   1111 {
   1112 	struct hashhead *hashp;
   1113 	vnode_impl_t *vip;
   1114 
   1115 	KASSERT(mutex_owned(&vcache_lock));
   1116 
   1117 	hashp = &vcache_hashtab[hash & vcache_hashmask];
   1118 	SLIST_FOREACH(vip, hashp, vi_hash) {
   1119 		if (key->vk_mount != vip->vi_key.vk_mount)
   1120 			continue;
   1121 		if (key->vk_key_len != vip->vi_key.vk_key_len)
   1122 			continue;
   1123 		if (memcmp(key->vk_key, vip->vi_key.vk_key, key->vk_key_len))
   1124 			continue;
   1125 		return vip;
   1126 	}
   1127 	return NULL;
   1128 }
   1129 
   1130 /*
   1131  * Allocate a new, uninitialized vcache node.
   1132  */
   1133 static vnode_impl_t *
   1134 vcache_alloc(void)
   1135 {
   1136 	vnode_impl_t *vip;
   1137 	vnode_t *vp;
   1138 
   1139 	vip = pool_cache_get(vcache_pool, PR_WAITOK);
   1140 	memset(vip, 0, sizeof(*vip));
   1141 
   1142 	vip->vi_lock = rw_obj_alloc();
   1143 
   1144 	vp = VIMPL_TO_VNODE(vip);
   1145 	uvm_obj_init(&vp->v_uobj, &uvm_vnodeops, true, 0);
   1146 	cv_init(&vp->v_cv, "vnode");
   1147 	cache_vnode_init(vp);
   1148 
   1149 	vp->v_usecount = 1;
   1150 	vp->v_type = VNON;
   1151 	vp->v_size = vp->v_writesize = VSIZENOTSET;
   1152 
   1153 	vip->vi_state = VS_LOADING;
   1154 
   1155 	lru_requeue(vp, &lru_list[LRU_FREE]);
   1156 
   1157 	return vip;
   1158 }
   1159 
   1160 /*
   1161  * Deallocate a vcache node in state VS_LOADING.
   1162  *
   1163  * vcache_lock held on entry and released on return.
   1164  */
   1165 static void
   1166 vcache_dealloc(vnode_impl_t *vip)
   1167 {
   1168 	vnode_t *vp;
   1169 
   1170 	KASSERT(mutex_owned(&vcache_lock));
   1171 
   1172 	vp = VIMPL_TO_VNODE(vip);
   1173 	vfs_ref(dead_rootmount);
   1174 	vfs_insmntque(vp, dead_rootmount);
   1175 	mutex_enter(vp->v_interlock);
   1176 	vp->v_op = dead_vnodeop_p;
   1177 	VSTATE_CHANGE(vp, VS_LOADING, VS_RECLAIMED);
   1178 	mutex_exit(&vcache_lock);
   1179 	vrelel(vp, 0, LK_NONE);
   1180 }
   1181 
   1182 /*
   1183  * Free an unused, unreferenced vcache node.
   1184  * v_interlock locked on entry.
   1185  */
   1186 static void
   1187 vcache_free(vnode_impl_t *vip)
   1188 {
   1189 	vnode_t *vp;
   1190 
   1191 	vp = VIMPL_TO_VNODE(vip);
   1192 	KASSERT(mutex_owned(vp->v_interlock));
   1193 
   1194 	KASSERT(vp->v_usecount == 0);
   1195 	KASSERT(vp->v_holdcnt == 0);
   1196 	KASSERT(vp->v_writecount == 0);
   1197 	lru_requeue(vp, NULL);
   1198 	mutex_exit(vp->v_interlock);
   1199 
   1200 	vfs_insmntque(vp, NULL);
   1201 	if (vp->v_type == VBLK || vp->v_type == VCHR)
   1202 		spec_node_destroy(vp);
   1203 
   1204 	rw_obj_free(vip->vi_lock);
   1205 	uvm_obj_destroy(&vp->v_uobj, true);
   1206 	cv_destroy(&vp->v_cv);
   1207 	cache_vnode_fini(vp);
   1208 	pool_cache_put(vcache_pool, vip);
   1209 }
   1210 
   1211 /*
   1212  * Try to get an initial reference on this cached vnode.
   1213  * Returns zero on success,  ENOENT if the vnode has been reclaimed and
   1214  * EBUSY if the vnode state is unstable.
   1215  *
   1216  * v_interlock locked on entry and unlocked on exit.
   1217  */
   1218 int
   1219 vcache_tryvget(vnode_t *vp)
   1220 {
   1221 	int error = 0;
   1222 
   1223 	KASSERT(mutex_owned(vp->v_interlock));
   1224 
   1225 	if (__predict_false(VSTATE_GET(vp) == VS_RECLAIMED))
   1226 		error = ENOENT;
   1227 	else if (__predict_false(VSTATE_GET(vp) != VS_LOADED))
   1228 		error = EBUSY;
   1229 	else
   1230 		vp->v_usecount++;
   1231 
   1232 	mutex_exit(vp->v_interlock);
   1233 
   1234 	return error;
   1235 }
   1236 
   1237 /*
   1238  * Try to get an initial reference on this cached vnode.
   1239  * Returns zero on success and  ENOENT if the vnode has been reclaimed.
   1240  * Will wait for the vnode state to be stable.
   1241  *
   1242  * v_interlock locked on entry and unlocked on exit.
   1243  */
   1244 int
   1245 vcache_vget(vnode_t *vp)
   1246 {
   1247 
   1248 	KASSERT(mutex_owned(vp->v_interlock));
   1249 
   1250 	/* Increment hold count to prevent vnode from disappearing. */
   1251 	vp->v_holdcnt++;
   1252 	VSTATE_WAIT_STABLE(vp);
   1253 	vp->v_holdcnt--;
   1254 
   1255 	/* If this was the last reference to a reclaimed vnode free it now. */
   1256 	if (__predict_false(VSTATE_GET(vp) == VS_RECLAIMED)) {
   1257 		if (vp->v_holdcnt == 0 && vp->v_usecount == 0)
   1258 			vcache_free(VNODE_TO_VIMPL(vp));
   1259 		else
   1260 			mutex_exit(vp->v_interlock);
   1261 		return ENOENT;
   1262 	}
   1263 	VSTATE_ASSERT(vp, VS_LOADED);
   1264 	vp->v_usecount++;
   1265 	mutex_exit(vp->v_interlock);
   1266 
   1267 	return 0;
   1268 }
   1269 
   1270 /*
   1271  * Get a vnode / fs node pair by key and return it referenced through vpp.
   1272  */
   1273 int
   1274 vcache_get(struct mount *mp, const void *key, size_t key_len,
   1275     struct vnode **vpp)
   1276 {
   1277 	int error;
   1278 	uint32_t hash;
   1279 	const void *new_key;
   1280 	struct vnode *vp;
   1281 	struct vcache_key vcache_key;
   1282 	vnode_impl_t *vip, *new_vip;
   1283 
   1284 	new_key = NULL;
   1285 	*vpp = NULL;
   1286 
   1287 	vcache_key.vk_mount = mp;
   1288 	vcache_key.vk_key = key;
   1289 	vcache_key.vk_key_len = key_len;
   1290 	hash = vcache_hash(&vcache_key);
   1291 
   1292 again:
   1293 	mutex_enter(&vcache_lock);
   1294 	vip = vcache_hash_lookup(&vcache_key, hash);
   1295 
   1296 	/* If found, take a reference or retry. */
   1297 	if (__predict_true(vip != NULL)) {
   1298 		/*
   1299 		 * If the vnode is loading we cannot take the v_interlock
   1300 		 * here as it might change during load (see uvm_obj_setlock()).
   1301 		 * As changing state from VS_LOADING requires both vcache_lock
   1302 		 * and v_interlock it is safe to test with vcache_lock held.
   1303 		 *
   1304 		 * Wait for vnodes changing state from VS_LOADING and retry.
   1305 		 */
   1306 		if (__predict_false(vip->vi_state == VS_LOADING)) {
   1307 			cv_wait(&vcache_cv, &vcache_lock);
   1308 			mutex_exit(&vcache_lock);
   1309 			goto again;
   1310 		}
   1311 		vp = VIMPL_TO_VNODE(vip);
   1312 		mutex_enter(vp->v_interlock);
   1313 		mutex_exit(&vcache_lock);
   1314 		error = vcache_vget(vp);
   1315 		if (error == ENOENT)
   1316 			goto again;
   1317 		if (error == 0)
   1318 			*vpp = vp;
   1319 		KASSERT((error != 0) == (*vpp == NULL));
   1320 		return error;
   1321 	}
   1322 	mutex_exit(&vcache_lock);
   1323 
   1324 	/* Allocate and initialize a new vcache / vnode pair. */
   1325 	error = vfs_busy(mp);
   1326 	if (error)
   1327 		return error;
   1328 	new_vip = vcache_alloc();
   1329 	new_vip->vi_key = vcache_key;
   1330 	vp = VIMPL_TO_VNODE(new_vip);
   1331 	mutex_enter(&vcache_lock);
   1332 	vip = vcache_hash_lookup(&vcache_key, hash);
   1333 	if (vip == NULL) {
   1334 		SLIST_INSERT_HEAD(&vcache_hashtab[hash & vcache_hashmask],
   1335 		    new_vip, vi_hash);
   1336 		vip = new_vip;
   1337 	}
   1338 
   1339 	/* If another thread beat us inserting this node, retry. */
   1340 	if (vip != new_vip) {
   1341 		vcache_dealloc(new_vip);
   1342 		vfs_unbusy(mp);
   1343 		goto again;
   1344 	}
   1345 	mutex_exit(&vcache_lock);
   1346 
   1347 	/* Load the fs node.  Exclusive as new_node is VS_LOADING. */
   1348 	error = VFS_LOADVNODE(mp, vp, key, key_len, &new_key);
   1349 	if (error) {
   1350 		mutex_enter(&vcache_lock);
   1351 		SLIST_REMOVE(&vcache_hashtab[hash & vcache_hashmask],
   1352 		    new_vip, vnode_impl, vi_hash);
   1353 		vcache_dealloc(new_vip);
   1354 		vfs_unbusy(mp);
   1355 		KASSERT(*vpp == NULL);
   1356 		return error;
   1357 	}
   1358 	KASSERT(new_key != NULL);
   1359 	KASSERT(memcmp(key, new_key, key_len) == 0);
   1360 	KASSERT(vp->v_op != NULL);
   1361 	vfs_insmntque(vp, mp);
   1362 	if ((mp->mnt_iflag & IMNT_MPSAFE) != 0)
   1363 		vp->v_vflag |= VV_MPSAFE;
   1364 	vfs_ref(mp);
   1365 	vfs_unbusy(mp);
   1366 
   1367 	/* Finished loading, finalize node. */
   1368 	mutex_enter(&vcache_lock);
   1369 	new_vip->vi_key.vk_key = new_key;
   1370 	mutex_enter(vp->v_interlock);
   1371 	VSTATE_CHANGE(vp, VS_LOADING, VS_LOADED);
   1372 	mutex_exit(vp->v_interlock);
   1373 	mutex_exit(&vcache_lock);
   1374 	*vpp = vp;
   1375 	return 0;
   1376 }
   1377 
   1378 /*
   1379  * Create a new vnode / fs node pair and return it referenced through vpp.
   1380  */
   1381 int
   1382 vcache_new(struct mount *mp, struct vnode *dvp, struct vattr *vap,
   1383     kauth_cred_t cred, void *extra, struct vnode **vpp)
   1384 {
   1385 	int error;
   1386 	uint32_t hash;
   1387 	struct vnode *vp, *ovp;
   1388 	vnode_impl_t *vip, *ovip;
   1389 
   1390 	*vpp = NULL;
   1391 
   1392 	/* Allocate and initialize a new vcache / vnode pair. */
   1393 	error = vfs_busy(mp);
   1394 	if (error)
   1395 		return error;
   1396 	vip = vcache_alloc();
   1397 	vip->vi_key.vk_mount = mp;
   1398 	vp = VIMPL_TO_VNODE(vip);
   1399 
   1400 	/* Create and load the fs node. */
   1401 	error = VFS_NEWVNODE(mp, dvp, vp, vap, cred, extra,
   1402 	    &vip->vi_key.vk_key_len, &vip->vi_key.vk_key);
   1403 	if (error) {
   1404 		mutex_enter(&vcache_lock);
   1405 		vcache_dealloc(vip);
   1406 		vfs_unbusy(mp);
   1407 		KASSERT(*vpp == NULL);
   1408 		return error;
   1409 	}
   1410 	KASSERT(vp->v_op != NULL);
   1411 	KASSERT((vip->vi_key.vk_key_len == 0) == (mp == dead_rootmount));
   1412 	if (vip->vi_key.vk_key_len > 0) {
   1413 		KASSERT(vip->vi_key.vk_key != NULL);
   1414 		hash = vcache_hash(&vip->vi_key);
   1415 
   1416 		/*
   1417 		 * Wait for previous instance to be reclaimed,
   1418 		 * then insert new node.
   1419 		 */
   1420 		mutex_enter(&vcache_lock);
   1421 		while ((ovip = vcache_hash_lookup(&vip->vi_key, hash))) {
   1422 			ovp = VIMPL_TO_VNODE(ovip);
   1423 			mutex_enter(ovp->v_interlock);
   1424 			mutex_exit(&vcache_lock);
   1425 			error = vcache_vget(ovp);
   1426 			KASSERT(error == ENOENT);
   1427 			mutex_enter(&vcache_lock);
   1428 		}
   1429 		SLIST_INSERT_HEAD(&vcache_hashtab[hash & vcache_hashmask],
   1430 		    vip, vi_hash);
   1431 		mutex_exit(&vcache_lock);
   1432 	}
   1433 	vfs_insmntque(vp, mp);
   1434 	if ((mp->mnt_iflag & IMNT_MPSAFE) != 0)
   1435 		vp->v_vflag |= VV_MPSAFE;
   1436 	vfs_ref(mp);
   1437 	vfs_unbusy(mp);
   1438 
   1439 	/* Finished loading, finalize node. */
   1440 	mutex_enter(&vcache_lock);
   1441 	mutex_enter(vp->v_interlock);
   1442 	VSTATE_CHANGE(vp, VS_LOADING, VS_LOADED);
   1443 	mutex_exit(&vcache_lock);
   1444 	mutex_exit(vp->v_interlock);
   1445 	*vpp = vp;
   1446 	return 0;
   1447 }
   1448 
   1449 /*
   1450  * Prepare key change: update old cache nodes key and lock new cache node.
   1451  * Return an error if the new node already exists.
   1452  */
   1453 int
   1454 vcache_rekey_enter(struct mount *mp, struct vnode *vp,
   1455     const void *old_key, size_t old_key_len,
   1456     const void *new_key, size_t new_key_len)
   1457 {
   1458 	uint32_t old_hash, new_hash;
   1459 	struct vcache_key old_vcache_key, new_vcache_key;
   1460 	vnode_impl_t *vip, *new_vip;
   1461 
   1462 	old_vcache_key.vk_mount = mp;
   1463 	old_vcache_key.vk_key = old_key;
   1464 	old_vcache_key.vk_key_len = old_key_len;
   1465 	old_hash = vcache_hash(&old_vcache_key);
   1466 
   1467 	new_vcache_key.vk_mount = mp;
   1468 	new_vcache_key.vk_key = new_key;
   1469 	new_vcache_key.vk_key_len = new_key_len;
   1470 	new_hash = vcache_hash(&new_vcache_key);
   1471 
   1472 	new_vip = vcache_alloc();
   1473 	new_vip->vi_key = new_vcache_key;
   1474 
   1475 	/* Insert locked new node used as placeholder. */
   1476 	mutex_enter(&vcache_lock);
   1477 	vip = vcache_hash_lookup(&new_vcache_key, new_hash);
   1478 	if (vip != NULL) {
   1479 		vcache_dealloc(new_vip);
   1480 		return EEXIST;
   1481 	}
   1482 	SLIST_INSERT_HEAD(&vcache_hashtab[new_hash & vcache_hashmask],
   1483 	    new_vip, vi_hash);
   1484 
   1485 	/* Replace old nodes key with the temporary copy. */
   1486 	vip = vcache_hash_lookup(&old_vcache_key, old_hash);
   1487 	KASSERT(vip != NULL);
   1488 	KASSERT(VIMPL_TO_VNODE(vip) == vp);
   1489 	KASSERT(vip->vi_key.vk_key != old_vcache_key.vk_key);
   1490 	vip->vi_key = old_vcache_key;
   1491 	mutex_exit(&vcache_lock);
   1492 	return 0;
   1493 }
   1494 
   1495 /*
   1496  * Key change complete: update old node and remove placeholder.
   1497  */
   1498 void
   1499 vcache_rekey_exit(struct mount *mp, struct vnode *vp,
   1500     const void *old_key, size_t old_key_len,
   1501     const void *new_key, size_t new_key_len)
   1502 {
   1503 	uint32_t old_hash, new_hash;
   1504 	struct vcache_key old_vcache_key, new_vcache_key;
   1505 	vnode_impl_t *vip, *new_vip;
   1506 	struct vnode *new_vp;
   1507 
   1508 	old_vcache_key.vk_mount = mp;
   1509 	old_vcache_key.vk_key = old_key;
   1510 	old_vcache_key.vk_key_len = old_key_len;
   1511 	old_hash = vcache_hash(&old_vcache_key);
   1512 
   1513 	new_vcache_key.vk_mount = mp;
   1514 	new_vcache_key.vk_key = new_key;
   1515 	new_vcache_key.vk_key_len = new_key_len;
   1516 	new_hash = vcache_hash(&new_vcache_key);
   1517 
   1518 	mutex_enter(&vcache_lock);
   1519 
   1520 	/* Lookup old and new node. */
   1521 	vip = vcache_hash_lookup(&old_vcache_key, old_hash);
   1522 	KASSERT(vip != NULL);
   1523 	KASSERT(VIMPL_TO_VNODE(vip) == vp);
   1524 
   1525 	new_vip = vcache_hash_lookup(&new_vcache_key, new_hash);
   1526 	KASSERT(new_vip != NULL);
   1527 	KASSERT(new_vip->vi_key.vk_key_len == new_key_len);
   1528 	new_vp = VIMPL_TO_VNODE(new_vip);
   1529 	mutex_enter(new_vp->v_interlock);
   1530 	VSTATE_ASSERT(VIMPL_TO_VNODE(new_vip), VS_LOADING);
   1531 	mutex_exit(new_vp->v_interlock);
   1532 
   1533 	/* Rekey old node and put it onto its new hashlist. */
   1534 	vip->vi_key = new_vcache_key;
   1535 	if (old_hash != new_hash) {
   1536 		SLIST_REMOVE(&vcache_hashtab[old_hash & vcache_hashmask],
   1537 		    vip, vnode_impl, vi_hash);
   1538 		SLIST_INSERT_HEAD(&vcache_hashtab[new_hash & vcache_hashmask],
   1539 		    vip, vi_hash);
   1540 	}
   1541 
   1542 	/* Remove new node used as placeholder. */
   1543 	SLIST_REMOVE(&vcache_hashtab[new_hash & vcache_hashmask],
   1544 	    new_vip, vnode_impl, vi_hash);
   1545 	vcache_dealloc(new_vip);
   1546 }
   1547 
   1548 /*
   1549  * Disassociate the underlying file system from a vnode.
   1550  *
   1551  * Must be called with vnode locked and will return unlocked.
   1552  * Must be called with the interlock held, and will return with it held.
   1553  */
   1554 static void
   1555 vcache_reclaim(vnode_t *vp)
   1556 {
   1557 	lwp_t *l = curlwp;
   1558 	vnode_impl_t *vip = VNODE_TO_VIMPL(vp);
   1559 	struct mount *mp = vp->v_mount;
   1560 	uint32_t hash;
   1561 	uint8_t temp_buf[64], *temp_key;
   1562 	size_t temp_key_len;
   1563 	bool recycle, active;
   1564 	int error;
   1565 
   1566 	KASSERT((vp->v_vflag & VV_LOCKSWORK) == 0 ||
   1567 	    VOP_ISLOCKED(vp) == LK_EXCLUSIVE);
   1568 	KASSERT(mutex_owned(vp->v_interlock));
   1569 	KASSERT(vp->v_usecount != 0);
   1570 
   1571 	active = (vp->v_usecount > 1);
   1572 	temp_key_len = vip->vi_key.vk_key_len;
   1573 	/*
   1574 	 * Prevent the vnode from being recycled or brought into use
   1575 	 * while we clean it out.
   1576 	 */
   1577 	VSTATE_CHANGE(vp, VS_LOADED, VS_RECLAIMING);
   1578 	if ((vp->v_iflag & VI_EXECMAP) != 0 && vp->v_uobj.uo_npages != 0) {
   1579 		cpu_count(CPU_COUNT_EXECPAGES, -vp->v_uobj.uo_npages);
   1580 		cpu_count(CPU_COUNT_FILEPAGES, vp->v_uobj.uo_npages);
   1581 	}
   1582 	vp->v_iflag &= ~(VI_TEXT|VI_EXECMAP);
   1583 	mutex_exit(vp->v_interlock);
   1584 
   1585 	/* Replace the vnode key with a temporary copy. */
   1586 	if (vip->vi_key.vk_key_len > sizeof(temp_buf)) {
   1587 		temp_key = kmem_alloc(temp_key_len, KM_SLEEP);
   1588 	} else {
   1589 		temp_key = temp_buf;
   1590 	}
   1591 	if (vip->vi_key.vk_key_len > 0) {
   1592 		mutex_enter(&vcache_lock);
   1593 		memcpy(temp_key, vip->vi_key.vk_key, temp_key_len);
   1594 		vip->vi_key.vk_key = temp_key;
   1595 		mutex_exit(&vcache_lock);
   1596 	}
   1597 
   1598 	fstrans_start(mp);
   1599 
   1600 	/*
   1601 	 * Clean out any cached data associated with the vnode.
   1602 	 * If purging an active vnode, it must be closed and
   1603 	 * deactivated before being reclaimed.
   1604 	 */
   1605 	error = vinvalbuf(vp, V_SAVE, NOCRED, l, 0, 0);
   1606 	if (error != 0) {
   1607 		if (wapbl_vphaswapbl(vp))
   1608 			WAPBL_DISCARD(wapbl_vptomp(vp));
   1609 		error = vinvalbuf(vp, 0, NOCRED, l, 0, 0);
   1610 	}
   1611 	KASSERTMSG((error == 0), "vinvalbuf failed: %d", error);
   1612 	KASSERT((vp->v_iflag & VI_ONWORKLST) == 0);
   1613 	if (active && (vp->v_type == VBLK || vp->v_type == VCHR)) {
   1614 		 spec_node_revoke(vp);
   1615 	}
   1616 
   1617 	/*
   1618 	 * Disassociate the underlying file system from the vnode.
   1619 	 * VOP_INACTIVE leaves the vnode locked; VOP_RECLAIM unlocks
   1620 	 * the vnode, and may destroy the vnode so that VOP_UNLOCK
   1621 	 * would no longer function.
   1622 	 */
   1623 	VOP_INACTIVE(vp, &recycle);
   1624 	KASSERT((vp->v_vflag & VV_LOCKSWORK) == 0 ||
   1625 	    VOP_ISLOCKED(vp) == LK_EXCLUSIVE);
   1626 	if (VOP_RECLAIM(vp)) {
   1627 		vnpanic(vp, "%s: cannot reclaim", __func__);
   1628 	}
   1629 
   1630 	KASSERT(vp->v_data == NULL);
   1631 	KASSERT(vp->v_uobj.uo_npages == 0);
   1632 
   1633 	if (vp->v_type == VREG && vp->v_ractx != NULL) {
   1634 		uvm_ra_freectx(vp->v_ractx);
   1635 		vp->v_ractx = NULL;
   1636 	}
   1637 
   1638 	/* Purge name cache. */
   1639 	cache_purge(vp);
   1640 
   1641 	if (vip->vi_key.vk_key_len > 0) {
   1642 	/* Remove from vnode cache. */
   1643 		hash = vcache_hash(&vip->vi_key);
   1644 		mutex_enter(&vcache_lock);
   1645 		KASSERT(vip == vcache_hash_lookup(&vip->vi_key, hash));
   1646 		SLIST_REMOVE(&vcache_hashtab[hash & vcache_hashmask],
   1647 		    vip, vnode_impl, vi_hash);
   1648 		mutex_exit(&vcache_lock);
   1649 	}
   1650 	if (temp_key != temp_buf)
   1651 		kmem_free(temp_key, temp_key_len);
   1652 
   1653 	/* Done with purge, notify sleepers of the grim news. */
   1654 	mutex_enter(vp->v_interlock);
   1655 	vp->v_op = dead_vnodeop_p;
   1656 	vp->v_vflag |= VV_LOCKSWORK;
   1657 	VSTATE_CHANGE(vp, VS_RECLAIMING, VS_RECLAIMED);
   1658 	vp->v_tag = VT_NON;
   1659 	KNOTE(&vp->v_klist, NOTE_REVOKE);
   1660 	mutex_exit(vp->v_interlock);
   1661 
   1662 	/*
   1663 	 * Move to dead mount.  Must be after changing the operations
   1664 	 * vector as vnode operations enter the mount before using the
   1665 	 * operations vector.  See sys/kern/vnode_if.c.
   1666 	 */
   1667 	vp->v_vflag &= ~VV_ROOT;
   1668 	vfs_ref(dead_rootmount);
   1669 	vfs_insmntque(vp, dead_rootmount);
   1670 
   1671 	mutex_enter(vp->v_interlock);
   1672 	fstrans_done(mp);
   1673 	KASSERT((vp->v_iflag & VI_ONWORKLST) == 0);
   1674 }
   1675 
   1676 /*
   1677  * Disassociate the underlying file system from an open device vnode
   1678  * and make it anonymous.
   1679  *
   1680  * Vnode unlocked on entry, drops a reference to the vnode.
   1681  */
   1682 void
   1683 vcache_make_anon(vnode_t *vp)
   1684 {
   1685 	vnode_impl_t *vip = VNODE_TO_VIMPL(vp);
   1686 	uint32_t hash;
   1687 	bool recycle;
   1688 
   1689 	KASSERT(vp->v_type == VBLK || vp->v_type == VCHR);
   1690 	KASSERT(vp->v_mount == dead_rootmount || fstrans_is_owner(vp->v_mount));
   1691 	VSTATE_ASSERT_UNLOCKED(vp, VS_ACTIVE);
   1692 
   1693 	/* Remove from vnode cache. */
   1694 	hash = vcache_hash(&vip->vi_key);
   1695 	mutex_enter(&vcache_lock);
   1696 	KASSERT(vip == vcache_hash_lookup(&vip->vi_key, hash));
   1697 	SLIST_REMOVE(&vcache_hashtab[hash & vcache_hashmask],
   1698 	    vip, vnode_impl, vi_hash);
   1699 	vip->vi_key.vk_mount = dead_rootmount;
   1700 	vip->vi_key.vk_key_len = 0;
   1701 	vip->vi_key.vk_key = NULL;
   1702 	mutex_exit(&vcache_lock);
   1703 
   1704 	/*
   1705 	 * Disassociate the underlying file system from the vnode.
   1706 	 * VOP_INACTIVE leaves the vnode locked; VOP_RECLAIM unlocks
   1707 	 * the vnode, and may destroy the vnode so that VOP_UNLOCK
   1708 	 * would no longer function.
   1709 	 */
   1710 	if (vn_lock(vp, LK_EXCLUSIVE)) {
   1711 		vnpanic(vp, "%s: cannot lock", __func__);
   1712 	}
   1713 	VOP_INACTIVE(vp, &recycle);
   1714 	KASSERT((vp->v_vflag & VV_LOCKSWORK) == 0 ||
   1715 	    VOP_ISLOCKED(vp) == LK_EXCLUSIVE);
   1716 	if (VOP_RECLAIM(vp)) {
   1717 		vnpanic(vp, "%s: cannot reclaim", __func__);
   1718 	}
   1719 
   1720 	/* Purge name cache. */
   1721 	cache_purge(vp);
   1722 
   1723 	/* Done with purge, change operations vector. */
   1724 	mutex_enter(vp->v_interlock);
   1725 	vp->v_op = spec_vnodeop_p;
   1726 	vp->v_vflag |= VV_MPSAFE;
   1727 	vp->v_vflag &= ~VV_LOCKSWORK;
   1728 	mutex_exit(vp->v_interlock);
   1729 
   1730 	/*
   1731 	 * Move to dead mount.  Must be after changing the operations
   1732 	 * vector as vnode operations enter the mount before using the
   1733 	 * operations vector.  See sys/kern/vnode_if.c.
   1734 	 */
   1735 	vfs_ref(dead_rootmount);
   1736 	vfs_insmntque(vp, dead_rootmount);
   1737 
   1738 	vrele(vp);
   1739 }
   1740 
   1741 /*
   1742  * Update outstanding I/O count and do wakeup if requested.
   1743  */
   1744 void
   1745 vwakeup(struct buf *bp)
   1746 {
   1747 	vnode_t *vp;
   1748 
   1749 	if ((vp = bp->b_vp) == NULL)
   1750 		return;
   1751 
   1752 	KASSERT(bp->b_objlock == vp->v_interlock);
   1753 	KASSERT(mutex_owned(bp->b_objlock));
   1754 
   1755 	if (--vp->v_numoutput < 0)
   1756 		vnpanic(vp, "%s: neg numoutput, vp %p", __func__, vp);
   1757 	if (vp->v_numoutput == 0)
   1758 		cv_broadcast(&vp->v_cv);
   1759 }
   1760 
   1761 /*
   1762  * Test a vnode for being or becoming dead.  Returns one of:
   1763  * EBUSY:  vnode is becoming dead, with "flags == VDEAD_NOWAIT" only.
   1764  * ENOENT: vnode is dead.
   1765  * 0:      otherwise.
   1766  *
   1767  * Whenever this function returns a non-zero value all future
   1768  * calls will also return a non-zero value.
   1769  */
   1770 int
   1771 vdead_check(struct vnode *vp, int flags)
   1772 {
   1773 
   1774 	KASSERT(mutex_owned(vp->v_interlock));
   1775 
   1776 	if (! ISSET(flags, VDEAD_NOWAIT))
   1777 		VSTATE_WAIT_STABLE(vp);
   1778 
   1779 	if (VSTATE_GET(vp) == VS_RECLAIMING) {
   1780 		KASSERT(ISSET(flags, VDEAD_NOWAIT));
   1781 		return EBUSY;
   1782 	} else if (VSTATE_GET(vp) == VS_RECLAIMED) {
   1783 		return ENOENT;
   1784 	}
   1785 
   1786 	return 0;
   1787 }
   1788 
   1789 int
   1790 vfs_drainvnodes(void)
   1791 {
   1792 	int i, gen;
   1793 
   1794 	mutex_enter(&vdrain_lock);
   1795 	for (i = 0; i < 2; i++) {
   1796 		gen = vdrain_gen;
   1797 		while (gen == vdrain_gen) {
   1798 			cv_broadcast(&vdrain_cv);
   1799 			cv_wait(&vdrain_gen_cv, &vdrain_lock);
   1800 		}
   1801 	}
   1802 	mutex_exit(&vdrain_lock);
   1803 
   1804 	if (numvnodes >= desiredvnodes)
   1805 		return EBUSY;
   1806 
   1807 	if (vcache_hashsize != desiredvnodes)
   1808 		vcache_reinit();
   1809 
   1810 	return 0;
   1811 }
   1812 
   1813 void
   1814 vnpanic(vnode_t *vp, const char *fmt, ...)
   1815 {
   1816 	va_list ap;
   1817 
   1818 #ifdef DIAGNOSTIC
   1819 	vprint(NULL, vp);
   1820 #endif
   1821 	va_start(ap, fmt);
   1822 	vpanic(fmt, ap);
   1823 	va_end(ap);
   1824 }
   1825