Home | History | Annotate | Line # | Download | only in kern
vfs_vnode.c revision 1.105.2.9
      1  1.105.2.9        ad /*	$NetBSD: vfs_vnode.c,v 1.105.2.9 2020/02/29 20:21:03 ad Exp $	*/
      2        1.1     rmind 
      3        1.1     rmind /*-
      4  1.105.2.9        ad  * Copyright (c) 1997-2011, 2019, 2020 The NetBSD Foundation, Inc.
      5        1.1     rmind  * All rights reserved.
      6        1.1     rmind  *
      7        1.1     rmind  * This code is derived from software contributed to The NetBSD Foundation
      8        1.1     rmind  * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
      9        1.1     rmind  * NASA Ames Research Center, by Charles M. Hannum, and by Andrew Doran.
     10        1.1     rmind  *
     11        1.1     rmind  * Redistribution and use in source and binary forms, with or without
     12        1.1     rmind  * modification, are permitted provided that the following conditions
     13        1.1     rmind  * are met:
     14        1.1     rmind  * 1. Redistributions of source code must retain the above copyright
     15        1.1     rmind  *    notice, this list of conditions and the following disclaimer.
     16        1.1     rmind  * 2. Redistributions in binary form must reproduce the above copyright
     17        1.1     rmind  *    notice, this list of conditions and the following disclaimer in the
     18        1.1     rmind  *    documentation and/or other materials provided with the distribution.
     19        1.1     rmind  *
     20        1.1     rmind  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     21        1.1     rmind  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     22        1.1     rmind  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     23        1.1     rmind  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     24        1.1     rmind  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     25        1.1     rmind  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     26        1.1     rmind  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     27        1.1     rmind  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     28        1.1     rmind  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     29        1.1     rmind  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     30        1.1     rmind  * POSSIBILITY OF SUCH DAMAGE.
     31        1.1     rmind  */
     32        1.1     rmind 
     33        1.1     rmind /*
     34        1.1     rmind  * Copyright (c) 1989, 1993
     35        1.1     rmind  *	The Regents of the University of California.  All rights reserved.
     36        1.1     rmind  * (c) UNIX System Laboratories, Inc.
     37        1.1     rmind  * All or some portions of this file are derived from material licensed
     38        1.1     rmind  * to the University of California by American Telephone and Telegraph
     39        1.1     rmind  * Co. or Unix System Laboratories, Inc. and are reproduced herein with
     40        1.1     rmind  * the permission of UNIX System Laboratories, Inc.
     41        1.1     rmind  *
     42        1.1     rmind  * Redistribution and use in source and binary forms, with or without
     43        1.1     rmind  * modification, are permitted provided that the following conditions
     44        1.1     rmind  * are met:
     45        1.1     rmind  * 1. Redistributions of source code must retain the above copyright
     46        1.1     rmind  *    notice, this list of conditions and the following disclaimer.
     47        1.1     rmind  * 2. Redistributions in binary form must reproduce the above copyright
     48        1.1     rmind  *    notice, this list of conditions and the following disclaimer in the
     49        1.1     rmind  *    documentation and/or other materials provided with the distribution.
     50        1.1     rmind  * 3. Neither the name of the University nor the names of its contributors
     51        1.1     rmind  *    may be used to endorse or promote products derived from this software
     52        1.1     rmind  *    without specific prior written permission.
     53        1.1     rmind  *
     54        1.1     rmind  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
     55        1.1     rmind  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
     56        1.1     rmind  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
     57        1.1     rmind  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
     58        1.1     rmind  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
     59        1.1     rmind  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
     60        1.1     rmind  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
     61        1.1     rmind  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
     62        1.1     rmind  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
     63        1.1     rmind  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
     64        1.1     rmind  * SUCH DAMAGE.
     65        1.1     rmind  *
     66        1.1     rmind  *	@(#)vfs_subr.c	8.13 (Berkeley) 4/18/94
     67        1.1     rmind  */
     68        1.1     rmind 
     69        1.1     rmind /*
     70        1.8     rmind  * The vnode cache subsystem.
     71        1.1     rmind  *
     72        1.8     rmind  * Life-cycle
     73        1.1     rmind  *
     74        1.8     rmind  *	Normally, there are two points where new vnodes are created:
     75        1.8     rmind  *	VOP_CREATE(9) and VOP_LOOKUP(9).  The life-cycle of a vnode
     76        1.8     rmind  *	starts in one of the following ways:
     77        1.8     rmind  *
     78       1.45   hannken  *	- Allocation, via vcache_get(9) or vcache_new(9).
     79       1.66   hannken  *	- Reclamation of inactive vnode, via vcache_vget(9).
     80        1.8     rmind  *
     81       1.16     rmind  *	Recycle from a free list, via getnewvnode(9) -> getcleanvnode(9)
     82       1.16     rmind  *	was another, traditional way.  Currently, only the draining thread
     83       1.16     rmind  *	recycles the vnodes.  This behaviour might be revisited.
     84       1.16     rmind  *
     85        1.8     rmind  *	The life-cycle ends when the last reference is dropped, usually
     86        1.8     rmind  *	in VOP_REMOVE(9).  In such case, VOP_INACTIVE(9) is called to inform
     87        1.8     rmind  *	the file system that vnode is inactive.  Via this call, file system
     88       1.16     rmind  *	indicates whether vnode can be recycled (usually, it checks its own
     89       1.16     rmind  *	references, e.g. count of links, whether the file was removed).
     90        1.8     rmind  *
     91        1.8     rmind  *	Depending on indication, vnode can be put into a free list (cache),
     92       1.54   hannken  *	or cleaned via vcache_reclaim, which calls VOP_RECLAIM(9) to
     93       1.54   hannken  *	disassociate underlying file system from the vnode, and finally
     94       1.54   hannken  *	destroyed.
     95        1.8     rmind  *
     96       1.52   hannken  * Vnode state
     97       1.52   hannken  *
     98       1.52   hannken  *	Vnode is always in one of six states:
     99       1.52   hannken  *	- MARKER	This is a marker vnode to help list traversal.  It
    100       1.52   hannken  *			will never change its state.
    101       1.52   hannken  *	- LOADING	Vnode is associating underlying file system and not
    102       1.52   hannken  *			yet ready to use.
    103       1.94   hannken  *	- LOADED	Vnode has associated underlying file system and is
    104       1.52   hannken  *			ready to use.
    105       1.52   hannken  *	- BLOCKED	Vnode is active but cannot get new references.
    106       1.52   hannken  *	- RECLAIMING	Vnode is disassociating from the underlying file
    107       1.52   hannken  *			system.
    108       1.52   hannken  *	- RECLAIMED	Vnode has disassociated from underlying file system
    109       1.52   hannken  *			and is dead.
    110       1.52   hannken  *
    111       1.52   hannken  *	Valid state changes are:
    112       1.94   hannken  *	LOADING -> LOADED
    113       1.52   hannken  *			Vnode has been initialised in vcache_get() or
    114       1.52   hannken  *			vcache_new() and is ready to use.
    115       1.94   hannken  *	LOADED -> RECLAIMING
    116       1.52   hannken  *			Vnode starts disassociation from underlying file
    117       1.54   hannken  *			system in vcache_reclaim().
    118       1.52   hannken  *	RECLAIMING -> RECLAIMED
    119       1.52   hannken  *			Vnode finished disassociation from underlying file
    120       1.54   hannken  *			system in vcache_reclaim().
    121       1.94   hannken  *	LOADED -> BLOCKED
    122  1.105.2.3        ad  *			vcache_rekey*() is changing the vnode key.
    123       1.94   hannken  *	BLOCKED -> LOADED
    124       1.52   hannken  *			The block condition is over.
    125       1.52   hannken  *	LOADING -> RECLAIMED
    126       1.52   hannken  *			Either vcache_get() or vcache_new() failed to
    127       1.52   hannken  *			associate the underlying file system or vcache_rekey*()
    128       1.52   hannken  *			drops a vnode used as placeholder.
    129       1.52   hannken  *
    130       1.52   hannken  *	Of these states LOADING, BLOCKED and RECLAIMING are intermediate
    131       1.52   hannken  *	and it is possible to wait for state change.
    132       1.52   hannken  *
    133       1.52   hannken  *	State is protected with v_interlock with one exception:
    134       1.69   hannken  *	to change from LOADING both v_interlock and vcache_lock must be held
    135       1.52   hannken  *	so it is possible to check "state == LOADING" without holding
    136       1.52   hannken  *	v_interlock.  See vcache_get() for details.
    137       1.52   hannken  *
    138        1.8     rmind  * Reference counting
    139        1.8     rmind  *
    140        1.8     rmind  *	Vnode is considered active, if reference count (vnode_t::v_usecount)
    141        1.8     rmind  *	is non-zero.  It is maintained using: vref(9) and vrele(9), as well
    142        1.8     rmind  *	as vput(9), routines.  Common points holding references are e.g.
    143        1.8     rmind  *	file openings, current working directory, mount points, etc.
    144        1.8     rmind  *
    145  1.105.2.8        ad  * Note on v_usecount and its locking
    146  1.105.2.5        ad  *
    147  1.105.2.8        ad  *	At nearly all points it is known that v_usecount could be zero,
    148  1.105.2.8        ad  *	the vnode_t::v_interlock will be held.  To change the count away
    149  1.105.2.5        ad  *	from zero, the interlock must be held.  To change from a non-zero
    150  1.105.2.5        ad  *	value to zero, again the interlock must be held.
    151  1.105.2.5        ad  *
    152  1.105.2.5        ad  *	Changing the usecount from a non-zero value to a non-zero value can
    153  1.105.2.5        ad  *	safely be done using atomic operations, without the interlock held.
    154        1.1     rmind  */
    155        1.1     rmind 
    156        1.1     rmind #include <sys/cdefs.h>
    157  1.105.2.9        ad __KERNEL_RCSID(0, "$NetBSD: vfs_vnode.c,v 1.105.2.9 2020/02/29 20:21:03 ad Exp $");
    158  1.105.2.7        ad 
    159  1.105.2.7        ad #ifdef _KERNEL_OPT
    160  1.105.2.7        ad #include "opt_pax.h"
    161  1.105.2.7        ad #endif
    162        1.1     rmind 
    163        1.1     rmind #include <sys/param.h>
    164        1.1     rmind #include <sys/kernel.h>
    165        1.1     rmind 
    166        1.1     rmind #include <sys/atomic.h>
    167        1.1     rmind #include <sys/buf.h>
    168        1.1     rmind #include <sys/conf.h>
    169        1.1     rmind #include <sys/device.h>
    170       1.36   hannken #include <sys/hash.h>
    171        1.1     rmind #include <sys/kauth.h>
    172        1.1     rmind #include <sys/kmem.h>
    173        1.1     rmind #include <sys/kthread.h>
    174        1.1     rmind #include <sys/module.h>
    175        1.1     rmind #include <sys/mount.h>
    176        1.1     rmind #include <sys/namei.h>
    177  1.105.2.7        ad #include <sys/pax.h>
    178        1.1     rmind #include <sys/syscallargs.h>
    179        1.1     rmind #include <sys/sysctl.h>
    180        1.1     rmind #include <sys/systm.h>
    181       1.58   hannken #include <sys/vnode_impl.h>
    182        1.1     rmind #include <sys/wapbl.h>
    183       1.24   hannken #include <sys/fstrans.h>
    184        1.1     rmind 
    185        1.1     rmind #include <uvm/uvm.h>
    186        1.1     rmind #include <uvm/uvm_readahead.h>
    187      1.104        ad #include <uvm/uvm_stat.h>
    188        1.1     rmind 
    189       1.23   hannken /* Flags to vrelel. */
    190      1.104        ad #define	VRELEL_ASYNC	0x0001	/* Always defer to vrele thread. */
    191      1.104        ad 
    192      1.104        ad #define	LRU_VRELE	0
    193      1.104        ad #define	LRU_FREE	1
    194      1.104        ad #define	LRU_HOLD	2
    195      1.104        ad #define	LRU_COUNT	3
    196        1.1     rmind 
    197       1.16     rmind /*
    198       1.63   hannken  * There are three lru lists: one holds vnodes waiting for async release,
    199      1.104        ad  * one is for vnodes which have no buffer/page references and one for those
    200      1.104        ad  * which do (i.e.  v_holdcnt is non-zero).  We put the lists into a single,
    201      1.104        ad  * private cache line as vnodes migrate between them while under the same
    202      1.104        ad  * lock (vdrain_lock).
    203       1.63   hannken  */
    204      1.104        ad u_int			numvnodes		__cacheline_aligned;
    205      1.104        ad static vnodelst_t	lru_list[LRU_COUNT]	__cacheline_aligned;
    206       1.63   hannken static kmutex_t		vdrain_lock		__cacheline_aligned;
    207      1.104        ad static kcondvar_t	vdrain_cv;
    208       1.63   hannken static int		vdrain_gen;
    209       1.63   hannken static kcondvar_t	vdrain_gen_cv;
    210       1.63   hannken static bool		vdrain_retry;
    211       1.63   hannken static lwp_t *		vdrain_lwp;
    212       1.57   hannken SLIST_HEAD(hashhead, vnode_impl);
    213       1.69   hannken static kmutex_t		vcache_lock		__cacheline_aligned;
    214      1.104        ad static kcondvar_t	vcache_cv;
    215       1.69   hannken static u_int		vcache_hashsize;
    216       1.69   hannken static u_long		vcache_hashmask;
    217      1.104        ad static struct hashhead	*vcache_hashtab;
    218       1.69   hannken static pool_cache_t	vcache_pool;
    219       1.63   hannken static void		lru_requeue(vnode_t *, vnodelst_t *);
    220       1.63   hannken static vnodelst_t *	lru_which(vnode_t *);
    221       1.63   hannken static vnode_impl_t *	vcache_alloc(void);
    222       1.79   hannken static void		vcache_dealloc(vnode_impl_t *);
    223       1.57   hannken static void		vcache_free(vnode_impl_t *);
    224       1.36   hannken static void		vcache_init(void);
    225       1.36   hannken static void		vcache_reinit(void);
    226       1.54   hannken static void		vcache_reclaim(vnode_t *);
    227  1.105.2.2        ad static void		vrelel(vnode_t *, int, int);
    228       1.12   hannken static void		vdrain_thread(void *);
    229       1.11  christos static void		vnpanic(vnode_t *, const char *, ...)
    230       1.18  christos     __printflike(2, 3);
    231        1.1     rmind 
    232        1.1     rmind /* Routines having to do with the management of the vnode table. */
    233       1.44   hannken extern struct mount	*dead_rootmount;
    234        1.1     rmind extern int		(**dead_vnodeop_p)(void *);
    235       1.98   hannken extern int		(**spec_vnodeop_p)(void *);
    236       1.31   hannken extern struct vfsops	dead_vfsops;
    237        1.1     rmind 
    238       1.51   hannken /* Vnode state operations and diagnostics. */
    239       1.51   hannken 
    240       1.51   hannken #if defined(DIAGNOSTIC)
    241       1.51   hannken 
    242       1.94   hannken #define VSTATE_VALID(state) \
    243       1.94   hannken 	((state) != VS_ACTIVE && (state) != VS_MARKER)
    244       1.51   hannken #define VSTATE_GET(vp) \
    245       1.51   hannken 	vstate_assert_get((vp), __func__, __LINE__)
    246       1.51   hannken #define VSTATE_CHANGE(vp, from, to) \
    247       1.51   hannken 	vstate_assert_change((vp), (from), (to), __func__, __LINE__)
    248       1.51   hannken #define VSTATE_WAIT_STABLE(vp) \
    249       1.51   hannken 	vstate_assert_wait_stable((vp), __func__, __LINE__)
    250       1.51   hannken 
    251       1.94   hannken void
    252       1.99     joerg _vstate_assert(vnode_t *vp, enum vnode_state state, const char *func, int line,
    253       1.99     joerg     bool has_lock)
    254       1.51   hannken {
    255       1.70   hannken 	vnode_impl_t *vip = VNODE_TO_VIMPL(vp);
    256       1.51   hannken 
    257       1.99     joerg 	if (!has_lock) {
    258       1.99     joerg 		/*
    259       1.99     joerg 		 * Prevent predictive loads from the CPU, but check the state
    260       1.99     joerg 		 * without loooking first.
    261       1.99     joerg 		 */
    262       1.99     joerg 		membar_enter();
    263       1.99     joerg 		if (state == VS_ACTIVE && vp->v_usecount > 0 &&
    264       1.99     joerg 		    (vip->vi_state == VS_LOADED || vip->vi_state == VS_BLOCKED))
    265       1.99     joerg 			return;
    266       1.99     joerg 		if (vip->vi_state == state)
    267       1.99     joerg 			return;
    268       1.99     joerg 		mutex_enter((vp)->v_interlock);
    269       1.99     joerg 	}
    270       1.99     joerg 
    271       1.51   hannken 	KASSERTMSG(mutex_owned(vp->v_interlock), "at %s:%d", func, line);
    272       1.51   hannken 
    273       1.99     joerg 	if ((state == VS_ACTIVE && vp->v_usecount > 0 &&
    274       1.99     joerg 	    (vip->vi_state == VS_LOADED || vip->vi_state == VS_BLOCKED)) ||
    275       1.99     joerg 	    vip->vi_state == state) {
    276       1.99     joerg 		if (!has_lock)
    277       1.99     joerg 			mutex_exit((vp)->v_interlock);
    278       1.94   hannken 		return;
    279       1.99     joerg 	}
    280       1.94   hannken 	vnpanic(vp, "state is %s, usecount %d, expected %s at %s:%d",
    281       1.94   hannken 	    vstate_name(vip->vi_state), vp->v_usecount,
    282       1.94   hannken 	    vstate_name(state), func, line);
    283       1.51   hannken }
    284       1.51   hannken 
    285       1.57   hannken static enum vnode_state
    286       1.51   hannken vstate_assert_get(vnode_t *vp, const char *func, int line)
    287       1.51   hannken {
    288       1.70   hannken 	vnode_impl_t *vip = VNODE_TO_VIMPL(vp);
    289       1.51   hannken 
    290       1.51   hannken 	KASSERTMSG(mutex_owned(vp->v_interlock), "at %s:%d", func, line);
    291       1.94   hannken 	if (! VSTATE_VALID(vip->vi_state))
    292       1.51   hannken 		vnpanic(vp, "state is %s at %s:%d",
    293       1.70   hannken 		    vstate_name(vip->vi_state), func, line);
    294       1.51   hannken 
    295       1.70   hannken 	return vip->vi_state;
    296       1.51   hannken }
    297       1.51   hannken 
    298       1.52   hannken static void
    299       1.51   hannken vstate_assert_wait_stable(vnode_t *vp, const char *func, int line)
    300       1.51   hannken {
    301       1.70   hannken 	vnode_impl_t *vip = VNODE_TO_VIMPL(vp);
    302       1.51   hannken 
    303       1.51   hannken 	KASSERTMSG(mutex_owned(vp->v_interlock), "at %s:%d", func, line);
    304       1.94   hannken 	if (! VSTATE_VALID(vip->vi_state))
    305       1.51   hannken 		vnpanic(vp, "state is %s at %s:%d",
    306       1.70   hannken 		    vstate_name(vip->vi_state), func, line);
    307       1.51   hannken 
    308       1.94   hannken 	while (vip->vi_state != VS_LOADED && vip->vi_state != VS_RECLAIMED)
    309       1.51   hannken 		cv_wait(&vp->v_cv, vp->v_interlock);
    310       1.51   hannken 
    311       1.94   hannken 	if (! VSTATE_VALID(vip->vi_state))
    312       1.51   hannken 		vnpanic(vp, "state is %s at %s:%d",
    313       1.70   hannken 		    vstate_name(vip->vi_state), func, line);
    314       1.51   hannken }
    315       1.51   hannken 
    316       1.52   hannken static void
    317       1.57   hannken vstate_assert_change(vnode_t *vp, enum vnode_state from, enum vnode_state to,
    318       1.51   hannken     const char *func, int line)
    319       1.51   hannken {
    320       1.70   hannken 	vnode_impl_t *vip = VNODE_TO_VIMPL(vp);
    321       1.51   hannken 
    322       1.51   hannken 	KASSERTMSG(mutex_owned(vp->v_interlock), "at %s:%d", func, line);
    323       1.57   hannken 	if (from == VS_LOADING)
    324       1.69   hannken 		KASSERTMSG(mutex_owned(&vcache_lock), "at %s:%d", func, line);
    325       1.51   hannken 
    326       1.94   hannken 	if (! VSTATE_VALID(from))
    327       1.51   hannken 		vnpanic(vp, "from is %s at %s:%d",
    328       1.51   hannken 		    vstate_name(from), func, line);
    329       1.94   hannken 	if (! VSTATE_VALID(to))
    330       1.51   hannken 		vnpanic(vp, "to is %s at %s:%d",
    331       1.51   hannken 		    vstate_name(to), func, line);
    332       1.70   hannken 	if (vip->vi_state != from)
    333       1.51   hannken 		vnpanic(vp, "from is %s, expected %s at %s:%d\n",
    334       1.70   hannken 		    vstate_name(vip->vi_state), vstate_name(from), func, line);
    335       1.68   hannken 	if ((from == VS_BLOCKED || to == VS_BLOCKED) && vp->v_usecount != 1)
    336       1.68   hannken 		vnpanic(vp, "%s to %s with usecount %d at %s:%d",
    337       1.68   hannken 		    vstate_name(from), vstate_name(to), vp->v_usecount,
    338       1.68   hannken 		    func, line);
    339       1.51   hannken 
    340       1.70   hannken 	vip->vi_state = to;
    341       1.57   hannken 	if (from == VS_LOADING)
    342       1.69   hannken 		cv_broadcast(&vcache_cv);
    343       1.94   hannken 	if (to == VS_LOADED || to == VS_RECLAIMED)
    344       1.51   hannken 		cv_broadcast(&vp->v_cv);
    345       1.51   hannken }
    346       1.51   hannken 
    347       1.51   hannken #else /* defined(DIAGNOSTIC) */
    348       1.51   hannken 
    349       1.51   hannken #define VSTATE_GET(vp) \
    350       1.57   hannken 	(VNODE_TO_VIMPL((vp))->vi_state)
    351       1.51   hannken #define VSTATE_CHANGE(vp, from, to) \
    352       1.51   hannken 	vstate_change((vp), (from), (to))
    353       1.51   hannken #define VSTATE_WAIT_STABLE(vp) \
    354       1.51   hannken 	vstate_wait_stable((vp))
    355       1.94   hannken void
    356      1.100     joerg _vstate_assert(vnode_t *vp, enum vnode_state state, const char *func, int line,
    357      1.100     joerg     bool has_lock)
    358       1.94   hannken {
    359       1.94   hannken 
    360       1.94   hannken }
    361       1.51   hannken 
    362       1.52   hannken static void
    363       1.51   hannken vstate_wait_stable(vnode_t *vp)
    364       1.51   hannken {
    365       1.70   hannken 	vnode_impl_t *vip = VNODE_TO_VIMPL(vp);
    366       1.51   hannken 
    367       1.94   hannken 	while (vip->vi_state != VS_LOADED && vip->vi_state != VS_RECLAIMED)
    368       1.51   hannken 		cv_wait(&vp->v_cv, vp->v_interlock);
    369       1.51   hannken }
    370       1.51   hannken 
    371       1.52   hannken static void
    372       1.57   hannken vstate_change(vnode_t *vp, enum vnode_state from, enum vnode_state to)
    373       1.51   hannken {
    374       1.70   hannken 	vnode_impl_t *vip = VNODE_TO_VIMPL(vp);
    375       1.51   hannken 
    376       1.70   hannken 	vip->vi_state = to;
    377       1.57   hannken 	if (from == VS_LOADING)
    378       1.69   hannken 		cv_broadcast(&vcache_cv);
    379       1.94   hannken 	if (to == VS_LOADED || to == VS_RECLAIMED)
    380       1.51   hannken 		cv_broadcast(&vp->v_cv);
    381       1.51   hannken }
    382       1.51   hannken 
    383       1.51   hannken #endif /* defined(DIAGNOSTIC) */
    384       1.51   hannken 
    385        1.1     rmind void
    386        1.1     rmind vfs_vnode_sysinit(void)
    387        1.1     rmind {
    388      1.104        ad 	int error __diagused, i;
    389        1.1     rmind 
    390       1.44   hannken 	dead_rootmount = vfs_mountalloc(&dead_vfsops, NULL);
    391       1.44   hannken 	KASSERT(dead_rootmount != NULL);
    392      1.103   hannken 	dead_rootmount->mnt_iflag |= IMNT_MPSAFE;
    393       1.31   hannken 
    394       1.63   hannken 	mutex_init(&vdrain_lock, MUTEX_DEFAULT, IPL_NONE);
    395      1.104        ad 	for (i = 0; i < LRU_COUNT; i++) {
    396      1.104        ad 		TAILQ_INIT(&lru_list[i]);
    397      1.104        ad 	}
    398       1.36   hannken 	vcache_init();
    399       1.36   hannken 
    400       1.12   hannken 	cv_init(&vdrain_cv, "vdrain");
    401       1.63   hannken 	cv_init(&vdrain_gen_cv, "vdrainwt");
    402       1.12   hannken 	error = kthread_create(PRI_VM, KTHREAD_MPSAFE, NULL, vdrain_thread,
    403       1.63   hannken 	    NULL, &vdrain_lwp, "vdrain");
    404       1.47  riastrad 	KASSERTMSG((error == 0), "kthread_create(vdrain) failed: %d", error);
    405        1.1     rmind }
    406        1.1     rmind 
    407        1.1     rmind /*
    408       1.48   hannken  * Allocate a new marker vnode.
    409       1.48   hannken  */
    410       1.48   hannken vnode_t *
    411       1.48   hannken vnalloc_marker(struct mount *mp)
    412       1.48   hannken {
    413       1.70   hannken 	vnode_impl_t *vip;
    414       1.50   hannken 	vnode_t *vp;
    415       1.50   hannken 
    416       1.70   hannken 	vip = pool_cache_get(vcache_pool, PR_WAITOK);
    417       1.70   hannken 	memset(vip, 0, sizeof(*vip));
    418       1.70   hannken 	vp = VIMPL_TO_VNODE(vip);
    419  1.105.2.9        ad 	uvm_obj_init(&vp->v_uobj, &uvm_vnodeops, true, 1);
    420       1.50   hannken 	vp->v_mount = mp;
    421       1.50   hannken 	vp->v_type = VBAD;
    422  1.105.2.9        ad 	vp->v_interlock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE);
    423       1.70   hannken 	vip->vi_state = VS_MARKER;
    424       1.48   hannken 
    425       1.50   hannken 	return vp;
    426       1.48   hannken }
    427       1.48   hannken 
    428       1.48   hannken /*
    429       1.48   hannken  * Free a marker vnode.
    430       1.48   hannken  */
    431       1.48   hannken void
    432       1.48   hannken vnfree_marker(vnode_t *vp)
    433       1.48   hannken {
    434       1.70   hannken 	vnode_impl_t *vip;
    435       1.48   hannken 
    436       1.70   hannken 	vip = VNODE_TO_VIMPL(vp);
    437       1.70   hannken 	KASSERT(vip->vi_state == VS_MARKER);
    438  1.105.2.9        ad 	mutex_obj_free(vp->v_interlock);
    439       1.50   hannken 	uvm_obj_destroy(&vp->v_uobj, true);
    440       1.70   hannken 	pool_cache_put(vcache_pool, vip);
    441       1.48   hannken }
    442       1.48   hannken 
    443       1.48   hannken /*
    444       1.48   hannken  * Test a vnode for being a marker vnode.
    445       1.48   hannken  */
    446       1.48   hannken bool
    447       1.48   hannken vnis_marker(vnode_t *vp)
    448       1.48   hannken {
    449       1.48   hannken 
    450       1.57   hannken 	return (VNODE_TO_VIMPL(vp)->vi_state == VS_MARKER);
    451       1.48   hannken }
    452       1.48   hannken 
    453       1.48   hannken /*
    454       1.63   hannken  * Return the lru list this node should be on.
    455       1.63   hannken  */
    456       1.63   hannken static vnodelst_t *
    457       1.63   hannken lru_which(vnode_t *vp)
    458       1.63   hannken {
    459       1.63   hannken 
    460       1.63   hannken 	KASSERT(mutex_owned(vp->v_interlock));
    461       1.63   hannken 
    462       1.63   hannken 	if (vp->v_holdcnt > 0)
    463      1.104        ad 		return &lru_list[LRU_HOLD];
    464       1.63   hannken 	else
    465      1.104        ad 		return &lru_list[LRU_FREE];
    466       1.63   hannken }
    467       1.63   hannken 
    468       1.63   hannken /*
    469       1.63   hannken  * Put vnode to end of given list.
    470       1.63   hannken  * Both the current and the new list may be NULL, used on vnode alloc/free.
    471       1.63   hannken  * Adjust numvnodes and signal vdrain thread if there is work.
    472       1.63   hannken  */
    473       1.63   hannken static void
    474       1.63   hannken lru_requeue(vnode_t *vp, vnodelst_t *listhd)
    475       1.63   hannken {
    476       1.70   hannken 	vnode_impl_t *vip;
    477      1.104        ad 	int d;
    478      1.104        ad 
    479      1.104        ad 	/*
    480      1.104        ad 	 * If the vnode is on the correct list, and was put there recently,
    481      1.104        ad 	 * then leave it be, thus avoiding huge cache and lock contention.
    482      1.104        ad 	 */
    483      1.104        ad 	vip = VNODE_TO_VIMPL(vp);
    484      1.104        ad 	if (listhd == vip->vi_lrulisthd &&
    485      1.104        ad 	    (hardclock_ticks - vip->vi_lrulisttm) < hz) {
    486      1.104        ad 	    	return;
    487      1.104        ad 	}
    488       1.63   hannken 
    489       1.63   hannken 	mutex_enter(&vdrain_lock);
    490      1.104        ad 	d = 0;
    491       1.70   hannken 	if (vip->vi_lrulisthd != NULL)
    492       1.70   hannken 		TAILQ_REMOVE(vip->vi_lrulisthd, vip, vi_lrulist);
    493       1.63   hannken 	else
    494      1.104        ad 		d++;
    495       1.70   hannken 	vip->vi_lrulisthd = listhd;
    496      1.104        ad 	vip->vi_lrulisttm = hardclock_ticks;
    497       1.70   hannken 	if (vip->vi_lrulisthd != NULL)
    498       1.70   hannken 		TAILQ_INSERT_TAIL(vip->vi_lrulisthd, vip, vi_lrulist);
    499       1.63   hannken 	else
    500      1.104        ad 		d--;
    501      1.104        ad 	if (d != 0) {
    502      1.104        ad 		/*
    503      1.104        ad 		 * Looks strange?  This is not a bug.  Don't store
    504      1.104        ad 		 * numvnodes unless there is a change - avoid false
    505      1.104        ad 		 * sharing on MP.
    506      1.104        ad 		 */
    507      1.104        ad 		numvnodes += d;
    508      1.104        ad 	}
    509      1.104        ad 	if (numvnodes > desiredvnodes || listhd == &lru_list[LRU_VRELE])
    510       1.63   hannken 		cv_broadcast(&vdrain_cv);
    511       1.63   hannken 	mutex_exit(&vdrain_lock);
    512       1.63   hannken }
    513       1.63   hannken 
    514       1.63   hannken /*
    515       1.75   hannken  * Release deferred vrele vnodes for this mount.
    516       1.75   hannken  * Called with file system suspended.
    517       1.75   hannken  */
    518       1.75   hannken void
    519       1.75   hannken vrele_flush(struct mount *mp)
    520       1.75   hannken {
    521       1.75   hannken 	vnode_impl_t *vip, *marker;
    522      1.104        ad 	vnode_t *vp;
    523       1.75   hannken 
    524       1.75   hannken 	KASSERT(fstrans_is_owner(mp));
    525       1.75   hannken 
    526       1.75   hannken 	marker = VNODE_TO_VIMPL(vnalloc_marker(NULL));
    527       1.75   hannken 
    528       1.75   hannken 	mutex_enter(&vdrain_lock);
    529      1.104        ad 	TAILQ_INSERT_HEAD(&lru_list[LRU_VRELE], marker, vi_lrulist);
    530       1.75   hannken 
    531       1.75   hannken 	while ((vip = TAILQ_NEXT(marker, vi_lrulist))) {
    532      1.104        ad 		TAILQ_REMOVE(&lru_list[LRU_VRELE], marker, vi_lrulist);
    533      1.104        ad 		TAILQ_INSERT_AFTER(&lru_list[LRU_VRELE], vip, marker,
    534      1.104        ad 		    vi_lrulist);
    535      1.104        ad 		vp = VIMPL_TO_VNODE(vip);
    536      1.104        ad 		if (vnis_marker(vp))
    537       1.75   hannken 			continue;
    538       1.75   hannken 
    539      1.104        ad 		KASSERT(vip->vi_lrulisthd == &lru_list[LRU_VRELE]);
    540       1.75   hannken 		TAILQ_REMOVE(vip->vi_lrulisthd, vip, vi_lrulist);
    541      1.104        ad 		vip->vi_lrulisthd = &lru_list[LRU_HOLD];
    542      1.104        ad 		vip->vi_lrulisttm = hardclock_ticks;
    543       1.75   hannken 		TAILQ_INSERT_TAIL(vip->vi_lrulisthd, vip, vi_lrulist);
    544       1.75   hannken 		mutex_exit(&vdrain_lock);
    545       1.75   hannken 
    546  1.105.2.2        ad 		vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
    547      1.104        ad 		mutex_enter(vp->v_interlock);
    548  1.105.2.2        ad 		vrelel(vp, 0, LK_EXCLUSIVE);
    549       1.75   hannken 
    550       1.75   hannken 		mutex_enter(&vdrain_lock);
    551       1.75   hannken 	}
    552       1.75   hannken 
    553      1.104        ad 	TAILQ_REMOVE(&lru_list[LRU_VRELE], marker, vi_lrulist);
    554       1.75   hannken 	mutex_exit(&vdrain_lock);
    555       1.75   hannken 
    556       1.75   hannken 	vnfree_marker(VIMPL_TO_VNODE(marker));
    557       1.75   hannken }
    558       1.75   hannken 
    559       1.75   hannken /*
    560       1.63   hannken  * Reclaim a cached vnode.  Used from vdrain_thread only.
    561        1.1     rmind  */
    562       1.63   hannken static __inline void
    563       1.63   hannken vdrain_remove(vnode_t *vp)
    564        1.1     rmind {
    565       1.24   hannken 	struct mount *mp;
    566        1.1     rmind 
    567       1.63   hannken 	KASSERT(mutex_owned(&vdrain_lock));
    568       1.24   hannken 
    569       1.63   hannken 	/* Probe usecount (unlocked). */
    570       1.63   hannken 	if (vp->v_usecount > 0)
    571       1.63   hannken 		return;
    572       1.63   hannken 	/* Try v_interlock -- we lock the wrong direction! */
    573       1.63   hannken 	if (!mutex_tryenter(vp->v_interlock))
    574       1.63   hannken 		return;
    575       1.63   hannken 	/* Probe usecount and state. */
    576       1.94   hannken 	if (vp->v_usecount > 0 || VSTATE_GET(vp) != VS_LOADED) {
    577       1.63   hannken 		mutex_exit(vp->v_interlock);
    578       1.63   hannken 		return;
    579        1.1     rmind 	}
    580       1.63   hannken 	mp = vp->v_mount;
    581       1.96   hannken 	if (fstrans_start_nowait(mp) != 0) {
    582       1.63   hannken 		mutex_exit(vp->v_interlock);
    583       1.63   hannken 		return;
    584        1.1     rmind 	}
    585       1.63   hannken 	vdrain_retry = true;
    586       1.63   hannken 	mutex_exit(&vdrain_lock);
    587        1.1     rmind 
    588       1.66   hannken 	if (vcache_vget(vp) == 0) {
    589       1.78   hannken 		if (!vrecycle(vp)) {
    590  1.105.2.2        ad 			vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
    591       1.78   hannken 			mutex_enter(vp->v_interlock);
    592  1.105.2.2        ad 			vrelel(vp, 0, LK_EXCLUSIVE);
    593       1.78   hannken 		}
    594       1.60   hannken 	}
    595       1.24   hannken 	fstrans_done(mp);
    596       1.12   hannken 
    597       1.63   hannken 	mutex_enter(&vdrain_lock);
    598        1.1     rmind }
    599        1.1     rmind 
    600        1.1     rmind /*
    601       1.63   hannken  * Release a cached vnode.  Used from vdrain_thread only.
    602       1.12   hannken  */
    603       1.63   hannken static __inline void
    604       1.63   hannken vdrain_vrele(vnode_t *vp)
    605       1.12   hannken {
    606       1.70   hannken 	vnode_impl_t *vip = VNODE_TO_VIMPL(vp);
    607       1.63   hannken 	struct mount *mp;
    608       1.12   hannken 
    609       1.63   hannken 	KASSERT(mutex_owned(&vdrain_lock));
    610       1.12   hannken 
    611       1.63   hannken 	mp = vp->v_mount;
    612       1.96   hannken 	if (fstrans_start_nowait(mp) != 0)
    613       1.63   hannken 		return;
    614       1.63   hannken 
    615       1.64   hannken 	/*
    616       1.64   hannken 	 * First remove the vnode from the vrele list.
    617       1.64   hannken 	 * Put it on the last lru list, the last vrele()
    618       1.64   hannken 	 * will put it back onto the right list before
    619       1.64   hannken 	 * its v_usecount reaches zero.
    620       1.64   hannken 	 */
    621      1.104        ad 	KASSERT(vip->vi_lrulisthd == &lru_list[LRU_VRELE]);
    622       1.70   hannken 	TAILQ_REMOVE(vip->vi_lrulisthd, vip, vi_lrulist);
    623      1.104        ad 	vip->vi_lrulisthd = &lru_list[LRU_HOLD];
    624      1.104        ad 	vip->vi_lrulisttm = hardclock_ticks;
    625       1.70   hannken 	TAILQ_INSERT_TAIL(vip->vi_lrulisthd, vip, vi_lrulist);
    626       1.63   hannken 
    627       1.63   hannken 	vdrain_retry = true;
    628       1.63   hannken 	mutex_exit(&vdrain_lock);
    629       1.63   hannken 
    630  1.105.2.2        ad 	vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
    631       1.64   hannken 	mutex_enter(vp->v_interlock);
    632  1.105.2.2        ad 	vrelel(vp, 0, LK_EXCLUSIVE);
    633       1.63   hannken 	fstrans_done(mp);
    634       1.63   hannken 
    635       1.63   hannken 	mutex_enter(&vdrain_lock);
    636       1.12   hannken }
    637       1.12   hannken 
    638       1.12   hannken /*
    639       1.63   hannken  * Helper thread to keep the number of vnodes below desiredvnodes
    640       1.63   hannken  * and release vnodes from asynchronous vrele.
    641        1.1     rmind  */
    642       1.63   hannken static void
    643       1.63   hannken vdrain_thread(void *cookie)
    644        1.1     rmind {
    645       1.63   hannken 	int i;
    646       1.63   hannken 	u_int target;
    647       1.70   hannken 	vnode_impl_t *vip, *marker;
    648       1.63   hannken 
    649       1.63   hannken 	marker = VNODE_TO_VIMPL(vnalloc_marker(NULL));
    650       1.63   hannken 
    651       1.63   hannken 	mutex_enter(&vdrain_lock);
    652       1.63   hannken 
    653       1.63   hannken 	for (;;) {
    654       1.63   hannken 		vdrain_retry = false;
    655       1.63   hannken 		target = desiredvnodes - desiredvnodes/10;
    656        1.1     rmind 
    657      1.104        ad 		for (i = 0; i < LRU_COUNT; i++) {
    658      1.104        ad 			TAILQ_INSERT_HEAD(&lru_list[i], marker, vi_lrulist);
    659       1.70   hannken 			while ((vip = TAILQ_NEXT(marker, vi_lrulist))) {
    660      1.104        ad 				TAILQ_REMOVE(&lru_list[i], marker, vi_lrulist);
    661      1.104        ad 				TAILQ_INSERT_AFTER(&lru_list[i], vip, marker,
    662       1.63   hannken 				    vi_lrulist);
    663       1.75   hannken 				if (vnis_marker(VIMPL_TO_VNODE(vip)))
    664       1.75   hannken 					continue;
    665      1.104        ad 				if (i == LRU_VRELE)
    666       1.70   hannken 					vdrain_vrele(VIMPL_TO_VNODE(vip));
    667       1.63   hannken 				else if (numvnodes < target)
    668       1.63   hannken 					break;
    669       1.63   hannken 				else
    670       1.70   hannken 					vdrain_remove(VIMPL_TO_VNODE(vip));
    671       1.63   hannken 			}
    672      1.104        ad 			TAILQ_REMOVE(&lru_list[i], marker, vi_lrulist);
    673       1.63   hannken 		}
    674        1.1     rmind 
    675       1.63   hannken 		if (vdrain_retry) {
    676       1.63   hannken 			mutex_exit(&vdrain_lock);
    677       1.63   hannken 			yield();
    678       1.63   hannken 			mutex_enter(&vdrain_lock);
    679       1.63   hannken 		} else {
    680       1.63   hannken 			vdrain_gen++;
    681       1.63   hannken 			cv_broadcast(&vdrain_gen_cv);
    682       1.63   hannken 			cv_wait(&vdrain_cv, &vdrain_lock);
    683       1.63   hannken 		}
    684        1.1     rmind 	}
    685        1.1     rmind }
    686        1.1     rmind 
    687        1.1     rmind /*
    688  1.105.2.5        ad  * Try to drop reference on a vnode.  Abort if we are releasing the
    689  1.105.2.5        ad  * last reference.  Note: this _must_ succeed if not the last reference.
    690  1.105.2.5        ad  */
    691  1.105.2.5        ad static bool
    692  1.105.2.5        ad vtryrele(vnode_t *vp)
    693  1.105.2.5        ad {
    694  1.105.2.5        ad 	u_int use, next;
    695  1.105.2.5        ad 
    696  1.105.2.9        ad 	for (use = atomic_load_relaxed(&vp->v_usecount);; use = next) {
    697  1.105.2.5        ad 		if (__predict_false(use == 1)) {
    698  1.105.2.5        ad 			return false;
    699  1.105.2.5        ad 		}
    700  1.105.2.5        ad 		KASSERT(use > 1);
    701  1.105.2.5        ad 		next = atomic_cas_uint(&vp->v_usecount, use, use - 1);
    702  1.105.2.5        ad 		if (__predict_true(next == use)) {
    703  1.105.2.5        ad 			return true;
    704  1.105.2.5        ad 		}
    705  1.105.2.5        ad 	}
    706  1.105.2.5        ad }
    707  1.105.2.5        ad 
    708  1.105.2.5        ad /*
    709        1.4     rmind  * vput: unlock and release the reference.
    710        1.1     rmind  */
    711        1.1     rmind void
    712        1.1     rmind vput(vnode_t *vp)
    713        1.1     rmind {
    714  1.105.2.2        ad 	int lktype;
    715        1.1     rmind 
    716  1.105.2.5        ad 	/*
    717  1.105.2.5        ad 	 * Do an unlocked check of v_usecount.  If it looks like we're not
    718  1.105.2.5        ad 	 * about to drop the last reference, then unlock the vnode and try
    719  1.105.2.5        ad 	 * to drop the reference.  If it ends up being the last reference
    720  1.105.2.9        ad 	 * after all, vrelel() can fix it all up.  Most of the time this
    721  1.105.2.9        ad 	 * will all go to plan.
    722  1.105.2.5        ad 	 */
    723  1.105.2.9        ad 	if (atomic_load_relaxed(&vp->v_usecount) > 1) {
    724  1.105.2.5        ad 		VOP_UNLOCK(vp);
    725  1.105.2.5        ad 		if (vtryrele(vp)) {
    726  1.105.2.5        ad 			return;
    727  1.105.2.5        ad 		}
    728  1.105.2.5        ad 		lktype = LK_NONE;
    729  1.105.2.5        ad 	} else if ((vp->v_vflag & VV_LOCKSWORK) == 0) {
    730  1.105.2.2        ad 		lktype = LK_EXCLUSIVE;
    731  1.105.2.2        ad 	} else {
    732  1.105.2.2        ad 		lktype = VOP_ISLOCKED(vp);
    733  1.105.2.2        ad 		KASSERT(lktype != LK_NONE);
    734  1.105.2.2        ad 	}
    735  1.105.2.2        ad 	mutex_enter(vp->v_interlock);
    736  1.105.2.2        ad 	vrelel(vp, 0, lktype);
    737        1.1     rmind }
    738        1.1     rmind 
    739        1.1     rmind /*
    740        1.1     rmind  * Vnode release.  If reference count drops to zero, call inactive
    741        1.1     rmind  * routine and either return to freelist or free to the pool.
    742        1.1     rmind  */
    743       1.23   hannken static void
    744  1.105.2.2        ad vrelel(vnode_t *vp, int flags, int lktype)
    745        1.1     rmind {
    746      1.104        ad 	const bool async = ((flags & VRELEL_ASYNC) != 0);
    747        1.1     rmind 	bool recycle, defer;
    748        1.1     rmind 	int error;
    749        1.1     rmind 
    750        1.9     rmind 	KASSERT(mutex_owned(vp->v_interlock));
    751        1.1     rmind 
    752        1.1     rmind 	if (__predict_false(vp->v_op == dead_vnodeop_p &&
    753       1.57   hannken 	    VSTATE_GET(vp) != VS_RECLAIMED)) {
    754       1.11  christos 		vnpanic(vp, "dead but not clean");
    755        1.1     rmind 	}
    756        1.1     rmind 
    757        1.1     rmind 	/*
    758  1.105.2.9        ad 	 * If not the last reference, just drop the reference count and
    759  1.105.2.9        ad 	 * unlock.  VOP_UNLOCK() is called here without a vnode reference
    760  1.105.2.9        ad 	 * held, but is ok as the hold of v_interlock will stop the vnode
    761  1.105.2.9        ad 	 * from disappearing.
    762        1.1     rmind 	 */
    763  1.105.2.5        ad 	if (vtryrele(vp)) {
    764  1.105.2.2        ad 		if (lktype != LK_NONE) {
    765  1.105.2.2        ad 			VOP_UNLOCK(vp);
    766  1.105.2.2        ad 		}
    767        1.9     rmind 		mutex_exit(vp->v_interlock);
    768        1.1     rmind 		return;
    769        1.1     rmind 	}
    770        1.1     rmind 	if (vp->v_usecount <= 0 || vp->v_writecount != 0) {
    771       1.11  christos 		vnpanic(vp, "%s: bad ref count", __func__);
    772        1.1     rmind 	}
    773        1.1     rmind 
    774       1.15   hannken #ifdef DIAGNOSTIC
    775       1.15   hannken 	if ((vp->v_type == VBLK || vp->v_type == VCHR) &&
    776       1.15   hannken 	    vp->v_specnode != NULL && vp->v_specnode->sn_opencnt != 0) {
    777       1.15   hannken 		vprint("vrelel: missing VOP_CLOSE()", vp);
    778       1.15   hannken 	}
    779       1.15   hannken #endif
    780       1.15   hannken 
    781        1.1     rmind 	/*
    782       1.79   hannken 	 * First try to get the vnode locked for VOP_INACTIVE().
    783       1.79   hannken 	 * Defer vnode release to vdrain_thread if caller requests
    784       1.79   hannken 	 * it explicitly, is the pagedaemon or the lock failed.
    785        1.1     rmind 	 */
    786  1.105.2.2        ad 	defer = false;
    787       1.79   hannken 	if ((curlwp == uvm.pagedaemon_lwp) || async) {
    788       1.79   hannken 		defer = true;
    789  1.105.2.2        ad 	} else if (lktype == LK_SHARED) {
    790  1.105.2.2        ad 		/* Excellent chance of getting, if the last ref. */
    791  1.105.2.2        ad 		error = vn_lock(vp, LK_UPGRADE | LK_RETRY |
    792  1.105.2.2        ad 		    LK_NOWAIT);
    793  1.105.2.2        ad 		if (error != 0) {
    794  1.105.2.2        ad 			defer = true;
    795  1.105.2.2        ad 		} else {
    796  1.105.2.2        ad 			lktype = LK_EXCLUSIVE;
    797  1.105.2.2        ad 		}
    798  1.105.2.2        ad 	} else if (lktype == LK_NONE) {
    799  1.105.2.2        ad 		/* Excellent chance of getting, if the last ref. */
    800  1.105.2.2        ad 		error = vn_lock(vp, LK_EXCLUSIVE | LK_RETRY |
    801  1.105.2.2        ad 		    LK_NOWAIT);
    802  1.105.2.2        ad 		if (error != 0) {
    803  1.105.2.2        ad 			defer = true;
    804  1.105.2.2        ad 		} else {
    805  1.105.2.2        ad 			lktype = LK_EXCLUSIVE;
    806  1.105.2.2        ad 		}
    807       1.79   hannken 	}
    808       1.79   hannken 	KASSERT(mutex_owned(vp->v_interlock));
    809       1.79   hannken 	if (defer) {
    810        1.1     rmind 		/*
    811       1.79   hannken 		 * Defer reclaim to the kthread; it's not safe to
    812       1.79   hannken 		 * clean it here.  We donate it our last reference.
    813        1.1     rmind 		 */
    814  1.105.2.2        ad 		if (lktype != LK_NONE) {
    815  1.105.2.2        ad 			VOP_UNLOCK(vp);
    816  1.105.2.2        ad 		}
    817      1.104        ad 		lru_requeue(vp, &lru_list[LRU_VRELE]);
    818       1.79   hannken 		mutex_exit(vp->v_interlock);
    819       1.79   hannken 		return;
    820       1.79   hannken 	}
    821  1.105.2.2        ad 	KASSERT(lktype == LK_EXCLUSIVE);
    822       1.30   hannken 
    823       1.79   hannken 	/*
    824       1.79   hannken 	 * If not clean, deactivate the vnode, but preserve
    825       1.79   hannken 	 * our reference across the call to VOP_INACTIVE().
    826       1.79   hannken 	 */
    827       1.79   hannken 	if (VSTATE_GET(vp) == VS_RECLAIMED) {
    828       1.79   hannken 		VOP_UNLOCK(vp);
    829       1.79   hannken 	} else {
    830        1.1     rmind 		/*
    831  1.105.2.3        ad 		 * If VOP_INACTIVE() indicates that the described file has
    832  1.105.2.3        ad 		 * been deleted, then recycle the vnode.  Note that
    833  1.105.2.3        ad 		 * VOP_INACTIVE() will not drop the vnode lock.
    834        1.1     rmind 		 *
    835  1.105.2.3        ad 		 * If the file has been deleted, this is a lingering
    836  1.105.2.3        ad 		 * reference and there is no need to worry about new
    837  1.105.2.3        ad 		 * references looking to do real work with the vnode (as it
    838  1.105.2.3        ad 		 * will have been purged from directories, caches, etc).
    839        1.1     rmind 		 */
    840       1.79   hannken 		recycle = false;
    841  1.105.2.3        ad 		mutex_exit(vp->v_interlock);
    842        1.1     rmind 		VOP_INACTIVE(vp, &recycle);
    843  1.105.2.9        ad 		rw_enter(vp->v_uobj.vmobjlock, RW_WRITER);
    844        1.9     rmind 		mutex_enter(vp->v_interlock);
    845        1.1     rmind 		if (!recycle) {
    846  1.105.2.3        ad 			VOP_UNLOCK(vp);
    847  1.105.2.5        ad 			if (vtryrele(vp)) {
    848        1.9     rmind 				mutex_exit(vp->v_interlock);
    849  1.105.2.9        ad 				rw_exit(vp->v_uobj.vmobjlock);
    850        1.1     rmind 				return;
    851        1.1     rmind 			}
    852        1.1     rmind 		}
    853        1.1     rmind 
    854        1.1     rmind 		/* Take care of space accounting. */
    855      1.104        ad 		if ((vp->v_iflag & VI_EXECMAP) != 0 &&
    856      1.104        ad 		    vp->v_uobj.uo_npages != 0) {
    857      1.105        ad 			cpu_count(CPU_COUNT_EXECPAGES, -vp->v_uobj.uo_npages);
    858      1.105        ad 			cpu_count(CPU_COUNT_FILEPAGES, vp->v_uobj.uo_npages);
    859        1.1     rmind 		}
    860        1.1     rmind 		vp->v_iflag &= ~(VI_TEXT|VI_EXECMAP|VI_WRMAP);
    861        1.1     rmind 		vp->v_vflag &= ~VV_MAPPED;
    862  1.105.2.9        ad 		rw_exit(vp->v_uobj.vmobjlock);
    863        1.1     rmind 
    864        1.1     rmind 		/*
    865        1.1     rmind 		 * Recycle the vnode if the file is now unused (unlinked),
    866        1.1     rmind 		 * otherwise just free it.
    867        1.1     rmind 		 */
    868        1.1     rmind 		if (recycle) {
    869       1.94   hannken 			VSTATE_ASSERT(vp, VS_LOADED);
    870       1.83  riastrad 			/* vcache_reclaim drops the lock. */
    871       1.54   hannken 			vcache_reclaim(vp);
    872        1.1     rmind 		}
    873        1.1     rmind 		KASSERT(vp->v_usecount > 0);
    874        1.1     rmind 	}
    875        1.1     rmind 
    876  1.105.2.5        ad 	if (atomic_dec_uint_nv(&vp->v_usecount) != 0) {
    877        1.1     rmind 		/* Gained another reference while being reclaimed. */
    878        1.9     rmind 		mutex_exit(vp->v_interlock);
    879        1.1     rmind 		return;
    880        1.1     rmind 	}
    881        1.1     rmind 
    882       1.67   hannken 	if (VSTATE_GET(vp) == VS_RECLAIMED && vp->v_holdcnt == 0) {
    883        1.1     rmind 		/*
    884        1.1     rmind 		 * It's clean so destroy it.  It isn't referenced
    885        1.1     rmind 		 * anywhere since it has been reclaimed.
    886        1.1     rmind 		 */
    887       1.57   hannken 		vcache_free(VNODE_TO_VIMPL(vp));
    888        1.1     rmind 	} else {
    889        1.1     rmind 		/*
    890        1.1     rmind 		 * Otherwise, put it back onto the freelist.  It
    891        1.1     rmind 		 * can't be destroyed while still associated with
    892        1.1     rmind 		 * a file system.
    893        1.1     rmind 		 */
    894       1.63   hannken 		lru_requeue(vp, lru_which(vp));
    895        1.9     rmind 		mutex_exit(vp->v_interlock);
    896        1.1     rmind 	}
    897        1.1     rmind }
    898        1.1     rmind 
    899        1.1     rmind void
    900        1.1     rmind vrele(vnode_t *vp)
    901        1.1     rmind {
    902        1.1     rmind 
    903  1.105.2.5        ad 	if (vtryrele(vp)) {
    904  1.105.2.5        ad 		return;
    905  1.105.2.5        ad 	}
    906        1.9     rmind 	mutex_enter(vp->v_interlock);
    907  1.105.2.2        ad 	vrelel(vp, 0, LK_NONE);
    908        1.1     rmind }
    909        1.1     rmind 
    910        1.1     rmind /*
    911        1.1     rmind  * Asynchronous vnode release, vnode is released in different context.
    912        1.1     rmind  */
    913        1.1     rmind void
    914        1.1     rmind vrele_async(vnode_t *vp)
    915        1.1     rmind {
    916        1.1     rmind 
    917  1.105.2.5        ad 	if (vtryrele(vp)) {
    918  1.105.2.5        ad 		return;
    919  1.105.2.5        ad 	}
    920        1.9     rmind 	mutex_enter(vp->v_interlock);
    921  1.105.2.2        ad 	vrelel(vp, VRELEL_ASYNC, LK_NONE);
    922        1.1     rmind }
    923        1.1     rmind 
    924        1.1     rmind /*
    925        1.1     rmind  * Vnode reference, where a reference is already held by some other
    926        1.1     rmind  * object (for example, a file structure).
    927  1.105.2.6        ad  *
    928  1.105.2.6        ad  * NB: we have lockless code sequences that rely on this not blocking.
    929        1.1     rmind  */
    930        1.1     rmind void
    931        1.1     rmind vref(vnode_t *vp)
    932        1.1     rmind {
    933        1.1     rmind 
    934  1.105.2.9        ad 	KASSERT(atomic_load_relaxed(&vp->v_usecount) != 0);
    935        1.1     rmind 
    936  1.105.2.5        ad 	atomic_inc_uint(&vp->v_usecount);
    937        1.1     rmind }
    938        1.1     rmind 
    939        1.1     rmind /*
    940        1.1     rmind  * Page or buffer structure gets a reference.
    941        1.1     rmind  * Called with v_interlock held.
    942        1.1     rmind  */
    943        1.1     rmind void
    944        1.1     rmind vholdl(vnode_t *vp)
    945        1.1     rmind {
    946        1.1     rmind 
    947        1.9     rmind 	KASSERT(mutex_owned(vp->v_interlock));
    948        1.1     rmind 
    949  1.105.2.8        ad 	if (vp->v_holdcnt++ == 0 && vp->v_usecount == 0)
    950       1.63   hannken 		lru_requeue(vp, lru_which(vp));
    951        1.1     rmind }
    952        1.1     rmind 
    953        1.1     rmind /*
    954  1.105.2.5        ad  * Page or buffer structure gets a reference.
    955  1.105.2.5        ad  */
    956  1.105.2.5        ad void
    957  1.105.2.5        ad vhold(vnode_t *vp)
    958  1.105.2.5        ad {
    959  1.105.2.5        ad 
    960  1.105.2.5        ad 	mutex_enter(vp->v_interlock);
    961  1.105.2.5        ad 	vholdl(vp);
    962  1.105.2.5        ad 	mutex_exit(vp->v_interlock);
    963  1.105.2.5        ad }
    964  1.105.2.5        ad 
    965  1.105.2.5        ad /*
    966        1.1     rmind  * Page or buffer structure frees a reference.
    967        1.1     rmind  * Called with v_interlock held.
    968        1.1     rmind  */
    969        1.1     rmind void
    970        1.1     rmind holdrelel(vnode_t *vp)
    971        1.1     rmind {
    972        1.1     rmind 
    973        1.9     rmind 	KASSERT(mutex_owned(vp->v_interlock));
    974        1.1     rmind 
    975        1.1     rmind 	if (vp->v_holdcnt <= 0) {
    976       1.11  christos 		vnpanic(vp, "%s: holdcnt vp %p", __func__, vp);
    977        1.1     rmind 	}
    978        1.1     rmind 
    979  1.105.2.8        ad 	vp->v_holdcnt--;
    980  1.105.2.8        ad 	if (vp->v_holdcnt == 0 && vp->v_usecount == 0)
    981       1.63   hannken 		lru_requeue(vp, lru_which(vp));
    982        1.1     rmind }
    983        1.1     rmind 
    984        1.1     rmind /*
    985  1.105.2.5        ad  * Page or buffer structure frees a reference.
    986  1.105.2.5        ad  */
    987  1.105.2.5        ad void
    988  1.105.2.5        ad holdrele(vnode_t *vp)
    989  1.105.2.5        ad {
    990  1.105.2.5        ad 
    991  1.105.2.5        ad 	mutex_enter(vp->v_interlock);
    992  1.105.2.5        ad 	holdrelel(vp);
    993  1.105.2.5        ad 	mutex_exit(vp->v_interlock);
    994  1.105.2.5        ad }
    995  1.105.2.5        ad 
    996  1.105.2.5        ad /*
    997       1.33   hannken  * Recycle an unused vnode if caller holds the last reference.
    998        1.1     rmind  */
    999       1.33   hannken bool
   1000       1.33   hannken vrecycle(vnode_t *vp)
   1001        1.1     rmind {
   1002       1.60   hannken 	int error __diagused;
   1003       1.46   hannken 
   1004       1.33   hannken 	mutex_enter(vp->v_interlock);
   1005       1.33   hannken 
   1006       1.60   hannken 	/* Make sure we hold the last reference. */
   1007       1.60   hannken 	VSTATE_WAIT_STABLE(vp);
   1008       1.33   hannken 	if (vp->v_usecount != 1) {
   1009       1.33   hannken 		mutex_exit(vp->v_interlock);
   1010       1.33   hannken 		return false;
   1011        1.1     rmind 	}
   1012       1.60   hannken 
   1013       1.60   hannken 	/* If the vnode is already clean we're done. */
   1014       1.94   hannken 	if (VSTATE_GET(vp) != VS_LOADED) {
   1015       1.60   hannken 		VSTATE_ASSERT(vp, VS_RECLAIMED);
   1016  1.105.2.2        ad 		vrelel(vp, 0, LK_NONE);
   1017       1.60   hannken 		return true;
   1018       1.60   hannken 	}
   1019       1.60   hannken 
   1020       1.60   hannken 	/* Prevent further references until the vnode is locked. */
   1021       1.94   hannken 	VSTATE_CHANGE(vp, VS_LOADED, VS_BLOCKED);
   1022       1.60   hannken 	mutex_exit(vp->v_interlock);
   1023       1.60   hannken 
   1024       1.73   hannken 	/*
   1025       1.73   hannken 	 * On a leaf file system this lock will always succeed as we hold
   1026       1.73   hannken 	 * the last reference and prevent further references.
   1027       1.73   hannken 	 * On layered file systems waiting for the lock would open a can of
   1028       1.73   hannken 	 * deadlocks as the lower vnodes may have other active references.
   1029       1.73   hannken 	 */
   1030       1.76   hannken 	error = vn_lock(vp, LK_EXCLUSIVE | LK_RETRY | LK_NOWAIT);
   1031       1.60   hannken 
   1032       1.60   hannken 	mutex_enter(vp->v_interlock);
   1033       1.94   hannken 	VSTATE_CHANGE(vp, VS_BLOCKED, VS_LOADED);
   1034       1.60   hannken 
   1035       1.73   hannken 	if (error) {
   1036       1.73   hannken 		mutex_exit(vp->v_interlock);
   1037       1.73   hannken 		return false;
   1038       1.73   hannken 	}
   1039       1.73   hannken 
   1040       1.68   hannken 	KASSERT(vp->v_usecount == 1);
   1041       1.54   hannken 	vcache_reclaim(vp);
   1042  1.105.2.2        ad 	vrelel(vp, 0, LK_NONE);
   1043       1.60   hannken 
   1044       1.33   hannken 	return true;
   1045        1.1     rmind }
   1046        1.1     rmind 
   1047        1.1     rmind /*
   1048       1.92   hannken  * Helper for vrevoke() to propagate suspension from lastmp
   1049       1.92   hannken  * to thismp.  Both args may be NULL.
   1050       1.92   hannken  * Returns the currently suspended file system or NULL.
   1051       1.92   hannken  */
   1052       1.92   hannken static struct mount *
   1053       1.92   hannken vrevoke_suspend_next(struct mount *lastmp, struct mount *thismp)
   1054       1.92   hannken {
   1055       1.92   hannken 	int error;
   1056       1.92   hannken 
   1057       1.92   hannken 	if (lastmp == thismp)
   1058       1.92   hannken 		return thismp;
   1059       1.92   hannken 
   1060       1.92   hannken 	if (lastmp != NULL)
   1061       1.92   hannken 		vfs_resume(lastmp);
   1062       1.92   hannken 
   1063       1.92   hannken 	if (thismp == NULL)
   1064       1.92   hannken 		return NULL;
   1065       1.92   hannken 
   1066       1.92   hannken 	do {
   1067       1.92   hannken 		error = vfs_suspend(thismp, 0);
   1068       1.92   hannken 	} while (error == EINTR || error == ERESTART);
   1069       1.92   hannken 
   1070       1.92   hannken 	if (error == 0)
   1071       1.92   hannken 		return thismp;
   1072       1.92   hannken 
   1073       1.92   hannken 	KASSERT(error == EOPNOTSUPP);
   1074       1.92   hannken 	return NULL;
   1075       1.92   hannken }
   1076       1.92   hannken 
   1077       1.92   hannken /*
   1078        1.1     rmind  * Eliminate all activity associated with the requested vnode
   1079        1.1     rmind  * and with all vnodes aliased to the requested vnode.
   1080        1.1     rmind  */
   1081        1.1     rmind void
   1082        1.1     rmind vrevoke(vnode_t *vp)
   1083        1.1     rmind {
   1084       1.88   hannken 	struct mount *mp;
   1085       1.19   hannken 	vnode_t *vq;
   1086        1.1     rmind 	enum vtype type;
   1087        1.1     rmind 	dev_t dev;
   1088        1.1     rmind 
   1089        1.1     rmind 	KASSERT(vp->v_usecount > 0);
   1090        1.1     rmind 
   1091       1.92   hannken 	mp = vrevoke_suspend_next(NULL, vp->v_mount);
   1092       1.88   hannken 
   1093        1.9     rmind 	mutex_enter(vp->v_interlock);
   1094       1.52   hannken 	VSTATE_WAIT_STABLE(vp);
   1095       1.57   hannken 	if (VSTATE_GET(vp) == VS_RECLAIMED) {
   1096        1.9     rmind 		mutex_exit(vp->v_interlock);
   1097        1.1     rmind 	} else if (vp->v_type != VBLK && vp->v_type != VCHR) {
   1098  1.105.2.5        ad 		atomic_inc_uint(&vp->v_usecount);
   1099       1.29  christos 		mutex_exit(vp->v_interlock);
   1100       1.29  christos 		vgone(vp);
   1101        1.1     rmind 	} else {
   1102        1.1     rmind 		dev = vp->v_rdev;
   1103        1.1     rmind 		type = vp->v_type;
   1104        1.9     rmind 		mutex_exit(vp->v_interlock);
   1105        1.1     rmind 
   1106       1.88   hannken 		while (spec_node_lookup_by_dev(type, dev, &vq) == 0) {
   1107       1.92   hannken 			mp = vrevoke_suspend_next(mp, vq->v_mount);
   1108       1.88   hannken 			vgone(vq);
   1109       1.88   hannken 		}
   1110        1.1     rmind 	}
   1111       1.92   hannken 	vrevoke_suspend_next(mp, NULL);
   1112        1.1     rmind }
   1113        1.1     rmind 
   1114        1.1     rmind /*
   1115        1.1     rmind  * Eliminate all activity associated with a vnode in preparation for
   1116        1.1     rmind  * reuse.  Drops a reference from the vnode.
   1117        1.1     rmind  */
   1118        1.1     rmind void
   1119        1.1     rmind vgone(vnode_t *vp)
   1120        1.1     rmind {
   1121  1.105.2.2        ad 	int lktype;
   1122        1.1     rmind 
   1123      1.103   hannken 	KASSERT(vp->v_mount == dead_rootmount || fstrans_is_owner(vp->v_mount));
   1124       1.93   hannken 
   1125       1.76   hannken 	vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
   1126  1.105.2.2        ad 	lktype = LK_EXCLUSIVE;
   1127        1.9     rmind 	mutex_enter(vp->v_interlock);
   1128       1.76   hannken 	VSTATE_WAIT_STABLE(vp);
   1129  1.105.2.2        ad 	if (VSTATE_GET(vp) == VS_LOADED) {
   1130       1.76   hannken 		vcache_reclaim(vp);
   1131  1.105.2.2        ad 		lktype = LK_NONE;
   1132  1.105.2.2        ad 	}
   1133       1.76   hannken 	VSTATE_ASSERT(vp, VS_RECLAIMED);
   1134  1.105.2.2        ad 	vrelel(vp, 0, lktype);
   1135        1.1     rmind }
   1136        1.1     rmind 
   1137       1.36   hannken static inline uint32_t
   1138       1.36   hannken vcache_hash(const struct vcache_key *key)
   1139       1.36   hannken {
   1140       1.36   hannken 	uint32_t hash = HASH32_BUF_INIT;
   1141       1.36   hannken 
   1142       1.97   hannken 	KASSERT(key->vk_key_len > 0);
   1143       1.97   hannken 
   1144       1.36   hannken 	hash = hash32_buf(&key->vk_mount, sizeof(struct mount *), hash);
   1145       1.36   hannken 	hash = hash32_buf(key->vk_key, key->vk_key_len, hash);
   1146       1.36   hannken 	return hash;
   1147       1.36   hannken }
   1148       1.36   hannken 
   1149       1.36   hannken static void
   1150       1.36   hannken vcache_init(void)
   1151       1.36   hannken {
   1152       1.36   hannken 
   1153  1.105.2.5        ad 	vcache_pool = pool_cache_init(sizeof(vnode_impl_t), coherency_unit,
   1154  1.105.2.5        ad 	    0, 0, "vcachepl", NULL, IPL_NONE, NULL, NULL, NULL);
   1155       1.69   hannken 	KASSERT(vcache_pool != NULL);
   1156       1.69   hannken 	mutex_init(&vcache_lock, MUTEX_DEFAULT, IPL_NONE);
   1157       1.69   hannken 	cv_init(&vcache_cv, "vcache");
   1158       1.69   hannken 	vcache_hashsize = desiredvnodes;
   1159       1.69   hannken 	vcache_hashtab = hashinit(desiredvnodes, HASH_SLIST, true,
   1160       1.69   hannken 	    &vcache_hashmask);
   1161       1.36   hannken }
   1162       1.36   hannken 
   1163       1.36   hannken static void
   1164       1.36   hannken vcache_reinit(void)
   1165       1.36   hannken {
   1166       1.36   hannken 	int i;
   1167       1.36   hannken 	uint32_t hash;
   1168       1.36   hannken 	u_long oldmask, newmask;
   1169       1.36   hannken 	struct hashhead *oldtab, *newtab;
   1170       1.70   hannken 	vnode_impl_t *vip;
   1171       1.36   hannken 
   1172       1.36   hannken 	newtab = hashinit(desiredvnodes, HASH_SLIST, true, &newmask);
   1173       1.69   hannken 	mutex_enter(&vcache_lock);
   1174       1.69   hannken 	oldtab = vcache_hashtab;
   1175       1.69   hannken 	oldmask = vcache_hashmask;
   1176       1.69   hannken 	vcache_hashsize = desiredvnodes;
   1177       1.69   hannken 	vcache_hashtab = newtab;
   1178       1.69   hannken 	vcache_hashmask = newmask;
   1179       1.36   hannken 	for (i = 0; i <= oldmask; i++) {
   1180       1.70   hannken 		while ((vip = SLIST_FIRST(&oldtab[i])) != NULL) {
   1181       1.70   hannken 			SLIST_REMOVE(&oldtab[i], vip, vnode_impl, vi_hash);
   1182       1.70   hannken 			hash = vcache_hash(&vip->vi_key);
   1183       1.69   hannken 			SLIST_INSERT_HEAD(&newtab[hash & vcache_hashmask],
   1184       1.70   hannken 			    vip, vi_hash);
   1185       1.36   hannken 		}
   1186       1.36   hannken 	}
   1187       1.69   hannken 	mutex_exit(&vcache_lock);
   1188       1.36   hannken 	hashdone(oldtab, HASH_SLIST, oldmask);
   1189       1.36   hannken }
   1190       1.36   hannken 
   1191       1.57   hannken static inline vnode_impl_t *
   1192       1.36   hannken vcache_hash_lookup(const struct vcache_key *key, uint32_t hash)
   1193       1.36   hannken {
   1194       1.36   hannken 	struct hashhead *hashp;
   1195       1.70   hannken 	vnode_impl_t *vip;
   1196       1.36   hannken 
   1197       1.69   hannken 	KASSERT(mutex_owned(&vcache_lock));
   1198       1.36   hannken 
   1199       1.69   hannken 	hashp = &vcache_hashtab[hash & vcache_hashmask];
   1200       1.70   hannken 	SLIST_FOREACH(vip, hashp, vi_hash) {
   1201       1.70   hannken 		if (key->vk_mount != vip->vi_key.vk_mount)
   1202       1.36   hannken 			continue;
   1203       1.70   hannken 		if (key->vk_key_len != vip->vi_key.vk_key_len)
   1204       1.36   hannken 			continue;
   1205       1.70   hannken 		if (memcmp(key->vk_key, vip->vi_key.vk_key, key->vk_key_len))
   1206       1.36   hannken 			continue;
   1207       1.70   hannken 		return vip;
   1208       1.36   hannken 	}
   1209       1.36   hannken 	return NULL;
   1210       1.36   hannken }
   1211       1.36   hannken 
   1212       1.36   hannken /*
   1213       1.50   hannken  * Allocate a new, uninitialized vcache node.
   1214       1.50   hannken  */
   1215       1.57   hannken static vnode_impl_t *
   1216       1.50   hannken vcache_alloc(void)
   1217       1.50   hannken {
   1218       1.70   hannken 	vnode_impl_t *vip;
   1219       1.50   hannken 	vnode_t *vp;
   1220       1.50   hannken 
   1221       1.70   hannken 	vip = pool_cache_get(vcache_pool, PR_WAITOK);
   1222  1.105.2.9        ad 	vp = VIMPL_TO_VNODE(vip);
   1223       1.70   hannken 	memset(vip, 0, sizeof(*vip));
   1224       1.50   hannken 
   1225  1.105.2.5        ad 	rw_init(&vip->vi_lock);
   1226  1.105.2.9        ad 	vp->v_interlock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE);
   1227       1.50   hannken 
   1228  1.105.2.9        ad 	uvm_obj_init(&vp->v_uobj, &uvm_vnodeops, true, 1);
   1229       1.50   hannken 	cv_init(&vp->v_cv, "vnode");
   1230  1.105.2.1        ad 	cache_vnode_init(vp);
   1231       1.50   hannken 
   1232       1.50   hannken 	vp->v_usecount = 1;
   1233       1.50   hannken 	vp->v_type = VNON;
   1234       1.50   hannken 	vp->v_size = vp->v_writesize = VSIZENOTSET;
   1235       1.50   hannken 
   1236       1.70   hannken 	vip->vi_state = VS_LOADING;
   1237       1.51   hannken 
   1238      1.104        ad 	lru_requeue(vp, &lru_list[LRU_FREE]);
   1239       1.63   hannken 
   1240       1.70   hannken 	return vip;
   1241       1.50   hannken }
   1242       1.50   hannken 
   1243       1.50   hannken /*
   1244       1.79   hannken  * Deallocate a vcache node in state VS_LOADING.
   1245       1.79   hannken  *
   1246       1.79   hannken  * vcache_lock held on entry and released on return.
   1247       1.79   hannken  */
   1248       1.79   hannken static void
   1249       1.79   hannken vcache_dealloc(vnode_impl_t *vip)
   1250       1.79   hannken {
   1251       1.79   hannken 	vnode_t *vp;
   1252       1.79   hannken 
   1253       1.79   hannken 	KASSERT(mutex_owned(&vcache_lock));
   1254       1.79   hannken 
   1255       1.79   hannken 	vp = VIMPL_TO_VNODE(vip);
   1256      1.102   hannken 	vfs_ref(dead_rootmount);
   1257      1.102   hannken 	vfs_insmntque(vp, dead_rootmount);
   1258       1.79   hannken 	mutex_enter(vp->v_interlock);
   1259       1.79   hannken 	vp->v_op = dead_vnodeop_p;
   1260       1.79   hannken 	VSTATE_CHANGE(vp, VS_LOADING, VS_RECLAIMED);
   1261       1.79   hannken 	mutex_exit(&vcache_lock);
   1262  1.105.2.2        ad 	vrelel(vp, 0, LK_NONE);
   1263       1.79   hannken }
   1264       1.79   hannken 
   1265       1.79   hannken /*
   1266       1.50   hannken  * Free an unused, unreferenced vcache node.
   1267       1.67   hannken  * v_interlock locked on entry.
   1268       1.50   hannken  */
   1269       1.50   hannken static void
   1270       1.70   hannken vcache_free(vnode_impl_t *vip)
   1271       1.50   hannken {
   1272       1.50   hannken 	vnode_t *vp;
   1273       1.50   hannken 
   1274       1.70   hannken 	vp = VIMPL_TO_VNODE(vip);
   1275       1.67   hannken 	KASSERT(mutex_owned(vp->v_interlock));
   1276       1.50   hannken 
   1277       1.50   hannken 	KASSERT(vp->v_usecount == 0);
   1278       1.67   hannken 	KASSERT(vp->v_holdcnt == 0);
   1279       1.67   hannken 	KASSERT(vp->v_writecount == 0);
   1280       1.67   hannken 	lru_requeue(vp, NULL);
   1281       1.67   hannken 	mutex_exit(vp->v_interlock);
   1282       1.67   hannken 
   1283       1.67   hannken 	vfs_insmntque(vp, NULL);
   1284       1.67   hannken 	if (vp->v_type == VBLK || vp->v_type == VCHR)
   1285       1.67   hannken 		spec_node_destroy(vp);
   1286       1.50   hannken 
   1287  1.105.2.9        ad 	mutex_obj_free(vp->v_interlock);
   1288  1.105.2.5        ad 	rw_destroy(&vip->vi_lock);
   1289       1.50   hannken 	uvm_obj_destroy(&vp->v_uobj, true);
   1290       1.50   hannken 	cv_destroy(&vp->v_cv);
   1291  1.105.2.1        ad 	cache_vnode_fini(vp);
   1292       1.70   hannken 	pool_cache_put(vcache_pool, vip);
   1293       1.50   hannken }
   1294       1.50   hannken 
   1295       1.50   hannken /*
   1296       1.66   hannken  * Try to get an initial reference on this cached vnode.
   1297       1.66   hannken  * Returns zero on success,  ENOENT if the vnode has been reclaimed and
   1298       1.66   hannken  * EBUSY if the vnode state is unstable.
   1299       1.66   hannken  *
   1300       1.66   hannken  * v_interlock locked on entry and unlocked on exit.
   1301       1.66   hannken  */
   1302       1.66   hannken int
   1303       1.66   hannken vcache_tryvget(vnode_t *vp)
   1304       1.66   hannken {
   1305       1.67   hannken 	int error = 0;
   1306       1.66   hannken 
   1307       1.66   hannken 	KASSERT(mutex_owned(vp->v_interlock));
   1308       1.66   hannken 
   1309       1.67   hannken 	if (__predict_false(VSTATE_GET(vp) == VS_RECLAIMED))
   1310       1.67   hannken 		error = ENOENT;
   1311       1.94   hannken 	else if (__predict_false(VSTATE_GET(vp) != VS_LOADED))
   1312       1.67   hannken 		error = EBUSY;
   1313  1.105.2.5        ad 	else if (vp->v_usecount == 0)
   1314  1.105.2.5        ad 		vp->v_usecount = 1;
   1315       1.67   hannken 	else
   1316  1.105.2.5        ad 		atomic_inc_uint(&vp->v_usecount);
   1317       1.66   hannken 
   1318       1.66   hannken 	mutex_exit(vp->v_interlock);
   1319       1.66   hannken 
   1320       1.67   hannken 	return error;
   1321       1.66   hannken }
   1322       1.66   hannken 
   1323       1.66   hannken /*
   1324       1.66   hannken  * Try to get an initial reference on this cached vnode.
   1325       1.66   hannken  * Returns zero on success and  ENOENT if the vnode has been reclaimed.
   1326       1.66   hannken  * Will wait for the vnode state to be stable.
   1327       1.66   hannken  *
   1328       1.66   hannken  * v_interlock locked on entry and unlocked on exit.
   1329       1.66   hannken  */
   1330       1.66   hannken int
   1331       1.66   hannken vcache_vget(vnode_t *vp)
   1332       1.66   hannken {
   1333       1.66   hannken 
   1334       1.66   hannken 	KASSERT(mutex_owned(vp->v_interlock));
   1335       1.66   hannken 
   1336       1.67   hannken 	/* Increment hold count to prevent vnode from disappearing. */
   1337       1.67   hannken 	vp->v_holdcnt++;
   1338       1.67   hannken 	VSTATE_WAIT_STABLE(vp);
   1339       1.67   hannken 	vp->v_holdcnt--;
   1340       1.66   hannken 
   1341       1.67   hannken 	/* If this was the last reference to a reclaimed vnode free it now. */
   1342       1.67   hannken 	if (__predict_false(VSTATE_GET(vp) == VS_RECLAIMED)) {
   1343       1.67   hannken 		if (vp->v_holdcnt == 0 && vp->v_usecount == 0)
   1344       1.67   hannken 			vcache_free(VNODE_TO_VIMPL(vp));
   1345       1.67   hannken 		else
   1346       1.67   hannken 			mutex_exit(vp->v_interlock);
   1347       1.66   hannken 		return ENOENT;
   1348       1.66   hannken 	}
   1349       1.94   hannken 	VSTATE_ASSERT(vp, VS_LOADED);
   1350  1.105.2.5        ad 	if (vp->v_usecount == 0)
   1351  1.105.2.5        ad 		vp->v_usecount = 1;
   1352  1.105.2.5        ad 	else
   1353  1.105.2.5        ad 		atomic_inc_uint(&vp->v_usecount);
   1354       1.66   hannken 	mutex_exit(vp->v_interlock);
   1355       1.66   hannken 
   1356       1.66   hannken 	return 0;
   1357       1.66   hannken }
   1358       1.66   hannken 
   1359       1.66   hannken /*
   1360       1.36   hannken  * Get a vnode / fs node pair by key and return it referenced through vpp.
   1361       1.36   hannken  */
   1362       1.36   hannken int
   1363       1.36   hannken vcache_get(struct mount *mp, const void *key, size_t key_len,
   1364       1.36   hannken     struct vnode **vpp)
   1365       1.36   hannken {
   1366       1.36   hannken 	int error;
   1367       1.36   hannken 	uint32_t hash;
   1368       1.36   hannken 	const void *new_key;
   1369       1.36   hannken 	struct vnode *vp;
   1370       1.36   hannken 	struct vcache_key vcache_key;
   1371       1.70   hannken 	vnode_impl_t *vip, *new_vip;
   1372       1.36   hannken 
   1373       1.36   hannken 	new_key = NULL;
   1374       1.36   hannken 	*vpp = NULL;
   1375       1.36   hannken 
   1376       1.36   hannken 	vcache_key.vk_mount = mp;
   1377       1.36   hannken 	vcache_key.vk_key = key;
   1378       1.36   hannken 	vcache_key.vk_key_len = key_len;
   1379       1.36   hannken 	hash = vcache_hash(&vcache_key);
   1380       1.36   hannken 
   1381       1.36   hannken again:
   1382       1.69   hannken 	mutex_enter(&vcache_lock);
   1383       1.70   hannken 	vip = vcache_hash_lookup(&vcache_key, hash);
   1384       1.36   hannken 
   1385       1.36   hannken 	/* If found, take a reference or retry. */
   1386       1.70   hannken 	if (__predict_true(vip != NULL)) {
   1387       1.52   hannken 		/*
   1388       1.52   hannken 		 * If the vnode is loading we cannot take the v_interlock
   1389       1.52   hannken 		 * here as it might change during load (see uvm_obj_setlock()).
   1390       1.69   hannken 		 * As changing state from VS_LOADING requires both vcache_lock
   1391       1.69   hannken 		 * and v_interlock it is safe to test with vcache_lock held.
   1392       1.52   hannken 		 *
   1393       1.57   hannken 		 * Wait for vnodes changing state from VS_LOADING and retry.
   1394       1.52   hannken 		 */
   1395       1.70   hannken 		if (__predict_false(vip->vi_state == VS_LOADING)) {
   1396       1.69   hannken 			cv_wait(&vcache_cv, &vcache_lock);
   1397       1.69   hannken 			mutex_exit(&vcache_lock);
   1398       1.52   hannken 			goto again;
   1399       1.52   hannken 		}
   1400       1.70   hannken 		vp = VIMPL_TO_VNODE(vip);
   1401       1.36   hannken 		mutex_enter(vp->v_interlock);
   1402       1.69   hannken 		mutex_exit(&vcache_lock);
   1403       1.66   hannken 		error = vcache_vget(vp);
   1404       1.36   hannken 		if (error == ENOENT)
   1405       1.36   hannken 			goto again;
   1406       1.36   hannken 		if (error == 0)
   1407       1.36   hannken 			*vpp = vp;
   1408       1.36   hannken 		KASSERT((error != 0) == (*vpp == NULL));
   1409       1.36   hannken 		return error;
   1410       1.36   hannken 	}
   1411       1.69   hannken 	mutex_exit(&vcache_lock);
   1412       1.36   hannken 
   1413       1.36   hannken 	/* Allocate and initialize a new vcache / vnode pair. */
   1414       1.87   hannken 	error = vfs_busy(mp);
   1415       1.36   hannken 	if (error)
   1416       1.36   hannken 		return error;
   1417       1.70   hannken 	new_vip = vcache_alloc();
   1418       1.70   hannken 	new_vip->vi_key = vcache_key;
   1419       1.70   hannken 	vp = VIMPL_TO_VNODE(new_vip);
   1420       1.69   hannken 	mutex_enter(&vcache_lock);
   1421       1.70   hannken 	vip = vcache_hash_lookup(&vcache_key, hash);
   1422       1.70   hannken 	if (vip == NULL) {
   1423       1.69   hannken 		SLIST_INSERT_HEAD(&vcache_hashtab[hash & vcache_hashmask],
   1424       1.70   hannken 		    new_vip, vi_hash);
   1425       1.70   hannken 		vip = new_vip;
   1426       1.36   hannken 	}
   1427       1.36   hannken 
   1428       1.36   hannken 	/* If another thread beat us inserting this node, retry. */
   1429       1.70   hannken 	if (vip != new_vip) {
   1430       1.79   hannken 		vcache_dealloc(new_vip);
   1431       1.87   hannken 		vfs_unbusy(mp);
   1432       1.36   hannken 		goto again;
   1433       1.36   hannken 	}
   1434       1.69   hannken 	mutex_exit(&vcache_lock);
   1435       1.36   hannken 
   1436       1.57   hannken 	/* Load the fs node.  Exclusive as new_node is VS_LOADING. */
   1437       1.36   hannken 	error = VFS_LOADVNODE(mp, vp, key, key_len, &new_key);
   1438       1.36   hannken 	if (error) {
   1439       1.69   hannken 		mutex_enter(&vcache_lock);
   1440       1.69   hannken 		SLIST_REMOVE(&vcache_hashtab[hash & vcache_hashmask],
   1441       1.70   hannken 		    new_vip, vnode_impl, vi_hash);
   1442       1.79   hannken 		vcache_dealloc(new_vip);
   1443       1.87   hannken 		vfs_unbusy(mp);
   1444       1.36   hannken 		KASSERT(*vpp == NULL);
   1445       1.36   hannken 		return error;
   1446       1.36   hannken 	}
   1447       1.36   hannken 	KASSERT(new_key != NULL);
   1448       1.36   hannken 	KASSERT(memcmp(key, new_key, key_len) == 0);
   1449       1.36   hannken 	KASSERT(vp->v_op != NULL);
   1450       1.36   hannken 	vfs_insmntque(vp, mp);
   1451       1.36   hannken 	if ((mp->mnt_iflag & IMNT_MPSAFE) != 0)
   1452       1.36   hannken 		vp->v_vflag |= VV_MPSAFE;
   1453       1.87   hannken 	vfs_ref(mp);
   1454       1.87   hannken 	vfs_unbusy(mp);
   1455       1.36   hannken 
   1456       1.36   hannken 	/* Finished loading, finalize node. */
   1457       1.69   hannken 	mutex_enter(&vcache_lock);
   1458       1.70   hannken 	new_vip->vi_key.vk_key = new_key;
   1459       1.39   hannken 	mutex_enter(vp->v_interlock);
   1460       1.94   hannken 	VSTATE_CHANGE(vp, VS_LOADING, VS_LOADED);
   1461       1.39   hannken 	mutex_exit(vp->v_interlock);
   1462       1.69   hannken 	mutex_exit(&vcache_lock);
   1463       1.36   hannken 	*vpp = vp;
   1464       1.36   hannken 	return 0;
   1465       1.36   hannken }
   1466       1.36   hannken 
   1467       1.36   hannken /*
   1468       1.40   hannken  * Create a new vnode / fs node pair and return it referenced through vpp.
   1469       1.40   hannken  */
   1470       1.40   hannken int
   1471       1.40   hannken vcache_new(struct mount *mp, struct vnode *dvp, struct vattr *vap,
   1472      1.101   hannken     kauth_cred_t cred, void *extra, struct vnode **vpp)
   1473       1.40   hannken {
   1474       1.40   hannken 	int error;
   1475       1.40   hannken 	uint32_t hash;
   1476       1.70   hannken 	struct vnode *vp, *ovp;
   1477       1.70   hannken 	vnode_impl_t *vip, *ovip;
   1478       1.40   hannken 
   1479       1.40   hannken 	*vpp = NULL;
   1480       1.40   hannken 
   1481       1.40   hannken 	/* Allocate and initialize a new vcache / vnode pair. */
   1482       1.87   hannken 	error = vfs_busy(mp);
   1483       1.40   hannken 	if (error)
   1484       1.40   hannken 		return error;
   1485       1.70   hannken 	vip = vcache_alloc();
   1486       1.70   hannken 	vip->vi_key.vk_mount = mp;
   1487       1.70   hannken 	vp = VIMPL_TO_VNODE(vip);
   1488       1.40   hannken 
   1489       1.40   hannken 	/* Create and load the fs node. */
   1490      1.101   hannken 	error = VFS_NEWVNODE(mp, dvp, vp, vap, cred, extra,
   1491       1.70   hannken 	    &vip->vi_key.vk_key_len, &vip->vi_key.vk_key);
   1492       1.40   hannken 	if (error) {
   1493       1.69   hannken 		mutex_enter(&vcache_lock);
   1494       1.79   hannken 		vcache_dealloc(vip);
   1495       1.87   hannken 		vfs_unbusy(mp);
   1496       1.40   hannken 		KASSERT(*vpp == NULL);
   1497       1.40   hannken 		return error;
   1498       1.40   hannken 	}
   1499       1.40   hannken 	KASSERT(vp->v_op != NULL);
   1500       1.97   hannken 	KASSERT((vip->vi_key.vk_key_len == 0) == (mp == dead_rootmount));
   1501       1.97   hannken 	if (vip->vi_key.vk_key_len > 0) {
   1502       1.97   hannken 		KASSERT(vip->vi_key.vk_key != NULL);
   1503       1.97   hannken 		hash = vcache_hash(&vip->vi_key);
   1504       1.40   hannken 
   1505       1.97   hannken 		/*
   1506       1.97   hannken 		 * Wait for previous instance to be reclaimed,
   1507       1.97   hannken 		 * then insert new node.
   1508       1.97   hannken 		 */
   1509       1.97   hannken 		mutex_enter(&vcache_lock);
   1510       1.97   hannken 		while ((ovip = vcache_hash_lookup(&vip->vi_key, hash))) {
   1511       1.97   hannken 			ovp = VIMPL_TO_VNODE(ovip);
   1512       1.97   hannken 			mutex_enter(ovp->v_interlock);
   1513       1.97   hannken 			mutex_exit(&vcache_lock);
   1514       1.97   hannken 			error = vcache_vget(ovp);
   1515       1.97   hannken 			KASSERT(error == ENOENT);
   1516       1.97   hannken 			mutex_enter(&vcache_lock);
   1517       1.97   hannken 		}
   1518       1.97   hannken 		SLIST_INSERT_HEAD(&vcache_hashtab[hash & vcache_hashmask],
   1519       1.97   hannken 		    vip, vi_hash);
   1520       1.69   hannken 		mutex_exit(&vcache_lock);
   1521       1.40   hannken 	}
   1522       1.40   hannken 	vfs_insmntque(vp, mp);
   1523       1.40   hannken 	if ((mp->mnt_iflag & IMNT_MPSAFE) != 0)
   1524       1.40   hannken 		vp->v_vflag |= VV_MPSAFE;
   1525       1.87   hannken 	vfs_ref(mp);
   1526       1.87   hannken 	vfs_unbusy(mp);
   1527       1.40   hannken 
   1528       1.40   hannken 	/* Finished loading, finalize node. */
   1529       1.69   hannken 	mutex_enter(&vcache_lock);
   1530       1.52   hannken 	mutex_enter(vp->v_interlock);
   1531       1.94   hannken 	VSTATE_CHANGE(vp, VS_LOADING, VS_LOADED);
   1532       1.69   hannken 	mutex_exit(&vcache_lock);
   1533       1.40   hannken 	mutex_exit(vp->v_interlock);
   1534       1.40   hannken 	*vpp = vp;
   1535       1.40   hannken 	return 0;
   1536       1.40   hannken }
   1537       1.40   hannken 
   1538       1.40   hannken /*
   1539       1.65   hannken  * Prepare key change: update old cache nodes key and lock new cache node.
   1540       1.37   hannken  * Return an error if the new node already exists.
   1541       1.37   hannken  */
   1542       1.37   hannken int
   1543       1.37   hannken vcache_rekey_enter(struct mount *mp, struct vnode *vp,
   1544       1.37   hannken     const void *old_key, size_t old_key_len,
   1545       1.37   hannken     const void *new_key, size_t new_key_len)
   1546       1.37   hannken {
   1547       1.37   hannken 	uint32_t old_hash, new_hash;
   1548       1.37   hannken 	struct vcache_key old_vcache_key, new_vcache_key;
   1549       1.70   hannken 	vnode_impl_t *vip, *new_vip;
   1550       1.37   hannken 
   1551       1.37   hannken 	old_vcache_key.vk_mount = mp;
   1552       1.37   hannken 	old_vcache_key.vk_key = old_key;
   1553       1.37   hannken 	old_vcache_key.vk_key_len = old_key_len;
   1554       1.37   hannken 	old_hash = vcache_hash(&old_vcache_key);
   1555       1.37   hannken 
   1556       1.37   hannken 	new_vcache_key.vk_mount = mp;
   1557       1.37   hannken 	new_vcache_key.vk_key = new_key;
   1558       1.37   hannken 	new_vcache_key.vk_key_len = new_key_len;
   1559       1.37   hannken 	new_hash = vcache_hash(&new_vcache_key);
   1560       1.37   hannken 
   1561       1.70   hannken 	new_vip = vcache_alloc();
   1562       1.70   hannken 	new_vip->vi_key = new_vcache_key;
   1563       1.37   hannken 
   1564       1.52   hannken 	/* Insert locked new node used as placeholder. */
   1565       1.69   hannken 	mutex_enter(&vcache_lock);
   1566       1.70   hannken 	vip = vcache_hash_lookup(&new_vcache_key, new_hash);
   1567       1.70   hannken 	if (vip != NULL) {
   1568       1.79   hannken 		vcache_dealloc(new_vip);
   1569       1.37   hannken 		return EEXIST;
   1570       1.37   hannken 	}
   1571       1.69   hannken 	SLIST_INSERT_HEAD(&vcache_hashtab[new_hash & vcache_hashmask],
   1572       1.70   hannken 	    new_vip, vi_hash);
   1573       1.49   hannken 
   1574       1.65   hannken 	/* Replace old nodes key with the temporary copy. */
   1575       1.70   hannken 	vip = vcache_hash_lookup(&old_vcache_key, old_hash);
   1576       1.70   hannken 	KASSERT(vip != NULL);
   1577       1.70   hannken 	KASSERT(VIMPL_TO_VNODE(vip) == vp);
   1578       1.70   hannken 	KASSERT(vip->vi_key.vk_key != old_vcache_key.vk_key);
   1579       1.70   hannken 	vip->vi_key = old_vcache_key;
   1580       1.69   hannken 	mutex_exit(&vcache_lock);
   1581       1.37   hannken 	return 0;
   1582       1.37   hannken }
   1583       1.37   hannken 
   1584       1.37   hannken /*
   1585       1.65   hannken  * Key change complete: update old node and remove placeholder.
   1586       1.37   hannken  */
   1587       1.37   hannken void
   1588       1.37   hannken vcache_rekey_exit(struct mount *mp, struct vnode *vp,
   1589       1.37   hannken     const void *old_key, size_t old_key_len,
   1590       1.37   hannken     const void *new_key, size_t new_key_len)
   1591       1.37   hannken {
   1592       1.37   hannken 	uint32_t old_hash, new_hash;
   1593       1.37   hannken 	struct vcache_key old_vcache_key, new_vcache_key;
   1594       1.70   hannken 	vnode_impl_t *vip, *new_vip;
   1595       1.70   hannken 	struct vnode *new_vp;
   1596       1.37   hannken 
   1597       1.37   hannken 	old_vcache_key.vk_mount = mp;
   1598       1.37   hannken 	old_vcache_key.vk_key = old_key;
   1599       1.37   hannken 	old_vcache_key.vk_key_len = old_key_len;
   1600       1.37   hannken 	old_hash = vcache_hash(&old_vcache_key);
   1601       1.37   hannken 
   1602       1.37   hannken 	new_vcache_key.vk_mount = mp;
   1603       1.37   hannken 	new_vcache_key.vk_key = new_key;
   1604       1.37   hannken 	new_vcache_key.vk_key_len = new_key_len;
   1605       1.37   hannken 	new_hash = vcache_hash(&new_vcache_key);
   1606       1.37   hannken 
   1607       1.69   hannken 	mutex_enter(&vcache_lock);
   1608       1.49   hannken 
   1609       1.49   hannken 	/* Lookup old and new node. */
   1610       1.70   hannken 	vip = vcache_hash_lookup(&old_vcache_key, old_hash);
   1611       1.70   hannken 	KASSERT(vip != NULL);
   1612       1.70   hannken 	KASSERT(VIMPL_TO_VNODE(vip) == vp);
   1613       1.70   hannken 
   1614       1.70   hannken 	new_vip = vcache_hash_lookup(&new_vcache_key, new_hash);
   1615       1.70   hannken 	KASSERT(new_vip != NULL);
   1616       1.70   hannken 	KASSERT(new_vip->vi_key.vk_key_len == new_key_len);
   1617       1.70   hannken 	new_vp = VIMPL_TO_VNODE(new_vip);
   1618       1.70   hannken 	mutex_enter(new_vp->v_interlock);
   1619       1.70   hannken 	VSTATE_ASSERT(VIMPL_TO_VNODE(new_vip), VS_LOADING);
   1620       1.79   hannken 	mutex_exit(new_vp->v_interlock);
   1621       1.49   hannken 
   1622       1.49   hannken 	/* Rekey old node and put it onto its new hashlist. */
   1623       1.70   hannken 	vip->vi_key = new_vcache_key;
   1624       1.49   hannken 	if (old_hash != new_hash) {
   1625       1.69   hannken 		SLIST_REMOVE(&vcache_hashtab[old_hash & vcache_hashmask],
   1626       1.70   hannken 		    vip, vnode_impl, vi_hash);
   1627       1.69   hannken 		SLIST_INSERT_HEAD(&vcache_hashtab[new_hash & vcache_hashmask],
   1628       1.70   hannken 		    vip, vi_hash);
   1629       1.49   hannken 	}
   1630       1.49   hannken 
   1631       1.49   hannken 	/* Remove new node used as placeholder. */
   1632       1.69   hannken 	SLIST_REMOVE(&vcache_hashtab[new_hash & vcache_hashmask],
   1633       1.70   hannken 	    new_vip, vnode_impl, vi_hash);
   1634       1.79   hannken 	vcache_dealloc(new_vip);
   1635       1.37   hannken }
   1636       1.37   hannken 
   1637       1.37   hannken /*
   1638       1.54   hannken  * Disassociate the underlying file system from a vnode.
   1639       1.54   hannken  *
   1640       1.54   hannken  * Must be called with vnode locked and will return unlocked.
   1641       1.54   hannken  * Must be called with the interlock held, and will return with it held.
   1642       1.54   hannken  */
   1643       1.54   hannken static void
   1644       1.54   hannken vcache_reclaim(vnode_t *vp)
   1645       1.54   hannken {
   1646       1.54   hannken 	lwp_t *l = curlwp;
   1647       1.70   hannken 	vnode_impl_t *vip = VNODE_TO_VIMPL(vp);
   1648       1.74   hannken 	struct mount *mp = vp->v_mount;
   1649       1.55   hannken 	uint32_t hash;
   1650       1.55   hannken 	uint8_t temp_buf[64], *temp_key;
   1651       1.55   hannken 	size_t temp_key_len;
   1652       1.54   hannken 	bool recycle, active;
   1653       1.54   hannken 	int error;
   1654       1.54   hannken 
   1655       1.54   hannken 	KASSERT((vp->v_vflag & VV_LOCKSWORK) == 0 ||
   1656       1.54   hannken 	    VOP_ISLOCKED(vp) == LK_EXCLUSIVE);
   1657       1.54   hannken 	KASSERT(mutex_owned(vp->v_interlock));
   1658       1.54   hannken 	KASSERT(vp->v_usecount != 0);
   1659       1.54   hannken 
   1660       1.54   hannken 	active = (vp->v_usecount > 1);
   1661       1.70   hannken 	temp_key_len = vip->vi_key.vk_key_len;
   1662       1.54   hannken 	/*
   1663       1.54   hannken 	 * Prevent the vnode from being recycled or brought into use
   1664       1.54   hannken 	 * while we clean it out.
   1665       1.54   hannken 	 */
   1666       1.94   hannken 	VSTATE_CHANGE(vp, VS_LOADED, VS_RECLAIMING);
   1667  1.105.2.9        ad 	mutex_exit(vp->v_interlock);
   1668  1.105.2.9        ad 
   1669  1.105.2.9        ad 	rw_enter(vp->v_uobj.vmobjlock, RW_WRITER);
   1670  1.105.2.9        ad 	mutex_enter(vp->v_interlock);
   1671      1.104        ad 	if ((vp->v_iflag & VI_EXECMAP) != 0 && vp->v_uobj.uo_npages != 0) {
   1672      1.105        ad 		cpu_count(CPU_COUNT_EXECPAGES, -vp->v_uobj.uo_npages);
   1673      1.105        ad 		cpu_count(CPU_COUNT_FILEPAGES, vp->v_uobj.uo_npages);
   1674       1.54   hannken 	}
   1675       1.54   hannken 	vp->v_iflag &= ~(VI_TEXT|VI_EXECMAP);
   1676       1.54   hannken 	mutex_exit(vp->v_interlock);
   1677  1.105.2.9        ad 	rw_exit(vp->v_uobj.vmobjlock);
   1678       1.54   hannken 
   1679  1.105.2.4        ad 	/*
   1680  1.105.2.4        ad 	 * With vnode state set to reclaiming, purge name cache immediately
   1681  1.105.2.4        ad 	 * to prevent new handles on vnode, and wait for existing threads
   1682  1.105.2.4        ad 	 * trying to get a handle to notice VS_RECLAIMED status and abort.
   1683  1.105.2.4        ad 	 */
   1684  1.105.2.4        ad 	cache_purge(vp);
   1685  1.105.2.4        ad 
   1686       1.55   hannken 	/* Replace the vnode key with a temporary copy. */
   1687       1.70   hannken 	if (vip->vi_key.vk_key_len > sizeof(temp_buf)) {
   1688       1.55   hannken 		temp_key = kmem_alloc(temp_key_len, KM_SLEEP);
   1689       1.55   hannken 	} else {
   1690       1.55   hannken 		temp_key = temp_buf;
   1691       1.55   hannken 	}
   1692       1.97   hannken 	if (vip->vi_key.vk_key_len > 0) {
   1693       1.97   hannken 		mutex_enter(&vcache_lock);
   1694       1.97   hannken 		memcpy(temp_key, vip->vi_key.vk_key, temp_key_len);
   1695       1.97   hannken 		vip->vi_key.vk_key = temp_key;
   1696       1.97   hannken 		mutex_exit(&vcache_lock);
   1697       1.97   hannken 	}
   1698       1.55   hannken 
   1699       1.96   hannken 	fstrans_start(mp);
   1700       1.74   hannken 
   1701       1.54   hannken 	/*
   1702       1.54   hannken 	 * Clean out any cached data associated with the vnode.
   1703       1.54   hannken 	 * If purging an active vnode, it must be closed and
   1704       1.60   hannken 	 * deactivated before being reclaimed.
   1705       1.54   hannken 	 */
   1706       1.54   hannken 	error = vinvalbuf(vp, V_SAVE, NOCRED, l, 0, 0);
   1707       1.54   hannken 	if (error != 0) {
   1708       1.54   hannken 		if (wapbl_vphaswapbl(vp))
   1709       1.54   hannken 			WAPBL_DISCARD(wapbl_vptomp(vp));
   1710       1.54   hannken 		error = vinvalbuf(vp, 0, NOCRED, l, 0, 0);
   1711       1.54   hannken 	}
   1712       1.54   hannken 	KASSERTMSG((error == 0), "vinvalbuf failed: %d", error);
   1713       1.54   hannken 	KASSERT((vp->v_iflag & VI_ONWORKLST) == 0);
   1714       1.54   hannken 	if (active && (vp->v_type == VBLK || vp->v_type == VCHR)) {
   1715       1.54   hannken 		 spec_node_revoke(vp);
   1716       1.54   hannken 	}
   1717       1.54   hannken 
   1718       1.60   hannken 	/*
   1719       1.60   hannken 	 * Disassociate the underlying file system from the vnode.
   1720       1.90  riastrad 	 * VOP_INACTIVE leaves the vnode locked; VOP_RECLAIM unlocks
   1721       1.90  riastrad 	 * the vnode, and may destroy the vnode so that VOP_UNLOCK
   1722       1.90  riastrad 	 * would no longer function.
   1723       1.60   hannken 	 */
   1724       1.60   hannken 	VOP_INACTIVE(vp, &recycle);
   1725       1.91  riastrad 	KASSERT((vp->v_vflag & VV_LOCKSWORK) == 0 ||
   1726       1.91  riastrad 	    VOP_ISLOCKED(vp) == LK_EXCLUSIVE);
   1727       1.54   hannken 	if (VOP_RECLAIM(vp)) {
   1728       1.54   hannken 		vnpanic(vp, "%s: cannot reclaim", __func__);
   1729       1.54   hannken 	}
   1730       1.54   hannken 
   1731       1.54   hannken 	KASSERT(vp->v_data == NULL);
   1732  1.105.2.9        ad 	KASSERT((vp->v_iflag & VI_PAGES) == 0);
   1733       1.54   hannken 
   1734       1.54   hannken 	if (vp->v_type == VREG && vp->v_ractx != NULL) {
   1735       1.54   hannken 		uvm_ra_freectx(vp->v_ractx);
   1736       1.54   hannken 		vp->v_ractx = NULL;
   1737       1.54   hannken 	}
   1738       1.54   hannken 
   1739       1.97   hannken 	if (vip->vi_key.vk_key_len > 0) {
   1740       1.55   hannken 	/* Remove from vnode cache. */
   1741       1.97   hannken 		hash = vcache_hash(&vip->vi_key);
   1742       1.97   hannken 		mutex_enter(&vcache_lock);
   1743       1.97   hannken 		KASSERT(vip == vcache_hash_lookup(&vip->vi_key, hash));
   1744       1.97   hannken 		SLIST_REMOVE(&vcache_hashtab[hash & vcache_hashmask],
   1745       1.97   hannken 		    vip, vnode_impl, vi_hash);
   1746       1.97   hannken 		mutex_exit(&vcache_lock);
   1747       1.97   hannken 	}
   1748       1.55   hannken 	if (temp_key != temp_buf)
   1749       1.55   hannken 		kmem_free(temp_key, temp_key_len);
   1750       1.55   hannken 
   1751       1.54   hannken 	/* Done with purge, notify sleepers of the grim news. */
   1752       1.54   hannken 	mutex_enter(vp->v_interlock);
   1753       1.54   hannken 	vp->v_op = dead_vnodeop_p;
   1754       1.54   hannken 	vp->v_vflag |= VV_LOCKSWORK;
   1755       1.57   hannken 	VSTATE_CHANGE(vp, VS_RECLAIMING, VS_RECLAIMED);
   1756       1.54   hannken 	vp->v_tag = VT_NON;
   1757       1.54   hannken 	KNOTE(&vp->v_klist, NOTE_REVOKE);
   1758       1.80   hannken 	mutex_exit(vp->v_interlock);
   1759       1.54   hannken 
   1760       1.80   hannken 	/*
   1761       1.80   hannken 	 * Move to dead mount.  Must be after changing the operations
   1762       1.80   hannken 	 * vector as vnode operations enter the mount before using the
   1763       1.80   hannken 	 * operations vector.  See sys/kern/vnode_if.c.
   1764       1.80   hannken 	 */
   1765       1.80   hannken 	vp->v_vflag &= ~VV_ROOT;
   1766       1.86   hannken 	vfs_ref(dead_rootmount);
   1767       1.80   hannken 	vfs_insmntque(vp, dead_rootmount);
   1768       1.80   hannken 
   1769  1.105.2.7        ad #ifdef PAX_SEGVGUARD
   1770  1.105.2.7        ad 	pax_segvguard_cleanup(vp);
   1771  1.105.2.7        ad #endif /* PAX_SEGVGUARD */
   1772  1.105.2.7        ad 
   1773       1.80   hannken 	mutex_enter(vp->v_interlock);
   1774       1.74   hannken 	fstrans_done(mp);
   1775       1.54   hannken 	KASSERT((vp->v_iflag & VI_ONWORKLST) == 0);
   1776       1.54   hannken }
   1777       1.54   hannken 
   1778       1.54   hannken /*
   1779       1.98   hannken  * Disassociate the underlying file system from an open device vnode
   1780       1.98   hannken  * and make it anonymous.
   1781       1.98   hannken  *
   1782       1.98   hannken  * Vnode unlocked on entry, drops a reference to the vnode.
   1783       1.98   hannken  */
   1784       1.98   hannken void
   1785       1.98   hannken vcache_make_anon(vnode_t *vp)
   1786       1.98   hannken {
   1787       1.98   hannken 	vnode_impl_t *vip = VNODE_TO_VIMPL(vp);
   1788       1.98   hannken 	uint32_t hash;
   1789       1.98   hannken 	bool recycle;
   1790       1.98   hannken 
   1791       1.98   hannken 	KASSERT(vp->v_type == VBLK || vp->v_type == VCHR);
   1792      1.103   hannken 	KASSERT(vp->v_mount == dead_rootmount || fstrans_is_owner(vp->v_mount));
   1793       1.98   hannken 	VSTATE_ASSERT_UNLOCKED(vp, VS_ACTIVE);
   1794       1.98   hannken 
   1795       1.98   hannken 	/* Remove from vnode cache. */
   1796       1.98   hannken 	hash = vcache_hash(&vip->vi_key);
   1797       1.98   hannken 	mutex_enter(&vcache_lock);
   1798       1.98   hannken 	KASSERT(vip == vcache_hash_lookup(&vip->vi_key, hash));
   1799       1.98   hannken 	SLIST_REMOVE(&vcache_hashtab[hash & vcache_hashmask],
   1800       1.98   hannken 	    vip, vnode_impl, vi_hash);
   1801       1.98   hannken 	vip->vi_key.vk_mount = dead_rootmount;
   1802       1.98   hannken 	vip->vi_key.vk_key_len = 0;
   1803       1.98   hannken 	vip->vi_key.vk_key = NULL;
   1804       1.98   hannken 	mutex_exit(&vcache_lock);
   1805       1.98   hannken 
   1806       1.98   hannken 	/*
   1807       1.98   hannken 	 * Disassociate the underlying file system from the vnode.
   1808       1.98   hannken 	 * VOP_INACTIVE leaves the vnode locked; VOP_RECLAIM unlocks
   1809       1.98   hannken 	 * the vnode, and may destroy the vnode so that VOP_UNLOCK
   1810       1.98   hannken 	 * would no longer function.
   1811       1.98   hannken 	 */
   1812       1.98   hannken 	if (vn_lock(vp, LK_EXCLUSIVE)) {
   1813       1.98   hannken 		vnpanic(vp, "%s: cannot lock", __func__);
   1814       1.98   hannken 	}
   1815       1.98   hannken 	VOP_INACTIVE(vp, &recycle);
   1816       1.98   hannken 	KASSERT((vp->v_vflag & VV_LOCKSWORK) == 0 ||
   1817       1.98   hannken 	    VOP_ISLOCKED(vp) == LK_EXCLUSIVE);
   1818       1.98   hannken 	if (VOP_RECLAIM(vp)) {
   1819       1.98   hannken 		vnpanic(vp, "%s: cannot reclaim", __func__);
   1820       1.98   hannken 	}
   1821       1.98   hannken 
   1822       1.98   hannken 	/* Purge name cache. */
   1823       1.98   hannken 	cache_purge(vp);
   1824       1.98   hannken 
   1825       1.98   hannken 	/* Done with purge, change operations vector. */
   1826       1.98   hannken 	mutex_enter(vp->v_interlock);
   1827       1.98   hannken 	vp->v_op = spec_vnodeop_p;
   1828       1.98   hannken 	vp->v_vflag |= VV_MPSAFE;
   1829       1.98   hannken 	vp->v_vflag &= ~VV_LOCKSWORK;
   1830       1.98   hannken 	mutex_exit(vp->v_interlock);
   1831       1.98   hannken 
   1832       1.98   hannken 	/*
   1833       1.98   hannken 	 * Move to dead mount.  Must be after changing the operations
   1834       1.98   hannken 	 * vector as vnode operations enter the mount before using the
   1835       1.98   hannken 	 * operations vector.  See sys/kern/vnode_if.c.
   1836       1.98   hannken 	 */
   1837       1.98   hannken 	vfs_ref(dead_rootmount);
   1838       1.98   hannken 	vfs_insmntque(vp, dead_rootmount);
   1839       1.98   hannken 
   1840       1.98   hannken 	vrele(vp);
   1841       1.98   hannken }
   1842       1.98   hannken 
   1843       1.98   hannken /*
   1844        1.1     rmind  * Update outstanding I/O count and do wakeup if requested.
   1845        1.1     rmind  */
   1846        1.1     rmind void
   1847        1.1     rmind vwakeup(struct buf *bp)
   1848        1.1     rmind {
   1849        1.1     rmind 	vnode_t *vp;
   1850        1.1     rmind 
   1851        1.1     rmind 	if ((vp = bp->b_vp) == NULL)
   1852        1.1     rmind 		return;
   1853        1.1     rmind 
   1854        1.9     rmind 	KASSERT(bp->b_objlock == vp->v_interlock);
   1855        1.1     rmind 	KASSERT(mutex_owned(bp->b_objlock));
   1856        1.1     rmind 
   1857        1.1     rmind 	if (--vp->v_numoutput < 0)
   1858       1.11  christos 		vnpanic(vp, "%s: neg numoutput, vp %p", __func__, vp);
   1859        1.1     rmind 	if (vp->v_numoutput == 0)
   1860        1.1     rmind 		cv_broadcast(&vp->v_cv);
   1861        1.1     rmind }
   1862        1.1     rmind 
   1863        1.1     rmind /*
   1864       1.35   hannken  * Test a vnode for being or becoming dead.  Returns one of:
   1865       1.35   hannken  * EBUSY:  vnode is becoming dead, with "flags == VDEAD_NOWAIT" only.
   1866       1.35   hannken  * ENOENT: vnode is dead.
   1867       1.35   hannken  * 0:      otherwise.
   1868       1.35   hannken  *
   1869       1.35   hannken  * Whenever this function returns a non-zero value all future
   1870       1.35   hannken  * calls will also return a non-zero value.
   1871       1.35   hannken  */
   1872       1.35   hannken int
   1873       1.35   hannken vdead_check(struct vnode *vp, int flags)
   1874       1.35   hannken {
   1875       1.35   hannken 
   1876       1.35   hannken 	KASSERT(mutex_owned(vp->v_interlock));
   1877       1.35   hannken 
   1878       1.52   hannken 	if (! ISSET(flags, VDEAD_NOWAIT))
   1879       1.52   hannken 		VSTATE_WAIT_STABLE(vp);
   1880        1.1     rmind 
   1881       1.57   hannken 	if (VSTATE_GET(vp) == VS_RECLAIMING) {
   1882       1.52   hannken 		KASSERT(ISSET(flags, VDEAD_NOWAIT));
   1883       1.52   hannken 		return EBUSY;
   1884       1.57   hannken 	} else if (VSTATE_GET(vp) == VS_RECLAIMED) {
   1885       1.52   hannken 		return ENOENT;
   1886       1.52   hannken 	}
   1887        1.1     rmind 
   1888       1.52   hannken 	return 0;
   1889        1.1     rmind }
   1890        1.1     rmind 
   1891        1.1     rmind int
   1892       1.61   hannken vfs_drainvnodes(void)
   1893        1.1     rmind {
   1894       1.63   hannken 	int i, gen;
   1895       1.61   hannken 
   1896       1.63   hannken 	mutex_enter(&vdrain_lock);
   1897       1.63   hannken 	for (i = 0; i < 2; i++) {
   1898       1.63   hannken 		gen = vdrain_gen;
   1899       1.63   hannken 		while (gen == vdrain_gen) {
   1900       1.63   hannken 			cv_broadcast(&vdrain_cv);
   1901       1.63   hannken 			cv_wait(&vdrain_gen_cv, &vdrain_lock);
   1902       1.63   hannken 		}
   1903       1.61   hannken 	}
   1904       1.63   hannken 	mutex_exit(&vdrain_lock);
   1905       1.12   hannken 
   1906       1.63   hannken 	if (numvnodes >= desiredvnodes)
   1907       1.63   hannken 		return EBUSY;
   1908       1.12   hannken 
   1909       1.69   hannken 	if (vcache_hashsize != desiredvnodes)
   1910       1.61   hannken 		vcache_reinit();
   1911       1.36   hannken 
   1912        1.1     rmind 	return 0;
   1913        1.1     rmind }
   1914        1.1     rmind 
   1915        1.1     rmind void
   1916       1.11  christos vnpanic(vnode_t *vp, const char *fmt, ...)
   1917        1.1     rmind {
   1918       1.11  christos 	va_list ap;
   1919       1.11  christos 
   1920        1.1     rmind #ifdef DIAGNOSTIC
   1921        1.1     rmind 	vprint(NULL, vp);
   1922        1.1     rmind #endif
   1923       1.11  christos 	va_start(ap, fmt);
   1924       1.11  christos 	vpanic(fmt, ap);
   1925       1.11  christos 	va_end(ap);
   1926        1.1     rmind }
   1927  1.105.2.9        ad 
   1928  1.105.2.9        ad void
   1929  1.105.2.9        ad vshareilock(vnode_t *tvp, vnode_t *fvp)
   1930  1.105.2.9        ad {
   1931  1.105.2.9        ad 	kmutex_t *oldlock;
   1932  1.105.2.9        ad 
   1933  1.105.2.9        ad 	oldlock = tvp->v_interlock;
   1934  1.105.2.9        ad 	mutex_obj_hold(fvp->v_interlock);
   1935  1.105.2.9        ad 	tvp->v_interlock = fvp->v_interlock;
   1936  1.105.2.9        ad 	mutex_obj_free(oldlock);
   1937  1.105.2.9        ad }
   1938