Home | History | Annotate | Line # | Download | only in kern
vfs_vnode.c revision 1.109
      1  1.109        ad /*	$NetBSD: vfs_vnode.c,v 1.109 2020/01/23 11:17:09 ad Exp $	*/
      2    1.1     rmind 
      3    1.1     rmind /*-
      4  1.104        ad  * Copyright (c) 1997-2011, 2019 The NetBSD Foundation, Inc.
      5    1.1     rmind  * All rights reserved.
      6    1.1     rmind  *
      7    1.1     rmind  * This code is derived from software contributed to The NetBSD Foundation
      8    1.1     rmind  * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
      9    1.1     rmind  * NASA Ames Research Center, by Charles M. Hannum, and by Andrew Doran.
     10    1.1     rmind  *
     11    1.1     rmind  * Redistribution and use in source and binary forms, with or without
     12    1.1     rmind  * modification, are permitted provided that the following conditions
     13    1.1     rmind  * are met:
     14    1.1     rmind  * 1. Redistributions of source code must retain the above copyright
     15    1.1     rmind  *    notice, this list of conditions and the following disclaimer.
     16    1.1     rmind  * 2. Redistributions in binary form must reproduce the above copyright
     17    1.1     rmind  *    notice, this list of conditions and the following disclaimer in the
     18    1.1     rmind  *    documentation and/or other materials provided with the distribution.
     19    1.1     rmind  *
     20    1.1     rmind  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     21    1.1     rmind  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     22    1.1     rmind  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     23    1.1     rmind  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     24    1.1     rmind  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     25    1.1     rmind  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     26    1.1     rmind  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     27    1.1     rmind  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     28    1.1     rmind  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     29    1.1     rmind  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     30    1.1     rmind  * POSSIBILITY OF SUCH DAMAGE.
     31    1.1     rmind  */
     32    1.1     rmind 
     33    1.1     rmind /*
     34    1.1     rmind  * Copyright (c) 1989, 1993
     35    1.1     rmind  *	The Regents of the University of California.  All rights reserved.
     36    1.1     rmind  * (c) UNIX System Laboratories, Inc.
     37    1.1     rmind  * All or some portions of this file are derived from material licensed
     38    1.1     rmind  * to the University of California by American Telephone and Telegraph
     39    1.1     rmind  * Co. or Unix System Laboratories, Inc. and are reproduced herein with
     40    1.1     rmind  * the permission of UNIX System Laboratories, Inc.
     41    1.1     rmind  *
     42    1.1     rmind  * Redistribution and use in source and binary forms, with or without
     43    1.1     rmind  * modification, are permitted provided that the following conditions
     44    1.1     rmind  * are met:
     45    1.1     rmind  * 1. Redistributions of source code must retain the above copyright
     46    1.1     rmind  *    notice, this list of conditions and the following disclaimer.
     47    1.1     rmind  * 2. Redistributions in binary form must reproduce the above copyright
     48    1.1     rmind  *    notice, this list of conditions and the following disclaimer in the
     49    1.1     rmind  *    documentation and/or other materials provided with the distribution.
     50    1.1     rmind  * 3. Neither the name of the University nor the names of its contributors
     51    1.1     rmind  *    may be used to endorse or promote products derived from this software
     52    1.1     rmind  *    without specific prior written permission.
     53    1.1     rmind  *
     54    1.1     rmind  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
     55    1.1     rmind  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
     56    1.1     rmind  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
     57    1.1     rmind  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
     58    1.1     rmind  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
     59    1.1     rmind  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
     60    1.1     rmind  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
     61    1.1     rmind  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
     62    1.1     rmind  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
     63    1.1     rmind  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
     64    1.1     rmind  * SUCH DAMAGE.
     65    1.1     rmind  *
     66    1.1     rmind  *	@(#)vfs_subr.c	8.13 (Berkeley) 4/18/94
     67    1.1     rmind  */
     68    1.1     rmind 
     69    1.1     rmind /*
     70    1.8     rmind  * The vnode cache subsystem.
     71    1.1     rmind  *
     72    1.8     rmind  * Life-cycle
     73    1.1     rmind  *
     74    1.8     rmind  *	Normally, there are two points where new vnodes are created:
     75    1.8     rmind  *	VOP_CREATE(9) and VOP_LOOKUP(9).  The life-cycle of a vnode
     76    1.8     rmind  *	starts in one of the following ways:
     77    1.8     rmind  *
     78   1.45   hannken  *	- Allocation, via vcache_get(9) or vcache_new(9).
     79   1.66   hannken  *	- Reclamation of inactive vnode, via vcache_vget(9).
     80    1.8     rmind  *
     81   1.16     rmind  *	Recycle from a free list, via getnewvnode(9) -> getcleanvnode(9)
     82   1.16     rmind  *	was another, traditional way.  Currently, only the draining thread
     83   1.16     rmind  *	recycles the vnodes.  This behaviour might be revisited.
     84   1.16     rmind  *
     85    1.8     rmind  *	The life-cycle ends when the last reference is dropped, usually
     86    1.8     rmind  *	in VOP_REMOVE(9).  In such case, VOP_INACTIVE(9) is called to inform
     87    1.8     rmind  *	the file system that vnode is inactive.  Via this call, file system
     88   1.16     rmind  *	indicates whether vnode can be recycled (usually, it checks its own
     89   1.16     rmind  *	references, e.g. count of links, whether the file was removed).
     90    1.8     rmind  *
     91    1.8     rmind  *	Depending on indication, vnode can be put into a free list (cache),
     92   1.54   hannken  *	or cleaned via vcache_reclaim, which calls VOP_RECLAIM(9) to
     93   1.54   hannken  *	disassociate underlying file system from the vnode, and finally
     94   1.54   hannken  *	destroyed.
     95    1.8     rmind  *
     96   1.52   hannken  * Vnode state
     97   1.52   hannken  *
     98   1.52   hannken  *	Vnode is always in one of six states:
     99   1.52   hannken  *	- MARKER	This is a marker vnode to help list traversal.  It
    100   1.52   hannken  *			will never change its state.
    101   1.52   hannken  *	- LOADING	Vnode is associating underlying file system and not
    102   1.52   hannken  *			yet ready to use.
    103   1.94   hannken  *	- LOADED	Vnode has associated underlying file system and is
    104   1.52   hannken  *			ready to use.
    105   1.52   hannken  *	- BLOCKED	Vnode is active but cannot get new references.
    106   1.52   hannken  *	- RECLAIMING	Vnode is disassociating from the underlying file
    107   1.52   hannken  *			system.
    108   1.52   hannken  *	- RECLAIMED	Vnode has disassociated from underlying file system
    109   1.52   hannken  *			and is dead.
    110   1.52   hannken  *
    111   1.52   hannken  *	Valid state changes are:
    112   1.94   hannken  *	LOADING -> LOADED
    113   1.52   hannken  *			Vnode has been initialised in vcache_get() or
    114   1.52   hannken  *			vcache_new() and is ready to use.
    115   1.94   hannken  *	LOADED -> RECLAIMING
    116   1.52   hannken  *			Vnode starts disassociation from underlying file
    117   1.54   hannken  *			system in vcache_reclaim().
    118   1.52   hannken  *	RECLAIMING -> RECLAIMED
    119   1.52   hannken  *			Vnode finished disassociation from underlying file
    120   1.54   hannken  *			system in vcache_reclaim().
    121   1.94   hannken  *	LOADED -> BLOCKED
    122   1.52   hannken  *			Either vcache_rekey*() is changing the vnode key or
    123   1.52   hannken  *			vrelel() is about to call VOP_INACTIVE().
    124   1.94   hannken  *	BLOCKED -> LOADED
    125   1.52   hannken  *			The block condition is over.
    126   1.52   hannken  *	LOADING -> RECLAIMED
    127   1.52   hannken  *			Either vcache_get() or vcache_new() failed to
    128   1.52   hannken  *			associate the underlying file system or vcache_rekey*()
    129   1.52   hannken  *			drops a vnode used as placeholder.
    130   1.52   hannken  *
    131   1.52   hannken  *	Of these states LOADING, BLOCKED and RECLAIMING are intermediate
    132   1.52   hannken  *	and it is possible to wait for state change.
    133   1.52   hannken  *
    134   1.52   hannken  *	State is protected with v_interlock with one exception:
    135   1.69   hannken  *	to change from LOADING both v_interlock and vcache_lock must be held
    136   1.52   hannken  *	so it is possible to check "state == LOADING" without holding
    137   1.52   hannken  *	v_interlock.  See vcache_get() for details.
    138   1.52   hannken  *
    139    1.8     rmind  * Reference counting
    140    1.8     rmind  *
    141    1.8     rmind  *	Vnode is considered active, if reference count (vnode_t::v_usecount)
    142    1.8     rmind  *	is non-zero.  It is maintained using: vref(9) and vrele(9), as well
    143    1.8     rmind  *	as vput(9), routines.  Common points holding references are e.g.
    144    1.8     rmind  *	file openings, current working directory, mount points, etc.
    145    1.8     rmind  *
    146    1.1     rmind  */
    147    1.1     rmind 
    148    1.1     rmind #include <sys/cdefs.h>
    149  1.109        ad __KERNEL_RCSID(0, "$NetBSD: vfs_vnode.c,v 1.109 2020/01/23 11:17:09 ad Exp $");
    150  1.108        ad 
    151  1.109        ad #ifdef _KERNEL_OPT
    152  1.108        ad #include "opt_pax.h"
    153  1.109        ad #endif
    154    1.1     rmind 
    155    1.1     rmind #include <sys/param.h>
    156    1.1     rmind #include <sys/kernel.h>
    157    1.1     rmind 
    158    1.1     rmind #include <sys/atomic.h>
    159    1.1     rmind #include <sys/buf.h>
    160    1.1     rmind #include <sys/conf.h>
    161    1.1     rmind #include <sys/device.h>
    162   1.36   hannken #include <sys/hash.h>
    163    1.1     rmind #include <sys/kauth.h>
    164    1.1     rmind #include <sys/kmem.h>
    165    1.1     rmind #include <sys/kthread.h>
    166    1.1     rmind #include <sys/module.h>
    167    1.1     rmind #include <sys/mount.h>
    168    1.1     rmind #include <sys/namei.h>
    169  1.108        ad #include <sys/pax.h>
    170    1.1     rmind #include <sys/syscallargs.h>
    171    1.1     rmind #include <sys/sysctl.h>
    172    1.1     rmind #include <sys/systm.h>
    173   1.58   hannken #include <sys/vnode_impl.h>
    174    1.1     rmind #include <sys/wapbl.h>
    175   1.24   hannken #include <sys/fstrans.h>
    176    1.1     rmind 
    177    1.1     rmind #include <uvm/uvm.h>
    178    1.1     rmind #include <uvm/uvm_readahead.h>
    179  1.104        ad #include <uvm/uvm_stat.h>
    180    1.1     rmind 
    181   1.23   hannken /* Flags to vrelel. */
    182  1.104        ad #define	VRELEL_ASYNC	0x0001	/* Always defer to vrele thread. */
    183  1.104        ad 
    184  1.104        ad #define	LRU_VRELE	0
    185  1.104        ad #define	LRU_FREE	1
    186  1.104        ad #define	LRU_HOLD	2
    187  1.104        ad #define	LRU_COUNT	3
    188    1.1     rmind 
    189   1.16     rmind /*
    190   1.63   hannken  * There are three lru lists: one holds vnodes waiting for async release,
    191  1.104        ad  * one is for vnodes which have no buffer/page references and one for those
    192  1.104        ad  * which do (i.e.  v_holdcnt is non-zero).  We put the lists into a single,
    193  1.104        ad  * private cache line as vnodes migrate between them while under the same
    194  1.104        ad  * lock (vdrain_lock).
    195   1.63   hannken  */
    196  1.104        ad u_int			numvnodes		__cacheline_aligned;
    197  1.104        ad static vnodelst_t	lru_list[LRU_COUNT]	__cacheline_aligned;
    198   1.63   hannken static kmutex_t		vdrain_lock		__cacheline_aligned;
    199  1.104        ad static kcondvar_t	vdrain_cv;
    200   1.63   hannken static int		vdrain_gen;
    201   1.63   hannken static kcondvar_t	vdrain_gen_cv;
    202   1.63   hannken static bool		vdrain_retry;
    203   1.63   hannken static lwp_t *		vdrain_lwp;
    204   1.57   hannken SLIST_HEAD(hashhead, vnode_impl);
    205   1.69   hannken static kmutex_t		vcache_lock		__cacheline_aligned;
    206  1.104        ad static kcondvar_t	vcache_cv;
    207   1.69   hannken static u_int		vcache_hashsize;
    208   1.69   hannken static u_long		vcache_hashmask;
    209  1.104        ad static struct hashhead	*vcache_hashtab;
    210   1.69   hannken static pool_cache_t	vcache_pool;
    211   1.63   hannken static void		lru_requeue(vnode_t *, vnodelst_t *);
    212   1.63   hannken static vnodelst_t *	lru_which(vnode_t *);
    213   1.63   hannken static vnode_impl_t *	vcache_alloc(void);
    214   1.79   hannken static void		vcache_dealloc(vnode_impl_t *);
    215   1.57   hannken static void		vcache_free(vnode_impl_t *);
    216   1.36   hannken static void		vcache_init(void);
    217   1.36   hannken static void		vcache_reinit(void);
    218   1.54   hannken static void		vcache_reclaim(vnode_t *);
    219  1.107        ad static void		vrelel(vnode_t *, int, int);
    220   1.12   hannken static void		vdrain_thread(void *);
    221   1.11  christos static void		vnpanic(vnode_t *, const char *, ...)
    222   1.18  christos     __printflike(2, 3);
    223    1.1     rmind 
    224    1.1     rmind /* Routines having to do with the management of the vnode table. */
    225   1.44   hannken extern struct mount	*dead_rootmount;
    226    1.1     rmind extern int		(**dead_vnodeop_p)(void *);
    227   1.98   hannken extern int		(**spec_vnodeop_p)(void *);
    228   1.31   hannken extern struct vfsops	dead_vfsops;
    229    1.1     rmind 
    230   1.51   hannken /* Vnode state operations and diagnostics. */
    231   1.51   hannken 
    232   1.51   hannken #if defined(DIAGNOSTIC)
    233   1.51   hannken 
    234   1.94   hannken #define VSTATE_VALID(state) \
    235   1.94   hannken 	((state) != VS_ACTIVE && (state) != VS_MARKER)
    236   1.51   hannken #define VSTATE_GET(vp) \
    237   1.51   hannken 	vstate_assert_get((vp), __func__, __LINE__)
    238   1.51   hannken #define VSTATE_CHANGE(vp, from, to) \
    239   1.51   hannken 	vstate_assert_change((vp), (from), (to), __func__, __LINE__)
    240   1.51   hannken #define VSTATE_WAIT_STABLE(vp) \
    241   1.51   hannken 	vstate_assert_wait_stable((vp), __func__, __LINE__)
    242   1.51   hannken 
    243   1.94   hannken void
    244   1.99     joerg _vstate_assert(vnode_t *vp, enum vnode_state state, const char *func, int line,
    245   1.99     joerg     bool has_lock)
    246   1.51   hannken {
    247   1.70   hannken 	vnode_impl_t *vip = VNODE_TO_VIMPL(vp);
    248   1.51   hannken 
    249   1.99     joerg 	if (!has_lock) {
    250   1.99     joerg 		/*
    251   1.99     joerg 		 * Prevent predictive loads from the CPU, but check the state
    252   1.99     joerg 		 * without loooking first.
    253   1.99     joerg 		 */
    254   1.99     joerg 		membar_enter();
    255   1.99     joerg 		if (state == VS_ACTIVE && vp->v_usecount > 0 &&
    256   1.99     joerg 		    (vip->vi_state == VS_LOADED || vip->vi_state == VS_BLOCKED))
    257   1.99     joerg 			return;
    258   1.99     joerg 		if (vip->vi_state == state)
    259   1.99     joerg 			return;
    260   1.99     joerg 		mutex_enter((vp)->v_interlock);
    261   1.99     joerg 	}
    262   1.99     joerg 
    263   1.51   hannken 	KASSERTMSG(mutex_owned(vp->v_interlock), "at %s:%d", func, line);
    264   1.51   hannken 
    265   1.99     joerg 	if ((state == VS_ACTIVE && vp->v_usecount > 0 &&
    266   1.99     joerg 	    (vip->vi_state == VS_LOADED || vip->vi_state == VS_BLOCKED)) ||
    267   1.99     joerg 	    vip->vi_state == state) {
    268   1.99     joerg 		if (!has_lock)
    269   1.99     joerg 			mutex_exit((vp)->v_interlock);
    270   1.94   hannken 		return;
    271   1.99     joerg 	}
    272   1.94   hannken 	vnpanic(vp, "state is %s, usecount %d, expected %s at %s:%d",
    273   1.94   hannken 	    vstate_name(vip->vi_state), vp->v_usecount,
    274   1.94   hannken 	    vstate_name(state), func, line);
    275   1.51   hannken }
    276   1.51   hannken 
    277   1.57   hannken static enum vnode_state
    278   1.51   hannken vstate_assert_get(vnode_t *vp, const char *func, int line)
    279   1.51   hannken {
    280   1.70   hannken 	vnode_impl_t *vip = VNODE_TO_VIMPL(vp);
    281   1.51   hannken 
    282   1.51   hannken 	KASSERTMSG(mutex_owned(vp->v_interlock), "at %s:%d", func, line);
    283   1.94   hannken 	if (! VSTATE_VALID(vip->vi_state))
    284   1.51   hannken 		vnpanic(vp, "state is %s at %s:%d",
    285   1.70   hannken 		    vstate_name(vip->vi_state), func, line);
    286   1.51   hannken 
    287   1.70   hannken 	return vip->vi_state;
    288   1.51   hannken }
    289   1.51   hannken 
    290   1.52   hannken static void
    291   1.51   hannken vstate_assert_wait_stable(vnode_t *vp, const char *func, int line)
    292   1.51   hannken {
    293   1.70   hannken 	vnode_impl_t *vip = VNODE_TO_VIMPL(vp);
    294   1.51   hannken 
    295   1.51   hannken 	KASSERTMSG(mutex_owned(vp->v_interlock), "at %s:%d", func, line);
    296   1.94   hannken 	if (! VSTATE_VALID(vip->vi_state))
    297   1.51   hannken 		vnpanic(vp, "state is %s at %s:%d",
    298   1.70   hannken 		    vstate_name(vip->vi_state), func, line);
    299   1.51   hannken 
    300   1.94   hannken 	while (vip->vi_state != VS_LOADED && vip->vi_state != VS_RECLAIMED)
    301   1.51   hannken 		cv_wait(&vp->v_cv, vp->v_interlock);
    302   1.51   hannken 
    303   1.94   hannken 	if (! VSTATE_VALID(vip->vi_state))
    304   1.51   hannken 		vnpanic(vp, "state is %s at %s:%d",
    305   1.70   hannken 		    vstate_name(vip->vi_state), func, line);
    306   1.51   hannken }
    307   1.51   hannken 
    308   1.52   hannken static void
    309   1.57   hannken vstate_assert_change(vnode_t *vp, enum vnode_state from, enum vnode_state to,
    310   1.51   hannken     const char *func, int line)
    311   1.51   hannken {
    312   1.70   hannken 	vnode_impl_t *vip = VNODE_TO_VIMPL(vp);
    313   1.51   hannken 
    314   1.51   hannken 	KASSERTMSG(mutex_owned(vp->v_interlock), "at %s:%d", func, line);
    315   1.57   hannken 	if (from == VS_LOADING)
    316   1.69   hannken 		KASSERTMSG(mutex_owned(&vcache_lock), "at %s:%d", func, line);
    317   1.51   hannken 
    318   1.94   hannken 	if (! VSTATE_VALID(from))
    319   1.51   hannken 		vnpanic(vp, "from is %s at %s:%d",
    320   1.51   hannken 		    vstate_name(from), func, line);
    321   1.94   hannken 	if (! VSTATE_VALID(to))
    322   1.51   hannken 		vnpanic(vp, "to is %s at %s:%d",
    323   1.51   hannken 		    vstate_name(to), func, line);
    324   1.70   hannken 	if (vip->vi_state != from)
    325   1.51   hannken 		vnpanic(vp, "from is %s, expected %s at %s:%d\n",
    326   1.70   hannken 		    vstate_name(vip->vi_state), vstate_name(from), func, line);
    327   1.68   hannken 	if ((from == VS_BLOCKED || to == VS_BLOCKED) && vp->v_usecount != 1)
    328   1.68   hannken 		vnpanic(vp, "%s to %s with usecount %d at %s:%d",
    329   1.68   hannken 		    vstate_name(from), vstate_name(to), vp->v_usecount,
    330   1.68   hannken 		    func, line);
    331   1.51   hannken 
    332   1.70   hannken 	vip->vi_state = to;
    333   1.57   hannken 	if (from == VS_LOADING)
    334   1.69   hannken 		cv_broadcast(&vcache_cv);
    335   1.94   hannken 	if (to == VS_LOADED || to == VS_RECLAIMED)
    336   1.51   hannken 		cv_broadcast(&vp->v_cv);
    337   1.51   hannken }
    338   1.51   hannken 
    339   1.51   hannken #else /* defined(DIAGNOSTIC) */
    340   1.51   hannken 
    341   1.51   hannken #define VSTATE_GET(vp) \
    342   1.57   hannken 	(VNODE_TO_VIMPL((vp))->vi_state)
    343   1.51   hannken #define VSTATE_CHANGE(vp, from, to) \
    344   1.51   hannken 	vstate_change((vp), (from), (to))
    345   1.51   hannken #define VSTATE_WAIT_STABLE(vp) \
    346   1.51   hannken 	vstate_wait_stable((vp))
    347   1.94   hannken void
    348  1.100     joerg _vstate_assert(vnode_t *vp, enum vnode_state state, const char *func, int line,
    349  1.100     joerg     bool has_lock)
    350   1.94   hannken {
    351   1.94   hannken 
    352   1.94   hannken }
    353   1.51   hannken 
    354   1.52   hannken static void
    355   1.51   hannken vstate_wait_stable(vnode_t *vp)
    356   1.51   hannken {
    357   1.70   hannken 	vnode_impl_t *vip = VNODE_TO_VIMPL(vp);
    358   1.51   hannken 
    359   1.94   hannken 	while (vip->vi_state != VS_LOADED && vip->vi_state != VS_RECLAIMED)
    360   1.51   hannken 		cv_wait(&vp->v_cv, vp->v_interlock);
    361   1.51   hannken }
    362   1.51   hannken 
    363   1.52   hannken static void
    364   1.57   hannken vstate_change(vnode_t *vp, enum vnode_state from, enum vnode_state to)
    365   1.51   hannken {
    366   1.70   hannken 	vnode_impl_t *vip = VNODE_TO_VIMPL(vp);
    367   1.51   hannken 
    368   1.70   hannken 	vip->vi_state = to;
    369   1.57   hannken 	if (from == VS_LOADING)
    370   1.69   hannken 		cv_broadcast(&vcache_cv);
    371   1.94   hannken 	if (to == VS_LOADED || to == VS_RECLAIMED)
    372   1.51   hannken 		cv_broadcast(&vp->v_cv);
    373   1.51   hannken }
    374   1.51   hannken 
    375   1.51   hannken #endif /* defined(DIAGNOSTIC) */
    376   1.51   hannken 
    377    1.1     rmind void
    378    1.1     rmind vfs_vnode_sysinit(void)
    379    1.1     rmind {
    380  1.104        ad 	int error __diagused, i;
    381    1.1     rmind 
    382   1.44   hannken 	dead_rootmount = vfs_mountalloc(&dead_vfsops, NULL);
    383   1.44   hannken 	KASSERT(dead_rootmount != NULL);
    384  1.103   hannken 	dead_rootmount->mnt_iflag |= IMNT_MPSAFE;
    385   1.31   hannken 
    386   1.63   hannken 	mutex_init(&vdrain_lock, MUTEX_DEFAULT, IPL_NONE);
    387  1.104        ad 	for (i = 0; i < LRU_COUNT; i++) {
    388  1.104        ad 		TAILQ_INIT(&lru_list[i]);
    389  1.104        ad 	}
    390   1.36   hannken 	vcache_init();
    391   1.36   hannken 
    392   1.12   hannken 	cv_init(&vdrain_cv, "vdrain");
    393   1.63   hannken 	cv_init(&vdrain_gen_cv, "vdrainwt");
    394   1.12   hannken 	error = kthread_create(PRI_VM, KTHREAD_MPSAFE, NULL, vdrain_thread,
    395   1.63   hannken 	    NULL, &vdrain_lwp, "vdrain");
    396   1.47  riastrad 	KASSERTMSG((error == 0), "kthread_create(vdrain) failed: %d", error);
    397    1.1     rmind }
    398    1.1     rmind 
    399    1.1     rmind /*
    400   1.48   hannken  * Allocate a new marker vnode.
    401   1.48   hannken  */
    402   1.48   hannken vnode_t *
    403   1.48   hannken vnalloc_marker(struct mount *mp)
    404   1.48   hannken {
    405   1.70   hannken 	vnode_impl_t *vip;
    406   1.50   hannken 	vnode_t *vp;
    407   1.50   hannken 
    408   1.70   hannken 	vip = pool_cache_get(vcache_pool, PR_WAITOK);
    409   1.70   hannken 	memset(vip, 0, sizeof(*vip));
    410   1.70   hannken 	vp = VIMPL_TO_VNODE(vip);
    411   1.50   hannken 	uvm_obj_init(&vp->v_uobj, &uvm_vnodeops, true, 0);
    412   1.50   hannken 	vp->v_mount = mp;
    413   1.50   hannken 	vp->v_type = VBAD;
    414   1.70   hannken 	vip->vi_state = VS_MARKER;
    415   1.48   hannken 
    416   1.50   hannken 	return vp;
    417   1.48   hannken }
    418   1.48   hannken 
    419   1.48   hannken /*
    420   1.48   hannken  * Free a marker vnode.
    421   1.48   hannken  */
    422   1.48   hannken void
    423   1.48   hannken vnfree_marker(vnode_t *vp)
    424   1.48   hannken {
    425   1.70   hannken 	vnode_impl_t *vip;
    426   1.48   hannken 
    427   1.70   hannken 	vip = VNODE_TO_VIMPL(vp);
    428   1.70   hannken 	KASSERT(vip->vi_state == VS_MARKER);
    429   1.50   hannken 	uvm_obj_destroy(&vp->v_uobj, true);
    430   1.70   hannken 	pool_cache_put(vcache_pool, vip);
    431   1.48   hannken }
    432   1.48   hannken 
    433   1.48   hannken /*
    434   1.48   hannken  * Test a vnode for being a marker vnode.
    435   1.48   hannken  */
    436   1.48   hannken bool
    437   1.48   hannken vnis_marker(vnode_t *vp)
    438   1.48   hannken {
    439   1.48   hannken 
    440   1.57   hannken 	return (VNODE_TO_VIMPL(vp)->vi_state == VS_MARKER);
    441   1.48   hannken }
    442   1.48   hannken 
    443   1.48   hannken /*
    444   1.63   hannken  * Return the lru list this node should be on.
    445   1.63   hannken  */
    446   1.63   hannken static vnodelst_t *
    447   1.63   hannken lru_which(vnode_t *vp)
    448   1.63   hannken {
    449   1.63   hannken 
    450   1.63   hannken 	KASSERT(mutex_owned(vp->v_interlock));
    451   1.63   hannken 
    452   1.63   hannken 	if (vp->v_holdcnt > 0)
    453  1.104        ad 		return &lru_list[LRU_HOLD];
    454   1.63   hannken 	else
    455  1.104        ad 		return &lru_list[LRU_FREE];
    456   1.63   hannken }
    457   1.63   hannken 
    458   1.63   hannken /*
    459   1.63   hannken  * Put vnode to end of given list.
    460   1.63   hannken  * Both the current and the new list may be NULL, used on vnode alloc/free.
    461   1.63   hannken  * Adjust numvnodes and signal vdrain thread if there is work.
    462   1.63   hannken  */
    463   1.63   hannken static void
    464   1.63   hannken lru_requeue(vnode_t *vp, vnodelst_t *listhd)
    465   1.63   hannken {
    466   1.70   hannken 	vnode_impl_t *vip;
    467  1.104        ad 	int d;
    468  1.104        ad 
    469  1.104        ad 	/*
    470  1.104        ad 	 * If the vnode is on the correct list, and was put there recently,
    471  1.104        ad 	 * then leave it be, thus avoiding huge cache and lock contention.
    472  1.104        ad 	 */
    473  1.104        ad 	vip = VNODE_TO_VIMPL(vp);
    474  1.104        ad 	if (listhd == vip->vi_lrulisthd &&
    475  1.104        ad 	    (hardclock_ticks - vip->vi_lrulisttm) < hz) {
    476  1.104        ad 	    	return;
    477  1.104        ad 	}
    478   1.63   hannken 
    479   1.63   hannken 	mutex_enter(&vdrain_lock);
    480  1.104        ad 	d = 0;
    481   1.70   hannken 	if (vip->vi_lrulisthd != NULL)
    482   1.70   hannken 		TAILQ_REMOVE(vip->vi_lrulisthd, vip, vi_lrulist);
    483   1.63   hannken 	else
    484  1.104        ad 		d++;
    485   1.70   hannken 	vip->vi_lrulisthd = listhd;
    486  1.104        ad 	vip->vi_lrulisttm = hardclock_ticks;
    487   1.70   hannken 	if (vip->vi_lrulisthd != NULL)
    488   1.70   hannken 		TAILQ_INSERT_TAIL(vip->vi_lrulisthd, vip, vi_lrulist);
    489   1.63   hannken 	else
    490  1.104        ad 		d--;
    491  1.104        ad 	if (d != 0) {
    492  1.104        ad 		/*
    493  1.104        ad 		 * Looks strange?  This is not a bug.  Don't store
    494  1.104        ad 		 * numvnodes unless there is a change - avoid false
    495  1.104        ad 		 * sharing on MP.
    496  1.104        ad 		 */
    497  1.104        ad 		numvnodes += d;
    498  1.104        ad 	}
    499  1.104        ad 	if (numvnodes > desiredvnodes || listhd == &lru_list[LRU_VRELE])
    500   1.63   hannken 		cv_broadcast(&vdrain_cv);
    501   1.63   hannken 	mutex_exit(&vdrain_lock);
    502   1.63   hannken }
    503   1.63   hannken 
    504   1.63   hannken /*
    505   1.75   hannken  * Release deferred vrele vnodes for this mount.
    506   1.75   hannken  * Called with file system suspended.
    507   1.75   hannken  */
    508   1.75   hannken void
    509   1.75   hannken vrele_flush(struct mount *mp)
    510   1.75   hannken {
    511   1.75   hannken 	vnode_impl_t *vip, *marker;
    512  1.104        ad 	vnode_t *vp;
    513   1.75   hannken 
    514   1.75   hannken 	KASSERT(fstrans_is_owner(mp));
    515   1.75   hannken 
    516   1.75   hannken 	marker = VNODE_TO_VIMPL(vnalloc_marker(NULL));
    517   1.75   hannken 
    518   1.75   hannken 	mutex_enter(&vdrain_lock);
    519  1.104        ad 	TAILQ_INSERT_HEAD(&lru_list[LRU_VRELE], marker, vi_lrulist);
    520   1.75   hannken 
    521   1.75   hannken 	while ((vip = TAILQ_NEXT(marker, vi_lrulist))) {
    522  1.104        ad 		TAILQ_REMOVE(&lru_list[LRU_VRELE], marker, vi_lrulist);
    523  1.104        ad 		TAILQ_INSERT_AFTER(&lru_list[LRU_VRELE], vip, marker,
    524  1.104        ad 		    vi_lrulist);
    525  1.104        ad 		vp = VIMPL_TO_VNODE(vip);
    526  1.104        ad 		if (vnis_marker(vp))
    527   1.75   hannken 			continue;
    528   1.75   hannken 
    529  1.104        ad 		KASSERT(vip->vi_lrulisthd == &lru_list[LRU_VRELE]);
    530   1.75   hannken 		TAILQ_REMOVE(vip->vi_lrulisthd, vip, vi_lrulist);
    531  1.104        ad 		vip->vi_lrulisthd = &lru_list[LRU_HOLD];
    532  1.104        ad 		vip->vi_lrulisttm = hardclock_ticks;
    533   1.75   hannken 		TAILQ_INSERT_TAIL(vip->vi_lrulisthd, vip, vi_lrulist);
    534   1.75   hannken 		mutex_exit(&vdrain_lock);
    535   1.75   hannken 
    536  1.107        ad 		vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
    537  1.104        ad 		mutex_enter(vp->v_interlock);
    538  1.107        ad 		vrelel(vp, 0, LK_EXCLUSIVE);
    539   1.75   hannken 
    540   1.75   hannken 		mutex_enter(&vdrain_lock);
    541   1.75   hannken 	}
    542   1.75   hannken 
    543  1.104        ad 	TAILQ_REMOVE(&lru_list[LRU_VRELE], marker, vi_lrulist);
    544   1.75   hannken 	mutex_exit(&vdrain_lock);
    545   1.75   hannken 
    546   1.75   hannken 	vnfree_marker(VIMPL_TO_VNODE(marker));
    547   1.75   hannken }
    548   1.75   hannken 
    549   1.75   hannken /*
    550   1.63   hannken  * Reclaim a cached vnode.  Used from vdrain_thread only.
    551    1.1     rmind  */
    552   1.63   hannken static __inline void
    553   1.63   hannken vdrain_remove(vnode_t *vp)
    554    1.1     rmind {
    555   1.24   hannken 	struct mount *mp;
    556    1.1     rmind 
    557   1.63   hannken 	KASSERT(mutex_owned(&vdrain_lock));
    558   1.24   hannken 
    559   1.63   hannken 	/* Probe usecount (unlocked). */
    560   1.63   hannken 	if (vp->v_usecount > 0)
    561   1.63   hannken 		return;
    562   1.63   hannken 	/* Try v_interlock -- we lock the wrong direction! */
    563   1.63   hannken 	if (!mutex_tryenter(vp->v_interlock))
    564   1.63   hannken 		return;
    565   1.63   hannken 	/* Probe usecount and state. */
    566   1.94   hannken 	if (vp->v_usecount > 0 || VSTATE_GET(vp) != VS_LOADED) {
    567   1.63   hannken 		mutex_exit(vp->v_interlock);
    568   1.63   hannken 		return;
    569    1.1     rmind 	}
    570   1.63   hannken 	mp = vp->v_mount;
    571   1.96   hannken 	if (fstrans_start_nowait(mp) != 0) {
    572   1.63   hannken 		mutex_exit(vp->v_interlock);
    573   1.63   hannken 		return;
    574    1.1     rmind 	}
    575   1.63   hannken 	vdrain_retry = true;
    576   1.63   hannken 	mutex_exit(&vdrain_lock);
    577    1.1     rmind 
    578   1.66   hannken 	if (vcache_vget(vp) == 0) {
    579   1.78   hannken 		if (!vrecycle(vp)) {
    580  1.107        ad 			vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
    581   1.78   hannken 			mutex_enter(vp->v_interlock);
    582  1.107        ad 			vrelel(vp, 0, LK_EXCLUSIVE);
    583   1.78   hannken 		}
    584   1.60   hannken 	}
    585   1.24   hannken 	fstrans_done(mp);
    586   1.12   hannken 
    587   1.63   hannken 	mutex_enter(&vdrain_lock);
    588    1.1     rmind }
    589    1.1     rmind 
    590    1.1     rmind /*
    591   1.63   hannken  * Release a cached vnode.  Used from vdrain_thread only.
    592   1.12   hannken  */
    593   1.63   hannken static __inline void
    594   1.63   hannken vdrain_vrele(vnode_t *vp)
    595   1.12   hannken {
    596   1.70   hannken 	vnode_impl_t *vip = VNODE_TO_VIMPL(vp);
    597   1.63   hannken 	struct mount *mp;
    598   1.12   hannken 
    599   1.63   hannken 	KASSERT(mutex_owned(&vdrain_lock));
    600   1.12   hannken 
    601   1.63   hannken 	mp = vp->v_mount;
    602   1.96   hannken 	if (fstrans_start_nowait(mp) != 0)
    603   1.63   hannken 		return;
    604   1.63   hannken 
    605   1.64   hannken 	/*
    606   1.64   hannken 	 * First remove the vnode from the vrele list.
    607   1.64   hannken 	 * Put it on the last lru list, the last vrele()
    608   1.64   hannken 	 * will put it back onto the right list before
    609   1.64   hannken 	 * its v_usecount reaches zero.
    610   1.64   hannken 	 */
    611  1.104        ad 	KASSERT(vip->vi_lrulisthd == &lru_list[LRU_VRELE]);
    612   1.70   hannken 	TAILQ_REMOVE(vip->vi_lrulisthd, vip, vi_lrulist);
    613  1.104        ad 	vip->vi_lrulisthd = &lru_list[LRU_HOLD];
    614  1.104        ad 	vip->vi_lrulisttm = hardclock_ticks;
    615   1.70   hannken 	TAILQ_INSERT_TAIL(vip->vi_lrulisthd, vip, vi_lrulist);
    616   1.63   hannken 
    617   1.63   hannken 	vdrain_retry = true;
    618   1.63   hannken 	mutex_exit(&vdrain_lock);
    619   1.63   hannken 
    620  1.107        ad 	vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
    621   1.64   hannken 	mutex_enter(vp->v_interlock);
    622  1.107        ad 	vrelel(vp, 0, LK_EXCLUSIVE);
    623   1.63   hannken 	fstrans_done(mp);
    624   1.63   hannken 
    625   1.63   hannken 	mutex_enter(&vdrain_lock);
    626   1.12   hannken }
    627   1.12   hannken 
    628   1.12   hannken /*
    629   1.63   hannken  * Helper thread to keep the number of vnodes below desiredvnodes
    630   1.63   hannken  * and release vnodes from asynchronous vrele.
    631    1.1     rmind  */
    632   1.63   hannken static void
    633   1.63   hannken vdrain_thread(void *cookie)
    634    1.1     rmind {
    635   1.63   hannken 	int i;
    636   1.63   hannken 	u_int target;
    637   1.70   hannken 	vnode_impl_t *vip, *marker;
    638   1.63   hannken 
    639   1.63   hannken 	marker = VNODE_TO_VIMPL(vnalloc_marker(NULL));
    640   1.63   hannken 
    641   1.63   hannken 	mutex_enter(&vdrain_lock);
    642   1.63   hannken 
    643   1.63   hannken 	for (;;) {
    644   1.63   hannken 		vdrain_retry = false;
    645   1.63   hannken 		target = desiredvnodes - desiredvnodes/10;
    646    1.1     rmind 
    647  1.104        ad 		for (i = 0; i < LRU_COUNT; i++) {
    648  1.104        ad 			TAILQ_INSERT_HEAD(&lru_list[i], marker, vi_lrulist);
    649   1.70   hannken 			while ((vip = TAILQ_NEXT(marker, vi_lrulist))) {
    650  1.104        ad 				TAILQ_REMOVE(&lru_list[i], marker, vi_lrulist);
    651  1.104        ad 				TAILQ_INSERT_AFTER(&lru_list[i], vip, marker,
    652   1.63   hannken 				    vi_lrulist);
    653   1.75   hannken 				if (vnis_marker(VIMPL_TO_VNODE(vip)))
    654   1.75   hannken 					continue;
    655  1.104        ad 				if (i == LRU_VRELE)
    656   1.70   hannken 					vdrain_vrele(VIMPL_TO_VNODE(vip));
    657   1.63   hannken 				else if (numvnodes < target)
    658   1.63   hannken 					break;
    659   1.63   hannken 				else
    660   1.70   hannken 					vdrain_remove(VIMPL_TO_VNODE(vip));
    661   1.63   hannken 			}
    662  1.104        ad 			TAILQ_REMOVE(&lru_list[i], marker, vi_lrulist);
    663   1.63   hannken 		}
    664    1.1     rmind 
    665   1.63   hannken 		if (vdrain_retry) {
    666   1.63   hannken 			mutex_exit(&vdrain_lock);
    667   1.63   hannken 			yield();
    668   1.63   hannken 			mutex_enter(&vdrain_lock);
    669   1.63   hannken 		} else {
    670   1.63   hannken 			vdrain_gen++;
    671   1.63   hannken 			cv_broadcast(&vdrain_gen_cv);
    672   1.63   hannken 			cv_wait(&vdrain_cv, &vdrain_lock);
    673   1.63   hannken 		}
    674    1.1     rmind 	}
    675    1.1     rmind }
    676    1.1     rmind 
    677    1.1     rmind /*
    678    1.4     rmind  * vput: unlock and release the reference.
    679    1.1     rmind  */
    680    1.1     rmind void
    681    1.1     rmind vput(vnode_t *vp)
    682    1.1     rmind {
    683  1.107        ad 	int lktype;
    684    1.1     rmind 
    685  1.107        ad 	if ((vp->v_vflag & VV_LOCKSWORK) == 0) {
    686  1.107        ad 		lktype = LK_EXCLUSIVE;
    687  1.107        ad 	} else {
    688  1.107        ad 		lktype = VOP_ISLOCKED(vp);
    689  1.107        ad 		KASSERT(lktype != LK_NONE);
    690  1.107        ad 	}
    691  1.107        ad 	mutex_enter(vp->v_interlock);
    692  1.107        ad 	vrelel(vp, 0, lktype);
    693    1.1     rmind }
    694    1.1     rmind 
    695    1.1     rmind /*
    696    1.1     rmind  * Vnode release.  If reference count drops to zero, call inactive
    697    1.1     rmind  * routine and either return to freelist or free to the pool.
    698    1.1     rmind  */
    699   1.23   hannken static void
    700  1.107        ad vrelel(vnode_t *vp, int flags, int lktype)
    701    1.1     rmind {
    702  1.104        ad 	const bool async = ((flags & VRELEL_ASYNC) != 0);
    703    1.1     rmind 	bool recycle, defer;
    704    1.1     rmind 	int error;
    705    1.1     rmind 
    706    1.9     rmind 	KASSERT(mutex_owned(vp->v_interlock));
    707    1.1     rmind 
    708    1.1     rmind 	if (__predict_false(vp->v_op == dead_vnodeop_p &&
    709   1.57   hannken 	    VSTATE_GET(vp) != VS_RECLAIMED)) {
    710   1.11  christos 		vnpanic(vp, "dead but not clean");
    711    1.1     rmind 	}
    712    1.1     rmind 
    713    1.1     rmind 	/*
    714    1.1     rmind 	 * If not the last reference, just drop the reference count
    715    1.1     rmind 	 * and unlock.
    716    1.1     rmind 	 */
    717  1.104        ad 	if (vp->v_usecount > 1) {
    718  1.107        ad 		if (lktype != LK_NONE) {
    719  1.107        ad 			VOP_UNLOCK(vp);
    720  1.107        ad 		}
    721  1.104        ad 		vp->v_usecount--;
    722    1.9     rmind 		mutex_exit(vp->v_interlock);
    723    1.1     rmind 		return;
    724    1.1     rmind 	}
    725    1.1     rmind 	if (vp->v_usecount <= 0 || vp->v_writecount != 0) {
    726   1.11  christos 		vnpanic(vp, "%s: bad ref count", __func__);
    727    1.1     rmind 	}
    728    1.1     rmind 
    729   1.15   hannken #ifdef DIAGNOSTIC
    730   1.15   hannken 	if ((vp->v_type == VBLK || vp->v_type == VCHR) &&
    731   1.15   hannken 	    vp->v_specnode != NULL && vp->v_specnode->sn_opencnt != 0) {
    732   1.15   hannken 		vprint("vrelel: missing VOP_CLOSE()", vp);
    733   1.15   hannken 	}
    734   1.15   hannken #endif
    735   1.15   hannken 
    736    1.1     rmind 	/*
    737   1.79   hannken 	 * First try to get the vnode locked for VOP_INACTIVE().
    738   1.79   hannken 	 * Defer vnode release to vdrain_thread if caller requests
    739   1.79   hannken 	 * it explicitly, is the pagedaemon or the lock failed.
    740    1.1     rmind 	 */
    741  1.107        ad 	defer = false;
    742   1.79   hannken 	if ((curlwp == uvm.pagedaemon_lwp) || async) {
    743   1.79   hannken 		defer = true;
    744  1.107        ad 	} else if (lktype == LK_SHARED) {
    745  1.107        ad 		/* Excellent chance of getting, if the last ref. */
    746  1.107        ad 		error = vn_lock(vp, LK_UPGRADE | LK_RETRY |
    747  1.107        ad 		    LK_NOWAIT);
    748  1.107        ad 		if (error != 0) {
    749  1.107        ad 			defer = true;
    750  1.107        ad 		} else {
    751  1.107        ad 			lktype = LK_EXCLUSIVE;
    752  1.107        ad 		}
    753  1.107        ad 	} else if (lktype == LK_NONE) {
    754  1.107        ad 		/* Excellent chance of getting, if the last ref. */
    755  1.107        ad 		error = vn_lock(vp, LK_EXCLUSIVE | LK_RETRY |
    756  1.107        ad 		    LK_NOWAIT);
    757  1.107        ad 		if (error != 0) {
    758  1.107        ad 			defer = true;
    759  1.107        ad 		} else {
    760  1.107        ad 			lktype = LK_EXCLUSIVE;
    761  1.107        ad 		}
    762   1.79   hannken 	}
    763   1.79   hannken 	KASSERT(mutex_owned(vp->v_interlock));
    764   1.79   hannken 	if (defer) {
    765    1.1     rmind 		/*
    766   1.79   hannken 		 * Defer reclaim to the kthread; it's not safe to
    767   1.79   hannken 		 * clean it here.  We donate it our last reference.
    768    1.1     rmind 		 */
    769  1.107        ad 		if (lktype != LK_NONE) {
    770  1.107        ad 			VOP_UNLOCK(vp);
    771  1.107        ad 		}
    772  1.104        ad 		lru_requeue(vp, &lru_list[LRU_VRELE]);
    773   1.79   hannken 		mutex_exit(vp->v_interlock);
    774   1.79   hannken 		return;
    775   1.79   hannken 	}
    776  1.107        ad 	KASSERT(lktype == LK_EXCLUSIVE);
    777   1.30   hannken 
    778   1.79   hannken 	/*
    779   1.79   hannken 	 * If not clean, deactivate the vnode, but preserve
    780   1.79   hannken 	 * our reference across the call to VOP_INACTIVE().
    781   1.79   hannken 	 */
    782   1.79   hannken 	if (VSTATE_GET(vp) == VS_RECLAIMED) {
    783   1.79   hannken 		VOP_UNLOCK(vp);
    784   1.79   hannken 	} else {
    785   1.94   hannken 		VSTATE_CHANGE(vp, VS_LOADED, VS_BLOCKED);
    786   1.29  christos 		mutex_exit(vp->v_interlock);
    787   1.29  christos 
    788    1.1     rmind 		/*
    789   1.52   hannken 		 * The vnode must not gain another reference while being
    790    1.1     rmind 		 * deactivated.  If VOP_INACTIVE() indicates that
    791    1.1     rmind 		 * the described file has been deleted, then recycle
    792   1.52   hannken 		 * the vnode.
    793    1.1     rmind 		 *
    794   1.82  riastrad 		 * Note that VOP_INACTIVE() will not drop the vnode lock.
    795    1.1     rmind 		 */
    796   1.79   hannken 		recycle = false;
    797    1.1     rmind 		VOP_INACTIVE(vp, &recycle);
    798   1.83  riastrad 		if (!recycle)
    799   1.83  riastrad 			VOP_UNLOCK(vp);
    800    1.9     rmind 		mutex_enter(vp->v_interlock);
    801   1.94   hannken 		VSTATE_CHANGE(vp, VS_BLOCKED, VS_LOADED);
    802    1.1     rmind 		if (!recycle) {
    803  1.104        ad 			if (vp->v_usecount > 1) {
    804  1.104        ad 				vp->v_usecount--;
    805    1.9     rmind 				mutex_exit(vp->v_interlock);
    806    1.1     rmind 				return;
    807    1.1     rmind 			}
    808    1.1     rmind 		}
    809    1.1     rmind 
    810    1.1     rmind 		/* Take care of space accounting. */
    811  1.104        ad 		if ((vp->v_iflag & VI_EXECMAP) != 0 &&
    812  1.104        ad 		    vp->v_uobj.uo_npages != 0) {
    813  1.105        ad 			cpu_count(CPU_COUNT_EXECPAGES, -vp->v_uobj.uo_npages);
    814  1.105        ad 			cpu_count(CPU_COUNT_FILEPAGES, vp->v_uobj.uo_npages);
    815    1.1     rmind 		}
    816    1.1     rmind 		vp->v_iflag &= ~(VI_TEXT|VI_EXECMAP|VI_WRMAP);
    817    1.1     rmind 		vp->v_vflag &= ~VV_MAPPED;
    818    1.1     rmind 
    819    1.1     rmind 		/*
    820    1.1     rmind 		 * Recycle the vnode if the file is now unused (unlinked),
    821    1.1     rmind 		 * otherwise just free it.
    822    1.1     rmind 		 */
    823    1.1     rmind 		if (recycle) {
    824   1.94   hannken 			VSTATE_ASSERT(vp, VS_LOADED);
    825   1.83  riastrad 			/* vcache_reclaim drops the lock. */
    826   1.54   hannken 			vcache_reclaim(vp);
    827    1.1     rmind 		}
    828    1.1     rmind 		KASSERT(vp->v_usecount > 0);
    829    1.1     rmind 	}
    830    1.1     rmind 
    831  1.104        ad 	vp->v_usecount--;
    832  1.104        ad 	if (vp->v_usecount != 0) {
    833    1.1     rmind 		/* Gained another reference while being reclaimed. */
    834    1.9     rmind 		mutex_exit(vp->v_interlock);
    835    1.1     rmind 		return;
    836    1.1     rmind 	}
    837    1.1     rmind 
    838   1.67   hannken 	if (VSTATE_GET(vp) == VS_RECLAIMED && vp->v_holdcnt == 0) {
    839    1.1     rmind 		/*
    840    1.1     rmind 		 * It's clean so destroy it.  It isn't referenced
    841    1.1     rmind 		 * anywhere since it has been reclaimed.
    842    1.1     rmind 		 */
    843   1.57   hannken 		vcache_free(VNODE_TO_VIMPL(vp));
    844    1.1     rmind 	} else {
    845    1.1     rmind 		/*
    846    1.1     rmind 		 * Otherwise, put it back onto the freelist.  It
    847    1.1     rmind 		 * can't be destroyed while still associated with
    848    1.1     rmind 		 * a file system.
    849    1.1     rmind 		 */
    850   1.63   hannken 		lru_requeue(vp, lru_which(vp));
    851    1.9     rmind 		mutex_exit(vp->v_interlock);
    852    1.1     rmind 	}
    853    1.1     rmind }
    854    1.1     rmind 
    855    1.1     rmind void
    856    1.1     rmind vrele(vnode_t *vp)
    857    1.1     rmind {
    858    1.1     rmind 
    859    1.9     rmind 	mutex_enter(vp->v_interlock);
    860  1.107        ad 	vrelel(vp, 0, LK_NONE);
    861    1.1     rmind }
    862    1.1     rmind 
    863    1.1     rmind /*
    864    1.1     rmind  * Asynchronous vnode release, vnode is released in different context.
    865    1.1     rmind  */
    866    1.1     rmind void
    867    1.1     rmind vrele_async(vnode_t *vp)
    868    1.1     rmind {
    869    1.1     rmind 
    870    1.9     rmind 	mutex_enter(vp->v_interlock);
    871  1.107        ad 	vrelel(vp, VRELEL_ASYNC, LK_NONE);
    872    1.1     rmind }
    873    1.1     rmind 
    874    1.1     rmind /*
    875    1.1     rmind  * Vnode reference, where a reference is already held by some other
    876    1.1     rmind  * object (for example, a file structure).
    877    1.1     rmind  */
    878    1.1     rmind void
    879    1.1     rmind vref(vnode_t *vp)
    880    1.1     rmind {
    881    1.1     rmind 
    882    1.1     rmind 	KASSERT(vp->v_usecount != 0);
    883    1.1     rmind 
    884  1.104        ad 	mutex_enter(vp->v_interlock);
    885  1.104        ad 	vp->v_usecount++;
    886  1.104        ad 	mutex_exit(vp->v_interlock);
    887    1.1     rmind }
    888    1.1     rmind 
    889    1.1     rmind /*
    890    1.1     rmind  * Page or buffer structure gets a reference.
    891    1.1     rmind  * Called with v_interlock held.
    892    1.1     rmind  */
    893    1.1     rmind void
    894    1.1     rmind vholdl(vnode_t *vp)
    895    1.1     rmind {
    896    1.1     rmind 
    897    1.9     rmind 	KASSERT(mutex_owned(vp->v_interlock));
    898    1.1     rmind 
    899   1.63   hannken 	if (vp->v_holdcnt++ == 0 && vp->v_usecount == 0)
    900   1.63   hannken 		lru_requeue(vp, lru_which(vp));
    901    1.1     rmind }
    902    1.1     rmind 
    903    1.1     rmind /*
    904    1.1     rmind  * Page or buffer structure frees a reference.
    905    1.1     rmind  * Called with v_interlock held.
    906    1.1     rmind  */
    907    1.1     rmind void
    908    1.1     rmind holdrelel(vnode_t *vp)
    909    1.1     rmind {
    910    1.1     rmind 
    911    1.9     rmind 	KASSERT(mutex_owned(vp->v_interlock));
    912    1.1     rmind 
    913    1.1     rmind 	if (vp->v_holdcnt <= 0) {
    914   1.11  christos 		vnpanic(vp, "%s: holdcnt vp %p", __func__, vp);
    915    1.1     rmind 	}
    916    1.1     rmind 
    917    1.1     rmind 	vp->v_holdcnt--;
    918   1.63   hannken 	if (vp->v_holdcnt == 0 && vp->v_usecount == 0)
    919   1.63   hannken 		lru_requeue(vp, lru_which(vp));
    920    1.1     rmind }
    921    1.1     rmind 
    922    1.1     rmind /*
    923   1.33   hannken  * Recycle an unused vnode if caller holds the last reference.
    924    1.1     rmind  */
    925   1.33   hannken bool
    926   1.33   hannken vrecycle(vnode_t *vp)
    927    1.1     rmind {
    928   1.60   hannken 	int error __diagused;
    929   1.46   hannken 
    930   1.33   hannken 	mutex_enter(vp->v_interlock);
    931   1.33   hannken 
    932   1.60   hannken 	/* Make sure we hold the last reference. */
    933   1.60   hannken 	VSTATE_WAIT_STABLE(vp);
    934   1.33   hannken 	if (vp->v_usecount != 1) {
    935   1.33   hannken 		mutex_exit(vp->v_interlock);
    936   1.33   hannken 		return false;
    937    1.1     rmind 	}
    938   1.60   hannken 
    939   1.60   hannken 	/* If the vnode is already clean we're done. */
    940   1.94   hannken 	if (VSTATE_GET(vp) != VS_LOADED) {
    941   1.60   hannken 		VSTATE_ASSERT(vp, VS_RECLAIMED);
    942  1.107        ad 		vrelel(vp, 0, LK_NONE);
    943   1.60   hannken 		return true;
    944   1.60   hannken 	}
    945   1.60   hannken 
    946   1.60   hannken 	/* Prevent further references until the vnode is locked. */
    947   1.94   hannken 	VSTATE_CHANGE(vp, VS_LOADED, VS_BLOCKED);
    948   1.60   hannken 	mutex_exit(vp->v_interlock);
    949   1.60   hannken 
    950   1.73   hannken 	/*
    951   1.73   hannken 	 * On a leaf file system this lock will always succeed as we hold
    952   1.73   hannken 	 * the last reference and prevent further references.
    953   1.73   hannken 	 * On layered file systems waiting for the lock would open a can of
    954   1.73   hannken 	 * deadlocks as the lower vnodes may have other active references.
    955   1.73   hannken 	 */
    956   1.76   hannken 	error = vn_lock(vp, LK_EXCLUSIVE | LK_RETRY | LK_NOWAIT);
    957   1.60   hannken 
    958   1.60   hannken 	mutex_enter(vp->v_interlock);
    959   1.94   hannken 	VSTATE_CHANGE(vp, VS_BLOCKED, VS_LOADED);
    960   1.60   hannken 
    961   1.73   hannken 	if (error) {
    962   1.73   hannken 		mutex_exit(vp->v_interlock);
    963   1.73   hannken 		return false;
    964   1.73   hannken 	}
    965   1.73   hannken 
    966   1.68   hannken 	KASSERT(vp->v_usecount == 1);
    967   1.54   hannken 	vcache_reclaim(vp);
    968  1.107        ad 	vrelel(vp, 0, LK_NONE);
    969   1.60   hannken 
    970   1.33   hannken 	return true;
    971    1.1     rmind }
    972    1.1     rmind 
    973    1.1     rmind /*
    974   1.92   hannken  * Helper for vrevoke() to propagate suspension from lastmp
    975   1.92   hannken  * to thismp.  Both args may be NULL.
    976   1.92   hannken  * Returns the currently suspended file system or NULL.
    977   1.92   hannken  */
    978   1.92   hannken static struct mount *
    979   1.92   hannken vrevoke_suspend_next(struct mount *lastmp, struct mount *thismp)
    980   1.92   hannken {
    981   1.92   hannken 	int error;
    982   1.92   hannken 
    983   1.92   hannken 	if (lastmp == thismp)
    984   1.92   hannken 		return thismp;
    985   1.92   hannken 
    986   1.92   hannken 	if (lastmp != NULL)
    987   1.92   hannken 		vfs_resume(lastmp);
    988   1.92   hannken 
    989   1.92   hannken 	if (thismp == NULL)
    990   1.92   hannken 		return NULL;
    991   1.92   hannken 
    992   1.92   hannken 	do {
    993   1.92   hannken 		error = vfs_suspend(thismp, 0);
    994   1.92   hannken 	} while (error == EINTR || error == ERESTART);
    995   1.92   hannken 
    996   1.92   hannken 	if (error == 0)
    997   1.92   hannken 		return thismp;
    998   1.92   hannken 
    999   1.92   hannken 	KASSERT(error == EOPNOTSUPP);
   1000   1.92   hannken 	return NULL;
   1001   1.92   hannken }
   1002   1.92   hannken 
   1003   1.92   hannken /*
   1004    1.1     rmind  * Eliminate all activity associated with the requested vnode
   1005    1.1     rmind  * and with all vnodes aliased to the requested vnode.
   1006    1.1     rmind  */
   1007    1.1     rmind void
   1008    1.1     rmind vrevoke(vnode_t *vp)
   1009    1.1     rmind {
   1010   1.88   hannken 	struct mount *mp;
   1011   1.19   hannken 	vnode_t *vq;
   1012    1.1     rmind 	enum vtype type;
   1013    1.1     rmind 	dev_t dev;
   1014    1.1     rmind 
   1015    1.1     rmind 	KASSERT(vp->v_usecount > 0);
   1016    1.1     rmind 
   1017   1.92   hannken 	mp = vrevoke_suspend_next(NULL, vp->v_mount);
   1018   1.88   hannken 
   1019    1.9     rmind 	mutex_enter(vp->v_interlock);
   1020   1.52   hannken 	VSTATE_WAIT_STABLE(vp);
   1021   1.57   hannken 	if (VSTATE_GET(vp) == VS_RECLAIMED) {
   1022    1.9     rmind 		mutex_exit(vp->v_interlock);
   1023    1.1     rmind 	} else if (vp->v_type != VBLK && vp->v_type != VCHR) {
   1024  1.104        ad 		vp->v_usecount++;
   1025   1.29  christos 		mutex_exit(vp->v_interlock);
   1026   1.29  christos 		vgone(vp);
   1027    1.1     rmind 	} else {
   1028    1.1     rmind 		dev = vp->v_rdev;
   1029    1.1     rmind 		type = vp->v_type;
   1030    1.9     rmind 		mutex_exit(vp->v_interlock);
   1031    1.1     rmind 
   1032   1.88   hannken 		while (spec_node_lookup_by_dev(type, dev, &vq) == 0) {
   1033   1.92   hannken 			mp = vrevoke_suspend_next(mp, vq->v_mount);
   1034   1.88   hannken 			vgone(vq);
   1035   1.88   hannken 		}
   1036    1.1     rmind 	}
   1037   1.92   hannken 	vrevoke_suspend_next(mp, NULL);
   1038    1.1     rmind }
   1039    1.1     rmind 
   1040    1.1     rmind /*
   1041    1.1     rmind  * Eliminate all activity associated with a vnode in preparation for
   1042    1.1     rmind  * reuse.  Drops a reference from the vnode.
   1043    1.1     rmind  */
   1044    1.1     rmind void
   1045    1.1     rmind vgone(vnode_t *vp)
   1046    1.1     rmind {
   1047  1.107        ad 	int lktype;
   1048    1.1     rmind 
   1049  1.103   hannken 	KASSERT(vp->v_mount == dead_rootmount || fstrans_is_owner(vp->v_mount));
   1050   1.93   hannken 
   1051   1.76   hannken 	vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
   1052  1.107        ad 	lktype = LK_EXCLUSIVE;
   1053    1.9     rmind 	mutex_enter(vp->v_interlock);
   1054   1.76   hannken 	VSTATE_WAIT_STABLE(vp);
   1055  1.107        ad 	if (VSTATE_GET(vp) == VS_LOADED) {
   1056   1.76   hannken 		vcache_reclaim(vp);
   1057  1.107        ad 		lktype = LK_NONE;
   1058  1.107        ad 	}
   1059   1.76   hannken 	VSTATE_ASSERT(vp, VS_RECLAIMED);
   1060  1.107        ad 	vrelel(vp, 0, lktype);
   1061    1.1     rmind }
   1062    1.1     rmind 
   1063   1.36   hannken static inline uint32_t
   1064   1.36   hannken vcache_hash(const struct vcache_key *key)
   1065   1.36   hannken {
   1066   1.36   hannken 	uint32_t hash = HASH32_BUF_INIT;
   1067   1.36   hannken 
   1068   1.97   hannken 	KASSERT(key->vk_key_len > 0);
   1069   1.97   hannken 
   1070   1.36   hannken 	hash = hash32_buf(&key->vk_mount, sizeof(struct mount *), hash);
   1071   1.36   hannken 	hash = hash32_buf(key->vk_key, key->vk_key_len, hash);
   1072   1.36   hannken 	return hash;
   1073   1.36   hannken }
   1074   1.36   hannken 
   1075   1.36   hannken static void
   1076   1.36   hannken vcache_init(void)
   1077   1.36   hannken {
   1078   1.36   hannken 
   1079   1.69   hannken 	vcache_pool = pool_cache_init(sizeof(vnode_impl_t), 0, 0, 0,
   1080   1.36   hannken 	    "vcachepl", NULL, IPL_NONE, NULL, NULL, NULL);
   1081   1.69   hannken 	KASSERT(vcache_pool != NULL);
   1082   1.69   hannken 	mutex_init(&vcache_lock, MUTEX_DEFAULT, IPL_NONE);
   1083   1.69   hannken 	cv_init(&vcache_cv, "vcache");
   1084   1.69   hannken 	vcache_hashsize = desiredvnodes;
   1085   1.69   hannken 	vcache_hashtab = hashinit(desiredvnodes, HASH_SLIST, true,
   1086   1.69   hannken 	    &vcache_hashmask);
   1087   1.36   hannken }
   1088   1.36   hannken 
   1089   1.36   hannken static void
   1090   1.36   hannken vcache_reinit(void)
   1091   1.36   hannken {
   1092   1.36   hannken 	int i;
   1093   1.36   hannken 	uint32_t hash;
   1094   1.36   hannken 	u_long oldmask, newmask;
   1095   1.36   hannken 	struct hashhead *oldtab, *newtab;
   1096   1.70   hannken 	vnode_impl_t *vip;
   1097   1.36   hannken 
   1098   1.36   hannken 	newtab = hashinit(desiredvnodes, HASH_SLIST, true, &newmask);
   1099   1.69   hannken 	mutex_enter(&vcache_lock);
   1100   1.69   hannken 	oldtab = vcache_hashtab;
   1101   1.69   hannken 	oldmask = vcache_hashmask;
   1102   1.69   hannken 	vcache_hashsize = desiredvnodes;
   1103   1.69   hannken 	vcache_hashtab = newtab;
   1104   1.69   hannken 	vcache_hashmask = newmask;
   1105   1.36   hannken 	for (i = 0; i <= oldmask; i++) {
   1106   1.70   hannken 		while ((vip = SLIST_FIRST(&oldtab[i])) != NULL) {
   1107   1.70   hannken 			SLIST_REMOVE(&oldtab[i], vip, vnode_impl, vi_hash);
   1108   1.70   hannken 			hash = vcache_hash(&vip->vi_key);
   1109   1.69   hannken 			SLIST_INSERT_HEAD(&newtab[hash & vcache_hashmask],
   1110   1.70   hannken 			    vip, vi_hash);
   1111   1.36   hannken 		}
   1112   1.36   hannken 	}
   1113   1.69   hannken 	mutex_exit(&vcache_lock);
   1114   1.36   hannken 	hashdone(oldtab, HASH_SLIST, oldmask);
   1115   1.36   hannken }
   1116   1.36   hannken 
   1117   1.57   hannken static inline vnode_impl_t *
   1118   1.36   hannken vcache_hash_lookup(const struct vcache_key *key, uint32_t hash)
   1119   1.36   hannken {
   1120   1.36   hannken 	struct hashhead *hashp;
   1121   1.70   hannken 	vnode_impl_t *vip;
   1122   1.36   hannken 
   1123   1.69   hannken 	KASSERT(mutex_owned(&vcache_lock));
   1124   1.36   hannken 
   1125   1.69   hannken 	hashp = &vcache_hashtab[hash & vcache_hashmask];
   1126   1.70   hannken 	SLIST_FOREACH(vip, hashp, vi_hash) {
   1127   1.70   hannken 		if (key->vk_mount != vip->vi_key.vk_mount)
   1128   1.36   hannken 			continue;
   1129   1.70   hannken 		if (key->vk_key_len != vip->vi_key.vk_key_len)
   1130   1.36   hannken 			continue;
   1131   1.70   hannken 		if (memcmp(key->vk_key, vip->vi_key.vk_key, key->vk_key_len))
   1132   1.36   hannken 			continue;
   1133   1.70   hannken 		return vip;
   1134   1.36   hannken 	}
   1135   1.36   hannken 	return NULL;
   1136   1.36   hannken }
   1137   1.36   hannken 
   1138   1.36   hannken /*
   1139   1.50   hannken  * Allocate a new, uninitialized vcache node.
   1140   1.50   hannken  */
   1141   1.57   hannken static vnode_impl_t *
   1142   1.50   hannken vcache_alloc(void)
   1143   1.50   hannken {
   1144   1.70   hannken 	vnode_impl_t *vip;
   1145   1.50   hannken 	vnode_t *vp;
   1146   1.50   hannken 
   1147   1.70   hannken 	vip = pool_cache_get(vcache_pool, PR_WAITOK);
   1148   1.70   hannken 	memset(vip, 0, sizeof(*vip));
   1149   1.50   hannken 
   1150  1.104        ad 	vip->vi_lock = rw_obj_alloc();
   1151   1.70   hannken 	/* SLIST_INIT(&vip->vi_hash); */
   1152  1.106        ad 	TAILQ_INIT(&vip->vi_nclist);
   1153   1.71   hannken 	/* LIST_INIT(&vip->vi_dnclist); */
   1154   1.50   hannken 
   1155   1.70   hannken 	vp = VIMPL_TO_VNODE(vip);
   1156   1.50   hannken 	uvm_obj_init(&vp->v_uobj, &uvm_vnodeops, true, 0);
   1157   1.50   hannken 	cv_init(&vp->v_cv, "vnode");
   1158   1.50   hannken 
   1159   1.50   hannken 	vp->v_usecount = 1;
   1160   1.50   hannken 	vp->v_type = VNON;
   1161   1.50   hannken 	vp->v_size = vp->v_writesize = VSIZENOTSET;
   1162   1.50   hannken 
   1163   1.70   hannken 	vip->vi_state = VS_LOADING;
   1164   1.51   hannken 
   1165  1.104        ad 	lru_requeue(vp, &lru_list[LRU_FREE]);
   1166   1.63   hannken 
   1167   1.70   hannken 	return vip;
   1168   1.50   hannken }
   1169   1.50   hannken 
   1170   1.50   hannken /*
   1171   1.79   hannken  * Deallocate a vcache node in state VS_LOADING.
   1172   1.79   hannken  *
   1173   1.79   hannken  * vcache_lock held on entry and released on return.
   1174   1.79   hannken  */
   1175   1.79   hannken static void
   1176   1.79   hannken vcache_dealloc(vnode_impl_t *vip)
   1177   1.79   hannken {
   1178   1.79   hannken 	vnode_t *vp;
   1179   1.79   hannken 
   1180   1.79   hannken 	KASSERT(mutex_owned(&vcache_lock));
   1181   1.79   hannken 
   1182   1.79   hannken 	vp = VIMPL_TO_VNODE(vip);
   1183  1.102   hannken 	vfs_ref(dead_rootmount);
   1184  1.102   hannken 	vfs_insmntque(vp, dead_rootmount);
   1185   1.79   hannken 	mutex_enter(vp->v_interlock);
   1186   1.79   hannken 	vp->v_op = dead_vnodeop_p;
   1187   1.79   hannken 	VSTATE_CHANGE(vp, VS_LOADING, VS_RECLAIMED);
   1188   1.79   hannken 	mutex_exit(&vcache_lock);
   1189  1.107        ad 	vrelel(vp, 0, LK_NONE);
   1190   1.79   hannken }
   1191   1.79   hannken 
   1192   1.79   hannken /*
   1193   1.50   hannken  * Free an unused, unreferenced vcache node.
   1194   1.67   hannken  * v_interlock locked on entry.
   1195   1.50   hannken  */
   1196   1.50   hannken static void
   1197   1.70   hannken vcache_free(vnode_impl_t *vip)
   1198   1.50   hannken {
   1199   1.50   hannken 	vnode_t *vp;
   1200   1.50   hannken 
   1201   1.70   hannken 	vp = VIMPL_TO_VNODE(vip);
   1202   1.67   hannken 	KASSERT(mutex_owned(vp->v_interlock));
   1203   1.50   hannken 
   1204   1.50   hannken 	KASSERT(vp->v_usecount == 0);
   1205   1.67   hannken 	KASSERT(vp->v_holdcnt == 0);
   1206   1.67   hannken 	KASSERT(vp->v_writecount == 0);
   1207   1.67   hannken 	lru_requeue(vp, NULL);
   1208   1.67   hannken 	mutex_exit(vp->v_interlock);
   1209   1.67   hannken 
   1210   1.67   hannken 	vfs_insmntque(vp, NULL);
   1211   1.67   hannken 	if (vp->v_type == VBLK || vp->v_type == VCHR)
   1212   1.67   hannken 		spec_node_destroy(vp);
   1213   1.50   hannken 
   1214  1.104        ad 	rw_obj_free(vip->vi_lock);
   1215   1.50   hannken 	uvm_obj_destroy(&vp->v_uobj, true);
   1216   1.50   hannken 	cv_destroy(&vp->v_cv);
   1217   1.70   hannken 	pool_cache_put(vcache_pool, vip);
   1218   1.50   hannken }
   1219   1.50   hannken 
   1220   1.50   hannken /*
   1221   1.66   hannken  * Try to get an initial reference on this cached vnode.
   1222   1.66   hannken  * Returns zero on success,  ENOENT if the vnode has been reclaimed and
   1223   1.66   hannken  * EBUSY if the vnode state is unstable.
   1224   1.66   hannken  *
   1225   1.66   hannken  * v_interlock locked on entry and unlocked on exit.
   1226   1.66   hannken  */
   1227   1.66   hannken int
   1228   1.66   hannken vcache_tryvget(vnode_t *vp)
   1229   1.66   hannken {
   1230   1.67   hannken 	int error = 0;
   1231   1.66   hannken 
   1232   1.66   hannken 	KASSERT(mutex_owned(vp->v_interlock));
   1233   1.66   hannken 
   1234   1.67   hannken 	if (__predict_false(VSTATE_GET(vp) == VS_RECLAIMED))
   1235   1.67   hannken 		error = ENOENT;
   1236   1.94   hannken 	else if (__predict_false(VSTATE_GET(vp) != VS_LOADED))
   1237   1.67   hannken 		error = EBUSY;
   1238   1.67   hannken 	else
   1239  1.104        ad 		vp->v_usecount++;
   1240   1.66   hannken 
   1241   1.66   hannken 	mutex_exit(vp->v_interlock);
   1242   1.66   hannken 
   1243   1.67   hannken 	return error;
   1244   1.66   hannken }
   1245   1.66   hannken 
   1246   1.66   hannken /*
   1247   1.66   hannken  * Try to get an initial reference on this cached vnode.
   1248   1.66   hannken  * Returns zero on success and  ENOENT if the vnode has been reclaimed.
   1249   1.66   hannken  * Will wait for the vnode state to be stable.
   1250   1.66   hannken  *
   1251   1.66   hannken  * v_interlock locked on entry and unlocked on exit.
   1252   1.66   hannken  */
   1253   1.66   hannken int
   1254   1.66   hannken vcache_vget(vnode_t *vp)
   1255   1.66   hannken {
   1256   1.66   hannken 
   1257   1.66   hannken 	KASSERT(mutex_owned(vp->v_interlock));
   1258   1.66   hannken 
   1259   1.67   hannken 	/* Increment hold count to prevent vnode from disappearing. */
   1260   1.67   hannken 	vp->v_holdcnt++;
   1261   1.67   hannken 	VSTATE_WAIT_STABLE(vp);
   1262   1.67   hannken 	vp->v_holdcnt--;
   1263   1.66   hannken 
   1264   1.67   hannken 	/* If this was the last reference to a reclaimed vnode free it now. */
   1265   1.67   hannken 	if (__predict_false(VSTATE_GET(vp) == VS_RECLAIMED)) {
   1266   1.67   hannken 		if (vp->v_holdcnt == 0 && vp->v_usecount == 0)
   1267   1.67   hannken 			vcache_free(VNODE_TO_VIMPL(vp));
   1268   1.67   hannken 		else
   1269   1.67   hannken 			mutex_exit(vp->v_interlock);
   1270   1.66   hannken 		return ENOENT;
   1271   1.66   hannken 	}
   1272   1.94   hannken 	VSTATE_ASSERT(vp, VS_LOADED);
   1273  1.104        ad 	vp->v_usecount++;
   1274   1.66   hannken 	mutex_exit(vp->v_interlock);
   1275   1.66   hannken 
   1276   1.66   hannken 	return 0;
   1277   1.66   hannken }
   1278   1.66   hannken 
   1279   1.66   hannken /*
   1280   1.36   hannken  * Get a vnode / fs node pair by key and return it referenced through vpp.
   1281   1.36   hannken  */
   1282   1.36   hannken int
   1283   1.36   hannken vcache_get(struct mount *mp, const void *key, size_t key_len,
   1284   1.36   hannken     struct vnode **vpp)
   1285   1.36   hannken {
   1286   1.36   hannken 	int error;
   1287   1.36   hannken 	uint32_t hash;
   1288   1.36   hannken 	const void *new_key;
   1289   1.36   hannken 	struct vnode *vp;
   1290   1.36   hannken 	struct vcache_key vcache_key;
   1291   1.70   hannken 	vnode_impl_t *vip, *new_vip;
   1292   1.36   hannken 
   1293   1.36   hannken 	new_key = NULL;
   1294   1.36   hannken 	*vpp = NULL;
   1295   1.36   hannken 
   1296   1.36   hannken 	vcache_key.vk_mount = mp;
   1297   1.36   hannken 	vcache_key.vk_key = key;
   1298   1.36   hannken 	vcache_key.vk_key_len = key_len;
   1299   1.36   hannken 	hash = vcache_hash(&vcache_key);
   1300   1.36   hannken 
   1301   1.36   hannken again:
   1302   1.69   hannken 	mutex_enter(&vcache_lock);
   1303   1.70   hannken 	vip = vcache_hash_lookup(&vcache_key, hash);
   1304   1.36   hannken 
   1305   1.36   hannken 	/* If found, take a reference or retry. */
   1306   1.70   hannken 	if (__predict_true(vip != NULL)) {
   1307   1.52   hannken 		/*
   1308   1.52   hannken 		 * If the vnode is loading we cannot take the v_interlock
   1309   1.52   hannken 		 * here as it might change during load (see uvm_obj_setlock()).
   1310   1.69   hannken 		 * As changing state from VS_LOADING requires both vcache_lock
   1311   1.69   hannken 		 * and v_interlock it is safe to test with vcache_lock held.
   1312   1.52   hannken 		 *
   1313   1.57   hannken 		 * Wait for vnodes changing state from VS_LOADING and retry.
   1314   1.52   hannken 		 */
   1315   1.70   hannken 		if (__predict_false(vip->vi_state == VS_LOADING)) {
   1316   1.69   hannken 			cv_wait(&vcache_cv, &vcache_lock);
   1317   1.69   hannken 			mutex_exit(&vcache_lock);
   1318   1.52   hannken 			goto again;
   1319   1.52   hannken 		}
   1320   1.70   hannken 		vp = VIMPL_TO_VNODE(vip);
   1321   1.36   hannken 		mutex_enter(vp->v_interlock);
   1322   1.69   hannken 		mutex_exit(&vcache_lock);
   1323   1.66   hannken 		error = vcache_vget(vp);
   1324   1.36   hannken 		if (error == ENOENT)
   1325   1.36   hannken 			goto again;
   1326   1.36   hannken 		if (error == 0)
   1327   1.36   hannken 			*vpp = vp;
   1328   1.36   hannken 		KASSERT((error != 0) == (*vpp == NULL));
   1329   1.36   hannken 		return error;
   1330   1.36   hannken 	}
   1331   1.69   hannken 	mutex_exit(&vcache_lock);
   1332   1.36   hannken 
   1333   1.36   hannken 	/* Allocate and initialize a new vcache / vnode pair. */
   1334   1.87   hannken 	error = vfs_busy(mp);
   1335   1.36   hannken 	if (error)
   1336   1.36   hannken 		return error;
   1337   1.70   hannken 	new_vip = vcache_alloc();
   1338   1.70   hannken 	new_vip->vi_key = vcache_key;
   1339   1.70   hannken 	vp = VIMPL_TO_VNODE(new_vip);
   1340   1.69   hannken 	mutex_enter(&vcache_lock);
   1341   1.70   hannken 	vip = vcache_hash_lookup(&vcache_key, hash);
   1342   1.70   hannken 	if (vip == NULL) {
   1343   1.69   hannken 		SLIST_INSERT_HEAD(&vcache_hashtab[hash & vcache_hashmask],
   1344   1.70   hannken 		    new_vip, vi_hash);
   1345   1.70   hannken 		vip = new_vip;
   1346   1.36   hannken 	}
   1347   1.36   hannken 
   1348   1.36   hannken 	/* If another thread beat us inserting this node, retry. */
   1349   1.70   hannken 	if (vip != new_vip) {
   1350   1.79   hannken 		vcache_dealloc(new_vip);
   1351   1.87   hannken 		vfs_unbusy(mp);
   1352   1.36   hannken 		goto again;
   1353   1.36   hannken 	}
   1354   1.69   hannken 	mutex_exit(&vcache_lock);
   1355   1.36   hannken 
   1356   1.57   hannken 	/* Load the fs node.  Exclusive as new_node is VS_LOADING. */
   1357   1.36   hannken 	error = VFS_LOADVNODE(mp, vp, key, key_len, &new_key);
   1358   1.36   hannken 	if (error) {
   1359   1.69   hannken 		mutex_enter(&vcache_lock);
   1360   1.69   hannken 		SLIST_REMOVE(&vcache_hashtab[hash & vcache_hashmask],
   1361   1.70   hannken 		    new_vip, vnode_impl, vi_hash);
   1362   1.79   hannken 		vcache_dealloc(new_vip);
   1363   1.87   hannken 		vfs_unbusy(mp);
   1364   1.36   hannken 		KASSERT(*vpp == NULL);
   1365   1.36   hannken 		return error;
   1366   1.36   hannken 	}
   1367   1.36   hannken 	KASSERT(new_key != NULL);
   1368   1.36   hannken 	KASSERT(memcmp(key, new_key, key_len) == 0);
   1369   1.36   hannken 	KASSERT(vp->v_op != NULL);
   1370   1.36   hannken 	vfs_insmntque(vp, mp);
   1371   1.36   hannken 	if ((mp->mnt_iflag & IMNT_MPSAFE) != 0)
   1372   1.36   hannken 		vp->v_vflag |= VV_MPSAFE;
   1373   1.87   hannken 	vfs_ref(mp);
   1374   1.87   hannken 	vfs_unbusy(mp);
   1375   1.36   hannken 
   1376   1.36   hannken 	/* Finished loading, finalize node. */
   1377   1.69   hannken 	mutex_enter(&vcache_lock);
   1378   1.70   hannken 	new_vip->vi_key.vk_key = new_key;
   1379   1.39   hannken 	mutex_enter(vp->v_interlock);
   1380   1.94   hannken 	VSTATE_CHANGE(vp, VS_LOADING, VS_LOADED);
   1381   1.39   hannken 	mutex_exit(vp->v_interlock);
   1382   1.69   hannken 	mutex_exit(&vcache_lock);
   1383   1.36   hannken 	*vpp = vp;
   1384   1.36   hannken 	return 0;
   1385   1.36   hannken }
   1386   1.36   hannken 
   1387   1.36   hannken /*
   1388   1.40   hannken  * Create a new vnode / fs node pair and return it referenced through vpp.
   1389   1.40   hannken  */
   1390   1.40   hannken int
   1391   1.40   hannken vcache_new(struct mount *mp, struct vnode *dvp, struct vattr *vap,
   1392  1.101   hannken     kauth_cred_t cred, void *extra, struct vnode **vpp)
   1393   1.40   hannken {
   1394   1.40   hannken 	int error;
   1395   1.40   hannken 	uint32_t hash;
   1396   1.70   hannken 	struct vnode *vp, *ovp;
   1397   1.70   hannken 	vnode_impl_t *vip, *ovip;
   1398   1.40   hannken 
   1399   1.40   hannken 	*vpp = NULL;
   1400   1.40   hannken 
   1401   1.40   hannken 	/* Allocate and initialize a new vcache / vnode pair. */
   1402   1.87   hannken 	error = vfs_busy(mp);
   1403   1.40   hannken 	if (error)
   1404   1.40   hannken 		return error;
   1405   1.70   hannken 	vip = vcache_alloc();
   1406   1.70   hannken 	vip->vi_key.vk_mount = mp;
   1407   1.70   hannken 	vp = VIMPL_TO_VNODE(vip);
   1408   1.40   hannken 
   1409   1.40   hannken 	/* Create and load the fs node. */
   1410  1.101   hannken 	error = VFS_NEWVNODE(mp, dvp, vp, vap, cred, extra,
   1411   1.70   hannken 	    &vip->vi_key.vk_key_len, &vip->vi_key.vk_key);
   1412   1.40   hannken 	if (error) {
   1413   1.69   hannken 		mutex_enter(&vcache_lock);
   1414   1.79   hannken 		vcache_dealloc(vip);
   1415   1.87   hannken 		vfs_unbusy(mp);
   1416   1.40   hannken 		KASSERT(*vpp == NULL);
   1417   1.40   hannken 		return error;
   1418   1.40   hannken 	}
   1419   1.40   hannken 	KASSERT(vp->v_op != NULL);
   1420   1.97   hannken 	KASSERT((vip->vi_key.vk_key_len == 0) == (mp == dead_rootmount));
   1421   1.97   hannken 	if (vip->vi_key.vk_key_len > 0) {
   1422   1.97   hannken 		KASSERT(vip->vi_key.vk_key != NULL);
   1423   1.97   hannken 		hash = vcache_hash(&vip->vi_key);
   1424   1.40   hannken 
   1425   1.97   hannken 		/*
   1426   1.97   hannken 		 * Wait for previous instance to be reclaimed,
   1427   1.97   hannken 		 * then insert new node.
   1428   1.97   hannken 		 */
   1429   1.97   hannken 		mutex_enter(&vcache_lock);
   1430   1.97   hannken 		while ((ovip = vcache_hash_lookup(&vip->vi_key, hash))) {
   1431   1.97   hannken 			ovp = VIMPL_TO_VNODE(ovip);
   1432   1.97   hannken 			mutex_enter(ovp->v_interlock);
   1433   1.97   hannken 			mutex_exit(&vcache_lock);
   1434   1.97   hannken 			error = vcache_vget(ovp);
   1435   1.97   hannken 			KASSERT(error == ENOENT);
   1436   1.97   hannken 			mutex_enter(&vcache_lock);
   1437   1.97   hannken 		}
   1438   1.97   hannken 		SLIST_INSERT_HEAD(&vcache_hashtab[hash & vcache_hashmask],
   1439   1.97   hannken 		    vip, vi_hash);
   1440   1.69   hannken 		mutex_exit(&vcache_lock);
   1441   1.40   hannken 	}
   1442   1.40   hannken 	vfs_insmntque(vp, mp);
   1443   1.40   hannken 	if ((mp->mnt_iflag & IMNT_MPSAFE) != 0)
   1444   1.40   hannken 		vp->v_vflag |= VV_MPSAFE;
   1445   1.87   hannken 	vfs_ref(mp);
   1446   1.87   hannken 	vfs_unbusy(mp);
   1447   1.40   hannken 
   1448   1.40   hannken 	/* Finished loading, finalize node. */
   1449   1.69   hannken 	mutex_enter(&vcache_lock);
   1450   1.52   hannken 	mutex_enter(vp->v_interlock);
   1451   1.94   hannken 	VSTATE_CHANGE(vp, VS_LOADING, VS_LOADED);
   1452   1.69   hannken 	mutex_exit(&vcache_lock);
   1453   1.40   hannken 	mutex_exit(vp->v_interlock);
   1454   1.40   hannken 	*vpp = vp;
   1455   1.40   hannken 	return 0;
   1456   1.40   hannken }
   1457   1.40   hannken 
   1458   1.40   hannken /*
   1459   1.65   hannken  * Prepare key change: update old cache nodes key and lock new cache node.
   1460   1.37   hannken  * Return an error if the new node already exists.
   1461   1.37   hannken  */
   1462   1.37   hannken int
   1463   1.37   hannken vcache_rekey_enter(struct mount *mp, struct vnode *vp,
   1464   1.37   hannken     const void *old_key, size_t old_key_len,
   1465   1.37   hannken     const void *new_key, size_t new_key_len)
   1466   1.37   hannken {
   1467   1.37   hannken 	uint32_t old_hash, new_hash;
   1468   1.37   hannken 	struct vcache_key old_vcache_key, new_vcache_key;
   1469   1.70   hannken 	vnode_impl_t *vip, *new_vip;
   1470   1.37   hannken 
   1471   1.37   hannken 	old_vcache_key.vk_mount = mp;
   1472   1.37   hannken 	old_vcache_key.vk_key = old_key;
   1473   1.37   hannken 	old_vcache_key.vk_key_len = old_key_len;
   1474   1.37   hannken 	old_hash = vcache_hash(&old_vcache_key);
   1475   1.37   hannken 
   1476   1.37   hannken 	new_vcache_key.vk_mount = mp;
   1477   1.37   hannken 	new_vcache_key.vk_key = new_key;
   1478   1.37   hannken 	new_vcache_key.vk_key_len = new_key_len;
   1479   1.37   hannken 	new_hash = vcache_hash(&new_vcache_key);
   1480   1.37   hannken 
   1481   1.70   hannken 	new_vip = vcache_alloc();
   1482   1.70   hannken 	new_vip->vi_key = new_vcache_key;
   1483   1.37   hannken 
   1484   1.52   hannken 	/* Insert locked new node used as placeholder. */
   1485   1.69   hannken 	mutex_enter(&vcache_lock);
   1486   1.70   hannken 	vip = vcache_hash_lookup(&new_vcache_key, new_hash);
   1487   1.70   hannken 	if (vip != NULL) {
   1488   1.79   hannken 		vcache_dealloc(new_vip);
   1489   1.37   hannken 		return EEXIST;
   1490   1.37   hannken 	}
   1491   1.69   hannken 	SLIST_INSERT_HEAD(&vcache_hashtab[new_hash & vcache_hashmask],
   1492   1.70   hannken 	    new_vip, vi_hash);
   1493   1.49   hannken 
   1494   1.65   hannken 	/* Replace old nodes key with the temporary copy. */
   1495   1.70   hannken 	vip = vcache_hash_lookup(&old_vcache_key, old_hash);
   1496   1.70   hannken 	KASSERT(vip != NULL);
   1497   1.70   hannken 	KASSERT(VIMPL_TO_VNODE(vip) == vp);
   1498   1.70   hannken 	KASSERT(vip->vi_key.vk_key != old_vcache_key.vk_key);
   1499   1.70   hannken 	vip->vi_key = old_vcache_key;
   1500   1.69   hannken 	mutex_exit(&vcache_lock);
   1501   1.37   hannken 	return 0;
   1502   1.37   hannken }
   1503   1.37   hannken 
   1504   1.37   hannken /*
   1505   1.65   hannken  * Key change complete: update old node and remove placeholder.
   1506   1.37   hannken  */
   1507   1.37   hannken void
   1508   1.37   hannken vcache_rekey_exit(struct mount *mp, struct vnode *vp,
   1509   1.37   hannken     const void *old_key, size_t old_key_len,
   1510   1.37   hannken     const void *new_key, size_t new_key_len)
   1511   1.37   hannken {
   1512   1.37   hannken 	uint32_t old_hash, new_hash;
   1513   1.37   hannken 	struct vcache_key old_vcache_key, new_vcache_key;
   1514   1.70   hannken 	vnode_impl_t *vip, *new_vip;
   1515   1.70   hannken 	struct vnode *new_vp;
   1516   1.37   hannken 
   1517   1.37   hannken 	old_vcache_key.vk_mount = mp;
   1518   1.37   hannken 	old_vcache_key.vk_key = old_key;
   1519   1.37   hannken 	old_vcache_key.vk_key_len = old_key_len;
   1520   1.37   hannken 	old_hash = vcache_hash(&old_vcache_key);
   1521   1.37   hannken 
   1522   1.37   hannken 	new_vcache_key.vk_mount = mp;
   1523   1.37   hannken 	new_vcache_key.vk_key = new_key;
   1524   1.37   hannken 	new_vcache_key.vk_key_len = new_key_len;
   1525   1.37   hannken 	new_hash = vcache_hash(&new_vcache_key);
   1526   1.37   hannken 
   1527   1.69   hannken 	mutex_enter(&vcache_lock);
   1528   1.49   hannken 
   1529   1.49   hannken 	/* Lookup old and new node. */
   1530   1.70   hannken 	vip = vcache_hash_lookup(&old_vcache_key, old_hash);
   1531   1.70   hannken 	KASSERT(vip != NULL);
   1532   1.70   hannken 	KASSERT(VIMPL_TO_VNODE(vip) == vp);
   1533   1.70   hannken 
   1534   1.70   hannken 	new_vip = vcache_hash_lookup(&new_vcache_key, new_hash);
   1535   1.70   hannken 	KASSERT(new_vip != NULL);
   1536   1.70   hannken 	KASSERT(new_vip->vi_key.vk_key_len == new_key_len);
   1537   1.70   hannken 	new_vp = VIMPL_TO_VNODE(new_vip);
   1538   1.70   hannken 	mutex_enter(new_vp->v_interlock);
   1539   1.70   hannken 	VSTATE_ASSERT(VIMPL_TO_VNODE(new_vip), VS_LOADING);
   1540   1.79   hannken 	mutex_exit(new_vp->v_interlock);
   1541   1.49   hannken 
   1542   1.49   hannken 	/* Rekey old node and put it onto its new hashlist. */
   1543   1.70   hannken 	vip->vi_key = new_vcache_key;
   1544   1.49   hannken 	if (old_hash != new_hash) {
   1545   1.69   hannken 		SLIST_REMOVE(&vcache_hashtab[old_hash & vcache_hashmask],
   1546   1.70   hannken 		    vip, vnode_impl, vi_hash);
   1547   1.69   hannken 		SLIST_INSERT_HEAD(&vcache_hashtab[new_hash & vcache_hashmask],
   1548   1.70   hannken 		    vip, vi_hash);
   1549   1.49   hannken 	}
   1550   1.49   hannken 
   1551   1.49   hannken 	/* Remove new node used as placeholder. */
   1552   1.69   hannken 	SLIST_REMOVE(&vcache_hashtab[new_hash & vcache_hashmask],
   1553   1.70   hannken 	    new_vip, vnode_impl, vi_hash);
   1554   1.79   hannken 	vcache_dealloc(new_vip);
   1555   1.37   hannken }
   1556   1.37   hannken 
   1557   1.37   hannken /*
   1558   1.54   hannken  * Disassociate the underlying file system from a vnode.
   1559   1.54   hannken  *
   1560   1.54   hannken  * Must be called with vnode locked and will return unlocked.
   1561   1.54   hannken  * Must be called with the interlock held, and will return with it held.
   1562   1.54   hannken  */
   1563   1.54   hannken static void
   1564   1.54   hannken vcache_reclaim(vnode_t *vp)
   1565   1.54   hannken {
   1566   1.54   hannken 	lwp_t *l = curlwp;
   1567   1.70   hannken 	vnode_impl_t *vip = VNODE_TO_VIMPL(vp);
   1568   1.74   hannken 	struct mount *mp = vp->v_mount;
   1569   1.55   hannken 	uint32_t hash;
   1570   1.55   hannken 	uint8_t temp_buf[64], *temp_key;
   1571   1.55   hannken 	size_t temp_key_len;
   1572   1.54   hannken 	bool recycle, active;
   1573   1.54   hannken 	int error;
   1574   1.54   hannken 
   1575   1.54   hannken 	KASSERT((vp->v_vflag & VV_LOCKSWORK) == 0 ||
   1576   1.54   hannken 	    VOP_ISLOCKED(vp) == LK_EXCLUSIVE);
   1577   1.54   hannken 	KASSERT(mutex_owned(vp->v_interlock));
   1578   1.54   hannken 	KASSERT(vp->v_usecount != 0);
   1579   1.54   hannken 
   1580   1.54   hannken 	active = (vp->v_usecount > 1);
   1581   1.70   hannken 	temp_key_len = vip->vi_key.vk_key_len;
   1582   1.54   hannken 	/*
   1583   1.54   hannken 	 * Prevent the vnode from being recycled or brought into use
   1584   1.54   hannken 	 * while we clean it out.
   1585   1.54   hannken 	 */
   1586   1.94   hannken 	VSTATE_CHANGE(vp, VS_LOADED, VS_RECLAIMING);
   1587  1.104        ad 	if ((vp->v_iflag & VI_EXECMAP) != 0 && vp->v_uobj.uo_npages != 0) {
   1588  1.105        ad 		cpu_count(CPU_COUNT_EXECPAGES, -vp->v_uobj.uo_npages);
   1589  1.105        ad 		cpu_count(CPU_COUNT_FILEPAGES, vp->v_uobj.uo_npages);
   1590   1.54   hannken 	}
   1591   1.54   hannken 	vp->v_iflag &= ~(VI_TEXT|VI_EXECMAP);
   1592   1.54   hannken 	mutex_exit(vp->v_interlock);
   1593   1.54   hannken 
   1594   1.55   hannken 	/* Replace the vnode key with a temporary copy. */
   1595   1.70   hannken 	if (vip->vi_key.vk_key_len > sizeof(temp_buf)) {
   1596   1.55   hannken 		temp_key = kmem_alloc(temp_key_len, KM_SLEEP);
   1597   1.55   hannken 	} else {
   1598   1.55   hannken 		temp_key = temp_buf;
   1599   1.55   hannken 	}
   1600   1.97   hannken 	if (vip->vi_key.vk_key_len > 0) {
   1601   1.97   hannken 		mutex_enter(&vcache_lock);
   1602   1.97   hannken 		memcpy(temp_key, vip->vi_key.vk_key, temp_key_len);
   1603   1.97   hannken 		vip->vi_key.vk_key = temp_key;
   1604   1.97   hannken 		mutex_exit(&vcache_lock);
   1605   1.97   hannken 	}
   1606   1.55   hannken 
   1607   1.96   hannken 	fstrans_start(mp);
   1608   1.74   hannken 
   1609   1.54   hannken 	/*
   1610   1.54   hannken 	 * Clean out any cached data associated with the vnode.
   1611   1.54   hannken 	 * If purging an active vnode, it must be closed and
   1612   1.60   hannken 	 * deactivated before being reclaimed.
   1613   1.54   hannken 	 */
   1614   1.54   hannken 	error = vinvalbuf(vp, V_SAVE, NOCRED, l, 0, 0);
   1615   1.54   hannken 	if (error != 0) {
   1616   1.54   hannken 		if (wapbl_vphaswapbl(vp))
   1617   1.54   hannken 			WAPBL_DISCARD(wapbl_vptomp(vp));
   1618   1.54   hannken 		error = vinvalbuf(vp, 0, NOCRED, l, 0, 0);
   1619   1.54   hannken 	}
   1620   1.54   hannken 	KASSERTMSG((error == 0), "vinvalbuf failed: %d", error);
   1621   1.54   hannken 	KASSERT((vp->v_iflag & VI_ONWORKLST) == 0);
   1622   1.54   hannken 	if (active && (vp->v_type == VBLK || vp->v_type == VCHR)) {
   1623   1.54   hannken 		 spec_node_revoke(vp);
   1624   1.54   hannken 	}
   1625   1.54   hannken 
   1626   1.60   hannken 	/*
   1627   1.60   hannken 	 * Disassociate the underlying file system from the vnode.
   1628   1.90  riastrad 	 * VOP_INACTIVE leaves the vnode locked; VOP_RECLAIM unlocks
   1629   1.90  riastrad 	 * the vnode, and may destroy the vnode so that VOP_UNLOCK
   1630   1.90  riastrad 	 * would no longer function.
   1631   1.60   hannken 	 */
   1632   1.60   hannken 	VOP_INACTIVE(vp, &recycle);
   1633   1.91  riastrad 	KASSERT((vp->v_vflag & VV_LOCKSWORK) == 0 ||
   1634   1.91  riastrad 	    VOP_ISLOCKED(vp) == LK_EXCLUSIVE);
   1635   1.54   hannken 	if (VOP_RECLAIM(vp)) {
   1636   1.54   hannken 		vnpanic(vp, "%s: cannot reclaim", __func__);
   1637   1.54   hannken 	}
   1638   1.54   hannken 
   1639   1.54   hannken 	KASSERT(vp->v_data == NULL);
   1640   1.54   hannken 	KASSERT(vp->v_uobj.uo_npages == 0);
   1641   1.54   hannken 
   1642   1.54   hannken 	if (vp->v_type == VREG && vp->v_ractx != NULL) {
   1643   1.54   hannken 		uvm_ra_freectx(vp->v_ractx);
   1644   1.54   hannken 		vp->v_ractx = NULL;
   1645   1.54   hannken 	}
   1646   1.54   hannken 
   1647   1.54   hannken 	/* Purge name cache. */
   1648   1.54   hannken 	cache_purge(vp);
   1649   1.54   hannken 
   1650   1.97   hannken 	if (vip->vi_key.vk_key_len > 0) {
   1651   1.55   hannken 	/* Remove from vnode cache. */
   1652   1.97   hannken 		hash = vcache_hash(&vip->vi_key);
   1653   1.97   hannken 		mutex_enter(&vcache_lock);
   1654   1.97   hannken 		KASSERT(vip == vcache_hash_lookup(&vip->vi_key, hash));
   1655   1.97   hannken 		SLIST_REMOVE(&vcache_hashtab[hash & vcache_hashmask],
   1656   1.97   hannken 		    vip, vnode_impl, vi_hash);
   1657   1.97   hannken 		mutex_exit(&vcache_lock);
   1658   1.97   hannken 	}
   1659   1.55   hannken 	if (temp_key != temp_buf)
   1660   1.55   hannken 		kmem_free(temp_key, temp_key_len);
   1661   1.55   hannken 
   1662   1.54   hannken 	/* Done with purge, notify sleepers of the grim news. */
   1663   1.54   hannken 	mutex_enter(vp->v_interlock);
   1664   1.54   hannken 	vp->v_op = dead_vnodeop_p;
   1665   1.54   hannken 	vp->v_vflag |= VV_LOCKSWORK;
   1666   1.57   hannken 	VSTATE_CHANGE(vp, VS_RECLAIMING, VS_RECLAIMED);
   1667   1.54   hannken 	vp->v_tag = VT_NON;
   1668   1.54   hannken 	KNOTE(&vp->v_klist, NOTE_REVOKE);
   1669   1.80   hannken 	mutex_exit(vp->v_interlock);
   1670   1.54   hannken 
   1671   1.80   hannken 	/*
   1672   1.80   hannken 	 * Move to dead mount.  Must be after changing the operations
   1673   1.80   hannken 	 * vector as vnode operations enter the mount before using the
   1674   1.80   hannken 	 * operations vector.  See sys/kern/vnode_if.c.
   1675   1.80   hannken 	 */
   1676   1.80   hannken 	vp->v_vflag &= ~VV_ROOT;
   1677   1.86   hannken 	vfs_ref(dead_rootmount);
   1678   1.80   hannken 	vfs_insmntque(vp, dead_rootmount);
   1679   1.80   hannken 
   1680   1.80   hannken 	mutex_enter(vp->v_interlock);
   1681   1.74   hannken 	fstrans_done(mp);
   1682   1.54   hannken 	KASSERT((vp->v_iflag & VI_ONWORKLST) == 0);
   1683  1.108        ad 
   1684  1.108        ad #ifdef PAX_SEGVGUARD
   1685  1.108        ad 	pax_segvguard_cleanup(vp);
   1686  1.108        ad #endif /* PAX_SEGVGUARD */
   1687   1.54   hannken }
   1688   1.54   hannken 
   1689   1.54   hannken /*
   1690   1.98   hannken  * Disassociate the underlying file system from an open device vnode
   1691   1.98   hannken  * and make it anonymous.
   1692   1.98   hannken  *
   1693   1.98   hannken  * Vnode unlocked on entry, drops a reference to the vnode.
   1694   1.98   hannken  */
   1695   1.98   hannken void
   1696   1.98   hannken vcache_make_anon(vnode_t *vp)
   1697   1.98   hannken {
   1698   1.98   hannken 	vnode_impl_t *vip = VNODE_TO_VIMPL(vp);
   1699   1.98   hannken 	uint32_t hash;
   1700   1.98   hannken 	bool recycle;
   1701   1.98   hannken 
   1702   1.98   hannken 	KASSERT(vp->v_type == VBLK || vp->v_type == VCHR);
   1703  1.103   hannken 	KASSERT(vp->v_mount == dead_rootmount || fstrans_is_owner(vp->v_mount));
   1704   1.98   hannken 	VSTATE_ASSERT_UNLOCKED(vp, VS_ACTIVE);
   1705   1.98   hannken 
   1706   1.98   hannken 	/* Remove from vnode cache. */
   1707   1.98   hannken 	hash = vcache_hash(&vip->vi_key);
   1708   1.98   hannken 	mutex_enter(&vcache_lock);
   1709   1.98   hannken 	KASSERT(vip == vcache_hash_lookup(&vip->vi_key, hash));
   1710   1.98   hannken 	SLIST_REMOVE(&vcache_hashtab[hash & vcache_hashmask],
   1711   1.98   hannken 	    vip, vnode_impl, vi_hash);
   1712   1.98   hannken 	vip->vi_key.vk_mount = dead_rootmount;
   1713   1.98   hannken 	vip->vi_key.vk_key_len = 0;
   1714   1.98   hannken 	vip->vi_key.vk_key = NULL;
   1715   1.98   hannken 	mutex_exit(&vcache_lock);
   1716   1.98   hannken 
   1717   1.98   hannken 	/*
   1718   1.98   hannken 	 * Disassociate the underlying file system from the vnode.
   1719   1.98   hannken 	 * VOP_INACTIVE leaves the vnode locked; VOP_RECLAIM unlocks
   1720   1.98   hannken 	 * the vnode, and may destroy the vnode so that VOP_UNLOCK
   1721   1.98   hannken 	 * would no longer function.
   1722   1.98   hannken 	 */
   1723   1.98   hannken 	if (vn_lock(vp, LK_EXCLUSIVE)) {
   1724   1.98   hannken 		vnpanic(vp, "%s: cannot lock", __func__);
   1725   1.98   hannken 	}
   1726   1.98   hannken 	VOP_INACTIVE(vp, &recycle);
   1727   1.98   hannken 	KASSERT((vp->v_vflag & VV_LOCKSWORK) == 0 ||
   1728   1.98   hannken 	    VOP_ISLOCKED(vp) == LK_EXCLUSIVE);
   1729   1.98   hannken 	if (VOP_RECLAIM(vp)) {
   1730   1.98   hannken 		vnpanic(vp, "%s: cannot reclaim", __func__);
   1731   1.98   hannken 	}
   1732   1.98   hannken 
   1733   1.98   hannken 	/* Purge name cache. */
   1734   1.98   hannken 	cache_purge(vp);
   1735   1.98   hannken 
   1736   1.98   hannken 	/* Done with purge, change operations vector. */
   1737   1.98   hannken 	mutex_enter(vp->v_interlock);
   1738   1.98   hannken 	vp->v_op = spec_vnodeop_p;
   1739   1.98   hannken 	vp->v_vflag |= VV_MPSAFE;
   1740   1.98   hannken 	vp->v_vflag &= ~VV_LOCKSWORK;
   1741   1.98   hannken 	mutex_exit(vp->v_interlock);
   1742   1.98   hannken 
   1743   1.98   hannken 	/*
   1744   1.98   hannken 	 * Move to dead mount.  Must be after changing the operations
   1745   1.98   hannken 	 * vector as vnode operations enter the mount before using the
   1746   1.98   hannken 	 * operations vector.  See sys/kern/vnode_if.c.
   1747   1.98   hannken 	 */
   1748   1.98   hannken 	vfs_ref(dead_rootmount);
   1749   1.98   hannken 	vfs_insmntque(vp, dead_rootmount);
   1750   1.98   hannken 
   1751   1.98   hannken 	vrele(vp);
   1752   1.98   hannken }
   1753   1.98   hannken 
   1754   1.98   hannken /*
   1755    1.1     rmind  * Update outstanding I/O count and do wakeup if requested.
   1756    1.1     rmind  */
   1757    1.1     rmind void
   1758    1.1     rmind vwakeup(struct buf *bp)
   1759    1.1     rmind {
   1760    1.1     rmind 	vnode_t *vp;
   1761    1.1     rmind 
   1762    1.1     rmind 	if ((vp = bp->b_vp) == NULL)
   1763    1.1     rmind 		return;
   1764    1.1     rmind 
   1765    1.9     rmind 	KASSERT(bp->b_objlock == vp->v_interlock);
   1766    1.1     rmind 	KASSERT(mutex_owned(bp->b_objlock));
   1767    1.1     rmind 
   1768    1.1     rmind 	if (--vp->v_numoutput < 0)
   1769   1.11  christos 		vnpanic(vp, "%s: neg numoutput, vp %p", __func__, vp);
   1770    1.1     rmind 	if (vp->v_numoutput == 0)
   1771    1.1     rmind 		cv_broadcast(&vp->v_cv);
   1772    1.1     rmind }
   1773    1.1     rmind 
   1774    1.1     rmind /*
   1775   1.35   hannken  * Test a vnode for being or becoming dead.  Returns one of:
   1776   1.35   hannken  * EBUSY:  vnode is becoming dead, with "flags == VDEAD_NOWAIT" only.
   1777   1.35   hannken  * ENOENT: vnode is dead.
   1778   1.35   hannken  * 0:      otherwise.
   1779   1.35   hannken  *
   1780   1.35   hannken  * Whenever this function returns a non-zero value all future
   1781   1.35   hannken  * calls will also return a non-zero value.
   1782   1.35   hannken  */
   1783   1.35   hannken int
   1784   1.35   hannken vdead_check(struct vnode *vp, int flags)
   1785   1.35   hannken {
   1786   1.35   hannken 
   1787   1.35   hannken 	KASSERT(mutex_owned(vp->v_interlock));
   1788   1.35   hannken 
   1789   1.52   hannken 	if (! ISSET(flags, VDEAD_NOWAIT))
   1790   1.52   hannken 		VSTATE_WAIT_STABLE(vp);
   1791    1.1     rmind 
   1792   1.57   hannken 	if (VSTATE_GET(vp) == VS_RECLAIMING) {
   1793   1.52   hannken 		KASSERT(ISSET(flags, VDEAD_NOWAIT));
   1794   1.52   hannken 		return EBUSY;
   1795   1.57   hannken 	} else if (VSTATE_GET(vp) == VS_RECLAIMED) {
   1796   1.52   hannken 		return ENOENT;
   1797   1.52   hannken 	}
   1798    1.1     rmind 
   1799   1.52   hannken 	return 0;
   1800    1.1     rmind }
   1801    1.1     rmind 
   1802    1.1     rmind int
   1803   1.61   hannken vfs_drainvnodes(void)
   1804    1.1     rmind {
   1805   1.63   hannken 	int i, gen;
   1806   1.61   hannken 
   1807   1.63   hannken 	mutex_enter(&vdrain_lock);
   1808   1.63   hannken 	for (i = 0; i < 2; i++) {
   1809   1.63   hannken 		gen = vdrain_gen;
   1810   1.63   hannken 		while (gen == vdrain_gen) {
   1811   1.63   hannken 			cv_broadcast(&vdrain_cv);
   1812   1.63   hannken 			cv_wait(&vdrain_gen_cv, &vdrain_lock);
   1813   1.63   hannken 		}
   1814   1.61   hannken 	}
   1815   1.63   hannken 	mutex_exit(&vdrain_lock);
   1816   1.12   hannken 
   1817   1.63   hannken 	if (numvnodes >= desiredvnodes)
   1818   1.63   hannken 		return EBUSY;
   1819   1.12   hannken 
   1820   1.69   hannken 	if (vcache_hashsize != desiredvnodes)
   1821   1.61   hannken 		vcache_reinit();
   1822   1.36   hannken 
   1823    1.1     rmind 	return 0;
   1824    1.1     rmind }
   1825    1.1     rmind 
   1826    1.1     rmind void
   1827   1.11  christos vnpanic(vnode_t *vp, const char *fmt, ...)
   1828    1.1     rmind {
   1829   1.11  christos 	va_list ap;
   1830   1.11  christos 
   1831    1.1     rmind #ifdef DIAGNOSTIC
   1832    1.1     rmind 	vprint(NULL, vp);
   1833    1.1     rmind #endif
   1834   1.11  christos 	va_start(ap, fmt);
   1835   1.11  christos 	vpanic(fmt, ap);
   1836   1.11  christos 	va_end(ap);
   1837    1.1     rmind }
   1838