vfs_vnode.c revision 1.109 1 1.109 ad /* $NetBSD: vfs_vnode.c,v 1.109 2020/01/23 11:17:09 ad Exp $ */
2 1.1 rmind
3 1.1 rmind /*-
4 1.104 ad * Copyright (c) 1997-2011, 2019 The NetBSD Foundation, Inc.
5 1.1 rmind * All rights reserved.
6 1.1 rmind *
7 1.1 rmind * This code is derived from software contributed to The NetBSD Foundation
8 1.1 rmind * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
9 1.1 rmind * NASA Ames Research Center, by Charles M. Hannum, and by Andrew Doran.
10 1.1 rmind *
11 1.1 rmind * Redistribution and use in source and binary forms, with or without
12 1.1 rmind * modification, are permitted provided that the following conditions
13 1.1 rmind * are met:
14 1.1 rmind * 1. Redistributions of source code must retain the above copyright
15 1.1 rmind * notice, this list of conditions and the following disclaimer.
16 1.1 rmind * 2. Redistributions in binary form must reproduce the above copyright
17 1.1 rmind * notice, this list of conditions and the following disclaimer in the
18 1.1 rmind * documentation and/or other materials provided with the distribution.
19 1.1 rmind *
20 1.1 rmind * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
21 1.1 rmind * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
22 1.1 rmind * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
23 1.1 rmind * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
24 1.1 rmind * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 1.1 rmind * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 1.1 rmind * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 1.1 rmind * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 1.1 rmind * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 1.1 rmind * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 1.1 rmind * POSSIBILITY OF SUCH DAMAGE.
31 1.1 rmind */
32 1.1 rmind
33 1.1 rmind /*
34 1.1 rmind * Copyright (c) 1989, 1993
35 1.1 rmind * The Regents of the University of California. All rights reserved.
36 1.1 rmind * (c) UNIX System Laboratories, Inc.
37 1.1 rmind * All or some portions of this file are derived from material licensed
38 1.1 rmind * to the University of California by American Telephone and Telegraph
39 1.1 rmind * Co. or Unix System Laboratories, Inc. and are reproduced herein with
40 1.1 rmind * the permission of UNIX System Laboratories, Inc.
41 1.1 rmind *
42 1.1 rmind * Redistribution and use in source and binary forms, with or without
43 1.1 rmind * modification, are permitted provided that the following conditions
44 1.1 rmind * are met:
45 1.1 rmind * 1. Redistributions of source code must retain the above copyright
46 1.1 rmind * notice, this list of conditions and the following disclaimer.
47 1.1 rmind * 2. Redistributions in binary form must reproduce the above copyright
48 1.1 rmind * notice, this list of conditions and the following disclaimer in the
49 1.1 rmind * documentation and/or other materials provided with the distribution.
50 1.1 rmind * 3. Neither the name of the University nor the names of its contributors
51 1.1 rmind * may be used to endorse or promote products derived from this software
52 1.1 rmind * without specific prior written permission.
53 1.1 rmind *
54 1.1 rmind * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
55 1.1 rmind * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
56 1.1 rmind * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
57 1.1 rmind * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
58 1.1 rmind * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
59 1.1 rmind * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
60 1.1 rmind * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
61 1.1 rmind * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
62 1.1 rmind * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
63 1.1 rmind * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
64 1.1 rmind * SUCH DAMAGE.
65 1.1 rmind *
66 1.1 rmind * @(#)vfs_subr.c 8.13 (Berkeley) 4/18/94
67 1.1 rmind */
68 1.1 rmind
69 1.1 rmind /*
70 1.8 rmind * The vnode cache subsystem.
71 1.1 rmind *
72 1.8 rmind * Life-cycle
73 1.1 rmind *
74 1.8 rmind * Normally, there are two points where new vnodes are created:
75 1.8 rmind * VOP_CREATE(9) and VOP_LOOKUP(9). The life-cycle of a vnode
76 1.8 rmind * starts in one of the following ways:
77 1.8 rmind *
78 1.45 hannken * - Allocation, via vcache_get(9) or vcache_new(9).
79 1.66 hannken * - Reclamation of inactive vnode, via vcache_vget(9).
80 1.8 rmind *
81 1.16 rmind * Recycle from a free list, via getnewvnode(9) -> getcleanvnode(9)
82 1.16 rmind * was another, traditional way. Currently, only the draining thread
83 1.16 rmind * recycles the vnodes. This behaviour might be revisited.
84 1.16 rmind *
85 1.8 rmind * The life-cycle ends when the last reference is dropped, usually
86 1.8 rmind * in VOP_REMOVE(9). In such case, VOP_INACTIVE(9) is called to inform
87 1.8 rmind * the file system that vnode is inactive. Via this call, file system
88 1.16 rmind * indicates whether vnode can be recycled (usually, it checks its own
89 1.16 rmind * references, e.g. count of links, whether the file was removed).
90 1.8 rmind *
91 1.8 rmind * Depending on indication, vnode can be put into a free list (cache),
92 1.54 hannken * or cleaned via vcache_reclaim, which calls VOP_RECLAIM(9) to
93 1.54 hannken * disassociate underlying file system from the vnode, and finally
94 1.54 hannken * destroyed.
95 1.8 rmind *
96 1.52 hannken * Vnode state
97 1.52 hannken *
98 1.52 hannken * Vnode is always in one of six states:
99 1.52 hannken * - MARKER This is a marker vnode to help list traversal. It
100 1.52 hannken * will never change its state.
101 1.52 hannken * - LOADING Vnode is associating underlying file system and not
102 1.52 hannken * yet ready to use.
103 1.94 hannken * - LOADED Vnode has associated underlying file system and is
104 1.52 hannken * ready to use.
105 1.52 hannken * - BLOCKED Vnode is active but cannot get new references.
106 1.52 hannken * - RECLAIMING Vnode is disassociating from the underlying file
107 1.52 hannken * system.
108 1.52 hannken * - RECLAIMED Vnode has disassociated from underlying file system
109 1.52 hannken * and is dead.
110 1.52 hannken *
111 1.52 hannken * Valid state changes are:
112 1.94 hannken * LOADING -> LOADED
113 1.52 hannken * Vnode has been initialised in vcache_get() or
114 1.52 hannken * vcache_new() and is ready to use.
115 1.94 hannken * LOADED -> RECLAIMING
116 1.52 hannken * Vnode starts disassociation from underlying file
117 1.54 hannken * system in vcache_reclaim().
118 1.52 hannken * RECLAIMING -> RECLAIMED
119 1.52 hannken * Vnode finished disassociation from underlying file
120 1.54 hannken * system in vcache_reclaim().
121 1.94 hannken * LOADED -> BLOCKED
122 1.52 hannken * Either vcache_rekey*() is changing the vnode key or
123 1.52 hannken * vrelel() is about to call VOP_INACTIVE().
124 1.94 hannken * BLOCKED -> LOADED
125 1.52 hannken * The block condition is over.
126 1.52 hannken * LOADING -> RECLAIMED
127 1.52 hannken * Either vcache_get() or vcache_new() failed to
128 1.52 hannken * associate the underlying file system or vcache_rekey*()
129 1.52 hannken * drops a vnode used as placeholder.
130 1.52 hannken *
131 1.52 hannken * Of these states LOADING, BLOCKED and RECLAIMING are intermediate
132 1.52 hannken * and it is possible to wait for state change.
133 1.52 hannken *
134 1.52 hannken * State is protected with v_interlock with one exception:
135 1.69 hannken * to change from LOADING both v_interlock and vcache_lock must be held
136 1.52 hannken * so it is possible to check "state == LOADING" without holding
137 1.52 hannken * v_interlock. See vcache_get() for details.
138 1.52 hannken *
139 1.8 rmind * Reference counting
140 1.8 rmind *
141 1.8 rmind * Vnode is considered active, if reference count (vnode_t::v_usecount)
142 1.8 rmind * is non-zero. It is maintained using: vref(9) and vrele(9), as well
143 1.8 rmind * as vput(9), routines. Common points holding references are e.g.
144 1.8 rmind * file openings, current working directory, mount points, etc.
145 1.8 rmind *
146 1.1 rmind */
147 1.1 rmind
148 1.1 rmind #include <sys/cdefs.h>
149 1.109 ad __KERNEL_RCSID(0, "$NetBSD: vfs_vnode.c,v 1.109 2020/01/23 11:17:09 ad Exp $");
150 1.108 ad
151 1.109 ad #ifdef _KERNEL_OPT
152 1.108 ad #include "opt_pax.h"
153 1.109 ad #endif
154 1.1 rmind
155 1.1 rmind #include <sys/param.h>
156 1.1 rmind #include <sys/kernel.h>
157 1.1 rmind
158 1.1 rmind #include <sys/atomic.h>
159 1.1 rmind #include <sys/buf.h>
160 1.1 rmind #include <sys/conf.h>
161 1.1 rmind #include <sys/device.h>
162 1.36 hannken #include <sys/hash.h>
163 1.1 rmind #include <sys/kauth.h>
164 1.1 rmind #include <sys/kmem.h>
165 1.1 rmind #include <sys/kthread.h>
166 1.1 rmind #include <sys/module.h>
167 1.1 rmind #include <sys/mount.h>
168 1.1 rmind #include <sys/namei.h>
169 1.108 ad #include <sys/pax.h>
170 1.1 rmind #include <sys/syscallargs.h>
171 1.1 rmind #include <sys/sysctl.h>
172 1.1 rmind #include <sys/systm.h>
173 1.58 hannken #include <sys/vnode_impl.h>
174 1.1 rmind #include <sys/wapbl.h>
175 1.24 hannken #include <sys/fstrans.h>
176 1.1 rmind
177 1.1 rmind #include <uvm/uvm.h>
178 1.1 rmind #include <uvm/uvm_readahead.h>
179 1.104 ad #include <uvm/uvm_stat.h>
180 1.1 rmind
181 1.23 hannken /* Flags to vrelel. */
182 1.104 ad #define VRELEL_ASYNC 0x0001 /* Always defer to vrele thread. */
183 1.104 ad
184 1.104 ad #define LRU_VRELE 0
185 1.104 ad #define LRU_FREE 1
186 1.104 ad #define LRU_HOLD 2
187 1.104 ad #define LRU_COUNT 3
188 1.1 rmind
189 1.16 rmind /*
190 1.63 hannken * There are three lru lists: one holds vnodes waiting for async release,
191 1.104 ad * one is for vnodes which have no buffer/page references and one for those
192 1.104 ad * which do (i.e. v_holdcnt is non-zero). We put the lists into a single,
193 1.104 ad * private cache line as vnodes migrate between them while under the same
194 1.104 ad * lock (vdrain_lock).
195 1.63 hannken */
196 1.104 ad u_int numvnodes __cacheline_aligned;
197 1.104 ad static vnodelst_t lru_list[LRU_COUNT] __cacheline_aligned;
198 1.63 hannken static kmutex_t vdrain_lock __cacheline_aligned;
199 1.104 ad static kcondvar_t vdrain_cv;
200 1.63 hannken static int vdrain_gen;
201 1.63 hannken static kcondvar_t vdrain_gen_cv;
202 1.63 hannken static bool vdrain_retry;
203 1.63 hannken static lwp_t * vdrain_lwp;
204 1.57 hannken SLIST_HEAD(hashhead, vnode_impl);
205 1.69 hannken static kmutex_t vcache_lock __cacheline_aligned;
206 1.104 ad static kcondvar_t vcache_cv;
207 1.69 hannken static u_int vcache_hashsize;
208 1.69 hannken static u_long vcache_hashmask;
209 1.104 ad static struct hashhead *vcache_hashtab;
210 1.69 hannken static pool_cache_t vcache_pool;
211 1.63 hannken static void lru_requeue(vnode_t *, vnodelst_t *);
212 1.63 hannken static vnodelst_t * lru_which(vnode_t *);
213 1.63 hannken static vnode_impl_t * vcache_alloc(void);
214 1.79 hannken static void vcache_dealloc(vnode_impl_t *);
215 1.57 hannken static void vcache_free(vnode_impl_t *);
216 1.36 hannken static void vcache_init(void);
217 1.36 hannken static void vcache_reinit(void);
218 1.54 hannken static void vcache_reclaim(vnode_t *);
219 1.107 ad static void vrelel(vnode_t *, int, int);
220 1.12 hannken static void vdrain_thread(void *);
221 1.11 christos static void vnpanic(vnode_t *, const char *, ...)
222 1.18 christos __printflike(2, 3);
223 1.1 rmind
224 1.1 rmind /* Routines having to do with the management of the vnode table. */
225 1.44 hannken extern struct mount *dead_rootmount;
226 1.1 rmind extern int (**dead_vnodeop_p)(void *);
227 1.98 hannken extern int (**spec_vnodeop_p)(void *);
228 1.31 hannken extern struct vfsops dead_vfsops;
229 1.1 rmind
230 1.51 hannken /* Vnode state operations and diagnostics. */
231 1.51 hannken
232 1.51 hannken #if defined(DIAGNOSTIC)
233 1.51 hannken
234 1.94 hannken #define VSTATE_VALID(state) \
235 1.94 hannken ((state) != VS_ACTIVE && (state) != VS_MARKER)
236 1.51 hannken #define VSTATE_GET(vp) \
237 1.51 hannken vstate_assert_get((vp), __func__, __LINE__)
238 1.51 hannken #define VSTATE_CHANGE(vp, from, to) \
239 1.51 hannken vstate_assert_change((vp), (from), (to), __func__, __LINE__)
240 1.51 hannken #define VSTATE_WAIT_STABLE(vp) \
241 1.51 hannken vstate_assert_wait_stable((vp), __func__, __LINE__)
242 1.51 hannken
243 1.94 hannken void
244 1.99 joerg _vstate_assert(vnode_t *vp, enum vnode_state state, const char *func, int line,
245 1.99 joerg bool has_lock)
246 1.51 hannken {
247 1.70 hannken vnode_impl_t *vip = VNODE_TO_VIMPL(vp);
248 1.51 hannken
249 1.99 joerg if (!has_lock) {
250 1.99 joerg /*
251 1.99 joerg * Prevent predictive loads from the CPU, but check the state
252 1.99 joerg * without loooking first.
253 1.99 joerg */
254 1.99 joerg membar_enter();
255 1.99 joerg if (state == VS_ACTIVE && vp->v_usecount > 0 &&
256 1.99 joerg (vip->vi_state == VS_LOADED || vip->vi_state == VS_BLOCKED))
257 1.99 joerg return;
258 1.99 joerg if (vip->vi_state == state)
259 1.99 joerg return;
260 1.99 joerg mutex_enter((vp)->v_interlock);
261 1.99 joerg }
262 1.99 joerg
263 1.51 hannken KASSERTMSG(mutex_owned(vp->v_interlock), "at %s:%d", func, line);
264 1.51 hannken
265 1.99 joerg if ((state == VS_ACTIVE && vp->v_usecount > 0 &&
266 1.99 joerg (vip->vi_state == VS_LOADED || vip->vi_state == VS_BLOCKED)) ||
267 1.99 joerg vip->vi_state == state) {
268 1.99 joerg if (!has_lock)
269 1.99 joerg mutex_exit((vp)->v_interlock);
270 1.94 hannken return;
271 1.99 joerg }
272 1.94 hannken vnpanic(vp, "state is %s, usecount %d, expected %s at %s:%d",
273 1.94 hannken vstate_name(vip->vi_state), vp->v_usecount,
274 1.94 hannken vstate_name(state), func, line);
275 1.51 hannken }
276 1.51 hannken
277 1.57 hannken static enum vnode_state
278 1.51 hannken vstate_assert_get(vnode_t *vp, const char *func, int line)
279 1.51 hannken {
280 1.70 hannken vnode_impl_t *vip = VNODE_TO_VIMPL(vp);
281 1.51 hannken
282 1.51 hannken KASSERTMSG(mutex_owned(vp->v_interlock), "at %s:%d", func, line);
283 1.94 hannken if (! VSTATE_VALID(vip->vi_state))
284 1.51 hannken vnpanic(vp, "state is %s at %s:%d",
285 1.70 hannken vstate_name(vip->vi_state), func, line);
286 1.51 hannken
287 1.70 hannken return vip->vi_state;
288 1.51 hannken }
289 1.51 hannken
290 1.52 hannken static void
291 1.51 hannken vstate_assert_wait_stable(vnode_t *vp, const char *func, int line)
292 1.51 hannken {
293 1.70 hannken vnode_impl_t *vip = VNODE_TO_VIMPL(vp);
294 1.51 hannken
295 1.51 hannken KASSERTMSG(mutex_owned(vp->v_interlock), "at %s:%d", func, line);
296 1.94 hannken if (! VSTATE_VALID(vip->vi_state))
297 1.51 hannken vnpanic(vp, "state is %s at %s:%d",
298 1.70 hannken vstate_name(vip->vi_state), func, line);
299 1.51 hannken
300 1.94 hannken while (vip->vi_state != VS_LOADED && vip->vi_state != VS_RECLAIMED)
301 1.51 hannken cv_wait(&vp->v_cv, vp->v_interlock);
302 1.51 hannken
303 1.94 hannken if (! VSTATE_VALID(vip->vi_state))
304 1.51 hannken vnpanic(vp, "state is %s at %s:%d",
305 1.70 hannken vstate_name(vip->vi_state), func, line);
306 1.51 hannken }
307 1.51 hannken
308 1.52 hannken static void
309 1.57 hannken vstate_assert_change(vnode_t *vp, enum vnode_state from, enum vnode_state to,
310 1.51 hannken const char *func, int line)
311 1.51 hannken {
312 1.70 hannken vnode_impl_t *vip = VNODE_TO_VIMPL(vp);
313 1.51 hannken
314 1.51 hannken KASSERTMSG(mutex_owned(vp->v_interlock), "at %s:%d", func, line);
315 1.57 hannken if (from == VS_LOADING)
316 1.69 hannken KASSERTMSG(mutex_owned(&vcache_lock), "at %s:%d", func, line);
317 1.51 hannken
318 1.94 hannken if (! VSTATE_VALID(from))
319 1.51 hannken vnpanic(vp, "from is %s at %s:%d",
320 1.51 hannken vstate_name(from), func, line);
321 1.94 hannken if (! VSTATE_VALID(to))
322 1.51 hannken vnpanic(vp, "to is %s at %s:%d",
323 1.51 hannken vstate_name(to), func, line);
324 1.70 hannken if (vip->vi_state != from)
325 1.51 hannken vnpanic(vp, "from is %s, expected %s at %s:%d\n",
326 1.70 hannken vstate_name(vip->vi_state), vstate_name(from), func, line);
327 1.68 hannken if ((from == VS_BLOCKED || to == VS_BLOCKED) && vp->v_usecount != 1)
328 1.68 hannken vnpanic(vp, "%s to %s with usecount %d at %s:%d",
329 1.68 hannken vstate_name(from), vstate_name(to), vp->v_usecount,
330 1.68 hannken func, line);
331 1.51 hannken
332 1.70 hannken vip->vi_state = to;
333 1.57 hannken if (from == VS_LOADING)
334 1.69 hannken cv_broadcast(&vcache_cv);
335 1.94 hannken if (to == VS_LOADED || to == VS_RECLAIMED)
336 1.51 hannken cv_broadcast(&vp->v_cv);
337 1.51 hannken }
338 1.51 hannken
339 1.51 hannken #else /* defined(DIAGNOSTIC) */
340 1.51 hannken
341 1.51 hannken #define VSTATE_GET(vp) \
342 1.57 hannken (VNODE_TO_VIMPL((vp))->vi_state)
343 1.51 hannken #define VSTATE_CHANGE(vp, from, to) \
344 1.51 hannken vstate_change((vp), (from), (to))
345 1.51 hannken #define VSTATE_WAIT_STABLE(vp) \
346 1.51 hannken vstate_wait_stable((vp))
347 1.94 hannken void
348 1.100 joerg _vstate_assert(vnode_t *vp, enum vnode_state state, const char *func, int line,
349 1.100 joerg bool has_lock)
350 1.94 hannken {
351 1.94 hannken
352 1.94 hannken }
353 1.51 hannken
354 1.52 hannken static void
355 1.51 hannken vstate_wait_stable(vnode_t *vp)
356 1.51 hannken {
357 1.70 hannken vnode_impl_t *vip = VNODE_TO_VIMPL(vp);
358 1.51 hannken
359 1.94 hannken while (vip->vi_state != VS_LOADED && vip->vi_state != VS_RECLAIMED)
360 1.51 hannken cv_wait(&vp->v_cv, vp->v_interlock);
361 1.51 hannken }
362 1.51 hannken
363 1.52 hannken static void
364 1.57 hannken vstate_change(vnode_t *vp, enum vnode_state from, enum vnode_state to)
365 1.51 hannken {
366 1.70 hannken vnode_impl_t *vip = VNODE_TO_VIMPL(vp);
367 1.51 hannken
368 1.70 hannken vip->vi_state = to;
369 1.57 hannken if (from == VS_LOADING)
370 1.69 hannken cv_broadcast(&vcache_cv);
371 1.94 hannken if (to == VS_LOADED || to == VS_RECLAIMED)
372 1.51 hannken cv_broadcast(&vp->v_cv);
373 1.51 hannken }
374 1.51 hannken
375 1.51 hannken #endif /* defined(DIAGNOSTIC) */
376 1.51 hannken
377 1.1 rmind void
378 1.1 rmind vfs_vnode_sysinit(void)
379 1.1 rmind {
380 1.104 ad int error __diagused, i;
381 1.1 rmind
382 1.44 hannken dead_rootmount = vfs_mountalloc(&dead_vfsops, NULL);
383 1.44 hannken KASSERT(dead_rootmount != NULL);
384 1.103 hannken dead_rootmount->mnt_iflag |= IMNT_MPSAFE;
385 1.31 hannken
386 1.63 hannken mutex_init(&vdrain_lock, MUTEX_DEFAULT, IPL_NONE);
387 1.104 ad for (i = 0; i < LRU_COUNT; i++) {
388 1.104 ad TAILQ_INIT(&lru_list[i]);
389 1.104 ad }
390 1.36 hannken vcache_init();
391 1.36 hannken
392 1.12 hannken cv_init(&vdrain_cv, "vdrain");
393 1.63 hannken cv_init(&vdrain_gen_cv, "vdrainwt");
394 1.12 hannken error = kthread_create(PRI_VM, KTHREAD_MPSAFE, NULL, vdrain_thread,
395 1.63 hannken NULL, &vdrain_lwp, "vdrain");
396 1.47 riastrad KASSERTMSG((error == 0), "kthread_create(vdrain) failed: %d", error);
397 1.1 rmind }
398 1.1 rmind
399 1.1 rmind /*
400 1.48 hannken * Allocate a new marker vnode.
401 1.48 hannken */
402 1.48 hannken vnode_t *
403 1.48 hannken vnalloc_marker(struct mount *mp)
404 1.48 hannken {
405 1.70 hannken vnode_impl_t *vip;
406 1.50 hannken vnode_t *vp;
407 1.50 hannken
408 1.70 hannken vip = pool_cache_get(vcache_pool, PR_WAITOK);
409 1.70 hannken memset(vip, 0, sizeof(*vip));
410 1.70 hannken vp = VIMPL_TO_VNODE(vip);
411 1.50 hannken uvm_obj_init(&vp->v_uobj, &uvm_vnodeops, true, 0);
412 1.50 hannken vp->v_mount = mp;
413 1.50 hannken vp->v_type = VBAD;
414 1.70 hannken vip->vi_state = VS_MARKER;
415 1.48 hannken
416 1.50 hannken return vp;
417 1.48 hannken }
418 1.48 hannken
419 1.48 hannken /*
420 1.48 hannken * Free a marker vnode.
421 1.48 hannken */
422 1.48 hannken void
423 1.48 hannken vnfree_marker(vnode_t *vp)
424 1.48 hannken {
425 1.70 hannken vnode_impl_t *vip;
426 1.48 hannken
427 1.70 hannken vip = VNODE_TO_VIMPL(vp);
428 1.70 hannken KASSERT(vip->vi_state == VS_MARKER);
429 1.50 hannken uvm_obj_destroy(&vp->v_uobj, true);
430 1.70 hannken pool_cache_put(vcache_pool, vip);
431 1.48 hannken }
432 1.48 hannken
433 1.48 hannken /*
434 1.48 hannken * Test a vnode for being a marker vnode.
435 1.48 hannken */
436 1.48 hannken bool
437 1.48 hannken vnis_marker(vnode_t *vp)
438 1.48 hannken {
439 1.48 hannken
440 1.57 hannken return (VNODE_TO_VIMPL(vp)->vi_state == VS_MARKER);
441 1.48 hannken }
442 1.48 hannken
443 1.48 hannken /*
444 1.63 hannken * Return the lru list this node should be on.
445 1.63 hannken */
446 1.63 hannken static vnodelst_t *
447 1.63 hannken lru_which(vnode_t *vp)
448 1.63 hannken {
449 1.63 hannken
450 1.63 hannken KASSERT(mutex_owned(vp->v_interlock));
451 1.63 hannken
452 1.63 hannken if (vp->v_holdcnt > 0)
453 1.104 ad return &lru_list[LRU_HOLD];
454 1.63 hannken else
455 1.104 ad return &lru_list[LRU_FREE];
456 1.63 hannken }
457 1.63 hannken
458 1.63 hannken /*
459 1.63 hannken * Put vnode to end of given list.
460 1.63 hannken * Both the current and the new list may be NULL, used on vnode alloc/free.
461 1.63 hannken * Adjust numvnodes and signal vdrain thread if there is work.
462 1.63 hannken */
463 1.63 hannken static void
464 1.63 hannken lru_requeue(vnode_t *vp, vnodelst_t *listhd)
465 1.63 hannken {
466 1.70 hannken vnode_impl_t *vip;
467 1.104 ad int d;
468 1.104 ad
469 1.104 ad /*
470 1.104 ad * If the vnode is on the correct list, and was put there recently,
471 1.104 ad * then leave it be, thus avoiding huge cache and lock contention.
472 1.104 ad */
473 1.104 ad vip = VNODE_TO_VIMPL(vp);
474 1.104 ad if (listhd == vip->vi_lrulisthd &&
475 1.104 ad (hardclock_ticks - vip->vi_lrulisttm) < hz) {
476 1.104 ad return;
477 1.104 ad }
478 1.63 hannken
479 1.63 hannken mutex_enter(&vdrain_lock);
480 1.104 ad d = 0;
481 1.70 hannken if (vip->vi_lrulisthd != NULL)
482 1.70 hannken TAILQ_REMOVE(vip->vi_lrulisthd, vip, vi_lrulist);
483 1.63 hannken else
484 1.104 ad d++;
485 1.70 hannken vip->vi_lrulisthd = listhd;
486 1.104 ad vip->vi_lrulisttm = hardclock_ticks;
487 1.70 hannken if (vip->vi_lrulisthd != NULL)
488 1.70 hannken TAILQ_INSERT_TAIL(vip->vi_lrulisthd, vip, vi_lrulist);
489 1.63 hannken else
490 1.104 ad d--;
491 1.104 ad if (d != 0) {
492 1.104 ad /*
493 1.104 ad * Looks strange? This is not a bug. Don't store
494 1.104 ad * numvnodes unless there is a change - avoid false
495 1.104 ad * sharing on MP.
496 1.104 ad */
497 1.104 ad numvnodes += d;
498 1.104 ad }
499 1.104 ad if (numvnodes > desiredvnodes || listhd == &lru_list[LRU_VRELE])
500 1.63 hannken cv_broadcast(&vdrain_cv);
501 1.63 hannken mutex_exit(&vdrain_lock);
502 1.63 hannken }
503 1.63 hannken
504 1.63 hannken /*
505 1.75 hannken * Release deferred vrele vnodes for this mount.
506 1.75 hannken * Called with file system suspended.
507 1.75 hannken */
508 1.75 hannken void
509 1.75 hannken vrele_flush(struct mount *mp)
510 1.75 hannken {
511 1.75 hannken vnode_impl_t *vip, *marker;
512 1.104 ad vnode_t *vp;
513 1.75 hannken
514 1.75 hannken KASSERT(fstrans_is_owner(mp));
515 1.75 hannken
516 1.75 hannken marker = VNODE_TO_VIMPL(vnalloc_marker(NULL));
517 1.75 hannken
518 1.75 hannken mutex_enter(&vdrain_lock);
519 1.104 ad TAILQ_INSERT_HEAD(&lru_list[LRU_VRELE], marker, vi_lrulist);
520 1.75 hannken
521 1.75 hannken while ((vip = TAILQ_NEXT(marker, vi_lrulist))) {
522 1.104 ad TAILQ_REMOVE(&lru_list[LRU_VRELE], marker, vi_lrulist);
523 1.104 ad TAILQ_INSERT_AFTER(&lru_list[LRU_VRELE], vip, marker,
524 1.104 ad vi_lrulist);
525 1.104 ad vp = VIMPL_TO_VNODE(vip);
526 1.104 ad if (vnis_marker(vp))
527 1.75 hannken continue;
528 1.75 hannken
529 1.104 ad KASSERT(vip->vi_lrulisthd == &lru_list[LRU_VRELE]);
530 1.75 hannken TAILQ_REMOVE(vip->vi_lrulisthd, vip, vi_lrulist);
531 1.104 ad vip->vi_lrulisthd = &lru_list[LRU_HOLD];
532 1.104 ad vip->vi_lrulisttm = hardclock_ticks;
533 1.75 hannken TAILQ_INSERT_TAIL(vip->vi_lrulisthd, vip, vi_lrulist);
534 1.75 hannken mutex_exit(&vdrain_lock);
535 1.75 hannken
536 1.107 ad vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
537 1.104 ad mutex_enter(vp->v_interlock);
538 1.107 ad vrelel(vp, 0, LK_EXCLUSIVE);
539 1.75 hannken
540 1.75 hannken mutex_enter(&vdrain_lock);
541 1.75 hannken }
542 1.75 hannken
543 1.104 ad TAILQ_REMOVE(&lru_list[LRU_VRELE], marker, vi_lrulist);
544 1.75 hannken mutex_exit(&vdrain_lock);
545 1.75 hannken
546 1.75 hannken vnfree_marker(VIMPL_TO_VNODE(marker));
547 1.75 hannken }
548 1.75 hannken
549 1.75 hannken /*
550 1.63 hannken * Reclaim a cached vnode. Used from vdrain_thread only.
551 1.1 rmind */
552 1.63 hannken static __inline void
553 1.63 hannken vdrain_remove(vnode_t *vp)
554 1.1 rmind {
555 1.24 hannken struct mount *mp;
556 1.1 rmind
557 1.63 hannken KASSERT(mutex_owned(&vdrain_lock));
558 1.24 hannken
559 1.63 hannken /* Probe usecount (unlocked). */
560 1.63 hannken if (vp->v_usecount > 0)
561 1.63 hannken return;
562 1.63 hannken /* Try v_interlock -- we lock the wrong direction! */
563 1.63 hannken if (!mutex_tryenter(vp->v_interlock))
564 1.63 hannken return;
565 1.63 hannken /* Probe usecount and state. */
566 1.94 hannken if (vp->v_usecount > 0 || VSTATE_GET(vp) != VS_LOADED) {
567 1.63 hannken mutex_exit(vp->v_interlock);
568 1.63 hannken return;
569 1.1 rmind }
570 1.63 hannken mp = vp->v_mount;
571 1.96 hannken if (fstrans_start_nowait(mp) != 0) {
572 1.63 hannken mutex_exit(vp->v_interlock);
573 1.63 hannken return;
574 1.1 rmind }
575 1.63 hannken vdrain_retry = true;
576 1.63 hannken mutex_exit(&vdrain_lock);
577 1.1 rmind
578 1.66 hannken if (vcache_vget(vp) == 0) {
579 1.78 hannken if (!vrecycle(vp)) {
580 1.107 ad vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
581 1.78 hannken mutex_enter(vp->v_interlock);
582 1.107 ad vrelel(vp, 0, LK_EXCLUSIVE);
583 1.78 hannken }
584 1.60 hannken }
585 1.24 hannken fstrans_done(mp);
586 1.12 hannken
587 1.63 hannken mutex_enter(&vdrain_lock);
588 1.1 rmind }
589 1.1 rmind
590 1.1 rmind /*
591 1.63 hannken * Release a cached vnode. Used from vdrain_thread only.
592 1.12 hannken */
593 1.63 hannken static __inline void
594 1.63 hannken vdrain_vrele(vnode_t *vp)
595 1.12 hannken {
596 1.70 hannken vnode_impl_t *vip = VNODE_TO_VIMPL(vp);
597 1.63 hannken struct mount *mp;
598 1.12 hannken
599 1.63 hannken KASSERT(mutex_owned(&vdrain_lock));
600 1.12 hannken
601 1.63 hannken mp = vp->v_mount;
602 1.96 hannken if (fstrans_start_nowait(mp) != 0)
603 1.63 hannken return;
604 1.63 hannken
605 1.64 hannken /*
606 1.64 hannken * First remove the vnode from the vrele list.
607 1.64 hannken * Put it on the last lru list, the last vrele()
608 1.64 hannken * will put it back onto the right list before
609 1.64 hannken * its v_usecount reaches zero.
610 1.64 hannken */
611 1.104 ad KASSERT(vip->vi_lrulisthd == &lru_list[LRU_VRELE]);
612 1.70 hannken TAILQ_REMOVE(vip->vi_lrulisthd, vip, vi_lrulist);
613 1.104 ad vip->vi_lrulisthd = &lru_list[LRU_HOLD];
614 1.104 ad vip->vi_lrulisttm = hardclock_ticks;
615 1.70 hannken TAILQ_INSERT_TAIL(vip->vi_lrulisthd, vip, vi_lrulist);
616 1.63 hannken
617 1.63 hannken vdrain_retry = true;
618 1.63 hannken mutex_exit(&vdrain_lock);
619 1.63 hannken
620 1.107 ad vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
621 1.64 hannken mutex_enter(vp->v_interlock);
622 1.107 ad vrelel(vp, 0, LK_EXCLUSIVE);
623 1.63 hannken fstrans_done(mp);
624 1.63 hannken
625 1.63 hannken mutex_enter(&vdrain_lock);
626 1.12 hannken }
627 1.12 hannken
628 1.12 hannken /*
629 1.63 hannken * Helper thread to keep the number of vnodes below desiredvnodes
630 1.63 hannken * and release vnodes from asynchronous vrele.
631 1.1 rmind */
632 1.63 hannken static void
633 1.63 hannken vdrain_thread(void *cookie)
634 1.1 rmind {
635 1.63 hannken int i;
636 1.63 hannken u_int target;
637 1.70 hannken vnode_impl_t *vip, *marker;
638 1.63 hannken
639 1.63 hannken marker = VNODE_TO_VIMPL(vnalloc_marker(NULL));
640 1.63 hannken
641 1.63 hannken mutex_enter(&vdrain_lock);
642 1.63 hannken
643 1.63 hannken for (;;) {
644 1.63 hannken vdrain_retry = false;
645 1.63 hannken target = desiredvnodes - desiredvnodes/10;
646 1.1 rmind
647 1.104 ad for (i = 0; i < LRU_COUNT; i++) {
648 1.104 ad TAILQ_INSERT_HEAD(&lru_list[i], marker, vi_lrulist);
649 1.70 hannken while ((vip = TAILQ_NEXT(marker, vi_lrulist))) {
650 1.104 ad TAILQ_REMOVE(&lru_list[i], marker, vi_lrulist);
651 1.104 ad TAILQ_INSERT_AFTER(&lru_list[i], vip, marker,
652 1.63 hannken vi_lrulist);
653 1.75 hannken if (vnis_marker(VIMPL_TO_VNODE(vip)))
654 1.75 hannken continue;
655 1.104 ad if (i == LRU_VRELE)
656 1.70 hannken vdrain_vrele(VIMPL_TO_VNODE(vip));
657 1.63 hannken else if (numvnodes < target)
658 1.63 hannken break;
659 1.63 hannken else
660 1.70 hannken vdrain_remove(VIMPL_TO_VNODE(vip));
661 1.63 hannken }
662 1.104 ad TAILQ_REMOVE(&lru_list[i], marker, vi_lrulist);
663 1.63 hannken }
664 1.1 rmind
665 1.63 hannken if (vdrain_retry) {
666 1.63 hannken mutex_exit(&vdrain_lock);
667 1.63 hannken yield();
668 1.63 hannken mutex_enter(&vdrain_lock);
669 1.63 hannken } else {
670 1.63 hannken vdrain_gen++;
671 1.63 hannken cv_broadcast(&vdrain_gen_cv);
672 1.63 hannken cv_wait(&vdrain_cv, &vdrain_lock);
673 1.63 hannken }
674 1.1 rmind }
675 1.1 rmind }
676 1.1 rmind
677 1.1 rmind /*
678 1.4 rmind * vput: unlock and release the reference.
679 1.1 rmind */
680 1.1 rmind void
681 1.1 rmind vput(vnode_t *vp)
682 1.1 rmind {
683 1.107 ad int lktype;
684 1.1 rmind
685 1.107 ad if ((vp->v_vflag & VV_LOCKSWORK) == 0) {
686 1.107 ad lktype = LK_EXCLUSIVE;
687 1.107 ad } else {
688 1.107 ad lktype = VOP_ISLOCKED(vp);
689 1.107 ad KASSERT(lktype != LK_NONE);
690 1.107 ad }
691 1.107 ad mutex_enter(vp->v_interlock);
692 1.107 ad vrelel(vp, 0, lktype);
693 1.1 rmind }
694 1.1 rmind
695 1.1 rmind /*
696 1.1 rmind * Vnode release. If reference count drops to zero, call inactive
697 1.1 rmind * routine and either return to freelist or free to the pool.
698 1.1 rmind */
699 1.23 hannken static void
700 1.107 ad vrelel(vnode_t *vp, int flags, int lktype)
701 1.1 rmind {
702 1.104 ad const bool async = ((flags & VRELEL_ASYNC) != 0);
703 1.1 rmind bool recycle, defer;
704 1.1 rmind int error;
705 1.1 rmind
706 1.9 rmind KASSERT(mutex_owned(vp->v_interlock));
707 1.1 rmind
708 1.1 rmind if (__predict_false(vp->v_op == dead_vnodeop_p &&
709 1.57 hannken VSTATE_GET(vp) != VS_RECLAIMED)) {
710 1.11 christos vnpanic(vp, "dead but not clean");
711 1.1 rmind }
712 1.1 rmind
713 1.1 rmind /*
714 1.1 rmind * If not the last reference, just drop the reference count
715 1.1 rmind * and unlock.
716 1.1 rmind */
717 1.104 ad if (vp->v_usecount > 1) {
718 1.107 ad if (lktype != LK_NONE) {
719 1.107 ad VOP_UNLOCK(vp);
720 1.107 ad }
721 1.104 ad vp->v_usecount--;
722 1.9 rmind mutex_exit(vp->v_interlock);
723 1.1 rmind return;
724 1.1 rmind }
725 1.1 rmind if (vp->v_usecount <= 0 || vp->v_writecount != 0) {
726 1.11 christos vnpanic(vp, "%s: bad ref count", __func__);
727 1.1 rmind }
728 1.1 rmind
729 1.15 hannken #ifdef DIAGNOSTIC
730 1.15 hannken if ((vp->v_type == VBLK || vp->v_type == VCHR) &&
731 1.15 hannken vp->v_specnode != NULL && vp->v_specnode->sn_opencnt != 0) {
732 1.15 hannken vprint("vrelel: missing VOP_CLOSE()", vp);
733 1.15 hannken }
734 1.15 hannken #endif
735 1.15 hannken
736 1.1 rmind /*
737 1.79 hannken * First try to get the vnode locked for VOP_INACTIVE().
738 1.79 hannken * Defer vnode release to vdrain_thread if caller requests
739 1.79 hannken * it explicitly, is the pagedaemon or the lock failed.
740 1.1 rmind */
741 1.107 ad defer = false;
742 1.79 hannken if ((curlwp == uvm.pagedaemon_lwp) || async) {
743 1.79 hannken defer = true;
744 1.107 ad } else if (lktype == LK_SHARED) {
745 1.107 ad /* Excellent chance of getting, if the last ref. */
746 1.107 ad error = vn_lock(vp, LK_UPGRADE | LK_RETRY |
747 1.107 ad LK_NOWAIT);
748 1.107 ad if (error != 0) {
749 1.107 ad defer = true;
750 1.107 ad } else {
751 1.107 ad lktype = LK_EXCLUSIVE;
752 1.107 ad }
753 1.107 ad } else if (lktype == LK_NONE) {
754 1.107 ad /* Excellent chance of getting, if the last ref. */
755 1.107 ad error = vn_lock(vp, LK_EXCLUSIVE | LK_RETRY |
756 1.107 ad LK_NOWAIT);
757 1.107 ad if (error != 0) {
758 1.107 ad defer = true;
759 1.107 ad } else {
760 1.107 ad lktype = LK_EXCLUSIVE;
761 1.107 ad }
762 1.79 hannken }
763 1.79 hannken KASSERT(mutex_owned(vp->v_interlock));
764 1.79 hannken if (defer) {
765 1.1 rmind /*
766 1.79 hannken * Defer reclaim to the kthread; it's not safe to
767 1.79 hannken * clean it here. We donate it our last reference.
768 1.1 rmind */
769 1.107 ad if (lktype != LK_NONE) {
770 1.107 ad VOP_UNLOCK(vp);
771 1.107 ad }
772 1.104 ad lru_requeue(vp, &lru_list[LRU_VRELE]);
773 1.79 hannken mutex_exit(vp->v_interlock);
774 1.79 hannken return;
775 1.79 hannken }
776 1.107 ad KASSERT(lktype == LK_EXCLUSIVE);
777 1.30 hannken
778 1.79 hannken /*
779 1.79 hannken * If not clean, deactivate the vnode, but preserve
780 1.79 hannken * our reference across the call to VOP_INACTIVE().
781 1.79 hannken */
782 1.79 hannken if (VSTATE_GET(vp) == VS_RECLAIMED) {
783 1.79 hannken VOP_UNLOCK(vp);
784 1.79 hannken } else {
785 1.94 hannken VSTATE_CHANGE(vp, VS_LOADED, VS_BLOCKED);
786 1.29 christos mutex_exit(vp->v_interlock);
787 1.29 christos
788 1.1 rmind /*
789 1.52 hannken * The vnode must not gain another reference while being
790 1.1 rmind * deactivated. If VOP_INACTIVE() indicates that
791 1.1 rmind * the described file has been deleted, then recycle
792 1.52 hannken * the vnode.
793 1.1 rmind *
794 1.82 riastrad * Note that VOP_INACTIVE() will not drop the vnode lock.
795 1.1 rmind */
796 1.79 hannken recycle = false;
797 1.1 rmind VOP_INACTIVE(vp, &recycle);
798 1.83 riastrad if (!recycle)
799 1.83 riastrad VOP_UNLOCK(vp);
800 1.9 rmind mutex_enter(vp->v_interlock);
801 1.94 hannken VSTATE_CHANGE(vp, VS_BLOCKED, VS_LOADED);
802 1.1 rmind if (!recycle) {
803 1.104 ad if (vp->v_usecount > 1) {
804 1.104 ad vp->v_usecount--;
805 1.9 rmind mutex_exit(vp->v_interlock);
806 1.1 rmind return;
807 1.1 rmind }
808 1.1 rmind }
809 1.1 rmind
810 1.1 rmind /* Take care of space accounting. */
811 1.104 ad if ((vp->v_iflag & VI_EXECMAP) != 0 &&
812 1.104 ad vp->v_uobj.uo_npages != 0) {
813 1.105 ad cpu_count(CPU_COUNT_EXECPAGES, -vp->v_uobj.uo_npages);
814 1.105 ad cpu_count(CPU_COUNT_FILEPAGES, vp->v_uobj.uo_npages);
815 1.1 rmind }
816 1.1 rmind vp->v_iflag &= ~(VI_TEXT|VI_EXECMAP|VI_WRMAP);
817 1.1 rmind vp->v_vflag &= ~VV_MAPPED;
818 1.1 rmind
819 1.1 rmind /*
820 1.1 rmind * Recycle the vnode if the file is now unused (unlinked),
821 1.1 rmind * otherwise just free it.
822 1.1 rmind */
823 1.1 rmind if (recycle) {
824 1.94 hannken VSTATE_ASSERT(vp, VS_LOADED);
825 1.83 riastrad /* vcache_reclaim drops the lock. */
826 1.54 hannken vcache_reclaim(vp);
827 1.1 rmind }
828 1.1 rmind KASSERT(vp->v_usecount > 0);
829 1.1 rmind }
830 1.1 rmind
831 1.104 ad vp->v_usecount--;
832 1.104 ad if (vp->v_usecount != 0) {
833 1.1 rmind /* Gained another reference while being reclaimed. */
834 1.9 rmind mutex_exit(vp->v_interlock);
835 1.1 rmind return;
836 1.1 rmind }
837 1.1 rmind
838 1.67 hannken if (VSTATE_GET(vp) == VS_RECLAIMED && vp->v_holdcnt == 0) {
839 1.1 rmind /*
840 1.1 rmind * It's clean so destroy it. It isn't referenced
841 1.1 rmind * anywhere since it has been reclaimed.
842 1.1 rmind */
843 1.57 hannken vcache_free(VNODE_TO_VIMPL(vp));
844 1.1 rmind } else {
845 1.1 rmind /*
846 1.1 rmind * Otherwise, put it back onto the freelist. It
847 1.1 rmind * can't be destroyed while still associated with
848 1.1 rmind * a file system.
849 1.1 rmind */
850 1.63 hannken lru_requeue(vp, lru_which(vp));
851 1.9 rmind mutex_exit(vp->v_interlock);
852 1.1 rmind }
853 1.1 rmind }
854 1.1 rmind
855 1.1 rmind void
856 1.1 rmind vrele(vnode_t *vp)
857 1.1 rmind {
858 1.1 rmind
859 1.9 rmind mutex_enter(vp->v_interlock);
860 1.107 ad vrelel(vp, 0, LK_NONE);
861 1.1 rmind }
862 1.1 rmind
863 1.1 rmind /*
864 1.1 rmind * Asynchronous vnode release, vnode is released in different context.
865 1.1 rmind */
866 1.1 rmind void
867 1.1 rmind vrele_async(vnode_t *vp)
868 1.1 rmind {
869 1.1 rmind
870 1.9 rmind mutex_enter(vp->v_interlock);
871 1.107 ad vrelel(vp, VRELEL_ASYNC, LK_NONE);
872 1.1 rmind }
873 1.1 rmind
874 1.1 rmind /*
875 1.1 rmind * Vnode reference, where a reference is already held by some other
876 1.1 rmind * object (for example, a file structure).
877 1.1 rmind */
878 1.1 rmind void
879 1.1 rmind vref(vnode_t *vp)
880 1.1 rmind {
881 1.1 rmind
882 1.1 rmind KASSERT(vp->v_usecount != 0);
883 1.1 rmind
884 1.104 ad mutex_enter(vp->v_interlock);
885 1.104 ad vp->v_usecount++;
886 1.104 ad mutex_exit(vp->v_interlock);
887 1.1 rmind }
888 1.1 rmind
889 1.1 rmind /*
890 1.1 rmind * Page or buffer structure gets a reference.
891 1.1 rmind * Called with v_interlock held.
892 1.1 rmind */
893 1.1 rmind void
894 1.1 rmind vholdl(vnode_t *vp)
895 1.1 rmind {
896 1.1 rmind
897 1.9 rmind KASSERT(mutex_owned(vp->v_interlock));
898 1.1 rmind
899 1.63 hannken if (vp->v_holdcnt++ == 0 && vp->v_usecount == 0)
900 1.63 hannken lru_requeue(vp, lru_which(vp));
901 1.1 rmind }
902 1.1 rmind
903 1.1 rmind /*
904 1.1 rmind * Page or buffer structure frees a reference.
905 1.1 rmind * Called with v_interlock held.
906 1.1 rmind */
907 1.1 rmind void
908 1.1 rmind holdrelel(vnode_t *vp)
909 1.1 rmind {
910 1.1 rmind
911 1.9 rmind KASSERT(mutex_owned(vp->v_interlock));
912 1.1 rmind
913 1.1 rmind if (vp->v_holdcnt <= 0) {
914 1.11 christos vnpanic(vp, "%s: holdcnt vp %p", __func__, vp);
915 1.1 rmind }
916 1.1 rmind
917 1.1 rmind vp->v_holdcnt--;
918 1.63 hannken if (vp->v_holdcnt == 0 && vp->v_usecount == 0)
919 1.63 hannken lru_requeue(vp, lru_which(vp));
920 1.1 rmind }
921 1.1 rmind
922 1.1 rmind /*
923 1.33 hannken * Recycle an unused vnode if caller holds the last reference.
924 1.1 rmind */
925 1.33 hannken bool
926 1.33 hannken vrecycle(vnode_t *vp)
927 1.1 rmind {
928 1.60 hannken int error __diagused;
929 1.46 hannken
930 1.33 hannken mutex_enter(vp->v_interlock);
931 1.33 hannken
932 1.60 hannken /* Make sure we hold the last reference. */
933 1.60 hannken VSTATE_WAIT_STABLE(vp);
934 1.33 hannken if (vp->v_usecount != 1) {
935 1.33 hannken mutex_exit(vp->v_interlock);
936 1.33 hannken return false;
937 1.1 rmind }
938 1.60 hannken
939 1.60 hannken /* If the vnode is already clean we're done. */
940 1.94 hannken if (VSTATE_GET(vp) != VS_LOADED) {
941 1.60 hannken VSTATE_ASSERT(vp, VS_RECLAIMED);
942 1.107 ad vrelel(vp, 0, LK_NONE);
943 1.60 hannken return true;
944 1.60 hannken }
945 1.60 hannken
946 1.60 hannken /* Prevent further references until the vnode is locked. */
947 1.94 hannken VSTATE_CHANGE(vp, VS_LOADED, VS_BLOCKED);
948 1.60 hannken mutex_exit(vp->v_interlock);
949 1.60 hannken
950 1.73 hannken /*
951 1.73 hannken * On a leaf file system this lock will always succeed as we hold
952 1.73 hannken * the last reference and prevent further references.
953 1.73 hannken * On layered file systems waiting for the lock would open a can of
954 1.73 hannken * deadlocks as the lower vnodes may have other active references.
955 1.73 hannken */
956 1.76 hannken error = vn_lock(vp, LK_EXCLUSIVE | LK_RETRY | LK_NOWAIT);
957 1.60 hannken
958 1.60 hannken mutex_enter(vp->v_interlock);
959 1.94 hannken VSTATE_CHANGE(vp, VS_BLOCKED, VS_LOADED);
960 1.60 hannken
961 1.73 hannken if (error) {
962 1.73 hannken mutex_exit(vp->v_interlock);
963 1.73 hannken return false;
964 1.73 hannken }
965 1.73 hannken
966 1.68 hannken KASSERT(vp->v_usecount == 1);
967 1.54 hannken vcache_reclaim(vp);
968 1.107 ad vrelel(vp, 0, LK_NONE);
969 1.60 hannken
970 1.33 hannken return true;
971 1.1 rmind }
972 1.1 rmind
973 1.1 rmind /*
974 1.92 hannken * Helper for vrevoke() to propagate suspension from lastmp
975 1.92 hannken * to thismp. Both args may be NULL.
976 1.92 hannken * Returns the currently suspended file system or NULL.
977 1.92 hannken */
978 1.92 hannken static struct mount *
979 1.92 hannken vrevoke_suspend_next(struct mount *lastmp, struct mount *thismp)
980 1.92 hannken {
981 1.92 hannken int error;
982 1.92 hannken
983 1.92 hannken if (lastmp == thismp)
984 1.92 hannken return thismp;
985 1.92 hannken
986 1.92 hannken if (lastmp != NULL)
987 1.92 hannken vfs_resume(lastmp);
988 1.92 hannken
989 1.92 hannken if (thismp == NULL)
990 1.92 hannken return NULL;
991 1.92 hannken
992 1.92 hannken do {
993 1.92 hannken error = vfs_suspend(thismp, 0);
994 1.92 hannken } while (error == EINTR || error == ERESTART);
995 1.92 hannken
996 1.92 hannken if (error == 0)
997 1.92 hannken return thismp;
998 1.92 hannken
999 1.92 hannken KASSERT(error == EOPNOTSUPP);
1000 1.92 hannken return NULL;
1001 1.92 hannken }
1002 1.92 hannken
1003 1.92 hannken /*
1004 1.1 rmind * Eliminate all activity associated with the requested vnode
1005 1.1 rmind * and with all vnodes aliased to the requested vnode.
1006 1.1 rmind */
1007 1.1 rmind void
1008 1.1 rmind vrevoke(vnode_t *vp)
1009 1.1 rmind {
1010 1.88 hannken struct mount *mp;
1011 1.19 hannken vnode_t *vq;
1012 1.1 rmind enum vtype type;
1013 1.1 rmind dev_t dev;
1014 1.1 rmind
1015 1.1 rmind KASSERT(vp->v_usecount > 0);
1016 1.1 rmind
1017 1.92 hannken mp = vrevoke_suspend_next(NULL, vp->v_mount);
1018 1.88 hannken
1019 1.9 rmind mutex_enter(vp->v_interlock);
1020 1.52 hannken VSTATE_WAIT_STABLE(vp);
1021 1.57 hannken if (VSTATE_GET(vp) == VS_RECLAIMED) {
1022 1.9 rmind mutex_exit(vp->v_interlock);
1023 1.1 rmind } else if (vp->v_type != VBLK && vp->v_type != VCHR) {
1024 1.104 ad vp->v_usecount++;
1025 1.29 christos mutex_exit(vp->v_interlock);
1026 1.29 christos vgone(vp);
1027 1.1 rmind } else {
1028 1.1 rmind dev = vp->v_rdev;
1029 1.1 rmind type = vp->v_type;
1030 1.9 rmind mutex_exit(vp->v_interlock);
1031 1.1 rmind
1032 1.88 hannken while (spec_node_lookup_by_dev(type, dev, &vq) == 0) {
1033 1.92 hannken mp = vrevoke_suspend_next(mp, vq->v_mount);
1034 1.88 hannken vgone(vq);
1035 1.88 hannken }
1036 1.1 rmind }
1037 1.92 hannken vrevoke_suspend_next(mp, NULL);
1038 1.1 rmind }
1039 1.1 rmind
1040 1.1 rmind /*
1041 1.1 rmind * Eliminate all activity associated with a vnode in preparation for
1042 1.1 rmind * reuse. Drops a reference from the vnode.
1043 1.1 rmind */
1044 1.1 rmind void
1045 1.1 rmind vgone(vnode_t *vp)
1046 1.1 rmind {
1047 1.107 ad int lktype;
1048 1.1 rmind
1049 1.103 hannken KASSERT(vp->v_mount == dead_rootmount || fstrans_is_owner(vp->v_mount));
1050 1.93 hannken
1051 1.76 hannken vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
1052 1.107 ad lktype = LK_EXCLUSIVE;
1053 1.9 rmind mutex_enter(vp->v_interlock);
1054 1.76 hannken VSTATE_WAIT_STABLE(vp);
1055 1.107 ad if (VSTATE_GET(vp) == VS_LOADED) {
1056 1.76 hannken vcache_reclaim(vp);
1057 1.107 ad lktype = LK_NONE;
1058 1.107 ad }
1059 1.76 hannken VSTATE_ASSERT(vp, VS_RECLAIMED);
1060 1.107 ad vrelel(vp, 0, lktype);
1061 1.1 rmind }
1062 1.1 rmind
1063 1.36 hannken static inline uint32_t
1064 1.36 hannken vcache_hash(const struct vcache_key *key)
1065 1.36 hannken {
1066 1.36 hannken uint32_t hash = HASH32_BUF_INIT;
1067 1.36 hannken
1068 1.97 hannken KASSERT(key->vk_key_len > 0);
1069 1.97 hannken
1070 1.36 hannken hash = hash32_buf(&key->vk_mount, sizeof(struct mount *), hash);
1071 1.36 hannken hash = hash32_buf(key->vk_key, key->vk_key_len, hash);
1072 1.36 hannken return hash;
1073 1.36 hannken }
1074 1.36 hannken
1075 1.36 hannken static void
1076 1.36 hannken vcache_init(void)
1077 1.36 hannken {
1078 1.36 hannken
1079 1.69 hannken vcache_pool = pool_cache_init(sizeof(vnode_impl_t), 0, 0, 0,
1080 1.36 hannken "vcachepl", NULL, IPL_NONE, NULL, NULL, NULL);
1081 1.69 hannken KASSERT(vcache_pool != NULL);
1082 1.69 hannken mutex_init(&vcache_lock, MUTEX_DEFAULT, IPL_NONE);
1083 1.69 hannken cv_init(&vcache_cv, "vcache");
1084 1.69 hannken vcache_hashsize = desiredvnodes;
1085 1.69 hannken vcache_hashtab = hashinit(desiredvnodes, HASH_SLIST, true,
1086 1.69 hannken &vcache_hashmask);
1087 1.36 hannken }
1088 1.36 hannken
1089 1.36 hannken static void
1090 1.36 hannken vcache_reinit(void)
1091 1.36 hannken {
1092 1.36 hannken int i;
1093 1.36 hannken uint32_t hash;
1094 1.36 hannken u_long oldmask, newmask;
1095 1.36 hannken struct hashhead *oldtab, *newtab;
1096 1.70 hannken vnode_impl_t *vip;
1097 1.36 hannken
1098 1.36 hannken newtab = hashinit(desiredvnodes, HASH_SLIST, true, &newmask);
1099 1.69 hannken mutex_enter(&vcache_lock);
1100 1.69 hannken oldtab = vcache_hashtab;
1101 1.69 hannken oldmask = vcache_hashmask;
1102 1.69 hannken vcache_hashsize = desiredvnodes;
1103 1.69 hannken vcache_hashtab = newtab;
1104 1.69 hannken vcache_hashmask = newmask;
1105 1.36 hannken for (i = 0; i <= oldmask; i++) {
1106 1.70 hannken while ((vip = SLIST_FIRST(&oldtab[i])) != NULL) {
1107 1.70 hannken SLIST_REMOVE(&oldtab[i], vip, vnode_impl, vi_hash);
1108 1.70 hannken hash = vcache_hash(&vip->vi_key);
1109 1.69 hannken SLIST_INSERT_HEAD(&newtab[hash & vcache_hashmask],
1110 1.70 hannken vip, vi_hash);
1111 1.36 hannken }
1112 1.36 hannken }
1113 1.69 hannken mutex_exit(&vcache_lock);
1114 1.36 hannken hashdone(oldtab, HASH_SLIST, oldmask);
1115 1.36 hannken }
1116 1.36 hannken
1117 1.57 hannken static inline vnode_impl_t *
1118 1.36 hannken vcache_hash_lookup(const struct vcache_key *key, uint32_t hash)
1119 1.36 hannken {
1120 1.36 hannken struct hashhead *hashp;
1121 1.70 hannken vnode_impl_t *vip;
1122 1.36 hannken
1123 1.69 hannken KASSERT(mutex_owned(&vcache_lock));
1124 1.36 hannken
1125 1.69 hannken hashp = &vcache_hashtab[hash & vcache_hashmask];
1126 1.70 hannken SLIST_FOREACH(vip, hashp, vi_hash) {
1127 1.70 hannken if (key->vk_mount != vip->vi_key.vk_mount)
1128 1.36 hannken continue;
1129 1.70 hannken if (key->vk_key_len != vip->vi_key.vk_key_len)
1130 1.36 hannken continue;
1131 1.70 hannken if (memcmp(key->vk_key, vip->vi_key.vk_key, key->vk_key_len))
1132 1.36 hannken continue;
1133 1.70 hannken return vip;
1134 1.36 hannken }
1135 1.36 hannken return NULL;
1136 1.36 hannken }
1137 1.36 hannken
1138 1.36 hannken /*
1139 1.50 hannken * Allocate a new, uninitialized vcache node.
1140 1.50 hannken */
1141 1.57 hannken static vnode_impl_t *
1142 1.50 hannken vcache_alloc(void)
1143 1.50 hannken {
1144 1.70 hannken vnode_impl_t *vip;
1145 1.50 hannken vnode_t *vp;
1146 1.50 hannken
1147 1.70 hannken vip = pool_cache_get(vcache_pool, PR_WAITOK);
1148 1.70 hannken memset(vip, 0, sizeof(*vip));
1149 1.50 hannken
1150 1.104 ad vip->vi_lock = rw_obj_alloc();
1151 1.70 hannken /* SLIST_INIT(&vip->vi_hash); */
1152 1.106 ad TAILQ_INIT(&vip->vi_nclist);
1153 1.71 hannken /* LIST_INIT(&vip->vi_dnclist); */
1154 1.50 hannken
1155 1.70 hannken vp = VIMPL_TO_VNODE(vip);
1156 1.50 hannken uvm_obj_init(&vp->v_uobj, &uvm_vnodeops, true, 0);
1157 1.50 hannken cv_init(&vp->v_cv, "vnode");
1158 1.50 hannken
1159 1.50 hannken vp->v_usecount = 1;
1160 1.50 hannken vp->v_type = VNON;
1161 1.50 hannken vp->v_size = vp->v_writesize = VSIZENOTSET;
1162 1.50 hannken
1163 1.70 hannken vip->vi_state = VS_LOADING;
1164 1.51 hannken
1165 1.104 ad lru_requeue(vp, &lru_list[LRU_FREE]);
1166 1.63 hannken
1167 1.70 hannken return vip;
1168 1.50 hannken }
1169 1.50 hannken
1170 1.50 hannken /*
1171 1.79 hannken * Deallocate a vcache node in state VS_LOADING.
1172 1.79 hannken *
1173 1.79 hannken * vcache_lock held on entry and released on return.
1174 1.79 hannken */
1175 1.79 hannken static void
1176 1.79 hannken vcache_dealloc(vnode_impl_t *vip)
1177 1.79 hannken {
1178 1.79 hannken vnode_t *vp;
1179 1.79 hannken
1180 1.79 hannken KASSERT(mutex_owned(&vcache_lock));
1181 1.79 hannken
1182 1.79 hannken vp = VIMPL_TO_VNODE(vip);
1183 1.102 hannken vfs_ref(dead_rootmount);
1184 1.102 hannken vfs_insmntque(vp, dead_rootmount);
1185 1.79 hannken mutex_enter(vp->v_interlock);
1186 1.79 hannken vp->v_op = dead_vnodeop_p;
1187 1.79 hannken VSTATE_CHANGE(vp, VS_LOADING, VS_RECLAIMED);
1188 1.79 hannken mutex_exit(&vcache_lock);
1189 1.107 ad vrelel(vp, 0, LK_NONE);
1190 1.79 hannken }
1191 1.79 hannken
1192 1.79 hannken /*
1193 1.50 hannken * Free an unused, unreferenced vcache node.
1194 1.67 hannken * v_interlock locked on entry.
1195 1.50 hannken */
1196 1.50 hannken static void
1197 1.70 hannken vcache_free(vnode_impl_t *vip)
1198 1.50 hannken {
1199 1.50 hannken vnode_t *vp;
1200 1.50 hannken
1201 1.70 hannken vp = VIMPL_TO_VNODE(vip);
1202 1.67 hannken KASSERT(mutex_owned(vp->v_interlock));
1203 1.50 hannken
1204 1.50 hannken KASSERT(vp->v_usecount == 0);
1205 1.67 hannken KASSERT(vp->v_holdcnt == 0);
1206 1.67 hannken KASSERT(vp->v_writecount == 0);
1207 1.67 hannken lru_requeue(vp, NULL);
1208 1.67 hannken mutex_exit(vp->v_interlock);
1209 1.67 hannken
1210 1.67 hannken vfs_insmntque(vp, NULL);
1211 1.67 hannken if (vp->v_type == VBLK || vp->v_type == VCHR)
1212 1.67 hannken spec_node_destroy(vp);
1213 1.50 hannken
1214 1.104 ad rw_obj_free(vip->vi_lock);
1215 1.50 hannken uvm_obj_destroy(&vp->v_uobj, true);
1216 1.50 hannken cv_destroy(&vp->v_cv);
1217 1.70 hannken pool_cache_put(vcache_pool, vip);
1218 1.50 hannken }
1219 1.50 hannken
1220 1.50 hannken /*
1221 1.66 hannken * Try to get an initial reference on this cached vnode.
1222 1.66 hannken * Returns zero on success, ENOENT if the vnode has been reclaimed and
1223 1.66 hannken * EBUSY if the vnode state is unstable.
1224 1.66 hannken *
1225 1.66 hannken * v_interlock locked on entry and unlocked on exit.
1226 1.66 hannken */
1227 1.66 hannken int
1228 1.66 hannken vcache_tryvget(vnode_t *vp)
1229 1.66 hannken {
1230 1.67 hannken int error = 0;
1231 1.66 hannken
1232 1.66 hannken KASSERT(mutex_owned(vp->v_interlock));
1233 1.66 hannken
1234 1.67 hannken if (__predict_false(VSTATE_GET(vp) == VS_RECLAIMED))
1235 1.67 hannken error = ENOENT;
1236 1.94 hannken else if (__predict_false(VSTATE_GET(vp) != VS_LOADED))
1237 1.67 hannken error = EBUSY;
1238 1.67 hannken else
1239 1.104 ad vp->v_usecount++;
1240 1.66 hannken
1241 1.66 hannken mutex_exit(vp->v_interlock);
1242 1.66 hannken
1243 1.67 hannken return error;
1244 1.66 hannken }
1245 1.66 hannken
1246 1.66 hannken /*
1247 1.66 hannken * Try to get an initial reference on this cached vnode.
1248 1.66 hannken * Returns zero on success and ENOENT if the vnode has been reclaimed.
1249 1.66 hannken * Will wait for the vnode state to be stable.
1250 1.66 hannken *
1251 1.66 hannken * v_interlock locked on entry and unlocked on exit.
1252 1.66 hannken */
1253 1.66 hannken int
1254 1.66 hannken vcache_vget(vnode_t *vp)
1255 1.66 hannken {
1256 1.66 hannken
1257 1.66 hannken KASSERT(mutex_owned(vp->v_interlock));
1258 1.66 hannken
1259 1.67 hannken /* Increment hold count to prevent vnode from disappearing. */
1260 1.67 hannken vp->v_holdcnt++;
1261 1.67 hannken VSTATE_WAIT_STABLE(vp);
1262 1.67 hannken vp->v_holdcnt--;
1263 1.66 hannken
1264 1.67 hannken /* If this was the last reference to a reclaimed vnode free it now. */
1265 1.67 hannken if (__predict_false(VSTATE_GET(vp) == VS_RECLAIMED)) {
1266 1.67 hannken if (vp->v_holdcnt == 0 && vp->v_usecount == 0)
1267 1.67 hannken vcache_free(VNODE_TO_VIMPL(vp));
1268 1.67 hannken else
1269 1.67 hannken mutex_exit(vp->v_interlock);
1270 1.66 hannken return ENOENT;
1271 1.66 hannken }
1272 1.94 hannken VSTATE_ASSERT(vp, VS_LOADED);
1273 1.104 ad vp->v_usecount++;
1274 1.66 hannken mutex_exit(vp->v_interlock);
1275 1.66 hannken
1276 1.66 hannken return 0;
1277 1.66 hannken }
1278 1.66 hannken
1279 1.66 hannken /*
1280 1.36 hannken * Get a vnode / fs node pair by key and return it referenced through vpp.
1281 1.36 hannken */
1282 1.36 hannken int
1283 1.36 hannken vcache_get(struct mount *mp, const void *key, size_t key_len,
1284 1.36 hannken struct vnode **vpp)
1285 1.36 hannken {
1286 1.36 hannken int error;
1287 1.36 hannken uint32_t hash;
1288 1.36 hannken const void *new_key;
1289 1.36 hannken struct vnode *vp;
1290 1.36 hannken struct vcache_key vcache_key;
1291 1.70 hannken vnode_impl_t *vip, *new_vip;
1292 1.36 hannken
1293 1.36 hannken new_key = NULL;
1294 1.36 hannken *vpp = NULL;
1295 1.36 hannken
1296 1.36 hannken vcache_key.vk_mount = mp;
1297 1.36 hannken vcache_key.vk_key = key;
1298 1.36 hannken vcache_key.vk_key_len = key_len;
1299 1.36 hannken hash = vcache_hash(&vcache_key);
1300 1.36 hannken
1301 1.36 hannken again:
1302 1.69 hannken mutex_enter(&vcache_lock);
1303 1.70 hannken vip = vcache_hash_lookup(&vcache_key, hash);
1304 1.36 hannken
1305 1.36 hannken /* If found, take a reference or retry. */
1306 1.70 hannken if (__predict_true(vip != NULL)) {
1307 1.52 hannken /*
1308 1.52 hannken * If the vnode is loading we cannot take the v_interlock
1309 1.52 hannken * here as it might change during load (see uvm_obj_setlock()).
1310 1.69 hannken * As changing state from VS_LOADING requires both vcache_lock
1311 1.69 hannken * and v_interlock it is safe to test with vcache_lock held.
1312 1.52 hannken *
1313 1.57 hannken * Wait for vnodes changing state from VS_LOADING and retry.
1314 1.52 hannken */
1315 1.70 hannken if (__predict_false(vip->vi_state == VS_LOADING)) {
1316 1.69 hannken cv_wait(&vcache_cv, &vcache_lock);
1317 1.69 hannken mutex_exit(&vcache_lock);
1318 1.52 hannken goto again;
1319 1.52 hannken }
1320 1.70 hannken vp = VIMPL_TO_VNODE(vip);
1321 1.36 hannken mutex_enter(vp->v_interlock);
1322 1.69 hannken mutex_exit(&vcache_lock);
1323 1.66 hannken error = vcache_vget(vp);
1324 1.36 hannken if (error == ENOENT)
1325 1.36 hannken goto again;
1326 1.36 hannken if (error == 0)
1327 1.36 hannken *vpp = vp;
1328 1.36 hannken KASSERT((error != 0) == (*vpp == NULL));
1329 1.36 hannken return error;
1330 1.36 hannken }
1331 1.69 hannken mutex_exit(&vcache_lock);
1332 1.36 hannken
1333 1.36 hannken /* Allocate and initialize a new vcache / vnode pair. */
1334 1.87 hannken error = vfs_busy(mp);
1335 1.36 hannken if (error)
1336 1.36 hannken return error;
1337 1.70 hannken new_vip = vcache_alloc();
1338 1.70 hannken new_vip->vi_key = vcache_key;
1339 1.70 hannken vp = VIMPL_TO_VNODE(new_vip);
1340 1.69 hannken mutex_enter(&vcache_lock);
1341 1.70 hannken vip = vcache_hash_lookup(&vcache_key, hash);
1342 1.70 hannken if (vip == NULL) {
1343 1.69 hannken SLIST_INSERT_HEAD(&vcache_hashtab[hash & vcache_hashmask],
1344 1.70 hannken new_vip, vi_hash);
1345 1.70 hannken vip = new_vip;
1346 1.36 hannken }
1347 1.36 hannken
1348 1.36 hannken /* If another thread beat us inserting this node, retry. */
1349 1.70 hannken if (vip != new_vip) {
1350 1.79 hannken vcache_dealloc(new_vip);
1351 1.87 hannken vfs_unbusy(mp);
1352 1.36 hannken goto again;
1353 1.36 hannken }
1354 1.69 hannken mutex_exit(&vcache_lock);
1355 1.36 hannken
1356 1.57 hannken /* Load the fs node. Exclusive as new_node is VS_LOADING. */
1357 1.36 hannken error = VFS_LOADVNODE(mp, vp, key, key_len, &new_key);
1358 1.36 hannken if (error) {
1359 1.69 hannken mutex_enter(&vcache_lock);
1360 1.69 hannken SLIST_REMOVE(&vcache_hashtab[hash & vcache_hashmask],
1361 1.70 hannken new_vip, vnode_impl, vi_hash);
1362 1.79 hannken vcache_dealloc(new_vip);
1363 1.87 hannken vfs_unbusy(mp);
1364 1.36 hannken KASSERT(*vpp == NULL);
1365 1.36 hannken return error;
1366 1.36 hannken }
1367 1.36 hannken KASSERT(new_key != NULL);
1368 1.36 hannken KASSERT(memcmp(key, new_key, key_len) == 0);
1369 1.36 hannken KASSERT(vp->v_op != NULL);
1370 1.36 hannken vfs_insmntque(vp, mp);
1371 1.36 hannken if ((mp->mnt_iflag & IMNT_MPSAFE) != 0)
1372 1.36 hannken vp->v_vflag |= VV_MPSAFE;
1373 1.87 hannken vfs_ref(mp);
1374 1.87 hannken vfs_unbusy(mp);
1375 1.36 hannken
1376 1.36 hannken /* Finished loading, finalize node. */
1377 1.69 hannken mutex_enter(&vcache_lock);
1378 1.70 hannken new_vip->vi_key.vk_key = new_key;
1379 1.39 hannken mutex_enter(vp->v_interlock);
1380 1.94 hannken VSTATE_CHANGE(vp, VS_LOADING, VS_LOADED);
1381 1.39 hannken mutex_exit(vp->v_interlock);
1382 1.69 hannken mutex_exit(&vcache_lock);
1383 1.36 hannken *vpp = vp;
1384 1.36 hannken return 0;
1385 1.36 hannken }
1386 1.36 hannken
1387 1.36 hannken /*
1388 1.40 hannken * Create a new vnode / fs node pair and return it referenced through vpp.
1389 1.40 hannken */
1390 1.40 hannken int
1391 1.40 hannken vcache_new(struct mount *mp, struct vnode *dvp, struct vattr *vap,
1392 1.101 hannken kauth_cred_t cred, void *extra, struct vnode **vpp)
1393 1.40 hannken {
1394 1.40 hannken int error;
1395 1.40 hannken uint32_t hash;
1396 1.70 hannken struct vnode *vp, *ovp;
1397 1.70 hannken vnode_impl_t *vip, *ovip;
1398 1.40 hannken
1399 1.40 hannken *vpp = NULL;
1400 1.40 hannken
1401 1.40 hannken /* Allocate and initialize a new vcache / vnode pair. */
1402 1.87 hannken error = vfs_busy(mp);
1403 1.40 hannken if (error)
1404 1.40 hannken return error;
1405 1.70 hannken vip = vcache_alloc();
1406 1.70 hannken vip->vi_key.vk_mount = mp;
1407 1.70 hannken vp = VIMPL_TO_VNODE(vip);
1408 1.40 hannken
1409 1.40 hannken /* Create and load the fs node. */
1410 1.101 hannken error = VFS_NEWVNODE(mp, dvp, vp, vap, cred, extra,
1411 1.70 hannken &vip->vi_key.vk_key_len, &vip->vi_key.vk_key);
1412 1.40 hannken if (error) {
1413 1.69 hannken mutex_enter(&vcache_lock);
1414 1.79 hannken vcache_dealloc(vip);
1415 1.87 hannken vfs_unbusy(mp);
1416 1.40 hannken KASSERT(*vpp == NULL);
1417 1.40 hannken return error;
1418 1.40 hannken }
1419 1.40 hannken KASSERT(vp->v_op != NULL);
1420 1.97 hannken KASSERT((vip->vi_key.vk_key_len == 0) == (mp == dead_rootmount));
1421 1.97 hannken if (vip->vi_key.vk_key_len > 0) {
1422 1.97 hannken KASSERT(vip->vi_key.vk_key != NULL);
1423 1.97 hannken hash = vcache_hash(&vip->vi_key);
1424 1.40 hannken
1425 1.97 hannken /*
1426 1.97 hannken * Wait for previous instance to be reclaimed,
1427 1.97 hannken * then insert new node.
1428 1.97 hannken */
1429 1.97 hannken mutex_enter(&vcache_lock);
1430 1.97 hannken while ((ovip = vcache_hash_lookup(&vip->vi_key, hash))) {
1431 1.97 hannken ovp = VIMPL_TO_VNODE(ovip);
1432 1.97 hannken mutex_enter(ovp->v_interlock);
1433 1.97 hannken mutex_exit(&vcache_lock);
1434 1.97 hannken error = vcache_vget(ovp);
1435 1.97 hannken KASSERT(error == ENOENT);
1436 1.97 hannken mutex_enter(&vcache_lock);
1437 1.97 hannken }
1438 1.97 hannken SLIST_INSERT_HEAD(&vcache_hashtab[hash & vcache_hashmask],
1439 1.97 hannken vip, vi_hash);
1440 1.69 hannken mutex_exit(&vcache_lock);
1441 1.40 hannken }
1442 1.40 hannken vfs_insmntque(vp, mp);
1443 1.40 hannken if ((mp->mnt_iflag & IMNT_MPSAFE) != 0)
1444 1.40 hannken vp->v_vflag |= VV_MPSAFE;
1445 1.87 hannken vfs_ref(mp);
1446 1.87 hannken vfs_unbusy(mp);
1447 1.40 hannken
1448 1.40 hannken /* Finished loading, finalize node. */
1449 1.69 hannken mutex_enter(&vcache_lock);
1450 1.52 hannken mutex_enter(vp->v_interlock);
1451 1.94 hannken VSTATE_CHANGE(vp, VS_LOADING, VS_LOADED);
1452 1.69 hannken mutex_exit(&vcache_lock);
1453 1.40 hannken mutex_exit(vp->v_interlock);
1454 1.40 hannken *vpp = vp;
1455 1.40 hannken return 0;
1456 1.40 hannken }
1457 1.40 hannken
1458 1.40 hannken /*
1459 1.65 hannken * Prepare key change: update old cache nodes key and lock new cache node.
1460 1.37 hannken * Return an error if the new node already exists.
1461 1.37 hannken */
1462 1.37 hannken int
1463 1.37 hannken vcache_rekey_enter(struct mount *mp, struct vnode *vp,
1464 1.37 hannken const void *old_key, size_t old_key_len,
1465 1.37 hannken const void *new_key, size_t new_key_len)
1466 1.37 hannken {
1467 1.37 hannken uint32_t old_hash, new_hash;
1468 1.37 hannken struct vcache_key old_vcache_key, new_vcache_key;
1469 1.70 hannken vnode_impl_t *vip, *new_vip;
1470 1.37 hannken
1471 1.37 hannken old_vcache_key.vk_mount = mp;
1472 1.37 hannken old_vcache_key.vk_key = old_key;
1473 1.37 hannken old_vcache_key.vk_key_len = old_key_len;
1474 1.37 hannken old_hash = vcache_hash(&old_vcache_key);
1475 1.37 hannken
1476 1.37 hannken new_vcache_key.vk_mount = mp;
1477 1.37 hannken new_vcache_key.vk_key = new_key;
1478 1.37 hannken new_vcache_key.vk_key_len = new_key_len;
1479 1.37 hannken new_hash = vcache_hash(&new_vcache_key);
1480 1.37 hannken
1481 1.70 hannken new_vip = vcache_alloc();
1482 1.70 hannken new_vip->vi_key = new_vcache_key;
1483 1.37 hannken
1484 1.52 hannken /* Insert locked new node used as placeholder. */
1485 1.69 hannken mutex_enter(&vcache_lock);
1486 1.70 hannken vip = vcache_hash_lookup(&new_vcache_key, new_hash);
1487 1.70 hannken if (vip != NULL) {
1488 1.79 hannken vcache_dealloc(new_vip);
1489 1.37 hannken return EEXIST;
1490 1.37 hannken }
1491 1.69 hannken SLIST_INSERT_HEAD(&vcache_hashtab[new_hash & vcache_hashmask],
1492 1.70 hannken new_vip, vi_hash);
1493 1.49 hannken
1494 1.65 hannken /* Replace old nodes key with the temporary copy. */
1495 1.70 hannken vip = vcache_hash_lookup(&old_vcache_key, old_hash);
1496 1.70 hannken KASSERT(vip != NULL);
1497 1.70 hannken KASSERT(VIMPL_TO_VNODE(vip) == vp);
1498 1.70 hannken KASSERT(vip->vi_key.vk_key != old_vcache_key.vk_key);
1499 1.70 hannken vip->vi_key = old_vcache_key;
1500 1.69 hannken mutex_exit(&vcache_lock);
1501 1.37 hannken return 0;
1502 1.37 hannken }
1503 1.37 hannken
1504 1.37 hannken /*
1505 1.65 hannken * Key change complete: update old node and remove placeholder.
1506 1.37 hannken */
1507 1.37 hannken void
1508 1.37 hannken vcache_rekey_exit(struct mount *mp, struct vnode *vp,
1509 1.37 hannken const void *old_key, size_t old_key_len,
1510 1.37 hannken const void *new_key, size_t new_key_len)
1511 1.37 hannken {
1512 1.37 hannken uint32_t old_hash, new_hash;
1513 1.37 hannken struct vcache_key old_vcache_key, new_vcache_key;
1514 1.70 hannken vnode_impl_t *vip, *new_vip;
1515 1.70 hannken struct vnode *new_vp;
1516 1.37 hannken
1517 1.37 hannken old_vcache_key.vk_mount = mp;
1518 1.37 hannken old_vcache_key.vk_key = old_key;
1519 1.37 hannken old_vcache_key.vk_key_len = old_key_len;
1520 1.37 hannken old_hash = vcache_hash(&old_vcache_key);
1521 1.37 hannken
1522 1.37 hannken new_vcache_key.vk_mount = mp;
1523 1.37 hannken new_vcache_key.vk_key = new_key;
1524 1.37 hannken new_vcache_key.vk_key_len = new_key_len;
1525 1.37 hannken new_hash = vcache_hash(&new_vcache_key);
1526 1.37 hannken
1527 1.69 hannken mutex_enter(&vcache_lock);
1528 1.49 hannken
1529 1.49 hannken /* Lookup old and new node. */
1530 1.70 hannken vip = vcache_hash_lookup(&old_vcache_key, old_hash);
1531 1.70 hannken KASSERT(vip != NULL);
1532 1.70 hannken KASSERT(VIMPL_TO_VNODE(vip) == vp);
1533 1.70 hannken
1534 1.70 hannken new_vip = vcache_hash_lookup(&new_vcache_key, new_hash);
1535 1.70 hannken KASSERT(new_vip != NULL);
1536 1.70 hannken KASSERT(new_vip->vi_key.vk_key_len == new_key_len);
1537 1.70 hannken new_vp = VIMPL_TO_VNODE(new_vip);
1538 1.70 hannken mutex_enter(new_vp->v_interlock);
1539 1.70 hannken VSTATE_ASSERT(VIMPL_TO_VNODE(new_vip), VS_LOADING);
1540 1.79 hannken mutex_exit(new_vp->v_interlock);
1541 1.49 hannken
1542 1.49 hannken /* Rekey old node and put it onto its new hashlist. */
1543 1.70 hannken vip->vi_key = new_vcache_key;
1544 1.49 hannken if (old_hash != new_hash) {
1545 1.69 hannken SLIST_REMOVE(&vcache_hashtab[old_hash & vcache_hashmask],
1546 1.70 hannken vip, vnode_impl, vi_hash);
1547 1.69 hannken SLIST_INSERT_HEAD(&vcache_hashtab[new_hash & vcache_hashmask],
1548 1.70 hannken vip, vi_hash);
1549 1.49 hannken }
1550 1.49 hannken
1551 1.49 hannken /* Remove new node used as placeholder. */
1552 1.69 hannken SLIST_REMOVE(&vcache_hashtab[new_hash & vcache_hashmask],
1553 1.70 hannken new_vip, vnode_impl, vi_hash);
1554 1.79 hannken vcache_dealloc(new_vip);
1555 1.37 hannken }
1556 1.37 hannken
1557 1.37 hannken /*
1558 1.54 hannken * Disassociate the underlying file system from a vnode.
1559 1.54 hannken *
1560 1.54 hannken * Must be called with vnode locked and will return unlocked.
1561 1.54 hannken * Must be called with the interlock held, and will return with it held.
1562 1.54 hannken */
1563 1.54 hannken static void
1564 1.54 hannken vcache_reclaim(vnode_t *vp)
1565 1.54 hannken {
1566 1.54 hannken lwp_t *l = curlwp;
1567 1.70 hannken vnode_impl_t *vip = VNODE_TO_VIMPL(vp);
1568 1.74 hannken struct mount *mp = vp->v_mount;
1569 1.55 hannken uint32_t hash;
1570 1.55 hannken uint8_t temp_buf[64], *temp_key;
1571 1.55 hannken size_t temp_key_len;
1572 1.54 hannken bool recycle, active;
1573 1.54 hannken int error;
1574 1.54 hannken
1575 1.54 hannken KASSERT((vp->v_vflag & VV_LOCKSWORK) == 0 ||
1576 1.54 hannken VOP_ISLOCKED(vp) == LK_EXCLUSIVE);
1577 1.54 hannken KASSERT(mutex_owned(vp->v_interlock));
1578 1.54 hannken KASSERT(vp->v_usecount != 0);
1579 1.54 hannken
1580 1.54 hannken active = (vp->v_usecount > 1);
1581 1.70 hannken temp_key_len = vip->vi_key.vk_key_len;
1582 1.54 hannken /*
1583 1.54 hannken * Prevent the vnode from being recycled or brought into use
1584 1.54 hannken * while we clean it out.
1585 1.54 hannken */
1586 1.94 hannken VSTATE_CHANGE(vp, VS_LOADED, VS_RECLAIMING);
1587 1.104 ad if ((vp->v_iflag & VI_EXECMAP) != 0 && vp->v_uobj.uo_npages != 0) {
1588 1.105 ad cpu_count(CPU_COUNT_EXECPAGES, -vp->v_uobj.uo_npages);
1589 1.105 ad cpu_count(CPU_COUNT_FILEPAGES, vp->v_uobj.uo_npages);
1590 1.54 hannken }
1591 1.54 hannken vp->v_iflag &= ~(VI_TEXT|VI_EXECMAP);
1592 1.54 hannken mutex_exit(vp->v_interlock);
1593 1.54 hannken
1594 1.55 hannken /* Replace the vnode key with a temporary copy. */
1595 1.70 hannken if (vip->vi_key.vk_key_len > sizeof(temp_buf)) {
1596 1.55 hannken temp_key = kmem_alloc(temp_key_len, KM_SLEEP);
1597 1.55 hannken } else {
1598 1.55 hannken temp_key = temp_buf;
1599 1.55 hannken }
1600 1.97 hannken if (vip->vi_key.vk_key_len > 0) {
1601 1.97 hannken mutex_enter(&vcache_lock);
1602 1.97 hannken memcpy(temp_key, vip->vi_key.vk_key, temp_key_len);
1603 1.97 hannken vip->vi_key.vk_key = temp_key;
1604 1.97 hannken mutex_exit(&vcache_lock);
1605 1.97 hannken }
1606 1.55 hannken
1607 1.96 hannken fstrans_start(mp);
1608 1.74 hannken
1609 1.54 hannken /*
1610 1.54 hannken * Clean out any cached data associated with the vnode.
1611 1.54 hannken * If purging an active vnode, it must be closed and
1612 1.60 hannken * deactivated before being reclaimed.
1613 1.54 hannken */
1614 1.54 hannken error = vinvalbuf(vp, V_SAVE, NOCRED, l, 0, 0);
1615 1.54 hannken if (error != 0) {
1616 1.54 hannken if (wapbl_vphaswapbl(vp))
1617 1.54 hannken WAPBL_DISCARD(wapbl_vptomp(vp));
1618 1.54 hannken error = vinvalbuf(vp, 0, NOCRED, l, 0, 0);
1619 1.54 hannken }
1620 1.54 hannken KASSERTMSG((error == 0), "vinvalbuf failed: %d", error);
1621 1.54 hannken KASSERT((vp->v_iflag & VI_ONWORKLST) == 0);
1622 1.54 hannken if (active && (vp->v_type == VBLK || vp->v_type == VCHR)) {
1623 1.54 hannken spec_node_revoke(vp);
1624 1.54 hannken }
1625 1.54 hannken
1626 1.60 hannken /*
1627 1.60 hannken * Disassociate the underlying file system from the vnode.
1628 1.90 riastrad * VOP_INACTIVE leaves the vnode locked; VOP_RECLAIM unlocks
1629 1.90 riastrad * the vnode, and may destroy the vnode so that VOP_UNLOCK
1630 1.90 riastrad * would no longer function.
1631 1.60 hannken */
1632 1.60 hannken VOP_INACTIVE(vp, &recycle);
1633 1.91 riastrad KASSERT((vp->v_vflag & VV_LOCKSWORK) == 0 ||
1634 1.91 riastrad VOP_ISLOCKED(vp) == LK_EXCLUSIVE);
1635 1.54 hannken if (VOP_RECLAIM(vp)) {
1636 1.54 hannken vnpanic(vp, "%s: cannot reclaim", __func__);
1637 1.54 hannken }
1638 1.54 hannken
1639 1.54 hannken KASSERT(vp->v_data == NULL);
1640 1.54 hannken KASSERT(vp->v_uobj.uo_npages == 0);
1641 1.54 hannken
1642 1.54 hannken if (vp->v_type == VREG && vp->v_ractx != NULL) {
1643 1.54 hannken uvm_ra_freectx(vp->v_ractx);
1644 1.54 hannken vp->v_ractx = NULL;
1645 1.54 hannken }
1646 1.54 hannken
1647 1.54 hannken /* Purge name cache. */
1648 1.54 hannken cache_purge(vp);
1649 1.54 hannken
1650 1.97 hannken if (vip->vi_key.vk_key_len > 0) {
1651 1.55 hannken /* Remove from vnode cache. */
1652 1.97 hannken hash = vcache_hash(&vip->vi_key);
1653 1.97 hannken mutex_enter(&vcache_lock);
1654 1.97 hannken KASSERT(vip == vcache_hash_lookup(&vip->vi_key, hash));
1655 1.97 hannken SLIST_REMOVE(&vcache_hashtab[hash & vcache_hashmask],
1656 1.97 hannken vip, vnode_impl, vi_hash);
1657 1.97 hannken mutex_exit(&vcache_lock);
1658 1.97 hannken }
1659 1.55 hannken if (temp_key != temp_buf)
1660 1.55 hannken kmem_free(temp_key, temp_key_len);
1661 1.55 hannken
1662 1.54 hannken /* Done with purge, notify sleepers of the grim news. */
1663 1.54 hannken mutex_enter(vp->v_interlock);
1664 1.54 hannken vp->v_op = dead_vnodeop_p;
1665 1.54 hannken vp->v_vflag |= VV_LOCKSWORK;
1666 1.57 hannken VSTATE_CHANGE(vp, VS_RECLAIMING, VS_RECLAIMED);
1667 1.54 hannken vp->v_tag = VT_NON;
1668 1.54 hannken KNOTE(&vp->v_klist, NOTE_REVOKE);
1669 1.80 hannken mutex_exit(vp->v_interlock);
1670 1.54 hannken
1671 1.80 hannken /*
1672 1.80 hannken * Move to dead mount. Must be after changing the operations
1673 1.80 hannken * vector as vnode operations enter the mount before using the
1674 1.80 hannken * operations vector. See sys/kern/vnode_if.c.
1675 1.80 hannken */
1676 1.80 hannken vp->v_vflag &= ~VV_ROOT;
1677 1.86 hannken vfs_ref(dead_rootmount);
1678 1.80 hannken vfs_insmntque(vp, dead_rootmount);
1679 1.80 hannken
1680 1.80 hannken mutex_enter(vp->v_interlock);
1681 1.74 hannken fstrans_done(mp);
1682 1.54 hannken KASSERT((vp->v_iflag & VI_ONWORKLST) == 0);
1683 1.108 ad
1684 1.108 ad #ifdef PAX_SEGVGUARD
1685 1.108 ad pax_segvguard_cleanup(vp);
1686 1.108 ad #endif /* PAX_SEGVGUARD */
1687 1.54 hannken }
1688 1.54 hannken
1689 1.54 hannken /*
1690 1.98 hannken * Disassociate the underlying file system from an open device vnode
1691 1.98 hannken * and make it anonymous.
1692 1.98 hannken *
1693 1.98 hannken * Vnode unlocked on entry, drops a reference to the vnode.
1694 1.98 hannken */
1695 1.98 hannken void
1696 1.98 hannken vcache_make_anon(vnode_t *vp)
1697 1.98 hannken {
1698 1.98 hannken vnode_impl_t *vip = VNODE_TO_VIMPL(vp);
1699 1.98 hannken uint32_t hash;
1700 1.98 hannken bool recycle;
1701 1.98 hannken
1702 1.98 hannken KASSERT(vp->v_type == VBLK || vp->v_type == VCHR);
1703 1.103 hannken KASSERT(vp->v_mount == dead_rootmount || fstrans_is_owner(vp->v_mount));
1704 1.98 hannken VSTATE_ASSERT_UNLOCKED(vp, VS_ACTIVE);
1705 1.98 hannken
1706 1.98 hannken /* Remove from vnode cache. */
1707 1.98 hannken hash = vcache_hash(&vip->vi_key);
1708 1.98 hannken mutex_enter(&vcache_lock);
1709 1.98 hannken KASSERT(vip == vcache_hash_lookup(&vip->vi_key, hash));
1710 1.98 hannken SLIST_REMOVE(&vcache_hashtab[hash & vcache_hashmask],
1711 1.98 hannken vip, vnode_impl, vi_hash);
1712 1.98 hannken vip->vi_key.vk_mount = dead_rootmount;
1713 1.98 hannken vip->vi_key.vk_key_len = 0;
1714 1.98 hannken vip->vi_key.vk_key = NULL;
1715 1.98 hannken mutex_exit(&vcache_lock);
1716 1.98 hannken
1717 1.98 hannken /*
1718 1.98 hannken * Disassociate the underlying file system from the vnode.
1719 1.98 hannken * VOP_INACTIVE leaves the vnode locked; VOP_RECLAIM unlocks
1720 1.98 hannken * the vnode, and may destroy the vnode so that VOP_UNLOCK
1721 1.98 hannken * would no longer function.
1722 1.98 hannken */
1723 1.98 hannken if (vn_lock(vp, LK_EXCLUSIVE)) {
1724 1.98 hannken vnpanic(vp, "%s: cannot lock", __func__);
1725 1.98 hannken }
1726 1.98 hannken VOP_INACTIVE(vp, &recycle);
1727 1.98 hannken KASSERT((vp->v_vflag & VV_LOCKSWORK) == 0 ||
1728 1.98 hannken VOP_ISLOCKED(vp) == LK_EXCLUSIVE);
1729 1.98 hannken if (VOP_RECLAIM(vp)) {
1730 1.98 hannken vnpanic(vp, "%s: cannot reclaim", __func__);
1731 1.98 hannken }
1732 1.98 hannken
1733 1.98 hannken /* Purge name cache. */
1734 1.98 hannken cache_purge(vp);
1735 1.98 hannken
1736 1.98 hannken /* Done with purge, change operations vector. */
1737 1.98 hannken mutex_enter(vp->v_interlock);
1738 1.98 hannken vp->v_op = spec_vnodeop_p;
1739 1.98 hannken vp->v_vflag |= VV_MPSAFE;
1740 1.98 hannken vp->v_vflag &= ~VV_LOCKSWORK;
1741 1.98 hannken mutex_exit(vp->v_interlock);
1742 1.98 hannken
1743 1.98 hannken /*
1744 1.98 hannken * Move to dead mount. Must be after changing the operations
1745 1.98 hannken * vector as vnode operations enter the mount before using the
1746 1.98 hannken * operations vector. See sys/kern/vnode_if.c.
1747 1.98 hannken */
1748 1.98 hannken vfs_ref(dead_rootmount);
1749 1.98 hannken vfs_insmntque(vp, dead_rootmount);
1750 1.98 hannken
1751 1.98 hannken vrele(vp);
1752 1.98 hannken }
1753 1.98 hannken
1754 1.98 hannken /*
1755 1.1 rmind * Update outstanding I/O count and do wakeup if requested.
1756 1.1 rmind */
1757 1.1 rmind void
1758 1.1 rmind vwakeup(struct buf *bp)
1759 1.1 rmind {
1760 1.1 rmind vnode_t *vp;
1761 1.1 rmind
1762 1.1 rmind if ((vp = bp->b_vp) == NULL)
1763 1.1 rmind return;
1764 1.1 rmind
1765 1.9 rmind KASSERT(bp->b_objlock == vp->v_interlock);
1766 1.1 rmind KASSERT(mutex_owned(bp->b_objlock));
1767 1.1 rmind
1768 1.1 rmind if (--vp->v_numoutput < 0)
1769 1.11 christos vnpanic(vp, "%s: neg numoutput, vp %p", __func__, vp);
1770 1.1 rmind if (vp->v_numoutput == 0)
1771 1.1 rmind cv_broadcast(&vp->v_cv);
1772 1.1 rmind }
1773 1.1 rmind
1774 1.1 rmind /*
1775 1.35 hannken * Test a vnode for being or becoming dead. Returns one of:
1776 1.35 hannken * EBUSY: vnode is becoming dead, with "flags == VDEAD_NOWAIT" only.
1777 1.35 hannken * ENOENT: vnode is dead.
1778 1.35 hannken * 0: otherwise.
1779 1.35 hannken *
1780 1.35 hannken * Whenever this function returns a non-zero value all future
1781 1.35 hannken * calls will also return a non-zero value.
1782 1.35 hannken */
1783 1.35 hannken int
1784 1.35 hannken vdead_check(struct vnode *vp, int flags)
1785 1.35 hannken {
1786 1.35 hannken
1787 1.35 hannken KASSERT(mutex_owned(vp->v_interlock));
1788 1.35 hannken
1789 1.52 hannken if (! ISSET(flags, VDEAD_NOWAIT))
1790 1.52 hannken VSTATE_WAIT_STABLE(vp);
1791 1.1 rmind
1792 1.57 hannken if (VSTATE_GET(vp) == VS_RECLAIMING) {
1793 1.52 hannken KASSERT(ISSET(flags, VDEAD_NOWAIT));
1794 1.52 hannken return EBUSY;
1795 1.57 hannken } else if (VSTATE_GET(vp) == VS_RECLAIMED) {
1796 1.52 hannken return ENOENT;
1797 1.52 hannken }
1798 1.1 rmind
1799 1.52 hannken return 0;
1800 1.1 rmind }
1801 1.1 rmind
1802 1.1 rmind int
1803 1.61 hannken vfs_drainvnodes(void)
1804 1.1 rmind {
1805 1.63 hannken int i, gen;
1806 1.61 hannken
1807 1.63 hannken mutex_enter(&vdrain_lock);
1808 1.63 hannken for (i = 0; i < 2; i++) {
1809 1.63 hannken gen = vdrain_gen;
1810 1.63 hannken while (gen == vdrain_gen) {
1811 1.63 hannken cv_broadcast(&vdrain_cv);
1812 1.63 hannken cv_wait(&vdrain_gen_cv, &vdrain_lock);
1813 1.63 hannken }
1814 1.61 hannken }
1815 1.63 hannken mutex_exit(&vdrain_lock);
1816 1.12 hannken
1817 1.63 hannken if (numvnodes >= desiredvnodes)
1818 1.63 hannken return EBUSY;
1819 1.12 hannken
1820 1.69 hannken if (vcache_hashsize != desiredvnodes)
1821 1.61 hannken vcache_reinit();
1822 1.36 hannken
1823 1.1 rmind return 0;
1824 1.1 rmind }
1825 1.1 rmind
1826 1.1 rmind void
1827 1.11 christos vnpanic(vnode_t *vp, const char *fmt, ...)
1828 1.1 rmind {
1829 1.11 christos va_list ap;
1830 1.11 christos
1831 1.1 rmind #ifdef DIAGNOSTIC
1832 1.1 rmind vprint(NULL, vp);
1833 1.1 rmind #endif
1834 1.11 christos va_start(ap, fmt);
1835 1.11 christos vpanic(fmt, ap);
1836 1.11 christos va_end(ap);
1837 1.1 rmind }
1838