vfs_vnode.c revision 1.113 1 1.113 ad /* $NetBSD: vfs_vnode.c,v 1.113 2020/02/27 22:12:54 ad Exp $ */
2 1.1 rmind
3 1.1 rmind /*-
4 1.111 ad * Copyright (c) 1997-2011, 2019, 2020 The NetBSD Foundation, Inc.
5 1.1 rmind * All rights reserved.
6 1.1 rmind *
7 1.1 rmind * This code is derived from software contributed to The NetBSD Foundation
8 1.1 rmind * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
9 1.1 rmind * NASA Ames Research Center, by Charles M. Hannum, and by Andrew Doran.
10 1.1 rmind *
11 1.1 rmind * Redistribution and use in source and binary forms, with or without
12 1.1 rmind * modification, are permitted provided that the following conditions
13 1.1 rmind * are met:
14 1.1 rmind * 1. Redistributions of source code must retain the above copyright
15 1.1 rmind * notice, this list of conditions and the following disclaimer.
16 1.1 rmind * 2. Redistributions in binary form must reproduce the above copyright
17 1.1 rmind * notice, this list of conditions and the following disclaimer in the
18 1.1 rmind * documentation and/or other materials provided with the distribution.
19 1.1 rmind *
20 1.1 rmind * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
21 1.1 rmind * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
22 1.1 rmind * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
23 1.1 rmind * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
24 1.1 rmind * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 1.1 rmind * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 1.1 rmind * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 1.1 rmind * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 1.1 rmind * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 1.1 rmind * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 1.1 rmind * POSSIBILITY OF SUCH DAMAGE.
31 1.1 rmind */
32 1.1 rmind
33 1.1 rmind /*
34 1.1 rmind * Copyright (c) 1989, 1993
35 1.1 rmind * The Regents of the University of California. All rights reserved.
36 1.1 rmind * (c) UNIX System Laboratories, Inc.
37 1.1 rmind * All or some portions of this file are derived from material licensed
38 1.1 rmind * to the University of California by American Telephone and Telegraph
39 1.1 rmind * Co. or Unix System Laboratories, Inc. and are reproduced herein with
40 1.1 rmind * the permission of UNIX System Laboratories, Inc.
41 1.1 rmind *
42 1.1 rmind * Redistribution and use in source and binary forms, with or without
43 1.1 rmind * modification, are permitted provided that the following conditions
44 1.1 rmind * are met:
45 1.1 rmind * 1. Redistributions of source code must retain the above copyright
46 1.1 rmind * notice, this list of conditions and the following disclaimer.
47 1.1 rmind * 2. Redistributions in binary form must reproduce the above copyright
48 1.1 rmind * notice, this list of conditions and the following disclaimer in the
49 1.1 rmind * documentation and/or other materials provided with the distribution.
50 1.1 rmind * 3. Neither the name of the University nor the names of its contributors
51 1.1 rmind * may be used to endorse or promote products derived from this software
52 1.1 rmind * without specific prior written permission.
53 1.1 rmind *
54 1.1 rmind * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
55 1.1 rmind * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
56 1.1 rmind * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
57 1.1 rmind * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
58 1.1 rmind * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
59 1.1 rmind * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
60 1.1 rmind * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
61 1.1 rmind * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
62 1.1 rmind * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
63 1.1 rmind * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
64 1.1 rmind * SUCH DAMAGE.
65 1.1 rmind *
66 1.1 rmind * @(#)vfs_subr.c 8.13 (Berkeley) 4/18/94
67 1.1 rmind */
68 1.1 rmind
69 1.1 rmind /*
70 1.8 rmind * The vnode cache subsystem.
71 1.1 rmind *
72 1.8 rmind * Life-cycle
73 1.1 rmind *
74 1.8 rmind * Normally, there are two points where new vnodes are created:
75 1.8 rmind * VOP_CREATE(9) and VOP_LOOKUP(9). The life-cycle of a vnode
76 1.8 rmind * starts in one of the following ways:
77 1.8 rmind *
78 1.45 hannken * - Allocation, via vcache_get(9) or vcache_new(9).
79 1.66 hannken * - Reclamation of inactive vnode, via vcache_vget(9).
80 1.8 rmind *
81 1.16 rmind * Recycle from a free list, via getnewvnode(9) -> getcleanvnode(9)
82 1.16 rmind * was another, traditional way. Currently, only the draining thread
83 1.16 rmind * recycles the vnodes. This behaviour might be revisited.
84 1.16 rmind *
85 1.8 rmind * The life-cycle ends when the last reference is dropped, usually
86 1.8 rmind * in VOP_REMOVE(9). In such case, VOP_INACTIVE(9) is called to inform
87 1.8 rmind * the file system that vnode is inactive. Via this call, file system
88 1.16 rmind * indicates whether vnode can be recycled (usually, it checks its own
89 1.16 rmind * references, e.g. count of links, whether the file was removed).
90 1.8 rmind *
91 1.8 rmind * Depending on indication, vnode can be put into a free list (cache),
92 1.54 hannken * or cleaned via vcache_reclaim, which calls VOP_RECLAIM(9) to
93 1.54 hannken * disassociate underlying file system from the vnode, and finally
94 1.54 hannken * destroyed.
95 1.8 rmind *
96 1.52 hannken * Vnode state
97 1.52 hannken *
98 1.52 hannken * Vnode is always in one of six states:
99 1.52 hannken * - MARKER This is a marker vnode to help list traversal. It
100 1.52 hannken * will never change its state.
101 1.52 hannken * - LOADING Vnode is associating underlying file system and not
102 1.52 hannken * yet ready to use.
103 1.94 hannken * - LOADED Vnode has associated underlying file system and is
104 1.52 hannken * ready to use.
105 1.52 hannken * - BLOCKED Vnode is active but cannot get new references.
106 1.52 hannken * - RECLAIMING Vnode is disassociating from the underlying file
107 1.52 hannken * system.
108 1.52 hannken * - RECLAIMED Vnode has disassociated from underlying file system
109 1.52 hannken * and is dead.
110 1.52 hannken *
111 1.52 hannken * Valid state changes are:
112 1.94 hannken * LOADING -> LOADED
113 1.52 hannken * Vnode has been initialised in vcache_get() or
114 1.52 hannken * vcache_new() and is ready to use.
115 1.94 hannken * LOADED -> RECLAIMING
116 1.52 hannken * Vnode starts disassociation from underlying file
117 1.54 hannken * system in vcache_reclaim().
118 1.52 hannken * RECLAIMING -> RECLAIMED
119 1.52 hannken * Vnode finished disassociation from underlying file
120 1.54 hannken * system in vcache_reclaim().
121 1.94 hannken * LOADED -> BLOCKED
122 1.52 hannken * Either vcache_rekey*() is changing the vnode key or
123 1.52 hannken * vrelel() is about to call VOP_INACTIVE().
124 1.94 hannken * BLOCKED -> LOADED
125 1.52 hannken * The block condition is over.
126 1.52 hannken * LOADING -> RECLAIMED
127 1.52 hannken * Either vcache_get() or vcache_new() failed to
128 1.52 hannken * associate the underlying file system or vcache_rekey*()
129 1.52 hannken * drops a vnode used as placeholder.
130 1.52 hannken *
131 1.52 hannken * Of these states LOADING, BLOCKED and RECLAIMING are intermediate
132 1.52 hannken * and it is possible to wait for state change.
133 1.52 hannken *
134 1.52 hannken * State is protected with v_interlock with one exception:
135 1.69 hannken * to change from LOADING both v_interlock and vcache_lock must be held
136 1.52 hannken * so it is possible to check "state == LOADING" without holding
137 1.52 hannken * v_interlock. See vcache_get() for details.
138 1.52 hannken *
139 1.8 rmind * Reference counting
140 1.8 rmind *
141 1.8 rmind * Vnode is considered active, if reference count (vnode_t::v_usecount)
142 1.8 rmind * is non-zero. It is maintained using: vref(9) and vrele(9), as well
143 1.8 rmind * as vput(9), routines. Common points holding references are e.g.
144 1.8 rmind * file openings, current working directory, mount points, etc.
145 1.8 rmind *
146 1.112 ad * Note on v_usecount and its locking
147 1.112 ad *
148 1.112 ad * At nearly all points it is known that v_usecount could be zero,
149 1.112 ad * the vnode_t::v_interlock will be held. To change the count away
150 1.112 ad * from zero, the interlock must be held. To change from a non-zero
151 1.112 ad * value to zero, again the interlock must be held.
152 1.112 ad *
153 1.112 ad * Changing the usecount from a non-zero value to a non-zero value can
154 1.112 ad * safely be done using atomic operations, without the interlock held.
155 1.1 rmind */
156 1.1 rmind
157 1.1 rmind #include <sys/cdefs.h>
158 1.113 ad __KERNEL_RCSID(0, "$NetBSD: vfs_vnode.c,v 1.113 2020/02/27 22:12:54 ad Exp $");
159 1.108 ad
160 1.109 ad #ifdef _KERNEL_OPT
161 1.108 ad #include "opt_pax.h"
162 1.109 ad #endif
163 1.1 rmind
164 1.1 rmind #include <sys/param.h>
165 1.1 rmind #include <sys/kernel.h>
166 1.1 rmind
167 1.1 rmind #include <sys/atomic.h>
168 1.1 rmind #include <sys/buf.h>
169 1.1 rmind #include <sys/conf.h>
170 1.1 rmind #include <sys/device.h>
171 1.36 hannken #include <sys/hash.h>
172 1.1 rmind #include <sys/kauth.h>
173 1.1 rmind #include <sys/kmem.h>
174 1.1 rmind #include <sys/kthread.h>
175 1.1 rmind #include <sys/module.h>
176 1.1 rmind #include <sys/mount.h>
177 1.1 rmind #include <sys/namei.h>
178 1.108 ad #include <sys/pax.h>
179 1.1 rmind #include <sys/syscallargs.h>
180 1.1 rmind #include <sys/sysctl.h>
181 1.1 rmind #include <sys/systm.h>
182 1.58 hannken #include <sys/vnode_impl.h>
183 1.1 rmind #include <sys/wapbl.h>
184 1.24 hannken #include <sys/fstrans.h>
185 1.1 rmind
186 1.1 rmind #include <uvm/uvm.h>
187 1.1 rmind #include <uvm/uvm_readahead.h>
188 1.104 ad #include <uvm/uvm_stat.h>
189 1.1 rmind
190 1.23 hannken /* Flags to vrelel. */
191 1.104 ad #define VRELEL_ASYNC 0x0001 /* Always defer to vrele thread. */
192 1.104 ad
193 1.104 ad #define LRU_VRELE 0
194 1.104 ad #define LRU_FREE 1
195 1.104 ad #define LRU_HOLD 2
196 1.104 ad #define LRU_COUNT 3
197 1.1 rmind
198 1.16 rmind /*
199 1.63 hannken * There are three lru lists: one holds vnodes waiting for async release,
200 1.104 ad * one is for vnodes which have no buffer/page references and one for those
201 1.104 ad * which do (i.e. v_holdcnt is non-zero). We put the lists into a single,
202 1.104 ad * private cache line as vnodes migrate between them while under the same
203 1.104 ad * lock (vdrain_lock).
204 1.63 hannken */
205 1.104 ad u_int numvnodes __cacheline_aligned;
206 1.104 ad static vnodelst_t lru_list[LRU_COUNT] __cacheline_aligned;
207 1.63 hannken static kmutex_t vdrain_lock __cacheline_aligned;
208 1.104 ad static kcondvar_t vdrain_cv;
209 1.63 hannken static int vdrain_gen;
210 1.63 hannken static kcondvar_t vdrain_gen_cv;
211 1.63 hannken static bool vdrain_retry;
212 1.63 hannken static lwp_t * vdrain_lwp;
213 1.57 hannken SLIST_HEAD(hashhead, vnode_impl);
214 1.69 hannken static kmutex_t vcache_lock __cacheline_aligned;
215 1.104 ad static kcondvar_t vcache_cv;
216 1.69 hannken static u_int vcache_hashsize;
217 1.69 hannken static u_long vcache_hashmask;
218 1.104 ad static struct hashhead *vcache_hashtab;
219 1.69 hannken static pool_cache_t vcache_pool;
220 1.63 hannken static void lru_requeue(vnode_t *, vnodelst_t *);
221 1.63 hannken static vnodelst_t * lru_which(vnode_t *);
222 1.63 hannken static vnode_impl_t * vcache_alloc(void);
223 1.79 hannken static void vcache_dealloc(vnode_impl_t *);
224 1.57 hannken static void vcache_free(vnode_impl_t *);
225 1.36 hannken static void vcache_init(void);
226 1.36 hannken static void vcache_reinit(void);
227 1.54 hannken static void vcache_reclaim(vnode_t *);
228 1.107 ad static void vrelel(vnode_t *, int, int);
229 1.12 hannken static void vdrain_thread(void *);
230 1.11 christos static void vnpanic(vnode_t *, const char *, ...)
231 1.18 christos __printflike(2, 3);
232 1.1 rmind
233 1.1 rmind /* Routines having to do with the management of the vnode table. */
234 1.44 hannken extern struct mount *dead_rootmount;
235 1.1 rmind extern int (**dead_vnodeop_p)(void *);
236 1.98 hannken extern int (**spec_vnodeop_p)(void *);
237 1.31 hannken extern struct vfsops dead_vfsops;
238 1.1 rmind
239 1.51 hannken /* Vnode state operations and diagnostics. */
240 1.51 hannken
241 1.51 hannken #if defined(DIAGNOSTIC)
242 1.51 hannken
243 1.94 hannken #define VSTATE_VALID(state) \
244 1.94 hannken ((state) != VS_ACTIVE && (state) != VS_MARKER)
245 1.51 hannken #define VSTATE_GET(vp) \
246 1.51 hannken vstate_assert_get((vp), __func__, __LINE__)
247 1.51 hannken #define VSTATE_CHANGE(vp, from, to) \
248 1.51 hannken vstate_assert_change((vp), (from), (to), __func__, __LINE__)
249 1.51 hannken #define VSTATE_WAIT_STABLE(vp) \
250 1.51 hannken vstate_assert_wait_stable((vp), __func__, __LINE__)
251 1.51 hannken
252 1.94 hannken void
253 1.99 joerg _vstate_assert(vnode_t *vp, enum vnode_state state, const char *func, int line,
254 1.99 joerg bool has_lock)
255 1.51 hannken {
256 1.70 hannken vnode_impl_t *vip = VNODE_TO_VIMPL(vp);
257 1.51 hannken
258 1.99 joerg if (!has_lock) {
259 1.99 joerg /*
260 1.99 joerg * Prevent predictive loads from the CPU, but check the state
261 1.99 joerg * without loooking first.
262 1.99 joerg */
263 1.99 joerg membar_enter();
264 1.99 joerg if (state == VS_ACTIVE && vp->v_usecount > 0 &&
265 1.99 joerg (vip->vi_state == VS_LOADED || vip->vi_state == VS_BLOCKED))
266 1.99 joerg return;
267 1.99 joerg if (vip->vi_state == state)
268 1.99 joerg return;
269 1.99 joerg mutex_enter((vp)->v_interlock);
270 1.99 joerg }
271 1.99 joerg
272 1.51 hannken KASSERTMSG(mutex_owned(vp->v_interlock), "at %s:%d", func, line);
273 1.51 hannken
274 1.99 joerg if ((state == VS_ACTIVE && vp->v_usecount > 0 &&
275 1.99 joerg (vip->vi_state == VS_LOADED || vip->vi_state == VS_BLOCKED)) ||
276 1.99 joerg vip->vi_state == state) {
277 1.99 joerg if (!has_lock)
278 1.99 joerg mutex_exit((vp)->v_interlock);
279 1.94 hannken return;
280 1.99 joerg }
281 1.94 hannken vnpanic(vp, "state is %s, usecount %d, expected %s at %s:%d",
282 1.94 hannken vstate_name(vip->vi_state), vp->v_usecount,
283 1.94 hannken vstate_name(state), func, line);
284 1.51 hannken }
285 1.51 hannken
286 1.57 hannken static enum vnode_state
287 1.51 hannken vstate_assert_get(vnode_t *vp, const char *func, int line)
288 1.51 hannken {
289 1.70 hannken vnode_impl_t *vip = VNODE_TO_VIMPL(vp);
290 1.51 hannken
291 1.51 hannken KASSERTMSG(mutex_owned(vp->v_interlock), "at %s:%d", func, line);
292 1.94 hannken if (! VSTATE_VALID(vip->vi_state))
293 1.51 hannken vnpanic(vp, "state is %s at %s:%d",
294 1.70 hannken vstate_name(vip->vi_state), func, line);
295 1.51 hannken
296 1.70 hannken return vip->vi_state;
297 1.51 hannken }
298 1.51 hannken
299 1.52 hannken static void
300 1.51 hannken vstate_assert_wait_stable(vnode_t *vp, const char *func, int line)
301 1.51 hannken {
302 1.70 hannken vnode_impl_t *vip = VNODE_TO_VIMPL(vp);
303 1.51 hannken
304 1.51 hannken KASSERTMSG(mutex_owned(vp->v_interlock), "at %s:%d", func, line);
305 1.94 hannken if (! VSTATE_VALID(vip->vi_state))
306 1.51 hannken vnpanic(vp, "state is %s at %s:%d",
307 1.70 hannken vstate_name(vip->vi_state), func, line);
308 1.51 hannken
309 1.94 hannken while (vip->vi_state != VS_LOADED && vip->vi_state != VS_RECLAIMED)
310 1.51 hannken cv_wait(&vp->v_cv, vp->v_interlock);
311 1.51 hannken
312 1.94 hannken if (! VSTATE_VALID(vip->vi_state))
313 1.51 hannken vnpanic(vp, "state is %s at %s:%d",
314 1.70 hannken vstate_name(vip->vi_state), func, line);
315 1.51 hannken }
316 1.51 hannken
317 1.52 hannken static void
318 1.57 hannken vstate_assert_change(vnode_t *vp, enum vnode_state from, enum vnode_state to,
319 1.51 hannken const char *func, int line)
320 1.51 hannken {
321 1.70 hannken vnode_impl_t *vip = VNODE_TO_VIMPL(vp);
322 1.51 hannken
323 1.51 hannken KASSERTMSG(mutex_owned(vp->v_interlock), "at %s:%d", func, line);
324 1.57 hannken if (from == VS_LOADING)
325 1.69 hannken KASSERTMSG(mutex_owned(&vcache_lock), "at %s:%d", func, line);
326 1.51 hannken
327 1.94 hannken if (! VSTATE_VALID(from))
328 1.51 hannken vnpanic(vp, "from is %s at %s:%d",
329 1.51 hannken vstate_name(from), func, line);
330 1.94 hannken if (! VSTATE_VALID(to))
331 1.51 hannken vnpanic(vp, "to is %s at %s:%d",
332 1.51 hannken vstate_name(to), func, line);
333 1.70 hannken if (vip->vi_state != from)
334 1.51 hannken vnpanic(vp, "from is %s, expected %s at %s:%d\n",
335 1.70 hannken vstate_name(vip->vi_state), vstate_name(from), func, line);
336 1.68 hannken if ((from == VS_BLOCKED || to == VS_BLOCKED) && vp->v_usecount != 1)
337 1.68 hannken vnpanic(vp, "%s to %s with usecount %d at %s:%d",
338 1.68 hannken vstate_name(from), vstate_name(to), vp->v_usecount,
339 1.68 hannken func, line);
340 1.51 hannken
341 1.70 hannken vip->vi_state = to;
342 1.57 hannken if (from == VS_LOADING)
343 1.69 hannken cv_broadcast(&vcache_cv);
344 1.94 hannken if (to == VS_LOADED || to == VS_RECLAIMED)
345 1.51 hannken cv_broadcast(&vp->v_cv);
346 1.51 hannken }
347 1.51 hannken
348 1.51 hannken #else /* defined(DIAGNOSTIC) */
349 1.51 hannken
350 1.51 hannken #define VSTATE_GET(vp) \
351 1.57 hannken (VNODE_TO_VIMPL((vp))->vi_state)
352 1.51 hannken #define VSTATE_CHANGE(vp, from, to) \
353 1.51 hannken vstate_change((vp), (from), (to))
354 1.51 hannken #define VSTATE_WAIT_STABLE(vp) \
355 1.51 hannken vstate_wait_stable((vp))
356 1.94 hannken void
357 1.100 joerg _vstate_assert(vnode_t *vp, enum vnode_state state, const char *func, int line,
358 1.100 joerg bool has_lock)
359 1.94 hannken {
360 1.94 hannken
361 1.94 hannken }
362 1.51 hannken
363 1.52 hannken static void
364 1.51 hannken vstate_wait_stable(vnode_t *vp)
365 1.51 hannken {
366 1.70 hannken vnode_impl_t *vip = VNODE_TO_VIMPL(vp);
367 1.51 hannken
368 1.94 hannken while (vip->vi_state != VS_LOADED && vip->vi_state != VS_RECLAIMED)
369 1.51 hannken cv_wait(&vp->v_cv, vp->v_interlock);
370 1.51 hannken }
371 1.51 hannken
372 1.52 hannken static void
373 1.57 hannken vstate_change(vnode_t *vp, enum vnode_state from, enum vnode_state to)
374 1.51 hannken {
375 1.70 hannken vnode_impl_t *vip = VNODE_TO_VIMPL(vp);
376 1.51 hannken
377 1.70 hannken vip->vi_state = to;
378 1.57 hannken if (from == VS_LOADING)
379 1.69 hannken cv_broadcast(&vcache_cv);
380 1.94 hannken if (to == VS_LOADED || to == VS_RECLAIMED)
381 1.51 hannken cv_broadcast(&vp->v_cv);
382 1.51 hannken }
383 1.51 hannken
384 1.51 hannken #endif /* defined(DIAGNOSTIC) */
385 1.51 hannken
386 1.1 rmind void
387 1.1 rmind vfs_vnode_sysinit(void)
388 1.1 rmind {
389 1.104 ad int error __diagused, i;
390 1.1 rmind
391 1.44 hannken dead_rootmount = vfs_mountalloc(&dead_vfsops, NULL);
392 1.44 hannken KASSERT(dead_rootmount != NULL);
393 1.103 hannken dead_rootmount->mnt_iflag |= IMNT_MPSAFE;
394 1.31 hannken
395 1.63 hannken mutex_init(&vdrain_lock, MUTEX_DEFAULT, IPL_NONE);
396 1.104 ad for (i = 0; i < LRU_COUNT; i++) {
397 1.104 ad TAILQ_INIT(&lru_list[i]);
398 1.104 ad }
399 1.36 hannken vcache_init();
400 1.36 hannken
401 1.12 hannken cv_init(&vdrain_cv, "vdrain");
402 1.63 hannken cv_init(&vdrain_gen_cv, "vdrainwt");
403 1.12 hannken error = kthread_create(PRI_VM, KTHREAD_MPSAFE, NULL, vdrain_thread,
404 1.63 hannken NULL, &vdrain_lwp, "vdrain");
405 1.47 riastrad KASSERTMSG((error == 0), "kthread_create(vdrain) failed: %d", error);
406 1.1 rmind }
407 1.1 rmind
408 1.1 rmind /*
409 1.48 hannken * Allocate a new marker vnode.
410 1.48 hannken */
411 1.48 hannken vnode_t *
412 1.48 hannken vnalloc_marker(struct mount *mp)
413 1.48 hannken {
414 1.70 hannken vnode_impl_t *vip;
415 1.50 hannken vnode_t *vp;
416 1.50 hannken
417 1.70 hannken vip = pool_cache_get(vcache_pool, PR_WAITOK);
418 1.70 hannken memset(vip, 0, sizeof(*vip));
419 1.70 hannken vp = VIMPL_TO_VNODE(vip);
420 1.111 ad uvm_obj_init(&vp->v_uobj, &uvm_vnodeops, true, 1);
421 1.50 hannken vp->v_mount = mp;
422 1.50 hannken vp->v_type = VBAD;
423 1.111 ad vp->v_interlock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE);
424 1.70 hannken vip->vi_state = VS_MARKER;
425 1.48 hannken
426 1.50 hannken return vp;
427 1.48 hannken }
428 1.48 hannken
429 1.48 hannken /*
430 1.48 hannken * Free a marker vnode.
431 1.48 hannken */
432 1.48 hannken void
433 1.48 hannken vnfree_marker(vnode_t *vp)
434 1.48 hannken {
435 1.70 hannken vnode_impl_t *vip;
436 1.48 hannken
437 1.70 hannken vip = VNODE_TO_VIMPL(vp);
438 1.70 hannken KASSERT(vip->vi_state == VS_MARKER);
439 1.111 ad mutex_obj_free(vp->v_interlock);
440 1.50 hannken uvm_obj_destroy(&vp->v_uobj, true);
441 1.70 hannken pool_cache_put(vcache_pool, vip);
442 1.48 hannken }
443 1.48 hannken
444 1.48 hannken /*
445 1.48 hannken * Test a vnode for being a marker vnode.
446 1.48 hannken */
447 1.48 hannken bool
448 1.48 hannken vnis_marker(vnode_t *vp)
449 1.48 hannken {
450 1.48 hannken
451 1.57 hannken return (VNODE_TO_VIMPL(vp)->vi_state == VS_MARKER);
452 1.48 hannken }
453 1.48 hannken
454 1.48 hannken /*
455 1.63 hannken * Return the lru list this node should be on.
456 1.63 hannken */
457 1.63 hannken static vnodelst_t *
458 1.63 hannken lru_which(vnode_t *vp)
459 1.63 hannken {
460 1.63 hannken
461 1.63 hannken KASSERT(mutex_owned(vp->v_interlock));
462 1.63 hannken
463 1.63 hannken if (vp->v_holdcnt > 0)
464 1.104 ad return &lru_list[LRU_HOLD];
465 1.63 hannken else
466 1.104 ad return &lru_list[LRU_FREE];
467 1.63 hannken }
468 1.63 hannken
469 1.63 hannken /*
470 1.63 hannken * Put vnode to end of given list.
471 1.63 hannken * Both the current and the new list may be NULL, used on vnode alloc/free.
472 1.63 hannken * Adjust numvnodes and signal vdrain thread if there is work.
473 1.63 hannken */
474 1.63 hannken static void
475 1.63 hannken lru_requeue(vnode_t *vp, vnodelst_t *listhd)
476 1.63 hannken {
477 1.70 hannken vnode_impl_t *vip;
478 1.104 ad int d;
479 1.104 ad
480 1.104 ad /*
481 1.104 ad * If the vnode is on the correct list, and was put there recently,
482 1.104 ad * then leave it be, thus avoiding huge cache and lock contention.
483 1.104 ad */
484 1.104 ad vip = VNODE_TO_VIMPL(vp);
485 1.104 ad if (listhd == vip->vi_lrulisthd &&
486 1.104 ad (hardclock_ticks - vip->vi_lrulisttm) < hz) {
487 1.104 ad return;
488 1.104 ad }
489 1.63 hannken
490 1.63 hannken mutex_enter(&vdrain_lock);
491 1.104 ad d = 0;
492 1.70 hannken if (vip->vi_lrulisthd != NULL)
493 1.70 hannken TAILQ_REMOVE(vip->vi_lrulisthd, vip, vi_lrulist);
494 1.63 hannken else
495 1.104 ad d++;
496 1.70 hannken vip->vi_lrulisthd = listhd;
497 1.104 ad vip->vi_lrulisttm = hardclock_ticks;
498 1.70 hannken if (vip->vi_lrulisthd != NULL)
499 1.70 hannken TAILQ_INSERT_TAIL(vip->vi_lrulisthd, vip, vi_lrulist);
500 1.63 hannken else
501 1.104 ad d--;
502 1.104 ad if (d != 0) {
503 1.104 ad /*
504 1.104 ad * Looks strange? This is not a bug. Don't store
505 1.104 ad * numvnodes unless there is a change - avoid false
506 1.104 ad * sharing on MP.
507 1.104 ad */
508 1.104 ad numvnodes += d;
509 1.104 ad }
510 1.104 ad if (numvnodes > desiredvnodes || listhd == &lru_list[LRU_VRELE])
511 1.63 hannken cv_broadcast(&vdrain_cv);
512 1.63 hannken mutex_exit(&vdrain_lock);
513 1.63 hannken }
514 1.63 hannken
515 1.63 hannken /*
516 1.75 hannken * Release deferred vrele vnodes for this mount.
517 1.75 hannken * Called with file system suspended.
518 1.75 hannken */
519 1.75 hannken void
520 1.75 hannken vrele_flush(struct mount *mp)
521 1.75 hannken {
522 1.75 hannken vnode_impl_t *vip, *marker;
523 1.104 ad vnode_t *vp;
524 1.75 hannken
525 1.75 hannken KASSERT(fstrans_is_owner(mp));
526 1.75 hannken
527 1.75 hannken marker = VNODE_TO_VIMPL(vnalloc_marker(NULL));
528 1.75 hannken
529 1.75 hannken mutex_enter(&vdrain_lock);
530 1.104 ad TAILQ_INSERT_HEAD(&lru_list[LRU_VRELE], marker, vi_lrulist);
531 1.75 hannken
532 1.75 hannken while ((vip = TAILQ_NEXT(marker, vi_lrulist))) {
533 1.104 ad TAILQ_REMOVE(&lru_list[LRU_VRELE], marker, vi_lrulist);
534 1.104 ad TAILQ_INSERT_AFTER(&lru_list[LRU_VRELE], vip, marker,
535 1.104 ad vi_lrulist);
536 1.104 ad vp = VIMPL_TO_VNODE(vip);
537 1.104 ad if (vnis_marker(vp))
538 1.75 hannken continue;
539 1.75 hannken
540 1.104 ad KASSERT(vip->vi_lrulisthd == &lru_list[LRU_VRELE]);
541 1.75 hannken TAILQ_REMOVE(vip->vi_lrulisthd, vip, vi_lrulist);
542 1.104 ad vip->vi_lrulisthd = &lru_list[LRU_HOLD];
543 1.104 ad vip->vi_lrulisttm = hardclock_ticks;
544 1.75 hannken TAILQ_INSERT_TAIL(vip->vi_lrulisthd, vip, vi_lrulist);
545 1.75 hannken mutex_exit(&vdrain_lock);
546 1.75 hannken
547 1.107 ad vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
548 1.104 ad mutex_enter(vp->v_interlock);
549 1.107 ad vrelel(vp, 0, LK_EXCLUSIVE);
550 1.75 hannken
551 1.75 hannken mutex_enter(&vdrain_lock);
552 1.75 hannken }
553 1.75 hannken
554 1.104 ad TAILQ_REMOVE(&lru_list[LRU_VRELE], marker, vi_lrulist);
555 1.75 hannken mutex_exit(&vdrain_lock);
556 1.75 hannken
557 1.75 hannken vnfree_marker(VIMPL_TO_VNODE(marker));
558 1.75 hannken }
559 1.75 hannken
560 1.75 hannken /*
561 1.63 hannken * Reclaim a cached vnode. Used from vdrain_thread only.
562 1.1 rmind */
563 1.63 hannken static __inline void
564 1.63 hannken vdrain_remove(vnode_t *vp)
565 1.1 rmind {
566 1.24 hannken struct mount *mp;
567 1.1 rmind
568 1.63 hannken KASSERT(mutex_owned(&vdrain_lock));
569 1.24 hannken
570 1.63 hannken /* Probe usecount (unlocked). */
571 1.63 hannken if (vp->v_usecount > 0)
572 1.63 hannken return;
573 1.63 hannken /* Try v_interlock -- we lock the wrong direction! */
574 1.63 hannken if (!mutex_tryenter(vp->v_interlock))
575 1.63 hannken return;
576 1.63 hannken /* Probe usecount and state. */
577 1.94 hannken if (vp->v_usecount > 0 || VSTATE_GET(vp) != VS_LOADED) {
578 1.63 hannken mutex_exit(vp->v_interlock);
579 1.63 hannken return;
580 1.1 rmind }
581 1.63 hannken mp = vp->v_mount;
582 1.96 hannken if (fstrans_start_nowait(mp) != 0) {
583 1.63 hannken mutex_exit(vp->v_interlock);
584 1.63 hannken return;
585 1.1 rmind }
586 1.63 hannken vdrain_retry = true;
587 1.63 hannken mutex_exit(&vdrain_lock);
588 1.1 rmind
589 1.66 hannken if (vcache_vget(vp) == 0) {
590 1.78 hannken if (!vrecycle(vp)) {
591 1.107 ad vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
592 1.78 hannken mutex_enter(vp->v_interlock);
593 1.107 ad vrelel(vp, 0, LK_EXCLUSIVE);
594 1.78 hannken }
595 1.60 hannken }
596 1.24 hannken fstrans_done(mp);
597 1.12 hannken
598 1.63 hannken mutex_enter(&vdrain_lock);
599 1.1 rmind }
600 1.1 rmind
601 1.1 rmind /*
602 1.63 hannken * Release a cached vnode. Used from vdrain_thread only.
603 1.12 hannken */
604 1.63 hannken static __inline void
605 1.63 hannken vdrain_vrele(vnode_t *vp)
606 1.12 hannken {
607 1.70 hannken vnode_impl_t *vip = VNODE_TO_VIMPL(vp);
608 1.63 hannken struct mount *mp;
609 1.12 hannken
610 1.63 hannken KASSERT(mutex_owned(&vdrain_lock));
611 1.12 hannken
612 1.63 hannken mp = vp->v_mount;
613 1.96 hannken if (fstrans_start_nowait(mp) != 0)
614 1.63 hannken return;
615 1.63 hannken
616 1.64 hannken /*
617 1.64 hannken * First remove the vnode from the vrele list.
618 1.64 hannken * Put it on the last lru list, the last vrele()
619 1.64 hannken * will put it back onto the right list before
620 1.64 hannken * its v_usecount reaches zero.
621 1.64 hannken */
622 1.104 ad KASSERT(vip->vi_lrulisthd == &lru_list[LRU_VRELE]);
623 1.70 hannken TAILQ_REMOVE(vip->vi_lrulisthd, vip, vi_lrulist);
624 1.104 ad vip->vi_lrulisthd = &lru_list[LRU_HOLD];
625 1.104 ad vip->vi_lrulisttm = hardclock_ticks;
626 1.70 hannken TAILQ_INSERT_TAIL(vip->vi_lrulisthd, vip, vi_lrulist);
627 1.63 hannken
628 1.63 hannken vdrain_retry = true;
629 1.63 hannken mutex_exit(&vdrain_lock);
630 1.63 hannken
631 1.107 ad vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
632 1.64 hannken mutex_enter(vp->v_interlock);
633 1.107 ad vrelel(vp, 0, LK_EXCLUSIVE);
634 1.63 hannken fstrans_done(mp);
635 1.63 hannken
636 1.63 hannken mutex_enter(&vdrain_lock);
637 1.12 hannken }
638 1.12 hannken
639 1.12 hannken /*
640 1.63 hannken * Helper thread to keep the number of vnodes below desiredvnodes
641 1.63 hannken * and release vnodes from asynchronous vrele.
642 1.1 rmind */
643 1.63 hannken static void
644 1.63 hannken vdrain_thread(void *cookie)
645 1.1 rmind {
646 1.63 hannken int i;
647 1.63 hannken u_int target;
648 1.70 hannken vnode_impl_t *vip, *marker;
649 1.63 hannken
650 1.63 hannken marker = VNODE_TO_VIMPL(vnalloc_marker(NULL));
651 1.63 hannken
652 1.63 hannken mutex_enter(&vdrain_lock);
653 1.63 hannken
654 1.63 hannken for (;;) {
655 1.63 hannken vdrain_retry = false;
656 1.63 hannken target = desiredvnodes - desiredvnodes/10;
657 1.1 rmind
658 1.104 ad for (i = 0; i < LRU_COUNT; i++) {
659 1.104 ad TAILQ_INSERT_HEAD(&lru_list[i], marker, vi_lrulist);
660 1.70 hannken while ((vip = TAILQ_NEXT(marker, vi_lrulist))) {
661 1.104 ad TAILQ_REMOVE(&lru_list[i], marker, vi_lrulist);
662 1.104 ad TAILQ_INSERT_AFTER(&lru_list[i], vip, marker,
663 1.63 hannken vi_lrulist);
664 1.75 hannken if (vnis_marker(VIMPL_TO_VNODE(vip)))
665 1.75 hannken continue;
666 1.104 ad if (i == LRU_VRELE)
667 1.70 hannken vdrain_vrele(VIMPL_TO_VNODE(vip));
668 1.63 hannken else if (numvnodes < target)
669 1.63 hannken break;
670 1.63 hannken else
671 1.70 hannken vdrain_remove(VIMPL_TO_VNODE(vip));
672 1.63 hannken }
673 1.104 ad TAILQ_REMOVE(&lru_list[i], marker, vi_lrulist);
674 1.63 hannken }
675 1.1 rmind
676 1.63 hannken if (vdrain_retry) {
677 1.63 hannken mutex_exit(&vdrain_lock);
678 1.63 hannken yield();
679 1.63 hannken mutex_enter(&vdrain_lock);
680 1.63 hannken } else {
681 1.63 hannken vdrain_gen++;
682 1.63 hannken cv_broadcast(&vdrain_gen_cv);
683 1.63 hannken cv_wait(&vdrain_cv, &vdrain_lock);
684 1.63 hannken }
685 1.1 rmind }
686 1.1 rmind }
687 1.1 rmind
688 1.1 rmind /*
689 1.112 ad * Try to drop reference on a vnode. Abort if we are releasing the
690 1.112 ad * last reference. Note: this _must_ succeed if not the last reference.
691 1.112 ad */
692 1.112 ad static bool
693 1.112 ad vtryrele(vnode_t *vp)
694 1.112 ad {
695 1.112 ad u_int use, next;
696 1.112 ad
697 1.112 ad for (use = atomic_load_relaxed(&vp->v_usecount);; use = next) {
698 1.112 ad if (__predict_false(use == 1)) {
699 1.112 ad return false;
700 1.112 ad }
701 1.112 ad KASSERT(use > 1);
702 1.112 ad next = atomic_cas_uint(&vp->v_usecount, use, use - 1);
703 1.112 ad if (__predict_true(next == use)) {
704 1.112 ad return true;
705 1.112 ad }
706 1.112 ad }
707 1.112 ad }
708 1.112 ad
709 1.112 ad /*
710 1.4 rmind * vput: unlock and release the reference.
711 1.1 rmind */
712 1.1 rmind void
713 1.1 rmind vput(vnode_t *vp)
714 1.1 rmind {
715 1.107 ad int lktype;
716 1.1 rmind
717 1.112 ad /*
718 1.112 ad * Do an unlocked check of v_usecount. If it looks like we're not
719 1.112 ad * about to drop the last reference, then unlock the vnode and try
720 1.112 ad * to drop the reference. If it ends up being the last reference
721 1.112 ad * after all, vrelel() can fix it all up. Most of the time this
722 1.112 ad * will all go to plan.
723 1.112 ad */
724 1.112 ad if (atomic_load_relaxed(&vp->v_usecount) > 1) {
725 1.112 ad VOP_UNLOCK(vp);
726 1.112 ad if (vtryrele(vp)) {
727 1.112 ad return;
728 1.112 ad }
729 1.112 ad lktype = LK_NONE;
730 1.112 ad } else if ((vp->v_vflag & VV_LOCKSWORK) == 0) {
731 1.107 ad lktype = LK_EXCLUSIVE;
732 1.107 ad } else {
733 1.107 ad lktype = VOP_ISLOCKED(vp);
734 1.107 ad KASSERT(lktype != LK_NONE);
735 1.107 ad }
736 1.107 ad mutex_enter(vp->v_interlock);
737 1.107 ad vrelel(vp, 0, lktype);
738 1.1 rmind }
739 1.1 rmind
740 1.1 rmind /*
741 1.1 rmind * Vnode release. If reference count drops to zero, call inactive
742 1.1 rmind * routine and either return to freelist or free to the pool.
743 1.1 rmind */
744 1.23 hannken static void
745 1.107 ad vrelel(vnode_t *vp, int flags, int lktype)
746 1.1 rmind {
747 1.104 ad const bool async = ((flags & VRELEL_ASYNC) != 0);
748 1.1 rmind bool recycle, defer;
749 1.1 rmind int error;
750 1.1 rmind
751 1.9 rmind KASSERT(mutex_owned(vp->v_interlock));
752 1.1 rmind
753 1.1 rmind if (__predict_false(vp->v_op == dead_vnodeop_p &&
754 1.57 hannken VSTATE_GET(vp) != VS_RECLAIMED)) {
755 1.11 christos vnpanic(vp, "dead but not clean");
756 1.1 rmind }
757 1.1 rmind
758 1.1 rmind /*
759 1.112 ad * If not the last reference, just drop the reference count and
760 1.112 ad * unlock. VOP_UNLOCK() is called here without a vnode reference
761 1.112 ad * held, but is ok as the hold of v_interlock will stop the vnode
762 1.112 ad * from disappearing.
763 1.1 rmind */
764 1.112 ad if (vtryrele(vp)) {
765 1.107 ad if (lktype != LK_NONE) {
766 1.107 ad VOP_UNLOCK(vp);
767 1.107 ad }
768 1.9 rmind mutex_exit(vp->v_interlock);
769 1.1 rmind return;
770 1.1 rmind }
771 1.1 rmind if (vp->v_usecount <= 0 || vp->v_writecount != 0) {
772 1.11 christos vnpanic(vp, "%s: bad ref count", __func__);
773 1.1 rmind }
774 1.1 rmind
775 1.15 hannken #ifdef DIAGNOSTIC
776 1.15 hannken if ((vp->v_type == VBLK || vp->v_type == VCHR) &&
777 1.15 hannken vp->v_specnode != NULL && vp->v_specnode->sn_opencnt != 0) {
778 1.15 hannken vprint("vrelel: missing VOP_CLOSE()", vp);
779 1.15 hannken }
780 1.15 hannken #endif
781 1.15 hannken
782 1.1 rmind /*
783 1.79 hannken * First try to get the vnode locked for VOP_INACTIVE().
784 1.79 hannken * Defer vnode release to vdrain_thread if caller requests
785 1.79 hannken * it explicitly, is the pagedaemon or the lock failed.
786 1.1 rmind */
787 1.107 ad defer = false;
788 1.79 hannken if ((curlwp == uvm.pagedaemon_lwp) || async) {
789 1.79 hannken defer = true;
790 1.107 ad } else if (lktype == LK_SHARED) {
791 1.107 ad /* Excellent chance of getting, if the last ref. */
792 1.107 ad error = vn_lock(vp, LK_UPGRADE | LK_RETRY |
793 1.107 ad LK_NOWAIT);
794 1.107 ad if (error != 0) {
795 1.107 ad defer = true;
796 1.107 ad } else {
797 1.107 ad lktype = LK_EXCLUSIVE;
798 1.107 ad }
799 1.107 ad } else if (lktype == LK_NONE) {
800 1.107 ad /* Excellent chance of getting, if the last ref. */
801 1.107 ad error = vn_lock(vp, LK_EXCLUSIVE | LK_RETRY |
802 1.107 ad LK_NOWAIT);
803 1.107 ad if (error != 0) {
804 1.107 ad defer = true;
805 1.107 ad } else {
806 1.107 ad lktype = LK_EXCLUSIVE;
807 1.107 ad }
808 1.79 hannken }
809 1.79 hannken KASSERT(mutex_owned(vp->v_interlock));
810 1.79 hannken if (defer) {
811 1.1 rmind /*
812 1.79 hannken * Defer reclaim to the kthread; it's not safe to
813 1.79 hannken * clean it here. We donate it our last reference.
814 1.1 rmind */
815 1.107 ad if (lktype != LK_NONE) {
816 1.107 ad VOP_UNLOCK(vp);
817 1.107 ad }
818 1.104 ad lru_requeue(vp, &lru_list[LRU_VRELE]);
819 1.79 hannken mutex_exit(vp->v_interlock);
820 1.79 hannken return;
821 1.79 hannken }
822 1.107 ad KASSERT(lktype == LK_EXCLUSIVE);
823 1.30 hannken
824 1.79 hannken /*
825 1.79 hannken * If not clean, deactivate the vnode, but preserve
826 1.79 hannken * our reference across the call to VOP_INACTIVE().
827 1.79 hannken */
828 1.79 hannken if (VSTATE_GET(vp) == VS_RECLAIMED) {
829 1.79 hannken VOP_UNLOCK(vp);
830 1.79 hannken } else {
831 1.94 hannken VSTATE_CHANGE(vp, VS_LOADED, VS_BLOCKED);
832 1.29 christos mutex_exit(vp->v_interlock);
833 1.29 christos
834 1.1 rmind /*
835 1.52 hannken * The vnode must not gain another reference while being
836 1.1 rmind * deactivated. If VOP_INACTIVE() indicates that
837 1.1 rmind * the described file has been deleted, then recycle
838 1.52 hannken * the vnode.
839 1.1 rmind *
840 1.82 riastrad * Note that VOP_INACTIVE() will not drop the vnode lock.
841 1.1 rmind */
842 1.79 hannken recycle = false;
843 1.1 rmind VOP_INACTIVE(vp, &recycle);
844 1.83 riastrad if (!recycle)
845 1.83 riastrad VOP_UNLOCK(vp);
846 1.113 ad rw_enter(vp->v_uobj.vmobjlock, RW_WRITER);
847 1.9 rmind mutex_enter(vp->v_interlock);
848 1.94 hannken VSTATE_CHANGE(vp, VS_BLOCKED, VS_LOADED);
849 1.1 rmind if (!recycle) {
850 1.112 ad if (vtryrele(vp)) {
851 1.9 rmind mutex_exit(vp->v_interlock);
852 1.113 ad rw_exit(vp->v_uobj.vmobjlock);
853 1.1 rmind return;
854 1.1 rmind }
855 1.1 rmind }
856 1.1 rmind
857 1.113 ad /* Take care of space accounting. */
858 1.104 ad if ((vp->v_iflag & VI_EXECMAP) != 0 &&
859 1.104 ad vp->v_uobj.uo_npages != 0) {
860 1.105 ad cpu_count(CPU_COUNT_EXECPAGES, -vp->v_uobj.uo_npages);
861 1.105 ad cpu_count(CPU_COUNT_FILEPAGES, vp->v_uobj.uo_npages);
862 1.1 rmind }
863 1.1 rmind vp->v_iflag &= ~(VI_TEXT|VI_EXECMAP|VI_WRMAP);
864 1.1 rmind vp->v_vflag &= ~VV_MAPPED;
865 1.113 ad rw_exit(vp->v_uobj.vmobjlock);
866 1.1 rmind
867 1.1 rmind /*
868 1.1 rmind * Recycle the vnode if the file is now unused (unlinked),
869 1.1 rmind * otherwise just free it.
870 1.1 rmind */
871 1.1 rmind if (recycle) {
872 1.94 hannken VSTATE_ASSERT(vp, VS_LOADED);
873 1.83 riastrad /* vcache_reclaim drops the lock. */
874 1.54 hannken vcache_reclaim(vp);
875 1.1 rmind }
876 1.1 rmind KASSERT(vp->v_usecount > 0);
877 1.1 rmind }
878 1.1 rmind
879 1.112 ad if (atomic_dec_uint_nv(&vp->v_usecount) != 0) {
880 1.1 rmind /* Gained another reference while being reclaimed. */
881 1.9 rmind mutex_exit(vp->v_interlock);
882 1.1 rmind return;
883 1.1 rmind }
884 1.1 rmind
885 1.67 hannken if (VSTATE_GET(vp) == VS_RECLAIMED && vp->v_holdcnt == 0) {
886 1.1 rmind /*
887 1.1 rmind * It's clean so destroy it. It isn't referenced
888 1.1 rmind * anywhere since it has been reclaimed.
889 1.1 rmind */
890 1.57 hannken vcache_free(VNODE_TO_VIMPL(vp));
891 1.1 rmind } else {
892 1.1 rmind /*
893 1.1 rmind * Otherwise, put it back onto the freelist. It
894 1.1 rmind * can't be destroyed while still associated with
895 1.1 rmind * a file system.
896 1.1 rmind */
897 1.63 hannken lru_requeue(vp, lru_which(vp));
898 1.9 rmind mutex_exit(vp->v_interlock);
899 1.1 rmind }
900 1.1 rmind }
901 1.1 rmind
902 1.1 rmind void
903 1.1 rmind vrele(vnode_t *vp)
904 1.1 rmind {
905 1.1 rmind
906 1.112 ad if (vtryrele(vp)) {
907 1.112 ad return;
908 1.112 ad }
909 1.9 rmind mutex_enter(vp->v_interlock);
910 1.107 ad vrelel(vp, 0, LK_NONE);
911 1.1 rmind }
912 1.1 rmind
913 1.1 rmind /*
914 1.1 rmind * Asynchronous vnode release, vnode is released in different context.
915 1.1 rmind */
916 1.1 rmind void
917 1.1 rmind vrele_async(vnode_t *vp)
918 1.1 rmind {
919 1.1 rmind
920 1.112 ad if (vtryrele(vp)) {
921 1.112 ad return;
922 1.112 ad }
923 1.9 rmind mutex_enter(vp->v_interlock);
924 1.107 ad vrelel(vp, VRELEL_ASYNC, LK_NONE);
925 1.1 rmind }
926 1.1 rmind
927 1.1 rmind /*
928 1.1 rmind * Vnode reference, where a reference is already held by some other
929 1.1 rmind * object (for example, a file structure).
930 1.112 ad *
931 1.112 ad * NB: we have lockless code sequences that rely on this not blocking.
932 1.1 rmind */
933 1.1 rmind void
934 1.1 rmind vref(vnode_t *vp)
935 1.1 rmind {
936 1.1 rmind
937 1.112 ad KASSERT(atomic_load_relaxed(&vp->v_usecount) != 0);
938 1.1 rmind
939 1.112 ad atomic_inc_uint(&vp->v_usecount);
940 1.1 rmind }
941 1.1 rmind
942 1.1 rmind /*
943 1.1 rmind * Page or buffer structure gets a reference.
944 1.1 rmind * Called with v_interlock held.
945 1.1 rmind */
946 1.1 rmind void
947 1.1 rmind vholdl(vnode_t *vp)
948 1.1 rmind {
949 1.1 rmind
950 1.9 rmind KASSERT(mutex_owned(vp->v_interlock));
951 1.1 rmind
952 1.63 hannken if (vp->v_holdcnt++ == 0 && vp->v_usecount == 0)
953 1.63 hannken lru_requeue(vp, lru_which(vp));
954 1.1 rmind }
955 1.1 rmind
956 1.1 rmind /*
957 1.112 ad * Page or buffer structure gets a reference.
958 1.112 ad */
959 1.112 ad void
960 1.112 ad vhold(vnode_t *vp)
961 1.112 ad {
962 1.112 ad
963 1.112 ad mutex_enter(vp->v_interlock);
964 1.112 ad vholdl(vp);
965 1.112 ad mutex_exit(vp->v_interlock);
966 1.112 ad }
967 1.112 ad
968 1.112 ad /*
969 1.1 rmind * Page or buffer structure frees a reference.
970 1.1 rmind * Called with v_interlock held.
971 1.1 rmind */
972 1.1 rmind void
973 1.1 rmind holdrelel(vnode_t *vp)
974 1.1 rmind {
975 1.1 rmind
976 1.9 rmind KASSERT(mutex_owned(vp->v_interlock));
977 1.1 rmind
978 1.1 rmind if (vp->v_holdcnt <= 0) {
979 1.11 christos vnpanic(vp, "%s: holdcnt vp %p", __func__, vp);
980 1.1 rmind }
981 1.1 rmind
982 1.1 rmind vp->v_holdcnt--;
983 1.63 hannken if (vp->v_holdcnt == 0 && vp->v_usecount == 0)
984 1.63 hannken lru_requeue(vp, lru_which(vp));
985 1.1 rmind }
986 1.1 rmind
987 1.1 rmind /*
988 1.112 ad * Page or buffer structure frees a reference.
989 1.112 ad */
990 1.112 ad void
991 1.112 ad holdrele(vnode_t *vp)
992 1.112 ad {
993 1.112 ad
994 1.112 ad mutex_enter(vp->v_interlock);
995 1.112 ad holdrelel(vp);
996 1.112 ad mutex_exit(vp->v_interlock);
997 1.112 ad }
998 1.112 ad
999 1.112 ad /*
1000 1.33 hannken * Recycle an unused vnode if caller holds the last reference.
1001 1.1 rmind */
1002 1.33 hannken bool
1003 1.33 hannken vrecycle(vnode_t *vp)
1004 1.1 rmind {
1005 1.60 hannken int error __diagused;
1006 1.46 hannken
1007 1.33 hannken mutex_enter(vp->v_interlock);
1008 1.33 hannken
1009 1.60 hannken /* Make sure we hold the last reference. */
1010 1.60 hannken VSTATE_WAIT_STABLE(vp);
1011 1.33 hannken if (vp->v_usecount != 1) {
1012 1.33 hannken mutex_exit(vp->v_interlock);
1013 1.33 hannken return false;
1014 1.1 rmind }
1015 1.60 hannken
1016 1.60 hannken /* If the vnode is already clean we're done. */
1017 1.94 hannken if (VSTATE_GET(vp) != VS_LOADED) {
1018 1.60 hannken VSTATE_ASSERT(vp, VS_RECLAIMED);
1019 1.107 ad vrelel(vp, 0, LK_NONE);
1020 1.60 hannken return true;
1021 1.60 hannken }
1022 1.60 hannken
1023 1.60 hannken /* Prevent further references until the vnode is locked. */
1024 1.94 hannken VSTATE_CHANGE(vp, VS_LOADED, VS_BLOCKED);
1025 1.60 hannken mutex_exit(vp->v_interlock);
1026 1.60 hannken
1027 1.73 hannken /*
1028 1.73 hannken * On a leaf file system this lock will always succeed as we hold
1029 1.73 hannken * the last reference and prevent further references.
1030 1.73 hannken * On layered file systems waiting for the lock would open a can of
1031 1.73 hannken * deadlocks as the lower vnodes may have other active references.
1032 1.73 hannken */
1033 1.76 hannken error = vn_lock(vp, LK_EXCLUSIVE | LK_RETRY | LK_NOWAIT);
1034 1.60 hannken
1035 1.60 hannken mutex_enter(vp->v_interlock);
1036 1.94 hannken VSTATE_CHANGE(vp, VS_BLOCKED, VS_LOADED);
1037 1.60 hannken
1038 1.73 hannken if (error) {
1039 1.73 hannken mutex_exit(vp->v_interlock);
1040 1.73 hannken return false;
1041 1.73 hannken }
1042 1.73 hannken
1043 1.68 hannken KASSERT(vp->v_usecount == 1);
1044 1.54 hannken vcache_reclaim(vp);
1045 1.107 ad vrelel(vp, 0, LK_NONE);
1046 1.60 hannken
1047 1.33 hannken return true;
1048 1.1 rmind }
1049 1.1 rmind
1050 1.1 rmind /*
1051 1.92 hannken * Helper for vrevoke() to propagate suspension from lastmp
1052 1.92 hannken * to thismp. Both args may be NULL.
1053 1.92 hannken * Returns the currently suspended file system or NULL.
1054 1.92 hannken */
1055 1.92 hannken static struct mount *
1056 1.92 hannken vrevoke_suspend_next(struct mount *lastmp, struct mount *thismp)
1057 1.92 hannken {
1058 1.92 hannken int error;
1059 1.92 hannken
1060 1.92 hannken if (lastmp == thismp)
1061 1.92 hannken return thismp;
1062 1.92 hannken
1063 1.92 hannken if (lastmp != NULL)
1064 1.92 hannken vfs_resume(lastmp);
1065 1.92 hannken
1066 1.92 hannken if (thismp == NULL)
1067 1.92 hannken return NULL;
1068 1.92 hannken
1069 1.92 hannken do {
1070 1.92 hannken error = vfs_suspend(thismp, 0);
1071 1.92 hannken } while (error == EINTR || error == ERESTART);
1072 1.92 hannken
1073 1.92 hannken if (error == 0)
1074 1.92 hannken return thismp;
1075 1.92 hannken
1076 1.92 hannken KASSERT(error == EOPNOTSUPP);
1077 1.92 hannken return NULL;
1078 1.92 hannken }
1079 1.92 hannken
1080 1.92 hannken /*
1081 1.1 rmind * Eliminate all activity associated with the requested vnode
1082 1.1 rmind * and with all vnodes aliased to the requested vnode.
1083 1.1 rmind */
1084 1.1 rmind void
1085 1.1 rmind vrevoke(vnode_t *vp)
1086 1.1 rmind {
1087 1.88 hannken struct mount *mp;
1088 1.19 hannken vnode_t *vq;
1089 1.1 rmind enum vtype type;
1090 1.1 rmind dev_t dev;
1091 1.1 rmind
1092 1.1 rmind KASSERT(vp->v_usecount > 0);
1093 1.1 rmind
1094 1.92 hannken mp = vrevoke_suspend_next(NULL, vp->v_mount);
1095 1.88 hannken
1096 1.9 rmind mutex_enter(vp->v_interlock);
1097 1.52 hannken VSTATE_WAIT_STABLE(vp);
1098 1.57 hannken if (VSTATE_GET(vp) == VS_RECLAIMED) {
1099 1.9 rmind mutex_exit(vp->v_interlock);
1100 1.1 rmind } else if (vp->v_type != VBLK && vp->v_type != VCHR) {
1101 1.112 ad atomic_inc_uint(&vp->v_usecount);
1102 1.29 christos mutex_exit(vp->v_interlock);
1103 1.29 christos vgone(vp);
1104 1.1 rmind } else {
1105 1.1 rmind dev = vp->v_rdev;
1106 1.1 rmind type = vp->v_type;
1107 1.9 rmind mutex_exit(vp->v_interlock);
1108 1.1 rmind
1109 1.88 hannken while (spec_node_lookup_by_dev(type, dev, &vq) == 0) {
1110 1.92 hannken mp = vrevoke_suspend_next(mp, vq->v_mount);
1111 1.88 hannken vgone(vq);
1112 1.88 hannken }
1113 1.1 rmind }
1114 1.92 hannken vrevoke_suspend_next(mp, NULL);
1115 1.1 rmind }
1116 1.1 rmind
1117 1.1 rmind /*
1118 1.1 rmind * Eliminate all activity associated with a vnode in preparation for
1119 1.1 rmind * reuse. Drops a reference from the vnode.
1120 1.1 rmind */
1121 1.1 rmind void
1122 1.1 rmind vgone(vnode_t *vp)
1123 1.1 rmind {
1124 1.107 ad int lktype;
1125 1.1 rmind
1126 1.103 hannken KASSERT(vp->v_mount == dead_rootmount || fstrans_is_owner(vp->v_mount));
1127 1.93 hannken
1128 1.76 hannken vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
1129 1.107 ad lktype = LK_EXCLUSIVE;
1130 1.9 rmind mutex_enter(vp->v_interlock);
1131 1.76 hannken VSTATE_WAIT_STABLE(vp);
1132 1.107 ad if (VSTATE_GET(vp) == VS_LOADED) {
1133 1.76 hannken vcache_reclaim(vp);
1134 1.107 ad lktype = LK_NONE;
1135 1.107 ad }
1136 1.76 hannken VSTATE_ASSERT(vp, VS_RECLAIMED);
1137 1.107 ad vrelel(vp, 0, lktype);
1138 1.1 rmind }
1139 1.1 rmind
1140 1.36 hannken static inline uint32_t
1141 1.36 hannken vcache_hash(const struct vcache_key *key)
1142 1.36 hannken {
1143 1.36 hannken uint32_t hash = HASH32_BUF_INIT;
1144 1.36 hannken
1145 1.97 hannken KASSERT(key->vk_key_len > 0);
1146 1.97 hannken
1147 1.36 hannken hash = hash32_buf(&key->vk_mount, sizeof(struct mount *), hash);
1148 1.36 hannken hash = hash32_buf(key->vk_key, key->vk_key_len, hash);
1149 1.36 hannken return hash;
1150 1.36 hannken }
1151 1.36 hannken
1152 1.36 hannken static void
1153 1.36 hannken vcache_init(void)
1154 1.36 hannken {
1155 1.36 hannken
1156 1.112 ad vcache_pool = pool_cache_init(sizeof(vnode_impl_t), coherency_unit,
1157 1.112 ad 0, 0, "vcachepl", NULL, IPL_NONE, NULL, NULL, NULL);
1158 1.69 hannken KASSERT(vcache_pool != NULL);
1159 1.69 hannken mutex_init(&vcache_lock, MUTEX_DEFAULT, IPL_NONE);
1160 1.69 hannken cv_init(&vcache_cv, "vcache");
1161 1.69 hannken vcache_hashsize = desiredvnodes;
1162 1.69 hannken vcache_hashtab = hashinit(desiredvnodes, HASH_SLIST, true,
1163 1.69 hannken &vcache_hashmask);
1164 1.36 hannken }
1165 1.36 hannken
1166 1.36 hannken static void
1167 1.36 hannken vcache_reinit(void)
1168 1.36 hannken {
1169 1.36 hannken int i;
1170 1.36 hannken uint32_t hash;
1171 1.36 hannken u_long oldmask, newmask;
1172 1.36 hannken struct hashhead *oldtab, *newtab;
1173 1.70 hannken vnode_impl_t *vip;
1174 1.36 hannken
1175 1.36 hannken newtab = hashinit(desiredvnodes, HASH_SLIST, true, &newmask);
1176 1.69 hannken mutex_enter(&vcache_lock);
1177 1.69 hannken oldtab = vcache_hashtab;
1178 1.69 hannken oldmask = vcache_hashmask;
1179 1.69 hannken vcache_hashsize = desiredvnodes;
1180 1.69 hannken vcache_hashtab = newtab;
1181 1.69 hannken vcache_hashmask = newmask;
1182 1.36 hannken for (i = 0; i <= oldmask; i++) {
1183 1.70 hannken while ((vip = SLIST_FIRST(&oldtab[i])) != NULL) {
1184 1.70 hannken SLIST_REMOVE(&oldtab[i], vip, vnode_impl, vi_hash);
1185 1.70 hannken hash = vcache_hash(&vip->vi_key);
1186 1.69 hannken SLIST_INSERT_HEAD(&newtab[hash & vcache_hashmask],
1187 1.70 hannken vip, vi_hash);
1188 1.36 hannken }
1189 1.36 hannken }
1190 1.69 hannken mutex_exit(&vcache_lock);
1191 1.36 hannken hashdone(oldtab, HASH_SLIST, oldmask);
1192 1.36 hannken }
1193 1.36 hannken
1194 1.57 hannken static inline vnode_impl_t *
1195 1.36 hannken vcache_hash_lookup(const struct vcache_key *key, uint32_t hash)
1196 1.36 hannken {
1197 1.36 hannken struct hashhead *hashp;
1198 1.70 hannken vnode_impl_t *vip;
1199 1.36 hannken
1200 1.69 hannken KASSERT(mutex_owned(&vcache_lock));
1201 1.36 hannken
1202 1.69 hannken hashp = &vcache_hashtab[hash & vcache_hashmask];
1203 1.70 hannken SLIST_FOREACH(vip, hashp, vi_hash) {
1204 1.70 hannken if (key->vk_mount != vip->vi_key.vk_mount)
1205 1.36 hannken continue;
1206 1.70 hannken if (key->vk_key_len != vip->vi_key.vk_key_len)
1207 1.36 hannken continue;
1208 1.70 hannken if (memcmp(key->vk_key, vip->vi_key.vk_key, key->vk_key_len))
1209 1.36 hannken continue;
1210 1.70 hannken return vip;
1211 1.36 hannken }
1212 1.36 hannken return NULL;
1213 1.36 hannken }
1214 1.36 hannken
1215 1.36 hannken /*
1216 1.50 hannken * Allocate a new, uninitialized vcache node.
1217 1.50 hannken */
1218 1.57 hannken static vnode_impl_t *
1219 1.50 hannken vcache_alloc(void)
1220 1.50 hannken {
1221 1.70 hannken vnode_impl_t *vip;
1222 1.50 hannken vnode_t *vp;
1223 1.50 hannken
1224 1.70 hannken vip = pool_cache_get(vcache_pool, PR_WAITOK);
1225 1.111 ad vp = VIMPL_TO_VNODE(vip);
1226 1.70 hannken memset(vip, 0, sizeof(*vip));
1227 1.50 hannken
1228 1.112 ad rw_init(&vip->vi_lock);
1229 1.111 ad vp->v_interlock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE);
1230 1.111 ad
1231 1.70 hannken /* SLIST_INIT(&vip->vi_hash); */
1232 1.106 ad TAILQ_INIT(&vip->vi_nclist);
1233 1.71 hannken /* LIST_INIT(&vip->vi_dnclist); */
1234 1.50 hannken
1235 1.111 ad uvm_obj_init(&vp->v_uobj, &uvm_vnodeops, true, 1);
1236 1.50 hannken cv_init(&vp->v_cv, "vnode");
1237 1.50 hannken
1238 1.50 hannken vp->v_usecount = 1;
1239 1.50 hannken vp->v_type = VNON;
1240 1.50 hannken vp->v_size = vp->v_writesize = VSIZENOTSET;
1241 1.50 hannken
1242 1.70 hannken vip->vi_state = VS_LOADING;
1243 1.51 hannken
1244 1.104 ad lru_requeue(vp, &lru_list[LRU_FREE]);
1245 1.63 hannken
1246 1.70 hannken return vip;
1247 1.50 hannken }
1248 1.50 hannken
1249 1.50 hannken /*
1250 1.79 hannken * Deallocate a vcache node in state VS_LOADING.
1251 1.79 hannken *
1252 1.79 hannken * vcache_lock held on entry and released on return.
1253 1.79 hannken */
1254 1.79 hannken static void
1255 1.79 hannken vcache_dealloc(vnode_impl_t *vip)
1256 1.79 hannken {
1257 1.79 hannken vnode_t *vp;
1258 1.79 hannken
1259 1.79 hannken KASSERT(mutex_owned(&vcache_lock));
1260 1.79 hannken
1261 1.79 hannken vp = VIMPL_TO_VNODE(vip);
1262 1.102 hannken vfs_ref(dead_rootmount);
1263 1.102 hannken vfs_insmntque(vp, dead_rootmount);
1264 1.79 hannken mutex_enter(vp->v_interlock);
1265 1.79 hannken vp->v_op = dead_vnodeop_p;
1266 1.79 hannken VSTATE_CHANGE(vp, VS_LOADING, VS_RECLAIMED);
1267 1.79 hannken mutex_exit(&vcache_lock);
1268 1.107 ad vrelel(vp, 0, LK_NONE);
1269 1.79 hannken }
1270 1.79 hannken
1271 1.79 hannken /*
1272 1.50 hannken * Free an unused, unreferenced vcache node.
1273 1.67 hannken * v_interlock locked on entry.
1274 1.50 hannken */
1275 1.50 hannken static void
1276 1.70 hannken vcache_free(vnode_impl_t *vip)
1277 1.50 hannken {
1278 1.50 hannken vnode_t *vp;
1279 1.50 hannken
1280 1.70 hannken vp = VIMPL_TO_VNODE(vip);
1281 1.67 hannken KASSERT(mutex_owned(vp->v_interlock));
1282 1.50 hannken
1283 1.50 hannken KASSERT(vp->v_usecount == 0);
1284 1.67 hannken KASSERT(vp->v_holdcnt == 0);
1285 1.67 hannken KASSERT(vp->v_writecount == 0);
1286 1.67 hannken lru_requeue(vp, NULL);
1287 1.67 hannken mutex_exit(vp->v_interlock);
1288 1.67 hannken
1289 1.67 hannken vfs_insmntque(vp, NULL);
1290 1.67 hannken if (vp->v_type == VBLK || vp->v_type == VCHR)
1291 1.67 hannken spec_node_destroy(vp);
1292 1.50 hannken
1293 1.111 ad mutex_obj_free(vp->v_interlock);
1294 1.112 ad rw_destroy(&vip->vi_lock);
1295 1.50 hannken uvm_obj_destroy(&vp->v_uobj, true);
1296 1.50 hannken cv_destroy(&vp->v_cv);
1297 1.70 hannken pool_cache_put(vcache_pool, vip);
1298 1.50 hannken }
1299 1.50 hannken
1300 1.50 hannken /*
1301 1.66 hannken * Try to get an initial reference on this cached vnode.
1302 1.66 hannken * Returns zero on success, ENOENT if the vnode has been reclaimed and
1303 1.66 hannken * EBUSY if the vnode state is unstable.
1304 1.66 hannken *
1305 1.66 hannken * v_interlock locked on entry and unlocked on exit.
1306 1.66 hannken */
1307 1.66 hannken int
1308 1.66 hannken vcache_tryvget(vnode_t *vp)
1309 1.66 hannken {
1310 1.67 hannken int error = 0;
1311 1.66 hannken
1312 1.66 hannken KASSERT(mutex_owned(vp->v_interlock));
1313 1.66 hannken
1314 1.67 hannken if (__predict_false(VSTATE_GET(vp) == VS_RECLAIMED))
1315 1.67 hannken error = ENOENT;
1316 1.94 hannken else if (__predict_false(VSTATE_GET(vp) != VS_LOADED))
1317 1.67 hannken error = EBUSY;
1318 1.112 ad else if (vp->v_usecount == 0)
1319 1.112 ad vp->v_usecount = 1;
1320 1.67 hannken else
1321 1.112 ad atomic_inc_uint(&vp->v_usecount);
1322 1.66 hannken
1323 1.66 hannken mutex_exit(vp->v_interlock);
1324 1.66 hannken
1325 1.67 hannken return error;
1326 1.66 hannken }
1327 1.66 hannken
1328 1.66 hannken /*
1329 1.66 hannken * Try to get an initial reference on this cached vnode.
1330 1.66 hannken * Returns zero on success and ENOENT if the vnode has been reclaimed.
1331 1.66 hannken * Will wait for the vnode state to be stable.
1332 1.66 hannken *
1333 1.66 hannken * v_interlock locked on entry and unlocked on exit.
1334 1.66 hannken */
1335 1.66 hannken int
1336 1.66 hannken vcache_vget(vnode_t *vp)
1337 1.66 hannken {
1338 1.66 hannken
1339 1.66 hannken KASSERT(mutex_owned(vp->v_interlock));
1340 1.66 hannken
1341 1.67 hannken /* Increment hold count to prevent vnode from disappearing. */
1342 1.67 hannken vp->v_holdcnt++;
1343 1.67 hannken VSTATE_WAIT_STABLE(vp);
1344 1.67 hannken vp->v_holdcnt--;
1345 1.66 hannken
1346 1.67 hannken /* If this was the last reference to a reclaimed vnode free it now. */
1347 1.67 hannken if (__predict_false(VSTATE_GET(vp) == VS_RECLAIMED)) {
1348 1.67 hannken if (vp->v_holdcnt == 0 && vp->v_usecount == 0)
1349 1.67 hannken vcache_free(VNODE_TO_VIMPL(vp));
1350 1.67 hannken else
1351 1.67 hannken mutex_exit(vp->v_interlock);
1352 1.66 hannken return ENOENT;
1353 1.66 hannken }
1354 1.94 hannken VSTATE_ASSERT(vp, VS_LOADED);
1355 1.112 ad if (vp->v_usecount == 0)
1356 1.112 ad vp->v_usecount = 1;
1357 1.112 ad else
1358 1.112 ad atomic_inc_uint(&vp->v_usecount);
1359 1.66 hannken mutex_exit(vp->v_interlock);
1360 1.66 hannken
1361 1.66 hannken return 0;
1362 1.66 hannken }
1363 1.66 hannken
1364 1.66 hannken /*
1365 1.36 hannken * Get a vnode / fs node pair by key and return it referenced through vpp.
1366 1.36 hannken */
1367 1.36 hannken int
1368 1.36 hannken vcache_get(struct mount *mp, const void *key, size_t key_len,
1369 1.36 hannken struct vnode **vpp)
1370 1.36 hannken {
1371 1.36 hannken int error;
1372 1.36 hannken uint32_t hash;
1373 1.36 hannken const void *new_key;
1374 1.36 hannken struct vnode *vp;
1375 1.36 hannken struct vcache_key vcache_key;
1376 1.70 hannken vnode_impl_t *vip, *new_vip;
1377 1.36 hannken
1378 1.36 hannken new_key = NULL;
1379 1.36 hannken *vpp = NULL;
1380 1.36 hannken
1381 1.36 hannken vcache_key.vk_mount = mp;
1382 1.36 hannken vcache_key.vk_key = key;
1383 1.36 hannken vcache_key.vk_key_len = key_len;
1384 1.36 hannken hash = vcache_hash(&vcache_key);
1385 1.36 hannken
1386 1.36 hannken again:
1387 1.69 hannken mutex_enter(&vcache_lock);
1388 1.70 hannken vip = vcache_hash_lookup(&vcache_key, hash);
1389 1.36 hannken
1390 1.36 hannken /* If found, take a reference or retry. */
1391 1.70 hannken if (__predict_true(vip != NULL)) {
1392 1.52 hannken /*
1393 1.52 hannken * If the vnode is loading we cannot take the v_interlock
1394 1.52 hannken * here as it might change during load (see uvm_obj_setlock()).
1395 1.69 hannken * As changing state from VS_LOADING requires both vcache_lock
1396 1.69 hannken * and v_interlock it is safe to test with vcache_lock held.
1397 1.52 hannken *
1398 1.57 hannken * Wait for vnodes changing state from VS_LOADING and retry.
1399 1.52 hannken */
1400 1.70 hannken if (__predict_false(vip->vi_state == VS_LOADING)) {
1401 1.69 hannken cv_wait(&vcache_cv, &vcache_lock);
1402 1.69 hannken mutex_exit(&vcache_lock);
1403 1.52 hannken goto again;
1404 1.52 hannken }
1405 1.70 hannken vp = VIMPL_TO_VNODE(vip);
1406 1.36 hannken mutex_enter(vp->v_interlock);
1407 1.69 hannken mutex_exit(&vcache_lock);
1408 1.66 hannken error = vcache_vget(vp);
1409 1.36 hannken if (error == ENOENT)
1410 1.36 hannken goto again;
1411 1.36 hannken if (error == 0)
1412 1.36 hannken *vpp = vp;
1413 1.36 hannken KASSERT((error != 0) == (*vpp == NULL));
1414 1.36 hannken return error;
1415 1.36 hannken }
1416 1.69 hannken mutex_exit(&vcache_lock);
1417 1.36 hannken
1418 1.36 hannken /* Allocate and initialize a new vcache / vnode pair. */
1419 1.87 hannken error = vfs_busy(mp);
1420 1.36 hannken if (error)
1421 1.36 hannken return error;
1422 1.70 hannken new_vip = vcache_alloc();
1423 1.70 hannken new_vip->vi_key = vcache_key;
1424 1.70 hannken vp = VIMPL_TO_VNODE(new_vip);
1425 1.69 hannken mutex_enter(&vcache_lock);
1426 1.70 hannken vip = vcache_hash_lookup(&vcache_key, hash);
1427 1.70 hannken if (vip == NULL) {
1428 1.69 hannken SLIST_INSERT_HEAD(&vcache_hashtab[hash & vcache_hashmask],
1429 1.70 hannken new_vip, vi_hash);
1430 1.70 hannken vip = new_vip;
1431 1.36 hannken }
1432 1.36 hannken
1433 1.36 hannken /* If another thread beat us inserting this node, retry. */
1434 1.70 hannken if (vip != new_vip) {
1435 1.79 hannken vcache_dealloc(new_vip);
1436 1.87 hannken vfs_unbusy(mp);
1437 1.36 hannken goto again;
1438 1.36 hannken }
1439 1.69 hannken mutex_exit(&vcache_lock);
1440 1.36 hannken
1441 1.57 hannken /* Load the fs node. Exclusive as new_node is VS_LOADING. */
1442 1.36 hannken error = VFS_LOADVNODE(mp, vp, key, key_len, &new_key);
1443 1.36 hannken if (error) {
1444 1.69 hannken mutex_enter(&vcache_lock);
1445 1.69 hannken SLIST_REMOVE(&vcache_hashtab[hash & vcache_hashmask],
1446 1.70 hannken new_vip, vnode_impl, vi_hash);
1447 1.79 hannken vcache_dealloc(new_vip);
1448 1.87 hannken vfs_unbusy(mp);
1449 1.36 hannken KASSERT(*vpp == NULL);
1450 1.36 hannken return error;
1451 1.36 hannken }
1452 1.36 hannken KASSERT(new_key != NULL);
1453 1.36 hannken KASSERT(memcmp(key, new_key, key_len) == 0);
1454 1.36 hannken KASSERT(vp->v_op != NULL);
1455 1.36 hannken vfs_insmntque(vp, mp);
1456 1.36 hannken if ((mp->mnt_iflag & IMNT_MPSAFE) != 0)
1457 1.36 hannken vp->v_vflag |= VV_MPSAFE;
1458 1.87 hannken vfs_ref(mp);
1459 1.87 hannken vfs_unbusy(mp);
1460 1.36 hannken
1461 1.36 hannken /* Finished loading, finalize node. */
1462 1.69 hannken mutex_enter(&vcache_lock);
1463 1.70 hannken new_vip->vi_key.vk_key = new_key;
1464 1.39 hannken mutex_enter(vp->v_interlock);
1465 1.94 hannken VSTATE_CHANGE(vp, VS_LOADING, VS_LOADED);
1466 1.39 hannken mutex_exit(vp->v_interlock);
1467 1.69 hannken mutex_exit(&vcache_lock);
1468 1.36 hannken *vpp = vp;
1469 1.36 hannken return 0;
1470 1.36 hannken }
1471 1.36 hannken
1472 1.36 hannken /*
1473 1.40 hannken * Create a new vnode / fs node pair and return it referenced through vpp.
1474 1.40 hannken */
1475 1.40 hannken int
1476 1.40 hannken vcache_new(struct mount *mp, struct vnode *dvp, struct vattr *vap,
1477 1.101 hannken kauth_cred_t cred, void *extra, struct vnode **vpp)
1478 1.40 hannken {
1479 1.40 hannken int error;
1480 1.40 hannken uint32_t hash;
1481 1.70 hannken struct vnode *vp, *ovp;
1482 1.70 hannken vnode_impl_t *vip, *ovip;
1483 1.40 hannken
1484 1.40 hannken *vpp = NULL;
1485 1.40 hannken
1486 1.40 hannken /* Allocate and initialize a new vcache / vnode pair. */
1487 1.87 hannken error = vfs_busy(mp);
1488 1.40 hannken if (error)
1489 1.40 hannken return error;
1490 1.70 hannken vip = vcache_alloc();
1491 1.70 hannken vip->vi_key.vk_mount = mp;
1492 1.70 hannken vp = VIMPL_TO_VNODE(vip);
1493 1.40 hannken
1494 1.40 hannken /* Create and load the fs node. */
1495 1.101 hannken error = VFS_NEWVNODE(mp, dvp, vp, vap, cred, extra,
1496 1.70 hannken &vip->vi_key.vk_key_len, &vip->vi_key.vk_key);
1497 1.40 hannken if (error) {
1498 1.69 hannken mutex_enter(&vcache_lock);
1499 1.79 hannken vcache_dealloc(vip);
1500 1.87 hannken vfs_unbusy(mp);
1501 1.40 hannken KASSERT(*vpp == NULL);
1502 1.40 hannken return error;
1503 1.40 hannken }
1504 1.40 hannken KASSERT(vp->v_op != NULL);
1505 1.97 hannken KASSERT((vip->vi_key.vk_key_len == 0) == (mp == dead_rootmount));
1506 1.97 hannken if (vip->vi_key.vk_key_len > 0) {
1507 1.97 hannken KASSERT(vip->vi_key.vk_key != NULL);
1508 1.97 hannken hash = vcache_hash(&vip->vi_key);
1509 1.40 hannken
1510 1.97 hannken /*
1511 1.97 hannken * Wait for previous instance to be reclaimed,
1512 1.97 hannken * then insert new node.
1513 1.97 hannken */
1514 1.97 hannken mutex_enter(&vcache_lock);
1515 1.97 hannken while ((ovip = vcache_hash_lookup(&vip->vi_key, hash))) {
1516 1.97 hannken ovp = VIMPL_TO_VNODE(ovip);
1517 1.97 hannken mutex_enter(ovp->v_interlock);
1518 1.97 hannken mutex_exit(&vcache_lock);
1519 1.97 hannken error = vcache_vget(ovp);
1520 1.97 hannken KASSERT(error == ENOENT);
1521 1.97 hannken mutex_enter(&vcache_lock);
1522 1.97 hannken }
1523 1.97 hannken SLIST_INSERT_HEAD(&vcache_hashtab[hash & vcache_hashmask],
1524 1.97 hannken vip, vi_hash);
1525 1.69 hannken mutex_exit(&vcache_lock);
1526 1.40 hannken }
1527 1.40 hannken vfs_insmntque(vp, mp);
1528 1.40 hannken if ((mp->mnt_iflag & IMNT_MPSAFE) != 0)
1529 1.40 hannken vp->v_vflag |= VV_MPSAFE;
1530 1.87 hannken vfs_ref(mp);
1531 1.87 hannken vfs_unbusy(mp);
1532 1.40 hannken
1533 1.40 hannken /* Finished loading, finalize node. */
1534 1.69 hannken mutex_enter(&vcache_lock);
1535 1.52 hannken mutex_enter(vp->v_interlock);
1536 1.94 hannken VSTATE_CHANGE(vp, VS_LOADING, VS_LOADED);
1537 1.69 hannken mutex_exit(&vcache_lock);
1538 1.40 hannken mutex_exit(vp->v_interlock);
1539 1.40 hannken *vpp = vp;
1540 1.40 hannken return 0;
1541 1.40 hannken }
1542 1.40 hannken
1543 1.40 hannken /*
1544 1.65 hannken * Prepare key change: update old cache nodes key and lock new cache node.
1545 1.37 hannken * Return an error if the new node already exists.
1546 1.37 hannken */
1547 1.37 hannken int
1548 1.37 hannken vcache_rekey_enter(struct mount *mp, struct vnode *vp,
1549 1.37 hannken const void *old_key, size_t old_key_len,
1550 1.37 hannken const void *new_key, size_t new_key_len)
1551 1.37 hannken {
1552 1.37 hannken uint32_t old_hash, new_hash;
1553 1.37 hannken struct vcache_key old_vcache_key, new_vcache_key;
1554 1.70 hannken vnode_impl_t *vip, *new_vip;
1555 1.37 hannken
1556 1.37 hannken old_vcache_key.vk_mount = mp;
1557 1.37 hannken old_vcache_key.vk_key = old_key;
1558 1.37 hannken old_vcache_key.vk_key_len = old_key_len;
1559 1.37 hannken old_hash = vcache_hash(&old_vcache_key);
1560 1.37 hannken
1561 1.37 hannken new_vcache_key.vk_mount = mp;
1562 1.37 hannken new_vcache_key.vk_key = new_key;
1563 1.37 hannken new_vcache_key.vk_key_len = new_key_len;
1564 1.37 hannken new_hash = vcache_hash(&new_vcache_key);
1565 1.37 hannken
1566 1.70 hannken new_vip = vcache_alloc();
1567 1.70 hannken new_vip->vi_key = new_vcache_key;
1568 1.37 hannken
1569 1.52 hannken /* Insert locked new node used as placeholder. */
1570 1.69 hannken mutex_enter(&vcache_lock);
1571 1.70 hannken vip = vcache_hash_lookup(&new_vcache_key, new_hash);
1572 1.70 hannken if (vip != NULL) {
1573 1.79 hannken vcache_dealloc(new_vip);
1574 1.37 hannken return EEXIST;
1575 1.37 hannken }
1576 1.69 hannken SLIST_INSERT_HEAD(&vcache_hashtab[new_hash & vcache_hashmask],
1577 1.70 hannken new_vip, vi_hash);
1578 1.49 hannken
1579 1.65 hannken /* Replace old nodes key with the temporary copy. */
1580 1.70 hannken vip = vcache_hash_lookup(&old_vcache_key, old_hash);
1581 1.70 hannken KASSERT(vip != NULL);
1582 1.70 hannken KASSERT(VIMPL_TO_VNODE(vip) == vp);
1583 1.70 hannken KASSERT(vip->vi_key.vk_key != old_vcache_key.vk_key);
1584 1.70 hannken vip->vi_key = old_vcache_key;
1585 1.69 hannken mutex_exit(&vcache_lock);
1586 1.37 hannken return 0;
1587 1.37 hannken }
1588 1.37 hannken
1589 1.37 hannken /*
1590 1.65 hannken * Key change complete: update old node and remove placeholder.
1591 1.37 hannken */
1592 1.37 hannken void
1593 1.37 hannken vcache_rekey_exit(struct mount *mp, struct vnode *vp,
1594 1.37 hannken const void *old_key, size_t old_key_len,
1595 1.37 hannken const void *new_key, size_t new_key_len)
1596 1.37 hannken {
1597 1.37 hannken uint32_t old_hash, new_hash;
1598 1.37 hannken struct vcache_key old_vcache_key, new_vcache_key;
1599 1.70 hannken vnode_impl_t *vip, *new_vip;
1600 1.70 hannken struct vnode *new_vp;
1601 1.37 hannken
1602 1.37 hannken old_vcache_key.vk_mount = mp;
1603 1.37 hannken old_vcache_key.vk_key = old_key;
1604 1.37 hannken old_vcache_key.vk_key_len = old_key_len;
1605 1.37 hannken old_hash = vcache_hash(&old_vcache_key);
1606 1.37 hannken
1607 1.37 hannken new_vcache_key.vk_mount = mp;
1608 1.37 hannken new_vcache_key.vk_key = new_key;
1609 1.37 hannken new_vcache_key.vk_key_len = new_key_len;
1610 1.37 hannken new_hash = vcache_hash(&new_vcache_key);
1611 1.37 hannken
1612 1.69 hannken mutex_enter(&vcache_lock);
1613 1.49 hannken
1614 1.49 hannken /* Lookup old and new node. */
1615 1.70 hannken vip = vcache_hash_lookup(&old_vcache_key, old_hash);
1616 1.70 hannken KASSERT(vip != NULL);
1617 1.70 hannken KASSERT(VIMPL_TO_VNODE(vip) == vp);
1618 1.70 hannken
1619 1.70 hannken new_vip = vcache_hash_lookup(&new_vcache_key, new_hash);
1620 1.70 hannken KASSERT(new_vip != NULL);
1621 1.70 hannken KASSERT(new_vip->vi_key.vk_key_len == new_key_len);
1622 1.70 hannken new_vp = VIMPL_TO_VNODE(new_vip);
1623 1.70 hannken mutex_enter(new_vp->v_interlock);
1624 1.70 hannken VSTATE_ASSERT(VIMPL_TO_VNODE(new_vip), VS_LOADING);
1625 1.79 hannken mutex_exit(new_vp->v_interlock);
1626 1.49 hannken
1627 1.49 hannken /* Rekey old node and put it onto its new hashlist. */
1628 1.70 hannken vip->vi_key = new_vcache_key;
1629 1.49 hannken if (old_hash != new_hash) {
1630 1.69 hannken SLIST_REMOVE(&vcache_hashtab[old_hash & vcache_hashmask],
1631 1.70 hannken vip, vnode_impl, vi_hash);
1632 1.69 hannken SLIST_INSERT_HEAD(&vcache_hashtab[new_hash & vcache_hashmask],
1633 1.70 hannken vip, vi_hash);
1634 1.49 hannken }
1635 1.49 hannken
1636 1.49 hannken /* Remove new node used as placeholder. */
1637 1.69 hannken SLIST_REMOVE(&vcache_hashtab[new_hash & vcache_hashmask],
1638 1.70 hannken new_vip, vnode_impl, vi_hash);
1639 1.79 hannken vcache_dealloc(new_vip);
1640 1.37 hannken }
1641 1.37 hannken
1642 1.37 hannken /*
1643 1.54 hannken * Disassociate the underlying file system from a vnode.
1644 1.54 hannken *
1645 1.54 hannken * Must be called with vnode locked and will return unlocked.
1646 1.54 hannken * Must be called with the interlock held, and will return with it held.
1647 1.54 hannken */
1648 1.54 hannken static void
1649 1.54 hannken vcache_reclaim(vnode_t *vp)
1650 1.54 hannken {
1651 1.54 hannken lwp_t *l = curlwp;
1652 1.70 hannken vnode_impl_t *vip = VNODE_TO_VIMPL(vp);
1653 1.74 hannken struct mount *mp = vp->v_mount;
1654 1.55 hannken uint32_t hash;
1655 1.55 hannken uint8_t temp_buf[64], *temp_key;
1656 1.55 hannken size_t temp_key_len;
1657 1.54 hannken bool recycle, active;
1658 1.54 hannken int error;
1659 1.54 hannken
1660 1.54 hannken KASSERT((vp->v_vflag & VV_LOCKSWORK) == 0 ||
1661 1.54 hannken VOP_ISLOCKED(vp) == LK_EXCLUSIVE);
1662 1.54 hannken KASSERT(mutex_owned(vp->v_interlock));
1663 1.54 hannken KASSERT(vp->v_usecount != 0);
1664 1.54 hannken
1665 1.54 hannken active = (vp->v_usecount > 1);
1666 1.70 hannken temp_key_len = vip->vi_key.vk_key_len;
1667 1.54 hannken /*
1668 1.54 hannken * Prevent the vnode from being recycled or brought into use
1669 1.54 hannken * while we clean it out.
1670 1.54 hannken */
1671 1.94 hannken VSTATE_CHANGE(vp, VS_LOADED, VS_RECLAIMING);
1672 1.111 ad mutex_exit(vp->v_interlock);
1673 1.111 ad
1674 1.111 ad rw_enter(vp->v_uobj.vmobjlock, RW_WRITER);
1675 1.111 ad mutex_enter(vp->v_interlock);
1676 1.104 ad if ((vp->v_iflag & VI_EXECMAP) != 0 && vp->v_uobj.uo_npages != 0) {
1677 1.105 ad cpu_count(CPU_COUNT_EXECPAGES, -vp->v_uobj.uo_npages);
1678 1.105 ad cpu_count(CPU_COUNT_FILEPAGES, vp->v_uobj.uo_npages);
1679 1.54 hannken }
1680 1.54 hannken vp->v_iflag &= ~(VI_TEXT|VI_EXECMAP);
1681 1.54 hannken mutex_exit(vp->v_interlock);
1682 1.111 ad rw_exit(vp->v_uobj.vmobjlock);
1683 1.54 hannken
1684 1.55 hannken /* Replace the vnode key with a temporary copy. */
1685 1.70 hannken if (vip->vi_key.vk_key_len > sizeof(temp_buf)) {
1686 1.55 hannken temp_key = kmem_alloc(temp_key_len, KM_SLEEP);
1687 1.55 hannken } else {
1688 1.55 hannken temp_key = temp_buf;
1689 1.55 hannken }
1690 1.97 hannken if (vip->vi_key.vk_key_len > 0) {
1691 1.97 hannken mutex_enter(&vcache_lock);
1692 1.97 hannken memcpy(temp_key, vip->vi_key.vk_key, temp_key_len);
1693 1.97 hannken vip->vi_key.vk_key = temp_key;
1694 1.97 hannken mutex_exit(&vcache_lock);
1695 1.97 hannken }
1696 1.55 hannken
1697 1.96 hannken fstrans_start(mp);
1698 1.74 hannken
1699 1.54 hannken /*
1700 1.54 hannken * Clean out any cached data associated with the vnode.
1701 1.54 hannken * If purging an active vnode, it must be closed and
1702 1.60 hannken * deactivated before being reclaimed.
1703 1.54 hannken */
1704 1.54 hannken error = vinvalbuf(vp, V_SAVE, NOCRED, l, 0, 0);
1705 1.54 hannken if (error != 0) {
1706 1.54 hannken if (wapbl_vphaswapbl(vp))
1707 1.54 hannken WAPBL_DISCARD(wapbl_vptomp(vp));
1708 1.54 hannken error = vinvalbuf(vp, 0, NOCRED, l, 0, 0);
1709 1.54 hannken }
1710 1.54 hannken KASSERTMSG((error == 0), "vinvalbuf failed: %d", error);
1711 1.54 hannken KASSERT((vp->v_iflag & VI_ONWORKLST) == 0);
1712 1.54 hannken if (active && (vp->v_type == VBLK || vp->v_type == VCHR)) {
1713 1.54 hannken spec_node_revoke(vp);
1714 1.54 hannken }
1715 1.54 hannken
1716 1.60 hannken /*
1717 1.60 hannken * Disassociate the underlying file system from the vnode.
1718 1.90 riastrad * VOP_INACTIVE leaves the vnode locked; VOP_RECLAIM unlocks
1719 1.90 riastrad * the vnode, and may destroy the vnode so that VOP_UNLOCK
1720 1.90 riastrad * would no longer function.
1721 1.60 hannken */
1722 1.60 hannken VOP_INACTIVE(vp, &recycle);
1723 1.91 riastrad KASSERT((vp->v_vflag & VV_LOCKSWORK) == 0 ||
1724 1.91 riastrad VOP_ISLOCKED(vp) == LK_EXCLUSIVE);
1725 1.54 hannken if (VOP_RECLAIM(vp)) {
1726 1.54 hannken vnpanic(vp, "%s: cannot reclaim", __func__);
1727 1.54 hannken }
1728 1.54 hannken
1729 1.54 hannken KASSERT(vp->v_data == NULL);
1730 1.113 ad KASSERT((vp->v_iflag & VI_PAGES) == 0);
1731 1.54 hannken
1732 1.54 hannken if (vp->v_type == VREG && vp->v_ractx != NULL) {
1733 1.54 hannken uvm_ra_freectx(vp->v_ractx);
1734 1.54 hannken vp->v_ractx = NULL;
1735 1.54 hannken }
1736 1.54 hannken
1737 1.54 hannken /* Purge name cache. */
1738 1.54 hannken cache_purge(vp);
1739 1.54 hannken
1740 1.97 hannken if (vip->vi_key.vk_key_len > 0) {
1741 1.55 hannken /* Remove from vnode cache. */
1742 1.97 hannken hash = vcache_hash(&vip->vi_key);
1743 1.97 hannken mutex_enter(&vcache_lock);
1744 1.97 hannken KASSERT(vip == vcache_hash_lookup(&vip->vi_key, hash));
1745 1.97 hannken SLIST_REMOVE(&vcache_hashtab[hash & vcache_hashmask],
1746 1.97 hannken vip, vnode_impl, vi_hash);
1747 1.97 hannken mutex_exit(&vcache_lock);
1748 1.97 hannken }
1749 1.55 hannken if (temp_key != temp_buf)
1750 1.55 hannken kmem_free(temp_key, temp_key_len);
1751 1.55 hannken
1752 1.54 hannken /* Done with purge, notify sleepers of the grim news. */
1753 1.54 hannken mutex_enter(vp->v_interlock);
1754 1.54 hannken vp->v_op = dead_vnodeop_p;
1755 1.54 hannken vp->v_vflag |= VV_LOCKSWORK;
1756 1.57 hannken VSTATE_CHANGE(vp, VS_RECLAIMING, VS_RECLAIMED);
1757 1.54 hannken vp->v_tag = VT_NON;
1758 1.54 hannken KNOTE(&vp->v_klist, NOTE_REVOKE);
1759 1.80 hannken mutex_exit(vp->v_interlock);
1760 1.54 hannken
1761 1.80 hannken /*
1762 1.80 hannken * Move to dead mount. Must be after changing the operations
1763 1.80 hannken * vector as vnode operations enter the mount before using the
1764 1.80 hannken * operations vector. See sys/kern/vnode_if.c.
1765 1.80 hannken */
1766 1.80 hannken vp->v_vflag &= ~VV_ROOT;
1767 1.86 hannken vfs_ref(dead_rootmount);
1768 1.80 hannken vfs_insmntque(vp, dead_rootmount);
1769 1.80 hannken
1770 1.110 ad #ifdef PAX_SEGVGUARD
1771 1.110 ad pax_segvguard_cleanup(vp);
1772 1.110 ad #endif /* PAX_SEGVGUARD */
1773 1.110 ad
1774 1.80 hannken mutex_enter(vp->v_interlock);
1775 1.74 hannken fstrans_done(mp);
1776 1.54 hannken KASSERT((vp->v_iflag & VI_ONWORKLST) == 0);
1777 1.54 hannken }
1778 1.54 hannken
1779 1.54 hannken /*
1780 1.98 hannken * Disassociate the underlying file system from an open device vnode
1781 1.98 hannken * and make it anonymous.
1782 1.98 hannken *
1783 1.98 hannken * Vnode unlocked on entry, drops a reference to the vnode.
1784 1.98 hannken */
1785 1.98 hannken void
1786 1.98 hannken vcache_make_anon(vnode_t *vp)
1787 1.98 hannken {
1788 1.98 hannken vnode_impl_t *vip = VNODE_TO_VIMPL(vp);
1789 1.98 hannken uint32_t hash;
1790 1.98 hannken bool recycle;
1791 1.98 hannken
1792 1.98 hannken KASSERT(vp->v_type == VBLK || vp->v_type == VCHR);
1793 1.103 hannken KASSERT(vp->v_mount == dead_rootmount || fstrans_is_owner(vp->v_mount));
1794 1.98 hannken VSTATE_ASSERT_UNLOCKED(vp, VS_ACTIVE);
1795 1.98 hannken
1796 1.98 hannken /* Remove from vnode cache. */
1797 1.98 hannken hash = vcache_hash(&vip->vi_key);
1798 1.98 hannken mutex_enter(&vcache_lock);
1799 1.98 hannken KASSERT(vip == vcache_hash_lookup(&vip->vi_key, hash));
1800 1.98 hannken SLIST_REMOVE(&vcache_hashtab[hash & vcache_hashmask],
1801 1.98 hannken vip, vnode_impl, vi_hash);
1802 1.98 hannken vip->vi_key.vk_mount = dead_rootmount;
1803 1.98 hannken vip->vi_key.vk_key_len = 0;
1804 1.98 hannken vip->vi_key.vk_key = NULL;
1805 1.98 hannken mutex_exit(&vcache_lock);
1806 1.98 hannken
1807 1.98 hannken /*
1808 1.98 hannken * Disassociate the underlying file system from the vnode.
1809 1.98 hannken * VOP_INACTIVE leaves the vnode locked; VOP_RECLAIM unlocks
1810 1.98 hannken * the vnode, and may destroy the vnode so that VOP_UNLOCK
1811 1.98 hannken * would no longer function.
1812 1.98 hannken */
1813 1.98 hannken if (vn_lock(vp, LK_EXCLUSIVE)) {
1814 1.98 hannken vnpanic(vp, "%s: cannot lock", __func__);
1815 1.98 hannken }
1816 1.98 hannken VOP_INACTIVE(vp, &recycle);
1817 1.98 hannken KASSERT((vp->v_vflag & VV_LOCKSWORK) == 0 ||
1818 1.98 hannken VOP_ISLOCKED(vp) == LK_EXCLUSIVE);
1819 1.98 hannken if (VOP_RECLAIM(vp)) {
1820 1.98 hannken vnpanic(vp, "%s: cannot reclaim", __func__);
1821 1.98 hannken }
1822 1.98 hannken
1823 1.98 hannken /* Purge name cache. */
1824 1.98 hannken cache_purge(vp);
1825 1.98 hannken
1826 1.98 hannken /* Done with purge, change operations vector. */
1827 1.98 hannken mutex_enter(vp->v_interlock);
1828 1.98 hannken vp->v_op = spec_vnodeop_p;
1829 1.98 hannken vp->v_vflag |= VV_MPSAFE;
1830 1.98 hannken vp->v_vflag &= ~VV_LOCKSWORK;
1831 1.98 hannken mutex_exit(vp->v_interlock);
1832 1.98 hannken
1833 1.98 hannken /*
1834 1.98 hannken * Move to dead mount. Must be after changing the operations
1835 1.98 hannken * vector as vnode operations enter the mount before using the
1836 1.98 hannken * operations vector. See sys/kern/vnode_if.c.
1837 1.98 hannken */
1838 1.98 hannken vfs_ref(dead_rootmount);
1839 1.98 hannken vfs_insmntque(vp, dead_rootmount);
1840 1.98 hannken
1841 1.98 hannken vrele(vp);
1842 1.98 hannken }
1843 1.98 hannken
1844 1.98 hannken /*
1845 1.1 rmind * Update outstanding I/O count and do wakeup if requested.
1846 1.1 rmind */
1847 1.1 rmind void
1848 1.1 rmind vwakeup(struct buf *bp)
1849 1.1 rmind {
1850 1.1 rmind vnode_t *vp;
1851 1.1 rmind
1852 1.1 rmind if ((vp = bp->b_vp) == NULL)
1853 1.1 rmind return;
1854 1.1 rmind
1855 1.9 rmind KASSERT(bp->b_objlock == vp->v_interlock);
1856 1.1 rmind KASSERT(mutex_owned(bp->b_objlock));
1857 1.1 rmind
1858 1.1 rmind if (--vp->v_numoutput < 0)
1859 1.11 christos vnpanic(vp, "%s: neg numoutput, vp %p", __func__, vp);
1860 1.1 rmind if (vp->v_numoutput == 0)
1861 1.1 rmind cv_broadcast(&vp->v_cv);
1862 1.1 rmind }
1863 1.1 rmind
1864 1.1 rmind /*
1865 1.35 hannken * Test a vnode for being or becoming dead. Returns one of:
1866 1.35 hannken * EBUSY: vnode is becoming dead, with "flags == VDEAD_NOWAIT" only.
1867 1.35 hannken * ENOENT: vnode is dead.
1868 1.35 hannken * 0: otherwise.
1869 1.35 hannken *
1870 1.35 hannken * Whenever this function returns a non-zero value all future
1871 1.35 hannken * calls will also return a non-zero value.
1872 1.35 hannken */
1873 1.35 hannken int
1874 1.35 hannken vdead_check(struct vnode *vp, int flags)
1875 1.35 hannken {
1876 1.35 hannken
1877 1.35 hannken KASSERT(mutex_owned(vp->v_interlock));
1878 1.35 hannken
1879 1.52 hannken if (! ISSET(flags, VDEAD_NOWAIT))
1880 1.52 hannken VSTATE_WAIT_STABLE(vp);
1881 1.1 rmind
1882 1.57 hannken if (VSTATE_GET(vp) == VS_RECLAIMING) {
1883 1.52 hannken KASSERT(ISSET(flags, VDEAD_NOWAIT));
1884 1.52 hannken return EBUSY;
1885 1.57 hannken } else if (VSTATE_GET(vp) == VS_RECLAIMED) {
1886 1.52 hannken return ENOENT;
1887 1.52 hannken }
1888 1.1 rmind
1889 1.52 hannken return 0;
1890 1.1 rmind }
1891 1.1 rmind
1892 1.1 rmind int
1893 1.61 hannken vfs_drainvnodes(void)
1894 1.1 rmind {
1895 1.63 hannken int i, gen;
1896 1.61 hannken
1897 1.63 hannken mutex_enter(&vdrain_lock);
1898 1.63 hannken for (i = 0; i < 2; i++) {
1899 1.63 hannken gen = vdrain_gen;
1900 1.63 hannken while (gen == vdrain_gen) {
1901 1.63 hannken cv_broadcast(&vdrain_cv);
1902 1.63 hannken cv_wait(&vdrain_gen_cv, &vdrain_lock);
1903 1.63 hannken }
1904 1.61 hannken }
1905 1.63 hannken mutex_exit(&vdrain_lock);
1906 1.12 hannken
1907 1.63 hannken if (numvnodes >= desiredvnodes)
1908 1.63 hannken return EBUSY;
1909 1.12 hannken
1910 1.69 hannken if (vcache_hashsize != desiredvnodes)
1911 1.61 hannken vcache_reinit();
1912 1.36 hannken
1913 1.1 rmind return 0;
1914 1.1 rmind }
1915 1.1 rmind
1916 1.1 rmind void
1917 1.11 christos vnpanic(vnode_t *vp, const char *fmt, ...)
1918 1.1 rmind {
1919 1.11 christos va_list ap;
1920 1.11 christos
1921 1.1 rmind #ifdef DIAGNOSTIC
1922 1.1 rmind vprint(NULL, vp);
1923 1.1 rmind #endif
1924 1.11 christos va_start(ap, fmt);
1925 1.11 christos vpanic(fmt, ap);
1926 1.11 christos va_end(ap);
1927 1.1 rmind }
1928 1.111 ad
1929 1.111 ad void
1930 1.111 ad vshareilock(vnode_t *tvp, vnode_t *fvp)
1931 1.111 ad {
1932 1.111 ad kmutex_t *oldlock;
1933 1.111 ad
1934 1.111 ad oldlock = tvp->v_interlock;
1935 1.111 ad mutex_obj_hold(fvp->v_interlock);
1936 1.111 ad tvp->v_interlock = fvp->v_interlock;
1937 1.111 ad mutex_obj_free(oldlock);
1938 1.111 ad }
1939