vfs_vnode.c revision 1.105.2.3 1 /* $NetBSD: vfs_vnode.c,v 1.105.2.3 2020/01/17 21:55:13 ad Exp $ */
2
3 /*-
4 * Copyright (c) 1997-2011, 2019 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
9 * NASA Ames Research Center, by Charles M. Hannum, and by Andrew Doran.
10 *
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
13 * are met:
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 * POSSIBILITY OF SUCH DAMAGE.
31 */
32
33 /*
34 * Copyright (c) 1989, 1993
35 * The Regents of the University of California. All rights reserved.
36 * (c) UNIX System Laboratories, Inc.
37 * All or some portions of this file are derived from material licensed
38 * to the University of California by American Telephone and Telegraph
39 * Co. or Unix System Laboratories, Inc. and are reproduced herein with
40 * the permission of UNIX System Laboratories, Inc.
41 *
42 * Redistribution and use in source and binary forms, with or without
43 * modification, are permitted provided that the following conditions
44 * are met:
45 * 1. Redistributions of source code must retain the above copyright
46 * notice, this list of conditions and the following disclaimer.
47 * 2. Redistributions in binary form must reproduce the above copyright
48 * notice, this list of conditions and the following disclaimer in the
49 * documentation and/or other materials provided with the distribution.
50 * 3. Neither the name of the University nor the names of its contributors
51 * may be used to endorse or promote products derived from this software
52 * without specific prior written permission.
53 *
54 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
55 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
56 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
57 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
58 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
59 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
60 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
61 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
62 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
63 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
64 * SUCH DAMAGE.
65 *
66 * @(#)vfs_subr.c 8.13 (Berkeley) 4/18/94
67 */
68
69 /*
70 * The vnode cache subsystem.
71 *
72 * Life-cycle
73 *
74 * Normally, there are two points where new vnodes are created:
75 * VOP_CREATE(9) and VOP_LOOKUP(9). The life-cycle of a vnode
76 * starts in one of the following ways:
77 *
78 * - Allocation, via vcache_get(9) or vcache_new(9).
79 * - Reclamation of inactive vnode, via vcache_vget(9).
80 *
81 * Recycle from a free list, via getnewvnode(9) -> getcleanvnode(9)
82 * was another, traditional way. Currently, only the draining thread
83 * recycles the vnodes. This behaviour might be revisited.
84 *
85 * The life-cycle ends when the last reference is dropped, usually
86 * in VOP_REMOVE(9). In such case, VOP_INACTIVE(9) is called to inform
87 * the file system that vnode is inactive. Via this call, file system
88 * indicates whether vnode can be recycled (usually, it checks its own
89 * references, e.g. count of links, whether the file was removed).
90 *
91 * Depending on indication, vnode can be put into a free list (cache),
92 * or cleaned via vcache_reclaim, which calls VOP_RECLAIM(9) to
93 * disassociate underlying file system from the vnode, and finally
94 * destroyed.
95 *
96 * Vnode state
97 *
98 * Vnode is always in one of six states:
99 * - MARKER This is a marker vnode to help list traversal. It
100 * will never change its state.
101 * - LOADING Vnode is associating underlying file system and not
102 * yet ready to use.
103 * - LOADED Vnode has associated underlying file system and is
104 * ready to use.
105 * - BLOCKED Vnode is active but cannot get new references.
106 * - RECLAIMING Vnode is disassociating from the underlying file
107 * system.
108 * - RECLAIMED Vnode has disassociated from underlying file system
109 * and is dead.
110 *
111 * Valid state changes are:
112 * LOADING -> LOADED
113 * Vnode has been initialised in vcache_get() or
114 * vcache_new() and is ready to use.
115 * LOADED -> RECLAIMING
116 * Vnode starts disassociation from underlying file
117 * system in vcache_reclaim().
118 * RECLAIMING -> RECLAIMED
119 * Vnode finished disassociation from underlying file
120 * system in vcache_reclaim().
121 * LOADED -> BLOCKED
122 * vcache_rekey*() is changing the vnode key.
123 * BLOCKED -> LOADED
124 * The block condition is over.
125 * LOADING -> RECLAIMED
126 * Either vcache_get() or vcache_new() failed to
127 * associate the underlying file system or vcache_rekey*()
128 * drops a vnode used as placeholder.
129 *
130 * Of these states LOADING, BLOCKED and RECLAIMING are intermediate
131 * and it is possible to wait for state change.
132 *
133 * State is protected with v_interlock with one exception:
134 * to change from LOADING both v_interlock and vcache_lock must be held
135 * so it is possible to check "state == LOADING" without holding
136 * v_interlock. See vcache_get() for details.
137 *
138 * Reference counting
139 *
140 * Vnode is considered active, if reference count (vnode_t::v_usecount)
141 * is non-zero. It is maintained using: vref(9) and vrele(9), as well
142 * as vput(9), routines. Common points holding references are e.g.
143 * file openings, current working directory, mount points, etc.
144 *
145 */
146
147 #include <sys/cdefs.h>
148 __KERNEL_RCSID(0, "$NetBSD: vfs_vnode.c,v 1.105.2.3 2020/01/17 21:55:13 ad Exp $");
149
150 #include <sys/param.h>
151 #include <sys/kernel.h>
152
153 #include <sys/atomic.h>
154 #include <sys/buf.h>
155 #include <sys/conf.h>
156 #include <sys/device.h>
157 #include <sys/hash.h>
158 #include <sys/kauth.h>
159 #include <sys/kmem.h>
160 #include <sys/kthread.h>
161 #include <sys/module.h>
162 #include <sys/mount.h>
163 #include <sys/namei.h>
164 #include <sys/syscallargs.h>
165 #include <sys/sysctl.h>
166 #include <sys/systm.h>
167 #include <sys/vnode_impl.h>
168 #include <sys/wapbl.h>
169 #include <sys/fstrans.h>
170
171 #include <uvm/uvm.h>
172 #include <uvm/uvm_readahead.h>
173 #include <uvm/uvm_stat.h>
174
175 /* Flags to vrelel. */
176 #define VRELEL_ASYNC 0x0001 /* Always defer to vrele thread. */
177
178 #define LRU_VRELE 0
179 #define LRU_FREE 1
180 #define LRU_HOLD 2
181 #define LRU_COUNT 3
182
183 /*
184 * There are three lru lists: one holds vnodes waiting for async release,
185 * one is for vnodes which have no buffer/page references and one for those
186 * which do (i.e. v_holdcnt is non-zero). We put the lists into a single,
187 * private cache line as vnodes migrate between them while under the same
188 * lock (vdrain_lock).
189 */
190 u_int numvnodes __cacheline_aligned;
191 static vnodelst_t lru_list[LRU_COUNT] __cacheline_aligned;
192 static kmutex_t vdrain_lock __cacheline_aligned;
193 static kcondvar_t vdrain_cv;
194 static int vdrain_gen;
195 static kcondvar_t vdrain_gen_cv;
196 static bool vdrain_retry;
197 static lwp_t * vdrain_lwp;
198 SLIST_HEAD(hashhead, vnode_impl);
199 static kmutex_t vcache_lock __cacheline_aligned;
200 static kcondvar_t vcache_cv;
201 static u_int vcache_hashsize;
202 static u_long vcache_hashmask;
203 static struct hashhead *vcache_hashtab;
204 static pool_cache_t vcache_pool;
205 static void lru_requeue(vnode_t *, vnodelst_t *);
206 static vnodelst_t * lru_which(vnode_t *);
207 static vnode_impl_t * vcache_alloc(void);
208 static void vcache_dealloc(vnode_impl_t *);
209 static void vcache_free(vnode_impl_t *);
210 static void vcache_init(void);
211 static void vcache_reinit(void);
212 static void vcache_reclaim(vnode_t *);
213 static void vrelel(vnode_t *, int, int);
214 static void vdrain_thread(void *);
215 static void vnpanic(vnode_t *, const char *, ...)
216 __printflike(2, 3);
217
218 /* Routines having to do with the management of the vnode table. */
219 extern struct mount *dead_rootmount;
220 extern int (**dead_vnodeop_p)(void *);
221 extern int (**spec_vnodeop_p)(void *);
222 extern struct vfsops dead_vfsops;
223
224 /* Vnode state operations and diagnostics. */
225
226 #if defined(DIAGNOSTIC)
227
228 #define VSTATE_VALID(state) \
229 ((state) != VS_ACTIVE && (state) != VS_MARKER)
230 #define VSTATE_GET(vp) \
231 vstate_assert_get((vp), __func__, __LINE__)
232 #define VSTATE_CHANGE(vp, from, to) \
233 vstate_assert_change((vp), (from), (to), __func__, __LINE__)
234 #define VSTATE_WAIT_STABLE(vp) \
235 vstate_assert_wait_stable((vp), __func__, __LINE__)
236
237 void
238 _vstate_assert(vnode_t *vp, enum vnode_state state, const char *func, int line,
239 bool has_lock)
240 {
241 vnode_impl_t *vip = VNODE_TO_VIMPL(vp);
242
243 if (!has_lock) {
244 /*
245 * Prevent predictive loads from the CPU, but check the state
246 * without loooking first.
247 */
248 membar_enter();
249 if (state == VS_ACTIVE && vp->v_usecount > 0 &&
250 (vip->vi_state == VS_LOADED || vip->vi_state == VS_BLOCKED))
251 return;
252 if (vip->vi_state == state)
253 return;
254 mutex_enter((vp)->v_interlock);
255 }
256
257 KASSERTMSG(mutex_owned(vp->v_interlock), "at %s:%d", func, line);
258
259 if ((state == VS_ACTIVE && vp->v_usecount > 0 &&
260 (vip->vi_state == VS_LOADED || vip->vi_state == VS_BLOCKED)) ||
261 vip->vi_state == state) {
262 if (!has_lock)
263 mutex_exit((vp)->v_interlock);
264 return;
265 }
266 vnpanic(vp, "state is %s, usecount %d, expected %s at %s:%d",
267 vstate_name(vip->vi_state), vp->v_usecount,
268 vstate_name(state), func, line);
269 }
270
271 static enum vnode_state
272 vstate_assert_get(vnode_t *vp, const char *func, int line)
273 {
274 vnode_impl_t *vip = VNODE_TO_VIMPL(vp);
275
276 KASSERTMSG(mutex_owned(vp->v_interlock), "at %s:%d", func, line);
277 if (! VSTATE_VALID(vip->vi_state))
278 vnpanic(vp, "state is %s at %s:%d",
279 vstate_name(vip->vi_state), func, line);
280
281 return vip->vi_state;
282 }
283
284 static void
285 vstate_assert_wait_stable(vnode_t *vp, const char *func, int line)
286 {
287 vnode_impl_t *vip = VNODE_TO_VIMPL(vp);
288
289 KASSERTMSG(mutex_owned(vp->v_interlock), "at %s:%d", func, line);
290 if (! VSTATE_VALID(vip->vi_state))
291 vnpanic(vp, "state is %s at %s:%d",
292 vstate_name(vip->vi_state), func, line);
293
294 while (vip->vi_state != VS_LOADED && vip->vi_state != VS_RECLAIMED)
295 cv_wait(&vp->v_cv, vp->v_interlock);
296
297 if (! VSTATE_VALID(vip->vi_state))
298 vnpanic(vp, "state is %s at %s:%d",
299 vstate_name(vip->vi_state), func, line);
300 }
301
302 static void
303 vstate_assert_change(vnode_t *vp, enum vnode_state from, enum vnode_state to,
304 const char *func, int line)
305 {
306 vnode_impl_t *vip = VNODE_TO_VIMPL(vp);
307
308 KASSERTMSG(mutex_owned(vp->v_interlock), "at %s:%d", func, line);
309 if (from == VS_LOADING)
310 KASSERTMSG(mutex_owned(&vcache_lock), "at %s:%d", func, line);
311
312 if (! VSTATE_VALID(from))
313 vnpanic(vp, "from is %s at %s:%d",
314 vstate_name(from), func, line);
315 if (! VSTATE_VALID(to))
316 vnpanic(vp, "to is %s at %s:%d",
317 vstate_name(to), func, line);
318 if (vip->vi_state != from)
319 vnpanic(vp, "from is %s, expected %s at %s:%d\n",
320 vstate_name(vip->vi_state), vstate_name(from), func, line);
321 if ((from == VS_BLOCKED || to == VS_BLOCKED) && vp->v_usecount != 1)
322 vnpanic(vp, "%s to %s with usecount %d at %s:%d",
323 vstate_name(from), vstate_name(to), vp->v_usecount,
324 func, line);
325
326 vip->vi_state = to;
327 if (from == VS_LOADING)
328 cv_broadcast(&vcache_cv);
329 if (to == VS_LOADED || to == VS_RECLAIMED)
330 cv_broadcast(&vp->v_cv);
331 }
332
333 #else /* defined(DIAGNOSTIC) */
334
335 #define VSTATE_GET(vp) \
336 (VNODE_TO_VIMPL((vp))->vi_state)
337 #define VSTATE_CHANGE(vp, from, to) \
338 vstate_change((vp), (from), (to))
339 #define VSTATE_WAIT_STABLE(vp) \
340 vstate_wait_stable((vp))
341 void
342 _vstate_assert(vnode_t *vp, enum vnode_state state, const char *func, int line,
343 bool has_lock)
344 {
345
346 }
347
348 static void
349 vstate_wait_stable(vnode_t *vp)
350 {
351 vnode_impl_t *vip = VNODE_TO_VIMPL(vp);
352
353 while (vip->vi_state != VS_LOADED && vip->vi_state != VS_RECLAIMED)
354 cv_wait(&vp->v_cv, vp->v_interlock);
355 }
356
357 static void
358 vstate_change(vnode_t *vp, enum vnode_state from, enum vnode_state to)
359 {
360 vnode_impl_t *vip = VNODE_TO_VIMPL(vp);
361
362 vip->vi_state = to;
363 if (from == VS_LOADING)
364 cv_broadcast(&vcache_cv);
365 if (to == VS_LOADED || to == VS_RECLAIMED)
366 cv_broadcast(&vp->v_cv);
367 }
368
369 #endif /* defined(DIAGNOSTIC) */
370
371 void
372 vfs_vnode_sysinit(void)
373 {
374 int error __diagused, i;
375
376 dead_rootmount = vfs_mountalloc(&dead_vfsops, NULL);
377 KASSERT(dead_rootmount != NULL);
378 dead_rootmount->mnt_iflag |= IMNT_MPSAFE;
379
380 mutex_init(&vdrain_lock, MUTEX_DEFAULT, IPL_NONE);
381 for (i = 0; i < LRU_COUNT; i++) {
382 TAILQ_INIT(&lru_list[i]);
383 }
384 vcache_init();
385
386 cv_init(&vdrain_cv, "vdrain");
387 cv_init(&vdrain_gen_cv, "vdrainwt");
388 error = kthread_create(PRI_VM, KTHREAD_MPSAFE, NULL, vdrain_thread,
389 NULL, &vdrain_lwp, "vdrain");
390 KASSERTMSG((error == 0), "kthread_create(vdrain) failed: %d", error);
391 }
392
393 /*
394 * Allocate a new marker vnode.
395 */
396 vnode_t *
397 vnalloc_marker(struct mount *mp)
398 {
399 vnode_impl_t *vip;
400 vnode_t *vp;
401
402 vip = pool_cache_get(vcache_pool, PR_WAITOK);
403 memset(vip, 0, sizeof(*vip));
404 vp = VIMPL_TO_VNODE(vip);
405 uvm_obj_init(&vp->v_uobj, &uvm_vnodeops, true, 0);
406 vp->v_mount = mp;
407 vp->v_type = VBAD;
408 vip->vi_state = VS_MARKER;
409
410 return vp;
411 }
412
413 /*
414 * Free a marker vnode.
415 */
416 void
417 vnfree_marker(vnode_t *vp)
418 {
419 vnode_impl_t *vip;
420
421 vip = VNODE_TO_VIMPL(vp);
422 KASSERT(vip->vi_state == VS_MARKER);
423 uvm_obj_destroy(&vp->v_uobj, true);
424 pool_cache_put(vcache_pool, vip);
425 }
426
427 /*
428 * Test a vnode for being a marker vnode.
429 */
430 bool
431 vnis_marker(vnode_t *vp)
432 {
433
434 return (VNODE_TO_VIMPL(vp)->vi_state == VS_MARKER);
435 }
436
437 /*
438 * Return the lru list this node should be on.
439 */
440 static vnodelst_t *
441 lru_which(vnode_t *vp)
442 {
443
444 KASSERT(mutex_owned(vp->v_interlock));
445
446 if (vp->v_holdcnt > 0)
447 return &lru_list[LRU_HOLD];
448 else
449 return &lru_list[LRU_FREE];
450 }
451
452 /*
453 * Put vnode to end of given list.
454 * Both the current and the new list may be NULL, used on vnode alloc/free.
455 * Adjust numvnodes and signal vdrain thread if there is work.
456 */
457 static void
458 lru_requeue(vnode_t *vp, vnodelst_t *listhd)
459 {
460 vnode_impl_t *vip;
461 int d;
462
463 /*
464 * If the vnode is on the correct list, and was put there recently,
465 * then leave it be, thus avoiding huge cache and lock contention.
466 */
467 vip = VNODE_TO_VIMPL(vp);
468 if (listhd == vip->vi_lrulisthd &&
469 (hardclock_ticks - vip->vi_lrulisttm) < hz) {
470 return;
471 }
472
473 mutex_enter(&vdrain_lock);
474 d = 0;
475 if (vip->vi_lrulisthd != NULL)
476 TAILQ_REMOVE(vip->vi_lrulisthd, vip, vi_lrulist);
477 else
478 d++;
479 vip->vi_lrulisthd = listhd;
480 vip->vi_lrulisttm = hardclock_ticks;
481 if (vip->vi_lrulisthd != NULL)
482 TAILQ_INSERT_TAIL(vip->vi_lrulisthd, vip, vi_lrulist);
483 else
484 d--;
485 if (d != 0) {
486 /*
487 * Looks strange? This is not a bug. Don't store
488 * numvnodes unless there is a change - avoid false
489 * sharing on MP.
490 */
491 numvnodes += d;
492 }
493 if (numvnodes > desiredvnodes || listhd == &lru_list[LRU_VRELE])
494 cv_broadcast(&vdrain_cv);
495 mutex_exit(&vdrain_lock);
496 }
497
498 /*
499 * Release deferred vrele vnodes for this mount.
500 * Called with file system suspended.
501 */
502 void
503 vrele_flush(struct mount *mp)
504 {
505 vnode_impl_t *vip, *marker;
506 vnode_t *vp;
507
508 KASSERT(fstrans_is_owner(mp));
509
510 marker = VNODE_TO_VIMPL(vnalloc_marker(NULL));
511
512 mutex_enter(&vdrain_lock);
513 TAILQ_INSERT_HEAD(&lru_list[LRU_VRELE], marker, vi_lrulist);
514
515 while ((vip = TAILQ_NEXT(marker, vi_lrulist))) {
516 TAILQ_REMOVE(&lru_list[LRU_VRELE], marker, vi_lrulist);
517 TAILQ_INSERT_AFTER(&lru_list[LRU_VRELE], vip, marker,
518 vi_lrulist);
519 vp = VIMPL_TO_VNODE(vip);
520 if (vnis_marker(vp))
521 continue;
522
523 KASSERT(vip->vi_lrulisthd == &lru_list[LRU_VRELE]);
524 TAILQ_REMOVE(vip->vi_lrulisthd, vip, vi_lrulist);
525 vip->vi_lrulisthd = &lru_list[LRU_HOLD];
526 vip->vi_lrulisttm = hardclock_ticks;
527 TAILQ_INSERT_TAIL(vip->vi_lrulisthd, vip, vi_lrulist);
528 mutex_exit(&vdrain_lock);
529
530 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
531 mutex_enter(vp->v_interlock);
532 vrelel(vp, 0, LK_EXCLUSIVE);
533
534 mutex_enter(&vdrain_lock);
535 }
536
537 TAILQ_REMOVE(&lru_list[LRU_VRELE], marker, vi_lrulist);
538 mutex_exit(&vdrain_lock);
539
540 vnfree_marker(VIMPL_TO_VNODE(marker));
541 }
542
543 /*
544 * Reclaim a cached vnode. Used from vdrain_thread only.
545 */
546 static __inline void
547 vdrain_remove(vnode_t *vp)
548 {
549 struct mount *mp;
550
551 KASSERT(mutex_owned(&vdrain_lock));
552
553 /* Probe usecount (unlocked). */
554 if (vp->v_usecount > 0)
555 return;
556 /* Try v_interlock -- we lock the wrong direction! */
557 if (!mutex_tryenter(vp->v_interlock))
558 return;
559 /* Probe usecount and state. */
560 if (vp->v_usecount > 0 || VSTATE_GET(vp) != VS_LOADED) {
561 mutex_exit(vp->v_interlock);
562 return;
563 }
564 mp = vp->v_mount;
565 if (fstrans_start_nowait(mp) != 0) {
566 mutex_exit(vp->v_interlock);
567 return;
568 }
569 vdrain_retry = true;
570 mutex_exit(&vdrain_lock);
571
572 if (vcache_vget(vp) == 0) {
573 if (!vrecycle(vp)) {
574 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
575 mutex_enter(vp->v_interlock);
576 vrelel(vp, 0, LK_EXCLUSIVE);
577 }
578 }
579 fstrans_done(mp);
580
581 mutex_enter(&vdrain_lock);
582 }
583
584 /*
585 * Release a cached vnode. Used from vdrain_thread only.
586 */
587 static __inline void
588 vdrain_vrele(vnode_t *vp)
589 {
590 vnode_impl_t *vip = VNODE_TO_VIMPL(vp);
591 struct mount *mp;
592
593 KASSERT(mutex_owned(&vdrain_lock));
594
595 mp = vp->v_mount;
596 if (fstrans_start_nowait(mp) != 0)
597 return;
598
599 /*
600 * First remove the vnode from the vrele list.
601 * Put it on the last lru list, the last vrele()
602 * will put it back onto the right list before
603 * its v_usecount reaches zero.
604 */
605 KASSERT(vip->vi_lrulisthd == &lru_list[LRU_VRELE]);
606 TAILQ_REMOVE(vip->vi_lrulisthd, vip, vi_lrulist);
607 vip->vi_lrulisthd = &lru_list[LRU_HOLD];
608 vip->vi_lrulisttm = hardclock_ticks;
609 TAILQ_INSERT_TAIL(vip->vi_lrulisthd, vip, vi_lrulist);
610
611 vdrain_retry = true;
612 mutex_exit(&vdrain_lock);
613
614 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
615 mutex_enter(vp->v_interlock);
616 vrelel(vp, 0, LK_EXCLUSIVE);
617 fstrans_done(mp);
618
619 mutex_enter(&vdrain_lock);
620 }
621
622 /*
623 * Helper thread to keep the number of vnodes below desiredvnodes
624 * and release vnodes from asynchronous vrele.
625 */
626 static void
627 vdrain_thread(void *cookie)
628 {
629 int i;
630 u_int target;
631 vnode_impl_t *vip, *marker;
632
633 marker = VNODE_TO_VIMPL(vnalloc_marker(NULL));
634
635 mutex_enter(&vdrain_lock);
636
637 for (;;) {
638 vdrain_retry = false;
639 target = desiredvnodes - desiredvnodes/10;
640
641 for (i = 0; i < LRU_COUNT; i++) {
642 TAILQ_INSERT_HEAD(&lru_list[i], marker, vi_lrulist);
643 while ((vip = TAILQ_NEXT(marker, vi_lrulist))) {
644 TAILQ_REMOVE(&lru_list[i], marker, vi_lrulist);
645 TAILQ_INSERT_AFTER(&lru_list[i], vip, marker,
646 vi_lrulist);
647 if (vnis_marker(VIMPL_TO_VNODE(vip)))
648 continue;
649 if (i == LRU_VRELE)
650 vdrain_vrele(VIMPL_TO_VNODE(vip));
651 else if (numvnodes < target)
652 break;
653 else
654 vdrain_remove(VIMPL_TO_VNODE(vip));
655 }
656 TAILQ_REMOVE(&lru_list[i], marker, vi_lrulist);
657 }
658
659 if (vdrain_retry) {
660 mutex_exit(&vdrain_lock);
661 yield();
662 mutex_enter(&vdrain_lock);
663 } else {
664 vdrain_gen++;
665 cv_broadcast(&vdrain_gen_cv);
666 cv_wait(&vdrain_cv, &vdrain_lock);
667 }
668 }
669 }
670
671 /*
672 * vput: unlock and release the reference.
673 */
674 void
675 vput(vnode_t *vp)
676 {
677 int lktype;
678
679 if ((vp->v_vflag & VV_LOCKSWORK) == 0) {
680 lktype = LK_EXCLUSIVE;
681 } else {
682 lktype = VOP_ISLOCKED(vp);
683 KASSERT(lktype != LK_NONE);
684 }
685 mutex_enter(vp->v_interlock);
686 vrelel(vp, 0, lktype);
687 }
688
689 /*
690 * Vnode release. If reference count drops to zero, call inactive
691 * routine and either return to freelist or free to the pool.
692 */
693 static void
694 vrelel(vnode_t *vp, int flags, int lktype)
695 {
696 const bool async = ((flags & VRELEL_ASYNC) != 0);
697 bool recycle, defer;
698 int error;
699
700 KASSERT(mutex_owned(vp->v_interlock));
701
702 if (__predict_false(vp->v_op == dead_vnodeop_p &&
703 VSTATE_GET(vp) != VS_RECLAIMED)) {
704 vnpanic(vp, "dead but not clean");
705 }
706
707 /*
708 * If not the last reference, just drop the reference count
709 * and unlock.
710 */
711 if (vp->v_usecount > 1) {
712 if (lktype != LK_NONE) {
713 VOP_UNLOCK(vp);
714 }
715 vp->v_usecount--;
716 mutex_exit(vp->v_interlock);
717 return;
718 }
719 if (vp->v_usecount <= 0 || vp->v_writecount != 0) {
720 vnpanic(vp, "%s: bad ref count", __func__);
721 }
722
723 #ifdef DIAGNOSTIC
724 if ((vp->v_type == VBLK || vp->v_type == VCHR) &&
725 vp->v_specnode != NULL && vp->v_specnode->sn_opencnt != 0) {
726 vprint("vrelel: missing VOP_CLOSE()", vp);
727 }
728 #endif
729
730 /*
731 * First try to get the vnode locked for VOP_INACTIVE().
732 * Defer vnode release to vdrain_thread if caller requests
733 * it explicitly, is the pagedaemon or the lock failed.
734 */
735 defer = false;
736 if ((curlwp == uvm.pagedaemon_lwp) || async) {
737 defer = true;
738 } else if (lktype == LK_SHARED) {
739 /* Excellent chance of getting, if the last ref. */
740 error = vn_lock(vp, LK_UPGRADE | LK_RETRY |
741 LK_NOWAIT);
742 if (error != 0) {
743 defer = true;
744 } else {
745 lktype = LK_EXCLUSIVE;
746 }
747 } else if (lktype == LK_NONE) {
748 /* Excellent chance of getting, if the last ref. */
749 error = vn_lock(vp, LK_EXCLUSIVE | LK_RETRY |
750 LK_NOWAIT);
751 if (error != 0) {
752 defer = true;
753 } else {
754 lktype = LK_EXCLUSIVE;
755 }
756 }
757 KASSERT(mutex_owned(vp->v_interlock));
758 if (defer) {
759 /*
760 * Defer reclaim to the kthread; it's not safe to
761 * clean it here. We donate it our last reference.
762 */
763 if (lktype != LK_NONE) {
764 VOP_UNLOCK(vp);
765 }
766 lru_requeue(vp, &lru_list[LRU_VRELE]);
767 mutex_exit(vp->v_interlock);
768 return;
769 }
770 KASSERT(lktype == LK_EXCLUSIVE);
771
772 /*
773 * If not clean, deactivate the vnode, but preserve
774 * our reference across the call to VOP_INACTIVE().
775 */
776 if (VSTATE_GET(vp) == VS_RECLAIMED) {
777 VOP_UNLOCK(vp);
778 } else {
779 /*
780 * If VOP_INACTIVE() indicates that the described file has
781 * been deleted, then recycle the vnode. Note that
782 * VOP_INACTIVE() will not drop the vnode lock.
783 *
784 * If the file has been deleted, this is a lingering
785 * reference and there is no need to worry about new
786 * references looking to do real work with the vnode (as it
787 * will have been purged from directories, caches, etc).
788 */
789 recycle = false;
790 mutex_exit(vp->v_interlock);
791 VOP_INACTIVE(vp, &recycle);
792 mutex_enter(vp->v_interlock);
793 if (!recycle) {
794 VOP_UNLOCK(vp);
795 if (vp->v_usecount > 1) {
796 vp->v_usecount--;
797 mutex_exit(vp->v_interlock);
798 return;
799 }
800 }
801
802 /* Take care of space accounting. */
803 if ((vp->v_iflag & VI_EXECMAP) != 0 &&
804 vp->v_uobj.uo_npages != 0) {
805 cpu_count(CPU_COUNT_EXECPAGES, -vp->v_uobj.uo_npages);
806 cpu_count(CPU_COUNT_FILEPAGES, vp->v_uobj.uo_npages);
807 }
808 vp->v_iflag &= ~(VI_TEXT|VI_EXECMAP|VI_WRMAP);
809 vp->v_vflag &= ~VV_MAPPED;
810
811 /*
812 * Recycle the vnode if the file is now unused (unlinked),
813 * otherwise just free it.
814 */
815 if (recycle) {
816 VSTATE_ASSERT(vp, VS_LOADED);
817 /* vcache_reclaim drops the lock. */
818 vcache_reclaim(vp);
819 }
820 KASSERT(vp->v_usecount > 0);
821 }
822
823 vp->v_usecount--;
824 if (vp->v_usecount != 0) {
825 /* Gained another reference while being reclaimed. */
826 mutex_exit(vp->v_interlock);
827 return;
828 }
829
830 if (VSTATE_GET(vp) == VS_RECLAIMED && vp->v_holdcnt == 0) {
831 /*
832 * It's clean so destroy it. It isn't referenced
833 * anywhere since it has been reclaimed.
834 */
835 vcache_free(VNODE_TO_VIMPL(vp));
836 } else {
837 /*
838 * Otherwise, put it back onto the freelist. It
839 * can't be destroyed while still associated with
840 * a file system.
841 */
842 lru_requeue(vp, lru_which(vp));
843 mutex_exit(vp->v_interlock);
844 }
845 }
846
847 void
848 vrele(vnode_t *vp)
849 {
850
851 mutex_enter(vp->v_interlock);
852 vrelel(vp, 0, LK_NONE);
853 }
854
855 /*
856 * Asynchronous vnode release, vnode is released in different context.
857 */
858 void
859 vrele_async(vnode_t *vp)
860 {
861
862 mutex_enter(vp->v_interlock);
863 vrelel(vp, VRELEL_ASYNC, LK_NONE);
864 }
865
866 /*
867 * Vnode reference, where a reference is already held by some other
868 * object (for example, a file structure).
869 */
870 void
871 vref(vnode_t *vp)
872 {
873
874 KASSERT(vp->v_usecount != 0);
875
876 mutex_enter(vp->v_interlock);
877 vp->v_usecount++;
878 mutex_exit(vp->v_interlock);
879 }
880
881 /*
882 * Page or buffer structure gets a reference.
883 * Called with v_interlock held.
884 */
885 void
886 vholdl(vnode_t *vp)
887 {
888
889 KASSERT(mutex_owned(vp->v_interlock));
890
891 if (vp->v_holdcnt++ == 0 && vp->v_usecount == 0)
892 lru_requeue(vp, lru_which(vp));
893 }
894
895 /*
896 * Page or buffer structure frees a reference.
897 * Called with v_interlock held.
898 */
899 void
900 holdrelel(vnode_t *vp)
901 {
902
903 KASSERT(mutex_owned(vp->v_interlock));
904
905 if (vp->v_holdcnt <= 0) {
906 vnpanic(vp, "%s: holdcnt vp %p", __func__, vp);
907 }
908
909 vp->v_holdcnt--;
910 if (vp->v_holdcnt == 0 && vp->v_usecount == 0)
911 lru_requeue(vp, lru_which(vp));
912 }
913
914 /*
915 * Recycle an unused vnode if caller holds the last reference.
916 */
917 bool
918 vrecycle(vnode_t *vp)
919 {
920 int error __diagused;
921
922 mutex_enter(vp->v_interlock);
923
924 /* Make sure we hold the last reference. */
925 VSTATE_WAIT_STABLE(vp);
926 if (vp->v_usecount != 1) {
927 mutex_exit(vp->v_interlock);
928 return false;
929 }
930
931 /* If the vnode is already clean we're done. */
932 if (VSTATE_GET(vp) != VS_LOADED) {
933 VSTATE_ASSERT(vp, VS_RECLAIMED);
934 vrelel(vp, 0, LK_NONE);
935 return true;
936 }
937
938 /* Prevent further references until the vnode is locked. */
939 VSTATE_CHANGE(vp, VS_LOADED, VS_BLOCKED);
940 mutex_exit(vp->v_interlock);
941
942 /*
943 * On a leaf file system this lock will always succeed as we hold
944 * the last reference and prevent further references.
945 * On layered file systems waiting for the lock would open a can of
946 * deadlocks as the lower vnodes may have other active references.
947 */
948 error = vn_lock(vp, LK_EXCLUSIVE | LK_RETRY | LK_NOWAIT);
949
950 mutex_enter(vp->v_interlock);
951 VSTATE_CHANGE(vp, VS_BLOCKED, VS_LOADED);
952
953 if (error) {
954 mutex_exit(vp->v_interlock);
955 return false;
956 }
957
958 KASSERT(vp->v_usecount == 1);
959 vcache_reclaim(vp);
960 vrelel(vp, 0, LK_NONE);
961
962 return true;
963 }
964
965 /*
966 * Helper for vrevoke() to propagate suspension from lastmp
967 * to thismp. Both args may be NULL.
968 * Returns the currently suspended file system or NULL.
969 */
970 static struct mount *
971 vrevoke_suspend_next(struct mount *lastmp, struct mount *thismp)
972 {
973 int error;
974
975 if (lastmp == thismp)
976 return thismp;
977
978 if (lastmp != NULL)
979 vfs_resume(lastmp);
980
981 if (thismp == NULL)
982 return NULL;
983
984 do {
985 error = vfs_suspend(thismp, 0);
986 } while (error == EINTR || error == ERESTART);
987
988 if (error == 0)
989 return thismp;
990
991 KASSERT(error == EOPNOTSUPP);
992 return NULL;
993 }
994
995 /*
996 * Eliminate all activity associated with the requested vnode
997 * and with all vnodes aliased to the requested vnode.
998 */
999 void
1000 vrevoke(vnode_t *vp)
1001 {
1002 struct mount *mp;
1003 vnode_t *vq;
1004 enum vtype type;
1005 dev_t dev;
1006
1007 KASSERT(vp->v_usecount > 0);
1008
1009 mp = vrevoke_suspend_next(NULL, vp->v_mount);
1010
1011 mutex_enter(vp->v_interlock);
1012 VSTATE_WAIT_STABLE(vp);
1013 if (VSTATE_GET(vp) == VS_RECLAIMED) {
1014 mutex_exit(vp->v_interlock);
1015 } else if (vp->v_type != VBLK && vp->v_type != VCHR) {
1016 vp->v_usecount++;
1017 mutex_exit(vp->v_interlock);
1018 vgone(vp);
1019 } else {
1020 dev = vp->v_rdev;
1021 type = vp->v_type;
1022 mutex_exit(vp->v_interlock);
1023
1024 while (spec_node_lookup_by_dev(type, dev, &vq) == 0) {
1025 mp = vrevoke_suspend_next(mp, vq->v_mount);
1026 vgone(vq);
1027 }
1028 }
1029 vrevoke_suspend_next(mp, NULL);
1030 }
1031
1032 /*
1033 * Eliminate all activity associated with a vnode in preparation for
1034 * reuse. Drops a reference from the vnode.
1035 */
1036 void
1037 vgone(vnode_t *vp)
1038 {
1039 int lktype;
1040
1041 KASSERT(vp->v_mount == dead_rootmount || fstrans_is_owner(vp->v_mount));
1042
1043 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
1044 lktype = LK_EXCLUSIVE;
1045 mutex_enter(vp->v_interlock);
1046 VSTATE_WAIT_STABLE(vp);
1047 if (VSTATE_GET(vp) == VS_LOADED) {
1048 vcache_reclaim(vp);
1049 lktype = LK_NONE;
1050 }
1051 VSTATE_ASSERT(vp, VS_RECLAIMED);
1052 vrelel(vp, 0, lktype);
1053 }
1054
1055 static inline uint32_t
1056 vcache_hash(const struct vcache_key *key)
1057 {
1058 uint32_t hash = HASH32_BUF_INIT;
1059
1060 KASSERT(key->vk_key_len > 0);
1061
1062 hash = hash32_buf(&key->vk_mount, sizeof(struct mount *), hash);
1063 hash = hash32_buf(key->vk_key, key->vk_key_len, hash);
1064 return hash;
1065 }
1066
1067 static void
1068 vcache_init(void)
1069 {
1070
1071 vcache_pool = pool_cache_init(sizeof(vnode_impl_t), 0, 0, 0,
1072 "vcachepl", NULL, IPL_NONE, NULL, NULL, NULL);
1073 KASSERT(vcache_pool != NULL);
1074 mutex_init(&vcache_lock, MUTEX_DEFAULT, IPL_NONE);
1075 cv_init(&vcache_cv, "vcache");
1076 vcache_hashsize = desiredvnodes;
1077 vcache_hashtab = hashinit(desiredvnodes, HASH_SLIST, true,
1078 &vcache_hashmask);
1079 }
1080
1081 static void
1082 vcache_reinit(void)
1083 {
1084 int i;
1085 uint32_t hash;
1086 u_long oldmask, newmask;
1087 struct hashhead *oldtab, *newtab;
1088 vnode_impl_t *vip;
1089
1090 newtab = hashinit(desiredvnodes, HASH_SLIST, true, &newmask);
1091 mutex_enter(&vcache_lock);
1092 oldtab = vcache_hashtab;
1093 oldmask = vcache_hashmask;
1094 vcache_hashsize = desiredvnodes;
1095 vcache_hashtab = newtab;
1096 vcache_hashmask = newmask;
1097 for (i = 0; i <= oldmask; i++) {
1098 while ((vip = SLIST_FIRST(&oldtab[i])) != NULL) {
1099 SLIST_REMOVE(&oldtab[i], vip, vnode_impl, vi_hash);
1100 hash = vcache_hash(&vip->vi_key);
1101 SLIST_INSERT_HEAD(&newtab[hash & vcache_hashmask],
1102 vip, vi_hash);
1103 }
1104 }
1105 mutex_exit(&vcache_lock);
1106 hashdone(oldtab, HASH_SLIST, oldmask);
1107 }
1108
1109 static inline vnode_impl_t *
1110 vcache_hash_lookup(const struct vcache_key *key, uint32_t hash)
1111 {
1112 struct hashhead *hashp;
1113 vnode_impl_t *vip;
1114
1115 KASSERT(mutex_owned(&vcache_lock));
1116
1117 hashp = &vcache_hashtab[hash & vcache_hashmask];
1118 SLIST_FOREACH(vip, hashp, vi_hash) {
1119 if (key->vk_mount != vip->vi_key.vk_mount)
1120 continue;
1121 if (key->vk_key_len != vip->vi_key.vk_key_len)
1122 continue;
1123 if (memcmp(key->vk_key, vip->vi_key.vk_key, key->vk_key_len))
1124 continue;
1125 return vip;
1126 }
1127 return NULL;
1128 }
1129
1130 /*
1131 * Allocate a new, uninitialized vcache node.
1132 */
1133 static vnode_impl_t *
1134 vcache_alloc(void)
1135 {
1136 vnode_impl_t *vip;
1137 vnode_t *vp;
1138
1139 vip = pool_cache_get(vcache_pool, PR_WAITOK);
1140 memset(vip, 0, sizeof(*vip));
1141
1142 vip->vi_lock = rw_obj_alloc();
1143
1144 vp = VIMPL_TO_VNODE(vip);
1145 uvm_obj_init(&vp->v_uobj, &uvm_vnodeops, true, 0);
1146 cv_init(&vp->v_cv, "vnode");
1147 cache_vnode_init(vp);
1148
1149 vp->v_usecount = 1;
1150 vp->v_type = VNON;
1151 vp->v_size = vp->v_writesize = VSIZENOTSET;
1152
1153 vip->vi_state = VS_LOADING;
1154
1155 lru_requeue(vp, &lru_list[LRU_FREE]);
1156
1157 return vip;
1158 }
1159
1160 /*
1161 * Deallocate a vcache node in state VS_LOADING.
1162 *
1163 * vcache_lock held on entry and released on return.
1164 */
1165 static void
1166 vcache_dealloc(vnode_impl_t *vip)
1167 {
1168 vnode_t *vp;
1169
1170 KASSERT(mutex_owned(&vcache_lock));
1171
1172 vp = VIMPL_TO_VNODE(vip);
1173 vfs_ref(dead_rootmount);
1174 vfs_insmntque(vp, dead_rootmount);
1175 mutex_enter(vp->v_interlock);
1176 vp->v_op = dead_vnodeop_p;
1177 VSTATE_CHANGE(vp, VS_LOADING, VS_RECLAIMED);
1178 mutex_exit(&vcache_lock);
1179 vrelel(vp, 0, LK_NONE);
1180 }
1181
1182 /*
1183 * Free an unused, unreferenced vcache node.
1184 * v_interlock locked on entry.
1185 */
1186 static void
1187 vcache_free(vnode_impl_t *vip)
1188 {
1189 vnode_t *vp;
1190
1191 vp = VIMPL_TO_VNODE(vip);
1192 KASSERT(mutex_owned(vp->v_interlock));
1193
1194 KASSERT(vp->v_usecount == 0);
1195 KASSERT(vp->v_holdcnt == 0);
1196 KASSERT(vp->v_writecount == 0);
1197 lru_requeue(vp, NULL);
1198 mutex_exit(vp->v_interlock);
1199
1200 vfs_insmntque(vp, NULL);
1201 if (vp->v_type == VBLK || vp->v_type == VCHR)
1202 spec_node_destroy(vp);
1203
1204 rw_obj_free(vip->vi_lock);
1205 uvm_obj_destroy(&vp->v_uobj, true);
1206 cv_destroy(&vp->v_cv);
1207 cache_vnode_fini(vp);
1208 pool_cache_put(vcache_pool, vip);
1209 }
1210
1211 /*
1212 * Try to get an initial reference on this cached vnode.
1213 * Returns zero on success, ENOENT if the vnode has been reclaimed and
1214 * EBUSY if the vnode state is unstable.
1215 *
1216 * v_interlock locked on entry and unlocked on exit.
1217 */
1218 int
1219 vcache_tryvget(vnode_t *vp)
1220 {
1221 int error = 0;
1222
1223 KASSERT(mutex_owned(vp->v_interlock));
1224
1225 if (__predict_false(VSTATE_GET(vp) == VS_RECLAIMED))
1226 error = ENOENT;
1227 else if (__predict_false(VSTATE_GET(vp) != VS_LOADED))
1228 error = EBUSY;
1229 else
1230 vp->v_usecount++;
1231
1232 mutex_exit(vp->v_interlock);
1233
1234 return error;
1235 }
1236
1237 /*
1238 * Try to get an initial reference on this cached vnode.
1239 * Returns zero on success and ENOENT if the vnode has been reclaimed.
1240 * Will wait for the vnode state to be stable.
1241 *
1242 * v_interlock locked on entry and unlocked on exit.
1243 */
1244 int
1245 vcache_vget(vnode_t *vp)
1246 {
1247
1248 KASSERT(mutex_owned(vp->v_interlock));
1249
1250 /* Increment hold count to prevent vnode from disappearing. */
1251 vp->v_holdcnt++;
1252 VSTATE_WAIT_STABLE(vp);
1253 vp->v_holdcnt--;
1254
1255 /* If this was the last reference to a reclaimed vnode free it now. */
1256 if (__predict_false(VSTATE_GET(vp) == VS_RECLAIMED)) {
1257 if (vp->v_holdcnt == 0 && vp->v_usecount == 0)
1258 vcache_free(VNODE_TO_VIMPL(vp));
1259 else
1260 mutex_exit(vp->v_interlock);
1261 return ENOENT;
1262 }
1263 VSTATE_ASSERT(vp, VS_LOADED);
1264 vp->v_usecount++;
1265 mutex_exit(vp->v_interlock);
1266
1267 return 0;
1268 }
1269
1270 /*
1271 * Get a vnode / fs node pair by key and return it referenced through vpp.
1272 */
1273 int
1274 vcache_get(struct mount *mp, const void *key, size_t key_len,
1275 struct vnode **vpp)
1276 {
1277 int error;
1278 uint32_t hash;
1279 const void *new_key;
1280 struct vnode *vp;
1281 struct vcache_key vcache_key;
1282 vnode_impl_t *vip, *new_vip;
1283
1284 new_key = NULL;
1285 *vpp = NULL;
1286
1287 vcache_key.vk_mount = mp;
1288 vcache_key.vk_key = key;
1289 vcache_key.vk_key_len = key_len;
1290 hash = vcache_hash(&vcache_key);
1291
1292 again:
1293 mutex_enter(&vcache_lock);
1294 vip = vcache_hash_lookup(&vcache_key, hash);
1295
1296 /* If found, take a reference or retry. */
1297 if (__predict_true(vip != NULL)) {
1298 /*
1299 * If the vnode is loading we cannot take the v_interlock
1300 * here as it might change during load (see uvm_obj_setlock()).
1301 * As changing state from VS_LOADING requires both vcache_lock
1302 * and v_interlock it is safe to test with vcache_lock held.
1303 *
1304 * Wait for vnodes changing state from VS_LOADING and retry.
1305 */
1306 if (__predict_false(vip->vi_state == VS_LOADING)) {
1307 cv_wait(&vcache_cv, &vcache_lock);
1308 mutex_exit(&vcache_lock);
1309 goto again;
1310 }
1311 vp = VIMPL_TO_VNODE(vip);
1312 mutex_enter(vp->v_interlock);
1313 mutex_exit(&vcache_lock);
1314 error = vcache_vget(vp);
1315 if (error == ENOENT)
1316 goto again;
1317 if (error == 0)
1318 *vpp = vp;
1319 KASSERT((error != 0) == (*vpp == NULL));
1320 return error;
1321 }
1322 mutex_exit(&vcache_lock);
1323
1324 /* Allocate and initialize a new vcache / vnode pair. */
1325 error = vfs_busy(mp);
1326 if (error)
1327 return error;
1328 new_vip = vcache_alloc();
1329 new_vip->vi_key = vcache_key;
1330 vp = VIMPL_TO_VNODE(new_vip);
1331 mutex_enter(&vcache_lock);
1332 vip = vcache_hash_lookup(&vcache_key, hash);
1333 if (vip == NULL) {
1334 SLIST_INSERT_HEAD(&vcache_hashtab[hash & vcache_hashmask],
1335 new_vip, vi_hash);
1336 vip = new_vip;
1337 }
1338
1339 /* If another thread beat us inserting this node, retry. */
1340 if (vip != new_vip) {
1341 vcache_dealloc(new_vip);
1342 vfs_unbusy(mp);
1343 goto again;
1344 }
1345 mutex_exit(&vcache_lock);
1346
1347 /* Load the fs node. Exclusive as new_node is VS_LOADING. */
1348 error = VFS_LOADVNODE(mp, vp, key, key_len, &new_key);
1349 if (error) {
1350 mutex_enter(&vcache_lock);
1351 SLIST_REMOVE(&vcache_hashtab[hash & vcache_hashmask],
1352 new_vip, vnode_impl, vi_hash);
1353 vcache_dealloc(new_vip);
1354 vfs_unbusy(mp);
1355 KASSERT(*vpp == NULL);
1356 return error;
1357 }
1358 KASSERT(new_key != NULL);
1359 KASSERT(memcmp(key, new_key, key_len) == 0);
1360 KASSERT(vp->v_op != NULL);
1361 vfs_insmntque(vp, mp);
1362 if ((mp->mnt_iflag & IMNT_MPSAFE) != 0)
1363 vp->v_vflag |= VV_MPSAFE;
1364 vfs_ref(mp);
1365 vfs_unbusy(mp);
1366
1367 /* Finished loading, finalize node. */
1368 mutex_enter(&vcache_lock);
1369 new_vip->vi_key.vk_key = new_key;
1370 mutex_enter(vp->v_interlock);
1371 VSTATE_CHANGE(vp, VS_LOADING, VS_LOADED);
1372 mutex_exit(vp->v_interlock);
1373 mutex_exit(&vcache_lock);
1374 *vpp = vp;
1375 return 0;
1376 }
1377
1378 /*
1379 * Create a new vnode / fs node pair and return it referenced through vpp.
1380 */
1381 int
1382 vcache_new(struct mount *mp, struct vnode *dvp, struct vattr *vap,
1383 kauth_cred_t cred, void *extra, struct vnode **vpp)
1384 {
1385 int error;
1386 uint32_t hash;
1387 struct vnode *vp, *ovp;
1388 vnode_impl_t *vip, *ovip;
1389
1390 *vpp = NULL;
1391
1392 /* Allocate and initialize a new vcache / vnode pair. */
1393 error = vfs_busy(mp);
1394 if (error)
1395 return error;
1396 vip = vcache_alloc();
1397 vip->vi_key.vk_mount = mp;
1398 vp = VIMPL_TO_VNODE(vip);
1399
1400 /* Create and load the fs node. */
1401 error = VFS_NEWVNODE(mp, dvp, vp, vap, cred, extra,
1402 &vip->vi_key.vk_key_len, &vip->vi_key.vk_key);
1403 if (error) {
1404 mutex_enter(&vcache_lock);
1405 vcache_dealloc(vip);
1406 vfs_unbusy(mp);
1407 KASSERT(*vpp == NULL);
1408 return error;
1409 }
1410 KASSERT(vp->v_op != NULL);
1411 KASSERT((vip->vi_key.vk_key_len == 0) == (mp == dead_rootmount));
1412 if (vip->vi_key.vk_key_len > 0) {
1413 KASSERT(vip->vi_key.vk_key != NULL);
1414 hash = vcache_hash(&vip->vi_key);
1415
1416 /*
1417 * Wait for previous instance to be reclaimed,
1418 * then insert new node.
1419 */
1420 mutex_enter(&vcache_lock);
1421 while ((ovip = vcache_hash_lookup(&vip->vi_key, hash))) {
1422 ovp = VIMPL_TO_VNODE(ovip);
1423 mutex_enter(ovp->v_interlock);
1424 mutex_exit(&vcache_lock);
1425 error = vcache_vget(ovp);
1426 KASSERT(error == ENOENT);
1427 mutex_enter(&vcache_lock);
1428 }
1429 SLIST_INSERT_HEAD(&vcache_hashtab[hash & vcache_hashmask],
1430 vip, vi_hash);
1431 mutex_exit(&vcache_lock);
1432 }
1433 vfs_insmntque(vp, mp);
1434 if ((mp->mnt_iflag & IMNT_MPSAFE) != 0)
1435 vp->v_vflag |= VV_MPSAFE;
1436 vfs_ref(mp);
1437 vfs_unbusy(mp);
1438
1439 /* Finished loading, finalize node. */
1440 mutex_enter(&vcache_lock);
1441 mutex_enter(vp->v_interlock);
1442 VSTATE_CHANGE(vp, VS_LOADING, VS_LOADED);
1443 mutex_exit(&vcache_lock);
1444 mutex_exit(vp->v_interlock);
1445 *vpp = vp;
1446 return 0;
1447 }
1448
1449 /*
1450 * Prepare key change: update old cache nodes key and lock new cache node.
1451 * Return an error if the new node already exists.
1452 */
1453 int
1454 vcache_rekey_enter(struct mount *mp, struct vnode *vp,
1455 const void *old_key, size_t old_key_len,
1456 const void *new_key, size_t new_key_len)
1457 {
1458 uint32_t old_hash, new_hash;
1459 struct vcache_key old_vcache_key, new_vcache_key;
1460 vnode_impl_t *vip, *new_vip;
1461
1462 old_vcache_key.vk_mount = mp;
1463 old_vcache_key.vk_key = old_key;
1464 old_vcache_key.vk_key_len = old_key_len;
1465 old_hash = vcache_hash(&old_vcache_key);
1466
1467 new_vcache_key.vk_mount = mp;
1468 new_vcache_key.vk_key = new_key;
1469 new_vcache_key.vk_key_len = new_key_len;
1470 new_hash = vcache_hash(&new_vcache_key);
1471
1472 new_vip = vcache_alloc();
1473 new_vip->vi_key = new_vcache_key;
1474
1475 /* Insert locked new node used as placeholder. */
1476 mutex_enter(&vcache_lock);
1477 vip = vcache_hash_lookup(&new_vcache_key, new_hash);
1478 if (vip != NULL) {
1479 vcache_dealloc(new_vip);
1480 return EEXIST;
1481 }
1482 SLIST_INSERT_HEAD(&vcache_hashtab[new_hash & vcache_hashmask],
1483 new_vip, vi_hash);
1484
1485 /* Replace old nodes key with the temporary copy. */
1486 vip = vcache_hash_lookup(&old_vcache_key, old_hash);
1487 KASSERT(vip != NULL);
1488 KASSERT(VIMPL_TO_VNODE(vip) == vp);
1489 KASSERT(vip->vi_key.vk_key != old_vcache_key.vk_key);
1490 vip->vi_key = old_vcache_key;
1491 mutex_exit(&vcache_lock);
1492 return 0;
1493 }
1494
1495 /*
1496 * Key change complete: update old node and remove placeholder.
1497 */
1498 void
1499 vcache_rekey_exit(struct mount *mp, struct vnode *vp,
1500 const void *old_key, size_t old_key_len,
1501 const void *new_key, size_t new_key_len)
1502 {
1503 uint32_t old_hash, new_hash;
1504 struct vcache_key old_vcache_key, new_vcache_key;
1505 vnode_impl_t *vip, *new_vip;
1506 struct vnode *new_vp;
1507
1508 old_vcache_key.vk_mount = mp;
1509 old_vcache_key.vk_key = old_key;
1510 old_vcache_key.vk_key_len = old_key_len;
1511 old_hash = vcache_hash(&old_vcache_key);
1512
1513 new_vcache_key.vk_mount = mp;
1514 new_vcache_key.vk_key = new_key;
1515 new_vcache_key.vk_key_len = new_key_len;
1516 new_hash = vcache_hash(&new_vcache_key);
1517
1518 mutex_enter(&vcache_lock);
1519
1520 /* Lookup old and new node. */
1521 vip = vcache_hash_lookup(&old_vcache_key, old_hash);
1522 KASSERT(vip != NULL);
1523 KASSERT(VIMPL_TO_VNODE(vip) == vp);
1524
1525 new_vip = vcache_hash_lookup(&new_vcache_key, new_hash);
1526 KASSERT(new_vip != NULL);
1527 KASSERT(new_vip->vi_key.vk_key_len == new_key_len);
1528 new_vp = VIMPL_TO_VNODE(new_vip);
1529 mutex_enter(new_vp->v_interlock);
1530 VSTATE_ASSERT(VIMPL_TO_VNODE(new_vip), VS_LOADING);
1531 mutex_exit(new_vp->v_interlock);
1532
1533 /* Rekey old node and put it onto its new hashlist. */
1534 vip->vi_key = new_vcache_key;
1535 if (old_hash != new_hash) {
1536 SLIST_REMOVE(&vcache_hashtab[old_hash & vcache_hashmask],
1537 vip, vnode_impl, vi_hash);
1538 SLIST_INSERT_HEAD(&vcache_hashtab[new_hash & vcache_hashmask],
1539 vip, vi_hash);
1540 }
1541
1542 /* Remove new node used as placeholder. */
1543 SLIST_REMOVE(&vcache_hashtab[new_hash & vcache_hashmask],
1544 new_vip, vnode_impl, vi_hash);
1545 vcache_dealloc(new_vip);
1546 }
1547
1548 /*
1549 * Disassociate the underlying file system from a vnode.
1550 *
1551 * Must be called with vnode locked and will return unlocked.
1552 * Must be called with the interlock held, and will return with it held.
1553 */
1554 static void
1555 vcache_reclaim(vnode_t *vp)
1556 {
1557 lwp_t *l = curlwp;
1558 vnode_impl_t *vip = VNODE_TO_VIMPL(vp);
1559 struct mount *mp = vp->v_mount;
1560 uint32_t hash;
1561 uint8_t temp_buf[64], *temp_key;
1562 size_t temp_key_len;
1563 bool recycle, active;
1564 int error;
1565
1566 KASSERT((vp->v_vflag & VV_LOCKSWORK) == 0 ||
1567 VOP_ISLOCKED(vp) == LK_EXCLUSIVE);
1568 KASSERT(mutex_owned(vp->v_interlock));
1569 KASSERT(vp->v_usecount != 0);
1570
1571 active = (vp->v_usecount > 1);
1572 temp_key_len = vip->vi_key.vk_key_len;
1573 /*
1574 * Prevent the vnode from being recycled or brought into use
1575 * while we clean it out.
1576 */
1577 VSTATE_CHANGE(vp, VS_LOADED, VS_RECLAIMING);
1578 if ((vp->v_iflag & VI_EXECMAP) != 0 && vp->v_uobj.uo_npages != 0) {
1579 cpu_count(CPU_COUNT_EXECPAGES, -vp->v_uobj.uo_npages);
1580 cpu_count(CPU_COUNT_FILEPAGES, vp->v_uobj.uo_npages);
1581 }
1582 vp->v_iflag &= ~(VI_TEXT|VI_EXECMAP);
1583 mutex_exit(vp->v_interlock);
1584
1585 /* Replace the vnode key with a temporary copy. */
1586 if (vip->vi_key.vk_key_len > sizeof(temp_buf)) {
1587 temp_key = kmem_alloc(temp_key_len, KM_SLEEP);
1588 } else {
1589 temp_key = temp_buf;
1590 }
1591 if (vip->vi_key.vk_key_len > 0) {
1592 mutex_enter(&vcache_lock);
1593 memcpy(temp_key, vip->vi_key.vk_key, temp_key_len);
1594 vip->vi_key.vk_key = temp_key;
1595 mutex_exit(&vcache_lock);
1596 }
1597
1598 fstrans_start(mp);
1599
1600 /*
1601 * Clean out any cached data associated with the vnode.
1602 * If purging an active vnode, it must be closed and
1603 * deactivated before being reclaimed.
1604 */
1605 error = vinvalbuf(vp, V_SAVE, NOCRED, l, 0, 0);
1606 if (error != 0) {
1607 if (wapbl_vphaswapbl(vp))
1608 WAPBL_DISCARD(wapbl_vptomp(vp));
1609 error = vinvalbuf(vp, 0, NOCRED, l, 0, 0);
1610 }
1611 KASSERTMSG((error == 0), "vinvalbuf failed: %d", error);
1612 KASSERT((vp->v_iflag & VI_ONWORKLST) == 0);
1613 if (active && (vp->v_type == VBLK || vp->v_type == VCHR)) {
1614 spec_node_revoke(vp);
1615 }
1616
1617 /*
1618 * Disassociate the underlying file system from the vnode.
1619 * VOP_INACTIVE leaves the vnode locked; VOP_RECLAIM unlocks
1620 * the vnode, and may destroy the vnode so that VOP_UNLOCK
1621 * would no longer function.
1622 */
1623 VOP_INACTIVE(vp, &recycle);
1624 KASSERT((vp->v_vflag & VV_LOCKSWORK) == 0 ||
1625 VOP_ISLOCKED(vp) == LK_EXCLUSIVE);
1626 if (VOP_RECLAIM(vp)) {
1627 vnpanic(vp, "%s: cannot reclaim", __func__);
1628 }
1629
1630 KASSERT(vp->v_data == NULL);
1631 KASSERT(vp->v_uobj.uo_npages == 0);
1632
1633 if (vp->v_type == VREG && vp->v_ractx != NULL) {
1634 uvm_ra_freectx(vp->v_ractx);
1635 vp->v_ractx = NULL;
1636 }
1637
1638 /* Purge name cache. */
1639 cache_purge(vp);
1640
1641 if (vip->vi_key.vk_key_len > 0) {
1642 /* Remove from vnode cache. */
1643 hash = vcache_hash(&vip->vi_key);
1644 mutex_enter(&vcache_lock);
1645 KASSERT(vip == vcache_hash_lookup(&vip->vi_key, hash));
1646 SLIST_REMOVE(&vcache_hashtab[hash & vcache_hashmask],
1647 vip, vnode_impl, vi_hash);
1648 mutex_exit(&vcache_lock);
1649 }
1650 if (temp_key != temp_buf)
1651 kmem_free(temp_key, temp_key_len);
1652
1653 /* Done with purge, notify sleepers of the grim news. */
1654 mutex_enter(vp->v_interlock);
1655 vp->v_op = dead_vnodeop_p;
1656 vp->v_vflag |= VV_LOCKSWORK;
1657 VSTATE_CHANGE(vp, VS_RECLAIMING, VS_RECLAIMED);
1658 vp->v_tag = VT_NON;
1659 KNOTE(&vp->v_klist, NOTE_REVOKE);
1660 mutex_exit(vp->v_interlock);
1661
1662 /*
1663 * Move to dead mount. Must be after changing the operations
1664 * vector as vnode operations enter the mount before using the
1665 * operations vector. See sys/kern/vnode_if.c.
1666 */
1667 vp->v_vflag &= ~VV_ROOT;
1668 vfs_ref(dead_rootmount);
1669 vfs_insmntque(vp, dead_rootmount);
1670
1671 mutex_enter(vp->v_interlock);
1672 fstrans_done(mp);
1673 KASSERT((vp->v_iflag & VI_ONWORKLST) == 0);
1674 }
1675
1676 /*
1677 * Disassociate the underlying file system from an open device vnode
1678 * and make it anonymous.
1679 *
1680 * Vnode unlocked on entry, drops a reference to the vnode.
1681 */
1682 void
1683 vcache_make_anon(vnode_t *vp)
1684 {
1685 vnode_impl_t *vip = VNODE_TO_VIMPL(vp);
1686 uint32_t hash;
1687 bool recycle;
1688
1689 KASSERT(vp->v_type == VBLK || vp->v_type == VCHR);
1690 KASSERT(vp->v_mount == dead_rootmount || fstrans_is_owner(vp->v_mount));
1691 VSTATE_ASSERT_UNLOCKED(vp, VS_ACTIVE);
1692
1693 /* Remove from vnode cache. */
1694 hash = vcache_hash(&vip->vi_key);
1695 mutex_enter(&vcache_lock);
1696 KASSERT(vip == vcache_hash_lookup(&vip->vi_key, hash));
1697 SLIST_REMOVE(&vcache_hashtab[hash & vcache_hashmask],
1698 vip, vnode_impl, vi_hash);
1699 vip->vi_key.vk_mount = dead_rootmount;
1700 vip->vi_key.vk_key_len = 0;
1701 vip->vi_key.vk_key = NULL;
1702 mutex_exit(&vcache_lock);
1703
1704 /*
1705 * Disassociate the underlying file system from the vnode.
1706 * VOP_INACTIVE leaves the vnode locked; VOP_RECLAIM unlocks
1707 * the vnode, and may destroy the vnode so that VOP_UNLOCK
1708 * would no longer function.
1709 */
1710 if (vn_lock(vp, LK_EXCLUSIVE)) {
1711 vnpanic(vp, "%s: cannot lock", __func__);
1712 }
1713 VOP_INACTIVE(vp, &recycle);
1714 KASSERT((vp->v_vflag & VV_LOCKSWORK) == 0 ||
1715 VOP_ISLOCKED(vp) == LK_EXCLUSIVE);
1716 if (VOP_RECLAIM(vp)) {
1717 vnpanic(vp, "%s: cannot reclaim", __func__);
1718 }
1719
1720 /* Purge name cache. */
1721 cache_purge(vp);
1722
1723 /* Done with purge, change operations vector. */
1724 mutex_enter(vp->v_interlock);
1725 vp->v_op = spec_vnodeop_p;
1726 vp->v_vflag |= VV_MPSAFE;
1727 vp->v_vflag &= ~VV_LOCKSWORK;
1728 mutex_exit(vp->v_interlock);
1729
1730 /*
1731 * Move to dead mount. Must be after changing the operations
1732 * vector as vnode operations enter the mount before using the
1733 * operations vector. See sys/kern/vnode_if.c.
1734 */
1735 vfs_ref(dead_rootmount);
1736 vfs_insmntque(vp, dead_rootmount);
1737
1738 vrele(vp);
1739 }
1740
1741 /*
1742 * Update outstanding I/O count and do wakeup if requested.
1743 */
1744 void
1745 vwakeup(struct buf *bp)
1746 {
1747 vnode_t *vp;
1748
1749 if ((vp = bp->b_vp) == NULL)
1750 return;
1751
1752 KASSERT(bp->b_objlock == vp->v_interlock);
1753 KASSERT(mutex_owned(bp->b_objlock));
1754
1755 if (--vp->v_numoutput < 0)
1756 vnpanic(vp, "%s: neg numoutput, vp %p", __func__, vp);
1757 if (vp->v_numoutput == 0)
1758 cv_broadcast(&vp->v_cv);
1759 }
1760
1761 /*
1762 * Test a vnode for being or becoming dead. Returns one of:
1763 * EBUSY: vnode is becoming dead, with "flags == VDEAD_NOWAIT" only.
1764 * ENOENT: vnode is dead.
1765 * 0: otherwise.
1766 *
1767 * Whenever this function returns a non-zero value all future
1768 * calls will also return a non-zero value.
1769 */
1770 int
1771 vdead_check(struct vnode *vp, int flags)
1772 {
1773
1774 KASSERT(mutex_owned(vp->v_interlock));
1775
1776 if (! ISSET(flags, VDEAD_NOWAIT))
1777 VSTATE_WAIT_STABLE(vp);
1778
1779 if (VSTATE_GET(vp) == VS_RECLAIMING) {
1780 KASSERT(ISSET(flags, VDEAD_NOWAIT));
1781 return EBUSY;
1782 } else if (VSTATE_GET(vp) == VS_RECLAIMED) {
1783 return ENOENT;
1784 }
1785
1786 return 0;
1787 }
1788
1789 int
1790 vfs_drainvnodes(void)
1791 {
1792 int i, gen;
1793
1794 mutex_enter(&vdrain_lock);
1795 for (i = 0; i < 2; i++) {
1796 gen = vdrain_gen;
1797 while (gen == vdrain_gen) {
1798 cv_broadcast(&vdrain_cv);
1799 cv_wait(&vdrain_gen_cv, &vdrain_lock);
1800 }
1801 }
1802 mutex_exit(&vdrain_lock);
1803
1804 if (numvnodes >= desiredvnodes)
1805 return EBUSY;
1806
1807 if (vcache_hashsize != desiredvnodes)
1808 vcache_reinit();
1809
1810 return 0;
1811 }
1812
1813 void
1814 vnpanic(vnode_t *vp, const char *fmt, ...)
1815 {
1816 va_list ap;
1817
1818 #ifdef DIAGNOSTIC
1819 vprint(NULL, vp);
1820 #endif
1821 va_start(ap, fmt);
1822 vpanic(fmt, ap);
1823 va_end(ap);
1824 }
1825