Home | History | Annotate | Line # | Download | only in kern
vfs_mount.c revision 1.81
      1 /*	$NetBSD: vfs_mount.c,v 1.81 2020/04/21 21:42:47 ad Exp $	*/
      2 
      3 /*-
      4  * Copyright (c) 1997-2020 The NetBSD Foundation, Inc.
      5  * All rights reserved.
      6  *
      7  * This code is derived from software contributed to The NetBSD Foundation
      8  * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
      9  * NASA Ames Research Center, by Charles M. Hannum, and by Andrew Doran.
     10  *
     11  * Redistribution and use in source and binary forms, with or without
     12  * modification, are permitted provided that the following conditions
     13  * are met:
     14  * 1. Redistributions of source code must retain the above copyright
     15  *    notice, this list of conditions and the following disclaimer.
     16  * 2. Redistributions in binary form must reproduce the above copyright
     17  *    notice, this list of conditions and the following disclaimer in the
     18  *    documentation and/or other materials provided with the distribution.
     19  *
     20  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     21  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     22  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     23  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     24  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     25  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     26  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     27  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     28  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     29  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     30  * POSSIBILITY OF SUCH DAMAGE.
     31  */
     32 
     33 /*
     34  * Copyright (c) 1989, 1993
     35  *	The Regents of the University of California.  All rights reserved.
     36  * (c) UNIX System Laboratories, Inc.
     37  * All or some portions of this file are derived from material licensed
     38  * to the University of California by American Telephone and Telegraph
     39  * Co. or Unix System Laboratories, Inc. and are reproduced herein with
     40  * the permission of UNIX System Laboratories, Inc.
     41  *
     42  * Redistribution and use in source and binary forms, with or without
     43  * modification, are permitted provided that the following conditions
     44  * are met:
     45  * 1. Redistributions of source code must retain the above copyright
     46  *    notice, this list of conditions and the following disclaimer.
     47  * 2. Redistributions in binary form must reproduce the above copyright
     48  *    notice, this list of conditions and the following disclaimer in the
     49  *    documentation and/or other materials provided with the distribution.
     50  * 3. Neither the name of the University nor the names of its contributors
     51  *    may be used to endorse or promote products derived from this software
     52  *    without specific prior written permission.
     53  *
     54  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
     55  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
     56  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
     57  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
     58  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
     59  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
     60  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
     61  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
     62  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
     63  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
     64  * SUCH DAMAGE.
     65  *
     66  *	@(#)vfs_subr.c	8.13 (Berkeley) 4/18/94
     67  */
     68 
     69 #include <sys/cdefs.h>
     70 __KERNEL_RCSID(0, "$NetBSD: vfs_mount.c,v 1.81 2020/04/21 21:42:47 ad Exp $");
     71 
     72 #include <sys/param.h>
     73 #include <sys/kernel.h>
     74 
     75 #include <sys/atomic.h>
     76 #include <sys/buf.h>
     77 #include <sys/conf.h>
     78 #include <sys/fcntl.h>
     79 #include <sys/filedesc.h>
     80 #include <sys/device.h>
     81 #include <sys/kauth.h>
     82 #include <sys/kmem.h>
     83 #include <sys/module.h>
     84 #include <sys/mount.h>
     85 #include <sys/fstrans.h>
     86 #include <sys/namei.h>
     87 #include <sys/extattr.h>
     88 #include <sys/syscallargs.h>
     89 #include <sys/sysctl.h>
     90 #include <sys/systm.h>
     91 #include <sys/vfs_syscalls.h>
     92 #include <sys/vnode_impl.h>
     93 
     94 #include <miscfs/genfs/genfs.h>
     95 #include <miscfs/specfs/specdev.h>
     96 
     97 enum mountlist_type {
     98 	ME_MOUNT,
     99 	ME_MARKER
    100 };
    101 struct mountlist_entry {
    102 	TAILQ_ENTRY(mountlist_entry) me_list;	/* Mount list. */
    103 	struct mount *me_mount;			/* Actual mount if ME_MOUNT,
    104 						   current mount else. */
    105 	enum mountlist_type me_type;		/* Mount or marker. */
    106 };
    107 struct mount_iterator {
    108 	struct mountlist_entry mi_entry;
    109 };
    110 
    111 static struct vnode *vfs_vnode_iterator_next1(struct vnode_iterator *,
    112     bool (*)(void *, struct vnode *), void *, bool);
    113 
    114 /* Root filesystem. */
    115 vnode_t *			rootvnode;
    116 
    117 extern struct mount		*dead_rootmount;
    118 
    119 /* Mounted filesystem list. */
    120 static TAILQ_HEAD(mountlist, mountlist_entry) mountlist;
    121 static kmutex_t			mountlist_lock __cacheline_aligned;
    122 int vnode_offset_next_by_lru	/* XXX: ugly hack for pstat.c */
    123     = offsetof(vnode_impl_t, vi_lrulist.tqe_next);
    124 
    125 kmutex_t			vfs_list_lock __cacheline_aligned;
    126 
    127 static specificdata_domain_t	mount_specificdata_domain;
    128 static kmutex_t			mntid_lock;
    129 
    130 static kmutex_t			mountgen_lock __cacheline_aligned;
    131 static uint64_t			mountgen;
    132 
    133 void
    134 vfs_mount_sysinit(void)
    135 {
    136 
    137 	TAILQ_INIT(&mountlist);
    138 	mutex_init(&mountlist_lock, MUTEX_DEFAULT, IPL_NONE);
    139 	mutex_init(&vfs_list_lock, MUTEX_DEFAULT, IPL_NONE);
    140 
    141 	mount_specificdata_domain = specificdata_domain_create();
    142 	mutex_init(&mntid_lock, MUTEX_DEFAULT, IPL_NONE);
    143 	mutex_init(&mountgen_lock, MUTEX_DEFAULT, IPL_NONE);
    144 	mountgen = 0;
    145 }
    146 
    147 struct mount *
    148 vfs_mountalloc(struct vfsops *vfsops, vnode_t *vp)
    149 {
    150 	struct mount *mp;
    151 	int error __diagused;
    152 
    153 	mp = kmem_zalloc(sizeof(*mp), KM_SLEEP);
    154 	mp->mnt_op = vfsops;
    155 	mp->mnt_refcnt = 1;
    156 	TAILQ_INIT(&mp->mnt_vnodelist);
    157 	mp->mnt_renamelock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE);
    158 	mp->mnt_vnodelock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE);
    159 	mp->mnt_updating = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE);
    160 	mp->mnt_vnodecovered = vp;
    161 	mount_initspecific(mp);
    162 
    163 	error = fstrans_mount(mp);
    164 	KASSERT(error == 0);
    165 
    166 	mutex_enter(&mountgen_lock);
    167 	mp->mnt_gen = mountgen++;
    168 	mutex_exit(&mountgen_lock);
    169 
    170 	return mp;
    171 }
    172 
    173 /*
    174  * vfs_rootmountalloc: lookup a filesystem type, and if found allocate and
    175  * initialize a mount structure for it.
    176  *
    177  * Devname is usually updated by mount(8) after booting.
    178  */
    179 int
    180 vfs_rootmountalloc(const char *fstypename, const char *devname,
    181     struct mount **mpp)
    182 {
    183 	struct vfsops *vfsp = NULL;
    184 	struct mount *mp;
    185 	int error __diagused;
    186 
    187 	mutex_enter(&vfs_list_lock);
    188 	LIST_FOREACH(vfsp, &vfs_list, vfs_list)
    189 		if (!strncmp(vfsp->vfs_name, fstypename,
    190 		    sizeof(mp->mnt_stat.f_fstypename)))
    191 			break;
    192 	if (vfsp == NULL) {
    193 		mutex_exit(&vfs_list_lock);
    194 		return (ENODEV);
    195 	}
    196 	vfsp->vfs_refcount++;
    197 	mutex_exit(&vfs_list_lock);
    198 
    199 	if ((mp = vfs_mountalloc(vfsp, NULL)) == NULL)
    200 		return ENOMEM;
    201 	error = vfs_busy(mp);
    202 	KASSERT(error == 0);
    203 	mp->mnt_flag = MNT_RDONLY;
    204 	(void)strlcpy(mp->mnt_stat.f_fstypename, vfsp->vfs_name,
    205 	    sizeof(mp->mnt_stat.f_fstypename));
    206 	mp->mnt_stat.f_mntonname[0] = '/';
    207 	mp->mnt_stat.f_mntonname[1] = '\0';
    208 	mp->mnt_stat.f_mntfromname[sizeof(mp->mnt_stat.f_mntfromname) - 1] =
    209 	    '\0';
    210 	(void)copystr(devname, mp->mnt_stat.f_mntfromname,
    211 	    sizeof(mp->mnt_stat.f_mntfromname) - 1, 0);
    212 	*mpp = mp;
    213 	return 0;
    214 }
    215 
    216 /*
    217  * vfs_getnewfsid: get a new unique fsid.
    218  */
    219 void
    220 vfs_getnewfsid(struct mount *mp)
    221 {
    222 	static u_short xxxfs_mntid;
    223 	fsid_t tfsid;
    224 	int mtype;
    225 
    226 	mutex_enter(&mntid_lock);
    227 	mtype = makefstype(mp->mnt_op->vfs_name);
    228 	mp->mnt_stat.f_fsidx.__fsid_val[0] = makedev(mtype, 0);
    229 	mp->mnt_stat.f_fsidx.__fsid_val[1] = mtype;
    230 	mp->mnt_stat.f_fsid = mp->mnt_stat.f_fsidx.__fsid_val[0];
    231 	if (xxxfs_mntid == 0)
    232 		++xxxfs_mntid;
    233 	tfsid.__fsid_val[0] = makedev(mtype & 0xff, xxxfs_mntid);
    234 	tfsid.__fsid_val[1] = mtype;
    235 	while (vfs_getvfs(&tfsid)) {
    236 		tfsid.__fsid_val[0]++;
    237 		xxxfs_mntid++;
    238 	}
    239 	mp->mnt_stat.f_fsidx.__fsid_val[0] = tfsid.__fsid_val[0];
    240 	mp->mnt_stat.f_fsid = mp->mnt_stat.f_fsidx.__fsid_val[0];
    241 	mutex_exit(&mntid_lock);
    242 }
    243 
    244 /*
    245  * Lookup a mount point by filesystem identifier.
    246  *
    247  * XXX Needs to add a reference to the mount point.
    248  */
    249 struct mount *
    250 vfs_getvfs(fsid_t *fsid)
    251 {
    252 	mount_iterator_t *iter;
    253 	struct mount *mp;
    254 
    255 	mountlist_iterator_init(&iter);
    256 	while ((mp = mountlist_iterator_next(iter)) != NULL) {
    257 		if (mp->mnt_stat.f_fsidx.__fsid_val[0] == fsid->__fsid_val[0] &&
    258 		    mp->mnt_stat.f_fsidx.__fsid_val[1] == fsid->__fsid_val[1]) {
    259 			mountlist_iterator_destroy(iter);
    260 			return mp;
    261 		}
    262 	}
    263 	mountlist_iterator_destroy(iter);
    264 	return NULL;
    265 }
    266 
    267 /*
    268  * Take a reference to a mount structure.
    269  */
    270 void
    271 vfs_ref(struct mount *mp)
    272 {
    273 
    274 	KASSERT(mp->mnt_refcnt > 0 || mutex_owned(&mountlist_lock));
    275 
    276 	atomic_inc_uint(&mp->mnt_refcnt);
    277 }
    278 
    279 /*
    280  * Drop a reference to a mount structure, freeing if the last reference.
    281  */
    282 void
    283 vfs_rele(struct mount *mp)
    284 {
    285 
    286 	if (__predict_true((int)atomic_dec_uint_nv(&mp->mnt_refcnt) > 0)) {
    287 		return;
    288 	}
    289 
    290 	/*
    291 	 * Nothing else has visibility of the mount: we can now
    292 	 * free the data structures.
    293 	 */
    294 	KASSERT(mp->mnt_refcnt == 0);
    295 	specificdata_fini(mount_specificdata_domain, &mp->mnt_specdataref);
    296 	mutex_obj_free(mp->mnt_updating);
    297 	mutex_obj_free(mp->mnt_renamelock);
    298 	mutex_obj_free(mp->mnt_vnodelock);
    299 	if (mp->mnt_op != NULL) {
    300 		vfs_delref(mp->mnt_op);
    301 	}
    302 	fstrans_unmount(mp);
    303 	/*
    304 	 * Final free of mp gets done from fstrans_mount_dtor().
    305 	 *
    306 	 * Prevents this memory to be reused as a mount before
    307 	 * fstrans releases all references to it.
    308 	 */
    309 }
    310 
    311 /*
    312  * Mark a mount point as busy, and gain a new reference to it.  Used to
    313  * prevent the file system from being unmounted during critical sections.
    314  *
    315  * vfs_busy can be called multiple times and by multiple threads
    316  * and must be accompanied by the same number of vfs_unbusy calls.
    317  *
    318  * => The caller must hold a pre-existing reference to the mount.
    319  * => Will fail if the file system is being unmounted, or is unmounted.
    320  */
    321 static inline int
    322 _vfs_busy(struct mount *mp, bool wait)
    323 {
    324 
    325 	KASSERT(mp->mnt_refcnt > 0);
    326 
    327 	if (wait) {
    328 		fstrans_start(mp);
    329 	} else {
    330 		if (fstrans_start_nowait(mp))
    331 			return EBUSY;
    332 	}
    333 	if (__predict_false((mp->mnt_iflag & IMNT_GONE) != 0)) {
    334 		fstrans_done(mp);
    335 		return ENOENT;
    336 	}
    337 	vfs_ref(mp);
    338 	return 0;
    339 }
    340 
    341 int
    342 vfs_busy(struct mount *mp)
    343 {
    344 
    345 	return _vfs_busy(mp, true);
    346 }
    347 
    348 int
    349 vfs_trybusy(struct mount *mp)
    350 {
    351 
    352 	return _vfs_busy(mp, false);
    353 }
    354 
    355 /*
    356  * Unbusy a busy filesystem.
    357  *
    358  * Every successful vfs_busy() call must be undone by a vfs_unbusy() call.
    359  */
    360 void
    361 vfs_unbusy(struct mount *mp)
    362 {
    363 
    364 	KASSERT(mp->mnt_refcnt > 0);
    365 
    366 	fstrans_done(mp);
    367 	vfs_rele(mp);
    368 }
    369 
    370 struct vnode_iterator {
    371 	vnode_impl_t vi_vnode;
    372 };
    373 
    374 void
    375 vfs_vnode_iterator_init(struct mount *mp, struct vnode_iterator **vnip)
    376 {
    377 	vnode_t *vp;
    378 	vnode_impl_t *vip;
    379 
    380 	vp = vnalloc_marker(mp);
    381 	vip = VNODE_TO_VIMPL(vp);
    382 
    383 	mutex_enter(mp->mnt_vnodelock);
    384 	TAILQ_INSERT_HEAD(&mp->mnt_vnodelist, vip, vi_mntvnodes);
    385 	vp->v_usecount = 1;
    386 	mutex_exit(mp->mnt_vnodelock);
    387 
    388 	*vnip = (struct vnode_iterator *)vip;
    389 }
    390 
    391 void
    392 vfs_vnode_iterator_destroy(struct vnode_iterator *vni)
    393 {
    394 	vnode_impl_t *mvip = &vni->vi_vnode;
    395 	vnode_t *mvp = VIMPL_TO_VNODE(mvip);
    396 	kmutex_t *lock;
    397 
    398 	KASSERT(vnis_marker(mvp));
    399 	if (vrefcnt(mvp) != 0) {
    400 		lock = mvp->v_mount->mnt_vnodelock;
    401 		mutex_enter(lock);
    402 		TAILQ_REMOVE(&mvp->v_mount->mnt_vnodelist, mvip, vi_mntvnodes);
    403 		mvp->v_usecount = 0;
    404 		mutex_exit(lock);
    405 	}
    406 	vnfree_marker(mvp);
    407 }
    408 
    409 static struct vnode *
    410 vfs_vnode_iterator_next1(struct vnode_iterator *vni,
    411     bool (*f)(void *, struct vnode *), void *cl, bool do_wait)
    412 {
    413 	vnode_impl_t *mvip = &vni->vi_vnode;
    414 	struct mount *mp = VIMPL_TO_VNODE(mvip)->v_mount;
    415 	vnode_t *vp;
    416 	vnode_impl_t *vip;
    417 	kmutex_t *lock;
    418 	int error;
    419 
    420 	KASSERT(vnis_marker(VIMPL_TO_VNODE(mvip)));
    421 
    422 	lock = mp->mnt_vnodelock;
    423 	do {
    424 		mutex_enter(lock);
    425 		vip = TAILQ_NEXT(mvip, vi_mntvnodes);
    426 		TAILQ_REMOVE(&mp->mnt_vnodelist, mvip, vi_mntvnodes);
    427 		VIMPL_TO_VNODE(mvip)->v_usecount = 0;
    428 again:
    429 		if (vip == NULL) {
    430 			mutex_exit(lock);
    431 	       		return NULL;
    432 		}
    433 		vp = VIMPL_TO_VNODE(vip);
    434 		KASSERT(vp != NULL);
    435 		mutex_enter(vp->v_interlock);
    436 		if (vnis_marker(vp) ||
    437 		    vdead_check(vp, (do_wait ? 0 : VDEAD_NOWAIT)) ||
    438 		    (f && !(*f)(cl, vp))) {
    439 			mutex_exit(vp->v_interlock);
    440 			vip = TAILQ_NEXT(vip, vi_mntvnodes);
    441 			goto again;
    442 		}
    443 
    444 		TAILQ_INSERT_AFTER(&mp->mnt_vnodelist, vip, mvip, vi_mntvnodes);
    445 		VIMPL_TO_VNODE(mvip)->v_usecount = 1;
    446 		mutex_exit(lock);
    447 		error = vcache_vget(vp);
    448 		KASSERT(error == 0 || error == ENOENT);
    449 	} while (error != 0);
    450 
    451 	return vp;
    452 }
    453 
    454 struct vnode *
    455 vfs_vnode_iterator_next(struct vnode_iterator *vni,
    456     bool (*f)(void *, struct vnode *), void *cl)
    457 {
    458 
    459 	return vfs_vnode_iterator_next1(vni, f, cl, false);
    460 }
    461 
    462 /*
    463  * Move a vnode from one mount queue to another.
    464  */
    465 void
    466 vfs_insmntque(vnode_t *vp, struct mount *mp)
    467 {
    468 	vnode_impl_t *vip = VNODE_TO_VIMPL(vp);
    469 	struct mount *omp;
    470 	kmutex_t *lock;
    471 
    472 	KASSERT(mp == NULL || (mp->mnt_iflag & IMNT_UNMOUNT) == 0 ||
    473 	    vp->v_tag == VT_VFS);
    474 
    475 	/*
    476 	 * Delete from old mount point vnode list, if on one.
    477 	 */
    478 	if ((omp = vp->v_mount) != NULL) {
    479 		lock = omp->mnt_vnodelock;
    480 		mutex_enter(lock);
    481 		TAILQ_REMOVE(&vp->v_mount->mnt_vnodelist, vip, vi_mntvnodes);
    482 		mutex_exit(lock);
    483 	}
    484 
    485 	/*
    486 	 * Insert into list of vnodes for the new mount point, if
    487 	 * available.  The caller must take a reference on the mount
    488 	 * structure and donate to the vnode.
    489 	 */
    490 	if ((vp->v_mount = mp) != NULL) {
    491 		lock = mp->mnt_vnodelock;
    492 		mutex_enter(lock);
    493 		TAILQ_INSERT_TAIL(&mp->mnt_vnodelist, vip, vi_mntvnodes);
    494 		mutex_exit(lock);
    495 	}
    496 
    497 	if (omp != NULL) {
    498 		/* Release reference to old mount. */
    499 		vfs_rele(omp);
    500 	}
    501 }
    502 
    503 /*
    504  * Remove any vnodes in the vnode table belonging to mount point mp.
    505  *
    506  * If FORCECLOSE is not specified, there should not be any active ones,
    507  * return error if any are found (nb: this is a user error, not a
    508  * system error). If FORCECLOSE is specified, detach any active vnodes
    509  * that are found.
    510  *
    511  * If WRITECLOSE is set, only flush out regular file vnodes open for
    512  * writing.
    513  *
    514  * SKIPSYSTEM causes any vnodes marked VV_SYSTEM to be skipped.
    515  */
    516 #ifdef DEBUG
    517 int busyprt = 0;	/* print out busy vnodes */
    518 struct ctldebug debug1 = { "busyprt", &busyprt };
    519 #endif
    520 
    521 static vnode_t *
    522 vflushnext(struct vnode_iterator *marker, int *when)
    523 {
    524 	if (getticks() > *when) {
    525 		yield();
    526 		*when = getticks() + hz / 10;
    527 	}
    528 	return vfs_vnode_iterator_next1(marker, NULL, NULL, true);
    529 }
    530 
    531 /*
    532  * Flush one vnode.  Referenced on entry, unreferenced on return.
    533  */
    534 static int
    535 vflush_one(vnode_t *vp, vnode_t *skipvp, int flags)
    536 {
    537 	int error;
    538 	struct vattr vattr;
    539 
    540 	if (vp == skipvp ||
    541 	    ((flags & SKIPSYSTEM) && (vp->v_vflag & VV_SYSTEM))) {
    542 		vrele(vp);
    543 		return 0;
    544 	}
    545 	/*
    546 	 * If WRITECLOSE is set, only flush out regular file
    547 	 * vnodes open for writing or open and unlinked.
    548 	 */
    549 	if ((flags & WRITECLOSE)) {
    550 		if (vp->v_type != VREG) {
    551 			vrele(vp);
    552 			return 0;
    553 		}
    554 		error = vn_lock(vp, LK_EXCLUSIVE);
    555 		if (error) {
    556 			KASSERT(error == ENOENT);
    557 			vrele(vp);
    558 			return 0;
    559 		}
    560 		error = VOP_FSYNC(vp, curlwp->l_cred, FSYNC_WAIT, 0, 0);
    561 		if (error == 0)
    562 			error = VOP_GETATTR(vp, &vattr, curlwp->l_cred);
    563 		VOP_UNLOCK(vp);
    564 		if (error) {
    565 			vrele(vp);
    566 			return error;
    567 		}
    568 		if (vp->v_writecount == 0 && vattr.va_nlink > 0) {
    569 			vrele(vp);
    570 			return 0;
    571 		}
    572 	}
    573 	/*
    574 	 * First try to recycle the vnode.
    575 	 */
    576 	if (vrecycle(vp))
    577 		return 0;
    578 	/*
    579 	 * If FORCECLOSE is set, forcibly close the vnode.
    580 	 * For block or character devices, revert to an
    581 	 * anonymous device.  For all other files, just
    582 	 * kill them.
    583 	 */
    584 	if (flags & FORCECLOSE) {
    585 		if (vrefcnt(vp) > 1 &&
    586 		    (vp->v_type == VBLK || vp->v_type == VCHR))
    587 			vcache_make_anon(vp);
    588 		else
    589 			vgone(vp);
    590 		return 0;
    591 	}
    592 	vrele(vp);
    593 	return EBUSY;
    594 }
    595 
    596 int
    597 vflush(struct mount *mp, vnode_t *skipvp, int flags)
    598 {
    599 	vnode_t *vp;
    600 	struct vnode_iterator *marker;
    601 	int busy, error, when, retries = 2;
    602 
    603 	do {
    604 		busy = error = when = 0;
    605 
    606 		/*
    607 		 * First, flush out any vnode references from the
    608 		 * deferred vrele list.
    609 		 */
    610 		vrele_flush(mp);
    611 
    612 		vfs_vnode_iterator_init(mp, &marker);
    613 
    614 		while ((vp = vflushnext(marker, &when)) != NULL) {
    615 			error = vflush_one(vp, skipvp, flags);
    616 			if (error == EBUSY) {
    617 				error = 0;
    618 				busy++;
    619 #ifdef DEBUG
    620 				if (busyprt && retries == 0)
    621 					vprint("vflush: busy vnode", vp);
    622 #endif
    623 			} else if (error != 0) {
    624 				break;
    625 			}
    626 		}
    627 
    628 		vfs_vnode_iterator_destroy(marker);
    629 	} while (error == 0 && busy > 0 && retries-- > 0);
    630 
    631 	if (error)
    632 		return error;
    633 	if (busy)
    634 		return EBUSY;
    635 	return 0;
    636 }
    637 
    638 /*
    639  * Mount a file system.
    640  */
    641 
    642 /*
    643  * Scan all active processes to see if any of them have a current or root
    644  * directory onto which the new filesystem has just been  mounted. If so,
    645  * replace them with the new mount point.
    646  */
    647 static void
    648 mount_checkdirs(vnode_t *olddp)
    649 {
    650 	vnode_t *newdp, *rele1, *rele2;
    651 	struct cwdinfo *cwdi;
    652 	struct proc *p;
    653 	bool retry;
    654 
    655 	if (vrefcnt(olddp) == 1) {
    656 		return;
    657 	}
    658 	if (VFS_ROOT(olddp->v_mountedhere, LK_EXCLUSIVE, &newdp))
    659 		panic("mount: lost mount");
    660 
    661 	do {
    662 		retry = false;
    663 		mutex_enter(proc_lock);
    664 		PROCLIST_FOREACH(p, &allproc) {
    665 			if ((cwdi = p->p_cwdi) == NULL)
    666 				continue;
    667 			/*
    668 			 * Cannot change to the old directory any more,
    669 			 * so even if we see a stale value it is not a
    670 			 * problem.
    671 			 */
    672 			if (cwdi->cwdi_cdir != olddp &&
    673 			    cwdi->cwdi_rdir != olddp)
    674 				continue;
    675 			retry = true;
    676 			rele1 = NULL;
    677 			rele2 = NULL;
    678 			atomic_inc_uint(&cwdi->cwdi_refcnt);
    679 			mutex_exit(proc_lock);
    680 			rw_enter(&cwdi->cwdi_lock, RW_WRITER);
    681 			if (cwdi->cwdi_cdir == olddp) {
    682 				rele1 = cwdi->cwdi_cdir;
    683 				vref(newdp);
    684 				cwdi->cwdi_cdir = newdp;
    685 			}
    686 			if (cwdi->cwdi_rdir == olddp) {
    687 				rele2 = cwdi->cwdi_rdir;
    688 				vref(newdp);
    689 				cwdi->cwdi_rdir = newdp;
    690 			}
    691 			rw_exit(&cwdi->cwdi_lock);
    692 			cwdfree(cwdi);
    693 			if (rele1 != NULL)
    694 				vrele(rele1);
    695 			if (rele2 != NULL)
    696 				vrele(rele2);
    697 			mutex_enter(proc_lock);
    698 			break;
    699 		}
    700 		mutex_exit(proc_lock);
    701 	} while (retry);
    702 
    703 	if (rootvnode == olddp) {
    704 		vrele(rootvnode);
    705 		vref(newdp);
    706 		rootvnode = newdp;
    707 	}
    708 	vput(newdp);
    709 }
    710 
    711 /*
    712  * Start extended attributes
    713  */
    714 static int
    715 start_extattr(struct mount *mp)
    716 {
    717 	int error;
    718 
    719 	error = VFS_EXTATTRCTL(mp, EXTATTR_CMD_START, NULL, 0, NULL);
    720 	if (error)
    721 		printf("%s: failed to start extattr: error = %d\n",
    722 		       mp->mnt_stat.f_mntonname, error);
    723 
    724 	return error;
    725 }
    726 
    727 int
    728 mount_domount(struct lwp *l, vnode_t **vpp, struct vfsops *vfsops,
    729     const char *path, int flags, void *data, size_t *data_len)
    730 {
    731 	vnode_t *vp = *vpp;
    732 	struct mount *mp;
    733 	struct pathbuf *pb;
    734 	struct nameidata nd;
    735 	int error;
    736 
    737 	error = kauth_authorize_system(l->l_cred, KAUTH_SYSTEM_MOUNT,
    738 	    KAUTH_REQ_SYSTEM_MOUNT_NEW, vp, KAUTH_ARG(flags), data);
    739 	if (error) {
    740 		vfs_delref(vfsops);
    741 		return error;
    742 	}
    743 
    744 	/* Cannot make a non-dir a mount-point (from here anyway). */
    745 	if (vp->v_type != VDIR) {
    746 		vfs_delref(vfsops);
    747 		return ENOTDIR;
    748 	}
    749 
    750 	if (flags & MNT_EXPORTED) {
    751 		vfs_delref(vfsops);
    752 		return EINVAL;
    753 	}
    754 
    755 	if ((mp = vfs_mountalloc(vfsops, vp)) == NULL) {
    756 		vfs_delref(vfsops);
    757 		return ENOMEM;
    758 	}
    759 
    760 	mp->mnt_stat.f_owner = kauth_cred_geteuid(l->l_cred);
    761 
    762 	/*
    763 	 * The underlying file system may refuse the mount for
    764 	 * various reasons.  Allow the user to force it to happen.
    765 	 *
    766 	 * Set the mount level flags.
    767 	 */
    768 	mp->mnt_flag = flags & (MNT_BASIC_FLAGS | MNT_FORCE | MNT_IGNORE);
    769 
    770 	mutex_enter(mp->mnt_updating);
    771 	error = VFS_MOUNT(mp, path, data, data_len);
    772 	mp->mnt_flag &= ~MNT_OP_FLAGS;
    773 
    774 	if (error != 0)
    775 		goto err_unmounted;
    776 
    777 	/*
    778 	 * Validate and prepare the mount point.
    779 	 */
    780 	error = pathbuf_copyin(path, &pb);
    781 	if (error != 0) {
    782 		goto err_mounted;
    783 	}
    784 	NDINIT(&nd, LOOKUP, FOLLOW | LOCKLEAF | TRYEMULROOT, pb);
    785 	error = namei(&nd);
    786 	pathbuf_destroy(pb);
    787 	if (error != 0) {
    788 		goto err_mounted;
    789 	}
    790 	if (nd.ni_vp != vp) {
    791 		vput(nd.ni_vp);
    792 		error = EINVAL;
    793 		goto err_mounted;
    794 	}
    795 	if (vp->v_mountedhere != NULL) {
    796 		vput(nd.ni_vp);
    797 		error = EBUSY;
    798 		goto err_mounted;
    799 	}
    800 	error = vinvalbuf(vp, V_SAVE, l->l_cred, l, 0, 0);
    801 	if (error != 0) {
    802 		vput(nd.ni_vp);
    803 		goto err_mounted;
    804 	}
    805 
    806 	/*
    807 	 * Put the new filesystem on the mount list after root.
    808 	 */
    809 	cache_purge(vp);
    810 	mp->mnt_iflag &= ~IMNT_WANTRDWR;
    811 
    812 	mountlist_append(mp);
    813 	if ((mp->mnt_flag & (MNT_RDONLY | MNT_ASYNC)) == 0)
    814 		vfs_syncer_add_to_worklist(mp);
    815 	vp->v_mountedhere = mp;
    816 	vput(nd.ni_vp);
    817 
    818 	mount_checkdirs(vp);
    819 	mutex_exit(mp->mnt_updating);
    820 
    821 	/* Hold an additional reference to the mount across VFS_START(). */
    822 	vfs_ref(mp);
    823 	(void) VFS_STATVFS(mp, &mp->mnt_stat);
    824 	error = VFS_START(mp, 0);
    825 	if (error) {
    826 		vrele(vp);
    827 	} else if (flags & MNT_EXTATTR) {
    828 		if (start_extattr(mp) != 0)
    829 			mp->mnt_flag &= ~MNT_EXTATTR;
    830 	}
    831 	/* Drop reference held for VFS_START(). */
    832 	vfs_rele(mp);
    833 	*vpp = NULL;
    834 	return error;
    835 
    836 err_mounted:
    837 	if (VFS_UNMOUNT(mp, MNT_FORCE) != 0)
    838 		panic("Unmounting fresh file system failed");
    839 
    840 err_unmounted:
    841 	vp->v_mountedhere = NULL;
    842 	mutex_exit(mp->mnt_updating);
    843 	vfs_rele(mp);
    844 
    845 	return error;
    846 }
    847 
    848 /*
    849  * Do the actual file system unmount.  File system is assumed to have
    850  * been locked by the caller.
    851  *
    852  * => Caller hold reference to the mount, explicitly for dounmount().
    853  */
    854 int
    855 dounmount(struct mount *mp, int flags, struct lwp *l)
    856 {
    857 	vnode_t *coveredvp;
    858 	int error, async, used_syncer, used_extattr;
    859 	const bool was_suspended = fstrans_is_owner(mp);
    860 
    861 #if NVERIEXEC > 0
    862 	error = veriexec_unmountchk(mp);
    863 	if (error)
    864 		return (error);
    865 #endif /* NVERIEXEC > 0 */
    866 
    867 	if (!was_suspended) {
    868 		error = vfs_suspend(mp, 0);
    869 		if (error) {
    870 			return error;
    871 		}
    872 	}
    873 
    874 	KASSERT((mp->mnt_iflag & IMNT_GONE) == 0);
    875 
    876 	used_syncer = (mp->mnt_iflag & IMNT_ONWORKLIST) != 0;
    877 	used_extattr = mp->mnt_flag & MNT_EXTATTR;
    878 
    879 	mp->mnt_iflag |= IMNT_UNMOUNT;
    880 	mutex_enter(mp->mnt_updating);
    881 	async = mp->mnt_flag & MNT_ASYNC;
    882 	mp->mnt_flag &= ~MNT_ASYNC;
    883 	cache_purgevfs(mp);	/* remove cache entries for this file sys */
    884 	if (used_syncer)
    885 		vfs_syncer_remove_from_worklist(mp);
    886 	error = 0;
    887 	if (((mp->mnt_flag & MNT_RDONLY) == 0) && ((flags & MNT_FORCE) == 0)) {
    888 		error = VFS_SYNC(mp, MNT_WAIT, l->l_cred);
    889 	}
    890 	if (error == 0 || (flags & MNT_FORCE)) {
    891 		error = VFS_UNMOUNT(mp, flags);
    892 	}
    893 	if (error) {
    894 		mp->mnt_iflag &= ~IMNT_UNMOUNT;
    895 		if ((mp->mnt_flag & (MNT_RDONLY | MNT_ASYNC)) == 0)
    896 			vfs_syncer_add_to_worklist(mp);
    897 		mp->mnt_flag |= async;
    898 		mutex_exit(mp->mnt_updating);
    899 		if (!was_suspended)
    900 			vfs_resume(mp);
    901 		if (used_extattr) {
    902 			if (start_extattr(mp) != 0)
    903 				mp->mnt_flag &= ~MNT_EXTATTR;
    904 			else
    905 				mp->mnt_flag |= MNT_EXTATTR;
    906 		}
    907 		return (error);
    908 	}
    909 	mutex_exit(mp->mnt_updating);
    910 
    911 	/*
    912 	 * mark filesystem as gone to prevent further umounts
    913 	 * after mnt_umounting lock is gone, this also prevents
    914 	 * vfs_busy() from succeeding.
    915 	 */
    916 	mp->mnt_iflag |= IMNT_GONE;
    917 	if (!was_suspended)
    918 		vfs_resume(mp);
    919 
    920 	if ((coveredvp = mp->mnt_vnodecovered) != NULLVP) {
    921 		vn_lock(coveredvp, LK_EXCLUSIVE | LK_RETRY);
    922 		coveredvp->v_mountedhere = NULL;
    923 		VOP_UNLOCK(coveredvp);
    924 	}
    925 	mountlist_remove(mp);
    926 	if (TAILQ_FIRST(&mp->mnt_vnodelist) != NULL)
    927 		panic("unmount: dangling vnode");
    928 	vfs_hooks_unmount(mp);
    929 
    930 	vfs_rele(mp);	/* reference from mount() */
    931 	if (coveredvp != NULLVP) {
    932 		vrele(coveredvp);
    933 	}
    934 	return (0);
    935 }
    936 
    937 /*
    938  * Unmount all file systems.
    939  * We traverse the list in reverse order under the assumption that doing so
    940  * will avoid needing to worry about dependencies.
    941  */
    942 bool
    943 vfs_unmountall(struct lwp *l)
    944 {
    945 
    946 	printf("unmounting file systems...\n");
    947 	return vfs_unmountall1(l, true, true);
    948 }
    949 
    950 static void
    951 vfs_unmount_print(struct mount *mp, const char *pfx)
    952 {
    953 
    954 	aprint_verbose("%sunmounted %s on %s type %s\n", pfx,
    955 	    mp->mnt_stat.f_mntfromname, mp->mnt_stat.f_mntonname,
    956 	    mp->mnt_stat.f_fstypename);
    957 }
    958 
    959 /*
    960  * Return the mount with the highest generation less than "gen".
    961  */
    962 static struct mount *
    963 vfs_unmount_next(uint64_t gen)
    964 {
    965 	mount_iterator_t *iter;
    966 	struct mount *mp, *nmp;
    967 
    968 	nmp = NULL;
    969 
    970 	mountlist_iterator_init(&iter);
    971 	while ((mp = mountlist_iterator_next(iter)) != NULL) {
    972 		if ((nmp == NULL || mp->mnt_gen > nmp->mnt_gen) &&
    973 		    mp->mnt_gen < gen) {
    974 			if (nmp != NULL)
    975 				vfs_rele(nmp);
    976 			nmp = mp;
    977 			vfs_ref(nmp);
    978 		}
    979 	}
    980 	mountlist_iterator_destroy(iter);
    981 
    982 	return nmp;
    983 }
    984 
    985 bool
    986 vfs_unmount_forceone(struct lwp *l)
    987 {
    988 	struct mount *mp;
    989 	int error;
    990 
    991 	mp = vfs_unmount_next(mountgen);
    992 	if (mp == NULL) {
    993 		return false;
    994 	}
    995 
    996 #ifdef DEBUG
    997 	printf("forcefully unmounting %s (%s)...\n",
    998 	    mp->mnt_stat.f_mntonname, mp->mnt_stat.f_mntfromname);
    999 #endif
   1000 	if ((error = dounmount(mp, MNT_FORCE, l)) == 0) {
   1001 		vfs_unmount_print(mp, "forcefully ");
   1002 		return true;
   1003 	} else {
   1004 		vfs_rele(mp);
   1005 	}
   1006 
   1007 #ifdef DEBUG
   1008 	printf("forceful unmount of %s failed with error %d\n",
   1009 	    mp->mnt_stat.f_mntonname, error);
   1010 #endif
   1011 
   1012 	return false;
   1013 }
   1014 
   1015 bool
   1016 vfs_unmountall1(struct lwp *l, bool force, bool verbose)
   1017 {
   1018 	struct mount *mp;
   1019 	mount_iterator_t *iter;
   1020 	bool any_error = false, progress = false;
   1021 	uint64_t gen;
   1022 	int error;
   1023 
   1024 	gen = mountgen;
   1025 	for (;;) {
   1026 		mp = vfs_unmount_next(gen);
   1027 		if (mp == NULL)
   1028 			break;
   1029 		gen = mp->mnt_gen;
   1030 
   1031 #ifdef DEBUG
   1032 		printf("unmounting %p %s (%s)...\n",
   1033 		    (void *)mp, mp->mnt_stat.f_mntonname,
   1034 		    mp->mnt_stat.f_mntfromname);
   1035 #endif
   1036 		if ((error = dounmount(mp, force ? MNT_FORCE : 0, l)) == 0) {
   1037 			vfs_unmount_print(mp, "");
   1038 			progress = true;
   1039 		} else {
   1040 			vfs_rele(mp);
   1041 			if (verbose) {
   1042 				printf("unmount of %s failed with error %d\n",
   1043 				    mp->mnt_stat.f_mntonname, error);
   1044 			}
   1045 			any_error = true;
   1046 		}
   1047 	}
   1048 	if (verbose) {
   1049 		printf("unmounting done\n");
   1050 	}
   1051 	if (any_error && verbose) {
   1052 		printf("WARNING: some file systems would not unmount\n");
   1053 	}
   1054 
   1055 	/* If the mountlist is empty destroy anonymous device vnodes. */
   1056 	mountlist_iterator_init(&iter);
   1057 	if (mountlist_iterator_next(iter) == NULL) {
   1058 		struct vnode_iterator *marker;
   1059 		vnode_t *vp;
   1060 
   1061 		vfs_vnode_iterator_init(dead_rootmount, &marker);
   1062 		while ((vp = vfs_vnode_iterator_next(marker, NULL, NULL))) {
   1063 			if (vp->v_type == VCHR || vp->v_type == VBLK)
   1064 				vgone(vp);
   1065 			else
   1066 				vrele(vp);
   1067 		}
   1068 		vfs_vnode_iterator_destroy(marker);
   1069 	}
   1070 	mountlist_iterator_destroy(iter);
   1071 
   1072 	return progress;
   1073 }
   1074 
   1075 void
   1076 vfs_sync_all(struct lwp *l)
   1077 {
   1078 	printf("syncing disks... ");
   1079 
   1080 	/* remove user processes from run queue */
   1081 	suspendsched();
   1082 	(void)spl0();
   1083 
   1084 	/* avoid coming back this way again if we panic. */
   1085 	doing_shutdown = 1;
   1086 
   1087 	do_sys_sync(l);
   1088 
   1089 	/* Wait for sync to finish. */
   1090 	if (vfs_syncwait() != 0) {
   1091 #if defined(DDB) && defined(DEBUG_HALT_BUSY)
   1092 		Debugger();
   1093 #endif
   1094 		printf("giving up\n");
   1095 		return;
   1096 	} else
   1097 		printf("done\n");
   1098 }
   1099 
   1100 /*
   1101  * Sync and unmount file systems before shutting down.
   1102  */
   1103 void
   1104 vfs_shutdown(void)
   1105 {
   1106 	lwp_t *l = curlwp;
   1107 
   1108 	vfs_sync_all(l);
   1109 
   1110 	/*
   1111 	 * If we have paniced - do not make the situation potentially
   1112 	 * worse by unmounting the file systems.
   1113 	 */
   1114 	if (panicstr != NULL) {
   1115 		return;
   1116 	}
   1117 
   1118 	/* Unmount file systems. */
   1119 	vfs_unmountall(l);
   1120 }
   1121 
   1122 /*
   1123  * Print a list of supported file system types (used by vfs_mountroot)
   1124  */
   1125 static void
   1126 vfs_print_fstypes(void)
   1127 {
   1128 	struct vfsops *v;
   1129 	int cnt = 0;
   1130 
   1131 	mutex_enter(&vfs_list_lock);
   1132 	LIST_FOREACH(v, &vfs_list, vfs_list)
   1133 		++cnt;
   1134 	mutex_exit(&vfs_list_lock);
   1135 
   1136 	if (cnt == 0) {
   1137 		printf("WARNING: No file system modules have been loaded.\n");
   1138 		return;
   1139 	}
   1140 
   1141 	printf("Supported file systems:");
   1142 	mutex_enter(&vfs_list_lock);
   1143 	LIST_FOREACH(v, &vfs_list, vfs_list) {
   1144 		printf(" %s", v->vfs_name);
   1145 	}
   1146 	mutex_exit(&vfs_list_lock);
   1147 	printf("\n");
   1148 }
   1149 
   1150 /*
   1151  * Mount the root file system.  If the operator didn't specify a
   1152  * file system to use, try all possible file systems until one
   1153  * succeeds.
   1154  */
   1155 int
   1156 vfs_mountroot(void)
   1157 {
   1158 	struct vfsops *v;
   1159 	int error = ENODEV;
   1160 
   1161 	if (root_device == NULL)
   1162 		panic("vfs_mountroot: root device unknown");
   1163 
   1164 	switch (device_class(root_device)) {
   1165 	case DV_IFNET:
   1166 		if (rootdev != NODEV)
   1167 			panic("vfs_mountroot: rootdev set for DV_IFNET "
   1168 			    "(0x%llx -> %llu,%llu)",
   1169 			    (unsigned long long)rootdev,
   1170 			    (unsigned long long)major(rootdev),
   1171 			    (unsigned long long)minor(rootdev));
   1172 		break;
   1173 
   1174 	case DV_DISK:
   1175 		if (rootdev == NODEV)
   1176 			panic("vfs_mountroot: rootdev not set for DV_DISK");
   1177 	        if (bdevvp(rootdev, &rootvp))
   1178 	                panic("vfs_mountroot: can't get vnode for rootdev");
   1179 		error = VOP_OPEN(rootvp, FREAD, FSCRED);
   1180 		if (error) {
   1181 			printf("vfs_mountroot: can't open root device\n");
   1182 			return (error);
   1183 		}
   1184 		break;
   1185 
   1186 	case DV_VIRTUAL:
   1187 		break;
   1188 
   1189 	default:
   1190 		printf("%s: inappropriate for root file system\n",
   1191 		    device_xname(root_device));
   1192 		return (ENODEV);
   1193 	}
   1194 
   1195 	/*
   1196 	 * If user specified a root fs type, use it.  Make sure the
   1197 	 * specified type exists and has a mount_root()
   1198 	 */
   1199 	if (strcmp(rootfstype, ROOT_FSTYPE_ANY) != 0) {
   1200 		v = vfs_getopsbyname(rootfstype);
   1201 		error = EFTYPE;
   1202 		if (v != NULL) {
   1203 			if (v->vfs_mountroot != NULL) {
   1204 				error = (v->vfs_mountroot)();
   1205 			}
   1206 			v->vfs_refcount--;
   1207 		}
   1208 		goto done;
   1209 	}
   1210 
   1211 	/*
   1212 	 * Try each file system currently configured into the kernel.
   1213 	 */
   1214 	mutex_enter(&vfs_list_lock);
   1215 	LIST_FOREACH(v, &vfs_list, vfs_list) {
   1216 		if (v->vfs_mountroot == NULL)
   1217 			continue;
   1218 #ifdef DEBUG
   1219 		aprint_normal("mountroot: trying %s...\n", v->vfs_name);
   1220 #endif
   1221 		v->vfs_refcount++;
   1222 		mutex_exit(&vfs_list_lock);
   1223 		error = (*v->vfs_mountroot)();
   1224 		mutex_enter(&vfs_list_lock);
   1225 		v->vfs_refcount--;
   1226 		if (!error) {
   1227 			aprint_normal("root file system type: %s\n",
   1228 			    v->vfs_name);
   1229 			break;
   1230 		}
   1231 	}
   1232 	mutex_exit(&vfs_list_lock);
   1233 
   1234 	if (v == NULL) {
   1235 		vfs_print_fstypes();
   1236 		printf("no file system for %s", device_xname(root_device));
   1237 		if (device_class(root_device) == DV_DISK)
   1238 			printf(" (dev 0x%llx)", (unsigned long long)rootdev);
   1239 		printf("\n");
   1240 		error = EFTYPE;
   1241 	}
   1242 
   1243 done:
   1244 	if (error && device_class(root_device) == DV_DISK) {
   1245 		VOP_CLOSE(rootvp, FREAD, FSCRED);
   1246 		vrele(rootvp);
   1247 	}
   1248 	if (error == 0) {
   1249 		mount_iterator_t *iter;
   1250 		struct mount *mp;
   1251 		extern struct cwdinfo cwdi0;
   1252 
   1253 		mountlist_iterator_init(&iter);
   1254 		mp = mountlist_iterator_next(iter);
   1255 		KASSERT(mp != NULL);
   1256 		mountlist_iterator_destroy(iter);
   1257 
   1258 		mp->mnt_flag |= MNT_ROOTFS;
   1259 		mp->mnt_op->vfs_refcount++;
   1260 
   1261 		/*
   1262 		 * Get the vnode for '/'.  Set cwdi0.cwdi_cdir to
   1263 		 * reference it, and donate it the reference grabbed
   1264 		 * with VFS_ROOT().
   1265 		 */
   1266 		error = VFS_ROOT(mp, LK_NONE, &rootvnode);
   1267 		if (error)
   1268 			panic("cannot find root vnode, error=%d", error);
   1269 		cwdi0.cwdi_cdir = rootvnode;
   1270 		cwdi0.cwdi_rdir = NULL;
   1271 
   1272 		/*
   1273 		 * Now that root is mounted, we can fixup initproc's CWD
   1274 		 * info.  All other processes are kthreads, which merely
   1275 		 * share proc0's CWD info.
   1276 		 */
   1277 		initproc->p_cwdi->cwdi_cdir = rootvnode;
   1278 		vref(initproc->p_cwdi->cwdi_cdir);
   1279 		initproc->p_cwdi->cwdi_rdir = NULL;
   1280 		/*
   1281 		 * Enable loading of modules from the filesystem
   1282 		 */
   1283 		module_load_vfs_init();
   1284 
   1285 	}
   1286 	return (error);
   1287 }
   1288 
   1289 /*
   1290  * mount_specific_key_create --
   1291  *	Create a key for subsystem mount-specific data.
   1292  */
   1293 int
   1294 mount_specific_key_create(specificdata_key_t *keyp, specificdata_dtor_t dtor)
   1295 {
   1296 
   1297 	return specificdata_key_create(mount_specificdata_domain, keyp, dtor);
   1298 }
   1299 
   1300 /*
   1301  * mount_specific_key_delete --
   1302  *	Delete a key for subsystem mount-specific data.
   1303  */
   1304 void
   1305 mount_specific_key_delete(specificdata_key_t key)
   1306 {
   1307 
   1308 	specificdata_key_delete(mount_specificdata_domain, key);
   1309 }
   1310 
   1311 /*
   1312  * mount_initspecific --
   1313  *	Initialize a mount's specificdata container.
   1314  */
   1315 void
   1316 mount_initspecific(struct mount *mp)
   1317 {
   1318 	int error __diagused;
   1319 
   1320 	error = specificdata_init(mount_specificdata_domain,
   1321 				  &mp->mnt_specdataref);
   1322 	KASSERT(error == 0);
   1323 }
   1324 
   1325 /*
   1326  * mount_finispecific --
   1327  *	Finalize a mount's specificdata container.
   1328  */
   1329 void
   1330 mount_finispecific(struct mount *mp)
   1331 {
   1332 
   1333 	specificdata_fini(mount_specificdata_domain, &mp->mnt_specdataref);
   1334 }
   1335 
   1336 /*
   1337  * mount_getspecific --
   1338  *	Return mount-specific data corresponding to the specified key.
   1339  */
   1340 void *
   1341 mount_getspecific(struct mount *mp, specificdata_key_t key)
   1342 {
   1343 
   1344 	return specificdata_getspecific(mount_specificdata_domain,
   1345 					 &mp->mnt_specdataref, key);
   1346 }
   1347 
   1348 /*
   1349  * mount_setspecific --
   1350  *	Set mount-specific data corresponding to the specified key.
   1351  */
   1352 void
   1353 mount_setspecific(struct mount *mp, specificdata_key_t key, void *data)
   1354 {
   1355 
   1356 	specificdata_setspecific(mount_specificdata_domain,
   1357 				 &mp->mnt_specdataref, key, data);
   1358 }
   1359 
   1360 /*
   1361  * Check to see if a filesystem is mounted on a block device.
   1362  */
   1363 int
   1364 vfs_mountedon(vnode_t *vp)
   1365 {
   1366 	vnode_t *vq;
   1367 	int error = 0;
   1368 
   1369 	if (vp->v_type != VBLK)
   1370 		return ENOTBLK;
   1371 	if (spec_node_getmountedfs(vp) != NULL)
   1372 		return EBUSY;
   1373 	if (spec_node_lookup_by_dev(vp->v_type, vp->v_rdev, &vq) == 0) {
   1374 		if (spec_node_getmountedfs(vq) != NULL)
   1375 			error = EBUSY;
   1376 		vrele(vq);
   1377 	}
   1378 
   1379 	return error;
   1380 }
   1381 
   1382 /*
   1383  * Check if a device pointed to by vp is mounted.
   1384  *
   1385  * Returns:
   1386  *   EINVAL	if it's not a disk
   1387  *   EBUSY	if it's a disk and mounted
   1388  *   0		if it's a disk and not mounted
   1389  */
   1390 int
   1391 rawdev_mounted(vnode_t *vp, vnode_t **bvpp)
   1392 {
   1393 	vnode_t *bvp;
   1394 	dev_t dev;
   1395 	int d_type;
   1396 
   1397 	bvp = NULL;
   1398 	d_type = D_OTHER;
   1399 
   1400 	if (iskmemvp(vp))
   1401 		return EINVAL;
   1402 
   1403 	switch (vp->v_type) {
   1404 	case VCHR: {
   1405 		const struct cdevsw *cdev;
   1406 
   1407 		dev = vp->v_rdev;
   1408 		cdev = cdevsw_lookup(dev);
   1409 		if (cdev != NULL) {
   1410 			dev_t blkdev;
   1411 
   1412 			blkdev = devsw_chr2blk(dev);
   1413 			if (blkdev != NODEV) {
   1414 				if (vfinddev(blkdev, VBLK, &bvp) != 0) {
   1415 					d_type = (cdev->d_flag & D_TYPEMASK);
   1416 					/* XXX: what if bvp disappears? */
   1417 					vrele(bvp);
   1418 				}
   1419 			}
   1420 		}
   1421 
   1422 		break;
   1423 		}
   1424 
   1425 	case VBLK: {
   1426 		const struct bdevsw *bdev;
   1427 
   1428 		dev = vp->v_rdev;
   1429 		bdev = bdevsw_lookup(dev);
   1430 		if (bdev != NULL)
   1431 			d_type = (bdev->d_flag & D_TYPEMASK);
   1432 
   1433 		bvp = vp;
   1434 
   1435 		break;
   1436 		}
   1437 
   1438 	default:
   1439 		break;
   1440 	}
   1441 
   1442 	if (d_type != D_DISK)
   1443 		return EINVAL;
   1444 
   1445 	if (bvpp != NULL)
   1446 		*bvpp = bvp;
   1447 
   1448 	/*
   1449 	 * XXX: This is bogus. We should be failing the request
   1450 	 * XXX: not only if this specific slice is mounted, but
   1451 	 * XXX: if it's on a disk with any other mounted slice.
   1452 	 */
   1453 	if (vfs_mountedon(bvp))
   1454 		return EBUSY;
   1455 
   1456 	return 0;
   1457 }
   1458 
   1459 /*
   1460  * Make a 'unique' number from a mount type name.
   1461  */
   1462 long
   1463 makefstype(const char *type)
   1464 {
   1465 	long rv;
   1466 
   1467 	for (rv = 0; *type; type++) {
   1468 		rv <<= 2;
   1469 		rv ^= *type;
   1470 	}
   1471 	return rv;
   1472 }
   1473 
   1474 static struct mountlist_entry *
   1475 mountlist_alloc(enum mountlist_type type, struct mount *mp)
   1476 {
   1477 	struct mountlist_entry *me;
   1478 
   1479 	me = kmem_zalloc(sizeof(*me), KM_SLEEP);
   1480 	me->me_mount = mp;
   1481 	me->me_type = type;
   1482 
   1483 	return me;
   1484 }
   1485 
   1486 static void
   1487 mountlist_free(struct mountlist_entry *me)
   1488 {
   1489 
   1490 	kmem_free(me, sizeof(*me));
   1491 }
   1492 
   1493 void
   1494 mountlist_iterator_init(mount_iterator_t **mip)
   1495 {
   1496 	struct mountlist_entry *me;
   1497 
   1498 	me = mountlist_alloc(ME_MARKER, NULL);
   1499 	mutex_enter(&mountlist_lock);
   1500 	TAILQ_INSERT_HEAD(&mountlist, me, me_list);
   1501 	mutex_exit(&mountlist_lock);
   1502 	*mip = (mount_iterator_t *)me;
   1503 }
   1504 
   1505 void
   1506 mountlist_iterator_destroy(mount_iterator_t *mi)
   1507 {
   1508 	struct mountlist_entry *marker = &mi->mi_entry;
   1509 
   1510 	if (marker->me_mount != NULL)
   1511 		vfs_unbusy(marker->me_mount);
   1512 
   1513 	mutex_enter(&mountlist_lock);
   1514 	TAILQ_REMOVE(&mountlist, marker, me_list);
   1515 	mutex_exit(&mountlist_lock);
   1516 
   1517 	mountlist_free(marker);
   1518 
   1519 }
   1520 
   1521 /*
   1522  * Return the next mount or NULL for this iterator.
   1523  * Mark it busy on success.
   1524  */
   1525 static inline struct mount *
   1526 _mountlist_iterator_next(mount_iterator_t *mi, bool wait)
   1527 {
   1528 	struct mountlist_entry *me, *marker = &mi->mi_entry;
   1529 	struct mount *mp;
   1530 	int error;
   1531 
   1532 	if (marker->me_mount != NULL) {
   1533 		vfs_unbusy(marker->me_mount);
   1534 		marker->me_mount = NULL;
   1535 	}
   1536 
   1537 	mutex_enter(&mountlist_lock);
   1538 	for (;;) {
   1539 		KASSERT(marker->me_type == ME_MARKER);
   1540 
   1541 		me = TAILQ_NEXT(marker, me_list);
   1542 		if (me == NULL) {
   1543 			/* End of list: keep marker and return. */
   1544 			mutex_exit(&mountlist_lock);
   1545 			return NULL;
   1546 		}
   1547 		TAILQ_REMOVE(&mountlist, marker, me_list);
   1548 		TAILQ_INSERT_AFTER(&mountlist, me, marker, me_list);
   1549 
   1550 		/* Skip other markers. */
   1551 		if (me->me_type != ME_MOUNT)
   1552 			continue;
   1553 
   1554 		/* Take an initial reference for vfs_busy() below. */
   1555 		mp = me->me_mount;
   1556 		KASSERT(mp != NULL);
   1557 		vfs_ref(mp);
   1558 		mutex_exit(&mountlist_lock);
   1559 
   1560 		/* Try to mark this mount busy and return on success. */
   1561 		if (wait)
   1562 			error = vfs_busy(mp);
   1563 		else
   1564 			error = vfs_trybusy(mp);
   1565 		if (error == 0) {
   1566 			vfs_rele(mp);
   1567 			marker->me_mount = mp;
   1568 			return mp;
   1569 		}
   1570 		vfs_rele(mp);
   1571 		mutex_enter(&mountlist_lock);
   1572 	}
   1573 }
   1574 
   1575 struct mount *
   1576 mountlist_iterator_next(mount_iterator_t *mi)
   1577 {
   1578 
   1579 	return _mountlist_iterator_next(mi, true);
   1580 }
   1581 
   1582 struct mount *
   1583 mountlist_iterator_trynext(mount_iterator_t *mi)
   1584 {
   1585 
   1586 	return _mountlist_iterator_next(mi, false);
   1587 }
   1588 
   1589 /*
   1590  * Attach new mount to the end of the mount list.
   1591  */
   1592 void
   1593 mountlist_append(struct mount *mp)
   1594 {
   1595 	struct mountlist_entry *me;
   1596 
   1597 	me = mountlist_alloc(ME_MOUNT, mp);
   1598 	mutex_enter(&mountlist_lock);
   1599 	TAILQ_INSERT_TAIL(&mountlist, me, me_list);
   1600 	mutex_exit(&mountlist_lock);
   1601 }
   1602 
   1603 /*
   1604  * Remove mount from mount list.
   1605  */void
   1606 mountlist_remove(struct mount *mp)
   1607 {
   1608 	struct mountlist_entry *me;
   1609 
   1610 	mutex_enter(&mountlist_lock);
   1611 	TAILQ_FOREACH(me, &mountlist, me_list)
   1612 		if (me->me_type == ME_MOUNT && me->me_mount == mp)
   1613 			break;
   1614 	KASSERT(me != NULL);
   1615 	TAILQ_REMOVE(&mountlist, me, me_list);
   1616 	mutex_exit(&mountlist_lock);
   1617 	mountlist_free(me);
   1618 }
   1619 
   1620 /*
   1621  * Unlocked variant to traverse the mountlist.
   1622  * To be used from DDB only.
   1623  */
   1624 struct mount *
   1625 _mountlist_next(struct mount *mp)
   1626 {
   1627 	struct mountlist_entry *me;
   1628 
   1629 	if (mp == NULL) {
   1630 		me = TAILQ_FIRST(&mountlist);
   1631 	} else {
   1632 		TAILQ_FOREACH(me, &mountlist, me_list)
   1633 			if (me->me_type == ME_MOUNT && me->me_mount == mp)
   1634 				break;
   1635 		if (me != NULL)
   1636 			me = TAILQ_NEXT(me, me_list);
   1637 	}
   1638 
   1639 	while (me != NULL && me->me_type != ME_MOUNT)
   1640 		me = TAILQ_NEXT(me, me_list);
   1641 
   1642 	return (me ? me->me_mount : NULL);
   1643 }
   1644