Home | History | Annotate | Line # | Download | only in kern
vfs_trans.c revision 1.34
      1 /*	$NetBSD: vfs_trans.c,v 1.34 2015/08/24 22:50:32 pooka Exp $	*/
      2 
      3 /*-
      4  * Copyright (c) 2007 The NetBSD Foundation, Inc.
      5  * All rights reserved.
      6  *
      7  * This code is derived from software contributed to The NetBSD Foundation
      8  * by Juergen Hannken-Illjes.
      9  *
     10  * Redistribution and use in source and binary forms, with or without
     11  * modification, are permitted provided that the following conditions
     12  * are met:
     13  * 1. Redistributions of source code must retain the above copyright
     14  *    notice, this list of conditions and the following disclaimer.
     15  * 2. Redistributions in binary form must reproduce the above copyright
     16  *    notice, this list of conditions and the following disclaimer in the
     17  *    documentation and/or other materials provided with the distribution.
     18  *
     19  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     20  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     21  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     22  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     23  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     24  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     25  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     26  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     27  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     28  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     29  * POSSIBILITY OF SUCH DAMAGE.
     30  */
     31 
     32 #include <sys/cdefs.h>
     33 __KERNEL_RCSID(0, "$NetBSD: vfs_trans.c,v 1.34 2015/08/24 22:50:32 pooka Exp $");
     34 
     35 /*
     36  * File system transaction operations.
     37  */
     38 
     39 #ifdef _KERNEL_OPT
     40 #include "opt_ddb.h"
     41 #endif
     42 
     43 #include <sys/param.h>
     44 #include <sys/systm.h>
     45 #include <sys/atomic.h>
     46 #include <sys/buf.h>
     47 #include <sys/kmem.h>
     48 #include <sys/mount.h>
     49 #include <sys/pserialize.h>
     50 #include <sys/vnode.h>
     51 #define _FSTRANS_API_PRIVATE
     52 #include <sys/fstrans.h>
     53 #include <sys/proc.h>
     54 
     55 #include <miscfs/specfs/specdev.h>
     56 
     57 struct fscow_handler {
     58 	LIST_ENTRY(fscow_handler) ch_list;
     59 	int (*ch_func)(void *, struct buf *, bool);
     60 	void *ch_arg;
     61 };
     62 struct fstrans_lwp_info {
     63 	struct fstrans_lwp_info *fli_succ;
     64 	struct lwp *fli_self;
     65 	struct mount *fli_mount;
     66 	int fli_trans_cnt;
     67 	int fli_cow_cnt;
     68 	enum fstrans_lock_type fli_lock_type;
     69 	LIST_ENTRY(fstrans_lwp_info) fli_list;
     70 };
     71 struct fstrans_mount_info {
     72 	enum fstrans_state fmi_state;
     73 	unsigned int fmi_ref_cnt;
     74 	bool fmi_cow_change;
     75 	LIST_HEAD(, fscow_handler) fmi_cow_handler;
     76 };
     77 
     78 static specificdata_key_t lwp_data_key;	/* Our specific data key. */
     79 static kmutex_t vfs_suspend_lock;	/* Serialize suspensions. */
     80 static kmutex_t fstrans_lock;		/* Fstrans big lock. */
     81 static kcondvar_t fstrans_state_cv;	/* Fstrans or cow state changed. */
     82 static kcondvar_t fstrans_count_cv;	/* Fstrans or cow count changed. */
     83 static pserialize_t fstrans_psz;	/* Pserialize state. */
     84 static LIST_HEAD(fstrans_lwp_head, fstrans_lwp_info) fstrans_fli_head;
     85 					/* List of all fstrans_lwp_info. */
     86 
     87 static void fstrans_lwp_dtor(void *);
     88 static void fstrans_mount_dtor(struct mount *);
     89 static struct fstrans_lwp_info *fstrans_get_lwp_info(struct mount *, bool);
     90 static bool grant_lock(const enum fstrans_state, const enum fstrans_lock_type);
     91 static bool state_change_done(const struct mount *);
     92 static bool cow_state_change_done(const struct mount *);
     93 static void cow_change_enter(const struct mount *);
     94 static void cow_change_done(const struct mount *);
     95 
     96 /*
     97  * Initialize.
     98  */
     99 void
    100 fstrans_init(void)
    101 {
    102 	int error __diagused;
    103 
    104 	error = lwp_specific_key_create(&lwp_data_key, fstrans_lwp_dtor);
    105 	KASSERT(error == 0);
    106 
    107 	mutex_init(&vfs_suspend_lock, MUTEX_DEFAULT, IPL_NONE);
    108 	mutex_init(&fstrans_lock, MUTEX_DEFAULT, IPL_NONE);
    109 	cv_init(&fstrans_state_cv, "fstchg");
    110 	cv_init(&fstrans_count_cv, "fstcnt");
    111 	fstrans_psz = pserialize_create();
    112 	LIST_INIT(&fstrans_fli_head);
    113 }
    114 
    115 /*
    116  * Deallocate lwp state.
    117  */
    118 static void
    119 fstrans_lwp_dtor(void *arg)
    120 {
    121 	struct fstrans_lwp_info *fli, *fli_next;
    122 
    123 	for (fli = arg; fli; fli = fli_next) {
    124 		KASSERT(fli->fli_trans_cnt == 0);
    125 		KASSERT(fli->fli_cow_cnt == 0);
    126 		if (fli->fli_mount != NULL)
    127 			fstrans_mount_dtor(fli->fli_mount);
    128 		fli_next = fli->fli_succ;
    129 		fli->fli_mount = NULL;
    130 		membar_sync();
    131 		fli->fli_self = NULL;
    132 	}
    133 }
    134 
    135 /*
    136  * Dereference mount state.
    137  */
    138 static void
    139 fstrans_mount_dtor(struct mount *mp)
    140 {
    141 	struct fstrans_mount_info *fmi;
    142 
    143 	fmi = mp->mnt_transinfo;
    144 	if (atomic_dec_uint_nv(&fmi->fmi_ref_cnt) > 0)
    145 		return;
    146 
    147 	KASSERT(fmi->fmi_state == FSTRANS_NORMAL);
    148 	KASSERT(LIST_FIRST(&fmi->fmi_cow_handler) == NULL);
    149 
    150 	kmem_free(fmi, sizeof(*fmi));
    151 	mp->mnt_iflag &= ~IMNT_HAS_TRANS;
    152 	mp->mnt_transinfo = NULL;
    153 
    154 	vfs_destroy(mp);
    155 }
    156 
    157 /*
    158  * Allocate mount state.
    159  */
    160 int
    161 fstrans_mount(struct mount *mp)
    162 {
    163 	int error;
    164 	struct fstrans_mount_info *newfmi;
    165 
    166 	error = vfs_busy(mp, NULL);
    167 	if (error)
    168 		return error;
    169 	newfmi = kmem_alloc(sizeof(*newfmi), KM_SLEEP);
    170 	newfmi->fmi_state = FSTRANS_NORMAL;
    171 	newfmi->fmi_ref_cnt = 1;
    172 	LIST_INIT(&newfmi->fmi_cow_handler);
    173 	newfmi->fmi_cow_change = false;
    174 
    175 	mp->mnt_transinfo = newfmi;
    176 	mp->mnt_iflag |= IMNT_HAS_TRANS;
    177 
    178 	vfs_unbusy(mp, true, NULL);
    179 
    180 	return 0;
    181 }
    182 
    183 /*
    184  * Deallocate mount state.
    185  */
    186 void
    187 fstrans_unmount(struct mount *mp)
    188 {
    189 
    190 	KASSERT(mp->mnt_transinfo != NULL);
    191 
    192 	fstrans_mount_dtor(mp);
    193 }
    194 
    195 /*
    196  * Retrieve the per lwp info for this mount allocating if necessary.
    197  */
    198 static struct fstrans_lwp_info *
    199 fstrans_get_lwp_info(struct mount *mp, bool do_alloc)
    200 {
    201 	struct fstrans_lwp_info *fli, *res;
    202 	struct fstrans_mount_info *fmi;
    203 
    204 	/*
    205 	 * Scan our list for a match clearing entries whose mount is gone.
    206 	 */
    207 	res = NULL;
    208 	for (fli = lwp_getspecific(lwp_data_key); fli; fli = fli->fli_succ) {
    209 		if (fli->fli_mount == mp) {
    210 			KASSERT(res == NULL);
    211 			res = fli;
    212 		} else if (fli->fli_mount != NULL &&
    213 		    (fli->fli_mount->mnt_iflag & IMNT_GONE) != 0 &&
    214 		    fli->fli_trans_cnt == 0 && fli->fli_cow_cnt == 0) {
    215 			fstrans_mount_dtor(fli->fli_mount);
    216 			fli->fli_mount = NULL;
    217 		}
    218 	}
    219 	if (__predict_true(res != NULL))
    220 		return res;
    221 
    222 	if (! do_alloc)
    223 		return NULL;
    224 
    225 	/*
    226 	 * Try to reuse a cleared entry or allocate a new one.
    227 	 */
    228 	for (fli = lwp_getspecific(lwp_data_key); fli; fli = fli->fli_succ) {
    229 		if (fli->fli_mount == NULL) {
    230 			KASSERT(fli->fli_trans_cnt == 0);
    231 			KASSERT(fli->fli_cow_cnt == 0);
    232 			break;
    233 		}
    234 	}
    235 	if (fli == NULL) {
    236 		mutex_enter(&fstrans_lock);
    237 		LIST_FOREACH(fli, &fstrans_fli_head, fli_list) {
    238 			if (fli->fli_self == NULL) {
    239 				KASSERT(fli->fli_trans_cnt == 0);
    240 				KASSERT(fli->fli_cow_cnt == 0);
    241 				fli->fli_self = curlwp;
    242 				fli->fli_succ = lwp_getspecific(lwp_data_key);
    243 				lwp_setspecific(lwp_data_key, fli);
    244 				break;
    245 			}
    246 		}
    247 		mutex_exit(&fstrans_lock);
    248 	}
    249 	if (fli == NULL) {
    250 		fli = kmem_alloc(sizeof(*fli), KM_SLEEP);
    251 		mutex_enter(&fstrans_lock);
    252 		memset(fli, 0, sizeof(*fli));
    253 		fli->fli_self = curlwp;
    254 		LIST_INSERT_HEAD(&fstrans_fli_head, fli, fli_list);
    255 		mutex_exit(&fstrans_lock);
    256 		fli->fli_succ = lwp_getspecific(lwp_data_key);
    257 		lwp_setspecific(lwp_data_key, fli);
    258 	}
    259 
    260 	/*
    261 	 * Attach the entry to the mount.
    262 	 */
    263 	fmi = mp->mnt_transinfo;
    264 	fli->fli_mount = mp;
    265 	atomic_inc_uint(&fmi->fmi_ref_cnt);
    266 
    267 	return fli;
    268 }
    269 
    270 /*
    271  * Check if this lock type is granted at this state.
    272  */
    273 static bool
    274 grant_lock(const enum fstrans_state state, const enum fstrans_lock_type type)
    275 {
    276 
    277 	if (__predict_true(state == FSTRANS_NORMAL))
    278 		return true;
    279 	if (type == FSTRANS_EXCL)
    280 		return true;
    281 	if  (state == FSTRANS_SUSPENDING && type == FSTRANS_LAZY)
    282 		return true;
    283 
    284 	return false;
    285 }
    286 
    287 /*
    288  * Start a transaction.  If this thread already has a transaction on this
    289  * file system increment the reference counter.
    290  */
    291 int
    292 _fstrans_start(struct mount *mp, enum fstrans_lock_type lock_type, int wait)
    293 {
    294 	int s;
    295 	struct fstrans_lwp_info *fli;
    296 	struct fstrans_mount_info *fmi;
    297 
    298 	ASSERT_SLEEPABLE();
    299 
    300 	if (mp == NULL || (mp->mnt_iflag & IMNT_HAS_TRANS) == 0)
    301 		return 0;
    302 
    303 	fli = fstrans_get_lwp_info(mp, true);
    304 
    305 	if (fli->fli_trans_cnt > 0) {
    306 		KASSERT(lock_type != FSTRANS_EXCL);
    307 		fli->fli_trans_cnt += 1;
    308 
    309 		return 0;
    310 	}
    311 
    312 	s = pserialize_read_enter();
    313 	fmi = mp->mnt_transinfo;
    314 	if (__predict_true(grant_lock(fmi->fmi_state, lock_type))) {
    315 		fli->fli_trans_cnt = 1;
    316 		fli->fli_lock_type = lock_type;
    317 		pserialize_read_exit(s);
    318 
    319 		return 0;
    320 	}
    321 	pserialize_read_exit(s);
    322 
    323 	if (! wait)
    324 		return EBUSY;
    325 
    326 	mutex_enter(&fstrans_lock);
    327 	while (! grant_lock(fmi->fmi_state, lock_type))
    328 		cv_wait(&fstrans_state_cv, &fstrans_lock);
    329 	fli->fli_trans_cnt = 1;
    330 	fli->fli_lock_type = lock_type;
    331 	mutex_exit(&fstrans_lock);
    332 
    333 	return 0;
    334 }
    335 
    336 /*
    337  * Finish a transaction.
    338  */
    339 void
    340 fstrans_done(struct mount *mp)
    341 {
    342 	int s;
    343 	struct fstrans_lwp_info *fli;
    344 	struct fstrans_mount_info *fmi;
    345 
    346 	if (mp == NULL || (mp->mnt_iflag & IMNT_HAS_TRANS) == 0)
    347 		return;
    348 
    349 	fli = fstrans_get_lwp_info(mp, false);
    350 	KASSERT(fli != NULL);
    351 	KASSERT(fli->fli_trans_cnt > 0);
    352 
    353 	if (fli->fli_trans_cnt > 1) {
    354 		fli->fli_trans_cnt -= 1;
    355 
    356 		return;
    357 	}
    358 
    359 	s = pserialize_read_enter();
    360 	fmi = mp->mnt_transinfo;
    361 	if (__predict_true(fmi->fmi_state == FSTRANS_NORMAL)) {
    362 		fli->fli_trans_cnt = 0;
    363 		pserialize_read_exit(s);
    364 
    365 		return;
    366 	}
    367 	pserialize_read_exit(s);
    368 
    369 	mutex_enter(&fstrans_lock);
    370 	fli->fli_trans_cnt = 0;
    371 	cv_signal(&fstrans_count_cv);
    372 	mutex_exit(&fstrans_lock);
    373 }
    374 
    375 /*
    376  * Check if this thread has an exclusive lock.
    377  */
    378 int
    379 fstrans_is_owner(struct mount *mp)
    380 {
    381 	struct fstrans_lwp_info *fli;
    382 
    383 	if (mp == NULL || (mp->mnt_iflag & IMNT_HAS_TRANS) == 0)
    384 		return 0;
    385 
    386 	fli = fstrans_get_lwp_info(mp, false);
    387 	if (fli == NULL || fli->fli_trans_cnt == 0)
    388 		return 0;
    389 
    390 	KASSERT(fli->fli_mount == mp);
    391 	KASSERT(fli->fli_trans_cnt > 0);
    392 
    393 	return (fli->fli_lock_type == FSTRANS_EXCL);
    394 }
    395 
    396 /*
    397  * True, if no thread is in a transaction not granted at the current state.
    398  */
    399 static bool
    400 state_change_done(const struct mount *mp)
    401 {
    402 	struct fstrans_lwp_info *fli;
    403 	struct fstrans_mount_info *fmi;
    404 
    405 	KASSERT(mutex_owned(&fstrans_lock));
    406 
    407 	fmi = mp->mnt_transinfo;
    408 	LIST_FOREACH(fli, &fstrans_fli_head, fli_list) {
    409 		if (fli->fli_mount != mp)
    410 			continue;
    411 		if (fli->fli_trans_cnt == 0)
    412 			continue;
    413 		if (grant_lock(fmi->fmi_state, fli->fli_lock_type))
    414 			continue;
    415 
    416 		return false;
    417 	}
    418 
    419 	return true;
    420 }
    421 
    422 /*
    423  * Set new file system state.
    424  */
    425 int
    426 fstrans_setstate(struct mount *mp, enum fstrans_state new_state)
    427 {
    428 	int error;
    429 	enum fstrans_state old_state;
    430 	struct fstrans_mount_info *fmi;
    431 
    432 	fmi = mp->mnt_transinfo;
    433 	old_state = fmi->fmi_state;
    434 	if (old_state == new_state)
    435 		return 0;
    436 
    437 	mutex_enter(&fstrans_lock);
    438 	fmi->fmi_state = new_state;
    439 	pserialize_perform(fstrans_psz);
    440 
    441 	/*
    442 	 * All threads see the new state now.
    443 	 * Wait for transactions invalid at this state to leave.
    444 	 */
    445 	error = 0;
    446 	while (! state_change_done(mp)) {
    447 		error = cv_wait_sig(&fstrans_count_cv, &fstrans_lock);
    448 		if (error) {
    449 			new_state = fmi->fmi_state = FSTRANS_NORMAL;
    450 			break;
    451 		}
    452 	}
    453 	cv_broadcast(&fstrans_state_cv);
    454 	mutex_exit(&fstrans_lock);
    455 
    456 	if (old_state != new_state) {
    457 		if (old_state == FSTRANS_NORMAL)
    458 			fstrans_start(mp, FSTRANS_EXCL);
    459 		if (new_state == FSTRANS_NORMAL)
    460 			fstrans_done(mp);
    461 	}
    462 
    463 	return error;
    464 }
    465 
    466 /*
    467  * Get current file system state.
    468  */
    469 enum fstrans_state
    470 fstrans_getstate(struct mount *mp)
    471 {
    472 	struct fstrans_mount_info *fmi;
    473 
    474 	fmi = mp->mnt_transinfo;
    475 	KASSERT(fmi != NULL);
    476 
    477 	return fmi->fmi_state;
    478 }
    479 
    480 /*
    481  * Request a filesystem to suspend all operations.
    482  */
    483 int
    484 vfs_suspend(struct mount *mp, int nowait)
    485 {
    486 	int error;
    487 
    488 	if (nowait) {
    489 		if (!mutex_tryenter(&vfs_suspend_lock))
    490 			return EWOULDBLOCK;
    491 	} else
    492 		mutex_enter(&vfs_suspend_lock);
    493 
    494 	mutex_enter(&syncer_mutex);
    495 	if ((error = VFS_SUSPENDCTL(mp, SUSPEND_SUSPEND)) != 0) {
    496 		mutex_exit(&syncer_mutex);
    497 		mutex_exit(&vfs_suspend_lock);
    498 	}
    499 
    500 	return error;
    501 }
    502 
    503 /*
    504  * Request a filesystem to resume all operations.
    505  */
    506 void
    507 vfs_resume(struct mount *mp)
    508 {
    509 
    510 	VFS_SUSPENDCTL(mp, SUSPEND_RESUME);
    511 	mutex_exit(&syncer_mutex);
    512 	mutex_exit(&vfs_suspend_lock);
    513 }
    514 
    515 
    516 /*
    517  * True, if no thread is running a cow handler.
    518  */
    519 static bool
    520 cow_state_change_done(const struct mount *mp)
    521 {
    522 	struct fstrans_lwp_info *fli;
    523 	struct fstrans_mount_info *fmi __diagused;
    524 
    525 	fmi = mp->mnt_transinfo;
    526 
    527 	KASSERT(mutex_owned(&fstrans_lock));
    528 	KASSERT(fmi->fmi_cow_change);
    529 
    530 	LIST_FOREACH(fli, &fstrans_fli_head, fli_list) {
    531 		if (fli->fli_mount != mp)
    532 			continue;
    533 		if (fli->fli_cow_cnt == 0)
    534 			continue;
    535 
    536 		return false;
    537 	}
    538 
    539 	return true;
    540 }
    541 
    542 /*
    543  * Prepare for changing this mounts cow list.
    544  * Returns with fstrans_lock locked.
    545  */
    546 static void
    547 cow_change_enter(const struct mount *mp)
    548 {
    549 	struct fstrans_mount_info *fmi;
    550 
    551 	fmi = mp->mnt_transinfo;
    552 
    553 	mutex_enter(&fstrans_lock);
    554 
    555 	/*
    556 	 * Wait for other threads changing the list.
    557 	 */
    558 	while (fmi->fmi_cow_change)
    559 		cv_wait(&fstrans_state_cv, &fstrans_lock);
    560 
    561 	/*
    562 	 * Wait until all threads are aware of a state change.
    563 	 */
    564 	fmi->fmi_cow_change = true;
    565 	pserialize_perform(fstrans_psz);
    566 
    567 	while (! cow_state_change_done(mp))
    568 		cv_wait(&fstrans_count_cv, &fstrans_lock);
    569 }
    570 
    571 /*
    572  * Done changing this mounts cow list.
    573  */
    574 static void
    575 cow_change_done(const struct mount *mp)
    576 {
    577 	struct fstrans_mount_info *fmi;
    578 
    579 	KASSERT(mutex_owned(&fstrans_lock));
    580 
    581 	fmi = mp->mnt_transinfo;
    582 
    583 	fmi->fmi_cow_change = false;
    584 	pserialize_perform(fstrans_psz);
    585 
    586 	cv_broadcast(&fstrans_state_cv);
    587 
    588 	mutex_exit(&fstrans_lock);
    589 }
    590 
    591 /*
    592  * Add a handler to this mount.
    593  */
    594 int
    595 fscow_establish(struct mount *mp, int (*func)(void *, struct buf *, bool),
    596     void *arg)
    597 {
    598 	struct fstrans_mount_info *fmi;
    599 	struct fscow_handler *newch;
    600 
    601 	if ((mp->mnt_iflag & IMNT_HAS_TRANS) == 0)
    602 		return EINVAL;
    603 
    604 	fmi = mp->mnt_transinfo;
    605 	KASSERT(fmi != NULL);
    606 
    607 	newch = kmem_alloc(sizeof(*newch), KM_SLEEP);
    608 	newch->ch_func = func;
    609 	newch->ch_arg = arg;
    610 
    611 	cow_change_enter(mp);
    612 	LIST_INSERT_HEAD(&fmi->fmi_cow_handler, newch, ch_list);
    613 	cow_change_done(mp);
    614 
    615 	return 0;
    616 }
    617 
    618 /*
    619  * Remove a handler from this mount.
    620  */
    621 int
    622 fscow_disestablish(struct mount *mp, int (*func)(void *, struct buf *, bool),
    623     void *arg)
    624 {
    625 	struct fstrans_mount_info *fmi;
    626 	struct fscow_handler *hp = NULL;
    627 
    628 	if ((mp->mnt_iflag & IMNT_HAS_TRANS) == 0)
    629 		return EINVAL;
    630 
    631 	fmi = mp->mnt_transinfo;
    632 	KASSERT(fmi != NULL);
    633 
    634 	cow_change_enter(mp);
    635 	LIST_FOREACH(hp, &fmi->fmi_cow_handler, ch_list)
    636 		if (hp->ch_func == func && hp->ch_arg == arg)
    637 			break;
    638 	if (hp != NULL) {
    639 		LIST_REMOVE(hp, ch_list);
    640 		kmem_free(hp, sizeof(*hp));
    641 	}
    642 	cow_change_done(mp);
    643 
    644 	return hp ? 0 : EINVAL;
    645 }
    646 
    647 /*
    648  * Check for need to copy block that is about to be written.
    649  */
    650 int
    651 fscow_run(struct buf *bp, bool data_valid)
    652 {
    653 	int error, s;
    654 	struct mount *mp;
    655 	struct fstrans_lwp_info *fli;
    656 	struct fstrans_mount_info *fmi;
    657 	struct fscow_handler *hp;
    658 
    659 	/*
    660 	 * First check if we need run the copy-on-write handler.
    661 	 */
    662 	if ((bp->b_flags & B_COWDONE))
    663 		return 0;
    664 	if (bp->b_vp == NULL) {
    665 		bp->b_flags |= B_COWDONE;
    666 		return 0;
    667 	}
    668 	if (bp->b_vp->v_type == VBLK)
    669 		mp = spec_node_getmountedfs(bp->b_vp);
    670 	else
    671 		mp = bp->b_vp->v_mount;
    672 	if (mp == NULL || (mp->mnt_iflag & IMNT_HAS_TRANS) == 0) {
    673 		bp->b_flags |= B_COWDONE;
    674 		return 0;
    675 	}
    676 
    677 	fli = fstrans_get_lwp_info(mp, true);
    678 	fmi = mp->mnt_transinfo;
    679 
    680 	/*
    681 	 * On non-recursed run check if other threads
    682 	 * want to change the list.
    683 	 */
    684 	if (fli->fli_cow_cnt == 0) {
    685 		s = pserialize_read_enter();
    686 		if (__predict_false(fmi->fmi_cow_change)) {
    687 			pserialize_read_exit(s);
    688 			mutex_enter(&fstrans_lock);
    689 			while (fmi->fmi_cow_change)
    690 				cv_wait(&fstrans_state_cv, &fstrans_lock);
    691 			fli->fli_cow_cnt = 1;
    692 			mutex_exit(&fstrans_lock);
    693 		} else {
    694 			fli->fli_cow_cnt = 1;
    695 			pserialize_read_exit(s);
    696 		}
    697 	} else
    698 		fli->fli_cow_cnt += 1;
    699 
    700 	/*
    701 	 * Run all copy-on-write handlers, stop on error.
    702 	 */
    703 	error = 0;
    704 	LIST_FOREACH(hp, &fmi->fmi_cow_handler, ch_list)
    705 		if ((error = (*hp->ch_func)(hp->ch_arg, bp, data_valid)) != 0)
    706 			break;
    707  	if (error == 0)
    708  		bp->b_flags |= B_COWDONE;
    709 
    710 	/*
    711 	 * Check if other threads want to change the list.
    712 	 */
    713 	if (fli->fli_cow_cnt > 1) {
    714 		fli->fli_cow_cnt -= 1;
    715 	} else {
    716 		s = pserialize_read_enter();
    717 		if (__predict_false(fmi->fmi_cow_change)) {
    718 			pserialize_read_exit(s);
    719 			mutex_enter(&fstrans_lock);
    720 			fli->fli_cow_cnt = 0;
    721 			cv_signal(&fstrans_count_cv);
    722 			mutex_exit(&fstrans_lock);
    723 		} else {
    724 			fli->fli_cow_cnt = 0;
    725 			pserialize_read_exit(s);
    726 		}
    727 	}
    728 
    729 	return error;
    730 }
    731 
    732 #if defined(DDB)
    733 void fstrans_dump(int);
    734 
    735 static void
    736 fstrans_print_lwp(struct proc *p, struct lwp *l, int verbose)
    737 {
    738 	char prefix[9];
    739 	struct fstrans_lwp_info *fli;
    740 
    741 	snprintf(prefix, sizeof(prefix), "%d.%d", p->p_pid, l->l_lid);
    742 	LIST_FOREACH(fli, &fstrans_fli_head, fli_list) {
    743 		if (fli->fli_self != l)
    744 			continue;
    745 		if (fli->fli_trans_cnt == 0 && fli->fli_cow_cnt == 0) {
    746 			if (! verbose)
    747 				continue;
    748 		}
    749 		printf("%-8s", prefix);
    750 		if (verbose)
    751 			printf(" @%p", fli);
    752 		if (fli->fli_mount != NULL)
    753 			printf(" (%s)", fli->fli_mount->mnt_stat.f_mntonname);
    754 		else
    755 			printf(" NULL");
    756 		if (fli->fli_trans_cnt == 0) {
    757 			printf(" -");
    758 		} else {
    759 			switch (fli->fli_lock_type) {
    760 			case FSTRANS_LAZY:
    761 				printf(" lazy");
    762 				break;
    763 			case FSTRANS_SHARED:
    764 				printf(" shared");
    765 				break;
    766 			case FSTRANS_EXCL:
    767 				printf(" excl");
    768 				break;
    769 			default:
    770 				printf(" %#x", fli->fli_lock_type);
    771 				break;
    772 			}
    773 		}
    774 		printf(" %d cow %d\n", fli->fli_trans_cnt, fli->fli_cow_cnt);
    775 		prefix[0] = '\0';
    776 	}
    777 }
    778 
    779 static void
    780 fstrans_print_mount(struct mount *mp, int verbose)
    781 {
    782 	struct fstrans_mount_info *fmi;
    783 
    784 	fmi = mp->mnt_transinfo;
    785 	if (!verbose && (fmi == NULL || fmi->fmi_state == FSTRANS_NORMAL))
    786 		return;
    787 
    788 	printf("%-16s ", mp->mnt_stat.f_mntonname);
    789 	if (fmi == NULL) {
    790 		printf("(null)\n");
    791 		return;
    792 	}
    793 	switch (fmi->fmi_state) {
    794 	case FSTRANS_NORMAL:
    795 		printf("state normal\n");
    796 		break;
    797 	case FSTRANS_SUSPENDING:
    798 		printf("state suspending\n");
    799 		break;
    800 	case FSTRANS_SUSPENDED:
    801 		printf("state suspended\n");
    802 		break;
    803 	default:
    804 		printf("state %#x\n", fmi->fmi_state);
    805 		break;
    806 	}
    807 }
    808 
    809 void
    810 fstrans_dump(int full)
    811 {
    812 	const struct proclist_desc *pd;
    813 	struct proc *p;
    814 	struct lwp *l;
    815 	struct mount *mp;
    816 
    817 	printf("Fstrans locks by lwp:\n");
    818 	for (pd = proclists; pd->pd_list != NULL; pd++)
    819 		PROCLIST_FOREACH(p, pd->pd_list)
    820 			LIST_FOREACH(l, &p->p_lwps, l_sibling)
    821 				fstrans_print_lwp(p, l, full == 1);
    822 
    823 	printf("Fstrans state by mount:\n");
    824 	TAILQ_FOREACH(mp, &mountlist, mnt_list)
    825 		fstrans_print_mount(mp, full == 1);
    826 }
    827 #endif /* defined(DDB) */
    828