Home | History | Annotate | Line # | Download | only in kern
vfs_trans.c revision 1.26.2.1
      1 /*	$NetBSD: vfs_trans.c,v 1.26.2.1 2014/05/18 17:46:08 rmind Exp $	*/
      2 
      3 /*-
      4  * Copyright (c) 2007 The NetBSD Foundation, Inc.
      5  * All rights reserved.
      6  *
      7  * This code is derived from software contributed to The NetBSD Foundation
      8  * by Juergen Hannken-Illjes.
      9  *
     10  * Redistribution and use in source and binary forms, with or without
     11  * modification, are permitted provided that the following conditions
     12  * are met:
     13  * 1. Redistributions of source code must retain the above copyright
     14  *    notice, this list of conditions and the following disclaimer.
     15  * 2. Redistributions in binary form must reproduce the above copyright
     16  *    notice, this list of conditions and the following disclaimer in the
     17  *    documentation and/or other materials provided with the distribution.
     18  *
     19  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     20  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     21  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     22  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     23  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     24  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     25  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     26  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     27  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     28  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     29  * POSSIBILITY OF SUCH DAMAGE.
     30  */
     31 
     32 #include <sys/cdefs.h>
     33 __KERNEL_RCSID(0, "$NetBSD: vfs_trans.c,v 1.26.2.1 2014/05/18 17:46:08 rmind Exp $");
     34 
     35 /*
     36  * File system transaction operations.
     37  */
     38 
     39 #include "opt_ddb.h"
     40 
     41 #include <sys/param.h>
     42 #include <sys/systm.h>
     43 #include <sys/atomic.h>
     44 #include <sys/buf.h>
     45 #include <sys/kmem.h>
     46 #include <sys/mount.h>
     47 #include <sys/pserialize.h>
     48 #include <sys/vnode.h>
     49 #define _FSTRANS_API_PRIVATE
     50 #include <sys/fstrans.h>
     51 #include <sys/proc.h>
     52 
     53 #include <miscfs/specfs/specdev.h>
     54 #include <miscfs/syncfs/syncfs.h>
     55 
     56 struct fscow_handler {
     57 	LIST_ENTRY(fscow_handler) ch_list;
     58 	int (*ch_func)(void *, struct buf *, bool);
     59 	void *ch_arg;
     60 };
     61 struct fstrans_lwp_info {
     62 	struct fstrans_lwp_info *fli_succ;
     63 	struct lwp *fli_self;
     64 	struct mount *fli_mount;
     65 	int fli_trans_cnt;
     66 	int fli_cow_cnt;
     67 	enum fstrans_lock_type fli_lock_type;
     68 	LIST_ENTRY(fstrans_lwp_info) fli_list;
     69 };
     70 struct fstrans_mount_info {
     71 	enum fstrans_state fmi_state;
     72 	unsigned int fmi_ref_cnt;
     73 	bool fmi_cow_change;
     74 	LIST_HEAD(, fscow_handler) fmi_cow_handler;
     75 };
     76 
     77 static specificdata_key_t lwp_data_key;	/* Our specific data key. */
     78 static kmutex_t vfs_suspend_lock;	/* Serialize suspensions. */
     79 static kmutex_t fstrans_lock;		/* Fstrans big lock. */
     80 static kcondvar_t fstrans_state_cv;	/* Fstrans or cow state changed. */
     81 static kcondvar_t fstrans_count_cv;	/* Fstrans or cow count changed. */
     82 static pserialize_t fstrans_psz;	/* Pserialize state. */
     83 static LIST_HEAD(fstrans_lwp_head, fstrans_lwp_info) fstrans_fli_head;
     84 					/* List of all fstrans_lwp_info. */
     85 
     86 static void fstrans_lwp_dtor(void *);
     87 static void fstrans_mount_dtor(struct mount *);
     88 static struct fstrans_lwp_info *fstrans_get_lwp_info(struct mount *, bool);
     89 static bool grant_lock(const enum fstrans_state, const enum fstrans_lock_type);
     90 static bool state_change_done(const struct mount *);
     91 static bool cow_state_change_done(const struct mount *);
     92 static void cow_change_enter(const struct mount *);
     93 static void cow_change_done(const struct mount *);
     94 
     95 /*
     96  * Initialize.
     97  */
     98 void
     99 fstrans_init(void)
    100 {
    101 	int error __diagused;
    102 
    103 	error = lwp_specific_key_create(&lwp_data_key, fstrans_lwp_dtor);
    104 	KASSERT(error == 0);
    105 
    106 	mutex_init(&vfs_suspend_lock, MUTEX_DEFAULT, IPL_NONE);
    107 	mutex_init(&fstrans_lock, MUTEX_DEFAULT, IPL_NONE);
    108 	cv_init(&fstrans_state_cv, "fstchg");
    109 	cv_init(&fstrans_count_cv, "fstcnt");
    110 	fstrans_psz = pserialize_create();
    111 	LIST_INIT(&fstrans_fli_head);
    112 }
    113 
    114 /*
    115  * Deallocate lwp state.
    116  */
    117 static void
    118 fstrans_lwp_dtor(void *arg)
    119 {
    120 	struct fstrans_lwp_info *fli, *fli_next;
    121 
    122 	for (fli = arg; fli; fli = fli_next) {
    123 		KASSERT(fli->fli_trans_cnt == 0);
    124 		KASSERT(fli->fli_cow_cnt == 0);
    125 		if (fli->fli_mount != NULL)
    126 			fstrans_mount_dtor(fli->fli_mount);
    127 		fli_next = fli->fli_succ;
    128 		fli->fli_mount = NULL;
    129 		membar_sync();
    130 		fli->fli_self = NULL;
    131 	}
    132 }
    133 
    134 /*
    135  * Dereference mount state.
    136  */
    137 static void
    138 fstrans_mount_dtor(struct mount *mp)
    139 {
    140 	struct fstrans_mount_info *fmi;
    141 
    142 	fmi = mp->mnt_transinfo;
    143 	if (atomic_dec_uint_nv(&fmi->fmi_ref_cnt) > 0)
    144 		return;
    145 
    146 	KASSERT(fmi->fmi_state == FSTRANS_NORMAL);
    147 	KASSERT(LIST_FIRST(&fmi->fmi_cow_handler) == NULL);
    148 
    149 	kmem_free(fmi, sizeof(*fmi));
    150 	mp->mnt_iflag &= ~IMNT_HAS_TRANS;
    151 	mp->mnt_transinfo = NULL;
    152 
    153 	vfs_destroy(mp);
    154 }
    155 
    156 /*
    157  * Allocate mount state.
    158  */
    159 int
    160 fstrans_mount(struct mount *mp)
    161 {
    162 	int error;
    163 	struct fstrans_mount_info *new;
    164 
    165 	error = vfs_busy(mp, NULL);
    166 	if (error)
    167 		return error;
    168 	if ((new = kmem_alloc(sizeof(*new), KM_SLEEP)) == NULL)
    169 		return ENOMEM;
    170 	new->fmi_state = FSTRANS_NORMAL;
    171 	new->fmi_ref_cnt = 1;
    172 	LIST_INIT(&new->fmi_cow_handler);
    173 	new->fmi_cow_change = false;
    174 
    175 	mp->mnt_transinfo = new;
    176 	mp->mnt_iflag |= IMNT_HAS_TRANS;
    177 
    178 	vfs_unbusy(mp, true, NULL);
    179 
    180 	return 0;
    181 }
    182 
    183 /*
    184  * Deallocate mount state.
    185  */
    186 void
    187 fstrans_unmount(struct mount *mp)
    188 {
    189 
    190 	KASSERT(mp->mnt_transinfo != NULL);
    191 
    192 	fstrans_mount_dtor(mp);
    193 }
    194 
    195 /*
    196  * Retrieve the per lwp info for this mount allocating if necessary.
    197  */
    198 static struct fstrans_lwp_info *
    199 fstrans_get_lwp_info(struct mount *mp, bool do_alloc)
    200 {
    201 	struct fstrans_lwp_info *fli, *res;
    202 	struct fstrans_mount_info *fmi;
    203 
    204 	/*
    205 	 * Scan our list for a match clearing entries whose mount is gone.
    206 	 */
    207 	res = NULL;
    208 	for (fli = lwp_getspecific(lwp_data_key); fli; fli = fli->fli_succ) {
    209 		if (fli->fli_mount == mp) {
    210 			KASSERT(res == NULL);
    211 			res = fli;
    212 		} else if (fli->fli_mount != NULL &&
    213 		    (fli->fli_mount->mnt_iflag & IMNT_GONE) != 0 &&
    214 		    fli->fli_trans_cnt == 0 && fli->fli_cow_cnt == 0) {
    215 			fstrans_mount_dtor(fli->fli_mount);
    216 			fli->fli_mount = NULL;
    217 		}
    218 	}
    219 	if (__predict_true(res != NULL))
    220 		return res;
    221 
    222 	if (! do_alloc)
    223 		return NULL;
    224 
    225 	/*
    226 	 * Try to reuse a cleared entry or allocate a new one.
    227 	 */
    228 	for (fli = lwp_getspecific(lwp_data_key); fli; fli = fli->fli_succ) {
    229 		if (fli->fli_mount == NULL) {
    230 			KASSERT(fli->fli_trans_cnt == 0);
    231 			KASSERT(fli->fli_cow_cnt == 0);
    232 			break;
    233 		}
    234 	}
    235 	if (fli == NULL) {
    236 		mutex_enter(&fstrans_lock);
    237 		LIST_FOREACH(fli, &fstrans_fli_head, fli_list) {
    238 			if (fli->fli_self == NULL) {
    239 				KASSERT(fli->fli_trans_cnt == 0);
    240 				KASSERT(fli->fli_cow_cnt == 0);
    241 				fli->fli_self = curlwp;
    242 				fli->fli_succ = lwp_getspecific(lwp_data_key);
    243 				lwp_setspecific(lwp_data_key, fli);
    244 				break;
    245 			}
    246 		}
    247 		mutex_exit(&fstrans_lock);
    248 	}
    249 	if (fli == NULL) {
    250 		fli = kmem_alloc(sizeof(*fli), KM_SLEEP);
    251 		mutex_enter(&fstrans_lock);
    252 		memset(fli, 0, sizeof(*fli));
    253 		fli->fli_self = curlwp;
    254 		LIST_INSERT_HEAD(&fstrans_fli_head, fli, fli_list);
    255 		mutex_exit(&fstrans_lock);
    256 		fli->fli_succ = lwp_getspecific(lwp_data_key);
    257 		lwp_setspecific(lwp_data_key, fli);
    258 	}
    259 
    260 	/*
    261 	 * Attach the entry to the mount.
    262 	 */
    263 	fmi = mp->mnt_transinfo;
    264 	fli->fli_mount = mp;
    265 	atomic_inc_uint(&fmi->fmi_ref_cnt);
    266 
    267 	return fli;
    268 }
    269 
    270 /*
    271  * Check if this lock type is granted at this state.
    272  */
    273 static bool
    274 grant_lock(const enum fstrans_state state, const enum fstrans_lock_type type)
    275 {
    276 
    277 	if (__predict_true(state == FSTRANS_NORMAL))
    278 		return true;
    279 	if (type == FSTRANS_EXCL)
    280 		return true;
    281 	if  (state == FSTRANS_SUSPENDING && type == FSTRANS_LAZY)
    282 		return true;
    283 
    284 	return false;
    285 }
    286 
    287 /*
    288  * Start a transaction.  If this thread already has a transaction on this
    289  * file system increment the reference counter.
    290  */
    291 int
    292 _fstrans_start(struct mount *mp, enum fstrans_lock_type lock_type, int wait)
    293 {
    294 	int s;
    295 	struct fstrans_lwp_info *fli;
    296 	struct fstrans_mount_info *fmi;
    297 
    298 	ASSERT_SLEEPABLE();
    299 
    300 	if (mp == NULL || (mp->mnt_iflag & IMNT_HAS_TRANS) == 0)
    301 		return 0;
    302 
    303 	fli = fstrans_get_lwp_info(mp, true);
    304 
    305 	if (fli->fli_trans_cnt > 0) {
    306 		KASSERT(lock_type != FSTRANS_EXCL);
    307 		fli->fli_trans_cnt += 1;
    308 
    309 		return 0;
    310 	}
    311 
    312 	s = pserialize_read_enter();
    313 	fmi = mp->mnt_transinfo;
    314 	if (__predict_true(grant_lock(fmi->fmi_state, lock_type))) {
    315 		fli->fli_trans_cnt = 1;
    316 		fli->fli_lock_type = lock_type;
    317 		pserialize_read_exit(s);
    318 
    319 		return 0;
    320 	}
    321 	pserialize_read_exit(s);
    322 
    323 	if (! wait)
    324 		return EBUSY;
    325 
    326 	mutex_enter(&fstrans_lock);
    327 	while (! grant_lock(fmi->fmi_state, lock_type))
    328 		cv_wait(&fstrans_state_cv, &fstrans_lock);
    329 	fli->fli_trans_cnt = 1;
    330 	fli->fli_lock_type = lock_type;
    331 	mutex_exit(&fstrans_lock);
    332 
    333 	return 0;
    334 }
    335 
    336 /*
    337  * Finish a transaction.
    338  */
    339 void
    340 fstrans_done(struct mount *mp)
    341 {
    342 	int s;
    343 	struct fstrans_lwp_info *fli;
    344 	struct fstrans_mount_info *fmi;
    345 
    346 	if (mp == NULL || (mp->mnt_iflag & IMNT_HAS_TRANS) == 0)
    347 		return;
    348 
    349 	fli = fstrans_get_lwp_info(mp, false);
    350 	KASSERT(fli != NULL);
    351 	KASSERT(fli->fli_trans_cnt > 0);
    352 
    353 	if (fli->fli_trans_cnt > 1) {
    354 		fli->fli_trans_cnt -= 1;
    355 
    356 		return;
    357 	}
    358 
    359 	s = pserialize_read_enter();
    360 	fmi = mp->mnt_transinfo;
    361 	if (__predict_true(fmi->fmi_state == FSTRANS_NORMAL)) {
    362 		fli->fli_trans_cnt = 0;
    363 		pserialize_read_exit(s);
    364 
    365 		return;
    366 	}
    367 	pserialize_read_exit(s);
    368 
    369 	mutex_enter(&fstrans_lock);
    370 	fli->fli_trans_cnt = 0;
    371 	cv_signal(&fstrans_count_cv);
    372 	mutex_exit(&fstrans_lock);
    373 }
    374 
    375 /*
    376  * Check if this thread has an exclusive lock.
    377  */
    378 int
    379 fstrans_is_owner(struct mount *mp)
    380 {
    381 	struct fstrans_lwp_info *fli;
    382 
    383 	if (mp == NULL || (mp->mnt_iflag & IMNT_HAS_TRANS) == 0)
    384 		return 0;
    385 
    386 	fli = fstrans_get_lwp_info(mp, false);
    387 	if (fli == NULL || fli->fli_trans_cnt == 0)
    388 		return 0;
    389 
    390 	KASSERT(fli->fli_mount == mp);
    391 	KASSERT(fli->fli_trans_cnt > 0);
    392 
    393 	return (fli->fli_lock_type == FSTRANS_EXCL);
    394 }
    395 
    396 /*
    397  * True, if no thread is in a transaction not granted at the current state.
    398  */
    399 static bool
    400 state_change_done(const struct mount *mp)
    401 {
    402 	struct fstrans_lwp_info *fli;
    403 	struct fstrans_mount_info *fmi;
    404 
    405 	KASSERT(mutex_owned(&fstrans_lock));
    406 
    407 	fmi = mp->mnt_transinfo;
    408 	LIST_FOREACH(fli, &fstrans_fli_head, fli_list) {
    409 		if (fli->fli_mount != mp)
    410 			continue;
    411 		if (fli->fli_trans_cnt == 0)
    412 			continue;
    413 		if (grant_lock(fmi->fmi_state, fli->fli_lock_type))
    414 			continue;
    415 
    416 		return false;
    417 	}
    418 
    419 	return true;
    420 }
    421 
    422 /*
    423  * Set new file system state.
    424  */
    425 int
    426 fstrans_setstate(struct mount *mp, enum fstrans_state new_state)
    427 {
    428 	int error;
    429 	enum fstrans_state old_state;
    430 	struct fstrans_mount_info *fmi;
    431 
    432 	fmi = mp->mnt_transinfo;
    433 	old_state = fmi->fmi_state;
    434 	if (old_state == new_state)
    435 		return 0;
    436 
    437 	mutex_enter(&fstrans_lock);
    438 	fmi->fmi_state = new_state;
    439 	pserialize_perform(fstrans_psz);
    440 
    441 	/*
    442 	 * All threads see the new state now.
    443 	 * Wait for transactions invalid at this state to leave.
    444 	 */
    445 	error = 0;
    446 	while (! state_change_done(mp)) {
    447 		error = cv_wait_sig(&fstrans_count_cv, &fstrans_lock);
    448 		if (error) {
    449 			new_state = fmi->fmi_state = FSTRANS_NORMAL;
    450 			break;
    451 		}
    452 	}
    453 	cv_broadcast(&fstrans_state_cv);
    454 	mutex_exit(&fstrans_lock);
    455 
    456 	if (old_state != new_state) {
    457 		if (old_state == FSTRANS_NORMAL)
    458 			fstrans_start(mp, FSTRANS_EXCL);
    459 		if (new_state == FSTRANS_NORMAL)
    460 			fstrans_done(mp);
    461 	}
    462 
    463 	return error;
    464 }
    465 
    466 /*
    467  * Get current file system state.
    468  */
    469 enum fstrans_state
    470 fstrans_getstate(struct mount *mp)
    471 {
    472 	struct fstrans_mount_info *fmi;
    473 
    474 	fmi = mp->mnt_transinfo;
    475 	KASSERT(fmi != NULL);
    476 
    477 	return fmi->fmi_state;
    478 }
    479 
    480 /*
    481  * Request a filesystem to suspend all operations.
    482  */
    483 int
    484 vfs_suspend(struct mount *mp, int nowait)
    485 {
    486 	int error;
    487 
    488 	if (nowait) {
    489 		if (!mutex_tryenter(&vfs_suspend_lock))
    490 			return EWOULDBLOCK;
    491 	} else
    492 		mutex_enter(&vfs_suspend_lock);
    493 
    494 	mutex_enter(&syncer_mutex);
    495 	if ((error = VFS_SUSPENDCTL(mp, SUSPEND_SUSPEND)) != 0) {
    496 		mutex_exit(&syncer_mutex);
    497 		mutex_exit(&vfs_suspend_lock);
    498 	}
    499 
    500 	return error;
    501 }
    502 
    503 /*
    504  * Request a filesystem to resume all operations.
    505  */
    506 void
    507 vfs_resume(struct mount *mp)
    508 {
    509 
    510 	VFS_SUSPENDCTL(mp, SUSPEND_RESUME);
    511 	mutex_exit(&syncer_mutex);
    512 	mutex_exit(&vfs_suspend_lock);
    513 }
    514 
    515 
    516 /*
    517  * True, if no thread is running a cow handler.
    518  */
    519 static bool
    520 cow_state_change_done(const struct mount *mp)
    521 {
    522 	struct fstrans_lwp_info *fli;
    523 	struct fstrans_mount_info *fmi __diagused;
    524 
    525 	fmi = mp->mnt_transinfo;
    526 
    527 	KASSERT(mutex_owned(&fstrans_lock));
    528 	KASSERT(fmi->fmi_cow_change);
    529 
    530 	LIST_FOREACH(fli, &fstrans_fli_head, fli_list) {
    531 		if (fli->fli_mount != mp)
    532 			continue;
    533 		if (fli->fli_cow_cnt == 0)
    534 			continue;
    535 
    536 		return false;
    537 	}
    538 
    539 	return true;
    540 }
    541 
    542 /*
    543  * Prepare for changing this mounts cow list.
    544  * Returns with fstrans_lock locked.
    545  */
    546 static void
    547 cow_change_enter(const struct mount *mp)
    548 {
    549 	struct fstrans_mount_info *fmi;
    550 
    551 	fmi = mp->mnt_transinfo;
    552 
    553 	mutex_enter(&fstrans_lock);
    554 
    555 	/*
    556 	 * Wait for other threads changing the list.
    557 	 */
    558 	while (fmi->fmi_cow_change)
    559 		cv_wait(&fstrans_state_cv, &fstrans_lock);
    560 
    561 	/*
    562 	 * Wait until all threads are aware of a state change.
    563 	 */
    564 	fmi->fmi_cow_change = true;
    565 	pserialize_perform(fstrans_psz);
    566 
    567 	while (! cow_state_change_done(mp))
    568 		cv_wait(&fstrans_count_cv, &fstrans_lock);
    569 }
    570 
    571 /*
    572  * Done changing this mounts cow list.
    573  */
    574 static void
    575 cow_change_done(const struct mount *mp)
    576 {
    577 	struct fstrans_mount_info *fmi;
    578 
    579 	KASSERT(mutex_owned(&fstrans_lock));
    580 
    581 	fmi = mp->mnt_transinfo;
    582 
    583 	fmi->fmi_cow_change = false;
    584 	pserialize_perform(fstrans_psz);
    585 
    586 	cv_broadcast(&fstrans_state_cv);
    587 
    588 	mutex_exit(&fstrans_lock);
    589 }
    590 
    591 /*
    592  * Add a handler to this mount.
    593  */
    594 int
    595 fscow_establish(struct mount *mp, int (*func)(void *, struct buf *, bool),
    596     void *arg)
    597 {
    598 	struct fstrans_mount_info *fmi;
    599 	struct fscow_handler *new;
    600 
    601 	if ((mp->mnt_iflag & IMNT_HAS_TRANS) == 0)
    602 		return EINVAL;
    603 
    604 	fmi = mp->mnt_transinfo;
    605 	KASSERT(fmi != NULL);
    606 
    607 	if ((new = kmem_alloc(sizeof(*new), KM_SLEEP)) == NULL)
    608 		return ENOMEM;
    609 	new->ch_func = func;
    610 	new->ch_arg = arg;
    611 
    612 	cow_change_enter(mp);
    613 	LIST_INSERT_HEAD(&fmi->fmi_cow_handler, new, ch_list);
    614 	cow_change_done(mp);
    615 
    616 	return 0;
    617 }
    618 
    619 /*
    620  * Remove a handler from this mount.
    621  */
    622 int
    623 fscow_disestablish(struct mount *mp, int (*func)(void *, struct buf *, bool),
    624     void *arg)
    625 {
    626 	struct fstrans_mount_info *fmi;
    627 	struct fscow_handler *hp = NULL;
    628 
    629 	if ((mp->mnt_iflag & IMNT_HAS_TRANS) == 0)
    630 		return EINVAL;
    631 
    632 	fmi = mp->mnt_transinfo;
    633 	KASSERT(fmi != NULL);
    634 
    635 	cow_change_enter(mp);
    636 	LIST_FOREACH(hp, &fmi->fmi_cow_handler, ch_list)
    637 		if (hp->ch_func == func && hp->ch_arg == arg)
    638 			break;
    639 	if (hp != NULL) {
    640 		LIST_REMOVE(hp, ch_list);
    641 		kmem_free(hp, sizeof(*hp));
    642 	}
    643 	cow_change_done(mp);
    644 
    645 	return hp ? 0 : EINVAL;
    646 }
    647 
    648 /*
    649  * Check for need to copy block that is about to be written.
    650  */
    651 int
    652 fscow_run(struct buf *bp, bool data_valid)
    653 {
    654 	int error, s;
    655 	struct mount *mp;
    656 	struct fstrans_lwp_info *fli;
    657 	struct fstrans_mount_info *fmi;
    658 	struct fscow_handler *hp;
    659 
    660 	/*
    661 	 * First check if we need run the copy-on-write handler.
    662 	 */
    663 	if ((bp->b_flags & B_COWDONE))
    664 		return 0;
    665 	if (bp->b_vp == NULL) {
    666 		bp->b_flags |= B_COWDONE;
    667 		return 0;
    668 	}
    669 	if (bp->b_vp->v_type == VBLK)
    670 		mp = spec_node_getmountedfs(bp->b_vp);
    671 	else
    672 		mp = bp->b_vp->v_mount;
    673 	if (mp == NULL || (mp->mnt_iflag & IMNT_HAS_TRANS) == 0) {
    674 		bp->b_flags |= B_COWDONE;
    675 		return 0;
    676 	}
    677 
    678 	fli = fstrans_get_lwp_info(mp, true);
    679 	fmi = mp->mnt_transinfo;
    680 
    681 	/*
    682 	 * On non-recursed run check if other threads
    683 	 * want to change the list.
    684 	 */
    685 	if (fli->fli_cow_cnt == 0) {
    686 		s = pserialize_read_enter();
    687 		if (__predict_false(fmi->fmi_cow_change)) {
    688 			pserialize_read_exit(s);
    689 			mutex_enter(&fstrans_lock);
    690 			while (fmi->fmi_cow_change)
    691 				cv_wait(&fstrans_state_cv, &fstrans_lock);
    692 			fli->fli_cow_cnt = 1;
    693 			mutex_exit(&fstrans_lock);
    694 		} else {
    695 			fli->fli_cow_cnt = 1;
    696 			pserialize_read_exit(s);
    697 		}
    698 	} else
    699 		fli->fli_cow_cnt += 1;
    700 
    701 	/*
    702 	 * Run all copy-on-write handlers, stop on error.
    703 	 */
    704 	error = 0;
    705 	LIST_FOREACH(hp, &fmi->fmi_cow_handler, ch_list)
    706 		if ((error = (*hp->ch_func)(hp->ch_arg, bp, data_valid)) != 0)
    707 			break;
    708  	if (error == 0)
    709  		bp->b_flags |= B_COWDONE;
    710 
    711 	/*
    712 	 * Check if other threads want to change the list.
    713 	 */
    714 	if (fli->fli_cow_cnt > 1) {
    715 		fli->fli_cow_cnt -= 1;
    716 	} else {
    717 		s = pserialize_read_enter();
    718 		if (__predict_false(fmi->fmi_cow_change)) {
    719 			pserialize_read_exit(s);
    720 			mutex_enter(&fstrans_lock);
    721 			fli->fli_cow_cnt = 0;
    722 			cv_signal(&fstrans_count_cv);
    723 			mutex_exit(&fstrans_lock);
    724 		} else {
    725 			fli->fli_cow_cnt = 0;
    726 			pserialize_read_exit(s);
    727 		}
    728 	}
    729 
    730 	return error;
    731 }
    732 
    733 #if defined(DDB)
    734 void fstrans_dump(int);
    735 
    736 static void
    737 fstrans_print_lwp(struct proc *p, struct lwp *l, int verbose)
    738 {
    739 	char prefix[9];
    740 	struct fstrans_lwp_info *fli;
    741 
    742 	snprintf(prefix, sizeof(prefix), "%d.%d", p->p_pid, l->l_lid);
    743 	LIST_FOREACH(fli, &fstrans_fli_head, fli_list) {
    744 		if (fli->fli_self != l)
    745 			continue;
    746 		if (fli->fli_trans_cnt == 0 && fli->fli_cow_cnt == 0) {
    747 			if (! verbose)
    748 				continue;
    749 		}
    750 		printf("%-8s", prefix);
    751 		if (verbose)
    752 			printf(" @%p", fli);
    753 		if (fli->fli_mount != NULL)
    754 			printf(" (%s)", fli->fli_mount->mnt_stat.f_mntonname);
    755 		else
    756 			printf(" NULL");
    757 		if (fli->fli_trans_cnt == 0) {
    758 			printf(" -");
    759 		} else {
    760 			switch (fli->fli_lock_type) {
    761 			case FSTRANS_LAZY:
    762 				printf(" lazy");
    763 				break;
    764 			case FSTRANS_SHARED:
    765 				printf(" shared");
    766 				break;
    767 			case FSTRANS_EXCL:
    768 				printf(" excl");
    769 				break;
    770 			default:
    771 				printf(" %#x", fli->fli_lock_type);
    772 				break;
    773 			}
    774 		}
    775 		printf(" %d cow %d\n", fli->fli_trans_cnt, fli->fli_cow_cnt);
    776 		prefix[0] = '\0';
    777 	}
    778 }
    779 
    780 static void
    781 fstrans_print_mount(struct mount *mp, int verbose)
    782 {
    783 	struct fstrans_mount_info *fmi;
    784 
    785 	fmi = mp->mnt_transinfo;
    786 	if (!verbose && (fmi == NULL || fmi->fmi_state == FSTRANS_NORMAL))
    787 		return;
    788 
    789 	printf("%-16s ", mp->mnt_stat.f_mntonname);
    790 	if (fmi == NULL) {
    791 		printf("(null)\n");
    792 		return;
    793 	}
    794 	switch (fmi->fmi_state) {
    795 	case FSTRANS_NORMAL:
    796 		printf("state normal\n");
    797 		break;
    798 	case FSTRANS_SUSPENDING:
    799 		printf("state suspending\n");
    800 		break;
    801 	case FSTRANS_SUSPENDED:
    802 		printf("state suspended\n");
    803 		break;
    804 	default:
    805 		printf("state %#x\n", fmi->fmi_state);
    806 		break;
    807 	}
    808 }
    809 
    810 void
    811 fstrans_dump(int full)
    812 {
    813 	const struct proclist_desc *pd;
    814 	struct proc *p;
    815 	struct lwp *l;
    816 	struct mount *mp;
    817 
    818 	printf("Fstrans locks by lwp:\n");
    819 	for (pd = proclists; pd->pd_list != NULL; pd++)
    820 		PROCLIST_FOREACH(p, pd->pd_list)
    821 			LIST_FOREACH(l, &p->p_lwps, l_sibling)
    822 				fstrans_print_lwp(p, l, full == 1);
    823 
    824 	printf("Fstrans state by mount:\n");
    825 	TAILQ_FOREACH(mp, &mountlist, mnt_list)
    826 		fstrans_print_mount(mp, full == 1);
    827 }
    828 #endif /* defined(DDB) */
    829