Home | History | Annotate | Line # | Download | only in kern
vfs_trans.c revision 1.36
      1 /*	$NetBSD: vfs_trans.c,v 1.36 2017/02/17 08:25:15 hannken Exp $	*/
      2 
      3 /*-
      4  * Copyright (c) 2007 The NetBSD Foundation, Inc.
      5  * All rights reserved.
      6  *
      7  * This code is derived from software contributed to The NetBSD Foundation
      8  * by Juergen Hannken-Illjes.
      9  *
     10  * Redistribution and use in source and binary forms, with or without
     11  * modification, are permitted provided that the following conditions
     12  * are met:
     13  * 1. Redistributions of source code must retain the above copyright
     14  *    notice, this list of conditions and the following disclaimer.
     15  * 2. Redistributions in binary form must reproduce the above copyright
     16  *    notice, this list of conditions and the following disclaimer in the
     17  *    documentation and/or other materials provided with the distribution.
     18  *
     19  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     20  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     21  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     22  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     23  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     24  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     25  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     26  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     27  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     28  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     29  * POSSIBILITY OF SUCH DAMAGE.
     30  */
     31 
     32 #include <sys/cdefs.h>
     33 __KERNEL_RCSID(0, "$NetBSD: vfs_trans.c,v 1.36 2017/02/17 08:25:15 hannken Exp $");
     34 
     35 /*
     36  * File system transaction operations.
     37  */
     38 
     39 #ifdef _KERNEL_OPT
     40 #include "opt_ddb.h"
     41 #endif
     42 
     43 #include <sys/param.h>
     44 #include <sys/systm.h>
     45 #include <sys/atomic.h>
     46 #include <sys/buf.h>
     47 #include <sys/kmem.h>
     48 #include <sys/mount.h>
     49 #include <sys/pserialize.h>
     50 #include <sys/vnode.h>
     51 #define _FSTRANS_API_PRIVATE
     52 #include <sys/fstrans.h>
     53 #include <sys/proc.h>
     54 
     55 #include <miscfs/specfs/specdev.h>
     56 
     57 struct fscow_handler {
     58 	LIST_ENTRY(fscow_handler) ch_list;
     59 	int (*ch_func)(void *, struct buf *, bool);
     60 	void *ch_arg;
     61 };
     62 struct fstrans_lwp_info {
     63 	struct fstrans_lwp_info *fli_succ;
     64 	struct lwp *fli_self;
     65 	struct mount *fli_mount;
     66 	int fli_trans_cnt;
     67 	int fli_cow_cnt;
     68 	enum fstrans_lock_type fli_lock_type;
     69 	LIST_ENTRY(fstrans_lwp_info) fli_list;
     70 };
     71 struct fstrans_mount_info {
     72 	enum fstrans_state fmi_state;
     73 	unsigned int fmi_ref_cnt;
     74 	bool fmi_cow_change;
     75 	LIST_HEAD(, fscow_handler) fmi_cow_handler;
     76 };
     77 
     78 static specificdata_key_t lwp_data_key;	/* Our specific data key. */
     79 static kmutex_t vfs_suspend_lock;	/* Serialize suspensions. */
     80 static kmutex_t fstrans_lock;		/* Fstrans big lock. */
     81 static kmutex_t fstrans_mount_lock;	/* Fstrans mount big lock. */
     82 static kcondvar_t fstrans_state_cv;	/* Fstrans or cow state changed. */
     83 static kcondvar_t fstrans_count_cv;	/* Fstrans or cow count changed. */
     84 static pserialize_t fstrans_psz;	/* Pserialize state. */
     85 static LIST_HEAD(fstrans_lwp_head, fstrans_lwp_info) fstrans_fli_head;
     86 					/* List of all fstrans_lwp_info. */
     87 
     88 static void fstrans_lwp_dtor(void *);
     89 static void fstrans_mount_dtor(struct mount *);
     90 static struct fstrans_lwp_info *fstrans_get_lwp_info(struct mount *, bool);
     91 static bool grant_lock(const enum fstrans_state, const enum fstrans_lock_type);
     92 static bool state_change_done(const struct mount *);
     93 static bool cow_state_change_done(const struct mount *);
     94 static void cow_change_enter(const struct mount *);
     95 static void cow_change_done(const struct mount *);
     96 
     97 /*
     98  * Initialize.
     99  */
    100 void
    101 fstrans_init(void)
    102 {
    103 	int error __diagused;
    104 
    105 	error = lwp_specific_key_create(&lwp_data_key, fstrans_lwp_dtor);
    106 	KASSERT(error == 0);
    107 
    108 	mutex_init(&vfs_suspend_lock, MUTEX_DEFAULT, IPL_NONE);
    109 	mutex_init(&fstrans_lock, MUTEX_DEFAULT, IPL_NONE);
    110 	mutex_init(&fstrans_mount_lock, MUTEX_DEFAULT, IPL_NONE);
    111 	cv_init(&fstrans_state_cv, "fstchg");
    112 	cv_init(&fstrans_count_cv, "fstcnt");
    113 	fstrans_psz = pserialize_create();
    114 	LIST_INIT(&fstrans_fli_head);
    115 }
    116 
    117 /*
    118  * Deallocate lwp state.
    119  */
    120 static void
    121 fstrans_lwp_dtor(void *arg)
    122 {
    123 	struct fstrans_lwp_info *fli, *fli_next;
    124 
    125 	for (fli = arg; fli; fli = fli_next) {
    126 		KASSERT(fli->fli_trans_cnt == 0);
    127 		KASSERT(fli->fli_cow_cnt == 0);
    128 		if (fli->fli_mount != NULL)
    129 			fstrans_mount_dtor(fli->fli_mount);
    130 		fli_next = fli->fli_succ;
    131 		fli->fli_mount = NULL;
    132 		membar_sync();
    133 		fli->fli_self = NULL;
    134 	}
    135 }
    136 
    137 /*
    138  * Dereference mount state.
    139  */
    140 static void
    141 fstrans_mount_dtor(struct mount *mp)
    142 {
    143 	struct fstrans_mount_info *fmi;
    144 
    145 	mutex_enter(&fstrans_mount_lock);
    146 
    147 	fmi = mp->mnt_transinfo;
    148 	KASSERT(fmi != NULL);
    149 	fmi->fmi_ref_cnt -= 1;
    150 	if (fmi->fmi_ref_cnt > 0) {
    151 		mutex_exit(&fstrans_mount_lock);
    152 		return;
    153 	}
    154 
    155 	KASSERT(fmi->fmi_state == FSTRANS_NORMAL);
    156 	KASSERT(LIST_FIRST(&fmi->fmi_cow_handler) == NULL);
    157 
    158 	mp->mnt_iflag &= ~IMNT_HAS_TRANS;
    159 	mp->mnt_transinfo = NULL;
    160 
    161 	mutex_exit(&fstrans_mount_lock);
    162 
    163 	kmem_free(fmi, sizeof(*fmi));
    164 	vfs_destroy(mp);
    165 }
    166 
    167 /*
    168  * Allocate mount state.
    169  */
    170 int
    171 fstrans_mount(struct mount *mp)
    172 {
    173 	int error;
    174 	struct fstrans_mount_info *newfmi;
    175 
    176 	error = vfs_busy(mp, NULL);
    177 	if (error)
    178 		return error;
    179 	newfmi = kmem_alloc(sizeof(*newfmi), KM_SLEEP);
    180 	newfmi->fmi_state = FSTRANS_NORMAL;
    181 	newfmi->fmi_ref_cnt = 1;
    182 	LIST_INIT(&newfmi->fmi_cow_handler);
    183 	newfmi->fmi_cow_change = false;
    184 
    185 	mutex_enter(&fstrans_mount_lock);
    186 	mp->mnt_transinfo = newfmi;
    187 	mp->mnt_iflag |= IMNT_HAS_TRANS;
    188 	mutex_exit(&fstrans_mount_lock);
    189 
    190 	vfs_unbusy(mp, true, NULL);
    191 
    192 	return 0;
    193 }
    194 
    195 /*
    196  * Deallocate mount state.
    197  */
    198 void
    199 fstrans_unmount(struct mount *mp)
    200 {
    201 
    202 	KASSERT(mp->mnt_transinfo != NULL);
    203 
    204 	fstrans_mount_dtor(mp);
    205 }
    206 
    207 /*
    208  * Retrieve the per lwp info for this mount allocating if necessary.
    209  */
    210 static struct fstrans_lwp_info *
    211 fstrans_get_lwp_info(struct mount *mp, bool do_alloc)
    212 {
    213 	struct fstrans_lwp_info *fli, *res;
    214 	struct fstrans_mount_info *fmi;
    215 
    216 	/*
    217 	 * Scan our list for a match clearing entries whose mount is gone.
    218 	 */
    219 	res = NULL;
    220 	for (fli = lwp_getspecific(lwp_data_key); fli; fli = fli->fli_succ) {
    221 		if (fli->fli_mount == mp) {
    222 			KASSERT(res == NULL);
    223 			res = fli;
    224 		} else if (fli->fli_mount != NULL &&
    225 		    (fli->fli_mount->mnt_iflag & IMNT_GONE) != 0 &&
    226 		    fli->fli_trans_cnt == 0 && fli->fli_cow_cnt == 0) {
    227 			fstrans_mount_dtor(fli->fli_mount);
    228 			fli->fli_mount = NULL;
    229 		}
    230 	}
    231 	if (__predict_true(res != NULL))
    232 		return res;
    233 
    234 	if (! do_alloc)
    235 		return NULL;
    236 
    237 	/*
    238 	 * Try to reuse a cleared entry or allocate a new one.
    239 	 */
    240 	for (fli = lwp_getspecific(lwp_data_key); fli; fli = fli->fli_succ) {
    241 		if (fli->fli_mount == NULL) {
    242 			KASSERT(fli->fli_trans_cnt == 0);
    243 			KASSERT(fli->fli_cow_cnt == 0);
    244 			break;
    245 		}
    246 	}
    247 	if (fli == NULL) {
    248 		mutex_enter(&fstrans_lock);
    249 		LIST_FOREACH(fli, &fstrans_fli_head, fli_list) {
    250 			if (fli->fli_self == NULL) {
    251 				KASSERT(fli->fli_mount == NULL);
    252 				KASSERT(fli->fli_trans_cnt == 0);
    253 				KASSERT(fli->fli_cow_cnt == 0);
    254 				fli->fli_self = curlwp;
    255 				fli->fli_succ = lwp_getspecific(lwp_data_key);
    256 				lwp_setspecific(lwp_data_key, fli);
    257 				break;
    258 			}
    259 		}
    260 		mutex_exit(&fstrans_lock);
    261 	}
    262 	if (fli == NULL) {
    263 		fli = kmem_alloc(sizeof(*fli), KM_SLEEP);
    264 		mutex_enter(&fstrans_lock);
    265 		memset(fli, 0, sizeof(*fli));
    266 		fli->fli_self = curlwp;
    267 		LIST_INSERT_HEAD(&fstrans_fli_head, fli, fli_list);
    268 		mutex_exit(&fstrans_lock);
    269 		fli->fli_succ = lwp_getspecific(lwp_data_key);
    270 		lwp_setspecific(lwp_data_key, fli);
    271 	}
    272 
    273 	/*
    274 	 * Attach the entry to the mount.
    275 	 */
    276 	mutex_enter(&fstrans_mount_lock);
    277 	if (mp == NULL || (mp->mnt_iflag & IMNT_HAS_TRANS) == 0) {
    278 		fli = NULL;
    279 	} else {
    280 		fmi = mp->mnt_transinfo;
    281 		KASSERT(fmi != NULL);
    282 		fli->fli_mount = mp;
    283 		fmi->fmi_ref_cnt += 1;
    284 	}
    285 	mutex_exit(&fstrans_mount_lock);
    286 
    287 	return fli;
    288 }
    289 
    290 /*
    291  * Check if this lock type is granted at this state.
    292  */
    293 static bool
    294 grant_lock(const enum fstrans_state state, const enum fstrans_lock_type type)
    295 {
    296 
    297 	if (__predict_true(state == FSTRANS_NORMAL))
    298 		return true;
    299 	if (type == FSTRANS_EXCL)
    300 		return true;
    301 	if  (state == FSTRANS_SUSPENDING && type == FSTRANS_LAZY)
    302 		return true;
    303 
    304 	return false;
    305 }
    306 
    307 /*
    308  * Start a transaction.  If this thread already has a transaction on this
    309  * file system increment the reference counter.
    310  */
    311 int
    312 _fstrans_start(struct mount *mp, enum fstrans_lock_type lock_type, int wait)
    313 {
    314 	int s;
    315 	struct fstrans_lwp_info *fli;
    316 	struct fstrans_mount_info *fmi;
    317 
    318 	ASSERT_SLEEPABLE();
    319 
    320 	if (mp == NULL || (fli = fstrans_get_lwp_info(mp, true)) == NULL)
    321 		return 0;
    322 
    323 	if (fli->fli_trans_cnt > 0) {
    324 		KASSERT(lock_type != FSTRANS_EXCL);
    325 		fli->fli_trans_cnt += 1;
    326 
    327 		return 0;
    328 	}
    329 
    330 	s = pserialize_read_enter();
    331 	fmi = mp->mnt_transinfo;
    332 	if (__predict_true(grant_lock(fmi->fmi_state, lock_type))) {
    333 		fli->fli_trans_cnt = 1;
    334 		fli->fli_lock_type = lock_type;
    335 		pserialize_read_exit(s);
    336 
    337 		return 0;
    338 	}
    339 	pserialize_read_exit(s);
    340 
    341 	if (! wait)
    342 		return EBUSY;
    343 
    344 	mutex_enter(&fstrans_lock);
    345 	while (! grant_lock(fmi->fmi_state, lock_type))
    346 		cv_wait(&fstrans_state_cv, &fstrans_lock);
    347 	fli->fli_trans_cnt = 1;
    348 	fli->fli_lock_type = lock_type;
    349 	mutex_exit(&fstrans_lock);
    350 
    351 	return 0;
    352 }
    353 
    354 /*
    355  * Finish a transaction.
    356  */
    357 void
    358 fstrans_done(struct mount *mp)
    359 {
    360 	int s;
    361 	struct fstrans_lwp_info *fli;
    362 	struct fstrans_mount_info *fmi;
    363 
    364 	if (mp == NULL || (fli = fstrans_get_lwp_info(mp, true)) == NULL)
    365 		return;
    366 
    367 	KASSERT(fli->fli_trans_cnt > 0);
    368 
    369 	if (fli->fli_trans_cnt > 1) {
    370 		fli->fli_trans_cnt -= 1;
    371 
    372 		return;
    373 	}
    374 
    375 	s = pserialize_read_enter();
    376 	fmi = mp->mnt_transinfo;
    377 	if (__predict_true(fmi->fmi_state == FSTRANS_NORMAL)) {
    378 		fli->fli_trans_cnt = 0;
    379 		pserialize_read_exit(s);
    380 
    381 		return;
    382 	}
    383 	pserialize_read_exit(s);
    384 
    385 	mutex_enter(&fstrans_lock);
    386 	fli->fli_trans_cnt = 0;
    387 	cv_signal(&fstrans_count_cv);
    388 	mutex_exit(&fstrans_lock);
    389 }
    390 
    391 /*
    392  * Check if this thread has an exclusive lock.
    393  */
    394 int
    395 fstrans_is_owner(struct mount *mp)
    396 {
    397 	struct fstrans_lwp_info *fli;
    398 
    399 	if (mp == NULL || (fli = fstrans_get_lwp_info(mp, false)) == NULL)
    400 		return 0;
    401 
    402 	if (fli->fli_trans_cnt == 0)
    403 		return 0;
    404 
    405 	KASSERT(fli->fli_mount == mp);
    406 	KASSERT(fli->fli_trans_cnt > 0);
    407 
    408 	return (fli->fli_lock_type == FSTRANS_EXCL);
    409 }
    410 
    411 /*
    412  * True, if no thread is in a transaction not granted at the current state.
    413  */
    414 static bool
    415 state_change_done(const struct mount *mp)
    416 {
    417 	struct fstrans_lwp_info *fli;
    418 	struct fstrans_mount_info *fmi;
    419 
    420 	KASSERT(mutex_owned(&fstrans_lock));
    421 
    422 	fmi = mp->mnt_transinfo;
    423 	LIST_FOREACH(fli, &fstrans_fli_head, fli_list) {
    424 		if (fli->fli_mount != mp)
    425 			continue;
    426 		if (fli->fli_trans_cnt == 0)
    427 			continue;
    428 		if (grant_lock(fmi->fmi_state, fli->fli_lock_type))
    429 			continue;
    430 
    431 		return false;
    432 	}
    433 
    434 	return true;
    435 }
    436 
    437 /*
    438  * Set new file system state.
    439  */
    440 int
    441 fstrans_setstate(struct mount *mp, enum fstrans_state new_state)
    442 {
    443 	int error;
    444 	enum fstrans_state old_state;
    445 	struct fstrans_mount_info *fmi;
    446 
    447 	fmi = mp->mnt_transinfo;
    448 	old_state = fmi->fmi_state;
    449 	if (old_state == new_state)
    450 		return 0;
    451 
    452 	mutex_enter(&fstrans_lock);
    453 	fmi->fmi_state = new_state;
    454 	pserialize_perform(fstrans_psz);
    455 
    456 	/*
    457 	 * All threads see the new state now.
    458 	 * Wait for transactions invalid at this state to leave.
    459 	 */
    460 	error = 0;
    461 	while (! state_change_done(mp)) {
    462 		error = cv_wait_sig(&fstrans_count_cv, &fstrans_lock);
    463 		if (error) {
    464 			new_state = fmi->fmi_state = FSTRANS_NORMAL;
    465 			break;
    466 		}
    467 	}
    468 	cv_broadcast(&fstrans_state_cv);
    469 	mutex_exit(&fstrans_lock);
    470 
    471 	if (old_state != new_state) {
    472 		if (old_state == FSTRANS_NORMAL)
    473 			fstrans_start(mp, FSTRANS_EXCL);
    474 		if (new_state == FSTRANS_NORMAL)
    475 			fstrans_done(mp);
    476 	}
    477 
    478 	return error;
    479 }
    480 
    481 /*
    482  * Get current file system state.
    483  */
    484 enum fstrans_state
    485 fstrans_getstate(struct mount *mp)
    486 {
    487 	struct fstrans_mount_info *fmi;
    488 
    489 	fmi = mp->mnt_transinfo;
    490 	KASSERT(fmi != NULL);
    491 
    492 	return fmi->fmi_state;
    493 }
    494 
    495 /*
    496  * Request a filesystem to suspend all operations.
    497  */
    498 int
    499 vfs_suspend(struct mount *mp, int nowait)
    500 {
    501 	int error;
    502 
    503 	if (nowait) {
    504 		if (!mutex_tryenter(&vfs_suspend_lock))
    505 			return EWOULDBLOCK;
    506 	} else
    507 		mutex_enter(&vfs_suspend_lock);
    508 
    509 	if ((error = VFS_SUSPENDCTL(mp, SUSPEND_SUSPEND)) != 0)
    510 		mutex_exit(&vfs_suspend_lock);
    511 
    512 	return error;
    513 }
    514 
    515 /*
    516  * Request a filesystem to resume all operations.
    517  */
    518 void
    519 vfs_resume(struct mount *mp)
    520 {
    521 
    522 	VFS_SUSPENDCTL(mp, SUSPEND_RESUME);
    523 	mutex_exit(&vfs_suspend_lock);
    524 }
    525 
    526 
    527 /*
    528  * True, if no thread is running a cow handler.
    529  */
    530 static bool
    531 cow_state_change_done(const struct mount *mp)
    532 {
    533 	struct fstrans_lwp_info *fli;
    534 	struct fstrans_mount_info *fmi __diagused;
    535 
    536 	fmi = mp->mnt_transinfo;
    537 
    538 	KASSERT(mutex_owned(&fstrans_lock));
    539 	KASSERT(fmi->fmi_cow_change);
    540 
    541 	LIST_FOREACH(fli, &fstrans_fli_head, fli_list) {
    542 		if (fli->fli_mount != mp)
    543 			continue;
    544 		if (fli->fli_cow_cnt == 0)
    545 			continue;
    546 
    547 		return false;
    548 	}
    549 
    550 	return true;
    551 }
    552 
    553 /*
    554  * Prepare for changing this mounts cow list.
    555  * Returns with fstrans_lock locked.
    556  */
    557 static void
    558 cow_change_enter(const struct mount *mp)
    559 {
    560 	struct fstrans_mount_info *fmi;
    561 
    562 	fmi = mp->mnt_transinfo;
    563 
    564 	mutex_enter(&fstrans_lock);
    565 
    566 	/*
    567 	 * Wait for other threads changing the list.
    568 	 */
    569 	while (fmi->fmi_cow_change)
    570 		cv_wait(&fstrans_state_cv, &fstrans_lock);
    571 
    572 	/*
    573 	 * Wait until all threads are aware of a state change.
    574 	 */
    575 	fmi->fmi_cow_change = true;
    576 	pserialize_perform(fstrans_psz);
    577 
    578 	while (! cow_state_change_done(mp))
    579 		cv_wait(&fstrans_count_cv, &fstrans_lock);
    580 }
    581 
    582 /*
    583  * Done changing this mounts cow list.
    584  */
    585 static void
    586 cow_change_done(const struct mount *mp)
    587 {
    588 	struct fstrans_mount_info *fmi;
    589 
    590 	KASSERT(mutex_owned(&fstrans_lock));
    591 
    592 	fmi = mp->mnt_transinfo;
    593 
    594 	fmi->fmi_cow_change = false;
    595 	pserialize_perform(fstrans_psz);
    596 
    597 	cv_broadcast(&fstrans_state_cv);
    598 
    599 	mutex_exit(&fstrans_lock);
    600 }
    601 
    602 /*
    603  * Add a handler to this mount.
    604  */
    605 int
    606 fscow_establish(struct mount *mp, int (*func)(void *, struct buf *, bool),
    607     void *arg)
    608 {
    609 	struct fstrans_mount_info *fmi;
    610 	struct fscow_handler *newch;
    611 
    612 	if ((mp->mnt_iflag & IMNT_HAS_TRANS) == 0)
    613 		return EINVAL;
    614 
    615 	fmi = mp->mnt_transinfo;
    616 	KASSERT(fmi != NULL);
    617 
    618 	newch = kmem_alloc(sizeof(*newch), KM_SLEEP);
    619 	newch->ch_func = func;
    620 	newch->ch_arg = arg;
    621 
    622 	cow_change_enter(mp);
    623 	LIST_INSERT_HEAD(&fmi->fmi_cow_handler, newch, ch_list);
    624 	cow_change_done(mp);
    625 
    626 	return 0;
    627 }
    628 
    629 /*
    630  * Remove a handler from this mount.
    631  */
    632 int
    633 fscow_disestablish(struct mount *mp, int (*func)(void *, struct buf *, bool),
    634     void *arg)
    635 {
    636 	struct fstrans_mount_info *fmi;
    637 	struct fscow_handler *hp = NULL;
    638 
    639 	if ((mp->mnt_iflag & IMNT_HAS_TRANS) == 0)
    640 		return EINVAL;
    641 
    642 	fmi = mp->mnt_transinfo;
    643 	KASSERT(fmi != NULL);
    644 
    645 	cow_change_enter(mp);
    646 	LIST_FOREACH(hp, &fmi->fmi_cow_handler, ch_list)
    647 		if (hp->ch_func == func && hp->ch_arg == arg)
    648 			break;
    649 	if (hp != NULL) {
    650 		LIST_REMOVE(hp, ch_list);
    651 		kmem_free(hp, sizeof(*hp));
    652 	}
    653 	cow_change_done(mp);
    654 
    655 	return hp ? 0 : EINVAL;
    656 }
    657 
    658 /*
    659  * Check for need to copy block that is about to be written.
    660  */
    661 int
    662 fscow_run(struct buf *bp, bool data_valid)
    663 {
    664 	int error, s;
    665 	struct mount *mp;
    666 	struct fstrans_lwp_info *fli;
    667 	struct fstrans_mount_info *fmi;
    668 	struct fscow_handler *hp;
    669 
    670 	/*
    671 	 * First check if we need run the copy-on-write handler.
    672 	 */
    673 	if ((bp->b_flags & B_COWDONE))
    674 		return 0;
    675 	if (bp->b_vp == NULL) {
    676 		bp->b_flags |= B_COWDONE;
    677 		return 0;
    678 	}
    679 	if (bp->b_vp->v_type == VBLK)
    680 		mp = spec_node_getmountedfs(bp->b_vp);
    681 	else
    682 		mp = bp->b_vp->v_mount;
    683 	if (mp == NULL || (mp->mnt_iflag & IMNT_HAS_TRANS) == 0) {
    684 		bp->b_flags |= B_COWDONE;
    685 		return 0;
    686 	}
    687 
    688 	fli = fstrans_get_lwp_info(mp, true);
    689 	fmi = mp->mnt_transinfo;
    690 
    691 	/*
    692 	 * On non-recursed run check if other threads
    693 	 * want to change the list.
    694 	 */
    695 	if (fli->fli_cow_cnt == 0) {
    696 		s = pserialize_read_enter();
    697 		if (__predict_false(fmi->fmi_cow_change)) {
    698 			pserialize_read_exit(s);
    699 			mutex_enter(&fstrans_lock);
    700 			while (fmi->fmi_cow_change)
    701 				cv_wait(&fstrans_state_cv, &fstrans_lock);
    702 			fli->fli_cow_cnt = 1;
    703 			mutex_exit(&fstrans_lock);
    704 		} else {
    705 			fli->fli_cow_cnt = 1;
    706 			pserialize_read_exit(s);
    707 		}
    708 	} else
    709 		fli->fli_cow_cnt += 1;
    710 
    711 	/*
    712 	 * Run all copy-on-write handlers, stop on error.
    713 	 */
    714 	error = 0;
    715 	LIST_FOREACH(hp, &fmi->fmi_cow_handler, ch_list)
    716 		if ((error = (*hp->ch_func)(hp->ch_arg, bp, data_valid)) != 0)
    717 			break;
    718  	if (error == 0)
    719  		bp->b_flags |= B_COWDONE;
    720 
    721 	/*
    722 	 * Check if other threads want to change the list.
    723 	 */
    724 	if (fli->fli_cow_cnt > 1) {
    725 		fli->fli_cow_cnt -= 1;
    726 	} else {
    727 		s = pserialize_read_enter();
    728 		if (__predict_false(fmi->fmi_cow_change)) {
    729 			pserialize_read_exit(s);
    730 			mutex_enter(&fstrans_lock);
    731 			fli->fli_cow_cnt = 0;
    732 			cv_signal(&fstrans_count_cv);
    733 			mutex_exit(&fstrans_lock);
    734 		} else {
    735 			fli->fli_cow_cnt = 0;
    736 			pserialize_read_exit(s);
    737 		}
    738 	}
    739 
    740 	return error;
    741 }
    742 
    743 #if defined(DDB)
    744 void fstrans_dump(int);
    745 
    746 static void
    747 fstrans_print_lwp(struct proc *p, struct lwp *l, int verbose)
    748 {
    749 	char prefix[9];
    750 	struct fstrans_lwp_info *fli;
    751 
    752 	snprintf(prefix, sizeof(prefix), "%d.%d", p->p_pid, l->l_lid);
    753 	LIST_FOREACH(fli, &fstrans_fli_head, fli_list) {
    754 		if (fli->fli_self != l)
    755 			continue;
    756 		if (fli->fli_trans_cnt == 0 && fli->fli_cow_cnt == 0) {
    757 			if (! verbose)
    758 				continue;
    759 		}
    760 		printf("%-8s", prefix);
    761 		if (verbose)
    762 			printf(" @%p", fli);
    763 		if (fli->fli_mount != NULL)
    764 			printf(" (%s)", fli->fli_mount->mnt_stat.f_mntonname);
    765 		else
    766 			printf(" NULL");
    767 		if (fli->fli_trans_cnt == 0) {
    768 			printf(" -");
    769 		} else {
    770 			switch (fli->fli_lock_type) {
    771 			case FSTRANS_LAZY:
    772 				printf(" lazy");
    773 				break;
    774 			case FSTRANS_SHARED:
    775 				printf(" shared");
    776 				break;
    777 			case FSTRANS_EXCL:
    778 				printf(" excl");
    779 				break;
    780 			default:
    781 				printf(" %#x", fli->fli_lock_type);
    782 				break;
    783 			}
    784 		}
    785 		printf(" %d cow %d\n", fli->fli_trans_cnt, fli->fli_cow_cnt);
    786 		prefix[0] = '\0';
    787 	}
    788 }
    789 
    790 static void
    791 fstrans_print_mount(struct mount *mp, int verbose)
    792 {
    793 	struct fstrans_mount_info *fmi;
    794 
    795 	fmi = mp->mnt_transinfo;
    796 	if (!verbose && (fmi == NULL || fmi->fmi_state == FSTRANS_NORMAL))
    797 		return;
    798 
    799 	printf("%-16s ", mp->mnt_stat.f_mntonname);
    800 	if (fmi == NULL) {
    801 		printf("(null)\n");
    802 		return;
    803 	}
    804 	switch (fmi->fmi_state) {
    805 	case FSTRANS_NORMAL:
    806 		printf("state normal\n");
    807 		break;
    808 	case FSTRANS_SUSPENDING:
    809 		printf("state suspending\n");
    810 		break;
    811 	case FSTRANS_SUSPENDED:
    812 		printf("state suspended\n");
    813 		break;
    814 	default:
    815 		printf("state %#x\n", fmi->fmi_state);
    816 		break;
    817 	}
    818 }
    819 
    820 void
    821 fstrans_dump(int full)
    822 {
    823 	const struct proclist_desc *pd;
    824 	struct proc *p;
    825 	struct lwp *l;
    826 	struct mount *mp;
    827 
    828 	printf("Fstrans locks by lwp:\n");
    829 	for (pd = proclists; pd->pd_list != NULL; pd++)
    830 		PROCLIST_FOREACH(p, pd->pd_list)
    831 			LIST_FOREACH(l, &p->p_lwps, l_sibling)
    832 				fstrans_print_lwp(p, l, full == 1);
    833 
    834 	printf("Fstrans state by mount:\n");
    835 	TAILQ_FOREACH(mp, &mountlist, mnt_list)
    836 		fstrans_print_mount(mp, full == 1);
    837 }
    838 #endif /* defined(DDB) */
    839