xref: /src/sys/ufs/ffs/ffs_alloc.c

/*	$NetBSD: ffs_alloc.c,v 1.169 2020/09/05 16:30:13 riastradh Exp $	*/

/*-
 * Copyright (c) 2008, 2009 The NetBSD Foundation, Inc.
 * All rights reserved.
 *
 * This code is derived from software contributed to The NetBSD Foundation
 * by Wasabi Systems, Inc.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

/*
 * Copyright (c) 2002 Networks Associates Technology, Inc.
 * All rights reserved.
 *
 * This software was developed for the FreeBSD Project by Marshall
 * Kirk McKusick and Network Associates Laboratories, the Security
 * Research Division of Network Associates, Inc. under DARPA/SPAWAR
 * contract N66001-01-C-8035 ("CBOSS"), as part of the DARPA CHATS
 * research program
 *
 * Copyright (c) 1982, 1986, 1989, 1993
 *	The Regents of the University of California.  All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 *	@(#)ffs_alloc.c	8.19 (Berkeley) 7/13/95
 */

#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: ffs_alloc.c,v 1.169 2020/09/05 16:30:13 riastradh Exp $");

#if defined(_KERNEL_OPT)
#include "opt_ffs.h"
#include "opt_quota.h"
#include "opt_uvm_page_trkown.h"
#endif

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/buf.h>
#include <sys/cprng.h>
#include <sys/kauth.h>
#include <sys/kernel.h>
#include <sys/mount.h>
#include <sys/proc.h>
#include <sys/syslog.h>
#include <sys/vnode.h>
#include <sys/wapbl.h>
#include <sys/cprng.h>

#include <miscfs/specfs/specdev.h>
#include <ufs/ufs/quota.h>
#include <ufs/ufs/ufsmount.h>
#include <ufs/ufs/inode.h>
#include <ufs/ufs/ufs_extern.h>
#include <ufs/ufs/ufs_bswap.h>
#include <ufs/ufs/ufs_wapbl.h>

#include <ufs/ffs/fs.h>
#include <ufs/ffs/ffs_extern.h>

#ifdef UVM_PAGE_TRKOWN
#include <uvm/uvm_object.h>
#include <uvm/uvm_page.h>
#endif

static daddr_t ffs_alloccg(struct inode *, int, daddr_t, int, int, int);
static daddr_t ffs_alloccgblk(struct inode *, struct buf *, daddr_t, int, int);
static ino_t ffs_dirpref(struct inode *);
static daddr_t ffs_fragextend(struct inode *, int, daddr_t, int, int);
static void ffs_fserr(struct fs *, kauth_cred_t, const char *);
static daddr_t ffs_hashalloc(struct inode *, int, daddr_t, int, int, int,
    daddr_t (*)(struct inode *, int, daddr_t, int, int, int));
static daddr_t ffs_nodealloccg(struct inode *, int, daddr_t, int, int, int);
static int32_t ffs_mapsearch(struct fs *, struct cg *,
				      daddr_t, int);
static void ffs_blkfree_common(struct ufsmount *, struct fs *, dev_t, struct buf *,
    daddr_t, long, bool);
static void ffs_freefile_common(struct ufsmount *, struct fs *, dev_t, struct buf *, ino_t,
    int, bool);

/* if 1, changes in optimalization strategy are logged */
int ffs_log_changeopt = 0;

/* in ffs_tables.c */
extern const int inside[], around[];
extern const u_char * const fragtbl[];

/* Basic consistency check for block allocations */
static int
ffs_check_bad_allocation(const char *func, struct fs *fs, daddr_t bno,
    long size, dev_t dev, ino_t inum)
{
	if ((u_int)size > fs->fs_bsize || ffs_fragoff(fs, size) != 0 ||
	    ffs_fragnum(fs, bno) + ffs_numfrags(fs, size) > fs->fs_frag) {
		panic("%s: bad size: dev = 0x%llx, bno = %" PRId64
		    " bsize = %d, size = %ld, fs = %s", func,
		    (long long)dev, bno, fs->fs_bsize, size, fs->fs_fsmnt);
	}

	if (bno >= fs->fs_size) {
		printf("%s: bad block %" PRId64 ", ino %llu\n", func, bno,
		    (unsigned long long)inum);
		ffs_fserr(fs, NOCRED, "bad block");
		return EINVAL;
	}
	return 0;
}

/*
 * Allocate a block in the file system.
 *
 * The size of the requested block is given, which must be some
 * multiple of fs_fsize and <= fs_bsize.
 * A preference may be optionally specified. If a preference is given
 * the following hierarchy is used to allocate a block:
 *   1) allocate the requested block.
 *   2) allocate a rotationally optimal block in the same cylinder.
 *   3) allocate a block in the same cylinder group.
 *   4) quadradically rehash into other cylinder groups, until an
 *      available block is located.
 * If no block preference is given the following hierarchy is used
 * to allocate a block:
 *   1) allocate a block in the cylinder group that contains the
 *      inode for the file.
 *   2) quadradically rehash into other cylinder groups, until an
 *      available block is located.
 *
 * => called with um_lock held
 * => releases um_lock before returning
 */
int
ffs_alloc(struct inode *ip, daddr_t lbn, daddr_t bpref, int size,
    int flags, kauth_cred_t cred, daddr_t *bnp)
{
	struct ufsmount *ump;
	struct fs *fs;
	daddr_t bno;
	int cg;
#if defined(QUOTA) || defined(QUOTA2)
	int error;
#endif

	fs = ip->i_fs;
	ump = ip->i_ump;

	KASSERT(mutex_owned(&ump->um_lock));

#ifdef UVM_PAGE_TRKOWN

	/*
	 * Sanity-check that allocations within the file size
	 * do not allow other threads to read the stale contents
	 * of newly allocated blocks.
	 * Usually pages will exist to cover the new allocation.
	 * There is an optimization in ffs_write() where we skip
	 * creating pages if several conditions are met:
	 *  - the file must not be mapped (in any user address space).
	 *  - the write must cover whole pages and whole blocks.
	 * If those conditions are not met then pages must exist and
	 * be locked by the current thread.
	 */

	struct vnode *vp = ITOV(ip);
	if (vp->v_type == VREG && (flags & IO_EXT) == 0 &&
	    ffs_lblktosize(fs, (voff_t)lbn) < round_page(vp->v_size) &&
	    ((vp->v_vflag & VV_MAPPED) != 0 || (size & PAGE_MASK) != 0 ||
	     ffs_blkoff(fs, size) != 0)) {
		struct vm_page *pg __diagused;
		struct uvm_object *uobj = &vp->v_uobj;
		voff_t off = trunc_page(ffs_lblktosize(fs, lbn));
		voff_t endoff = round_page(ffs_lblktosize(fs, lbn) + size);

		rw_enter(uobj->vmobjlock, RW_WRITER);
		while (off < endoff) {
			pg = uvm_pagelookup(uobj, off);
			KASSERT((pg != NULL && pg->owner_tag != NULL &&
				 pg->owner == curproc->p_pid &&
				 pg->lowner == curlwp->l_lid));
			off += PAGE_SIZE;
		}
		rw_exit(uobj->vmobjlock);
	}
#endif

	*bnp = 0;

	KASSERTMSG((cred != NOCRED), "missing credential");
	KASSERTMSG(((u_int)size <= fs->fs_bsize),
	    "bad size: dev = 0x%llx, bsize = %d, size = %d, fs = %s",
	    (unsigned long long)ip->i_dev, fs->fs_bsize, size, fs->fs_fsmnt);
	KASSERTMSG((ffs_fragoff(fs, size) == 0),
	    "bad size: dev = 0x%llx, bsize = %d, size = %d, fs = %s",
	    (unsigned long long)ip->i_dev, fs->fs_bsize, size, fs->fs_fsmnt);

	if (size == fs->fs_bsize && fs->fs_cstotal.cs_nbfree == 0)
		goto nospace;
	if (freespace(fs, fs->fs_minfree) <= 0 &&
	    kauth_authorize_system(cred, KAUTH_SYSTEM_FS_RESERVEDSPACE, 0, NULL,
	    NULL, NULL) != 0)
		goto nospace;
#if defined(QUOTA) || defined(QUOTA2)
	mutex_exit(&ump->um_lock);
	if ((error = chkdq(ip, btodb(size), cred, 0)) != 0)
		return (error);
	mutex_enter(&ump->um_lock);
#endif

	if (bpref >= fs->fs_size)
		bpref = 0;
	if (bpref == 0)
		cg = ino_to_cg(fs, ip->i_number);
	else
		cg = dtog(fs, bpref);
	bno = ffs_hashalloc(ip, cg, bpref, size, 0, flags, ffs_alloccg);
	if (bno > 0) {
		DIP_ADD(ip, blocks, btodb(size));
		if (flags & IO_EXT)
			ip->i_flag |= IN_CHANGE;
		else
			ip->i_flag |= IN_CHANGE | IN_UPDATE;
		*bnp = bno;
		return (0);
	}
#if defined(QUOTA) || defined(QUOTA2)
	/*
	 * Restore user's disk quota because allocation failed.
	 */
	(void) chkdq(ip, -btodb(size), cred, FORCE);
#endif
	if (flags & B_CONTIG) {
		/*
		 * XXX ump->um_lock handling is "suspect" at best.
		 * For the case where ffs_hashalloc() fails early
		 * in the B_CONTIG case we reach here with um_lock
		 * already unlocked, so we can't release it again
		 * like in the normal error path.  See kern/39206.
		 *
		 *
		 * Fail silently - it's up to our caller to report
		 * errors.
		 */
		return (ENOSPC);
	}
nospace:
	mutex_exit(&ump->um_lock);
	ffs_fserr(fs, cred, "file system full");
	uprintf("\n%s: write failed, file system is full\n", fs->fs_fsmnt);
	return (ENOSPC);
}

/*
 * Reallocate a fragment to a bigger size
 *
 * The number and size of the old block is given, and a preference
 * and new size is also specified. The allocator attempts to extend
 * the original block. Failing that, the regular block allocator is
 * invoked to get an appropriate block.
 *
 * => called with um_lock held
 * => return with um_lock released
 */
int
ffs_realloccg(struct inode *ip, daddr_t lbprev, daddr_t bprev, daddr_t bpref,
    int osize, int nsize, int flags, kauth_cred_t cred, struct buf **bpp,
    daddr_t *blknop)
{
	struct ufsmount *ump;
	struct fs *fs;
	struct buf *bp;
	int cg, request, error;
	daddr_t bno;

	fs = ip->i_fs;
	ump = ip->i_ump;

	KASSERT(mutex_owned(&ump->um_lock));

#ifdef UVM_PAGE_TRKOWN

	/*
	 * Sanity-check that allocations within the file size
	 * do not allow other threads to read the stale contents
	 * of newly allocated blocks.
	 * Unlike in ffs_alloc(), here pages must always exist
	 * for such allocations, because only the last block of a file
	 * can be a fragment and ffs_write() will reallocate the
	 * fragment to the new size using ufs_balloc_range(),
	 * which always creates pages to cover blocks it allocates.
	 */

	if (ITOV(ip)->v_type == VREG) {
		struct vm_page *pg __diagused;
		struct uvm_object *uobj = &ITOV(ip)->v_uobj;
		voff_t off = trunc_page(ffs_lblktosize(fs, lbprev));
		voff_t endoff = round_page(ffs_lblktosize(fs, lbprev) + osize);

		rw_enter(uobj->vmobjlock, RW_WRITER);
		while (off < endoff) {
			pg = uvm_pagelookup(uobj, off);
			KASSERT(pg->owner == curproc->p_pid &&
				pg->lowner == curlwp->l_lid);
			off += PAGE_SIZE;
		}
		rw_exit(uobj->vmobjlock);
	}
#endif

	KASSERTMSG((cred != NOCRED), "missing credential");
	KASSERTMSG(((u_int)osize <= fs->fs_bsize),
	    "bad size: dev=0x%llx, bsize=%d, osize=%d, nsize=%d, fs=%s",
	    (unsigned long long)ip->i_dev, fs->fs_bsize, osize, nsize,
	    fs->fs_fsmnt);
	KASSERTMSG((ffs_fragoff(fs, osize) == 0),
	    "bad size: dev=0x%llx, bsize=%d, osize=%d, nsize=%d, fs=%s",
	    (unsigned long long)ip->i_dev, fs->fs_bsize, osize, nsize,
	    fs->fs_fsmnt);
	KASSERTMSG(((u_int)nsize <= fs->fs_bsize),
	    "bad size: dev=0x%llx, bsize=%d, osize=%d, nsize=%d, fs=%s",
	    (unsigned long long)ip->i_dev, fs->fs_bsize, osize, nsize,
	    fs->fs_fsmnt);
	KASSERTMSG((ffs_fragoff(fs, nsize) == 0),
	    "bad size: dev=0x%llx, bsize=%d, osize=%d, nsize=%d, fs=%s",
	    (unsigned long long)ip->i_dev, fs->fs_bsize, osize, nsize,
	    fs->fs_fsmnt);

	if (freespace(fs, fs->fs_minfree) <= 0 &&
	    kauth_authorize_system(cred, KAUTH_SYSTEM_FS_RESERVEDSPACE, 0, NULL,
	    NULL, NULL) != 0) {
		mutex_exit(&ump->um_lock);
		goto nospace;
	}

	if (bprev == 0) {
		panic("%s: bad bprev: dev = 0x%llx, bsize = %d, bprev = %"
		    PRId64 ", fs = %s", __func__,
		    (unsigned long long)ip->i_dev, fs->fs_bsize, bprev,
		    fs->fs_fsmnt);
	}
	mutex_exit(&ump->um_lock);

	/*
	 * Allocate the extra space in the buffer.
	 */
	if (bpp != NULL &&
	    (error = bread(ITOV(ip), lbprev, osize, 0, &bp)) != 0) {
		return (error);
	}
#if defined(QUOTA) || defined(QUOTA2)
	if ((error = chkdq(ip, btodb(nsize - osize), cred, 0)) != 0) {
		if (bpp != NULL) {
			brelse(bp, 0);
		}
		return (error);
	}
#endif
	/*
	 * Check for extension in the existing location.
	 */
	cg = dtog(fs, bprev);
	mutex_enter(&ump->um_lock);
	if ((bno = ffs_fragextend(ip, cg, bprev, osize, nsize)) != 0) {
		DIP_ADD(ip, blocks, btodb(nsize - osize));
		if (flags & IO_EXT)
			ip->i_flag |= IN_CHANGE;
		else
			ip->i_flag |= IN_CHANGE | IN_UPDATE;

		if (bpp != NULL) {
			if (bp->b_blkno != FFS_FSBTODB(fs, bno)) {
				panic("%s: bad blockno %#llx != %#llx",
				    __func__, (unsigned long long) bp->b_blkno,
				    (unsigned long long)FFS_FSBTODB(fs, bno));
			}
			allocbuf(bp, nsize, 1);
			memset((char *)bp->b_data + osize, 0, nsize - osize);
			mutex_enter(bp->b_objlock);
			KASSERT(!cv_has_waiters(&bp->b_done));
			bp->b_oflags |= BO_DONE;
			mutex_exit(bp->b_objlock);
			*bpp = bp;
		}
		if (blknop != NULL) {
			*blknop = bno;
		}
		return (0);
	}
	/*
	 * Allocate a new disk location.
	 */
	if (bpref >= fs->fs_size)
		bpref = 0;
	switch ((int)fs->fs_optim) {
	case FS_OPTSPACE:
		/*
		 * Allocate an exact sized fragment. Although this makes
		 * best use of space, we will waste time relocating it if
		 * the file continues to grow. If the fragmentation is
		 * less than half of the minimum free reserve, we choose
		 * to begin optimizing for time.
		 */
		request = nsize;
		if (fs->fs_minfree < 5 ||
		    fs->fs_cstotal.cs_nffree >
		    fs->fs_dsize * fs->fs_minfree / (2 * 100))
			break;

		if (ffs_log_changeopt) {
			log(LOG_NOTICE,
				"%s: optimization changed from SPACE to TIME\n",
				fs->fs_fsmnt);
		}

		fs->fs_optim = FS_OPTTIME;
		break;
	case FS_OPTTIME:
		/*
		 * At this point we have discovered a file that is trying to
		 * grow a small fragment to a larger fragment. To save time,
		 * we allocate a full sized block, then free the unused portion.
		 * If the file continues to grow, the `ffs_fragextend' call
		 * above will be able to grow it in place without further
		 * copying. If aberrant programs cause disk fragmentation to
		 * grow within 2% of the free reserve, we choose to begin
		 * optimizing for space.
		 */
		request = fs->fs_bsize;
		if (fs->fs_cstotal.cs_nffree <
		    fs->fs_dsize * (fs->fs_minfree - 2) / 100)
			break;

		if (ffs_log_changeopt) {
			log(LOG_NOTICE,
				"%s: optimization changed from TIME to SPACE\n",
				fs->fs_fsmnt);
		}

		fs->fs_optim = FS_OPTSPACE;
		break;
	default:
		panic("%s: bad optim: dev = 0x%llx, optim = %d, fs = %s",
		    __func__, (unsigned long long)ip->i_dev, fs->fs_optim,
		    fs->fs_fsmnt);
		/* NOTREACHED */
	}
	bno = ffs_hashalloc(ip, cg, bpref, request, nsize, 0, ffs_alloccg);
	if (bno > 0) {
		/*
		 * Use forced deallocation registration, we can't handle
		 * failure here. This is safe, as this place is ever hit
		 * maximum once per write operation, when fragment is extended
		 * to longer fragment, or a full block.
		 */
		if ((ip->i_ump->um_mountp->mnt_wapbl) &&
		    (ITOV(ip)->v_type != VREG)) {
			/* this should never fail */
			error = UFS_WAPBL_REGISTER_DEALLOCATION_FORCE(
			    ip->i_ump->um_mountp, FFS_FSBTODB(fs, bprev),
			    osize);
			if (error)
				panic("ffs_realloccg: dealloc registration failed");
		} else {
			ffs_blkfree(fs, ip->i_devvp, bprev, (long)osize,
			    ip->i_number);
		}
		DIP_ADD(ip, blocks, btodb(nsize - osize));
		if (flags & IO_EXT)
			ip->i_flag |= IN_CHANGE;
		else
			ip->i_flag |= IN_CHANGE | IN_UPDATE;
		if (bpp != NULL) {
			bp->b_blkno = FFS_FSBTODB(fs, bno);
			allocbuf(bp, nsize, 1);
			memset((char *)bp->b_data + osize, 0, (u_int)nsize - osize);
			mutex_enter(bp->b_objlock);
			KASSERT(!cv_has_waiters(&bp->b_done));
			bp->b_oflags |= BO_DONE;
			mutex_exit(bp->b_objlock);
			*bpp = bp;
		}
		if (blknop != NULL) {
			*blknop = bno;
		}
		return (0);
	}
	mutex_exit(&ump->um_lock);

#if defined(QUOTA) || defined(QUOTA2)
	/*
	 * Restore user's disk quota because allocation failed.
	 */
	(void) chkdq(ip, -btodb(nsize - osize), cred, FORCE);
#endif
	if (bpp != NULL) {
		brelse(bp, 0);
	}

nospace:
	/*
	 * no space available
	 */
	ffs_fserr(fs, cred, "file system full");
	uprintf("\n%s: write failed, file system is full\n", fs->fs_fsmnt);
	return (ENOSPC);
}

/*
 * Allocate an inode in the file system.
 *
 * If allocating a directory, use ffs_dirpref to select the inode.
 * If allocating in a directory, the following hierarchy is followed:
 *   1) allocate the preferred inode.
 *   2) allocate an inode in the same cylinder group.
 *   3) quadradically rehash into other cylinder groups, until an
 *      available inode is located.
 * If no inode preference is given the following hierarchy is used
 * to allocate an inode:
 *   1) allocate an inode in cylinder group 0.
 *   2) quadradically rehash into other cylinder groups, until an
 *      available inode is located.
 *
 * => um_lock not held upon entry or return
 */
int
ffs_valloc(struct vnode *pvp, int mode, kauth_cred_t cred, ino_t *inop)
{
	struct ufsmount *ump;
	struct inode *pip;
	struct fs *fs;
	ino_t ino, ipref;
	int cg, error;

	UFS_WAPBL_JUNLOCK_ASSERT(pvp->v_mount);

	pip = VTOI(pvp);
	fs = pip->i_fs;
	ump = pip->i_ump;

	error = UFS_WAPBL_BEGIN(pvp->v_mount);
	if (error) {
		return error;
	}
	mutex_enter(&ump->um_lock);
	if (fs->fs_cstotal.cs_nifree == 0)
		goto noinodes;

	if ((mode & IFMT) == IFDIR)
		ipref = ffs_dirpref(pip);
	else
		ipref = pip->i_number;
	if (ipref >= fs->fs_ncg * fs->fs_ipg)
		ipref = 0;
	cg = ino_to_cg(fs, ipref);
	/*
	 * Track number of dirs created one after another
	 * in a same cg without intervening by files.
	 */
	if ((mode & IFMT) == IFDIR) {
		if (fs->fs_contigdirs[cg] < 255)
			fs->fs_contigdirs[cg]++;
	} else {
		if (fs->fs_contigdirs[cg] > 0)
			fs->fs_contigdirs[cg]--;
	}
	ino = (ino_t)ffs_hashalloc(pip, cg, ipref, mode, 0, 0, ffs_nodealloccg);
	if (ino == 0)
		goto noinodes;
	UFS_WAPBL_END(pvp->v_mount);
	*inop = ino;
	return 0;

noinodes:
	mutex_exit(&ump->um_lock);
	UFS_WAPBL_END(pvp->v_mount);
	ffs_fserr(fs, cred, "out of inodes");
	uprintf("\n%s: create/symlink failed, no inodes free\n", fs->fs_fsmnt);
	return ENOSPC;
}

/*
 * Find a cylinder group in which to place a directory.
 *
 * The policy implemented by this algorithm is to allocate a
 * directory inode in the same cylinder group as its parent
 * directory, but also to reserve space for its files inodes
 * and data. Restrict the number of directories which may be
 * allocated one after another in the same cylinder group
 * without intervening allocation of files.
 *
 * If we allocate a first level directory then force allocation
 * in another cylinder group.
 */
static ino_t
ffs_dirpref(struct inode *pip)
{
	register struct fs *fs;
	int cg, prefcg;
	int64_t dirsize, cgsize, curdsz;
	int avgifree, avgbfree, avgndir;
	int minifree, minbfree, maxndir;
	int mincg, minndir;
	int maxcontigdirs;

	KASSERT(mutex_owned(&pip->i_ump->um_lock));

	fs = pip->i_fs;

	avgifree = fs->fs_cstotal.cs_nifree / fs->fs_ncg;
	avgbfree = fs->fs_cstotal.cs_nbfree / fs->fs_ncg;
	avgndir = fs->fs_cstotal.cs_ndir / fs->fs_ncg;

	/*
	 * Force allocation in another cg if creating a first level dir.
	 */
	if (ITOV(pip)->v_vflag & VV_ROOT) {
		prefcg = cprng_fast32() % fs->fs_ncg;
		mincg = prefcg;
		minndir = fs->fs_ipg;
		for (cg = prefcg; cg < fs->fs_ncg; cg++)
			if (fs->fs_cs(fs, cg).cs_ndir < minndir &&
			    fs->fs_cs(fs, cg).cs_nifree >= avgifree &&
			    fs->fs_cs(fs, cg).cs_nbfree >= avgbfree) {
				mincg = cg;
				minndir = fs->fs_cs(fs, cg).cs_ndir;
			}
		for (cg = 0; cg < prefcg; cg++)
			if (fs->fs_cs(fs, cg).cs_ndir < minndir &&
			    fs->fs_cs(fs, cg).cs_nifree >= avgifree &&
			    fs->fs_cs(fs, cg).cs_nbfree >= avgbfree) {
				mincg = cg;
				minndir = fs->fs_cs(fs, cg).cs_ndir;
			}
		return ((ino_t)(fs->fs_ipg * mincg));
	}

	/*
	 * Count various limits which used for
	 * optimal allocation of a directory inode.
	 * Try cylinder groups with >75% avgifree and avgbfree.
	 * Avoid cylinder groups with no free blocks or inodes as that
	 * triggers an I/O-expensive cylinder group scan.
	 */
	maxndir = uimin(avgndir + fs->fs_ipg / 16, fs->fs_ipg);
	minifree = avgifree - avgifree / 4;
	if (minifree < 1)
		minifree = 1;
	minbfree = avgbfree - avgbfree / 4;
	if (minbfree < 1)
		minbfree = 1;
	cgsize = (int64_t)fs->fs_fsize * fs->fs_fpg;
	dirsize = (int64_t)fs->fs_avgfilesize * fs->fs_avgfpdir;
	if (avgndir != 0) {
		curdsz = (cgsize - (int64_t)avgbfree * fs->fs_bsize) / avgndir;
		if (dirsize < curdsz)
			dirsize = curdsz;
	}
	if (cgsize < dirsize * 255)
		maxcontigdirs = (avgbfree * fs->fs_bsize) / dirsize;
	else
		maxcontigdirs = 255;
	if (fs->fs_avgfpdir > 0)
		maxcontigdirs = uimin(maxcontigdirs,
				    fs->fs_ipg / fs->fs_avgfpdir);
	if (maxcontigdirs == 0)
		maxcontigdirs = 1;

	/*
	 * Limit number of dirs in one cg and reserve space for
	 * regular files, but only if we have no deficit in
	 * inodes or space.
	 */
	prefcg = ino_to_cg(fs, pip->i_number);
	for (cg = prefcg; cg < fs->fs_ncg; cg++)
		if (fs->fs_cs(fs, cg).cs_ndir < maxndir &&
		    fs->fs_cs(fs, cg).cs_nifree >= minifree &&
	    	    fs->fs_cs(fs, cg).cs_nbfree >= minbfree) {
			if (fs->fs_contigdirs[cg] < maxcontigdirs)
				return ((ino_t)(fs->fs_ipg * cg));
		}
	for (cg = 0; cg < prefcg; cg++)
		if (fs->fs_cs(fs, cg).cs_ndir < maxndir &&
		    fs->fs_cs(fs, cg).cs_nifree >= minifree &&
	    	    fs->fs_cs(fs, cg).cs_nbfree >= minbfree) {
			if (fs->fs_contigdirs[cg] < maxcontigdirs)
				return ((ino_t)(fs->fs_ipg * cg));
		}
	/*
	 * This is a backstop when we are deficient in space.
	 */
	for (cg = prefcg; cg < fs->fs_ncg; cg++)
		if (fs->fs_cs(fs, cg).cs_nifree >= avgifree)
			return ((ino_t)(fs->fs_ipg * cg));
	for (cg = 0; cg < prefcg; cg++)
		if (fs->fs_cs(fs, cg).cs_nifree >= avgifree)
			break;
	return ((ino_t)(fs->fs_ipg * cg));
}

/*
 * Select the desired position for the next block in a file.  The file is
 * logically divided into sections. The first section is composed of the
 * direct blocks. Each additional section contains fs_maxbpg blocks.
 *
 * If no blocks have been allocated in the first section, the policy is to
 * request a block in the same cylinder group as the inode that describes
 * the file. If no blocks have been allocated in any other section, the
 * policy is to place the section in a cylinder group with a greater than
 * average number of free blocks.  An appropriate cylinder group is found
 * by using a rotor that sweeps the cylinder groups. When a new group of
 * blocks is needed, the sweep begins in the cylinder group following the
 * cylinder group from which the previous allocation was made. The sweep
 * continues until a cylinder group with greater than the average number
 * of free blocks is found. If the allocation is for the first block in an
 * indirect block, the information on the previous allocation is unavailable;
 * here a best guess is made based upon the logical block number being
 * allocated.
 *
 * If a section is already partially allocated, the policy is to
 * contiguously allocate fs_maxcontig blocks.  The end of one of these
 * contiguous blocks and the beginning of the next is laid out
 * contigously if possible.
 *
 * => um_lock held on entry and exit
 */
daddr_t
ffs_blkpref_ufs1(struct inode *ip, daddr_t lbn, int indx, int flags,
    int32_t *bap /* XXX ondisk32 */)
{
	struct fs *fs;
	int cg;
	int avgbfree, startcg;

	KASSERT(mutex_owned(&ip->i_ump->um_lock));

	fs = ip->i_fs;

	/*
	 * If allocating a contiguous file with B_CONTIG, use the hints
	 * in the inode extentions to return the desired block.
	 *
	 * For metadata (indirect blocks) return the address of where
	 * the first indirect block resides - we'll scan for the next
	 * available slot if we need to allocate more than one indirect
	 * block.  For data, return the address of the actual block
	 * relative to the address of the first data block.
	 */
	if (flags & B_CONTIG) {
		KASSERT(ip->i_ffs_first_data_blk != 0);
		KASSERT(ip->i_ffs_first_indir_blk != 0);
		if (flags & B_METAONLY)
			return ip->i_ffs_first_indir_blk;
		else
			return ip->i_ffs_first_data_blk + ffs_blkstofrags(fs, lbn);
	}

	if (indx % fs->fs_maxbpg == 0 || bap[indx - 1] == 0) {
		if (lbn < UFS_NDADDR + FFS_NINDIR(fs)) {
			cg = ino_to_cg(fs, ip->i_number);
			return (cgbase(fs, cg) + fs->fs_frag);
		}
		/*
		 * Find a cylinder with greater than average number of
		 * unused data blocks.
		 */
		if (indx == 0 || bap[indx - 1] == 0)
			startcg =
			    ino_to_cg(fs, ip->i_number) + lbn / fs->fs_maxbpg;
		else
			startcg = dtog(fs,
				ufs_rw32(bap[indx - 1], UFS_FSNEEDSWAP(fs)) + 1);
		startcg %= fs->fs_ncg;
		avgbfree = fs->fs_cstotal.cs_nbfree / fs->fs_ncg;
		for (cg = startcg; cg < fs->fs_ncg; cg++)
			if (fs->fs_cs(fs, cg).cs_nbfree >= avgbfree) {
				return (cgbase(fs, cg) + fs->fs_frag);
			}
		for (cg = 0; cg < startcg; cg++)
			if (fs->fs_cs(fs, cg).cs_nbfree >= avgbfree) {
				return (cgbase(fs, cg) + fs->fs_frag);
			}
		return (0);
	}
	/*
	 * We just always try to lay things out contiguously.
	 */
	return ufs_rw32(bap[indx - 1], UFS_FSNEEDSWAP(fs)) + fs->fs_frag;
}

daddr_t
ffs_blkpref_ufs2(struct inode *ip, daddr_t lbn, int indx, int flags,
    int64_t *bap)
{
	struct fs *fs;
	int cg;
	int avgbfree, startcg;

	KASSERT(mutex_owned(&ip->i_ump->um_lock));

	fs = ip->i_fs;

	/*
	 * If allocating a contiguous file with B_CONTIG, use the hints
	 * in the inode extentions to return the desired block.
	 *
	 * For metadata (indirect blocks) return the address of where
	 * the first indirect block resides - we'll scan for the next
	 * available slot if we need to allocate more than one indirect
	 * block.  For data, return the address of the actual block
	 * relative to the address of the first data block.
	 */
	if (flags & B_CONTIG) {
		KASSERT(ip->i_ffs_first_data_blk != 0);
		KASSERT(ip->i_ffs_first_indir_blk != 0);
		if (flags & B_METAONLY)
			return ip->i_ffs_first_indir_blk;
		else
			return ip->i_ffs_first_data_blk + ffs_blkstofrags(fs, lbn);
	}

	if (indx % fs->fs_maxbpg == 0 || bap[indx - 1] == 0) {
		if (lbn < UFS_NDADDR + FFS_NINDIR(fs)) {
			cg = ino_to_cg(fs, ip->i_number);
			return (cgbase(fs, cg) + fs->fs_frag);
		}
		/*
		 * Find a cylinder with greater than average number of
		 * unused data blocks.
		 */
		if (indx == 0 || bap[indx - 1] == 0)
			startcg =
			    ino_to_cg(fs, ip->i_number) + lbn / fs->fs_maxbpg;
		else
			startcg = dtog(fs,
				ufs_rw64(bap[indx - 1], UFS_FSNEEDSWAP(fs)) + 1);
		startcg %= fs->fs_ncg;
		avgbfree = fs->fs_cstotal.cs_nbfree / fs->fs_ncg;
		for (cg = startcg; cg < fs->fs_ncg; cg++)
			if (fs->fs_cs(fs, cg).cs_nbfree >= avgbfree) {
				return (cgbase(fs, cg) + fs->fs_frag);
			}
		for (cg = 0; cg < startcg; cg++)
			if (fs->fs_cs(fs, cg).cs_nbfree >= avgbfree) {
				return (cgbase(fs, cg) + fs->fs_frag);
			}
		return (0);
	}
	/*
	 * We just always try to lay things out contiguously.
	 */
	return ufs_rw64(bap[indx - 1], UFS_FSNEEDSWAP(fs)) + fs->fs_frag;
}


/*
 * Implement the cylinder overflow algorithm.
 *
 * The policy implemented by this algorithm is:
 *   1) allocate the block in its requested cylinder group.
 *   2) quadradically rehash on the cylinder group number.
 *   3) brute force search for a free block.
 *
 * => called with um_lock held
 * => returns with um_lock released on success, held on failure
 *    (*allocator releases lock on success, retains lock on failure)
 */
/*VARARGS5*/
static daddr_t
ffs_hashalloc(struct inode *ip, int cg, daddr_t pref,
    int size /* size for data blocks, mode for inodes */,
    int realsize,
    int flags,
    daddr_t (*allocator)(struct inode *, int, daddr_t, int, int, int))
{
	struct fs *fs;
	daddr_t result;
	int i, icg = cg;

	fs = ip->i_fs;
	/*
	 * 1: preferred cylinder group
	 */
	result = (*allocator)(ip, cg, pref, size, realsize, flags);
	if (result)
		return (result);

	if (flags & B_CONTIG)
		return (result);
	/*
	 * 2: quadratic rehash
	 */
	for (i = 1; i < fs->fs_ncg; i *= 2) {
		cg += i;
		if (cg >= fs->fs_ncg)
			cg -= fs->fs_ncg;
		result = (*allocator)(ip, cg, 0, size, realsize, flags);
		if (result)
			return (result);
	}
	/*
	 * 3: brute force search
	 * Note that we start at i == 2, since 0 was checked initially,
	 * and 1 is always checked in the quadratic rehash.
	 */
	cg = (icg + 2) % fs->fs_ncg;
	for (i = 2; i < fs->fs_ncg; i++) {
		result = (*allocator)(ip, cg, 0, size, realsize, flags);
		if (result)
			return (result);
		cg++;
		if (cg == fs->fs_ncg)
			cg = 0;
	}
	return (0);
}

/*
 * Determine whether a fragment can be extended.
 *
 * Check to see if the necessary fragments are available, and
 * if they are, allocate them.
 *
 * => called with um_lock held
 * => returns with um_lock released on success, held on failure
 */
static daddr_t
ffs_fragextend(struct inode *ip, int cg, daddr_t bprev, int osize, int nsize)
{
	struct ufsmount *ump;
	struct fs *fs;
	struct cg *cgp;
	struct buf *bp;
	daddr_t bno;
	int frags, bbase;
	int i, error;
	u_int8_t *blksfree;

	fs = ip->i_fs;
	ump = ip->i_ump;

	KASSERT(mutex_owned(&ump->um_lock));

	if (fs->fs_cs(fs, cg).cs_nffree < ffs_numfrags(fs, nsize - osize))
		return (0);
	frags = ffs_numfrags(fs, nsize);
	bbase = ffs_fragnum(fs, bprev);
	if (bbase > ffs_fragnum(fs, (bprev + frags - 1))) {
		/* cannot extend across a block boundary */
		return (0);
	}
	mutex_exit(&ump->um_lock);
	error = bread(ip->i_devvp, FFS_FSBTODB(fs, cgtod(fs, cg)),
		(int)fs->fs_cgsize, B_MODIFY, &bp);
	if (error)
		goto fail;
	cgp = (struct cg *)bp->b_data;
	if (!cg_chkmagic(cgp, UFS_FSNEEDSWAP(fs)))
		goto fail;
	cgp->cg_old_time = ufs_rw32(time_second, UFS_FSNEEDSWAP(fs));
	if ((fs->fs_magic != FS_UFS1_MAGIC) ||
	    (fs->fs_old_flags & FS_FLAGS_UPDATED))
		cgp->cg_time = ufs_rw64(time_second, UFS_FSNEEDSWAP(fs));
	bno = dtogd(fs, bprev);
	blksfree = cg_blksfree(cgp, UFS_FSNEEDSWAP(fs));
	for (i = ffs_numfrags(fs, osize); i < frags; i++)
		if (isclr(blksfree, bno + i))
			goto fail;
	/*
	 * the current fragment can be extended
	 * deduct the count on fragment being extended into
	 * increase the count on the remaining fragment (if any)
	 * allocate the extended piece
	 */
	for (i = frags; i < fs->fs_frag - bbase; i++)
		if (isclr(blksfree, bno + i))
			break;
	ufs_add32(cgp->cg_frsum[i - ffs_numfrags(fs, osize)], -1, UFS_FSNEEDSWAP(fs));
	if (i != frags)
		ufs_add32(cgp->cg_frsum[i - frags], 1, UFS_FSNEEDSWAP(fs));
	mutex_enter(&ump->um_lock);
	for (i = ffs_numfrags(fs, osize); i < frags; i++) {
		clrbit(blksfree, bno + i);
		ufs_add32(cgp->cg_cs.cs_nffree, -1, UFS_FSNEEDSWAP(fs));
		fs->fs_cstotal.cs_nffree--;
		fs->fs_cs(fs, cg).cs_nffree--;
	}
	fs->fs_fmod = 1;
	ACTIVECG_CLR(fs, cg);
	mutex_exit(&ump->um_lock);
	bdwrite(bp);
	return (bprev);

 fail:
 	if (bp != NULL)
		brelse(bp, 0);
 	mutex_enter(&ump->um_lock);
 	return (0);
}

/*
 * Determine whether a block can be allocated.
 *
 * Check to see if a block of the appropriate size is available,
 * and if it is, allocate it.
 */
static daddr_t
ffs_alloccg(struct inode *ip, int cg, daddr_t bpref, int size, int realsize,
    int flags)
{
	struct ufsmount *ump;
	struct fs *fs = ip->i_fs;
	struct cg *cgp;
	struct buf *bp;
	int32_t bno;
	daddr_t blkno;
	int error, frags, allocsiz, i;
	u_int8_t *blksfree;
	const int needswap = UFS_FSNEEDSWAP(fs);

	ump = ip->i_ump;

	KASSERT(mutex_owned(&ump->um_lock));

	if (fs->fs_cs(fs, cg).cs_nbfree == 0 && size == fs->fs_bsize)
		return (0);
	mutex_exit(&ump->um_lock);
	error = bread(ip->i_devvp, FFS_FSBTODB(fs, cgtod(fs, cg)),
		(int)fs->fs_cgsize, B_MODIFY, &bp);
	if (error)
		goto fail;
	cgp = (struct cg *)bp->b_data;
	if (!cg_chkmagic(cgp, needswap) ||
	    (cgp->cg_cs.cs_nbfree == 0 && size == fs->fs_bsize))
		goto fail;
	cgp->cg_old_time = ufs_rw32(time_second, needswap);
	if ((fs->fs_magic != FS_UFS1_MAGIC) ||
	    (fs->fs_old_flags & FS_FLAGS_UPDATED))
		cgp->cg_time = ufs_rw64(time_second, needswap);
	if (size == fs->fs_bsize) {
		mutex_enter(&ump->um_lock);
		blkno = ffs_alloccgblk(ip, bp, bpref, realsize, flags);
		ACTIVECG_CLR(fs, cg);
		mutex_exit(&ump->um_lock);

		/*
		 * If actually needed size is lower, free the extra blocks now.
		 * This is safe to call here, there is no outside reference
		 * to this block yet. It is not necessary to keep um_lock
		 * locked.
		 */
		if (realsize != 0 && realsize < size) {
			ffs_blkfree_common(ip->i_ump, ip->i_fs,
			    ip->i_devvp->v_rdev,
			    bp, blkno + ffs_numfrags(fs, realsize),
			    (long)(size - realsize), false);
		}

		bdwrite(bp);
		return (blkno);
	}
	/*
	 * check to see if any fragments are already available
	 * allocsiz is the size which will be allocated, hacking
	 * it down to a smaller size if necessary
	 */
	blksfree = cg_blksfree(cgp, needswap);
	frags = ffs_numfrags(fs, size);
	for (allocsiz = frags; allocsiz < fs->fs_frag; allocsiz++)
		if (cgp->cg_frsum[allocsiz] != 0)
			break;
	if (allocsiz == fs->fs_frag) {
		/*
		 * no fragments were available, so a block will be
		 * allocated, and hacked up
		 */
		if (cgp->cg_cs.cs_nbfree == 0)
			goto fail;
		mutex_enter(&ump->um_lock);
		blkno = ffs_alloccgblk(ip, bp, bpref, realsize, flags);
		bno = dtogd(fs, blkno);
		for (i = frags; i < fs->fs_frag; i++)
			setbit(blksfree, bno + i);
		i = fs->fs_frag - frags;
		ufs_add32(cgp->cg_cs.cs_nffree, i, needswap);
		fs->fs_cstotal.cs_nffree += i;
		fs->fs_cs(fs, cg).cs_nffree += i;
		fs->fs_fmod = 1;
		ufs_add32(cgp->cg_frsum[i], 1, needswap);
		ACTIVECG_CLR(fs, cg);
		mutex_exit(&ump->um_lock);
		bdwrite(bp);
		return (blkno);
	}
	bno = ffs_mapsearch(fs, cgp, bpref, allocsiz);
#if 0
	/*
	 * XXX fvdl mapsearch will panic, and never return -1
	 *          also: returning NULL as daddr_t ?
	 */
	if (bno < 0)
		goto fail;
#endif
	for (i = 0; i < frags; i++)
		clrbit(blksfree, bno + i);
	mutex_enter(&ump->um_lock);
	ufs_add32(cgp->cg_cs.cs_nffree, -frags, needswap);
	fs->fs_cstotal.cs_nffree -= frags;
	fs->fs_cs(fs, cg).cs_nffree -= frags;
	fs->fs_fmod = 1;
	ufs_add32(cgp->cg_frsum[allocsiz], -1, needswap);
	if (frags != allocsiz)
		ufs_add32(cgp->cg_frsum[allocsiz - frags], 1, needswap);
	blkno = cgbase(fs, cg) + bno;
	ACTIVECG_CLR(fs, cg);
	mutex_exit(&ump->um_lock);
	bdwrite(bp);
	return blkno;

 fail:
 	if (bp != NULL)
		brelse(bp, 0);
 	mutex_enter(&ump->um_lock);
 	return (0);
}

/*
 * Allocate a block in a cylinder group.
 *
 * This algorithm implements the following policy:
 *   1) allocate the requested block.
 *   2) allocate a rotationally optimal block in the same cylinder.
 *   3) allocate the next available block on the block rotor for the
 *      specified cylinder group.
 * Note that this routine only allocates fs_bsize blocks; these
 * blocks may be fragmented by the routine that allocates them.
 */
static daddr_t
ffs_alloccgblk(struct inode *ip, struct buf *bp, daddr_t bpref, int realsize,
    int flags)
{
	struct fs *fs = ip->i_fs;
	struct cg *cgp;
	int cg;
	daddr_t blkno;
	int32_t bno;
	u_int8_t *blksfree;
	const int needswap = UFS_FSNEEDSWAP(fs);

	KASSERT(mutex_owned(&ip->i_ump->um_lock));

	cgp = (struct cg *)bp->b_data;
	blksfree = cg_blksfree(cgp, needswap);
	if (bpref == 0 || dtog(fs, bpref) != ufs_rw32(cgp->cg_cgx, needswap)) {
		bpref = ufs_rw32(cgp->cg_rotor, needswap);
	} else {
		bpref = ffs_blknum(fs, bpref);
		bno = dtogd(fs, bpref);
		/*
		 * if the requested block is available, use it
		 */
		if (ffs_isblock(fs, blksfree, ffs_fragstoblks(fs, bno)))
			goto gotit;
		/*
		 * if the requested data block isn't available and we are
		 * trying to allocate a contiguous file, return an error.
		 */
		if ((flags & (B_CONTIG | B_METAONLY)) == B_CONTIG)
			return (0);
	}

	/*
	 * Take the next available block in this cylinder group.
	 */
	bno = ffs_mapsearch(fs, cgp, bpref, (int)fs->fs_frag);
#if 0
	/*
	 * XXX jdolecek ffs_mapsearch() succeeds or panics
	 */
	if (bno < 0)
		return (0);
#endif
	cgp->cg_rotor = ufs_rw32(bno, needswap);
gotit:
	blkno = ffs_fragstoblks(fs, bno);
	ffs_clrblock(fs, blksfree, blkno);
	ffs_clusteracct(fs, cgp, blkno, -1);
	ufs_add32(cgp->cg_cs.cs_nbfree, -1, needswap);
	fs->fs_cstotal.cs_nbfree--;
	fs->fs_cs(fs, ufs_rw32(cgp->cg_cgx, needswap)).cs_nbfree--;
	if ((fs->fs_magic == FS_UFS1_MAGIC) &&
	    ((fs->fs_old_flags & FS_FLAGS_UPDATED) == 0)) {
		int cylno;
		cylno = old_cbtocylno(fs, bno);
		KASSERT(cylno >= 0);
		KASSERT(cylno < fs->fs_old_ncyl);
		KASSERT(old_cbtorpos(fs, bno) >= 0);
		KASSERT(fs->fs_old_nrpos == 0 || old_cbtorpos(fs, bno) < fs->fs_old_nrpos);
		ufs_add16(old_cg_blks(fs, cgp, cylno, needswap)[old_cbtorpos(fs, bno)], -1,
		    needswap);
		ufs_add32(old_cg_blktot(cgp, needswap)[cylno], -1, needswap);
	}
	fs->fs_fmod = 1;
	cg = ufs_rw32(cgp->cg_cgx, needswap);
	blkno = cgbase(fs, cg) + bno;
	return (blkno);
}

/*
 * Determine whether an inode can be allocated.
 *
 * Check to see if an inode is available, and if it is,
 * allocate it using the following policy:
 *   1) allocate the requested inode.
 *   2) allocate the next available inode after the requested
 *      inode in the specified cylinder group.
 */
static daddr_t
ffs_nodealloccg(struct inode *ip, int cg, daddr_t ipref, int mode, int realsize,
    int flags)
{
	struct ufsmount *ump = ip->i_ump;
	struct fs *fs = ip->i_fs;
	struct cg *cgp;
	struct buf *bp, *ibp;
	u_int8_t *inosused;
	int error, start, len, loc, map, i;
	int32_t initediblk, maxiblk, irotor;
	daddr_t nalloc;
	struct ufs2_dinode *dp2;
	const int needswap = UFS_FSNEEDSWAP(fs);

	KASSERT(mutex_owned(&ump->um_lock));
	UFS_WAPBL_JLOCK_ASSERT(ip->i_ump->um_mountp);

	if (fs->fs_cs(fs, cg).cs_nifree == 0)
		return (0);
	mutex_exit(&ump->um_lock);
	ibp = NULL;
	if (fs->fs_magic == FS_UFS2_MAGIC) {
		initediblk = -1;
	} else {
		initediblk = fs->fs_ipg;
	}
	maxiblk = initediblk;

retry:
	error = bread(ip->i_devvp, FFS_FSBTODB(fs, cgtod(fs, cg)),
		(int)fs->fs_cgsize, B_MODIFY, &bp);
	if (error)
		goto fail;
	cgp = (struct cg *)bp->b_data;
	if (!cg_chkmagic(cgp, needswap) || cgp->cg_cs.cs_nifree == 0)
		goto fail;

	if (ibp != NULL &&
	    initediblk != ufs_rw32(cgp->cg_initediblk, needswap)) {
		/* Another thread allocated more inodes so we retry the test. */
		brelse(ibp, 0);
		ibp = NULL;
	}
	/*
	 * Check to see if we need to initialize more inodes.
	 */
	if (fs->fs_magic == FS_UFS2_MAGIC && ibp == NULL) {
	        initediblk = ufs_rw32(cgp->cg_initediblk, needswap);
		maxiblk = initediblk;
		nalloc = fs->fs_ipg - ufs_rw32(cgp->cg_cs.cs_nifree, needswap);
		if (nalloc + FFS_INOPB(fs) > initediblk &&
		    initediblk < ufs_rw32(cgp->cg_niblk, needswap)) {
			/*
			 * We have to release the cg buffer here to prevent
			 * a deadlock when reading the inode block will
			 * run a copy-on-write that might use this cg.
			 */
			brelse(bp, 0);
			bp = NULL;
			error = ffs_getblk(ip->i_devvp, FFS_FSBTODB(fs,
			    ino_to_fsba(fs, cg * fs->fs_ipg + initediblk)),
			    FFS_NOBLK, fs->fs_bsize, false, &ibp);
			if (error)
				goto fail;

			maxiblk += FFS_INOPB(fs);

			goto retry;
		}
	}

	cgp->cg_old_time = ufs_rw32(time_second, needswap);
	if ((fs->fs_magic != FS_UFS1_MAGIC) ||
	    (fs->fs_old_flags & FS_FLAGS_UPDATED))
		cgp->cg_time = ufs_rw64(time_second, needswap);
	inosused = cg_inosused(cgp, needswap);

	if (ipref) {
		ipref %= fs->fs_ipg;
		/* safeguard to stay in (to be) allocated range */
		if (ipref < maxiblk && isclr(inosused, ipref))
			goto gotit;
	}

	irotor = ufs_rw32(cgp->cg_irotor, needswap);

	KASSERTMSG(irotor < initediblk, "%s: allocation botch: cg=%d, irotor %d"
		   " out of bounds, initediblk=%d",
		   __func__, cg, irotor, initediblk);

	start = irotor / NBBY;
	len = howmany(maxiblk - irotor, NBBY);
	loc = skpc(0xff, len, &inosused[start]);
	if (loc == 0) {
		len = start + 1;
		start = 0;
		loc = skpc(0xff, len, &inosused[0]);
		if (loc == 0) {
			panic("%s: map corrupted: cg=%d, irotor=%d, fs=%s",
			    __func__, cg, ufs_rw32(cgp->cg_irotor, needswap),
			    fs->fs_fsmnt);
			/* NOTREACHED */
		}
	}
	i = start + len - loc;
	map = inosused[i] ^ 0xff;
	if (map == 0) {
		panic("%s: block not in map: fs=%s", __func__, fs->fs_fsmnt);
	}

	ipref = i * NBBY + ffs(map) - 1;

	cgp->cg_irotor = ufs_rw32(ipref, needswap);

gotit:
	KASSERTMSG(ipref < maxiblk, "%s: allocation botch: cg=%d attempt to "
		   "allocate inode index %d beyond max allocated index %d"
		   " of %d inodes/cg",
		   __func__, cg, (int)ipref, maxiblk, cgp->cg_niblk);

	UFS_WAPBL_REGISTER_INODE(ip->i_ump->um_mountp, cg * fs->fs_ipg + ipref,
	    mode);
	/*
	 * Check to see if we need to initialize more inodes.
	 */
	if (ibp != NULL) {
		KASSERT(initediblk == ufs_rw32(cgp->cg_initediblk, needswap));
		memset(ibp->b_data, 0, fs->fs_bsize);
		dp2 = (struct ufs2_dinode *)(ibp->b_data);
		for (i = 0; i < FFS_INOPB(fs); i++) {
			/*
			 * Don't bother to swap, it's supposed to be
			 * random, after all.
			 */
			dp2->di_gen = (cprng_fast32() & INT32_MAX) / 2 + 1;
			dp2++;
		}
		initediblk += FFS_INOPB(fs);
		cgp->cg_initediblk = ufs_rw32(initediblk, needswap);
	}

	mutex_enter(&ump->um_lock);
	ACTIVECG_CLR(fs, cg);
	setbit(inosused, ipref);
	ufs_add32(cgp->cg_cs.cs_nifree, -1, needswap);
	fs->fs_cstotal.cs_nifree--;
	fs->fs_cs(fs, cg).cs_nifree--;
	fs->fs_fmod = 1;
	if ((mode & IFMT) == IFDIR) {
		ufs_add32(cgp->cg_cs.cs_ndir, 1, needswap);
		fs->fs_cstotal.cs_ndir++;
		fs->fs_cs(fs, cg).cs_ndir++;
	}
	mutex_exit(&ump->um_lock);
	if (ibp != NULL) {
		bwrite(ibp);
		bwrite(bp);
	} else
		bdwrite(bp);
	return (cg * fs->fs_ipg + ipref);
 fail:
	if (bp != NULL)
		brelse(bp, 0);
	if (ibp != NULL)
		brelse(ibp, 0);
	mutex_enter(&ump->um_lock);
	return (0);
}

/*
 * Allocate a block or fragment.
 *
 * The specified block or fragment is removed from the
 * free map, possibly fragmenting a block in the process.
 *
 * This implementation should mirror fs_blkfree
 *
 * => um_lock not held on entry or exit
 */
int
ffs_blkalloc(struct inode *ip, daddr_t bno, long size)
{
	int error;

	error = ffs_check_bad_allocation(__func__, ip->i_fs, bno, size,
	    ip->i_dev, ip->i_uid);
	if (error)
		return error;

	return ffs_blkalloc_ump(ip->i_ump, bno, size);
}

int
ffs_blkalloc_ump(struct ufsmount *ump, daddr_t bno, long size)
{
	struct fs *fs = ump->um_fs;
	struct cg *cgp;
	struct buf *bp;
	int32_t fragno, cgbno;
	int i, error, cg, blk, frags, bbase;
	u_int8_t *blksfree;
	const int needswap = UFS_FSNEEDSWAP(fs);

	KASSERT((u_int)size <= fs->fs_bsize && ffs_fragoff(fs, size) == 0 &&
	    ffs_fragnum(fs, bno) + ffs_numfrags(fs, size) <= fs->fs_frag);
	KASSERT(bno < fs->fs_size);

	cg = dtog(fs, bno);
	error = bread(ump->um_devvp, FFS_FSBTODB(fs, cgtod(fs, cg)),
		(int)fs->fs_cgsize, B_MODIFY, &bp);
	if (error) {
		return error;
	}
	cgp = (struct cg *)bp->b_data;
	if (!cg_chkmagic(cgp, needswap)) {
		brelse(bp, 0);
		return EIO;
	}
	cgp->cg_old_time = ufs_rw32(time_second, needswap);
	cgp->cg_time = ufs_rw64(time_second, needswap);
	cgbno = dtogd(fs, bno);
	blksfree = cg_blksfree(cgp, needswap);

	mutex_enter(&ump->um_lock);
	if (size == fs->fs_bsize) {
		fragno = ffs_fragstoblks(fs, cgbno);
		if (!ffs_isblock(fs, blksfree, fragno)) {
			mutex_exit(&ump->um_lock);
			brelse(bp, 0);
			return EBUSY;
		}
		ffs_clrblock(fs, blksfree, fragno);
		ffs_clusteracct(fs, cgp, fragno, -1);
		ufs_add32(cgp->cg_cs.cs_nbfree, -1, needswap);
		fs->fs_cstotal.cs_nbfree--;
		fs->fs_cs(fs, cg).cs_nbfree--;
	} else {
		bbase = cgbno - ffs_fragnum(fs, cgbno);

		frags = ffs_numfrags(fs, size);
		for (i = 0; i < frags; i++) {
			if (isclr(blksfree, cgbno + i)) {
				mutex_exit(&ump->um_lock);
				brelse(bp, 0);
				return EBUSY;
			}
		}
		/*
		 * if a complete block is being split, account for it
		 */
		fragno = ffs_fragstoblks(fs, bbase);
		if (ffs_isblock(fs, blksfree, fragno)) {
			ufs_add32(cgp->cg_cs.cs_nffree, fs->fs_frag, needswap);
			fs->fs_cstotal.cs_nffree += fs->fs_frag;
			fs->fs_cs(fs, cg).cs_nffree += fs->fs_frag;
			ffs_clusteracct(fs, cgp, fragno, -1);
			ufs_add32(cgp->cg_cs.cs_nbfree, -1, needswap);
			fs->fs_cstotal.cs_nbfree--;
			fs->fs_cs(fs, cg).cs_nbfree--;
		}
		/*
		 * decrement the counts associated with the old frags
		 */
		blk = blkmap(fs, blksfree, bbase);
		ffs_fragacct(fs, blk, cgp->cg_frsum, -1, needswap);
		/*
		 * allocate the fragment
		 */
		for (i = 0; i < frags; i++) {
			clrbit(blksfree, cgbno + i);
		}
		ufs_add32(cgp->cg_cs.cs_nffree, -i, needswap);
		fs->fs_cstotal.cs_nffree -= i;
		fs->fs_cs(fs, cg).cs_nffree -= i;
		/*
		 * add back in counts associated with the new frags
		 */
		blk = blkmap(fs, blksfree, bbase);
		ffs_fragacct(fs, blk, cgp->cg_frsum, 1, needswap);
	}
	fs->fs_fmod = 1;
	ACTIVECG_CLR(fs, cg);
	mutex_exit(&ump->um_lock);
	bdwrite(bp);
	return 0;
}

/*
 * Free a block or fragment.
 *
 * The specified block or fragment is placed back in the
 * free map. If a fragment is deallocated, a possible
 * block reassembly is checked.
 *
 * => um_lock not held on entry or exit
 */
static void
ffs_blkfree_cg(struct fs *fs, struct vnode *devvp, daddr_t bno, long size)
{
	struct cg *cgp;
	struct buf *bp;
	struct ufsmount *ump;
	daddr_t cgblkno;
	int error, cg;
	dev_t dev;
	const bool devvp_is_snapshot = (devvp->v_type != VBLK);
	const int needswap = UFS_FSNEEDSWAP(fs);

	KASSERT(!devvp_is_snapshot);

	cg = dtog(fs, bno);
	dev = devvp->v_rdev;
	ump = VFSTOUFS(spec_node_getmountedfs(devvp));
	KASSERT(fs == ump->um_fs);
	cgblkno = FFS_FSBTODB(fs, cgtod(fs, cg));

	error = bread(devvp, cgblkno, (int)fs->fs_cgsize,
	    B_MODIFY, &bp);
	if (error) {
		return;
	}
	cgp = (struct cg *)bp->b_data;
	if (!cg_chkmagic(cgp, needswap)) {
		brelse(bp, 0);
		return;
	}

	ffs_blkfree_common(ump, fs, dev, bp, bno, size, devvp_is_snapshot);

	bdwrite(bp);
}

struct discardopdata {
	struct work wk; /* must be first */
	struct vnode *devvp;
	daddr_t bno;
	long size;
};

struct discarddata {
	struct fs *fs;
	struct discardopdata *entry;
	long maxsize;
	kmutex_t entrylk;
	struct workqueue *wq;
	int wqcnt, wqdraining;
	kmutex_t wqlk;
	kcondvar_t wqcv;
	/* timer for flush? */
};

static void
ffs_blkfree_td(struct fs *fs, struct discardopdata *td)
{
	struct mount *mp = spec_node_getmountedfs(td->devvp);
	long todo;
	int error;

	while (td->size) {
		todo = uimin(td->size,
		  ffs_lfragtosize(fs, (fs->fs_frag - ffs_fragnum(fs, td->bno))));
		error = UFS_WAPBL_BEGIN(mp);
		if (error) {
			printf("ffs: failed to begin wapbl transaction"
			    " for discard: %d\n", error);
			break;
		}
		ffs_blkfree_cg(fs, td->devvp, td->bno, todo);
		UFS_WAPBL_END(mp);
		td->bno += ffs_numfrags(fs, todo);
		td->size -= todo;
	}
}

static void
ffs_discardcb(struct work *wk, void *arg)
{
	struct discardopdata *td = (void *)wk;
	struct discarddata *ts = arg;
	struct fs *fs = ts->fs;
	off_t start, len;
#ifdef TRIMDEBUG
	int error;
#endif

/* like FSBTODB but emits bytes; XXX move to fs.h */
#ifndef FFS_FSBTOBYTES
#define FFS_FSBTOBYTES(fs, b) ((b) << (fs)->fs_fshift)
#endif

	start = FFS_FSBTOBYTES(fs, td->bno);
	len = td->size;
#ifdef TRIMDEBUG
	error =
#endif
		VOP_FDISCARD(td->devvp, start, len);
#ifdef TRIMDEBUG
	printf("trim(%" PRId64 ",%ld):%d\n", td->bno, td->size, error);
#endif

	ffs_blkfree_td(fs, td);
	kmem_free(td, sizeof(*td));
	mutex_enter(&ts->wqlk);
	ts->wqcnt--;
	if (ts->wqdraining && !ts->wqcnt)
		cv_signal(&ts->wqcv);
	mutex_exit(&ts->wqlk);
}

void *
ffs_discard_init(struct vnode *devvp, struct fs *fs)
{
	struct discarddata *ts;
	int error;

	ts = kmem_zalloc(sizeof (*ts), KM_SLEEP);
	error = workqueue_create(&ts->wq, "trimwq", ffs_discardcb, ts,
				 PRI_USER, IPL_NONE, 0);
	if (error) {
		kmem_free(ts, sizeof (*ts));
		return NULL;
	}
	mutex_init(&ts->entrylk, MUTEX_DEFAULT, IPL_NONE);
	mutex_init(&ts->wqlk, MUTEX_DEFAULT, IPL_NONE);
	cv_init(&ts->wqcv, "trimwqcv");
	ts->maxsize = 100*1024; /* XXX */
	ts->fs = fs;
	return ts;
}

void
ffs_discard_finish(void *vts, int flags)
{
	struct discarddata *ts = vts;
	struct discardopdata *td = NULL;

	/* wait for workqueue to drain */
	mutex_enter(&ts->wqlk);
	if (ts->wqcnt) {
		ts->wqdraining = 1;
		cv_wait(&ts->wqcv, &ts->wqlk);
	}
	mutex_exit(&ts->wqlk);

	mutex_enter(&ts->entrylk);
	if (ts->entry) {
		td = ts->entry;
		ts->entry = NULL;
	}
	mutex_exit(&ts->entrylk);
	if (td) {
		/* XXX don't tell disk, its optional */
		ffs_blkfree_td(ts->fs, td);
#ifdef TRIMDEBUG
		printf("finish(%" PRId64 ",%ld)\n", td->bno, td->size);
#endif
		kmem_free(td, sizeof(*td));
	}

	cv_destroy(&ts->wqcv);
	mutex_destroy(&ts->entrylk);
	mutex_destroy(&ts->wqlk);
	workqueue_destroy(ts->wq);
	kmem_free(ts, sizeof(*ts));
}

void
ffs_blkfree(struct fs *fs, struct vnode *devvp, daddr_t bno, long size,
    ino_t inum)
{
	struct ufsmount *ump;
	int error;
	dev_t dev;
	struct discarddata *ts;
	struct discardopdata *td;

	dev = devvp->v_rdev;
	ump = VFSTOUFS(spec_node_getmountedfs(devvp));
	if (ffs_snapblkfree(fs, devvp, bno, size, inum))
		return;

	error = ffs_check_bad_allocation(__func__, fs, bno, size, dev, inum);
	if (error)
		return;

	if (!ump->um_discarddata) {
		ffs_blkfree_cg(fs, devvp, bno, size);
		return;
	}

#ifdef TRIMDEBUG
	printf("blkfree(%" PRId64 ",%ld)\n", bno, size);
#endif
	ts = ump->um_discarddata;
	td = NULL;

	mutex_enter(&ts->entrylk);
	if (ts->entry) {
		td = ts->entry;
		/* ffs deallocs backwards, check for prepend only */
		if (td->bno == bno + ffs_numfrags(fs, size)
		    && td->size + size <= ts->maxsize) {
			td->bno = bno;
			td->size += size;
			if (td->size < ts->maxsize) {
#ifdef TRIMDEBUG
				printf("defer(%" PRId64 ",%ld)\n", td->bno, td->size);
#endif
				mutex_exit(&ts->entrylk);
				return;
			}
			size = 0; /* mark done */
		}
		ts->entry = NULL;
	}
	mutex_exit(&ts->entrylk);

	if (td) {
#ifdef TRIMDEBUG
		printf("enq old(%" PRId64 ",%ld)\n", td->bno, td->size);
#endif
		mutex_enter(&ts->wqlk);
		ts->wqcnt++;
		mutex_exit(&ts->wqlk);
		workqueue_enqueue(ts->wq, &td->wk, NULL);
	}
	if (!size)
		return;

	td = kmem_alloc(sizeof(*td), KM_SLEEP);
	td->devvp = devvp;
	td->bno = bno;
	td->size = size;

	if (td->size < ts->maxsize) { /* XXX always the case */
		mutex_enter(&ts->entrylk);
		if (!ts->entry) { /* possible race? */
#ifdef TRIMDEBUG
			printf("defer(%" PRId64 ",%ld)\n", td->bno, td->size);
#endif
			ts->entry = td;
			td = NULL;
		}
		mutex_exit(&ts->entrylk);
	}
	if (td) {
#ifdef TRIMDEBUG
		printf("enq new(%" PRId64 ",%ld)\n", td->bno, td->size);
#endif
		mutex_enter(&ts->wqlk);
		ts->wqcnt++;
		mutex_exit(&ts->wqlk);
		workqueue_enqueue(ts->wq, &td->wk, NULL);
	}
}

/*
 * Free a block or fragment from a snapshot cg copy.
 *
 * The specified block or fragment is placed back in the
 * free map. If a fragment is deallocated, a possible
 * block reassembly is checked.
 *
 * => um_lock not held on entry or exit
 */
void
ffs_blkfree_snap(struct fs *fs, struct vnode *devvp, daddr_t bno, long size,
    ino_t inum)
{
	struct cg *cgp;
	struct buf *bp;
	struct ufsmount *ump;
	daddr_t cgblkno;
	int error, cg;
	dev_t dev;
	const bool devvp_is_snapshot = (devvp->v_type != VBLK);
	const int needswap = UFS_FSNEEDSWAP(fs);

	KASSERT(devvp_is_snapshot);

	cg = dtog(fs, bno);
	dev = VTOI(devvp)->i_devvp->v_rdev;
	ump = VFSTOUFS(devvp->v_mount);
	cgblkno = ffs_fragstoblks(fs, cgtod(fs, cg));

	error = ffs_check_bad_allocation(__func__, fs, bno, size, dev, inum);
	if (error)
		return;

	error = bread(devvp, cgblkno, (int)fs->fs_cgsize,
	    B_MODIFY, &bp);
	if (error) {
		return;
	}
	cgp = (struct cg *)bp->b_data;
	if (!cg_chkmagic(cgp, needswap)) {
		brelse(bp, 0);
		return;
	}

	ffs_blkfree_common(ump, fs, dev, bp, bno, size, devvp_is_snapshot);

	bdwrite(bp);
}

static void
ffs_blkfree_common(struct ufsmount *ump, struct fs *fs, dev_t dev,
    struct buf *bp, daddr_t bno, long size, bool devvp_is_snapshot)
{
	struct cg *cgp;
	int32_t fragno, cgbno;
	int i, cg, blk, frags, bbase;
	u_int8_t *blksfree;
	const int needswap = UFS_FSNEEDSWAP(fs);

	cg = dtog(fs, bno);
	cgp = (struct cg *)bp->b_data;
	cgp->cg_old_time = ufs_rw32(time_second, needswap);
	if ((fs->fs_magic != FS_UFS1_MAGIC) ||
	    (fs->fs_old_flags & FS_FLAGS_UPDATED))
		cgp->cg_time = ufs_rw64(time_second, needswap);
	cgbno = dtogd(fs, bno);
	blksfree = cg_blksfree(cgp, needswap);
	mutex_enter(&ump->um_lock);
	if (size == fs->fs_bsize) {
		fragno = ffs_fragstoblks(fs, cgbno);
		if (!ffs_isfreeblock(fs, blksfree, fragno)) {
			if (devvp_is_snapshot) {
				mutex_exit(&ump->um_lock);
				return;
			}
			panic("%s: freeing free block: dev = 0x%llx, block = %"
			    PRId64 ", fs = %s", __func__,
			    (unsigned long long)dev, bno, fs->fs_fsmnt);
		}
		ffs_setblock(fs, blksfree, fragno);
		ffs_clusteracct(fs, cgp, fragno, 1);
		ufs_add32(cgp->cg_cs.cs_nbfree, 1, needswap);
		fs->fs_cstotal.cs_nbfree++;
		fs->fs_cs(fs, cg).cs_nbfree++;
		if ((fs->fs_magic == FS_UFS1_MAGIC) &&
		    ((fs->fs_old_flags & FS_FLAGS_UPDATED) == 0)) {
			i = old_cbtocylno(fs, cgbno);
			KASSERT(i >= 0);
			KASSERT(i < fs->fs_old_ncyl);
			KASSERT(old_cbtorpos(fs, cgbno) >= 0);
			KASSERT(fs->fs_old_nrpos == 0 || old_cbtorpos(fs, cgbno) < fs->fs_old_nrpos);
			ufs_add16(old_cg_blks(fs, cgp, i, needswap)[old_cbtorpos(fs, cgbno)], 1,
			    needswap);
			ufs_add32(old_cg_blktot(cgp, needswap)[i], 1, needswap);
		}
	} else {
		bbase = cgbno - ffs_fragnum(fs, cgbno);
		/*
		 * decrement the counts associated with the old frags
		 */
		blk = blkmap(fs, blksfree, bbase);
		ffs_fragacct(fs, blk, cgp->cg_frsum, -1, needswap);
		/*
		 * deallocate the fragment
		 */
		frags = ffs_numfrags(fs, size);
		for (i = 0; i < frags; i++) {
			if (isset(blksfree, cgbno + i)) {
				panic("%s: freeing free frag: "
				    "dev = 0x%llx, block = %" PRId64
				    ", fs = %s", __func__,
				    (unsigned long long)dev, bno + i,
				    fs->fs_fsmnt);
			}
			setbit(blksfree, cgbno + i);
		}
		ufs_add32(cgp->cg_cs.cs_nffree, i, needswap);
		fs->fs_cstotal.cs_nffree += i;
		fs->fs_cs(fs, cg).cs_nffree += i;
		/*
		 * add back in counts associated with the new frags
		 */
		blk = blkmap(fs, blksfree, bbase);
		ffs_fragacct(fs, blk, cgp->cg_frsum, 1, needswap);
		/*
		 * if a complete block has been reassembled, account for it
		 */
		fragno = ffs_fragstoblks(fs, bbase);
		if (ffs_isblock(fs, blksfree, fragno)) {
			ufs_add32(cgp->cg_cs.cs_nffree, -fs->fs_frag, needswap);
			fs->fs_cstotal.cs_nffree -= fs->fs_frag;
			fs->fs_cs(fs, cg).cs_nffree -= fs->fs_frag;
			ffs_clusteracct(fs, cgp, fragno, 1);
			ufs_add32(cgp->cg_cs.cs_nbfree, 1, needswap);
			fs->fs_cstotal.cs_nbfree++;
			fs->fs_cs(fs, cg).cs_nbfree++;
			if ((fs->fs_magic == FS_UFS1_MAGIC) &&
			    ((fs->fs_old_flags & FS_FLAGS_UPDATED) == 0)) {
				i = old_cbtocylno(fs, bbase);
				KASSERT(i >= 0);
				KASSERT(i < fs->fs_old_ncyl);
				KASSERT(old_cbtorpos(fs, bbase) >= 0);
				KASSERT(fs->fs_old_nrpos == 0 || old_cbtorpos(fs, bbase) < fs->fs_old_nrpos);
				ufs_add16(old_cg_blks(fs, cgp, i, needswap)[old_cbtorpos(fs,
				    bbase)], 1, needswap);
				ufs_add32(old_cg_blktot(cgp, needswap)[i], 1, needswap);
			}
		}
	}
	fs->fs_fmod = 1;
	ACTIVECG_CLR(fs, cg);
	mutex_exit(&ump->um_lock);
}

/*
 * Free an inode.
 */
int
ffs_vfree(struct vnode *vp, ino_t ino, int mode)
{

	return ffs_freefile(vp->v_mount, ino, mode);
}

/*
 * Do the actual free operation.
 * The specified inode is placed back in the free map.
 *
 * => um_lock not held on entry or exit
 */
int
ffs_freefile(struct mount *mp, ino_t ino, int mode)
{
	struct ufsmount *ump = VFSTOUFS(mp);
	struct fs *fs = ump->um_fs;
	struct vnode *devvp;
	struct cg *cgp;
	struct buf *bp;
	int error, cg;
	daddr_t cgbno;
	dev_t dev;
	const int needswap = UFS_FSNEEDSWAP(fs);

	cg = ino_to_cg(fs, ino);
	devvp = ump->um_devvp;
	dev = devvp->v_rdev;
	cgbno = FFS_FSBTODB(fs, cgtod(fs, cg));

	if ((u_int)ino >= fs->fs_ipg * fs->fs_ncg)
		panic("%s: range: dev = 0x%llx, ino = %llu, fs = %s", __func__,
		    (long long)dev, (unsigned long long)ino, fs->fs_fsmnt);
	error = bread(devvp, cgbno, (int)fs->fs_cgsize,
	    B_MODIFY, &bp);
	if (error) {
		return (error);
	}
	cgp = (struct cg *)bp->b_data;
	if (!cg_chkmagic(cgp, needswap)) {
		brelse(bp, 0);
		return (0);
	}

	ffs_freefile_common(ump, fs, dev, bp, ino, mode, false);

	bdwrite(bp);

	return 0;
}

int
ffs_freefile_snap(struct fs *fs, struct vnode *devvp, ino_t ino, int mode)
{
	struct ufsmount *ump;
	struct cg *cgp;
	struct buf *bp;
	int error, cg;
	daddr_t cgbno;
	dev_t dev;
	const int needswap = UFS_FSNEEDSWAP(fs);

	KASSERT(devvp->v_type != VBLK);

	cg = ino_to_cg(fs, ino);
	dev = VTOI(devvp)->i_devvp->v_rdev;
	ump = VFSTOUFS(devvp->v_mount);
	cgbno = ffs_fragstoblks(fs, cgtod(fs, cg));
	if ((u_int)ino >= fs->fs_ipg * fs->fs_ncg)
		panic("%s: range: dev = 0x%llx, ino = %llu, fs = %s", __func__,
		    (unsigned long long)dev, (unsigned long long)ino,
		    fs->fs_fsmnt);
	error = bread(devvp, cgbno, (int)fs->fs_cgsize,
	    B_MODIFY, &bp);
	if (error) {
		return (error);
	}
	cgp = (struct cg *)bp->b_data;
	if (!cg_chkmagic(cgp, needswap)) {
		brelse(bp, 0);
		return (0);
	}
	ffs_freefile_common(ump, fs, dev, bp, ino, mode, true);

	bdwrite(bp);

	return 0;
}

static void
ffs_freefile_common(struct ufsmount *ump, struct fs *fs, dev_t dev,
    struct buf *bp, ino_t ino, int mode, bool devvp_is_snapshot)
{
	int cg;
	struct cg *cgp;
	u_int8_t *inosused;
	const int needswap = UFS_FSNEEDSWAP(fs);

	cg = ino_to_cg(fs, ino);
	cgp = (struct cg *)bp->b_data;
	cgp->cg_old_time = ufs_rw32(time_second, needswap);
	if ((fs->fs_magic != FS_UFS1_MAGIC) ||
	    (fs->fs_old_flags & FS_FLAGS_UPDATED))
		cgp->cg_time = ufs_rw64(time_second, needswap);
	inosused = cg_inosused(cgp, needswap);
	ino %= fs->fs_ipg;
	if (isclr(inosused, ino)) {
		printf("ifree: dev = 0x%llx, ino = %llu, fs = %s\n",
		    (unsigned long long)dev, (unsigned long long)ino +
		    cg * fs->fs_ipg, fs->fs_fsmnt);
		if (fs->fs_ronly == 0)
			panic("%s: freeing free inode", __func__);
	}
	clrbit(inosused, ino);
	if (!devvp_is_snapshot)
		UFS_WAPBL_UNREGISTER_INODE(ump->um_mountp,
		    ino + cg * fs->fs_ipg, mode);
	if (ino < ufs_rw32(cgp->cg_irotor, needswap))
		cgp->cg_irotor = ufs_rw32(ino, needswap);
	ufs_add32(cgp->cg_cs.cs_nifree, 1, needswap);
	mutex_enter(&ump->um_lock);
	fs->fs_cstotal.cs_nifree++;
	fs->fs_cs(fs, cg).cs_nifree++;
	if ((mode & IFMT) == IFDIR) {
		ufs_add32(cgp->cg_cs.cs_ndir, -1, needswap);
		fs->fs_cstotal.cs_ndir--;
		fs->fs_cs(fs, cg).cs_ndir--;
	}
	fs->fs_fmod = 1;
	ACTIVECG_CLR(fs, cg);
	mutex_exit(&ump->um_lock);
}

/*
 * Check to see if a file is free.
 */
int
ffs_checkfreefile(struct fs *fs, struct vnode *devvp, ino_t ino)
{
	struct cg *cgp;
	struct buf *bp;
	daddr_t cgbno;
	int ret, cg;
	u_int8_t *inosused;
	const bool devvp_is_snapshot = (devvp->v_type != VBLK);

	KASSERT(devvp_is_snapshot);

	cg = ino_to_cg(fs, ino);
	if (devvp_is_snapshot)
		cgbno = ffs_fragstoblks(fs, cgtod(fs, cg));
	else
		cgbno = FFS_FSBTODB(fs, cgtod(fs, cg));
	if ((u_int)ino >= fs->fs_ipg * fs->fs_ncg)
		return 1;
	if (bread(devvp, cgbno, (int)fs->fs_cgsize, 0, &bp)) {
		return 1;
	}
	cgp = (struct cg *)bp->b_data;
	if (!cg_chkmagic(cgp, UFS_FSNEEDSWAP(fs))) {
		brelse(bp, 0);
		return 1;
	}
	inosused = cg_inosused(cgp, UFS_FSNEEDSWAP(fs));
	ino %= fs->fs_ipg;
	ret = isclr(inosused, ino);
	brelse(bp, 0);
	return ret;
}

/*
 * Find a block of the specified size in the specified cylinder group.
 *
 * It is a panic if a request is made to find a block if none are
 * available.
 */
static int32_t
ffs_mapsearch(struct fs *fs, struct cg *cgp, daddr_t bpref, int allocsiz)
{
	int32_t bno;
	int start, len, loc, i;
	int blk, field, subfield, pos;
	int ostart, olen;
	u_int8_t *blksfree;
	const int needswap = UFS_FSNEEDSWAP(fs);

	/* KASSERT(mutex_owned(&ump->um_lock)); */

	/*
	 * find the fragment by searching through the free block
	 * map for an appropriate bit pattern
	 */
	if (bpref)
		start = dtogd(fs, bpref) / NBBY;
	else
		start = ufs_rw32(cgp->cg_frotor, needswap) / NBBY;
	blksfree = cg_blksfree(cgp, needswap);
	len = howmany(fs->fs_fpg, NBBY) - start;
	ostart = start;
	olen = len;
	loc = scanc((u_int)len,
		(const u_char *)&blksfree[start],
		(const u_char *)fragtbl[fs->fs_frag],
		(1 << (allocsiz - 1 + (fs->fs_frag & (NBBY - 1)))));
	if (loc == 0) {
		len = start + 1;
		start = 0;
		loc = scanc((u_int)len,
			(const u_char *)&blksfree[0],
			(const u_char *)fragtbl[fs->fs_frag],
			(1 << (allocsiz - 1 + (fs->fs_frag & (NBBY - 1)))));
		if (loc == 0) {
			panic("%s: map corrupted: start=%d, len=%d, "
			    "fs = %s, offset=%d/%ld, cg %d", __func__,
			    ostart, olen, fs->fs_fsmnt,
			    ufs_rw32(cgp->cg_freeoff, needswap),
			    (long)blksfree - (long)cgp, cgp->cg_cgx);
			/* NOTREACHED */
		}
	}
	bno = (start + len - loc) * NBBY;
	cgp->cg_frotor = ufs_rw32(bno, needswap);
	/*
	 * found the byte in the map
	 * sift through the bits to find the selected frag
	 */
	for (i = bno + NBBY; bno < i; bno += fs->fs_frag) {
		blk = blkmap(fs, blksfree, bno);
		blk <<= 1;
		field = around[allocsiz];
		subfield = inside[allocsiz];
		for (pos = 0; pos <= fs->fs_frag - allocsiz; pos++) {
			if ((blk & field) == subfield)
				return (bno + pos);
			field <<= 1;
			subfield <<= 1;
		}
	}
	panic("%s: block not in map: bno=%d, fs=%s", __func__,
	    bno, fs->fs_fsmnt);
	/* return (-1); */
}

/*
 * Fserr prints the name of a file system with an error diagnostic.
 *
 * The form of the error message is:
 *	fs: error message
 */
static void
ffs_fserr(struct fs *fs, kauth_cred_t cred, const char *cp)
{
	KASSERT(cred != NULL);

	if (cred == NOCRED || cred == FSCRED) {
		log(LOG_ERR, "pid %d, command %s, on %s: %s\n",
		    curproc->p_pid, curproc->p_comm,
		    fs->fs_fsmnt, cp);
	} else {
		log(LOG_ERR, "uid %d, pid %d, command %s, on %s: %s\n",
		    kauth_cred_getuid(cred), curproc->p_pid, curproc->p_comm,
		    fs->fs_fsmnt, cp);
	}
}