xref: /src/sys/uvm/uvm_swap.c

/*	$NetBSD: uvm_swap.c,v 1.200.2.1 2021/04/03 22:29:03 thorpej Exp $	*/

/*
 * Copyright (c) 1995, 1996, 1997, 2009 Matthew R. Green
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 * from: NetBSD: vm_swap.c,v 1.52 1997/12/02 13:47:37 pk Exp
 * from: Id: uvm_swap.c,v 1.1.2.42 1998/02/02 20:38:06 chuck Exp
 */

#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: uvm_swap.c,v 1.200.2.1 2021/04/03 22:29:03 thorpej Exp $");

#include "opt_uvmhist.h"
#include "opt_compat_netbsd.h"
#include "opt_ddb.h"

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/atomic.h>
#include <sys/buf.h>
#include <sys/bufq.h>
#include <sys/conf.h>
#include <sys/cprng.h>
#include <sys/proc.h>
#include <sys/namei.h>
#include <sys/disklabel.h>
#include <sys/errno.h>
#include <sys/kernel.h>
#include <sys/vnode.h>
#include <sys/file.h>
#include <sys/vmem.h>
#include <sys/blist.h>
#include <sys/mount.h>
#include <sys/pool.h>
#include <sys/kmem.h>
#include <sys/syscallargs.h>
#include <sys/swap.h>
#include <sys/kauth.h>
#include <sys/sysctl.h>
#include <sys/workqueue.h>

#include <uvm/uvm.h>

#include <miscfs/specfs/specdev.h>

#include <crypto/aes/aes.h>
#include <crypto/aes/aes_cbc.h>

/*
 * uvm_swap.c: manage configuration and i/o to swap space.
 */

/*
 * swap space is managed in the following way:
 *
 * each swap partition or file is described by a "swapdev" structure.
 * each "swapdev" structure contains a "swapent" structure which contains
 * information that is passed up to the user (via system calls).
 *
 * each swap partition is assigned a "priority" (int) which controls
 * swap partition usage.
 *
 * the system maintains a global data structure describing all swap
 * partitions/files.   there is a sorted LIST of "swappri" structures
 * which describe "swapdev"'s at that priority.   this LIST is headed
 * by the "swap_priority" global var.    each "swappri" contains a
 * TAILQ of "swapdev" structures at that priority.
 *
 * locking:
 *  - swap_syscall_lock (krwlock_t): this lock serializes the swapctl
 *    system call and prevents the swap priority list from changing
 *    while we are in the middle of a system call (e.g. SWAP_STATS).
 *  - uvm_swap_data_lock (kmutex_t): this lock protects all swap data
 *    structures including the priority list, the swapdev structures,
 *    and the swapmap arena.
 *
 * each swap device has the following info:
 *  - swap device in use (could be disabled, preventing future use)
 *  - swap enabled (allows new allocations on swap)
 *  - map info in /dev/drum
 *  - vnode pointer
 * for swap files only:
 *  - block size
 *  - max byte count in buffer
 *  - buffer
 *
 * userland controls and configures swap with the swapctl(2) system call.
 * the sys_swapctl performs the following operations:
 *  [1] SWAP_NSWAP: returns the number of swap devices currently configured
 *  [2] SWAP_STATS: given a pointer to an array of swapent structures
 *	(passed in via "arg") of a size passed in via "misc" ... we load
 *	the current swap config into the array. The actual work is done
 *	in the uvm_swap_stats() function.
 *  [3] SWAP_ON: given a pathname in arg (could be device or file) and a
 *	priority in "misc", start swapping on it.
 *  [4] SWAP_OFF: as SWAP_ON, but stops swapping to a device
 *  [5] SWAP_CTL: changes the priority of a swap device (new priority in
 *	"misc")
 */

/*
 * swapdev: describes a single swap partition/file
 *
 * note the following should be true:
 * swd_inuse <= swd_nblks  [number of blocks in use is <= total blocks]
 * swd_nblks <= swd_mapsize [because mapsize includes miniroot+disklabel]
 */
struct swapdev {
	dev_t			swd_dev;	/* device id */
	int			swd_flags;	/* flags:inuse/enable/fake */
	int			swd_priority;	/* our priority */
	int			swd_nblks;	/* blocks in this device */
	char			*swd_path;	/* saved pathname of device */
	int			swd_pathlen;	/* length of pathname */
	int			swd_npages;	/* #pages we can use */
	int			swd_npginuse;	/* #pages in use */
	int			swd_npgbad;	/* #pages bad */
	int			swd_drumoffset;	/* page0 offset in drum */
	int			swd_drumsize;	/* #pages in drum */
	blist_t			swd_blist;	/* blist for this swapdev */
	struct vnode		*swd_vp;	/* backing vnode */
	TAILQ_ENTRY(swapdev)	swd_next;	/* priority tailq */

	int			swd_bsize;	/* blocksize (bytes) */
	int			swd_maxactive;	/* max active i/o reqs */
	struct bufq_state	*swd_tab;	/* buffer list */
	int			swd_active;	/* number of active buffers */

	volatile uint32_t	*swd_encmap;	/* bitmap of encrypted slots */
	struct aesenc		swd_enckey;	/* AES key expanded for enc */
	struct aesdec		swd_deckey;	/* AES key expanded for dec */
	bool			swd_encinit;	/* true if keys initialized */
};

/*
 * swap device priority entry; the list is kept sorted on `spi_priority'.
 */
struct swappri {
	int			spi_priority;     /* priority */
	TAILQ_HEAD(spi_swapdev, swapdev)	spi_swapdev;
	/* tailq of swapdevs at this priority */
	LIST_ENTRY(swappri)	spi_swappri;      /* global list of pri's */
};

/*
 * The following two structures are used to keep track of data transfers
 * on swap devices associated with regular files.
 * NOTE: this code is more or less a copy of vnd.c; we use the same
 * structure names here to ease porting..
 */
struct vndxfer {
	struct buf	*vx_bp;		/* Pointer to parent buffer */
	struct swapdev	*vx_sdp;
	int		vx_error;
	int		vx_pending;	/* # of pending aux buffers */
	int		vx_flags;
#define VX_BUSY		1
#define VX_DEAD		2
};

struct vndbuf {
	struct buf	vb_buf;
	struct vndxfer	*vb_xfer;
};

/*
 * We keep a of pool vndbuf's and vndxfer structures.
 */
static struct pool vndxfer_pool, vndbuf_pool;

/*
 * local variables
 */
static vmem_t *swapmap;	/* controls the mapping of /dev/drum */

/* list of all active swap devices [by priority] */
LIST_HEAD(swap_priority, swappri);
static struct swap_priority swap_priority;

/* locks */
static kmutex_t uvm_swap_data_lock __cacheline_aligned;
static krwlock_t swap_syscall_lock;

/* workqueue and use counter for swap to regular files */
static int sw_reg_count = 0;
static struct workqueue *sw_reg_workqueue;

/* tuneables */
u_int uvm_swapisfull_factor = 99;
bool uvm_swap_encrypt = false;

/*
 * prototypes
 */
static struct swapdev	*swapdrum_getsdp(int);

static struct swapdev	*swaplist_find(struct vnode *, bool);
static void		 swaplist_insert(struct swapdev *,
					 struct swappri *, int);
static void		 swaplist_trim(void);

static int swap_on(struct lwp *, struct swapdev *);
static int swap_off(struct lwp *, struct swapdev *);

static void sw_reg_strategy(struct swapdev *, struct buf *, int);
static void sw_reg_biodone(struct buf *);
static void sw_reg_iodone(struct work *wk, void *dummy);
static void sw_reg_start(struct swapdev *);

static int uvm_swap_io(struct vm_page **, int, int, int);

static void uvm_swap_genkey(struct swapdev *);
static void uvm_swap_encryptpage(struct swapdev *, void *, int);
static void uvm_swap_decryptpage(struct swapdev *, void *, int);

static size_t
encmap_size(size_t npages)
{
	struct swapdev *sdp;
	const size_t bytesperword = sizeof(sdp->swd_encmap[0]);
	const size_t bitsperword = NBBY * bytesperword;
	const size_t nbits = npages; /* one bit for each page */
	const size_t nwords = howmany(nbits, bitsperword);
	const size_t nbytes = nwords * bytesperword;

	return nbytes;
}

/*
 * uvm_swap_init: init the swap system data structures and locks
 *
 * => called at boot time from init_main.c after the filesystems
 *	are brought up (which happens after uvm_init())
 */
void
uvm_swap_init(void)
{
	UVMHIST_FUNC(__func__);

	UVMHIST_CALLED(pdhist);
	/*
	 * first, init the swap list, its counter, and its lock.
	 * then get a handle on the vnode for /dev/drum by using
	 * the its dev_t number ("swapdev", from MD conf.c).
	 */

	LIST_INIT(&swap_priority);
	uvmexp.nswapdev = 0;
	rw_init(&swap_syscall_lock);
	mutex_init(&uvm_swap_data_lock, MUTEX_DEFAULT, IPL_NONE);

	if (bdevvp(swapdev, &swapdev_vp))
		panic("%s: can't get vnode for swap device", __func__);
	if (vn_lock(swapdev_vp, LK_EXCLUSIVE | LK_RETRY))
		panic("%s: can't lock swap device", __func__);
	if (VOP_OPEN(swapdev_vp, FREAD | FWRITE, NOCRED))
		panic("%s: can't open swap device", __func__);
	VOP_UNLOCK(swapdev_vp);

	/*
	 * create swap block resource map to map /dev/drum.   the range
	 * from 1 to INT_MAX allows 2 gigablocks of swap space.  note
	 * that block 0 is reserved (used to indicate an allocation
	 * failure, or no allocation).
	 */
	swapmap = vmem_create("swapmap", 1, INT_MAX - 1, 1, NULL, NULL, NULL, 0,
	    VM_NOSLEEP, IPL_NONE);
	if (swapmap == 0) {
		panic("%s: vmem_create failed", __func__);
	}

	pool_init(&vndxfer_pool, sizeof(struct vndxfer), 0, 0, 0, "swp vnx",
	    NULL, IPL_BIO);
	pool_init(&vndbuf_pool, sizeof(struct vndbuf), 0, 0, 0, "swp vnd",
	    NULL, IPL_BIO);

	UVMHIST_LOG(pdhist, "<- done", 0, 0, 0, 0);
}

/*
 * swaplist functions: functions that operate on the list of swap
 * devices on the system.
 */

/*
 * swaplist_insert: insert swap device "sdp" into the global list
 *
 * => caller must hold both swap_syscall_lock and uvm_swap_data_lock
 * => caller must provide a newly allocated swappri structure (we will
 *	FREE it if we don't need it... this it to prevent allocation
 *	blocking here while adding swap)
 */
static void
swaplist_insert(struct swapdev *sdp, struct swappri *newspp, int priority)
{
	struct swappri *spp, *pspp;
	UVMHIST_FUNC(__func__); UVMHIST_CALLED(pdhist);

	KASSERT(rw_write_held(&swap_syscall_lock));
	KASSERT(mutex_owned(&uvm_swap_data_lock));

	/*
	 * find entry at or after which to insert the new device.
	 */
	pspp = NULL;
	LIST_FOREACH(spp, &swap_priority, spi_swappri) {
		if (priority <= spp->spi_priority)
			break;
		pspp = spp;
	}

	/*
	 * new priority?
	 */
	if (spp == NULL || spp->spi_priority != priority) {
		spp = newspp;  /* use newspp! */
		UVMHIST_LOG(pdhist, "created new swappri = %jd",
			    priority, 0, 0, 0);

		spp->spi_priority = priority;
		TAILQ_INIT(&spp->spi_swapdev);

		if (pspp)
			LIST_INSERT_AFTER(pspp, spp, spi_swappri);
		else
			LIST_INSERT_HEAD(&swap_priority, spp, spi_swappri);
	} else {
	  	/* we don't need a new priority structure, free it */
		kmem_free(newspp, sizeof(*newspp));
	}

	/*
	 * priority found (or created).   now insert on the priority's
	 * tailq list and bump the total number of swapdevs.
	 */
	sdp->swd_priority = priority;
	TAILQ_INSERT_TAIL(&spp->spi_swapdev, sdp, swd_next);
	uvmexp.nswapdev++;
}

/*
 * swaplist_find: find and optionally remove a swap device from the
 *	global list.
 *
 * => caller must hold both swap_syscall_lock and uvm_swap_data_lock
 * => we return the swapdev we found (and removed)
 */
static struct swapdev *
swaplist_find(struct vnode *vp, bool remove)
{
	struct swapdev *sdp;
	struct swappri *spp;

	KASSERT(rw_lock_held(&swap_syscall_lock));
	KASSERT(remove ? rw_write_held(&swap_syscall_lock) : 1);
	KASSERT(mutex_owned(&uvm_swap_data_lock));

	/*
	 * search the lists for the requested vp
	 */

	LIST_FOREACH(spp, &swap_priority, spi_swappri) {
		TAILQ_FOREACH(sdp, &spp->spi_swapdev, swd_next) {
			if (sdp->swd_vp == vp) {
				if (remove) {
					TAILQ_REMOVE(&spp->spi_swapdev,
					    sdp, swd_next);
					uvmexp.nswapdev--;
				}
				return(sdp);
			}
		}
	}
	return (NULL);
}

/*
 * swaplist_trim: scan priority list for empty priority entries and kill
 *	them.
 *
 * => caller must hold both swap_syscall_lock and uvm_swap_data_lock
 */
static void
swaplist_trim(void)
{
	struct swappri *spp, *nextspp;

	KASSERT(rw_write_held(&swap_syscall_lock));
	KASSERT(mutex_owned(&uvm_swap_data_lock));

	LIST_FOREACH_SAFE(spp, &swap_priority, spi_swappri, nextspp) {
		if (!TAILQ_EMPTY(&spp->spi_swapdev))
			continue;
		LIST_REMOVE(spp, spi_swappri);
		kmem_free(spp, sizeof(*spp));
	}
}

/*
 * swapdrum_getsdp: given a page offset in /dev/drum, convert it back
 *	to the "swapdev" that maps that section of the drum.
 *
 * => each swapdev takes one big contig chunk of the drum
 * => caller must hold uvm_swap_data_lock
 */
static struct swapdev *
swapdrum_getsdp(int pgno)
{
	struct swapdev *sdp;
	struct swappri *spp;

	KASSERT(mutex_owned(&uvm_swap_data_lock));

	LIST_FOREACH(spp, &swap_priority, spi_swappri) {
		TAILQ_FOREACH(sdp, &spp->spi_swapdev, swd_next) {
			if (sdp->swd_flags & SWF_FAKE)
				continue;
			if (pgno >= sdp->swd_drumoffset &&
			    pgno < (sdp->swd_drumoffset + sdp->swd_drumsize)) {
				return sdp;
			}
		}
	}
	return NULL;
}

/*
 * swapdrum_sdp_is: true iff the swap device for pgno is sdp
 *
 * => for use in positive assertions only; result is not stable
 */
static bool __debugused
swapdrum_sdp_is(int pgno, struct swapdev *sdp)
{
	bool result;

	mutex_enter(&uvm_swap_data_lock);
	result = swapdrum_getsdp(pgno) == sdp;
	mutex_exit(&uvm_swap_data_lock);

	return result;
}

void swapsys_lock(krw_t op)
{
	rw_enter(&swap_syscall_lock, op);
}

void swapsys_unlock(void)
{
	rw_exit(&swap_syscall_lock);
}

static void
swapent_cvt(struct swapent *se, const struct swapdev *sdp, int inuse)
{
	se->se_dev = sdp->swd_dev;
	se->se_flags = sdp->swd_flags;
	se->se_nblks = sdp->swd_nblks;
	se->se_inuse = inuse;
	se->se_priority = sdp->swd_priority;
	KASSERT(sdp->swd_pathlen < sizeof(se->se_path));
	strcpy(se->se_path, sdp->swd_path);
}

int (*uvm_swap_stats13)(const struct sys_swapctl_args *, register_t *) =
    (void *)enosys;
int (*uvm_swap_stats50)(const struct sys_swapctl_args *, register_t *) =
    (void *)enosys;

/*
 * sys_swapctl: main entry point for swapctl(2) system call
 * 	[with two helper functions: swap_on and swap_off]
 */
int
sys_swapctl(struct lwp *l, const struct sys_swapctl_args *uap, register_t *retval)
{
	/* {
		syscallarg(int) cmd;
		syscallarg(void *) arg;
		syscallarg(int) misc;
	} */
	struct vnode *vp;
	struct nameidata nd;
	struct swappri *spp;
	struct swapdev *sdp;
#define SWAP_PATH_MAX (PATH_MAX + 1)
	char	*userpath;
	size_t	len = 0;
	int	error;
	int	priority;
	UVMHIST_FUNC(__func__); UVMHIST_CALLED(pdhist);

	/*
	 * we handle the non-priv NSWAP and STATS request first.
	 *
	 * SWAP_NSWAP: return number of config'd swap devices
	 * [can also be obtained with uvmexp sysctl]
	 */
	if (SCARG(uap, cmd) == SWAP_NSWAP) {
		const int nswapdev = uvmexp.nswapdev;
		UVMHIST_LOG(pdhist, "<- done SWAP_NSWAP=%jd", nswapdev,
		    0, 0, 0);
		*retval = nswapdev;
		return 0;
	}

	userpath = kmem_alloc(SWAP_PATH_MAX, KM_SLEEP);

	/*
	 * ensure serialized syscall access by grabbing the swap_syscall_lock
	 */
	rw_enter(&swap_syscall_lock, RW_WRITER);

	/*
	 * SWAP_STATS: get stats on current # of configured swap devs
	 *
	 * note that the swap_priority list can't change as long
	 * as we are holding the swap_syscall_lock.  we don't want
	 * to grab the uvm_swap_data_lock because we may fault&sleep during
	 * copyout() and we don't want to be holding that lock then!
	 */
	switch (SCARG(uap, cmd)) {
	case SWAP_STATS13:
		error = (*uvm_swap_stats13)(uap, retval);
		goto out;
	case SWAP_STATS50:
		error = (*uvm_swap_stats50)(uap, retval);
		goto out;
	case SWAP_STATS:
		error = uvm_swap_stats(SCARG(uap, arg), SCARG(uap, misc),
		    NULL, sizeof(struct swapent), retval);
		UVMHIST_LOG(pdhist, "<- done SWAP_STATS", 0, 0, 0, 0);
		goto out;

	case SWAP_GETDUMPDEV:
		error = copyout(&dumpdev, SCARG(uap, arg), sizeof(dumpdev));
		goto out;
	default:
		break;
	}

	/*
	 * all other requests require superuser privs.   verify.
	 */
	if ((error = kauth_authorize_system(l->l_cred, KAUTH_SYSTEM_SWAPCTL,
	    0, NULL, NULL, NULL)))
		goto out;

	if (SCARG(uap, cmd) == SWAP_DUMPOFF) {
		/* drop the current dump device */
		dumpdev = NODEV;
		dumpcdev = NODEV;
		cpu_dumpconf();
		goto out;
	}

	/*
	 * at this point we expect a path name in arg.   we will
	 * use namei() to gain a vnode reference (vref), and lock
	 * the vnode (VOP_LOCK).
	 *
	 * XXX: a NULL arg means use the root vnode pointer (e.g. for
	 * miniroot)
	 */
	if (SCARG(uap, arg) == NULL) {
		vp = rootvp;		/* miniroot */
		vref(vp);
		if (vn_lock(vp, LK_EXCLUSIVE)) {
			vrele(vp);
			error = EBUSY;
			goto out;
		}
		if (SCARG(uap, cmd) == SWAP_ON &&
		    copystr("miniroot", userpath, SWAP_PATH_MAX, &len))
			panic("swapctl: miniroot copy failed");
	} else {
		struct pathbuf *pb;

		/*
		 * This used to allow copying in one extra byte
		 * (SWAP_PATH_MAX instead of PATH_MAX) for SWAP_ON.
		 * This was completely pointless because if anyone
		 * used that extra byte namei would fail with
		 * ENAMETOOLONG anyway, so I've removed the excess
		 * logic. - dholland 20100215
		 */

		error = pathbuf_copyin(SCARG(uap, arg), &pb);
		if (error) {
			goto out;
		}
		if (SCARG(uap, cmd) == SWAP_ON) {
			/* get a copy of the string */
			pathbuf_copystring(pb, userpath, SWAP_PATH_MAX);
			len = strlen(userpath) + 1;
		}
		NDINIT(&nd, LOOKUP, FOLLOW | LOCKLEAF | TRYEMULROOT, pb);
		if ((error = namei(&nd))) {
			pathbuf_destroy(pb);
			goto out;
		}
		vp = nd.ni_vp;
		pathbuf_destroy(pb);
	}
	/* note: "vp" is referenced and locked */

	error = 0;		/* assume no error */
	switch(SCARG(uap, cmd)) {

	case SWAP_DUMPDEV:
		if (vp->v_type != VBLK) {
			error = ENOTBLK;
			break;
		}
		if (bdevsw_lookup(vp->v_rdev)) {
			dumpdev = vp->v_rdev;
			dumpcdev = devsw_blk2chr(dumpdev);
		} else
			dumpdev = NODEV;
		cpu_dumpconf();
		break;

	case SWAP_CTL:
		/*
		 * get new priority, remove old entry (if any) and then
		 * reinsert it in the correct place.  finally, prune out
		 * any empty priority structures.
		 */
		priority = SCARG(uap, misc);
		spp = kmem_alloc(sizeof(*spp), KM_SLEEP);
		mutex_enter(&uvm_swap_data_lock);
		if ((sdp = swaplist_find(vp, true)) == NULL) {
			error = ENOENT;
		} else {
			swaplist_insert(sdp, spp, priority);
			swaplist_trim();
		}
		mutex_exit(&uvm_swap_data_lock);
		if (error)
			kmem_free(spp, sizeof(*spp));
		break;

	case SWAP_ON:

		/*
		 * check for duplicates.   if none found, then insert a
		 * dummy entry on the list to prevent someone else from
		 * trying to enable this device while we are working on
		 * it.
		 */

		priority = SCARG(uap, misc);
		sdp = kmem_zalloc(sizeof(*sdp), KM_SLEEP);
		spp = kmem_alloc(sizeof(*spp), KM_SLEEP);
		sdp->swd_flags = SWF_FAKE;
		sdp->swd_vp = vp;
		sdp->swd_dev = (vp->v_type == VBLK) ? vp->v_rdev : NODEV;
		bufq_alloc(&sdp->swd_tab, "disksort", BUFQ_SORT_RAWBLOCK);
		mutex_enter(&uvm_swap_data_lock);
		if (swaplist_find(vp, false) != NULL) {
			error = EBUSY;
			mutex_exit(&uvm_swap_data_lock);
			bufq_free(sdp->swd_tab);
			kmem_free(sdp, sizeof(*sdp));
			kmem_free(spp, sizeof(*spp));
			break;
		}
		swaplist_insert(sdp, spp, priority);
		mutex_exit(&uvm_swap_data_lock);

		KASSERT(len > 0);
		sdp->swd_pathlen = len;
		sdp->swd_path = kmem_alloc(len, KM_SLEEP);
		if (copystr(userpath, sdp->swd_path, len, 0) != 0)
			panic("swapctl: copystr");

		/*
		 * we've now got a FAKE placeholder in the swap list.
		 * now attempt to enable swap on it.  if we fail, undo
		 * what we've done and kill the fake entry we just inserted.
		 * if swap_on is a success, it will clear the SWF_FAKE flag
		 */

		if ((error = swap_on(l, sdp)) != 0) {
			mutex_enter(&uvm_swap_data_lock);
			(void) swaplist_find(vp, true);  /* kill fake entry */
			swaplist_trim();
			mutex_exit(&uvm_swap_data_lock);
			bufq_free(sdp->swd_tab);
			kmem_free(sdp->swd_path, sdp->swd_pathlen);
			kmem_free(sdp, sizeof(*sdp));
			break;
		}
		break;

	case SWAP_OFF:
		mutex_enter(&uvm_swap_data_lock);
		if ((sdp = swaplist_find(vp, false)) == NULL) {
			mutex_exit(&uvm_swap_data_lock);
			error = ENXIO;
			break;
		}

		/*
		 * If a device isn't in use or enabled, we
		 * can't stop swapping from it (again).
		 */
		if ((sdp->swd_flags & (SWF_INUSE|SWF_ENABLE)) == 0) {
			mutex_exit(&uvm_swap_data_lock);
			error = EBUSY;
			break;
		}

		/*
		 * do the real work.
		 */
		error = swap_off(l, sdp);
		break;

	default:
		error = EINVAL;
	}

	/*
	 * done!  release the ref gained by namei() and unlock.
	 */
	vput(vp);
out:
	rw_exit(&swap_syscall_lock);
	kmem_free(userpath, SWAP_PATH_MAX);

	UVMHIST_LOG(pdhist, "<- done!  error=%jd", error, 0, 0, 0);
	return (error);
}

/*
 * uvm_swap_stats: implements swapctl(SWAP_STATS). The function is kept
 * away from sys_swapctl() in order to allow COMPAT_* swapctl()
 * emulation to use it directly without going through sys_swapctl().
 * The problem with using sys_swapctl() there is that it involves
 * copying the swapent array to the stackgap, and this array's size
 * is not known at build time. Hence it would not be possible to
 * ensure it would fit in the stackgap in any case.
 */
int
uvm_swap_stats(char *ptr, int misc,
    void (*f)(void *, const struct swapent *), size_t len,
    register_t *retval)
{
	struct swappri *spp;
	struct swapdev *sdp;
	struct swapent sep;
	int count = 0;
	int error;

	KASSERT(len <= sizeof(sep));
	if (len == 0)
		return ENOSYS;

	if (misc < 0)
		return EINVAL;

	if (misc == 0 || uvmexp.nswapdev == 0)
		return 0;

	/* Make sure userland cannot exhaust kernel memory */
	if ((size_t)misc > (size_t)uvmexp.nswapdev)
		misc = uvmexp.nswapdev;

	KASSERT(rw_lock_held(&swap_syscall_lock));

	LIST_FOREACH(spp, &swap_priority, spi_swappri) {
		TAILQ_FOREACH(sdp, &spp->spi_swapdev, swd_next) {
			int inuse;

			if (misc-- <= 0)
				break;

			inuse = btodb((uint64_t)sdp->swd_npginuse <<
			    PAGE_SHIFT);

			memset(&sep, 0, sizeof(sep));
			swapent_cvt(&sep, sdp, inuse);
			if (f)
				(*f)(&sep, &sep);
			if ((error = copyout(&sep, ptr, len)) != 0)
				return error;
			ptr += len;
			count++;
		}
	}
	*retval = count;
	return 0;
}

/*
 * swap_on: attempt to enable a swapdev for swapping.   note that the
 *	swapdev is already on the global list, but disabled (marked
 *	SWF_FAKE).
 *
 * => we avoid the start of the disk (to protect disk labels)
 * => we also avoid the miniroot, if we are swapping to root.
 * => caller should leave uvm_swap_data_lock unlocked, we may lock it
 *	if needed.
 */
static int
swap_on(struct lwp *l, struct swapdev *sdp)
{
	struct vnode *vp;
	int error, npages, nblocks, size;
	long addr;
	vmem_addr_t result;
	struct vattr va;
	dev_t dev;
	UVMHIST_FUNC(__func__); UVMHIST_CALLED(pdhist);

	/*
	 * we want to enable swapping on sdp.   the swd_vp contains
	 * the vnode we want (locked and ref'd), and the swd_dev
	 * contains the dev_t of the file, if it a block device.
	 */

	vp = sdp->swd_vp;
	dev = sdp->swd_dev;

	/*
	 * open the swap file (mostly useful for block device files to
	 * let device driver know what is up).
	 *
	 * we skip the open/close for root on swap because the root
	 * has already been opened when root was mounted (mountroot).
	 */
	if (vp != rootvp) {
		if ((error = VOP_OPEN(vp, FREAD|FWRITE, l->l_cred)))
			return (error);
	}

	/* XXX this only works for block devices */
	UVMHIST_LOG(pdhist, "  dev=%jd, major(dev)=%jd", dev, major(dev), 0, 0);

	/*
	 * we now need to determine the size of the swap area.   for
	 * block specials we can call the d_psize function.
	 * for normal files, we must stat [get attrs].
	 *
	 * we put the result in nblks.
	 * for normal files, we also want the filesystem block size
	 * (which we get with statfs).
	 */
	switch (vp->v_type) {
	case VBLK:
		if ((nblocks = bdev_size(dev)) == -1) {
			error = ENXIO;
			goto bad;
		}
		break;

	case VREG:
		if ((error = VOP_GETATTR(vp, &va, l->l_cred)))
			goto bad;
		nblocks = (int)btodb(va.va_size);
		sdp->swd_bsize = 1 << vp->v_mount->mnt_fs_bshift;
		/*
		 * limit the max # of outstanding I/O requests we issue
		 * at any one time.   take it easy on NFS servers.
		 */
		if (vp->v_tag == VT_NFS)
			sdp->swd_maxactive = 2; /* XXX */
		else
			sdp->swd_maxactive = 8; /* XXX */
		break;

	default:
		error = ENXIO;
		goto bad;
	}

	/*
	 * save nblocks in a safe place and convert to pages.
	 */

	sdp->swd_nblks = nblocks;
	npages = dbtob((uint64_t)nblocks) >> PAGE_SHIFT;

	/*
	 * for block special files, we want to make sure that leave
	 * the disklabel and bootblocks alone, so we arrange to skip
	 * over them (arbitrarily choosing to skip PAGE_SIZE bytes).
	 * note that because of this the "size" can be less than the
	 * actual number of blocks on the device.
	 */
	if (vp->v_type == VBLK) {
		/* we use pages 1 to (size - 1) [inclusive] */
		size = npages - 1;
		addr = 1;
	} else {
		/* we use pages 0 to (size - 1) [inclusive] */
		size = npages;
		addr = 0;
	}

	/*
	 * make sure we have enough blocks for a reasonable sized swap
	 * area.   we want at least one page.
	 */

	if (size < 1) {
		UVMHIST_LOG(pdhist, "  size <= 1!!", 0, 0, 0, 0);
		error = EINVAL;
		goto bad;
	}

	UVMHIST_LOG(pdhist, "  dev=%#jx: size=%jd addr=%jd", dev, size, addr, 0);

	/*
	 * now we need to allocate an extent to manage this swap device
	 */

	sdp->swd_blist = blist_create(npages);
	/* mark all expect the `saved' region free. */
	blist_free(sdp->swd_blist, addr, size);

	/*
	 * allocate space to for swap encryption state and mark the
	 * keys uninitialized so we generate them lazily
	 */
	sdp->swd_encmap = kmem_zalloc(encmap_size(npages), KM_SLEEP);
	sdp->swd_encinit = false;

	/*
	 * if the vnode we are swapping to is the root vnode
	 * (i.e. we are swapping to the miniroot) then we want
	 * to make sure we don't overwrite it.   do a statfs to
	 * find its size and skip over it.
	 */
	if (vp == rootvp) {
		struct mount *mp;
		struct statvfs *sp;
		int rootblocks, rootpages;

		mp = rootvnode->v_mount;
		sp = &mp->mnt_stat;
		rootblocks = sp->f_blocks * btodb(sp->f_frsize);
		/*
		 * XXX: sp->f_blocks isn't the total number of
		 * blocks in the filesystem, it's the number of
		 * data blocks.  so, our rootblocks almost
		 * definitely underestimates the total size
		 * of the filesystem - how badly depends on the
		 * details of the filesystem type.  there isn't
		 * an obvious way to deal with this cleanly
		 * and perfectly, so for now we just pad our
		 * rootblocks estimate with an extra 5 percent.
		 */
		rootblocks += (rootblocks >> 5) +
			(rootblocks >> 6) +
			(rootblocks >> 7);
		rootpages = round_page(dbtob(rootblocks)) >> PAGE_SHIFT;
		if (rootpages > size)
			panic("swap_on: miniroot larger than swap?");

		if (rootpages != blist_fill(sdp->swd_blist, addr, rootpages)) {
			panic("swap_on: unable to preserve miniroot");
		}

		size -= rootpages;
		printf("Preserved %d pages of miniroot ", rootpages);
		printf("leaving %d pages of swap\n", size);
	}

	/*
	 * add a ref to vp to reflect usage as a swap device.
	 */
	vref(vp);

	/*
	 * now add the new swapdev to the drum and enable.
	 */
	error = vmem_alloc(swapmap, npages, VM_BESTFIT | VM_SLEEP, &result);
	if (error != 0)
		panic("swapdrum_add");
	/*
	 * If this is the first regular swap create the workqueue.
	 * => Protected by swap_syscall_lock.
	 */
	if (vp->v_type != VBLK) {
		if (sw_reg_count++ == 0) {
			KASSERT(sw_reg_workqueue == NULL);
			if (workqueue_create(&sw_reg_workqueue, "swapiod",
			    sw_reg_iodone, NULL, PRIBIO, IPL_BIO, 0) != 0)
				panic("%s: workqueue_create failed", __func__);
		}
	}

	sdp->swd_drumoffset = (int)result;
	sdp->swd_drumsize = npages;
	sdp->swd_npages = size;
	mutex_enter(&uvm_swap_data_lock);
	sdp->swd_flags &= ~SWF_FAKE;	/* going live */
	sdp->swd_flags |= (SWF_INUSE|SWF_ENABLE);
	uvmexp.swpages += size;
	uvmexp.swpgavail += size;
	mutex_exit(&uvm_swap_data_lock);
	return (0);

	/*
	 * failure: clean up and return error.
	 */

bad:
	if (sdp->swd_blist) {
		blist_destroy(sdp->swd_blist);
	}
	if (vp != rootvp) {
		(void)VOP_CLOSE(vp, FREAD|FWRITE, l->l_cred);
	}
	return (error);
}

/*
 * swap_off: stop swapping on swapdev
 *
 * => swap data should be locked, we will unlock.
 */
static int
swap_off(struct lwp *l, struct swapdev *sdp)
{
	int npages = sdp->swd_npages;
	int error = 0;

	UVMHIST_FUNC(__func__);
	UVMHIST_CALLARGS(pdhist, "  dev=%#jx, npages=%jd", sdp->swd_dev,npages, 0, 0);

	KASSERT(rw_write_held(&swap_syscall_lock));
	KASSERT(mutex_owned(&uvm_swap_data_lock));

	/* disable the swap area being removed */
	sdp->swd_flags &= ~SWF_ENABLE;
	uvmexp.swpgavail -= npages;
	mutex_exit(&uvm_swap_data_lock);

	/*
	 * the idea is to find all the pages that are paged out to this
	 * device, and page them all in.  in uvm, swap-backed pageable
	 * memory can take two forms: aobjs and anons.  call the
	 * swapoff hook for each subsystem to bring in pages.
	 */

	if (uao_swap_off(sdp->swd_drumoffset,
			 sdp->swd_drumoffset + sdp->swd_drumsize) ||
	    amap_swap_off(sdp->swd_drumoffset,
			  sdp->swd_drumoffset + sdp->swd_drumsize)) {
		error = ENOMEM;
	} else if (sdp->swd_npginuse > sdp->swd_npgbad) {
		error = EBUSY;
	}

	if (error) {
		mutex_enter(&uvm_swap_data_lock);
		sdp->swd_flags |= SWF_ENABLE;
		uvmexp.swpgavail += npages;
		mutex_exit(&uvm_swap_data_lock);

		return error;
	}

	/*
	 * If this is the last regular swap destroy the workqueue.
	 * => Protected by swap_syscall_lock.
	 */
	if (sdp->swd_vp->v_type != VBLK) {
		KASSERT(sw_reg_count > 0);
		KASSERT(sw_reg_workqueue != NULL);
		if (--sw_reg_count == 0) {
			workqueue_destroy(sw_reg_workqueue);
			sw_reg_workqueue = NULL;
		}
	}

	/*
	 * done with the vnode.
	 * drop our ref on the vnode before calling VOP_CLOSE()
	 * so that spec_close() can tell if this is the last close.
	 */
	vrele(sdp->swd_vp);
	if (sdp->swd_vp != rootvp) {
		(void) VOP_CLOSE(sdp->swd_vp, FREAD|FWRITE, l->l_cred);
	}

	mutex_enter(&uvm_swap_data_lock);
	uvmexp.swpages -= npages;
	uvmexp.swpginuse -= sdp->swd_npgbad;

	if (swaplist_find(sdp->swd_vp, true) == NULL)
		panic("%s: swapdev not in list", __func__);
	swaplist_trim();
	mutex_exit(&uvm_swap_data_lock);

	/*
	 * free all resources!
	 */
	vmem_free(swapmap, sdp->swd_drumoffset, sdp->swd_drumsize);
	blist_destroy(sdp->swd_blist);
	bufq_free(sdp->swd_tab);
	kmem_free(__UNVOLATILE(sdp->swd_encmap),
	    encmap_size(sdp->swd_drumsize));
	explicit_memset(&sdp->swd_enckey, 0, sizeof sdp->swd_enckey);
	explicit_memset(&sdp->swd_deckey, 0, sizeof sdp->swd_deckey);
	kmem_free(sdp, sizeof(*sdp));
	return (0);
}

void
uvm_swap_shutdown(struct lwp *l)
{
	struct swapdev *sdp;
	struct swappri *spp;
	struct vnode *vp;
	int error;

	printf("turning off swap...");
	rw_enter(&swap_syscall_lock, RW_WRITER);
	mutex_enter(&uvm_swap_data_lock);
again:
	LIST_FOREACH(spp, &swap_priority, spi_swappri)
		TAILQ_FOREACH(sdp, &spp->spi_swapdev, swd_next) {
			if (sdp->swd_flags & SWF_FAKE)
				continue;
			if ((sdp->swd_flags & (SWF_INUSE|SWF_ENABLE)) == 0)
				continue;
#ifdef DEBUG
			printf("\nturning off swap on %s...", sdp->swd_path);
#endif
			/* Have to lock and reference vnode for swap_off(). */
			vn_lock(vp = sdp->swd_vp, LK_EXCLUSIVE|LK_RETRY);
			vref(vp);
			error = swap_off(l, sdp);
			vput(vp);
			mutex_enter(&uvm_swap_data_lock);
			if (error) {
				printf("stopping swap on %s failed "
				    "with error %d\n", sdp->swd_path, error);
				TAILQ_REMOVE(&spp->spi_swapdev, sdp, swd_next);
				uvmexp.nswapdev--;
				swaplist_trim();
			}
			goto again;
		}
	printf(" done\n");
	mutex_exit(&uvm_swap_data_lock);
	rw_exit(&swap_syscall_lock);
}


/*
 * /dev/drum interface and i/o functions
 */

/*
 * swstrategy: perform I/O on the drum
 *
 * => we must map the i/o request from the drum to the correct swapdev.
 */
static void
swstrategy(struct buf *bp)
{
	struct swapdev *sdp;
	struct vnode *vp;
	int pageno, bn;
	UVMHIST_FUNC(__func__); UVMHIST_CALLED(pdhist);

	/*
	 * convert block number to swapdev.   note that swapdev can't
	 * be yanked out from under us because we are holding resources
	 * in it (i.e. the blocks we are doing I/O on).
	 */
	pageno = dbtob((int64_t)bp->b_blkno) >> PAGE_SHIFT;
	mutex_enter(&uvm_swap_data_lock);
	sdp = swapdrum_getsdp(pageno);
	mutex_exit(&uvm_swap_data_lock);
	if (sdp == NULL) {
		bp->b_error = EINVAL;
		bp->b_resid = bp->b_bcount;
		biodone(bp);
		UVMHIST_LOG(pdhist, "  failed to get swap device", 0, 0, 0, 0);
		return;
	}

	/*
	 * convert drum page number to block number on this swapdev.
	 */

	pageno -= sdp->swd_drumoffset;	/* page # on swapdev */
	bn = btodb((uint64_t)pageno << PAGE_SHIFT); /* convert to diskblock */

	UVMHIST_LOG(pdhist, "  Rd/Wr (0/1) %jd: mapoff=%#jx bn=%#jx bcount=%jd",
		((bp->b_flags & B_READ) == 0) ? 1 : 0,
		sdp->swd_drumoffset, bn, bp->b_bcount);

	/*
	 * for block devices we finish up here.
	 * for regular files we have to do more work which we delegate
	 * to sw_reg_strategy().
	 */

	vp = sdp->swd_vp;		/* swapdev vnode pointer */
	switch (vp->v_type) {
	default:
		panic("%s: vnode type 0x%x", __func__, vp->v_type);

	case VBLK:

		/*
		 * must convert "bp" from an I/O on /dev/drum to an I/O
		 * on the swapdev (sdp).
		 */
		bp->b_blkno = bn;		/* swapdev block number */
		bp->b_dev = sdp->swd_dev;	/* swapdev dev_t */

		/*
		 * if we are doing a write, we have to redirect the i/o on
		 * drum's v_numoutput counter to the swapdevs.
		 */
		if ((bp->b_flags & B_READ) == 0) {
			mutex_enter(bp->b_objlock);
			vwakeup(bp);	/* kills one 'v_numoutput' on drum */
			mutex_exit(bp->b_objlock);
			mutex_enter(vp->v_interlock);
			vp->v_numoutput++;	/* put it on swapdev */
			mutex_exit(vp->v_interlock);
		}

		/*
		 * finally plug in swapdev vnode and start I/O
		 */
		bp->b_vp = vp;
		bp->b_objlock = vp->v_interlock;
		VOP_STRATEGY(vp, bp);
		return;

	case VREG:
		/*
		 * delegate to sw_reg_strategy function.
		 */
		sw_reg_strategy(sdp, bp, bn);
		return;
	}
	/* NOTREACHED */
}

/*
 * swread: the read function for the drum (just a call to physio)
 */
/*ARGSUSED*/
static int
swread(dev_t dev, struct uio *uio, int ioflag)
{
	UVMHIST_FUNC(__func__);
	UVMHIST_CALLARGS(pdhist, "  dev=%#jx offset=%#jx", dev, uio->uio_offset, 0, 0);

	return (physio(swstrategy, NULL, dev, B_READ, minphys, uio));
}

/*
 * swwrite: the write function for the drum (just a call to physio)
 */
/*ARGSUSED*/
static int
swwrite(dev_t dev, struct uio *uio, int ioflag)
{
	UVMHIST_FUNC(__func__);
	UVMHIST_CALLARGS(pdhist, "  dev=%#jx offset=%#jx", dev, uio->uio_offset, 0, 0);

	return (physio(swstrategy, NULL, dev, B_WRITE, minphys, uio));
}

const struct bdevsw swap_bdevsw = {
	.d_open = nullopen,
	.d_close = nullclose,
	.d_strategy = swstrategy,
	.d_ioctl = noioctl,
	.d_dump = nodump,
	.d_psize = nosize,
	.d_discard = nodiscard,
	.d_flag = D_OTHER
};

const struct cdevsw swap_cdevsw = {
	.d_open = nullopen,
	.d_close = nullclose,
	.d_read = swread,
	.d_write = swwrite,
	.d_ioctl = noioctl,
	.d_stop = nostop,
	.d_tty = notty,
	.d_poll = nopoll,
	.d_mmap = nommap,
	.d_kqfilter = nokqfilter,
	.d_discard = nodiscard,
	.d_flag = D_OTHER,
};

/*
 * sw_reg_strategy: handle swap i/o to regular files
 */
static void
sw_reg_strategy(struct swapdev *sdp, struct buf *bp, int bn)
{
	struct vnode	*vp;
	struct vndxfer	*vnx;
	daddr_t		nbn;
	char 		*addr;
	off_t		byteoff;
	int		s, off, nra, error, sz, resid;
	UVMHIST_FUNC(__func__); UVMHIST_CALLED(pdhist);

	/*
	 * allocate a vndxfer head for this transfer and point it to
	 * our buffer.
	 */
	vnx = pool_get(&vndxfer_pool, PR_WAITOK);
	vnx->vx_flags = VX_BUSY;
	vnx->vx_error = 0;
	vnx->vx_pending = 0;
	vnx->vx_bp = bp;
	vnx->vx_sdp = sdp;

	/*
	 * setup for main loop where we read filesystem blocks into
	 * our buffer.
	 */
	error = 0;
	bp->b_resid = bp->b_bcount;	/* nothing transferred yet! */
	addr = bp->b_data;		/* current position in buffer */
	byteoff = dbtob((uint64_t)bn);

	for (resid = bp->b_resid; resid; resid -= sz) {
		struct vndbuf	*nbp;

		/*
		 * translate byteoffset into block number.  return values:
		 *   vp = vnode of underlying device
		 *  nbn = new block number (on underlying vnode dev)
		 *  nra = num blocks we can read-ahead (excludes requested
		 *	block)
		 */
		nra = 0;
		error = VOP_BMAP(sdp->swd_vp, byteoff / sdp->swd_bsize,
				 	&vp, &nbn, &nra);

		if (error == 0 && nbn == (daddr_t)-1) {
			/*
			 * this used to just set error, but that doesn't
			 * do the right thing.  Instead, it causes random
			 * memory errors.  The panic() should remain until
			 * this condition doesn't destabilize the system.
			 */
#if 1
			panic("%s: swap to sparse file", __func__);
#else
			error = EIO;	/* failure */
#endif
		}

		/*
		 * punt if there was an error or a hole in the file.
		 * we must wait for any i/o ops we have already started
		 * to finish before returning.
		 *
		 * XXX we could deal with holes here but it would be
		 * a hassle (in the write case).
		 */
		if (error) {
			s = splbio();
			vnx->vx_error = error;	/* pass error up */
			goto out;
		}

		/*
		 * compute the size ("sz") of this transfer (in bytes).
		 */
		off = byteoff % sdp->swd_bsize;
		sz = (1 + nra) * sdp->swd_bsize - off;
		if (sz > resid)
			sz = resid;

		UVMHIST_LOG(pdhist, "sw_reg_strategy: "
		    "vp %#jx/%#jx offset %#jx/%#jx",
		    (uintptr_t)sdp->swd_vp, (uintptr_t)vp, byteoff, nbn);

		/*
		 * now get a buf structure.   note that the vb_buf is
		 * at the front of the nbp structure so that you can
		 * cast pointers between the two structure easily.
		 */
		nbp = pool_get(&vndbuf_pool, PR_WAITOK);
		buf_init(&nbp->vb_buf);
		nbp->vb_buf.b_flags    = bp->b_flags;
		nbp->vb_buf.b_cflags   = bp->b_cflags;
		nbp->vb_buf.b_oflags   = bp->b_oflags;
		nbp->vb_buf.b_bcount   = sz;
		nbp->vb_buf.b_bufsize  = sz;
		nbp->vb_buf.b_error    = 0;
		nbp->vb_buf.b_data     = addr;
		nbp->vb_buf.b_lblkno   = 0;
		nbp->vb_buf.b_blkno    = nbn + btodb(off);
		nbp->vb_buf.b_rawblkno = nbp->vb_buf.b_blkno;
		nbp->vb_buf.b_iodone   = sw_reg_biodone;
		nbp->vb_buf.b_vp       = vp;
		nbp->vb_buf.b_objlock  = vp->v_interlock;
		if (vp->v_type == VBLK) {
			nbp->vb_buf.b_dev = vp->v_rdev;
		}

		nbp->vb_xfer = vnx;	/* patch it back in to vnx */

		/*
		 * Just sort by block number
		 */
		s = splbio();
		if (vnx->vx_error != 0) {
			buf_destroy(&nbp->vb_buf);
			pool_put(&vndbuf_pool, nbp);
			goto out;
		}
		vnx->vx_pending++;

		/* sort it in and start I/O if we are not over our limit */
		/* XXXAD locking */
		bufq_put(sdp->swd_tab, &nbp->vb_buf);
		sw_reg_start(sdp);
		splx(s);

		/*
		 * advance to the next I/O
		 */
		byteoff += sz;
		addr += sz;
	}

	s = splbio();

out: /* Arrive here at splbio */
	vnx->vx_flags &= ~VX_BUSY;
	if (vnx->vx_pending == 0) {
		error = vnx->vx_error;
		pool_put(&vndxfer_pool, vnx);
		bp->b_error = error;
		biodone(bp);
	}
	splx(s);
}

/*
 * sw_reg_start: start an I/O request on the requested swapdev
 *
 * => reqs are sorted by b_rawblkno (above)
 */
static void
sw_reg_start(struct swapdev *sdp)
{
	struct buf	*bp;
	struct vnode	*vp;
	UVMHIST_FUNC(__func__); UVMHIST_CALLED(pdhist);

	/* recursion control */
	if ((sdp->swd_flags & SWF_BUSY) != 0)
		return;

	sdp->swd_flags |= SWF_BUSY;

	while (sdp->swd_active < sdp->swd_maxactive) {
		bp = bufq_get(sdp->swd_tab);
		if (bp == NULL)
			break;
		sdp->swd_active++;

		UVMHIST_LOG(pdhist,
		    "sw_reg_start:  bp %#jx vp %#jx blkno %#jx cnt %#jx",
		    (uintptr_t)bp, (uintptr_t)bp->b_vp, (uintptr_t)bp->b_blkno,
		    bp->b_bcount);
		vp = bp->b_vp;
		KASSERT(bp->b_objlock == vp->v_interlock);
		if ((bp->b_flags & B_READ) == 0) {
			mutex_enter(vp->v_interlock);
			vp->v_numoutput++;
			mutex_exit(vp->v_interlock);
		}
		VOP_STRATEGY(vp, bp);
	}
	sdp->swd_flags &= ~SWF_BUSY;
}

/*
 * sw_reg_biodone: one of our i/o's has completed
 */
static void
sw_reg_biodone(struct buf *bp)
{
	workqueue_enqueue(sw_reg_workqueue, &bp->b_work, NULL);
}

/*
 * sw_reg_iodone: one of our i/o's has completed and needs post-i/o cleanup
 *
 * => note that we can recover the vndbuf struct by casting the buf ptr
 */
static void
sw_reg_iodone(struct work *wk, void *dummy)
{
	struct vndbuf *vbp = (void *)wk;
	struct vndxfer *vnx = vbp->vb_xfer;
	struct buf *pbp = vnx->vx_bp;		/* parent buffer */
	struct swapdev	*sdp = vnx->vx_sdp;
	int s, resid, error;
	KASSERT(&vbp->vb_buf.b_work == wk);
	UVMHIST_FUNC(__func__);
	UVMHIST_CALLARGS(pdhist, "  vbp=%#jx vp=%#jx blkno=%#jx addr=%#jx",
	    (uintptr_t)vbp, (uintptr_t)vbp->vb_buf.b_vp, vbp->vb_buf.b_blkno,
	    (uintptr_t)vbp->vb_buf.b_data);
	UVMHIST_LOG(pdhist, "  cnt=%#jx resid=%#jx",
	    vbp->vb_buf.b_bcount, vbp->vb_buf.b_resid, 0, 0);

	/*
	 * protect vbp at splbio and update.
	 */

	s = splbio();
	resid = vbp->vb_buf.b_bcount - vbp->vb_buf.b_resid;
	pbp->b_resid -= resid;
	vnx->vx_pending--;

	if (vbp->vb_buf.b_error != 0) {
		/* pass error upward */
		error = vbp->vb_buf.b_error ? vbp->vb_buf.b_error : EIO;
		UVMHIST_LOG(pdhist, "  got error=%jd !", error, 0, 0, 0);
		vnx->vx_error = error;
	}

	/*
	 * kill vbp structure
	 */
	buf_destroy(&vbp->vb_buf);
	pool_put(&vndbuf_pool, vbp);

	/*
	 * wrap up this transaction if it has run to completion or, in
	 * case of an error, when all auxiliary buffers have returned.
	 */
	if (vnx->vx_error != 0) {
		/* pass error upward */
		error = vnx->vx_error;
		if ((vnx->vx_flags & VX_BUSY) == 0 && vnx->vx_pending == 0) {
			pbp->b_error = error;
			biodone(pbp);
			pool_put(&vndxfer_pool, vnx);
		}
	} else if (pbp->b_resid == 0) {
		KASSERT(vnx->vx_pending == 0);
		if ((vnx->vx_flags & VX_BUSY) == 0) {
			UVMHIST_LOG(pdhist, "  iodone, pbp=%#jx error=%jd !",
			    (uintptr_t)pbp, vnx->vx_error, 0, 0);
			biodone(pbp);
			pool_put(&vndxfer_pool, vnx);
		}
	}

	/*
	 * done!   start next swapdev I/O if one is pending
	 */
	sdp->swd_active--;
	sw_reg_start(sdp);
	splx(s);
}


/*
 * uvm_swap_alloc: allocate space on swap
 *
 * => allocation is done "round robin" down the priority list, as we
 *	allocate in a priority we "rotate" the circle queue.
 * => space can be freed with uvm_swap_free
 * => we return the page slot number in /dev/drum (0 == invalid slot)
 * => we lock uvm_swap_data_lock
 * => XXXMRG: "LESSOK" INTERFACE NEEDED TO EXTENT SYSTEM
 */
int
uvm_swap_alloc(int *nslots /* IN/OUT */, bool lessok)
{
	struct swapdev *sdp;
	struct swappri *spp;
	UVMHIST_FUNC(__func__); UVMHIST_CALLED(pdhist);

	/*
	 * no swap devices configured yet?   definite failure.
	 */
	if (uvmexp.nswapdev < 1)
		return 0;

	/*
	 * XXXJAK: BEGIN HACK
	 *
	 * blist_alloc() in subr_blist.c will panic if we try to allocate
	 * too many slots.
	 */
	if (*nslots > BLIST_MAX_ALLOC) {
		if (__predict_false(lessok == false))
			return 0;
		*nslots = BLIST_MAX_ALLOC;
	}
	/* XXXJAK: END HACK */

	/*
	 * lock data lock, convert slots into blocks, and enter loop
	 */
	mutex_enter(&uvm_swap_data_lock);

ReTry:	/* XXXMRG */
	LIST_FOREACH(spp, &swap_priority, spi_swappri) {
		TAILQ_FOREACH(sdp, &spp->spi_swapdev, swd_next) {
			uint64_t result;

			/* if it's not enabled, then we can't swap from it */
			if ((sdp->swd_flags & SWF_ENABLE) == 0)
				continue;
			if (sdp->swd_npginuse + *nslots > sdp->swd_npages)
				continue;
			result = blist_alloc(sdp->swd_blist, *nslots);
			if (result == BLIST_NONE) {
				continue;
			}
			KASSERT(result < sdp->swd_drumsize);

			/*
			 * successful allocation!  now rotate the tailq.
			 */
			TAILQ_REMOVE(&spp->spi_swapdev, sdp, swd_next);
			TAILQ_INSERT_TAIL(&spp->spi_swapdev, sdp, swd_next);
			sdp->swd_npginuse += *nslots;
			uvmexp.swpginuse += *nslots;
			mutex_exit(&uvm_swap_data_lock);
			/* done!  return drum slot number */
			UVMHIST_LOG(pdhist,
			    "success!  returning %jd slots starting at %jd",
			    *nslots, result + sdp->swd_drumoffset, 0, 0);
			return (result + sdp->swd_drumoffset);
		}
	}

	/* XXXMRG: BEGIN HACK */
	if (*nslots > 1 && lessok) {
		*nslots = 1;
		/* XXXMRG: ugh!  blist should support this for us */
		goto ReTry;
	}
	/* XXXMRG: END HACK */

	mutex_exit(&uvm_swap_data_lock);
	return 0;
}

/*
 * uvm_swapisfull: return true if most of available swap is allocated
 * and in use.  we don't count some small portion as it may be inaccessible
 * to us at any given moment, for example if there is lock contention or if
 * pages are busy.
 */
bool
uvm_swapisfull(void)
{
	int swpgonly;
	bool rv;

	if (uvmexp.swpages == 0) {
		return true;
	}

	mutex_enter(&uvm_swap_data_lock);
	KASSERT(uvmexp.swpgonly <= uvmexp.swpages);
	swpgonly = (int)((uint64_t)uvmexp.swpgonly * 100 /
	    uvm_swapisfull_factor);
	rv = (swpgonly >= uvmexp.swpgavail);
	mutex_exit(&uvm_swap_data_lock);

	return (rv);
}

/*
 * uvm_swap_markbad: keep track of swap ranges where we've had i/o errors
 *
 * => we lock uvm_swap_data_lock
 */
void
uvm_swap_markbad(int startslot, int nslots)
{
	struct swapdev *sdp;
	UVMHIST_FUNC(__func__); UVMHIST_CALLED(pdhist);

	mutex_enter(&uvm_swap_data_lock);
	sdp = swapdrum_getsdp(startslot);
	KASSERT(sdp != NULL);

	/*
	 * we just keep track of how many pages have been marked bad
	 * in this device, to make everything add up in swap_off().
	 * we assume here that the range of slots will all be within
	 * one swap device.
	 */

	KASSERT(uvmexp.swpgonly >= nslots);
	atomic_add_int(&uvmexp.swpgonly, -nslots);
	sdp->swd_npgbad += nslots;
	UVMHIST_LOG(pdhist, "now %jd bad", sdp->swd_npgbad, 0,0,0);
	mutex_exit(&uvm_swap_data_lock);
}

/*
 * uvm_swap_free: free swap slots
 *
 * => this can be all or part of an allocation made by uvm_swap_alloc
 * => we lock uvm_swap_data_lock
 */
void
uvm_swap_free(int startslot, int nslots)
{
	struct swapdev *sdp;
	UVMHIST_FUNC(__func__);
	UVMHIST_CALLARGS(pdhist, "freeing %jd slots starting at %jd", nslots,
	    startslot, 0, 0);

	/*
	 * ignore attempts to free the "bad" slot.
	 */

	if (startslot == SWSLOT_BAD) {
		return;
	}

	/*
	 * convert drum slot offset back to sdp, free the blocks
	 * in the extent, and return.   must hold pri lock to do
	 * lookup and access the extent.
	 */

	mutex_enter(&uvm_swap_data_lock);
	sdp = swapdrum_getsdp(startslot);
	KASSERT(uvmexp.nswapdev >= 1);
	KASSERT(sdp != NULL);
	KASSERT(sdp->swd_npginuse >= nslots);
	blist_free(sdp->swd_blist, startslot - sdp->swd_drumoffset, nslots);
	sdp->swd_npginuse -= nslots;
	uvmexp.swpginuse -= nslots;
	mutex_exit(&uvm_swap_data_lock);
}

/*
 * uvm_swap_put: put any number of pages into a contig place on swap
 *
 * => can be sync or async
 */

int
uvm_swap_put(int swslot, struct vm_page **ppsp, int npages, int flags)
{
	int error;

	error = uvm_swap_io(ppsp, swslot, npages, B_WRITE |
	    ((flags & PGO_SYNCIO) ? 0 : B_ASYNC));
	return error;
}

/*
 * uvm_swap_get: get a single page from swap
 *
 * => usually a sync op (from fault)
 */

int
uvm_swap_get(struct vm_page *page, int swslot, int flags)
{
	int error;

	atomic_inc_uint(&uvmexp.nswget);
	KASSERT(flags & PGO_SYNCIO);
	if (swslot == SWSLOT_BAD) {
		return EIO;
	}

	error = uvm_swap_io(&page, swslot, 1, B_READ |
	    ((flags & PGO_SYNCIO) ? 0 : B_ASYNC));
	if (error == 0) {

		/*
		 * this page is no longer only in swap.
		 */

		KASSERT(uvmexp.swpgonly > 0);
		atomic_dec_uint(&uvmexp.swpgonly);
	}
	return error;
}

/*
 * uvm_swap_io: do an i/o operation to swap
 */

static int
uvm_swap_io(struct vm_page **pps, int startslot, int npages, int flags)
{
	daddr_t startblk;
	struct	buf *bp;
	vaddr_t kva;
	int	error, mapinflags;
	bool write, async, swap_encrypt;
	UVMHIST_FUNC(__func__);
	UVMHIST_CALLARGS(pdhist, "<- called, startslot=%jd, npages=%jd, flags=%#jx",
	    startslot, npages, flags, 0);

	write = (flags & B_READ) == 0;
	async = (flags & B_ASYNC) != 0;
	swap_encrypt = atomic_load_relaxed(&uvm_swap_encrypt);

	/*
	 * allocate a buf for the i/o.
	 */

	KASSERT(curlwp != uvm.pagedaemon_lwp || (write && async));
	bp = getiobuf(swapdev_vp, curlwp != uvm.pagedaemon_lwp);
	if (bp == NULL) {
		uvm_aio_aiodone_pages(pps, npages, true, ENOMEM);
		return ENOMEM;
	}

	/*
	 * convert starting drum slot to block number
	 */

	startblk = btodb((uint64_t)startslot << PAGE_SHIFT);

	/*
	 * first, map the pages into the kernel.
	 */

	mapinflags = !write ?
		UVMPAGER_MAPIN_WAITOK|UVMPAGER_MAPIN_READ :
		UVMPAGER_MAPIN_WAITOK|UVMPAGER_MAPIN_WRITE;
	if (write && swap_encrypt)	/* need to encrypt in-place */
		mapinflags |= UVMPAGER_MAPIN_READ;
	kva = uvm_pagermapin(pps, npages, mapinflags);

	/*
	 * encrypt writes in place if requested
	 */

	if (write) do {
		struct swapdev *sdp;
		int i;

		/*
		 * Get the swapdev so we can discriminate on the
		 * encryption state.  There may or may not be an
		 * encryption key generated; we may or may not be asked
		 * to encrypt swap.
		 *
		 * 1. NO KEY, NO ENCRYPTION: Nothing to do.
		 *
		 * 2. NO KEY, BUT ENCRYPTION: Generate a key, encrypt,
		 *    and mark the slots encrypted.
		 *
		 * 3. KEY, BUT NO ENCRYPTION: The slots may already be
		 *    marked encrypted from a past life.  Mark them not
		 *    encrypted.
		 *
		 * 4. KEY, ENCRYPTION: Encrypt and mark the slots
		 *    encrypted.
		 */
		mutex_enter(&uvm_swap_data_lock);
		sdp = swapdrum_getsdp(startslot);
		if (!sdp->swd_encinit) {
			if (!swap_encrypt) {
				mutex_exit(&uvm_swap_data_lock);
				break;
			}
			uvm_swap_genkey(sdp);
		}
		KASSERT(sdp->swd_encinit);
		mutex_exit(&uvm_swap_data_lock);

		for (i = 0; i < npages; i++) {
			int s = startslot + i;
			KDASSERT(swapdrum_sdp_is(s, sdp));
			KASSERT(s >= sdp->swd_drumoffset);
			s -= sdp->swd_drumoffset;
			KASSERT(s < sdp->swd_drumsize);

			if (swap_encrypt) {
				uvm_swap_encryptpage(sdp,
				    (void *)(kva + (vsize_t)i*PAGE_SIZE), s);
				atomic_or_32(&sdp->swd_encmap[s/32],
				    __BIT(s%32));
			} else {
				atomic_and_32(&sdp->swd_encmap[s/32],
				    ~__BIT(s%32));
			}
		}
	} while (0);

	/*
	 * fill in the bp/sbp.   we currently route our i/o through
	 * /dev/drum's vnode [swapdev_vp].
	 */

	bp->b_cflags = BC_BUSY | BC_NOCACHE;
	bp->b_flags = (flags & (B_READ|B_ASYNC));
	bp->b_proc = &proc0;	/* XXX */
	bp->b_vnbufs.le_next = NOLIST;
	bp->b_data = (void *)kva;
	bp->b_blkno = startblk;
	bp->b_bufsize = bp->b_bcount = npages << PAGE_SHIFT;

	/*
	 * bump v_numoutput (counter of number of active outputs).
	 */

	if (write) {
		mutex_enter(swapdev_vp->v_interlock);
		swapdev_vp->v_numoutput++;
		mutex_exit(swapdev_vp->v_interlock);
	}

	/*
	 * for async ops we must set up the iodone handler.
	 */

	if (async) {
		bp->b_iodone = uvm_aio_aiodone;
		UVMHIST_LOG(pdhist, "doing async!", 0, 0, 0, 0);
		if (curlwp == uvm.pagedaemon_lwp)
			BIO_SETPRIO(bp, BPRIO_TIMECRITICAL);
		else
			BIO_SETPRIO(bp, BPRIO_TIMELIMITED);
	} else {
		bp->b_iodone = NULL;
		BIO_SETPRIO(bp, BPRIO_TIMECRITICAL);
	}
	UVMHIST_LOG(pdhist,
	    "about to start io: data = %#jx blkno = %#jx, bcount = %jd",
	    (uintptr_t)bp->b_data, bp->b_blkno, bp->b_bcount, 0);

	/*
	 * now we start the I/O, and if async, return.
	 */

	VOP_STRATEGY(swapdev_vp, bp);
	if (async) {
		/*
		 * Reads are always synchronous; if this changes, we
		 * need to add an asynchronous path for decryption.
		 */
		KASSERT(write);
		return 0;
	}

	/*
	 * must be sync i/o.   wait for it to finish
	 */

	error = biowait(bp);
	if (error)
		goto out;

	/*
	 * decrypt reads in place if needed
	 */

	if (!write) do {
		struct swapdev *sdp;
		bool encinit;
		int i;

		/*
		 * Get the sdp.  Everything about it except the encinit
		 * bit, saying whether the encryption key is
		 * initialized or not, and the encrypted bit for each
		 * page, is stable until all swap pages have been
		 * released and the device is removed.
		 */
		mutex_enter(&uvm_swap_data_lock);
		sdp = swapdrum_getsdp(startslot);
		encinit = sdp->swd_encinit;
		mutex_exit(&uvm_swap_data_lock);

		if (!encinit)
			/*
			 * If there's no encryption key, there's no way
			 * any of these slots can be encrypted, so
			 * nothing to do here.
			 */
			break;
		for (i = 0; i < npages; i++) {
			int s = startslot + i;
			KDASSERT(swapdrum_sdp_is(s, sdp));
			KASSERT(s >= sdp->swd_drumoffset);
			s -= sdp->swd_drumoffset;
			KASSERT(s < sdp->swd_drumsize);
			if ((atomic_load_relaxed(&sdp->swd_encmap[s/32]) &
				__BIT(s%32)) == 0)
				continue;
			uvm_swap_decryptpage(sdp,
			    (void *)(kva + (vsize_t)i*PAGE_SIZE), s);
		}
	} while (0);
out:
	/*
	 * kill the pager mapping
	 */

	uvm_pagermapout(kva, npages);

	/*
	 * now dispose of the buf and we're done.
	 */

	if (write) {
		mutex_enter(swapdev_vp->v_interlock);
		vwakeup(bp);
		mutex_exit(swapdev_vp->v_interlock);
	}
	putiobuf(bp);
	UVMHIST_LOG(pdhist, "<- done (sync)  error=%jd", error, 0, 0, 0);

	return (error);
}

/*
 * uvm_swap_genkey(sdp)
 *
 *	Generate a key for swap encryption.
 */
static void
uvm_swap_genkey(struct swapdev *sdp)
{
	uint8_t key[32];

	KASSERT(!sdp->swd_encinit);

	cprng_strong(kern_cprng, key, sizeof key, 0);
	aes_setenckey256(&sdp->swd_enckey, key);
	aes_setdeckey256(&sdp->swd_deckey, key);
	explicit_memset(key, 0, sizeof key);

	sdp->swd_encinit = true;
}

/*
 * uvm_swap_encryptpage(sdp, kva, slot)
 *
 *	Encrypt one page of data at kva for the specified slot number
 *	in the swap device.
 */
static void
uvm_swap_encryptpage(struct swapdev *sdp, void *kva, int slot)
{
	uint8_t preiv[16] __aligned(16) = {0}, iv[16] __aligned(16);

	/* iv := AES_k(le32enc(slot) || 0^96) */
	le32enc(preiv, slot);
	aes_enc(&sdp->swd_enckey, (const void *)preiv, iv, AES_256_NROUNDS);

	/* *kva := AES-CBC_k(iv, *kva) */
	aes_cbc_enc(&sdp->swd_enckey, kva, kva, PAGE_SIZE, iv,
	    AES_256_NROUNDS);

	explicit_memset(&iv, 0, sizeof iv);
}

/*
 * uvm_swap_decryptpage(sdp, kva, slot)
 *
 *	Decrypt one page of data at kva for the specified slot number
 *	in the swap device.
 */
static void
uvm_swap_decryptpage(struct swapdev *sdp, void *kva, int slot)
{
	uint8_t preiv[16] __aligned(16) = {0}, iv[16] __aligned(16);

	/* iv := AES_k(le32enc(slot) || 0^96) */
	le32enc(preiv, slot);
	aes_enc(&sdp->swd_enckey, (const void *)preiv, iv, AES_256_NROUNDS);

	/* *kva := AES-CBC^{-1}_k(iv, *kva) */
	aes_cbc_dec(&sdp->swd_deckey, kva, kva, PAGE_SIZE, iv,
	    AES_256_NROUNDS);

	explicit_memset(&iv, 0, sizeof iv);
}

SYSCTL_SETUP(sysctl_uvmswap_setup, "sysctl uvmswap setup")
{

	sysctl_createv(clog, 0, NULL, NULL,
	    CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_BOOL, "swap_encrypt",
	    SYSCTL_DESCR("Encrypt data when swapped out to disk"),
	    NULL, 0, &uvm_swap_encrypt, 0,
	    CTL_VM, CTL_CREATE, CTL_EOL);
}