xref: /src/sys/arch/arm/sunxi/sun8i_crypto.c

/*	$NetBSD: sun8i_crypto.c,v 1.16 2020/06/13 18:57:54 riastradh Exp $	*/

/*-
 * Copyright (c) 2019 The NetBSD Foundation, Inc.
 * All rights reserved.
 *
 * This code is derived from software contributed to The NetBSD Foundation
 * by Taylor R. Campbell.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

/*
 * sun8i_crypto -- Allwinner Crypto Engine driver
 *
 * The Crypto Engine is documented in Sec. 3.15 of the Allwinner A64
 * User Manual v1.1, on pp. 230--241.  We only use it for the TRNG at
 * the moment, but in principle it could be wired up with opencrypto(9)
 * to compute AES, DES, 3DES, MD5, SHA-1, SHA-224, SHA-256, HMAC-SHA1,
 * HMAC-HA256, RSA, and an undocumented PRNG.  It also seems to support
 * AES keys in SRAM (for some kind of HDMI HDCP stuff?).
 *
 * https://linux-sunxi.org/images/b/b4/Allwinner_A64_User_Manual_V1.1.pdf
 */

#include <sys/cdefs.h>
__KERNEL_RCSID(1, "$NetBSD: sun8i_crypto.c,v 1.16 2020/06/13 18:57:54 riastradh Exp $");

#include <sys/types.h>
#include <sys/param.h>
#include <sys/atomic.h>
#include <sys/bus.h>
#include <sys/callout.h>
#include <sys/conf.h>
#include <sys/device.h>
#include <sys/kernel.h>
#include <sys/kmem.h>
#include <sys/mutex.h>
#include <sys/rndsource.h>
#include <sys/sysctl.h>
#include <sys/workqueue.h>

#include <dev/fdt/fdtvar.h>

#include <arm/sunxi/sun8i_crypto.h>

#define	SUN8I_CRYPTO_TIMEOUT	hz
#define	SUN8I_CRYPTO_RNGENTROPY	100 /* estimated bits per bit of entropy */
#define	SUN8I_CRYPTO_RNGBYTES	PAGE_SIZE

struct sun8i_crypto_task;

struct sun8i_crypto_buf {
	bus_dma_segment_t	cb_seg[1];
	int			cb_nsegs;
	void			*cb_kva;
};

struct sun8i_crypto_softc {
	device_t			sc_dev;
	bus_space_tag_t			sc_bst;
	bus_space_handle_t		sc_bsh;
	bus_dma_tag_t			sc_dmat;
	struct pool_cache		*sc_taskpool;
	kmutex_t			sc_lock;
	struct sun8i_crypto_chan {
		struct sun8i_crypto_task	*cc_task;
		unsigned			cc_starttime;
	}				sc_chan[SUN8I_CRYPTO_NCHAN];
	struct callout			sc_timeout;
	struct workqueue		*sc_wq;
	struct work			sc_work;
	void				*sc_ih;
	uint32_t			sc_done;
	uint32_t			sc_esr;
	bool				sc_work_pending;
	struct sun8i_crypto_rng {
		struct sun8i_crypto_buf		cr_buf;
		struct sun8i_crypto_task	*cr_task;
		struct krndsource		cr_rndsource;
		bool				cr_pending;
	}				sc_rng;
	struct sun8i_crypto_selftest {
		struct sun8i_crypto_buf		cs_in;
		struct sun8i_crypto_buf		cs_key;
		struct sun8i_crypto_buf		cs_out;
		struct sun8i_crypto_task	*cs_task;
	}				sc_selftest;
	struct sun8i_crypto_sysctl {
		struct sysctllog		*cy_log;
		const struct sysctlnode		*cy_root_node;
		const struct sysctlnode		*cy_trng_node;
	}				sc_sysctl;
};

struct sun8i_crypto_task {
	struct sun8i_crypto_buf	ct_descbuf;
	struct sun8i_crypto_taskdesc *ct_desc;
	bus_dmamap_t		ct_descmap;
	bus_dmamap_t		ct_keymap;
	bus_dmamap_t		ct_ivmap;
	bus_dmamap_t		ct_ctrmap;
	bus_dmamap_t		ct_srcmap;
	bus_dmamap_t		ct_dstmap;
	uint32_t		ct_nbytes;
	int			ct_flags;
#define	TASK_KEY		__BIT(0)
#define	TASK_IV			__BIT(1)
#define	TASK_CTR		__BIT(2)
#define	TASK_SRC		__BIT(3)
#define	TASK_BYTES		__BIT(4) /* datalen is in bytes, not words */
	void			(*ct_callback)(struct sun8i_crypto_softc *,
				    struct sun8i_crypto_task *, void *, int);
	void			*ct_cookie;
};

#define	SUN8I_CRYPTO_MAXDMASIZE		PAGE_SIZE
#define	SUN8I_CRYPTO_MAXDMASEGSIZE	PAGE_SIZE

CTASSERT(SUN8I_CRYPTO_MAXDMASIZE <= SUN8I_CRYPTO_MAXDATALEN);
CTASSERT(SUN8I_CRYPTO_MAXDMASEGSIZE <= SUN8I_CRYPTO_MAXSEGLEN);

/*
 * Forward declarations
 */

static int	sun8i_crypto_match(device_t, cfdata_t, void *);
static void	sun8i_crypto_attach(device_t, device_t, void *);

static int	sun8i_crypto_task_ctor(void *, void *, int);
static void	sun8i_crypto_task_dtor(void *, void *);
static struct sun8i_crypto_task *
		sun8i_crypto_task_get(struct sun8i_crypto_softc *,
		    void (*)(struct sun8i_crypto_softc *,
			struct sun8i_crypto_task *, void *, int),
		    void *, int);
static void	sun8i_crypto_task_put(struct sun8i_crypto_softc *,
		    struct sun8i_crypto_task *);

static int	sun8i_crypto_task_load(struct sun8i_crypto_softc *,
		    struct sun8i_crypto_task *, uint32_t,
		    uint32_t, uint32_t, uint32_t);
static int	sun8i_crypto_task_scatter(struct sun8i_crypto_adrlen *,
		    bus_dmamap_t, uint32_t);

static int	sun8i_crypto_task_load_trng(struct sun8i_crypto_softc *,
		    struct sun8i_crypto_task *, uint32_t);
static int	sun8i_crypto_task_load_aesecb(struct sun8i_crypto_softc *,
		    struct sun8i_crypto_task *, uint32_t, uint32_t, uint32_t);

static int	sun8i_crypto_submit(struct sun8i_crypto_softc *,
		    struct sun8i_crypto_task *);

static void	sun8i_crypto_timeout(void *);
static int	sun8i_crypto_intr(void *);
static void	sun8i_crypto_schedule_worker(struct sun8i_crypto_softc *);
static void	sun8i_crypto_worker(struct work *, void *);
static void	sun8i_crypto_chan_done(struct sun8i_crypto_softc *, unsigned,
		    int);

static int	sun8i_crypto_allocbuf(struct sun8i_crypto_softc *, size_t,
		    struct sun8i_crypto_buf *, int);
static void	sun8i_crypto_freebuf(struct sun8i_crypto_softc *, size_t,
		    struct sun8i_crypto_buf *);

static void	sun8i_crypto_rng_attach(struct sun8i_crypto_softc *);
static void	sun8i_crypto_rng_get(size_t, void *);
static void	sun8i_crypto_rng_done(struct sun8i_crypto_softc *,
		    struct sun8i_crypto_task *, void *, int);

static void	sun8i_crypto_selftest(device_t);
static void	sun8i_crypto_selftest_done(struct sun8i_crypto_softc *,
		    struct sun8i_crypto_task *, void *, int);

static void	sun8i_crypto_sysctl_attach(struct sun8i_crypto_softc *);
static int	sun8i_crypto_sysctl_rng(SYSCTLFN_ARGS);
static void	sun8i_crypto_sysctl_rng_done(struct sun8i_crypto_softc *,
		    struct sun8i_crypto_task *, void *, int);

/*
 * Register access
 */

static uint32_t
sun8i_crypto_read(struct sun8i_crypto_softc *sc, bus_addr_t reg)
{
	return bus_space_read_4(sc->sc_bst, sc->sc_bsh, reg);
}

static void
sun8i_crypto_write(struct sun8i_crypto_softc *sc, bus_addr_t reg, uint32_t v)
{
	bus_space_write_4(sc->sc_bst, sc->sc_bsh, reg, v);
}

/*
 * Autoconf goo
 */

CFATTACH_DECL_NEW(sun8i_crypto, sizeof(struct sun8i_crypto_softc),
    sun8i_crypto_match, sun8i_crypto_attach, NULL, NULL);

static const struct of_compat_data compat_data[] = {
	{"allwinner,sun50i-a64-crypto", 0},
	{NULL}
};

static int
sun8i_crypto_match(device_t parent, cfdata_t cf, void *aux)
{
	const struct fdt_attach_args *const faa = aux;

	return of_match_compat_data(faa->faa_phandle, compat_data);
}

static void
sun8i_crypto_attach(device_t parent, device_t self, void *aux)
{
	struct sun8i_crypto_softc *const sc = device_private(self);
	const struct fdt_attach_args *const faa = aux;
	bus_addr_t addr;
	bus_size_t size;
	const int phandle = faa->faa_phandle;
	char intrstr[128];
	struct clk *clk;
	struct fdtbus_reset *rst;

	sc->sc_dev = self;
	sc->sc_dmat = faa->faa_dmat;
	sc->sc_bst = faa->faa_bst;
	sc->sc_taskpool = pool_cache_init(sizeof(struct sun8i_crypto_task),
	    0, 0, 0, "sun8icry", NULL, IPL_VM,
	    &sun8i_crypto_task_ctor, &sun8i_crypto_task_dtor, sc);
	mutex_init(&sc->sc_lock, MUTEX_DEFAULT, IPL_VM);
	callout_init(&sc->sc_timeout, CALLOUT_MPSAFE);
	callout_setfunc(&sc->sc_timeout, &sun8i_crypto_timeout, sc);
	if (workqueue_create(&sc->sc_wq, device_xname(self),
		&sun8i_crypto_worker, sc, PRI_NONE, IPL_VM, WQ_MPSAFE) != 0) {
		aprint_error(": couldn't create workqueue\n");
		return;
	}

	/* Get and map device registers.  */
	if (fdtbus_get_reg(phandle, 0, &addr, &size) != 0) {
		aprint_error(": couldn't get registers\n");
		return;
	}
	if (bus_space_map(sc->sc_bst, addr, size, 0, &sc->sc_bsh) != 0) {
		aprint_error(": couldn't map registers\n");
		return;
	}

	/* Get an interrupt handle.  */
	if (!fdtbus_intr_str(phandle, 0, intrstr, sizeof(intrstr))) {
		aprint_error(": failed to decode interrupt\n");
		return;
	}

	/* Enable the bus clock.  */
	if (fdtbus_clock_enable(phandle, "bus", true) != 0) {
		aprint_error(": couldn't enable bus clock\n");
		return;
	}

	/* Get the module clock and set it to 300 MHz.  */
	if ((clk = fdtbus_clock_get(phandle, "mod")) != NULL) {
		if (clk_enable(clk) != 0) {
			aprint_error(": couldn't enable CE clock\n");
			return;
		}
		if (clk_set_rate(clk, 300*1000*1000) != 0) {
			aprint_error(": couldn't set CE clock to 300MHz\n");
			return;
		}
	}

	/* Get a reset handle if we need and try to deassert it.  */
	if ((rst = fdtbus_reset_get_index(phandle, 0)) != NULL) {
		if (fdtbus_reset_deassert(rst) != 0) {
			aprint_error(": couldn't de-assert reset\n");
			return;
		}
	}

	aprint_naive("\n");
	aprint_normal(": Crypto Engine\n");
	aprint_debug_dev(self, ": clock freq %d\n", clk_get_rate(clk));

	/* Disable and clear interrupts.  */
	sun8i_crypto_write(sc, SUN8I_CRYPTO_ICR, 0);
	sun8i_crypto_write(sc, SUN8I_CRYPTO_ISR, 0);

	/* Establish an interrupt handler.  */
	sc->sc_ih = fdtbus_intr_establish(phandle, 0, IPL_VM, FDT_INTR_MPSAFE,
	    &sun8i_crypto_intr, sc);
	if (sc->sc_ih == NULL) {
		aprint_error_dev(self, "failed to establish interrupt on %s\n",
		    intrstr);
		return;
	}
	aprint_normal_dev(self, "interrupting on %s\n", intrstr);

	/* Set up the RNG.  */
	sun8i_crypto_rng_attach(sc);

	/* Attach the sysctl.  */
	sun8i_crypto_sysctl_attach(sc);

	/* Perform self-tests.  */
	config_interrupts(self, sun8i_crypto_selftest);
}

static int
sun8i_crypto_task_ctor(void *cookie, void *vtask, int pflags)
{
	struct sun8i_crypto_softc *sc = cookie;
	struct sun8i_crypto_task *task = vtask;
	int dmaflags = (pflags & PR_WAITOK) ? BUS_DMA_WAITOK : BUS_DMA_NOWAIT;
	int error;

	/* Create a DMA buffer for the task descriptor.  */
	error = sun8i_crypto_allocbuf(sc, sizeof(*task->ct_desc),
	    &task->ct_descbuf, dmaflags);
	if (error)
		goto fail0;
	task->ct_desc = task->ct_descbuf.cb_kva;

	/* Create a DMA map for the task descriptor and preload it.  */
	error = bus_dmamap_create(sc->sc_dmat, sizeof(*task->ct_desc), 1,
	    sizeof(*task->ct_desc), 0, dmaflags, &task->ct_descmap);
	if (error)
		goto fail1;
	error = bus_dmamap_load(sc->sc_dmat, task->ct_descmap, task->ct_desc,
	    sizeof(*task->ct_desc), NULL, BUS_DMA_WAITOK);
	if (error)
		goto fail2;

	/* Create DMA maps for the key, IV, and CTR.  */
	error = bus_dmamap_create(sc->sc_dmat, SUN8I_CRYPTO_MAXKEYBYTES, 1,
	    SUN8I_CRYPTO_MAXKEYBYTES, 0, dmaflags, &task->ct_keymap);
	if (error)
		goto fail3;
	error = bus_dmamap_create(sc->sc_dmat, SUN8I_CRYPTO_MAXIVBYTES, 1,
	    SUN8I_CRYPTO_MAXIVBYTES, 0, dmaflags, &task->ct_ivmap);
	if (error)
		goto fail4;
	error = bus_dmamap_create(sc->sc_dmat, SUN8I_CRYPTO_MAXCTRBYTES, 1,
	    SUN8I_CRYPTO_MAXCTRBYTES, 0, dmaflags, &task->ct_ctrmap);
	if (error)
		goto fail5;

	/* Create DMA maps for the src and dst scatter/gather vectors.  */
	error = bus_dmamap_create(sc->sc_dmat, SUN8I_CRYPTO_MAXDMASIZE,
	    SUN8I_CRYPTO_MAXSEGS, SUN8I_CRYPTO_MAXDMASEGSIZE, 0, dmaflags,
	    &task->ct_srcmap);
	if (error)
		goto fail6;
	error = bus_dmamap_create(sc->sc_dmat, SUN8I_CRYPTO_MAXDMASIZE,
	    SUN8I_CRYPTO_MAXSEGS, SUN8I_CRYPTO_MAXDMASEGSIZE, 0, dmaflags,
	    &task->ct_dstmap);
	if (error)
		goto fail7;

	/* Success!  */
	return 0;

fail8: __unused
	bus_dmamap_destroy(sc->sc_dmat, task->ct_dstmap);
fail7:	bus_dmamap_destroy(sc->sc_dmat, task->ct_srcmap);
fail6:	bus_dmamap_destroy(sc->sc_dmat, task->ct_ctrmap);
fail5:	bus_dmamap_destroy(sc->sc_dmat, task->ct_ivmap);
fail4:	bus_dmamap_destroy(sc->sc_dmat, task->ct_keymap);
fail3:	bus_dmamap_unload(sc->sc_dmat, task->ct_descmap);
fail2:	bus_dmamap_destroy(sc->sc_dmat, task->ct_descmap);
fail1:	sun8i_crypto_freebuf(sc, sizeof(*task->ct_desc), &task->ct_descbuf);
fail0:	return error;
}

static void
sun8i_crypto_task_dtor(void *cookie, void *vtask)
{
	struct sun8i_crypto_softc *sc = cookie;
	struct sun8i_crypto_task *task = vtask;

	/* XXX Zero the bounce buffers if there are any.  */

	bus_dmamap_destroy(sc->sc_dmat, task->ct_dstmap);
	bus_dmamap_destroy(sc->sc_dmat, task->ct_srcmap);
	bus_dmamap_destroy(sc->sc_dmat, task->ct_ctrmap);
	bus_dmamap_destroy(sc->sc_dmat, task->ct_ivmap);
	bus_dmamap_destroy(sc->sc_dmat, task->ct_keymap);
	bus_dmamap_unload(sc->sc_dmat, task->ct_descmap);
	bus_dmamap_destroy(sc->sc_dmat, task->ct_descmap);
	sun8i_crypto_freebuf(sc, sizeof(*task->ct_desc), &task->ct_descbuf);
}

/*
 * sun8i_crypto_task_get(sc, callback, cookie, pflags)
 *
 *	Allocate a task that will call callback(sc, task, cookie,
 *	error) when done.  pflags is PR_WAITOK or PR_NOWAIT; if
 *	PR_NOWAIT, may fail and return NULL.  No further allocation is
 *	needed to submit the task if this succeeds (although task
 *	submission may still fail if all channels are busy).
 */
static struct sun8i_crypto_task *
sun8i_crypto_task_get(struct sun8i_crypto_softc *sc,
    void (*callback)(struct sun8i_crypto_softc *, struct sun8i_crypto_task *,
	void *, int),
    void *cookie, int pflags)
{
	struct sun8i_crypto_task *task;

	/* Allocate a task, or fail if we can't.  */
	task = pool_cache_get(sc->sc_taskpool, pflags);
	if (task == NULL)
		return NULL;

	/* Set up flags and the callback.  */
	task->ct_flags = 0;
	task->ct_callback = callback;
	task->ct_cookie = cookie;
	return task;
}

/*
 * sun8i_crypto_task_invalid(sc, task, cookie, error)
 *
 *	Callback for a task not currently in use, to detect errors.
 */
static void
sun8i_crypto_task_invalid(struct sun8i_crypto_softc *sc,
    struct sun8i_crypto_task *task, void *cookie, int error)
{
	void (*callback)(struct sun8i_crypto_softc *,
	    struct sun8i_crypto_task *, void *, int) = cookie;

	panic("task for callback %p used after free", callback);
}

/*
 * sun8i_crypto_task_put(sc, task)
 *
 *	Free a task obtained with sun8i_crypto_task_get.
 */
static void
sun8i_crypto_task_put(struct sun8i_crypto_softc *sc,
    struct sun8i_crypto_task *task)
{

	task->ct_cookie = task->ct_callback;
	task->ct_callback = &sun8i_crypto_task_invalid;
	pool_cache_put(sc->sc_taskpool, task);
}

/*
 * sun8i_crypto_task_load(sc, task, nbytes, tdqc, tdqs, tdqa)
 *
 *	Set up the task descriptor after the relevant DMA maps have
 *	been loaded for a transfer of nbytes.  bus_dmamap_sync matches
 *	sun8i_crypto_chan_done.  May fail if input is inadequately
 *	aligned.
 *
 *	XXX Teach this to fail gracefully if any alignment is wrong.
 *	XXX Teach this to support task chains.
 */
static int
sun8i_crypto_task_load(struct sun8i_crypto_softc *sc,
    struct sun8i_crypto_task *task, uint32_t nbytes,
    uint32_t tdqc, uint32_t tdqs, uint32_t tdqa)
{
	struct sun8i_crypto_taskdesc *desc = task->ct_desc;
	int error;

	KASSERT(tdqs == 0 || tdqa == 0);
	KASSERT(nbytes % 4 == 0);

	memset(desc, 0, sizeof(*desc));

	desc->td_tdqc = htole32(tdqc);
	desc->td_tdqs = htole32(tdqs);
	desc->td_tdqa = htole32(tdqa);

	if (task->ct_flags & TASK_KEY) {
		bus_dmamap_t keymap = task->ct_keymap;
		KASSERT(keymap->dm_nsegs == 1);
		desc->td_keydesc = htole32(keymap->dm_segs[0].ds_addr);
		bus_dmamap_sync(sc->sc_dmat, keymap, 0,
		    keymap->dm_segs[0].ds_len, BUS_DMASYNC_PREWRITE);
	}
	if (task->ct_flags & TASK_IV) {
		bus_dmamap_t ivmap = task->ct_ivmap;
		KASSERT(ivmap->dm_nsegs == 1);
		desc->td_ivdesc = htole32(ivmap->dm_segs[0].ds_addr);
		bus_dmamap_sync(sc->sc_dmat, ivmap, 0,
		    ivmap->dm_segs[0].ds_len, BUS_DMASYNC_PREWRITE);
	}
	if (task->ct_flags & TASK_CTR) {
		bus_dmamap_t ctrmap = task->ct_ctrmap;
		KASSERT(ctrmap->dm_nsegs == 1);
		desc->td_ctrdesc = htole32(ctrmap->dm_segs[0].ds_addr);
		bus_dmamap_sync(sc->sc_dmat, ctrmap, 0,
		    ctrmap->dm_segs[0].ds_len, BUS_DMASYNC_PREWRITE);
	}

	if (task->ct_flags & TASK_BYTES)
		desc->td_datalen = htole32(nbytes);
	else
		desc->td_datalen = htole32(nbytes/4);

	if (task->ct_flags & TASK_SRC) {
		bus_dmamap_t srcmap = task->ct_srcmap;
		KASSERT(srcmap->dm_mapsize == task->ct_dstmap->dm_mapsize);
		error = sun8i_crypto_task_scatter(desc->td_src, srcmap,
		    nbytes);
		if (error)
			return error;
		bus_dmamap_sync(sc->sc_dmat, srcmap, 0, nbytes,
		    BUS_DMASYNC_PREWRITE);
	}

	error = sun8i_crypto_task_scatter(desc->td_dst, task->ct_dstmap,
	    nbytes);
	if (error)
		return error;
	bus_dmamap_sync(sc->sc_dmat, task->ct_dstmap, 0, nbytes,
	    BUS_DMASYNC_PREREAD);

	task->ct_nbytes = nbytes;

	/* Success!  */
	return 0;
}

/*
 * sun8i_crypto_task_scatter(adrlen, map, nbytes)
 *
 *	Set up a task's scatter/gather vector -- src or dst -- with the
 *	given DMA map for a transfer of nbytes.  May fail if input is
 *	inadequately aligned.
 */
static int
sun8i_crypto_task_scatter(struct sun8i_crypto_adrlen *adrlen, bus_dmamap_t map,
    uint32_t nbytes __diagused)
{
	uint32_t total __diagused = 0;
	unsigned i;

	KASSERT(map->dm_nsegs <= SUN8I_CRYPTO_MAXSEGS);
	for (i = 0; i < map->dm_nsegs; i++) {
		KASSERT((map->dm_segs[i].ds_addr % 4) == 0);
		KASSERT(map->dm_segs[i].ds_addr <= UINT32_MAX);
		KASSERT(map->dm_segs[i].ds_len <= UINT32_MAX - total);
		adrlen[i].adr = htole32(map->dm_segs[i].ds_addr);
		adrlen[i].len = htole32(map->dm_segs[i].ds_len/4);
		total += map->dm_segs[i].ds_len;
	}

	/* Verify the remainder are zero.  */
	for (; i < SUN8I_CRYPTO_MAXSEGS; i++) {
		KASSERT(adrlen[i].adr == 0);
		KASSERT(adrlen[i].len == 0);
	}

	/* Verify the total size matches the transfer length.  */
	KASSERT(total == nbytes);

	/* Success!  */
	return 0;
}

/*
 * sun8i_crypto_task_load_trng(task, nbytes)
 *
 *	Set up the task descriptor for a transfer of nbytes from the
 *	TRNG.
 */
static int
sun8i_crypto_task_load_trng(struct sun8i_crypto_softc *sc,
    struct sun8i_crypto_task *task, uint32_t nbytes)
{
	uint32_t tdqc = 0;

	/* Caller must provide dst only.  */
	KASSERT((task->ct_flags & TASK_KEY) == 0);
	KASSERT((task->ct_flags & TASK_IV) == 0);
	KASSERT((task->ct_flags & TASK_CTR) == 0);
	KASSERT((task->ct_flags & TASK_SRC) == 0);

	/* Set up the task descriptor queue control words.  */
	tdqc |= SUN8I_CRYPTO_TDQC_INTR_EN;
	tdqc |= __SHIFTIN(SUN8I_CRYPTO_TDQC_METHOD_TRNG,
	    SUN8I_CRYPTO_TDQC_METHOD);

	/* Fill in the descriptor.  */
	return sun8i_crypto_task_load(sc, task, nbytes, tdqc, 0, 0);
}

static int
sun8i_crypto_task_load_aesecb(struct sun8i_crypto_softc *sc,
    struct sun8i_crypto_task *task,
    uint32_t nbytes, uint32_t keysize, uint32_t dir)
{
	uint32_t tdqc = 0, tdqs = 0;

	/* Caller must provide key, src, and dst only.  */
	KASSERT(task->ct_flags & TASK_KEY);
	KASSERT((task->ct_flags & TASK_IV) == 0);
	KASSERT((task->ct_flags & TASK_CTR) == 0);
	KASSERT(task->ct_flags & TASK_SRC);

	/* Set up the task descriptor queue control word.  */
	tdqc |= SUN8I_CRYPTO_TDQC_INTR_EN;
	tdqc |= __SHIFTIN(SUN8I_CRYPTO_TDQC_METHOD_AES,
	    SUN8I_CRYPTO_TDQC_METHOD);

#ifdef DIAGNOSTIC
	switch (keysize) {
	case SUN8I_CRYPTO_TDQS_AES_KEYSIZE_128:
		KASSERT(task->ct_keymap->dm_segs[0].ds_len == 16);
		break;
	case SUN8I_CRYPTO_TDQS_AES_KEYSIZE_192:
		KASSERT(task->ct_keymap->dm_segs[0].ds_len == 24);
		break;
	case SUN8I_CRYPTO_TDQS_AES_KEYSIZE_256:
		KASSERT(task->ct_keymap->dm_segs[0].ds_len == 32);
		break;
	}
#endif

	/* Set up the symmetric control word.  */
	tdqs |= __SHIFTIN(SUN8I_CRYPTO_TDQS_SKEY_SELECT_SS_KEYx,
	    SUN8I_CRYPTO_TDQS_SKEY_SELECT);
	tdqs |= __SHIFTIN(SUN8I_CRYPTO_TDQS_OP_MODE_ECB,
	    SUN8I_CRYPTO_TDQS_OP_MODE);
	tdqs |= __SHIFTIN(keysize, SUN8I_CRYPTO_TDQS_AES_KEYSIZE);

	/* Fill in the descriptor.  */
	return sun8i_crypto_task_load(sc, task, nbytes, tdqc, tdqs, 0);
}

/*
 * sun8i_crypto_submit(sc, task)
 *
 *	Submit a task to the crypto engine after it has been loaded
 *	with sun8i_crypto_task_load.  On success, guarantees to
 *	eventually call the task's callback.
 */
static int
sun8i_crypto_submit(struct sun8i_crypto_softc *sc,
    struct sun8i_crypto_task *task)
{
	unsigned i, retries = 0;
	uint32_t icr;
	int error = 0;

	/* One at a time at the device registers, please.  */
	mutex_enter(&sc->sc_lock);

	/* Find a channel.  */
	for (i = 0; i < SUN8I_CRYPTO_NCHAN; i++) {
		if (sc->sc_chan[i].cc_task == NULL)
			break;
	}
	if (i == SUN8I_CRYPTO_NCHAN) {
		device_printf(sc->sc_dev, "no free channels\n");
		error = ERESTART;
		goto out;
	}

	/*
	 * Set the channel id.  Caller is responsible for setting up
	 * all other parts of the descriptor.
	 */
	task->ct_desc->td_cid = htole32(i);

	/*
	 * Prepare to send the descriptor to the device by DMA.
	 * Matches POSTWRITE in sun8i_crypto_chan_done.
	 */
	bus_dmamap_sync(sc->sc_dmat, task->ct_descmap, 0,
	    sizeof(*task->ct_desc), BUS_DMASYNC_PREWRITE);

	/* Confirm we're ready to go.  */
	if (sun8i_crypto_read(sc, SUN8I_CRYPTO_TLR) & SUN8I_CRYPTO_TLR_LOAD) {
		device_printf(sc->sc_dev, "TLR not clear\n");
		error = EIO;
		goto out;
	}

	/* Enable interrupts for this channel.  */
	icr = sun8i_crypto_read(sc, SUN8I_CRYPTO_ICR);
	icr |= __SHIFTIN(SUN8I_CRYPTO_ICR_INTR_EN_CHAN(i),
	    SUN8I_CRYPTO_ICR_INTR_EN);
	sun8i_crypto_write(sc, SUN8I_CRYPTO_ICR, icr);

	/* Set the task descriptor queue address.  */
	sun8i_crypto_write(sc, SUN8I_CRYPTO_TDQ,
	    task->ct_descmap->dm_segs[0].ds_addr);

	/* Notify the engine to load it, and wait for acknowledgement.  */
	sun8i_crypto_write(sc, SUN8I_CRYPTO_TLR, SUN8I_CRYPTO_TLR_LOAD);
	while (sun8i_crypto_read(sc, SUN8I_CRYPTO_TLR) & SUN8I_CRYPTO_TLR_LOAD)
	{
		/*
		 * XXX Timeout pulled from arse.  Is it even important
		 * to wait here?
		 */
		if (++retries == 1000) {
			device_printf(sc->sc_dev, "TLR didn't clear: %08x\n",
			    sun8i_crypto_read(sc, SUN8I_CRYPTO_TLR));
			/*
			 * Hope it clears eventually; if not, we'll
			 * time out.
			 */
			break;
		}
		DELAY(1);
	}

	/*
	 * Loaded up and ready to go.  Start a timer ticking if it's
	 * not already.
	 */
	sc->sc_chan[i].cc_task = task;
	sc->sc_chan[i].cc_starttime = getticks();
	if (!callout_pending(&sc->sc_timeout))
		callout_schedule(&sc->sc_timeout, SUN8I_CRYPTO_TIMEOUT);

	/* XXX Consider polling if cold to get entropy earlier.  */

out:	/* Done!  */
	mutex_exit(&sc->sc_lock);
	return error;
}

/*
 * sun8i_crypto_timeout(cookie)
 *
 *	Timeout handler.  Schedules work in a thread to cancel all
 *	pending tasks that were started long enough ago we're bored of
 *	waiting for them, and reschedules another timeout unless the
 *	channels are all idle.
 */
static void
sun8i_crypto_timeout(void *cookie)
{
	struct sun8i_crypto_softc *sc = cookie;
	unsigned i;

	mutex_enter(&sc->sc_lock);

	/* Check whether there are any tasks pending.  */
	for (i = 0; i < SUN8I_CRYPTO_NCHAN; i++) {
		if (sc->sc_chan[i].cc_task)
			break;
	}
	if (i == SUN8I_CRYPTO_NCHAN)
		/* None pending, so nothing to do.  */
		goto out;

	/*
	 * Schedule the worker to check for timeouts, and schedule
	 * another timeout in case we need it.
	 */
	sun8i_crypto_schedule_worker(sc);
	callout_schedule(&sc->sc_timeout, SUN8I_CRYPTO_TIMEOUT);

out:	mutex_exit(&sc->sc_lock);
}

/*
 * sun8i_crypto_intr(cookie)
 *
 *	Device interrupt handler.  Find what channels have completed,
 *	whether with success or with failure, and schedule work in
 *	thread context to invoke the appropriate callbacks.
 */
static int
sun8i_crypto_intr(void *cookie)
{
	struct sun8i_crypto_softc *sc = cookie;
	uint32_t isr, esr;

	mutex_enter(&sc->sc_lock);

	/*
	 * Get and acknowledge the interrupts and error status.
	 *
	 * XXX Data sheet says the error status register is read-only,
	 * but then advises writing 1 to bit x1xx (keysram access error
	 * for AES, SUN8I_CRYPTO_ESR_KEYSRAMERR) to clear it.  What do?
	 */
	isr = sun8i_crypto_read(sc, SUN8I_CRYPTO_ISR);
	esr = sun8i_crypto_read(sc, SUN8I_CRYPTO_ESR);
	sun8i_crypto_write(sc, SUN8I_CRYPTO_ISR, isr);
	sun8i_crypto_write(sc, SUN8I_CRYPTO_ESR, esr);

	/* Start the worker if necessary.  */
	sun8i_crypto_schedule_worker(sc);

	/* Tell the worker what to do.  */
	sc->sc_done |= __SHIFTOUT(isr, SUN8I_CRYPTO_ISR_DONE);
	sc->sc_esr |= esr;

	mutex_exit(&sc->sc_lock);

	return __SHIFTOUT(isr, SUN8I_CRYPTO_ISR_DONE) != 0;
}

/*
 * sun8i_crypto_schedule_worker(sc)
 *
 *	Ensure that crypto engine thread context work to invoke task
 *	callbacks will run promptly.  Idempotent.
 */
static void
sun8i_crypto_schedule_worker(struct sun8i_crypto_softc *sc)
{

	KASSERT(mutex_owned(&sc->sc_lock));

	/* Start the worker if necessary.  */
	if (!sc->sc_work_pending) {
		workqueue_enqueue(sc->sc_wq, &sc->sc_work, NULL);
		sc->sc_work_pending = true;
	}
}

/*
 * sun8i_crypto_worker(wk, cookie)
 *
 *	Thread-context worker: Invoke all task callbacks for which the
 *	device has notified us of completion or for which we gave up
 *	waiting.
 */
static void
sun8i_crypto_worker(struct work *wk, void *cookie)
{
	struct sun8i_crypto_softc *sc = cookie;
	uint32_t done, esr, esr_chan;
	unsigned i, now;
	int error;

	/*
	 * Acquire the lock.  Note: We will be releasing and
	 * reacquiring it throughout the loop.
	 */
	mutex_enter(&sc->sc_lock);

	/* Acknowledge the work.  */
	KASSERT(sc->sc_work_pending);
	sc->sc_work_pending = false;

	/*
	 * Claim the done mask and error status once; we will be
	 * releasing and reacquiring the lock for the callbacks, so
	 * they may change.
	 */
	done = sc->sc_done;
	esr = sc->sc_esr;
	sc->sc_done = 0;
	sc->sc_esr = 0;

	/* Check the time to determine what's timed out.  */
	now = getticks();

	/* Process the channels.  */
	for (i = 0; i < SUN8I_CRYPTO_NCHAN; i++) {
		/* Check whether the channel is done.  */
		if (!ISSET(done, SUN8I_CRYPTO_ISR_DONE_CHAN(i))) {
			/* Nope.  Do we have a task to time out?  */
			if ((sc->sc_chan[i].cc_task != NULL) &&
			    ((now - sc->sc_chan[i].cc_starttime) >=
				SUN8I_CRYPTO_TIMEOUT))
				sun8i_crypto_chan_done(sc, i, ETIMEDOUT);
			continue;
		}

		/* Channel is done.  Interpret the error if any.  */
		esr_chan = __SHIFTOUT(esr, SUN8I_CRYPTO_ESR_CHAN(i));
		if (esr_chan & SUN8I_CRYPTO_ESR_CHAN_ALGNOTSUP) {
			device_printf(sc->sc_dev, "channel %u:"
			    " alg not supported\n", i);
			error = ENODEV;
		} else if (esr_chan & SUN8I_CRYPTO_ESR_CHAN_DATALENERR) {
			device_printf(sc->sc_dev, "channel %u:"
			    " data length error\n", i);
			error = EIO;	/* XXX */
		} else if (esr_chan & SUN8I_CRYPTO_ESR_CHAN_KEYSRAMERR) {
			device_printf(sc->sc_dev, "channel %u:"
			    " key sram error\n", i);
			error = EIO;	/* XXX */
		} else if (esr_chan != 0) {
			error = EIO;	/* generic I/O error */
		} else {
			error = 0;
		}

		/*
		 * Notify the task of completion.  May release the lock
		 * to invoke a callback.
		 */
		sun8i_crypto_chan_done(sc, i, error);
	}

	/* All one; release the lock one last time.  */
	mutex_exit(&sc->sc_lock);
}

/*
 * sun8i_crypto_chan_done(sc, i, error)
 *
 *	Notify the callback for the task on channel i, if there is one,
 *	of the specified error, or 0 for success.
 */
static void
sun8i_crypto_chan_done(struct sun8i_crypto_softc *sc, unsigned i, int error)
{
	struct sun8i_crypto_task *task;
	uint32_t nbytes;
	uint32_t icr;

	KASSERT(mutex_owned(&sc->sc_lock));

	/* Claim the task if there is one; bail if not.  */
	if ((task = sc->sc_chan[i].cc_task) == NULL) {
		device_printf(sc->sc_dev, "channel %u: no task but error=%d\n",
		    i, error);
		return;
	}
	sc->sc_chan[i].cc_task = NULL;

	/* Disable interrupts on this channel.  */
	icr = sun8i_crypto_read(sc, SUN8I_CRYPTO_ICR);
	icr &= ~__SHIFTIN(SUN8I_CRYPTO_ICR_INTR_EN_CHAN(i),
	    SUN8I_CRYPTO_ICR_INTR_EN);
	sun8i_crypto_write(sc, SUN8I_CRYPTO_ICR, icr);

	/*
	 * Finished sending the descriptor to the device by DMA.
	 * Matches PREWRITE in sun8i_crypto_task_submit.
	 */
	bus_dmamap_sync(sc->sc_dmat, task->ct_descmap, 0,
	    sizeof(*task->ct_desc), BUS_DMASYNC_POSTWRITE);

	/*
	 * Finished with all the other bits of DMA too.  Matches
	 * sun8i_crypto_task_load.
	 */
	nbytes = task->ct_nbytes;
	bus_dmamap_sync(sc->sc_dmat, task->ct_dstmap, 0, nbytes,
	    BUS_DMASYNC_POSTREAD);
	if (task->ct_flags & TASK_SRC)
		bus_dmamap_sync(sc->sc_dmat, task->ct_srcmap, 0, nbytes,
		    BUS_DMASYNC_POSTWRITE);
	if (task->ct_flags & TASK_CTR)
		bus_dmamap_sync(sc->sc_dmat, task->ct_ctrmap, 0,
		    task->ct_ctrmap->dm_segs[0].ds_len, BUS_DMASYNC_POSTWRITE);
	if (task->ct_flags & TASK_IV)
		bus_dmamap_sync(sc->sc_dmat, task->ct_ivmap, 0,
		    task->ct_ivmap->dm_segs[0].ds_len, BUS_DMASYNC_POSTWRITE);
	if (task->ct_flags & TASK_KEY)
		/* XXX Can we zero the bounce buffer if there is one?  */
		bus_dmamap_sync(sc->sc_dmat, task->ct_keymap, 0,
		    task->ct_keymap->dm_segs[0].ds_len, BUS_DMASYNC_POSTWRITE);

	/* Temporarily release the lock to invoke the callback.  */
	mutex_exit(&sc->sc_lock);
	(*task->ct_callback)(sc, task, task->ct_cookie, error);
	mutex_enter(&sc->sc_lock);
}

/*
 * sun8i_crypto_allocbuf(sc, size, buf, dmaflags)
 *
 *	Allocate a single-segment DMA-safe buffer and map it into KVA.
 *	May fail if dmaflags is BUS_DMA_NOWAIT.
 */
static int
sun8i_crypto_allocbuf(struct sun8i_crypto_softc *sc, size_t size,
    struct sun8i_crypto_buf *buf, int dmaflags)
{
	int error;

	/* Allocate a DMA-safe buffer.  */
	error = bus_dmamem_alloc(sc->sc_dmat, size, 0, 0, buf->cb_seg,
	    __arraycount(buf->cb_seg), &buf->cb_nsegs, dmaflags);
	if (error)
		goto fail0;

	/* Map the buffer into kernel virtual address space.  */
	error = bus_dmamem_map(sc->sc_dmat, buf->cb_seg, buf->cb_nsegs,
	    size, &buf->cb_kva, dmaflags);
	if (error)
		goto fail1;

	/* Success!  */
	return 0;

fail2: __unused
	bus_dmamem_unmap(sc->sc_dmat, buf->cb_kva, size);
fail1:	bus_dmamem_free(sc->sc_dmat, buf->cb_seg, buf->cb_nsegs);
fail0:	return error;
}

/*
 * sun8i_crypto_freebuf(sc, buf)
 *
 *	Unmap buf and free it.
 */
static void
sun8i_crypto_freebuf(struct sun8i_crypto_softc *sc, size_t size,
    struct sun8i_crypto_buf *buf)
{

	bus_dmamem_unmap(sc->sc_dmat, buf->cb_kva, size);
	bus_dmamem_free(sc->sc_dmat, buf->cb_seg, buf->cb_nsegs);
}

/*
 * sun8i_crypto_rng_attach(sc)
 *
 *	Attach an rndsource for the crypto engine's TRNG.
 */
static void
sun8i_crypto_rng_attach(struct sun8i_crypto_softc *sc)
{
	device_t self = sc->sc_dev;
	struct sun8i_crypto_rng *rng = &sc->sc_rng;
	struct sun8i_crypto_task *task;
	int error;

	/* Preallocate a buffer to reuse.  */
	error = sun8i_crypto_allocbuf(sc, SUN8I_CRYPTO_RNGBYTES, &rng->cr_buf,
	    BUS_DMA_WAITOK);
	if (error)
		goto fail0;

	/* Create a task to reuse.  */
	task = rng->cr_task = sun8i_crypto_task_get(sc, sun8i_crypto_rng_done,
	    rng, PR_WAITOK);
	if (rng->cr_task == NULL) {
		error = ENOMEM;
		goto fail1;
	}

	/* Preload the destination map.  */
	error = bus_dmamap_load(sc->sc_dmat, task->ct_dstmap,
	    rng->cr_buf.cb_kva, SUN8I_CRYPTO_RNGBYTES, NULL, BUS_DMA_NOWAIT);
	if (error)
		goto fail2;

	/*
	 * Attach the rndsource.  This is _not_ marked as RND_TYPE_RNG
	 * because the output is not uniformly distributed.  The bits
	 * are heavily weighted toward 0 or 1, at different times, and
	 * I haven't scienced a satisfactory story out of it yet.
	 */
	rndsource_setcb(&rng->cr_rndsource, sun8i_crypto_rng_get, sc);
	rnd_attach_source(&rng->cr_rndsource, device_xname(self),
	    RND_TYPE_UNKNOWN,
	    RND_FLAG_COLLECT_VALUE|RND_FLAG_ESTIMATE_VALUE|RND_FLAG_HASCB);

	/* Success!  */
	return;

fail3: __unused
	bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap);
fail2:	sun8i_crypto_task_put(sc, task);
fail1:	sun8i_crypto_freebuf(sc, SUN8I_CRYPTO_RNGBYTES, &rng->cr_buf);
fail0:	aprint_error_dev(self, "failed to set up RNG, error=%d\n", error);
}

/*
 * sun8i_crypto_rng_get(nbytes, cookie)
 *
 *	On-demand rndsource callback: try to gather nbytes of entropy
 *	and enter them into the pool ASAP.
 */
static void
sun8i_crypto_rng_get(size_t nbytes, void *cookie)
{
	struct sun8i_crypto_softc *sc = cookie;
	struct sun8i_crypto_rng *rng = &sc->sc_rng;
	struct sun8i_crypto_task *task = rng->cr_task;
	bool pending;
	int error;

	/*
	 * Test and set the RNG-pending flag.  If it's already in
	 * progress, nothing to do here.
	 */
	mutex_enter(&sc->sc_lock);
	pending = rng->cr_pending;
	rng->cr_pending = true;
	mutex_exit(&sc->sc_lock);
	if (pending)
		return;

	/* Load the task descriptor.  */
	error = sun8i_crypto_task_load_trng(sc, task, SUN8I_CRYPTO_RNGBYTES);
	if (error)
		goto fail;

	/* Submit!  */
	error = sun8i_crypto_submit(sc, task);
	if (error)
		goto fail;

	/* All done!  */
	return;

fail:	mutex_enter(&sc->sc_lock);
	rng->cr_pending = false;
	mutex_exit(&sc->sc_lock);
}

static void
sun8i_crypto_rng_done(struct sun8i_crypto_softc *sc,
    struct sun8i_crypto_task *task, void *cookie, int error)
{
	struct sun8i_crypto_rng *rng = cookie;
	uint8_t *buf = rng->cr_buf.cb_kva;
	uint32_t entropybits;

	KASSERT(rng == &sc->sc_rng);

	/* If anything went wrong, forget about it.  */
	if (error)
		goto out;

	/*
	 * This TRNG has quite low entropy at best.  But if it fails a
	 * repeated output test, then assume it's busted.
	 */
	CTASSERT(SUN8I_CRYPTO_RNGBYTES <= UINT32_MAX/NBBY);
	entropybits = (NBBY*SUN8I_CRYPTO_RNGBYTES)/SUN8I_CRYPTO_RNGENTROPY;
	if (consttime_memequal(buf, buf + SUN8I_CRYPTO_RNGBYTES/2,
		SUN8I_CRYPTO_RNGBYTES/2)) {
		device_printf(sc->sc_dev, "failed repeated output test\n");
		entropybits = 0;
	}

	/*
	 * Actually we don't believe in any of the entropy until this
	 * device has had more scrutiny.
	 */
	entropybits = 0;

	/* Success!  Enter and erase the data.  */
	rnd_add_data(&rng->cr_rndsource, buf, SUN8I_CRYPTO_RNGBYTES,
	    entropybits);
	explicit_memset(buf, 0, SUN8I_CRYPTO_RNGBYTES);

out:	/* Done -- clear the RNG-pending flag.  */
	mutex_enter(&sc->sc_lock);
	rng->cr_pending = false;
	mutex_exit(&sc->sc_lock);
}

/*
 * Self-test
 */

static const uint8_t selftest_input[16];
static const uint8_t selftest_key[16];
static const uint8_t selftest_output[16] = {
	0x66,0xe9,0x4b,0xd4,0xef,0x8a,0x2c,0x3b,
	0x88,0x4c,0xfa,0x59,0xca,0x34,0x2b,0x2e,
};

static void
sun8i_crypto_selftest(device_t self)
{
	const size_t keybytes = sizeof selftest_key;
	const size_t nbytes = sizeof selftest_input;
	struct sun8i_crypto_softc *sc = device_private(self);
	struct sun8i_crypto_selftest *selftest = &sc->sc_selftest;
	struct sun8i_crypto_task *task;
	int error;

	CTASSERT(sizeof selftest_input == sizeof selftest_output);

	/* Allocate an input buffer.  */
	error = sun8i_crypto_allocbuf(sc, nbytes, &selftest->cs_in,
	    BUS_DMA_WAITOK);
	if (error)
		goto fail0;

	/* Allocate a key buffer.  */
	error = sun8i_crypto_allocbuf(sc, keybytes, &selftest->cs_key,
	    BUS_DMA_WAITOK);
	if (error)
		goto fail1;

	/* Allocate an output buffer.  */
	error = sun8i_crypto_allocbuf(sc, nbytes, &selftest->cs_out,
	    BUS_DMA_WAITOK);
	if (error)
		goto fail2;

	/* Allocate a task descriptor.  */
	task = selftest->cs_task = sun8i_crypto_task_get(sc,
	    sun8i_crypto_selftest_done, selftest, PR_WAITOK);
	if (selftest->cs_task == NULL) {
		error = ENOMEM;
		goto fail3;
	}

	/* Copy the input and key into their buffers.  */
	memcpy(selftest->cs_in.cb_kva, selftest_input, nbytes);
	memcpy(selftest->cs_key.cb_kva, selftest_key, keybytes);

	/* Load the key, src, and dst for DMA transfers.  */
	error = bus_dmamap_load(sc->sc_dmat, task->ct_keymap,
	    selftest->cs_key.cb_kva, keybytes, NULL, BUS_DMA_WAITOK);
	if (error)
		goto fail4;
	task->ct_flags |= TASK_KEY;

	error = bus_dmamap_load(sc->sc_dmat, task->ct_srcmap,
	    selftest->cs_in.cb_kva, nbytes, NULL, BUS_DMA_WAITOK);
	if (error)
		goto fail5;
	task->ct_flags |= TASK_SRC;

	error = bus_dmamap_load(sc->sc_dmat, task->ct_dstmap,
	    selftest->cs_out.cb_kva, nbytes, NULL, BUS_DMA_WAITOK);
	if (error)
		goto fail6;

	/* Set up the task descriptor.  */
	sun8i_crypto_task_load_aesecb(sc, task, nbytes,
	    SUN8I_CRYPTO_TDQS_AES_KEYSIZE_128, SUN8I_CRYPTO_TDQC_OP_DIR_ENC);

	/* Submit!  */
	error = sun8i_crypto_submit(sc, task);
	if (error)
		goto fail7;

	device_printf(sc->sc_dev, "AES-128 self-test initiated\n");

	/* Success!  */
	return;

fail7:	bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap);
fail6:	bus_dmamap_unload(sc->sc_dmat, task->ct_srcmap);
fail5:	bus_dmamap_unload(sc->sc_dmat, task->ct_keymap);
fail4:	sun8i_crypto_task_put(sc, task);
fail3:	sun8i_crypto_freebuf(sc, nbytes, &selftest->cs_out);
fail2:	sun8i_crypto_freebuf(sc, keybytes, &selftest->cs_key);
fail1:	sun8i_crypto_freebuf(sc, nbytes, &selftest->cs_in);
fail0:	aprint_error_dev(self, "failed to run self-test, error=%d\n", error);
}

static bool
sun8i_crypto_selftest_check(struct sun8i_crypto_softc *sc, const char *title,
    size_t n, const void *expected, const void *actual)
{
	const uint8_t *e = expected;
	const uint8_t *a = actual;
	size_t i;

	if (memcmp(e, a, n) == 0)
		return true;

	device_printf(sc->sc_dev, "self-test: %s\n", title);
	printf("expected: ");
	for (i = 0; i < n; i++)
		printf("%02hhx", e[i]);
	printf("\n");
	printf("actual:   ");
	for (i = 0; i < n; i++)
		printf("%02hhx", a[i]);
	printf("\n");
	return false;
}

static void
sun8i_crypto_selftest_done(struct sun8i_crypto_softc *sc,
    struct sun8i_crypto_task *task, void *cookie, int error)
{
	const size_t keybytes = sizeof selftest_key;
	const size_t nbytes = sizeof selftest_input;
	struct sun8i_crypto_selftest *selftest = cookie;
	bool ok = true;

	KASSERT(selftest == &sc->sc_selftest);

	/* If anything went wrong, fail now.  */
	if (error) {
		device_printf(sc->sc_dev, "self-test error=%d\n", error);
		goto out;
	}

	/*
	 * Verify the input and key weren't clobbered, and verify the
	 * output matches what we expect.
	 */
	ok &= sun8i_crypto_selftest_check(sc, "input clobbered", nbytes,
	    selftest_input, selftest->cs_in.cb_kva);
	ok &= sun8i_crypto_selftest_check(sc, "key clobbered", keybytes,
	    selftest_key, selftest->cs_key.cb_kva);
	ok &= sun8i_crypto_selftest_check(sc, "output mismatch", nbytes,
	    selftest_output, selftest->cs_out.cb_kva);

	/* XXX Disable the RNG and other stuff if this fails...  */
	if (ok)
		device_printf(sc->sc_dev, "AES-128 self-test passed\n");

out:	bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap);
	bus_dmamap_unload(sc->sc_dmat, task->ct_srcmap);
	bus_dmamap_unload(sc->sc_dmat, task->ct_keymap);
	sun8i_crypto_task_put(sc, task);
	sun8i_crypto_freebuf(sc, nbytes, &selftest->cs_out);
	sun8i_crypto_freebuf(sc, keybytes, &selftest->cs_key);
	sun8i_crypto_freebuf(sc, nbytes, &selftest->cs_in);
}

/*
 * Sysctl for testing
 */

struct sun8i_crypto_userreq {
	kmutex_t			cu_lock;
	kcondvar_t			cu_cv;
	size_t				cu_size;
	struct sun8i_crypto_buf		cu_buf;
	struct sun8i_crypto_task	*cu_task;
	int				cu_error;
	bool				cu_done;
	bool				cu_cancel;
};

static void
sun8i_crypto_sysctl_attach(struct sun8i_crypto_softc *sc)
{
	struct sun8i_crypto_sysctl *cy = &sc->sc_sysctl;
	int error;

	/* hw.sun8icryptoN (node) */
	error = sysctl_createv(&cy->cy_log, 0, NULL, &cy->cy_root_node,
	    CTLFLAG_PERMANENT, CTLTYPE_NODE, device_xname(sc->sc_dev),
	    SYSCTL_DESCR("sun8i crypto engine knobs"),
	    NULL, 0, NULL, 0,
	    CTL_HW, CTL_CREATE, CTL_EOL);
	if (error) {
		aprint_error_dev(sc->sc_dev,
		    "failed to set up sysctl hw.%s: %d\n",
		    device_xname(sc->sc_dev), error);
		return;
	}

	/* hw.sun8icryptoN.rng (`struct', 4096-byte array) */
	sysctl_createv(&cy->cy_log, 0, &cy->cy_root_node, &cy->cy_trng_node,
	    CTLFLAG_PERMANENT|CTLFLAG_READONLY|CTLFLAG_PRIVATE, CTLTYPE_STRUCT,
	    "rng", SYSCTL_DESCR("Read up to 4096 bytes out of the TRNG"),
	    &sun8i_crypto_sysctl_rng, 0, sc, 0, CTL_CREATE, CTL_EOL);
	if (error) {
		aprint_error_dev(sc->sc_dev,
		    "failed to set up sysctl hw.%s.rng: %d\n",
		    device_xname(sc->sc_dev), error);
		return;
	}
}

static int
sun8i_crypto_sysctl_rng(SYSCTLFN_ARGS)
{
	struct sysctlnode node = *rnode;
	struct sun8i_crypto_softc *sc = node.sysctl_data;
	struct sun8i_crypto_userreq *req;
	struct sun8i_crypto_task *task;
	size_t size;
	int error;

	/* If oldp == NULL, the caller wants to learn the size.  */
	if (oldp == NULL) {
		*oldlenp = 4096;
		return 0;
	}

	/* Truncate to 4096 bytes.  */
	size = MIN(4096, *oldlenp);
	if (size == 0)
		return 0;	/* nothing to do */

	/* Allocate a request context.  */
	req = kmem_alloc(sizeof(*req), KM_NOSLEEP);
	if (req == NULL)
		return ENOMEM;

	/* Initialize the request context.  */
	mutex_init(&req->cu_lock, MUTEX_DEFAULT, IPL_NONE);
	cv_init(&req->cu_cv, "sun8isy");
	req->cu_size = size;
	req->cu_error = EIO;
	req->cu_done = false;
	req->cu_cancel = false;

	/* Allocate a buffer for the RNG output.  */
	error = sun8i_crypto_allocbuf(sc, size, &req->cu_buf, BUS_DMA_NOWAIT);
	if (error)
		goto out0;

	/* Allocate a task.  */
	task = req->cu_task = sun8i_crypto_task_get(sc,
	    sun8i_crypto_sysctl_rng_done, req, PR_NOWAIT);
	if (task == NULL) {
		error = ENOMEM;
		goto out1;
	}

	/* Set the task up for TRNG to our buffer.  */
	error = bus_dmamap_load(sc->sc_dmat, task->ct_dstmap,
	    req->cu_buf.cb_kva, SUN8I_CRYPTO_RNGBYTES, NULL, BUS_DMA_NOWAIT);
	if (error)
		goto out2;
	error = sun8i_crypto_task_load_trng(sc, task, SUN8I_CRYPTO_RNGBYTES);
	if (error)
		goto out3;

	/* Submit!  */
	error = sun8i_crypto_submit(sc, task);
	if (error) {
		/* Make sure we don't restart the syscall -- just fail.  */
		if (error == ERESTART)
			error = EBUSY;
		goto out3;
	}

	/* Wait for the request to complete.  */
	mutex_enter(&req->cu_lock);
	while (!req->cu_done) {
		error = cv_wait_sig(&req->cu_cv, &req->cu_lock);
		if (error) {
			/*
			 * If we finished while waiting to acquire the
			 * lock, ignore the error and just return now.
			 * Otherwise, notify the callback that it has
			 * to clean up after us.
			 */
			if (req->cu_done)
				error = 0;
			else
				req->cu_cancel = true;
			break;
		}
	}
	mutex_exit(&req->cu_lock);

	/*
	 * Return early on error from cv_wait_sig, which means
	 * interruption; the callback will clean up instead.
	 */
	if (error)
		return error;

	/* Check for error from the device.  */
	error = req->cu_error;
	if (error)
		goto out3;

	/* Copy out the data.  */
	node.sysctl_data = req->cu_buf.cb_kva;
	node.sysctl_size = size;
	error = sysctl_lookup(SYSCTLFN_CALL(&node));

	/* Clear the buffer.  */
	explicit_memset(req->cu_buf.cb_kva, 0, size);

	/* Clean up.  */
out3:	bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap);
out2:	sun8i_crypto_task_put(sc, task);
out1:	sun8i_crypto_freebuf(sc, req->cu_size, &req->cu_buf);
out0:	cv_destroy(&req->cu_cv);
	mutex_destroy(&req->cu_lock);
	kmem_free(req, sizeof(*req));
	return error;
}

static void
sun8i_crypto_sysctl_rng_done(struct sun8i_crypto_softc *sc,
    struct sun8i_crypto_task *task, void *cookie, int error)
{
	struct sun8i_crypto_userreq *req = cookie;
	bool cancel;

	/*
	 * Notify the waiting thread of the error, and find out whether
	 * that thread cancelled.
	 */
	mutex_enter(&req->cu_lock);
	cancel = req->cu_cancel;
	req->cu_error = error;
	req->cu_done = true;
	cv_broadcast(&req->cu_cv);
	mutex_exit(&req->cu_lock);

	/*
	 * If it wasn't cancelled, we're done -- the main thread will
	 * clean up after itself.
	 */
	if (!cancel)
		return;

	/* Clean up after the main thread cancelled.  */
	bus_dmamap_unload(sc->sc_dmat, task->ct_dstmap);
	sun8i_crypto_task_put(sc, task);
	sun8i_crypto_freebuf(sc, req->cu_size, &req->cu_buf);
	cv_destroy(&req->cu_cv);
	mutex_destroy(&req->cu_lock);
	kmem_free(req, sizeof(*req));
}