xref: /src/sys/arch/arm/xscale/pxa2x0_dmac.c

/*	$NetBSD: pxa2x0_dmac.c,v 1.13 2015/02/05 13:27:18 nonaka Exp $	*/

/*
 * Copyright (c) 2003, 2005 Wasabi Systems, Inc.
 * All rights reserved.
 *
 * Written by Steve C. Woodford for Wasabi Systems, Inc.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. All advertising materials mentioning features or use of this software
 *    must display the following acknowledgement:
 *	This product includes software developed for the NetBSD Project by
 *	Wasabi Systems, Inc.
 * 4. The name of Wasabi Systems, Inc. may not be used to endorse
 *    or promote products derived from this software without specific prior
 *    written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL WASABI SYSTEMS, INC
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#include "opt_pxa2x0_dmac.h"

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/device.h>
#include <sys/kernel.h>
#include <sys/kmem.h>
#include <sys/queue.h>

#include <uvm/uvm_param.h>	/* For PAGE_SIZE */

#include <machine/intr.h>
#include <sys/bus.h>

#include <dev/dmover/dmovervar.h>

#include <arm/xscale/pxa2x0reg.h>
#include <arm/xscale/pxa2x0var.h>
#include <arm/xscale/pxa2x0cpu.h>

#include <arm/xscale/pxa2x0_dmac.h>

#include "locators.h"

#undef DMAC_N_PRIORITIES
#ifndef PXA2X0_DMAC_FIXED_PRIORITY
#define DMAC_N_PRIORITIES 3
#define DMAC_PRI(p)   (p)
#else
#define DMAC_N_PRIORITIES 1
#define DMAC_PRI(p)   (0)
#endif

struct dmac_desc {
	SLIST_ENTRY(dmac_desc) d_link;
	struct pxa2x0_dma_desc *d_desc;
	paddr_t d_desc_pa;
};

/*
 * This is used to maintain state for an in-progress transfer.
 * It tracks the current DMA segment, and offset within the segment
 * in the case where we had to split a request into several DMA
 * operations due to a shortage of DMAC descriptors.
 */
struct dmac_desc_segs {
	bus_dma_segment_t *ds_curseg;		/* Current segment */
	u_int ds_nsegs;				/* Remaining segments */
	bus_size_t ds_offset;			/* Offset within current seg */
};

SIMPLEQ_HEAD(dmac_xfer_state_head, dmac_xfer_state);

struct dmac_xfer_state {
	struct dmac_xfer dxs_xfer;
#define	dxs_cookie	dxs_xfer.dx_cookie
#define	dxs_done	dxs_xfer.dx_done
#define	dxs_priority	dxs_xfer.dx_priority
#define	dxs_peripheral	dxs_xfer.dx_peripheral
#define	dxs_flow	dxs_xfer.dx_flow
#define	dxs_dev_width	dxs_xfer.dx_dev_width
#define	dxs_burst_size	dxs_xfer.dx_burst_size
#define	dxs_loop_notify	dxs_xfer.dx_loop_notify
#define	dxs_desc	dxs_xfer.dx_desc
	SIMPLEQ_ENTRY(dmac_xfer_state) dxs_link;
	SLIST_HEAD(, dmac_desc) dxs_descs;
	struct dmac_xfer_state_head *dxs_queue;
	u_int dxs_channel;
#define	DMAC_NO_CHANNEL	(~0)
	uint32_t dxs_dcmd;
	struct dmac_desc_segs dxs_segs[2];
	bool dxs_misaligned_flag;
};


#if (PXA2X0_DMAC_DMOVER_CONCURRENCY > 0)
/*
 * This structure is used to maintain state for the dmover(9) backend
 * part of the driver. We can have a number of concurrent dmover
 * requests in progress at any given time. The exact number is given
 * by the PXA2X0_DMAC_DMOVER_CONCURRENCY compile-time constant. One of
 * these structures is allocated for each concurrent request.
 */
struct dmac_dmover_state {
	LIST_ENTRY(dmac_dmover_state) ds_link;	/* List of idle dmover chans */
	struct pxadmac_softc *ds_sc;		/* Uplink to pxadmac softc */
	struct dmover_request *ds_current;	/* Current dmover request */
	struct dmac_xfer_state ds_xfer;
	bus_dmamap_t ds_src_dmap;
	bus_dmamap_t ds_dst_dmap;
/*
 * There is no inherent size limit in the DMA engine.
 * The following limit is somewhat arbitrary.
 */
#define	DMAC_DMOVER_MAX_XFER	(8*1024*1024)
#if 0
/* This would require 16KB * 2 just for segments... */
#define DMAC_DMOVER_NSEGS	((DMAC_DMOVER_MAX_XFER / PAGE_SIZE) + 1)
#else
#define DMAC_DMOVER_NSEGS	512		/* XXX: Only enough for 2MB */
#endif
	bus_dma_segment_t ds_zero_seg;		/* Used for zero-fill ops */
	void *ds_zero_va;
	bus_dma_segment_t ds_fill_seg;		/* Used for fill8 ops */
	void *ds_fill_va;

#define	ds_src_addr_hold	ds_xfer.dxs_desc[DMAC_DESC_SRC].xd_addr_hold
#define	ds_dst_addr_hold	ds_xfer.dxs_desc[DMAC_DESC_DST].xd_addr_hold
#define	ds_src_burst		ds_xfer.dxs_desc[DMAC_DESC_SRC].xd_burst_size
#define	ds_dst_burst		ds_xfer.dxs_desc[DMAC_DESC_DST].xd_burst_size
#define	ds_src_dma_segs		ds_xfer.dxs_desc[DMAC_DESC_SRC].xd_dma_segs
#define	ds_dst_dma_segs		ds_xfer.dxs_desc[DMAC_DESC_DST].xd_dma_segs
#define	ds_src_nsegs		ds_xfer.dxs_desc[DMAC_DESC_SRC].xd_nsegs
#define	ds_dst_nsegs		ds_xfer.dxs_desc[DMAC_DESC_DST].xd_nsegs
};

/*
 * Overall dmover(9) backend state
 */
struct dmac_dmover {
	struct dmover_backend dd_backend;
	int dd_busy;
	LIST_HEAD(, dmac_dmover_state) dd_free;
	struct dmac_dmover_state dd_state[PXA2X0_DMAC_DMOVER_CONCURRENCY];
};
#endif

struct pxadmac_softc {
	device_t sc_dev;
	bus_space_tag_t sc_bust;
	bus_dma_tag_t sc_dmat;
	bus_space_handle_t sc_bush;
	void *sc_irqcookie;

	/*
	 * Queue of pending requests, per priority
	 */
	struct dmac_xfer_state_head sc_queue[DMAC_N_PRIORITIES];

	/*
	 * Queue of pending requests, per peripheral
	 */
	struct {
		struct dmac_xfer_state_head sp_queue;
		u_int sp_busy;
	} sc_periph[DMAC_N_PERIPH];

	/*
	 * Active requests, per channel.
	 */
	struct dmac_xfer_state *sc_active[DMAC_N_CHANNELS];

	/*
	 * Channel Priority Allocation
	 */
	struct {
		uint8_t p_first;
		uint8_t p_pri[DMAC_N_CHANNELS];
	} sc_prio[DMAC_N_PRIORITIES];
#define	DMAC_PRIO_END		(~0)
	uint8_t sc_channel_priority[DMAC_N_CHANNELS];

	/*
	 * DMA descriptor management
	 */
	bus_dmamap_t sc_desc_map;
	bus_dma_segment_t sc_segs;
#define	DMAC_N_DESCS	((PAGE_SIZE * 2) / sizeof(struct pxa2x0_dma_desc))
#define	DMAC_DESCS_SIZE	(DMAC_N_DESCS * sizeof(struct pxa2x0_dma_desc))
	struct dmac_desc sc_all_descs[DMAC_N_DESCS];
	u_int sc_free_descs;
	SLIST_HEAD(, dmac_desc) sc_descs;

#if (PXA2X0_DMAC_DMOVER_CONCURRENCY > 0)
	/*
	 * dmover(9) backend state
	 */
	struct dmac_dmover sc_dmover;
#endif
};

static int	pxadmac_match(device_t, cfdata_t, void *);
static void	pxadmac_attach(device_t, device_t, void *);

CFATTACH_DECL_NEW(pxadmac, sizeof(struct pxadmac_softc),
    pxadmac_match, pxadmac_attach, NULL, NULL);

static struct pxadmac_softc *pxadmac_sc;

static void dmac_start(struct pxadmac_softc *, dmac_priority_t);
static int dmac_continue_xfer(struct pxadmac_softc *, struct dmac_xfer_state *);
static u_int dmac_channel_intr(struct pxadmac_softc *, u_int);
static int dmac_intr(void *);

#if (PXA2X0_DMAC_DMOVER_CONCURRENCY > 0)
static void dmac_dmover_attach(struct pxadmac_softc *);
static void dmac_dmover_process(struct dmover_backend *);
static void dmac_dmover_run(struct dmover_backend *);
static void dmac_dmover_done(struct dmac_xfer *, int);
#endif

static inline uint32_t
dmac_reg_read(struct pxadmac_softc *sc, int reg)
{

	return (bus_space_read_4(sc->sc_bust, sc->sc_bush, reg));
}

static inline void
dmac_reg_write(struct pxadmac_softc *sc, int reg, uint32_t val)
{

	bus_space_write_4(sc->sc_bust, sc->sc_bush, reg, val);
}

static inline int
dmac_allocate_channel(struct pxadmac_softc *sc, dmac_priority_t priority,
    u_int *chanp)
{
	u_int channel;

	KDASSERT((u_int)priority < DMAC_N_PRIORITIES);

	if ((channel = sc->sc_prio[priority].p_first) == DMAC_PRIO_END)
		return (-1);
	sc->sc_prio[priority].p_first = sc->sc_prio[priority].p_pri[channel];

	*chanp = channel;
	return (0);
}

static inline void
dmac_free_channel(struct pxadmac_softc *sc, dmac_priority_t priority,
    u_int channel)
{

	KDASSERT((u_int)priority < DMAC_N_PRIORITIES);

	sc->sc_prio[priority].p_pri[channel] = sc->sc_prio[priority].p_first;
	sc->sc_prio[priority].p_first = channel;
}

static int
pxadmac_match(device_t parent, cfdata_t cf, void *aux)
{
	struct pxaip_attach_args *pxa = aux;

	if (pxadmac_sc || pxa->pxa_addr != PXA2X0_DMAC_BASE ||
	    pxa->pxa_intr != PXA2X0_INT_DMA)
		return (0);

	pxa->pxa_size = PXA2X0_DMAC_SIZE;

	return (1);
}

static void
pxadmac_attach(device_t parent, device_t self, void *aux)
{
	struct pxadmac_softc *sc = device_private(self);
	struct pxaip_attach_args *pxa = aux;
	struct pxa2x0_dma_desc *dd;
	int i, nsegs;

	sc->sc_dev = self;
	sc->sc_bust = pxa->pxa_iot;
	sc->sc_dmat = pxa->pxa_dmat;

	aprint_normal(": DMA Controller\n");

	if (bus_space_map(sc->sc_bust, pxa->pxa_addr, pxa->pxa_size, 0,
	    &sc->sc_bush)) {
		aprint_error_dev(self, "Can't map registers!\n");
		return;
	}

	pxadmac_sc = sc;

	/*
	 * Make sure the DMAC is quiescent
	 */
	for (i = 0; i < DMAC_N_CHANNELS; i++) {
		dmac_reg_write(sc, DMAC_DCSR(i), 0);
		dmac_reg_write(sc, DMAC_DRCMR(i), 0);
		sc->sc_active[i] = NULL;
	}
	if (!CPU_IS_PXA270)
		dmac_reg_write(sc, DMAC_DINT,
		    dmac_reg_read(sc, DMAC_DINT) & DMAC_DINT_MASK);

	/*
	 * Initialise the request queues
	 */
	for (i = 0; i < DMAC_N_PRIORITIES; i++)
		SIMPLEQ_INIT(&sc->sc_queue[i]);

	/*
	 * Initialise the request queues
	 */
	for (i = 0; i < DMAC_N_PERIPH; i++) {
		sc->sc_periph[i].sp_busy = 0;
		SIMPLEQ_INIT(&sc->sc_periph[i].sp_queue);
	}

	/*
	 * Initialise the channel priority metadata
	 */
	memset(sc->sc_prio, DMAC_PRIO_END, sizeof(sc->sc_prio));
	for (i = 0; i < DMAC_N_CHANNELS; i++) {
#if (DMAC_N_PRIORITIES > 1)
		if (i <= 3)
			dmac_free_channel(sc, DMAC_PRIORITY_HIGH, i);
		else
		if (i <= 7)
			dmac_free_channel(sc, DMAC_PRIORITY_MED, i);
		else
			dmac_free_channel(sc, DMAC_PRIORITY_LOW, i);
#else
		dmac_free_channel(sc, DMAC_PRIORITY_NORMAL, i);
#endif
	}

	/*
	 * Initialise DMA descriptors and associated metadata
	 */
	if (bus_dmamem_alloc(sc->sc_dmat, DMAC_DESCS_SIZE, DMAC_DESCS_SIZE, 0,
	    &sc->sc_segs, 1, &nsegs, BUS_DMA_NOWAIT))
		panic("dmac_pxaip_attach: bus_dmamem_alloc failed");

	if (bus_dmamem_map(sc->sc_dmat, &sc->sc_segs, 1, DMAC_DESCS_SIZE,
	    (void *)&dd, BUS_DMA_COHERENT|BUS_DMA_NOCACHE))
		panic("dmac_pxaip_attach: bus_dmamem_map failed");

	if (bus_dmamap_create(sc->sc_dmat, DMAC_DESCS_SIZE, 1,
	    DMAC_DESCS_SIZE, 0, BUS_DMA_NOWAIT, &sc->sc_desc_map))
		panic("dmac_pxaip_attach: bus_dmamap_create failed");

	if (bus_dmamap_load(sc->sc_dmat, sc->sc_desc_map, (void *)dd,
	    DMAC_DESCS_SIZE, NULL, BUS_DMA_NOWAIT))
		panic("dmac_pxaip_attach: bus_dmamap_load failed");

	SLIST_INIT(&sc->sc_descs);
	sc->sc_free_descs = DMAC_N_DESCS;
	for (i = 0; i < DMAC_N_DESCS; i++, dd++) {
		SLIST_INSERT_HEAD(&sc->sc_descs, &sc->sc_all_descs[i], d_link);
		sc->sc_all_descs[i].d_desc = dd;
		sc->sc_all_descs[i].d_desc_pa =
		    sc->sc_segs.ds_addr + (sizeof(struct pxa2x0_dma_desc) * i);
	}

	sc->sc_irqcookie = pxa2x0_intr_establish(pxa->pxa_intr, IPL_BIO,
	    dmac_intr, sc);
	KASSERT(sc->sc_irqcookie != NULL);

#if (PXA2X0_DMAC_DMOVER_CONCURRENCY > 0)
	dmac_dmover_attach(sc);
#endif
}

#if (PXA2X0_DMAC_DMOVER_CONCURRENCY > 0)
/*
 * We support the following dmover(9) operations
 */
static const struct dmover_algdesc dmac_dmover_algdescs[] = {
	{DMOVER_FUNC_ZERO, NULL, 0},	/* Zero-fill */
	{DMOVER_FUNC_FILL8, NULL, 0},	/* Fill with 8-bit immediate value */
	{DMOVER_FUNC_COPY, NULL, 1}	/* Copy */
};
#define	DMAC_DMOVER_ALGDESC_COUNT \
	(sizeof(dmac_dmover_algdescs) / sizeof(dmac_dmover_algdescs[0]))

static void
dmac_dmover_attach(struct pxadmac_softc *sc)
{
	struct dmac_dmover *dd = &sc->sc_dmover;
	struct dmac_dmover_state *ds;
	int i, dummy;

	/*
	 * Describe ourselves to the dmover(9) code
	 */
	dd->dd_backend.dmb_name = "pxadmac";
	dd->dd_backend.dmb_speed = 100*1024*1024;	/* XXX */
	dd->dd_backend.dmb_cookie = sc;
	dd->dd_backend.dmb_algdescs = dmac_dmover_algdescs;
	dd->dd_backend.dmb_nalgdescs = DMAC_DMOVER_ALGDESC_COUNT;
	dd->dd_backend.dmb_process = dmac_dmover_process;
	dd->dd_busy = 0;
	LIST_INIT(&dd->dd_free);

	for (i = 0; i < PXA2X0_DMAC_DMOVER_CONCURRENCY; i++) {
		ds = &dd->dd_state[i];
		ds->ds_sc = sc;
		ds->ds_current = NULL;
		ds->ds_xfer.dxs_cookie = ds;
		ds->ds_xfer.dxs_done = dmac_dmover_done;
		ds->ds_xfer.dxs_priority = DMAC_PRIORITY_NORMAL;
		ds->ds_xfer.dxs_peripheral = DMAC_PERIPH_NONE;
		ds->ds_xfer.dxs_flow = DMAC_FLOW_CTRL_NONE;
		ds->ds_xfer.dxs_dev_width = DMAC_DEV_WIDTH_DEFAULT;
		ds->ds_xfer.dxs_burst_size = DMAC_BURST_SIZE_8;	/* XXX */
		ds->ds_xfer.dxs_loop_notify = DMAC_DONT_LOOP;
		ds->ds_src_addr_hold = false;
		ds->ds_dst_addr_hold = false;
		ds->ds_src_nsegs = 0;
		ds->ds_dst_nsegs = 0;
		LIST_INSERT_HEAD(&dd->dd_free, ds, ds_link);

		/*
		 * Create dma maps for both source and destination buffers.
		 */
		if (bus_dmamap_create(sc->sc_dmat, DMAC_DMOVER_MAX_XFER,
				DMAC_DMOVER_NSEGS, DMAC_DMOVER_MAX_XFER,
				0, BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW,
				&ds->ds_src_dmap) ||
		    bus_dmamap_create(sc->sc_dmat, DMAC_DMOVER_MAX_XFER,
				DMAC_DMOVER_NSEGS, DMAC_DMOVER_MAX_XFER,
				0, BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW,
				&ds->ds_dst_dmap)) {
			panic("dmac_dmover_attach: bus_dmamap_create failed");
		}

		/*
		 * Allocate some dma memory to be used as source buffers
		 * for the zero-fill and fill-8 operations. We only need
		 * small buffers here, since we set up the DMAC source
		 * descriptor with 'ds_addr_hold' set to true.
		 */
		if (bus_dmamem_alloc(sc->sc_dmat, arm_pcache.dcache_line_size,
				arm_pcache.dcache_line_size, 0,
				&ds->ds_zero_seg, 1, &dummy, BUS_DMA_NOWAIT) ||
		    bus_dmamem_alloc(sc->sc_dmat, arm_pcache.dcache_line_size,
				arm_pcache.dcache_line_size, 0,
				&ds->ds_fill_seg, 1, &dummy, BUS_DMA_NOWAIT)) {
			panic("dmac_dmover_attach: bus_dmamem_alloc failed");
		}

		if (bus_dmamem_map(sc->sc_dmat, &ds->ds_zero_seg, 1,
				arm_pcache.dcache_line_size, &ds->ds_zero_va,
				BUS_DMA_NOWAIT) ||
		    bus_dmamem_map(sc->sc_dmat, &ds->ds_fill_seg, 1,
				arm_pcache.dcache_line_size, &ds->ds_fill_va,
				BUS_DMA_NOWAIT)) {
			panic("dmac_dmover_attach: bus_dmamem_map failed");
		}

		/*
		 * Make sure the zero-fill source buffer really is zero filled
		 */
		memset(ds->ds_zero_va, 0, arm_pcache.dcache_line_size);
	}

	dmover_backend_register(&sc->sc_dmover.dd_backend);
}

static void
dmac_dmover_process(struct dmover_backend *dmb)
{
	struct pxadmac_softc *sc = dmb->dmb_cookie;
	int s = splbio();

	/*
	 * If the backend is currently idle, go process the queue.
	 */
	if (sc->sc_dmover.dd_busy == 0)
		dmac_dmover_run(&sc->sc_dmover.dd_backend);
	splx(s);
}

static void
dmac_dmover_run(struct dmover_backend *dmb)
{
	struct dmover_request *dreq;
	struct pxadmac_softc *sc;
	struct dmac_dmover *dd;
	struct dmac_dmover_state *ds;
	size_t len_src, len_dst;
	int rv;

	sc = dmb->dmb_cookie;
	dd = &sc->sc_dmover;
	sc->sc_dmover.dd_busy = 1;

	/*
	 * As long as we can queue up dmover requests...
	 */
	while ((dreq = TAILQ_FIRST(&dmb->dmb_pendreqs)) != NULL &&
	    (ds = LIST_FIRST(&dd->dd_free)) != NULL) {
		/*
		 * Pull the request off the queue, mark it 'running',
		 * and make it 'current'.
		 */
		dmover_backend_remque(dmb, dreq);
		dreq->dreq_flags |= DMOVER_REQ_RUNNING;
		LIST_REMOVE(ds, ds_link);
		ds->ds_current = dreq;

		switch (dreq->dreq_outbuf_type) {
		case DMOVER_BUF_LINEAR:
			len_dst = dreq->dreq_outbuf.dmbuf_linear.l_len;
			break;
		case DMOVER_BUF_UIO:
			len_dst = dreq->dreq_outbuf.dmbuf_uio->uio_resid;
			break;
		default:
			goto error;
		}

		/*
		 * Fix up the appropriate DMA 'source' buffer
		 */
		if (dreq->dreq_assignment->das_algdesc->dad_ninputs) {
			struct uio *uio;
			/*
			 * This is a 'copy' operation.
			 * Load up the specified source buffer
			 */
			switch (dreq->dreq_inbuf_type) {
			case DMOVER_BUF_LINEAR:
				len_src= dreq->dreq_inbuf[0].dmbuf_linear.l_len;
				if (len_src != len_dst)
					goto error;
				if (bus_dmamap_load(sc->sc_dmat,ds->ds_src_dmap,
				    dreq->dreq_inbuf[0].dmbuf_linear.l_addr,
				    len_src, NULL,
				    BUS_DMA_NOWAIT | BUS_DMA_STREAMING |
				    BUS_DMA_READ))
					goto error;
				break;

			case DMOVER_BUF_UIO:
				uio = dreq->dreq_inbuf[0].dmbuf_uio;
				len_src = uio->uio_resid;
				if (uio->uio_rw != UIO_WRITE ||
				    len_src != len_dst)
					goto error;
				if (bus_dmamap_load_uio(sc->sc_dmat,
				    ds->ds_src_dmap, uio,
				    BUS_DMA_NOWAIT | BUS_DMA_STREAMING |
				    BUS_DMA_READ))
					goto error;
				break;

			default:
				goto error;
			}

			ds->ds_src_addr_hold = false;
		} else
		if (dreq->dreq_assignment->das_algdesc->dad_name ==
		    DMOVER_FUNC_ZERO) {
			/*
			 * Zero-fill operation.
			 * Simply load up the pre-zeroed source buffer
			 */
			if (bus_dmamap_load(sc->sc_dmat, ds->ds_src_dmap,
			    ds->ds_zero_va, arm_pcache.dcache_line_size, NULL,
			    BUS_DMA_NOWAIT | BUS_DMA_STREAMING | BUS_DMA_READ))
				goto error;

			ds->ds_src_addr_hold = true;
		} else
		if (dreq->dreq_assignment->das_algdesc->dad_name ==
		    DMOVER_FUNC_FILL8) {
			/*
			 * Fill-8 operation.
			 * Initialise our fill-8 buffer, and load it up.
			 *
			 * XXX: Experiment with exactly how much of the
			 * source buffer needs to be filled. Particularly WRT
			 * burst size (which is hardcoded to 8 for dmover).
			 */
			memset(ds->ds_fill_va, dreq->dreq_immediate[0],
			    arm_pcache.dcache_line_size);

			if (bus_dmamap_load(sc->sc_dmat, ds->ds_src_dmap,
			    ds->ds_fill_va, arm_pcache.dcache_line_size, NULL,
			    BUS_DMA_NOWAIT | BUS_DMA_STREAMING | BUS_DMA_READ))
				goto error;

			ds->ds_src_addr_hold = true;
		} else {
			goto error;
		}

		/*
		 * Now do the same for the destination buffer
		 */
		switch (dreq->dreq_outbuf_type) {
		case DMOVER_BUF_LINEAR:
			if (bus_dmamap_load(sc->sc_dmat, ds->ds_dst_dmap,
			    dreq->dreq_outbuf.dmbuf_linear.l_addr,
			    len_dst, NULL,
			    BUS_DMA_NOWAIT | BUS_DMA_STREAMING | BUS_DMA_WRITE))
				goto error_unload_src;
			break;

		case DMOVER_BUF_UIO:
			if (dreq->dreq_outbuf.dmbuf_uio->uio_rw != UIO_READ)
				goto error_unload_src;
			if (bus_dmamap_load_uio(sc->sc_dmat, ds->ds_dst_dmap,
			    dreq->dreq_outbuf.dmbuf_uio,
			    BUS_DMA_NOWAIT | BUS_DMA_STREAMING | BUS_DMA_WRITE))
				goto error_unload_src;
			break;

		default:
		error_unload_src:
			bus_dmamap_unload(sc->sc_dmat, ds->ds_src_dmap);
		error:
			dreq->dreq_error = EINVAL;
			dreq->dreq_flags |= DMOVER_REQ_ERROR;
			ds->ds_current = NULL;
			LIST_INSERT_HEAD(&dd->dd_free, ds, ds_link);
			dmover_done(dreq);
			continue;
		}

		/*
		 * The last step before shipping the request off to the
		 * DMAC driver is to sync the dma maps.
		 */
		bus_dmamap_sync(sc->sc_dmat, ds->ds_src_dmap, 0,
		    ds->ds_src_dmap->dm_mapsize, BUS_DMASYNC_PREWRITE);
		ds->ds_src_dma_segs = ds->ds_src_dmap->dm_segs;
		ds->ds_src_nsegs = ds->ds_src_dmap->dm_nsegs;

		bus_dmamap_sync(sc->sc_dmat, ds->ds_dst_dmap, 0,
		    ds->ds_dst_dmap->dm_mapsize, BUS_DMASYNC_PREREAD);
		ds->ds_dst_dma_segs = ds->ds_dst_dmap->dm_segs;
		ds->ds_dst_nsegs = ds->ds_dst_dmap->dm_nsegs;

		/*
		 * Hand the request over to the dmac section of the driver.
		 */
		if ((rv = pxa2x0_dmac_start_xfer(&ds->ds_xfer.dxs_xfer)) != 0) {
			bus_dmamap_unload(sc->sc_dmat, ds->ds_src_dmap);
			bus_dmamap_unload(sc->sc_dmat, ds->ds_dst_dmap);
			dreq->dreq_error = rv;
			dreq->dreq_flags |= DMOVER_REQ_ERROR;
			ds->ds_current = NULL;
			LIST_INSERT_HEAD(&dd->dd_free, ds, ds_link);
			dmover_done(dreq);
		}
	}

	/* All done */
	sc->sc_dmover.dd_busy = 0;
}

static void
dmac_dmover_done(struct dmac_xfer *dx, int error)
{
	struct dmac_dmover_state *ds = dx->dx_cookie;
	struct pxadmac_softc *sc = ds->ds_sc;
	struct dmover_request *dreq = ds->ds_current;

	/*
	 * A dmover(9) request has just completed.
	 */

	KDASSERT(dreq != NULL);

	/*
	 * Sync and unload the DMA maps
	 */
	bus_dmamap_sync(sc->sc_dmat, ds->ds_src_dmap, 0,
	    ds->ds_src_dmap->dm_mapsize, BUS_DMASYNC_POSTREAD);
	bus_dmamap_sync(sc->sc_dmat, ds->ds_dst_dmap, 0,
	    ds->ds_dst_dmap->dm_mapsize, BUS_DMASYNC_POSTWRITE);

	bus_dmamap_unload(sc->sc_dmat, ds->ds_src_dmap);
	bus_dmamap_unload(sc->sc_dmat, ds->ds_dst_dmap);

	ds->ds_current = NULL;
	LIST_INSERT_HEAD(&sc->sc_dmover.dd_free, ds, ds_link);

	/*
	 * Record the completion status of the transfer
	 */
	if (error) {
		dreq->dreq_error = error;
		dreq->dreq_flags |= DMOVER_REQ_ERROR;
	} else {
		if (dreq->dreq_outbuf_type == DMOVER_BUF_UIO)
			dreq->dreq_outbuf.dmbuf_uio->uio_resid = 0;
		if (dreq->dreq_assignment->das_algdesc->dad_ninputs &&
		    dreq->dreq_inbuf_type == DMOVER_BUF_UIO)
			dreq->dreq_inbuf[0].dmbuf_uio->uio_resid = 0;
	}

	/*
	 * Done!
	 */
	dmover_done(dreq);

	/*
	 * See if we can start some more dmover(9) requests.
	 *
	 * Note: We're already at splbio() here.
	 */
	if (sc->sc_dmover.dd_busy == 0)
		dmac_dmover_run(&sc->sc_dmover.dd_backend);
}
#endif

struct dmac_xfer *
pxa2x0_dmac_allocate_xfer(void)
{
	struct dmac_xfer_state *dxs;

	dxs = kmem_alloc(sizeof(*dxs), KM_SLEEP);

	return ((struct dmac_xfer *)dxs);
}

void
pxa2x0_dmac_free_xfer(struct dmac_xfer *dx)
{
	struct dmac_xfer_state *dxs = (struct dmac_xfer_state *)dx;

	/*
	 * XXX: Should verify the DMAC is not actively using this
	 * structure before freeing...
	 */
	kmem_free(dxs, sizeof(*dxs));
}

static inline int
dmac_validate_desc(struct dmac_xfer_desc *xd, size_t *psize,
    bool *misaligned_flag)
{
	bus_dma_segment_t *dma_segs = xd->xd_dma_segs;
	bus_addr_t periph_end;
	bus_size_t align;
	size_t size;
	int i, nsegs = xd->xd_nsegs;

	/*
	 * Make sure the transfer parameters are acceptable.
	 */

	if (xd->xd_addr_hold &&
	    (nsegs != 1 || dma_segs[0].ds_len == 0))
		return (EINVAL);

	periph_end = CPU_IS_PXA270 ? PXA270_PERIPH_END : PXA250_PERIPH_END;
	for (i = 0, size = 0; i < nsegs; i++) {
		if (dma_segs[i].ds_addr >= PXA2X0_PERIPH_START &&
		    dma_segs[i].ds_addr + dma_segs[i].ds_len < periph_end)
			/* Internal Peripherals. */
			align = 0x03;
		else /* Companion-Chip/External Peripherals/External Memory. */
			align = 0x07;
		/*
		 * XXXX:
		 * Also PXA27x has more constraints by pairs Source/Target.
		 */

		if (dma_segs[i].ds_addr & align) {
			if (!CPU_IS_PXA270)
				return (EFAULT);
			*misaligned_flag = true;
		}
		size += dma_segs[i].ds_len;
	}

	*psize = size;
	return (0);
}

static inline int
dmac_init_desc(struct dmac_desc_segs *ds, struct dmac_xfer_desc *xd,
    size_t *psize, bool *misaligned_flag)
{
	int err;

	if ((err = dmac_validate_desc(xd, psize, misaligned_flag)))
		return (err);

	ds->ds_curseg = xd->xd_dma_segs;
	ds->ds_nsegs = xd->xd_nsegs;
	ds->ds_offset = 0;
	return (0);
}

int
pxa2x0_dmac_start_xfer(struct dmac_xfer *dx)
{
	struct pxadmac_softc *sc = pxadmac_sc;
	struct dmac_xfer_state *dxs = (struct dmac_xfer_state *)dx;
	struct dmac_xfer_desc *src, *dst;
	size_t size;
	int err, s;

	if (dxs->dxs_peripheral != DMAC_PERIPH_NONE &&
	    dxs->dxs_peripheral >= DMAC_N_PERIPH)
		return (EINVAL);

	src = &dxs->dxs_desc[DMAC_DESC_SRC];
	dst = &dxs->dxs_desc[DMAC_DESC_DST];

	dxs->dxs_misaligned_flag = false;

	if ((err = dmac_init_desc(&dxs->dxs_segs[DMAC_DESC_SRC], src, &size,
	    &dxs->dxs_misaligned_flag)))
		return (err);
	if (src->xd_addr_hold == false &&
	    dxs->dxs_loop_notify != DMAC_DONT_LOOP &&
	    (size % dxs->dxs_loop_notify) != 0)
		return (EINVAL);

	if ((err = dmac_init_desc(&dxs->dxs_segs[DMAC_DESC_DST], dst, &size,
	    &dxs->dxs_misaligned_flag)))
		return (err);
	if (dst->xd_addr_hold == false &&
	    dxs->dxs_loop_notify != DMAC_DONT_LOOP &&
	    (size % dxs->dxs_loop_notify) != 0)
		return (EINVAL);

	SLIST_INIT(&dxs->dxs_descs);
	dxs->dxs_channel = DMAC_NO_CHANNEL;
	dxs->dxs_dcmd = (((uint32_t)dxs->dxs_dev_width) << DCMD_WIDTH_SHIFT) |
	    (((uint32_t)dxs->dxs_burst_size) << DCMD_SIZE_SHIFT);

	switch (dxs->dxs_flow) {
	case DMAC_FLOW_CTRL_NONE:
		break;
	case DMAC_FLOW_CTRL_SRC:
		dxs->dxs_dcmd |= DCMD_FLOWSRC;
		break;
	case DMAC_FLOW_CTRL_DEST:
		dxs->dxs_dcmd |= DCMD_FLOWTRG;
		break;
	}

	if (src->xd_addr_hold == false)
		dxs->dxs_dcmd |= DCMD_INCSRCADDR;
	if (dst->xd_addr_hold == false)
		dxs->dxs_dcmd |= DCMD_INCTRGADDR;

	s = splbio();
	if (dxs->dxs_peripheral == DMAC_PERIPH_NONE ||
	    sc->sc_periph[dxs->dxs_peripheral].sp_busy == 0) {
		dxs->dxs_queue = &sc->sc_queue[DMAC_PRI(dxs->dxs_priority)];
		SIMPLEQ_INSERT_TAIL(dxs->dxs_queue, dxs, dxs_link);
		if (dxs->dxs_peripheral != DMAC_PERIPH_NONE)
			sc->sc_periph[dxs->dxs_peripheral].sp_busy++;
		dmac_start(sc, DMAC_PRI(dxs->dxs_priority));
	} else {
		dxs->dxs_queue = &sc->sc_periph[dxs->dxs_peripheral].sp_queue;
		SIMPLEQ_INSERT_TAIL(dxs->dxs_queue, dxs, dxs_link);
		sc->sc_periph[dxs->dxs_peripheral].sp_busy++;
	}
	splx(s);

	return (0);
}

void
pxa2x0_dmac_abort_xfer(struct dmac_xfer *dx)
{
	struct pxadmac_softc *sc = pxadmac_sc;
	struct dmac_xfer_state *ndxs, *dxs = (struct dmac_xfer_state *)dx;
	struct dmac_desc *desc, *ndesc;
	struct dmac_xfer_state_head *queue;
	uint32_t rv;
	int s, timeout, need_start = 0;

	s = splbio();

	queue = dxs->dxs_queue;

	if (dxs->dxs_channel == DMAC_NO_CHANNEL) {
		/*
		 * The request has not yet started, or it has already
		 * completed. If the request is not on a queue, just
		 * return.
		 */
		if (queue == NULL) {
			splx(s);
			return;
		}

		dxs->dxs_queue = NULL;
		SIMPLEQ_REMOVE(queue, dxs, dmac_xfer_state, dxs_link);
	} else {
		/*
		 * The request is in progress. This is a bit trickier.
		 */
		dmac_reg_write(sc, DMAC_DCSR(dxs->dxs_channel), 0);

		for (timeout = 5000; timeout; timeout--) {
			rv = dmac_reg_read(sc, DMAC_DCSR(dxs->dxs_channel));
			if (rv & DCSR_STOPSTATE)
				break;
			delay(1);
		}

		if ((rv & DCSR_STOPSTATE) == 0)
			panic(
			   "pxa2x0_dmac_abort_xfer: channel %d failed to abort",
			    dxs->dxs_channel);

		/*
		 * Free resources allocated to the request
		 */
		for (desc = SLIST_FIRST(&dxs->dxs_descs); desc; desc = ndesc) {
			ndesc = SLIST_NEXT(desc, d_link);
			SLIST_INSERT_HEAD(&sc->sc_descs, desc, d_link);
			sc->sc_free_descs++;
		}

		sc->sc_active[dxs->dxs_channel] = NULL;
		dmac_free_channel(sc, DMAC_PRI(dxs->dxs_priority),
		    dxs->dxs_channel);

		if (dxs->dxs_peripheral != DMAC_PERIPH_NONE)
			dmac_reg_write(sc, DMAC_DRCMR(dxs->dxs_peripheral), 0);

		need_start = 1;
		dxs->dxs_queue = NULL;
	}

	if (dxs->dxs_peripheral == DMAC_PERIPH_NONE ||
	    sc->sc_periph[dxs->dxs_peripheral].sp_busy-- == 1 ||
	    queue == &sc->sc_periph[dxs->dxs_peripheral].sp_queue)
		goto out;

	/*
	 * We've just removed the current item for this
	 * peripheral, and there is at least one more
	 * pending item waiting. Make it current.
	 */
	ndxs = SIMPLEQ_FIRST(&sc->sc_periph[dxs->dxs_peripheral].sp_queue);
	dxs = ndxs;
	KDASSERT(dxs != NULL);
	SIMPLEQ_REMOVE_HEAD(&sc->sc_periph[dxs->dxs_peripheral].sp_queue,
	    dxs_link);

	dxs->dxs_queue = &sc->sc_queue[DMAC_PRI(dxs->dxs_priority)];
	SIMPLEQ_INSERT_TAIL(dxs->dxs_queue, dxs, dxs_link);
	need_start = 1;

	/*
	 * Try to start any pending requests with the same
	 * priority.
	 */
out:
	if (need_start)
		dmac_start(sc, DMAC_PRI(dxs->dxs_priority));
	splx(s);
}

static void
dmac_start(struct pxadmac_softc *sc, dmac_priority_t priority)
{
	struct dmac_xfer_state *dxs;
	u_int channel;

	while (sc->sc_free_descs &&
	    (dxs = SIMPLEQ_FIRST(&sc->sc_queue[priority])) != NULL &&
	    dmac_allocate_channel(sc, priority, &channel) == 0) {
		/*
		 * Yay, got some descriptors, a transfer request, and
		 * an available DMA channel.
		 */
		KDASSERT(sc->sc_active[channel] == NULL);
		SIMPLEQ_REMOVE_HEAD(&sc->sc_queue[priority], dxs_link);

		/* set DMA alignment register */
		if (CPU_IS_PXA270) {
			uint32_t dalgn;

			dalgn = dmac_reg_read(sc, DMAC_DALGN);
			dalgn &= ~(1U << channel);
			if (dxs->dxs_misaligned_flag)
				dalgn |= (1U << channel);
			dmac_reg_write(sc, DMAC_DALGN, dalgn);
		}

		dxs->dxs_channel = channel;
		sc->sc_active[channel] = dxs;
		(void) dmac_continue_xfer(sc, dxs);
		/*
		 * XXX: Deal with descriptor allocation failure for loops
		 */
	}
}

static int
dmac_continue_xfer(struct pxadmac_softc *sc, struct dmac_xfer_state *dxs)
{
	struct dmac_desc *desc, *prev_desc;
	struct pxa2x0_dma_desc *dd;
	struct dmac_desc_segs *src_ds, *dst_ds;
	struct dmac_xfer_desc *src_xd, *dst_xd;
	bus_dma_segment_t *src_seg, *dst_seg;
	bus_addr_t src_mem_addr, dst_mem_addr;
	bus_size_t src_size, dst_size, this_size;

	desc = NULL;
	prev_desc = NULL;
	dd = NULL;
	src_ds = &dxs->dxs_segs[DMAC_DESC_SRC];
	dst_ds = &dxs->dxs_segs[DMAC_DESC_DST];
	src_xd = &dxs->dxs_desc[DMAC_DESC_SRC];
	dst_xd = &dxs->dxs_desc[DMAC_DESC_DST];
	SLIST_INIT(&dxs->dxs_descs);

	/*
	 * As long as the source/destination buffers have DMA segments,
	 * and we have free descriptors, build a DMA chain.
	 */
	while (src_ds->ds_nsegs && dst_ds->ds_nsegs && sc->sc_free_descs) {
		src_seg = src_ds->ds_curseg;
		src_mem_addr = src_seg->ds_addr + src_ds->ds_offset;
		if (src_xd->xd_addr_hold == false &&
		    dxs->dxs_loop_notify != DMAC_DONT_LOOP)
			src_size = dxs->dxs_loop_notify;
		else
			src_size = src_seg->ds_len - src_ds->ds_offset;

		dst_seg = dst_ds->ds_curseg;
		dst_mem_addr = dst_seg->ds_addr + dst_ds->ds_offset;
		if (dst_xd->xd_addr_hold == false &&
		    dxs->dxs_loop_notify != DMAC_DONT_LOOP)
			dst_size = dxs->dxs_loop_notify;
		else
			dst_size = dst_seg->ds_len - dst_ds->ds_offset;

		/*
		 * We may need to split a source or destination segment
		 * across two or more DMAC descriptors.
		 */
		while (src_size && dst_size &&
		    (desc = SLIST_FIRST(&sc->sc_descs)) != NULL) {
			SLIST_REMOVE_HEAD(&sc->sc_descs, d_link);
			sc->sc_free_descs--;

			/*
			 * Decide how much data we're going to transfer
			 * using this DMAC descriptor.
			 */
			if (src_xd->xd_addr_hold)
				this_size = dst_size;
			else
			if (dst_xd->xd_addr_hold)
				this_size = src_size;
			else
				this_size = min(dst_size, src_size);

			/*
			 * But clamp the transfer size to the DMAC
			 * descriptor's maximum.
			 */
			this_size = min(this_size, DCMD_LENGTH_MASK & ~0x1f);

			/*
			 * Fill in the DMAC descriptor
			 */
			dd = desc->d_desc;
			dd->dd_dsadr = src_mem_addr;
			dd->dd_dtadr = dst_mem_addr;
			dd->dd_dcmd = dxs->dxs_dcmd | this_size;

			/*
			 * Link it into the chain
			 */
			if (prev_desc) {
				SLIST_INSERT_AFTER(prev_desc, desc, d_link);
				prev_desc->d_desc->dd_ddadr = desc->d_desc_pa;
			} else {
				SLIST_INSERT_HEAD(&dxs->dxs_descs, desc,
				    d_link);
			}
			prev_desc = desc;

			/*
			 * Update the source/destination pointers
			 */
			if (src_xd->xd_addr_hold == false) {
				src_size -= this_size;
				src_ds->ds_offset += this_size;
				if (src_ds->ds_offset == src_seg->ds_len) {
					KDASSERT(src_size == 0);
					src_ds->ds_curseg = ++src_seg;
					src_ds->ds_offset = 0;
					src_ds->ds_nsegs--;
				} else
					src_mem_addr += this_size;
			}

			if (dst_xd->xd_addr_hold == false) {
				dst_size -= this_size;
				dst_ds->ds_offset += this_size;
				if (dst_ds->ds_offset == dst_seg->ds_len) {
					KDASSERT(dst_size == 0);
					dst_ds->ds_curseg = ++dst_seg;
					dst_ds->ds_offset = 0;
					dst_ds->ds_nsegs--;
				} else
					dst_mem_addr += this_size;
			}
		}

		if (dxs->dxs_loop_notify != DMAC_DONT_LOOP) {
			/*
			 * We must be able to allocate descriptors for the
			 * entire loop. Otherwise, return them to the pool
			 * and bail.
			 */
			if (desc == NULL) {
				struct dmac_desc *ndesc;
				for (desc = SLIST_FIRST(&dxs->dxs_descs);
				    desc; desc = ndesc) {
					ndesc = SLIST_NEXT(desc, d_link);
					SLIST_INSERT_HEAD(&sc->sc_descs, desc,
					    d_link);
					sc->sc_free_descs++;
				}

				return (0);
			}

			KASSERT(dd != NULL);
			dd->dd_dcmd |= DCMD_ENDIRQEN;
		}
	}

	/*
	 * Did we manage to build a chain?
	 * If not, just return.
	 */
	if (dd == NULL)
		return (0);

	if (dxs->dxs_loop_notify == DMAC_DONT_LOOP) {
		dd->dd_dcmd |= DCMD_ENDIRQEN;
		dd->dd_ddadr = DMAC_DESC_LAST;
	} else
		dd->dd_ddadr = SLIST_FIRST(&dxs->dxs_descs)->d_desc_pa;

	if (dxs->dxs_peripheral != DMAC_PERIPH_NONE) {
		dmac_reg_write(sc, DMAC_DRCMR(dxs->dxs_peripheral),
		    dxs->dxs_channel | DRCMR_MAPVLD);
	}
	dmac_reg_write(sc, DMAC_DDADR(dxs->dxs_channel),
	    SLIST_FIRST(&dxs->dxs_descs)->d_desc_pa);
	dmac_reg_write(sc, DMAC_DCSR(dxs->dxs_channel),
	    DCSR_ENDINTR | DCSR_RUN);

	return (1);
}

static u_int
dmac_channel_intr(struct pxadmac_softc *sc, u_int channel)
{
	struct dmac_xfer_state *dxs;
	struct dmac_desc *desc, *ndesc;
	uint32_t dcsr;
	u_int rv = 0;

	dcsr = dmac_reg_read(sc, DMAC_DCSR(channel));
	dmac_reg_write(sc, DMAC_DCSR(channel), dcsr);
	if (dmac_reg_read(sc, DMAC_DCSR(channel)) & DCSR_STOPSTATE)
		dmac_reg_write(sc, DMAC_DCSR(channel), dcsr & ~DCSR_RUN);

	if ((dxs = sc->sc_active[channel]) == NULL) {
		aprint_error_dev(sc->sc_dev,
		    "Stray DMAC interrupt for unallocated channel %d\n",
		    channel);
		return (0);
	}

	/*
	 * Clear down the interrupt in the DMA Interrupt Register
	 */
	if (!CPU_IS_PXA270)
		dmac_reg_write(sc, DMAC_DINT, (1u << channel));

	/*
	 * If this is a looping request, invoke the 'done' callback and
	 * return immediately.
	 */
	if (dxs->dxs_loop_notify != DMAC_DONT_LOOP &&
	    (dcsr & DCSR_BUSERRINTR) == 0) {
		(dxs->dxs_done)(&dxs->dxs_xfer, 0);
		return (0);
	}

	/*
	 * Free the descriptors allocated to the completed transfer
	 *
	 * XXX: If there is more data to transfer in this request,
	 * we could simply reuse some or all of the descriptors
	 * already allocated for the transfer which just completed.
	 */
	for (desc = SLIST_FIRST(&dxs->dxs_descs); desc; desc = ndesc) {
		ndesc = SLIST_NEXT(desc, d_link);
		SLIST_INSERT_HEAD(&sc->sc_descs, desc, d_link);
		sc->sc_free_descs++;
	}

	if ((dcsr & DCSR_BUSERRINTR) || dmac_continue_xfer(sc, dxs) == 0) {
		/*
		 * The transfer completed (possibly due to an error),
		 * -OR- we were unable to continue any remaining
		 * segment of the transfer due to a lack of descriptors.
		 *
		 * In either case, we have to free up DMAC resources
		 * allocated to the request.
		 */
		sc->sc_active[channel] = NULL;
		dmac_free_channel(sc, DMAC_PRI(dxs->dxs_priority), channel);
		dxs->dxs_channel = DMAC_NO_CHANNEL;
		if (dxs->dxs_peripheral != DMAC_PERIPH_NONE)
			dmac_reg_write(sc, DMAC_DRCMR(dxs->dxs_peripheral), 0);

		if (dxs->dxs_segs[DMAC_DESC_SRC].ds_nsegs == 0 ||
		    dxs->dxs_segs[DMAC_DESC_DST].ds_nsegs == 0 ||
		    (dcsr & DCSR_BUSERRINTR)) {

			/*
			 * The transfer is complete.
			 */
			dxs->dxs_queue = NULL;
			rv = 1u << DMAC_PRI(dxs->dxs_priority);

			if (dxs->dxs_peripheral != DMAC_PERIPH_NONE &&
			    --sc->sc_periph[dxs->dxs_peripheral].sp_busy != 0) {
				struct dmac_xfer_state *ndxs;
				/*
				 * We've just removed the current item for this
				 * peripheral, and there is at least one more
				 * pending item waiting. Make it current.
				 */
				ndxs = SIMPLEQ_FIRST(
				  &sc->sc_periph[dxs->dxs_peripheral].sp_queue);
				KDASSERT(ndxs != NULL);
				SIMPLEQ_REMOVE_HEAD(
				   &sc->sc_periph[dxs->dxs_peripheral].sp_queue,
				    dxs_link);

				ndxs->dxs_queue =
				    &sc->sc_queue[DMAC_PRI(dxs->dxs_priority)];
				SIMPLEQ_INSERT_TAIL(ndxs->dxs_queue, ndxs,
				    dxs_link);
			}

			(dxs->dxs_done)(&dxs->dxs_xfer,
			    (dcsr & DCSR_BUSERRINTR) ? EFAULT : 0);
		} else {
			/*
			 * The request is not yet complete, but we were unable
			 * to make any headway at this time because there are
			 * no free descriptors. Put the request back at the
			 * head of the appropriate priority queue. It'll be
			 * dealt with as other in-progress transfers complete.
			 */
			SIMPLEQ_INSERT_HEAD(
			    &sc->sc_queue[DMAC_PRI(dxs->dxs_priority)], dxs,
			    dxs_link);
		}
	}

	return (rv);
}

static int
dmac_intr(void *arg)
{
	struct pxadmac_softc *sc = arg;
	uint32_t rv, mask;
	u_int chan, pri;

	rv = dmac_reg_read(sc, DMAC_DINT);
	if ((rv & DMAC_DINT_MASK) == 0)
		return (0);

	/*
	 * Deal with completed transfers
	 */
	for (chan = 0, mask = 1u, pri = 0;
	    chan < DMAC_N_CHANNELS; chan++, mask <<= 1) {
		if (rv & mask)
			pri |= dmac_channel_intr(sc, chan);
	}

	/*
	 * Now try to start any queued transfers
	 */
#if (DMAC_N_PRIORITIES > 1)
	if (pri & (1u << DMAC_PRIORITY_HIGH))
		dmac_start(sc, DMAC_PRIORITY_HIGH);
	if (pri & (1u << DMAC_PRIORITY_MED))
		dmac_start(sc, DMAC_PRIORITY_MED);
	if (pri & (1u << DMAC_PRIORITY_LOW))
		dmac_start(sc, DMAC_PRIORITY_LOW);
#else
	if (pri)
		dmac_start(sc, DMAC_PRIORITY_NORMAL);
#endif

	return (1);
}