xref: /src/sys/arch/powerpc/ibm4xx/dev/dwcsata.c

/*	$NetBSD: dwcsata.c,v 1.2 2026/06/15 19:21:14 rkujawa Exp $	*/

/*
 * Copyright (c) 2025, 2026 The NetBSD Foundation, Inc.
 * All rights reserved.
 *
 * This code is derived from software contributed to The NetBSD Foundation
 * by Radoslaw Kujawa.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

/*
 * 460EX on-chip Synopsys DWC SATA-II host
 *
 * SATA PHY shares its SerDes lane with PCIE0; firmware routes
 * and initializes it. Only one of these can work at a given time.
 *
 * DMA support is ugly, we mess with USB host controller schedule enables to
 * work around the DMAC FIFO. Honestly, this is just papering over the true
 * issue, which is that the DMAC FIFO drain is starved by the AHB bus-mastering.
 * Or at least, that's what it looks like, we may never know without an NDA
 * with company that does not exist anymore.
 *
 * The on-chip OHCI and EHCI bus-master their periodic/async schedules on
 * the shared AHB and starve the SATA DMAC FIFO drain.
 *
 * sysctl hw.dwcsata.usb_workaround tries to workaround DMAC stalls by pausing
 * USB for the duration of each SATA DMA burst:
 * 1=EHCI periodic 2=EHCI async 4=OHCI; 0=off 7=all
 *
 * Note that even with EHCI periodic schedule paused, the async schedule can
 * still cause stall, so don't be fooled into thinking that DMA is working
 * stable when USB is enabled. It will break sooner or later, depending on the
 * USB and SATA traffic.
 *
 * The only stable option is to either:
 * - disable USB entirely - then you can use DWC SATA DMA
 * - drive DWC SATA in PIO mode - then you can use USB and DWC SATA at the same time
 */

#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: dwcsata.c,v 1.2 2026/06/15 19:21:14 rkujawa Exp $");

#ifdef _KERNEL_OPT
#include "opt_dwcsata.h"
#endif

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/device.h>
#include <sys/extent.h>
#include <sys/bus.h>
#include <sys/sysctl.h>

#include <powerpc/ibm4xx/cpu.h>
#include <powerpc/ibm4xx/amcc460ex.h>
#include <powerpc/ibm4xx/dev/plbvar.h>
#include <powerpc/ibm4xx/dev/dwcsatareg.h>
#include <powerpc/ibm4xx/dev/dwcdmacreg.h>

#include <dev/ata/atavar.h>
#include <dev/ata/atareg.h>
#include <dev/ic/wdcreg.h>
#include <dev/ic/wdcvar.h>

#include "locators.h"

/* the DMAC channel wired to the SATA core's handshake interface */
#define	DWCSATA_DMACH		0
/*
 * Burst sizing (CTL SRC/DST_MSIZE encoding).
 */
#define	DWCSATA_DMA_MEM_MSIZE	3	/* 16 items = 64 bytes, memory side */
#define	DWCSATA_DMA_FIFO_MSIZE	0	/* single DWORD, FIFO side */
/* matching FIFO-side burst in bytes for the SATA core's DBTSR */
#define	DWCSATA_DMA_FIFO_BURST	4	/* single DWORD */
/*
 * Worst case for a MAXPHYS transfer is one LLI per map segment plus
 * one extra per 8KB FIS boundary split; 64 is comfortably above both.
 */
#define	DWCSATA_NLLI		64

#ifndef DWCSATA_PIO_ONLY
#define	DWCSATA_USB_OFFSET	0x1000
#define	DWCSATA_USB_SIZE	0x800
#define	DWCSATA_OHCI_HCCONTROL	0x0004
#define	 DWCSATA_OHCI_PLE	0x00000004
#define	 DWCSATA_OHCI_CLE	0x00000010
#define	 DWCSATA_OHCI_BLE	0x00000020
#define	DWCSATA_EHCI_CAPLEN	0x0400	/* EHCI base; USBCMD = +CAPLENGTH */
#define	 DWCSATA_EHCI_PSE	0x00000010
#define	 DWCSATA_EHCI_ASE	0x00000020
#define	DWCSATA_Q_EHCI_PSE	0x1
#define	DWCSATA_Q_EHCI_ASE	0x2
#define	DWCSATA_Q_OHCI		0x4
/* SATA PHY soft-reset registers (SDR space), for dwcsata_unwedge() */
#define	SATA_PESDR0_L0CDRCTL	0x030a
#define	SATA_PESDR0_L0DRV	0x030b
#define	SATA_PESDR0_L0CLK	0x030e
#define	SATA_PESDR0_PHY_CTL_RST	0x030f
#endif

struct dwcsata_softc {
	struct wdc_softc sc_wdcdev;
	struct ata_channel *sc_chanlist[1];
	struct ata_channel sc_channel;
	struct wdc_regs sc_wdc_regs;

	bus_space_tag_t sc_iot;
	bus_space_handle_t sc_ioh;

	bus_dma_tag_t sc_dmat;
	int sc_irq;		/* for interrupt-driven operation, later */

#ifndef DWCSATA_PIO_ONLY
	bus_space_handle_t sc_dmac_ioh;	/* companion AHB DMAC */
	bus_dmamap_t sc_dmamap_xfer;	/* current data buffer */
	struct dwcdmac_lli *sc_lli;	/* uncached LLI table */
	bus_addr_t sc_lli_phys;
	bus_dmamap_t sc_lli_map;
	bus_dma_segment_t sc_lli_seg;
	int sc_lli_nseg;
	uint32_t sc_dmadr_phys;		/* AHB address of the DMADR window */
	int sc_dma_flags;		/* WDC_DMA_* of the loaded transfer */
	int sc_dma_nlli;		/* LLIs in the loaded chain */
	bool sc_dma_loaded;		/* sc_dmamap_xfer is loaded */
	bool sc_dma_ok;			/* resources up, hooks registered */
	bool sc_dma_active;		/* between dma_start and teardown */

	/* USB schedule-quiesce */
	bus_space_tag_t sc_usb_iot;
	bus_space_handle_t sc_usb_ioh;
	bus_size_t sc_ehci_cmd_off;
	bool sc_usb_mapped;
	bool sc_usb_quiesced;
	bool sc_usb_q_ehci;
	bool sc_usb_q_ohci;
	uint32_t sc_usb_ohci_ctl;
	uint32_t sc_usb_ehci_cmd;
#endif
};

static int	dwcsata_match(device_t, cfdata_t, void *);
static void	dwcsata_attach(device_t, device_t, void *);
static void	dwcsata_probe(struct ata_channel *);
static void	dwcsata_reset(struct ata_channel *, int);

#ifndef DWCSATA_PIO_ONLY
CTASSERT(sizeof(struct dwcdmac_lli) == 28);

static bool	dwcsata_dma_setup(device_t, struct dwcsata_softc *,
		    struct plb_attach_args *);
static int	dwcsata_dma_init(void *, int, int, void *, size_t, int);
static void	dwcsata_dma_start(void *, int, int);
static int	dwcsata_dma_finish(void *, int, int, int);
static int	dwcsata_intr(void *);
static int	dwcsata_dmac_intr(void *);
static void	dwcsata_unwedge(struct dwcsata_softc *);
static void	dwcsata_usb_quiesce(struct dwcsata_softc *);
static void	dwcsata_usb_restore(struct dwcsata_softc *);
#endif

CFATTACH_DECL_NEW(dwcsata, sizeof(struct dwcsata_softc),
    dwcsata_match, dwcsata_attach, NULL, NULL);

static struct powerpc_bus_space dwcsata_tag = {
	_BUS_SPACE_LITTLE_ENDIAN | _BUS_SPACE_MEM_TYPE,
	0x00000000,
};
static char dwcsata_ex_storage[EXTENT_FIXED_STORAGE_SIZE(8)]
    __attribute__((aligned(8)));

#ifndef DWCSATA_PIO_ONLY
/* separate little-endian tag for the OHCI/EHCI host window (below the core) */
static struct powerpc_bus_space dwcsata_usb_tag = {
	_BUS_SPACE_LITTLE_ENDIAN | _BUS_SPACE_MEM_TYPE,
	0x00000000,
};
static char dwcsata_usb_ex_storage[EXTENT_FIXED_STORAGE_SIZE(8)]
    __attribute__((aligned(8)));
static int dwcsata_usb_workaround = 0;	/* hw.dwcsata.usb_workaround */
#endif

static int
dwcsata_match(device_t parent, cfdata_t match, void *aux)
{
	struct plb_attach_args *paa = aux;

	if (strcmp(paa->plb_name, match->cf_name) != 0)
		return 0;

	if (match->cf_loc[PLBCF_ADDR] == PLBCF_ADDR_DEFAULT)
		panic("dwcsata_match: wildcard addr not allowed");
	if (match->cf_loc[PLBCF_IRQ] == PLBCF_IRQ_DEFAULT)
		panic("dwcsata_match: wildcard IRQ not allowed");

	paa->plb_addr = match->cf_loc[PLBCF_ADDR];
	paa->plb_irq = match->cf_loc[PLBCF_IRQ];
	return 1;
}

#ifndef DWCSATA_PIO_ONLY

static inline uint32_t
dwcdmac_read(struct dwcsata_softc *sc, bus_size_t reg)
{

	return bus_space_read_4(sc->sc_iot, sc->sc_dmac_ioh, reg);
}

static inline void
dwcdmac_write(struct dwcsata_softc *sc, bus_size_t reg, uint32_t val)
{

	bus_space_write_4(sc->sc_iot, sc->sc_dmac_ioh, reg, val);
}

static void
dwcdmac_clear_intrs(struct dwcsata_softc *sc)
{

	dwcdmac_write(sc, DWCDMAC_CLEAR_TFR, DWCDMAC_CHANBIT(DWCSATA_DMACH));
	dwcdmac_write(sc, DWCDMAC_CLEAR_BLOCK, DWCDMAC_CHANBIT(DWCSATA_DMACH));
	dwcdmac_write(sc, DWCDMAC_CLEAR_SRCTRAN,
	    DWCDMAC_CHANBIT(DWCSATA_DMACH));
	dwcdmac_write(sc, DWCDMAC_CLEAR_DSTTRAN,
	    DWCDMAC_CHANBIT(DWCSATA_DMACH));
	dwcdmac_write(sc, DWCDMAC_CLEAR_ERR, DWCDMAC_CHANBIT(DWCSATA_DMACH));
}

static bool
dwcsata_dma_setup(device_t self, struct dwcsata_softc *sc,
    struct plb_attach_args *paa)
{
	int error;

	if (bus_space_map(sc->sc_iot, paa->plb_addr - DWCDMAC_OFFSET,
	    DWCDMAC_SIZE, 0, &sc->sc_dmac_ioh)) {
		aprint_error_dev(self, "can't map the AHB DMAC\n");
		return false;
	}

	aprint_debug_dev(self, "AHB DMAC id 0x%08x\n",
	    dwcdmac_read(sc, DWCDMAC_ID));

	/*
	 * 460EX errata workaround for concurrent AHB use
	 */
	{
		uint32_t ahbcfg = mfsdr(DCR_SDR0_AHB_CFG);

		aprint_normal_dev(self, "SDR0_AHB_CFG 0x%08x", ahbcfg);
		ahbcfg |= SDR0_AHB_CFG_A2P_INCR4;
		ahbcfg &= ~SDR0_AHB_CFG_A2P_PROT2;
		mtsdr(DCR_SDR0_AHB_CFG, ahbcfg);
		aprint_normal(" -> 0x%08x (460EX AHB errata)\n",
		    mfsdr(DCR_SDR0_AHB_CFG));

		aprint_normal_dev(self,
		    "arbiter: PLB4A0_ACR 0x%08x PLB4A1_ACR 0x%08x "
		    "SDR0_USB2HOST_CFG 0x%08x\n",
		    mfdcr(DCR_PLB4A0_ACR), mfdcr(DCR_PLB4A1_ACR),
		    mfsdr(DCR_SDR0_USB2HOST_CFG));
	}

	/* map the on-chip OHCI + EHCI host-control registers for the USB
	 * schedule-quiesce; EHCI operational regs are at base + CAPLENGTH */
	dwcsata_usb_tag.pbs_base = paa->plb_addr - DWCSATA_USB_OFFSET;
	dwcsata_usb_tag.pbs_limit = paa->plb_addr - DWCSATA_USB_OFFSET +
	    DWCSATA_USB_SIZE;
	if (bus_space_init(&dwcsata_usb_tag, "dwcsataub", dwcsata_usb_ex_storage,
	      sizeof(dwcsata_usb_ex_storage)) == 0 &&
	    bus_space_map(&dwcsata_usb_tag, paa->plb_addr - DWCSATA_USB_OFFSET,
	      DWCSATA_USB_SIZE, 0, &sc->sc_usb_ioh) == 0) {
		uint8_t caplen = bus_space_read_1(&dwcsata_usb_tag,
		    sc->sc_usb_ioh, DWCSATA_EHCI_CAPLEN);
		sc->sc_usb_iot = &dwcsata_usb_tag;
		sc->sc_ehci_cmd_off = DWCSATA_EHCI_CAPLEN + caplen;
		sc->sc_usb_mapped = true;
	} else
		sc->sc_usb_mapped = false;

	dwcdmac_write(sc, DWCDMAC_MASK_TFR, DWCDMAC_CH_ENABLE(DWCSATA_DMACH));
	dwcdmac_write(sc, DWCDMAC_MASK_BLOCK,
	    DWCDMAC_CH_DISABLE(DWCSATA_DMACH));
	dwcdmac_write(sc, DWCDMAC_MASK_SRCTRAN,
	    DWCDMAC_CH_DISABLE(DWCSATA_DMACH));
	dwcdmac_write(sc, DWCDMAC_MASK_DSTTRAN,
	    DWCDMAC_CH_DISABLE(DWCSATA_DMACH));
	dwcdmac_write(sc, DWCDMAC_MASK_ERR, DWCDMAC_CH_ENABLE(DWCSATA_DMACH));
	dwcdmac_clear_intrs(sc);
	dwcdmac_write(sc, DWCDMAC_CHEN, DWCDMAC_CH_DISABLE(DWCSATA_DMACH));
	dwcdmac_write(sc, DWCDMAC_DMACFG, DWCDMAC_DMACFG_EN);

	/* burst sizes the LLIs are built for */
	bus_space_write_4(sc->sc_iot, sc->sc_ioh, DWCSATA_DBTSR,
	    DWCSATA_DBTSR_MWR(DWCSATA_DMA_FIFO_BURST) |
	    DWCSATA_DBTSR_MRD(DWCSATA_DMA_FIFO_BURST));

	sc->sc_dmadr_phys = DWCSATA_DMADR_PHYS(paa->plb_addr);

	/*
	 * The LLI table is fetched by the DMAC behind the CPU's back...
	 * map it uncached so building it needs no cache gymnastics.
	 */
	error = bus_dmamem_alloc(sc->sc_dmat,
	    DWCSATA_NLLI * sizeof(struct dwcdmac_lli), 8, 0,
	    &sc->sc_lli_seg, 1, &sc->sc_lli_nseg, BUS_DMA_NOWAIT);
	if (error)
		goto fail0;
	error = bus_dmamem_map(sc->sc_dmat, &sc->sc_lli_seg, sc->sc_lli_nseg,
	    DWCSATA_NLLI * sizeof(struct dwcdmac_lli), (void **)&sc->sc_lli,
	    BUS_DMA_NOWAIT | BUS_DMA_DONTCACHE);
	if (error)
		goto fail1;
	error = bus_dmamap_create(sc->sc_dmat,
	    DWCSATA_NLLI * sizeof(struct dwcdmac_lli), 1,
	    DWCSATA_NLLI * sizeof(struct dwcdmac_lli), 0, BUS_DMA_NOWAIT,
	    &sc->sc_lli_map);
	if (error)
		goto fail2;
	error = bus_dmamap_load(sc->sc_dmat, sc->sc_lli_map, sc->sc_lli,
	    DWCSATA_NLLI * sizeof(struct dwcdmac_lli), NULL, BUS_DMA_NOWAIT);
	if (error)
		goto fail3;
	sc->sc_lli_phys = sc->sc_lli_map->dm_segs[0].ds_addr;

	error = bus_dmamap_create(sc->sc_dmat, MAXPHYS,
	    MAXPHYS / PAGE_SIZE + 8, 0x2000, 0x2000, BUS_DMA_NOWAIT,
	    &sc->sc_dmamap_xfer);
	if (error)
		goto fail4;

	return true;

fail4:
	bus_dmamap_unload(sc->sc_dmat, sc->sc_lli_map);
fail3:
	bus_dmamap_destroy(sc->sc_dmat, sc->sc_lli_map);
fail2:
	bus_dmamem_unmap(sc->sc_dmat, sc->sc_lli,
	    DWCSATA_NLLI * sizeof(struct dwcdmac_lli));
fail1:
	bus_dmamem_free(sc->sc_dmat, &sc->sc_lli_seg, sc->sc_lli_nseg);
fail0:
	bus_space_unmap(sc->sc_iot, sc->sc_dmac_ioh, DWCDMAC_SIZE);
	return false;
}

static int
dwcsata_dma_init(void *v, int channel, int drive, void *databuf,
    size_t datalen, int flags)
{
	struct dwcsata_softc *sc = v;
	bus_dmamap_t map = sc->sc_dmamap_xfer;
	const bool read = (flags & WDC_DMA_READ) != 0;
	const u_int lsize = curcpu()->ci_ci.dcache_line_size;
	uint32_t ctl, serror;
	u_int idx, fis_len;
	int i, error;

	KASSERT(channel == 0 && drive == 0);

	/*
	 * The channel enable self-clears at the end of the chain; if it
	 * is still set here, a previous abort failed to stop the engine.
	 */
	if (dwcdmac_read(sc, DWCDMAC_CHEN) & DWCDMAC_CHANBIT(DWCSATA_DMACH)) {
		aprint_error_dev(sc->sc_wdcdev.sc_atac.atac_dev,
		    "dma_init: channel still enabled (CHEN 0x%08x)\n",
		    dwcdmac_read(sc, DWCDMAC_CHEN));
		return EBUSY;
	}

	/*
	 * If the MI code bailed between dma_init and dma_start (e.g. a
	 * not-ready timeout) it never calls dma_finish, leaking the
	 * loaded map; recover instead of failing every later load.
	 */
	if (sc->sc_dma_loaded) {
		aprint_error_dev(sc->sc_wdcdev.sc_atac.atac_dev,
		    "dma_init: recovering leaked dmamap\n");
		bus_dmamap_unload(sc->sc_dmat, map);
		sc->sc_dma_loaded = false;
	}

	/* drop any latched SError bits before issuing the command */
	serror = bus_space_read_4(sc->sc_iot, sc->sc_ioh, DWCSATA_SERROR);
	if (serror & DWCSATA_ERRMR_ERR_BITS)
		bus_space_write_4(sc->sc_iot, sc->sc_ioh, DWCSATA_SERROR,
		    serror);

	error = bus_dmamap_load(sc->sc_dmat, map, databuf, datalen, NULL,
	    BUS_DMA_NOWAIT | BUS_DMA_STREAMING |
	    (read ? BUS_DMA_READ : BUS_DMA_WRITE));
	if (error) {
		aprint_error_dev(sc->sc_wdcdev.sc_atac.atac_dev,
		    "dma_init: dmamap_load failed: error %d "
		    "(buf %p len %zu)\n", error, databuf, datalen);
		return error;
	}
	sc->sc_dma_loaded = true;

	/*
	 * The DMAC moves 32-bit items, so segments must be word-aligned.
	 */
	for (i = 0; i < map->dm_nsegs; i++) {
		if ((map->dm_segs[i].ds_addr | map->dm_segs[i].ds_len) & 3)
			goto fallback;
		if (read && ((map->dm_segs[i].ds_addr |
		    map->dm_segs[i].ds_len) & (lsize - 1)))
			goto fallback;
	}

	bus_dmamap_sync(sc->sc_dmat, map, 0, datalen,
	    read ? BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE);

	if (read) {
		/* device->memory: source is the FIFO */
		ctl = DWCDMAC_CTL_TTFC(DWCDMAC_TTFC_P2M_DMAC) |
		    DWCDMAC_CTL_SMS(DWCDMAC_MS_PERIPH) |
		    DWCDMAC_CTL_DMS(DWCDMAC_MS_MEM) |
		    DWCDMAC_CTL_SINC_NOCHANGE |
		    DWCDMAC_CTL_SRC_MSIZE(DWCSATA_DMA_FIFO_MSIZE) |
		    DWCDMAC_CTL_DST_MSIZE(DWCSATA_DMA_MEM_MSIZE);
	} else {
		/* memory->device: destination is the FIFO */
		ctl = DWCDMAC_CTL_TTFC(DWCDMAC_TTFC_M2P_PER) |
		    DWCDMAC_CTL_SMS(DWCDMAC_MS_MEM) |
		    DWCDMAC_CTL_DMS(DWCDMAC_MS_PERIPH) |
		    DWCDMAC_CTL_DINC_NOCHANGE |
		    DWCDMAC_CTL_SRC_MSIZE(DWCSATA_DMA_MEM_MSIZE) |
		    DWCDMAC_CTL_DST_MSIZE(DWCSATA_DMA_FIFO_MSIZE);
	}
	ctl |= DWCDMAC_CTL_SRC_TRWID(2) | DWCDMAC_CTL_DST_TRWID(2) |
	    DWCDMAC_CTL_INT_EN |
	    DWCDMAC_CTL_LLP_SRC_EN | DWCDMAC_CTL_LLP_DST_EN;

	idx = 0;
	fis_len = 0;
	for (i = 0; i < map->dm_nsegs; i++) {
		uint32_t addr = map->dm_segs[i].ds_addr;
		bus_size_t left = map->dm_segs[i].ds_len;

		while (left > 0) {
			struct dwcdmac_lli *lli;
			uint32_t len;

			if (idx >= DWCSATA_NLLI) {
				aprint_error_dev(
				    sc->sc_wdcdev.sc_atac.atac_dev,
				    "dma_init: LLI overflow (len %zu, "
				    "%d segs)\n", datalen, map->dm_nsegs);
				goto fallback;
			}

			len = uimin(left, DWCDMAC_MAX_BLOCK_ITEMS * 4);
			if (fis_len + len > 8192) {
				len = 8192 - fis_len;
				fis_len = 0;
			} else {
				fis_len += len;
				if (fis_len == 8192)
					fis_len = 0;
			}

			lli = &sc->sc_lli[idx];
			if (read) {
				lli->sar = htole32(sc->sc_dmadr_phys);
				lli->dar = htole32(addr);
			} else {
				lli->sar = htole32(addr);
				lli->dar = htole32(sc->sc_dmadr_phys);
			}
			lli->llp = htole32((sc->sc_lli_phys +
			    (idx + 1) * sizeof(struct dwcdmac_lli)) |
			    DWCDMAC_MS_MEM);
			lli->ctl_lo = htole32(ctl);
			lli->ctl_hi = htole32(DWCDMAC_CTL_BLOCK_TS(len / 4));
			lli->dstat_lo = 0;
			lli->dstat_hi = 0;

			idx++;
			addr += len;
			left -= len;
		}
	}
	KASSERT(idx > 0);
	sc->sc_lli[idx - 1].llp = 0;
	sc->sc_lli[idx - 1].ctl_lo &=
	    htole32(~(DWCDMAC_CTL_LLP_SRC_EN | DWCDMAC_CTL_LLP_DST_EN));
	/* the table is uncached; order the stores before the enables */
	__asm volatile("sync" ::: "memory");

	dwcdmac_clear_intrs(sc);
	dwcdmac_write(sc, DWCDMAC_CFG_HI(DWCSATA_DMACH),
	    DWCDMAC_CFG_HS_SRC(DWCSATA_DMACH) |
	    DWCDMAC_CFG_HS_DST(DWCSATA_DMACH) |
	    DWCDMAC_CFG_PROTCTL | DWCDMAC_CFG_FCMODE_REQ);
	dwcdmac_write(sc, DWCDMAC_CFG(DWCSATA_DMACH),
	    DWCDMAC_CFG_CH_PRIOR(DWCSATA_DMACH));
	dwcdmac_write(sc, DWCDMAC_LLP(DWCSATA_DMACH),
	    sc->sc_lli_phys | DWCDMAC_MS_MEM);
	dwcdmac_write(sc, DWCDMAC_CTL(DWCSATA_DMACH),
	    DWCDMAC_CTL_LLP_SRC_EN | DWCDMAC_CTL_LLP_DST_EN);
	dwcdmac_write(sc, DWCDMAC_CTL_HI(DWCSATA_DMACH), 0);

	sc->sc_dma_flags = flags;
	sc->sc_dma_nlli = idx;
	return 0;

fallback:
	bus_dmamap_unload(sc->sc_dmat, map);
	sc->sc_dma_loaded = false;
	return EINVAL;
}

static void
dwcsata_dma_dump(struct dwcsata_softc *sc, uint32_t tfr, uint32_t err)
{
	device_t dev = sc->sc_wdcdev.sc_atac.atac_dev;
	uint32_t intpr;
	int i;

	intpr = bus_space_read_4(sc->sc_iot, sc->sc_ioh, DWCSATA_INTPR);
	aprint_error_dev(dev,
	    "DMA state: CHEN 0x%08x tfr/blk/err 0x%08x/0x%08x/0x%08x "
	    "CTL 0x%08x.%08x LLP 0x%08x CFG 0x%08x.%08x DMACR 0x%08x "
	    "INTPR 0x%08x (erraddr 0x%03x) SError 0x%08x\n",
	    dwcdmac_read(sc, DWCDMAC_CHEN), tfr,
	    dwcdmac_read(sc, DWCDMAC_RAW_BLOCK), err,
	    dwcdmac_read(sc, DWCDMAC_CTL_HI(DWCSATA_DMACH)),
	    dwcdmac_read(sc, DWCDMAC_CTL(DWCSATA_DMACH)),
	    dwcdmac_read(sc, DWCDMAC_LLP(DWCSATA_DMACH)),
	    dwcdmac_read(sc, DWCDMAC_CFG_HI(DWCSATA_DMACH)),
	    dwcdmac_read(sc, DWCDMAC_CFG(DWCSATA_DMACH)),
	    bus_space_read_4(sc->sc_iot, sc->sc_ioh, DWCSATA_DMACR),
	    intpr, DWCSATA_INTPR_ERRADDR_GET(intpr),
	    bus_space_read_4(sc->sc_iot, sc->sc_ioh, DWCSATA_SERROR));
	for (i = 0; i < sc->sc_dma_nlli; i++) {
		const struct dwcdmac_lli *lli = &sc->sc_lli[i];

		aprint_error_dev(dev,
		    "  lli[%d]: sar 0x%08x dar 0x%08x llp 0x%08x "
		    "ctl 0x%08x.%08x\n", i,
		    le32toh(lli->sar), le32toh(lli->dar), le32toh(lli->llp),
		    le32toh(lli->ctl_hi), le32toh(lli->ctl_lo));
	}
}

/*
 * SDR/DCR SATA reset
 *
 * Tries to recover a hung DMAC channel without touching the poisoned AHB
 * master, so the abort path can proceed without machine check (drop to
 * DDB or panic).
 */
static void
dwcsata_unwedge(struct dwcsata_softc *sc)
{
	uint32_t reg;

	mtsdr(DCR_SDR0_SRST1, SDR0_SRST1_SATA_RESET);
	reg = mfsdr(SATA_PESDR0_PHY_CTL_RST);
	mtsdr(SATA_PESDR0_PHY_CTL_RST, (reg & 0xeffffffc) | 0x00000001);
	reg = mfsdr(SATA_PESDR0_L0CLK);
	mtsdr(SATA_PESDR0_L0CLK, (reg & 0xfffffff8) | 0x00000007);
	mtsdr(SATA_PESDR0_L0CDRCTL, 0x00003111);
	mtsdr(SATA_PESDR0_L0DRV, 0x00000104);
	mtsdr(DCR_SDR0_SRST1, 0);
	delay(10000);
	reg = mfsdr(DCR_SDR0_AHB_CFG);	/* errata is lost across the reset */
	mtsdr(DCR_SDR0_AHB_CFG,
	    (reg | SDR0_AHB_CFG_A2P_INCR4) & ~SDR0_AHB_CFG_A2P_PROT2);
	(void)sc;
}

/* clear the selected USB schedule-enable bits; keep the controllers RUNning */
static void
dwcsata_usb_quiesce(struct dwcsata_softc *sc)
{
	bus_space_tag_t t = sc->sc_usb_iot;
	bus_space_handle_t h = sc->sc_usb_ioh;
	const int mask = dwcsata_usb_workaround;
	uint32_t ehci_clr, ohci_clr;

	if (mask == 0 || !sc->sc_usb_mapped || sc->sc_usb_quiesced)
		return;
	ehci_clr = ((mask & DWCSATA_Q_EHCI_PSE) ? DWCSATA_EHCI_PSE : 0) |
		   ((mask & DWCSATA_Q_EHCI_ASE) ? DWCSATA_EHCI_ASE : 0);
	ohci_clr = (mask & DWCSATA_Q_OHCI) ?
	    (DWCSATA_OHCI_PLE | DWCSATA_OHCI_CLE | DWCSATA_OHCI_BLE) : 0;
	if (ohci_clr != 0) {
		sc->sc_usb_ohci_ctl =
		    bus_space_read_4(t, h, DWCSATA_OHCI_HCCONTROL);
		bus_space_write_4(t, h, DWCSATA_OHCI_HCCONTROL,
		    sc->sc_usb_ohci_ctl & ~ohci_clr);
		sc->sc_usb_q_ohci = true;
	}
	if (ehci_clr != 0) {
		sc->sc_usb_ehci_cmd =
		    bus_space_read_4(t, h, sc->sc_ehci_cmd_off);
		bus_space_write_4(t, h, sc->sc_ehci_cmd_off,
		    sc->sc_usb_ehci_cmd & ~ehci_clr);
		sc->sc_usb_q_ehci = true;
	}
	sc->sc_usb_quiesced = true;
}

static void
dwcsata_usb_restore(struct dwcsata_softc *sc)
{
	if (!sc->sc_usb_quiesced)
		return;
	if (sc->sc_usb_q_ohci) {
		bus_space_write_4(sc->sc_usb_iot, sc->sc_usb_ioh,
		    DWCSATA_OHCI_HCCONTROL, sc->sc_usb_ohci_ctl);
		sc->sc_usb_q_ohci = false;
	}
	if (sc->sc_usb_q_ehci) {
		bus_space_write_4(sc->sc_usb_iot, sc->sc_usb_ioh,
		    sc->sc_ehci_cmd_off, sc->sc_usb_ehci_cmd);
		sc->sc_usb_q_ehci = false;
	}
	sc->sc_usb_quiesced = false;
}

static void
dwcsata_dma_start(void *v, int channel, int drive)
{
	struct dwcsata_softc *sc = v;
	const bool read = (sc->sc_dma_flags & WDC_DMA_READ) != 0;

	dwcsata_usb_quiesce(sc);

	/* open the FIS data channel in the SATA core first... */
	bus_space_write_4(sc->sc_iot, sc->sc_ioh, DWCSATA_DMACR,
	    read ? DWCSATA_DMACR_RX_START : DWCSATA_DMACR_TX_START);
	/* ...then let the AHB DMAC run the linked list */
	dwcdmac_write(sc, DWCDMAC_CHEN, DWCDMAC_CH_ENABLE(DWCSATA_DMACH));
	sc->sc_dma_active = true;
}

static int
dwcsata_dma_finish(void *v, int channel, int drive, int force)
{
	struct dwcsata_softc *sc = v;
	const bool read = (sc->sc_dma_flags & WDC_DMA_READ) != 0;
	uint32_t tfr, err, dmacr, intpr;
	int i, status = 0;

	if (!sc->sc_dma_active)
		return 0;

	if (force == WDC_DMAEND_END) {
		/*
		 * Plain register polling: this core does not write
		 * the DONE bit back to the LLI (measured: the memory
		 * fast path never fired), and bridge traffic during
		 * the transfer has been ruled out as the stall
		 * trigger.
		 */
		tfr = dwcdmac_read(sc, DWCDMAC_RAW_TFR);
		err = dwcdmac_read(sc, DWCDMAC_RAW_ERR);
		/* still running?  wdc_dmawait() polls us again */
		if (((tfr | err) & DWCDMAC_CHANBIT(DWCSATA_DMACH)) == 0)
			return WDC_DMAST_NOIRQ;
		/* a detectable error deserves the full dump */
		if (err & DWCDMAC_CHANBIT(DWCSATA_DMACH))
			dwcsata_dma_dump(sc, tfr, err);
	} else {
		const uint32_t chanbit = DWCDMAC_CHANBIT(DWCSATA_DMACH);

		/*
		 * If the channel wedged, clear it through SDR/DCR first
		 * (bridge-safe) before any AHB register access can machine-
		 * check. CHEN stuck with no TFR/ERR is the wedge signature.
		 */
		if (dwcdmac_read(sc, DWCDMAC_CHEN) & chanbit) {
			if (force != WDC_DMAEND_ABRT_QUIET)
				aprint_error_dev(sc->sc_wdcdev.sc_atac.atac_dev,
				    "DMA wedge: SDR un-wedge before abort\n");
			dwcsata_unwedge(sc);
		}

		tfr = dwcdmac_read(sc, DWCDMAC_RAW_TFR);
		err = dwcdmac_read(sc, DWCDMAC_RAW_ERR);

		/*
		 * Abort. Close the SATA-side channel first so the
		 * handshake stops feeding the DMAC, then follow the
		 * databook: suspend the channel, let its FIFO drain,
		 * and only then clear the enable.
		 */
		bus_space_write_4(sc->sc_iot, sc->sc_ioh, DWCSATA_DMACR,
		    DWCSATA_DMACR_TXRXCH_CLEAR);
		dwcdmac_write(sc, DWCDMAC_CFG(DWCSATA_DMACH),
		    dwcdmac_read(sc, DWCDMAC_CFG(DWCSATA_DMACH)) |
		    DWCDMAC_CFG_CH_SUSP);
		for (i = 0; i < 100; i++) {
			if (dwcdmac_read(sc, DWCDMAC_CFG(DWCSATA_DMACH)) &
			    DWCDMAC_CFG_FIFO_EMPTY)
				break;
			delay(10);
		}
		dwcdmac_write(sc, DWCDMAC_CHEN,
		    DWCDMAC_CH_DISABLE(DWCSATA_DMACH));
		for (i = 0; i < 100; i++) {
			if ((dwcdmac_read(sc, DWCDMAC_CHEN) & chanbit) == 0)
				break;
			delay(10);
		}
		if (dwcdmac_read(sc, DWCDMAC_CHEN) & chanbit) {
			/*
			 * This happens. Not sure why.
			 */
			dwcdmac_write(sc, DWCDMAC_DMACFG, 0);
			delay(100);
			dwcdmac_write(sc, DWCDMAC_DMACFG, DWCDMAC_DMACFG_EN);
			if (force != WDC_DMAEND_ABRT_QUIET)
				aprint_error_dev(
				    sc->sc_wdcdev.sc_atac.atac_dev,
				    "DMA channel wedged, reset the DMAC\n");
		}
		/* drop the suspend again for the next transfer */
		dwcdmac_write(sc, DWCDMAC_CFG(DWCSATA_DMACH),
		    dwcdmac_read(sc, DWCDMAC_CFG(DWCSATA_DMACH)) &
		    ~DWCDMAC_CFG_CH_SUSP);

		if (force != WDC_DMAEND_ABRT_QUIET) {
			dwcsata_dma_dump(sc, tfr, err);
			if ((tfr & chanbit) == 0)
				status |= WDC_DMAST_NOIRQ;
		}
	}

	if (err & DWCDMAC_CHANBIT(DWCSATA_DMACH))
		status |= WDC_DMAST_ERR;

	/* close the SATA core side, keeping TXMODE set */
	dmacr = bus_space_read_4(sc->sc_iot, sc->sc_ioh, DWCSATA_DMACR);
	bus_space_write_4(sc->sc_iot, sc->sc_ioh, DWCSATA_DMACR,
	    read ? DWCSATA_DMACR_RX_CLEAR(dmacr) :
	    DWCSATA_DMACR_TX_CLEAR(dmacr));

	intpr = bus_space_read_4(sc->sc_iot, sc->sc_ioh, DWCSATA_INTPR);
	if (intpr & DWCSATA_INTPR_ERR)
		status |= WDC_DMAST_ERR;
	bus_space_write_4(sc->sc_iot, sc->sc_ioh, DWCSATA_INTPR,
	    DWCSATA_INTPR_DMAT | DWCSATA_INTPR_ERR);
	if ((status & WDC_DMAST_ERR) != 0 &&
	    force != WDC_DMAEND_ABRT_QUIET) {
		uint32_t serror = bus_space_read_4(sc->sc_iot, sc->sc_ioh,
		    DWCSATA_SERROR);
		aprint_error_dev(sc->sc_wdcdev.sc_atac.atac_dev,
		    "DMA error: DMAC err 0x%08x INTPR 0x%08x SError 0x%08x\n",
		    err, intpr, serror);
		bus_space_write_4(sc->sc_iot, sc->sc_ioh, DWCSATA_SERROR,
		    serror);
	}

	dwcdmac_clear_intrs(sc);

	bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap_xfer, 0,
	    sc->sc_dmamap_xfer->dm_mapsize,
	    read ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE);
	bus_dmamap_unload(sc->sc_dmat, sc->sc_dmamap_xfer);
	sc->sc_dma_loaded = false;
	dwcsata_usb_restore(sc);
	sc->sc_dma_active = false;

	return status;
}

static int
dwcsata_intr(void *arg)
{
	struct dwcsata_softc *sc = arg;
	struct ata_channel *chp = &sc->sc_channel;
	struct wdc_regs *wdr = &sc->sc_wdc_regs;
	uint32_t intpr;

	intpr = bus_space_read_4(sc->sc_iot, sc->sc_ioh, DWCSATA_INTPR);
	if (intpr == 0)
		return 0;		/* nothing pending; not our line */

	if (intpr & (DWCSATA_INTPR_ERR | DWCSATA_INTPR_CMDABORT |
	    DWCSATA_INTPR_PRIMERR)) {
		uint32_t serror = bus_space_read_4(sc->sc_iot, sc->sc_ioh,
		    DWCSATA_SERROR);
		aprint_error_dev(sc->sc_wdcdev.sc_atac.atac_dev,
		    "controller error interrupt: INTPR 0x%08x SError 0x%08x\n",
		    intpr, serror);
		if (serror & DWCSATA_ERRMR_ERR_BITS)
			bus_space_write_4(sc->sc_iot, sc->sc_ioh,
			    DWCSATA_SERROR, serror);
	}

	/*
	 * Acknowledge the latched INTPR status bits
	 */
	bus_space_write_4(sc->sc_iot, sc->sc_ioh, DWCSATA_INTPR,
	    intpr & ~DWCSATA_INTPR_ERRADDR);

	if (chp->ch_flags & ATACH_DMA_WAIT) {
		/*
		 * A DMA transfer is in flight
		 */
		(void)bus_space_read_1(wdr->cmd_iot,
		    wdr->cmd_iohs[wd_status], 0);
		return 1;
	}

	/*
	 * No DMA pending: a PIO command completion or a stray drive
	 * interrupt. wdcintr reads the status register.
	 */
	return wdcintr(chp);
}

static int
dwcsata_dmac_intr(void *arg)
{
	struct dwcsata_softc *sc = arg;
	struct ata_channel *chp = &sc->sc_channel;
	const uint32_t chanbit = DWCDMAC_CHANBIT(DWCSATA_DMACH);
	int rv;

	if (((dwcdmac_read(sc, DWCDMAC_RAW_TFR) |
	    dwcdmac_read(sc, DWCDMAC_RAW_ERR)) & chanbit) == 0)
		return 0;		/* not our channel */

	rv = wdcintr(chp);

	if ((dwcdmac_read(sc, DWCDMAC_RAW_TFR) |
	    dwcdmac_read(sc, DWCDMAC_RAW_ERR)) & chanbit) {
		dwcdmac_write(sc, DWCDMAC_CLEAR_TFR, chanbit);
		dwcdmac_write(sc, DWCDMAC_CLEAR_ERR, chanbit);
	}
	return rv ? rv : 1;
}

#endif	/* !DWCSATA_PIO_ONLY */

static void
dwcsata_attach(device_t parent, device_t self, void *aux)
{
	struct dwcsata_softc *sc = device_private(self);
	struct plb_attach_args *paa = aux;
	struct wdc_regs *wdr;
	struct ata_channel *chp = &sc->sc_channel;
	uint32_t idr, versionr;
	int i;

	sc->sc_dmat = paa->plb_dmat;
	sc->sc_irq = paa->plb_irq;

	/* the window also covers the companion AHB DMAC below the core */
	dwcsata_tag.pbs_base = paa->plb_addr - DWCDMAC_OFFSET;
	dwcsata_tag.pbs_limit = paa->plb_addr + DWCSATA_SIZE;
	sc->sc_iot = &dwcsata_tag;

	if (bus_space_init(&dwcsata_tag, "dwcsata", dwcsata_ex_storage,
	      sizeof(dwcsata_ex_storage)) ||
	    bus_space_map(sc->sc_iot, paa->plb_addr, DWCSATA_SIZE, 0,
	      &sc->sc_ioh)) {
		aprint_error(": can't map registers\n");
		return;
	}

	versionr = bus_space_read_4(sc->sc_iot, sc->sc_ioh,
	    DWCSATA_VERSIONR);
	if (versionr == 0 || versionr == 0xffffffff) {
		aprint_normal(": DWC SATA core not responding\n");
		return;
	}
	idr = bus_space_read_4(sc->sc_iot, sc->sc_ioh, DWCSATA_IDR);
	aprint_normal(": DWC SATA-II controller, core version %c.%c%c "
	    "id 0x%02x\n",
	    (int)(versionr >> 24) & 0xff, (int)(versionr >> 16) & 0xff,
	    (int)(versionr >> 8) & 0xff, idr & 0xff);

	sc->sc_wdcdev.regs = wdr = &sc->sc_wdc_regs;

	wdr->cmd_iot = wdr->ctl_iot = sc->sc_iot;
	wdr->cmd_baseioh = sc->sc_ioh;
	wdr->cmd_ios = DWCSATA_SIZE;
	for (i = 0; i < WDC_NREG; i++) {
		if (bus_space_subregion(wdr->cmd_iot, wdr->cmd_baseioh,
		    DWCSATA_CDR_BASE + i * DWCSATA_CDR_STRIDE,
		    DWCSATA_CDR_STRIDE, &wdr->cmd_iohs[i]) != 0) {
			aprint_error_dev(self,
			    "couldn't subregion registers\n");
			return;
		}
	}
	wdc_init_shadow_regs(wdr);

	if (bus_space_subregion(wdr->ctl_iot, sc->sc_ioh, DWCSATA_CLR0, 4,
	      &wdr->ctl_ioh) ||
	    bus_space_subregion(sc->sc_iot, sc->sc_ioh, DWCSATA_SSTATUS, 4,
	      &wdr->sata_status) ||
	    bus_space_subregion(sc->sc_iot, sc->sc_ioh, DWCSATA_SERROR, 4,
	      &wdr->sata_error) ||
	    bus_space_subregion(sc->sc_iot, sc->sc_ioh, DWCSATA_SCONTROL, 4,
	      &wdr->sata_control)) {
		aprint_error_dev(self, "couldn't subregion registers\n");
		return;
	}
	wdr->ctl_ios = 4;
	wdr->sata_iot = sc->sc_iot;
	wdr->sata_baseioh = sc->sc_ioh;

	bus_space_write_4(sc->sc_iot, sc->sc_ioh, DWCSATA_INTMR, 0);
	bus_space_write_4(sc->sc_iot, sc->sc_ioh, DWCSATA_ERRMR,
	    DWCSATA_ERRMR_ERR_BITS);
	bus_space_write_4(sc->sc_iot, sc->sc_ioh, DWCSATA_DMACR,
	    DWCSATA_DMACR_TXRXCH_CLEAR);
	bus_space_write_4(sc->sc_iot, sc->sc_ioh, DWCSATA_SERROR,
	    0xffffffff);
	bus_space_write_4(sc->sc_iot, sc->sc_ioh, DWCSATA_INTPR,
	    0xffffffff);

	sc->sc_wdcdev.sc_atac.atac_dev = self;
	sc->sc_wdcdev.sc_atac.atac_cap = ATAC_CAP_DATA16 | ATAC_CAP_NOIRQ;
	sc->sc_wdcdev.sc_atac.atac_pio_cap = 4;

#ifndef DWCSATA_PIO_ONLY
	/*
	 * No atac_set_modes: SATA carries the mode in the FIS, there are
	 * no host timings to program, and without the hook the MI code
	 * also skips the (unneeded) SET FEATURES xfer-mode command.
	 */
	if (dwcsata_dma_setup(self, sc, paa)) {
		sc->sc_wdcdev.sc_atac.atac_cap |= ATAC_CAP_DMA |
		    ATAC_CAP_UDMA;
		sc->sc_wdcdev.sc_atac.atac_dma_cap = 2;
		sc->sc_wdcdev.sc_atac.atac_udma_cap = 6;
		sc->sc_wdcdev.dma_arg = sc;
		sc->sc_wdcdev.dma_init = dwcsata_dma_init;
		sc->sc_wdcdev.dma_start = dwcsata_dma_start;
		sc->sc_wdcdev.dma_finish = dwcsata_dma_finish;
		chp->ch_flags |= ATACH_DMA_BEFORE_CMD;
		sc->sc_dma_ok = true;
	} else
		aprint_error_dev(self, "DMA setup failed, PIO only\n");

	if (sc->sc_dma_ok) {
		const struct sysctlnode *rnode = NULL;

		sysctl_createv(NULL, 0, NULL, &rnode,
		    CTLFLAG_PERMANENT, CTLTYPE_NODE, "dwcsata",
		    SYSCTL_DESCR("DWC SATA-II controller"),
		    NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL);
		if (rnode != NULL)
			sysctl_createv(NULL, 0, &rnode, NULL,
			    CTLFLAG_READWRITE, CTLTYPE_INT, "usb_workaround",
			    SYSCTL_DESCR("pause USB schedules during SATA DMA "
			    "(1=EHCI periodic 2=async 4=OHCI; 0=off 7=all)"),
			    NULL, 0, &dwcsata_usb_workaround, 0,
			    CTL_CREATE, CTL_EOL);
	}
#endif

	sc->sc_chanlist[0] = chp;
	sc->sc_wdcdev.sc_atac.atac_channels = sc->sc_chanlist;
	sc->sc_wdcdev.sc_atac.atac_nchannels = 1;
	sc->sc_wdcdev.sc_atac.atac_probe = dwcsata_probe;
	sc->sc_wdcdev.wdc_maxdrives = 1;	/* point-to-point, no PMP */
	sc->sc_wdcdev.reset = dwcsata_reset;	/* settle the core post-SRST */

	chp->ch_channel = 0;
	chp->ch_atac = &sc->sc_wdcdev.sc_atac;

#ifndef DWCSATA_PIO_ONLY
	/*
	 * With the DMA engine up, switch to interrupt-driven operation.
	 */
	if (sc->sc_dma_ok) {
		sc->sc_wdcdev.sc_atac.atac_cap &= ~ATAC_CAP_NOIRQ;

		bus_space_write_4(sc->sc_iot, sc->sc_ioh, DWCSATA_SERROR,
		    0xffffffff);
		bus_space_write_4(sc->sc_iot, sc->sc_ioh, DWCSATA_INTPR,
		    0xffffffff);
		(void)bus_space_read_1(wdr->cmd_iot, wdr->cmd_iohs[wd_status],
		    0);

		intr_establish_xname(sc->sc_irq, IST_LEVEL, IPL_BIO,
		    dwcsata_intr, sc, device_xname(self));
		intr_establish_xname(AMCC460EX_SATA_DMA_IRQ, IST_LEVEL,
		    IPL_BIO, dwcsata_dmac_intr, sc, "dwcsata dmac");

		/* let SATA link/transport errors raise irq 96 */
		bus_space_write_4(sc->sc_iot, sc->sc_ioh, DWCSATA_INTMR,
		    DWCSATA_INTMR_ERRM);
	}
#endif

	wdcattach(chp);
}

/*
 * Soft-reset hook (installed as wdc->reset, replacing wdc_do_reset).
 */
static void
dwcsata_reset(struct ata_channel *chp, int poll)
{
	struct wdc_softc *wdc = CHAN_TO_WDC(chp);
	struct wdc_regs *wdr = &wdc->regs[chp->ch_channel];
	uint8_t st = 0;
	int i;

	/* SRST pulse, as in wdc_do_reset() */
	bus_space_write_1(wdr->cmd_iot, wdr->cmd_iohs[wd_sdh], 0, WDSD_IBM);
	delay(10);
	bus_space_write_1(wdr->ctl_iot, wdr->ctl_ioh, wd_aux_ctlr,
	    WDCTL_RST | WDCTL_IDS | WDCTL_4BIT);
	delay(2000);
	(void)bus_space_read_1(wdr->cmd_iot, wdr->cmd_iohs[wd_error], 0);
	bus_space_write_1(wdr->ctl_iot, wdr->ctl_ioh, wd_aux_ctlr,
	    WDCTL_4BIT | WDCTL_IDS);

	/*
	 * Let the core finish the reset before the taskfile is touched
	 * again.
	 */
	delay(150000);
	for (i = 0; i < 100; i++) {
		st = bus_space_read_1(wdr->cmd_iot, wdr->cmd_iohs[wd_status],
		    0);
		if ((st & WDCS_BSY) == 0)
			break;
		delay(10000);
	}
	aprint_normal_dev(wdc->sc_atac.atac_dev,
	    "soft reset: BSY cleared after %dms, status 0x%02x\n",
	    150 + i * 10, st);
}

static void
dwcsata_probe(struct ata_channel *chp)
{
	struct dwcsata_softc *sc = (struct dwcsata_softc *)CHAN_TO_WDC(chp);

	wdc_sataprobe(chp);

	/* drop the diagnostics the PHY bring-up latched into SError */
	bus_space_write_4(sc->sc_iot, sc->sc_ioh, DWCSATA_SERROR,
	    0xffffffff);
	bus_space_write_4(sc->sc_iot, sc->sc_ioh, DWCSATA_INTPR,
	    0xffffffff);

#ifndef DWCSATA_PIO_ONLY
	/* link resets do not touch DMACR/DBTSR, but re-init regardless */
	if (sc->sc_dma_ok) {
		bus_space_write_4(sc->sc_iot, sc->sc_ioh, DWCSATA_DMACR,
		    DWCSATA_DMACR_TXRXCH_CLEAR);
		bus_space_write_4(sc->sc_iot, sc->sc_ioh, DWCSATA_DBTSR,
		    DWCSATA_DBTSR_MWR(DWCSATA_DMA_FIFO_BURST) |
		    DWCSATA_DBTSR_MRD(DWCSATA_DMA_FIFO_BURST));
	}
#endif
}