xref: /src/sys/dev/ata/ata.c

/*	$NetBSD: ata.c,v 1.134 2017/10/08 04:52:33 mlelstv Exp $	*/

/*
 * Copyright (c) 1998, 2001 Manuel Bouyer.  All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: ata.c,v 1.134 2017/10/08 04:52:33 mlelstv Exp $");

#include "opt_ata.h"

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/malloc.h>
#include <sys/device.h>
#include <sys/conf.h>
#include <sys/fcntl.h>
#include <sys/proc.h>
#include <sys/kthread.h>
#include <sys/errno.h>
#include <sys/ataio.h>
#include <sys/kmem.h>
#include <sys/intr.h>
#include <sys/bus.h>
#include <sys/once.h>
#include <sys/bitops.h>

#define ATABUS_PRIVATE

#include <dev/ata/ataconf.h>
#include <dev/ata/atareg.h>
#include <dev/ata/atavar.h>
#include <dev/ic/wdcvar.h>	/* for PIOBM */

#include "locators.h"

#include "atapibus.h"
#include "ataraid.h"
#include "sata_pmp.h"

#if NATARAID > 0
#include <dev/ata/ata_raidvar.h>
#endif
#if NSATA_PMP > 0
#include <dev/ata/satapmpvar.h>
#endif
#include <dev/ata/satapmpreg.h>

#define DEBUG_FUNCS  0x08
#define DEBUG_PROBE  0x10
#define DEBUG_DETACH 0x20
#define	DEBUG_XFERS  0x40
#ifdef ATADEBUG
#ifndef ATADEBUG_MASK
#define ATADEBUG_MASK 0
#endif
int atadebug_mask = ATADEBUG_MASK;
#define ATADEBUG_PRINT(args, level) \
	if (atadebug_mask & (level)) \
		printf args
#else
#define ATADEBUG_PRINT(args, level)
#endif

static ONCE_DECL(ata_init_ctrl);

/*
 * A queue of atabus instances, used to ensure the same bus probe order
 * for a given hardware configuration at each boot.  Kthread probing
 * devices on a atabus.  Only one probing at once.
 */
static TAILQ_HEAD(, atabus_initq)	atabus_initq_head;
static kmutex_t				atabus_qlock;
static kcondvar_t			atabus_qcv;
static lwp_t *				atabus_cfg_lwp;

/*****************************************************************************
 * ATA bus layer.
 *
 * ATA controllers attach an atabus instance, which handles probing the bus
 * for drives, etc.
 *****************************************************************************/

dev_type_open(atabusopen);
dev_type_close(atabusclose);
dev_type_ioctl(atabusioctl);

const struct cdevsw atabus_cdevsw = {
	.d_open = atabusopen,
	.d_close = atabusclose,
	.d_read = noread,
	.d_write = nowrite,
	.d_ioctl = atabusioctl,
	.d_stop = nostop,
	.d_tty = notty,
	.d_poll = nopoll,
	.d_mmap = nommap,
	.d_kqfilter = nokqfilter,
	.d_discard = nodiscard,
	.d_flag = D_OTHER
};

extern struct cfdriver atabus_cd;

static void atabus_childdetached(device_t, device_t);
static int atabus_rescan(device_t, const char *, const int *);
static bool atabus_resume(device_t, const pmf_qual_t *);
static bool atabus_suspend(device_t, const pmf_qual_t *);
static void atabusconfig_thread(void *);

static void ata_channel_idle(struct ata_channel *);
static void ata_channel_thaw_locked(struct ata_channel *);
static void ata_activate_xfer_locked(struct ata_channel *, struct ata_xfer *);
static void ata_channel_freeze_locked(struct ata_channel *);
static struct ata_xfer *ata_queue_get_active_xfer_locked(struct ata_channel *);
static void ata_thread_wake_locked(struct ata_channel *);

/*
 * atabus_init:
 *
 *	Initialize ATA subsystem structures.
 */
static int
atabus_init(void)
{

	TAILQ_INIT(&atabus_initq_head);
	mutex_init(&atabus_qlock, MUTEX_DEFAULT, IPL_NONE);
	cv_init(&atabus_qcv, "atainitq");
	return 0;
}

/*
 * atabusprint:
 *
 *	Autoconfiguration print routine used by ATA controllers when
 *	attaching an atabus instance.
 */
int
atabusprint(void *aux, const char *pnp)
{
	struct ata_channel *chan = aux;

	if (pnp)
		aprint_normal("atabus at %s", pnp);
	aprint_normal(" channel %d", chan->ch_channel);

	return (UNCONF);
}

/*
 * ataprint:
 *
 *	Autoconfiguration print routine.
 */
int
ataprint(void *aux, const char *pnp)
{
	struct ata_device *adev = aux;

	if (pnp)
		aprint_normal("wd at %s", pnp);
	aprint_normal(" drive %d", adev->adev_drv_data->drive);

	return (UNCONF);
}

static void
ata_queue_reset(struct ata_queue *chq)
{
	/* make sure that we can use polled commands */
	TAILQ_INIT(&chq->queue_xfer);
	TAILQ_INIT(&chq->active_xfers);
	chq->queue_freeze = 0;
	chq->queue_active = 0;
	chq->active_xfers_used = 0;
	chq->queue_xfers_avail = __BIT(chq->queue_openings) - 1;
}

struct ata_xfer *
ata_queue_hwslot_to_xfer(struct ata_channel *chp, int hwslot)
{
	struct ata_queue *chq = chp->ch_queue;
	struct ata_xfer *xfer = NULL;

	ata_channel_lock(chp);

	KASSERTMSG(hwslot < chq->queue_openings, "hwslot %d > openings %d",
	    hwslot, chq->queue_openings);
	KASSERTMSG((chq->active_xfers_used & __BIT(hwslot)) != 0,
	    "hwslot %d not active", hwslot);

	/* Usually the first entry will be the one */
	TAILQ_FOREACH(xfer, &chq->active_xfers, c_activechain) {
		if (xfer->c_slot == hwslot)
			break;
	}

	ata_channel_unlock(chp);

	KASSERTMSG((xfer != NULL),
	    "%s: xfer with slot %d not found (active %x)", __func__,
	    hwslot, chq->active_xfers_used);

	return xfer;
}

static struct ata_xfer *
ata_queue_get_active_xfer_locked(struct ata_channel *chp)
{
	struct ata_xfer *xfer;

	KASSERT(mutex_owned(&chp->ch_lock));
	xfer = TAILQ_FIRST(&chp->ch_queue->active_xfers);

	if (xfer && ISSET(xfer->c_flags, C_NCQ)) {
		/* Spurious call, never return NCQ xfer from this interface */
		xfer = NULL;
	}

	return xfer;
}

/*
 * This interface is supposed only to be used when there is exactly
 * one outstanding command, when there is no information about the slot,
 * which triggered the command. ata_queue_hwslot_to_xfer() interface
 * is preferred in all NCQ cases.
 */
struct ata_xfer *
ata_queue_get_active_xfer(struct ata_channel *chp)
{
	struct ata_xfer *xfer = NULL;

	ata_channel_lock(chp);
	xfer = ata_queue_get_active_xfer_locked(chp);
	ata_channel_unlock(chp);

	return xfer;
}

struct ata_xfer *
ata_queue_drive_active_xfer(struct ata_channel *chp, int drive)
{
	struct ata_xfer *xfer = NULL;

	ata_channel_lock(chp);

	TAILQ_FOREACH(xfer, &chp->ch_queue->active_xfers, c_activechain) {
		if (xfer->c_drive == drive)
			break;
	}
	KASSERT(xfer != NULL);

	ata_channel_unlock(chp);

	return xfer;
}

static void
ata_xfer_init(struct ata_xfer *xfer, uint8_t slot)
{
	memset(xfer, 0, sizeof(*xfer));

	xfer->c_slot = slot;

	cv_init(&xfer->c_active, "ataact");
	cv_init(&xfer->c_finish, "atafin");
	callout_init(&xfer->c_timo_callout, 0); 	/* XXX MPSAFE */
	callout_init(&xfer->c_retry_callout, 0); 	/* XXX MPSAFE */
}

static void
ata_xfer_destroy(struct ata_xfer *xfer)
{
	callout_halt(&xfer->c_timo_callout, NULL);	/* XXX MPSAFE */
	callout_destroy(&xfer->c_timo_callout);
	callout_halt(&xfer->c_retry_callout, NULL);	/* XXX MPSAFE */
	callout_destroy(&xfer->c_retry_callout);
	cv_destroy(&xfer->c_active);
	cv_destroy(&xfer->c_finish);
}

struct ata_queue *
ata_queue_alloc(uint8_t openings)
{
	if (openings == 0)
		openings = 1;

	if (openings > ATA_MAX_OPENINGS)
		openings = ATA_MAX_OPENINGS;

	struct ata_queue *chq = malloc(offsetof(struct ata_queue, queue_xfers[openings]),
	    M_DEVBUF, M_WAITOK | M_ZERO);

	chq->queue_openings = openings;
	ata_queue_reset(chq);

	cv_init(&chq->queue_busy, "ataqbusy");
	cv_init(&chq->queue_drain, "atdrn");
	cv_init(&chq->queue_idle, "qidl");

	for (uint8_t i = 0; i < openings; i++)
		ata_xfer_init(&chq->queue_xfers[i], i);

	return chq;
}

void
ata_queue_free(struct ata_queue *chq)
{
	for (uint8_t i = 0; i < chq->queue_openings; i++)
		ata_xfer_destroy(&chq->queue_xfers[i]);

	cv_destroy(&chq->queue_busy);
	cv_destroy(&chq->queue_drain);
	cv_destroy(&chq->queue_idle);

	free(chq, M_DEVBUF);
}

void
ata_channel_init(struct ata_channel *chp)
{
	mutex_init(&chp->ch_lock, MUTEX_DEFAULT, IPL_BIO);
	cv_init(&chp->ch_thr_idle, "atath");
}

/*
 * ata_channel_attach:
 *
 *	Common parts of attaching an atabus to an ATA controller channel.
 */
void
ata_channel_attach(struct ata_channel *chp)
{
	if (chp->ch_flags & ATACH_DISABLED)
		return;

	KASSERT(chp->ch_queue != NULL);

	ata_channel_init(chp);

	chp->atabus = config_found_ia(chp->ch_atac->atac_dev, "ata", chp,
		atabusprint);
}

void
ata_channel_destroy(struct ata_channel *chp)
{
	mutex_destroy(&chp->ch_lock);
	cv_destroy(&chp->ch_thr_idle);
}

/*
 * ata_channel_detach:
 *
 *	Common parts of detaching an atabus to an ATA controller channel.
 */
void
ata_channel_detach(struct ata_channel *chp)
{
	if (chp->ch_flags & ATACH_DISABLED)
		return;

	ata_channel_destroy(chp);
}

static void
atabusconfig(struct atabus_softc *atabus_sc)
{
	struct ata_channel *chp = atabus_sc->sc_chan;
	struct atac_softc *atac = chp->ch_atac;
	struct atabus_initq *atabus_initq = NULL;
	int i, error;

	/* we are in the atabus's thread context */
	ata_channel_lock(chp);
	chp->ch_flags |= ATACH_TH_RUN;
	ata_channel_unlock(chp);

	/*
	 * Probe for the drives attached to controller, unless a PMP
	 * is already known
	 */
	/* XXX for SATA devices we will power up all drives at once */
	if (chp->ch_satapmp_nports == 0)
		(*atac->atac_probe)(chp);

	if (chp->ch_ndrives >= 2) {
		ATADEBUG_PRINT(("atabusattach: ch_drive_type 0x%x 0x%x\n",
		    chp->ch_drive[0].drive_type, chp->ch_drive[1].drive_type),
		    DEBUG_PROBE);
	}

	/* next operations will occurs in a separate thread */
	ata_channel_lock(chp);
	chp->ch_flags &= ~ATACH_TH_RUN;
	ata_channel_unlock(chp);

	/* Make sure the devices probe in atabus order to avoid jitter. */
	mutex_enter(&atabus_qlock);
	for (;;) {
		atabus_initq = TAILQ_FIRST(&atabus_initq_head);
		if (atabus_initq->atabus_sc == atabus_sc)
			break;
		cv_wait(&atabus_qcv, &atabus_qlock);
	}
	mutex_exit(&atabus_qlock);

	ata_channel_lock(chp);

	/* If no drives, abort here */
	if (chp->ch_drive == NULL)
		goto out;
	KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL);
	for (i = 0; i < chp->ch_ndrives; i++)
		if (chp->ch_drive[i].drive_type != ATA_DRIVET_NONE)
			break;
	if (i == chp->ch_ndrives)
		goto out;

	/* Shortcut in case we've been shutdown */
	if (chp->ch_flags & ATACH_SHUTDOWN)
		goto out;

	ata_channel_unlock(chp);

	if ((error = kthread_create(PRI_NONE, 0, NULL, atabusconfig_thread,
	    atabus_sc, &atabus_cfg_lwp,
	    "%scnf", device_xname(atac->atac_dev))) != 0)
		aprint_error_dev(atac->atac_dev,
		    "unable to create config thread: error %d\n", error);
	return;

 out:
	ata_channel_unlock(chp);

	mutex_enter(&atabus_qlock);
	TAILQ_REMOVE(&atabus_initq_head, atabus_initq, atabus_initq);
	cv_broadcast(&atabus_qcv);
	mutex_exit(&atabus_qlock);

	free(atabus_initq, M_DEVBUF);

	ata_delref(chp);

	config_pending_decr(atac->atac_dev);
}

/*
 * atabus_configthread: finish attach of atabus's childrens, in a separate
 * kernel thread.
 */
static void
atabusconfig_thread(void *arg)
{
	struct atabus_softc *atabus_sc = arg;
	struct ata_channel *chp = atabus_sc->sc_chan;
	struct atac_softc *atac = chp->ch_atac;
	struct atabus_initq *atabus_initq = NULL;
	int i, s;

	/* XXX seems wrong */
	mutex_enter(&atabus_qlock);
	atabus_initq = TAILQ_FIRST(&atabus_initq_head);
	KASSERT(atabus_initq->atabus_sc == atabus_sc);
	mutex_exit(&atabus_qlock);

	/*
	 * First look for a port multiplier
	 */
	if (chp->ch_ndrives == PMP_MAX_DRIVES &&
	    chp->ch_drive[PMP_PORT_CTL].drive_type == ATA_DRIVET_PM) {
#if NSATA_PMP > 0
		satapmp_attach(chp);
#else
		aprint_error_dev(atabus_sc->sc_dev,
		    "SATA port multiplier not supported\n");
		/* no problems going on, all drives are ATA_DRIVET_NONE */
#endif
	}

	/*
	 * Attach an ATAPI bus, if needed.
	 */
	KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL);
	for (i = 0; i < chp->ch_ndrives && chp->atapibus == NULL; i++) {
		if (chp->ch_drive[i].drive_type == ATA_DRIVET_ATAPI) {
#if NATAPIBUS > 0
			(*atac->atac_atapibus_attach)(atabus_sc);
#else
			/*
			 * Fake the autoconfig "not configured" message
			 */
			aprint_normal("atapibus at %s not configured\n",
			    device_xname(atac->atac_dev));
			chp->atapibus = NULL;
			s = splbio();
			for (i = 0; i < chp->ch_ndrives; i++) {
				if (chp->ch_drive[i].drive_type == ATA_DRIVET_ATAPI)
					chp->ch_drive[i].drive_type = ATA_DRIVET_NONE;
			}
			splx(s);
#endif
			break;
		}
	}

	for (i = 0; i < chp->ch_ndrives; i++) {
		struct ata_device adev;
		if (chp->ch_drive[i].drive_type != ATA_DRIVET_ATA &&
		    chp->ch_drive[i].drive_type != ATA_DRIVET_OLD) {
			continue;
		}
		if (chp->ch_drive[i].drv_softc != NULL)
			continue;
		memset(&adev, 0, sizeof(struct ata_device));
		adev.adev_bustype = atac->atac_bustype_ata;
		adev.adev_channel = chp->ch_channel;
		adev.adev_drv_data = &chp->ch_drive[i];
		chp->ch_drive[i].drv_softc = config_found_ia(atabus_sc->sc_dev,
		    "ata_hl", &adev, ataprint);
		if (chp->ch_drive[i].drv_softc != NULL) {
			ata_probe_caps(&chp->ch_drive[i]);
		} else {
			s = splbio();
			chp->ch_drive[i].drive_type = ATA_DRIVET_NONE;
			splx(s);
		}
	}

	/* now that we know the drives, the controller can set its modes */
	if (atac->atac_set_modes) {
		(*atac->atac_set_modes)(chp);
		ata_print_modes(chp);
	}
#if NATARAID > 0
	if (atac->atac_cap & ATAC_CAP_RAID) {
		for (i = 0; i < chp->ch_ndrives; i++) {
			if (chp->ch_drive[i].drive_type == ATA_DRIVET_ATA) {
				ata_raid_check_component(
				    chp->ch_drive[i].drv_softc);
			}
		}
	}
#endif /* NATARAID > 0 */

	/*
	 * reset drive_flags for unattached devices, reset state for attached
	 * ones
	 */
	s = splbio();
	for (i = 0; i < chp->ch_ndrives; i++) {
		if (chp->ch_drive[i].drive_type == ATA_DRIVET_PM)
			continue;
		if (chp->ch_drive[i].drv_softc == NULL) {
			chp->ch_drive[i].drive_flags = 0;
			chp->ch_drive[i].drive_type = ATA_DRIVET_NONE;
		} else
			chp->ch_drive[i].state = 0;
	}
	splx(s);

	mutex_enter(&atabus_qlock);
	TAILQ_REMOVE(&atabus_initq_head, atabus_initq, atabus_initq);
	cv_broadcast(&atabus_qcv);
	mutex_exit(&atabus_qlock);

	free(atabus_initq, M_DEVBUF);

	ata_delref(chp);

	config_pending_decr(atac->atac_dev);
	kthread_exit(0);
}

/*
 * atabus_thread:
 *
 *	Worker thread for the ATA bus.
 */
static void
atabus_thread(void *arg)
{
	struct atabus_softc *sc = arg;
	struct ata_channel *chp = sc->sc_chan;
	struct ata_queue *chq = chp->ch_queue;
	struct ata_xfer *xfer;
	int i, rv, s;

	ata_channel_lock(chp);
	chp->ch_flags |= ATACH_TH_RUN;

	/*
	 * Probe the drives.  Reset type to indicate to controllers
	 * that can re-probe that all drives must be probed..
	 *
	 * Note: ch_ndrives may be changed during the probe.
	 */
	KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL);
	for (i = 0; i < chp->ch_ndrives; i++) {
		chp->ch_drive[i].drive_flags = 0;
		chp->ch_drive[i].drive_type = ATA_DRIVET_NONE;
	}
	ata_channel_unlock(chp);

	atabusconfig(sc);

	ata_channel_lock(chp);
	for (;;) {
		if ((chp->ch_flags & (ATACH_TH_RESET | ATACH_SHUTDOWN)) == 0 &&
		    (chq->queue_active == 0 || chq->queue_freeze == 0)) {
			chp->ch_flags &= ~ATACH_TH_RUN;
			cv_wait(&chp->ch_thr_idle, &chp->ch_lock);
			chp->ch_flags |= ATACH_TH_RUN;
		}
		if (chp->ch_flags & ATACH_SHUTDOWN) {
			break;
		}
		if (chp->ch_flags & ATACH_TH_RESCAN) {
			chp->ch_flags &= ~ATACH_TH_RESCAN;
			ata_channel_unlock(chp);
			atabusconfig(sc);
			ata_channel_lock(chp);
		}
		if (chp->ch_flags & ATACH_TH_RESET) {
			/*
			 * ata_reset_channel() will freeze 2 times, so
			 * unfreeze one time. Not a problem as we're at splbio
			 */
			ata_channel_thaw_locked(chp);
			ata_channel_unlock(chp);
			s = splbio();
			ata_reset_channel(chp, AT_WAIT | chp->ch_reset_flags);
			splx(s);
			ata_channel_lock(chp);
		} else if (chq->queue_active > 0 && chq->queue_freeze == 1) {
			/*
			 * Caller has bumped queue_freeze, decrease it. This
			 * flow shalt never be executed for NCQ commands.
			 */
			KASSERT((chp->ch_flags & ATACH_NCQ) == 0);
			KASSERT(chq->queue_active == 1);

			ata_channel_thaw_locked(chp);
			xfer = ata_queue_get_active_xfer_locked(chp);

			KASSERT(xfer != NULL);
			KASSERT((xfer->c_flags & C_POLL) == 0);

			switch ((rv = ata_xfer_start(xfer))) {
			case ATASTART_STARTED:
			case ATASTART_POLL:
			case ATASTART_ABORT:
				break;
			case ATASTART_TH:
			default:
				panic("%s: ata_xfer_start() unexpected rv %d",
				    __func__, rv);
				/* NOTREACHED */
			}
		} else if (chq->queue_freeze > 1)
			panic("%s: queue_freeze", __func__);
	}
	chp->ch_thread = NULL;
	cv_signal(&chp->ch_thr_idle);
	ata_channel_unlock(chp);
	kthread_exit(0);
}

static void
ata_thread_wake_locked(struct ata_channel *chp)
{
	KASSERT(mutex_owned(&chp->ch_lock));
	ata_channel_freeze_locked(chp);
	cv_signal(&chp->ch_thr_idle);
}

/*
 * atabus_match:
 *
 *	Autoconfiguration match routine.
 */
static int
atabus_match(device_t parent, cfdata_t cf, void *aux)
{
	struct ata_channel *chp = aux;

	if (chp == NULL)
		return (0);

	if (cf->cf_loc[ATACF_CHANNEL] != chp->ch_channel &&
	    cf->cf_loc[ATACF_CHANNEL] != ATACF_CHANNEL_DEFAULT)
		return (0);

	return (1);
}

/*
 * atabus_attach:
 *
 *	Autoconfiguration attach routine.
 */
static void
atabus_attach(device_t parent, device_t self, void *aux)
{
	struct atabus_softc *sc = device_private(self);
	struct ata_channel *chp = aux;
	struct atabus_initq *initq;
	int error;

	sc->sc_chan = chp;

	aprint_normal("\n");
	aprint_naive("\n");

	sc->sc_dev = self;

	if (ata_addref(chp))
		return;

	RUN_ONCE(&ata_init_ctrl, atabus_init);

	initq = malloc(sizeof(*initq), M_DEVBUF, M_WAITOK);
	initq->atabus_sc = sc;
	mutex_enter(&atabus_qlock);
	TAILQ_INSERT_TAIL(&atabus_initq_head, initq, atabus_initq);
	mutex_exit(&atabus_qlock);
	config_pending_incr(sc->sc_dev);

	if ((error = kthread_create(PRI_NONE, 0, NULL, atabus_thread, sc,
	    &chp->ch_thread, "%s", device_xname(self))) != 0)
		aprint_error_dev(self,
		    "unable to create kernel thread: error %d\n", error);

	if (!pmf_device_register(self, atabus_suspend, atabus_resume))
		aprint_error_dev(self, "couldn't establish power handler\n");
}

/*
 * atabus_detach:
 *
 *	Autoconfiguration detach routine.
 */
static int
atabus_detach(device_t self, int flags)
{
	struct atabus_softc *sc = device_private(self);
	struct ata_channel *chp = sc->sc_chan;
	device_t dev = NULL;
	int i, error = 0;

	/* Shutdown the channel. */
	ata_channel_lock(chp);
	chp->ch_flags |= ATACH_SHUTDOWN;
	while (chp->ch_thread != NULL) {
		cv_signal(&chp->ch_thr_idle);
		cv_wait(&chp->ch_thr_idle, &chp->ch_lock);
	}
	ata_channel_unlock(chp);

	/*
	 * Detach atapibus and its children.
	 */
	if ((dev = chp->atapibus) != NULL) {
		ATADEBUG_PRINT(("atabus_detach: %s: detaching %s\n",
		    device_xname(self), device_xname(dev)), DEBUG_DETACH);

		error = config_detach(dev, flags);
		if (error)
			goto out;
		KASSERT(chp->atapibus == NULL);
	}

	KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL);

	/*
	 * Detach our other children.
	 */
	for (i = 0; i < chp->ch_ndrives; i++) {
		if (chp->ch_drive[i].drive_type == ATA_DRIVET_ATAPI)
			continue;
		if (chp->ch_drive[i].drive_type == ATA_DRIVET_PM)
			chp->ch_drive[i].drive_type = ATA_DRIVET_NONE;
		if ((dev = chp->ch_drive[i].drv_softc) != NULL) {
			ATADEBUG_PRINT(("%s.%d: %s: detaching %s\n", __func__,
			    __LINE__, device_xname(self), device_xname(dev)),
			    DEBUG_DETACH);
			error = config_detach(dev, flags);
			if (error)
				goto out;
			KASSERT(chp->ch_drive[i].drv_softc == NULL);
			KASSERT(chp->ch_drive[i].drive_type == 0);
		}
	}
	atabus_free_drives(chp);

 out:
#ifdef ATADEBUG
	if (dev != NULL && error != 0)
		ATADEBUG_PRINT(("%s: %s: error %d detaching %s\n", __func__,
		    device_xname(self), error, device_xname(dev)),
		    DEBUG_DETACH);
#endif /* ATADEBUG */

	return (error);
}

void
atabus_childdetached(device_t self, device_t child)
{
	bool found = false;
	struct atabus_softc *sc = device_private(self);
	struct ata_channel *chp = sc->sc_chan;
	int i;

	KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL);
	/*
	 * atapibus detached.
	 */
	if (child == chp->atapibus) {
		chp->atapibus = NULL;
		found = true;
		for (i = 0; i < chp->ch_ndrives; i++) {
			if (chp->ch_drive[i].drive_type != ATA_DRIVET_ATAPI)
				continue;
			KASSERT(chp->ch_drive[i].drv_softc != NULL);
			chp->ch_drive[i].drv_softc = NULL;
			chp->ch_drive[i].drive_flags = 0;
			chp->ch_drive[i].drive_type = ATA_DRIVET_NONE;
		}
	}

	/*
	 * Detach our other children.
	 */
	for (i = 0; i < chp->ch_ndrives; i++) {
		if (chp->ch_drive[i].drive_type == ATA_DRIVET_ATAPI)
			continue;
		if (child == chp->ch_drive[i].drv_softc) {
			chp->ch_drive[i].drv_softc = NULL;
			chp->ch_drive[i].drive_flags = 0;
			if (chp->ch_drive[i].drive_type == ATA_DRIVET_PM)
				chp->ch_satapmp_nports = 0;
			chp->ch_drive[i].drive_type = ATA_DRIVET_NONE;
			found = true;
		}
	}

	if (!found)
		panic("%s: unknown child %p", device_xname(self),
		    (const void *)child);
}

CFATTACH_DECL3_NEW(atabus, sizeof(struct atabus_softc),
    atabus_match, atabus_attach, atabus_detach, NULL, atabus_rescan,
    atabus_childdetached, DVF_DETACH_SHUTDOWN);

/*****************************************************************************
 * Common ATA bus operations.
 *****************************************************************************/

/* allocate/free the channel's ch_drive[] array */
int
atabus_alloc_drives(struct ata_channel *chp, int ndrives)
{
	int i;
	if (chp->ch_ndrives != ndrives)
		atabus_free_drives(chp);
	if (chp->ch_drive == NULL) {
		chp->ch_drive = malloc(
		    sizeof(struct ata_drive_datas) * ndrives,
		    M_DEVBUF, M_NOWAIT | M_ZERO);
	}
	if (chp->ch_drive == NULL) {
	    aprint_error_dev(chp->ch_atac->atac_dev,
		"can't alloc drive array\n");
	    chp->ch_ndrives = 0;
	    return ENOMEM;
	};
	for (i = 0; i < ndrives; i++) {
		chp->ch_drive[i].chnl_softc = chp;
		chp->ch_drive[i].drive = i;
	}
	chp->ch_ndrives = ndrives;
	return 0;
}

void
atabus_free_drives(struct ata_channel *chp)
{
#ifdef DIAGNOSTIC
	int i;
	int dopanic = 0;
	KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL);
	for (i = 0; i < chp->ch_ndrives; i++) {
		if (chp->ch_drive[i].drive_type != ATA_DRIVET_NONE) {
			printf("%s: ch_drive[%d] type %d != ATA_DRIVET_NONE\n",
			    device_xname(chp->atabus), i,
			    chp->ch_drive[i].drive_type);
			dopanic = 1;
		}
		if (chp->ch_drive[i].drv_softc != NULL) {
			printf("%s: ch_drive[%d] attached to %s\n",
			    device_xname(chp->atabus), i,
			    device_xname(chp->ch_drive[i].drv_softc));
			dopanic = 1;
		}
	}
	if (dopanic)
		panic("atabus_free_drives");
#endif

	if (chp->ch_drive == NULL)
		return;
	chp->ch_ndrives = 0;
	free(chp->ch_drive, M_DEVBUF);
	chp->ch_drive = NULL;
}

/* Get the disk's parameters */
int
ata_get_params(struct ata_drive_datas *drvp, uint8_t flags,
    struct ataparams *prms)
{
	struct ata_xfer *xfer;
	struct ata_channel *chp = drvp->chnl_softc;
	struct atac_softc *atac = chp->ch_atac;
	char *tb;
	int i, rv;
	uint16_t *p;

	ATADEBUG_PRINT(("%s\n", __func__), DEBUG_FUNCS);

	xfer = ata_get_xfer(chp);
	if (xfer == NULL) {
		ATADEBUG_PRINT(("%s: no xfer\n", __func__),
		    DEBUG_FUNCS|DEBUG_PROBE);
		return CMD_AGAIN;
	}

	tb = kmem_zalloc(ATA_BSIZE, KM_SLEEP);
	memset(prms, 0, sizeof(struct ataparams));

	if (drvp->drive_type == ATA_DRIVET_ATA) {
		xfer->c_ata_c.r_command = WDCC_IDENTIFY;
		xfer->c_ata_c.r_st_bmask = WDCS_DRDY;
		xfer->c_ata_c.r_st_pmask = WDCS_DRQ;
		xfer->c_ata_c.timeout = 3000; /* 3s */
	} else if (drvp->drive_type == ATA_DRIVET_ATAPI) {
		xfer->c_ata_c.r_command = ATAPI_IDENTIFY_DEVICE;
		xfer->c_ata_c.r_st_bmask = 0;
		xfer->c_ata_c.r_st_pmask = WDCS_DRQ;
		xfer->c_ata_c.timeout = 10000; /* 10s */
	} else {
		ATADEBUG_PRINT(("ata_get_parms: no disks\n"),
		    DEBUG_FUNCS|DEBUG_PROBE);
		rv = CMD_ERR;
		goto out;
	}
	xfer->c_ata_c.flags = AT_READ | flags;
	xfer->c_ata_c.data = tb;
	xfer->c_ata_c.bcount = ATA_BSIZE;
	if ((*atac->atac_bustype_ata->ata_exec_command)(drvp,
						xfer) != ATACMD_COMPLETE) {
		ATADEBUG_PRINT(("ata_get_parms: wdc_exec_command failed\n"),
		    DEBUG_FUNCS|DEBUG_PROBE);
		rv = CMD_AGAIN;
		goto out;
	}
	if (xfer->c_ata_c.flags & (AT_ERROR | AT_TIMEOU | AT_DF)) {
		ATADEBUG_PRINT(("ata_get_parms: ata_c.flags=0x%x\n",
		    xfer->c_ata_c.flags), DEBUG_FUNCS|DEBUG_PROBE);
		rv = CMD_ERR;
		goto out;
	}
	/* if we didn't read any data something is wrong */
	if ((xfer->c_ata_c.flags & AT_XFDONE) == 0) {
		rv = CMD_ERR;
		goto out;
	}

	/* Read in parameter block. */
	memcpy(prms, tb, sizeof(struct ataparams));

	/*
	 * Shuffle string byte order.
	 * ATAPI NEC, Mitsumi and Pioneer drives and
	 * old ATA TDK CompactFlash cards
	 * have different byte order.
	 */
#if BYTE_ORDER == BIG_ENDIAN
# define M(n)	prms->atap_model[(n) ^ 1]
#else
# define M(n)	prms->atap_model[n]
#endif
	if (
#if BYTE_ORDER == BIG_ENDIAN
	    !
#endif
	    ((drvp->drive_type == ATA_DRIVET_ATAPI) ?
	     ((M(0) == 'N' && M(1) == 'E') ||
	      (M(0) == 'F' && M(1) == 'X') ||
	      (M(0) == 'P' && M(1) == 'i')) :
	     ((M(0) == 'T' && M(1) == 'D' && M(2) == 'K')))) {
		rv = CMD_OK;
		goto out;
	     }
#undef M
	for (i = 0; i < sizeof(prms->atap_model); i += 2) {
		p = (uint16_t *)(prms->atap_model + i);
		*p = bswap16(*p);
	}
	for (i = 0; i < sizeof(prms->atap_serial); i += 2) {
		p = (uint16_t *)(prms->atap_serial + i);
		*p = bswap16(*p);
	}
	for (i = 0; i < sizeof(prms->atap_revision); i += 2) {
		p = (uint16_t *)(prms->atap_revision + i);
		*p = bswap16(*p);
	}

	rv = CMD_OK;
 out:
	kmem_free(tb, ATA_BSIZE);
	ata_free_xfer(chp, xfer);
	return rv;
}

int
ata_set_mode(struct ata_drive_datas *drvp, uint8_t mode, uint8_t flags)
{
	struct ata_xfer *xfer;
	int rv;
	struct ata_channel *chp = drvp->chnl_softc;
	struct atac_softc *atac = chp->ch_atac;

	ATADEBUG_PRINT(("ata_set_mode=0x%x\n", mode), DEBUG_FUNCS);

	xfer = ata_get_xfer(chp);
	if (xfer == NULL) {
		ATADEBUG_PRINT(("%s: no xfer\n", __func__),
		    DEBUG_FUNCS|DEBUG_PROBE);
		return CMD_AGAIN;
	}

	xfer->c_ata_c.r_command = SET_FEATURES;
	xfer->c_ata_c.r_st_bmask = 0;
	xfer->c_ata_c.r_st_pmask = 0;
	xfer->c_ata_c.r_features = WDSF_SET_MODE;
	xfer->c_ata_c.r_count = mode;
	xfer->c_ata_c.flags = flags;
	xfer->c_ata_c.timeout = 1000; /* 1s */
	if ((*atac->atac_bustype_ata->ata_exec_command)(drvp,
						xfer) != ATACMD_COMPLETE) {
		rv = CMD_AGAIN;
		goto out;
	}
	if (xfer->c_ata_c.flags & (AT_ERROR | AT_TIMEOU | AT_DF)) {
		rv = CMD_ERR;
		goto out;
	}

	rv = CMD_OK;

out:
	ata_free_xfer(chp, xfer);
	return rv;
}

int
ata_read_log_ext_ncq(struct ata_drive_datas *drvp, uint8_t flags,
    uint8_t *slot, uint8_t *status, uint8_t *err)
{
	struct ata_xfer *xfer;
	int rv;
	struct ata_channel *chp = drvp->chnl_softc;
	struct atac_softc *atac = chp->ch_atac;
	uint8_t *tb, cksum, page;

	ATADEBUG_PRINT(("%s\n", __func__), DEBUG_FUNCS);

	/* Only NCQ ATA drives support/need this */
	if (drvp->drive_type != ATA_DRIVET_ATA ||
	    (drvp->drive_flags & ATA_DRIVE_NCQ) == 0)
		return EOPNOTSUPP;

	xfer = ata_get_xfer_ext(chp, C_RECOVERY, 0);

	tb = drvp->recovery_blk;
	memset(tb, 0, sizeof(drvp->recovery_blk));

	/*
	 * We could use READ LOG DMA EXT if drive supports it (i.e.
	 * when it supports Streaming feature) to avoid PIO command,
	 * and to make this a little faster. Realistically, it
	 * should not matter.
	 */
	xfer->c_flags |= C_RECOVERY;
	xfer->c_ata_c.r_command = WDCC_READ_LOG_EXT;
	xfer->c_ata_c.r_lba = page = WDCC_LOG_PAGE_NCQ;
	xfer->c_ata_c.r_st_bmask = WDCS_DRDY;
	xfer->c_ata_c.r_st_pmask = WDCS_DRDY;
	xfer->c_ata_c.r_count = 1;
	xfer->c_ata_c.r_device = WDSD_LBA;
	xfer->c_ata_c.flags = AT_READ | AT_LBA | AT_LBA48 | flags;
	xfer->c_ata_c.timeout = 1000; /* 1s */
	xfer->c_ata_c.data = tb;
	xfer->c_ata_c.bcount = sizeof(drvp->recovery_blk);

	if ((*atac->atac_bustype_ata->ata_exec_command)(drvp,
						xfer) != ATACMD_COMPLETE) {
		rv = EAGAIN;
		goto out;
	}
	if (xfer->c_ata_c.flags & (AT_ERROR | AT_TIMEOU | AT_DF)) {
		rv = EINVAL;
		goto out;
	}

	cksum = 0;
	for (int i = 0; i < sizeof(drvp->recovery_blk); i++)
		cksum += tb[i];
	if (cksum != 0) {
		aprint_error_dev(drvp->drv_softc,
		    "invalid checksum %x for READ LOG EXT page %x\n",
		    cksum, page);
		rv = EINVAL;
		goto out;
	}

	if (tb[0] & WDCC_LOG_NQ) {
		/* not queued command */
		rv = EOPNOTSUPP;
		goto out;
	}

	*slot = tb[0] & 0x1f;
	*status = tb[2];
	*err = tb[3];

	KASSERTMSG((*status & WDCS_ERR),
	    "%s: non-error command slot %d reported by READ LOG EXT page %x: "
	    "err %x status %x\n",
	    device_xname(drvp->drv_softc), *slot, page, *err, *status);

	rv = 0;

out:
	ata_free_xfer(chp, xfer);
	return rv;
}

#if NATA_DMA
void
ata_dmaerr(struct ata_drive_datas *drvp, int flags)
{
	/*
	 * Downgrade decision: if we get NERRS_MAX in NXFER.
	 * We start with n_dmaerrs set to NERRS_MAX-1 so that the
	 * first error within the first NXFER ops will immediatly trigger
	 * a downgrade.
	 * If we got an error and n_xfers is bigger than NXFER reset counters.
	 */
	drvp->n_dmaerrs++;
	if (drvp->n_dmaerrs >= NERRS_MAX && drvp->n_xfers <= NXFER) {
		ata_downgrade_mode(drvp, flags);
		drvp->n_dmaerrs = NERRS_MAX-1;
		drvp->n_xfers = 0;
		return;
	}
	if (drvp->n_xfers > NXFER) {
		drvp->n_dmaerrs = 1; /* just got an error */
		drvp->n_xfers = 1; /* restart counting from this error */
	}
}
#endif	/* NATA_DMA */

/*
 * freeze the queue and wait for the controller to be idle. Caller has to
 * unfreeze/restart the queue
 */
static void
ata_channel_idle(struct ata_channel *chp)
{
	ata_channel_lock(chp);
	ata_channel_freeze_locked(chp);
	while (chp->ch_queue->queue_active > 0) {
		chp->ch_queue->queue_flags |= QF_IDLE_WAIT;
		cv_timedwait(&chp->ch_queue->queue_idle, &chp->ch_lock, 1);
	}
	ata_channel_unlock(chp);
}

/*
 * Add a command to the queue and start controller.
 *
 * MUST BE CALLED AT splbio()!
 */
void
ata_exec_xfer(struct ata_channel *chp, struct ata_xfer *xfer)
{

	ATADEBUG_PRINT(("ata_exec_xfer %p channel %d drive %d\n", xfer,
	    chp->ch_channel, xfer->c_drive), DEBUG_XFERS);

	/* complete xfer setup */
	xfer->c_chp = chp;

	ata_channel_lock(chp);

	/*
	 * Standard commands are added to the end of command list, but
	 * recovery commands must be run immediatelly.
	 */
	if ((xfer->c_flags & C_RECOVERY) == 0)
		TAILQ_INSERT_TAIL(&chp->ch_queue->queue_xfer, xfer,
		    c_xferchain);
	else
		TAILQ_INSERT_HEAD(&chp->ch_queue->queue_xfer, xfer,
		    c_xferchain);

	/*
	 * if polling and can sleep, wait for the xfer to be at head of queue
	 */
	if ((xfer->c_flags & (C_POLL | C_WAIT)) ==  (C_POLL | C_WAIT)) {
		while (chp->ch_queue->queue_active > 0 ||
		    TAILQ_FIRST(&chp->ch_queue->queue_xfer) != xfer) {
			xfer->c_flags |= C_WAITACT;
			cv_wait(&xfer->c_active, &chp->ch_lock);
			xfer->c_flags &= ~C_WAITACT;

			/*
			 * Free xfer now if it there was attempt to free it
			 * while we were waiting.
			 */
			if ((xfer->c_flags & (C_FREE|C_WAITTIMO)) == C_FREE) {
				ata_channel_unlock(chp);

				ata_free_xfer(chp, xfer);
				return;
			}
		}
	}

	ata_channel_unlock(chp);

	ATADEBUG_PRINT(("atastart from ata_exec_xfer, flags 0x%x\n",
	    chp->ch_flags), DEBUG_XFERS);
	atastart(chp);
}

/*
 * Start I/O on a controller, for the given channel.
 * The first xfer may be not for our channel if the channel queues
 * are shared.
 *
 * MUST BE CALLED AT splbio()!
 */
void
atastart(struct ata_channel *chp)
{
	struct atac_softc *atac = chp->ch_atac;
	struct ata_queue *chq = chp->ch_queue;
	struct ata_xfer *xfer, *axfer;
	bool recovery;

#ifdef ATA_DEBUG
	int spl1, spl2;

	spl1 = splbio();
	spl2 = splbio();
	if (spl2 != spl1) {
		printf("atastart: not at splbio()\n");
		panic("atastart");
	}
	splx(spl2);
	splx(spl1);
#endif /* ATA_DEBUG */

	ata_channel_lock(chp);

again:
	KASSERT(chq->queue_active <= chq->queue_openings);
	if (chq->queue_active == chq->queue_openings) {
		goto out; /* channel completely busy */
	}

	/* is there a xfer ? */
	if ((xfer = TAILQ_FIRST(&chp->ch_queue->queue_xfer)) == NULL)
		goto out;

	recovery = ISSET(xfer->c_flags, C_RECOVERY);

	/* is the queue frozen? */
	if (__predict_false(!recovery && chq->queue_freeze > 0)) {
		if (chq->queue_flags & QF_IDLE_WAIT) {
			chq->queue_flags &= ~QF_IDLE_WAIT;
			cv_signal(&chp->ch_queue->queue_idle);
		}
		goto out; /* queue frozen */
	}

	/* all xfers on same queue must belong to the same channel */
	KASSERT(xfer->c_chp == chp);

	/*
	 * Can only take the command if there are no current active
	 * commands, or if the command is NCQ and the active commands are also
	 * NCQ. If PM is in use and HBA driver doesn't support/use FIS-based
	 * switching, can only send commands to single drive.
	 * Need only check first xfer.
	 * XXX FIS-based switching - revisit
	 */
	if (!recovery && (axfer = TAILQ_FIRST(&chp->ch_queue->active_xfers))) {
		if (!ISSET(xfer->c_flags, C_NCQ) ||
		    !ISSET(axfer->c_flags, C_NCQ) ||
		    xfer->c_drive != axfer->c_drive)
			goto out;
	}

	struct ata_drive_datas * const drvp = &chp->ch_drive[xfer->c_drive];

	/*
	 * if someone is waiting for the command to be active, wake it up
	 * and let it process the command
	 */
	if (xfer->c_flags & C_WAITACT) {
		ATADEBUG_PRINT(("atastart: xfer %p channel %d drive %d "
		    "wait active\n", xfer, chp->ch_channel, xfer->c_drive),
		    DEBUG_XFERS);
		cv_signal(&xfer->c_active);
		goto out;
	}

	if (atac->atac_claim_hw)
		if (!atac->atac_claim_hw(chp, 0))
			goto out;

	ATADEBUG_PRINT(("atastart: xfer %p channel %d drive %d\n", xfer,
	    chp->ch_channel, xfer->c_drive), DEBUG_XFERS);
	if (drvp->drive_flags & ATA_DRIVE_RESET) {
		drvp->drive_flags &= ~ATA_DRIVE_RESET;
		drvp->state = 0;
	}

	if (ISSET(xfer->c_flags, C_NCQ))
		SET(chp->ch_flags, ATACH_NCQ);
	else
		CLR(chp->ch_flags, ATACH_NCQ);

	ata_activate_xfer_locked(chp, xfer);

	if (atac->atac_cap & ATAC_CAP_NOIRQ)
		KASSERT(xfer->c_flags & C_POLL);

	switch (ata_xfer_start(xfer)) {
	case ATASTART_TH:
	case ATASTART_ABORT:
		/* don't start any further commands in this case */
		goto out;
	default:
		/* nothing to do */
		break;
	}

	/* Queue more commands if possible, but not during recovery */
	if (!recovery && chq->queue_active < chq->queue_openings)
		goto again;

out:
	ata_channel_unlock(chp);
}

int
ata_xfer_start(struct ata_xfer *xfer)
{
	struct ata_channel *chp = xfer->c_chp;
	int rv;

	KASSERT(mutex_owned(&chp->ch_lock));

	rv = xfer->c_start(chp, xfer);
	switch (rv) {
	case ATASTART_STARTED:
		/* nothing to do */
		break;
	case ATASTART_TH:
		/* postpone xfer to thread */
		ata_thread_wake_locked(chp);
		break;
	case ATASTART_POLL:
		/* can happen even in thread context for some ATAPI devices */
		ata_channel_unlock(chp);
		KASSERT(xfer->c_poll != NULL);
		xfer->c_poll(chp, xfer);
		ata_channel_lock(chp);
		break;
	case ATASTART_ABORT:
		ata_channel_unlock(chp);
		KASSERT(xfer->c_abort != NULL);
		xfer->c_abort(chp, xfer);
		ata_channel_lock(chp);
		break;
	}

	return rv;
}

/*
 * Does it's own locking, does not require splbio().
 * flags - whether to block waiting for free xfer
 * openings - limit of openings supported by device, <= 0 means tag not
 *     relevant, and any available xfer can be returned
 */
struct ata_xfer *
ata_get_xfer_ext(struct ata_channel *chp, int flags, uint8_t openings)
{
	struct ata_queue *chq = chp->ch_queue;
	struct ata_xfer *xfer = NULL;
	uint32_t avail, slot, mask;
	int error;

	ATADEBUG_PRINT(("%s: channel %d flags %x openings %d\n",
	    __func__, chp->ch_channel, flags, openings),
	    DEBUG_XFERS);

	ata_channel_lock(chp);

	/*
	 * When openings is just 1, can't reserve anything for
	 * recovery. KASSERT() here is to catch code which naively
	 * relies on C_RECOVERY to work under this condition.
	 */
	KASSERT((flags & C_RECOVERY) == 0 || chq->queue_openings > 1);

	if (flags & C_RECOVERY) {
		mask = UINT32_MAX;
	} else {
		if (openings <= 0 || openings > chq->queue_openings)
			openings = chq->queue_openings;

		if (openings > 1) {
			mask = __BIT(openings - 1) - 1;
		} else {
			mask = UINT32_MAX;
		}
	}

retry:
	avail = ffs32(chq->queue_xfers_avail & mask);
	if (avail == 0) {
		/*
		 * Catch code which tries to get another recovery xfer while
		 * already holding one (wrong recursion).
		 */
		KASSERTMSG((flags & C_RECOVERY) == 0,
		    "recovery xfer busy openings %d mask %x avail %x",
		    openings, mask, chq->queue_xfers_avail);

		if (flags & C_WAIT) {
			chq->queue_flags |= QF_NEED_XFER;
			error = cv_wait_sig(&chq->queue_busy, &chp->ch_lock);
			if (error == 0)
				goto retry;
		}

		goto out;
	}

	slot = avail - 1;
	xfer = &chq->queue_xfers[slot];
	chq->queue_xfers_avail &= ~__BIT(slot);

	KASSERT((chq->active_xfers_used & __BIT(slot)) == 0);

	/* zero everything after the callout member */
	memset(&xfer->c_startzero, 0,
	    sizeof(struct ata_xfer) - offsetof(struct ata_xfer, c_startzero));

out:
	ata_channel_unlock(chp);
	return xfer;
}

/*
 * ata_deactivate_xfer() must be always called prior to ata_free_xfer()
 */
void
ata_free_xfer(struct ata_channel *chp, struct ata_xfer *xfer)
{
	struct ata_queue *chq = chp->ch_queue;

	ata_channel_lock(chp);

	if (xfer->c_flags & (C_WAITACT|C_WAITTIMO)) {
		/* Someone is waiting for this xfer, so we can't free now */
		xfer->c_flags |= C_FREE;
		cv_signal(&xfer->c_active);
		goto out;
	}

#if NATA_PIOBM		/* XXX wdc dependent code */
	if (xfer->c_flags & C_PIOBM) {
		struct wdc_softc *wdc = CHAN_TO_WDC(chp);

		/* finish the busmastering PIO */
		(*wdc->piobm_done)(wdc->dma_arg,
		    chp->ch_channel, xfer->c_drive);
		chp->ch_flags &= ~(ATACH_DMA_WAIT | ATACH_PIOBM_WAIT);
	}
#endif

	if (chp->ch_atac->atac_free_hw)
		chp->ch_atac->atac_free_hw(chp);

	KASSERT((chq->active_xfers_used & __BIT(xfer->c_slot)) == 0);
	KASSERT((chq->queue_xfers_avail & __BIT(xfer->c_slot)) == 0);
	chq->queue_xfers_avail |= __BIT(xfer->c_slot);

out:
	if (chq->queue_flags & QF_NEED_XFER) {
		chq->queue_flags &= ~QF_NEED_XFER;
		cv_broadcast(&chq->queue_busy);
	}

	ata_channel_unlock(chp);
}

static void
ata_activate_xfer_locked(struct ata_channel *chp, struct ata_xfer *xfer)
{
	struct ata_queue * const chq = chp->ch_queue;

	KASSERT(mutex_owned(&chp->ch_lock));

	KASSERT(chq->queue_active < chq->queue_openings);
	KASSERT((chq->active_xfers_used & __BIT(xfer->c_slot)) == 0);

	TAILQ_REMOVE(&chq->queue_xfer, xfer, c_xferchain);
	if ((xfer->c_flags & C_RECOVERY) == 0)
		TAILQ_INSERT_TAIL(&chq->active_xfers, xfer, c_activechain);
	else {
		/*
		 * Must go to head, so that ata_queue_get_active_xfer()
		 * returns the recovery command, and not some other
		 * random active transfer.
		 */
		TAILQ_INSERT_HEAD(&chq->active_xfers, xfer, c_activechain);
	}
	chq->active_xfers_used |= __BIT(xfer->c_slot);
	chq->queue_active++;
}

void
ata_deactivate_xfer(struct ata_channel *chp, struct ata_xfer *xfer)
{
	struct ata_queue * const chq = chp->ch_queue;

	ata_channel_lock(chp);

	KASSERT(chq->queue_active > 0);
	KASSERT((chq->active_xfers_used & __BIT(xfer->c_slot)) != 0);

	callout_stop(&xfer->c_timo_callout);

	if (callout_invoking(&xfer->c_timo_callout))
		xfer->c_flags |= C_WAITTIMO;

	TAILQ_REMOVE(&chq->active_xfers, xfer, c_activechain);
	chq->active_xfers_used &= ~__BIT(xfer->c_slot);
	chq->queue_active--;

	ata_channel_unlock(chp);
}

/*
 * Called in c_intr hook. Must be called before before any deactivations
 * are done - if there is drain pending, it calls c_kill_xfer hook which
 * deactivates the xfer.
 * Calls c_kill_xfer with channel lock free.
 * Returns true if caller should just exit without further processing.
 * Caller must not further access any part of xfer or any related controller
 * structures in that case, it should just return.
 */
bool
ata_waitdrain_xfer_check(struct ata_channel *chp, struct ata_xfer *xfer)
{
	int drive = xfer->c_drive;
	bool draining = false;

	ata_channel_lock(chp);

	if (chp->ch_drive[drive].drive_flags & ATA_DRIVE_WAITDRAIN) {
		ata_channel_unlock(chp);

		(*xfer->c_kill_xfer)(chp, xfer, KILL_GONE);

		ata_channel_lock(chp);
		chp->ch_drive[drive].drive_flags &= ~ATA_DRIVE_WAITDRAIN;
		cv_signal(&chp->ch_queue->queue_drain);
		draining = true;
	}

	ata_channel_unlock(chp);

	return draining;
}

/*
 * Check for race of normal transfer handling vs. timeout.
 */
bool
ata_timo_xfer_check(struct ata_xfer *xfer)
{
	struct ata_channel *chp = xfer->c_chp;
	struct ata_drive_datas *drvp = &chp->ch_drive[xfer->c_drive];

	ata_channel_lock(chp);

	callout_ack(&xfer->c_timo_callout);

	if (xfer->c_flags & C_WAITTIMO) {
		xfer->c_flags &= ~C_WAITTIMO;

		/* Handle race vs. ata_free_xfer() */
		if (xfer->c_flags & C_FREE) {
			xfer->c_flags &= ~C_FREE;
			ata_channel_unlock(chp);

	    		aprint_normal_dev(drvp->drv_softc,
			    "xfer %d freed while invoking timeout\n",
			    xfer->c_slot);

			ata_free_xfer(chp, xfer);
			return true;
		}

		/* Race vs. callout_stop() in ata_deactivate_xfer() */
		ata_channel_unlock(chp);

	    	aprint_normal_dev(drvp->drv_softc,
		    "xfer %d deactivated while invoking timeout\n",
		    xfer->c_slot);
		return true;
	}

	ata_channel_unlock(chp);

	/* No race, proceed with timeout handling */
	return false;
}

void
ata_timeout(void *v)
{
	struct ata_xfer *xfer = v;
	int s;

	ATADEBUG_PRINT(("%s: slot %d\n", __func__, xfer->c_slot),
	    DEBUG_FUNCS|DEBUG_XFERS);

	s = splbio();				/* XXX MPSAFE */

	if (ata_timo_xfer_check(xfer)) {
		/* Already logged */
		goto out;
	}

	/* Mark as timed out. Do not print anything, wd(4) will. */
	xfer->c_flags |= C_TIMEOU;
	xfer->c_intr(xfer->c_chp, xfer, 0);

out:
	splx(s);
}

/*
 * Kill off all active xfers for a ata_channel.
 *
 * Must be called with channel lock held.
 */
void
ata_kill_active(struct ata_channel *chp, int reason, int flags)
{
	struct ata_queue * const chq = chp->ch_queue;
	struct ata_xfer *xfer, *xfernext;

	KASSERT(mutex_owned(&chp->ch_lock));

	TAILQ_FOREACH_SAFE(xfer, &chq->active_xfers, c_activechain, xfernext) {
		(*xfer->c_kill_xfer)(xfer->c_chp, xfer, reason);
	}

	if (flags & AT_RST_EMERG)
		ata_queue_reset(chq);
}

/*
 * Kill off all pending xfers for a drive.
 */
void
ata_kill_pending(struct ata_drive_datas *drvp)
{
	struct ata_channel * const chp = drvp->chnl_softc;
	struct ata_queue * const chq = chp->ch_queue;
	struct ata_xfer *xfer, *xfernext;

	ata_channel_lock(chp);

	/* Kill all pending transfers */
	TAILQ_FOREACH_SAFE(xfer, &chq->queue_xfer, c_xferchain, xfernext) {
		KASSERT(xfer->c_chp == chp);

		if (xfer->c_drive != drvp->drive)
			continue;

		TAILQ_REMOVE(&chp->ch_queue->queue_xfer, xfer, c_xferchain);

		/*
		 * Keep the lock, so that we get deadlock (and 'locking against
		 * myself' with LOCKDEBUG), instead of silent
		 * data corruption, if the hook tries to call back into
		 * middle layer for inactive xfer.
		 */
		(*xfer->c_kill_xfer)(chp, xfer, KILL_GONE_INACTIVE);
	}

	/* Wait until all active transfers on the drive finish */
	while (chq->queue_active > 0) {
		bool drv_active = false;

		TAILQ_FOREACH(xfer, &chq->active_xfers, c_activechain) {
			KASSERT(xfer->c_chp == chp);

			if (xfer->c_drive == drvp->drive) {
				drv_active = true;
				break;
			}
		}

		if (!drv_active) {
			/* all finished */
			break;
		}

		drvp->drive_flags |= ATA_DRIVE_WAITDRAIN;
		cv_wait(&chq->queue_drain, &chp->ch_lock);
	}

	ata_channel_unlock(chp);
}

static void
ata_channel_freeze_locked(struct ata_channel *chp)
{
	chp->ch_queue->queue_freeze++;
}

void
ata_channel_freeze(struct ata_channel *chp)
{
	ata_channel_lock(chp);
	ata_channel_freeze_locked(chp);
	ata_channel_unlock(chp);
}

static void
ata_channel_thaw_locked(struct ata_channel *chp)
{
	KASSERT(mutex_owned(&chp->ch_lock));

	chp->ch_queue->queue_freeze--;
}

void
ata_channel_thaw(struct ata_channel *chp)
{
	ata_channel_lock(chp);
	ata_channel_thaw_locked(chp);
	ata_channel_unlock(chp);
}

/*
 * ata_reset_channel:
 *
 *	Reset and ATA channel.
 *
 *	MUST BE CALLED AT splbio()!
 */
void
ata_reset_channel(struct ata_channel *chp, int flags)
{
	struct atac_softc *atac = chp->ch_atac;
	int drive;

#ifdef ATA_DEBUG
	int spl1, spl2;

	spl1 = splbio();
	spl2 = splbio();
	if (spl2 != spl1) {
		printf("ata_reset_channel: not at splbio()\n");
		panic("ata_reset_channel");
	}
	splx(spl2);
	splx(spl1);
#endif /* ATA_DEBUG */

	ata_channel_freeze(chp);

	/*
	 * If we can poll or wait it's OK, otherwise wake up the
	 * kernel thread to do it for us.
	 */
	ATADEBUG_PRINT(("ata_reset_channel flags 0x%x ch_flags 0x%x\n",
	    flags, chp->ch_flags), DEBUG_FUNCS | DEBUG_XFERS);
	if ((flags & (AT_POLL | AT_WAIT)) == 0) {
		if (chp->ch_flags & ATACH_TH_RESET) {
			/* No need to schedule a reset more than one time. */
			ata_channel_thaw(chp);
			return;
		}
		ata_channel_lock(chp);
		chp->ch_flags |= ATACH_TH_RESET;
		chp->ch_reset_flags = flags & AT_RST_EMERG;
		cv_signal(&chp->ch_thr_idle);
		ata_channel_unlock(chp);
		return;
	}

	(*atac->atac_bustype_ata->ata_reset_channel)(chp, flags);

	ata_channel_lock(chp);
	KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL);
	for (drive = 0; drive < chp->ch_ndrives; drive++)
		chp->ch_drive[drive].state = 0;

	chp->ch_flags &= ~ATACH_TH_RESET;
	ata_channel_unlock(chp);

	if (flags & AT_RST_EMERG) {
		/* make sure that we can use polled commands */
		ata_queue_reset(chp->ch_queue);
	} else {
		ata_channel_thaw(chp);
		atastart(chp);
	}
}

int
ata_addref(struct ata_channel *chp)
{
	struct atac_softc *atac = chp->ch_atac;
	struct scsipi_adapter *adapt = &atac->atac_atapi_adapter._generic;
	int s, error = 0;

	s = splbio();
	if (adapt->adapt_refcnt++ == 0 &&
	    adapt->adapt_enable != NULL) {
		error = (*adapt->adapt_enable)(atac->atac_dev, 1);
		if (error)
			adapt->adapt_refcnt--;
	}
	splx(s);
	return (error);
}

void
ata_delref(struct ata_channel *chp)
{
	struct atac_softc *atac = chp->ch_atac;
	struct scsipi_adapter *adapt = &atac->atac_atapi_adapter._generic;
	int s;

	s = splbio();
	if (adapt->adapt_refcnt-- == 1 &&
	    adapt->adapt_enable != NULL)
		(void) (*adapt->adapt_enable)(atac->atac_dev, 0);
	splx(s);
}

void
ata_print_modes(struct ata_channel *chp)
{
	struct atac_softc *atac = chp->ch_atac;
	int drive;
	struct ata_drive_datas *drvp;

	KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL);
	for (drive = 0; drive < chp->ch_ndrives; drive++) {
		drvp = &chp->ch_drive[drive];
		if (drvp->drive_type == ATA_DRIVET_NONE ||
		    drvp->drv_softc == NULL)
			continue;
		aprint_verbose("%s(%s:%d:%d): using PIO mode %d",
			device_xname(drvp->drv_softc),
			device_xname(atac->atac_dev),
			chp->ch_channel, drvp->drive, drvp->PIO_mode);
#if NATA_DMA
		if (drvp->drive_flags & ATA_DRIVE_DMA)
			aprint_verbose(", DMA mode %d", drvp->DMA_mode);
#if NATA_UDMA
		if (drvp->drive_flags & ATA_DRIVE_UDMA) {
			aprint_verbose(", Ultra-DMA mode %d", drvp->UDMA_mode);
			if (drvp->UDMA_mode == 2)
				aprint_verbose(" (Ultra/33)");
			else if (drvp->UDMA_mode == 4)
				aprint_verbose(" (Ultra/66)");
			else if (drvp->UDMA_mode == 5)
				aprint_verbose(" (Ultra/100)");
			else if (drvp->UDMA_mode == 6)
				aprint_verbose(" (Ultra/133)");
		}
#endif	/* NATA_UDMA */
#endif	/* NATA_DMA */
#if NATA_DMA || NATA_PIOBM
		if (0
#if NATA_DMA
		    || (drvp->drive_flags & (ATA_DRIVE_DMA | ATA_DRIVE_UDMA))
#endif
#if NATA_PIOBM
		    /* PIOBM capable controllers use DMA for PIO commands */
		    || (atac->atac_cap & ATAC_CAP_PIOBM)
#endif
		    )
			aprint_verbose(" (using DMA)");

		if (drvp->drive_flags & ATA_DRIVE_NCQ) {
			aprint_verbose(", NCQ (%d tags)%s",
			    ATA_REAL_OPENINGS(chp->ch_queue->queue_openings),
			    (drvp->drive_flags & ATA_DRIVE_NCQ_PRIO)
			    ? " w/PRIO" : "");
		} else if (drvp->drive_flags & ATA_DRIVE_WFUA)
			aprint_verbose(", WRITE DMA FUA EXT");

#endif	/* NATA_DMA || NATA_PIOBM */
		aprint_verbose("\n");
	}
}

#if NATA_DMA
/*
 * downgrade the transfer mode of a drive after an error. return 1 if
 * downgrade was possible, 0 otherwise.
 *
 * MUST BE CALLED AT splbio()!
 */
int
ata_downgrade_mode(struct ata_drive_datas *drvp, int flags)
{
	struct ata_channel *chp = drvp->chnl_softc;
	struct atac_softc *atac = chp->ch_atac;
	device_t drv_dev = drvp->drv_softc;
	int cf_flags = device_cfdata(drv_dev)->cf_flags;

	/* if drive or controller don't know its mode, we can't do much */
	if ((drvp->drive_flags & ATA_DRIVE_MODE) == 0 ||
	    (atac->atac_set_modes == NULL))
		return 0;
	/* current drive mode was set by a config flag, let it this way */
	if ((cf_flags & ATA_CONFIG_PIO_SET) ||
	    (cf_flags & ATA_CONFIG_DMA_SET) ||
	    (cf_flags & ATA_CONFIG_UDMA_SET))
		return 0;

#if NATA_UDMA
	/*
	 * If we were using Ultra-DMA mode, downgrade to the next lower mode.
	 */
	if ((drvp->drive_flags & ATA_DRIVE_UDMA) && drvp->UDMA_mode >= 2) {
		drvp->UDMA_mode--;
		aprint_error_dev(drv_dev,
		    "transfer error, downgrading to Ultra-DMA mode %d\n",
		    drvp->UDMA_mode);
	}
#endif

	/*
	 * If we were using ultra-DMA, don't downgrade to multiword DMA.
	 */
	else if (drvp->drive_flags & (ATA_DRIVE_DMA | ATA_DRIVE_UDMA)) {
		drvp->drive_flags &= ~(ATA_DRIVE_DMA | ATA_DRIVE_UDMA);
		drvp->PIO_mode = drvp->PIO_cap;
		aprint_error_dev(drv_dev,
		    "transfer error, downgrading to PIO mode %d\n",
		    drvp->PIO_mode);
	} else /* already using PIO, can't downgrade */
		return 0;

	(*atac->atac_set_modes)(chp);
	ata_print_modes(chp);
	/* reset the channel, which will schedule all drives for setup */
	ata_reset_channel(chp, flags);
	return 1;
}
#endif	/* NATA_DMA */

/*
 * Probe drive's capabilities, for use by the controller later
 * Assumes drvp points to an existing drive.
 */
void
ata_probe_caps(struct ata_drive_datas *drvp)
{
	struct ataparams params, params2;
	struct ata_channel *chp = drvp->chnl_softc;
	struct atac_softc *atac = chp->ch_atac;
	device_t drv_dev = drvp->drv_softc;
	int i, printed = 0;
	const char *sep = "";
	int cf_flags;

	if (ata_get_params(drvp, AT_WAIT, &params) != CMD_OK) {
		/* IDENTIFY failed. Can't tell more about the device */
		return;
	}
	if ((atac->atac_cap & (ATAC_CAP_DATA16 | ATAC_CAP_DATA32)) ==
	    (ATAC_CAP_DATA16 | ATAC_CAP_DATA32)) {
		/*
		 * Controller claims 16 and 32 bit transfers.
		 * Re-do an IDENTIFY with 32-bit transfers,
		 * and compare results.
		 */
		ata_channel_lock(chp);
		drvp->drive_flags |= ATA_DRIVE_CAP32;
		ata_channel_unlock(chp);
		ata_get_params(drvp, AT_WAIT, &params2);
		if (memcmp(&params, &params2, sizeof(struct ataparams)) != 0) {
			/* Not good. fall back to 16bits */
			ata_channel_lock(chp);
			drvp->drive_flags &= ~ATA_DRIVE_CAP32;
			ata_channel_unlock(chp);
		} else {
			aprint_verbose_dev(drv_dev, "32-bit data port\n");
		}
	}
#if 0 /* Some ultra-DMA drives claims to only support ATA-3. sigh */
	if (params.atap_ata_major > 0x01 &&
	    params.atap_ata_major != 0xffff) {
		for (i = 14; i > 0; i--) {
			if (params.atap_ata_major & (1 << i)) {
				aprint_verbose_dev(drv_dev,
				    "ATA version %d\n", i);
				drvp->ata_vers = i;
				break;
			}
		}
	}
#endif

	/* An ATAPI device is at last PIO mode 3 */
	if (drvp->drive_type == ATA_DRIVET_ATAPI)
		drvp->PIO_mode = 3;

	/*
	 * It's not in the specs, but it seems that some drive
	 * returns 0xffff in atap_extensions when this field is invalid
	 */
	if (params.atap_extensions != 0xffff &&
	    (params.atap_extensions & WDC_EXT_MODES)) {
		/*
		 * XXX some drives report something wrong here (they claim to
		 * support PIO mode 8 !). As mode is coded on 3 bits in
		 * SET FEATURE, limit it to 7 (so limit i to 4).
		 * If higher mode than 7 is found, abort.
		 */
		for (i = 7; i >= 0; i--) {
			if ((params.atap_piomode_supp & (1 << i)) == 0)
				continue;
			if (i > 4)
				return;
			/*
			 * See if mode is accepted.
			 * If the controller can't set its PIO mode,
			 * assume the defaults are good, so don't try
			 * to set it
			 */
			if (atac->atac_set_modes)
				/*
				 * It's OK to pool here, it's fast enough
				 * to not bother waiting for interrupt
				 */
				if (ata_set_mode(drvp, 0x08 | (i + 3),
				   AT_WAIT) != CMD_OK)
					continue;
			if (!printed) {
				aprint_verbose_dev(drv_dev,
				    "drive supports PIO mode %d", i + 3);
				sep = ",";
				printed = 1;
			}
			/*
			 * If controller's driver can't set its PIO mode,
			 * get the highter one for the drive.
			 */
			if (atac->atac_set_modes == NULL ||
			    atac->atac_pio_cap >= i + 3) {
				drvp->PIO_mode = i + 3;
				drvp->PIO_cap = i + 3;
				break;
			}
		}
		if (!printed) {
			/*
			 * We didn't find a valid PIO mode.
			 * Assume the values returned for DMA are buggy too
			 */
			return;
		}
		ata_channel_lock(chp);
		drvp->drive_flags |= ATA_DRIVE_MODE;
		ata_channel_unlock(chp);
		printed = 0;
		for (i = 7; i >= 0; i--) {
			if ((params.atap_dmamode_supp & (1 << i)) == 0)
				continue;
#if NATA_DMA
			if ((atac->atac_cap & ATAC_CAP_DMA) &&
			    atac->atac_set_modes != NULL)
				if (ata_set_mode(drvp, 0x20 | i, AT_WAIT)
				    != CMD_OK)
					continue;
#endif
			if (!printed) {
				aprint_verbose("%s DMA mode %d", sep, i);
				sep = ",";
				printed = 1;
			}
#if NATA_DMA
			if (atac->atac_cap & ATAC_CAP_DMA) {
				if (atac->atac_set_modes != NULL &&
				    atac->atac_dma_cap < i)
					continue;
				drvp->DMA_mode = i;
				drvp->DMA_cap = i;
				ata_channel_lock(chp);
				drvp->drive_flags |= ATA_DRIVE_DMA;
				ata_channel_unlock(chp);
			}
#endif
			break;
		}
		if (params.atap_extensions & WDC_EXT_UDMA_MODES) {
			printed = 0;
			for (i = 7; i >= 0; i--) {
				if ((params.atap_udmamode_supp & (1 << i))
				    == 0)
					continue;
#if NATA_UDMA
				if (atac->atac_set_modes != NULL &&
				    (atac->atac_cap & ATAC_CAP_UDMA))
					if (ata_set_mode(drvp, 0x40 | i,
					    AT_WAIT) != CMD_OK)
						continue;
#endif
				if (!printed) {
					aprint_verbose("%s Ultra-DMA mode %d",
					    sep, i);
					if (i == 2)
						aprint_verbose(" (Ultra/33)");
					else if (i == 4)
						aprint_verbose(" (Ultra/66)");
					else if (i == 5)
						aprint_verbose(" (Ultra/100)");
					else if (i == 6)
						aprint_verbose(" (Ultra/133)");
					sep = ",";
					printed = 1;
				}
#if NATA_UDMA
				if (atac->atac_cap & ATAC_CAP_UDMA) {
					if (atac->atac_set_modes != NULL &&
					    atac->atac_udma_cap < i)
						continue;
					drvp->UDMA_mode = i;
					drvp->UDMA_cap = i;
					ata_channel_lock(chp);
					drvp->drive_flags |= ATA_DRIVE_UDMA;
					ata_channel_unlock(chp);
				}
#endif
				break;
			}
		}
	}

	ata_channel_lock(chp);
	drvp->drive_flags &= ~ATA_DRIVE_NOSTREAM;
	if (drvp->drive_type == ATA_DRIVET_ATAPI) {
		if (atac->atac_cap & ATAC_CAP_ATAPI_NOSTREAM)
			drvp->drive_flags |= ATA_DRIVE_NOSTREAM;
	} else {
		if (atac->atac_cap & ATAC_CAP_ATA_NOSTREAM)
			drvp->drive_flags |= ATA_DRIVE_NOSTREAM;
	}
	ata_channel_unlock(chp);

	/* Try to guess ATA version here, if it didn't get reported */
	if (drvp->ata_vers == 0) {
#if NATA_UDMA
		if (drvp->drive_flags & ATA_DRIVE_UDMA)
			drvp->ata_vers = 4; /* should be at last ATA-4 */
		else
#endif
		if (drvp->PIO_cap > 2)
			drvp->ata_vers = 2; /* should be at last ATA-2 */
	}
	cf_flags = device_cfdata(drv_dev)->cf_flags;
	if (cf_flags & ATA_CONFIG_PIO_SET) {
		ata_channel_lock(chp);
		drvp->PIO_mode =
		    (cf_flags & ATA_CONFIG_PIO_MODES) >> ATA_CONFIG_PIO_OFF;
		drvp->drive_flags |= ATA_DRIVE_MODE;
		ata_channel_unlock(chp);
	}
#if NATA_DMA
	if ((atac->atac_cap & ATAC_CAP_DMA) == 0) {
		/* don't care about DMA modes */
		return;
	}
	if (cf_flags & ATA_CONFIG_DMA_SET) {
		ata_channel_lock(chp);
		if ((cf_flags & ATA_CONFIG_DMA_MODES) ==
		    ATA_CONFIG_DMA_DISABLE) {
			drvp->drive_flags &= ~ATA_DRIVE_DMA;
		} else {
			drvp->DMA_mode = (cf_flags & ATA_CONFIG_DMA_MODES) >>
			    ATA_CONFIG_DMA_OFF;
			drvp->drive_flags |= ATA_DRIVE_DMA | ATA_DRIVE_MODE;
		}
		ata_channel_unlock(chp);
	}

	/*
	 * Probe WRITE DMA FUA EXT. Support is mandatory for devices
	 * supporting LBA48, but nevertheless confirm with the feature flag.
	 */
	if (drvp->drive_flags & ATA_DRIVE_DMA) {
		if ((params.atap_cmd2_en & ATA_CMD2_LBA48) != 0
		    && (params.atap_cmd_def & ATA_CMDE_WFE)) {
			drvp->drive_flags |= ATA_DRIVE_WFUA;
			aprint_verbose("%s WRITE DMA FUA", sep);
			sep = ",";
		}
	}

	/* Probe NCQ support - READ/WRITE FPDMA QUEUED command support */
	ata_channel_lock(chp);
	drvp->drv_openings = 1;
	if (params.atap_sata_caps & SATA_NATIVE_CMDQ) {
		if (atac->atac_cap & ATAC_CAP_NCQ)
			drvp->drive_flags |= ATA_DRIVE_NCQ;
		drvp->drv_openings =
		    (params.atap_queuedepth & WDC_QUEUE_DEPTH_MASK) + 1;
		aprint_verbose("%s NCQ (%d tags)", sep, drvp->drv_openings);
		sep = ",";

		if (params.atap_sata_caps & SATA_NCQ_PRIO) {
			drvp->drive_flags |= ATA_DRIVE_NCQ_PRIO;
			aprint_verbose(" w/PRIO");
		}
	}
	ata_channel_unlock(chp);

	if (printed)
		aprint_verbose("\n");

#if NATA_UDMA
	if ((atac->atac_cap & ATAC_CAP_UDMA) == 0) {
		/* don't care about UDMA modes */
		return;
	}
	if (cf_flags & ATA_CONFIG_UDMA_SET) {
		ata_channel_lock(chp);
		if ((cf_flags & ATA_CONFIG_UDMA_MODES) ==
		    ATA_CONFIG_UDMA_DISABLE) {
			drvp->drive_flags &= ~ATA_DRIVE_UDMA;
		} else {
			drvp->UDMA_mode = (cf_flags & ATA_CONFIG_UDMA_MODES) >>
			    ATA_CONFIG_UDMA_OFF;
			drvp->drive_flags |= ATA_DRIVE_UDMA | ATA_DRIVE_MODE;
		}
		ata_channel_unlock(chp);
	}
#endif	/* NATA_UDMA */
#endif	/* NATA_DMA */
}

/* management of the /dev/atabus* devices */
int
atabusopen(dev_t dev, int flag, int fmt, struct lwp *l)
{
	struct atabus_softc *sc;
	int error;

	sc = device_lookup_private(&atabus_cd, minor(dev));
	if (sc == NULL)
		return (ENXIO);

	if (sc->sc_flags & ATABUSCF_OPEN)
		return (EBUSY);

	if ((error = ata_addref(sc->sc_chan)) != 0)
		return (error);

	sc->sc_flags |= ATABUSCF_OPEN;

	return (0);
}


int
atabusclose(dev_t dev, int flag, int fmt, struct lwp *l)
{
	struct atabus_softc *sc =
	    device_lookup_private(&atabus_cd, minor(dev));

	ata_delref(sc->sc_chan);

	sc->sc_flags &= ~ATABUSCF_OPEN;

	return (0);
}

int
atabusioctl(dev_t dev, u_long cmd, void *addr, int flag, struct lwp *l)
{
	struct atabus_softc *sc =
	    device_lookup_private(&atabus_cd, minor(dev));
	struct ata_channel *chp = sc->sc_chan;
	int min_drive, max_drive, drive;
	int error;
	int s;

	/*
	 * Enforce write permission for ioctls that change the
	 * state of the bus.  Host adapter specific ioctls must
	 * be checked by the adapter driver.
	 */
	switch (cmd) {
	case ATABUSIOSCAN:
	case ATABUSIODETACH:
	case ATABUSIORESET:
		if ((flag & FWRITE) == 0)
			return (EBADF);
	}

	switch (cmd) {
	case ATABUSIORESET:
		s = splbio();
		ata_reset_channel(sc->sc_chan, AT_WAIT | AT_POLL);
		splx(s);
		return 0;
	case ATABUSIOSCAN:
	{
#if 0
		struct atabusioscan_args *a=
		    (struct atabusioscan_args *)addr;
#endif
		if ((chp->ch_drive[0].drive_type == ATA_DRIVET_OLD) ||
		    (chp->ch_drive[1].drive_type == ATA_DRIVET_OLD))
			return (EOPNOTSUPP);
		return (EOPNOTSUPP);
	}
	case ATABUSIODETACH:
	{
		struct atabusiodetach_args *a=
		    (struct atabusiodetach_args *)addr;
		if ((chp->ch_drive[0].drive_type == ATA_DRIVET_OLD) ||
		    (chp->ch_drive[1].drive_type == ATA_DRIVET_OLD))
			return (EOPNOTSUPP);
		switch (a->at_dev) {
		case -1:
			min_drive = 0;
			max_drive = 1;
			break;
		case 0:
		case 1:
			min_drive = max_drive = a->at_dev;
			break;
		default:
			return (EINVAL);
		}
		for (drive = min_drive; drive <= max_drive; drive++) {
			if (chp->ch_drive[drive].drv_softc != NULL) {
				error = config_detach(
				    chp->ch_drive[drive].drv_softc, 0);
				if (error)
					return (error);
				KASSERT(chp->ch_drive[drive].drv_softc == NULL);
			}
		}
		return 0;
	}
	default:
		return ENOTTY;
	}
}

static bool
atabus_suspend(device_t dv, const pmf_qual_t *qual)
{
	struct atabus_softc *sc = device_private(dv);
	struct ata_channel *chp = sc->sc_chan;

	ata_channel_idle(chp);

	return true;
}

static bool
atabus_resume(device_t dv, const pmf_qual_t *qual)
{
	struct atabus_softc *sc = device_private(dv);
	struct ata_channel *chp = sc->sc_chan;

	/*
	 * XXX joerg: with wdc, the first channel unfreezes the controler.
	 * Move this the reset and queue idling into wdc.
	 */
	ata_channel_lock(chp);
	if (chp->ch_queue->queue_freeze == 0) {
		ata_channel_unlock(chp);
		goto out;
	}
	KASSERT(chp->ch_queue->queue_freeze > 0);
	ata_channel_unlock(chp);

	/* unfreeze the queue and reset drives */
	ata_channel_thaw(chp);

	/* reset channel only if there are drives attached */
	if (chp->ch_ndrives > 0)
		ata_reset_channel(chp, AT_WAIT);

out:
	return true;
}

static int
atabus_rescan(device_t self, const char *ifattr, const int *locators)
{
	struct atabus_softc *sc = device_private(self);
	struct ata_channel *chp = sc->sc_chan;
	struct atabus_initq *initq;
	int i;

	/*
	 * we can rescan a port multiplier atabus, even if some devices are
	 * still attached
	 */
	if (chp->ch_satapmp_nports == 0) {
		if (chp->atapibus != NULL) {
			return EBUSY;
		}

		KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL);
		for (i = 0; i < chp->ch_ndrives; i++) {
			if (chp->ch_drive[i].drv_softc != NULL) {
				return EBUSY;
			}
		}
	}

	initq = malloc(sizeof(*initq), M_DEVBUF, M_WAITOK);
	initq->atabus_sc = sc;
	mutex_enter(&atabus_qlock);
	TAILQ_INSERT_TAIL(&atabus_initq_head, initq, atabus_initq);
	mutex_exit(&atabus_qlock);
	config_pending_incr(sc->sc_dev);

	ata_channel_lock(chp);
	chp->ch_flags |= ATACH_TH_RESCAN;
	cv_signal(&chp->ch_thr_idle);
	ata_channel_unlock(chp);

	return 0;
}

void
ata_delay(struct ata_channel *chp, int ms, const char *msg, int flags)
{

	if ((flags & (AT_WAIT | AT_POLL)) == AT_POLL) {
		/*
		 * can't use kpause(), we may be in interrupt context
		 * or taking a crash dump
		 */
		delay(ms * 1000);
	} else {
		int pause = mstohz(ms);

		KASSERT(mutex_owned(&chp->ch_lock));
		kpause(msg, false, pause > 0 ? pause : 1, &chp->ch_lock);
	}
}

void
atacmd_toncq(struct ata_xfer *xfer, uint8_t *cmd, uint16_t *count,
    uint16_t *features, uint8_t *device)
{
	if ((xfer->c_flags & C_NCQ) == 0) {
		/* FUA handling for non-NCQ drives */
		if (xfer->c_bio.flags & ATA_FUA
		    && *cmd == WDCC_WRITEDMA_EXT)
			*cmd = WDCC_WRITEDMA_FUA_EXT;

		return;
	}

	*cmd = (xfer->c_bio.flags & ATA_READ) ?
	    WDCC_READ_FPDMA_QUEUED : WDCC_WRITE_FPDMA_QUEUED;

	/* for FPDMA the block count is in features */
	*features = *count;

	/* NCQ tag */
	*count = (xfer->c_slot << 3);

	if (xfer->c_bio.flags & ATA_PRIO_HIGH)
		*count |= WDSC_PRIO_HIGH;

	/* other device flags */
	if (xfer->c_bio.flags & ATA_FUA)
		*device |= WDSD_FUA;
}

/*
 * Must be called without any locks, i.e. with both drive and channel locks
 * released.
 */
void
ata_channel_start(struct ata_channel *chp, int drive)
{
	int i, s;
	struct ata_drive_datas *drvp;

	s = splbio();

	KASSERT(chp->ch_ndrives > 0);

#define ATA_DRIVE_START(chp, drive) \
	do {							\
		KASSERT(drive < chp->ch_ndrives);		\
		drvp = &chp->ch_drive[drive];			\
								\
		if (drvp->drive_type != ATA_DRIVET_ATA &&	\
		    drvp->drive_type != ATA_DRIVET_ATAPI &&	\
		    drvp->drive_type != ATA_DRIVET_OLD)		\
			continue;				\
								\
		if (drvp->drv_start != NULL)			\
			(*drvp->drv_start)(drvp->drv_softc);	\
	} while (0)

	/*
	 * Process drives in round robin fashion starting with next one after
	 * the one which finished transfer. Thus no single drive would
	 * completely starve other drives on same channel.
	 * This loop processes all but the current drive, so won't do anything
	 * if there is only one drive in channel.
	 */
	for (i = (drive + 1) % chp->ch_ndrives; i != drive;
	    i = (i + 1) % chp->ch_ndrives) {
		ATA_DRIVE_START(chp, i);
	}

	/* Now try to kick off xfers on the current drive */
	ATA_DRIVE_START(chp, drive);

	splx(s);
#undef ATA_DRIVE_START
}

void
ata_channel_lock(struct ata_channel *chp)
{
	mutex_enter(&chp->ch_lock);
}

void
ata_channel_unlock(struct ata_channel *chp)
{
	mutex_exit(&chp->ch_lock);
}

void
ata_channel_lock_owned(struct ata_channel *chp)
{
	KASSERT(mutex_owned(&chp->ch_lock));
}

void
ata_wait_xfer(struct ata_channel *chp, struct ata_xfer *xfer)
{
	KASSERT(mutex_owned(&chp->ch_lock));

	cv_wait(&xfer->c_finish, &chp->ch_lock);
}

void
ata_wake_xfer(struct ata_channel *chp, struct ata_xfer *xfer)
{
	KASSERT(mutex_owned(&chp->ch_lock));

	cv_signal(&xfer->c_finish);
}