Home | History | Annotate | Line # | Download | only in ata
ata.c revision 1.144
      1 /*	$NetBSD: ata.c,v 1.144 2018/10/24 20:01:13 jdolecek Exp $	*/
      2 
      3 /*
      4  * Copyright (c) 1998, 2001 Manuel Bouyer.  All rights reserved.
      5  *
      6  * Redistribution and use in source and binary forms, with or without
      7  * modification, are permitted provided that the following conditions
      8  * are met:
      9  * 1. Redistributions of source code must retain the above copyright
     10  *    notice, this list of conditions and the following disclaimer.
     11  * 2. Redistributions in binary form must reproduce the above copyright
     12  *    notice, this list of conditions and the following disclaimer in the
     13  *    documentation and/or other materials provided with the distribution.
     14  *
     15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
     16  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
     17  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
     18  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
     19  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
     20  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
     21  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
     22  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
     23  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
     24  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     25  */
     26 
     27 #include <sys/cdefs.h>
     28 __KERNEL_RCSID(0, "$NetBSD: ata.c,v 1.144 2018/10/24 20:01:13 jdolecek Exp $");
     29 
     30 #include "opt_ata.h"
     31 
     32 #include <sys/param.h>
     33 #include <sys/systm.h>
     34 #include <sys/kernel.h>
     35 #include <sys/device.h>
     36 #include <sys/conf.h>
     37 #include <sys/fcntl.h>
     38 #include <sys/proc.h>
     39 #include <sys/kthread.h>
     40 #include <sys/errno.h>
     41 #include <sys/ataio.h>
     42 #include <sys/kmem.h>
     43 #include <sys/intr.h>
     44 #include <sys/bus.h>
     45 #include <sys/once.h>
     46 #include <sys/bitops.h>
     47 
     48 #define ATABUS_PRIVATE
     49 
     50 #include <dev/ata/ataconf.h>
     51 #include <dev/ata/atareg.h>
     52 #include <dev/ata/atavar.h>
     53 #include <dev/ic/wdcvar.h>	/* for PIOBM */
     54 
     55 #include "ioconf.h"
     56 #include "locators.h"
     57 
     58 #include "atapibus.h"
     59 #include "ataraid.h"
     60 #include "sata_pmp.h"
     61 
     62 #if NATARAID > 0
     63 #include <dev/ata/ata_raidvar.h>
     64 #endif
     65 #if NSATA_PMP > 0
     66 #include <dev/ata/satapmpvar.h>
     67 #endif
     68 #include <dev/ata/satapmpreg.h>
     69 
     70 #define DEBUG_FUNCS  0x08
     71 #define DEBUG_PROBE  0x10
     72 #define DEBUG_DETACH 0x20
     73 #define	DEBUG_XFERS  0x40
     74 #ifdef ATADEBUG
     75 #ifndef ATADEBUG_MASK
     76 #define ATADEBUG_MASK 0
     77 #endif
     78 int atadebug_mask = ATADEBUG_MASK;
     79 #define ATADEBUG_PRINT(args, level) \
     80 	if (atadebug_mask & (level)) \
     81 		printf args
     82 #else
     83 #define ATADEBUG_PRINT(args, level)
     84 #endif
     85 
     86 static ONCE_DECL(ata_init_ctrl);
     87 static struct pool ata_xfer_pool;
     88 
     89 /*
     90  * A queue of atabus instances, used to ensure the same bus probe order
     91  * for a given hardware configuration at each boot.  Kthread probing
     92  * devices on a atabus.  Only one probing at once.
     93  */
     94 static TAILQ_HEAD(, atabus_initq)	atabus_initq_head;
     95 static kmutex_t				atabus_qlock;
     96 static kcondvar_t			atabus_qcv;
     97 static lwp_t *				atabus_cfg_lwp;
     98 
     99 /*****************************************************************************
    100  * ATA bus layer.
    101  *
    102  * ATA controllers attach an atabus instance, which handles probing the bus
    103  * for drives, etc.
    104  *****************************************************************************/
    105 
    106 dev_type_open(atabusopen);
    107 dev_type_close(atabusclose);
    108 dev_type_ioctl(atabusioctl);
    109 
    110 const struct cdevsw atabus_cdevsw = {
    111 	.d_open = atabusopen,
    112 	.d_close = atabusclose,
    113 	.d_read = noread,
    114 	.d_write = nowrite,
    115 	.d_ioctl = atabusioctl,
    116 	.d_stop = nostop,
    117 	.d_tty = notty,
    118 	.d_poll = nopoll,
    119 	.d_mmap = nommap,
    120 	.d_kqfilter = nokqfilter,
    121 	.d_discard = nodiscard,
    122 	.d_flag = D_OTHER
    123 };
    124 
    125 static void atabus_childdetached(device_t, device_t);
    126 static int atabus_rescan(device_t, const char *, const int *);
    127 static bool atabus_resume(device_t, const pmf_qual_t *);
    128 static bool atabus_suspend(device_t, const pmf_qual_t *);
    129 static void atabusconfig_thread(void *);
    130 
    131 static void ata_channel_idle(struct ata_channel *);
    132 static void ata_activate_xfer_locked(struct ata_channel *, struct ata_xfer *);
    133 static void ata_channel_freeze_locked(struct ata_channel *);
    134 static void ata_thread_wake_locked(struct ata_channel *);
    135 
    136 /*
    137  * atabus_init:
    138  *
    139  *	Initialize ATA subsystem structures.
    140  */
    141 static int
    142 atabus_init(void)
    143 {
    144 
    145 	pool_init(&ata_xfer_pool, sizeof(struct ata_xfer), 0, 0, 0,
    146 	    "ataspl", NULL, IPL_BIO);
    147 	TAILQ_INIT(&atabus_initq_head);
    148 	mutex_init(&atabus_qlock, MUTEX_DEFAULT, IPL_NONE);
    149 	cv_init(&atabus_qcv, "atainitq");
    150 	return 0;
    151 }
    152 
    153 /*
    154  * atabusprint:
    155  *
    156  *	Autoconfiguration print routine used by ATA controllers when
    157  *	attaching an atabus instance.
    158  */
    159 int
    160 atabusprint(void *aux, const char *pnp)
    161 {
    162 	struct ata_channel *chan = aux;
    163 
    164 	if (pnp)
    165 		aprint_normal("atabus at %s", pnp);
    166 	aprint_normal(" channel %d", chan->ch_channel);
    167 
    168 	return (UNCONF);
    169 }
    170 
    171 /*
    172  * ataprint:
    173  *
    174  *	Autoconfiguration print routine.
    175  */
    176 int
    177 ataprint(void *aux, const char *pnp)
    178 {
    179 	struct ata_device *adev = aux;
    180 
    181 	if (pnp)
    182 		aprint_normal("wd at %s", pnp);
    183 	aprint_normal(" drive %d", adev->adev_drv_data->drive);
    184 
    185 	return (UNCONF);
    186 }
    187 
    188 /*
    189  * ata_channel_attach:
    190  *
    191  *	Common parts of attaching an atabus to an ATA controller channel.
    192  */
    193 void
    194 ata_channel_attach(struct ata_channel *chp)
    195 {
    196 	if (chp->ch_flags & ATACH_DISABLED)
    197 		return;
    198 
    199 	ata_channel_init(chp);
    200 
    201 	KASSERT(chp->ch_queue != NULL);
    202 
    203 	chp->atabus = config_found_ia(chp->ch_atac->atac_dev, "ata", chp,
    204 		atabusprint);
    205 }
    206 
    207 /*
    208  * ata_channel_detach:
    209  *
    210  *	Common parts of detaching an atabus to an ATA controller channel.
    211  */
    212 void
    213 ata_channel_detach(struct ata_channel *chp)
    214 {
    215 	if (chp->ch_flags & ATACH_DISABLED)
    216 		return;
    217 
    218 	ata_channel_destroy(chp);
    219 
    220 	chp->ch_flags |= ATACH_DETACHED;
    221 }
    222 
    223 static void
    224 atabusconfig(struct atabus_softc *atabus_sc)
    225 {
    226 	struct ata_channel *chp = atabus_sc->sc_chan;
    227 	struct atac_softc *atac = chp->ch_atac;
    228 	struct atabus_initq *atabus_initq = NULL;
    229 	int i, error;
    230 
    231 	/* we are in the atabus's thread context */
    232 	ata_channel_lock(chp);
    233 	chp->ch_flags |= ATACH_TH_RUN;
    234 	ata_channel_unlock(chp);
    235 
    236 	/*
    237 	 * Probe for the drives attached to controller, unless a PMP
    238 	 * is already known
    239 	 */
    240 	/* XXX for SATA devices we will power up all drives at once */
    241 	if (chp->ch_satapmp_nports == 0)
    242 		(*atac->atac_probe)(chp);
    243 
    244 	if (chp->ch_ndrives >= 2) {
    245 		ATADEBUG_PRINT(("atabusattach: ch_drive_type 0x%x 0x%x\n",
    246 		    chp->ch_drive[0].drive_type, chp->ch_drive[1].drive_type),
    247 		    DEBUG_PROBE);
    248 	}
    249 
    250 	/* next operations will occurs in a separate thread */
    251 	ata_channel_lock(chp);
    252 	chp->ch_flags &= ~ATACH_TH_RUN;
    253 	ata_channel_unlock(chp);
    254 
    255 	/* Make sure the devices probe in atabus order to avoid jitter. */
    256 	mutex_enter(&atabus_qlock);
    257 	for (;;) {
    258 		atabus_initq = TAILQ_FIRST(&atabus_initq_head);
    259 		if (atabus_initq->atabus_sc == atabus_sc)
    260 			break;
    261 		cv_wait(&atabus_qcv, &atabus_qlock);
    262 	}
    263 	mutex_exit(&atabus_qlock);
    264 
    265 	ata_channel_lock(chp);
    266 
    267 	/* If no drives, abort here */
    268 	if (chp->ch_drive == NULL)
    269 		goto out;
    270 	KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL);
    271 	for (i = 0; i < chp->ch_ndrives; i++)
    272 		if (chp->ch_drive[i].drive_type != ATA_DRIVET_NONE)
    273 			break;
    274 	if (i == chp->ch_ndrives)
    275 		goto out;
    276 
    277 	/* Shortcut in case we've been shutdown */
    278 	if (chp->ch_flags & ATACH_SHUTDOWN)
    279 		goto out;
    280 
    281 	ata_channel_unlock(chp);
    282 
    283 	if ((error = kthread_create(PRI_NONE, 0, NULL, atabusconfig_thread,
    284 	    atabus_sc, &atabus_cfg_lwp,
    285 	    "%scnf", device_xname(atac->atac_dev))) != 0)
    286 		aprint_error_dev(atac->atac_dev,
    287 		    "unable to create config thread: error %d\n", error);
    288 	return;
    289 
    290  out:
    291 	ata_channel_unlock(chp);
    292 
    293 	mutex_enter(&atabus_qlock);
    294 	TAILQ_REMOVE(&atabus_initq_head, atabus_initq, atabus_initq);
    295 	cv_broadcast(&atabus_qcv);
    296 	mutex_exit(&atabus_qlock);
    297 
    298 	kmem_free(atabus_initq, sizeof(*atabus_initq));
    299 
    300 	ata_delref(chp);
    301 
    302 	config_pending_decr(atac->atac_dev);
    303 }
    304 
    305 /*
    306  * atabus_configthread: finish attach of atabus's childrens, in a separate
    307  * kernel thread.
    308  */
    309 static void
    310 atabusconfig_thread(void *arg)
    311 {
    312 	struct atabus_softc *atabus_sc = arg;
    313 	struct ata_channel *chp = atabus_sc->sc_chan;
    314 	struct atac_softc *atac = chp->ch_atac;
    315 	struct atabus_initq *atabus_initq = NULL;
    316 	int i, s;
    317 
    318 	/* XXX seems wrong */
    319 	mutex_enter(&atabus_qlock);
    320 	atabus_initq = TAILQ_FIRST(&atabus_initq_head);
    321 	KASSERT(atabus_initq->atabus_sc == atabus_sc);
    322 	mutex_exit(&atabus_qlock);
    323 
    324 	/*
    325 	 * First look for a port multiplier
    326 	 */
    327 	if (chp->ch_ndrives == PMP_MAX_DRIVES &&
    328 	    chp->ch_drive[PMP_PORT_CTL].drive_type == ATA_DRIVET_PM) {
    329 #if NSATA_PMP > 0
    330 		satapmp_attach(chp);
    331 #else
    332 		aprint_error_dev(atabus_sc->sc_dev,
    333 		    "SATA port multiplier not supported\n");
    334 		/* no problems going on, all drives are ATA_DRIVET_NONE */
    335 #endif
    336 	}
    337 
    338 	/*
    339 	 * Attach an ATAPI bus, if needed.
    340 	 */
    341 	KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL);
    342 	for (i = 0; i < chp->ch_ndrives && chp->atapibus == NULL; i++) {
    343 		if (chp->ch_drive[i].drive_type == ATA_DRIVET_ATAPI) {
    344 #if NATAPIBUS > 0
    345 			(*atac->atac_atapibus_attach)(atabus_sc);
    346 #else
    347 			/*
    348 			 * Fake the autoconfig "not configured" message
    349 			 */
    350 			aprint_normal("atapibus at %s not configured\n",
    351 			    device_xname(atac->atac_dev));
    352 			chp->atapibus = NULL;
    353 			s = splbio();
    354 			for (i = 0; i < chp->ch_ndrives; i++) {
    355 				if (chp->ch_drive[i].drive_type == ATA_DRIVET_ATAPI)
    356 					chp->ch_drive[i].drive_type = ATA_DRIVET_NONE;
    357 			}
    358 			splx(s);
    359 #endif
    360 			break;
    361 		}
    362 	}
    363 
    364 	for (i = 0; i < chp->ch_ndrives; i++) {
    365 		struct ata_device adev;
    366 		if (chp->ch_drive[i].drive_type != ATA_DRIVET_ATA &&
    367 		    chp->ch_drive[i].drive_type != ATA_DRIVET_OLD) {
    368 			continue;
    369 		}
    370 		if (chp->ch_drive[i].drv_softc != NULL)
    371 			continue;
    372 		memset(&adev, 0, sizeof(struct ata_device));
    373 		adev.adev_bustype = atac->atac_bustype_ata;
    374 		adev.adev_channel = chp->ch_channel;
    375 		adev.adev_drv_data = &chp->ch_drive[i];
    376 		chp->ch_drive[i].drv_softc = config_found_ia(atabus_sc->sc_dev,
    377 		    "ata_hl", &adev, ataprint);
    378 		if (chp->ch_drive[i].drv_softc != NULL) {
    379 			ata_probe_caps(&chp->ch_drive[i]);
    380 		} else {
    381 			s = splbio();
    382 			chp->ch_drive[i].drive_type = ATA_DRIVET_NONE;
    383 			splx(s);
    384 		}
    385 	}
    386 
    387 	/* now that we know the drives, the controller can set its modes */
    388 	if (atac->atac_set_modes) {
    389 		(*atac->atac_set_modes)(chp);
    390 		ata_print_modes(chp);
    391 	}
    392 #if NATARAID > 0
    393 	if (atac->atac_cap & ATAC_CAP_RAID) {
    394 		for (i = 0; i < chp->ch_ndrives; i++) {
    395 			if (chp->ch_drive[i].drive_type == ATA_DRIVET_ATA) {
    396 				ata_raid_check_component(
    397 				    chp->ch_drive[i].drv_softc);
    398 			}
    399 		}
    400 	}
    401 #endif /* NATARAID > 0 */
    402 
    403 	/*
    404 	 * reset drive_flags for unattached devices, reset state for attached
    405 	 * ones
    406 	 */
    407 	s = splbio();
    408 	for (i = 0; i < chp->ch_ndrives; i++) {
    409 		if (chp->ch_drive[i].drive_type == ATA_DRIVET_PM)
    410 			continue;
    411 		if (chp->ch_drive[i].drv_softc == NULL) {
    412 			chp->ch_drive[i].drive_flags = 0;
    413 			chp->ch_drive[i].drive_type = ATA_DRIVET_NONE;
    414 		} else
    415 			chp->ch_drive[i].state = 0;
    416 	}
    417 	splx(s);
    418 
    419 	mutex_enter(&atabus_qlock);
    420 	TAILQ_REMOVE(&atabus_initq_head, atabus_initq, atabus_initq);
    421 	cv_broadcast(&atabus_qcv);
    422 	mutex_exit(&atabus_qlock);
    423 
    424 	kmem_free(atabus_initq, sizeof(*atabus_initq));
    425 
    426 	ata_delref(chp);
    427 
    428 	config_pending_decr(atac->atac_dev);
    429 	kthread_exit(0);
    430 }
    431 
    432 /*
    433  * atabus_thread:
    434  *
    435  *	Worker thread for the ATA bus.
    436  */
    437 static void
    438 atabus_thread(void *arg)
    439 {
    440 	struct atabus_softc *sc = arg;
    441 	struct ata_channel *chp = sc->sc_chan;
    442 	struct ata_queue *chq = chp->ch_queue;
    443 	struct ata_xfer *xfer;
    444 	int i, rv;
    445 
    446 	ata_channel_lock(chp);
    447 	chp->ch_flags |= ATACH_TH_RUN;
    448 
    449 	/*
    450 	 * Probe the drives.  Reset type to indicate to controllers
    451 	 * that can re-probe that all drives must be probed..
    452 	 *
    453 	 * Note: ch_ndrives may be changed during the probe.
    454 	 */
    455 	KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL);
    456 	for (i = 0; i < chp->ch_ndrives; i++) {
    457 		chp->ch_drive[i].drive_flags = 0;
    458 		chp->ch_drive[i].drive_type = ATA_DRIVET_NONE;
    459 	}
    460 	ata_channel_unlock(chp);
    461 
    462 	atabusconfig(sc);
    463 
    464 	ata_channel_lock(chp);
    465 	for (;;) {
    466 		if ((chp->ch_flags & (ATACH_TH_RESET | ATACH_TH_DRIVE_RESET
    467 		    | ATACH_TH_RECOVERY | ATACH_SHUTDOWN)) == 0 &&
    468 		    (chq->queue_active == 0 || chq->queue_freeze == 0)) {
    469 			chp->ch_flags &= ~ATACH_TH_RUN;
    470 			cv_wait(&chp->ch_thr_idle, &chp->ch_lock);
    471 			chp->ch_flags |= ATACH_TH_RUN;
    472 		}
    473 		if (chp->ch_flags & ATACH_SHUTDOWN) {
    474 			break;
    475 		}
    476 		if (chp->ch_flags & ATACH_TH_RESCAN) {
    477 			chp->ch_flags &= ~ATACH_TH_RESCAN;
    478 			ata_channel_unlock(chp);
    479 			atabusconfig(sc);
    480 			ata_channel_lock(chp);
    481 		}
    482 		if (chp->ch_flags & ATACH_TH_RESET) {
    483 			/* this will unfreeze the channel */
    484 			ata_thread_run(chp, AT_WAIT,
    485 			    ATACH_TH_RESET, ATACH_NODRIVE);
    486 		} else if (chp->ch_flags & ATACH_TH_DRIVE_RESET) {
    487 			/* this will unfreeze the channel */
    488 			for (i = 0; i < chp->ch_ndrives; i++) {
    489 				struct ata_drive_datas *drvp;
    490 
    491 				drvp = &chp->ch_drive[i];
    492 
    493 				if (drvp->drive_flags & ATA_DRIVE_TH_RESET) {
    494 					ata_thread_run(chp,
    495 					    AT_WAIT, ATACH_TH_DRIVE_RESET, i);
    496 				}
    497 			}
    498 			chp->ch_flags &= ~ATACH_TH_DRIVE_RESET;
    499 		} else if (chp->ch_flags & ATACH_TH_RECOVERY) {
    500 			/*
    501 			 * This will unfreeze the channel; drops locks during
    502 			 * run, so must wrap in splbio()/splx() to avoid
    503 			 * spurious interrupts. XXX MPSAFE
    504 			 */
    505 			int s = splbio();
    506 			ata_thread_run(chp, AT_WAIT, ATACH_TH_RECOVERY,
    507 			    chp->recovery_tfd);
    508 			splx(s);
    509 		} else if (chq->queue_active > 0 && chq->queue_freeze == 1) {
    510 			/*
    511 			 * Caller has bumped queue_freeze, decrease it. This
    512 			 * flow shalt never be executed for NCQ commands.
    513 			 */
    514 			KASSERT((chp->ch_flags & ATACH_NCQ) == 0);
    515 			KASSERT(chq->queue_active == 1);
    516 
    517 			ata_channel_thaw_locked(chp);
    518 			xfer = ata_queue_get_active_xfer_locked(chp);
    519 
    520 			KASSERT(xfer != NULL);
    521 			KASSERT((xfer->c_flags & C_POLL) == 0);
    522 
    523 			switch ((rv = ata_xfer_start(xfer))) {
    524 			case ATASTART_STARTED:
    525 			case ATASTART_POLL:
    526 			case ATASTART_ABORT:
    527 				break;
    528 			case ATASTART_TH:
    529 			default:
    530 				panic("%s: ata_xfer_start() unexpected rv %d",
    531 				    __func__, rv);
    532 				/* NOTREACHED */
    533 			}
    534 		} else if (chq->queue_freeze > 1)
    535 			panic("%s: queue_freeze", __func__);
    536 
    537 		/* Try to run down the queue once channel is unfrozen */
    538 		if (chq->queue_freeze == 0) {
    539 			ata_channel_unlock(chp);
    540 			atastart(chp);
    541 			ata_channel_lock(chp);
    542 		}
    543 	}
    544 	chp->ch_thread = NULL;
    545 	cv_signal(&chp->ch_thr_idle);
    546 	ata_channel_unlock(chp);
    547 	kthread_exit(0);
    548 }
    549 
    550 static void
    551 ata_thread_wake_locked(struct ata_channel *chp)
    552 {
    553 	KASSERT(mutex_owned(&chp->ch_lock));
    554 	ata_channel_freeze_locked(chp);
    555 	cv_signal(&chp->ch_thr_idle);
    556 }
    557 
    558 /*
    559  * atabus_match:
    560  *
    561  *	Autoconfiguration match routine.
    562  */
    563 static int
    564 atabus_match(device_t parent, cfdata_t cf, void *aux)
    565 {
    566 	struct ata_channel *chp = aux;
    567 
    568 	if (chp == NULL)
    569 		return (0);
    570 
    571 	if (cf->cf_loc[ATACF_CHANNEL] != chp->ch_channel &&
    572 	    cf->cf_loc[ATACF_CHANNEL] != ATACF_CHANNEL_DEFAULT)
    573 		return (0);
    574 
    575 	return (1);
    576 }
    577 
    578 /*
    579  * atabus_attach:
    580  *
    581  *	Autoconfiguration attach routine.
    582  */
    583 static void
    584 atabus_attach(device_t parent, device_t self, void *aux)
    585 {
    586 	struct atabus_softc *sc = device_private(self);
    587 	struct ata_channel *chp = aux;
    588 	struct atabus_initq *initq;
    589 	int error;
    590 
    591 	sc->sc_chan = chp;
    592 
    593 	aprint_normal("\n");
    594 	aprint_naive("\n");
    595 
    596 	sc->sc_dev = self;
    597 
    598 	if (ata_addref(chp))
    599 		return;
    600 
    601 	RUN_ONCE(&ata_init_ctrl, atabus_init);
    602 
    603 	initq = kmem_zalloc(sizeof(*initq), KM_SLEEP);
    604 	initq->atabus_sc = sc;
    605 	mutex_enter(&atabus_qlock);
    606 	TAILQ_INSERT_TAIL(&atabus_initq_head, initq, atabus_initq);
    607 	mutex_exit(&atabus_qlock);
    608 	config_pending_incr(sc->sc_dev);
    609 
    610 	/* XXX MPSAFE - no KTHREAD_MPSAFE, so protected by KERNEL_LOCK() */
    611 	if ((error = kthread_create(PRI_NONE, 0, NULL, atabus_thread, sc,
    612 	    &chp->ch_thread, "%s", device_xname(self))) != 0)
    613 		aprint_error_dev(self,
    614 		    "unable to create kernel thread: error %d\n", error);
    615 
    616 	if (!pmf_device_register(self, atabus_suspend, atabus_resume))
    617 		aprint_error_dev(self, "couldn't establish power handler\n");
    618 }
    619 
    620 /*
    621  * atabus_detach:
    622  *
    623  *	Autoconfiguration detach routine.
    624  */
    625 static int
    626 atabus_detach(device_t self, int flags)
    627 {
    628 	struct atabus_softc *sc = device_private(self);
    629 	struct ata_channel *chp = sc->sc_chan;
    630 	device_t dev = NULL;
    631 	int i, error = 0;
    632 
    633 	/* Shutdown the channel. */
    634 	ata_channel_lock(chp);
    635 	chp->ch_flags |= ATACH_SHUTDOWN;
    636 	while (chp->ch_thread != NULL) {
    637 		cv_signal(&chp->ch_thr_idle);
    638 		cv_wait(&chp->ch_thr_idle, &chp->ch_lock);
    639 	}
    640 	ata_channel_unlock(chp);
    641 
    642 	/*
    643 	 * Detach atapibus and its children.
    644 	 */
    645 	if ((dev = chp->atapibus) != NULL) {
    646 		ATADEBUG_PRINT(("atabus_detach: %s: detaching %s\n",
    647 		    device_xname(self), device_xname(dev)), DEBUG_DETACH);
    648 
    649 		error = config_detach(dev, flags);
    650 		if (error)
    651 			goto out;
    652 		KASSERT(chp->atapibus == NULL);
    653 	}
    654 
    655 	KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL);
    656 
    657 	/*
    658 	 * Detach our other children.
    659 	 */
    660 	for (i = 0; i < chp->ch_ndrives; i++) {
    661 		if (chp->ch_drive[i].drive_type == ATA_DRIVET_ATAPI)
    662 			continue;
    663 		if (chp->ch_drive[i].drive_type == ATA_DRIVET_PM)
    664 			chp->ch_drive[i].drive_type = ATA_DRIVET_NONE;
    665 		if ((dev = chp->ch_drive[i].drv_softc) != NULL) {
    666 			ATADEBUG_PRINT(("%s.%d: %s: detaching %s\n", __func__,
    667 			    __LINE__, device_xname(self), device_xname(dev)),
    668 			    DEBUG_DETACH);
    669 			error = config_detach(dev, flags);
    670 			if (error)
    671 				goto out;
    672 			KASSERT(chp->ch_drive[i].drv_softc == NULL);
    673 			KASSERT(chp->ch_drive[i].drive_type == 0);
    674 		}
    675 	}
    676 	atabus_free_drives(chp);
    677 
    678  out:
    679 #ifdef ATADEBUG
    680 	if (dev != NULL && error != 0)
    681 		ATADEBUG_PRINT(("%s: %s: error %d detaching %s\n", __func__,
    682 		    device_xname(self), error, device_xname(dev)),
    683 		    DEBUG_DETACH);
    684 #endif /* ATADEBUG */
    685 
    686 	return (error);
    687 }
    688 
    689 void
    690 atabus_childdetached(device_t self, device_t child)
    691 {
    692 	bool found = false;
    693 	struct atabus_softc *sc = device_private(self);
    694 	struct ata_channel *chp = sc->sc_chan;
    695 	int i;
    696 
    697 	KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL);
    698 	/*
    699 	 * atapibus detached.
    700 	 */
    701 	if (child == chp->atapibus) {
    702 		chp->atapibus = NULL;
    703 		found = true;
    704 		for (i = 0; i < chp->ch_ndrives; i++) {
    705 			if (chp->ch_drive[i].drive_type != ATA_DRIVET_ATAPI)
    706 				continue;
    707 			KASSERT(chp->ch_drive[i].drv_softc != NULL);
    708 			chp->ch_drive[i].drv_softc = NULL;
    709 			chp->ch_drive[i].drive_flags = 0;
    710 			chp->ch_drive[i].drive_type = ATA_DRIVET_NONE;
    711 		}
    712 	}
    713 
    714 	/*
    715 	 * Detach our other children.
    716 	 */
    717 	for (i = 0; i < chp->ch_ndrives; i++) {
    718 		if (chp->ch_drive[i].drive_type == ATA_DRIVET_ATAPI)
    719 			continue;
    720 		if (child == chp->ch_drive[i].drv_softc) {
    721 			chp->ch_drive[i].drv_softc = NULL;
    722 			chp->ch_drive[i].drive_flags = 0;
    723 			if (chp->ch_drive[i].drive_type == ATA_DRIVET_PM)
    724 				chp->ch_satapmp_nports = 0;
    725 			chp->ch_drive[i].drive_type = ATA_DRIVET_NONE;
    726 			found = true;
    727 		}
    728 	}
    729 
    730 	if (!found)
    731 		panic("%s: unknown child %p", device_xname(self),
    732 		    (const void *)child);
    733 }
    734 
    735 CFATTACH_DECL3_NEW(atabus, sizeof(struct atabus_softc),
    736     atabus_match, atabus_attach, atabus_detach, NULL, atabus_rescan,
    737     atabus_childdetached, DVF_DETACH_SHUTDOWN);
    738 
    739 /*****************************************************************************
    740  * Common ATA bus operations.
    741  *****************************************************************************/
    742 
    743 /* allocate/free the channel's ch_drive[] array */
    744 int
    745 atabus_alloc_drives(struct ata_channel *chp, int ndrives)
    746 {
    747 	int i;
    748 	if (chp->ch_ndrives != ndrives)
    749 		atabus_free_drives(chp);
    750 	if (chp->ch_drive == NULL) {
    751 		chp->ch_drive = kmem_zalloc(
    752 		    sizeof(struct ata_drive_datas) * ndrives, KM_NOSLEEP);
    753 	}
    754 	if (chp->ch_drive == NULL) {
    755 	    aprint_error_dev(chp->ch_atac->atac_dev,
    756 		"can't alloc drive array\n");
    757 	    chp->ch_ndrives = 0;
    758 	    return ENOMEM;
    759 	};
    760 	for (i = 0; i < ndrives; i++) {
    761 		chp->ch_drive[i].chnl_softc = chp;
    762 		chp->ch_drive[i].drive = i;
    763 	}
    764 	chp->ch_ndrives = ndrives;
    765 	return 0;
    766 }
    767 
    768 void
    769 atabus_free_drives(struct ata_channel *chp)
    770 {
    771 #ifdef DIAGNOSTIC
    772 	int i;
    773 	int dopanic = 0;
    774 	KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL);
    775 	for (i = 0; i < chp->ch_ndrives; i++) {
    776 		if (chp->ch_drive[i].drive_type != ATA_DRIVET_NONE) {
    777 			printf("%s: ch_drive[%d] type %d != ATA_DRIVET_NONE\n",
    778 			    device_xname(chp->atabus), i,
    779 			    chp->ch_drive[i].drive_type);
    780 			dopanic = 1;
    781 		}
    782 		if (chp->ch_drive[i].drv_softc != NULL) {
    783 			printf("%s: ch_drive[%d] attached to %s\n",
    784 			    device_xname(chp->atabus), i,
    785 			    device_xname(chp->ch_drive[i].drv_softc));
    786 			dopanic = 1;
    787 		}
    788 	}
    789 	if (dopanic)
    790 		panic("atabus_free_drives");
    791 #endif
    792 
    793 	if (chp->ch_drive == NULL)
    794 		return;
    795 	kmem_free(chp->ch_drive,
    796 	    sizeof(struct ata_drive_datas) * chp->ch_ndrives);
    797 	chp->ch_ndrives = 0;
    798 	chp->ch_drive = NULL;
    799 }
    800 
    801 /* Get the disk's parameters */
    802 int
    803 ata_get_params(struct ata_drive_datas *drvp, uint8_t flags,
    804     struct ataparams *prms)
    805 {
    806 	struct ata_xfer *xfer;
    807 	struct ata_channel *chp = drvp->chnl_softc;
    808 	struct atac_softc *atac = chp->ch_atac;
    809 	char *tb;
    810 	int i, rv;
    811 	uint16_t *p;
    812 
    813 	ATADEBUG_PRINT(("%s\n", __func__), DEBUG_FUNCS);
    814 
    815 	xfer = ata_get_xfer(chp, false);
    816 	if (xfer == NULL) {
    817 		ATADEBUG_PRINT(("%s: no xfer\n", __func__),
    818 		    DEBUG_FUNCS|DEBUG_PROBE);
    819 		return CMD_AGAIN;
    820 	}
    821 
    822 	tb = kmem_zalloc(ATA_BSIZE, KM_SLEEP);
    823 	memset(prms, 0, sizeof(struct ataparams));
    824 
    825 	if (drvp->drive_type == ATA_DRIVET_ATA) {
    826 		xfer->c_ata_c.r_command = WDCC_IDENTIFY;
    827 		xfer->c_ata_c.r_st_bmask = WDCS_DRDY;
    828 		xfer->c_ata_c.r_st_pmask = WDCS_DRQ;
    829 		xfer->c_ata_c.timeout = 3000; /* 3s */
    830 	} else if (drvp->drive_type == ATA_DRIVET_ATAPI) {
    831 		xfer->c_ata_c.r_command = ATAPI_IDENTIFY_DEVICE;
    832 		xfer->c_ata_c.r_st_bmask = 0;
    833 		xfer->c_ata_c.r_st_pmask = WDCS_DRQ;
    834 		xfer->c_ata_c.timeout = 10000; /* 10s */
    835 	} else {
    836 		ATADEBUG_PRINT(("ata_get_parms: no disks\n"),
    837 		    DEBUG_FUNCS|DEBUG_PROBE);
    838 		rv = CMD_ERR;
    839 		goto out;
    840 	}
    841 	xfer->c_ata_c.flags = AT_READ | flags;
    842 	xfer->c_ata_c.data = tb;
    843 	xfer->c_ata_c.bcount = ATA_BSIZE;
    844 	if ((*atac->atac_bustype_ata->ata_exec_command)(drvp,
    845 						xfer) != ATACMD_COMPLETE) {
    846 		ATADEBUG_PRINT(("ata_get_parms: wdc_exec_command failed\n"),
    847 		    DEBUG_FUNCS|DEBUG_PROBE);
    848 		rv = CMD_AGAIN;
    849 		goto out;
    850 	}
    851 	if (xfer->c_ata_c.flags & (AT_ERROR | AT_TIMEOU | AT_DF)) {
    852 		ATADEBUG_PRINT(("ata_get_parms: ata_c.flags=0x%x\n",
    853 		    xfer->c_ata_c.flags), DEBUG_FUNCS|DEBUG_PROBE);
    854 		rv = CMD_ERR;
    855 		goto out;
    856 	}
    857 	/* if we didn't read any data something is wrong */
    858 	if ((xfer->c_ata_c.flags & AT_XFDONE) == 0) {
    859 		rv = CMD_ERR;
    860 		goto out;
    861 	}
    862 
    863 	/* Read in parameter block. */
    864 	memcpy(prms, tb, sizeof(struct ataparams));
    865 
    866 	/*
    867 	 * Shuffle string byte order.
    868 	 * ATAPI NEC, Mitsumi and Pioneer drives and
    869 	 * old ATA TDK CompactFlash cards
    870 	 * have different byte order.
    871 	 */
    872 #if BYTE_ORDER == BIG_ENDIAN
    873 # define M(n)	prms->atap_model[(n) ^ 1]
    874 #else
    875 # define M(n)	prms->atap_model[n]
    876 #endif
    877 	if (
    878 #if BYTE_ORDER == BIG_ENDIAN
    879 	    !
    880 #endif
    881 	    ((drvp->drive_type == ATA_DRIVET_ATAPI) ?
    882 	     ((M(0) == 'N' && M(1) == 'E') ||
    883 	      (M(0) == 'F' && M(1) == 'X') ||
    884 	      (M(0) == 'P' && M(1) == 'i')) :
    885 	     ((M(0) == 'T' && M(1) == 'D' && M(2) == 'K')))) {
    886 		rv = CMD_OK;
    887 		goto out;
    888 	     }
    889 #undef M
    890 	for (i = 0; i < sizeof(prms->atap_model); i += 2) {
    891 		p = (uint16_t *)(prms->atap_model + i);
    892 		*p = bswap16(*p);
    893 	}
    894 	for (i = 0; i < sizeof(prms->atap_serial); i += 2) {
    895 		p = (uint16_t *)(prms->atap_serial + i);
    896 		*p = bswap16(*p);
    897 	}
    898 	for (i = 0; i < sizeof(prms->atap_revision); i += 2) {
    899 		p = (uint16_t *)(prms->atap_revision + i);
    900 		*p = bswap16(*p);
    901 	}
    902 
    903 	rv = CMD_OK;
    904  out:
    905 	kmem_free(tb, ATA_BSIZE);
    906 	ata_free_xfer(chp, xfer);
    907 	return rv;
    908 }
    909 
    910 int
    911 ata_set_mode(struct ata_drive_datas *drvp, uint8_t mode, uint8_t flags)
    912 {
    913 	struct ata_xfer *xfer;
    914 	int rv;
    915 	struct ata_channel *chp = drvp->chnl_softc;
    916 	struct atac_softc *atac = chp->ch_atac;
    917 
    918 	ATADEBUG_PRINT(("ata_set_mode=0x%x\n", mode), DEBUG_FUNCS);
    919 
    920 	xfer = ata_get_xfer(chp, false);
    921 	if (xfer == NULL) {
    922 		ATADEBUG_PRINT(("%s: no xfer\n", __func__),
    923 		    DEBUG_FUNCS|DEBUG_PROBE);
    924 		return CMD_AGAIN;
    925 	}
    926 
    927 	xfer->c_ata_c.r_command = SET_FEATURES;
    928 	xfer->c_ata_c.r_st_bmask = 0;
    929 	xfer->c_ata_c.r_st_pmask = 0;
    930 	xfer->c_ata_c.r_features = WDSF_SET_MODE;
    931 	xfer->c_ata_c.r_count = mode;
    932 	xfer->c_ata_c.flags = flags;
    933 	xfer->c_ata_c.timeout = 1000; /* 1s */
    934 	if ((*atac->atac_bustype_ata->ata_exec_command)(drvp,
    935 						xfer) != ATACMD_COMPLETE) {
    936 		rv = CMD_AGAIN;
    937 		goto out;
    938 	}
    939 	if (xfer->c_ata_c.flags & (AT_ERROR | AT_TIMEOU | AT_DF)) {
    940 		rv = CMD_ERR;
    941 		goto out;
    942 	}
    943 
    944 	rv = CMD_OK;
    945 
    946 out:
    947 	ata_free_xfer(chp, xfer);
    948 	return rv;
    949 }
    950 
    951 #if NATA_DMA
    952 void
    953 ata_dmaerr(struct ata_drive_datas *drvp, int flags)
    954 {
    955 	/*
    956 	 * Downgrade decision: if we get NERRS_MAX in NXFER.
    957 	 * We start with n_dmaerrs set to NERRS_MAX-1 so that the
    958 	 * first error within the first NXFER ops will immediatly trigger
    959 	 * a downgrade.
    960 	 * If we got an error and n_xfers is bigger than NXFER reset counters.
    961 	 */
    962 	drvp->n_dmaerrs++;
    963 	if (drvp->n_dmaerrs >= NERRS_MAX && drvp->n_xfers <= NXFER) {
    964 		ata_downgrade_mode(drvp, flags);
    965 		drvp->n_dmaerrs = NERRS_MAX-1;
    966 		drvp->n_xfers = 0;
    967 		return;
    968 	}
    969 	if (drvp->n_xfers > NXFER) {
    970 		drvp->n_dmaerrs = 1; /* just got an error */
    971 		drvp->n_xfers = 1; /* restart counting from this error */
    972 	}
    973 }
    974 #endif	/* NATA_DMA */
    975 
    976 /*
    977  * freeze the queue and wait for the controller to be idle. Caller has to
    978  * unfreeze/restart the queue
    979  */
    980 static void
    981 ata_channel_idle(struct ata_channel *chp)
    982 {
    983 	ata_channel_lock(chp);
    984 	ata_channel_freeze_locked(chp);
    985 	while (chp->ch_queue->queue_active > 0) {
    986 		chp->ch_queue->queue_flags |= QF_IDLE_WAIT;
    987 		cv_timedwait(&chp->ch_queue->queue_idle, &chp->ch_lock, 1);
    988 	}
    989 	ata_channel_unlock(chp);
    990 }
    991 
    992 /*
    993  * Add a command to the queue and start controller.
    994  *
    995  * MUST BE CALLED AT splbio()!
    996  */
    997 void
    998 ata_exec_xfer(struct ata_channel *chp, struct ata_xfer *xfer)
    999 {
   1000 
   1001 	ATADEBUG_PRINT(("ata_exec_xfer %p channel %d drive %d\n", xfer,
   1002 	    chp->ch_channel, xfer->c_drive), DEBUG_XFERS);
   1003 
   1004 	/* complete xfer setup */
   1005 	xfer->c_chp = chp;
   1006 
   1007 	ata_channel_lock(chp);
   1008 
   1009 	/*
   1010 	 * Standard commands are added to the end of command list, but
   1011 	 * recovery commands must be run immediatelly.
   1012 	 */
   1013 	if ((xfer->c_flags & C_SKIP_QUEUE) == 0)
   1014 		SIMPLEQ_INSERT_TAIL(&chp->ch_queue->queue_xfer, xfer,
   1015 		    c_xferchain);
   1016 	else
   1017 		SIMPLEQ_INSERT_HEAD(&chp->ch_queue->queue_xfer, xfer,
   1018 		    c_xferchain);
   1019 
   1020 	/*
   1021 	 * if polling and can sleep, wait for the xfer to be at head of queue
   1022 	 */
   1023 	if ((xfer->c_flags & (C_POLL | C_WAIT)) ==  (C_POLL | C_WAIT)) {
   1024 		while (chp->ch_queue->queue_active > 0 ||
   1025 		    SIMPLEQ_FIRST(&chp->ch_queue->queue_xfer) != xfer) {
   1026 			xfer->c_flags |= C_WAITACT;
   1027 			cv_wait(&chp->ch_queue->c_active, &chp->ch_lock);
   1028 			xfer->c_flags &= ~C_WAITACT;
   1029 		}
   1030 
   1031 		/*
   1032 		 * Free xfer now if it there was attempt to free it
   1033 		 * while we were waiting.
   1034 		 */
   1035 		if ((xfer->c_flags & (C_FREE|C_WAITTIMO)) == C_FREE) {
   1036 			ata_channel_unlock(chp);
   1037 
   1038 			ata_free_xfer(chp, xfer);
   1039 			return;
   1040 		}
   1041 	}
   1042 
   1043 	ata_channel_unlock(chp);
   1044 
   1045 	ATADEBUG_PRINT(("atastart from ata_exec_xfer, flags 0x%x\n",
   1046 	    chp->ch_flags), DEBUG_XFERS);
   1047 	atastart(chp);
   1048 }
   1049 
   1050 /*
   1051  * Start I/O on a controller, for the given channel.
   1052  * The first xfer may be not for our channel if the channel queues
   1053  * are shared.
   1054  *
   1055  * MUST BE CALLED AT splbio()!
   1056  *
   1057  * XXX FIS-based switching with PMP
   1058  * Currently atastart() never schedules concurrent NCQ transfers to more than
   1059  * one drive, even when channel has several SATA drives attached via PMP.
   1060  * To support concurrent transfers to different drives with PMP, it would be
   1061  * necessary to implement FIS-based switching support in controller driver,
   1062  * and then adjust error handling and recovery to stop assuming at most
   1063  * one active drive.
   1064  */
   1065 void
   1066 atastart(struct ata_channel *chp)
   1067 {
   1068 	struct atac_softc *atac = chp->ch_atac;
   1069 	struct ata_queue *chq = chp->ch_queue;
   1070 	struct ata_xfer *xfer, *axfer;
   1071 	bool skipq;
   1072 
   1073 #ifdef ATA_DEBUG
   1074 	int spl1, spl2;
   1075 
   1076 	spl1 = splbio();
   1077 	spl2 = splbio();
   1078 	if (spl2 != spl1) {
   1079 		printf("atastart: not at splbio()\n");
   1080 		panic("atastart");
   1081 	}
   1082 	splx(spl2);
   1083 	splx(spl1);
   1084 #endif /* ATA_DEBUG */
   1085 
   1086 	ata_channel_lock(chp);
   1087 
   1088 again:
   1089 	/* is there a xfer ? */
   1090 	if ((xfer = SIMPLEQ_FIRST(&chp->ch_queue->queue_xfer)) == NULL) {
   1091 		ATADEBUG_PRINT(("%s(chp=%p): channel %d queue_xfer is empty\n",
   1092 		    __func__, chp, chp->ch_channel), DEBUG_XFERS);
   1093 		goto out;
   1094 	}
   1095 
   1096 	/*
   1097 	 * if someone is waiting for the command to be active, wake it up
   1098 	 * and let it process the command
   1099 	 */
   1100 	if (__predict_false(xfer->c_flags & C_WAITACT)) {
   1101 		ATADEBUG_PRINT(("atastart: xfer %p channel %d drive %d "
   1102 		    "wait active\n", xfer, chp->ch_channel, xfer->c_drive),
   1103 		    DEBUG_XFERS);
   1104 		cv_broadcast(&chp->ch_queue->c_active);
   1105 		goto out;
   1106 	}
   1107 
   1108 	skipq = ISSET(xfer->c_flags, C_SKIP_QUEUE);
   1109 
   1110 	/* is the queue frozen? */
   1111 	if (__predict_false(!skipq && chq->queue_freeze > 0)) {
   1112 		if (chq->queue_flags & QF_IDLE_WAIT) {
   1113 			chq->queue_flags &= ~QF_IDLE_WAIT;
   1114 			cv_signal(&chp->ch_queue->queue_idle);
   1115 		}
   1116 		ATADEBUG_PRINT(("%s(chp=%p): channel %d drive %d "
   1117 		    "queue frozen: %d\n",
   1118 		    __func__, chp, chp->ch_channel, xfer->c_drive,
   1119 		    chq->queue_freeze),
   1120 		    DEBUG_XFERS);
   1121 		goto out;
   1122 	}
   1123 
   1124 	/* all xfers on same queue must belong to the same channel */
   1125 	KASSERT(xfer->c_chp == chp);
   1126 
   1127 	/*
   1128 	 * Can only take the command if there are no current active
   1129 	 * commands, or if the command is NCQ and the active commands are also
   1130 	 * NCQ. If PM is in use and HBA driver doesn't support/use FIS-based
   1131 	 * switching, can only send commands to single drive.
   1132 	 * Need only check first xfer.
   1133 	 * XXX FIS-based switching - revisit
   1134 	 */
   1135 	if (!skipq && (axfer = TAILQ_FIRST(&chp->ch_queue->active_xfers))) {
   1136 		if (!ISSET(xfer->c_flags, C_NCQ) ||
   1137 		    !ISSET(axfer->c_flags, C_NCQ) ||
   1138 		    xfer->c_drive != axfer->c_drive)
   1139 			goto out;
   1140 	}
   1141 
   1142 	struct ata_drive_datas * const drvp = &chp->ch_drive[xfer->c_drive];
   1143 
   1144 	/*
   1145 	 * Are we on limit of active xfers ? If the queue has more
   1146 	 * than 1 openings, we keep one slot reserved for recovery or dump.
   1147 	 */
   1148 	KASSERT(chq->queue_active <= chq->queue_openings);
   1149 	const uint8_t chq_openings = (!skipq && chq->queue_openings > 1)
   1150 	    ? (chq->queue_openings - 1) : chq->queue_openings;
   1151 	const uint8_t drv_openings = ISSET(xfer->c_flags, C_NCQ)
   1152 	    ? drvp->drv_openings : ATA_MAX_OPENINGS;
   1153 	if (chq->queue_active >= MIN(chq_openings, drv_openings)) {
   1154 		if (skipq) {
   1155 			panic("%s: channel %d busy, xfer not possible",
   1156 			    __func__, chp->ch_channel);
   1157 		}
   1158 
   1159 		ATADEBUG_PRINT(("%s(chp=%p): channel %d completely busy\n",
   1160 		    __func__, chp, chp->ch_channel), DEBUG_XFERS);
   1161 		goto out;
   1162 	}
   1163 
   1164 	/* Slot allocation can fail if drv_openings < ch_openings */
   1165 	if (!ata_queue_alloc_slot(chp, &xfer->c_slot, drv_openings))
   1166 		goto out;
   1167 
   1168 	if (__predict_false(atac->atac_claim_hw)) {
   1169 		if (!atac->atac_claim_hw(chp, 0)) {
   1170 			ata_queue_free_slot(chp, xfer->c_slot);
   1171 			goto out;
   1172 		}
   1173 	}
   1174 
   1175 	/* Now committed to start the xfer */
   1176 
   1177 	ATADEBUG_PRINT(("%s(chp=%p): xfer %p channel %d drive %d\n",
   1178 	    __func__, chp, xfer, chp->ch_channel, xfer->c_drive), DEBUG_XFERS);
   1179 	if (drvp->drive_flags & ATA_DRIVE_RESET) {
   1180 		drvp->drive_flags &= ~ATA_DRIVE_RESET;
   1181 		drvp->state = 0;
   1182 	}
   1183 
   1184 	if (ISSET(xfer->c_flags, C_NCQ))
   1185 		SET(chp->ch_flags, ATACH_NCQ);
   1186 	else
   1187 		CLR(chp->ch_flags, ATACH_NCQ);
   1188 
   1189 	SIMPLEQ_REMOVE_HEAD(&chq->queue_xfer, c_xferchain);
   1190 
   1191 	ata_activate_xfer_locked(chp, xfer);
   1192 
   1193 	if (atac->atac_cap & ATAC_CAP_NOIRQ)
   1194 		KASSERT(xfer->c_flags & C_POLL);
   1195 
   1196 	switch (ata_xfer_start(xfer)) {
   1197 	case ATASTART_TH:
   1198 	case ATASTART_ABORT:
   1199 		/* don't start any further commands in this case */
   1200 		goto out;
   1201 	default:
   1202 		/* nothing to do */
   1203 		break;
   1204 	}
   1205 
   1206 	/* Queue more commands if possible, but not during recovery or dump */
   1207 	if (!skipq && chq->queue_active < chq->queue_openings)
   1208 		goto again;
   1209 
   1210 out:
   1211 	ata_channel_unlock(chp);
   1212 }
   1213 
   1214 int
   1215 ata_xfer_start(struct ata_xfer *xfer)
   1216 {
   1217 	struct ata_channel *chp = xfer->c_chp;
   1218 	int rv;
   1219 
   1220 	KASSERT(mutex_owned(&chp->ch_lock));
   1221 
   1222 	rv = xfer->ops->c_start(chp, xfer);
   1223 	switch (rv) {
   1224 	case ATASTART_STARTED:
   1225 		/* nothing to do */
   1226 		break;
   1227 	case ATASTART_TH:
   1228 		/* postpone xfer to thread */
   1229 		ata_thread_wake_locked(chp);
   1230 		break;
   1231 	case ATASTART_POLL:
   1232 		/* can happen even in thread context for some ATAPI devices */
   1233 		ata_channel_unlock(chp);
   1234 		KASSERT(xfer->ops != NULL && xfer->ops->c_poll != NULL);
   1235 		xfer->ops->c_poll(chp, xfer);
   1236 		ata_channel_lock(chp);
   1237 		break;
   1238 	case ATASTART_ABORT:
   1239 		ata_channel_unlock(chp);
   1240 		KASSERT(xfer->ops != NULL && xfer->ops->c_abort != NULL);
   1241 		xfer->ops->c_abort(chp, xfer);
   1242 		ata_channel_lock(chp);
   1243 		break;
   1244 	}
   1245 
   1246 	return rv;
   1247 }
   1248 
   1249 static void
   1250 ata_activate_xfer_locked(struct ata_channel *chp, struct ata_xfer *xfer)
   1251 {
   1252 	struct ata_queue * const chq = chp->ch_queue;
   1253 
   1254 	KASSERT(mutex_owned(&chp->ch_lock));
   1255 	KASSERT((chq->active_xfers_used & __BIT(xfer->c_slot)) == 0);
   1256 
   1257 	if ((xfer->c_flags & C_SKIP_QUEUE) == 0)
   1258 		TAILQ_INSERT_TAIL(&chq->active_xfers, xfer, c_activechain);
   1259 	else {
   1260 		/*
   1261 		 * Must go to head, so that ata_queue_get_active_xfer()
   1262 		 * returns the recovery command, and not some other
   1263 		 * random active transfer.
   1264 		 */
   1265 		TAILQ_INSERT_HEAD(&chq->active_xfers, xfer, c_activechain);
   1266 	}
   1267 	chq->active_xfers_used |= __BIT(xfer->c_slot);
   1268 	chq->queue_active++;
   1269 }
   1270 
   1271 /*
   1272  * Does it's own locking, does not require splbio().
   1273  * flags - whether to block waiting for free xfer
   1274  */
   1275 struct ata_xfer *
   1276 ata_get_xfer(struct ata_channel *chp, bool waitok)
   1277 {
   1278 	struct ata_xfer *xfer;
   1279 
   1280 	xfer = pool_get(&ata_xfer_pool, waitok ? PR_WAITOK : PR_NOWAIT);
   1281 	KASSERT(!waitok || xfer != NULL);
   1282 
   1283 	if (xfer != NULL) {
   1284 		/* zero everything */
   1285 		memset(xfer, 0, sizeof(*xfer));
   1286 	}
   1287 
   1288 	return xfer;
   1289 }
   1290 
   1291 /*
   1292  * ata_deactivate_xfer() must be always called prior to ata_free_xfer()
   1293  */
   1294 void
   1295 ata_free_xfer(struct ata_channel *chp, struct ata_xfer *xfer)
   1296 {
   1297 	struct ata_queue *chq = chp->ch_queue;
   1298 
   1299 	ata_channel_lock(chp);
   1300 
   1301 	if (__predict_false(xfer->c_flags & (C_WAITACT|C_WAITTIMO))) {
   1302 		/* Someone is waiting for this xfer, so we can't free now */
   1303 		xfer->c_flags |= C_FREE;
   1304 		cv_broadcast(&chq->c_active);
   1305 		ata_channel_unlock(chp);
   1306 		return;
   1307 	}
   1308 
   1309 	/* XXX move PIOBM and free_gw to deactivate? */
   1310 #if NATA_PIOBM		/* XXX wdc dependent code */
   1311 	if (__predict_false(xfer->c_flags & C_PIOBM)) {
   1312 		struct wdc_softc *wdc = CHAN_TO_WDC(chp);
   1313 
   1314 		/* finish the busmastering PIO */
   1315 		(*wdc->piobm_done)(wdc->dma_arg,
   1316 		    chp->ch_channel, xfer->c_drive);
   1317 		chp->ch_flags &= ~(ATACH_DMA_WAIT | ATACH_PIOBM_WAIT | ATACH_IRQ_WAIT);
   1318 	}
   1319 #endif
   1320 
   1321 	if (__predict_false(chp->ch_atac->atac_free_hw))
   1322 		chp->ch_atac->atac_free_hw(chp);
   1323 
   1324 	ata_channel_unlock(chp);
   1325 
   1326 	if (__predict_true(!ISSET(xfer->c_flags, C_PRIVATE_ALLOC)))
   1327 		pool_put(&ata_xfer_pool, xfer);
   1328 }
   1329 
   1330 void
   1331 ata_deactivate_xfer(struct ata_channel *chp, struct ata_xfer *xfer)
   1332 {
   1333 	struct ata_queue * const chq = chp->ch_queue;
   1334 
   1335 	ata_channel_lock(chp);
   1336 
   1337 	KASSERT(chq->queue_active > 0);
   1338 	KASSERT((chq->active_xfers_used & __BIT(xfer->c_slot)) != 0);
   1339 
   1340 	/* Stop only when this is last active xfer */
   1341 	if (chq->queue_active == 1)
   1342 		callout_stop(&chp->c_timo_callout);
   1343 
   1344 	if (callout_invoking(&chp->c_timo_callout))
   1345 		xfer->c_flags |= C_WAITTIMO;
   1346 
   1347 	TAILQ_REMOVE(&chq->active_xfers, xfer, c_activechain);
   1348 	chq->active_xfers_used &= ~__BIT(xfer->c_slot);
   1349 	chq->queue_active--;
   1350 
   1351 	ata_queue_free_slot(chp, xfer->c_slot);
   1352 
   1353 	if (xfer->c_flags & C_WAIT)
   1354 		cv_broadcast(&chq->c_cmd_finish);
   1355 
   1356 	ata_channel_unlock(chp);
   1357 }
   1358 
   1359 /*
   1360  * Called in c_intr hook. Must be called before before any deactivations
   1361  * are done - if there is drain pending, it calls c_kill_xfer hook which
   1362  * deactivates the xfer.
   1363  * Calls c_kill_xfer with channel lock free.
   1364  * Returns true if caller should just exit without further processing.
   1365  * Caller must not further access any part of xfer or any related controller
   1366  * structures in that case, it should just return.
   1367  */
   1368 bool
   1369 ata_waitdrain_xfer_check(struct ata_channel *chp, struct ata_xfer *xfer)
   1370 {
   1371 	int drive = xfer->c_drive;
   1372 	bool draining = false;
   1373 
   1374 	ata_channel_lock(chp);
   1375 
   1376 	if (chp->ch_drive[drive].drive_flags & ATA_DRIVE_WAITDRAIN) {
   1377 		ata_channel_unlock(chp);
   1378 
   1379 		xfer->ops->c_kill_xfer(chp, xfer, KILL_GONE);
   1380 
   1381 		ata_channel_lock(chp);
   1382 		chp->ch_drive[drive].drive_flags &= ~ATA_DRIVE_WAITDRAIN;
   1383 		cv_signal(&chp->ch_queue->queue_drain);
   1384 		draining = true;
   1385 	}
   1386 
   1387 	ata_channel_unlock(chp);
   1388 
   1389 	return draining;
   1390 }
   1391 
   1392 /*
   1393  * Check for race of normal transfer handling vs. timeout.
   1394  */
   1395 bool
   1396 ata_timo_xfer_check(struct ata_xfer *xfer)
   1397 {
   1398 	struct ata_channel *chp = xfer->c_chp;
   1399 	struct ata_drive_datas *drvp = &chp->ch_drive[xfer->c_drive];
   1400 
   1401 	ata_channel_lock(chp);
   1402 
   1403 	if (xfer->c_flags & C_WAITTIMO) {
   1404 		xfer->c_flags &= ~C_WAITTIMO;
   1405 
   1406 		/* Handle race vs. ata_free_xfer() */
   1407 		if (xfer->c_flags & C_FREE) {
   1408 			xfer->c_flags &= ~C_FREE;
   1409 			ata_channel_unlock(chp);
   1410 
   1411 	    		device_printf(drvp->drv_softc,
   1412 			    "xfer %"PRIxPTR" freed while invoking timeout\n",
   1413 			    (intptr_t)xfer & PAGE_MASK);
   1414 
   1415 			ata_free_xfer(chp, xfer);
   1416 			return true;
   1417 		}
   1418 
   1419 		/* Race vs. callout_stop() in ata_deactivate_xfer() */
   1420 		ata_channel_unlock(chp);
   1421 
   1422 	    	device_printf(drvp->drv_softc,
   1423 		    "xfer %"PRIxPTR" deactivated while invoking timeout\n",
   1424 		    (intptr_t)xfer & PAGE_MASK);
   1425 		return true;
   1426 	}
   1427 
   1428 	ata_channel_unlock(chp);
   1429 
   1430 	/* No race, proceed with timeout handling */
   1431 	return false;
   1432 }
   1433 
   1434 /*
   1435  * Kill off all active xfers for a ata_channel.
   1436  *
   1437  * Must be called with channel lock held.
   1438  */
   1439 void
   1440 ata_kill_active(struct ata_channel *chp, int reason, int flags)
   1441 {
   1442 	struct ata_queue * const chq = chp->ch_queue;
   1443 	struct ata_xfer *xfer, *xfernext;
   1444 
   1445 	KASSERT(mutex_owned(&chp->ch_lock));
   1446 
   1447 	TAILQ_FOREACH_SAFE(xfer, &chq->active_xfers, c_activechain, xfernext) {
   1448 		xfer->ops->c_kill_xfer(xfer->c_chp, xfer, reason);
   1449 	}
   1450 }
   1451 
   1452 /*
   1453  * Kill off all pending xfers for a drive.
   1454  */
   1455 void
   1456 ata_kill_pending(struct ata_drive_datas *drvp)
   1457 {
   1458 	struct ata_channel * const chp = drvp->chnl_softc;
   1459 	struct ata_queue * const chq = chp->ch_queue;
   1460 	struct ata_xfer *xfer;
   1461 
   1462 	ata_channel_lock(chp);
   1463 
   1464 	/* Kill all pending transfers */
   1465 	while ((xfer = SIMPLEQ_FIRST(&chq->queue_xfer))) {
   1466 		KASSERT(xfer->c_chp == chp);
   1467 
   1468 		if (xfer->c_drive != drvp->drive)
   1469 			continue;
   1470 
   1471 		SIMPLEQ_REMOVE_HEAD(&chp->ch_queue->queue_xfer, c_xferchain);
   1472 
   1473 		/*
   1474 		 * Keep the lock, so that we get deadlock (and 'locking against
   1475 		 * myself' with LOCKDEBUG), instead of silent
   1476 		 * data corruption, if the hook tries to call back into
   1477 		 * middle layer for inactive xfer.
   1478 		 */
   1479 		xfer->ops->c_kill_xfer(chp, xfer, KILL_GONE_INACTIVE);
   1480 	}
   1481 
   1482 	/* Wait until all active transfers on the drive finish */
   1483 	while (chq->queue_active > 0) {
   1484 		bool drv_active = false;
   1485 
   1486 		TAILQ_FOREACH(xfer, &chq->active_xfers, c_activechain) {
   1487 			KASSERT(xfer->c_chp == chp);
   1488 
   1489 			if (xfer->c_drive == drvp->drive) {
   1490 				drv_active = true;
   1491 				break;
   1492 			}
   1493 		}
   1494 
   1495 		if (!drv_active) {
   1496 			/* all finished */
   1497 			break;
   1498 		}
   1499 
   1500 		drvp->drive_flags |= ATA_DRIVE_WAITDRAIN;
   1501 		cv_wait(&chq->queue_drain, &chp->ch_lock);
   1502 	}
   1503 
   1504 	ata_channel_unlock(chp);
   1505 }
   1506 
   1507 static void
   1508 ata_channel_freeze_locked(struct ata_channel *chp)
   1509 {
   1510 	chp->ch_queue->queue_freeze++;
   1511 
   1512 	ATADEBUG_PRINT(("%s(chp=%p) -> %d\n", __func__, chp,
   1513 	    chp->ch_queue->queue_freeze), DEBUG_FUNCS | DEBUG_XFERS);
   1514 }
   1515 
   1516 void
   1517 ata_channel_freeze(struct ata_channel *chp)
   1518 {
   1519 	ata_channel_lock(chp);
   1520 	ata_channel_freeze_locked(chp);
   1521 	ata_channel_unlock(chp);
   1522 }
   1523 
   1524 void
   1525 ata_channel_thaw_locked(struct ata_channel *chp)
   1526 {
   1527 	KASSERT(mutex_owned(&chp->ch_lock));
   1528 	KASSERT(chp->ch_queue->queue_freeze > 0);
   1529 
   1530 	chp->ch_queue->queue_freeze--;
   1531 
   1532 	ATADEBUG_PRINT(("%s(chp=%p) -> %d\n", __func__, chp,
   1533 	    chp->ch_queue->queue_freeze), DEBUG_FUNCS | DEBUG_XFERS);
   1534 }
   1535 
   1536 /*
   1537  * ata_thread_run:
   1538  *
   1539  *	Reset and ATA channel. Channel lock must be held. arg is type-specific.
   1540  */
   1541 void
   1542 ata_thread_run(struct ata_channel *chp, int flags, int type, int arg)
   1543 {
   1544 	struct atac_softc *atac = chp->ch_atac;
   1545 	bool threset = false;
   1546 	struct ata_drive_datas *drvp;
   1547 
   1548 	ata_channel_lock_owned(chp);
   1549 
   1550 	/*
   1551 	 * If we can poll or wait it's OK, otherwise wake up the
   1552 	 * kernel thread to do it for us.
   1553 	 */
   1554 	ATADEBUG_PRINT(("%s flags 0x%x ch_flags 0x%x\n",
   1555 	    __func__, flags, chp->ch_flags), DEBUG_FUNCS | DEBUG_XFERS);
   1556 	if ((flags & (AT_POLL | AT_WAIT)) == 0) {
   1557 		switch (type) {
   1558 		case ATACH_TH_RESET:
   1559 			if (chp->ch_flags & ATACH_TH_RESET) {
   1560 				/* No need to schedule another reset */
   1561 				return;
   1562 			}
   1563 			break;
   1564 		case ATACH_TH_DRIVE_RESET:
   1565 		    {
   1566 			int drive = arg;
   1567 
   1568 			KASSERT(drive <= chp->ch_ndrives);
   1569 			drvp = &chp->ch_drive[drive];
   1570 
   1571 			if (drvp->drive_flags & ATA_DRIVE_TH_RESET) {
   1572 				/* No need to schedule another reset */
   1573 				return;
   1574 			}
   1575 			drvp->drive_flags |= ATA_DRIVE_TH_RESET;
   1576 			break;
   1577 		    }
   1578 		case ATACH_TH_RECOVERY:
   1579 		    {
   1580 			uint32_t tfd = (uint32_t)arg;
   1581 
   1582 			KASSERT((chp->ch_flags & ATACH_RECOVERING) == 0);
   1583 			chp->recovery_tfd = tfd;
   1584 			break;
   1585 		    }
   1586 		default:
   1587 			panic("%s: unknown type: %x", __func__, type);
   1588 			/* NOTREACHED */
   1589 		}
   1590 
   1591 		/*
   1592 		 * Block execution of other commands while reset is scheduled
   1593 		 * to a thread.
   1594 		 */
   1595 		ata_channel_freeze_locked(chp);
   1596 		chp->ch_flags |= type;
   1597 
   1598 		cv_signal(&chp->ch_thr_idle);
   1599 		return;
   1600 	}
   1601 
   1602 	/* Block execution of other commands during reset */
   1603 	ata_channel_freeze_locked(chp);
   1604 
   1605 	/*
   1606 	 * If reset has been scheduled to a thread, then clear
   1607 	 * the flag now so that the thread won't try to execute it if
   1608 	 * we happen to sleep, and thaw one more time after the reset.
   1609 	 */
   1610 	if (chp->ch_flags & type) {
   1611 		chp->ch_flags &= ~type;
   1612 		threset = true;
   1613 	}
   1614 
   1615 	switch (type) {
   1616 	case ATACH_TH_RESET:
   1617 		(*atac->atac_bustype_ata->ata_reset_channel)(chp, flags);
   1618 
   1619 		KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL);
   1620 		for (int drive = 0; drive < chp->ch_ndrives; drive++)
   1621 			chp->ch_drive[drive].state = 0;
   1622 		break;
   1623 
   1624 	case ATACH_TH_DRIVE_RESET:
   1625 	    {
   1626 		int drive = arg;
   1627 
   1628 		KASSERT(drive <= chp->ch_ndrives);
   1629 		drvp = &chp->ch_drive[drive];
   1630 		(*atac->atac_bustype_ata->ata_reset_drive)(drvp, flags, NULL);
   1631 		drvp->state = 0;
   1632 		break;
   1633 	    }
   1634 
   1635 	case ATACH_TH_RECOVERY:
   1636 	    {
   1637 		uint32_t tfd = (uint32_t)arg;
   1638 
   1639 		KASSERT((chp->ch_flags & ATACH_RECOVERING) == 0);
   1640 		KASSERT(atac->atac_bustype_ata->ata_recovery != NULL);
   1641 
   1642 		SET(chp->ch_flags, ATACH_RECOVERING);
   1643 		(*atac->atac_bustype_ata->ata_recovery)(chp, flags, tfd);
   1644 		CLR(chp->ch_flags, ATACH_RECOVERING);
   1645 		break;
   1646 	    }
   1647 
   1648 	default:
   1649 		panic("%s: unknown type: %x", __func__, type);
   1650 		/* NOTREACHED */
   1651 	}
   1652 
   1653 	/*
   1654 	 * Thaw one extra time to clear the freeze done when the reset has
   1655 	 * been scheduled to the thread.
   1656 	 */
   1657 	if (threset)
   1658 		ata_channel_thaw_locked(chp);
   1659 
   1660 	/* Allow commands to run again */
   1661 	ata_channel_thaw_locked(chp);
   1662 
   1663 	/* Signal the thread in case there is an xfer to run */
   1664 	cv_signal(&chp->ch_thr_idle);
   1665 }
   1666 
   1667 int
   1668 ata_addref(struct ata_channel *chp)
   1669 {
   1670 	struct atac_softc *atac = chp->ch_atac;
   1671 	struct scsipi_adapter *adapt = &atac->atac_atapi_adapter._generic;
   1672 	int s, error = 0;
   1673 
   1674 	s = splbio();
   1675 	if (adapt->adapt_refcnt++ == 0 &&
   1676 	    adapt->adapt_enable != NULL) {
   1677 		error = (*adapt->adapt_enable)(atac->atac_dev, 1);
   1678 		if (error)
   1679 			adapt->adapt_refcnt--;
   1680 	}
   1681 	splx(s);
   1682 	return (error);
   1683 }
   1684 
   1685 void
   1686 ata_delref(struct ata_channel *chp)
   1687 {
   1688 	struct atac_softc *atac = chp->ch_atac;
   1689 	struct scsipi_adapter *adapt = &atac->atac_atapi_adapter._generic;
   1690 	int s;
   1691 
   1692 	s = splbio();
   1693 	if (adapt->adapt_refcnt-- == 1 &&
   1694 	    adapt->adapt_enable != NULL)
   1695 		(void) (*adapt->adapt_enable)(atac->atac_dev, 0);
   1696 	splx(s);
   1697 }
   1698 
   1699 void
   1700 ata_print_modes(struct ata_channel *chp)
   1701 {
   1702 	struct atac_softc *atac = chp->ch_atac;
   1703 	int drive;
   1704 	struct ata_drive_datas *drvp;
   1705 
   1706 	KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL);
   1707 	for (drive = 0; drive < chp->ch_ndrives; drive++) {
   1708 		drvp = &chp->ch_drive[drive];
   1709 		if (drvp->drive_type == ATA_DRIVET_NONE ||
   1710 		    drvp->drv_softc == NULL)
   1711 			continue;
   1712 		aprint_verbose("%s(%s:%d:%d): using PIO mode %d",
   1713 			device_xname(drvp->drv_softc),
   1714 			device_xname(atac->atac_dev),
   1715 			chp->ch_channel, drvp->drive, drvp->PIO_mode);
   1716 #if NATA_DMA
   1717 		if (drvp->drive_flags & ATA_DRIVE_DMA)
   1718 			aprint_verbose(", DMA mode %d", drvp->DMA_mode);
   1719 #if NATA_UDMA
   1720 		if (drvp->drive_flags & ATA_DRIVE_UDMA) {
   1721 			aprint_verbose(", Ultra-DMA mode %d", drvp->UDMA_mode);
   1722 			if (drvp->UDMA_mode == 2)
   1723 				aprint_verbose(" (Ultra/33)");
   1724 			else if (drvp->UDMA_mode == 4)
   1725 				aprint_verbose(" (Ultra/66)");
   1726 			else if (drvp->UDMA_mode == 5)
   1727 				aprint_verbose(" (Ultra/100)");
   1728 			else if (drvp->UDMA_mode == 6)
   1729 				aprint_verbose(" (Ultra/133)");
   1730 		}
   1731 #endif	/* NATA_UDMA */
   1732 #endif	/* NATA_DMA */
   1733 #if NATA_DMA || NATA_PIOBM
   1734 		if (0
   1735 #if NATA_DMA
   1736 		    || (drvp->drive_flags & (ATA_DRIVE_DMA | ATA_DRIVE_UDMA))
   1737 #endif
   1738 #if NATA_PIOBM
   1739 		    /* PIOBM capable controllers use DMA for PIO commands */
   1740 		    || (atac->atac_cap & ATAC_CAP_PIOBM)
   1741 #endif
   1742 		    )
   1743 			aprint_verbose(" (using DMA)");
   1744 
   1745 		if (drvp->drive_flags & ATA_DRIVE_NCQ) {
   1746 			aprint_verbose(", NCQ (%d tags)%s",
   1747 			    ATA_REAL_OPENINGS(chp->ch_queue->queue_openings),
   1748 			    (drvp->drive_flags & ATA_DRIVE_NCQ_PRIO)
   1749 			    ? " w/PRIO" : "");
   1750 		} else if (drvp->drive_flags & ATA_DRIVE_WFUA)
   1751 			aprint_verbose(", WRITE DMA FUA EXT");
   1752 
   1753 #endif	/* NATA_DMA || NATA_PIOBM */
   1754 		aprint_verbose("\n");
   1755 	}
   1756 }
   1757 
   1758 #if NATA_DMA
   1759 /*
   1760  * downgrade the transfer mode of a drive after an error. return 1 if
   1761  * downgrade was possible, 0 otherwise.
   1762  *
   1763  * MUST BE CALLED AT splbio()!
   1764  */
   1765 int
   1766 ata_downgrade_mode(struct ata_drive_datas *drvp, int flags)
   1767 {
   1768 	struct ata_channel *chp = drvp->chnl_softc;
   1769 	struct atac_softc *atac = chp->ch_atac;
   1770 	device_t drv_dev = drvp->drv_softc;
   1771 	int cf_flags = device_cfdata(drv_dev)->cf_flags;
   1772 
   1773 	/* if drive or controller don't know its mode, we can't do much */
   1774 	if ((drvp->drive_flags & ATA_DRIVE_MODE) == 0 ||
   1775 	    (atac->atac_set_modes == NULL))
   1776 		return 0;
   1777 	/* current drive mode was set by a config flag, let it this way */
   1778 	if ((cf_flags & ATA_CONFIG_PIO_SET) ||
   1779 	    (cf_flags & ATA_CONFIG_DMA_SET) ||
   1780 	    (cf_flags & ATA_CONFIG_UDMA_SET))
   1781 		return 0;
   1782 
   1783 #if NATA_UDMA
   1784 	/*
   1785 	 * If we were using Ultra-DMA mode, downgrade to the next lower mode.
   1786 	 */
   1787 	if ((drvp->drive_flags & ATA_DRIVE_UDMA) && drvp->UDMA_mode >= 2) {
   1788 		drvp->UDMA_mode--;
   1789 		aprint_error_dev(drv_dev,
   1790 		    "transfer error, downgrading to Ultra-DMA mode %d\n",
   1791 		    drvp->UDMA_mode);
   1792 	}
   1793 #endif
   1794 
   1795 	/*
   1796 	 * If we were using ultra-DMA, don't downgrade to multiword DMA.
   1797 	 */
   1798 	else if (drvp->drive_flags & (ATA_DRIVE_DMA | ATA_DRIVE_UDMA)) {
   1799 		drvp->drive_flags &= ~(ATA_DRIVE_DMA | ATA_DRIVE_UDMA);
   1800 		drvp->PIO_mode = drvp->PIO_cap;
   1801 		aprint_error_dev(drv_dev,
   1802 		    "transfer error, downgrading to PIO mode %d\n",
   1803 		    drvp->PIO_mode);
   1804 	} else /* already using PIO, can't downgrade */
   1805 		return 0;
   1806 
   1807 	(*atac->atac_set_modes)(chp);
   1808 	ata_print_modes(chp);
   1809 	/* reset the channel, which will schedule all drives for setup */
   1810 	ata_thread_run(chp, flags, ATACH_TH_RESET, ATACH_NODRIVE);
   1811 	return 1;
   1812 }
   1813 #endif	/* NATA_DMA */
   1814 
   1815 /*
   1816  * Probe drive's capabilities, for use by the controller later
   1817  * Assumes drvp points to an existing drive.
   1818  */
   1819 void
   1820 ata_probe_caps(struct ata_drive_datas *drvp)
   1821 {
   1822 	struct ataparams params, params2;
   1823 	struct ata_channel *chp = drvp->chnl_softc;
   1824 	struct atac_softc *atac = chp->ch_atac;
   1825 	device_t drv_dev = drvp->drv_softc;
   1826 	int i, printed = 0;
   1827 	const char *sep = "";
   1828 	int cf_flags;
   1829 
   1830 	if (ata_get_params(drvp, AT_WAIT, &params) != CMD_OK) {
   1831 		/* IDENTIFY failed. Can't tell more about the device */
   1832 		return;
   1833 	}
   1834 	if ((atac->atac_cap & (ATAC_CAP_DATA16 | ATAC_CAP_DATA32)) ==
   1835 	    (ATAC_CAP_DATA16 | ATAC_CAP_DATA32)) {
   1836 		/*
   1837 		 * Controller claims 16 and 32 bit transfers.
   1838 		 * Re-do an IDENTIFY with 32-bit transfers,
   1839 		 * and compare results.
   1840 		 */
   1841 		ata_channel_lock(chp);
   1842 		drvp->drive_flags |= ATA_DRIVE_CAP32;
   1843 		ata_channel_unlock(chp);
   1844 		ata_get_params(drvp, AT_WAIT, &params2);
   1845 		if (memcmp(&params, &params2, sizeof(struct ataparams)) != 0) {
   1846 			/* Not good. fall back to 16bits */
   1847 			ata_channel_lock(chp);
   1848 			drvp->drive_flags &= ~ATA_DRIVE_CAP32;
   1849 			ata_channel_unlock(chp);
   1850 		} else {
   1851 			aprint_verbose_dev(drv_dev, "32-bit data port\n");
   1852 		}
   1853 	}
   1854 #if 0 /* Some ultra-DMA drives claims to only support ATA-3. sigh */
   1855 	if (params.atap_ata_major > 0x01 &&
   1856 	    params.atap_ata_major != 0xffff) {
   1857 		for (i = 14; i > 0; i--) {
   1858 			if (params.atap_ata_major & (1 << i)) {
   1859 				aprint_verbose_dev(drv_dev,
   1860 				    "ATA version %d\n", i);
   1861 				drvp->ata_vers = i;
   1862 				break;
   1863 			}
   1864 		}
   1865 	}
   1866 #endif
   1867 
   1868 	/* An ATAPI device is at last PIO mode 3 */
   1869 	if (drvp->drive_type == ATA_DRIVET_ATAPI)
   1870 		drvp->PIO_mode = 3;
   1871 
   1872 	/*
   1873 	 * It's not in the specs, but it seems that some drive
   1874 	 * returns 0xffff in atap_extensions when this field is invalid
   1875 	 */
   1876 	if (params.atap_extensions != 0xffff &&
   1877 	    (params.atap_extensions & WDC_EXT_MODES)) {
   1878 		/*
   1879 		 * XXX some drives report something wrong here (they claim to
   1880 		 * support PIO mode 8 !). As mode is coded on 3 bits in
   1881 		 * SET FEATURE, limit it to 7 (so limit i to 4).
   1882 		 * If higher mode than 7 is found, abort.
   1883 		 */
   1884 		for (i = 7; i >= 0; i--) {
   1885 			if ((params.atap_piomode_supp & (1 << i)) == 0)
   1886 				continue;
   1887 			if (i > 4)
   1888 				return;
   1889 			/*
   1890 			 * See if mode is accepted.
   1891 			 * If the controller can't set its PIO mode,
   1892 			 * assume the defaults are good, so don't try
   1893 			 * to set it
   1894 			 */
   1895 			if (atac->atac_set_modes)
   1896 				/*
   1897 				 * It's OK to pool here, it's fast enough
   1898 				 * to not bother waiting for interrupt
   1899 				 */
   1900 				if (ata_set_mode(drvp, 0x08 | (i + 3),
   1901 				   AT_WAIT) != CMD_OK)
   1902 					continue;
   1903 			if (!printed) {
   1904 				aprint_verbose_dev(drv_dev,
   1905 				    "drive supports PIO mode %d", i + 3);
   1906 				sep = ",";
   1907 				printed = 1;
   1908 			}
   1909 			/*
   1910 			 * If controller's driver can't set its PIO mode,
   1911 			 * get the highter one for the drive.
   1912 			 */
   1913 			if (atac->atac_set_modes == NULL ||
   1914 			    atac->atac_pio_cap >= i + 3) {
   1915 				drvp->PIO_mode = i + 3;
   1916 				drvp->PIO_cap = i + 3;
   1917 				break;
   1918 			}
   1919 		}
   1920 		if (!printed) {
   1921 			/*
   1922 			 * We didn't find a valid PIO mode.
   1923 			 * Assume the values returned for DMA are buggy too
   1924 			 */
   1925 			return;
   1926 		}
   1927 		ata_channel_lock(chp);
   1928 		drvp->drive_flags |= ATA_DRIVE_MODE;
   1929 		ata_channel_unlock(chp);
   1930 		printed = 0;
   1931 		for (i = 7; i >= 0; i--) {
   1932 			if ((params.atap_dmamode_supp & (1 << i)) == 0)
   1933 				continue;
   1934 #if NATA_DMA
   1935 			if ((atac->atac_cap & ATAC_CAP_DMA) &&
   1936 			    atac->atac_set_modes != NULL)
   1937 				if (ata_set_mode(drvp, 0x20 | i, AT_WAIT)
   1938 				    != CMD_OK)
   1939 					continue;
   1940 #endif
   1941 			if (!printed) {
   1942 				aprint_verbose("%s DMA mode %d", sep, i);
   1943 				sep = ",";
   1944 				printed = 1;
   1945 			}
   1946 #if NATA_DMA
   1947 			if (atac->atac_cap & ATAC_CAP_DMA) {
   1948 				if (atac->atac_set_modes != NULL &&
   1949 				    atac->atac_dma_cap < i)
   1950 					continue;
   1951 				drvp->DMA_mode = i;
   1952 				drvp->DMA_cap = i;
   1953 				ata_channel_lock(chp);
   1954 				drvp->drive_flags |= ATA_DRIVE_DMA;
   1955 				ata_channel_unlock(chp);
   1956 			}
   1957 #endif
   1958 			break;
   1959 		}
   1960 		if (params.atap_extensions & WDC_EXT_UDMA_MODES) {
   1961 			printed = 0;
   1962 			for (i = 7; i >= 0; i--) {
   1963 				if ((params.atap_udmamode_supp & (1 << i))
   1964 				    == 0)
   1965 					continue;
   1966 #if NATA_UDMA
   1967 				if (atac->atac_set_modes != NULL &&
   1968 				    (atac->atac_cap & ATAC_CAP_UDMA))
   1969 					if (ata_set_mode(drvp, 0x40 | i,
   1970 					    AT_WAIT) != CMD_OK)
   1971 						continue;
   1972 #endif
   1973 				if (!printed) {
   1974 					aprint_verbose("%s Ultra-DMA mode %d",
   1975 					    sep, i);
   1976 					if (i == 2)
   1977 						aprint_verbose(" (Ultra/33)");
   1978 					else if (i == 4)
   1979 						aprint_verbose(" (Ultra/66)");
   1980 					else if (i == 5)
   1981 						aprint_verbose(" (Ultra/100)");
   1982 					else if (i == 6)
   1983 						aprint_verbose(" (Ultra/133)");
   1984 					sep = ",";
   1985 					printed = 1;
   1986 				}
   1987 #if NATA_UDMA
   1988 				if (atac->atac_cap & ATAC_CAP_UDMA) {
   1989 					if (atac->atac_set_modes != NULL &&
   1990 					    atac->atac_udma_cap < i)
   1991 						continue;
   1992 					drvp->UDMA_mode = i;
   1993 					drvp->UDMA_cap = i;
   1994 					ata_channel_lock(chp);
   1995 					drvp->drive_flags |= ATA_DRIVE_UDMA;
   1996 					ata_channel_unlock(chp);
   1997 				}
   1998 #endif
   1999 				break;
   2000 			}
   2001 		}
   2002 	}
   2003 
   2004 	ata_channel_lock(chp);
   2005 	drvp->drive_flags &= ~ATA_DRIVE_NOSTREAM;
   2006 	if (drvp->drive_type == ATA_DRIVET_ATAPI) {
   2007 		if (atac->atac_cap & ATAC_CAP_ATAPI_NOSTREAM)
   2008 			drvp->drive_flags |= ATA_DRIVE_NOSTREAM;
   2009 	} else {
   2010 		if (atac->atac_cap & ATAC_CAP_ATA_NOSTREAM)
   2011 			drvp->drive_flags |= ATA_DRIVE_NOSTREAM;
   2012 	}
   2013 	ata_channel_unlock(chp);
   2014 
   2015 	/* Try to guess ATA version here, if it didn't get reported */
   2016 	if (drvp->ata_vers == 0) {
   2017 #if NATA_UDMA
   2018 		if (drvp->drive_flags & ATA_DRIVE_UDMA)
   2019 			drvp->ata_vers = 4; /* should be at last ATA-4 */
   2020 		else
   2021 #endif
   2022 		if (drvp->PIO_cap > 2)
   2023 			drvp->ata_vers = 2; /* should be at last ATA-2 */
   2024 	}
   2025 	cf_flags = device_cfdata(drv_dev)->cf_flags;
   2026 	if (cf_flags & ATA_CONFIG_PIO_SET) {
   2027 		ata_channel_lock(chp);
   2028 		drvp->PIO_mode =
   2029 		    (cf_flags & ATA_CONFIG_PIO_MODES) >> ATA_CONFIG_PIO_OFF;
   2030 		drvp->drive_flags |= ATA_DRIVE_MODE;
   2031 		ata_channel_unlock(chp);
   2032 	}
   2033 #if NATA_DMA
   2034 	if ((atac->atac_cap & ATAC_CAP_DMA) == 0) {
   2035 		/* don't care about DMA modes */
   2036 		return;
   2037 	}
   2038 	if (cf_flags & ATA_CONFIG_DMA_SET) {
   2039 		ata_channel_lock(chp);
   2040 		if ((cf_flags & ATA_CONFIG_DMA_MODES) ==
   2041 		    ATA_CONFIG_DMA_DISABLE) {
   2042 			drvp->drive_flags &= ~ATA_DRIVE_DMA;
   2043 		} else {
   2044 			drvp->DMA_mode = (cf_flags & ATA_CONFIG_DMA_MODES) >>
   2045 			    ATA_CONFIG_DMA_OFF;
   2046 			drvp->drive_flags |= ATA_DRIVE_DMA | ATA_DRIVE_MODE;
   2047 		}
   2048 		ata_channel_unlock(chp);
   2049 	}
   2050 
   2051 	/*
   2052 	 * Probe WRITE DMA FUA EXT. Support is mandatory for devices
   2053 	 * supporting LBA48, but nevertheless confirm with the feature flag.
   2054 	 */
   2055 	if (drvp->drive_flags & ATA_DRIVE_DMA) {
   2056 		if ((params.atap_cmd2_en & ATA_CMD2_LBA48) != 0
   2057 		    && (params.atap_cmd_def & ATA_CMDE_WFE)) {
   2058 			drvp->drive_flags |= ATA_DRIVE_WFUA;
   2059 			aprint_verbose("%s WRITE DMA FUA", sep);
   2060 			sep = ",";
   2061 		}
   2062 	}
   2063 
   2064 	/* Probe NCQ support - READ/WRITE FPDMA QUEUED command support */
   2065 	ata_channel_lock(chp);
   2066 	drvp->drv_openings = 1;
   2067 	if (params.atap_sata_caps & SATA_NATIVE_CMDQ) {
   2068 		if (atac->atac_cap & ATAC_CAP_NCQ)
   2069 			drvp->drive_flags |= ATA_DRIVE_NCQ;
   2070 		drvp->drv_openings =
   2071 		    (params.atap_queuedepth & WDC_QUEUE_DEPTH_MASK) + 1;
   2072 		aprint_verbose("%s NCQ (%d tags)", sep, drvp->drv_openings);
   2073 		sep = ",";
   2074 
   2075 		if (params.atap_sata_caps & SATA_NCQ_PRIO) {
   2076 			drvp->drive_flags |= ATA_DRIVE_NCQ_PRIO;
   2077 			aprint_verbose(" w/PRIO");
   2078 		}
   2079 	}
   2080 	ata_channel_unlock(chp);
   2081 
   2082 	if (printed)
   2083 		aprint_verbose("\n");
   2084 
   2085 #if NATA_UDMA
   2086 	if ((atac->atac_cap & ATAC_CAP_UDMA) == 0) {
   2087 		/* don't care about UDMA modes */
   2088 		return;
   2089 	}
   2090 	if (cf_flags & ATA_CONFIG_UDMA_SET) {
   2091 		ata_channel_lock(chp);
   2092 		if ((cf_flags & ATA_CONFIG_UDMA_MODES) ==
   2093 		    ATA_CONFIG_UDMA_DISABLE) {
   2094 			drvp->drive_flags &= ~ATA_DRIVE_UDMA;
   2095 		} else {
   2096 			drvp->UDMA_mode = (cf_flags & ATA_CONFIG_UDMA_MODES) >>
   2097 			    ATA_CONFIG_UDMA_OFF;
   2098 			drvp->drive_flags |= ATA_DRIVE_UDMA | ATA_DRIVE_MODE;
   2099 		}
   2100 		ata_channel_unlock(chp);
   2101 	}
   2102 #endif	/* NATA_UDMA */
   2103 #endif	/* NATA_DMA */
   2104 }
   2105 
   2106 /* management of the /dev/atabus* devices */
   2107 int
   2108 atabusopen(dev_t dev, int flag, int fmt, struct lwp *l)
   2109 {
   2110 	struct atabus_softc *sc;
   2111 	int error;
   2112 
   2113 	sc = device_lookup_private(&atabus_cd, minor(dev));
   2114 	if (sc == NULL)
   2115 		return (ENXIO);
   2116 
   2117 	if (sc->sc_flags & ATABUSCF_OPEN)
   2118 		return (EBUSY);
   2119 
   2120 	if ((error = ata_addref(sc->sc_chan)) != 0)
   2121 		return (error);
   2122 
   2123 	sc->sc_flags |= ATABUSCF_OPEN;
   2124 
   2125 	return (0);
   2126 }
   2127 
   2128 
   2129 int
   2130 atabusclose(dev_t dev, int flag, int fmt, struct lwp *l)
   2131 {
   2132 	struct atabus_softc *sc =
   2133 	    device_lookup_private(&atabus_cd, minor(dev));
   2134 
   2135 	ata_delref(sc->sc_chan);
   2136 
   2137 	sc->sc_flags &= ~ATABUSCF_OPEN;
   2138 
   2139 	return (0);
   2140 }
   2141 
   2142 int
   2143 atabusioctl(dev_t dev, u_long cmd, void *addr, int flag, struct lwp *l)
   2144 {
   2145 	struct atabus_softc *sc =
   2146 	    device_lookup_private(&atabus_cd, minor(dev));
   2147 	struct ata_channel *chp = sc->sc_chan;
   2148 	int min_drive, max_drive, drive;
   2149 	int error;
   2150 
   2151 	/*
   2152 	 * Enforce write permission for ioctls that change the
   2153 	 * state of the bus.  Host adapter specific ioctls must
   2154 	 * be checked by the adapter driver.
   2155 	 */
   2156 	switch (cmd) {
   2157 	case ATABUSIOSCAN:
   2158 	case ATABUSIODETACH:
   2159 	case ATABUSIORESET:
   2160 		if ((flag & FWRITE) == 0)
   2161 			return (EBADF);
   2162 	}
   2163 
   2164 	switch (cmd) {
   2165 	case ATABUSIORESET:
   2166 		ata_channel_lock(chp);
   2167 		ata_thread_run(sc->sc_chan, AT_WAIT | AT_POLL,
   2168 		    ATACH_TH_RESET, ATACH_NODRIVE);
   2169 		ata_channel_unlock(chp);
   2170 		return 0;
   2171 	case ATABUSIOSCAN:
   2172 	{
   2173 #if 0
   2174 		struct atabusioscan_args *a=
   2175 		    (struct atabusioscan_args *)addr;
   2176 #endif
   2177 		if ((chp->ch_drive[0].drive_type == ATA_DRIVET_OLD) ||
   2178 		    (chp->ch_drive[1].drive_type == ATA_DRIVET_OLD))
   2179 			return (EOPNOTSUPP);
   2180 		return (EOPNOTSUPP);
   2181 	}
   2182 	case ATABUSIODETACH:
   2183 	{
   2184 		struct atabusiodetach_args *a=
   2185 		    (struct atabusiodetach_args *)addr;
   2186 		if ((chp->ch_drive[0].drive_type == ATA_DRIVET_OLD) ||
   2187 		    (chp->ch_drive[1].drive_type == ATA_DRIVET_OLD))
   2188 			return (EOPNOTSUPP);
   2189 		switch (a->at_dev) {
   2190 		case -1:
   2191 			min_drive = 0;
   2192 			max_drive = 1;
   2193 			break;
   2194 		case 0:
   2195 		case 1:
   2196 			min_drive = max_drive = a->at_dev;
   2197 			break;
   2198 		default:
   2199 			return (EINVAL);
   2200 		}
   2201 		for (drive = min_drive; drive <= max_drive; drive++) {
   2202 			if (chp->ch_drive[drive].drv_softc != NULL) {
   2203 				error = config_detach(
   2204 				    chp->ch_drive[drive].drv_softc, 0);
   2205 				if (error)
   2206 					return (error);
   2207 				KASSERT(chp->ch_drive[drive].drv_softc == NULL);
   2208 			}
   2209 		}
   2210 		return 0;
   2211 	}
   2212 	default:
   2213 		return ENOTTY;
   2214 	}
   2215 }
   2216 
   2217 static bool
   2218 atabus_suspend(device_t dv, const pmf_qual_t *qual)
   2219 {
   2220 	struct atabus_softc *sc = device_private(dv);
   2221 	struct ata_channel *chp = sc->sc_chan;
   2222 
   2223 	ata_channel_idle(chp);
   2224 
   2225 	return true;
   2226 }
   2227 
   2228 static bool
   2229 atabus_resume(device_t dv, const pmf_qual_t *qual)
   2230 {
   2231 	struct atabus_softc *sc = device_private(dv);
   2232 	struct ata_channel *chp = sc->sc_chan;
   2233 
   2234 	/*
   2235 	 * XXX joerg: with wdc, the first channel unfreezes the controler.
   2236 	 * Move this the reset and queue idling into wdc.
   2237 	 */
   2238 	ata_channel_lock(chp);
   2239 	if (chp->ch_queue->queue_freeze == 0) {
   2240 		ata_channel_unlock(chp);
   2241 		goto out;
   2242 	}
   2243 
   2244 	/* unfreeze the queue and reset drives */
   2245 	ata_channel_thaw_locked(chp);
   2246 
   2247 	/* reset channel only if there are drives attached */
   2248 	if (chp->ch_ndrives > 0)
   2249 		ata_thread_run(chp, AT_WAIT, ATACH_TH_RESET, ATACH_NODRIVE);
   2250 
   2251 	ata_channel_unlock(chp);
   2252 
   2253 out:
   2254 	return true;
   2255 }
   2256 
   2257 static int
   2258 atabus_rescan(device_t self, const char *ifattr, const int *locators)
   2259 {
   2260 	struct atabus_softc *sc = device_private(self);
   2261 	struct ata_channel *chp = sc->sc_chan;
   2262 	struct atabus_initq *initq;
   2263 	int i;
   2264 
   2265 	/*
   2266 	 * we can rescan a port multiplier atabus, even if some devices are
   2267 	 * still attached
   2268 	 */
   2269 	if (chp->ch_satapmp_nports == 0) {
   2270 		if (chp->atapibus != NULL) {
   2271 			return EBUSY;
   2272 		}
   2273 
   2274 		KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL);
   2275 		for (i = 0; i < chp->ch_ndrives; i++) {
   2276 			if (chp->ch_drive[i].drv_softc != NULL) {
   2277 				return EBUSY;
   2278 			}
   2279 		}
   2280 	}
   2281 
   2282 	initq = kmem_zalloc(sizeof(*initq), KM_SLEEP);
   2283 	initq->atabus_sc = sc;
   2284 	mutex_enter(&atabus_qlock);
   2285 	TAILQ_INSERT_TAIL(&atabus_initq_head, initq, atabus_initq);
   2286 	mutex_exit(&atabus_qlock);
   2287 	config_pending_incr(sc->sc_dev);
   2288 
   2289 	ata_channel_lock(chp);
   2290 	chp->ch_flags |= ATACH_TH_RESCAN;
   2291 	cv_signal(&chp->ch_thr_idle);
   2292 	ata_channel_unlock(chp);
   2293 
   2294 	return 0;
   2295 }
   2296 
   2297 void
   2298 ata_delay(struct ata_channel *chp, int ms, const char *msg, int flags)
   2299 {
   2300 	KASSERT(mutex_owned(&chp->ch_lock));
   2301 
   2302 	if ((flags & (AT_WAIT | AT_POLL)) == AT_POLL) {
   2303 		/*
   2304 		 * can't use kpause(), we may be in interrupt context
   2305 		 * or taking a crash dump
   2306 		 */
   2307 		delay(ms * 1000);
   2308 	} else {
   2309 		int pause = mstohz(ms);
   2310 
   2311 		kpause(msg, false, pause > 0 ? pause : 1, &chp->ch_lock);
   2312 	}
   2313 }
   2314 
   2315 void
   2316 atacmd_toncq(struct ata_xfer *xfer, uint8_t *cmd, uint16_t *count,
   2317     uint16_t *features, uint8_t *device)
   2318 {
   2319 	if ((xfer->c_flags & C_NCQ) == 0) {
   2320 		/* FUA handling for non-NCQ drives */
   2321 		if (xfer->c_bio.flags & ATA_FUA
   2322 		    && *cmd == WDCC_WRITEDMA_EXT)
   2323 			*cmd = WDCC_WRITEDMA_FUA_EXT;
   2324 
   2325 		return;
   2326 	}
   2327 
   2328 	*cmd = (xfer->c_bio.flags & ATA_READ) ?
   2329 	    WDCC_READ_FPDMA_QUEUED : WDCC_WRITE_FPDMA_QUEUED;
   2330 
   2331 	/* for FPDMA the block count is in features */
   2332 	*features = *count;
   2333 
   2334 	/* NCQ tag */
   2335 	*count = (xfer->c_slot << 3);
   2336 
   2337 	if (xfer->c_bio.flags & ATA_PRIO_HIGH)
   2338 		*count |= WDSC_PRIO_HIGH;
   2339 
   2340 	/* other device flags */
   2341 	if (xfer->c_bio.flags & ATA_FUA)
   2342 		*device |= WDSD_FUA;
   2343 }
   2344 
   2345 void
   2346 ata_wait_cmd(struct ata_channel *chp, struct ata_xfer *xfer)
   2347 {
   2348 	struct ata_queue *chq = chp->ch_queue;
   2349 	struct ata_command *ata_c = &xfer->c_ata_c;
   2350 
   2351 	ata_channel_lock(chp);
   2352 
   2353 	while ((ata_c->flags & AT_DONE) == 0)
   2354 		cv_wait(&chq->c_cmd_finish, &chp->ch_lock);
   2355 
   2356 	ata_channel_unlock(chp);
   2357 
   2358 	KASSERT((ata_c->flags & AT_DONE) != 0);
   2359 }
   2360