Home | History | Annotate | Line # | Download | only in ata
ata.c revision 1.134
      1  1.134   mlelstv /*	$NetBSD: ata.c,v 1.134 2017/10/08 04:52:33 mlelstv Exp $	*/
      2   1.21   thorpej 
      3    1.2    bouyer /*
      4   1.16    bouyer  * Copyright (c) 1998, 2001 Manuel Bouyer.  All rights reserved.
      5    1.2    bouyer  *
      6    1.2    bouyer  * Redistribution and use in source and binary forms, with or without
      7    1.2    bouyer  * modification, are permitted provided that the following conditions
      8    1.2    bouyer  * are met:
      9    1.2    bouyer  * 1. Redistributions of source code must retain the above copyright
     10    1.2    bouyer  *    notice, this list of conditions and the following disclaimer.
     11    1.2    bouyer  * 2. Redistributions in binary form must reproduce the above copyright
     12    1.2    bouyer  *    notice, this list of conditions and the following disclaimer in the
     13    1.2    bouyer  *    documentation and/or other materials provided with the distribution.
     14    1.2    bouyer  *
     15    1.2    bouyer  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
     16    1.2    bouyer  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
     17    1.2    bouyer  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
     18   1.65     perry  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
     19    1.2    bouyer  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
     20    1.2    bouyer  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
     21    1.2    bouyer  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
     22    1.2    bouyer  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
     23    1.2    bouyer  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
     24    1.2    bouyer  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     25    1.2    bouyer  */
     26   1.14     lukem 
     27   1.14     lukem #include <sys/cdefs.h>
     28  1.134   mlelstv __KERNEL_RCSID(0, "$NetBSD: ata.c,v 1.134 2017/10/08 04:52:33 mlelstv Exp $");
     29    1.2    bouyer 
     30   1.88    dyoung #include "opt_ata.h"
     31    1.2    bouyer 
     32    1.2    bouyer #include <sys/param.h>
     33    1.2    bouyer #include <sys/systm.h>
     34    1.2    bouyer #include <sys/kernel.h>
     35    1.2    bouyer #include <sys/malloc.h>
     36    1.2    bouyer #include <sys/device.h>
     37   1.30    bouyer #include <sys/conf.h>
     38   1.30    bouyer #include <sys/fcntl.h>
     39   1.23   thorpej #include <sys/proc.h>
     40   1.23   thorpej #include <sys/kthread.h>
     41   1.23   thorpej #include <sys/errno.h>
     42   1.30    bouyer #include <sys/ataio.h>
     43   1.93     pooka #include <sys/kmem.h>
     44   1.91        ad #include <sys/intr.h>
     45   1.91        ad #include <sys/bus.h>
     46  1.107     pooka #include <sys/once.h>
     47  1.133  jdolecek #include <sys/bitops.h>
     48  1.133  jdolecek 
     49  1.133  jdolecek #define ATABUS_PRIVATE
     50   1.15    bouyer 
     51   1.76     itohy #include <dev/ata/ataconf.h>
     52    1.2    bouyer #include <dev/ata/atareg.h>
     53    1.2    bouyer #include <dev/ata/atavar.h>
     54   1.76     itohy #include <dev/ic/wdcvar.h>	/* for PIOBM */
     55    1.2    bouyer 
     56   1.23   thorpej #include "locators.h"
     57   1.23   thorpej 
     58   1.51   thorpej #include "atapibus.h"
     59   1.51   thorpej #include "ataraid.h"
     60  1.124    bouyer #include "sata_pmp.h"
     61   1.51   thorpej 
     62   1.65     perry #if NATARAID > 0
     63   1.65     perry #include <dev/ata/ata_raidvar.h>
     64   1.51   thorpej #endif
     65  1.124    bouyer #if NSATA_PMP > 0
     66  1.124    bouyer #include <dev/ata/satapmpvar.h>
     67  1.124    bouyer #endif
     68  1.124    bouyer #include <dev/ata/satapmpreg.h>
     69   1.51   thorpej 
     70    1.2    bouyer #define DEBUG_FUNCS  0x08
     71    1.2    bouyer #define DEBUG_PROBE  0x10
     72   1.23   thorpej #define DEBUG_DETACH 0x20
     73   1.44   thorpej #define	DEBUG_XFERS  0x40
     74   1.47   thorpej #ifdef ATADEBUG
     75  1.126       abs #ifndef ATADEBUG_MASK
     76  1.126       abs #define ATADEBUG_MASK 0
     77  1.126       abs #endif
     78  1.126       abs int atadebug_mask = ATADEBUG_MASK;
     79   1.47   thorpej #define ATADEBUG_PRINT(args, level) \
     80   1.81     itohy 	if (atadebug_mask & (level)) \
     81    1.2    bouyer 		printf args
     82    1.2    bouyer #else
     83   1.47   thorpej #define ATADEBUG_PRINT(args, level)
     84    1.2    bouyer #endif
     85    1.2    bouyer 
     86  1.114     rmind static ONCE_DECL(ata_init_ctrl);
     87   1.41   thorpej 
     88   1.51   thorpej /*
     89   1.51   thorpej  * A queue of atabus instances, used to ensure the same bus probe order
     90  1.114     rmind  * for a given hardware configuration at each boot.  Kthread probing
     91  1.114     rmind  * devices on a atabus.  Only one probing at once.
     92   1.51   thorpej  */
     93  1.114     rmind static TAILQ_HEAD(, atabus_initq)	atabus_initq_head;
     94  1.114     rmind static kmutex_t				atabus_qlock;
     95  1.114     rmind static kcondvar_t			atabus_qcv;
     96  1.114     rmind static lwp_t *				atabus_cfg_lwp;
     97  1.100    bouyer 
     98   1.23   thorpej /*****************************************************************************
     99   1.23   thorpej  * ATA bus layer.
    100   1.23   thorpej  *
    101   1.23   thorpej  * ATA controllers attach an atabus instance, which handles probing the bus
    102   1.23   thorpej  * for drives, etc.
    103   1.23   thorpej  *****************************************************************************/
    104   1.23   thorpej 
    105   1.30    bouyer dev_type_open(atabusopen);
    106   1.30    bouyer dev_type_close(atabusclose);
    107   1.30    bouyer dev_type_ioctl(atabusioctl);
    108   1.30    bouyer 
    109   1.30    bouyer const struct cdevsw atabus_cdevsw = {
    110  1.130  dholland 	.d_open = atabusopen,
    111  1.130  dholland 	.d_close = atabusclose,
    112  1.130  dholland 	.d_read = noread,
    113  1.130  dholland 	.d_write = nowrite,
    114  1.130  dholland 	.d_ioctl = atabusioctl,
    115  1.130  dholland 	.d_stop = nostop,
    116  1.130  dholland 	.d_tty = notty,
    117  1.130  dholland 	.d_poll = nopoll,
    118  1.130  dholland 	.d_mmap = nommap,
    119  1.130  dholland 	.d_kqfilter = nokqfilter,
    120  1.131  dholland 	.d_discard = nodiscard,
    121  1.130  dholland 	.d_flag = D_OTHER
    122   1.30    bouyer };
    123   1.30    bouyer 
    124   1.30    bouyer extern struct cfdriver atabus_cd;
    125   1.30    bouyer 
    126   1.95    dyoung static void atabus_childdetached(device_t, device_t);
    127  1.115  jakllsch static int atabus_rescan(device_t, const char *, const int *);
    128  1.112    dyoung static bool atabus_resume(device_t, const pmf_qual_t *);
    129  1.112    dyoung static bool atabus_suspend(device_t, const pmf_qual_t *);
    130  1.100    bouyer static void atabusconfig_thread(void *);
    131   1.30    bouyer 
    132  1.133  jdolecek static void ata_channel_idle(struct ata_channel *);
    133  1.133  jdolecek static void ata_channel_thaw_locked(struct ata_channel *);
    134  1.133  jdolecek static void ata_activate_xfer_locked(struct ata_channel *, struct ata_xfer *);
    135  1.133  jdolecek static void ata_channel_freeze_locked(struct ata_channel *);
    136  1.133  jdolecek static struct ata_xfer *ata_queue_get_active_xfer_locked(struct ata_channel *);
    137  1.133  jdolecek static void ata_thread_wake_locked(struct ata_channel *);
    138  1.133  jdolecek 
    139   1.23   thorpej /*
    140  1.114     rmind  * atabus_init:
    141  1.114     rmind  *
    142  1.114     rmind  *	Initialize ATA subsystem structures.
    143  1.114     rmind  */
    144  1.114     rmind static int
    145  1.114     rmind atabus_init(void)
    146  1.114     rmind {
    147  1.114     rmind 
    148  1.114     rmind 	TAILQ_INIT(&atabus_initq_head);
    149  1.114     rmind 	mutex_init(&atabus_qlock, MUTEX_DEFAULT, IPL_NONE);
    150  1.114     rmind 	cv_init(&atabus_qcv, "atainitq");
    151  1.114     rmind 	return 0;
    152  1.114     rmind }
    153  1.114     rmind 
    154  1.114     rmind /*
    155   1.23   thorpej  * atabusprint:
    156   1.23   thorpej  *
    157   1.23   thorpej  *	Autoconfiguration print routine used by ATA controllers when
    158   1.23   thorpej  *	attaching an atabus instance.
    159   1.23   thorpej  */
    160   1.23   thorpej int
    161   1.23   thorpej atabusprint(void *aux, const char *pnp)
    162   1.23   thorpej {
    163   1.48   thorpej 	struct ata_channel *chan = aux;
    164   1.65     perry 
    165   1.23   thorpej 	if (pnp)
    166   1.23   thorpej 		aprint_normal("atabus at %s", pnp);
    167   1.26   thorpej 	aprint_normal(" channel %d", chan->ch_channel);
    168   1.23   thorpej 
    169   1.23   thorpej 	return (UNCONF);
    170   1.23   thorpej }
    171   1.23   thorpej 
    172   1.23   thorpej /*
    173   1.23   thorpej  * ataprint:
    174   1.23   thorpej  *
    175   1.23   thorpej  *	Autoconfiguration print routine.
    176   1.23   thorpej  */
    177   1.23   thorpej int
    178   1.23   thorpej ataprint(void *aux, const char *pnp)
    179   1.23   thorpej {
    180   1.23   thorpej 	struct ata_device *adev = aux;
    181   1.23   thorpej 
    182   1.23   thorpej 	if (pnp)
    183   1.23   thorpej 		aprint_normal("wd at %s", pnp);
    184   1.23   thorpej 	aprint_normal(" drive %d", adev->adev_drv_data->drive);
    185   1.23   thorpej 
    186   1.23   thorpej 	return (UNCONF);
    187   1.23   thorpej }
    188   1.23   thorpej 
    189  1.133  jdolecek static void
    190  1.133  jdolecek ata_queue_reset(struct ata_queue *chq)
    191  1.133  jdolecek {
    192  1.133  jdolecek 	/* make sure that we can use polled commands */
    193  1.133  jdolecek 	TAILQ_INIT(&chq->queue_xfer);
    194  1.133  jdolecek 	TAILQ_INIT(&chq->active_xfers);
    195  1.133  jdolecek 	chq->queue_freeze = 0;
    196  1.133  jdolecek 	chq->queue_active = 0;
    197  1.133  jdolecek 	chq->active_xfers_used = 0;
    198  1.133  jdolecek 	chq->queue_xfers_avail = __BIT(chq->queue_openings) - 1;
    199  1.133  jdolecek }
    200  1.133  jdolecek 
    201  1.133  jdolecek struct ata_xfer *
    202  1.133  jdolecek ata_queue_hwslot_to_xfer(struct ata_channel *chp, int hwslot)
    203  1.133  jdolecek {
    204  1.133  jdolecek 	struct ata_queue *chq = chp->ch_queue;
    205  1.133  jdolecek 	struct ata_xfer *xfer = NULL;
    206  1.133  jdolecek 
    207  1.133  jdolecek 	ata_channel_lock(chp);
    208  1.133  jdolecek 
    209  1.133  jdolecek 	KASSERTMSG(hwslot < chq->queue_openings, "hwslot %d > openings %d",
    210  1.133  jdolecek 	    hwslot, chq->queue_openings);
    211  1.133  jdolecek 	KASSERTMSG((chq->active_xfers_used & __BIT(hwslot)) != 0,
    212  1.133  jdolecek 	    "hwslot %d not active", hwslot);
    213  1.133  jdolecek 
    214  1.133  jdolecek 	/* Usually the first entry will be the one */
    215  1.133  jdolecek 	TAILQ_FOREACH(xfer, &chq->active_xfers, c_activechain) {
    216  1.133  jdolecek 		if (xfer->c_slot == hwslot)
    217  1.133  jdolecek 			break;
    218  1.133  jdolecek 	}
    219  1.133  jdolecek 
    220  1.133  jdolecek 	ata_channel_unlock(chp);
    221  1.133  jdolecek 
    222  1.133  jdolecek 	KASSERTMSG((xfer != NULL),
    223  1.133  jdolecek 	    "%s: xfer with slot %d not found (active %x)", __func__,
    224  1.133  jdolecek 	    hwslot, chq->active_xfers_used);
    225  1.133  jdolecek 
    226  1.133  jdolecek 	return xfer;
    227  1.133  jdolecek }
    228  1.133  jdolecek 
    229  1.133  jdolecek static struct ata_xfer *
    230  1.133  jdolecek ata_queue_get_active_xfer_locked(struct ata_channel *chp)
    231  1.133  jdolecek {
    232  1.133  jdolecek 	struct ata_xfer *xfer;
    233  1.133  jdolecek 
    234  1.133  jdolecek 	KASSERT(mutex_owned(&chp->ch_lock));
    235  1.133  jdolecek 	xfer = TAILQ_FIRST(&chp->ch_queue->active_xfers);
    236  1.133  jdolecek 
    237  1.133  jdolecek 	if (xfer && ISSET(xfer->c_flags, C_NCQ)) {
    238  1.133  jdolecek 		/* Spurious call, never return NCQ xfer from this interface */
    239  1.133  jdolecek 		xfer = NULL;
    240  1.133  jdolecek 	}
    241  1.133  jdolecek 
    242  1.133  jdolecek 	return xfer;
    243  1.133  jdolecek }
    244  1.133  jdolecek 
    245  1.133  jdolecek /*
    246  1.133  jdolecek  * This interface is supposed only to be used when there is exactly
    247  1.133  jdolecek  * one outstanding command, when there is no information about the slot,
    248  1.133  jdolecek  * which triggered the command. ata_queue_hwslot_to_xfer() interface
    249  1.133  jdolecek  * is preferred in all NCQ cases.
    250  1.133  jdolecek  */
    251  1.133  jdolecek struct ata_xfer *
    252  1.133  jdolecek ata_queue_get_active_xfer(struct ata_channel *chp)
    253  1.133  jdolecek {
    254  1.133  jdolecek 	struct ata_xfer *xfer = NULL;
    255  1.133  jdolecek 
    256  1.133  jdolecek 	ata_channel_lock(chp);
    257  1.133  jdolecek 	xfer = ata_queue_get_active_xfer_locked(chp);
    258  1.133  jdolecek 	ata_channel_unlock(chp);
    259  1.133  jdolecek 
    260  1.133  jdolecek 	return xfer;
    261  1.133  jdolecek }
    262  1.133  jdolecek 
    263  1.133  jdolecek struct ata_xfer *
    264  1.133  jdolecek ata_queue_drive_active_xfer(struct ata_channel *chp, int drive)
    265  1.133  jdolecek {
    266  1.133  jdolecek 	struct ata_xfer *xfer = NULL;
    267  1.133  jdolecek 
    268  1.133  jdolecek 	ata_channel_lock(chp);
    269  1.133  jdolecek 
    270  1.133  jdolecek 	TAILQ_FOREACH(xfer, &chp->ch_queue->active_xfers, c_activechain) {
    271  1.133  jdolecek 		if (xfer->c_drive == drive)
    272  1.133  jdolecek 			break;
    273  1.133  jdolecek 	}
    274  1.133  jdolecek 	KASSERT(xfer != NULL);
    275  1.133  jdolecek 
    276  1.133  jdolecek 	ata_channel_unlock(chp);
    277  1.133  jdolecek 
    278  1.133  jdolecek 	return xfer;
    279  1.133  jdolecek }
    280  1.133  jdolecek 
    281  1.133  jdolecek static void
    282  1.133  jdolecek ata_xfer_init(struct ata_xfer *xfer, uint8_t slot)
    283  1.133  jdolecek {
    284  1.133  jdolecek 	memset(xfer, 0, sizeof(*xfer));
    285  1.133  jdolecek 
    286  1.133  jdolecek 	xfer->c_slot = slot;
    287  1.133  jdolecek 
    288  1.133  jdolecek 	cv_init(&xfer->c_active, "ataact");
    289  1.133  jdolecek 	cv_init(&xfer->c_finish, "atafin");
    290  1.133  jdolecek 	callout_init(&xfer->c_timo_callout, 0); 	/* XXX MPSAFE */
    291  1.133  jdolecek 	callout_init(&xfer->c_retry_callout, 0); 	/* XXX MPSAFE */
    292  1.133  jdolecek }
    293  1.133  jdolecek 
    294  1.133  jdolecek static void
    295  1.133  jdolecek ata_xfer_destroy(struct ata_xfer *xfer)
    296  1.133  jdolecek {
    297  1.133  jdolecek 	callout_halt(&xfer->c_timo_callout, NULL);	/* XXX MPSAFE */
    298  1.133  jdolecek 	callout_destroy(&xfer->c_timo_callout);
    299  1.133  jdolecek 	callout_halt(&xfer->c_retry_callout, NULL);	/* XXX MPSAFE */
    300  1.133  jdolecek 	callout_destroy(&xfer->c_retry_callout);
    301  1.133  jdolecek 	cv_destroy(&xfer->c_active);
    302  1.133  jdolecek 	cv_destroy(&xfer->c_finish);
    303  1.133  jdolecek }
    304  1.133  jdolecek 
    305  1.133  jdolecek struct ata_queue *
    306  1.133  jdolecek ata_queue_alloc(uint8_t openings)
    307  1.133  jdolecek {
    308  1.133  jdolecek 	if (openings == 0)
    309  1.133  jdolecek 		openings = 1;
    310  1.133  jdolecek 
    311  1.133  jdolecek 	if (openings > ATA_MAX_OPENINGS)
    312  1.133  jdolecek 		openings = ATA_MAX_OPENINGS;
    313  1.133  jdolecek 
    314  1.133  jdolecek 	struct ata_queue *chq = malloc(offsetof(struct ata_queue, queue_xfers[openings]),
    315  1.133  jdolecek 	    M_DEVBUF, M_WAITOK | M_ZERO);
    316  1.133  jdolecek 
    317  1.133  jdolecek 	chq->queue_openings = openings;
    318  1.133  jdolecek 	ata_queue_reset(chq);
    319  1.133  jdolecek 
    320  1.133  jdolecek 	cv_init(&chq->queue_busy, "ataqbusy");
    321  1.133  jdolecek 	cv_init(&chq->queue_drain, "atdrn");
    322  1.133  jdolecek 	cv_init(&chq->queue_idle, "qidl");
    323  1.133  jdolecek 
    324  1.133  jdolecek 	for (uint8_t i = 0; i < openings; i++)
    325  1.133  jdolecek 		ata_xfer_init(&chq->queue_xfers[i], i);
    326  1.133  jdolecek 
    327  1.133  jdolecek 	return chq;
    328  1.133  jdolecek }
    329  1.133  jdolecek 
    330  1.133  jdolecek void
    331  1.133  jdolecek ata_queue_free(struct ata_queue *chq)
    332  1.133  jdolecek {
    333  1.133  jdolecek 	for (uint8_t i = 0; i < chq->queue_openings; i++)
    334  1.133  jdolecek 		ata_xfer_destroy(&chq->queue_xfers[i]);
    335  1.133  jdolecek 
    336  1.133  jdolecek 	cv_destroy(&chq->queue_busy);
    337  1.133  jdolecek 	cv_destroy(&chq->queue_drain);
    338  1.133  jdolecek 	cv_destroy(&chq->queue_idle);
    339  1.133  jdolecek 
    340  1.133  jdolecek 	free(chq, M_DEVBUF);
    341  1.133  jdolecek }
    342  1.133  jdolecek 
    343  1.133  jdolecek void
    344  1.133  jdolecek ata_channel_init(struct ata_channel *chp)
    345  1.133  jdolecek {
    346  1.133  jdolecek 	mutex_init(&chp->ch_lock, MUTEX_DEFAULT, IPL_BIO);
    347  1.133  jdolecek 	cv_init(&chp->ch_thr_idle, "atath");
    348  1.133  jdolecek }
    349  1.133  jdolecek 
    350   1.52   thorpej /*
    351   1.52   thorpej  * ata_channel_attach:
    352   1.52   thorpej  *
    353   1.52   thorpej  *	Common parts of attaching an atabus to an ATA controller channel.
    354   1.52   thorpej  */
    355   1.52   thorpej void
    356   1.52   thorpej ata_channel_attach(struct ata_channel *chp)
    357   1.52   thorpej {
    358   1.52   thorpej 	if (chp->ch_flags & ATACH_DISABLED)
    359   1.52   thorpej 		return;
    360   1.65     perry 
    361  1.133  jdolecek 	KASSERT(chp->ch_queue != NULL);
    362   1.52   thorpej 
    363  1.133  jdolecek 	ata_channel_init(chp);
    364   1.52   thorpej 
    365   1.98      cube 	chp->atabus = config_found_ia(chp->ch_atac->atac_dev, "ata", chp,
    366   1.71  drochner 		atabusprint);
    367   1.52   thorpej }
    368   1.52   thorpej 
    369  1.133  jdolecek void
    370  1.133  jdolecek ata_channel_destroy(struct ata_channel *chp)
    371  1.133  jdolecek {
    372  1.133  jdolecek 	mutex_destroy(&chp->ch_lock);
    373  1.133  jdolecek 	cv_destroy(&chp->ch_thr_idle);
    374  1.133  jdolecek }
    375  1.133  jdolecek 
    376  1.133  jdolecek /*
    377  1.133  jdolecek  * ata_channel_detach:
    378  1.133  jdolecek  *
    379  1.133  jdolecek  *	Common parts of detaching an atabus to an ATA controller channel.
    380  1.133  jdolecek  */
    381  1.133  jdolecek void
    382  1.133  jdolecek ata_channel_detach(struct ata_channel *chp)
    383  1.133  jdolecek {
    384  1.133  jdolecek 	if (chp->ch_flags & ATACH_DISABLED)
    385  1.133  jdolecek 		return;
    386  1.133  jdolecek 
    387  1.133  jdolecek 	ata_channel_destroy(chp);
    388  1.133  jdolecek }
    389  1.133  jdolecek 
    390   1.51   thorpej static void
    391   1.51   thorpej atabusconfig(struct atabus_softc *atabus_sc)
    392   1.51   thorpej {
    393   1.51   thorpej 	struct ata_channel *chp = atabus_sc->sc_chan;
    394   1.51   thorpej 	struct atac_softc *atac = chp->ch_atac;
    395   1.51   thorpej 	struct atabus_initq *atabus_initq = NULL;
    396  1.133  jdolecek 	int i, error;
    397  1.102    bouyer 
    398  1.102    bouyer 	/* we are in the atabus's thread context */
    399  1.133  jdolecek 	ata_channel_lock(chp);
    400  1.102    bouyer 	chp->ch_flags |= ATACH_TH_RUN;
    401  1.133  jdolecek 	ata_channel_unlock(chp);
    402   1.51   thorpej 
    403  1.124    bouyer 	/*
    404  1.124    bouyer 	 * Probe for the drives attached to controller, unless a PMP
    405  1.124    bouyer 	 * is already known
    406  1.124    bouyer 	 */
    407   1.82    bouyer 	/* XXX for SATA devices we will power up all drives at once */
    408  1.124    bouyer 	if (chp->ch_satapmp_nports == 0)
    409  1.124    bouyer 		(*atac->atac_probe)(chp);
    410   1.51   thorpej 
    411  1.125    bouyer 	if (chp->ch_ndrives >= 2) {
    412  1.124    bouyer 		ATADEBUG_PRINT(("atabusattach: ch_drive_type 0x%x 0x%x\n",
    413  1.124    bouyer 		    chp->ch_drive[0].drive_type, chp->ch_drive[1].drive_type),
    414  1.124    bouyer 		    DEBUG_PROBE);
    415  1.124    bouyer 	}
    416   1.51   thorpej 
    417  1.102    bouyer 	/* next operations will occurs in a separate thread */
    418  1.133  jdolecek 	ata_channel_lock(chp);
    419  1.102    bouyer 	chp->ch_flags &= ~ATACH_TH_RUN;
    420  1.133  jdolecek 	ata_channel_unlock(chp);
    421  1.102    bouyer 
    422  1.100    bouyer 	/* Make sure the devices probe in atabus order to avoid jitter. */
    423  1.114     rmind 	mutex_enter(&atabus_qlock);
    424  1.114     rmind 	for (;;) {
    425  1.100    bouyer 		atabus_initq = TAILQ_FIRST(&atabus_initq_head);
    426  1.100    bouyer 		if (atabus_initq->atabus_sc == atabus_sc)
    427  1.100    bouyer 			break;
    428  1.114     rmind 		cv_wait(&atabus_qcv, &atabus_qlock);
    429  1.100    bouyer 	}
    430  1.114     rmind 	mutex_exit(&atabus_qlock);
    431  1.100    bouyer 
    432  1.133  jdolecek 	ata_channel_lock(chp);
    433  1.133  jdolecek 
    434   1.51   thorpej 	/* If no drives, abort here */
    435  1.124    bouyer 	if (chp->ch_drive == NULL)
    436  1.124    bouyer 		goto out;
    437  1.124    bouyer 	KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL);
    438  1.124    bouyer 	for (i = 0; i < chp->ch_ndrives; i++)
    439  1.124    bouyer 		if (chp->ch_drive[i].drive_type != ATA_DRIVET_NONE)
    440   1.51   thorpej 			break;
    441  1.124    bouyer 	if (i == chp->ch_ndrives)
    442   1.51   thorpej 		goto out;
    443   1.51   thorpej 
    444   1.51   thorpej 	/* Shortcut in case we've been shutdown */
    445   1.51   thorpej 	if (chp->ch_flags & ATACH_SHUTDOWN)
    446   1.51   thorpej 		goto out;
    447   1.51   thorpej 
    448  1.133  jdolecek 	ata_channel_unlock(chp);
    449  1.133  jdolecek 
    450  1.100    bouyer 	if ((error = kthread_create(PRI_NONE, 0, NULL, atabusconfig_thread,
    451  1.114     rmind 	    atabus_sc, &atabus_cfg_lwp,
    452  1.100    bouyer 	    "%scnf", device_xname(atac->atac_dev))) != 0)
    453  1.100    bouyer 		aprint_error_dev(atac->atac_dev,
    454  1.100    bouyer 		    "unable to create config thread: error %d\n", error);
    455  1.100    bouyer 	return;
    456  1.100    bouyer 
    457  1.100    bouyer  out:
    458  1.133  jdolecek 	ata_channel_unlock(chp);
    459  1.133  jdolecek 
    460  1.114     rmind 	mutex_enter(&atabus_qlock);
    461  1.100    bouyer 	TAILQ_REMOVE(&atabus_initq_head, atabus_initq, atabus_initq);
    462  1.114     rmind 	cv_broadcast(&atabus_qcv);
    463  1.114     rmind 	mutex_exit(&atabus_qlock);
    464   1.51   thorpej 
    465  1.100    bouyer 	free(atabus_initq, M_DEVBUF);
    466  1.100    bouyer 
    467  1.100    bouyer 	ata_delref(chp);
    468  1.100    bouyer 
    469  1.129  christos 	config_pending_decr(atac->atac_dev);
    470  1.100    bouyer }
    471  1.100    bouyer 
    472  1.100    bouyer /*
    473  1.100    bouyer  * atabus_configthread: finish attach of atabus's childrens, in a separate
    474  1.100    bouyer  * kernel thread.
    475  1.100    bouyer  */
    476  1.100    bouyer static void
    477  1.100    bouyer atabusconfig_thread(void *arg)
    478  1.100    bouyer {
    479  1.100    bouyer 	struct atabus_softc *atabus_sc = arg;
    480  1.100    bouyer 	struct ata_channel *chp = atabus_sc->sc_chan;
    481  1.100    bouyer 	struct atac_softc *atac = chp->ch_atac;
    482  1.114     rmind 	struct atabus_initq *atabus_initq = NULL;
    483  1.100    bouyer 	int i, s;
    484  1.100    bouyer 
    485  1.114     rmind 	/* XXX seems wrong */
    486  1.114     rmind 	mutex_enter(&atabus_qlock);
    487  1.100    bouyer 	atabus_initq = TAILQ_FIRST(&atabus_initq_head);
    488  1.100    bouyer 	KASSERT(atabus_initq->atabus_sc == atabus_sc);
    489  1.114     rmind 	mutex_exit(&atabus_qlock);
    490  1.114     rmind 
    491   1.51   thorpej 	/*
    492  1.124    bouyer 	 * First look for a port multiplier
    493  1.124    bouyer 	 */
    494  1.124    bouyer 	if (chp->ch_ndrives == PMP_MAX_DRIVES &&
    495  1.124    bouyer 	    chp->ch_drive[PMP_PORT_CTL].drive_type == ATA_DRIVET_PM) {
    496  1.124    bouyer #if NSATA_PMP > 0
    497  1.124    bouyer 		satapmp_attach(chp);
    498  1.124    bouyer #else
    499  1.124    bouyer 		aprint_error_dev(atabus_sc->sc_dev,
    500  1.124    bouyer 		    "SATA port multiplier not supported\n");
    501  1.124    bouyer 		/* no problems going on, all drives are ATA_DRIVET_NONE */
    502  1.124    bouyer #endif
    503  1.124    bouyer 	}
    504  1.124    bouyer 
    505  1.124    bouyer 	/*
    506   1.51   thorpej 	 * Attach an ATAPI bus, if needed.
    507   1.51   thorpej 	 */
    508  1.124    bouyer 	KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL);
    509  1.124    bouyer 	for (i = 0; i < chp->ch_ndrives && chp->atapibus == NULL; i++) {
    510  1.124    bouyer 		if (chp->ch_drive[i].drive_type == ATA_DRIVET_ATAPI) {
    511   1.51   thorpej #if NATAPIBUS > 0
    512   1.51   thorpej 			(*atac->atac_atapibus_attach)(atabus_sc);
    513   1.51   thorpej #else
    514   1.51   thorpej 			/*
    515   1.51   thorpej 			 * Fake the autoconfig "not configured" message
    516   1.51   thorpej 			 */
    517   1.51   thorpej 			aprint_normal("atapibus at %s not configured\n",
    518   1.98      cube 			    device_xname(atac->atac_dev));
    519   1.51   thorpej 			chp->atapibus = NULL;
    520   1.58   thorpej 			s = splbio();
    521  1.124    bouyer 			for (i = 0; i < chp->ch_ndrives; i++) {
    522  1.124    bouyer 				if (chp->ch_drive[i].drive_type == ATA_DRIVET_ATAPI)
    523  1.124    bouyer 					chp->ch_drive[i].drive_type = ATA_DRIVET_NONE;
    524  1.124    bouyer 			}
    525   1.58   thorpej 			splx(s);
    526   1.51   thorpej #endif
    527   1.51   thorpej 			break;
    528   1.51   thorpej 		}
    529   1.51   thorpej 	}
    530   1.51   thorpej 
    531  1.124    bouyer 	for (i = 0; i < chp->ch_ndrives; i++) {
    532   1.51   thorpej 		struct ata_device adev;
    533  1.124    bouyer 		if (chp->ch_drive[i].drive_type != ATA_DRIVET_ATA &&
    534  1.124    bouyer 		    chp->ch_drive[i].drive_type != ATA_DRIVET_OLD) {
    535   1.51   thorpej 			continue;
    536   1.51   thorpej 		}
    537  1.124    bouyer 		if (chp->ch_drive[i].drv_softc != NULL)
    538  1.124    bouyer 			continue;
    539   1.51   thorpej 		memset(&adev, 0, sizeof(struct ata_device));
    540   1.51   thorpej 		adev.adev_bustype = atac->atac_bustype_ata;
    541   1.51   thorpej 		adev.adev_channel = chp->ch_channel;
    542   1.51   thorpej 		adev.adev_drv_data = &chp->ch_drive[i];
    543  1.123  jakllsch 		chp->ch_drive[i].drv_softc = config_found_ia(atabus_sc->sc_dev,
    544   1.71  drochner 		    "ata_hl", &adev, ataprint);
    545  1.124    bouyer 		if (chp->ch_drive[i].drv_softc != NULL) {
    546   1.51   thorpej 			ata_probe_caps(&chp->ch_drive[i]);
    547  1.124    bouyer 		} else {
    548   1.58   thorpej 			s = splbio();
    549  1.124    bouyer 			chp->ch_drive[i].drive_type = ATA_DRIVET_NONE;
    550   1.58   thorpej 			splx(s);
    551   1.58   thorpej 		}
    552   1.51   thorpej 	}
    553   1.51   thorpej 
    554   1.51   thorpej 	/* now that we know the drives, the controller can set its modes */
    555   1.51   thorpej 	if (atac->atac_set_modes) {
    556   1.51   thorpej 		(*atac->atac_set_modes)(chp);
    557   1.51   thorpej 		ata_print_modes(chp);
    558   1.51   thorpej 	}
    559   1.51   thorpej #if NATARAID > 0
    560  1.123  jakllsch 	if (atac->atac_cap & ATAC_CAP_RAID) {
    561  1.124    bouyer 		for (i = 0; i < chp->ch_ndrives; i++) {
    562  1.124    bouyer 			if (chp->ch_drive[i].drive_type == ATA_DRIVET_ATA) {
    563  1.123  jakllsch 				ata_raid_check_component(
    564  1.123  jakllsch 				    chp->ch_drive[i].drv_softc);
    565  1.123  jakllsch 			}
    566  1.123  jakllsch 		}
    567  1.123  jakllsch 	}
    568   1.51   thorpej #endif /* NATARAID > 0 */
    569   1.51   thorpej 
    570   1.51   thorpej 	/*
    571   1.51   thorpej 	 * reset drive_flags for unattached devices, reset state for attached
    572   1.51   thorpej 	 * ones
    573   1.51   thorpej 	 */
    574   1.58   thorpej 	s = splbio();
    575  1.124    bouyer 	for (i = 0; i < chp->ch_ndrives; i++) {
    576  1.124    bouyer 		if (chp->ch_drive[i].drive_type == ATA_DRIVET_PM)
    577  1.124    bouyer 			continue;
    578  1.124    bouyer 		if (chp->ch_drive[i].drv_softc == NULL) {
    579   1.51   thorpej 			chp->ch_drive[i].drive_flags = 0;
    580  1.124    bouyer 			chp->ch_drive[i].drive_type = ATA_DRIVET_NONE;
    581  1.124    bouyer 		} else
    582   1.51   thorpej 			chp->ch_drive[i].state = 0;
    583   1.51   thorpej 	}
    584   1.58   thorpej 	splx(s);
    585   1.51   thorpej 
    586  1.114     rmind 	mutex_enter(&atabus_qlock);
    587   1.81     itohy 	TAILQ_REMOVE(&atabus_initq_head, atabus_initq, atabus_initq);
    588  1.114     rmind 	cv_broadcast(&atabus_qcv);
    589  1.114     rmind 	mutex_exit(&atabus_qlock);
    590   1.51   thorpej 
    591   1.81     itohy 	free(atabus_initq, M_DEVBUF);
    592   1.51   thorpej 
    593   1.51   thorpej 	ata_delref(chp);
    594   1.51   thorpej 
    595  1.129  christos 	config_pending_decr(atac->atac_dev);
    596  1.100    bouyer 	kthread_exit(0);
    597   1.51   thorpej }
    598   1.51   thorpej 
    599   1.23   thorpej /*
    600   1.23   thorpej  * atabus_thread:
    601   1.23   thorpej  *
    602   1.23   thorpej  *	Worker thread for the ATA bus.
    603   1.23   thorpej  */
    604   1.23   thorpej static void
    605   1.23   thorpej atabus_thread(void *arg)
    606   1.23   thorpej {
    607   1.23   thorpej 	struct atabus_softc *sc = arg;
    608   1.48   thorpej 	struct ata_channel *chp = sc->sc_chan;
    609  1.133  jdolecek 	struct ata_queue *chq = chp->ch_queue;
    610   1.24   thorpej 	struct ata_xfer *xfer;
    611  1.133  jdolecek 	int i, rv, s;
    612   1.23   thorpej 
    613  1.133  jdolecek 	ata_channel_lock(chp);
    614  1.102    bouyer 	chp->ch_flags |= ATACH_TH_RUN;
    615  1.102    bouyer 
    616   1.48   thorpej 	/*
    617  1.124    bouyer 	 * Probe the drives.  Reset type to indicate to controllers
    618   1.48   thorpej 	 * that can re-probe that all drives must be probed..
    619   1.48   thorpej 	 *
    620  1.124    bouyer 	 * Note: ch_ndrives may be changed during the probe.
    621   1.48   thorpej 	 */
    622  1.124    bouyer 	KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL);
    623  1.124    bouyer 	for (i = 0; i < chp->ch_ndrives; i++) {
    624   1.48   thorpej 		chp->ch_drive[i].drive_flags = 0;
    625  1.124    bouyer 		chp->ch_drive[i].drive_type = ATA_DRIVET_NONE;
    626  1.124    bouyer 	}
    627  1.133  jdolecek 	ata_channel_unlock(chp);
    628   1.23   thorpej 
    629   1.68    bouyer 	atabusconfig(sc);
    630   1.23   thorpej 
    631  1.133  jdolecek 	ata_channel_lock(chp);
    632   1.23   thorpej 	for (;;) {
    633   1.48   thorpej 		if ((chp->ch_flags & (ATACH_TH_RESET | ATACH_SHUTDOWN)) == 0 &&
    634  1.133  jdolecek 		    (chq->queue_active == 0 || chq->queue_freeze == 0)) {
    635  1.102    bouyer 			chp->ch_flags &= ~ATACH_TH_RUN;
    636  1.133  jdolecek 			cv_wait(&chp->ch_thr_idle, &chp->ch_lock);
    637  1.102    bouyer 			chp->ch_flags |= ATACH_TH_RUN;
    638   1.23   thorpej 		}
    639   1.59   thorpej 		if (chp->ch_flags & ATACH_SHUTDOWN) {
    640   1.23   thorpej 			break;
    641   1.59   thorpej 		}
    642  1.115  jakllsch 		if (chp->ch_flags & ATACH_TH_RESCAN) {
    643  1.133  jdolecek 			chp->ch_flags &= ~ATACH_TH_RESCAN;
    644  1.133  jdolecek 			ata_channel_unlock(chp);
    645  1.115  jakllsch 			atabusconfig(sc);
    646  1.133  jdolecek 			ata_channel_lock(chp);
    647  1.115  jakllsch 		}
    648   1.48   thorpej 		if (chp->ch_flags & ATACH_TH_RESET) {
    649   1.31    bouyer 			/*
    650   1.55   thorpej 			 * ata_reset_channel() will freeze 2 times, so
    651   1.31    bouyer 			 * unfreeze one time. Not a problem as we're at splbio
    652   1.31    bouyer 			 */
    653  1.133  jdolecek 			ata_channel_thaw_locked(chp);
    654  1.133  jdolecek 			ata_channel_unlock(chp);
    655  1.133  jdolecek 			s = splbio();
    656   1.55   thorpej 			ata_reset_channel(chp, AT_WAIT | chp->ch_reset_flags);
    657  1.133  jdolecek 			splx(s);
    658  1.133  jdolecek 			ata_channel_lock(chp);
    659  1.133  jdolecek 		} else if (chq->queue_active > 0 && chq->queue_freeze == 1) {
    660   1.23   thorpej 			/*
    661  1.133  jdolecek 			 * Caller has bumped queue_freeze, decrease it. This
    662  1.133  jdolecek 			 * flow shalt never be executed for NCQ commands.
    663   1.23   thorpej 			 */
    664  1.133  jdolecek 			KASSERT((chp->ch_flags & ATACH_NCQ) == 0);
    665  1.133  jdolecek 			KASSERT(chq->queue_active == 1);
    666  1.133  jdolecek 
    667  1.133  jdolecek 			ata_channel_thaw_locked(chp);
    668  1.133  jdolecek 			xfer = ata_queue_get_active_xfer_locked(chp);
    669  1.133  jdolecek 
    670   1.23   thorpej 			KASSERT(xfer != NULL);
    671  1.133  jdolecek 			KASSERT((xfer->c_flags & C_POLL) == 0);
    672  1.133  jdolecek 
    673  1.133  jdolecek 			switch ((rv = ata_xfer_start(xfer))) {
    674  1.133  jdolecek 			case ATASTART_STARTED:
    675  1.133  jdolecek 			case ATASTART_POLL:
    676  1.133  jdolecek 			case ATASTART_ABORT:
    677  1.133  jdolecek 				break;
    678  1.133  jdolecek 			case ATASTART_TH:
    679  1.133  jdolecek 			default:
    680  1.133  jdolecek 				panic("%s: ata_xfer_start() unexpected rv %d",
    681  1.133  jdolecek 				    __func__, rv);
    682  1.133  jdolecek 				/* NOTREACHED */
    683  1.133  jdolecek 			}
    684  1.133  jdolecek 		} else if (chq->queue_freeze > 1)
    685  1.133  jdolecek 			panic("%s: queue_freeze", __func__);
    686   1.23   thorpej 	}
    687   1.27   thorpej 	chp->ch_thread = NULL;
    688  1.133  jdolecek 	cv_signal(&chp->ch_thr_idle);
    689  1.133  jdolecek 	ata_channel_unlock(chp);
    690   1.23   thorpej 	kthread_exit(0);
    691   1.23   thorpej }
    692   1.23   thorpej 
    693  1.133  jdolecek static void
    694  1.133  jdolecek ata_thread_wake_locked(struct ata_channel *chp)
    695  1.133  jdolecek {
    696  1.133  jdolecek 	KASSERT(mutex_owned(&chp->ch_lock));
    697  1.133  jdolecek 	ata_channel_freeze_locked(chp);
    698  1.133  jdolecek 	cv_signal(&chp->ch_thr_idle);
    699  1.133  jdolecek }
    700  1.133  jdolecek 
    701   1.23   thorpej /*
    702   1.23   thorpej  * atabus_match:
    703   1.23   thorpej  *
    704   1.23   thorpej  *	Autoconfiguration match routine.
    705   1.23   thorpej  */
    706   1.23   thorpej static int
    707  1.106    cegger atabus_match(device_t parent, cfdata_t cf, void *aux)
    708   1.23   thorpej {
    709   1.48   thorpej 	struct ata_channel *chp = aux;
    710   1.23   thorpej 
    711   1.23   thorpej 	if (chp == NULL)
    712   1.23   thorpej 		return (0);
    713   1.65     perry 
    714   1.26   thorpej 	if (cf->cf_loc[ATACF_CHANNEL] != chp->ch_channel &&
    715   1.23   thorpej 	    cf->cf_loc[ATACF_CHANNEL] != ATACF_CHANNEL_DEFAULT)
    716   1.81     itohy 		return (0);
    717   1.65     perry 
    718   1.23   thorpej 	return (1);
    719   1.23   thorpej }
    720   1.23   thorpej 
    721   1.23   thorpej /*
    722   1.23   thorpej  * atabus_attach:
    723   1.23   thorpej  *
    724   1.23   thorpej  *	Autoconfiguration attach routine.
    725   1.23   thorpej  */
    726   1.23   thorpej static void
    727   1.95    dyoung atabus_attach(device_t parent, device_t self, void *aux)
    728   1.23   thorpej {
    729   1.95    dyoung 	struct atabus_softc *sc = device_private(self);
    730   1.48   thorpej 	struct ata_channel *chp = aux;
    731   1.23   thorpej 	struct atabus_initq *initq;
    732   1.90        ad 	int error;
    733   1.23   thorpej 
    734   1.23   thorpej 	sc->sc_chan = chp;
    735   1.23   thorpej 
    736   1.23   thorpej 	aprint_normal("\n");
    737   1.23   thorpej 	aprint_naive("\n");
    738   1.23   thorpej 
    739   1.98      cube 	sc->sc_dev = self;
    740   1.98      cube 
    741   1.81     itohy 	if (ata_addref(chp))
    742   1.81     itohy 		return;
    743   1.35   mycroft 
    744  1.114     rmind 	RUN_ONCE(&ata_init_ctrl, atabus_init);
    745  1.107     pooka 
    746   1.23   thorpej 	initq = malloc(sizeof(*initq), M_DEVBUF, M_WAITOK);
    747   1.23   thorpej 	initq->atabus_sc = sc;
    748  1.133  jdolecek 	mutex_enter(&atabus_qlock);
    749   1.23   thorpej 	TAILQ_INSERT_TAIL(&atabus_initq_head, initq, atabus_initq);
    750  1.133  jdolecek 	mutex_exit(&atabus_qlock);
    751  1.129  christos 	config_pending_incr(sc->sc_dev);
    752   1.90        ad 
    753   1.90        ad 	if ((error = kthread_create(PRI_NONE, 0, NULL, atabus_thread, sc,
    754   1.95    dyoung 	    &chp->ch_thread, "%s", device_xname(self))) != 0)
    755   1.95    dyoung 		aprint_error_dev(self,
    756   1.95    dyoung 		    "unable to create kernel thread: error %d\n", error);
    757   1.64  jmcneill 
    758   1.92  jmcneill 	if (!pmf_device_register(self, atabus_suspend, atabus_resume))
    759   1.92  jmcneill 		aprint_error_dev(self, "couldn't establish power handler\n");
    760   1.23   thorpej }
    761   1.23   thorpej 
    762   1.23   thorpej /*
    763   1.23   thorpej  * atabus_detach:
    764   1.23   thorpej  *
    765   1.23   thorpej  *	Autoconfiguration detach routine.
    766   1.23   thorpej  */
    767   1.23   thorpej static int
    768   1.95    dyoung atabus_detach(device_t self, int flags)
    769   1.23   thorpej {
    770   1.95    dyoung 	struct atabus_softc *sc = device_private(self);
    771   1.48   thorpej 	struct ata_channel *chp = sc->sc_chan;
    772   1.95    dyoung 	device_t dev = NULL;
    773  1.133  jdolecek 	int i, error = 0;
    774   1.23   thorpej 
    775   1.23   thorpej 	/* Shutdown the channel. */
    776  1.133  jdolecek 	ata_channel_lock(chp);
    777   1.48   thorpej 	chp->ch_flags |= ATACH_SHUTDOWN;
    778  1.133  jdolecek 	while (chp->ch_thread != NULL) {
    779  1.133  jdolecek 		cv_signal(&chp->ch_thr_idle);
    780  1.133  jdolecek 		cv_wait(&chp->ch_thr_idle, &chp->ch_lock);
    781  1.133  jdolecek 	}
    782  1.133  jdolecek 	ata_channel_unlock(chp);
    783   1.95    dyoung 
    784   1.23   thorpej 	/*
    785   1.23   thorpej 	 * Detach atapibus and its children.
    786   1.23   thorpej 	 */
    787   1.23   thorpej 	if ((dev = chp->atapibus) != NULL) {
    788   1.47   thorpej 		ATADEBUG_PRINT(("atabus_detach: %s: detaching %s\n",
    789   1.95    dyoung 		    device_xname(self), device_xname(dev)), DEBUG_DETACH);
    790   1.95    dyoung 
    791   1.23   thorpej 		error = config_detach(dev, flags);
    792   1.23   thorpej 		if (error)
    793   1.23   thorpej 			goto out;
    794  1.104    dyoung 		KASSERT(chp->atapibus == NULL);
    795   1.23   thorpej 	}
    796   1.23   thorpej 
    797  1.124    bouyer 	KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL);
    798  1.124    bouyer 
    799   1.23   thorpej 	/*
    800   1.23   thorpej 	 * Detach our other children.
    801   1.23   thorpej 	 */
    802  1.124    bouyer 	for (i = 0; i < chp->ch_ndrives; i++) {
    803  1.124    bouyer 		if (chp->ch_drive[i].drive_type == ATA_DRIVET_ATAPI)
    804   1.23   thorpej 			continue;
    805  1.124    bouyer 		if (chp->ch_drive[i].drive_type == ATA_DRIVET_PM)
    806  1.124    bouyer 			chp->ch_drive[i].drive_type = ATA_DRIVET_NONE;
    807  1.123  jakllsch 		if ((dev = chp->ch_drive[i].drv_softc) != NULL) {
    808  1.104    dyoung 			ATADEBUG_PRINT(("%s.%d: %s: detaching %s\n", __func__,
    809  1.104    dyoung 			    __LINE__, device_xname(self), device_xname(dev)),
    810   1.23   thorpej 			    DEBUG_DETACH);
    811   1.23   thorpej 			error = config_detach(dev, flags);
    812   1.23   thorpej 			if (error)
    813   1.23   thorpej 				goto out;
    814  1.123  jakllsch 			KASSERT(chp->ch_drive[i].drv_softc == NULL);
    815  1.124    bouyer 			KASSERT(chp->ch_drive[i].drive_type == 0);
    816   1.23   thorpej 		}
    817   1.23   thorpej 	}
    818  1.124    bouyer 	atabus_free_drives(chp);
    819   1.23   thorpej 
    820   1.23   thorpej  out:
    821   1.47   thorpej #ifdef ATADEBUG
    822   1.23   thorpej 	if (dev != NULL && error != 0)
    823  1.104    dyoung 		ATADEBUG_PRINT(("%s: %s: error %d detaching %s\n", __func__,
    824   1.95    dyoung 		    device_xname(self), error, device_xname(dev)),
    825   1.23   thorpej 		    DEBUG_DETACH);
    826   1.47   thorpej #endif /* ATADEBUG */
    827   1.23   thorpej 
    828   1.23   thorpej 	return (error);
    829   1.23   thorpej }
    830   1.23   thorpej 
    831   1.95    dyoung void
    832   1.95    dyoung atabus_childdetached(device_t self, device_t child)
    833   1.95    dyoung {
    834  1.104    dyoung 	bool found = false;
    835   1.95    dyoung 	struct atabus_softc *sc = device_private(self);
    836   1.95    dyoung 	struct ata_channel *chp = sc->sc_chan;
    837   1.95    dyoung 	int i;
    838   1.95    dyoung 
    839  1.124    bouyer 	KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL);
    840   1.95    dyoung 	/*
    841   1.95    dyoung 	 * atapibus detached.
    842   1.95    dyoung 	 */
    843   1.95    dyoung 	if (child == chp->atapibus) {
    844   1.95    dyoung 		chp->atapibus = NULL;
    845  1.104    dyoung 		found = true;
    846  1.124    bouyer 		for (i = 0; i < chp->ch_ndrives; i++) {
    847  1.124    bouyer 			if (chp->ch_drive[i].drive_type != ATA_DRIVET_ATAPI)
    848  1.124    bouyer 				continue;
    849  1.124    bouyer 			KASSERT(chp->ch_drive[i].drv_softc != NULL);
    850  1.124    bouyer 			chp->ch_drive[i].drv_softc = NULL;
    851  1.124    bouyer 			chp->ch_drive[i].drive_flags = 0;
    852  1.124    bouyer 			chp->ch_drive[i].drive_type = ATA_DRIVET_NONE;
    853  1.124    bouyer 		}
    854   1.95    dyoung 	}
    855   1.95    dyoung 
    856   1.95    dyoung 	/*
    857   1.95    dyoung 	 * Detach our other children.
    858   1.95    dyoung 	 */
    859  1.124    bouyer 	for (i = 0; i < chp->ch_ndrives; i++) {
    860  1.124    bouyer 		if (chp->ch_drive[i].drive_type == ATA_DRIVET_ATAPI)
    861   1.95    dyoung 			continue;
    862  1.123  jakllsch 		if (child == chp->ch_drive[i].drv_softc) {
    863   1.95    dyoung 			chp->ch_drive[i].drv_softc = NULL;
    864   1.95    dyoung 			chp->ch_drive[i].drive_flags = 0;
    865  1.124    bouyer 			if (chp->ch_drive[i].drive_type == ATA_DRIVET_PM)
    866  1.124    bouyer 				chp->ch_satapmp_nports = 0;
    867  1.124    bouyer 			chp->ch_drive[i].drive_type = ATA_DRIVET_NONE;
    868  1.104    dyoung 			found = true;
    869   1.95    dyoung 		}
    870   1.95    dyoung 	}
    871   1.95    dyoung 
    872  1.104    dyoung 	if (!found)
    873  1.104    dyoung 		panic("%s: unknown child %p", device_xname(self),
    874  1.104    dyoung 		    (const void *)child);
    875   1.95    dyoung }
    876   1.95    dyoung 
    877  1.103    dyoung CFATTACH_DECL3_NEW(atabus, sizeof(struct atabus_softc),
    878  1.115  jakllsch     atabus_match, atabus_attach, atabus_detach, NULL, atabus_rescan,
    879  1.103    dyoung     atabus_childdetached, DVF_DETACH_SHUTDOWN);
    880   1.23   thorpej 
    881   1.23   thorpej /*****************************************************************************
    882   1.23   thorpej  * Common ATA bus operations.
    883   1.23   thorpej  *****************************************************************************/
    884   1.23   thorpej 
    885  1.124    bouyer /* allocate/free the channel's ch_drive[] array */
    886  1.124    bouyer int
    887  1.124    bouyer atabus_alloc_drives(struct ata_channel *chp, int ndrives)
    888  1.124    bouyer {
    889  1.124    bouyer 	int i;
    890  1.124    bouyer 	if (chp->ch_ndrives != ndrives)
    891  1.124    bouyer 		atabus_free_drives(chp);
    892  1.124    bouyer 	if (chp->ch_drive == NULL) {
    893  1.124    bouyer 		chp->ch_drive = malloc(
    894  1.124    bouyer 		    sizeof(struct ata_drive_datas) * ndrives,
    895  1.124    bouyer 		    M_DEVBUF, M_NOWAIT | M_ZERO);
    896  1.124    bouyer 	}
    897  1.124    bouyer 	if (chp->ch_drive == NULL) {
    898  1.124    bouyer 	    aprint_error_dev(chp->ch_atac->atac_dev,
    899  1.124    bouyer 		"can't alloc drive array\n");
    900  1.124    bouyer 	    chp->ch_ndrives = 0;
    901  1.124    bouyer 	    return ENOMEM;
    902  1.124    bouyer 	};
    903  1.124    bouyer 	for (i = 0; i < ndrives; i++) {
    904  1.124    bouyer 		chp->ch_drive[i].chnl_softc = chp;
    905  1.124    bouyer 		chp->ch_drive[i].drive = i;
    906  1.124    bouyer 	}
    907  1.124    bouyer 	chp->ch_ndrives = ndrives;
    908  1.124    bouyer 	return 0;
    909  1.124    bouyer }
    910  1.124    bouyer 
    911  1.124    bouyer void
    912  1.124    bouyer atabus_free_drives(struct ata_channel *chp)
    913  1.124    bouyer {
    914  1.124    bouyer #ifdef DIAGNOSTIC
    915  1.124    bouyer 	int i;
    916  1.124    bouyer 	int dopanic = 0;
    917  1.124    bouyer 	KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL);
    918  1.124    bouyer 	for (i = 0; i < chp->ch_ndrives; i++) {
    919  1.124    bouyer 		if (chp->ch_drive[i].drive_type != ATA_DRIVET_NONE) {
    920  1.124    bouyer 			printf("%s: ch_drive[%d] type %d != ATA_DRIVET_NONE\n",
    921  1.124    bouyer 			    device_xname(chp->atabus), i,
    922  1.124    bouyer 			    chp->ch_drive[i].drive_type);
    923  1.124    bouyer 			dopanic = 1;
    924  1.124    bouyer 		}
    925  1.124    bouyer 		if (chp->ch_drive[i].drv_softc != NULL) {
    926  1.124    bouyer 			printf("%s: ch_drive[%d] attached to %s\n",
    927  1.124    bouyer 			    device_xname(chp->atabus), i,
    928  1.124    bouyer 			    device_xname(chp->ch_drive[i].drv_softc));
    929  1.124    bouyer 			dopanic = 1;
    930  1.124    bouyer 		}
    931  1.124    bouyer 	}
    932  1.124    bouyer 	if (dopanic)
    933  1.124    bouyer 		panic("atabus_free_drives");
    934  1.124    bouyer #endif
    935  1.124    bouyer 
    936  1.124    bouyer 	if (chp->ch_drive == NULL)
    937  1.124    bouyer 		return;
    938  1.124    bouyer 	chp->ch_ndrives = 0;
    939  1.124    bouyer 	free(chp->ch_drive, M_DEVBUF);
    940  1.124    bouyer 	chp->ch_drive = NULL;
    941  1.124    bouyer }
    942  1.124    bouyer 
    943    1.2    bouyer /* Get the disk's parameters */
    944    1.2    bouyer int
    945  1.132      matt ata_get_params(struct ata_drive_datas *drvp, uint8_t flags,
    946   1.21   thorpej     struct ataparams *prms)
    947    1.2    bouyer {
    948  1.133  jdolecek 	struct ata_xfer *xfer;
    949   1.50   thorpej 	struct ata_channel *chp = drvp->chnl_softc;
    950   1.50   thorpej 	struct atac_softc *atac = chp->ch_atac;
    951   1.93     pooka 	char *tb;
    952   1.93     pooka 	int i, rv;
    953  1.132      matt 	uint16_t *p;
    954    1.2    bouyer 
    955   1.88    dyoung 	ATADEBUG_PRINT(("%s\n", __func__), DEBUG_FUNCS);
    956    1.2    bouyer 
    957  1.133  jdolecek 	xfer = ata_get_xfer(chp);
    958  1.133  jdolecek 	if (xfer == NULL) {
    959  1.133  jdolecek 		ATADEBUG_PRINT(("%s: no xfer\n", __func__),
    960  1.133  jdolecek 		    DEBUG_FUNCS|DEBUG_PROBE);
    961  1.133  jdolecek 		return CMD_AGAIN;
    962  1.133  jdolecek 	}
    963  1.133  jdolecek 
    964  1.133  jdolecek 	tb = kmem_zalloc(ATA_BSIZE, KM_SLEEP);
    965    1.2    bouyer 	memset(prms, 0, sizeof(struct ataparams));
    966    1.2    bouyer 
    967  1.124    bouyer 	if (drvp->drive_type == ATA_DRIVET_ATA) {
    968  1.133  jdolecek 		xfer->c_ata_c.r_command = WDCC_IDENTIFY;
    969  1.133  jdolecek 		xfer->c_ata_c.r_st_bmask = WDCS_DRDY;
    970  1.133  jdolecek 		xfer->c_ata_c.r_st_pmask = WDCS_DRQ;
    971  1.133  jdolecek 		xfer->c_ata_c.timeout = 3000; /* 3s */
    972  1.124    bouyer 	} else if (drvp->drive_type == ATA_DRIVET_ATAPI) {
    973  1.133  jdolecek 		xfer->c_ata_c.r_command = ATAPI_IDENTIFY_DEVICE;
    974  1.133  jdolecek 		xfer->c_ata_c.r_st_bmask = 0;
    975  1.133  jdolecek 		xfer->c_ata_c.r_st_pmask = WDCS_DRQ;
    976  1.133  jdolecek 		xfer->c_ata_c.timeout = 10000; /* 10s */
    977    1.7    bouyer 	} else {
    978   1.47   thorpej 		ATADEBUG_PRINT(("ata_get_parms: no disks\n"),
    979   1.10    bouyer 		    DEBUG_FUNCS|DEBUG_PROBE);
    980   1.93     pooka 		rv = CMD_ERR;
    981   1.93     pooka 		goto out;
    982    1.2    bouyer 	}
    983  1.133  jdolecek 	xfer->c_ata_c.flags = AT_READ | flags;
    984  1.133  jdolecek 	xfer->c_ata_c.data = tb;
    985  1.133  jdolecek 	xfer->c_ata_c.bcount = ATA_BSIZE;
    986   1.50   thorpej 	if ((*atac->atac_bustype_ata->ata_exec_command)(drvp,
    987  1.133  jdolecek 						xfer) != ATACMD_COMPLETE) {
    988   1.47   thorpej 		ATADEBUG_PRINT(("ata_get_parms: wdc_exec_command failed\n"),
    989   1.10    bouyer 		    DEBUG_FUNCS|DEBUG_PROBE);
    990   1.93     pooka 		rv = CMD_AGAIN;
    991   1.93     pooka 		goto out;
    992   1.10    bouyer 	}
    993  1.133  jdolecek 	if (xfer->c_ata_c.flags & (AT_ERROR | AT_TIMEOU | AT_DF)) {
    994   1.47   thorpej 		ATADEBUG_PRINT(("ata_get_parms: ata_c.flags=0x%x\n",
    995  1.133  jdolecek 		    xfer->c_ata_c.flags), DEBUG_FUNCS|DEBUG_PROBE);
    996   1.93     pooka 		rv = CMD_ERR;
    997   1.93     pooka 		goto out;
    998   1.89       dsl 	}
    999   1.89       dsl 	/* if we didn't read any data something is wrong */
   1000  1.133  jdolecek 	if ((xfer->c_ata_c.flags & AT_XFDONE) == 0) {
   1001   1.93     pooka 		rv = CMD_ERR;
   1002   1.93     pooka 		goto out;
   1003   1.93     pooka 	}
   1004   1.89       dsl 
   1005   1.89       dsl 	/* Read in parameter block. */
   1006   1.89       dsl 	memcpy(prms, tb, sizeof(struct ataparams));
   1007   1.80     itohy 
   1008   1.89       dsl 	/*
   1009   1.89       dsl 	 * Shuffle string byte order.
   1010   1.89       dsl 	 * ATAPI NEC, Mitsumi and Pioneer drives and
   1011   1.89       dsl 	 * old ATA TDK CompactFlash cards
   1012   1.89       dsl 	 * have different byte order.
   1013   1.89       dsl 	 */
   1014   1.80     itohy #if BYTE_ORDER == BIG_ENDIAN
   1015   1.80     itohy # define M(n)	prms->atap_model[(n) ^ 1]
   1016   1.80     itohy #else
   1017   1.80     itohy # define M(n)	prms->atap_model[n]
   1018   1.80     itohy #endif
   1019   1.89       dsl 	if (
   1020   1.80     itohy #if BYTE_ORDER == BIG_ENDIAN
   1021   1.89       dsl 	    !
   1022   1.80     itohy #endif
   1023  1.124    bouyer 	    ((drvp->drive_type == ATA_DRIVET_ATAPI) ?
   1024   1.89       dsl 	     ((M(0) == 'N' && M(1) == 'E') ||
   1025   1.89       dsl 	      (M(0) == 'F' && M(1) == 'X') ||
   1026   1.89       dsl 	      (M(0) == 'P' && M(1) == 'i')) :
   1027   1.93     pooka 	     ((M(0) == 'T' && M(1) == 'D' && M(2) == 'K')))) {
   1028   1.93     pooka 		rv = CMD_OK;
   1029   1.93     pooka 		goto out;
   1030   1.93     pooka 	     }
   1031   1.80     itohy #undef M
   1032   1.89       dsl 	for (i = 0; i < sizeof(prms->atap_model); i += 2) {
   1033  1.132      matt 		p = (uint16_t *)(prms->atap_model + i);
   1034   1.89       dsl 		*p = bswap16(*p);
   1035   1.89       dsl 	}
   1036   1.89       dsl 	for (i = 0; i < sizeof(prms->atap_serial); i += 2) {
   1037  1.132      matt 		p = (uint16_t *)(prms->atap_serial + i);
   1038   1.89       dsl 		*p = bswap16(*p);
   1039   1.89       dsl 	}
   1040   1.89       dsl 	for (i = 0; i < sizeof(prms->atap_revision); i += 2) {
   1041  1.132      matt 		p = (uint16_t *)(prms->atap_revision + i);
   1042   1.89       dsl 		*p = bswap16(*p);
   1043   1.89       dsl 	}
   1044   1.80     itohy 
   1045   1.93     pooka 	rv = CMD_OK;
   1046   1.93     pooka  out:
   1047  1.133  jdolecek 	kmem_free(tb, ATA_BSIZE);
   1048  1.133  jdolecek 	ata_free_xfer(chp, xfer);
   1049   1.93     pooka 	return rv;
   1050    1.2    bouyer }
   1051    1.2    bouyer 
   1052    1.2    bouyer int
   1053  1.132      matt ata_set_mode(struct ata_drive_datas *drvp, uint8_t mode, uint8_t flags)
   1054    1.2    bouyer {
   1055  1.133  jdolecek 	struct ata_xfer *xfer;
   1056  1.133  jdolecek 	int rv;
   1057   1.50   thorpej 	struct ata_channel *chp = drvp->chnl_softc;
   1058   1.50   thorpej 	struct atac_softc *atac = chp->ch_atac;
   1059    1.2    bouyer 
   1060   1.47   thorpej 	ATADEBUG_PRINT(("ata_set_mode=0x%x\n", mode), DEBUG_FUNCS);
   1061    1.2    bouyer 
   1062  1.133  jdolecek 	xfer = ata_get_xfer(chp);
   1063  1.133  jdolecek 	if (xfer == NULL) {
   1064  1.133  jdolecek 		ATADEBUG_PRINT(("%s: no xfer\n", __func__),
   1065  1.133  jdolecek 		    DEBUG_FUNCS|DEBUG_PROBE);
   1066  1.133  jdolecek 		return CMD_AGAIN;
   1067  1.133  jdolecek 	}
   1068  1.133  jdolecek 
   1069  1.133  jdolecek 	xfer->c_ata_c.r_command = SET_FEATURES;
   1070  1.133  jdolecek 	xfer->c_ata_c.r_st_bmask = 0;
   1071  1.133  jdolecek 	xfer->c_ata_c.r_st_pmask = 0;
   1072  1.133  jdolecek 	xfer->c_ata_c.r_features = WDSF_SET_MODE;
   1073  1.133  jdolecek 	xfer->c_ata_c.r_count = mode;
   1074  1.133  jdolecek 	xfer->c_ata_c.flags = flags;
   1075  1.133  jdolecek 	xfer->c_ata_c.timeout = 1000; /* 1s */
   1076   1.50   thorpej 	if ((*atac->atac_bustype_ata->ata_exec_command)(drvp,
   1077  1.133  jdolecek 						xfer) != ATACMD_COMPLETE) {
   1078  1.133  jdolecek 		rv = CMD_AGAIN;
   1079  1.133  jdolecek 		goto out;
   1080  1.133  jdolecek 	}
   1081  1.133  jdolecek 	if (xfer->c_ata_c.flags & (AT_ERROR | AT_TIMEOU | AT_DF)) {
   1082  1.133  jdolecek 		rv = CMD_ERR;
   1083  1.133  jdolecek 		goto out;
   1084  1.133  jdolecek 	}
   1085  1.133  jdolecek 
   1086  1.133  jdolecek 	rv = CMD_OK;
   1087  1.133  jdolecek 
   1088  1.133  jdolecek out:
   1089  1.133  jdolecek 	ata_free_xfer(chp, xfer);
   1090  1.133  jdolecek 	return rv;
   1091  1.133  jdolecek }
   1092  1.133  jdolecek 
   1093  1.133  jdolecek int
   1094  1.133  jdolecek ata_read_log_ext_ncq(struct ata_drive_datas *drvp, uint8_t flags,
   1095  1.133  jdolecek     uint8_t *slot, uint8_t *status, uint8_t *err)
   1096  1.133  jdolecek {
   1097  1.133  jdolecek 	struct ata_xfer *xfer;
   1098  1.133  jdolecek 	int rv;
   1099  1.133  jdolecek 	struct ata_channel *chp = drvp->chnl_softc;
   1100  1.133  jdolecek 	struct atac_softc *atac = chp->ch_atac;
   1101  1.133  jdolecek 	uint8_t *tb, cksum, page;
   1102  1.133  jdolecek 
   1103  1.133  jdolecek 	ATADEBUG_PRINT(("%s\n", __func__), DEBUG_FUNCS);
   1104  1.133  jdolecek 
   1105  1.133  jdolecek 	/* Only NCQ ATA drives support/need this */
   1106  1.133  jdolecek 	if (drvp->drive_type != ATA_DRIVET_ATA ||
   1107  1.133  jdolecek 	    (drvp->drive_flags & ATA_DRIVE_NCQ) == 0)
   1108  1.133  jdolecek 		return EOPNOTSUPP;
   1109  1.133  jdolecek 
   1110  1.133  jdolecek 	xfer = ata_get_xfer_ext(chp, C_RECOVERY, 0);
   1111  1.133  jdolecek 
   1112  1.133  jdolecek 	tb = drvp->recovery_blk;
   1113  1.133  jdolecek 	memset(tb, 0, sizeof(drvp->recovery_blk));
   1114  1.133  jdolecek 
   1115  1.133  jdolecek 	/*
   1116  1.133  jdolecek 	 * We could use READ LOG DMA EXT if drive supports it (i.e.
   1117  1.133  jdolecek 	 * when it supports Streaming feature) to avoid PIO command,
   1118  1.133  jdolecek 	 * and to make this a little faster. Realistically, it
   1119  1.133  jdolecek 	 * should not matter.
   1120  1.133  jdolecek 	 */
   1121  1.133  jdolecek 	xfer->c_flags |= C_RECOVERY;
   1122  1.133  jdolecek 	xfer->c_ata_c.r_command = WDCC_READ_LOG_EXT;
   1123  1.133  jdolecek 	xfer->c_ata_c.r_lba = page = WDCC_LOG_PAGE_NCQ;
   1124  1.133  jdolecek 	xfer->c_ata_c.r_st_bmask = WDCS_DRDY;
   1125  1.133  jdolecek 	xfer->c_ata_c.r_st_pmask = WDCS_DRDY;
   1126  1.133  jdolecek 	xfer->c_ata_c.r_count = 1;
   1127  1.133  jdolecek 	xfer->c_ata_c.r_device = WDSD_LBA;
   1128  1.133  jdolecek 	xfer->c_ata_c.flags = AT_READ | AT_LBA | AT_LBA48 | flags;
   1129  1.133  jdolecek 	xfer->c_ata_c.timeout = 1000; /* 1s */
   1130  1.133  jdolecek 	xfer->c_ata_c.data = tb;
   1131  1.133  jdolecek 	xfer->c_ata_c.bcount = sizeof(drvp->recovery_blk);
   1132  1.133  jdolecek 
   1133  1.133  jdolecek 	if ((*atac->atac_bustype_ata->ata_exec_command)(drvp,
   1134  1.133  jdolecek 						xfer) != ATACMD_COMPLETE) {
   1135  1.133  jdolecek 		rv = EAGAIN;
   1136  1.133  jdolecek 		goto out;
   1137  1.133  jdolecek 	}
   1138  1.133  jdolecek 	if (xfer->c_ata_c.flags & (AT_ERROR | AT_TIMEOU | AT_DF)) {
   1139  1.133  jdolecek 		rv = EINVAL;
   1140  1.133  jdolecek 		goto out;
   1141  1.133  jdolecek 	}
   1142  1.133  jdolecek 
   1143  1.133  jdolecek 	cksum = 0;
   1144  1.133  jdolecek 	for (int i = 0; i < sizeof(drvp->recovery_blk); i++)
   1145  1.133  jdolecek 		cksum += tb[i];
   1146  1.133  jdolecek 	if (cksum != 0) {
   1147  1.133  jdolecek 		aprint_error_dev(drvp->drv_softc,
   1148  1.133  jdolecek 		    "invalid checksum %x for READ LOG EXT page %x\n",
   1149  1.133  jdolecek 		    cksum, page);
   1150  1.133  jdolecek 		rv = EINVAL;
   1151  1.133  jdolecek 		goto out;
   1152  1.133  jdolecek 	}
   1153  1.133  jdolecek 
   1154  1.133  jdolecek 	if (tb[0] & WDCC_LOG_NQ) {
   1155  1.133  jdolecek 		/* not queued command */
   1156  1.133  jdolecek 		rv = EOPNOTSUPP;
   1157  1.133  jdolecek 		goto out;
   1158    1.2    bouyer 	}
   1159  1.133  jdolecek 
   1160  1.133  jdolecek 	*slot = tb[0] & 0x1f;
   1161  1.133  jdolecek 	*status = tb[2];
   1162  1.133  jdolecek 	*err = tb[3];
   1163  1.133  jdolecek 
   1164  1.133  jdolecek 	KASSERTMSG((*status & WDCS_ERR),
   1165  1.133  jdolecek 	    "%s: non-error command slot %d reported by READ LOG EXT page %x: "
   1166  1.133  jdolecek 	    "err %x status %x\n",
   1167  1.133  jdolecek 	    device_xname(drvp->drv_softc), *slot, page, *err, *status);
   1168  1.133  jdolecek 
   1169  1.133  jdolecek 	rv = 0;
   1170  1.133  jdolecek 
   1171  1.133  jdolecek out:
   1172  1.133  jdolecek 	ata_free_xfer(chp, xfer);
   1173  1.133  jdolecek 	return rv;
   1174   1.11    bouyer }
   1175   1.11    bouyer 
   1176   1.78     itohy #if NATA_DMA
   1177   1.11    bouyer void
   1178   1.21   thorpej ata_dmaerr(struct ata_drive_datas *drvp, int flags)
   1179   1.11    bouyer {
   1180   1.11    bouyer 	/*
   1181   1.11    bouyer 	 * Downgrade decision: if we get NERRS_MAX in NXFER.
   1182   1.11    bouyer 	 * We start with n_dmaerrs set to NERRS_MAX-1 so that the
   1183   1.11    bouyer 	 * first error within the first NXFER ops will immediatly trigger
   1184   1.11    bouyer 	 * a downgrade.
   1185   1.11    bouyer 	 * If we got an error and n_xfers is bigger than NXFER reset counters.
   1186   1.11    bouyer 	 */
   1187   1.11    bouyer 	drvp->n_dmaerrs++;
   1188   1.11    bouyer 	if (drvp->n_dmaerrs >= NERRS_MAX && drvp->n_xfers <= NXFER) {
   1189   1.39   thorpej 		ata_downgrade_mode(drvp, flags);
   1190   1.11    bouyer 		drvp->n_dmaerrs = NERRS_MAX-1;
   1191   1.11    bouyer 		drvp->n_xfers = 0;
   1192   1.11    bouyer 		return;
   1193   1.11    bouyer 	}
   1194   1.11    bouyer 	if (drvp->n_xfers > NXFER) {
   1195   1.11    bouyer 		drvp->n_dmaerrs = 1; /* just got an error */
   1196   1.11    bouyer 		drvp->n_xfers = 1; /* restart counting from this error */
   1197    1.4    bouyer 	}
   1198    1.2    bouyer }
   1199   1.78     itohy #endif	/* NATA_DMA */
   1200   1.30    bouyer 
   1201   1.44   thorpej /*
   1202   1.68    bouyer  * freeze the queue and wait for the controller to be idle. Caller has to
   1203   1.68    bouyer  * unfreeze/restart the queue
   1204   1.68    bouyer  */
   1205  1.133  jdolecek static void
   1206  1.133  jdolecek ata_channel_idle(struct ata_channel *chp)
   1207   1.68    bouyer {
   1208  1.133  jdolecek 	ata_channel_lock(chp);
   1209  1.133  jdolecek 	ata_channel_freeze_locked(chp);
   1210  1.133  jdolecek 	while (chp->ch_queue->queue_active > 0) {
   1211  1.133  jdolecek 		chp->ch_queue->queue_flags |= QF_IDLE_WAIT;
   1212  1.133  jdolecek 		cv_timedwait(&chp->ch_queue->queue_idle, &chp->ch_lock, 1);
   1213   1.68    bouyer 	}
   1214  1.133  jdolecek 	ata_channel_unlock(chp);
   1215   1.68    bouyer }
   1216   1.68    bouyer 
   1217   1.68    bouyer /*
   1218   1.57   thorpej  * Add a command to the queue and start controller.
   1219   1.57   thorpej  *
   1220   1.57   thorpej  * MUST BE CALLED AT splbio()!
   1221   1.44   thorpej  */
   1222   1.44   thorpej void
   1223   1.48   thorpej ata_exec_xfer(struct ata_channel *chp, struct ata_xfer *xfer)
   1224   1.44   thorpej {
   1225   1.44   thorpej 
   1226   1.47   thorpej 	ATADEBUG_PRINT(("ata_exec_xfer %p channel %d drive %d\n", xfer,
   1227   1.44   thorpej 	    chp->ch_channel, xfer->c_drive), DEBUG_XFERS);
   1228   1.44   thorpej 
   1229   1.44   thorpej 	/* complete xfer setup */
   1230   1.44   thorpej 	xfer->c_chp = chp;
   1231   1.44   thorpej 
   1232  1.133  jdolecek 	ata_channel_lock(chp);
   1233  1.133  jdolecek 
   1234  1.133  jdolecek 	/*
   1235  1.133  jdolecek 	 * Standard commands are added to the end of command list, but
   1236  1.133  jdolecek 	 * recovery commands must be run immediatelly.
   1237  1.133  jdolecek 	 */
   1238  1.133  jdolecek 	if ((xfer->c_flags & C_RECOVERY) == 0)
   1239  1.133  jdolecek 		TAILQ_INSERT_TAIL(&chp->ch_queue->queue_xfer, xfer,
   1240  1.133  jdolecek 		    c_xferchain);
   1241  1.133  jdolecek 	else
   1242  1.133  jdolecek 		TAILQ_INSERT_HEAD(&chp->ch_queue->queue_xfer, xfer,
   1243  1.133  jdolecek 		    c_xferchain);
   1244  1.133  jdolecek 
   1245   1.62    bouyer 	/*
   1246   1.62    bouyer 	 * if polling and can sleep, wait for the xfer to be at head of queue
   1247   1.62    bouyer 	 */
   1248   1.62    bouyer 	if ((xfer->c_flags & (C_POLL | C_WAIT)) ==  (C_POLL | C_WAIT)) {
   1249  1.133  jdolecek 		while (chp->ch_queue->queue_active > 0 ||
   1250   1.62    bouyer 		    TAILQ_FIRST(&chp->ch_queue->queue_xfer) != xfer) {
   1251   1.62    bouyer 			xfer->c_flags |= C_WAITACT;
   1252  1.133  jdolecek 			cv_wait(&xfer->c_active, &chp->ch_lock);
   1253   1.62    bouyer 			xfer->c_flags &= ~C_WAITACT;
   1254  1.133  jdolecek 
   1255  1.133  jdolecek 			/*
   1256  1.133  jdolecek 			 * Free xfer now if it there was attempt to free it
   1257  1.133  jdolecek 			 * while we were waiting.
   1258  1.133  jdolecek 			 */
   1259  1.133  jdolecek 			if ((xfer->c_flags & (C_FREE|C_WAITTIMO)) == C_FREE) {
   1260  1.133  jdolecek 				ata_channel_unlock(chp);
   1261  1.133  jdolecek 
   1262   1.62    bouyer 				ata_free_xfer(chp, xfer);
   1263   1.62    bouyer 				return;
   1264   1.62    bouyer 			}
   1265   1.62    bouyer 		}
   1266   1.62    bouyer 	}
   1267  1.133  jdolecek 
   1268  1.133  jdolecek 	ata_channel_unlock(chp);
   1269  1.133  jdolecek 
   1270  1.133  jdolecek 	ATADEBUG_PRINT(("atastart from ata_exec_xfer, flags 0x%x\n",
   1271  1.133  jdolecek 	    chp->ch_flags), DEBUG_XFERS);
   1272   1.45   thorpej 	atastart(chp);
   1273   1.45   thorpej }
   1274   1.45   thorpej 
   1275   1.45   thorpej /*
   1276   1.45   thorpej  * Start I/O on a controller, for the given channel.
   1277   1.45   thorpej  * The first xfer may be not for our channel if the channel queues
   1278   1.45   thorpej  * are shared.
   1279   1.57   thorpej  *
   1280   1.57   thorpej  * MUST BE CALLED AT splbio()!
   1281   1.45   thorpej  */
   1282   1.45   thorpej void
   1283   1.48   thorpej atastart(struct ata_channel *chp)
   1284   1.45   thorpej {
   1285   1.49   thorpej 	struct atac_softc *atac = chp->ch_atac;
   1286  1.133  jdolecek 	struct ata_queue *chq = chp->ch_queue;
   1287  1.133  jdolecek 	struct ata_xfer *xfer, *axfer;
   1288  1.133  jdolecek 	bool recovery;
   1289   1.45   thorpej 
   1290   1.56   thorpej #ifdef ATA_DEBUG
   1291   1.45   thorpej 	int spl1, spl2;
   1292   1.45   thorpej 
   1293   1.45   thorpej 	spl1 = splbio();
   1294   1.45   thorpej 	spl2 = splbio();
   1295   1.45   thorpej 	if (spl2 != spl1) {
   1296   1.45   thorpej 		printf("atastart: not at splbio()\n");
   1297   1.45   thorpej 		panic("atastart");
   1298   1.45   thorpej 	}
   1299   1.45   thorpej 	splx(spl2);
   1300   1.45   thorpej 	splx(spl1);
   1301   1.56   thorpej #endif /* ATA_DEBUG */
   1302   1.45   thorpej 
   1303  1.133  jdolecek 	ata_channel_lock(chp);
   1304  1.133  jdolecek 
   1305  1.133  jdolecek again:
   1306  1.133  jdolecek 	KASSERT(chq->queue_active <= chq->queue_openings);
   1307  1.133  jdolecek 	if (chq->queue_active == chq->queue_openings) {
   1308  1.133  jdolecek 		goto out; /* channel completely busy */
   1309  1.133  jdolecek 	}
   1310  1.133  jdolecek 
   1311   1.45   thorpej 	/* is there a xfer ? */
   1312   1.45   thorpej 	if ((xfer = TAILQ_FIRST(&chp->ch_queue->queue_xfer)) == NULL)
   1313  1.133  jdolecek 		goto out;
   1314   1.45   thorpej 
   1315  1.133  jdolecek 	recovery = ISSET(xfer->c_flags, C_RECOVERY);
   1316   1.45   thorpej 
   1317  1.133  jdolecek 	/* is the queue frozen? */
   1318  1.133  jdolecek 	if (__predict_false(!recovery && chq->queue_freeze > 0)) {
   1319  1.133  jdolecek 		if (chq->queue_flags & QF_IDLE_WAIT) {
   1320  1.133  jdolecek 			chq->queue_flags &= ~QF_IDLE_WAIT;
   1321  1.133  jdolecek 			cv_signal(&chp->ch_queue->queue_idle);
   1322  1.133  jdolecek 		}
   1323  1.133  jdolecek 		goto out; /* queue frozen */
   1324   1.45   thorpej 	}
   1325  1.133  jdolecek 
   1326  1.133  jdolecek 	/* all xfers on same queue must belong to the same channel */
   1327  1.133  jdolecek 	KASSERT(xfer->c_chp == chp);
   1328  1.133  jdolecek 
   1329  1.133  jdolecek 	/*
   1330  1.133  jdolecek 	 * Can only take the command if there are no current active
   1331  1.133  jdolecek 	 * commands, or if the command is NCQ and the active commands are also
   1332  1.133  jdolecek 	 * NCQ. If PM is in use and HBA driver doesn't support/use FIS-based
   1333  1.133  jdolecek 	 * switching, can only send commands to single drive.
   1334  1.133  jdolecek 	 * Need only check first xfer.
   1335  1.133  jdolecek 	 * XXX FIS-based switching - revisit
   1336  1.133  jdolecek 	 */
   1337  1.133  jdolecek 	if (!recovery && (axfer = TAILQ_FIRST(&chp->ch_queue->active_xfers))) {
   1338  1.133  jdolecek 		if (!ISSET(xfer->c_flags, C_NCQ) ||
   1339  1.133  jdolecek 		    !ISSET(axfer->c_flags, C_NCQ) ||
   1340  1.133  jdolecek 		    xfer->c_drive != axfer->c_drive)
   1341  1.133  jdolecek 			goto out;
   1342   1.45   thorpej 	}
   1343  1.133  jdolecek 
   1344  1.133  jdolecek 	struct ata_drive_datas * const drvp = &chp->ch_drive[xfer->c_drive];
   1345  1.133  jdolecek 
   1346   1.62    bouyer 	/*
   1347   1.62    bouyer 	 * if someone is waiting for the command to be active, wake it up
   1348   1.62    bouyer 	 * and let it process the command
   1349   1.62    bouyer 	 */
   1350   1.62    bouyer 	if (xfer->c_flags & C_WAITACT) {
   1351   1.62    bouyer 		ATADEBUG_PRINT(("atastart: xfer %p channel %d drive %d "
   1352   1.62    bouyer 		    "wait active\n", xfer, chp->ch_channel, xfer->c_drive),
   1353   1.62    bouyer 		    DEBUG_XFERS);
   1354  1.133  jdolecek 		cv_signal(&xfer->c_active);
   1355  1.133  jdolecek 		goto out;
   1356   1.62    bouyer 	}
   1357  1.133  jdolecek 
   1358   1.49   thorpej 	if (atac->atac_claim_hw)
   1359  1.133  jdolecek 		if (!atac->atac_claim_hw(chp, 0))
   1360  1.133  jdolecek 			goto out;
   1361   1.45   thorpej 
   1362   1.47   thorpej 	ATADEBUG_PRINT(("atastart: xfer %p channel %d drive %d\n", xfer,
   1363   1.45   thorpej 	    chp->ch_channel, xfer->c_drive), DEBUG_XFERS);
   1364  1.133  jdolecek 	if (drvp->drive_flags & ATA_DRIVE_RESET) {
   1365  1.133  jdolecek 		drvp->drive_flags &= ~ATA_DRIVE_RESET;
   1366  1.133  jdolecek 		drvp->state = 0;
   1367   1.45   thorpej 	}
   1368  1.133  jdolecek 
   1369  1.133  jdolecek 	if (ISSET(xfer->c_flags, C_NCQ))
   1370  1.133  jdolecek 		SET(chp->ch_flags, ATACH_NCQ);
   1371  1.133  jdolecek 	else
   1372  1.133  jdolecek 		CLR(chp->ch_flags, ATACH_NCQ);
   1373  1.133  jdolecek 
   1374  1.133  jdolecek 	ata_activate_xfer_locked(chp, xfer);
   1375   1.65     perry 
   1376   1.49   thorpej 	if (atac->atac_cap & ATAC_CAP_NOIRQ)
   1377   1.45   thorpej 		KASSERT(xfer->c_flags & C_POLL);
   1378   1.62    bouyer 
   1379  1.133  jdolecek 	switch (ata_xfer_start(xfer)) {
   1380  1.133  jdolecek 	case ATASTART_TH:
   1381  1.133  jdolecek 	case ATASTART_ABORT:
   1382  1.133  jdolecek 		/* don't start any further commands in this case */
   1383  1.133  jdolecek 		goto out;
   1384  1.133  jdolecek 	default:
   1385  1.133  jdolecek 		/* nothing to do */
   1386  1.133  jdolecek 		break;
   1387  1.133  jdolecek 	}
   1388  1.133  jdolecek 
   1389  1.133  jdolecek 	/* Queue more commands if possible, but not during recovery */
   1390  1.133  jdolecek 	if (!recovery && chq->queue_active < chq->queue_openings)
   1391  1.133  jdolecek 		goto again;
   1392  1.133  jdolecek 
   1393  1.133  jdolecek out:
   1394  1.133  jdolecek 	ata_channel_unlock(chp);
   1395   1.44   thorpej }
   1396   1.44   thorpej 
   1397  1.133  jdolecek int
   1398  1.133  jdolecek ata_xfer_start(struct ata_xfer *xfer)
   1399  1.133  jdolecek {
   1400  1.133  jdolecek 	struct ata_channel *chp = xfer->c_chp;
   1401  1.133  jdolecek 	int rv;
   1402  1.133  jdolecek 
   1403  1.133  jdolecek 	KASSERT(mutex_owned(&chp->ch_lock));
   1404  1.133  jdolecek 
   1405  1.133  jdolecek 	rv = xfer->c_start(chp, xfer);
   1406  1.133  jdolecek 	switch (rv) {
   1407  1.133  jdolecek 	case ATASTART_STARTED:
   1408  1.133  jdolecek 		/* nothing to do */
   1409  1.133  jdolecek 		break;
   1410  1.133  jdolecek 	case ATASTART_TH:
   1411  1.133  jdolecek 		/* postpone xfer to thread */
   1412  1.133  jdolecek 		ata_thread_wake_locked(chp);
   1413  1.133  jdolecek 		break;
   1414  1.133  jdolecek 	case ATASTART_POLL:
   1415  1.133  jdolecek 		/* can happen even in thread context for some ATAPI devices */
   1416  1.133  jdolecek 		ata_channel_unlock(chp);
   1417  1.133  jdolecek 		KASSERT(xfer->c_poll != NULL);
   1418  1.133  jdolecek 		xfer->c_poll(chp, xfer);
   1419  1.133  jdolecek 		ata_channel_lock(chp);
   1420  1.133  jdolecek 		break;
   1421  1.133  jdolecek 	case ATASTART_ABORT:
   1422  1.133  jdolecek 		ata_channel_unlock(chp);
   1423  1.133  jdolecek 		KASSERT(xfer->c_abort != NULL);
   1424  1.133  jdolecek 		xfer->c_abort(chp, xfer);
   1425  1.133  jdolecek 		ata_channel_lock(chp);
   1426  1.133  jdolecek 		break;
   1427  1.133  jdolecek 	}
   1428  1.133  jdolecek 
   1429  1.133  jdolecek 	return rv;
   1430  1.133  jdolecek }
   1431  1.133  jdolecek 
   1432  1.133  jdolecek /*
   1433  1.133  jdolecek  * Does it's own locking, does not require splbio().
   1434  1.133  jdolecek  * flags - whether to block waiting for free xfer
   1435  1.133  jdolecek  * openings - limit of openings supported by device, <= 0 means tag not
   1436  1.133  jdolecek  *     relevant, and any available xfer can be returned
   1437  1.133  jdolecek  */
   1438   1.41   thorpej struct ata_xfer *
   1439  1.133  jdolecek ata_get_xfer_ext(struct ata_channel *chp, int flags, uint8_t openings)
   1440   1.41   thorpej {
   1441  1.133  jdolecek 	struct ata_queue *chq = chp->ch_queue;
   1442  1.133  jdolecek 	struct ata_xfer *xfer = NULL;
   1443  1.133  jdolecek 	uint32_t avail, slot, mask;
   1444  1.133  jdolecek 	int error;
   1445  1.133  jdolecek 
   1446  1.133  jdolecek 	ATADEBUG_PRINT(("%s: channel %d flags %x openings %d\n",
   1447  1.133  jdolecek 	    __func__, chp->ch_channel, flags, openings),
   1448  1.133  jdolecek 	    DEBUG_XFERS);
   1449  1.133  jdolecek 
   1450  1.133  jdolecek 	ata_channel_lock(chp);
   1451  1.133  jdolecek 
   1452  1.133  jdolecek 	/*
   1453  1.133  jdolecek 	 * When openings is just 1, can't reserve anything for
   1454  1.133  jdolecek 	 * recovery. KASSERT() here is to catch code which naively
   1455  1.133  jdolecek 	 * relies on C_RECOVERY to work under this condition.
   1456  1.133  jdolecek 	 */
   1457  1.133  jdolecek 	KASSERT((flags & C_RECOVERY) == 0 || chq->queue_openings > 1);
   1458  1.133  jdolecek 
   1459  1.133  jdolecek 	if (flags & C_RECOVERY) {
   1460  1.133  jdolecek 		mask = UINT32_MAX;
   1461  1.133  jdolecek 	} else {
   1462  1.133  jdolecek 		if (openings <= 0 || openings > chq->queue_openings)
   1463  1.133  jdolecek 			openings = chq->queue_openings;
   1464  1.133  jdolecek 
   1465  1.133  jdolecek 		if (openings > 1) {
   1466  1.133  jdolecek 			mask = __BIT(openings - 1) - 1;
   1467  1.133  jdolecek 		} else {
   1468  1.133  jdolecek 			mask = UINT32_MAX;
   1469  1.133  jdolecek 		}
   1470  1.133  jdolecek 	}
   1471  1.133  jdolecek 
   1472  1.133  jdolecek retry:
   1473  1.133  jdolecek 	avail = ffs32(chq->queue_xfers_avail & mask);
   1474  1.133  jdolecek 	if (avail == 0) {
   1475  1.133  jdolecek 		/*
   1476  1.133  jdolecek 		 * Catch code which tries to get another recovery xfer while
   1477  1.133  jdolecek 		 * already holding one (wrong recursion).
   1478  1.133  jdolecek 		 */
   1479  1.133  jdolecek 		KASSERTMSG((flags & C_RECOVERY) == 0,
   1480  1.133  jdolecek 		    "recovery xfer busy openings %d mask %x avail %x",
   1481  1.133  jdolecek 		    openings, mask, chq->queue_xfers_avail);
   1482  1.133  jdolecek 
   1483  1.133  jdolecek 		if (flags & C_WAIT) {
   1484  1.133  jdolecek 			chq->queue_flags |= QF_NEED_XFER;
   1485  1.133  jdolecek 			error = cv_wait_sig(&chq->queue_busy, &chp->ch_lock);
   1486  1.133  jdolecek 			if (error == 0)
   1487  1.133  jdolecek 				goto retry;
   1488  1.133  jdolecek 		}
   1489   1.41   thorpej 
   1490  1.133  jdolecek 		goto out;
   1491   1.41   thorpej 	}
   1492  1.133  jdolecek 
   1493  1.133  jdolecek 	slot = avail - 1;
   1494  1.133  jdolecek 	xfer = &chq->queue_xfers[slot];
   1495  1.133  jdolecek 	chq->queue_xfers_avail &= ~__BIT(slot);
   1496  1.133  jdolecek 
   1497  1.133  jdolecek 	KASSERT((chq->active_xfers_used & __BIT(slot)) == 0);
   1498  1.133  jdolecek 
   1499  1.133  jdolecek 	/* zero everything after the callout member */
   1500  1.133  jdolecek 	memset(&xfer->c_startzero, 0,
   1501  1.133  jdolecek 	    sizeof(struct ata_xfer) - offsetof(struct ata_xfer, c_startzero));
   1502  1.133  jdolecek 
   1503  1.133  jdolecek out:
   1504  1.133  jdolecek 	ata_channel_unlock(chp);
   1505   1.41   thorpej 	return xfer;
   1506   1.41   thorpej }
   1507   1.41   thorpej 
   1508  1.133  jdolecek /*
   1509  1.133  jdolecek  * ata_deactivate_xfer() must be always called prior to ata_free_xfer()
   1510  1.133  jdolecek  */
   1511   1.41   thorpej void
   1512   1.48   thorpej ata_free_xfer(struct ata_channel *chp, struct ata_xfer *xfer)
   1513   1.41   thorpej {
   1514  1.133  jdolecek 	struct ata_queue *chq = chp->ch_queue;
   1515  1.133  jdolecek 
   1516  1.133  jdolecek 	ata_channel_lock(chp);
   1517   1.41   thorpej 
   1518  1.133  jdolecek 	if (xfer->c_flags & (C_WAITACT|C_WAITTIMO)) {
   1519   1.62    bouyer 		/* Someone is waiting for this xfer, so we can't free now */
   1520   1.62    bouyer 		xfer->c_flags |= C_FREE;
   1521  1.133  jdolecek 		cv_signal(&xfer->c_active);
   1522  1.133  jdolecek 		goto out;
   1523   1.62    bouyer 	}
   1524   1.62    bouyer 
   1525   1.76     itohy #if NATA_PIOBM		/* XXX wdc dependent code */
   1526   1.76     itohy 	if (xfer->c_flags & C_PIOBM) {
   1527   1.76     itohy 		struct wdc_softc *wdc = CHAN_TO_WDC(chp);
   1528   1.76     itohy 
   1529   1.76     itohy 		/* finish the busmastering PIO */
   1530   1.76     itohy 		(*wdc->piobm_done)(wdc->dma_arg,
   1531   1.76     itohy 		    chp->ch_channel, xfer->c_drive);
   1532  1.133  jdolecek 		chp->ch_flags &= ~(ATACH_DMA_WAIT | ATACH_PIOBM_WAIT);
   1533   1.76     itohy 	}
   1534   1.76     itohy #endif
   1535   1.76     itohy 
   1536  1.133  jdolecek 	if (chp->ch_atac->atac_free_hw)
   1537  1.133  jdolecek 		chp->ch_atac->atac_free_hw(chp);
   1538  1.133  jdolecek 
   1539  1.133  jdolecek 	KASSERT((chq->active_xfers_used & __BIT(xfer->c_slot)) == 0);
   1540  1.133  jdolecek 	KASSERT((chq->queue_xfers_avail & __BIT(xfer->c_slot)) == 0);
   1541  1.133  jdolecek 	chq->queue_xfers_avail |= __BIT(xfer->c_slot);
   1542  1.133  jdolecek 
   1543  1.133  jdolecek out:
   1544  1.133  jdolecek 	if (chq->queue_flags & QF_NEED_XFER) {
   1545  1.133  jdolecek 		chq->queue_flags &= ~QF_NEED_XFER;
   1546  1.133  jdolecek 		cv_broadcast(&chq->queue_busy);
   1547  1.133  jdolecek 	}
   1548  1.133  jdolecek 
   1549  1.133  jdolecek 	ata_channel_unlock(chp);
   1550  1.133  jdolecek }
   1551  1.133  jdolecek 
   1552  1.133  jdolecek static void
   1553  1.133  jdolecek ata_activate_xfer_locked(struct ata_channel *chp, struct ata_xfer *xfer)
   1554  1.133  jdolecek {
   1555  1.133  jdolecek 	struct ata_queue * const chq = chp->ch_queue;
   1556  1.133  jdolecek 
   1557  1.133  jdolecek 	KASSERT(mutex_owned(&chp->ch_lock));
   1558  1.133  jdolecek 
   1559  1.133  jdolecek 	KASSERT(chq->queue_active < chq->queue_openings);
   1560  1.133  jdolecek 	KASSERT((chq->active_xfers_used & __BIT(xfer->c_slot)) == 0);
   1561  1.133  jdolecek 
   1562  1.133  jdolecek 	TAILQ_REMOVE(&chq->queue_xfer, xfer, c_xferchain);
   1563  1.133  jdolecek 	if ((xfer->c_flags & C_RECOVERY) == 0)
   1564  1.133  jdolecek 		TAILQ_INSERT_TAIL(&chq->active_xfers, xfer, c_activechain);
   1565  1.133  jdolecek 	else {
   1566  1.133  jdolecek 		/*
   1567  1.133  jdolecek 		 * Must go to head, so that ata_queue_get_active_xfer()
   1568  1.133  jdolecek 		 * returns the recovery command, and not some other
   1569  1.133  jdolecek 		 * random active transfer.
   1570  1.133  jdolecek 		 */
   1571  1.133  jdolecek 		TAILQ_INSERT_HEAD(&chq->active_xfers, xfer, c_activechain);
   1572  1.133  jdolecek 	}
   1573  1.133  jdolecek 	chq->active_xfers_used |= __BIT(xfer->c_slot);
   1574  1.133  jdolecek 	chq->queue_active++;
   1575  1.133  jdolecek }
   1576  1.133  jdolecek 
   1577  1.133  jdolecek void
   1578  1.133  jdolecek ata_deactivate_xfer(struct ata_channel *chp, struct ata_xfer *xfer)
   1579  1.133  jdolecek {
   1580  1.133  jdolecek 	struct ata_queue * const chq = chp->ch_queue;
   1581  1.133  jdolecek 
   1582  1.133  jdolecek 	ata_channel_lock(chp);
   1583  1.133  jdolecek 
   1584  1.133  jdolecek 	KASSERT(chq->queue_active > 0);
   1585  1.133  jdolecek 	KASSERT((chq->active_xfers_used & __BIT(xfer->c_slot)) != 0);
   1586  1.133  jdolecek 
   1587  1.133  jdolecek 	callout_stop(&xfer->c_timo_callout);
   1588  1.133  jdolecek 
   1589  1.133  jdolecek 	if (callout_invoking(&xfer->c_timo_callout))
   1590  1.133  jdolecek 		xfer->c_flags |= C_WAITTIMO;
   1591  1.133  jdolecek 
   1592  1.133  jdolecek 	TAILQ_REMOVE(&chq->active_xfers, xfer, c_activechain);
   1593  1.133  jdolecek 	chq->active_xfers_used &= ~__BIT(xfer->c_slot);
   1594  1.133  jdolecek 	chq->queue_active--;
   1595  1.133  jdolecek 
   1596  1.133  jdolecek 	ata_channel_unlock(chp);
   1597  1.133  jdolecek }
   1598  1.133  jdolecek 
   1599  1.133  jdolecek /*
   1600  1.133  jdolecek  * Called in c_intr hook. Must be called before before any deactivations
   1601  1.133  jdolecek  * are done - if there is drain pending, it calls c_kill_xfer hook which
   1602  1.133  jdolecek  * deactivates the xfer.
   1603  1.133  jdolecek  * Calls c_kill_xfer with channel lock free.
   1604  1.133  jdolecek  * Returns true if caller should just exit without further processing.
   1605  1.133  jdolecek  * Caller must not further access any part of xfer or any related controller
   1606  1.133  jdolecek  * structures in that case, it should just return.
   1607  1.133  jdolecek  */
   1608  1.133  jdolecek bool
   1609  1.133  jdolecek ata_waitdrain_xfer_check(struct ata_channel *chp, struct ata_xfer *xfer)
   1610  1.133  jdolecek {
   1611  1.133  jdolecek 	int drive = xfer->c_drive;
   1612  1.133  jdolecek 	bool draining = false;
   1613  1.133  jdolecek 
   1614  1.133  jdolecek 	ata_channel_lock(chp);
   1615  1.133  jdolecek 
   1616  1.133  jdolecek 	if (chp->ch_drive[drive].drive_flags & ATA_DRIVE_WAITDRAIN) {
   1617  1.133  jdolecek 		ata_channel_unlock(chp);
   1618  1.133  jdolecek 
   1619  1.133  jdolecek 		(*xfer->c_kill_xfer)(chp, xfer, KILL_GONE);
   1620  1.133  jdolecek 
   1621  1.133  jdolecek 		ata_channel_lock(chp);
   1622  1.133  jdolecek 		chp->ch_drive[drive].drive_flags &= ~ATA_DRIVE_WAITDRAIN;
   1623  1.133  jdolecek 		cv_signal(&chp->ch_queue->queue_drain);
   1624  1.133  jdolecek 		draining = true;
   1625  1.133  jdolecek 	}
   1626  1.133  jdolecek 
   1627  1.133  jdolecek 	ata_channel_unlock(chp);
   1628  1.133  jdolecek 
   1629  1.133  jdolecek 	return draining;
   1630  1.133  jdolecek }
   1631  1.133  jdolecek 
   1632  1.133  jdolecek /*
   1633  1.133  jdolecek  * Check for race of normal transfer handling vs. timeout.
   1634  1.133  jdolecek  */
   1635  1.133  jdolecek bool
   1636  1.133  jdolecek ata_timo_xfer_check(struct ata_xfer *xfer)
   1637  1.133  jdolecek {
   1638  1.133  jdolecek 	struct ata_channel *chp = xfer->c_chp;
   1639  1.133  jdolecek 	struct ata_drive_datas *drvp = &chp->ch_drive[xfer->c_drive];
   1640  1.133  jdolecek 
   1641  1.133  jdolecek 	ata_channel_lock(chp);
   1642  1.133  jdolecek 
   1643  1.133  jdolecek 	callout_ack(&xfer->c_timo_callout);
   1644  1.133  jdolecek 
   1645  1.133  jdolecek 	if (xfer->c_flags & C_WAITTIMO) {
   1646  1.133  jdolecek 		xfer->c_flags &= ~C_WAITTIMO;
   1647  1.133  jdolecek 
   1648  1.133  jdolecek 		/* Handle race vs. ata_free_xfer() */
   1649  1.133  jdolecek 		if (xfer->c_flags & C_FREE) {
   1650  1.133  jdolecek 			xfer->c_flags &= ~C_FREE;
   1651  1.133  jdolecek 			ata_channel_unlock(chp);
   1652  1.133  jdolecek 
   1653  1.133  jdolecek 	    		aprint_normal_dev(drvp->drv_softc,
   1654  1.133  jdolecek 			    "xfer %d freed while invoking timeout\n",
   1655  1.133  jdolecek 			    xfer->c_slot);
   1656  1.133  jdolecek 
   1657  1.133  jdolecek 			ata_free_xfer(chp, xfer);
   1658  1.133  jdolecek 			return true;
   1659  1.133  jdolecek 		}
   1660  1.133  jdolecek 
   1661  1.133  jdolecek 		/* Race vs. callout_stop() in ata_deactivate_xfer() */
   1662  1.133  jdolecek 		ata_channel_unlock(chp);
   1663  1.133  jdolecek 
   1664  1.133  jdolecek 	    	aprint_normal_dev(drvp->drv_softc,
   1665  1.133  jdolecek 		    "xfer %d deactivated while invoking timeout\n",
   1666  1.133  jdolecek 		    xfer->c_slot);
   1667  1.133  jdolecek 		return true;
   1668  1.133  jdolecek 	}
   1669  1.133  jdolecek 
   1670  1.133  jdolecek 	ata_channel_unlock(chp);
   1671  1.133  jdolecek 
   1672  1.133  jdolecek 	/* No race, proceed with timeout handling */
   1673  1.133  jdolecek 	return false;
   1674  1.133  jdolecek }
   1675  1.133  jdolecek 
   1676  1.133  jdolecek void
   1677  1.133  jdolecek ata_timeout(void *v)
   1678  1.133  jdolecek {
   1679  1.133  jdolecek 	struct ata_xfer *xfer = v;
   1680  1.133  jdolecek 	int s;
   1681  1.133  jdolecek 
   1682  1.133  jdolecek 	ATADEBUG_PRINT(("%s: slot %d\n", __func__, xfer->c_slot),
   1683  1.133  jdolecek 	    DEBUG_FUNCS|DEBUG_XFERS);
   1684  1.133  jdolecek 
   1685  1.133  jdolecek 	s = splbio();				/* XXX MPSAFE */
   1686  1.133  jdolecek 
   1687  1.133  jdolecek 	if (ata_timo_xfer_check(xfer)) {
   1688  1.133  jdolecek 		/* Already logged */
   1689  1.133  jdolecek 		goto out;
   1690  1.133  jdolecek 	}
   1691  1.133  jdolecek 
   1692  1.133  jdolecek 	/* Mark as timed out. Do not print anything, wd(4) will. */
   1693  1.133  jdolecek 	xfer->c_flags |= C_TIMEOU;
   1694  1.133  jdolecek 	xfer->c_intr(xfer->c_chp, xfer, 0);
   1695  1.133  jdolecek 
   1696  1.133  jdolecek out:
   1697   1.41   thorpej 	splx(s);
   1698   1.41   thorpej }
   1699   1.41   thorpej 
   1700   1.42   thorpej /*
   1701  1.133  jdolecek  * Kill off all active xfers for a ata_channel.
   1702   1.42   thorpej  *
   1703  1.133  jdolecek  * Must be called with channel lock held.
   1704  1.133  jdolecek  */
   1705  1.133  jdolecek void
   1706  1.133  jdolecek ata_kill_active(struct ata_channel *chp, int reason, int flags)
   1707  1.133  jdolecek {
   1708  1.133  jdolecek 	struct ata_queue * const chq = chp->ch_queue;
   1709  1.133  jdolecek 	struct ata_xfer *xfer, *xfernext;
   1710  1.133  jdolecek 
   1711  1.133  jdolecek 	KASSERT(mutex_owned(&chp->ch_lock));
   1712  1.133  jdolecek 
   1713  1.133  jdolecek 	TAILQ_FOREACH_SAFE(xfer, &chq->active_xfers, c_activechain, xfernext) {
   1714  1.133  jdolecek 		(*xfer->c_kill_xfer)(xfer->c_chp, xfer, reason);
   1715  1.133  jdolecek 	}
   1716  1.133  jdolecek 
   1717  1.133  jdolecek 	if (flags & AT_RST_EMERG)
   1718  1.133  jdolecek 		ata_queue_reset(chq);
   1719  1.133  jdolecek }
   1720  1.133  jdolecek 
   1721  1.133  jdolecek /*
   1722  1.133  jdolecek  * Kill off all pending xfers for a drive.
   1723   1.42   thorpej  */
   1724   1.42   thorpej void
   1725   1.42   thorpej ata_kill_pending(struct ata_drive_datas *drvp)
   1726   1.42   thorpej {
   1727  1.133  jdolecek 	struct ata_channel * const chp = drvp->chnl_softc;
   1728  1.133  jdolecek 	struct ata_queue * const chq = chp->ch_queue;
   1729  1.133  jdolecek 	struct ata_xfer *xfer, *xfernext;
   1730  1.133  jdolecek 
   1731  1.133  jdolecek 	ata_channel_lock(chp);
   1732  1.133  jdolecek 
   1733  1.133  jdolecek 	/* Kill all pending transfers */
   1734  1.133  jdolecek 	TAILQ_FOREACH_SAFE(xfer, &chq->queue_xfer, c_xferchain, xfernext) {
   1735  1.133  jdolecek 		KASSERT(xfer->c_chp == chp);
   1736   1.42   thorpej 
   1737  1.133  jdolecek 		if (xfer->c_drive != drvp->drive)
   1738   1.42   thorpej 			continue;
   1739  1.133  jdolecek 
   1740   1.42   thorpej 		TAILQ_REMOVE(&chp->ch_queue->queue_xfer, xfer, c_xferchain);
   1741  1.133  jdolecek 
   1742  1.133  jdolecek 		/*
   1743  1.133  jdolecek 		 * Keep the lock, so that we get deadlock (and 'locking against
   1744  1.133  jdolecek 		 * myself' with LOCKDEBUG), instead of silent
   1745  1.133  jdolecek 		 * data corruption, if the hook tries to call back into
   1746  1.133  jdolecek 		 * middle layer for inactive xfer.
   1747  1.133  jdolecek 		 */
   1748  1.133  jdolecek 		(*xfer->c_kill_xfer)(chp, xfer, KILL_GONE_INACTIVE);
   1749   1.42   thorpej 	}
   1750   1.42   thorpej 
   1751  1.133  jdolecek 	/* Wait until all active transfers on the drive finish */
   1752  1.133  jdolecek 	while (chq->queue_active > 0) {
   1753  1.133  jdolecek 		bool drv_active = false;
   1754  1.133  jdolecek 
   1755  1.133  jdolecek 		TAILQ_FOREACH(xfer, &chq->active_xfers, c_activechain) {
   1756  1.133  jdolecek 			KASSERT(xfer->c_chp == chp);
   1757  1.133  jdolecek 
   1758  1.133  jdolecek 			if (xfer->c_drive == drvp->drive) {
   1759  1.133  jdolecek 				drv_active = true;
   1760  1.133  jdolecek 				break;
   1761  1.133  jdolecek 			}
   1762  1.133  jdolecek 		}
   1763  1.133  jdolecek 
   1764  1.133  jdolecek 		if (!drv_active) {
   1765  1.133  jdolecek 			/* all finished */
   1766   1.42   thorpej 			break;
   1767   1.42   thorpej 		}
   1768  1.133  jdolecek 
   1769  1.133  jdolecek 		drvp->drive_flags |= ATA_DRIVE_WAITDRAIN;
   1770  1.133  jdolecek 		cv_wait(&chq->queue_drain, &chp->ch_lock);
   1771   1.42   thorpej 	}
   1772  1.133  jdolecek 
   1773  1.133  jdolecek 	ata_channel_unlock(chp);
   1774  1.133  jdolecek }
   1775  1.133  jdolecek 
   1776  1.133  jdolecek static void
   1777  1.133  jdolecek ata_channel_freeze_locked(struct ata_channel *chp)
   1778  1.133  jdolecek {
   1779  1.133  jdolecek 	chp->ch_queue->queue_freeze++;
   1780  1.133  jdolecek }
   1781  1.133  jdolecek 
   1782  1.133  jdolecek void
   1783  1.133  jdolecek ata_channel_freeze(struct ata_channel *chp)
   1784  1.133  jdolecek {
   1785  1.133  jdolecek 	ata_channel_lock(chp);
   1786  1.133  jdolecek 	ata_channel_freeze_locked(chp);
   1787  1.133  jdolecek 	ata_channel_unlock(chp);
   1788  1.133  jdolecek }
   1789  1.133  jdolecek 
   1790  1.133  jdolecek static void
   1791  1.133  jdolecek ata_channel_thaw_locked(struct ata_channel *chp)
   1792  1.133  jdolecek {
   1793  1.133  jdolecek 	KASSERT(mutex_owned(&chp->ch_lock));
   1794  1.133  jdolecek 
   1795  1.133  jdolecek 	chp->ch_queue->queue_freeze--;
   1796  1.133  jdolecek }
   1797  1.133  jdolecek 
   1798  1.133  jdolecek void
   1799  1.133  jdolecek ata_channel_thaw(struct ata_channel *chp)
   1800  1.133  jdolecek {
   1801  1.133  jdolecek 	ata_channel_lock(chp);
   1802  1.133  jdolecek 	ata_channel_thaw_locked(chp);
   1803  1.133  jdolecek 	ata_channel_unlock(chp);
   1804   1.42   thorpej }
   1805   1.42   thorpej 
   1806   1.55   thorpej /*
   1807   1.55   thorpej  * ata_reset_channel:
   1808   1.55   thorpej  *
   1809   1.55   thorpej  *	Reset and ATA channel.
   1810   1.57   thorpej  *
   1811   1.57   thorpej  *	MUST BE CALLED AT splbio()!
   1812   1.55   thorpej  */
   1813   1.55   thorpej void
   1814   1.55   thorpej ata_reset_channel(struct ata_channel *chp, int flags)
   1815   1.55   thorpej {
   1816   1.55   thorpej 	struct atac_softc *atac = chp->ch_atac;
   1817   1.55   thorpej 	int drive;
   1818   1.55   thorpej 
   1819   1.57   thorpej #ifdef ATA_DEBUG
   1820   1.57   thorpej 	int spl1, spl2;
   1821   1.57   thorpej 
   1822   1.57   thorpej 	spl1 = splbio();
   1823   1.57   thorpej 	spl2 = splbio();
   1824   1.57   thorpej 	if (spl2 != spl1) {
   1825   1.57   thorpej 		printf("ata_reset_channel: not at splbio()\n");
   1826   1.57   thorpej 		panic("ata_reset_channel");
   1827   1.57   thorpej 	}
   1828   1.57   thorpej 	splx(spl2);
   1829   1.57   thorpej 	splx(spl1);
   1830   1.57   thorpej #endif /* ATA_DEBUG */
   1831   1.57   thorpej 
   1832  1.133  jdolecek 	ata_channel_freeze(chp);
   1833   1.55   thorpej 
   1834   1.55   thorpej 	/*
   1835   1.55   thorpej 	 * If we can poll or wait it's OK, otherwise wake up the
   1836   1.55   thorpej 	 * kernel thread to do it for us.
   1837   1.55   thorpej 	 */
   1838  1.100    bouyer 	ATADEBUG_PRINT(("ata_reset_channel flags 0x%x ch_flags 0x%x\n",
   1839  1.100    bouyer 	    flags, chp->ch_flags), DEBUG_FUNCS | DEBUG_XFERS);
   1840   1.55   thorpej 	if ((flags & (AT_POLL | AT_WAIT)) == 0) {
   1841   1.55   thorpej 		if (chp->ch_flags & ATACH_TH_RESET) {
   1842   1.55   thorpej 			/* No need to schedule a reset more than one time. */
   1843  1.133  jdolecek 			ata_channel_thaw(chp);
   1844   1.55   thorpej 			return;
   1845   1.55   thorpej 		}
   1846  1.133  jdolecek 		ata_channel_lock(chp);
   1847   1.55   thorpej 		chp->ch_flags |= ATACH_TH_RESET;
   1848  1.133  jdolecek 		chp->ch_reset_flags = flags & AT_RST_EMERG;
   1849  1.133  jdolecek 		cv_signal(&chp->ch_thr_idle);
   1850  1.133  jdolecek 		ata_channel_unlock(chp);
   1851   1.55   thorpej 		return;
   1852   1.55   thorpej 	}
   1853   1.55   thorpej 
   1854   1.55   thorpej 	(*atac->atac_bustype_ata->ata_reset_channel)(chp, flags);
   1855   1.55   thorpej 
   1856  1.133  jdolecek 	ata_channel_lock(chp);
   1857  1.124    bouyer 	KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL);
   1858  1.124    bouyer 	for (drive = 0; drive < chp->ch_ndrives; drive++)
   1859   1.55   thorpej 		chp->ch_drive[drive].state = 0;
   1860   1.55   thorpej 
   1861   1.55   thorpej 	chp->ch_flags &= ~ATACH_TH_RESET;
   1862  1.133  jdolecek 	ata_channel_unlock(chp);
   1863  1.133  jdolecek 
   1864  1.133  jdolecek 	if (flags & AT_RST_EMERG) {
   1865  1.133  jdolecek 		/* make sure that we can use polled commands */
   1866  1.133  jdolecek 		ata_queue_reset(chp->ch_queue);
   1867  1.133  jdolecek 	} else {
   1868  1.133  jdolecek 		ata_channel_thaw(chp);
   1869   1.55   thorpej 		atastart(chp);
   1870   1.55   thorpej 	}
   1871   1.55   thorpej }
   1872   1.55   thorpej 
   1873   1.43   thorpej int
   1874   1.48   thorpej ata_addref(struct ata_channel *chp)
   1875   1.43   thorpej {
   1876   1.65     perry 	struct atac_softc *atac = chp->ch_atac;
   1877   1.49   thorpej 	struct scsipi_adapter *adapt = &atac->atac_atapi_adapter._generic;
   1878   1.43   thorpej 	int s, error = 0;
   1879   1.43   thorpej 
   1880   1.43   thorpej 	s = splbio();
   1881   1.43   thorpej 	if (adapt->adapt_refcnt++ == 0 &&
   1882   1.43   thorpej 	    adapt->adapt_enable != NULL) {
   1883   1.98      cube 		error = (*adapt->adapt_enable)(atac->atac_dev, 1);
   1884   1.43   thorpej 		if (error)
   1885   1.43   thorpej 			adapt->adapt_refcnt--;
   1886   1.43   thorpej 	}
   1887   1.43   thorpej 	splx(s);
   1888   1.43   thorpej 	return (error);
   1889   1.43   thorpej }
   1890   1.43   thorpej 
   1891   1.43   thorpej void
   1892   1.48   thorpej ata_delref(struct ata_channel *chp)
   1893   1.43   thorpej {
   1894   1.49   thorpej 	struct atac_softc *atac = chp->ch_atac;
   1895   1.49   thorpej 	struct scsipi_adapter *adapt = &atac->atac_atapi_adapter._generic;
   1896   1.43   thorpej 	int s;
   1897   1.43   thorpej 
   1898   1.43   thorpej 	s = splbio();
   1899   1.43   thorpej 	if (adapt->adapt_refcnt-- == 1 &&
   1900   1.43   thorpej 	    adapt->adapt_enable != NULL)
   1901   1.98      cube 		(void) (*adapt->adapt_enable)(atac->atac_dev, 0);
   1902   1.43   thorpej 	splx(s);
   1903   1.43   thorpej }
   1904   1.43   thorpej 
   1905   1.38   thorpej void
   1906   1.48   thorpej ata_print_modes(struct ata_channel *chp)
   1907   1.38   thorpej {
   1908   1.49   thorpej 	struct atac_softc *atac = chp->ch_atac;
   1909   1.38   thorpej 	int drive;
   1910   1.38   thorpej 	struct ata_drive_datas *drvp;
   1911   1.38   thorpej 
   1912  1.124    bouyer 	KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL);
   1913  1.124    bouyer 	for (drive = 0; drive < chp->ch_ndrives; drive++) {
   1914   1.38   thorpej 		drvp = &chp->ch_drive[drive];
   1915  1.124    bouyer 		if (drvp->drive_type == ATA_DRIVET_NONE ||
   1916  1.124    bouyer 		    drvp->drv_softc == NULL)
   1917   1.38   thorpej 			continue;
   1918   1.85        ad 		aprint_verbose("%s(%s:%d:%d): using PIO mode %d",
   1919   1.95    dyoung 			device_xname(drvp->drv_softc),
   1920   1.98      cube 			device_xname(atac->atac_dev),
   1921   1.67      matt 			chp->ch_channel, drvp->drive, drvp->PIO_mode);
   1922   1.78     itohy #if NATA_DMA
   1923  1.124    bouyer 		if (drvp->drive_flags & ATA_DRIVE_DMA)
   1924   1.85        ad 			aprint_verbose(", DMA mode %d", drvp->DMA_mode);
   1925   1.78     itohy #if NATA_UDMA
   1926  1.124    bouyer 		if (drvp->drive_flags & ATA_DRIVE_UDMA) {
   1927   1.85        ad 			aprint_verbose(", Ultra-DMA mode %d", drvp->UDMA_mode);
   1928   1.38   thorpej 			if (drvp->UDMA_mode == 2)
   1929   1.85        ad 				aprint_verbose(" (Ultra/33)");
   1930   1.38   thorpej 			else if (drvp->UDMA_mode == 4)
   1931   1.85        ad 				aprint_verbose(" (Ultra/66)");
   1932   1.38   thorpej 			else if (drvp->UDMA_mode == 5)
   1933   1.85        ad 				aprint_verbose(" (Ultra/100)");
   1934   1.38   thorpej 			else if (drvp->UDMA_mode == 6)
   1935   1.85        ad 				aprint_verbose(" (Ultra/133)");
   1936   1.38   thorpej 		}
   1937   1.78     itohy #endif	/* NATA_UDMA */
   1938   1.78     itohy #endif	/* NATA_DMA */
   1939   1.78     itohy #if NATA_DMA || NATA_PIOBM
   1940   1.78     itohy 		if (0
   1941   1.78     itohy #if NATA_DMA
   1942  1.124    bouyer 		    || (drvp->drive_flags & (ATA_DRIVE_DMA | ATA_DRIVE_UDMA))
   1943   1.78     itohy #endif
   1944   1.76     itohy #if NATA_PIOBM
   1945   1.76     itohy 		    /* PIOBM capable controllers use DMA for PIO commands */
   1946   1.76     itohy 		    || (atac->atac_cap & ATAC_CAP_PIOBM)
   1947   1.76     itohy #endif
   1948   1.76     itohy 		    )
   1949   1.85        ad 			aprint_verbose(" (using DMA)");
   1950  1.133  jdolecek 
   1951  1.133  jdolecek 		if (drvp->drive_flags & ATA_DRIVE_NCQ) {
   1952  1.133  jdolecek 			aprint_verbose(", NCQ (%d tags)%s",
   1953  1.133  jdolecek 			    ATA_REAL_OPENINGS(chp->ch_queue->queue_openings),
   1954  1.133  jdolecek 			    (drvp->drive_flags & ATA_DRIVE_NCQ_PRIO)
   1955  1.133  jdolecek 			    ? " w/PRIO" : "");
   1956  1.133  jdolecek 		} else if (drvp->drive_flags & ATA_DRIVE_WFUA)
   1957  1.133  jdolecek 			aprint_verbose(", WRITE DMA FUA EXT");
   1958  1.133  jdolecek 
   1959   1.78     itohy #endif	/* NATA_DMA || NATA_PIOBM */
   1960   1.85        ad 		aprint_verbose("\n");
   1961   1.38   thorpej 	}
   1962   1.38   thorpej }
   1963   1.38   thorpej 
   1964   1.78     itohy #if NATA_DMA
   1965   1.39   thorpej /*
   1966   1.39   thorpej  * downgrade the transfer mode of a drive after an error. return 1 if
   1967   1.39   thorpej  * downgrade was possible, 0 otherwise.
   1968   1.57   thorpej  *
   1969   1.57   thorpej  * MUST BE CALLED AT splbio()!
   1970   1.39   thorpej  */
   1971   1.39   thorpej int
   1972   1.39   thorpej ata_downgrade_mode(struct ata_drive_datas *drvp, int flags)
   1973   1.39   thorpej {
   1974   1.48   thorpej 	struct ata_channel *chp = drvp->chnl_softc;
   1975   1.49   thorpej 	struct atac_softc *atac = chp->ch_atac;
   1976   1.95    dyoung 	device_t drv_dev = drvp->drv_softc;
   1977   1.74   thorpej 	int cf_flags = device_cfdata(drv_dev)->cf_flags;
   1978   1.39   thorpej 
   1979   1.39   thorpej 	/* if drive or controller don't know its mode, we can't do much */
   1980  1.124    bouyer 	if ((drvp->drive_flags & ATA_DRIVE_MODE) == 0 ||
   1981   1.49   thorpej 	    (atac->atac_set_modes == NULL))
   1982   1.39   thorpej 		return 0;
   1983   1.39   thorpej 	/* current drive mode was set by a config flag, let it this way */
   1984   1.39   thorpej 	if ((cf_flags & ATA_CONFIG_PIO_SET) ||
   1985   1.39   thorpej 	    (cf_flags & ATA_CONFIG_DMA_SET) ||
   1986   1.39   thorpej 	    (cf_flags & ATA_CONFIG_UDMA_SET))
   1987   1.39   thorpej 		return 0;
   1988   1.39   thorpej 
   1989   1.78     itohy #if NATA_UDMA
   1990   1.39   thorpej 	/*
   1991   1.39   thorpej 	 * If we were using Ultra-DMA mode, downgrade to the next lower mode.
   1992   1.39   thorpej 	 */
   1993  1.124    bouyer 	if ((drvp->drive_flags & ATA_DRIVE_UDMA) && drvp->UDMA_mode >= 2) {
   1994   1.39   thorpej 		drvp->UDMA_mode--;
   1995   1.98      cube 		aprint_error_dev(drv_dev,
   1996   1.98      cube 		    "transfer error, downgrading to Ultra-DMA mode %d\n",
   1997   1.98      cube 		    drvp->UDMA_mode);
   1998   1.39   thorpej 	}
   1999   1.78     itohy #endif
   2000   1.39   thorpej 
   2001   1.39   thorpej 	/*
   2002   1.39   thorpej 	 * If we were using ultra-DMA, don't downgrade to multiword DMA.
   2003   1.39   thorpej 	 */
   2004  1.124    bouyer 	else if (drvp->drive_flags & (ATA_DRIVE_DMA | ATA_DRIVE_UDMA)) {
   2005  1.124    bouyer 		drvp->drive_flags &= ~(ATA_DRIVE_DMA | ATA_DRIVE_UDMA);
   2006   1.39   thorpej 		drvp->PIO_mode = drvp->PIO_cap;
   2007   1.98      cube 		aprint_error_dev(drv_dev,
   2008   1.98      cube 		    "transfer error, downgrading to PIO mode %d\n",
   2009   1.98      cube 		    drvp->PIO_mode);
   2010   1.39   thorpej 	} else /* already using PIO, can't downgrade */
   2011   1.39   thorpej 		return 0;
   2012   1.39   thorpej 
   2013   1.49   thorpej 	(*atac->atac_set_modes)(chp);
   2014   1.39   thorpej 	ata_print_modes(chp);
   2015   1.84       wiz 	/* reset the channel, which will schedule all drives for setup */
   2016  1.133  jdolecek 	ata_reset_channel(chp, flags);
   2017   1.39   thorpej 	return 1;
   2018   1.39   thorpej }
   2019   1.78     itohy #endif	/* NATA_DMA */
   2020   1.39   thorpej 
   2021   1.40   thorpej /*
   2022   1.40   thorpej  * Probe drive's capabilities, for use by the controller later
   2023   1.65     perry  * Assumes drvp points to an existing drive.
   2024   1.40   thorpej  */
   2025   1.40   thorpej void
   2026   1.40   thorpej ata_probe_caps(struct ata_drive_datas *drvp)
   2027   1.40   thorpej {
   2028   1.40   thorpej 	struct ataparams params, params2;
   2029   1.48   thorpej 	struct ata_channel *chp = drvp->chnl_softc;
   2030   1.49   thorpej 	struct atac_softc *atac = chp->ch_atac;
   2031   1.95    dyoung 	device_t drv_dev = drvp->drv_softc;
   2032  1.133  jdolecek 	int i, printed = 0;
   2033   1.70  christos 	const char *sep = "";
   2034   1.40   thorpej 	int cf_flags;
   2035   1.40   thorpej 
   2036   1.40   thorpej 	if (ata_get_params(drvp, AT_WAIT, &params) != CMD_OK) {
   2037   1.40   thorpej 		/* IDENTIFY failed. Can't tell more about the device */
   2038   1.40   thorpej 		return;
   2039   1.40   thorpej 	}
   2040   1.49   thorpej 	if ((atac->atac_cap & (ATAC_CAP_DATA16 | ATAC_CAP_DATA32)) ==
   2041   1.49   thorpej 	    (ATAC_CAP_DATA16 | ATAC_CAP_DATA32)) {
   2042   1.40   thorpej 		/*
   2043   1.40   thorpej 		 * Controller claims 16 and 32 bit transfers.
   2044   1.40   thorpej 		 * Re-do an IDENTIFY with 32-bit transfers,
   2045   1.40   thorpej 		 * and compare results.
   2046   1.40   thorpej 		 */
   2047  1.133  jdolecek 		ata_channel_lock(chp);
   2048  1.124    bouyer 		drvp->drive_flags |= ATA_DRIVE_CAP32;
   2049  1.133  jdolecek 		ata_channel_unlock(chp);
   2050   1.40   thorpej 		ata_get_params(drvp, AT_WAIT, &params2);
   2051   1.40   thorpej 		if (memcmp(&params, &params2, sizeof(struct ataparams)) != 0) {
   2052   1.40   thorpej 			/* Not good. fall back to 16bits */
   2053  1.133  jdolecek 			ata_channel_lock(chp);
   2054  1.124    bouyer 			drvp->drive_flags &= ~ATA_DRIVE_CAP32;
   2055  1.133  jdolecek 			ata_channel_unlock(chp);
   2056   1.40   thorpej 		} else {
   2057   1.98      cube 			aprint_verbose_dev(drv_dev, "32-bit data port\n");
   2058   1.40   thorpej 		}
   2059   1.40   thorpej 	}
   2060   1.40   thorpej #if 0 /* Some ultra-DMA drives claims to only support ATA-3. sigh */
   2061   1.65     perry 	if (params.atap_ata_major > 0x01 &&
   2062   1.40   thorpej 	    params.atap_ata_major != 0xffff) {
   2063   1.40   thorpej 		for (i = 14; i > 0; i--) {
   2064   1.40   thorpej 			if (params.atap_ata_major & (1 << i)) {
   2065   1.98      cube 				aprint_verbose_dev(drv_dev,
   2066   1.98      cube 				    "ATA version %d\n", i);
   2067   1.40   thorpej 				drvp->ata_vers = i;
   2068   1.40   thorpej 				break;
   2069   1.40   thorpej 			}
   2070   1.40   thorpej 		}
   2071   1.40   thorpej 	}
   2072   1.40   thorpej #endif
   2073   1.40   thorpej 
   2074   1.40   thorpej 	/* An ATAPI device is at last PIO mode 3 */
   2075  1.124    bouyer 	if (drvp->drive_type == ATA_DRIVET_ATAPI)
   2076   1.40   thorpej 		drvp->PIO_mode = 3;
   2077   1.40   thorpej 
   2078   1.40   thorpej 	/*
   2079   1.65     perry 	 * It's not in the specs, but it seems that some drive
   2080   1.40   thorpej 	 * returns 0xffff in atap_extensions when this field is invalid
   2081   1.40   thorpej 	 */
   2082   1.40   thorpej 	if (params.atap_extensions != 0xffff &&
   2083   1.40   thorpej 	    (params.atap_extensions & WDC_EXT_MODES)) {
   2084   1.40   thorpej 		/*
   2085   1.40   thorpej 		 * XXX some drives report something wrong here (they claim to
   2086   1.40   thorpej 		 * support PIO mode 8 !). As mode is coded on 3 bits in
   2087   1.40   thorpej 		 * SET FEATURE, limit it to 7 (so limit i to 4).
   2088   1.40   thorpej 		 * If higher mode than 7 is found, abort.
   2089   1.40   thorpej 		 */
   2090   1.40   thorpej 		for (i = 7; i >= 0; i--) {
   2091   1.40   thorpej 			if ((params.atap_piomode_supp & (1 << i)) == 0)
   2092   1.40   thorpej 				continue;
   2093   1.40   thorpej 			if (i > 4)
   2094   1.40   thorpej 				return;
   2095   1.40   thorpej 			/*
   2096   1.40   thorpej 			 * See if mode is accepted.
   2097   1.40   thorpej 			 * If the controller can't set its PIO mode,
   2098   1.40   thorpej 			 * assume the defaults are good, so don't try
   2099   1.40   thorpej 			 * to set it
   2100   1.40   thorpej 			 */
   2101   1.49   thorpej 			if (atac->atac_set_modes)
   2102   1.40   thorpej 				/*
   2103  1.113       snj 				 * It's OK to pool here, it's fast enough
   2104   1.40   thorpej 				 * to not bother waiting for interrupt
   2105   1.40   thorpej 				 */
   2106   1.40   thorpej 				if (ata_set_mode(drvp, 0x08 | (i + 3),
   2107   1.40   thorpej 				   AT_WAIT) != CMD_OK)
   2108   1.40   thorpej 					continue;
   2109   1.65     perry 			if (!printed) {
   2110   1.98      cube 				aprint_verbose_dev(drv_dev,
   2111   1.98      cube 				    "drive supports PIO mode %d", i + 3);
   2112   1.40   thorpej 				sep = ",";
   2113   1.40   thorpej 				printed = 1;
   2114   1.40   thorpej 			}
   2115   1.40   thorpej 			/*
   2116   1.40   thorpej 			 * If controller's driver can't set its PIO mode,
   2117   1.40   thorpej 			 * get the highter one for the drive.
   2118   1.40   thorpej 			 */
   2119   1.49   thorpej 			if (atac->atac_set_modes == NULL ||
   2120   1.49   thorpej 			    atac->atac_pio_cap >= i + 3) {
   2121   1.40   thorpej 				drvp->PIO_mode = i + 3;
   2122   1.40   thorpej 				drvp->PIO_cap = i + 3;
   2123   1.40   thorpej 				break;
   2124   1.40   thorpej 			}
   2125   1.40   thorpej 		}
   2126   1.40   thorpej 		if (!printed) {
   2127   1.65     perry 			/*
   2128   1.40   thorpej 			 * We didn't find a valid PIO mode.
   2129   1.40   thorpej 			 * Assume the values returned for DMA are buggy too
   2130   1.40   thorpej 			 */
   2131   1.40   thorpej 			return;
   2132   1.40   thorpej 		}
   2133  1.133  jdolecek 		ata_channel_lock(chp);
   2134  1.124    bouyer 		drvp->drive_flags |= ATA_DRIVE_MODE;
   2135  1.133  jdolecek 		ata_channel_unlock(chp);
   2136   1.40   thorpej 		printed = 0;
   2137   1.40   thorpej 		for (i = 7; i >= 0; i--) {
   2138   1.40   thorpej 			if ((params.atap_dmamode_supp & (1 << i)) == 0)
   2139   1.40   thorpej 				continue;
   2140   1.78     itohy #if NATA_DMA
   2141   1.49   thorpej 			if ((atac->atac_cap & ATAC_CAP_DMA) &&
   2142   1.49   thorpej 			    atac->atac_set_modes != NULL)
   2143   1.40   thorpej 				if (ata_set_mode(drvp, 0x20 | i, AT_WAIT)
   2144   1.40   thorpej 				    != CMD_OK)
   2145   1.40   thorpej 					continue;
   2146   1.78     itohy #endif
   2147   1.40   thorpej 			if (!printed) {
   2148   1.85        ad 				aprint_verbose("%s DMA mode %d", sep, i);
   2149   1.40   thorpej 				sep = ",";
   2150   1.40   thorpej 				printed = 1;
   2151   1.40   thorpej 			}
   2152   1.78     itohy #if NATA_DMA
   2153   1.49   thorpej 			if (atac->atac_cap & ATAC_CAP_DMA) {
   2154   1.49   thorpej 				if (atac->atac_set_modes != NULL &&
   2155   1.49   thorpej 				    atac->atac_dma_cap < i)
   2156   1.40   thorpej 					continue;
   2157   1.40   thorpej 				drvp->DMA_mode = i;
   2158   1.40   thorpej 				drvp->DMA_cap = i;
   2159  1.133  jdolecek 				ata_channel_lock(chp);
   2160  1.124    bouyer 				drvp->drive_flags |= ATA_DRIVE_DMA;
   2161  1.133  jdolecek 				ata_channel_unlock(chp);
   2162   1.40   thorpej 			}
   2163   1.78     itohy #endif
   2164   1.40   thorpej 			break;
   2165   1.40   thorpej 		}
   2166   1.40   thorpej 		if (params.atap_extensions & WDC_EXT_UDMA_MODES) {
   2167   1.40   thorpej 			printed = 0;
   2168   1.40   thorpej 			for (i = 7; i >= 0; i--) {
   2169   1.40   thorpej 				if ((params.atap_udmamode_supp & (1 << i))
   2170   1.40   thorpej 				    == 0)
   2171   1.40   thorpej 					continue;
   2172   1.78     itohy #if NATA_UDMA
   2173   1.49   thorpej 				if (atac->atac_set_modes != NULL &&
   2174   1.49   thorpej 				    (atac->atac_cap & ATAC_CAP_UDMA))
   2175   1.40   thorpej 					if (ata_set_mode(drvp, 0x40 | i,
   2176   1.40   thorpej 					    AT_WAIT) != CMD_OK)
   2177   1.40   thorpej 						continue;
   2178   1.78     itohy #endif
   2179   1.40   thorpej 				if (!printed) {
   2180   1.85        ad 					aprint_verbose("%s Ultra-DMA mode %d",
   2181   1.40   thorpej 					    sep, i);
   2182   1.40   thorpej 					if (i == 2)
   2183   1.85        ad 						aprint_verbose(" (Ultra/33)");
   2184   1.40   thorpej 					else if (i == 4)
   2185   1.85        ad 						aprint_verbose(" (Ultra/66)");
   2186   1.40   thorpej 					else if (i == 5)
   2187   1.85        ad 						aprint_verbose(" (Ultra/100)");
   2188   1.40   thorpej 					else if (i == 6)
   2189   1.85        ad 						aprint_verbose(" (Ultra/133)");
   2190   1.40   thorpej 					sep = ",";
   2191   1.40   thorpej 					printed = 1;
   2192   1.40   thorpej 				}
   2193   1.78     itohy #if NATA_UDMA
   2194   1.49   thorpej 				if (atac->atac_cap & ATAC_CAP_UDMA) {
   2195   1.49   thorpej 					if (atac->atac_set_modes != NULL &&
   2196   1.49   thorpej 					    atac->atac_udma_cap < i)
   2197   1.40   thorpej 						continue;
   2198   1.40   thorpej 					drvp->UDMA_mode = i;
   2199   1.40   thorpej 					drvp->UDMA_cap = i;
   2200  1.133  jdolecek 					ata_channel_lock(chp);
   2201  1.124    bouyer 					drvp->drive_flags |= ATA_DRIVE_UDMA;
   2202  1.133  jdolecek 					ata_channel_unlock(chp);
   2203   1.40   thorpej 				}
   2204   1.78     itohy #endif
   2205   1.40   thorpej 				break;
   2206   1.40   thorpej 			}
   2207   1.40   thorpej 		}
   2208   1.40   thorpej 	}
   2209   1.40   thorpej 
   2210  1.133  jdolecek 	ata_channel_lock(chp);
   2211  1.124    bouyer 	drvp->drive_flags &= ~ATA_DRIVE_NOSTREAM;
   2212  1.124    bouyer 	if (drvp->drive_type == ATA_DRIVET_ATAPI) {
   2213   1.65     perry 		if (atac->atac_cap & ATAC_CAP_ATAPI_NOSTREAM)
   2214  1.124    bouyer 			drvp->drive_flags |= ATA_DRIVE_NOSTREAM;
   2215   1.40   thorpej 	} else {
   2216   1.65     perry 		if (atac->atac_cap & ATAC_CAP_ATA_NOSTREAM)
   2217  1.124    bouyer 			drvp->drive_flags |= ATA_DRIVE_NOSTREAM;
   2218   1.40   thorpej 	}
   2219  1.133  jdolecek 	ata_channel_unlock(chp);
   2220   1.40   thorpej 
   2221   1.40   thorpej 	/* Try to guess ATA version here, if it didn't get reported */
   2222   1.40   thorpej 	if (drvp->ata_vers == 0) {
   2223   1.78     itohy #if NATA_UDMA
   2224  1.124    bouyer 		if (drvp->drive_flags & ATA_DRIVE_UDMA)
   2225   1.40   thorpej 			drvp->ata_vers = 4; /* should be at last ATA-4 */
   2226   1.78     itohy 		else
   2227   1.78     itohy #endif
   2228   1.78     itohy 		if (drvp->PIO_cap > 2)
   2229   1.40   thorpej 			drvp->ata_vers = 2; /* should be at last ATA-2 */
   2230   1.40   thorpej 	}
   2231   1.74   thorpej 	cf_flags = device_cfdata(drv_dev)->cf_flags;
   2232   1.40   thorpej 	if (cf_flags & ATA_CONFIG_PIO_SET) {
   2233  1.133  jdolecek 		ata_channel_lock(chp);
   2234   1.40   thorpej 		drvp->PIO_mode =
   2235   1.40   thorpej 		    (cf_flags & ATA_CONFIG_PIO_MODES) >> ATA_CONFIG_PIO_OFF;
   2236  1.124    bouyer 		drvp->drive_flags |= ATA_DRIVE_MODE;
   2237  1.133  jdolecek 		ata_channel_unlock(chp);
   2238   1.40   thorpej 	}
   2239   1.78     itohy #if NATA_DMA
   2240   1.49   thorpej 	if ((atac->atac_cap & ATAC_CAP_DMA) == 0) {
   2241   1.40   thorpej 		/* don't care about DMA modes */
   2242   1.40   thorpej 		return;
   2243   1.40   thorpej 	}
   2244   1.40   thorpej 	if (cf_flags & ATA_CONFIG_DMA_SET) {
   2245  1.133  jdolecek 		ata_channel_lock(chp);
   2246   1.40   thorpej 		if ((cf_flags & ATA_CONFIG_DMA_MODES) ==
   2247   1.40   thorpej 		    ATA_CONFIG_DMA_DISABLE) {
   2248  1.124    bouyer 			drvp->drive_flags &= ~ATA_DRIVE_DMA;
   2249   1.40   thorpej 		} else {
   2250   1.40   thorpej 			drvp->DMA_mode = (cf_flags & ATA_CONFIG_DMA_MODES) >>
   2251   1.40   thorpej 			    ATA_CONFIG_DMA_OFF;
   2252  1.124    bouyer 			drvp->drive_flags |= ATA_DRIVE_DMA | ATA_DRIVE_MODE;
   2253   1.40   thorpej 		}
   2254  1.133  jdolecek 		ata_channel_unlock(chp);
   2255  1.133  jdolecek 	}
   2256  1.133  jdolecek 
   2257  1.133  jdolecek 	/*
   2258  1.133  jdolecek 	 * Probe WRITE DMA FUA EXT. Support is mandatory for devices
   2259  1.133  jdolecek 	 * supporting LBA48, but nevertheless confirm with the feature flag.
   2260  1.133  jdolecek 	 */
   2261  1.133  jdolecek 	if (drvp->drive_flags & ATA_DRIVE_DMA) {
   2262  1.133  jdolecek 		if ((params.atap_cmd2_en & ATA_CMD2_LBA48) != 0
   2263  1.133  jdolecek 		    && (params.atap_cmd_def & ATA_CMDE_WFE)) {
   2264  1.133  jdolecek 			drvp->drive_flags |= ATA_DRIVE_WFUA;
   2265  1.133  jdolecek 			aprint_verbose("%s WRITE DMA FUA", sep);
   2266  1.133  jdolecek 			sep = ",";
   2267  1.133  jdolecek 		}
   2268  1.133  jdolecek 	}
   2269  1.133  jdolecek 
   2270  1.133  jdolecek 	/* Probe NCQ support - READ/WRITE FPDMA QUEUED command support */
   2271  1.133  jdolecek 	ata_channel_lock(chp);
   2272  1.133  jdolecek 	drvp->drv_openings = 1;
   2273  1.133  jdolecek 	if (params.atap_sata_caps & SATA_NATIVE_CMDQ) {
   2274  1.133  jdolecek 		if (atac->atac_cap & ATAC_CAP_NCQ)
   2275  1.133  jdolecek 			drvp->drive_flags |= ATA_DRIVE_NCQ;
   2276  1.133  jdolecek 		drvp->drv_openings =
   2277  1.133  jdolecek 		    (params.atap_queuedepth & WDC_QUEUE_DEPTH_MASK) + 1;
   2278  1.133  jdolecek 		aprint_verbose("%s NCQ (%d tags)", sep, drvp->drv_openings);
   2279  1.133  jdolecek 		sep = ",";
   2280  1.133  jdolecek 
   2281  1.133  jdolecek 		if (params.atap_sata_caps & SATA_NCQ_PRIO) {
   2282  1.133  jdolecek 			drvp->drive_flags |= ATA_DRIVE_NCQ_PRIO;
   2283  1.133  jdolecek 			aprint_verbose(" w/PRIO");
   2284  1.133  jdolecek 		}
   2285   1.40   thorpej 	}
   2286  1.133  jdolecek 	ata_channel_unlock(chp);
   2287  1.133  jdolecek 
   2288  1.133  jdolecek 	if (printed)
   2289  1.133  jdolecek 		aprint_verbose("\n");
   2290  1.133  jdolecek 
   2291   1.78     itohy #if NATA_UDMA
   2292   1.49   thorpej 	if ((atac->atac_cap & ATAC_CAP_UDMA) == 0) {
   2293   1.40   thorpej 		/* don't care about UDMA modes */
   2294   1.40   thorpej 		return;
   2295   1.40   thorpej 	}
   2296   1.40   thorpej 	if (cf_flags & ATA_CONFIG_UDMA_SET) {
   2297  1.133  jdolecek 		ata_channel_lock(chp);
   2298   1.40   thorpej 		if ((cf_flags & ATA_CONFIG_UDMA_MODES) ==
   2299   1.40   thorpej 		    ATA_CONFIG_UDMA_DISABLE) {
   2300  1.124    bouyer 			drvp->drive_flags &= ~ATA_DRIVE_UDMA;
   2301   1.40   thorpej 		} else {
   2302   1.40   thorpej 			drvp->UDMA_mode = (cf_flags & ATA_CONFIG_UDMA_MODES) >>
   2303   1.40   thorpej 			    ATA_CONFIG_UDMA_OFF;
   2304  1.124    bouyer 			drvp->drive_flags |= ATA_DRIVE_UDMA | ATA_DRIVE_MODE;
   2305   1.40   thorpej 		}
   2306  1.133  jdolecek 		ata_channel_unlock(chp);
   2307   1.40   thorpej 	}
   2308   1.78     itohy #endif	/* NATA_UDMA */
   2309   1.78     itohy #endif	/* NATA_DMA */
   2310   1.40   thorpej }
   2311   1.40   thorpej 
   2312   1.30    bouyer /* management of the /dev/atabus* devices */
   2313   1.54   thorpej int
   2314  1.111    dyoung atabusopen(dev_t dev, int flag, int fmt, struct lwp *l)
   2315   1.30    bouyer {
   2316   1.81     itohy 	struct atabus_softc *sc;
   2317   1.99    cegger 	int error;
   2318   1.65     perry 
   2319   1.99    cegger 	sc = device_lookup_private(&atabus_cd, minor(dev));
   2320   1.99    cegger 	if (sc == NULL)
   2321   1.81     itohy 		return (ENXIO);
   2322   1.65     perry 
   2323   1.81     itohy 	if (sc->sc_flags & ATABUSCF_OPEN)
   2324   1.81     itohy 		return (EBUSY);
   2325   1.30    bouyer 
   2326   1.81     itohy 	if ((error = ata_addref(sc->sc_chan)) != 0)
   2327   1.81     itohy 		return (error);
   2328   1.30    bouyer 
   2329   1.81     itohy 	sc->sc_flags |= ATABUSCF_OPEN;
   2330   1.30    bouyer 
   2331   1.81     itohy 	return (0);
   2332   1.30    bouyer }
   2333   1.30    bouyer 
   2334   1.30    bouyer 
   2335   1.30    bouyer int
   2336  1.111    dyoung atabusclose(dev_t dev, int flag, int fmt, struct lwp *l)
   2337   1.30    bouyer {
   2338   1.98      cube 	struct atabus_softc *sc =
   2339   1.99    cegger 	    device_lookup_private(&atabus_cd, minor(dev));
   2340   1.30    bouyer 
   2341   1.81     itohy 	ata_delref(sc->sc_chan);
   2342   1.30    bouyer 
   2343   1.81     itohy 	sc->sc_flags &= ~ATABUSCF_OPEN;
   2344   1.30    bouyer 
   2345   1.81     itohy 	return (0);
   2346   1.30    bouyer }
   2347   1.30    bouyer 
   2348   1.30    bouyer int
   2349  1.111    dyoung atabusioctl(dev_t dev, u_long cmd, void *addr, int flag, struct lwp *l)
   2350   1.30    bouyer {
   2351   1.98      cube 	struct atabus_softc *sc =
   2352   1.99    cegger 	    device_lookup_private(&atabus_cd, minor(dev));
   2353   1.48   thorpej 	struct ata_channel *chp = sc->sc_chan;
   2354   1.32    bouyer 	int min_drive, max_drive, drive;
   2355   1.81     itohy 	int error;
   2356   1.30    bouyer 	int s;
   2357   1.30    bouyer 
   2358   1.81     itohy 	/*
   2359   1.81     itohy 	 * Enforce write permission for ioctls that change the
   2360   1.81     itohy 	 * state of the bus.  Host adapter specific ioctls must
   2361   1.81     itohy 	 * be checked by the adapter driver.
   2362   1.81     itohy 	 */
   2363   1.81     itohy 	switch (cmd) {
   2364   1.81     itohy 	case ATABUSIOSCAN:
   2365   1.81     itohy 	case ATABUSIODETACH:
   2366   1.81     itohy 	case ATABUSIORESET:
   2367   1.81     itohy 		if ((flag & FWRITE) == 0)
   2368   1.81     itohy 			return (EBADF);
   2369   1.81     itohy 	}
   2370   1.30    bouyer 
   2371   1.81     itohy 	switch (cmd) {
   2372   1.81     itohy 	case ATABUSIORESET:
   2373   1.30    bouyer 		s = splbio();
   2374   1.55   thorpej 		ata_reset_channel(sc->sc_chan, AT_WAIT | AT_POLL);
   2375   1.30    bouyer 		splx(s);
   2376  1.111    dyoung 		return 0;
   2377   1.32    bouyer 	case ATABUSIOSCAN:
   2378   1.32    bouyer 	{
   2379   1.32    bouyer #if 0
   2380   1.32    bouyer 		struct atabusioscan_args *a=
   2381   1.32    bouyer 		    (struct atabusioscan_args *)addr;
   2382   1.32    bouyer #endif
   2383  1.124    bouyer 		if ((chp->ch_drive[0].drive_type == ATA_DRIVET_OLD) ||
   2384  1.124    bouyer 		    (chp->ch_drive[1].drive_type == ATA_DRIVET_OLD))
   2385   1.32    bouyer 			return (EOPNOTSUPP);
   2386   1.32    bouyer 		return (EOPNOTSUPP);
   2387   1.32    bouyer 	}
   2388   1.32    bouyer 	case ATABUSIODETACH:
   2389   1.32    bouyer 	{
   2390  1.116     isaki 		struct atabusiodetach_args *a=
   2391  1.116     isaki 		    (struct atabusiodetach_args *)addr;
   2392  1.124    bouyer 		if ((chp->ch_drive[0].drive_type == ATA_DRIVET_OLD) ||
   2393  1.124    bouyer 		    (chp->ch_drive[1].drive_type == ATA_DRIVET_OLD))
   2394   1.32    bouyer 			return (EOPNOTSUPP);
   2395   1.32    bouyer 		switch (a->at_dev) {
   2396   1.32    bouyer 		case -1:
   2397   1.32    bouyer 			min_drive = 0;
   2398   1.32    bouyer 			max_drive = 1;
   2399   1.32    bouyer 			break;
   2400   1.32    bouyer 		case 0:
   2401   1.32    bouyer 		case 1:
   2402   1.32    bouyer 			min_drive = max_drive = a->at_dev;
   2403   1.32    bouyer 			break;
   2404   1.32    bouyer 		default:
   2405   1.32    bouyer 			return (EINVAL);
   2406   1.32    bouyer 		}
   2407   1.32    bouyer 		for (drive = min_drive; drive <= max_drive; drive++) {
   2408   1.32    bouyer 			if (chp->ch_drive[drive].drv_softc != NULL) {
   2409   1.32    bouyer 				error = config_detach(
   2410   1.32    bouyer 				    chp->ch_drive[drive].drv_softc, 0);
   2411   1.32    bouyer 				if (error)
   2412   1.32    bouyer 					return (error);
   2413   1.95    dyoung 				KASSERT(chp->ch_drive[drive].drv_softc == NULL);
   2414   1.32    bouyer 			}
   2415   1.32    bouyer 		}
   2416  1.111    dyoung 		return 0;
   2417   1.32    bouyer 	}
   2418   1.30    bouyer 	default:
   2419  1.111    dyoung 		return ENOTTY;
   2420   1.30    bouyer 	}
   2421  1.111    dyoung }
   2422   1.64  jmcneill 
   2423   1.92  jmcneill static bool
   2424  1.112    dyoung atabus_suspend(device_t dv, const pmf_qual_t *qual)
   2425   1.92  jmcneill {
   2426   1.92  jmcneill 	struct atabus_softc *sc = device_private(dv);
   2427   1.92  jmcneill 	struct ata_channel *chp = sc->sc_chan;
   2428   1.92  jmcneill 
   2429  1.133  jdolecek 	ata_channel_idle(chp);
   2430   1.92  jmcneill 
   2431   1.92  jmcneill 	return true;
   2432   1.92  jmcneill }
   2433   1.92  jmcneill 
   2434   1.92  jmcneill static bool
   2435  1.112    dyoung atabus_resume(device_t dv, const pmf_qual_t *qual)
   2436   1.64  jmcneill {
   2437   1.92  jmcneill 	struct atabus_softc *sc = device_private(dv);
   2438   1.64  jmcneill 	struct ata_channel *chp = sc->sc_chan;
   2439   1.64  jmcneill 
   2440   1.92  jmcneill 	/*
   2441   1.92  jmcneill 	 * XXX joerg: with wdc, the first channel unfreezes the controler.
   2442   1.92  jmcneill 	 * Move this the reset and queue idling into wdc.
   2443   1.92  jmcneill 	 */
   2444  1.133  jdolecek 	ata_channel_lock(chp);
   2445   1.92  jmcneill 	if (chp->ch_queue->queue_freeze == 0) {
   2446  1.133  jdolecek 		ata_channel_unlock(chp);
   2447  1.133  jdolecek 		goto out;
   2448   1.64  jmcneill 	}
   2449   1.92  jmcneill 	KASSERT(chp->ch_queue->queue_freeze > 0);
   2450  1.133  jdolecek 	ata_channel_unlock(chp);
   2451  1.133  jdolecek 
   2452   1.92  jmcneill 	/* unfreeze the queue and reset drives */
   2453  1.133  jdolecek 	ata_channel_thaw(chp);
   2454  1.128     blymn 
   2455  1.128     blymn 	/* reset channel only if there are drives attached */
   2456  1.128     blymn 	if (chp->ch_ndrives > 0)
   2457  1.128     blymn 		ata_reset_channel(chp, AT_WAIT);
   2458   1.64  jmcneill 
   2459  1.133  jdolecek out:
   2460   1.92  jmcneill 	return true;
   2461   1.64  jmcneill }
   2462  1.115  jakllsch 
   2463  1.115  jakllsch static int
   2464  1.115  jakllsch atabus_rescan(device_t self, const char *ifattr, const int *locators)
   2465  1.115  jakllsch {
   2466  1.115  jakllsch 	struct atabus_softc *sc = device_private(self);
   2467  1.115  jakllsch 	struct ata_channel *chp = sc->sc_chan;
   2468  1.115  jakllsch 	struct atabus_initq *initq;
   2469  1.115  jakllsch 	int i;
   2470  1.115  jakllsch 
   2471  1.124    bouyer 	/*
   2472  1.124    bouyer 	 * we can rescan a port multiplier atabus, even if some devices are
   2473  1.124    bouyer 	 * still attached
   2474  1.124    bouyer 	 */
   2475  1.124    bouyer 	if (chp->ch_satapmp_nports == 0) {
   2476  1.124    bouyer 		if (chp->atapibus != NULL) {
   2477  1.115  jakllsch 			return EBUSY;
   2478  1.115  jakllsch 		}
   2479  1.115  jakllsch 
   2480  1.124    bouyer 		KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL);
   2481  1.124    bouyer 		for (i = 0; i < chp->ch_ndrives; i++) {
   2482  1.124    bouyer 			if (chp->ch_drive[i].drv_softc != NULL) {
   2483  1.124    bouyer 				return EBUSY;
   2484  1.124    bouyer 			}
   2485  1.124    bouyer 		}
   2486  1.115  jakllsch 	}
   2487  1.115  jakllsch 
   2488  1.115  jakllsch 	initq = malloc(sizeof(*initq), M_DEVBUF, M_WAITOK);
   2489  1.115  jakllsch 	initq->atabus_sc = sc;
   2490  1.133  jdolecek 	mutex_enter(&atabus_qlock);
   2491  1.115  jakllsch 	TAILQ_INSERT_TAIL(&atabus_initq_head, initq, atabus_initq);
   2492  1.133  jdolecek 	mutex_exit(&atabus_qlock);
   2493  1.129  christos 	config_pending_incr(sc->sc_dev);
   2494  1.115  jakllsch 
   2495  1.133  jdolecek 	ata_channel_lock(chp);
   2496  1.115  jakllsch 	chp->ch_flags |= ATACH_TH_RESCAN;
   2497  1.133  jdolecek 	cv_signal(&chp->ch_thr_idle);
   2498  1.133  jdolecek 	ata_channel_unlock(chp);
   2499  1.115  jakllsch 
   2500  1.115  jakllsch 	return 0;
   2501  1.115  jakllsch }
   2502  1.127    bouyer 
   2503  1.127    bouyer void
   2504  1.133  jdolecek ata_delay(struct ata_channel *chp, int ms, const char *msg, int flags)
   2505  1.127    bouyer {
   2506  1.133  jdolecek 
   2507  1.127    bouyer 	if ((flags & (AT_WAIT | AT_POLL)) == AT_POLL) {
   2508  1.127    bouyer 		/*
   2509  1.133  jdolecek 		 * can't use kpause(), we may be in interrupt context
   2510  1.127    bouyer 		 * or taking a crash dump
   2511  1.127    bouyer 		 */
   2512  1.127    bouyer 		delay(ms * 1000);
   2513  1.127    bouyer 	} else {
   2514  1.133  jdolecek 		int pause = mstohz(ms);
   2515  1.134   mlelstv 
   2516  1.134   mlelstv 		KASSERT(mutex_owned(&chp->ch_lock));
   2517  1.133  jdolecek 		kpause(msg, false, pause > 0 ? pause : 1, &chp->ch_lock);
   2518  1.127    bouyer 	}
   2519  1.127    bouyer }
   2520  1.133  jdolecek 
   2521  1.133  jdolecek void
   2522  1.133  jdolecek atacmd_toncq(struct ata_xfer *xfer, uint8_t *cmd, uint16_t *count,
   2523  1.133  jdolecek     uint16_t *features, uint8_t *device)
   2524  1.133  jdolecek {
   2525  1.133  jdolecek 	if ((xfer->c_flags & C_NCQ) == 0) {
   2526  1.133  jdolecek 		/* FUA handling for non-NCQ drives */
   2527  1.133  jdolecek 		if (xfer->c_bio.flags & ATA_FUA
   2528  1.133  jdolecek 		    && *cmd == WDCC_WRITEDMA_EXT)
   2529  1.133  jdolecek 			*cmd = WDCC_WRITEDMA_FUA_EXT;
   2530  1.133  jdolecek 
   2531  1.133  jdolecek 		return;
   2532  1.133  jdolecek 	}
   2533  1.133  jdolecek 
   2534  1.133  jdolecek 	*cmd = (xfer->c_bio.flags & ATA_READ) ?
   2535  1.133  jdolecek 	    WDCC_READ_FPDMA_QUEUED : WDCC_WRITE_FPDMA_QUEUED;
   2536  1.133  jdolecek 
   2537  1.133  jdolecek 	/* for FPDMA the block count is in features */
   2538  1.133  jdolecek 	*features = *count;
   2539  1.133  jdolecek 
   2540  1.133  jdolecek 	/* NCQ tag */
   2541  1.133  jdolecek 	*count = (xfer->c_slot << 3);
   2542  1.133  jdolecek 
   2543  1.133  jdolecek 	if (xfer->c_bio.flags & ATA_PRIO_HIGH)
   2544  1.133  jdolecek 		*count |= WDSC_PRIO_HIGH;
   2545  1.133  jdolecek 
   2546  1.133  jdolecek 	/* other device flags */
   2547  1.133  jdolecek 	if (xfer->c_bio.flags & ATA_FUA)
   2548  1.133  jdolecek 		*device |= WDSD_FUA;
   2549  1.133  jdolecek }
   2550  1.133  jdolecek 
   2551  1.133  jdolecek /*
   2552  1.133  jdolecek  * Must be called without any locks, i.e. with both drive and channel locks
   2553  1.133  jdolecek  * released.
   2554  1.133  jdolecek  */
   2555  1.133  jdolecek void
   2556  1.133  jdolecek ata_channel_start(struct ata_channel *chp, int drive)
   2557  1.133  jdolecek {
   2558  1.133  jdolecek 	int i, s;
   2559  1.133  jdolecek 	struct ata_drive_datas *drvp;
   2560  1.133  jdolecek 
   2561  1.133  jdolecek 	s = splbio();
   2562  1.133  jdolecek 
   2563  1.133  jdolecek 	KASSERT(chp->ch_ndrives > 0);
   2564  1.133  jdolecek 
   2565  1.133  jdolecek #define ATA_DRIVE_START(chp, drive) \
   2566  1.133  jdolecek 	do {							\
   2567  1.133  jdolecek 		KASSERT(drive < chp->ch_ndrives);		\
   2568  1.133  jdolecek 		drvp = &chp->ch_drive[drive];			\
   2569  1.133  jdolecek 								\
   2570  1.133  jdolecek 		if (drvp->drive_type != ATA_DRIVET_ATA &&	\
   2571  1.133  jdolecek 		    drvp->drive_type != ATA_DRIVET_ATAPI &&	\
   2572  1.133  jdolecek 		    drvp->drive_type != ATA_DRIVET_OLD)		\
   2573  1.133  jdolecek 			continue;				\
   2574  1.133  jdolecek 								\
   2575  1.133  jdolecek 		if (drvp->drv_start != NULL)			\
   2576  1.133  jdolecek 			(*drvp->drv_start)(drvp->drv_softc);	\
   2577  1.133  jdolecek 	} while (0)
   2578  1.133  jdolecek 
   2579  1.133  jdolecek 	/*
   2580  1.133  jdolecek 	 * Process drives in round robin fashion starting with next one after
   2581  1.133  jdolecek 	 * the one which finished transfer. Thus no single drive would
   2582  1.133  jdolecek 	 * completely starve other drives on same channel.
   2583  1.133  jdolecek 	 * This loop processes all but the current drive, so won't do anything
   2584  1.133  jdolecek 	 * if there is only one drive in channel.
   2585  1.133  jdolecek 	 */
   2586  1.133  jdolecek 	for (i = (drive + 1) % chp->ch_ndrives; i != drive;
   2587  1.133  jdolecek 	    i = (i + 1) % chp->ch_ndrives) {
   2588  1.133  jdolecek 		ATA_DRIVE_START(chp, i);
   2589  1.133  jdolecek 	}
   2590  1.133  jdolecek 
   2591  1.133  jdolecek 	/* Now try to kick off xfers on the current drive */
   2592  1.133  jdolecek 	ATA_DRIVE_START(chp, drive);
   2593  1.133  jdolecek 
   2594  1.133  jdolecek 	splx(s);
   2595  1.133  jdolecek #undef ATA_DRIVE_START
   2596  1.133  jdolecek }
   2597  1.133  jdolecek 
   2598  1.133  jdolecek void
   2599  1.133  jdolecek ata_channel_lock(struct ata_channel *chp)
   2600  1.133  jdolecek {
   2601  1.133  jdolecek 	mutex_enter(&chp->ch_lock);
   2602  1.133  jdolecek }
   2603  1.133  jdolecek 
   2604  1.133  jdolecek void
   2605  1.133  jdolecek ata_channel_unlock(struct ata_channel *chp)
   2606  1.133  jdolecek {
   2607  1.133  jdolecek 	mutex_exit(&chp->ch_lock);
   2608  1.133  jdolecek }
   2609  1.133  jdolecek 
   2610  1.133  jdolecek void
   2611  1.133  jdolecek ata_channel_lock_owned(struct ata_channel *chp)
   2612  1.133  jdolecek {
   2613  1.133  jdolecek 	KASSERT(mutex_owned(&chp->ch_lock));
   2614  1.133  jdolecek }
   2615  1.133  jdolecek 
   2616  1.133  jdolecek void
   2617  1.133  jdolecek ata_wait_xfer(struct ata_channel *chp, struct ata_xfer *xfer)
   2618  1.133  jdolecek {
   2619  1.133  jdolecek 	KASSERT(mutex_owned(&chp->ch_lock));
   2620  1.133  jdolecek 
   2621  1.133  jdolecek 	cv_wait(&xfer->c_finish, &chp->ch_lock);
   2622  1.133  jdolecek }
   2623  1.133  jdolecek 
   2624  1.133  jdolecek void
   2625  1.133  jdolecek ata_wake_xfer(struct ata_channel *chp, struct ata_xfer *xfer)
   2626  1.133  jdolecek {
   2627  1.133  jdolecek 	KASSERT(mutex_owned(&chp->ch_lock));
   2628  1.133  jdolecek 
   2629  1.133  jdolecek 	cv_signal(&xfer->c_finish);
   2630  1.133  jdolecek }
   2631