Home | History | Annotate | Line # | Download | only in scsipi
atapi_wdc.c revision 1.133
      1 /*	$NetBSD: atapi_wdc.c,v 1.133 2018/11/12 20:54:03 jdolecek Exp $	*/
      2 
      3 /*
      4  * Copyright (c) 1998, 2001 Manuel Bouyer.
      5  *
      6  * Redistribution and use in source and binary forms, with or without
      7  * modification, are permitted provided that the following conditions
      8  * are met:
      9  * 1. Redistributions of source code must retain the above copyright
     10  *    notice, this list of conditions and the following disclaimer.
     11  * 2. Redistributions in binary form must reproduce the above copyright
     12  *    notice, this list of conditions and the following disclaimer in the
     13  *    documentation and/or other materials provided with the distribution.
     14  *
     15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
     16  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
     17  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
     18  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
     19  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
     20  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
     21  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
     22  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
     23  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
     24  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     25  */
     26 
     27 #include <sys/cdefs.h>
     28 __KERNEL_RCSID(0, "$NetBSD: atapi_wdc.c,v 1.133 2018/11/12 20:54:03 jdolecek Exp $");
     29 
     30 #ifndef ATADEBUG
     31 #define ATADEBUG
     32 #endif /* ATADEBUG */
     33 
     34 #include <sys/param.h>
     35 #include <sys/systm.h>
     36 #include <sys/kernel.h>
     37 #include <sys/file.h>
     38 #include <sys/stat.h>
     39 #include <sys/buf.h>
     40 #include <sys/malloc.h>
     41 #include <sys/device.h>
     42 #include <sys/syslog.h>
     43 #include <sys/proc.h>
     44 #include <sys/dvdio.h>
     45 
     46 #include <sys/intr.h>
     47 #include <sys/bus.h>
     48 
     49 #ifndef __BUS_SPACE_HAS_STREAM_METHODS
     50 #define	bus_space_write_multi_stream_2	bus_space_write_multi_2
     51 #define	bus_space_write_multi_stream_4	bus_space_write_multi_4
     52 #define	bus_space_read_multi_stream_2	bus_space_read_multi_2
     53 #define	bus_space_read_multi_stream_4	bus_space_read_multi_4
     54 #endif /* __BUS_SPACE_HAS_STREAM_METHODS */
     55 
     56 #include <dev/ata/ataconf.h>
     57 #include <dev/ata/atareg.h>
     58 #include <dev/ata/atavar.h>
     59 #include <dev/ic/wdcreg.h>
     60 #include <dev/ic/wdcvar.h>
     61 
     62 #include <dev/scsipi/scsi_all.h> /* for SCSI status */
     63 
     64 #define DEBUG_INTR   0x01
     65 #define DEBUG_XFERS  0x02
     66 #define DEBUG_STATUS 0x04
     67 #define DEBUG_FUNCS  0x08
     68 #define DEBUG_PROBE  0x10
     69 #ifdef ATADEBUG
     70 #ifndef ATADEBUG_ATAPI_MASK
     71 #define ATADEBUG_ATAPI_MASK 0x0
     72 #endif
     73 int wdcdebug_atapi_mask = ATADEBUG_ATAPI_MASK;
     74 #define ATADEBUG_PRINT(args, level) \
     75 	if (wdcdebug_atapi_mask & (level)) \
     76 		printf args
     77 #else
     78 #define ATADEBUG_PRINT(args, level)
     79 #endif
     80 
     81 #define ATAPI_DELAY 10	/* 10 ms, this is used only before sending a cmd */
     82 #define ATAPI_MODE_DELAY 1000	/* 1s, timeout for SET_FEATYRE cmds */
     83 
     84 static int	wdc_atapi_get_params(struct scsipi_channel *, int,
     85 				     struct ataparams *);
     86 static void	wdc_atapi_probe_device(struct atapibus_softc *, int);
     87 static void	wdc_atapi_minphys (struct buf *bp);
     88 static int	wdc_atapi_start(struct ata_channel *,struct ata_xfer *);
     89 static int	wdc_atapi_intr(struct ata_channel *, struct ata_xfer *, int);
     90 static void	wdc_atapi_kill_xfer(struct ata_channel *,
     91 				    struct ata_xfer *, int);
     92 static void	wdc_atapi_phase_complete(struct ata_xfer *, int);
     93 static void	wdc_atapi_poll(struct ata_channel *, struct ata_xfer *);
     94 static void	wdc_atapi_done(struct ata_channel *, struct ata_xfer *);
     95 static void	wdc_atapi_reset(struct ata_channel *, struct ata_xfer *);
     96 static void	wdc_atapi_scsipi_request(struct scsipi_channel *,
     97 					 scsipi_adapter_req_t, void *);
     98 static void	wdc_atapi_kill_pending(struct scsipi_periph *);
     99 static void	wdc_atapi_polldsc(void *arg);
    100 
    101 #define MAX_SIZE MAXPHYS
    102 
    103 static const struct scsipi_bustype wdc_atapi_bustype = {
    104 	SCSIPI_BUSTYPE_ATAPI,
    105 	atapi_scsipi_cmd,
    106 	atapi_interpret_sense,
    107 	atapi_print_addr,
    108 	wdc_atapi_kill_pending,
    109 	NULL,
    110 };
    111 
    112 void
    113 wdc_atapibus_attach(struct atabus_softc *ata_sc)
    114 {
    115 	struct ata_channel *chp = ata_sc->sc_chan;
    116 	struct atac_softc *atac = chp->ch_atac;
    117 	struct scsipi_adapter *adapt = &atac->atac_atapi_adapter._generic;
    118 	struct scsipi_channel *chan = &chp->ch_atapi_channel;
    119 
    120 	/*
    121 	 * Fill in the scsipi_adapter.
    122 	 */
    123 	adapt->adapt_dev = atac->atac_dev;
    124 	adapt->adapt_nchannels = atac->atac_nchannels;
    125 	adapt->adapt_request = wdc_atapi_scsipi_request;
    126 	adapt->adapt_minphys = wdc_atapi_minphys;
    127 	if (atac->atac_cap & ATAC_CAP_NOIRQ)
    128 		adapt->adapt_flags |= SCSIPI_ADAPT_POLL_ONLY;
    129 	atac->atac_atapi_adapter.atapi_probe_device = wdc_atapi_probe_device;
    130 
    131 	/*
    132 	 * Fill in the scsipi_channel.
    133 	 */
    134 	memset(chan, 0, sizeof(*chan));
    135 	chan->chan_adapter = adapt;
    136 	chan->chan_bustype = &wdc_atapi_bustype;
    137 	chan->chan_channel = chp->ch_channel;
    138 	chan->chan_flags = SCSIPI_CHAN_OPENINGS;
    139 	chan->chan_openings = 1;
    140 	chan->chan_max_periph = 1;
    141 	chan->chan_ntargets = 2;
    142 	chan->chan_nluns = 1;
    143 
    144 	chp->atapibus = config_found_ia(ata_sc->sc_dev, "atapi", chan,
    145 		atapiprint);
    146 }
    147 
    148 static void
    149 wdc_atapi_minphys(struct buf *bp)
    150 {
    151 
    152 	if (bp->b_bcount > MAX_SIZE)
    153 		bp->b_bcount = MAX_SIZE;
    154 	minphys(bp);
    155 }
    156 
    157 /*
    158  * Kill off all pending xfers for a periph.
    159  *
    160  * Must be called with adapter lock held
    161  */
    162 static void
    163 wdc_atapi_kill_pending(struct scsipi_periph *periph)
    164 {
    165 	struct atac_softc *atac =
    166 	    device_private(periph->periph_channel->chan_adapter->adapt_dev);
    167 	struct ata_channel *chp =
    168 	    atac->atac_channels[periph->periph_channel->chan_channel];
    169 
    170 	ata_kill_pending(&chp->ch_drive[periph->periph_target]);
    171 }
    172 
    173 static void
    174 wdc_atapi_kill_xfer(struct ata_channel *chp, struct ata_xfer *xfer, int reason)
    175 {
    176 	struct scsipi_xfer *sc_xfer = xfer->c_scsipi;
    177 	bool deactivate = true;
    178 
    179 	/* remove this command from xfer queue */
    180 	switch (reason) {
    181 	case KILL_GONE_INACTIVE:
    182 		deactivate = false;
    183 		/* FALLTHROUGH */
    184 	case KILL_GONE:
    185 		sc_xfer->error = XS_DRIVER_STUFFUP;
    186 		break;
    187 	case KILL_RESET:
    188 		sc_xfer->error = XS_RESET;
    189 		break;
    190 	default:
    191 		printf("wdc_ata_bio_kill_xfer: unknown reason %d\n",
    192 		    reason);
    193 		panic("wdc_ata_bio_kill_xfer");
    194 	}
    195 
    196 	if (deactivate)
    197 		ata_deactivate_xfer(chp, xfer);
    198 
    199 	ata_free_xfer(chp, xfer);
    200 	scsipi_done(sc_xfer);
    201 }
    202 
    203 static int
    204 wdc_atapi_get_params(struct scsipi_channel *chan, int drive,
    205     struct ataparams *id)
    206 {
    207 	struct wdc_softc *wdc = device_private(chan->chan_adapter->adapt_dev);
    208 	struct atac_softc *atac = &wdc->sc_atac;
    209 	struct wdc_regs *wdr = &wdc->regs[chan->chan_channel];
    210 	struct ata_channel *chp = atac->atac_channels[chan->chan_channel];
    211 	struct ata_xfer *xfer;
    212 	int rv;
    213 
    214 	xfer = ata_get_xfer(chp, false);
    215 	if (xfer == NULL) {
    216 		printf("wdc_atapi_get_params: no xfer\n");
    217 		return EBUSY;
    218 	}
    219 
    220 	xfer->c_ata_c.r_command = ATAPI_SOFT_RESET;
    221 	xfer->c_ata_c.r_st_bmask = 0;
    222 	xfer->c_ata_c.r_st_pmask = 0;
    223 	xfer->c_ata_c.flags = AT_WAIT | AT_POLL;
    224 	xfer->c_ata_c.timeout = WDC_RESET_WAIT;
    225 	if (wdc_exec_command(&chp->ch_drive[drive], xfer) != ATACMD_COMPLETE) {
    226 		printf("wdc_atapi_get_params: ATAPI_SOFT_RESET failed for"
    227 		    " drive %s:%d:%d: driver failed\n",
    228 		    device_xname(atac->atac_dev), chp->ch_channel, drive);
    229 		panic("wdc_atapi_get_params");
    230 	}
    231 	if (xfer->c_ata_c.flags & (AT_ERROR | AT_TIMEOU | AT_DF)) {
    232 		ATADEBUG_PRINT(("wdc_atapi_get_params: ATAPI_SOFT_RESET "
    233 		    "failed for drive %s:%d:%d: error 0x%x\n",
    234 		    device_xname(atac->atac_dev), chp->ch_channel, drive,
    235 		    xfer->c_ata_c.r_error), DEBUG_PROBE);
    236 		rv = -1;
    237 		goto out_xfer;
    238 	}
    239 	chp->ch_drive[drive].state = 0;
    240 
    241 	ata_free_xfer(chp, xfer);
    242 
    243 	(void)bus_space_read_1(wdr->cmd_iot, wdr->cmd_iohs[wd_status], 0);
    244 
    245 	/* Some ATAPI devices need a bit more time after software reset. */
    246 	delay(5000);
    247 	if (ata_get_params(&chp->ch_drive[drive], AT_WAIT, id) != 0) {
    248 		ATADEBUG_PRINT(("wdc_atapi_get_params: ATAPI_IDENTIFY_DEVICE "
    249 		    "failed for drive %s:%d:%d\n",
    250 		    device_xname(atac->atac_dev), chp->ch_channel, drive),
    251 		    DEBUG_PROBE);
    252 		rv = -1;
    253 		goto out;
    254 	}
    255 	rv = 0;
    256 out:
    257 	return rv;
    258 
    259 out_xfer:
    260 	ata_free_xfer(chp, xfer);
    261 	return rv;
    262 }
    263 
    264 static void
    265 wdc_atapi_probe_device(struct atapibus_softc *sc, int target)
    266 {
    267 	struct scsipi_channel *chan = sc->sc_channel;
    268 	struct scsipi_periph *periph;
    269 	struct ataparams ids;
    270 	struct ataparams *id = &ids;
    271 	struct wdc_softc *wdc = device_private(chan->chan_adapter->adapt_dev);
    272 	struct atac_softc *atac = &wdc->sc_atac;
    273 	struct ata_channel *chp = atac->atac_channels[chan->chan_channel];
    274 	struct ata_drive_datas *drvp = &chp->ch_drive[target];
    275 	struct scsipibus_attach_args sa;
    276 	char serial_number[21], model[41], firmware_revision[9];
    277 	int s;
    278 
    279 	/* skip if already attached */
    280 	if (scsipi_lookup_periph(chan, target, 0) != NULL)
    281 		return;
    282 
    283 	/* if no ATAPI device detected at wdc attach time, skip */
    284 	if (drvp->drive_type != ATA_DRIVET_ATAPI) {
    285 		ATADEBUG_PRINT(("wdc_atapi_probe_device: "
    286 		    "drive %d not present\n", target), DEBUG_PROBE);
    287 		return;
    288 	}
    289 
    290 	if (wdc_atapi_get_params(chan, target, id) == 0) {
    291 #ifdef ATAPI_DEBUG_PROBE
    292 		printf("%s drive %d: cmdsz 0x%x drqtype 0x%x\n",
    293 		    device_xname(sc->sc_dev), target,
    294 		    id->atap_config & ATAPI_CFG_CMD_MASK,
    295 		    id->atap_config & ATAPI_CFG_DRQ_MASK);
    296 #endif
    297 		periph = scsipi_alloc_periph(M_NOWAIT);
    298 		if (periph == NULL) {
    299 			aprint_error_dev(sc->sc_dev,
    300 			    "unable to allocate periph for drive %d\n",
    301 			    target);
    302 			return;
    303 		}
    304 		periph->periph_dev = NULL;
    305 		periph->periph_channel = chan;
    306 		periph->periph_switch = &atapi_probe_periphsw;
    307 		periph->periph_target = target;
    308 		periph->periph_lun = 0;
    309 		periph->periph_quirks = PQUIRK_ONLYBIG;
    310 
    311 #ifdef SCSIPI_DEBUG
    312 		if (SCSIPI_DEBUG_TYPE == SCSIPI_BUSTYPE_ATAPI &&
    313 		    SCSIPI_DEBUG_TARGET == target)
    314 			periph->periph_dbflags |= SCSIPI_DEBUG_FLAGS;
    315 #endif
    316 		periph->periph_type = ATAPI_CFG_TYPE(id->atap_config);
    317 		if (id->atap_config & ATAPI_CFG_REMOV)
    318 			periph->periph_flags |= PERIPH_REMOVABLE;
    319 		if (periph->periph_type == T_SEQUENTIAL) {
    320 			s = splbio();
    321 			drvp->drive_flags |= ATA_DRIVE_ATAPIDSCW;
    322 			splx(s);
    323 		}
    324 
    325 		sa.sa_periph = periph;
    326 		sa.sa_inqbuf.type =  ATAPI_CFG_TYPE(id->atap_config);
    327 		sa.sa_inqbuf.removable = id->atap_config & ATAPI_CFG_REMOV ?
    328 		    T_REMOV : T_FIXED;
    329 		strnvisx(model, sizeof(model), id->atap_model,
    330 		    sizeof(id->atap_model), VIS_TRIM|VIS_SAFE|VIS_OCTAL);
    331 		strnvisx(serial_number, sizeof(serial_number),
    332 		    id->atap_serial, sizeof(id->atap_serial),
    333 		    VIS_TRIM|VIS_SAFE|VIS_OCTAL);
    334 		strnvisx(firmware_revision, sizeof(firmware_revision),
    335 		    id->atap_revision, sizeof(id->atap_revision),
    336 		    VIS_TRIM|VIS_SAFE|VIS_OCTAL);
    337 		sa.sa_inqbuf.vendor = model;
    338 		sa.sa_inqbuf.product = serial_number;
    339 		sa.sa_inqbuf.revision = firmware_revision;
    340 
    341 		/*
    342 		 * Determine the operating mode capabilities of the device.
    343 		 */
    344 		if ((id->atap_config & ATAPI_CFG_CMD_MASK) == ATAPI_CFG_CMD_16)
    345 			periph->periph_cap |= PERIPH_CAP_CMD16;
    346 		/* XXX This is gross. */
    347 		periph->periph_cap |= (id->atap_config & ATAPI_CFG_DRQ_MASK);
    348 
    349 		drvp->drv_softc = atapi_probe_device(sc, target, periph, &sa);
    350 
    351 		if (drvp->drv_softc)
    352 			ata_probe_caps(drvp);
    353 		else {
    354 			s = splbio();
    355 			drvp->drive_type = ATA_DRIVET_NONE;
    356 			splx(s);
    357 		}
    358 	} else {
    359 		s = splbio();
    360 		drvp->drive_type = ATA_DRIVET_NONE;
    361 		splx(s);
    362 	}
    363 }
    364 
    365 static const struct ata_xfer_ops wdc_atapi_xfer_ops = {
    366 	.c_start = wdc_atapi_start,
    367 	.c_intr = wdc_atapi_intr,
    368 	.c_poll = wdc_atapi_poll,
    369 	.c_abort = wdc_atapi_reset,
    370 	.c_kill_xfer = wdc_atapi_kill_xfer,
    371 };
    372 
    373 static void
    374 wdc_atapi_scsipi_request(struct scsipi_channel *chan, scsipi_adapter_req_t req,
    375     void *arg)
    376 {
    377 	struct scsipi_adapter *adapt = chan->chan_adapter;
    378 	struct scsipi_periph *periph;
    379 	struct scsipi_xfer *sc_xfer;
    380 	struct wdc_softc *wdc = device_private(adapt->adapt_dev);
    381 	struct atac_softc *atac = &wdc->sc_atac;
    382 	struct ata_xfer *xfer;
    383 	int channel = chan->chan_channel;
    384 	int drive, s;
    385 
    386 	switch (req) {
    387 	case ADAPTER_REQ_RUN_XFER:
    388 		sc_xfer = arg;
    389 		periph = sc_xfer->xs_periph;
    390 		drive = periph->periph_target;
    391 
    392 		ATADEBUG_PRINT(("wdc_atapi_scsipi_request %s:%d:%d\n",
    393 		    device_xname(atac->atac_dev), channel, drive),
    394 		    DEBUG_XFERS);
    395 		if (!device_is_active(atac->atac_dev)) {
    396 			sc_xfer->error = XS_DRIVER_STUFFUP;
    397 			scsipi_done(sc_xfer);
    398 			return;
    399 		}
    400 
    401 		xfer = ata_get_xfer(atac->atac_channels[channel], false);
    402 		if (xfer == NULL) {
    403 			sc_xfer->error = XS_RESOURCE_SHORTAGE;
    404 			scsipi_done(sc_xfer);
    405 			return;
    406 		}
    407 
    408 		if (sc_xfer->xs_control & XS_CTL_POLL)
    409 			xfer->c_flags |= C_POLL;
    410 #if NATA_DMA
    411 		if ((atac->atac_channels[channel]->ch_drive[drive].drive_flags &
    412 		    (ATA_DRIVE_DMA | ATA_DRIVE_UDMA)) && sc_xfer->datalen > 0)
    413 			xfer->c_flags |= C_DMA;
    414 #endif
    415 #if NATA_DMA && NATA_PIOBM
    416 		else
    417 #endif
    418 #if NATA_PIOBM
    419 		if ((atac->atac_cap & ATAC_CAP_PIOBM) &&
    420 		    sc_xfer->datalen > 0)
    421 			xfer->c_flags |= C_PIOBM;
    422 #endif
    423 		xfer->c_drive = drive;
    424 		xfer->c_flags |= C_ATAPI;
    425 #if NATA_DMA
    426 		if (sc_xfer->cmd->opcode == GPCMD_REPORT_KEY ||
    427 		    sc_xfer->cmd->opcode == GPCMD_SEND_KEY ||
    428 		    sc_xfer->cmd->opcode == GPCMD_READ_DVD_STRUCTURE) {
    429 			/*
    430 			 * DVD authentication commands must always be done in
    431 			 * PIO mode.
    432 			 */
    433 			xfer->c_flags &= ~C_DMA;
    434 		}
    435 
    436 		/*
    437 		 * DMA normally can't deal with transfers which are not a
    438 		 * multiple of its databus width. It's a bug to request odd
    439 		 * length transfers for ATAPI.
    440 		 *
    441 		 * Some devices also can't cope with unaligned DMA xfers
    442 		 * either. Also some devices seem to not handle DMA xfers of
    443 		 * less than 4 bytes.
    444 		 *
    445 		 * By enforcing at least 4 byte aligned offset and length for
    446 		 * DMA, we might use PIO where DMA could be allowed but better
    447 		 * safe than sorry as recent problems proved.
    448 		 *
    449 		 * Offending structures that are thus done by PIO instead of
    450 		 * DMA are normally small structures since all bulkdata is
    451 		 * aligned. But as the request may come from userland, we have
    452 		 * to protect against it anyway.
    453 		 *
    454 		 * XXX check for the 32 bit wide flag?
    455 		 */
    456 
    457 		if (((uintptr_t) sc_xfer->data) & 0x03)
    458 			xfer->c_flags &= ~C_DMA;
    459 		if ((sc_xfer->datalen < 4) || (sc_xfer->datalen & 0x03))
    460 			xfer->c_flags &= ~C_DMA;
    461 #endif	/* NATA_DMA */
    462 
    463 		xfer->c_databuf = sc_xfer->data;
    464 		xfer->c_bcount = sc_xfer->datalen;
    465 		xfer->ops = &wdc_atapi_xfer_ops;
    466 		xfer->c_scsipi = sc_xfer;
    467 		xfer->c_atapi.c_dscpoll = 0;
    468 		s = splbio();
    469 		ata_exec_xfer(atac->atac_channels[channel], xfer);
    470 #ifdef DIAGNOSTIC
    471 		if ((sc_xfer->xs_control & XS_CTL_POLL) != 0 &&
    472 		    (sc_xfer->xs_status & XS_STS_DONE) == 0)
    473 			panic("wdc_atapi_scsipi_request: polled command "
    474 			    "not done");
    475 #endif
    476 		splx(s);
    477 		return;
    478 
    479 	default:
    480 		/* Not supported, nothing to do. */
    481 		;
    482 	}
    483 }
    484 
    485 static int
    486 wdc_atapi_start(struct ata_channel *chp, struct ata_xfer *xfer)
    487 {
    488 	struct atac_softc *atac = chp->ch_atac;
    489 	struct wdc_softc *wdc = CHAN_TO_WDC(chp);
    490 	struct wdc_regs *wdr = &wdc->regs[chp->ch_channel];
    491 	struct scsipi_xfer *sc_xfer = xfer->c_scsipi;
    492 	struct ata_drive_datas *drvp = &chp->ch_drive[xfer->c_drive];
    493 	int wait_flags = (sc_xfer->xs_control & XS_CTL_POLL) ? AT_POLL : 0;
    494 	int tfd;
    495 	const char *errstring;
    496 
    497 	ATADEBUG_PRINT(("wdc_atapi_start %s:%d:%d, scsi flags 0x%x \n",
    498 	    device_xname(atac->atac_dev), chp->ch_channel, drvp->drive,
    499 	    sc_xfer->xs_control), DEBUG_XFERS);
    500 
    501 	ata_channel_lock_owned(chp);
    502 
    503 #if NATA_DMA
    504 	if ((xfer->c_flags & C_DMA) && (drvp->n_xfers <= NXFER))
    505 		drvp->n_xfers++;
    506 #endif
    507 	/* Do control operations specially. */
    508 	if (__predict_false(drvp->state < READY)) {
    509 		/* If it's not a polled command, we need the kernel thread */
    510 		if ((sc_xfer->xs_control & XS_CTL_POLL) == 0 &&
    511 		    (chp->ch_flags & ATACH_TH_RUN) == 0) {
    512 			return ATASTART_TH;
    513 		}
    514 		/*
    515 		 * disable interrupts, all commands here should be quick
    516 		 * enough to be able to poll, and we don't go here that often
    517 		 */
    518 		bus_space_write_1(wdr->ctl_iot, wdr->ctl_ioh, wd_aux_ctlr,
    519 		     WDCTL_4BIT | WDCTL_IDS);
    520 		if (wdc->select)
    521 			wdc->select(chp, xfer->c_drive);
    522 		bus_space_write_1(wdr->cmd_iot, wdr->cmd_iohs[wd_sdh], 0,
    523 		    WDSD_IBM | (xfer->c_drive << 4));
    524 		/* Don't try to set mode if controller can't be adjusted */
    525 		if (atac->atac_set_modes == NULL)
    526 			goto ready;
    527 		/* Also don't try if the drive didn't report its mode */
    528 		if ((drvp->drive_flags & ATA_DRIVE_MODE) == 0)
    529 			goto ready;
    530 		errstring = "unbusy";
    531 		if (wdc_wait_for_unbusy(chp, ATAPI_DELAY, wait_flags, &tfd))
    532 			goto timeout;
    533 		wdccommand(chp, drvp->drive, SET_FEATURES, 0, 0, 0,
    534 		    0x08 | drvp->PIO_mode, WDSF_SET_MODE);
    535 		errstring = "piomode";
    536 		if (wdc_wait_for_unbusy(chp, ATAPI_MODE_DELAY, wait_flags,
    537 		    &tfd))
    538 			goto timeout;
    539 		if (ATACH_ST(tfd) & WDCS_ERR) {
    540 			if (ATACH_ST(tfd) == WDCE_ABRT) {
    541 				/*
    542 				 * Some ATAPI drives reject PIO settings.
    543 				 * Fall back to PIO mode 3 since that's the
    544 				 * minimum for ATAPI.
    545 				 */
    546 				printf("%s:%d:%d: PIO mode %d rejected, "
    547 				    "falling back to PIO mode 3\n",
    548 				    device_xname(atac->atac_dev),
    549 				    chp->ch_channel, xfer->c_drive,
    550 				    drvp->PIO_mode);
    551 				if (drvp->PIO_mode > 3)
    552 					drvp->PIO_mode = 3;
    553 			} else
    554 				goto error;
    555 		}
    556 #if NATA_DMA
    557 #if NATA_UDMA
    558 		if (drvp->drive_flags & ATA_DRIVE_UDMA) {
    559 			wdccommand(chp, drvp->drive, SET_FEATURES, 0, 0, 0,
    560 			    0x40 | drvp->UDMA_mode, WDSF_SET_MODE);
    561 		} else
    562 #endif
    563 		if (drvp->drive_flags & ATA_DRIVE_DMA) {
    564 			wdccommand(chp, drvp->drive, SET_FEATURES, 0, 0, 0,
    565 			    0x20 | drvp->DMA_mode, WDSF_SET_MODE);
    566 		} else {
    567 			goto ready;
    568 		}
    569 		errstring = "dmamode";
    570 		if (wdc_wait_for_unbusy(chp, ATAPI_MODE_DELAY, wait_flags,
    571 		    &tfd))
    572 			goto timeout;
    573 		if (ATACH_ST(tfd) & WDCS_ERR) {
    574 			if (ATACH_ERR(tfd) == WDCE_ABRT) {
    575 #if NATA_UDMA
    576 				if (drvp->drive_flags & ATA_DRIVE_UDMA)
    577 					goto error;
    578 				else
    579 #endif
    580 				{
    581 					/*
    582 					 * The drive rejected our DMA setting.
    583 					 * Fall back to mode 1.
    584 					 */
    585 					printf("%s:%d:%d: DMA mode %d rejected, "
    586 					    "falling back to DMA mode 0\n",
    587 					    device_xname(atac->atac_dev),
    588 					    chp->ch_channel, xfer->c_drive,
    589 					    drvp->DMA_mode);
    590 					if (drvp->DMA_mode > 0)
    591 						drvp->DMA_mode = 0;
    592 				}
    593 			} else
    594 				goto error;
    595 		}
    596 #endif	/* NATA_DMA */
    597 ready:
    598 		drvp->state = READY;
    599 		bus_space_write_1(wdr->ctl_iot, wdr->ctl_ioh, wd_aux_ctlr,
    600 		    WDCTL_4BIT);
    601 		delay(10); /* some drives need a little delay here */
    602 	}
    603 	/* start timeout machinery */
    604 	if ((sc_xfer->xs_control & XS_CTL_POLL) == 0)
    605 		callout_reset(&chp->c_timo_callout, mstohz(sc_xfer->timeout),
    606 		    wdctimeout, chp);
    607 
    608 	if (wdc->select)
    609 		wdc->select(chp, xfer->c_drive);
    610 	bus_space_write_1(wdr->cmd_iot, wdr->cmd_iohs[wd_sdh], 0,
    611 	    WDSD_IBM | (xfer->c_drive << 4));
    612 	switch (wdc_wait_for_unbusy(chp, ATAPI_DELAY, wait_flags, &tfd)) {
    613 	case WDCWAIT_OK:
    614 		break;
    615 	case WDCWAIT_TOUT:
    616 		printf("wdc_atapi_start: not ready, st = %02x\n",
    617 		    ATACH_ST(tfd));
    618 		sc_xfer->error = XS_TIMEOUT;
    619 		return ATASTART_ABORT;
    620 	case WDCWAIT_THR:
    621 		return ATASTART_TH;
    622 	}
    623 
    624 	/*
    625 	 * Even with WDCS_ERR, the device should accept a command packet
    626 	 * Limit length to what can be stuffed into the cylinder register
    627 	 * (16 bits).  Some CD-ROMs seem to interpret '0' as 65536,
    628 	 * but not all devices do that and it's not obvious from the
    629 	 * ATAPI spec that that behaviour should be expected.  If more
    630 	 * data is necessary, multiple data transfer phases will be done.
    631 	 */
    632 
    633 	wdccommand(chp, xfer->c_drive, ATAPI_PKT_CMD,
    634 	    xfer->c_bcount <= 0xffff ? xfer->c_bcount : 0xffff,
    635 	    0, 0, 0,
    636 #if NATA_DMA
    637 	    (xfer->c_flags & C_DMA) ? ATAPI_PKT_CMD_FTRE_DMA :
    638 #endif
    639 	    0
    640 	    );
    641 
    642 #if NATA_PIOBM
    643 	if (xfer->c_flags & C_PIOBM) {
    644 		int error;
    645 		int dma_flags = (sc_xfer->xs_control & XS_CTL_DATA_IN)
    646 		    ?  WDC_DMA_READ : 0;
    647 		if (xfer->c_flags & C_POLL) {
    648 			/* XXX not supported yet --- fall back to PIO */
    649 			xfer->c_flags &= ~C_PIOBM;
    650 		} else {
    651 			/* Init the DMA channel. */
    652 			error = (*wdc->dma_init)(wdc->dma_arg,
    653 			    chp->ch_channel, xfer->c_drive,
    654 			    (char *)xfer->c_databuf,
    655 			    xfer->c_bcount,
    656 			    dma_flags | WDC_DMA_PIOBM_ATAPI);
    657 			if (error) {
    658 				if (error == EINVAL) {
    659 					/*
    660 					 * We can't do DMA on this transfer
    661 					 * for some reason.  Fall back to
    662 					 * PIO.
    663 					 */
    664 					xfer->c_flags &= ~C_PIOBM;
    665 					error = 0;
    666 				} else {
    667 					sc_xfer->error = XS_DRIVER_STUFFUP;
    668 					errstring = "piobm";
    669 					goto error;
    670 				}
    671 			}
    672 		}
    673 	}
    674 #endif
    675 	/*
    676 	 * If there is no interrupt for CMD input, busy-wait for it (done in
    677 	 * the interrupt routine. Poll routine will exit early in this case.
    678 	 */
    679 	if ((sc_xfer->xs_periph->periph_cap & ATAPI_CFG_DRQ_MASK) !=
    680 	    ATAPI_CFG_IRQ_DRQ || (sc_xfer->xs_control & XS_CTL_POLL))
    681 		return ATASTART_POLL;
    682 	else {
    683 		chp->ch_flags |= ATACH_IRQ_WAIT;
    684 		return ATASTART_STARTED;
    685 	}
    686 
    687 timeout:
    688 	printf("%s:%d:%d: %s timed out\n",
    689 	    device_xname(atac->atac_dev), chp->ch_channel, xfer->c_drive,
    690 	    errstring);
    691 	sc_xfer->error = XS_TIMEOUT;
    692 	bus_space_write_1(wdr->ctl_iot, wdr->ctl_ioh, wd_aux_ctlr, WDCTL_4BIT);
    693 	delay(10); /* some drives need a little delay here */
    694 	return ATASTART_ABORT;
    695 
    696 error:
    697 	printf("%s:%d:%d: %s ",
    698 	    device_xname(atac->atac_dev), chp->ch_channel, xfer->c_drive,
    699 	    errstring);
    700 	printf("error (0x%x)\n", ATACH_ERR(tfd));
    701 	sc_xfer->error = XS_SHORTSENSE;
    702 	sc_xfer->sense.atapi_sense = ATACH_ERR(tfd);
    703 	bus_space_write_1(wdr->ctl_iot, wdr->ctl_ioh, wd_aux_ctlr, WDCTL_4BIT);
    704 	delay(10); /* some drives need a little delay here */
    705 	return ATASTART_ABORT;
    706 }
    707 
    708 static void
    709 wdc_atapi_poll(struct ata_channel *chp, struct ata_xfer *xfer)
    710 {
    711 	/*
    712 	 * If there is no interrupt for CMD input, busy-wait for it (done in
    713 	 * the interrupt routine. If it is a polled command, call the interrupt
    714 	 * routine until command is done.
    715 	 */
    716 	const bool poll = ((xfer->c_scsipi->xs_control & XS_CTL_POLL) != 0);
    717 
    718 	/* Wait for at last 400ns for status bit to be valid */
    719 	DELAY(1);
    720 	wdc_atapi_intr(chp, xfer, 0);
    721 
    722 	if (!poll)
    723 		return;
    724 
    725 #if NATA_DMA
    726 	if (chp->ch_flags & ATACH_DMA_WAIT) {
    727 		wdc_dmawait(chp, xfer, xfer->c_scsipi->timeout);
    728 		chp->ch_flags &= ~ATACH_DMA_WAIT;
    729 	}
    730 #endif
    731 	while ((xfer->c_scsipi->xs_status & XS_STS_DONE) == 0) {
    732 		/* Wait for at last 400ns for status bit to be valid */
    733 		DELAY(1);
    734 		wdc_atapi_intr(chp, xfer, 0);
    735 	}
    736 }
    737 
    738 static int
    739 wdc_atapi_intr(struct ata_channel *chp, struct ata_xfer *xfer, int irq)
    740 {
    741 	struct atac_softc *atac = chp->ch_atac;
    742 	struct wdc_softc *wdc = CHAN_TO_WDC(chp);
    743 	struct wdc_regs *wdr = &wdc->regs[chp->ch_channel];
    744 	struct scsipi_xfer *sc_xfer = xfer->c_scsipi;
    745 	struct ata_drive_datas *drvp = &chp->ch_drive[xfer->c_drive];
    746 	int len, phase, i, retries=0;
    747 	int ire, tfd;
    748 #if NATA_DMA
    749 	int error;
    750 #endif
    751 #if NATA_DMA || NATA_PIOBM
    752 	int dma_flags = 0;
    753 #endif
    754 	void *cmd;
    755 
    756 	ATADEBUG_PRINT(("wdc_atapi_intr %s:%d:%d\n",
    757 	    device_xname(atac->atac_dev), chp->ch_channel, drvp->drive),
    758 	    DEBUG_INTR);
    759 
    760 	ata_channel_lock(chp);
    761 
    762 	/* Is it not a transfer, but a control operation? */
    763 	if (drvp->state < READY) {
    764 		printf("%s:%d:%d: bad state %d in wdc_atapi_intr\n",
    765 		    device_xname(atac->atac_dev), chp->ch_channel,
    766 		    xfer->c_drive, drvp->state);
    767 		panic("wdc_atapi_intr: bad state");
    768 	}
    769 	/*
    770 	 * If we missed an interrupt in a PIO transfer, reset and restart.
    771 	 * Don't try to continue transfer, we may have missed cycles.
    772 	 */
    773 	if ((xfer->c_flags & (C_TIMEOU | C_DMA)) == C_TIMEOU) {
    774 		ata_channel_unlock(chp);
    775 		sc_xfer->error = XS_TIMEOUT;
    776 		wdc_atapi_reset(chp, xfer);
    777 		return 1;
    778 	}
    779 
    780 #if NATA_PIOBM
    781 	/* Transfer-done interrupt for busmastering PIO operation */
    782 	if ((xfer->c_flags & C_PIOBM) && (chp->ch_flags & ATACH_PIOBM_WAIT)) {
    783 		chp->ch_flags &= ~ATACH_PIOBM_WAIT;
    784 
    785 		/* restore transfer length */
    786 		len = xfer->c_bcount;
    787 		if (xfer->c_atapi.c_lenoff < 0)
    788 			len += xfer->c_atapi.c_lenoff;
    789 
    790 		if (sc_xfer->xs_control & XS_CTL_DATA_IN)
    791 			goto end_piobm_datain;
    792 		else
    793 			goto end_piobm_dataout;
    794 	}
    795 #endif
    796 
    797 	/* Ack interrupt done in wdc_wait_for_unbusy */
    798 	if (wdc->select)
    799 		wdc->select(chp, xfer->c_drive);
    800 	bus_space_write_1(wdr->cmd_iot, wdr->cmd_iohs[wd_sdh], 0,
    801 	    WDSD_IBM | (xfer->c_drive << 4));
    802 	if (wdc_wait_for_unbusy(chp,
    803 	    (irq == 0) ? sc_xfer->timeout : 0, AT_POLL, &tfd) == WDCWAIT_TOUT) {
    804 		if (irq && (xfer->c_flags & C_TIMEOU) == 0) {
    805 			ata_channel_unlock(chp);
    806 			return 0; /* IRQ was not for us */
    807 		}
    808 		printf("%s:%d:%d: device timeout, c_bcount=%d, c_skip=%d\n",
    809 		    device_xname(atac->atac_dev), chp->ch_channel,
    810 		    xfer->c_drive, xfer->c_bcount, xfer->c_skip);
    811 #if NATA_DMA
    812 		if (xfer->c_flags & C_DMA) {
    813 			ata_dmaerr(drvp,
    814 			    (xfer->c_flags & C_POLL) ? AT_POLL : 0);
    815 		}
    816 #endif
    817 		sc_xfer->error = XS_TIMEOUT;
    818 		ata_channel_unlock(chp);
    819 		wdc_atapi_reset(chp, xfer);
    820 		return 1;
    821 	}
    822 	if (wdc->irqack)
    823 		wdc->irqack(chp);
    824 
    825 #if NATA_DMA
    826 	/*
    827 	 * If we missed an IRQ and were using DMA, flag it as a DMA error
    828 	 * and reset device.
    829 	 */
    830 	if ((xfer->c_flags & C_TIMEOU) && (xfer->c_flags & C_DMA)) {
    831 		ata_dmaerr(drvp, (xfer->c_flags & C_POLL) ? AT_POLL : 0);
    832 		sc_xfer->error = XS_RESET;
    833 		ata_channel_unlock(chp);
    834 		wdc_atapi_reset(chp, xfer);
    835 		return (1);
    836 	}
    837 #endif
    838 	/*
    839 	 * if the request sense command was aborted, report the short sense
    840 	 * previously recorded, else continue normal processing
    841 	 */
    842 
    843 #if NATA_DMA || NATA_PIOBM
    844 	if (xfer->c_flags & (C_DMA | C_PIOBM))
    845 		dma_flags = (sc_xfer->xs_control & XS_CTL_DATA_IN)
    846 		    ?  WDC_DMA_READ : 0;
    847 #endif
    848 again:
    849 	len = bus_space_read_1(wdr->cmd_iot, wdr->cmd_iohs[wd_cyl_lo], 0) +
    850 	    256 * bus_space_read_1(wdr->cmd_iot, wdr->cmd_iohs[wd_cyl_hi], 0);
    851 	ire = bus_space_read_1(wdr->cmd_iot, wdr->cmd_iohs[wd_ireason], 0);
    852 	phase = (ire & (WDCI_CMD | WDCI_IN)) | (ATACH_ST(tfd) & WDCS_DRQ);
    853 	ATADEBUG_PRINT(("wdc_atapi_intr: c_bcount %d len %d st 0x%x err 0x%x "
    854 	    "ire 0x%x :", xfer->c_bcount,
    855 	    len, ATACH_ST(tfd), ATACH_ERR(tfd), ire), DEBUG_INTR);
    856 
    857 	switch (phase) {
    858 	case PHASE_CMDOUT:
    859 		cmd = sc_xfer->cmd;
    860 		ATADEBUG_PRINT(("PHASE_CMDOUT\n"), DEBUG_INTR);
    861 #if NATA_DMA
    862 		/* Init the DMA channel if necessary */
    863 		if (xfer->c_flags & C_DMA) {
    864 			error = (*wdc->dma_init)(wdc->dma_arg,
    865 			    chp->ch_channel, xfer->c_drive,
    866 			    xfer->c_databuf, xfer->c_bcount, dma_flags);
    867 			if (error) {
    868 				if (error == EINVAL) {
    869 					/*
    870 					 * We can't do DMA on this transfer
    871 					 * for some reason.  Fall back to
    872 					 * PIO.
    873 					 */
    874 					xfer->c_flags &= ~C_DMA;
    875 					error = 0;
    876 				} else {
    877 					sc_xfer->error = XS_DRIVER_STUFFUP;
    878 					break;
    879 				}
    880 			}
    881 		}
    882 #endif
    883 
    884 		/* send packet command */
    885 		/* Commands are 12 or 16 bytes long. It's 32-bit aligned */
    886 		wdc->dataout_pio(chp, drvp->drive_flags, cmd, sc_xfer->cmdlen);
    887 
    888 #if NATA_DMA
    889 		/* Start the DMA channel if necessary */
    890 		if (xfer->c_flags & C_DMA) {
    891 			(*wdc->dma_start)(wdc->dma_arg,
    892 			    chp->ch_channel, xfer->c_drive);
    893 			chp->ch_flags |= ATACH_DMA_WAIT;
    894 		}
    895 #endif
    896 
    897 		if ((sc_xfer->xs_control & XS_CTL_POLL) == 0) {
    898 			chp->ch_flags |= ATACH_IRQ_WAIT;
    899 		}
    900 
    901 		ata_channel_unlock(chp);
    902 		return 1;
    903 
    904 	 case PHASE_DATAOUT:
    905 		/* write data */
    906 		ATADEBUG_PRINT(("PHASE_DATAOUT\n"), DEBUG_INTR);
    907 #if NATA_DMA
    908 		if ((sc_xfer->xs_control & XS_CTL_DATA_OUT) == 0 ||
    909 		    (xfer->c_flags & C_DMA) != 0) {
    910 			printf("wdc_atapi_intr: bad data phase DATAOUT\n");
    911 			if (xfer->c_flags & C_DMA) {
    912 				ata_dmaerr(drvp,
    913 				    (xfer->c_flags & C_POLL) ? AT_POLL : 0);
    914 			}
    915 			sc_xfer->error = XS_TIMEOUT;
    916 			ata_channel_unlock(chp);
    917 			wdc_atapi_reset(chp, xfer);
    918 			return 1;
    919 		}
    920 #endif
    921 		xfer->c_atapi.c_lenoff = len - xfer->c_bcount;
    922 		if (xfer->c_bcount < len) {
    923 			printf("wdc_atapi_intr: warning: write only "
    924 			    "%d of %d requested bytes\n", xfer->c_bcount, len);
    925 			len = xfer->c_bcount;
    926 		}
    927 
    928 #if NATA_PIOBM
    929 		if (xfer->c_flags & C_PIOBM) {
    930 			/* start the busmastering PIO */
    931 			(*wdc->piobm_start)(wdc->dma_arg,
    932 			    chp->ch_channel, xfer->c_drive,
    933 			    xfer->c_skip, len, WDC_PIOBM_XFER_IRQ);
    934 			chp->ch_flags |= ATACH_DMA_WAIT | ATACH_IRQ_WAIT |
    935 			    ATACH_PIOBM_WAIT;
    936 			ata_channel_unlock(chp);
    937 			return 1;
    938 		}
    939 #endif
    940 		wdc->dataout_pio(chp, drvp->drive_flags,
    941 		    (char *)xfer->c_databuf + xfer->c_skip, len);
    942 
    943 #if NATA_PIOBM
    944 	end_piobm_dataout:
    945 #endif
    946 		for (i = xfer->c_atapi.c_lenoff; i > 0; i -= 2)
    947 			bus_space_write_2(wdr->cmd_iot,
    948 			    wdr->cmd_iohs[wd_data], 0, 0);
    949 
    950 		xfer->c_skip += len;
    951 		xfer->c_bcount -= len;
    952 		if ((sc_xfer->xs_control & XS_CTL_POLL) == 0) {
    953 			chp->ch_flags |= ATACH_IRQ_WAIT;
    954 		}
    955 		ata_channel_unlock(chp);
    956 		return 1;
    957 
    958 	case PHASE_DATAIN:
    959 		/* Read data */
    960 		ATADEBUG_PRINT(("PHASE_DATAIN\n"), DEBUG_INTR);
    961 #if NATA_DMA
    962 		if ((sc_xfer->xs_control & XS_CTL_DATA_IN) == 0 ||
    963 		    (xfer->c_flags & C_DMA) != 0) {
    964 			printf("wdc_atapi_intr: bad data phase DATAIN\n");
    965 			if (xfer->c_flags & C_DMA) {
    966 				ata_dmaerr(drvp,
    967 				    (xfer->c_flags & C_POLL) ? AT_POLL : 0);
    968 			}
    969 			sc_xfer->error = XS_TIMEOUT;
    970 			ata_channel_unlock(chp);
    971 			wdc_atapi_reset(chp, xfer);
    972 			return 1;
    973 		}
    974 #endif
    975 		xfer->c_atapi.c_lenoff = len - xfer->c_bcount;
    976 		if (xfer->c_bcount < len) {
    977 			printf("wdc_atapi_intr: warning: reading only "
    978 			    "%d of %d bytes\n", xfer->c_bcount, len);
    979 			len = xfer->c_bcount;
    980 		}
    981 
    982 #if NATA_PIOBM
    983 		if (xfer->c_flags & C_PIOBM) {
    984 			/* start the busmastering PIO */
    985 			(*wdc->piobm_start)(wdc->dma_arg,
    986 			    chp->ch_channel, xfer->c_drive,
    987 			    xfer->c_skip, len, WDC_PIOBM_XFER_IRQ);
    988 			chp->ch_flags |= ATACH_DMA_WAIT | ATACH_IRQ_WAIT |
    989 			    ATACH_PIOBM_WAIT;
    990 			ata_channel_unlock(chp);
    991 			return 1;
    992 		}
    993 #endif
    994 		wdc->datain_pio(chp, drvp->drive_flags,
    995 		    (char *)xfer->c_databuf + xfer->c_skip, len);
    996 
    997 #if NATA_PIOBM
    998 	end_piobm_datain:
    999 #endif
   1000 		if (xfer->c_atapi.c_lenoff > 0)
   1001 			wdcbit_bucket(chp, xfer->c_atapi.c_lenoff);
   1002 
   1003 		xfer->c_skip += len;
   1004 		xfer->c_bcount -= len;
   1005 		if ((sc_xfer->xs_control & XS_CTL_POLL) == 0) {
   1006 			chp->ch_flags |= ATACH_IRQ_WAIT;
   1007 		}
   1008 		ata_channel_unlock(chp);
   1009 		return 1;
   1010 
   1011 	case PHASE_ABORTED:
   1012 	case PHASE_COMPLETED:
   1013 		ATADEBUG_PRINT(("PHASE_COMPLETED\n"), DEBUG_INTR);
   1014 #if NATA_DMA
   1015 		if (xfer->c_flags & C_DMA) {
   1016 			xfer->c_bcount -= sc_xfer->datalen;
   1017 		}
   1018 #endif
   1019 		sc_xfer->resid = xfer->c_bcount;
   1020 		/* this will unlock channel lock too */
   1021 		wdc_atapi_phase_complete(xfer, tfd);
   1022 		return(1);
   1023 
   1024 	default:
   1025 		if (++retries<500) {
   1026 			DELAY(100);
   1027 			tfd = ATACH_ERR_ST(
   1028 			    bus_space_read_1(wdr->cmd_iot,
   1029 				wdr->cmd_iohs[wd_error], 0),
   1030 			    bus_space_read_1(wdr->cmd_iot,
   1031 				wdr->cmd_iohs[wd_status], 0)
   1032 			);
   1033 			goto again;
   1034 		}
   1035 		printf("wdc_atapi_intr: unknown phase 0x%x\n", phase);
   1036 		if (ATACH_ST(tfd) & WDCS_ERR) {
   1037 			sc_xfer->error = XS_SHORTSENSE;
   1038 			sc_xfer->sense.atapi_sense = ATACH_ERR(tfd);
   1039 		} else {
   1040 #if NATA_DMA
   1041 			if (xfer->c_flags & C_DMA) {
   1042 				ata_dmaerr(drvp,
   1043 				    (xfer->c_flags & C_POLL) ? AT_POLL : 0);
   1044 			}
   1045 #endif
   1046 			sc_xfer->error = XS_RESET;
   1047 			ata_channel_unlock(chp);
   1048 			wdc_atapi_reset(chp, xfer);
   1049 			return (1);
   1050 		}
   1051 	}
   1052 	ATADEBUG_PRINT(("wdc_atapi_intr: wdc_atapi_done() (end), error 0x%x "
   1053 	    "sense 0x%x\n", sc_xfer->error, sc_xfer->sense.atapi_sense),
   1054 	    DEBUG_INTR);
   1055 	ata_channel_unlock(chp);
   1056 	wdc_atapi_done(chp, xfer);
   1057 	return (1);
   1058 }
   1059 
   1060 static void
   1061 wdc_atapi_phase_complete(struct ata_xfer *xfer, int tfd)
   1062 {
   1063 	struct ata_channel *chp = xfer->c_chp;
   1064 	struct atac_softc *atac = chp->ch_atac;
   1065 #if NATA_DMA || NATA_PIOBM
   1066 	struct wdc_softc *wdc = CHAN_TO_WDC(chp);
   1067 #endif
   1068 	struct scsipi_xfer *sc_xfer = xfer->c_scsipi;
   1069 	struct ata_drive_datas *drvp = &chp->ch_drive[xfer->c_drive];
   1070 
   1071 	ata_channel_lock_owned(chp);
   1072 
   1073 	/* wait for DSC if needed */
   1074 	if (drvp->drive_flags & ATA_DRIVE_ATAPIDSCW) {
   1075 		ATADEBUG_PRINT(("wdc_atapi_phase_complete(%s:%d:%d) "
   1076 		    "polldsc %d\n", device_xname(atac->atac_dev),
   1077 		    chp->ch_channel,
   1078 		    xfer->c_drive, xfer->c_atapi.c_dscpoll), DEBUG_XFERS);
   1079 #if 1
   1080 		if (cold)
   1081 			panic("wdc_atapi_phase_complete: cold");
   1082 #endif
   1083 		if (wdcwait(chp, WDCS_DSC, WDCS_DSC, 10,
   1084 		    AT_POLL, &tfd) == WDCWAIT_TOUT) {
   1085 			/* 10ms not enough, try again in 1 tick */
   1086 			if (xfer->c_atapi.c_dscpoll++ >
   1087 			    mstohz(sc_xfer->timeout)) {
   1088 				printf("%s:%d:%d: wait_for_dsc "
   1089 				    "failed\n",
   1090 				    device_xname(atac->atac_dev),
   1091 				    chp->ch_channel, xfer->c_drive);
   1092 				ata_channel_unlock(chp);
   1093 				sc_xfer->error = XS_TIMEOUT;
   1094 				wdc_atapi_reset(chp, xfer);
   1095 			} else {
   1096 				callout_reset(&chp->c_timo_callout, 1,
   1097 				    wdc_atapi_polldsc, chp);
   1098 				ata_channel_unlock(chp);
   1099 			}
   1100 			return;
   1101 		}
   1102 	}
   1103 
   1104 	/*
   1105 	 * Some drive occasionally set WDCS_ERR with
   1106 	 * "ATA illegal length indication" in the error
   1107 	 * register. If we read some data the sense is valid
   1108 	 * anyway, so don't report the error.
   1109 	 */
   1110 	if (ATACH_ST(tfd) & WDCS_ERR &&
   1111 	    ((sc_xfer->xs_control & XS_CTL_REQSENSE) == 0 ||
   1112 	    sc_xfer->resid == sc_xfer->datalen)) {
   1113 		/* save the short sense */
   1114 		sc_xfer->error = XS_SHORTSENSE;
   1115 		sc_xfer->sense.atapi_sense = ATACH_ERR(tfd);
   1116 		if ((sc_xfer->xs_periph->periph_quirks &
   1117 		    PQUIRK_NOSENSE) == 0) {
   1118 			/* ask scsipi to send a REQUEST_SENSE */
   1119 			sc_xfer->error = XS_BUSY;
   1120 			sc_xfer->status = SCSI_CHECK;
   1121 		}
   1122 #if NATA_DMA || NATA_PIOBM
   1123 		else if (wdc->dma_status &
   1124 		    (WDC_DMAST_NOIRQ | WDC_DMAST_ERR)) {
   1125 #if NATA_DMA
   1126 			ata_dmaerr(drvp,
   1127 			    (xfer->c_flags & C_POLL) ? AT_POLL : 0);
   1128 #endif
   1129 			sc_xfer->error = XS_RESET;
   1130 			ata_channel_unlock(chp);
   1131 			wdc_atapi_reset(chp, xfer);
   1132 			return;
   1133 		}
   1134 #endif
   1135 	}
   1136 	if (xfer->c_bcount != 0) {
   1137 		ATADEBUG_PRINT(("wdc_atapi_intr: bcount value is "
   1138 		    "%d after io\n", xfer->c_bcount), DEBUG_XFERS);
   1139 	}
   1140 #ifdef DIAGNOSTIC
   1141 	if (xfer->c_bcount < 0) {
   1142 		printf("wdc_atapi_intr warning: bcount value "
   1143 		    "is %d after io\n", xfer->c_bcount);
   1144 	}
   1145 #endif
   1146 	ATADEBUG_PRINT(("wdc_atapi_phase_complete: wdc_atapi_done(), "
   1147 	    "error 0x%x sense 0x%x\n", sc_xfer->error,
   1148 	    sc_xfer->sense.atapi_sense), DEBUG_INTR);
   1149 	ata_channel_unlock(chp);
   1150 	wdc_atapi_done(chp, xfer);
   1151 }
   1152 
   1153 static void
   1154 wdc_atapi_done(struct ata_channel *chp, struct ata_xfer *xfer)
   1155 {
   1156 	struct atac_softc *atac = chp->ch_atac;
   1157 	struct scsipi_xfer *sc_xfer = xfer->c_scsipi;
   1158 
   1159 	ATADEBUG_PRINT(("wdc_atapi_done %s:%d:%d: flags 0x%x\n",
   1160 	    device_xname(atac->atac_dev), chp->ch_channel, xfer->c_drive,
   1161 	    (u_int)xfer->c_flags), DEBUG_XFERS);
   1162 
   1163 	if (ata_waitdrain_xfer_check(chp, xfer))
   1164 		return;
   1165 
   1166 	ata_deactivate_xfer(chp, xfer);
   1167 	ata_free_xfer(chp, xfer);
   1168 
   1169 	ATADEBUG_PRINT(("wdc_atapi_done: scsipi_done\n"), DEBUG_XFERS);
   1170 	scsipi_done(sc_xfer);
   1171 	ATADEBUG_PRINT(("atastart from wdc_atapi_done, flags 0x%x\n",
   1172 	    chp->ch_flags), DEBUG_XFERS);
   1173 	atastart(chp);
   1174 }
   1175 
   1176 static void
   1177 wdc_atapi_reset(struct ata_channel *chp, struct ata_xfer *xfer)
   1178 {
   1179 	struct atac_softc *atac = chp->ch_atac;
   1180 	struct ata_drive_datas *drvp = &chp->ch_drive[xfer->c_drive];
   1181 	struct scsipi_xfer *sc_xfer = xfer->c_scsipi;
   1182 	int tfd;
   1183 
   1184 	ata_channel_lock(chp);
   1185 	wdccommandshort(chp, xfer->c_drive, ATAPI_SOFT_RESET);
   1186 	drvp->state = 0;
   1187 	if (wdc_wait_for_unbusy(chp, WDC_RESET_WAIT, AT_POLL, &tfd) != 0) {
   1188 		printf("%s:%d:%d: reset failed\n",
   1189 		    device_xname(atac->atac_dev), chp->ch_channel,
   1190 		    xfer->c_drive);
   1191 		sc_xfer->error = XS_SELTIMEOUT;
   1192 	}
   1193 	ata_channel_unlock(chp);
   1194 	wdc_atapi_done(chp, xfer);
   1195 	return;
   1196 }
   1197 
   1198 static void
   1199 wdc_atapi_polldsc(void *arg)
   1200 {
   1201 	struct ata_channel *chp = arg;
   1202 	struct ata_xfer *xfer = ata_queue_get_active_xfer(chp);
   1203 
   1204 	KASSERT(xfer != NULL);
   1205 
   1206 	ata_channel_lock(chp);
   1207 
   1208 	/* this will unlock channel lock too */
   1209 	wdc_atapi_phase_complete(xfer, 0);
   1210 }
   1211