Home | History | Annotate | Line # | Download | only in scsipi
atapi_wdc.c revision 1.107
      1 /*	$NetBSD: atapi_wdc.c,v 1.107 2008/03/24 14:44:26 cube Exp $	*/
      2 
      3 /*
      4  * Copyright (c) 1998, 2001 Manuel Bouyer.
      5  *
      6  * Redistribution and use in source and binary forms, with or without
      7  * modification, are permitted provided that the following conditions
      8  * are met:
      9  * 1. Redistributions of source code must retain the above copyright
     10  *    notice, this list of conditions and the following disclaimer.
     11  * 2. Redistributions in binary form must reproduce the above copyright
     12  *    notice, this list of conditions and the following disclaimer in the
     13  *    documentation and/or other materials provided with the distribution.
     14  * 3. All advertising materials mentioning features or use of this software
     15  *    must display the following acknowledgement:
     16  *	This product includes software developed by Manuel Bouyer.
     17  * 4. The name of the author may not be used to endorse or promote products
     18  *    derived from this software without specific prior written permission.
     19  *
     20  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
     21  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
     22  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
     23  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
     24  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
     25  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
     26  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
     27  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
     28  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
     29  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     30  */
     31 
     32 #include <sys/cdefs.h>
     33 __KERNEL_RCSID(0, "$NetBSD: atapi_wdc.c,v 1.107 2008/03/24 14:44:26 cube Exp $");
     34 
     35 #ifndef ATADEBUG
     36 #define ATADEBUG
     37 #endif /* ATADEBUG */
     38 
     39 #include <sys/param.h>
     40 #include <sys/systm.h>
     41 #include <sys/kernel.h>
     42 #include <sys/file.h>
     43 #include <sys/stat.h>
     44 #include <sys/buf.h>
     45 #include <sys/malloc.h>
     46 #include <sys/device.h>
     47 #include <sys/syslog.h>
     48 #include <sys/proc.h>
     49 #include <sys/dvdio.h>
     50 
     51 #include <sys/intr.h>
     52 #include <sys/bus.h>
     53 
     54 #ifndef __BUS_SPACE_HAS_STREAM_METHODS
     55 #define	bus_space_write_multi_stream_2	bus_space_write_multi_2
     56 #define	bus_space_write_multi_stream_4	bus_space_write_multi_4
     57 #define	bus_space_read_multi_stream_2	bus_space_read_multi_2
     58 #define	bus_space_read_multi_stream_4	bus_space_read_multi_4
     59 #endif /* __BUS_SPACE_HAS_STREAM_METHODS */
     60 
     61 #include <dev/ata/ataconf.h>
     62 #include <dev/ata/atareg.h>
     63 #include <dev/ata/atavar.h>
     64 #include <dev/ic/wdcreg.h>
     65 #include <dev/ic/wdcvar.h>
     66 
     67 #include <dev/scsipi/scsi_all.h> /* for SCSI status */
     68 
     69 #define DEBUG_INTR   0x01
     70 #define DEBUG_XFERS  0x02
     71 #define DEBUG_STATUS 0x04
     72 #define DEBUG_FUNCS  0x08
     73 #define DEBUG_PROBE  0x10
     74 #ifdef ATADEBUG
     75 int wdcdebug_atapi_mask = 0;
     76 #define ATADEBUG_PRINT(args, level) \
     77 	if (wdcdebug_atapi_mask & (level)) \
     78 		printf args
     79 #else
     80 #define ATADEBUG_PRINT(args, level)
     81 #endif
     82 
     83 #define ATAPI_DELAY 10	/* 10 ms, this is used only before sending a cmd */
     84 #define ATAPI_MODE_DELAY 1000	/* 1s, timeout for SET_FEATYRE cmds */
     85 
     86 static int	wdc_atapi_get_params(struct scsipi_channel *, int,
     87 				     struct ataparams *);
     88 static void	wdc_atapi_probe_device(struct atapibus_softc *, int);
     89 static void	wdc_atapi_minphys (struct buf *bp);
     90 static void	wdc_atapi_start(struct ata_channel *,struct ata_xfer *);
     91 static int	wdc_atapi_intr(struct ata_channel *, struct ata_xfer *, int);
     92 static void	wdc_atapi_kill_xfer(struct ata_channel *,
     93 				    struct ata_xfer *, int);
     94 static void	wdc_atapi_phase_complete(struct ata_xfer *);
     95 static void	wdc_atapi_done(struct ata_channel *, struct ata_xfer *);
     96 static void	wdc_atapi_reset(struct ata_channel *, struct ata_xfer *);
     97 static void	wdc_atapi_scsipi_request(struct scsipi_channel *,
     98 					 scsipi_adapter_req_t, void *);
     99 static void	wdc_atapi_kill_pending(struct scsipi_periph *);
    100 static void	wdc_atapi_polldsc(void *arg);
    101 
    102 #define MAX_SIZE MAXPHYS
    103 
    104 static const struct scsipi_bustype wdc_atapi_bustype = {
    105 	SCSIPI_BUSTYPE_ATAPI,
    106 	atapi_scsipi_cmd,
    107 	atapi_interpret_sense,
    108 	atapi_print_addr,
    109 	wdc_atapi_kill_pending,
    110 };
    111 
    112 void
    113 wdc_atapibus_attach(struct atabus_softc *ata_sc)
    114 {
    115 	struct ata_channel *chp = ata_sc->sc_chan;
    116 	struct atac_softc *atac = chp->ch_atac;
    117 	struct scsipi_adapter *adapt = &atac->atac_atapi_adapter._generic;
    118 	struct scsipi_channel *chan = &chp->ch_atapi_channel;
    119 
    120 	/*
    121 	 * Fill in the scsipi_adapter.
    122 	 */
    123 	adapt->adapt_dev = atac->atac_dev;
    124 	adapt->adapt_nchannels = atac->atac_nchannels;
    125 	adapt->adapt_request = wdc_atapi_scsipi_request;
    126 	adapt->adapt_minphys = wdc_atapi_minphys;
    127 	if (atac->atac_cap & ATAC_CAP_NOIRQ)
    128 		adapt->adapt_flags |= SCSIPI_ADAPT_POLL_ONLY;
    129 	atac->atac_atapi_adapter.atapi_probe_device = wdc_atapi_probe_device;
    130 
    131 	/*
    132 	 * Fill in the scsipi_channel.
    133 	 */
    134 	memset(chan, 0, sizeof(*chan));
    135 	chan->chan_adapter = adapt;
    136 	chan->chan_bustype = &wdc_atapi_bustype;
    137 	chan->chan_channel = chp->ch_channel;
    138 	chan->chan_flags = SCSIPI_CHAN_OPENINGS;
    139 	chan->chan_openings = 1;
    140 	chan->chan_max_periph = 1;
    141 	chan->chan_ntargets = 2;
    142 	chan->chan_nluns = 1;
    143 
    144 	chp->atapibus = config_found_ia(ata_sc->sc_dev, "atapi", chan,
    145 		atapiprint);
    146 }
    147 
    148 static void
    149 wdc_atapi_minphys(struct buf *bp)
    150 {
    151 
    152 	if (bp->b_bcount > MAX_SIZE)
    153 		bp->b_bcount = MAX_SIZE;
    154 	minphys(bp);
    155 }
    156 
    157 /*
    158  * Kill off all pending xfers for a periph.
    159  *
    160  * Must be called at splbio().
    161  */
    162 static void
    163 wdc_atapi_kill_pending(struct scsipi_periph *periph)
    164 {
    165 	struct atac_softc *atac =
    166 	    device_private(periph->periph_channel->chan_adapter->adapt_dev);
    167 	struct ata_channel *chp =
    168 	    atac->atac_channels[periph->periph_channel->chan_channel];
    169 
    170 	ata_kill_pending(&chp->ch_drive[periph->periph_target]);
    171 }
    172 
    173 static void
    174 wdc_atapi_kill_xfer(struct ata_channel *chp, struct ata_xfer *xfer, int reason)
    175 {
    176 	struct scsipi_xfer *sc_xfer = xfer->c_cmd;
    177 
    178 	/* remove this command from xfer queue */
    179 	switch (reason) {
    180 	case KILL_GONE:
    181 		sc_xfer->error = XS_DRIVER_STUFFUP;
    182 		break;
    183 	case KILL_RESET:
    184 		sc_xfer->error = XS_RESET;
    185 		break;
    186 	default:
    187 		printf("wdc_ata_bio_kill_xfer: unknown reason %d\n",
    188 		    reason);
    189 		panic("wdc_ata_bio_kill_xfer");
    190 	}
    191 	ata_free_xfer(chp, xfer);
    192 	scsipi_done(sc_xfer);
    193 }
    194 
    195 static int
    196 wdc_atapi_get_params(struct scsipi_channel *chan, int drive,
    197     struct ataparams *id)
    198 {
    199 	struct wdc_softc *wdc = device_private(chan->chan_adapter->adapt_dev);
    200 	struct atac_softc *atac = &wdc->sc_atac;
    201 	struct wdc_regs *wdr = &wdc->regs[chan->chan_channel];
    202 	struct ata_channel *chp = atac->atac_channels[chan->chan_channel];
    203 	struct ata_command ata_c;
    204 
    205 	/* if no ATAPI device detected at wdc attach time, skip */
    206 	if ((chp->ch_drive[drive].drive_flags & DRIVE_ATAPI) == 0) {
    207 		ATADEBUG_PRINT(("wdc_atapi_get_params: drive %d not present\n",
    208 		    drive), DEBUG_PROBE);
    209 		return -1;
    210 	}
    211 
    212 	memset(&ata_c, 0, sizeof(struct ata_command));
    213 	ata_c.r_command = ATAPI_SOFT_RESET;
    214 	ata_c.r_st_bmask = 0;
    215 	ata_c.r_st_pmask = 0;
    216 	ata_c.flags = AT_WAIT | AT_POLL;
    217 	ata_c.timeout = WDC_RESET_WAIT;
    218 	if (wdc_exec_command(&chp->ch_drive[drive], &ata_c) != ATACMD_COMPLETE) {
    219 		printf("wdc_atapi_get_params: ATAPI_SOFT_RESET failed for"
    220 		    " drive %s:%d:%d: driver failed\n",
    221 		    device_xname(atac->atac_dev), chp->ch_channel, drive);
    222 		panic("wdc_atapi_get_params");
    223 	}
    224 	if (ata_c.flags & (AT_ERROR | AT_TIMEOU | AT_DF)) {
    225 		ATADEBUG_PRINT(("wdc_atapi_get_params: ATAPI_SOFT_RESET "
    226 		    "failed for drive %s:%d:%d: error 0x%x\n",
    227 		    device_xname(atac->atac_dev), chp->ch_channel, drive,
    228 		    ata_c.r_error), DEBUG_PROBE);
    229 		return -1;
    230 	}
    231 	chp->ch_drive[drive].state = 0;
    232 
    233 	(void)bus_space_read_1(wdr->cmd_iot, wdr->cmd_iohs[wd_status], 0);
    234 
    235 	/* Some ATAPI devices need a bit more time after software reset. */
    236 	delay(5000);
    237 	if (ata_get_params(&chp->ch_drive[drive], AT_WAIT, id) != 0) {
    238 		ATADEBUG_PRINT(("wdc_atapi_get_params: ATAPI_IDENTIFY_DEVICE "
    239 		    "failed for drive %s:%d:%d: error 0x%x\n",
    240 		    device_xname(atac->atac_dev), chp->ch_channel, drive,
    241 		    ata_c.r_error), DEBUG_PROBE);
    242 		return -1;
    243 	}
    244 	return 0;
    245 }
    246 
    247 static void
    248 wdc_atapi_probe_device(struct atapibus_softc *sc, int target)
    249 {
    250 	struct scsipi_channel *chan = sc->sc_channel;
    251 	struct scsipi_periph *periph;
    252 	struct ataparams ids;
    253 	struct ataparams *id = &ids;
    254 	struct wdc_softc *wdc = device_private(chan->chan_adapter->adapt_dev);
    255 	struct atac_softc *atac = &wdc->sc_atac;
    256 	struct ata_channel *chp = atac->atac_channels[chan->chan_channel];
    257 	struct ata_drive_datas *drvp = &chp->ch_drive[target];
    258 	struct scsipibus_attach_args sa;
    259 	char serial_number[21], model[41], firmware_revision[9];
    260 	int s;
    261 
    262 	/* skip if already attached */
    263 	if (scsipi_lookup_periph(chan, target, 0) != NULL)
    264 		return;
    265 
    266 	if (wdc_atapi_get_params(chan, target, id) == 0) {
    267 #ifdef ATAPI_DEBUG_PROBE
    268 		printf("%s drive %d: cmdsz 0x%x drqtype 0x%x\n",
    269 		    device_xname(sc->sc_dev), target,
    270 		    id->atap_config & ATAPI_CFG_CMD_MASK,
    271 		    id->atap_config & ATAPI_CFG_DRQ_MASK);
    272 #endif
    273 		periph = scsipi_alloc_periph(M_NOWAIT);
    274 		if (periph == NULL) {
    275 			aprint_error_dev(sc->sc_dev,
    276 			    "unable to allocate periph for drive %d\n",
    277 			    target);
    278 			return;
    279 		}
    280 		periph->periph_dev = NULL;
    281 		periph->periph_channel = chan;
    282 		periph->periph_switch = &atapi_probe_periphsw;
    283 		periph->periph_target = target;
    284 		periph->periph_lun = 0;
    285 		periph->periph_quirks = PQUIRK_ONLYBIG;
    286 
    287 #ifdef SCSIPI_DEBUG
    288 		if (SCSIPI_DEBUG_TYPE == SCSIPI_BUSTYPE_ATAPI &&
    289 		    SCSIPI_DEBUG_TARGET == target)
    290 			periph->periph_dbflags |= SCSIPI_DEBUG_FLAGS;
    291 #endif
    292 		periph->periph_type = ATAPI_CFG_TYPE(id->atap_config);
    293 		if (id->atap_config & ATAPI_CFG_REMOV)
    294 			periph->periph_flags |= PERIPH_REMOVABLE;
    295 		if (periph->periph_type == T_SEQUENTIAL) {
    296 			s = splbio();
    297 			drvp->drive_flags |= DRIVE_ATAPIST;
    298 			splx(s);
    299 		}
    300 
    301 		sa.sa_periph = periph;
    302 		sa.sa_inqbuf.type =  ATAPI_CFG_TYPE(id->atap_config);
    303 		sa.sa_inqbuf.removable = id->atap_config & ATAPI_CFG_REMOV ?
    304 		    T_REMOV : T_FIXED;
    305 		scsipi_strvis((u_char *)model, 40, id->atap_model, 40);
    306 		scsipi_strvis((u_char *)serial_number, 20, id->atap_serial,
    307 		    20);
    308 		scsipi_strvis((u_char *)firmware_revision, 8,
    309 		    id->atap_revision, 8);
    310 		sa.sa_inqbuf.vendor = model;
    311 		sa.sa_inqbuf.product = serial_number;
    312 		sa.sa_inqbuf.revision = firmware_revision;
    313 
    314 		/*
    315 		 * Determine the operating mode capabilities of the device.
    316 		 */
    317 		if ((id->atap_config & ATAPI_CFG_CMD_MASK) == ATAPI_CFG_CMD_16)
    318 			periph->periph_cap |= PERIPH_CAP_CMD16;
    319 		/* XXX This is gross. */
    320 		periph->periph_cap |= (id->atap_config & ATAPI_CFG_DRQ_MASK);
    321 
    322 		drvp->drv_softc = atapi_probe_device(sc, target, periph, &sa);
    323 
    324 		if (drvp->drv_softc)
    325 			ata_probe_caps(drvp);
    326 		else {
    327 			s = splbio();
    328 			drvp->drive_flags &= ~DRIVE_ATAPI;
    329 			splx(s);
    330 		}
    331 	} else {
    332 		s = splbio();
    333 		drvp->drive_flags &= ~DRIVE_ATAPI;
    334 		splx(s);
    335 	}
    336 }
    337 
    338 static void
    339 wdc_atapi_scsipi_request(struct scsipi_channel *chan, scsipi_adapter_req_t req,
    340     void *arg)
    341 {
    342 	struct scsipi_adapter *adapt = chan->chan_adapter;
    343 	struct scsipi_periph *periph;
    344 	struct scsipi_xfer *sc_xfer;
    345 	struct wdc_softc *wdc = device_private(adapt->adapt_dev);
    346 	struct atac_softc *atac = &wdc->sc_atac;
    347 	struct ata_xfer *xfer;
    348 	int channel = chan->chan_channel;
    349 	int drive, s;
    350 
    351 	switch (req) {
    352 	case ADAPTER_REQ_RUN_XFER:
    353 		sc_xfer = arg;
    354 		periph = sc_xfer->xs_periph;
    355 		drive = periph->periph_target;
    356 
    357 		ATADEBUG_PRINT(("wdc_atapi_scsipi_request %s:%d:%d\n",
    358 		    device_xname(atac->atac_dev), channel, drive),
    359 		    DEBUG_XFERS);
    360 		if (!device_is_active(atac->atac_dev)) {
    361 			sc_xfer->error = XS_DRIVER_STUFFUP;
    362 			scsipi_done(sc_xfer);
    363 			return;
    364 		}
    365 
    366 		xfer = ata_get_xfer(ATAXF_NOSLEEP);
    367 		if (xfer == NULL) {
    368 			sc_xfer->error = XS_RESOURCE_SHORTAGE;
    369 			scsipi_done(sc_xfer);
    370 			return;
    371 		}
    372 
    373 		if (sc_xfer->xs_control & XS_CTL_POLL)
    374 			xfer->c_flags |= C_POLL;
    375 #if NATA_DMA
    376 		if ((atac->atac_channels[channel]->ch_drive[drive].drive_flags &
    377 		    (DRIVE_DMA | DRIVE_UDMA)) && sc_xfer->datalen > 0)
    378 			xfer->c_flags |= C_DMA;
    379 #endif
    380 #if NATA_DMA && NATA_PIOBM
    381 		else
    382 #endif
    383 #if NATA_PIOBM
    384 		if ((atac->atac_cap & ATAC_CAP_PIOBM) &&
    385 		    sc_xfer->datalen > 0)
    386 			xfer->c_flags |= C_PIOBM;
    387 #endif
    388 		xfer->c_drive = drive;
    389 		xfer->c_flags |= C_ATAPI;
    390 #if NATA_DMA
    391 		if (sc_xfer->cmd->opcode == GPCMD_REPORT_KEY ||
    392 		    sc_xfer->cmd->opcode == GPCMD_SEND_KEY ||
    393 		    sc_xfer->cmd->opcode == GPCMD_READ_DVD_STRUCTURE) {
    394 			/*
    395 			 * DVD authentication commands must always be done in
    396 			 * PIO mode.
    397 			 */
    398 			xfer->c_flags &= ~C_DMA;
    399 		}
    400 
    401 		/*
    402 		 * DMA normally can't deal with transfers which are not a
    403 		 * multiple of its databus width. It's a bug to request odd
    404 		 * length transfers for ATAPI.
    405 		 *
    406 		 * Some devices also can't cope with unaligned DMA xfers
    407 		 * either. Also some devices seem to not handle DMA xfers of
    408 		 * less than 4 bytes.
    409 		 *
    410 		 * By enforcing at least 4 byte aligned offset and length for
    411 		 * DMA, we might use PIO where DMA could be allowed but better
    412 		 * safe than sorry as recent problems proved.
    413 		 *
    414 		 * Offending structures that are thus done by PIO instead of
    415 		 * DMA are normally small structures since all bulkdata is
    416 		 * aligned. But as the request may come from userland, we have
    417 		 * to protect against it anyway.
    418 		 *
    419 		 * XXX check for the 32 bit wide flag?
    420 		 */
    421 
    422 		if (((uintptr_t) sc_xfer->data) & 0x03)
    423 			xfer->c_flags &= ~C_DMA;
    424 		if ((sc_xfer->datalen < 4) || (sc_xfer->datalen & 0x03))
    425 			xfer->c_flags &= ~C_DMA;
    426 #endif	/* NATA_DMA */
    427 
    428 		xfer->c_cmd = sc_xfer;
    429 		xfer->c_databuf = sc_xfer->data;
    430 		xfer->c_bcount = sc_xfer->datalen;
    431 		xfer->c_start = wdc_atapi_start;
    432 		xfer->c_intr = wdc_atapi_intr;
    433 		xfer->c_kill_xfer = wdc_atapi_kill_xfer;
    434 		xfer->c_dscpoll = 0;
    435 		s = splbio();
    436 		ata_exec_xfer(atac->atac_channels[channel], xfer);
    437 #ifdef DIAGNOSTIC
    438 		if ((sc_xfer->xs_control & XS_CTL_POLL) != 0 &&
    439 		    (sc_xfer->xs_status & XS_STS_DONE) == 0)
    440 			panic("wdc_atapi_scsipi_request: polled command "
    441 			    "not done");
    442 #endif
    443 		splx(s);
    444 		return;
    445 
    446 	default:
    447 		/* Not supported, nothing to do. */
    448 		;
    449 	}
    450 }
    451 
    452 static void
    453 wdc_atapi_start(struct ata_channel *chp, struct ata_xfer *xfer)
    454 {
    455 	struct atac_softc *atac = chp->ch_atac;
    456 	struct wdc_softc *wdc = CHAN_TO_WDC(chp);
    457 	struct wdc_regs *wdr = &wdc->regs[chp->ch_channel];
    458 	struct scsipi_xfer *sc_xfer = xfer->c_cmd;
    459 	struct ata_drive_datas *drvp = &chp->ch_drive[xfer->c_drive];
    460 	int wait_flags = (sc_xfer->xs_control & XS_CTL_POLL) ? AT_POLL : 0;
    461 	const char *errstring;
    462 
    463 	ATADEBUG_PRINT(("wdc_atapi_start %s:%d:%d, scsi flags 0x%x \n",
    464 	    device_xname(atac->atac_dev), chp->ch_channel, drvp->drive,
    465 	    sc_xfer->xs_control), DEBUG_XFERS);
    466 #if NATA_DMA
    467 	if ((xfer->c_flags & C_DMA) && (drvp->n_xfers <= NXFER))
    468 		drvp->n_xfers++;
    469 #endif
    470 	/* Do control operations specially. */
    471 	if (__predict_false(drvp->state < READY)) {
    472 		/* If it's not a polled command, we need the kernel thread */
    473 		if ((sc_xfer->xs_control & XS_CTL_POLL) == 0 &&
    474 		    (chp->ch_flags & ATACH_TH_RUN) == 0) {
    475 			chp->ch_queue->queue_freeze++;
    476 			wakeup(&chp->ch_thread);
    477 			return;
    478 		}
    479 		/*
    480 		 * disable interrupts, all commands here should be quick
    481 		 * enouth to be able to poll, and we don't go here that often
    482 		 */
    483 		 bus_space_write_1(wdr->ctl_iot, wdr->ctl_ioh, wd_aux_ctlr,
    484 		     WDCTL_4BIT | WDCTL_IDS);
    485 		if (wdc->select)
    486 			wdc->select(chp, xfer->c_drive);
    487 		bus_space_write_1(wdr->cmd_iot, wdr->cmd_iohs[wd_sdh], 0,
    488 		    WDSD_IBM | (xfer->c_drive << 4));
    489 		/* Don't try to set mode if controller can't be adjusted */
    490 		if (atac->atac_set_modes == NULL)
    491 			goto ready;
    492 		/* Also don't try if the drive didn't report its mode */
    493 		if ((drvp->drive_flags & DRIVE_MODE) == 0)
    494 			goto ready;
    495 		errstring = "unbusy";
    496 		if (wdc_wait_for_unbusy(chp, ATAPI_DELAY, wait_flags))
    497 			goto timeout;
    498 		wdccommand(chp, drvp->drive, SET_FEATURES, 0, 0, 0,
    499 		    0x08 | drvp->PIO_mode, WDSF_SET_MODE);
    500 		errstring = "piomode";
    501 		if (wdc_wait_for_unbusy(chp, ATAPI_MODE_DELAY, wait_flags))
    502 			goto timeout;
    503 		if (chp->ch_status & WDCS_ERR) {
    504 			if (chp->ch_error == WDCE_ABRT) {
    505 				/*
    506 				 * Some ATAPI drives reject PIO settings.
    507 				 * Fall back to PIO mode 3 since that's the
    508 				 * minimum for ATAPI.
    509 				 */
    510 				printf("%s:%d:%d: PIO mode %d rejected, "
    511 				    "falling back to PIO mode 3\n",
    512 				    device_xname(atac->atac_dev),
    513 				    chp->ch_channel, xfer->c_drive,
    514 				    drvp->PIO_mode);
    515 				if (drvp->PIO_mode > 3)
    516 					drvp->PIO_mode = 3;
    517 			} else
    518 				goto error;
    519 		}
    520 #if NATA_DMA
    521 #if NATA_UDMA
    522 		if (drvp->drive_flags & DRIVE_UDMA) {
    523 			wdccommand(chp, drvp->drive, SET_FEATURES, 0, 0, 0,
    524 			    0x40 | drvp->UDMA_mode, WDSF_SET_MODE);
    525 		} else
    526 #endif
    527 		if (drvp->drive_flags & DRIVE_DMA) {
    528 			wdccommand(chp, drvp->drive, SET_FEATURES, 0, 0, 0,
    529 			    0x20 | drvp->DMA_mode, WDSF_SET_MODE);
    530 		} else {
    531 			goto ready;
    532 		}
    533 		errstring = "dmamode";
    534 		if (wdc_wait_for_unbusy(chp, ATAPI_MODE_DELAY, wait_flags))
    535 			goto timeout;
    536 		if (chp->ch_status & WDCS_ERR) {
    537 			if (chp->ch_error == WDCE_ABRT) {
    538 #if NATA_UDMA
    539 				if (drvp->drive_flags & DRIVE_UDMA)
    540 					goto error;
    541 				else
    542 #endif
    543 				{
    544 					/*
    545 					 * The drive rejected our DMA setting.
    546 					 * Fall back to mode 1.
    547 					 */
    548 					printf("%s:%d:%d: DMA mode %d rejected, "
    549 					    "falling back to DMA mode 0\n",
    550 					    device_xname(atac->atac_dev),
    551 					    chp->ch_channel, xfer->c_drive,
    552 					    drvp->DMA_mode);
    553 					if (drvp->DMA_mode > 0)
    554 						drvp->DMA_mode = 0;
    555 				}
    556 			} else
    557 				goto error;
    558 		}
    559 #endif	/* NATA_DMA */
    560 ready:
    561 		drvp->state = READY;
    562 		bus_space_write_1(wdr->ctl_iot, wdr->ctl_ioh, wd_aux_ctlr,
    563 		    WDCTL_4BIT);
    564 		delay(10); /* some drives need a little delay here */
    565 	}
    566 	/* start timeout machinery */
    567 	if ((sc_xfer->xs_control & XS_CTL_POLL) == 0)
    568 		callout_reset(&chp->ch_callout, mstohz(sc_xfer->timeout),
    569 		    wdctimeout, chp);
    570 
    571 	if (wdc->select)
    572 		wdc->select(chp, xfer->c_drive);
    573 	bus_space_write_1(wdr->cmd_iot, wdr->cmd_iohs[wd_sdh], 0,
    574 	    WDSD_IBM | (xfer->c_drive << 4));
    575 	switch (wdc_wait_for_unbusy(chp, ATAPI_DELAY, wait_flags)  < 0) {
    576 	case WDCWAIT_OK:
    577 		break;
    578 	case WDCWAIT_TOUT:
    579 		printf("wdc_atapi_start: not ready, st = %02x\n",
    580 		    chp->ch_status);
    581 		sc_xfer->error = XS_TIMEOUT;
    582 		wdc_atapi_reset(chp, xfer);
    583 		return;
    584 	case WDCWAIT_THR:
    585 		return;
    586 	}
    587 
    588 	/*
    589 	 * Even with WDCS_ERR, the device should accept a command packet
    590 	 * Limit length to what can be stuffed into the cylinder register
    591 	 * (16 bits).  Some CD-ROMs seem to interpret '0' as 65536,
    592 	 * but not all devices do that and it's not obvious from the
    593 	 * ATAPI spec that that behaviour should be expected.  If more
    594 	 * data is necessary, multiple data transfer phases will be done.
    595 	 */
    596 
    597 	wdccommand(chp, xfer->c_drive, ATAPI_PKT_CMD,
    598 	    xfer->c_bcount <= 0xffff ? xfer->c_bcount : 0xffff,
    599 	    0, 0, 0,
    600 #if NATA_DMA
    601 	    (xfer->c_flags & C_DMA) ? ATAPI_PKT_CMD_FTRE_DMA :
    602 #endif
    603 	    0
    604 	    );
    605 
    606 #if NATA_PIOBM
    607 	if (xfer->c_flags & C_PIOBM) {
    608 		int error;
    609 		int dma_flags = (sc_xfer->xs_control & XS_CTL_DATA_IN)
    610 		    ?  WDC_DMA_READ : 0;
    611 		if (xfer->c_flags & C_POLL) {
    612 			/* XXX not supported yet --- fall back to PIO */
    613 			xfer->c_flags &= ~C_PIOBM;
    614 		} else {
    615 			/* Init the DMA channel. */
    616 			error = (*wdc->dma_init)(wdc->dma_arg,
    617 			    chp->ch_channel, xfer->c_drive,
    618 			    (char *)xfer->c_databuf,
    619 			    xfer->c_bcount,
    620 			    dma_flags | WDC_DMA_PIOBM_ATAPI);
    621 			if (error) {
    622 				if (error == EINVAL) {
    623 					/*
    624 					 * We can't do DMA on this transfer
    625 					 * for some reason.  Fall back to
    626 					 * PIO.
    627 					 */
    628 					xfer->c_flags &= ~C_PIOBM;
    629 					error = 0;
    630 				} else {
    631 					sc_xfer->error = XS_DRIVER_STUFFUP;
    632 					errstring = "piobm";
    633 					goto error;
    634 				}
    635 			}
    636 		}
    637 	}
    638 #endif
    639 	/*
    640 	 * If there is no interrupt for CMD input, busy-wait for it (done in
    641 	 * the interrupt routine. If it is a polled command, call the interrupt
    642 	 * routine until command is done.
    643 	 */
    644 	if ((sc_xfer->xs_periph->periph_cap & ATAPI_CFG_DRQ_MASK) !=
    645 	    ATAPI_CFG_IRQ_DRQ || (sc_xfer->xs_control & XS_CTL_POLL)) {
    646 		/* Wait for at last 400ns for status bit to be valid */
    647 		DELAY(1);
    648 		wdc_atapi_intr(chp, xfer, 0);
    649 	} else {
    650 		chp->ch_flags |= ATACH_IRQ_WAIT;
    651 	}
    652 	if (sc_xfer->xs_control & XS_CTL_POLL) {
    653 #if NATA_DMA
    654 		if (chp->ch_flags & ATACH_DMA_WAIT) {
    655 			wdc_dmawait(chp, xfer, sc_xfer->timeout);
    656 			chp->ch_flags &= ~ATACH_DMA_WAIT;
    657 		}
    658 #endif
    659 		while ((sc_xfer->xs_status & XS_STS_DONE) == 0) {
    660 			/* Wait for at last 400ns for status bit to be valid */
    661 			DELAY(1);
    662 			wdc_atapi_intr(chp, xfer, 0);
    663 		}
    664 	}
    665 	return;
    666 timeout:
    667 	printf("%s:%d:%d: %s timed out\n",
    668 	    device_xname(atac->atac_dev), chp->ch_channel, xfer->c_drive,
    669 	    errstring);
    670 	sc_xfer->error = XS_TIMEOUT;
    671 	bus_space_write_1(wdr->ctl_iot, wdr->ctl_ioh, wd_aux_ctlr, WDCTL_4BIT);
    672 	delay(10); /* some drives need a little delay here */
    673 	wdc_atapi_reset(chp, xfer);
    674 	return;
    675 error:
    676 	printf("%s:%d:%d: %s ",
    677 	    device_xname(atac->atac_dev), chp->ch_channel, xfer->c_drive,
    678 	    errstring);
    679 	printf("error (0x%x)\n", chp->ch_error);
    680 	sc_xfer->error = XS_SHORTSENSE;
    681 	sc_xfer->sense.atapi_sense = chp->ch_error;
    682 	bus_space_write_1(wdr->ctl_iot, wdr->ctl_ioh, wd_aux_ctlr, WDCTL_4BIT);
    683 	delay(10); /* some drives need a little delay here */
    684 	wdc_atapi_reset(chp, xfer);
    685 	return;
    686 }
    687 
    688 static int
    689 wdc_atapi_intr(struct ata_channel *chp, struct ata_xfer *xfer, int irq)
    690 {
    691 	struct atac_softc *atac = chp->ch_atac;
    692 	struct wdc_softc *wdc = CHAN_TO_WDC(chp);
    693 	struct wdc_regs *wdr = &wdc->regs[chp->ch_channel];
    694 	struct scsipi_xfer *sc_xfer = xfer->c_cmd;
    695 	struct ata_drive_datas *drvp = &chp->ch_drive[xfer->c_drive];
    696 	int len, phase, i, retries=0;
    697 	int ire;
    698 #if NATA_DMA
    699 	int error;
    700 #endif
    701 #if NATA_DMA || NATA_PIOBM
    702 	int dma_flags = 0;
    703 #endif
    704 	void *cmd;
    705 
    706 	ATADEBUG_PRINT(("wdc_atapi_intr %s:%d:%d\n",
    707 	    device_xname(atac->atac_dev), chp->ch_channel, drvp->drive),
    708 	    DEBUG_INTR);
    709 
    710 	/* Is it not a transfer, but a control operation? */
    711 	if (drvp->state < READY) {
    712 		printf("%s:%d:%d: bad state %d in wdc_atapi_intr\n",
    713 		    device_xname(atac->atac_dev), chp->ch_channel,
    714 		    xfer->c_drive, drvp->state);
    715 		panic("wdc_atapi_intr: bad state");
    716 	}
    717 	/*
    718 	 * If we missed an interrupt in a PIO transfer, reset and restart.
    719 	 * Don't try to continue transfer, we may have missed cycles.
    720 	 */
    721 	if ((xfer->c_flags & (C_TIMEOU | C_DMA)) == C_TIMEOU) {
    722 		sc_xfer->error = XS_TIMEOUT;
    723 		wdc_atapi_reset(chp, xfer);
    724 		return 1;
    725 	}
    726 
    727 #if NATA_PIOBM
    728 	/* Transfer-done interrupt for busmastering PIO operation */
    729 	if ((xfer->c_flags & C_PIOBM) && (chp->ch_flags & ATACH_PIOBM_WAIT)) {
    730 		chp->ch_flags &= ~ATACH_PIOBM_WAIT;
    731 
    732 		/* restore transfer length */
    733 		len = xfer->c_bcount;
    734 		if (xfer->c_lenoff < 0)
    735 			len += xfer->c_lenoff;
    736 
    737 		if (sc_xfer->xs_control & XS_CTL_DATA_IN)
    738 			goto end_piobm_datain;
    739 		else
    740 			goto end_piobm_dataout;
    741 	}
    742 #endif
    743 
    744 	/* Ack interrupt done in wdc_wait_for_unbusy */
    745 	if (wdc->select)
    746 		wdc->select(chp, xfer->c_drive);
    747 	bus_space_write_1(wdr->cmd_iot, wdr->cmd_iohs[wd_sdh], 0,
    748 	    WDSD_IBM | (xfer->c_drive << 4));
    749 	if (wdc_wait_for_unbusy(chp,
    750 	    (irq == 0) ? sc_xfer->timeout : 0, AT_POLL) == WDCWAIT_TOUT) {
    751 		if (irq && (xfer->c_flags & C_TIMEOU) == 0)
    752 			return 0; /* IRQ was not for us */
    753 		printf("%s:%d:%d: device timeout, c_bcount=%d, c_skip=%d\n",
    754 		    device_xname(atac->atac_dev), chp->ch_channel,
    755 		    xfer->c_drive, xfer->c_bcount, xfer->c_skip);
    756 #if NATA_DMA
    757 		if (xfer->c_flags & C_DMA) {
    758 			ata_dmaerr(drvp,
    759 			    (xfer->c_flags & C_POLL) ? AT_POLL : 0);
    760 		}
    761 #endif
    762 		sc_xfer->error = XS_TIMEOUT;
    763 		wdc_atapi_reset(chp, xfer);
    764 		return 1;
    765 	}
    766 	if (wdc->irqack)
    767 		wdc->irqack(chp);
    768 
    769 #if NATA_DMA
    770 	/*
    771 	 * If we missed an IRQ and were using DMA, flag it as a DMA error
    772 	 * and reset device.
    773 	 */
    774 	if ((xfer->c_flags & C_TIMEOU) && (xfer->c_flags & C_DMA)) {
    775 		ata_dmaerr(drvp, (xfer->c_flags & C_POLL) ? AT_POLL : 0);
    776 		sc_xfer->error = XS_RESET;
    777 		wdc_atapi_reset(chp, xfer);
    778 		return (1);
    779 	}
    780 #endif
    781 	/*
    782 	 * if the request sense command was aborted, report the short sense
    783 	 * previously recorded, else continue normal processing
    784 	 */
    785 
    786 #if NATA_DMA || NATA_PIOBM
    787 	if (xfer->c_flags & (C_DMA | C_PIOBM))
    788 		dma_flags = (sc_xfer->xs_control & XS_CTL_DATA_IN)
    789 		    ?  WDC_DMA_READ : 0;
    790 #endif
    791 again:
    792 	len = bus_space_read_1(wdr->cmd_iot, wdr->cmd_iohs[wd_cyl_lo], 0) +
    793 	    256 * bus_space_read_1(wdr->cmd_iot, wdr->cmd_iohs[wd_cyl_hi], 0);
    794 	ire = bus_space_read_1(wdr->cmd_iot, wdr->cmd_iohs[wd_ireason], 0);
    795 	phase = (ire & (WDCI_CMD | WDCI_IN)) | (chp->ch_status & WDCS_DRQ);
    796 	ATADEBUG_PRINT(("wdc_atapi_intr: c_bcount %d len %d st 0x%x err 0x%x "
    797 	    "ire 0x%x :", xfer->c_bcount,
    798 	    len, chp->ch_status, chp->ch_error, ire), DEBUG_INTR);
    799 
    800 	switch (phase) {
    801 	case PHASE_CMDOUT:
    802 		cmd = sc_xfer->cmd;
    803 		ATADEBUG_PRINT(("PHASE_CMDOUT\n"), DEBUG_INTR);
    804 #if NATA_DMA
    805 		/* Init the DMA channel if necessary */
    806 		if (xfer->c_flags & C_DMA) {
    807 			error = (*wdc->dma_init)(wdc->dma_arg,
    808 			    chp->ch_channel, xfer->c_drive,
    809 			    xfer->c_databuf, xfer->c_bcount, dma_flags);
    810 			if (error) {
    811 				if (error == EINVAL) {
    812 					/*
    813 					 * We can't do DMA on this transfer
    814 					 * for some reason.  Fall back to
    815 					 * PIO.
    816 					 */
    817 					xfer->c_flags &= ~C_DMA;
    818 					error = 0;
    819 				} else {
    820 					sc_xfer->error = XS_DRIVER_STUFFUP;
    821 					break;
    822 				}
    823 			}
    824 		}
    825 #endif
    826 
    827 		/* send packet command */
    828 		/* Commands are 12 or 16 bytes long. It's 32-bit aligned */
    829 		wdc->dataout_pio(chp, drvp->drive_flags, cmd, sc_xfer->cmdlen);
    830 
    831 #if NATA_DMA
    832 		/* Start the DMA channel if necessary */
    833 		if (xfer->c_flags & C_DMA) {
    834 			(*wdc->dma_start)(wdc->dma_arg,
    835 			    chp->ch_channel, xfer->c_drive);
    836 			chp->ch_flags |= ATACH_DMA_WAIT;
    837 		}
    838 #endif
    839 
    840 		if ((sc_xfer->xs_control & XS_CTL_POLL) == 0) {
    841 			chp->ch_flags |= ATACH_IRQ_WAIT;
    842 		}
    843 		return 1;
    844 
    845 	 case PHASE_DATAOUT:
    846 		/* write data */
    847 		ATADEBUG_PRINT(("PHASE_DATAOUT\n"), DEBUG_INTR);
    848 #if NATA_DMA
    849 		if ((sc_xfer->xs_control & XS_CTL_DATA_OUT) == 0 ||
    850 		    (xfer->c_flags & C_DMA) != 0) {
    851 			printf("wdc_atapi_intr: bad data phase DATAOUT\n");
    852 			if (xfer->c_flags & C_DMA) {
    853 				ata_dmaerr(drvp,
    854 				    (xfer->c_flags & C_POLL) ? AT_POLL : 0);
    855 			}
    856 			sc_xfer->error = XS_TIMEOUT;
    857 			wdc_atapi_reset(chp, xfer);
    858 			return 1;
    859 		}
    860 #endif
    861 		xfer->c_lenoff = len - xfer->c_bcount;
    862 		if (xfer->c_bcount < len) {
    863 			printf("wdc_atapi_intr: warning: write only "
    864 			    "%d of %d requested bytes\n", xfer->c_bcount, len);
    865 			len = xfer->c_bcount;
    866 		}
    867 
    868 #if NATA_PIOBM
    869 		if (xfer->c_flags & C_PIOBM) {
    870 			/* start the busmastering PIO */
    871 			(*wdc->piobm_start)(wdc->dma_arg,
    872 			    chp->ch_channel, xfer->c_drive,
    873 			    xfer->c_skip, len, WDC_PIOBM_XFER_IRQ);
    874 			chp->ch_flags |= ATACH_DMA_WAIT | ATACH_IRQ_WAIT |
    875 			    ATACH_PIOBM_WAIT;
    876 			return 1;
    877 		}
    878 #endif
    879 		wdc->dataout_pio(chp, drvp->drive_flags,
    880 	    	    (char *)xfer->c_databuf + xfer->c_skip, len);
    881 
    882 #if NATA_PIOBM
    883 	end_piobm_dataout:
    884 #endif
    885 		for (i = xfer->c_lenoff; i > 0; i -= 2)
    886 			bus_space_write_2(wdr->cmd_iot,
    887 			    wdr->cmd_iohs[wd_data], 0, 0);
    888 
    889 		xfer->c_skip += len;
    890 		xfer->c_bcount -= len;
    891 		if ((sc_xfer->xs_control & XS_CTL_POLL) == 0) {
    892 			chp->ch_flags |= ATACH_IRQ_WAIT;
    893 		}
    894 		return 1;
    895 
    896 	case PHASE_DATAIN:
    897 		/* Read data */
    898 		ATADEBUG_PRINT(("PHASE_DATAIN\n"), DEBUG_INTR);
    899 #if NATA_DMA
    900 		if ((sc_xfer->xs_control & XS_CTL_DATA_IN) == 0 ||
    901 		    (xfer->c_flags & C_DMA) != 0) {
    902 			printf("wdc_atapi_intr: bad data phase DATAIN\n");
    903 			if (xfer->c_flags & C_DMA) {
    904 				ata_dmaerr(drvp,
    905 				    (xfer->c_flags & C_POLL) ? AT_POLL : 0);
    906 			}
    907 			sc_xfer->error = XS_TIMEOUT;
    908 			wdc_atapi_reset(chp, xfer);
    909 			return 1;
    910 		}
    911 #endif
    912 		xfer->c_lenoff = len - xfer->c_bcount;
    913 		if (xfer->c_bcount < len) {
    914 			printf("wdc_atapi_intr: warning: reading only "
    915 			    "%d of %d bytes\n", xfer->c_bcount, len);
    916 			len = xfer->c_bcount;
    917 		}
    918 
    919 #if NATA_PIOBM
    920 		if (xfer->c_flags & C_PIOBM) {
    921 			/* start the busmastering PIO */
    922 			(*wdc->piobm_start)(wdc->dma_arg,
    923 			    chp->ch_channel, xfer->c_drive,
    924 			    xfer->c_skip, len, WDC_PIOBM_XFER_IRQ);
    925 			chp->ch_flags |= ATACH_DMA_WAIT | ATACH_IRQ_WAIT |
    926 			    ATACH_PIOBM_WAIT;
    927 			return 1;
    928 		}
    929 #endif
    930 		wdc->datain_pio(chp, drvp->drive_flags,
    931 		    (char *)xfer->c_databuf + xfer->c_skip, len);
    932 
    933 #if NATA_PIOBM
    934 	end_piobm_datain:
    935 #endif
    936 		if (xfer->c_lenoff > 0)
    937 			wdcbit_bucket(chp, len - xfer->c_bcount);
    938 
    939 		xfer->c_skip += len;
    940 		xfer->c_bcount -= len;
    941 		if ((sc_xfer->xs_control & XS_CTL_POLL) == 0) {
    942 			chp->ch_flags |= ATACH_IRQ_WAIT;
    943 		}
    944 		return 1;
    945 
    946 	case PHASE_ABORTED:
    947 	case PHASE_COMPLETED:
    948 		ATADEBUG_PRINT(("PHASE_COMPLETED\n"), DEBUG_INTR);
    949 #if NATA_DMA
    950 		if (xfer->c_flags & C_DMA) {
    951 			xfer->c_bcount -= sc_xfer->datalen;
    952 		}
    953 #endif
    954 		sc_xfer->resid = xfer->c_bcount;
    955 		wdc_atapi_phase_complete(xfer);
    956 		return(1);
    957 
    958 	default:
    959 		if (++retries<500) {
    960 			DELAY(100);
    961 			chp->ch_status = bus_space_read_1(wdr->cmd_iot,
    962 			    wdr->cmd_iohs[wd_status], 0);
    963 			chp->ch_error = bus_space_read_1(wdr->cmd_iot,
    964 			    wdr->cmd_iohs[wd_error], 0);
    965 			goto again;
    966 		}
    967 		printf("wdc_atapi_intr: unknown phase 0x%x\n", phase);
    968 		if (chp->ch_status & WDCS_ERR) {
    969 			sc_xfer->error = XS_SHORTSENSE;
    970 			sc_xfer->sense.atapi_sense = chp->ch_error;
    971 		} else {
    972 #if NATA_DMA
    973 			if (xfer->c_flags & C_DMA) {
    974 				ata_dmaerr(drvp,
    975 				    (xfer->c_flags & C_POLL) ? AT_POLL : 0);
    976 			}
    977 #endif
    978 			sc_xfer->error = XS_RESET;
    979 			wdc_atapi_reset(chp, xfer);
    980 			return (1);
    981 		}
    982 	}
    983 	ATADEBUG_PRINT(("wdc_atapi_intr: wdc_atapi_done() (end), error 0x%x "
    984 	    "sense 0x%x\n", sc_xfer->error, sc_xfer->sense.atapi_sense),
    985 	    DEBUG_INTR);
    986 	wdc_atapi_done(chp, xfer);
    987 	return (1);
    988 }
    989 
    990 static void
    991 wdc_atapi_phase_complete(struct ata_xfer *xfer)
    992 {
    993 	struct ata_channel *chp = xfer->c_chp;
    994 	struct atac_softc *atac = chp->ch_atac;
    995 #if NATA_DMA || NATA_PIOBM
    996 	struct wdc_softc *wdc = CHAN_TO_WDC(chp);
    997 #endif
    998 	struct scsipi_xfer *sc_xfer = xfer->c_cmd;
    999 	struct ata_drive_datas *drvp = &chp->ch_drive[xfer->c_drive];
   1000 
   1001 	/* wait for DSC if needed */
   1002 	if (drvp->drive_flags & DRIVE_ATAPIST) {
   1003 		ATADEBUG_PRINT(("wdc_atapi_phase_complete(%s:%d:%d) "
   1004 		    "polldsc %d\n", device_xname(atac->atac_dev),
   1005 		    chp->ch_channel,
   1006 		    xfer->c_drive, xfer->c_dscpoll), DEBUG_XFERS);
   1007 #if 1
   1008 		if (cold)
   1009 			panic("wdc_atapi_phase_complete: cold");
   1010 #endif
   1011 		if (wdcwait(chp, WDCS_DSC, WDCS_DSC, 10,
   1012 		    AT_POLL) == WDCWAIT_TOUT) {
   1013 			/* 10ms not enough, try again in 1 tick */
   1014 			if (xfer->c_dscpoll++ >
   1015 			    mstohz(sc_xfer->timeout)) {
   1016 				printf("%s:%d:%d: wait_for_dsc "
   1017 				    "failed\n",
   1018 				    device_xname(atac->atac_dev),
   1019 				    chp->ch_channel, xfer->c_drive);
   1020 				sc_xfer->error = XS_TIMEOUT;
   1021 				wdc_atapi_reset(chp, xfer);
   1022 				return;
   1023 			} else
   1024 				callout_reset(&chp->ch_callout, 1,
   1025 				    wdc_atapi_polldsc, xfer);
   1026 			return;
   1027 		}
   1028 	}
   1029 
   1030 	/*
   1031 	 * Some drive occasionally set WDCS_ERR with
   1032 	 * "ATA illegal length indication" in the error
   1033 	 * register. If we read some data the sense is valid
   1034 	 * anyway, so don't report the error.
   1035 	 */
   1036 	if (chp->ch_status & WDCS_ERR &&
   1037 	    ((sc_xfer->xs_control & XS_CTL_REQSENSE) == 0 ||
   1038 	    sc_xfer->resid == sc_xfer->datalen)) {
   1039 		/* save the short sense */
   1040 		sc_xfer->error = XS_SHORTSENSE;
   1041 		sc_xfer->sense.atapi_sense = chp->ch_error;
   1042 		if ((sc_xfer->xs_periph->periph_quirks &
   1043 		    PQUIRK_NOSENSE) == 0) {
   1044 			/* ask scsipi to send a REQUEST_SENSE */
   1045 			sc_xfer->error = XS_BUSY;
   1046 			sc_xfer->status = SCSI_CHECK;
   1047 		}
   1048 #if NATA_DMA || NATA_PIOBM
   1049 		else if (wdc->dma_status &
   1050 		    (WDC_DMAST_NOIRQ | WDC_DMAST_ERR)) {
   1051 #if NATA_DMA
   1052 			ata_dmaerr(drvp,
   1053 			    (xfer->c_flags & C_POLL) ? AT_POLL : 0);
   1054 #endif
   1055 			sc_xfer->error = XS_RESET;
   1056 			wdc_atapi_reset(chp, xfer);
   1057 			return;
   1058 		}
   1059 #endif
   1060 	}
   1061 	if (xfer->c_bcount != 0) {
   1062 		ATADEBUG_PRINT(("wdc_atapi_intr: bcount value is "
   1063 		    "%d after io\n", xfer->c_bcount), DEBUG_XFERS);
   1064 	}
   1065 #ifdef DIAGNOSTIC
   1066 	if (xfer->c_bcount < 0) {
   1067 		printf("wdc_atapi_intr warning: bcount value "
   1068 		    "is %d after io\n", xfer->c_bcount);
   1069 	}
   1070 #endif
   1071 	ATADEBUG_PRINT(("wdc_atapi_phase_complete: wdc_atapi_done(), "
   1072 	    "error 0x%x sense 0x%x\n", sc_xfer->error,
   1073 	    sc_xfer->sense.atapi_sense), DEBUG_INTR);
   1074 	wdc_atapi_done(chp, xfer);
   1075 }
   1076 
   1077 static void
   1078 wdc_atapi_done(struct ata_channel *chp, struct ata_xfer *xfer)
   1079 {
   1080 	struct atac_softc *atac = chp->ch_atac;
   1081 	struct scsipi_xfer *sc_xfer = xfer->c_cmd;
   1082 	int drive = xfer->c_drive;
   1083 
   1084 	ATADEBUG_PRINT(("wdc_atapi_done %s:%d:%d: flags 0x%x\n",
   1085 	    device_xname(atac->atac_dev), chp->ch_channel, xfer->c_drive,
   1086 	    (u_int)xfer->c_flags), DEBUG_XFERS);
   1087 	callout_stop(&chp->ch_callout);
   1088 	/* mark controller inactive and free the command */
   1089 	chp->ch_queue->active_xfer = NULL;
   1090 	ata_free_xfer(chp, xfer);
   1091 
   1092 	if (chp->ch_drive[drive].drive_flags & DRIVE_WAITDRAIN) {
   1093 		sc_xfer->error = XS_DRIVER_STUFFUP;
   1094 		chp->ch_drive[drive].drive_flags &= ~DRIVE_WAITDRAIN;
   1095 		wakeup(&chp->ch_queue->active_xfer);
   1096 	}
   1097 
   1098 	ATADEBUG_PRINT(("wdc_atapi_done: scsipi_done\n"), DEBUG_XFERS);
   1099 	scsipi_done(sc_xfer);
   1100 	ATADEBUG_PRINT(("atastart from wdc_atapi_done, flags 0x%x\n",
   1101 	    chp->ch_flags), DEBUG_XFERS);
   1102 	atastart(chp);
   1103 }
   1104 
   1105 static void
   1106 wdc_atapi_reset(struct ata_channel *chp, struct ata_xfer *xfer)
   1107 {
   1108 	struct atac_softc *atac = chp->ch_atac;
   1109 	struct ata_drive_datas *drvp = &chp->ch_drive[xfer->c_drive];
   1110 	struct scsipi_xfer *sc_xfer = xfer->c_cmd;
   1111 
   1112 	wdccommandshort(chp, xfer->c_drive, ATAPI_SOFT_RESET);
   1113 	drvp->state = 0;
   1114 	if (wdc_wait_for_unbusy(chp, WDC_RESET_WAIT, AT_POLL) != 0) {
   1115 		printf("%s:%d:%d: reset failed\n",
   1116 		    device_xname(atac->atac_dev), chp->ch_channel,
   1117 		    xfer->c_drive);
   1118 		sc_xfer->error = XS_SELTIMEOUT;
   1119 	}
   1120 	wdc_atapi_done(chp, xfer);
   1121 	return;
   1122 }
   1123 
   1124 static void
   1125 wdc_atapi_polldsc(void *arg)
   1126 {
   1127 
   1128 	wdc_atapi_phase_complete(arg);
   1129 }
   1130