Home | History | Annotate | Line # | Download | only in scsipi
atapi_wdc.c revision 1.105
      1 /*	$NetBSD: atapi_wdc.c,v 1.105 2007/10/19 12:01:18 ad Exp $	*/
      2 
      3 /*
      4  * Copyright (c) 1998, 2001 Manuel Bouyer.
      5  *
      6  * Redistribution and use in source and binary forms, with or without
      7  * modification, are permitted provided that the following conditions
      8  * are met:
      9  * 1. Redistributions of source code must retain the above copyright
     10  *    notice, this list of conditions and the following disclaimer.
     11  * 2. Redistributions in binary form must reproduce the above copyright
     12  *    notice, this list of conditions and the following disclaimer in the
     13  *    documentation and/or other materials provided with the distribution.
     14  * 3. All advertising materials mentioning features or use of this software
     15  *    must display the following acknowledgement:
     16  *	This product includes software developed by Manuel Bouyer.
     17  * 4. The name of the author may not be used to endorse or promote products
     18  *    derived from this software without specific prior written permission.
     19  *
     20  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
     21  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
     22  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
     23  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
     24  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
     25  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
     26  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
     27  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
     28  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
     29  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     30  */
     31 
     32 #include <sys/cdefs.h>
     33 __KERNEL_RCSID(0, "$NetBSD: atapi_wdc.c,v 1.105 2007/10/19 12:01:18 ad Exp $");
     34 
     35 #ifndef ATADEBUG
     36 #define ATADEBUG
     37 #endif /* ATADEBUG */
     38 
     39 #include <sys/param.h>
     40 #include <sys/systm.h>
     41 #include <sys/kernel.h>
     42 #include <sys/file.h>
     43 #include <sys/stat.h>
     44 #include <sys/buf.h>
     45 #include <sys/malloc.h>
     46 #include <sys/device.h>
     47 #include <sys/syslog.h>
     48 #include <sys/proc.h>
     49 #include <sys/dvdio.h>
     50 
     51 #include <sys/intr.h>
     52 #include <sys/bus.h>
     53 
     54 #ifndef __BUS_SPACE_HAS_STREAM_METHODS
     55 #define	bus_space_write_multi_stream_2	bus_space_write_multi_2
     56 #define	bus_space_write_multi_stream_4	bus_space_write_multi_4
     57 #define	bus_space_read_multi_stream_2	bus_space_read_multi_2
     58 #define	bus_space_read_multi_stream_4	bus_space_read_multi_4
     59 #endif /* __BUS_SPACE_HAS_STREAM_METHODS */
     60 
     61 #include <dev/ata/ataconf.h>
     62 #include <dev/ata/atareg.h>
     63 #include <dev/ata/atavar.h>
     64 #include <dev/ic/wdcreg.h>
     65 #include <dev/ic/wdcvar.h>
     66 
     67 #include <dev/scsipi/scsi_all.h> /* for SCSI status */
     68 
     69 #define DEBUG_INTR   0x01
     70 #define DEBUG_XFERS  0x02
     71 #define DEBUG_STATUS 0x04
     72 #define DEBUG_FUNCS  0x08
     73 #define DEBUG_PROBE  0x10
     74 #ifdef ATADEBUG
     75 int wdcdebug_atapi_mask = 0;
     76 #define ATADEBUG_PRINT(args, level) \
     77 	if (wdcdebug_atapi_mask & (level)) \
     78 		printf args
     79 #else
     80 #define ATADEBUG_PRINT(args, level)
     81 #endif
     82 
     83 #define ATAPI_DELAY 10	/* 10 ms, this is used only before sending a cmd */
     84 #define ATAPI_MODE_DELAY 1000	/* 1s, timeout for SET_FEATYRE cmds */
     85 
     86 static int	wdc_atapi_get_params(struct scsipi_channel *, int,
     87 				     struct ataparams *);
     88 static void	wdc_atapi_probe_device(struct atapibus_softc *, int);
     89 static void	wdc_atapi_minphys (struct buf *bp);
     90 static void	wdc_atapi_start(struct ata_channel *,struct ata_xfer *);
     91 static int	wdc_atapi_intr(struct ata_channel *, struct ata_xfer *, int);
     92 static void	wdc_atapi_kill_xfer(struct ata_channel *,
     93 				    struct ata_xfer *, int);
     94 static void	wdc_atapi_phase_complete(struct ata_xfer *);
     95 static void	wdc_atapi_done(struct ata_channel *, struct ata_xfer *);
     96 static void	wdc_atapi_reset(struct ata_channel *, struct ata_xfer *);
     97 static void	wdc_atapi_scsipi_request(struct scsipi_channel *,
     98 					 scsipi_adapter_req_t, void *);
     99 static void	wdc_atapi_kill_pending(struct scsipi_periph *);
    100 static void	wdc_atapi_polldsc(void *arg);
    101 
    102 #define MAX_SIZE MAXPHYS
    103 
    104 static const struct scsipi_bustype wdc_atapi_bustype = {
    105 	SCSIPI_BUSTYPE_ATAPI,
    106 	atapi_scsipi_cmd,
    107 	atapi_interpret_sense,
    108 	atapi_print_addr,
    109 	wdc_atapi_kill_pending,
    110 };
    111 
    112 void
    113 wdc_atapibus_attach(struct atabus_softc *ata_sc)
    114 {
    115 	struct ata_channel *chp = ata_sc->sc_chan;
    116 	struct atac_softc *atac = chp->ch_atac;
    117 	struct scsipi_adapter *adapt = &atac->atac_atapi_adapter._generic;
    118 	struct scsipi_channel *chan = &chp->ch_atapi_channel;
    119 
    120 	/*
    121 	 * Fill in the scsipi_adapter.
    122 	 */
    123 	adapt->adapt_dev = &atac->atac_dev;
    124 	adapt->adapt_nchannels = atac->atac_nchannels;
    125 	adapt->adapt_request = wdc_atapi_scsipi_request;
    126 	adapt->adapt_minphys = wdc_atapi_minphys;
    127 	if (atac->atac_cap & ATAC_CAP_NOIRQ)
    128 		adapt->adapt_flags |= SCSIPI_ADAPT_POLL_ONLY;
    129 	atac->atac_atapi_adapter.atapi_probe_device = wdc_atapi_probe_device;
    130 
    131 	/*
    132 	 * Fill in the scsipi_channel.
    133 	 */
    134 	memset(chan, 0, sizeof(*chan));
    135 	chan->chan_adapter = adapt;
    136 	chan->chan_bustype = &wdc_atapi_bustype;
    137 	chan->chan_channel = chp->ch_channel;
    138 	chan->chan_flags = SCSIPI_CHAN_OPENINGS;
    139 	chan->chan_openings = 1;
    140 	chan->chan_max_periph = 1;
    141 	chan->chan_ntargets = 2;
    142 	chan->chan_nluns = 1;
    143 
    144 	chp->atapibus = config_found_ia(&ata_sc->sc_dev, "atapi", chan,
    145 		atapiprint);
    146 }
    147 
    148 static void
    149 wdc_atapi_minphys(struct buf *bp)
    150 {
    151 
    152 	if (bp->b_bcount > MAX_SIZE)
    153 		bp->b_bcount = MAX_SIZE;
    154 	minphys(bp);
    155 }
    156 
    157 /*
    158  * Kill off all pending xfers for a periph.
    159  *
    160  * Must be called at splbio().
    161  */
    162 static void
    163 wdc_atapi_kill_pending(struct scsipi_periph *periph)
    164 {
    165 	struct atac_softc *atac =
    166 	    (void *)periph->periph_channel->chan_adapter->adapt_dev;
    167 	struct ata_channel *chp =
    168 	    atac->atac_channels[periph->periph_channel->chan_channel];
    169 
    170 	ata_kill_pending(&chp->ch_drive[periph->periph_target]);
    171 }
    172 
    173 static void
    174 wdc_atapi_kill_xfer(struct ata_channel *chp, struct ata_xfer *xfer, int reason)
    175 {
    176 	struct scsipi_xfer *sc_xfer = xfer->c_cmd;
    177 
    178 	/* remove this command from xfer queue */
    179 	switch (reason) {
    180 	case KILL_GONE:
    181 		sc_xfer->error = XS_DRIVER_STUFFUP;
    182 		break;
    183 	case KILL_RESET:
    184 		sc_xfer->error = XS_RESET;
    185 		break;
    186 	default:
    187 		printf("wdc_ata_bio_kill_xfer: unknown reason %d\n",
    188 		    reason);
    189 		panic("wdc_ata_bio_kill_xfer");
    190 	}
    191 	ata_free_xfer(chp, xfer);
    192 	scsipi_done(sc_xfer);
    193 }
    194 
    195 static int
    196 wdc_atapi_get_params(struct scsipi_channel *chan, int drive,
    197     struct ataparams *id)
    198 {
    199 	struct wdc_softc *wdc = (void *)chan->chan_adapter->adapt_dev;
    200 	struct atac_softc *atac = &wdc->sc_atac;
    201 	struct wdc_regs *wdr = &wdc->regs[chan->chan_channel];
    202 	struct ata_channel *chp = atac->atac_channels[chan->chan_channel];
    203 	struct ata_command ata_c;
    204 
    205 	/* if no ATAPI device detected at wdc attach time, skip */
    206 	if ((chp->ch_drive[drive].drive_flags & DRIVE_ATAPI) == 0) {
    207 		ATADEBUG_PRINT(("wdc_atapi_get_params: drive %d not present\n",
    208 		    drive), DEBUG_PROBE);
    209 		return -1;
    210 	}
    211 
    212 	memset(&ata_c, 0, sizeof(struct ata_command));
    213 	ata_c.r_command = ATAPI_SOFT_RESET;
    214 	ata_c.r_st_bmask = 0;
    215 	ata_c.r_st_pmask = 0;
    216 	ata_c.flags = AT_WAIT | AT_POLL;
    217 	ata_c.timeout = WDC_RESET_WAIT;
    218 	if (wdc_exec_command(&chp->ch_drive[drive], &ata_c) != ATACMD_COMPLETE) {
    219 		printf("wdc_atapi_get_params: ATAPI_SOFT_RESET failed for"
    220 		    " drive %s:%d:%d: driver failed\n",
    221 		    atac->atac_dev.dv_xname, chp->ch_channel, drive);
    222 		panic("wdc_atapi_get_params");
    223 	}
    224 	if (ata_c.flags & (AT_ERROR | AT_TIMEOU | AT_DF)) {
    225 		ATADEBUG_PRINT(("wdc_atapi_get_params: ATAPI_SOFT_RESET "
    226 		    "failed for drive %s:%d:%d: error 0x%x\n",
    227 		    atac->atac_dev.dv_xname, chp->ch_channel, drive,
    228 		    ata_c.r_error), DEBUG_PROBE);
    229 		return -1;
    230 	}
    231 	chp->ch_drive[drive].state = 0;
    232 
    233 	(void)bus_space_read_1(wdr->cmd_iot, wdr->cmd_iohs[wd_status], 0);
    234 
    235 	/* Some ATAPI devices need a bit more time after software reset. */
    236 	delay(5000);
    237 	if (ata_get_params(&chp->ch_drive[drive], AT_WAIT, id) != 0) {
    238 		ATADEBUG_PRINT(("wdc_atapi_get_params: ATAPI_IDENTIFY_DEVICE "
    239 		    "failed for drive %s:%d:%d: error 0x%x\n",
    240 		    atac->atac_dev.dv_xname, chp->ch_channel, drive,
    241 		    ata_c.r_error), DEBUG_PROBE);
    242 		return -1;
    243 	}
    244 	return 0;
    245 }
    246 
    247 static void
    248 wdc_atapi_probe_device(struct atapibus_softc *sc, int target)
    249 {
    250 	struct scsipi_channel *chan = sc->sc_channel;
    251 	struct scsipi_periph *periph;
    252 	struct ataparams ids;
    253 	struct ataparams *id = &ids;
    254 	struct wdc_softc *wdc = (void *)chan->chan_adapter->adapt_dev;
    255 	struct atac_softc *atac = &wdc->sc_atac;
    256 	struct ata_channel *chp = atac->atac_channels[chan->chan_channel];
    257 	struct ata_drive_datas *drvp = &chp->ch_drive[target];
    258 	struct scsipibus_attach_args sa;
    259 	char serial_number[21], model[41], firmware_revision[9];
    260 	int s;
    261 
    262 	/* skip if already attached */
    263 	if (scsipi_lookup_periph(chan, target, 0) != NULL)
    264 		return;
    265 
    266 	if (wdc_atapi_get_params(chan, target, id) == 0) {
    267 #ifdef ATAPI_DEBUG_PROBE
    268 		printf("%s drive %d: cmdsz 0x%x drqtype 0x%x\n",
    269 		    sc->sc_dev.dv_xname, target,
    270 		    id->atap_config & ATAPI_CFG_CMD_MASK,
    271 		    id->atap_config & ATAPI_CFG_DRQ_MASK);
    272 #endif
    273 		periph = scsipi_alloc_periph(M_NOWAIT);
    274 		if (periph == NULL) {
    275 			printf("%s: unable to allocate periph for drive %d\n",
    276 			    sc->sc_dev.dv_xname, target);
    277 			return;
    278 		}
    279 		periph->periph_dev = NULL;
    280 		periph->periph_channel = chan;
    281 		periph->periph_switch = &atapi_probe_periphsw;
    282 		periph->periph_target = target;
    283 		periph->periph_lun = 0;
    284 		periph->periph_quirks = PQUIRK_ONLYBIG;
    285 
    286 #ifdef SCSIPI_DEBUG
    287 		if (SCSIPI_DEBUG_TYPE == SCSIPI_BUSTYPE_ATAPI &&
    288 		    SCSIPI_DEBUG_TARGET == target)
    289 			periph->periph_dbflags |= SCSIPI_DEBUG_FLAGS;
    290 #endif
    291 		periph->periph_type = ATAPI_CFG_TYPE(id->atap_config);
    292 		if (id->atap_config & ATAPI_CFG_REMOV)
    293 			periph->periph_flags |= PERIPH_REMOVABLE;
    294 		if (periph->periph_type == T_SEQUENTIAL) {
    295 			s = splbio();
    296 			drvp->drive_flags |= DRIVE_ATAPIST;
    297 			splx(s);
    298 		}
    299 
    300 		sa.sa_periph = periph;
    301 		sa.sa_inqbuf.type =  ATAPI_CFG_TYPE(id->atap_config);
    302 		sa.sa_inqbuf.removable = id->atap_config & ATAPI_CFG_REMOV ?
    303 		    T_REMOV : T_FIXED;
    304 		scsipi_strvis((u_char *)model, 40, id->atap_model, 40);
    305 		scsipi_strvis((u_char *)serial_number, 20, id->atap_serial,
    306 		    20);
    307 		scsipi_strvis((u_char *)firmware_revision, 8,
    308 		    id->atap_revision, 8);
    309 		sa.sa_inqbuf.vendor = model;
    310 		sa.sa_inqbuf.product = serial_number;
    311 		sa.sa_inqbuf.revision = firmware_revision;
    312 
    313 		/*
    314 		 * Determine the operating mode capabilities of the device.
    315 		 */
    316 		if ((id->atap_config & ATAPI_CFG_CMD_MASK) == ATAPI_CFG_CMD_16)
    317 			periph->periph_cap |= PERIPH_CAP_CMD16;
    318 		/* XXX This is gross. */
    319 		periph->periph_cap |= (id->atap_config & ATAPI_CFG_DRQ_MASK);
    320 
    321 		drvp->drv_softc = atapi_probe_device(sc, target, periph, &sa);
    322 
    323 		if (drvp->drv_softc)
    324 			ata_probe_caps(drvp);
    325 		else {
    326 			s = splbio();
    327 			drvp->drive_flags &= ~DRIVE_ATAPI;
    328 			splx(s);
    329 		}
    330 	} else {
    331 		s = splbio();
    332 		drvp->drive_flags &= ~DRIVE_ATAPI;
    333 		splx(s);
    334 	}
    335 }
    336 
    337 static void
    338 wdc_atapi_scsipi_request(struct scsipi_channel *chan, scsipi_adapter_req_t req,
    339     void *arg)
    340 {
    341 	struct scsipi_adapter *adapt = chan->chan_adapter;
    342 	struct scsipi_periph *periph;
    343 	struct scsipi_xfer *sc_xfer;
    344 	struct wdc_softc *wdc = (void *)adapt->adapt_dev;
    345 	struct atac_softc *atac = &wdc->sc_atac;
    346 	struct ata_xfer *xfer;
    347 	int channel = chan->chan_channel;
    348 	int drive, s;
    349 
    350 	switch (req) {
    351 	case ADAPTER_REQ_RUN_XFER:
    352 		sc_xfer = arg;
    353 		periph = sc_xfer->xs_periph;
    354 		drive = periph->periph_target;
    355 
    356 		ATADEBUG_PRINT(("wdc_atapi_scsipi_request %s:%d:%d\n",
    357 		    atac->atac_dev.dv_xname, channel, drive), DEBUG_XFERS);
    358 		if (!device_is_active(&atac->atac_dev)) {
    359 			sc_xfer->error = XS_DRIVER_STUFFUP;
    360 			scsipi_done(sc_xfer);
    361 			return;
    362 		}
    363 
    364 		xfer = ata_get_xfer(ATAXF_NOSLEEP);
    365 		if (xfer == NULL) {
    366 			sc_xfer->error = XS_RESOURCE_SHORTAGE;
    367 			scsipi_done(sc_xfer);
    368 			return;
    369 		}
    370 
    371 		if (sc_xfer->xs_control & XS_CTL_POLL)
    372 			xfer->c_flags |= C_POLL;
    373 #if NATA_DMA
    374 		if ((atac->atac_channels[channel]->ch_drive[drive].drive_flags &
    375 		    (DRIVE_DMA | DRIVE_UDMA)) && sc_xfer->datalen > 0)
    376 			xfer->c_flags |= C_DMA;
    377 #endif
    378 #if NATA_DMA && NATA_PIOBM
    379 		else
    380 #endif
    381 #if NATA_PIOBM
    382 		if ((atac->atac_cap & ATAC_CAP_PIOBM) &&
    383 		    sc_xfer->datalen > 0)
    384 			xfer->c_flags |= C_PIOBM;
    385 #endif
    386 		xfer->c_drive = drive;
    387 		xfer->c_flags |= C_ATAPI;
    388 #if NATA_DMA
    389 		if (sc_xfer->cmd->opcode == GPCMD_REPORT_KEY ||
    390 		    sc_xfer->cmd->opcode == GPCMD_SEND_KEY ||
    391 		    sc_xfer->cmd->opcode == GPCMD_READ_DVD_STRUCTURE) {
    392 			/*
    393 			 * DVD authentication commands must always be done in
    394 			 * PIO mode.
    395 			 */
    396 			xfer->c_flags &= ~C_DMA;
    397 		}
    398 
    399 		/*
    400 		 * DMA normally can't deal with transfers which are not a
    401 		 * multiple of its databus width. It's a bug to request odd
    402 		 * length transfers for ATAPI.
    403 		 *
    404 		 * Some devices also can't cope with unaligned DMA xfers
    405 		 * either. Also some devices seem to not handle DMA xfers of
    406 		 * less than 4 bytes.
    407 		 *
    408 		 * By enforcing at least 4 byte aligned offset and length for
    409 		 * DMA, we might use PIO where DMA could be allowed but better
    410 		 * safe than sorry as recent problems proved.
    411 		 *
    412 		 * Offending structures that are thus done by PIO instead of
    413 		 * DMA are normally small structures since all bulkdata is
    414 		 * aligned. But as the request may come from userland, we have
    415 		 * to protect against it anyway.
    416 		 *
    417 		 * XXX check for the 32 bit wide flag?
    418 		 */
    419 
    420 		if (((uintptr_t) sc_xfer->data) & 0x03)
    421 			xfer->c_flags &= ~C_DMA;
    422 		if ((sc_xfer->datalen < 4) || (sc_xfer->datalen & 0x03))
    423 			xfer->c_flags &= ~C_DMA;
    424 #endif	/* NATA_DMA */
    425 
    426 		xfer->c_cmd = sc_xfer;
    427 		xfer->c_databuf = sc_xfer->data;
    428 		xfer->c_bcount = sc_xfer->datalen;
    429 		xfer->c_start = wdc_atapi_start;
    430 		xfer->c_intr = wdc_atapi_intr;
    431 		xfer->c_kill_xfer = wdc_atapi_kill_xfer;
    432 		xfer->c_dscpoll = 0;
    433 		s = splbio();
    434 		ata_exec_xfer(atac->atac_channels[channel], xfer);
    435 #ifdef DIAGNOSTIC
    436 		if ((sc_xfer->xs_control & XS_CTL_POLL) != 0 &&
    437 		    (sc_xfer->xs_status & XS_STS_DONE) == 0)
    438 			panic("wdc_atapi_scsipi_request: polled command "
    439 			    "not done");
    440 #endif
    441 		splx(s);
    442 		return;
    443 
    444 	default:
    445 		/* Not supported, nothing to do. */
    446 		;
    447 	}
    448 }
    449 
    450 static void
    451 wdc_atapi_start(struct ata_channel *chp, struct ata_xfer *xfer)
    452 {
    453 	struct atac_softc *atac = chp->ch_atac;
    454 	struct wdc_softc *wdc = CHAN_TO_WDC(chp);
    455 	struct wdc_regs *wdr = &wdc->regs[chp->ch_channel];
    456 	struct scsipi_xfer *sc_xfer = xfer->c_cmd;
    457 	struct ata_drive_datas *drvp = &chp->ch_drive[xfer->c_drive];
    458 	int wait_flags = (sc_xfer->xs_control & XS_CTL_POLL) ? AT_POLL : 0;
    459 	const char *errstring;
    460 
    461 	ATADEBUG_PRINT(("wdc_atapi_start %s:%d:%d, scsi flags 0x%x \n",
    462 	    atac->atac_dev.dv_xname, chp->ch_channel, drvp->drive,
    463 	    sc_xfer->xs_control), DEBUG_XFERS);
    464 #if NATA_DMA
    465 	if ((xfer->c_flags & C_DMA) && (drvp->n_xfers <= NXFER))
    466 		drvp->n_xfers++;
    467 #endif
    468 	/* Do control operations specially. */
    469 	if (__predict_false(drvp->state < READY)) {
    470 		/* If it's not a polled command, we need the kernel thread */
    471 		if ((sc_xfer->xs_control & XS_CTL_POLL) == 0 &&
    472 		    (chp->ch_flags & ATACH_TH_RUN) == 0) {
    473 			chp->ch_queue->queue_freeze++;
    474 			wakeup(&chp->ch_thread);
    475 			return;
    476 		}
    477 		/*
    478 		 * disable interrupts, all commands here should be quick
    479 		 * enouth to be able to poll, and we don't go here that often
    480 		 */
    481 		 bus_space_write_1(wdr->ctl_iot, wdr->ctl_ioh, wd_aux_ctlr,
    482 		     WDCTL_4BIT | WDCTL_IDS);
    483 		if (wdc->select)
    484 			wdc->select(chp, xfer->c_drive);
    485 		bus_space_write_1(wdr->cmd_iot, wdr->cmd_iohs[wd_sdh], 0,
    486 		    WDSD_IBM | (xfer->c_drive << 4));
    487 		/* Don't try to set mode if controller can't be adjusted */
    488 		if (atac->atac_set_modes == NULL)
    489 			goto ready;
    490 		/* Also don't try if the drive didn't report its mode */
    491 		if ((drvp->drive_flags & DRIVE_MODE) == 0)
    492 			goto ready;
    493 		errstring = "unbusy";
    494 		if (wdc_wait_for_unbusy(chp, ATAPI_DELAY, wait_flags))
    495 			goto timeout;
    496 		wdccommand(chp, drvp->drive, SET_FEATURES, 0, 0, 0,
    497 		    0x08 | drvp->PIO_mode, WDSF_SET_MODE);
    498 		errstring = "piomode";
    499 		if (wdc_wait_for_unbusy(chp, ATAPI_MODE_DELAY, wait_flags))
    500 			goto timeout;
    501 		if (chp->ch_status & WDCS_ERR) {
    502 			if (chp->ch_error == WDCE_ABRT) {
    503 				/*
    504 				 * Some ATAPI drives reject PIO settings.
    505 				 * Fall back to PIO mode 3 since that's the
    506 				 * minimum for ATAPI.
    507 				 */
    508 				printf("%s:%d:%d: PIO mode %d rejected, "
    509 				    "falling back to PIO mode 3\n",
    510 				    atac->atac_dev.dv_xname,
    511 				    chp->ch_channel, xfer->c_drive,
    512 				    drvp->PIO_mode);
    513 				if (drvp->PIO_mode > 3)
    514 					drvp->PIO_mode = 3;
    515 			} else
    516 				goto error;
    517 		}
    518 #if NATA_DMA
    519 #if NATA_UDMA
    520 		if (drvp->drive_flags & DRIVE_UDMA) {
    521 			wdccommand(chp, drvp->drive, SET_FEATURES, 0, 0, 0,
    522 			    0x40 | drvp->UDMA_mode, WDSF_SET_MODE);
    523 		} else
    524 #endif
    525 		if (drvp->drive_flags & DRIVE_DMA) {
    526 			wdccommand(chp, drvp->drive, SET_FEATURES, 0, 0, 0,
    527 			    0x20 | drvp->DMA_mode, WDSF_SET_MODE);
    528 		} else {
    529 			goto ready;
    530 		}
    531 		errstring = "dmamode";
    532 		if (wdc_wait_for_unbusy(chp, ATAPI_MODE_DELAY, wait_flags))
    533 			goto timeout;
    534 		if (chp->ch_status & WDCS_ERR) {
    535 			if (chp->ch_error == WDCE_ABRT) {
    536 #if NATA_UDMA
    537 				if (drvp->drive_flags & DRIVE_UDMA)
    538 					goto error;
    539 				else
    540 #endif
    541 				{
    542 					/*
    543 					 * The drive rejected our DMA setting.
    544 					 * Fall back to mode 1.
    545 					 */
    546 					printf("%s:%d:%d: DMA mode %d rejected, "
    547 					    "falling back to DMA mode 0\n",
    548 					    atac->atac_dev.dv_xname,
    549 					    chp->ch_channel, xfer->c_drive,
    550 					    drvp->DMA_mode);
    551 					if (drvp->DMA_mode > 0)
    552 						drvp->DMA_mode = 0;
    553 				}
    554 			} else
    555 				goto error;
    556 		}
    557 #endif	/* NATA_DMA */
    558 ready:
    559 		drvp->state = READY;
    560 		bus_space_write_1(wdr->ctl_iot, wdr->ctl_ioh, wd_aux_ctlr,
    561 		    WDCTL_4BIT);
    562 		delay(10); /* some drives need a little delay here */
    563 	}
    564 	/* start timeout machinery */
    565 	if ((sc_xfer->xs_control & XS_CTL_POLL) == 0)
    566 		callout_reset(&chp->ch_callout, mstohz(sc_xfer->timeout),
    567 		    wdctimeout, chp);
    568 
    569 	if (wdc->select)
    570 		wdc->select(chp, xfer->c_drive);
    571 	bus_space_write_1(wdr->cmd_iot, wdr->cmd_iohs[wd_sdh], 0,
    572 	    WDSD_IBM | (xfer->c_drive << 4));
    573 	switch (wdc_wait_for_unbusy(chp, ATAPI_DELAY, wait_flags)  < 0) {
    574 	case WDCWAIT_OK:
    575 		break;
    576 	case WDCWAIT_TOUT:
    577 		printf("wdc_atapi_start: not ready, st = %02x\n",
    578 		    chp->ch_status);
    579 		sc_xfer->error = XS_TIMEOUT;
    580 		wdc_atapi_reset(chp, xfer);
    581 		return;
    582 	case WDCWAIT_THR:
    583 		return;
    584 	}
    585 
    586 	/*
    587 	 * Even with WDCS_ERR, the device should accept a command packet
    588 	 * Limit length to what can be stuffed into the cylinder register
    589 	 * (16 bits).  Some CD-ROMs seem to interpret '0' as 65536,
    590 	 * but not all devices do that and it's not obvious from the
    591 	 * ATAPI spec that that behaviour should be expected.  If more
    592 	 * data is necessary, multiple data transfer phases will be done.
    593 	 */
    594 
    595 	wdccommand(chp, xfer->c_drive, ATAPI_PKT_CMD,
    596 	    xfer->c_bcount <= 0xffff ? xfer->c_bcount : 0xffff,
    597 	    0, 0, 0,
    598 #if NATA_DMA
    599 	    (xfer->c_flags & C_DMA) ? ATAPI_PKT_CMD_FTRE_DMA :
    600 #endif
    601 	    0
    602 	    );
    603 
    604 #if NATA_PIOBM
    605 	if (xfer->c_flags & C_PIOBM) {
    606 		int error;
    607 		int dma_flags = (sc_xfer->xs_control & XS_CTL_DATA_IN)
    608 		    ?  WDC_DMA_READ : 0;
    609 		if (xfer->c_flags & C_POLL) {
    610 			/* XXX not supported yet --- fall back to PIO */
    611 			xfer->c_flags &= ~C_PIOBM;
    612 		} else {
    613 			/* Init the DMA channel. */
    614 			error = (*wdc->dma_init)(wdc->dma_arg,
    615 			    chp->ch_channel, xfer->c_drive,
    616 			    (char *)xfer->c_databuf,
    617 			    xfer->c_bcount,
    618 			    dma_flags | WDC_DMA_PIOBM_ATAPI);
    619 			if (error) {
    620 				if (error == EINVAL) {
    621 					/*
    622 					 * We can't do DMA on this transfer
    623 					 * for some reason.  Fall back to
    624 					 * PIO.
    625 					 */
    626 					xfer->c_flags &= ~C_PIOBM;
    627 					error = 0;
    628 				} else {
    629 					sc_xfer->error = XS_DRIVER_STUFFUP;
    630 					errstring = "piobm";
    631 					goto error;
    632 				}
    633 			}
    634 		}
    635 	}
    636 #endif
    637 	/*
    638 	 * If there is no interrupt for CMD input, busy-wait for it (done in
    639 	 * the interrupt routine. If it is a polled command, call the interrupt
    640 	 * routine until command is done.
    641 	 */
    642 	if ((sc_xfer->xs_periph->periph_cap & ATAPI_CFG_DRQ_MASK) !=
    643 	    ATAPI_CFG_IRQ_DRQ || (sc_xfer->xs_control & XS_CTL_POLL)) {
    644 		/* Wait for at last 400ns for status bit to be valid */
    645 		DELAY(1);
    646 		wdc_atapi_intr(chp, xfer, 0);
    647 	} else {
    648 		chp->ch_flags |= ATACH_IRQ_WAIT;
    649 	}
    650 	if (sc_xfer->xs_control & XS_CTL_POLL) {
    651 #if NATA_DMA
    652 		if (chp->ch_flags & ATACH_DMA_WAIT) {
    653 			wdc_dmawait(chp, xfer, sc_xfer->timeout);
    654 			chp->ch_flags &= ~ATACH_DMA_WAIT;
    655 		}
    656 #endif
    657 		while ((sc_xfer->xs_status & XS_STS_DONE) == 0) {
    658 			/* Wait for at last 400ns for status bit to be valid */
    659 			DELAY(1);
    660 			wdc_atapi_intr(chp, xfer, 0);
    661 		}
    662 	}
    663 	return;
    664 timeout:
    665 	printf("%s:%d:%d: %s timed out\n",
    666 	    atac->atac_dev.dv_xname, chp->ch_channel, xfer->c_drive,
    667 	    errstring);
    668 	sc_xfer->error = XS_TIMEOUT;
    669 	bus_space_write_1(wdr->ctl_iot, wdr->ctl_ioh, wd_aux_ctlr, WDCTL_4BIT);
    670 	delay(10); /* some drives need a little delay here */
    671 	wdc_atapi_reset(chp, xfer);
    672 	return;
    673 error:
    674 	printf("%s:%d:%d: %s ",
    675 	    atac->atac_dev.dv_xname, chp->ch_channel, xfer->c_drive,
    676 	    errstring);
    677 	printf("error (0x%x)\n", chp->ch_error);
    678 	sc_xfer->error = XS_SHORTSENSE;
    679 	sc_xfer->sense.atapi_sense = chp->ch_error;
    680 	bus_space_write_1(wdr->ctl_iot, wdr->ctl_ioh, wd_aux_ctlr, WDCTL_4BIT);
    681 	delay(10); /* some drives need a little delay here */
    682 	wdc_atapi_reset(chp, xfer);
    683 	return;
    684 }
    685 
    686 static int
    687 wdc_atapi_intr(struct ata_channel *chp, struct ata_xfer *xfer, int irq)
    688 {
    689 	struct atac_softc *atac = chp->ch_atac;
    690 	struct wdc_softc *wdc = CHAN_TO_WDC(chp);
    691 	struct wdc_regs *wdr = &wdc->regs[chp->ch_channel];
    692 	struct scsipi_xfer *sc_xfer = xfer->c_cmd;
    693 	struct ata_drive_datas *drvp = &chp->ch_drive[xfer->c_drive];
    694 	int len, phase, i, retries=0;
    695 	int ire;
    696 #if NATA_DMA
    697 	int error;
    698 #endif
    699 #if NATA_DMA || NATA_PIOBM
    700 	int dma_flags = 0;
    701 #endif
    702 	void *cmd;
    703 
    704 	ATADEBUG_PRINT(("wdc_atapi_intr %s:%d:%d\n",
    705 	    atac->atac_dev.dv_xname, chp->ch_channel, drvp->drive),
    706 	    DEBUG_INTR);
    707 
    708 	/* Is it not a transfer, but a control operation? */
    709 	if (drvp->state < READY) {
    710 		printf("%s:%d:%d: bad state %d in wdc_atapi_intr\n",
    711 		    atac->atac_dev.dv_xname, chp->ch_channel, xfer->c_drive,
    712 		    drvp->state);
    713 		panic("wdc_atapi_intr: bad state");
    714 	}
    715 	/*
    716 	 * If we missed an interrupt in a PIO transfer, reset and restart.
    717 	 * Don't try to continue transfer, we may have missed cycles.
    718 	 */
    719 	if ((xfer->c_flags & (C_TIMEOU | C_DMA)) == C_TIMEOU) {
    720 		sc_xfer->error = XS_TIMEOUT;
    721 		wdc_atapi_reset(chp, xfer);
    722 		return 1;
    723 	}
    724 
    725 #if NATA_PIOBM
    726 	/* Transfer-done interrupt for busmastering PIO operation */
    727 	if ((xfer->c_flags & C_PIOBM) && (chp->ch_flags & ATACH_PIOBM_WAIT)) {
    728 		chp->ch_flags &= ~ATACH_PIOBM_WAIT;
    729 
    730 		/* restore transfer length */
    731 		len = xfer->c_bcount;
    732 		if (xfer->c_lenoff < 0)
    733 			len += xfer->c_lenoff;
    734 
    735 		if (sc_xfer->xs_control & XS_CTL_DATA_IN)
    736 			goto end_piobm_datain;
    737 		else
    738 			goto end_piobm_dataout;
    739 	}
    740 #endif
    741 
    742 	/* Ack interrupt done in wdc_wait_for_unbusy */
    743 	if (wdc->select)
    744 		wdc->select(chp, xfer->c_drive);
    745 	bus_space_write_1(wdr->cmd_iot, wdr->cmd_iohs[wd_sdh], 0,
    746 	    WDSD_IBM | (xfer->c_drive << 4));
    747 	if (wdc_wait_for_unbusy(chp,
    748 	    (irq == 0) ? sc_xfer->timeout : 0, AT_POLL) == WDCWAIT_TOUT) {
    749 		if (irq && (xfer->c_flags & C_TIMEOU) == 0)
    750 			return 0; /* IRQ was not for us */
    751 		printf("%s:%d:%d: device timeout, c_bcount=%d, c_skip=%d\n",
    752 		    atac->atac_dev.dv_xname, chp->ch_channel, xfer->c_drive,
    753 		    xfer->c_bcount, xfer->c_skip);
    754 #if NATA_DMA
    755 		if (xfer->c_flags & C_DMA) {
    756 			ata_dmaerr(drvp,
    757 			    (xfer->c_flags & C_POLL) ? AT_POLL : 0);
    758 		}
    759 #endif
    760 		sc_xfer->error = XS_TIMEOUT;
    761 		wdc_atapi_reset(chp, xfer);
    762 		return 1;
    763 	}
    764 	if (wdc->irqack)
    765 		wdc->irqack(chp);
    766 
    767 #if NATA_DMA
    768 	/*
    769 	 * If we missed an IRQ and were using DMA, flag it as a DMA error
    770 	 * and reset device.
    771 	 */
    772 	if ((xfer->c_flags & C_TIMEOU) && (xfer->c_flags & C_DMA)) {
    773 		ata_dmaerr(drvp, (xfer->c_flags & C_POLL) ? AT_POLL : 0);
    774 		sc_xfer->error = XS_RESET;
    775 		wdc_atapi_reset(chp, xfer);
    776 		return (1);
    777 	}
    778 #endif
    779 	/*
    780 	 * if the request sense command was aborted, report the short sense
    781 	 * previously recorded, else continue normal processing
    782 	 */
    783 
    784 #if NATA_DMA || NATA_PIOBM
    785 	if (xfer->c_flags & (C_DMA | C_PIOBM))
    786 		dma_flags = (sc_xfer->xs_control & XS_CTL_DATA_IN)
    787 		    ?  WDC_DMA_READ : 0;
    788 #endif
    789 again:
    790 	len = bus_space_read_1(wdr->cmd_iot, wdr->cmd_iohs[wd_cyl_lo], 0) +
    791 	    256 * bus_space_read_1(wdr->cmd_iot, wdr->cmd_iohs[wd_cyl_hi], 0);
    792 	ire = bus_space_read_1(wdr->cmd_iot, wdr->cmd_iohs[wd_ireason], 0);
    793 	phase = (ire & (WDCI_CMD | WDCI_IN)) | (chp->ch_status & WDCS_DRQ);
    794 	ATADEBUG_PRINT(("wdc_atapi_intr: c_bcount %d len %d st 0x%x err 0x%x "
    795 	    "ire 0x%x :", xfer->c_bcount,
    796 	    len, chp->ch_status, chp->ch_error, ire), DEBUG_INTR);
    797 
    798 	switch (phase) {
    799 	case PHASE_CMDOUT:
    800 		cmd = sc_xfer->cmd;
    801 		ATADEBUG_PRINT(("PHASE_CMDOUT\n"), DEBUG_INTR);
    802 #if NATA_DMA
    803 		/* Init the DMA channel if necessary */
    804 		if (xfer->c_flags & C_DMA) {
    805 			error = (*wdc->dma_init)(wdc->dma_arg,
    806 			    chp->ch_channel, xfer->c_drive,
    807 			    xfer->c_databuf, xfer->c_bcount, dma_flags);
    808 			if (error) {
    809 				if (error == EINVAL) {
    810 					/*
    811 					 * We can't do DMA on this transfer
    812 					 * for some reason.  Fall back to
    813 					 * PIO.
    814 					 */
    815 					xfer->c_flags &= ~C_DMA;
    816 					error = 0;
    817 				} else {
    818 					sc_xfer->error = XS_DRIVER_STUFFUP;
    819 					break;
    820 				}
    821 			}
    822 		}
    823 #endif
    824 
    825 		/* send packet command */
    826 		/* Commands are 12 or 16 bytes long. It's 32-bit aligned */
    827 		wdc->dataout_pio(chp, drvp->drive_flags, cmd, sc_xfer->cmdlen);
    828 
    829 #if NATA_DMA
    830 		/* Start the DMA channel if necessary */
    831 		if (xfer->c_flags & C_DMA) {
    832 			(*wdc->dma_start)(wdc->dma_arg,
    833 			    chp->ch_channel, xfer->c_drive);
    834 			chp->ch_flags |= ATACH_DMA_WAIT;
    835 		}
    836 #endif
    837 
    838 		if ((sc_xfer->xs_control & XS_CTL_POLL) == 0) {
    839 			chp->ch_flags |= ATACH_IRQ_WAIT;
    840 		}
    841 		return 1;
    842 
    843 	 case PHASE_DATAOUT:
    844 		/* write data */
    845 		ATADEBUG_PRINT(("PHASE_DATAOUT\n"), DEBUG_INTR);
    846 #if NATA_DMA
    847 		if ((sc_xfer->xs_control & XS_CTL_DATA_OUT) == 0 ||
    848 		    (xfer->c_flags & C_DMA) != 0) {
    849 			printf("wdc_atapi_intr: bad data phase DATAOUT\n");
    850 			if (xfer->c_flags & C_DMA) {
    851 				ata_dmaerr(drvp,
    852 				    (xfer->c_flags & C_POLL) ? AT_POLL : 0);
    853 			}
    854 			sc_xfer->error = XS_TIMEOUT;
    855 			wdc_atapi_reset(chp, xfer);
    856 			return 1;
    857 		}
    858 #endif
    859 		xfer->c_lenoff = len - xfer->c_bcount;
    860 		if (xfer->c_bcount < len) {
    861 			printf("wdc_atapi_intr: warning: write only "
    862 			    "%d of %d requested bytes\n", xfer->c_bcount, len);
    863 			len = xfer->c_bcount;
    864 		}
    865 
    866 #if NATA_PIOBM
    867 		if (xfer->c_flags & C_PIOBM) {
    868 			/* start the busmastering PIO */
    869 			(*wdc->piobm_start)(wdc->dma_arg,
    870 			    chp->ch_channel, xfer->c_drive,
    871 			    xfer->c_skip, len, WDC_PIOBM_XFER_IRQ);
    872 			chp->ch_flags |= ATACH_DMA_WAIT | ATACH_IRQ_WAIT |
    873 			    ATACH_PIOBM_WAIT;
    874 			return 1;
    875 		}
    876 #endif
    877 		wdc->dataout_pio(chp, drvp->drive_flags,
    878 	    	    (char *)xfer->c_databuf + xfer->c_skip, len);
    879 
    880 #if NATA_PIOBM
    881 	end_piobm_dataout:
    882 #endif
    883 		for (i = xfer->c_lenoff; i > 0; i -= 2)
    884 			bus_space_write_2(wdr->cmd_iot,
    885 			    wdr->cmd_iohs[wd_data], 0, 0);
    886 
    887 		xfer->c_skip += len;
    888 		xfer->c_bcount -= len;
    889 		if ((sc_xfer->xs_control & XS_CTL_POLL) == 0) {
    890 			chp->ch_flags |= ATACH_IRQ_WAIT;
    891 		}
    892 		return 1;
    893 
    894 	case PHASE_DATAIN:
    895 		/* Read data */
    896 		ATADEBUG_PRINT(("PHASE_DATAIN\n"), DEBUG_INTR);
    897 #if NATA_DMA
    898 		if ((sc_xfer->xs_control & XS_CTL_DATA_IN) == 0 ||
    899 		    (xfer->c_flags & C_DMA) != 0) {
    900 			printf("wdc_atapi_intr: bad data phase DATAIN\n");
    901 			if (xfer->c_flags & C_DMA) {
    902 				ata_dmaerr(drvp,
    903 				    (xfer->c_flags & C_POLL) ? AT_POLL : 0);
    904 			}
    905 			sc_xfer->error = XS_TIMEOUT;
    906 			wdc_atapi_reset(chp, xfer);
    907 			return 1;
    908 		}
    909 #endif
    910 		xfer->c_lenoff = len - xfer->c_bcount;
    911 		if (xfer->c_bcount < len) {
    912 			printf("wdc_atapi_intr: warning: reading only "
    913 			    "%d of %d bytes\n", xfer->c_bcount, len);
    914 			len = xfer->c_bcount;
    915 		}
    916 
    917 #if NATA_PIOBM
    918 		if (xfer->c_flags & C_PIOBM) {
    919 			/* start the busmastering PIO */
    920 			(*wdc->piobm_start)(wdc->dma_arg,
    921 			    chp->ch_channel, xfer->c_drive,
    922 			    xfer->c_skip, len, WDC_PIOBM_XFER_IRQ);
    923 			chp->ch_flags |= ATACH_DMA_WAIT | ATACH_IRQ_WAIT |
    924 			    ATACH_PIOBM_WAIT;
    925 			return 1;
    926 		}
    927 #endif
    928 		wdc->datain_pio(chp, drvp->drive_flags,
    929 		    (char *)xfer->c_databuf + xfer->c_skip, len);
    930 
    931 #if NATA_PIOBM
    932 	end_piobm_datain:
    933 #endif
    934 		if (xfer->c_lenoff > 0)
    935 			wdcbit_bucket(chp, len - xfer->c_bcount);
    936 
    937 		xfer->c_skip += len;
    938 		xfer->c_bcount -= len;
    939 		if ((sc_xfer->xs_control & XS_CTL_POLL) == 0) {
    940 			chp->ch_flags |= ATACH_IRQ_WAIT;
    941 		}
    942 		return 1;
    943 
    944 	case PHASE_ABORTED:
    945 	case PHASE_COMPLETED:
    946 		ATADEBUG_PRINT(("PHASE_COMPLETED\n"), DEBUG_INTR);
    947 #if NATA_DMA
    948 		if (xfer->c_flags & C_DMA) {
    949 			xfer->c_bcount -= sc_xfer->datalen;
    950 		}
    951 #endif
    952 		sc_xfer->resid = xfer->c_bcount;
    953 		wdc_atapi_phase_complete(xfer);
    954 		return(1);
    955 
    956 	default:
    957 		if (++retries<500) {
    958 			DELAY(100);
    959 			chp->ch_status = bus_space_read_1(wdr->cmd_iot,
    960 			    wdr->cmd_iohs[wd_status], 0);
    961 			chp->ch_error = bus_space_read_1(wdr->cmd_iot,
    962 			    wdr->cmd_iohs[wd_error], 0);
    963 			goto again;
    964 		}
    965 		printf("wdc_atapi_intr: unknown phase 0x%x\n", phase);
    966 		if (chp->ch_status & WDCS_ERR) {
    967 			sc_xfer->error = XS_SHORTSENSE;
    968 			sc_xfer->sense.atapi_sense = chp->ch_error;
    969 		} else {
    970 #if NATA_DMA
    971 			if (xfer->c_flags & C_DMA) {
    972 				ata_dmaerr(drvp,
    973 				    (xfer->c_flags & C_POLL) ? AT_POLL : 0);
    974 			}
    975 #endif
    976 			sc_xfer->error = XS_RESET;
    977 			wdc_atapi_reset(chp, xfer);
    978 			return (1);
    979 		}
    980 	}
    981 	ATADEBUG_PRINT(("wdc_atapi_intr: wdc_atapi_done() (end), error 0x%x "
    982 	    "sense 0x%x\n", sc_xfer->error, sc_xfer->sense.atapi_sense),
    983 	    DEBUG_INTR);
    984 	wdc_atapi_done(chp, xfer);
    985 	return (1);
    986 }
    987 
    988 static void
    989 wdc_atapi_phase_complete(struct ata_xfer *xfer)
    990 {
    991 	struct ata_channel *chp = xfer->c_chp;
    992 	struct atac_softc *atac = chp->ch_atac;
    993 #if NATA_DMA || NATA_PIOBM
    994 	struct wdc_softc *wdc = CHAN_TO_WDC(chp);
    995 #endif
    996 	struct scsipi_xfer *sc_xfer = xfer->c_cmd;
    997 	struct ata_drive_datas *drvp = &chp->ch_drive[xfer->c_drive];
    998 
    999 	/* wait for DSC if needed */
   1000 	if (drvp->drive_flags & DRIVE_ATAPIST) {
   1001 		ATADEBUG_PRINT(("wdc_atapi_phase_complete(%s:%d:%d) "
   1002 		    "polldsc %d\n", atac->atac_dev.dv_xname, chp->ch_channel,
   1003 		    xfer->c_drive, xfer->c_dscpoll), DEBUG_XFERS);
   1004 #if 1
   1005 		if (cold)
   1006 			panic("wdc_atapi_phase_complete: cold");
   1007 #endif
   1008 		if (wdcwait(chp, WDCS_DSC, WDCS_DSC, 10,
   1009 		    AT_POLL) == WDCWAIT_TOUT) {
   1010 			/* 10ms not enough, try again in 1 tick */
   1011 			if (xfer->c_dscpoll++ >
   1012 			    mstohz(sc_xfer->timeout)) {
   1013 				printf("%s:%d:%d: wait_for_dsc "
   1014 				    "failed\n",
   1015 				    atac->atac_dev.dv_xname,
   1016 				    chp->ch_channel, xfer->c_drive);
   1017 				sc_xfer->error = XS_TIMEOUT;
   1018 				wdc_atapi_reset(chp, xfer);
   1019 				return;
   1020 			} else
   1021 				callout_reset(&chp->ch_callout, 1,
   1022 				    wdc_atapi_polldsc, xfer);
   1023 			return;
   1024 		}
   1025 	}
   1026 
   1027 	/*
   1028 	 * Some drive occasionally set WDCS_ERR with
   1029 	 * "ATA illegal length indication" in the error
   1030 	 * register. If we read some data the sense is valid
   1031 	 * anyway, so don't report the error.
   1032 	 */
   1033 	if (chp->ch_status & WDCS_ERR &&
   1034 	    ((sc_xfer->xs_control & XS_CTL_REQSENSE) == 0 ||
   1035 	    sc_xfer->resid == sc_xfer->datalen)) {
   1036 		/* save the short sense */
   1037 		sc_xfer->error = XS_SHORTSENSE;
   1038 		sc_xfer->sense.atapi_sense = chp->ch_error;
   1039 		if ((sc_xfer->xs_periph->periph_quirks &
   1040 		    PQUIRK_NOSENSE) == 0) {
   1041 			/* ask scsipi to send a REQUEST_SENSE */
   1042 			sc_xfer->error = XS_BUSY;
   1043 			sc_xfer->status = SCSI_CHECK;
   1044 		}
   1045 #if NATA_DMA || NATA_PIOBM
   1046 		else if (wdc->dma_status &
   1047 		    (WDC_DMAST_NOIRQ | WDC_DMAST_ERR)) {
   1048 #if NATA_DMA
   1049 			ata_dmaerr(drvp,
   1050 			    (xfer->c_flags & C_POLL) ? AT_POLL : 0);
   1051 #endif
   1052 			sc_xfer->error = XS_RESET;
   1053 			wdc_atapi_reset(chp, xfer);
   1054 			return;
   1055 		}
   1056 #endif
   1057 	}
   1058 	if (xfer->c_bcount != 0) {
   1059 		ATADEBUG_PRINT(("wdc_atapi_intr: bcount value is "
   1060 		    "%d after io\n", xfer->c_bcount), DEBUG_XFERS);
   1061 	}
   1062 #ifdef DIAGNOSTIC
   1063 	if (xfer->c_bcount < 0) {
   1064 		printf("wdc_atapi_intr warning: bcount value "
   1065 		    "is %d after io\n", xfer->c_bcount);
   1066 	}
   1067 #endif
   1068 	ATADEBUG_PRINT(("wdc_atapi_phase_complete: wdc_atapi_done(), "
   1069 	    "error 0x%x sense 0x%x\n", sc_xfer->error,
   1070 	    sc_xfer->sense.atapi_sense), DEBUG_INTR);
   1071 	wdc_atapi_done(chp, xfer);
   1072 }
   1073 
   1074 static void
   1075 wdc_atapi_done(struct ata_channel *chp, struct ata_xfer *xfer)
   1076 {
   1077 	struct atac_softc *atac = chp->ch_atac;
   1078 	struct scsipi_xfer *sc_xfer = xfer->c_cmd;
   1079 	int drive = xfer->c_drive;
   1080 
   1081 	ATADEBUG_PRINT(("wdc_atapi_done %s:%d:%d: flags 0x%x\n",
   1082 	    atac->atac_dev.dv_xname, chp->ch_channel, xfer->c_drive,
   1083 	    (u_int)xfer->c_flags), DEBUG_XFERS);
   1084 	callout_stop(&chp->ch_callout);
   1085 	/* mark controller inactive and free the command */
   1086 	chp->ch_queue->active_xfer = NULL;
   1087 	ata_free_xfer(chp, xfer);
   1088 
   1089 	if (chp->ch_drive[drive].drive_flags & DRIVE_WAITDRAIN) {
   1090 		sc_xfer->error = XS_DRIVER_STUFFUP;
   1091 		chp->ch_drive[drive].drive_flags &= ~DRIVE_WAITDRAIN;
   1092 		wakeup(&chp->ch_queue->active_xfer);
   1093 	}
   1094 
   1095 	ATADEBUG_PRINT(("wdc_atapi_done: scsipi_done\n"), DEBUG_XFERS);
   1096 	scsipi_done(sc_xfer);
   1097 	ATADEBUG_PRINT(("atastart from wdc_atapi_done, flags 0x%x\n",
   1098 	    chp->ch_flags), DEBUG_XFERS);
   1099 	atastart(chp);
   1100 }
   1101 
   1102 static void
   1103 wdc_atapi_reset(struct ata_channel *chp, struct ata_xfer *xfer)
   1104 {
   1105 	struct atac_softc *atac = chp->ch_atac;
   1106 	struct ata_drive_datas *drvp = &chp->ch_drive[xfer->c_drive];
   1107 	struct scsipi_xfer *sc_xfer = xfer->c_cmd;
   1108 
   1109 	wdccommandshort(chp, xfer->c_drive, ATAPI_SOFT_RESET);
   1110 	drvp->state = 0;
   1111 	if (wdc_wait_for_unbusy(chp, WDC_RESET_WAIT, AT_POLL) != 0) {
   1112 		printf("%s:%d:%d: reset failed\n",
   1113 		    atac->atac_dev.dv_xname, chp->ch_channel,
   1114 		    xfer->c_drive);
   1115 		sc_xfer->error = XS_SELTIMEOUT;
   1116 	}
   1117 	wdc_atapi_done(chp, xfer);
   1118 	return;
   1119 }
   1120 
   1121 static void
   1122 wdc_atapi_polldsc(void *arg)
   1123 {
   1124 
   1125 	wdc_atapi_phase_complete(arg);
   1126 }
   1127