Home | History | Annotate | Line # | Download | only in scsipi
atapi_wdc.c revision 1.106
      1 /*	$NetBSD: atapi_wdc.c,v 1.106 2008/03/18 20:46:37 cube Exp $	*/
      2 
      3 /*
      4  * Copyright (c) 1998, 2001 Manuel Bouyer.
      5  *
      6  * Redistribution and use in source and binary forms, with or without
      7  * modification, are permitted provided that the following conditions
      8  * are met:
      9  * 1. Redistributions of source code must retain the above copyright
     10  *    notice, this list of conditions and the following disclaimer.
     11  * 2. Redistributions in binary form must reproduce the above copyright
     12  *    notice, this list of conditions and the following disclaimer in the
     13  *    documentation and/or other materials provided with the distribution.
     14  * 3. All advertising materials mentioning features or use of this software
     15  *    must display the following acknowledgement:
     16  *	This product includes software developed by Manuel Bouyer.
     17  * 4. The name of the author may not be used to endorse or promote products
     18  *    derived from this software without specific prior written permission.
     19  *
     20  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
     21  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
     22  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
     23  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
     24  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
     25  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
     26  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
     27  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
     28  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
     29  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     30  */
     31 
     32 #include <sys/cdefs.h>
     33 __KERNEL_RCSID(0, "$NetBSD: atapi_wdc.c,v 1.106 2008/03/18 20:46:37 cube Exp $");
     34 
     35 #ifndef ATADEBUG
     36 #define ATADEBUG
     37 #endif /* ATADEBUG */
     38 
     39 #include <sys/param.h>
     40 #include <sys/systm.h>
     41 #include <sys/kernel.h>
     42 #include <sys/file.h>
     43 #include <sys/stat.h>
     44 #include <sys/buf.h>
     45 #include <sys/malloc.h>
     46 #include <sys/device.h>
     47 #include <sys/syslog.h>
     48 #include <sys/proc.h>
     49 #include <sys/dvdio.h>
     50 
     51 #include <sys/intr.h>
     52 #include <sys/bus.h>
     53 
     54 #ifndef __BUS_SPACE_HAS_STREAM_METHODS
     55 #define	bus_space_write_multi_stream_2	bus_space_write_multi_2
     56 #define	bus_space_write_multi_stream_4	bus_space_write_multi_4
     57 #define	bus_space_read_multi_stream_2	bus_space_read_multi_2
     58 #define	bus_space_read_multi_stream_4	bus_space_read_multi_4
     59 #endif /* __BUS_SPACE_HAS_STREAM_METHODS */
     60 
     61 #include <dev/ata/ataconf.h>
     62 #include <dev/ata/atareg.h>
     63 #include <dev/ata/atavar.h>
     64 #include <dev/ic/wdcreg.h>
     65 #include <dev/ic/wdcvar.h>
     66 
     67 #include <dev/scsipi/scsi_all.h> /* for SCSI status */
     68 
     69 #define DEBUG_INTR   0x01
     70 #define DEBUG_XFERS  0x02
     71 #define DEBUG_STATUS 0x04
     72 #define DEBUG_FUNCS  0x08
     73 #define DEBUG_PROBE  0x10
     74 #ifdef ATADEBUG
     75 int wdcdebug_atapi_mask = 0;
     76 #define ATADEBUG_PRINT(args, level) \
     77 	if (wdcdebug_atapi_mask & (level)) \
     78 		printf args
     79 #else
     80 #define ATADEBUG_PRINT(args, level)
     81 #endif
     82 
     83 #define ATAPI_DELAY 10	/* 10 ms, this is used only before sending a cmd */
     84 #define ATAPI_MODE_DELAY 1000	/* 1s, timeout for SET_FEATYRE cmds */
     85 
     86 static int	wdc_atapi_get_params(struct scsipi_channel *, int,
     87 				     struct ataparams *);
     88 static void	wdc_atapi_probe_device(struct atapibus_softc *, int);
     89 static void	wdc_atapi_minphys (struct buf *bp);
     90 static void	wdc_atapi_start(struct ata_channel *,struct ata_xfer *);
     91 static int	wdc_atapi_intr(struct ata_channel *, struct ata_xfer *, int);
     92 static void	wdc_atapi_kill_xfer(struct ata_channel *,
     93 				    struct ata_xfer *, int);
     94 static void	wdc_atapi_phase_complete(struct ata_xfer *);
     95 static void	wdc_atapi_done(struct ata_channel *, struct ata_xfer *);
     96 static void	wdc_atapi_reset(struct ata_channel *, struct ata_xfer *);
     97 static void	wdc_atapi_scsipi_request(struct scsipi_channel *,
     98 					 scsipi_adapter_req_t, void *);
     99 static void	wdc_atapi_kill_pending(struct scsipi_periph *);
    100 static void	wdc_atapi_polldsc(void *arg);
    101 
    102 #define MAX_SIZE MAXPHYS
    103 
    104 static const struct scsipi_bustype wdc_atapi_bustype = {
    105 	SCSIPI_BUSTYPE_ATAPI,
    106 	atapi_scsipi_cmd,
    107 	atapi_interpret_sense,
    108 	atapi_print_addr,
    109 	wdc_atapi_kill_pending,
    110 };
    111 
    112 void
    113 wdc_atapibus_attach(struct atabus_softc *ata_sc)
    114 {
    115 	struct ata_channel *chp = ata_sc->sc_chan;
    116 	struct atac_softc *atac = chp->ch_atac;
    117 	struct scsipi_adapter *adapt = &atac->atac_atapi_adapter._generic;
    118 	struct scsipi_channel *chan = &chp->ch_atapi_channel;
    119 
    120 	/*
    121 	 * Fill in the scsipi_adapter.
    122 	 */
    123 	adapt->adapt_dev = atac->atac_dev;
    124 	adapt->adapt_nchannels = atac->atac_nchannels;
    125 	adapt->adapt_request = wdc_atapi_scsipi_request;
    126 	adapt->adapt_minphys = wdc_atapi_minphys;
    127 	if (atac->atac_cap & ATAC_CAP_NOIRQ)
    128 		adapt->adapt_flags |= SCSIPI_ADAPT_POLL_ONLY;
    129 	atac->atac_atapi_adapter.atapi_probe_device = wdc_atapi_probe_device;
    130 
    131 	/*
    132 	 * Fill in the scsipi_channel.
    133 	 */
    134 	memset(chan, 0, sizeof(*chan));
    135 	chan->chan_adapter = adapt;
    136 	chan->chan_bustype = &wdc_atapi_bustype;
    137 	chan->chan_channel = chp->ch_channel;
    138 	chan->chan_flags = SCSIPI_CHAN_OPENINGS;
    139 	chan->chan_openings = 1;
    140 	chan->chan_max_periph = 1;
    141 	chan->chan_ntargets = 2;
    142 	chan->chan_nluns = 1;
    143 
    144 	chp->atapibus = config_found_ia(ata_sc->sc_dev, "atapi", chan,
    145 		atapiprint);
    146 }
    147 
    148 static void
    149 wdc_atapi_minphys(struct buf *bp)
    150 {
    151 
    152 	if (bp->b_bcount > MAX_SIZE)
    153 		bp->b_bcount = MAX_SIZE;
    154 	minphys(bp);
    155 }
    156 
    157 /*
    158  * Kill off all pending xfers for a periph.
    159  *
    160  * Must be called at splbio().
    161  */
    162 static void
    163 wdc_atapi_kill_pending(struct scsipi_periph *periph)
    164 {
    165 	struct atac_softc *atac =
    166 	    device_private(periph->periph_channel->chan_adapter->adapt_dev);
    167 	struct ata_channel *chp =
    168 	    atac->atac_channels[periph->periph_channel->chan_channel];
    169 
    170 	ata_kill_pending(&chp->ch_drive[periph->periph_target]);
    171 }
    172 
    173 static void
    174 wdc_atapi_kill_xfer(struct ata_channel *chp, struct ata_xfer *xfer, int reason)
    175 {
    176 	struct scsipi_xfer *sc_xfer = xfer->c_cmd;
    177 
    178 	/* remove this command from xfer queue */
    179 	switch (reason) {
    180 	case KILL_GONE:
    181 		sc_xfer->error = XS_DRIVER_STUFFUP;
    182 		break;
    183 	case KILL_RESET:
    184 		sc_xfer->error = XS_RESET;
    185 		break;
    186 	default:
    187 		printf("wdc_ata_bio_kill_xfer: unknown reason %d\n",
    188 		    reason);
    189 		panic("wdc_ata_bio_kill_xfer");
    190 	}
    191 	ata_free_xfer(chp, xfer);
    192 	scsipi_done(sc_xfer);
    193 }
    194 
    195 static int
    196 wdc_atapi_get_params(struct scsipi_channel *chan, int drive,
    197     struct ataparams *id)
    198 {
    199 	struct wdc_softc *wdc = device_private(chan->chan_adapter->adapt_dev);
    200 	struct atac_softc *atac = &wdc->sc_atac;
    201 	struct wdc_regs *wdr = &wdc->regs[chan->chan_channel];
    202 	struct ata_channel *chp = atac->atac_channels[chan->chan_channel];
    203 	struct ata_command ata_c;
    204 
    205 	/* if no ATAPI device detected at wdc attach time, skip */
    206 	if ((chp->ch_drive[drive].drive_flags & DRIVE_ATAPI) == 0) {
    207 		ATADEBUG_PRINT(("wdc_atapi_get_params: drive %d not present\n",
    208 		    drive), DEBUG_PROBE);
    209 		return -1;
    210 	}
    211 
    212 	memset(&ata_c, 0, sizeof(struct ata_command));
    213 	ata_c.r_command = ATAPI_SOFT_RESET;
    214 	ata_c.r_st_bmask = 0;
    215 	ata_c.r_st_pmask = 0;
    216 	ata_c.flags = AT_WAIT | AT_POLL;
    217 	ata_c.timeout = WDC_RESET_WAIT;
    218 	if (wdc_exec_command(&chp->ch_drive[drive], &ata_c) != ATACMD_COMPLETE) {
    219 		printf("wdc_atapi_get_params: ATAPI_SOFT_RESET failed for"
    220 		    " drive %s:%d:%d: driver failed\n",
    221 		    device_xname(atac->atac_dev), chp->ch_channel, drive);
    222 		panic("wdc_atapi_get_params");
    223 	}
    224 	if (ata_c.flags & (AT_ERROR | AT_TIMEOU | AT_DF)) {
    225 		ATADEBUG_PRINT(("wdc_atapi_get_params: ATAPI_SOFT_RESET "
    226 		    "failed for drive %s:%d:%d: error 0x%x\n",
    227 		    device_xname(atac->atac_dev), chp->ch_channel, drive,
    228 		    ata_c.r_error), DEBUG_PROBE);
    229 		return -1;
    230 	}
    231 	chp->ch_drive[drive].state = 0;
    232 
    233 	(void)bus_space_read_1(wdr->cmd_iot, wdr->cmd_iohs[wd_status], 0);
    234 
    235 	/* Some ATAPI devices need a bit more time after software reset. */
    236 	delay(5000);
    237 	if (ata_get_params(&chp->ch_drive[drive], AT_WAIT, id) != 0) {
    238 		ATADEBUG_PRINT(("wdc_atapi_get_params: ATAPI_IDENTIFY_DEVICE "
    239 		    "failed for drive %s:%d:%d: error 0x%x\n",
    240 		    device_xname(atac->atac_dev), chp->ch_channel, drive,
    241 		    ata_c.r_error), DEBUG_PROBE);
    242 		return -1;
    243 	}
    244 	return 0;
    245 }
    246 
    247 static void
    248 wdc_atapi_probe_device(struct atapibus_softc *sc, int target)
    249 {
    250 	struct scsipi_channel *chan = sc->sc_channel;
    251 	struct scsipi_periph *periph;
    252 	struct ataparams ids;
    253 	struct ataparams *id = &ids;
    254 	struct wdc_softc *wdc = device_private(chan->chan_adapter->adapt_dev);
    255 	struct atac_softc *atac = &wdc->sc_atac;
    256 	struct ata_channel *chp = atac->atac_channels[chan->chan_channel];
    257 	struct ata_drive_datas *drvp = &chp->ch_drive[target];
    258 	struct scsipibus_attach_args sa;
    259 	char serial_number[21], model[41], firmware_revision[9];
    260 	int s;
    261 
    262 	/* skip if already attached */
    263 	if (scsipi_lookup_periph(chan, target, 0) != NULL)
    264 		return;
    265 
    266 	if (wdc_atapi_get_params(chan, target, id) == 0) {
    267 #ifdef ATAPI_DEBUG_PROBE
    268 		printf("%s drive %d: cmdsz 0x%x drqtype 0x%x\n",
    269 		    sc->sc_dev.dv_xname, target,
    270 		    id->atap_config & ATAPI_CFG_CMD_MASK,
    271 		    id->atap_config & ATAPI_CFG_DRQ_MASK);
    272 #endif
    273 		periph = scsipi_alloc_periph(M_NOWAIT);
    274 		if (periph == NULL) {
    275 			printf("%s: unable to allocate periph for drive %d\n",
    276 			    device_xname(&sc->sc_dev), target);
    277 			return;
    278 		}
    279 		periph->periph_dev = NULL;
    280 		periph->periph_channel = chan;
    281 		periph->periph_switch = &atapi_probe_periphsw;
    282 		periph->periph_target = target;
    283 		periph->periph_lun = 0;
    284 		periph->periph_quirks = PQUIRK_ONLYBIG;
    285 
    286 #ifdef SCSIPI_DEBUG
    287 		if (SCSIPI_DEBUG_TYPE == SCSIPI_BUSTYPE_ATAPI &&
    288 		    SCSIPI_DEBUG_TARGET == target)
    289 			periph->periph_dbflags |= SCSIPI_DEBUG_FLAGS;
    290 #endif
    291 		periph->periph_type = ATAPI_CFG_TYPE(id->atap_config);
    292 		if (id->atap_config & ATAPI_CFG_REMOV)
    293 			periph->periph_flags |= PERIPH_REMOVABLE;
    294 		if (periph->periph_type == T_SEQUENTIAL) {
    295 			s = splbio();
    296 			drvp->drive_flags |= DRIVE_ATAPIST;
    297 			splx(s);
    298 		}
    299 
    300 		sa.sa_periph = periph;
    301 		sa.sa_inqbuf.type =  ATAPI_CFG_TYPE(id->atap_config);
    302 		sa.sa_inqbuf.removable = id->atap_config & ATAPI_CFG_REMOV ?
    303 		    T_REMOV : T_FIXED;
    304 		scsipi_strvis((u_char *)model, 40, id->atap_model, 40);
    305 		scsipi_strvis((u_char *)serial_number, 20, id->atap_serial,
    306 		    20);
    307 		scsipi_strvis((u_char *)firmware_revision, 8,
    308 		    id->atap_revision, 8);
    309 		sa.sa_inqbuf.vendor = model;
    310 		sa.sa_inqbuf.product = serial_number;
    311 		sa.sa_inqbuf.revision = firmware_revision;
    312 
    313 		/*
    314 		 * Determine the operating mode capabilities of the device.
    315 		 */
    316 		if ((id->atap_config & ATAPI_CFG_CMD_MASK) == ATAPI_CFG_CMD_16)
    317 			periph->periph_cap |= PERIPH_CAP_CMD16;
    318 		/* XXX This is gross. */
    319 		periph->periph_cap |= (id->atap_config & ATAPI_CFG_DRQ_MASK);
    320 
    321 		drvp->drv_softc = atapi_probe_device(sc, target, periph, &sa);
    322 
    323 		if (drvp->drv_softc)
    324 			ata_probe_caps(drvp);
    325 		else {
    326 			s = splbio();
    327 			drvp->drive_flags &= ~DRIVE_ATAPI;
    328 			splx(s);
    329 		}
    330 	} else {
    331 		s = splbio();
    332 		drvp->drive_flags &= ~DRIVE_ATAPI;
    333 		splx(s);
    334 	}
    335 }
    336 
    337 static void
    338 wdc_atapi_scsipi_request(struct scsipi_channel *chan, scsipi_adapter_req_t req,
    339     void *arg)
    340 {
    341 	struct scsipi_adapter *adapt = chan->chan_adapter;
    342 	struct scsipi_periph *periph;
    343 	struct scsipi_xfer *sc_xfer;
    344 	struct wdc_softc *wdc = device_private(adapt->adapt_dev);
    345 	struct atac_softc *atac = &wdc->sc_atac;
    346 	struct ata_xfer *xfer;
    347 	int channel = chan->chan_channel;
    348 	int drive, s;
    349 
    350 	switch (req) {
    351 	case ADAPTER_REQ_RUN_XFER:
    352 		sc_xfer = arg;
    353 		periph = sc_xfer->xs_periph;
    354 		drive = periph->periph_target;
    355 
    356 		ATADEBUG_PRINT(("wdc_atapi_scsipi_request %s:%d:%d\n",
    357 		    device_xname(atac->atac_dev), channel, drive),
    358 		    DEBUG_XFERS);
    359 		if (!device_is_active(atac->atac_dev)) {
    360 			sc_xfer->error = XS_DRIVER_STUFFUP;
    361 			scsipi_done(sc_xfer);
    362 			return;
    363 		}
    364 
    365 		xfer = ata_get_xfer(ATAXF_NOSLEEP);
    366 		if (xfer == NULL) {
    367 			sc_xfer->error = XS_RESOURCE_SHORTAGE;
    368 			scsipi_done(sc_xfer);
    369 			return;
    370 		}
    371 
    372 		if (sc_xfer->xs_control & XS_CTL_POLL)
    373 			xfer->c_flags |= C_POLL;
    374 #if NATA_DMA
    375 		if ((atac->atac_channels[channel]->ch_drive[drive].drive_flags &
    376 		    (DRIVE_DMA | DRIVE_UDMA)) && sc_xfer->datalen > 0)
    377 			xfer->c_flags |= C_DMA;
    378 #endif
    379 #if NATA_DMA && NATA_PIOBM
    380 		else
    381 #endif
    382 #if NATA_PIOBM
    383 		if ((atac->atac_cap & ATAC_CAP_PIOBM) &&
    384 		    sc_xfer->datalen > 0)
    385 			xfer->c_flags |= C_PIOBM;
    386 #endif
    387 		xfer->c_drive = drive;
    388 		xfer->c_flags |= C_ATAPI;
    389 #if NATA_DMA
    390 		if (sc_xfer->cmd->opcode == GPCMD_REPORT_KEY ||
    391 		    sc_xfer->cmd->opcode == GPCMD_SEND_KEY ||
    392 		    sc_xfer->cmd->opcode == GPCMD_READ_DVD_STRUCTURE) {
    393 			/*
    394 			 * DVD authentication commands must always be done in
    395 			 * PIO mode.
    396 			 */
    397 			xfer->c_flags &= ~C_DMA;
    398 		}
    399 
    400 		/*
    401 		 * DMA normally can't deal with transfers which are not a
    402 		 * multiple of its databus width. It's a bug to request odd
    403 		 * length transfers for ATAPI.
    404 		 *
    405 		 * Some devices also can't cope with unaligned DMA xfers
    406 		 * either. Also some devices seem to not handle DMA xfers of
    407 		 * less than 4 bytes.
    408 		 *
    409 		 * By enforcing at least 4 byte aligned offset and length for
    410 		 * DMA, we might use PIO where DMA could be allowed but better
    411 		 * safe than sorry as recent problems proved.
    412 		 *
    413 		 * Offending structures that are thus done by PIO instead of
    414 		 * DMA are normally small structures since all bulkdata is
    415 		 * aligned. But as the request may come from userland, we have
    416 		 * to protect against it anyway.
    417 		 *
    418 		 * XXX check for the 32 bit wide flag?
    419 		 */
    420 
    421 		if (((uintptr_t) sc_xfer->data) & 0x03)
    422 			xfer->c_flags &= ~C_DMA;
    423 		if ((sc_xfer->datalen < 4) || (sc_xfer->datalen & 0x03))
    424 			xfer->c_flags &= ~C_DMA;
    425 #endif	/* NATA_DMA */
    426 
    427 		xfer->c_cmd = sc_xfer;
    428 		xfer->c_databuf = sc_xfer->data;
    429 		xfer->c_bcount = sc_xfer->datalen;
    430 		xfer->c_start = wdc_atapi_start;
    431 		xfer->c_intr = wdc_atapi_intr;
    432 		xfer->c_kill_xfer = wdc_atapi_kill_xfer;
    433 		xfer->c_dscpoll = 0;
    434 		s = splbio();
    435 		ata_exec_xfer(atac->atac_channels[channel], xfer);
    436 #ifdef DIAGNOSTIC
    437 		if ((sc_xfer->xs_control & XS_CTL_POLL) != 0 &&
    438 		    (sc_xfer->xs_status & XS_STS_DONE) == 0)
    439 			panic("wdc_atapi_scsipi_request: polled command "
    440 			    "not done");
    441 #endif
    442 		splx(s);
    443 		return;
    444 
    445 	default:
    446 		/* Not supported, nothing to do. */
    447 		;
    448 	}
    449 }
    450 
    451 static void
    452 wdc_atapi_start(struct ata_channel *chp, struct ata_xfer *xfer)
    453 {
    454 	struct atac_softc *atac = chp->ch_atac;
    455 	struct wdc_softc *wdc = CHAN_TO_WDC(chp);
    456 	struct wdc_regs *wdr = &wdc->regs[chp->ch_channel];
    457 	struct scsipi_xfer *sc_xfer = xfer->c_cmd;
    458 	struct ata_drive_datas *drvp = &chp->ch_drive[xfer->c_drive];
    459 	int wait_flags = (sc_xfer->xs_control & XS_CTL_POLL) ? AT_POLL : 0;
    460 	const char *errstring;
    461 
    462 	ATADEBUG_PRINT(("wdc_atapi_start %s:%d:%d, scsi flags 0x%x \n",
    463 	    device_xname(atac->atac_dev), chp->ch_channel, drvp->drive,
    464 	    sc_xfer->xs_control), DEBUG_XFERS);
    465 #if NATA_DMA
    466 	if ((xfer->c_flags & C_DMA) && (drvp->n_xfers <= NXFER))
    467 		drvp->n_xfers++;
    468 #endif
    469 	/* Do control operations specially. */
    470 	if (__predict_false(drvp->state < READY)) {
    471 		/* If it's not a polled command, we need the kernel thread */
    472 		if ((sc_xfer->xs_control & XS_CTL_POLL) == 0 &&
    473 		    (chp->ch_flags & ATACH_TH_RUN) == 0) {
    474 			chp->ch_queue->queue_freeze++;
    475 			wakeup(&chp->ch_thread);
    476 			return;
    477 		}
    478 		/*
    479 		 * disable interrupts, all commands here should be quick
    480 		 * enouth to be able to poll, and we don't go here that often
    481 		 */
    482 		 bus_space_write_1(wdr->ctl_iot, wdr->ctl_ioh, wd_aux_ctlr,
    483 		     WDCTL_4BIT | WDCTL_IDS);
    484 		if (wdc->select)
    485 			wdc->select(chp, xfer->c_drive);
    486 		bus_space_write_1(wdr->cmd_iot, wdr->cmd_iohs[wd_sdh], 0,
    487 		    WDSD_IBM | (xfer->c_drive << 4));
    488 		/* Don't try to set mode if controller can't be adjusted */
    489 		if (atac->atac_set_modes == NULL)
    490 			goto ready;
    491 		/* Also don't try if the drive didn't report its mode */
    492 		if ((drvp->drive_flags & DRIVE_MODE) == 0)
    493 			goto ready;
    494 		errstring = "unbusy";
    495 		if (wdc_wait_for_unbusy(chp, ATAPI_DELAY, wait_flags))
    496 			goto timeout;
    497 		wdccommand(chp, drvp->drive, SET_FEATURES, 0, 0, 0,
    498 		    0x08 | drvp->PIO_mode, WDSF_SET_MODE);
    499 		errstring = "piomode";
    500 		if (wdc_wait_for_unbusy(chp, ATAPI_MODE_DELAY, wait_flags))
    501 			goto timeout;
    502 		if (chp->ch_status & WDCS_ERR) {
    503 			if (chp->ch_error == WDCE_ABRT) {
    504 				/*
    505 				 * Some ATAPI drives reject PIO settings.
    506 				 * Fall back to PIO mode 3 since that's the
    507 				 * minimum for ATAPI.
    508 				 */
    509 				printf("%s:%d:%d: PIO mode %d rejected, "
    510 				    "falling back to PIO mode 3\n",
    511 				    device_xname(atac->atac_dev),
    512 				    chp->ch_channel, xfer->c_drive,
    513 				    drvp->PIO_mode);
    514 				if (drvp->PIO_mode > 3)
    515 					drvp->PIO_mode = 3;
    516 			} else
    517 				goto error;
    518 		}
    519 #if NATA_DMA
    520 #if NATA_UDMA
    521 		if (drvp->drive_flags & DRIVE_UDMA) {
    522 			wdccommand(chp, drvp->drive, SET_FEATURES, 0, 0, 0,
    523 			    0x40 | drvp->UDMA_mode, WDSF_SET_MODE);
    524 		} else
    525 #endif
    526 		if (drvp->drive_flags & DRIVE_DMA) {
    527 			wdccommand(chp, drvp->drive, SET_FEATURES, 0, 0, 0,
    528 			    0x20 | drvp->DMA_mode, WDSF_SET_MODE);
    529 		} else {
    530 			goto ready;
    531 		}
    532 		errstring = "dmamode";
    533 		if (wdc_wait_for_unbusy(chp, ATAPI_MODE_DELAY, wait_flags))
    534 			goto timeout;
    535 		if (chp->ch_status & WDCS_ERR) {
    536 			if (chp->ch_error == WDCE_ABRT) {
    537 #if NATA_UDMA
    538 				if (drvp->drive_flags & DRIVE_UDMA)
    539 					goto error;
    540 				else
    541 #endif
    542 				{
    543 					/*
    544 					 * The drive rejected our DMA setting.
    545 					 * Fall back to mode 1.
    546 					 */
    547 					printf("%s:%d:%d: DMA mode %d rejected, "
    548 					    "falling back to DMA mode 0\n",
    549 					    device_xname(atac->atac_dev),
    550 					    chp->ch_channel, xfer->c_drive,
    551 					    drvp->DMA_mode);
    552 					if (drvp->DMA_mode > 0)
    553 						drvp->DMA_mode = 0;
    554 				}
    555 			} else
    556 				goto error;
    557 		}
    558 #endif	/* NATA_DMA */
    559 ready:
    560 		drvp->state = READY;
    561 		bus_space_write_1(wdr->ctl_iot, wdr->ctl_ioh, wd_aux_ctlr,
    562 		    WDCTL_4BIT);
    563 		delay(10); /* some drives need a little delay here */
    564 	}
    565 	/* start timeout machinery */
    566 	if ((sc_xfer->xs_control & XS_CTL_POLL) == 0)
    567 		callout_reset(&chp->ch_callout, mstohz(sc_xfer->timeout),
    568 		    wdctimeout, chp);
    569 
    570 	if (wdc->select)
    571 		wdc->select(chp, xfer->c_drive);
    572 	bus_space_write_1(wdr->cmd_iot, wdr->cmd_iohs[wd_sdh], 0,
    573 	    WDSD_IBM | (xfer->c_drive << 4));
    574 	switch (wdc_wait_for_unbusy(chp, ATAPI_DELAY, wait_flags)  < 0) {
    575 	case WDCWAIT_OK:
    576 		break;
    577 	case WDCWAIT_TOUT:
    578 		printf("wdc_atapi_start: not ready, st = %02x\n",
    579 		    chp->ch_status);
    580 		sc_xfer->error = XS_TIMEOUT;
    581 		wdc_atapi_reset(chp, xfer);
    582 		return;
    583 	case WDCWAIT_THR:
    584 		return;
    585 	}
    586 
    587 	/*
    588 	 * Even with WDCS_ERR, the device should accept a command packet
    589 	 * Limit length to what can be stuffed into the cylinder register
    590 	 * (16 bits).  Some CD-ROMs seem to interpret '0' as 65536,
    591 	 * but not all devices do that and it's not obvious from the
    592 	 * ATAPI spec that that behaviour should be expected.  If more
    593 	 * data is necessary, multiple data transfer phases will be done.
    594 	 */
    595 
    596 	wdccommand(chp, xfer->c_drive, ATAPI_PKT_CMD,
    597 	    xfer->c_bcount <= 0xffff ? xfer->c_bcount : 0xffff,
    598 	    0, 0, 0,
    599 #if NATA_DMA
    600 	    (xfer->c_flags & C_DMA) ? ATAPI_PKT_CMD_FTRE_DMA :
    601 #endif
    602 	    0
    603 	    );
    604 
    605 #if NATA_PIOBM
    606 	if (xfer->c_flags & C_PIOBM) {
    607 		int error;
    608 		int dma_flags = (sc_xfer->xs_control & XS_CTL_DATA_IN)
    609 		    ?  WDC_DMA_READ : 0;
    610 		if (xfer->c_flags & C_POLL) {
    611 			/* XXX not supported yet --- fall back to PIO */
    612 			xfer->c_flags &= ~C_PIOBM;
    613 		} else {
    614 			/* Init the DMA channel. */
    615 			error = (*wdc->dma_init)(wdc->dma_arg,
    616 			    chp->ch_channel, xfer->c_drive,
    617 			    (char *)xfer->c_databuf,
    618 			    xfer->c_bcount,
    619 			    dma_flags | WDC_DMA_PIOBM_ATAPI);
    620 			if (error) {
    621 				if (error == EINVAL) {
    622 					/*
    623 					 * We can't do DMA on this transfer
    624 					 * for some reason.  Fall back to
    625 					 * PIO.
    626 					 */
    627 					xfer->c_flags &= ~C_PIOBM;
    628 					error = 0;
    629 				} else {
    630 					sc_xfer->error = XS_DRIVER_STUFFUP;
    631 					errstring = "piobm";
    632 					goto error;
    633 				}
    634 			}
    635 		}
    636 	}
    637 #endif
    638 	/*
    639 	 * If there is no interrupt for CMD input, busy-wait for it (done in
    640 	 * the interrupt routine. If it is a polled command, call the interrupt
    641 	 * routine until command is done.
    642 	 */
    643 	if ((sc_xfer->xs_periph->periph_cap & ATAPI_CFG_DRQ_MASK) !=
    644 	    ATAPI_CFG_IRQ_DRQ || (sc_xfer->xs_control & XS_CTL_POLL)) {
    645 		/* Wait for at last 400ns for status bit to be valid */
    646 		DELAY(1);
    647 		wdc_atapi_intr(chp, xfer, 0);
    648 	} else {
    649 		chp->ch_flags |= ATACH_IRQ_WAIT;
    650 	}
    651 	if (sc_xfer->xs_control & XS_CTL_POLL) {
    652 #if NATA_DMA
    653 		if (chp->ch_flags & ATACH_DMA_WAIT) {
    654 			wdc_dmawait(chp, xfer, sc_xfer->timeout);
    655 			chp->ch_flags &= ~ATACH_DMA_WAIT;
    656 		}
    657 #endif
    658 		while ((sc_xfer->xs_status & XS_STS_DONE) == 0) {
    659 			/* Wait for at last 400ns for status bit to be valid */
    660 			DELAY(1);
    661 			wdc_atapi_intr(chp, xfer, 0);
    662 		}
    663 	}
    664 	return;
    665 timeout:
    666 	printf("%s:%d:%d: %s timed out\n",
    667 	    device_xname(atac->atac_dev), chp->ch_channel, xfer->c_drive,
    668 	    errstring);
    669 	sc_xfer->error = XS_TIMEOUT;
    670 	bus_space_write_1(wdr->ctl_iot, wdr->ctl_ioh, wd_aux_ctlr, WDCTL_4BIT);
    671 	delay(10); /* some drives need a little delay here */
    672 	wdc_atapi_reset(chp, xfer);
    673 	return;
    674 error:
    675 	printf("%s:%d:%d: %s ",
    676 	    device_xname(atac->atac_dev), chp->ch_channel, xfer->c_drive,
    677 	    errstring);
    678 	printf("error (0x%x)\n", chp->ch_error);
    679 	sc_xfer->error = XS_SHORTSENSE;
    680 	sc_xfer->sense.atapi_sense = chp->ch_error;
    681 	bus_space_write_1(wdr->ctl_iot, wdr->ctl_ioh, wd_aux_ctlr, WDCTL_4BIT);
    682 	delay(10); /* some drives need a little delay here */
    683 	wdc_atapi_reset(chp, xfer);
    684 	return;
    685 }
    686 
    687 static int
    688 wdc_atapi_intr(struct ata_channel *chp, struct ata_xfer *xfer, int irq)
    689 {
    690 	struct atac_softc *atac = chp->ch_atac;
    691 	struct wdc_softc *wdc = CHAN_TO_WDC(chp);
    692 	struct wdc_regs *wdr = &wdc->regs[chp->ch_channel];
    693 	struct scsipi_xfer *sc_xfer = xfer->c_cmd;
    694 	struct ata_drive_datas *drvp = &chp->ch_drive[xfer->c_drive];
    695 	int len, phase, i, retries=0;
    696 	int ire;
    697 #if NATA_DMA
    698 	int error;
    699 #endif
    700 #if NATA_DMA || NATA_PIOBM
    701 	int dma_flags = 0;
    702 #endif
    703 	void *cmd;
    704 
    705 	ATADEBUG_PRINT(("wdc_atapi_intr %s:%d:%d\n",
    706 	    device_xname(atac->atac_dev), chp->ch_channel, drvp->drive),
    707 	    DEBUG_INTR);
    708 
    709 	/* Is it not a transfer, but a control operation? */
    710 	if (drvp->state < READY) {
    711 		printf("%s:%d:%d: bad state %d in wdc_atapi_intr\n",
    712 		    device_xname(atac->atac_dev), chp->ch_channel,
    713 		    xfer->c_drive, drvp->state);
    714 		panic("wdc_atapi_intr: bad state");
    715 	}
    716 	/*
    717 	 * If we missed an interrupt in a PIO transfer, reset and restart.
    718 	 * Don't try to continue transfer, we may have missed cycles.
    719 	 */
    720 	if ((xfer->c_flags & (C_TIMEOU | C_DMA)) == C_TIMEOU) {
    721 		sc_xfer->error = XS_TIMEOUT;
    722 		wdc_atapi_reset(chp, xfer);
    723 		return 1;
    724 	}
    725 
    726 #if NATA_PIOBM
    727 	/* Transfer-done interrupt for busmastering PIO operation */
    728 	if ((xfer->c_flags & C_PIOBM) && (chp->ch_flags & ATACH_PIOBM_WAIT)) {
    729 		chp->ch_flags &= ~ATACH_PIOBM_WAIT;
    730 
    731 		/* restore transfer length */
    732 		len = xfer->c_bcount;
    733 		if (xfer->c_lenoff < 0)
    734 			len += xfer->c_lenoff;
    735 
    736 		if (sc_xfer->xs_control & XS_CTL_DATA_IN)
    737 			goto end_piobm_datain;
    738 		else
    739 			goto end_piobm_dataout;
    740 	}
    741 #endif
    742 
    743 	/* Ack interrupt done in wdc_wait_for_unbusy */
    744 	if (wdc->select)
    745 		wdc->select(chp, xfer->c_drive);
    746 	bus_space_write_1(wdr->cmd_iot, wdr->cmd_iohs[wd_sdh], 0,
    747 	    WDSD_IBM | (xfer->c_drive << 4));
    748 	if (wdc_wait_for_unbusy(chp,
    749 	    (irq == 0) ? sc_xfer->timeout : 0, AT_POLL) == WDCWAIT_TOUT) {
    750 		if (irq && (xfer->c_flags & C_TIMEOU) == 0)
    751 			return 0; /* IRQ was not for us */
    752 		printf("%s:%d:%d: device timeout, c_bcount=%d, c_skip=%d\n",
    753 		    device_xname(atac->atac_dev), chp->ch_channel,
    754 		    xfer->c_drive, xfer->c_bcount, xfer->c_skip);
    755 #if NATA_DMA
    756 		if (xfer->c_flags & C_DMA) {
    757 			ata_dmaerr(drvp,
    758 			    (xfer->c_flags & C_POLL) ? AT_POLL : 0);
    759 		}
    760 #endif
    761 		sc_xfer->error = XS_TIMEOUT;
    762 		wdc_atapi_reset(chp, xfer);
    763 		return 1;
    764 	}
    765 	if (wdc->irqack)
    766 		wdc->irqack(chp);
    767 
    768 #if NATA_DMA
    769 	/*
    770 	 * If we missed an IRQ and were using DMA, flag it as a DMA error
    771 	 * and reset device.
    772 	 */
    773 	if ((xfer->c_flags & C_TIMEOU) && (xfer->c_flags & C_DMA)) {
    774 		ata_dmaerr(drvp, (xfer->c_flags & C_POLL) ? AT_POLL : 0);
    775 		sc_xfer->error = XS_RESET;
    776 		wdc_atapi_reset(chp, xfer);
    777 		return (1);
    778 	}
    779 #endif
    780 	/*
    781 	 * if the request sense command was aborted, report the short sense
    782 	 * previously recorded, else continue normal processing
    783 	 */
    784 
    785 #if NATA_DMA || NATA_PIOBM
    786 	if (xfer->c_flags & (C_DMA | C_PIOBM))
    787 		dma_flags = (sc_xfer->xs_control & XS_CTL_DATA_IN)
    788 		    ?  WDC_DMA_READ : 0;
    789 #endif
    790 again:
    791 	len = bus_space_read_1(wdr->cmd_iot, wdr->cmd_iohs[wd_cyl_lo], 0) +
    792 	    256 * bus_space_read_1(wdr->cmd_iot, wdr->cmd_iohs[wd_cyl_hi], 0);
    793 	ire = bus_space_read_1(wdr->cmd_iot, wdr->cmd_iohs[wd_ireason], 0);
    794 	phase = (ire & (WDCI_CMD | WDCI_IN)) | (chp->ch_status & WDCS_DRQ);
    795 	ATADEBUG_PRINT(("wdc_atapi_intr: c_bcount %d len %d st 0x%x err 0x%x "
    796 	    "ire 0x%x :", xfer->c_bcount,
    797 	    len, chp->ch_status, chp->ch_error, ire), DEBUG_INTR);
    798 
    799 	switch (phase) {
    800 	case PHASE_CMDOUT:
    801 		cmd = sc_xfer->cmd;
    802 		ATADEBUG_PRINT(("PHASE_CMDOUT\n"), DEBUG_INTR);
    803 #if NATA_DMA
    804 		/* Init the DMA channel if necessary */
    805 		if (xfer->c_flags & C_DMA) {
    806 			error = (*wdc->dma_init)(wdc->dma_arg,
    807 			    chp->ch_channel, xfer->c_drive,
    808 			    xfer->c_databuf, xfer->c_bcount, dma_flags);
    809 			if (error) {
    810 				if (error == EINVAL) {
    811 					/*
    812 					 * We can't do DMA on this transfer
    813 					 * for some reason.  Fall back to
    814 					 * PIO.
    815 					 */
    816 					xfer->c_flags &= ~C_DMA;
    817 					error = 0;
    818 				} else {
    819 					sc_xfer->error = XS_DRIVER_STUFFUP;
    820 					break;
    821 				}
    822 			}
    823 		}
    824 #endif
    825 
    826 		/* send packet command */
    827 		/* Commands are 12 or 16 bytes long. It's 32-bit aligned */
    828 		wdc->dataout_pio(chp, drvp->drive_flags, cmd, sc_xfer->cmdlen);
    829 
    830 #if NATA_DMA
    831 		/* Start the DMA channel if necessary */
    832 		if (xfer->c_flags & C_DMA) {
    833 			(*wdc->dma_start)(wdc->dma_arg,
    834 			    chp->ch_channel, xfer->c_drive);
    835 			chp->ch_flags |= ATACH_DMA_WAIT;
    836 		}
    837 #endif
    838 
    839 		if ((sc_xfer->xs_control & XS_CTL_POLL) == 0) {
    840 			chp->ch_flags |= ATACH_IRQ_WAIT;
    841 		}
    842 		return 1;
    843 
    844 	 case PHASE_DATAOUT:
    845 		/* write data */
    846 		ATADEBUG_PRINT(("PHASE_DATAOUT\n"), DEBUG_INTR);
    847 #if NATA_DMA
    848 		if ((sc_xfer->xs_control & XS_CTL_DATA_OUT) == 0 ||
    849 		    (xfer->c_flags & C_DMA) != 0) {
    850 			printf("wdc_atapi_intr: bad data phase DATAOUT\n");
    851 			if (xfer->c_flags & C_DMA) {
    852 				ata_dmaerr(drvp,
    853 				    (xfer->c_flags & C_POLL) ? AT_POLL : 0);
    854 			}
    855 			sc_xfer->error = XS_TIMEOUT;
    856 			wdc_atapi_reset(chp, xfer);
    857 			return 1;
    858 		}
    859 #endif
    860 		xfer->c_lenoff = len - xfer->c_bcount;
    861 		if (xfer->c_bcount < len) {
    862 			printf("wdc_atapi_intr: warning: write only "
    863 			    "%d of %d requested bytes\n", xfer->c_bcount, len);
    864 			len = xfer->c_bcount;
    865 		}
    866 
    867 #if NATA_PIOBM
    868 		if (xfer->c_flags & C_PIOBM) {
    869 			/* start the busmastering PIO */
    870 			(*wdc->piobm_start)(wdc->dma_arg,
    871 			    chp->ch_channel, xfer->c_drive,
    872 			    xfer->c_skip, len, WDC_PIOBM_XFER_IRQ);
    873 			chp->ch_flags |= ATACH_DMA_WAIT | ATACH_IRQ_WAIT |
    874 			    ATACH_PIOBM_WAIT;
    875 			return 1;
    876 		}
    877 #endif
    878 		wdc->dataout_pio(chp, drvp->drive_flags,
    879 	    	    (char *)xfer->c_databuf + xfer->c_skip, len);
    880 
    881 #if NATA_PIOBM
    882 	end_piobm_dataout:
    883 #endif
    884 		for (i = xfer->c_lenoff; i > 0; i -= 2)
    885 			bus_space_write_2(wdr->cmd_iot,
    886 			    wdr->cmd_iohs[wd_data], 0, 0);
    887 
    888 		xfer->c_skip += len;
    889 		xfer->c_bcount -= len;
    890 		if ((sc_xfer->xs_control & XS_CTL_POLL) == 0) {
    891 			chp->ch_flags |= ATACH_IRQ_WAIT;
    892 		}
    893 		return 1;
    894 
    895 	case PHASE_DATAIN:
    896 		/* Read data */
    897 		ATADEBUG_PRINT(("PHASE_DATAIN\n"), DEBUG_INTR);
    898 #if NATA_DMA
    899 		if ((sc_xfer->xs_control & XS_CTL_DATA_IN) == 0 ||
    900 		    (xfer->c_flags & C_DMA) != 0) {
    901 			printf("wdc_atapi_intr: bad data phase DATAIN\n");
    902 			if (xfer->c_flags & C_DMA) {
    903 				ata_dmaerr(drvp,
    904 				    (xfer->c_flags & C_POLL) ? AT_POLL : 0);
    905 			}
    906 			sc_xfer->error = XS_TIMEOUT;
    907 			wdc_atapi_reset(chp, xfer);
    908 			return 1;
    909 		}
    910 #endif
    911 		xfer->c_lenoff = len - xfer->c_bcount;
    912 		if (xfer->c_bcount < len) {
    913 			printf("wdc_atapi_intr: warning: reading only "
    914 			    "%d of %d bytes\n", xfer->c_bcount, len);
    915 			len = xfer->c_bcount;
    916 		}
    917 
    918 #if NATA_PIOBM
    919 		if (xfer->c_flags & C_PIOBM) {
    920 			/* start the busmastering PIO */
    921 			(*wdc->piobm_start)(wdc->dma_arg,
    922 			    chp->ch_channel, xfer->c_drive,
    923 			    xfer->c_skip, len, WDC_PIOBM_XFER_IRQ);
    924 			chp->ch_flags |= ATACH_DMA_WAIT | ATACH_IRQ_WAIT |
    925 			    ATACH_PIOBM_WAIT;
    926 			return 1;
    927 		}
    928 #endif
    929 		wdc->datain_pio(chp, drvp->drive_flags,
    930 		    (char *)xfer->c_databuf + xfer->c_skip, len);
    931 
    932 #if NATA_PIOBM
    933 	end_piobm_datain:
    934 #endif
    935 		if (xfer->c_lenoff > 0)
    936 			wdcbit_bucket(chp, len - xfer->c_bcount);
    937 
    938 		xfer->c_skip += len;
    939 		xfer->c_bcount -= len;
    940 		if ((sc_xfer->xs_control & XS_CTL_POLL) == 0) {
    941 			chp->ch_flags |= ATACH_IRQ_WAIT;
    942 		}
    943 		return 1;
    944 
    945 	case PHASE_ABORTED:
    946 	case PHASE_COMPLETED:
    947 		ATADEBUG_PRINT(("PHASE_COMPLETED\n"), DEBUG_INTR);
    948 #if NATA_DMA
    949 		if (xfer->c_flags & C_DMA) {
    950 			xfer->c_bcount -= sc_xfer->datalen;
    951 		}
    952 #endif
    953 		sc_xfer->resid = xfer->c_bcount;
    954 		wdc_atapi_phase_complete(xfer);
    955 		return(1);
    956 
    957 	default:
    958 		if (++retries<500) {
    959 			DELAY(100);
    960 			chp->ch_status = bus_space_read_1(wdr->cmd_iot,
    961 			    wdr->cmd_iohs[wd_status], 0);
    962 			chp->ch_error = bus_space_read_1(wdr->cmd_iot,
    963 			    wdr->cmd_iohs[wd_error], 0);
    964 			goto again;
    965 		}
    966 		printf("wdc_atapi_intr: unknown phase 0x%x\n", phase);
    967 		if (chp->ch_status & WDCS_ERR) {
    968 			sc_xfer->error = XS_SHORTSENSE;
    969 			sc_xfer->sense.atapi_sense = chp->ch_error;
    970 		} else {
    971 #if NATA_DMA
    972 			if (xfer->c_flags & C_DMA) {
    973 				ata_dmaerr(drvp,
    974 				    (xfer->c_flags & C_POLL) ? AT_POLL : 0);
    975 			}
    976 #endif
    977 			sc_xfer->error = XS_RESET;
    978 			wdc_atapi_reset(chp, xfer);
    979 			return (1);
    980 		}
    981 	}
    982 	ATADEBUG_PRINT(("wdc_atapi_intr: wdc_atapi_done() (end), error 0x%x "
    983 	    "sense 0x%x\n", sc_xfer->error, sc_xfer->sense.atapi_sense),
    984 	    DEBUG_INTR);
    985 	wdc_atapi_done(chp, xfer);
    986 	return (1);
    987 }
    988 
    989 static void
    990 wdc_atapi_phase_complete(struct ata_xfer *xfer)
    991 {
    992 	struct ata_channel *chp = xfer->c_chp;
    993 	struct atac_softc *atac = chp->ch_atac;
    994 #if NATA_DMA || NATA_PIOBM
    995 	struct wdc_softc *wdc = CHAN_TO_WDC(chp);
    996 #endif
    997 	struct scsipi_xfer *sc_xfer = xfer->c_cmd;
    998 	struct ata_drive_datas *drvp = &chp->ch_drive[xfer->c_drive];
    999 
   1000 	/* wait for DSC if needed */
   1001 	if (drvp->drive_flags & DRIVE_ATAPIST) {
   1002 		ATADEBUG_PRINT(("wdc_atapi_phase_complete(%s:%d:%d) "
   1003 		    "polldsc %d\n", device_xname(atac->atac_dev),
   1004 		    chp->ch_channel,
   1005 		    xfer->c_drive, xfer->c_dscpoll), DEBUG_XFERS);
   1006 #if 1
   1007 		if (cold)
   1008 			panic("wdc_atapi_phase_complete: cold");
   1009 #endif
   1010 		if (wdcwait(chp, WDCS_DSC, WDCS_DSC, 10,
   1011 		    AT_POLL) == WDCWAIT_TOUT) {
   1012 			/* 10ms not enough, try again in 1 tick */
   1013 			if (xfer->c_dscpoll++ >
   1014 			    mstohz(sc_xfer->timeout)) {
   1015 				printf("%s:%d:%d: wait_for_dsc "
   1016 				    "failed\n",
   1017 				    device_xname(atac->atac_dev),
   1018 				    chp->ch_channel, xfer->c_drive);
   1019 				sc_xfer->error = XS_TIMEOUT;
   1020 				wdc_atapi_reset(chp, xfer);
   1021 				return;
   1022 			} else
   1023 				callout_reset(&chp->ch_callout, 1,
   1024 				    wdc_atapi_polldsc, xfer);
   1025 			return;
   1026 		}
   1027 	}
   1028 
   1029 	/*
   1030 	 * Some drive occasionally set WDCS_ERR with
   1031 	 * "ATA illegal length indication" in the error
   1032 	 * register. If we read some data the sense is valid
   1033 	 * anyway, so don't report the error.
   1034 	 */
   1035 	if (chp->ch_status & WDCS_ERR &&
   1036 	    ((sc_xfer->xs_control & XS_CTL_REQSENSE) == 0 ||
   1037 	    sc_xfer->resid == sc_xfer->datalen)) {
   1038 		/* save the short sense */
   1039 		sc_xfer->error = XS_SHORTSENSE;
   1040 		sc_xfer->sense.atapi_sense = chp->ch_error;
   1041 		if ((sc_xfer->xs_periph->periph_quirks &
   1042 		    PQUIRK_NOSENSE) == 0) {
   1043 			/* ask scsipi to send a REQUEST_SENSE */
   1044 			sc_xfer->error = XS_BUSY;
   1045 			sc_xfer->status = SCSI_CHECK;
   1046 		}
   1047 #if NATA_DMA || NATA_PIOBM
   1048 		else if (wdc->dma_status &
   1049 		    (WDC_DMAST_NOIRQ | WDC_DMAST_ERR)) {
   1050 #if NATA_DMA
   1051 			ata_dmaerr(drvp,
   1052 			    (xfer->c_flags & C_POLL) ? AT_POLL : 0);
   1053 #endif
   1054 			sc_xfer->error = XS_RESET;
   1055 			wdc_atapi_reset(chp, xfer);
   1056 			return;
   1057 		}
   1058 #endif
   1059 	}
   1060 	if (xfer->c_bcount != 0) {
   1061 		ATADEBUG_PRINT(("wdc_atapi_intr: bcount value is "
   1062 		    "%d after io\n", xfer->c_bcount), DEBUG_XFERS);
   1063 	}
   1064 #ifdef DIAGNOSTIC
   1065 	if (xfer->c_bcount < 0) {
   1066 		printf("wdc_atapi_intr warning: bcount value "
   1067 		    "is %d after io\n", xfer->c_bcount);
   1068 	}
   1069 #endif
   1070 	ATADEBUG_PRINT(("wdc_atapi_phase_complete: wdc_atapi_done(), "
   1071 	    "error 0x%x sense 0x%x\n", sc_xfer->error,
   1072 	    sc_xfer->sense.atapi_sense), DEBUG_INTR);
   1073 	wdc_atapi_done(chp, xfer);
   1074 }
   1075 
   1076 static void
   1077 wdc_atapi_done(struct ata_channel *chp, struct ata_xfer *xfer)
   1078 {
   1079 	struct atac_softc *atac = chp->ch_atac;
   1080 	struct scsipi_xfer *sc_xfer = xfer->c_cmd;
   1081 	int drive = xfer->c_drive;
   1082 
   1083 	ATADEBUG_PRINT(("wdc_atapi_done %s:%d:%d: flags 0x%x\n",
   1084 	    device_xname(atac->atac_dev), chp->ch_channel, xfer->c_drive,
   1085 	    (u_int)xfer->c_flags), DEBUG_XFERS);
   1086 	callout_stop(&chp->ch_callout);
   1087 	/* mark controller inactive and free the command */
   1088 	chp->ch_queue->active_xfer = NULL;
   1089 	ata_free_xfer(chp, xfer);
   1090 
   1091 	if (chp->ch_drive[drive].drive_flags & DRIVE_WAITDRAIN) {
   1092 		sc_xfer->error = XS_DRIVER_STUFFUP;
   1093 		chp->ch_drive[drive].drive_flags &= ~DRIVE_WAITDRAIN;
   1094 		wakeup(&chp->ch_queue->active_xfer);
   1095 	}
   1096 
   1097 	ATADEBUG_PRINT(("wdc_atapi_done: scsipi_done\n"), DEBUG_XFERS);
   1098 	scsipi_done(sc_xfer);
   1099 	ATADEBUG_PRINT(("atastart from wdc_atapi_done, flags 0x%x\n",
   1100 	    chp->ch_flags), DEBUG_XFERS);
   1101 	atastart(chp);
   1102 }
   1103 
   1104 static void
   1105 wdc_atapi_reset(struct ata_channel *chp, struct ata_xfer *xfer)
   1106 {
   1107 	struct atac_softc *atac = chp->ch_atac;
   1108 	struct ata_drive_datas *drvp = &chp->ch_drive[xfer->c_drive];
   1109 	struct scsipi_xfer *sc_xfer = xfer->c_cmd;
   1110 
   1111 	wdccommandshort(chp, xfer->c_drive, ATAPI_SOFT_RESET);
   1112 	drvp->state = 0;
   1113 	if (wdc_wait_for_unbusy(chp, WDC_RESET_WAIT, AT_POLL) != 0) {
   1114 		printf("%s:%d:%d: reset failed\n",
   1115 		    device_xname(atac->atac_dev), chp->ch_channel,
   1116 		    xfer->c_drive);
   1117 		sc_xfer->error = XS_SELTIMEOUT;
   1118 	}
   1119 	wdc_atapi_done(chp, xfer);
   1120 	return;
   1121 }
   1122 
   1123 static void
   1124 wdc_atapi_polldsc(void *arg)
   1125 {
   1126 
   1127 	wdc_atapi_phase_complete(arg);
   1128 }
   1129