Home | History | Annotate | Line # | Download | only in scsipi
atapi_wdc.c revision 1.127
      1 /*	$NetBSD: atapi_wdc.c,v 1.127 2017/10/08 21:33:38 christos Exp $	*/
      2 
      3 /*
      4  * Copyright (c) 1998, 2001 Manuel Bouyer.
      5  *
      6  * Redistribution and use in source and binary forms, with or without
      7  * modification, are permitted provided that the following conditions
      8  * are met:
      9  * 1. Redistributions of source code must retain the above copyright
     10  *    notice, this list of conditions and the following disclaimer.
     11  * 2. Redistributions in binary form must reproduce the above copyright
     12  *    notice, this list of conditions and the following disclaimer in the
     13  *    documentation and/or other materials provided with the distribution.
     14  *
     15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
     16  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
     17  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
     18  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
     19  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
     20  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
     21  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
     22  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
     23  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
     24  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     25  */
     26 
     27 #include <sys/cdefs.h>
     28 __KERNEL_RCSID(0, "$NetBSD: atapi_wdc.c,v 1.127 2017/10/08 21:33:38 christos Exp $");
     29 
     30 #ifndef ATADEBUG
     31 #define ATADEBUG
     32 #endif /* ATADEBUG */
     33 
     34 #include <sys/param.h>
     35 #include <sys/systm.h>
     36 #include <sys/kernel.h>
     37 #include <sys/file.h>
     38 #include <sys/stat.h>
     39 #include <sys/buf.h>
     40 #include <sys/malloc.h>
     41 #include <sys/device.h>
     42 #include <sys/syslog.h>
     43 #include <sys/proc.h>
     44 #include <sys/dvdio.h>
     45 
     46 #include <sys/intr.h>
     47 #include <sys/bus.h>
     48 
     49 #ifndef __BUS_SPACE_HAS_STREAM_METHODS
     50 #define	bus_space_write_multi_stream_2	bus_space_write_multi_2
     51 #define	bus_space_write_multi_stream_4	bus_space_write_multi_4
     52 #define	bus_space_read_multi_stream_2	bus_space_read_multi_2
     53 #define	bus_space_read_multi_stream_4	bus_space_read_multi_4
     54 #endif /* __BUS_SPACE_HAS_STREAM_METHODS */
     55 
     56 #include <dev/ata/ataconf.h>
     57 #include <dev/ata/atareg.h>
     58 #include <dev/ata/atavar.h>
     59 #include <dev/ic/wdcreg.h>
     60 #include <dev/ic/wdcvar.h>
     61 
     62 #include <dev/scsipi/scsi_all.h> /* for SCSI status */
     63 
     64 #define DEBUG_INTR   0x01
     65 #define DEBUG_XFERS  0x02
     66 #define DEBUG_STATUS 0x04
     67 #define DEBUG_FUNCS  0x08
     68 #define DEBUG_PROBE  0x10
     69 #ifdef ATADEBUG
     70 int wdcdebug_atapi_mask = 0;
     71 #define ATADEBUG_PRINT(args, level) \
     72 	if (wdcdebug_atapi_mask & (level)) \
     73 		printf args
     74 #else
     75 #define ATADEBUG_PRINT(args, level)
     76 #endif
     77 
     78 #define ATAPI_DELAY 10	/* 10 ms, this is used only before sending a cmd */
     79 #define ATAPI_MODE_DELAY 1000	/* 1s, timeout for SET_FEATYRE cmds */
     80 
     81 static int	wdc_atapi_get_params(struct scsipi_channel *, int,
     82 				     struct ataparams *);
     83 static void	wdc_atapi_probe_device(struct atapibus_softc *, int);
     84 static void	wdc_atapi_minphys (struct buf *bp);
     85 static int	wdc_atapi_start(struct ata_channel *,struct ata_xfer *);
     86 static int	wdc_atapi_intr(struct ata_channel *, struct ata_xfer *, int);
     87 static void	wdc_atapi_kill_xfer(struct ata_channel *,
     88 				    struct ata_xfer *, int);
     89 static void	wdc_atapi_phase_complete(struct ata_xfer *);
     90 static void	wdc_atapi_poll(struct ata_channel *, struct ata_xfer *);
     91 static void	wdc_atapi_done(struct ata_channel *, struct ata_xfer *);
     92 static void	wdc_atapi_reset(struct ata_channel *, struct ata_xfer *);
     93 static void	wdc_atapi_scsipi_request(struct scsipi_channel *,
     94 					 scsipi_adapter_req_t, void *);
     95 static void	wdc_atapi_kill_pending(struct scsipi_periph *);
     96 static void	wdc_atapi_polldsc(void *arg);
     97 
     98 #define MAX_SIZE MAXPHYS
     99 
    100 static const struct scsipi_bustype wdc_atapi_bustype = {
    101 	SCSIPI_BUSTYPE_ATAPI,
    102 	atapi_scsipi_cmd,
    103 	atapi_interpret_sense,
    104 	atapi_print_addr,
    105 	wdc_atapi_kill_pending,
    106 	NULL,
    107 };
    108 
    109 void
    110 wdc_atapibus_attach(struct atabus_softc *ata_sc)
    111 {
    112 	struct ata_channel *chp = ata_sc->sc_chan;
    113 	struct atac_softc *atac = chp->ch_atac;
    114 	struct scsipi_adapter *adapt = &atac->atac_atapi_adapter._generic;
    115 	struct scsipi_channel *chan = &chp->ch_atapi_channel;
    116 
    117 	/*
    118 	 * Fill in the scsipi_adapter.
    119 	 */
    120 	adapt->adapt_dev = atac->atac_dev;
    121 	adapt->adapt_nchannels = atac->atac_nchannels;
    122 	adapt->adapt_request = wdc_atapi_scsipi_request;
    123 	adapt->adapt_minphys = wdc_atapi_minphys;
    124 	if (atac->atac_cap & ATAC_CAP_NOIRQ)
    125 		adapt->adapt_flags |= SCSIPI_ADAPT_POLL_ONLY;
    126 	atac->atac_atapi_adapter.atapi_probe_device = wdc_atapi_probe_device;
    127 
    128 	/*
    129 	 * Fill in the scsipi_channel.
    130 	 */
    131 	memset(chan, 0, sizeof(*chan));
    132 	chan->chan_adapter = adapt;
    133 	chan->chan_bustype = &wdc_atapi_bustype;
    134 	chan->chan_channel = chp->ch_channel;
    135 	chan->chan_flags = SCSIPI_CHAN_OPENINGS;
    136 	chan->chan_openings = 1;
    137 	chan->chan_max_periph = 1;
    138 	chan->chan_ntargets = 2;
    139 	chan->chan_nluns = 1;
    140 
    141 	chp->atapibus = config_found_ia(ata_sc->sc_dev, "atapi", chan,
    142 		atapiprint);
    143 }
    144 
    145 static void
    146 wdc_atapi_minphys(struct buf *bp)
    147 {
    148 
    149 	if (bp->b_bcount > MAX_SIZE)
    150 		bp->b_bcount = MAX_SIZE;
    151 	minphys(bp);
    152 }
    153 
    154 /*
    155  * Kill off all pending xfers for a periph.
    156  *
    157  * Must be called with adapter lock held
    158  */
    159 static void
    160 wdc_atapi_kill_pending(struct scsipi_periph *periph)
    161 {
    162 	struct atac_softc *atac =
    163 	    device_private(periph->periph_channel->chan_adapter->adapt_dev);
    164 	struct ata_channel *chp =
    165 	    atac->atac_channels[periph->periph_channel->chan_channel];
    166 
    167 	ata_kill_pending(&chp->ch_drive[periph->periph_target]);
    168 }
    169 
    170 static void
    171 wdc_atapi_kill_xfer(struct ata_channel *chp, struct ata_xfer *xfer, int reason)
    172 {
    173 	struct scsipi_xfer *sc_xfer = xfer->c_scsipi;
    174 	bool deactivate = true;
    175 
    176 	/* remove this command from xfer queue */
    177 	switch (reason) {
    178 	case KILL_GONE_INACTIVE:
    179 		deactivate = false;
    180 		/* FALLTHROUGH */
    181 	case KILL_GONE:
    182 		sc_xfer->error = XS_DRIVER_STUFFUP;
    183 		break;
    184 	case KILL_RESET:
    185 		sc_xfer->error = XS_RESET;
    186 		break;
    187 	default:
    188 		printf("wdc_ata_bio_kill_xfer: unknown reason %d\n",
    189 		    reason);
    190 		panic("wdc_ata_bio_kill_xfer");
    191 	}
    192 
    193 	if (deactivate)
    194 		ata_deactivate_xfer(chp, xfer);
    195 
    196 	ata_free_xfer(chp, xfer);
    197 	scsipi_done(sc_xfer);
    198 }
    199 
    200 static int
    201 wdc_atapi_get_params(struct scsipi_channel *chan, int drive,
    202     struct ataparams *id)
    203 {
    204 	struct wdc_softc *wdc = device_private(chan->chan_adapter->adapt_dev);
    205 	struct atac_softc *atac = &wdc->sc_atac;
    206 	struct wdc_regs *wdr = &wdc->regs[chan->chan_channel];
    207 	struct ata_channel *chp = atac->atac_channels[chan->chan_channel];
    208 	struct ata_xfer *xfer;
    209 	int rv;
    210 
    211 	xfer = ata_get_xfer(chp);
    212 	if (xfer == NULL) {
    213 		printf("wdc_atapi_get_params: no xfer\n");
    214 		return EBUSY;
    215 	}
    216 
    217 	xfer->c_ata_c.r_command = ATAPI_SOFT_RESET;
    218 	xfer->c_ata_c.r_st_bmask = 0;
    219 	xfer->c_ata_c.r_st_pmask = 0;
    220 	xfer->c_ata_c.flags = AT_WAIT | AT_POLL;
    221 	xfer->c_ata_c.timeout = WDC_RESET_WAIT;
    222 	if (wdc_exec_command(&chp->ch_drive[drive], xfer) != ATACMD_COMPLETE) {
    223 		printf("wdc_atapi_get_params: ATAPI_SOFT_RESET failed for"
    224 		    " drive %s:%d:%d: driver failed\n",
    225 		    device_xname(atac->atac_dev), chp->ch_channel, drive);
    226 		panic("wdc_atapi_get_params");
    227 	}
    228 	if (xfer->c_ata_c.flags & (AT_ERROR | AT_TIMEOU | AT_DF)) {
    229 		ATADEBUG_PRINT(("wdc_atapi_get_params: ATAPI_SOFT_RESET "
    230 		    "failed for drive %s:%d:%d: error 0x%x\n",
    231 		    device_xname(atac->atac_dev), chp->ch_channel, drive,
    232 		    xfer->c_ata_c.r_error), DEBUG_PROBE);
    233 		rv = -1;
    234 		goto out_xfer;
    235 	}
    236 	chp->ch_drive[drive].state = 0;
    237 
    238 	ata_free_xfer(chp, xfer);
    239 
    240 	(void)bus_space_read_1(wdr->cmd_iot, wdr->cmd_iohs[wd_status], 0);
    241 
    242 	/* Some ATAPI devices need a bit more time after software reset. */
    243 	delay(5000);
    244 	if (ata_get_params(&chp->ch_drive[drive], AT_WAIT, id) != 0) {
    245 		ATADEBUG_PRINT(("wdc_atapi_get_params: ATAPI_IDENTIFY_DEVICE "
    246 		    "failed for drive %s:%d:%d\n",
    247 		    device_xname(atac->atac_dev), chp->ch_channel, drive),
    248 		    DEBUG_PROBE);
    249 		rv = -1;
    250 		goto out;
    251 	}
    252 	rv = 0;
    253 out:
    254 	return rv;
    255 
    256 out_xfer:
    257 	ata_free_xfer(chp, xfer);
    258 	return rv;
    259 }
    260 
    261 static void
    262 wdc_atapi_probe_device(struct atapibus_softc *sc, int target)
    263 {
    264 	struct scsipi_channel *chan = sc->sc_channel;
    265 	struct scsipi_periph *periph;
    266 	struct ataparams ids;
    267 	struct ataparams *id = &ids;
    268 	struct wdc_softc *wdc = device_private(chan->chan_adapter->adapt_dev);
    269 	struct atac_softc *atac = &wdc->sc_atac;
    270 	struct ata_channel *chp = atac->atac_channels[chan->chan_channel];
    271 	struct ata_drive_datas *drvp = &chp->ch_drive[target];
    272 	struct scsipibus_attach_args sa;
    273 	char serial_number[21], model[41], firmware_revision[9];
    274 	int s;
    275 
    276 	/* skip if already attached */
    277 	if (scsipi_lookup_periph(chan, target, 0) != NULL)
    278 		return;
    279 
    280 	/* if no ATAPI device detected at wdc attach time, skip */
    281 	if (drvp->drive_type != ATA_DRIVET_ATAPI) {
    282 		ATADEBUG_PRINT(("wdc_atapi_probe_device: "
    283 		    "drive %d not present\n", target), DEBUG_PROBE);
    284 		return;
    285 	}
    286 
    287 	if (wdc_atapi_get_params(chan, target, id) == 0) {
    288 #ifdef ATAPI_DEBUG_PROBE
    289 		printf("%s drive %d: cmdsz 0x%x drqtype 0x%x\n",
    290 		    device_xname(sc->sc_dev), target,
    291 		    id->atap_config & ATAPI_CFG_CMD_MASK,
    292 		    id->atap_config & ATAPI_CFG_DRQ_MASK);
    293 #endif
    294 		periph = scsipi_alloc_periph(M_NOWAIT);
    295 		if (periph == NULL) {
    296 			aprint_error_dev(sc->sc_dev,
    297 			    "unable to allocate periph for drive %d\n",
    298 			    target);
    299 			return;
    300 		}
    301 		periph->periph_dev = NULL;
    302 		periph->periph_channel = chan;
    303 		periph->periph_switch = &atapi_probe_periphsw;
    304 		periph->periph_target = target;
    305 		periph->periph_lun = 0;
    306 		periph->periph_quirks = PQUIRK_ONLYBIG;
    307 
    308 #ifdef SCSIPI_DEBUG
    309 		if (SCSIPI_DEBUG_TYPE == SCSIPI_BUSTYPE_ATAPI &&
    310 		    SCSIPI_DEBUG_TARGET == target)
    311 			periph->periph_dbflags |= SCSIPI_DEBUG_FLAGS;
    312 #endif
    313 		periph->periph_type = ATAPI_CFG_TYPE(id->atap_config);
    314 		if (id->atap_config & ATAPI_CFG_REMOV)
    315 			periph->periph_flags |= PERIPH_REMOVABLE;
    316 		if (periph->periph_type == T_SEQUENTIAL) {
    317 			s = splbio();
    318 			drvp->drive_flags |= ATA_DRIVE_ATAPIDSCW;
    319 			splx(s);
    320 		}
    321 
    322 		sa.sa_periph = periph;
    323 		sa.sa_inqbuf.type =  ATAPI_CFG_TYPE(id->atap_config);
    324 		sa.sa_inqbuf.removable = id->atap_config & ATAPI_CFG_REMOV ?
    325 		    T_REMOV : T_FIXED;
    326 		strnvisx(model, sizeof(model), id->atap_model,
    327 		    sizeof(id->atap_model), VIS_TRIM|VIS_SAFE|VIS_OCTAL);
    328 		strnvisx(serial_number, sizeof(serial_number),
    329 		    id->atap_serial, sizeof(id->atap_serial),
    330 		    VIS_TRIM|VIS_SAFE|VIS_OCTAL);
    331 		strnvisx(firmware_revision, sizeof(firmware_revision),
    332 		    id->atap_revision, sizeof(id->atap_revision),
    333 		    VIS_TRIM|VIS_SAFE|VIS_OCTAL);
    334 		sa.sa_inqbuf.vendor = model;
    335 		sa.sa_inqbuf.product = serial_number;
    336 		sa.sa_inqbuf.revision = firmware_revision;
    337 
    338 		/*
    339 		 * Determine the operating mode capabilities of the device.
    340 		 */
    341 		if ((id->atap_config & ATAPI_CFG_CMD_MASK) == ATAPI_CFG_CMD_16)
    342 			periph->periph_cap |= PERIPH_CAP_CMD16;
    343 		/* XXX This is gross. */
    344 		periph->periph_cap |= (id->atap_config & ATAPI_CFG_DRQ_MASK);
    345 
    346 		drvp->drv_softc = atapi_probe_device(sc, target, periph, &sa);
    347 
    348 		if (drvp->drv_softc)
    349 			ata_probe_caps(drvp);
    350 		else {
    351 			s = splbio();
    352 			drvp->drive_type = ATA_DRIVET_NONE;
    353 			splx(s);
    354 		}
    355 	} else {
    356 		s = splbio();
    357 		drvp->drive_type = ATA_DRIVET_NONE;
    358 		splx(s);
    359 	}
    360 }
    361 
    362 static void
    363 wdc_atapi_scsipi_request(struct scsipi_channel *chan, scsipi_adapter_req_t req,
    364     void *arg)
    365 {
    366 	struct scsipi_adapter *adapt = chan->chan_adapter;
    367 	struct scsipi_periph *periph;
    368 	struct scsipi_xfer *sc_xfer;
    369 	struct wdc_softc *wdc = device_private(adapt->adapt_dev);
    370 	struct atac_softc *atac = &wdc->sc_atac;
    371 	struct ata_xfer *xfer;
    372 	int channel = chan->chan_channel;
    373 	int drive, s;
    374 
    375 	switch (req) {
    376 	case ADAPTER_REQ_RUN_XFER:
    377 		sc_xfer = arg;
    378 		periph = sc_xfer->xs_periph;
    379 		drive = periph->periph_target;
    380 
    381 		ATADEBUG_PRINT(("wdc_atapi_scsipi_request %s:%d:%d\n",
    382 		    device_xname(atac->atac_dev), channel, drive),
    383 		    DEBUG_XFERS);
    384 		if (!device_is_active(atac->atac_dev)) {
    385 			sc_xfer->error = XS_DRIVER_STUFFUP;
    386 			scsipi_done(sc_xfer);
    387 			return;
    388 		}
    389 
    390 		xfer = ata_get_xfer_ext(atac->atac_channels[channel], false, 0);
    391 		if (xfer == NULL) {
    392 			sc_xfer->error = XS_RESOURCE_SHORTAGE;
    393 			scsipi_done(sc_xfer);
    394 			return;
    395 		}
    396 
    397 		if (sc_xfer->xs_control & XS_CTL_POLL)
    398 			xfer->c_flags |= C_POLL;
    399 #if NATA_DMA
    400 		if ((atac->atac_channels[channel]->ch_drive[drive].drive_flags &
    401 		    (ATA_DRIVE_DMA | ATA_DRIVE_UDMA)) && sc_xfer->datalen > 0)
    402 			xfer->c_flags |= C_DMA;
    403 #endif
    404 #if NATA_DMA && NATA_PIOBM
    405 		else
    406 #endif
    407 #if NATA_PIOBM
    408 		if ((atac->atac_cap & ATAC_CAP_PIOBM) &&
    409 		    sc_xfer->datalen > 0)
    410 			xfer->c_flags |= C_PIOBM;
    411 #endif
    412 		xfer->c_drive = drive;
    413 		xfer->c_flags |= C_ATAPI;
    414 #if NATA_DMA
    415 		if (sc_xfer->cmd->opcode == GPCMD_REPORT_KEY ||
    416 		    sc_xfer->cmd->opcode == GPCMD_SEND_KEY ||
    417 		    sc_xfer->cmd->opcode == GPCMD_READ_DVD_STRUCTURE) {
    418 			/*
    419 			 * DVD authentication commands must always be done in
    420 			 * PIO mode.
    421 			 */
    422 			xfer->c_flags &= ~C_DMA;
    423 		}
    424 
    425 		/*
    426 		 * DMA normally can't deal with transfers which are not a
    427 		 * multiple of its databus width. It's a bug to request odd
    428 		 * length transfers for ATAPI.
    429 		 *
    430 		 * Some devices also can't cope with unaligned DMA xfers
    431 		 * either. Also some devices seem to not handle DMA xfers of
    432 		 * less than 4 bytes.
    433 		 *
    434 		 * By enforcing at least 4 byte aligned offset and length for
    435 		 * DMA, we might use PIO where DMA could be allowed but better
    436 		 * safe than sorry as recent problems proved.
    437 		 *
    438 		 * Offending structures that are thus done by PIO instead of
    439 		 * DMA are normally small structures since all bulkdata is
    440 		 * aligned. But as the request may come from userland, we have
    441 		 * to protect against it anyway.
    442 		 *
    443 		 * XXX check for the 32 bit wide flag?
    444 		 */
    445 
    446 		if (((uintptr_t) sc_xfer->data) & 0x03)
    447 			xfer->c_flags &= ~C_DMA;
    448 		if ((sc_xfer->datalen < 4) || (sc_xfer->datalen & 0x03))
    449 			xfer->c_flags &= ~C_DMA;
    450 #endif	/* NATA_DMA */
    451 
    452 		xfer->c_scsipi = sc_xfer;
    453 		xfer->c_databuf = sc_xfer->data;
    454 		xfer->c_bcount = sc_xfer->datalen;
    455 		xfer->c_start = wdc_atapi_start;
    456 		xfer->c_intr = wdc_atapi_intr;
    457 		xfer->c_poll = wdc_atapi_poll;
    458 		xfer->c_abort = wdc_atapi_reset;
    459 		xfer->c_kill_xfer = wdc_atapi_kill_xfer;
    460 		xfer->c_dscpoll = 0;
    461 		s = splbio();
    462 		ata_exec_xfer(atac->atac_channels[channel], xfer);
    463 #ifdef DIAGNOSTIC
    464 		if ((sc_xfer->xs_control & XS_CTL_POLL) != 0 &&
    465 		    (sc_xfer->xs_status & XS_STS_DONE) == 0)
    466 			panic("wdc_atapi_scsipi_request: polled command "
    467 			    "not done");
    468 #endif
    469 		splx(s);
    470 		return;
    471 
    472 	default:
    473 		/* Not supported, nothing to do. */
    474 		;
    475 	}
    476 }
    477 
    478 static int
    479 wdc_atapi_start(struct ata_channel *chp, struct ata_xfer *xfer)
    480 {
    481 	struct atac_softc *atac = chp->ch_atac;
    482 	struct wdc_softc *wdc = CHAN_TO_WDC(chp);
    483 	struct wdc_regs *wdr = &wdc->regs[chp->ch_channel];
    484 	struct scsipi_xfer *sc_xfer = xfer->c_scsipi;
    485 	struct ata_drive_datas *drvp = &chp->ch_drive[xfer->c_drive];
    486 	int wait_flags = (sc_xfer->xs_control & XS_CTL_POLL) ? AT_POLL : 0;
    487 	int tfd;
    488 	const char *errstring;
    489 
    490 	ATADEBUG_PRINT(("wdc_atapi_start %s:%d:%d, scsi flags 0x%x \n",
    491 	    device_xname(atac->atac_dev), chp->ch_channel, drvp->drive,
    492 	    sc_xfer->xs_control), DEBUG_XFERS);
    493 
    494 	ata_channel_lock_owned(chp);
    495 
    496 #if NATA_DMA
    497 	if ((xfer->c_flags & C_DMA) && (drvp->n_xfers <= NXFER))
    498 		drvp->n_xfers++;
    499 #endif
    500 	/* Do control operations specially. */
    501 	if (__predict_false(drvp->state < READY)) {
    502 		/* If it's not a polled command, we need the kernel thread */
    503 		if ((sc_xfer->xs_control & XS_CTL_POLL) == 0 &&
    504 		    (chp->ch_flags & ATACH_TH_RUN) == 0) {
    505 			return ATASTART_TH;
    506 		}
    507 		/*
    508 		 * disable interrupts, all commands here should be quick
    509 		 * enough to be able to poll, and we don't go here that often
    510 		 */
    511 		bus_space_write_1(wdr->ctl_iot, wdr->ctl_ioh, wd_aux_ctlr,
    512 		     WDCTL_4BIT | WDCTL_IDS);
    513 		if (wdc->select)
    514 			wdc->select(chp, xfer->c_drive);
    515 		bus_space_write_1(wdr->cmd_iot, wdr->cmd_iohs[wd_sdh], 0,
    516 		    WDSD_IBM | (xfer->c_drive << 4));
    517 		/* Don't try to set mode if controller can't be adjusted */
    518 		if (atac->atac_set_modes == NULL)
    519 			goto ready;
    520 		/* Also don't try if the drive didn't report its mode */
    521 		if ((drvp->drive_flags & ATA_DRIVE_MODE) == 0)
    522 			goto ready;
    523 		errstring = "unbusy";
    524 		if (wdc_wait_for_unbusy(chp, ATAPI_DELAY, wait_flags, &tfd))
    525 			goto timeout;
    526 		wdccommand(chp, drvp->drive, SET_FEATURES, 0, 0, 0,
    527 		    0x08 | drvp->PIO_mode, WDSF_SET_MODE);
    528 		errstring = "piomode";
    529 		if (wdc_wait_for_unbusy(chp, ATAPI_MODE_DELAY, wait_flags,
    530 		    &tfd))
    531 			goto timeout;
    532 		if (ATACH_ST(tfd) & WDCS_ERR) {
    533 			if (ATACH_ST(tfd) == WDCE_ABRT) {
    534 				/*
    535 				 * Some ATAPI drives reject PIO settings.
    536 				 * Fall back to PIO mode 3 since that's the
    537 				 * minimum for ATAPI.
    538 				 */
    539 				printf("%s:%d:%d: PIO mode %d rejected, "
    540 				    "falling back to PIO mode 3\n",
    541 				    device_xname(atac->atac_dev),
    542 				    chp->ch_channel, xfer->c_drive,
    543 				    drvp->PIO_mode);
    544 				if (drvp->PIO_mode > 3)
    545 					drvp->PIO_mode = 3;
    546 			} else
    547 				goto error;
    548 		}
    549 #if NATA_DMA
    550 #if NATA_UDMA
    551 		if (drvp->drive_flags & ATA_DRIVE_UDMA) {
    552 			wdccommand(chp, drvp->drive, SET_FEATURES, 0, 0, 0,
    553 			    0x40 | drvp->UDMA_mode, WDSF_SET_MODE);
    554 		} else
    555 #endif
    556 		if (drvp->drive_flags & ATA_DRIVE_DMA) {
    557 			wdccommand(chp, drvp->drive, SET_FEATURES, 0, 0, 0,
    558 			    0x20 | drvp->DMA_mode, WDSF_SET_MODE);
    559 		} else {
    560 			goto ready;
    561 		}
    562 		errstring = "dmamode";
    563 		if (wdc_wait_for_unbusy(chp, ATAPI_MODE_DELAY, wait_flags,
    564 		    &tfd))
    565 			goto timeout;
    566 		if (ATACH_ST(tfd) & WDCS_ERR) {
    567 			if (ATACH_ERR(tfd) == WDCE_ABRT) {
    568 #if NATA_UDMA
    569 				if (drvp->drive_flags & ATA_DRIVE_UDMA)
    570 					goto error;
    571 				else
    572 #endif
    573 				{
    574 					/*
    575 					 * The drive rejected our DMA setting.
    576 					 * Fall back to mode 1.
    577 					 */
    578 					printf("%s:%d:%d: DMA mode %d rejected, "
    579 					    "falling back to DMA mode 0\n",
    580 					    device_xname(atac->atac_dev),
    581 					    chp->ch_channel, xfer->c_drive,
    582 					    drvp->DMA_mode);
    583 					if (drvp->DMA_mode > 0)
    584 						drvp->DMA_mode = 0;
    585 				}
    586 			} else
    587 				goto error;
    588 		}
    589 #endif	/* NATA_DMA */
    590 ready:
    591 		drvp->state = READY;
    592 		bus_space_write_1(wdr->ctl_iot, wdr->ctl_ioh, wd_aux_ctlr,
    593 		    WDCTL_4BIT);
    594 		delay(10); /* some drives need a little delay here */
    595 	}
    596 	/* start timeout machinery */
    597 	if ((sc_xfer->xs_control & XS_CTL_POLL) == 0)
    598 		callout_reset(&xfer->c_timo_callout, mstohz(sc_xfer->timeout),
    599 		    wdctimeout, xfer);
    600 
    601 	if (wdc->select)
    602 		wdc->select(chp, xfer->c_drive);
    603 	bus_space_write_1(wdr->cmd_iot, wdr->cmd_iohs[wd_sdh], 0,
    604 	    WDSD_IBM | (xfer->c_drive << 4));
    605 	switch (wdc_wait_for_unbusy(chp, ATAPI_DELAY, wait_flags, &tfd)) {
    606 	case WDCWAIT_OK:
    607 		break;
    608 	case WDCWAIT_TOUT:
    609 		printf("wdc_atapi_start: not ready, st = %02x\n",
    610 		    ATACH_ST(tfd));
    611 		sc_xfer->error = XS_TIMEOUT;
    612 		return ATASTART_ABORT;
    613 	case WDCWAIT_THR:
    614 		return ATASTART_TH;
    615 	}
    616 
    617 	/*
    618 	 * Even with WDCS_ERR, the device should accept a command packet
    619 	 * Limit length to what can be stuffed into the cylinder register
    620 	 * (16 bits).  Some CD-ROMs seem to interpret '0' as 65536,
    621 	 * but not all devices do that and it's not obvious from the
    622 	 * ATAPI spec that that behaviour should be expected.  If more
    623 	 * data is necessary, multiple data transfer phases will be done.
    624 	 */
    625 
    626 	wdccommand(chp, xfer->c_drive, ATAPI_PKT_CMD,
    627 	    xfer->c_bcount <= 0xffff ? xfer->c_bcount : 0xffff,
    628 	    0, 0, 0,
    629 #if NATA_DMA
    630 	    (xfer->c_flags & C_DMA) ? ATAPI_PKT_CMD_FTRE_DMA :
    631 #endif
    632 	    0
    633 	    );
    634 
    635 #if NATA_PIOBM
    636 	if (xfer->c_flags & C_PIOBM) {
    637 		int error;
    638 		int dma_flags = (sc_xfer->xs_control & XS_CTL_DATA_IN)
    639 		    ?  WDC_DMA_READ : 0;
    640 		if (xfer->c_flags & C_POLL) {
    641 			/* XXX not supported yet --- fall back to PIO */
    642 			xfer->c_flags &= ~C_PIOBM;
    643 		} else {
    644 			/* Init the DMA channel. */
    645 			error = (*wdc->dma_init)(wdc->dma_arg,
    646 			    chp->ch_channel, xfer->c_drive,
    647 			    (char *)xfer->c_databuf,
    648 			    xfer->c_bcount,
    649 			    dma_flags | WDC_DMA_PIOBM_ATAPI);
    650 			if (error) {
    651 				if (error == EINVAL) {
    652 					/*
    653 					 * We can't do DMA on this transfer
    654 					 * for some reason.  Fall back to
    655 					 * PIO.
    656 					 */
    657 					xfer->c_flags &= ~C_PIOBM;
    658 					error = 0;
    659 				} else {
    660 					sc_xfer->error = XS_DRIVER_STUFFUP;
    661 					errstring = "piobm";
    662 					goto error;
    663 				}
    664 			}
    665 		}
    666 	}
    667 #endif
    668 	/*
    669 	 * If there is no interrupt for CMD input, busy-wait for it (done in
    670 	 * the interrupt routine. Poll routine will exit early in this case.
    671 	 */
    672 	if ((sc_xfer->xs_periph->periph_cap & ATAPI_CFG_DRQ_MASK) !=
    673 	    ATAPI_CFG_IRQ_DRQ || (sc_xfer->xs_control & XS_CTL_POLL))
    674 		return ATASTART_POLL;
    675 	else
    676 		return ATASTART_STARTED;
    677 
    678 timeout:
    679 	printf("%s:%d:%d: %s timed out\n",
    680 	    device_xname(atac->atac_dev), chp->ch_channel, xfer->c_drive,
    681 	    errstring);
    682 	sc_xfer->error = XS_TIMEOUT;
    683 	bus_space_write_1(wdr->ctl_iot, wdr->ctl_ioh, wd_aux_ctlr, WDCTL_4BIT);
    684 	delay(10); /* some drives need a little delay here */
    685 	return ATASTART_ABORT;
    686 
    687 error:
    688 	printf("%s:%d:%d: %s ",
    689 	    device_xname(atac->atac_dev), chp->ch_channel, xfer->c_drive,
    690 	    errstring);
    691 	printf("error (0x%x)\n", ATACH_ERR(tfd));
    692 	sc_xfer->error = XS_SHORTSENSE;
    693 	sc_xfer->sense.atapi_sense = ATACH_ERR(tfd);
    694 	bus_space_write_1(wdr->ctl_iot, wdr->ctl_ioh, wd_aux_ctlr, WDCTL_4BIT);
    695 	delay(10); /* some drives need a little delay here */
    696 	return ATASTART_ABORT;
    697 }
    698 
    699 static void
    700 wdc_atapi_poll(struct ata_channel *chp, struct ata_xfer *xfer)
    701 {
    702 	/*
    703 	 * If there is no interrupt for CMD input, busy-wait for it (done in
    704 	 * the interrupt routine. If it is a polled command, call the interrupt
    705 	 * routine until command is done.
    706 	 */
    707 	const bool poll = ((xfer->c_scsipi->xs_control & XS_CTL_POLL) != 0);
    708 
    709 	/* Wait for at last 400ns for status bit to be valid */
    710 	DELAY(1);
    711 	wdc_atapi_intr(chp, xfer, 0);
    712 
    713 	if (!poll)
    714 		return;
    715 
    716 #if NATA_DMA
    717 	if (chp->ch_flags & ATACH_DMA_WAIT) {
    718 		wdc_dmawait(chp, xfer, xfer->c_scsipi->timeout);
    719 		chp->ch_flags &= ~ATACH_DMA_WAIT;
    720 	}
    721 #endif
    722 	while ((xfer->c_scsipi->xs_status & XS_STS_DONE) == 0) {
    723 		/* Wait for at last 400ns for status bit to be valid */
    724 		DELAY(1);
    725 		wdc_atapi_intr(chp, xfer, 0);
    726 	}
    727 }
    728 
    729 static int
    730 wdc_atapi_intr(struct ata_channel *chp, struct ata_xfer *xfer, int is)
    731 {
    732 	struct atac_softc *atac = chp->ch_atac;
    733 	struct wdc_softc *wdc = CHAN_TO_WDC(chp);
    734 	struct wdc_regs *wdr = &wdc->regs[chp->ch_channel];
    735 	struct scsipi_xfer *sc_xfer = xfer->c_scsipi;
    736 	struct ata_drive_datas *drvp = &chp->ch_drive[xfer->c_drive];
    737 	int len, phase, i, retries=0;
    738 	int ire, tfd;
    739 	int poll = ((xfer->c_flags & C_POLL) != 0);
    740 #if NATA_DMA
    741 	int error;
    742 #endif
    743 #if NATA_DMA || NATA_PIOBM
    744 	int dma_flags = 0;
    745 #endif
    746 	void *cmd;
    747 
    748 	ATADEBUG_PRINT(("wdc_atapi_intr %s:%d:%d\n",
    749 	    device_xname(atac->atac_dev), chp->ch_channel, drvp->drive),
    750 	    DEBUG_INTR);
    751 
    752 	ata_channel_lock(chp);
    753 
    754 	/* Is it not a transfer, but a control operation? */
    755 	if (drvp->state < READY) {
    756 		printf("%s:%d:%d: bad state %d in wdc_atapi_intr\n",
    757 		    device_xname(atac->atac_dev), chp->ch_channel,
    758 		    xfer->c_drive, drvp->state);
    759 		panic("wdc_atapi_intr: bad state");
    760 	}
    761 	/*
    762 	 * If we missed an interrupt in a PIO transfer, reset and restart.
    763 	 * Don't try to continue transfer, we may have missed cycles.
    764 	 */
    765 	if ((xfer->c_flags & (C_TIMEOU | C_DMA)) == C_TIMEOU) {
    766 		ata_channel_unlock(chp);
    767 		sc_xfer->error = XS_TIMEOUT;
    768 		wdc_atapi_reset(chp, xfer);
    769 		return 1;
    770 	}
    771 
    772 #if NATA_PIOBM
    773 	/* Transfer-done interrupt for busmastering PIO operation */
    774 	if ((xfer->c_flags & C_PIOBM) && (chp->ch_flags & ATACH_PIOBM_WAIT)) {
    775 		chp->ch_flags &= ~ATACH_PIOBM_WAIT;
    776 
    777 		/* restore transfer length */
    778 		len = xfer->c_bcount;
    779 		if (xfer->c_lenoff < 0)
    780 			len += xfer->c_lenoff;
    781 
    782 		if (sc_xfer->xs_control & XS_CTL_DATA_IN)
    783 			goto end_piobm_datain;
    784 		else
    785 			goto end_piobm_dataout;
    786 	}
    787 #endif
    788 
    789 	/* Ack interrupt done in wdc_wait_for_unbusy */
    790 	if (wdc->select)
    791 		wdc->select(chp, xfer->c_drive);
    792 	bus_space_write_1(wdr->cmd_iot, wdr->cmd_iohs[wd_sdh], 0,
    793 	    WDSD_IBM | (xfer->c_drive << 4));
    794 	if (wdc_wait_for_unbusy(chp,
    795 	    poll ? sc_xfer->timeout : 0, AT_POLL, &tfd) == WDCWAIT_TOUT) {
    796 		if (!poll && (xfer->c_flags & C_TIMEOU) == 0) {
    797 			ata_channel_unlock(chp);
    798 			return 0; /* IRQ was not for us */
    799 		}
    800 		printf("%s:%d:%d: device timeout, c_bcount=%d, c_skip=%d\n",
    801 		    device_xname(atac->atac_dev), chp->ch_channel,
    802 		    xfer->c_drive, xfer->c_bcount, xfer->c_skip);
    803 		ata_channel_unlock(chp);
    804 #if NATA_DMA
    805 		if (xfer->c_flags & C_DMA) {
    806 			ata_dmaerr(drvp,
    807 			    (xfer->c_flags & C_POLL) ? AT_POLL : 0);
    808 		}
    809 #endif
    810 		sc_xfer->error = XS_TIMEOUT;
    811 		wdc_atapi_reset(chp, xfer);
    812 		return 1;
    813 	}
    814 	if (wdc->irqack)
    815 		wdc->irqack(chp);
    816 
    817 #if NATA_DMA
    818 	/*
    819 	 * If we missed an IRQ and were using DMA, flag it as a DMA error
    820 	 * and reset device.
    821 	 */
    822 	if ((xfer->c_flags & C_TIMEOU) && (xfer->c_flags & C_DMA)) {
    823 		ata_channel_unlock(chp);
    824 		ata_dmaerr(drvp, (xfer->c_flags & C_POLL) ? AT_POLL : 0);
    825 		sc_xfer->error = XS_RESET;
    826 		wdc_atapi_reset(chp, xfer);
    827 		return (1);
    828 	}
    829 #endif
    830 	/*
    831 	 * if the request sense command was aborted, report the short sense
    832 	 * previously recorded, else continue normal processing
    833 	 */
    834 
    835 #if NATA_DMA || NATA_PIOBM
    836 	if (xfer->c_flags & (C_DMA | C_PIOBM))
    837 		dma_flags = (sc_xfer->xs_control & XS_CTL_DATA_IN)
    838 		    ?  WDC_DMA_READ : 0;
    839 #endif
    840 again:
    841 	len = bus_space_read_1(wdr->cmd_iot, wdr->cmd_iohs[wd_cyl_lo], 0) +
    842 	    256 * bus_space_read_1(wdr->cmd_iot, wdr->cmd_iohs[wd_cyl_hi], 0);
    843 	ire = bus_space_read_1(wdr->cmd_iot, wdr->cmd_iohs[wd_ireason], 0);
    844 	phase = (ire & (WDCI_CMD | WDCI_IN)) | (ATACH_ST(tfd) & WDCS_DRQ);
    845 	ATADEBUG_PRINT(("wdc_atapi_intr: c_bcount %d len %d st 0x%x err 0x%x "
    846 	    "ire 0x%x :", xfer->c_bcount,
    847 	    len, ATACH_ST(tfd), ATACH_ERR(tfd), ire), DEBUG_INTR);
    848 
    849 	switch (phase) {
    850 	case PHASE_CMDOUT:
    851 		cmd = sc_xfer->cmd;
    852 		ATADEBUG_PRINT(("PHASE_CMDOUT\n"), DEBUG_INTR);
    853 #if NATA_DMA
    854 		/* Init the DMA channel if necessary */
    855 		if (xfer->c_flags & C_DMA) {
    856 			error = (*wdc->dma_init)(wdc->dma_arg,
    857 			    chp->ch_channel, xfer->c_drive,
    858 			    xfer->c_databuf, xfer->c_bcount, dma_flags);
    859 			if (error) {
    860 				if (error == EINVAL) {
    861 					/*
    862 					 * We can't do DMA on this transfer
    863 					 * for some reason.  Fall back to
    864 					 * PIO.
    865 					 */
    866 					xfer->c_flags &= ~C_DMA;
    867 					error = 0;
    868 				} else {
    869 					sc_xfer->error = XS_DRIVER_STUFFUP;
    870 					break;
    871 				}
    872 			}
    873 		}
    874 #endif
    875 
    876 		/* send packet command */
    877 		/* Commands are 12 or 16 bytes long. It's 32-bit aligned */
    878 		wdc->dataout_pio(chp, drvp->drive_flags, cmd, sc_xfer->cmdlen);
    879 
    880 #if NATA_DMA
    881 		/* Start the DMA channel if necessary */
    882 		if (xfer->c_flags & C_DMA) {
    883 			(*wdc->dma_start)(wdc->dma_arg,
    884 			    chp->ch_channel, xfer->c_drive);
    885 			chp->ch_flags |= ATACH_DMA_WAIT;
    886 		}
    887 #endif
    888 		ata_channel_unlock(chp);
    889 		return 1;
    890 
    891 	 case PHASE_DATAOUT:
    892 		/* write data */
    893 		ATADEBUG_PRINT(("PHASE_DATAOUT\n"), DEBUG_INTR);
    894 #if NATA_DMA
    895 		if ((sc_xfer->xs_control & XS_CTL_DATA_OUT) == 0 ||
    896 		    (xfer->c_flags & C_DMA) != 0) {
    897 			printf("wdc_atapi_intr: bad data phase DATAOUT\n");
    898 			ata_channel_unlock(chp);
    899 			if (xfer->c_flags & C_DMA) {
    900 				ata_dmaerr(drvp,
    901 				    (xfer->c_flags & C_POLL) ? AT_POLL : 0);
    902 			}
    903 			sc_xfer->error = XS_TIMEOUT;
    904 			wdc_atapi_reset(chp, xfer);
    905 			return 1;
    906 		}
    907 #endif
    908 		xfer->c_lenoff = len - xfer->c_bcount;
    909 		if (xfer->c_bcount < len) {
    910 			printf("wdc_atapi_intr: warning: write only "
    911 			    "%d of %d requested bytes\n", xfer->c_bcount, len);
    912 			len = xfer->c_bcount;
    913 		}
    914 
    915 #if NATA_PIOBM
    916 		if (xfer->c_flags & C_PIOBM) {
    917 			/* start the busmastering PIO */
    918 			(*wdc->piobm_start)(wdc->dma_arg,
    919 			    chp->ch_channel, xfer->c_drive,
    920 			    xfer->c_skip, len, WDC_PIOBM_XFER_IRQ);
    921 			chp->ch_flags |= ATACH_DMA_WAIT | ATACH_PIOBM_WAIT;
    922 			ata_channel_unlock(chp);
    923 			return 1;
    924 		}
    925 #endif
    926 		wdc->dataout_pio(chp, drvp->drive_flags,
    927 		    (char *)xfer->c_databuf + xfer->c_skip, len);
    928 
    929 #if NATA_PIOBM
    930 	end_piobm_dataout:
    931 #endif
    932 		for (i = xfer->c_lenoff; i > 0; i -= 2)
    933 			bus_space_write_2(wdr->cmd_iot,
    934 			    wdr->cmd_iohs[wd_data], 0, 0);
    935 
    936 		xfer->c_skip += len;
    937 		xfer->c_bcount -= len;
    938 		ata_channel_unlock(chp);
    939 		return 1;
    940 
    941 	case PHASE_DATAIN:
    942 		/* Read data */
    943 		ATADEBUG_PRINT(("PHASE_DATAIN\n"), DEBUG_INTR);
    944 #if NATA_DMA
    945 		if ((sc_xfer->xs_control & XS_CTL_DATA_IN) == 0 ||
    946 		    (xfer->c_flags & C_DMA) != 0) {
    947 			printf("wdc_atapi_intr: bad data phase DATAIN\n");
    948 			ata_channel_unlock(chp);
    949 			if (xfer->c_flags & C_DMA) {
    950 				ata_dmaerr(drvp,
    951 				    (xfer->c_flags & C_POLL) ? AT_POLL : 0);
    952 			}
    953 			sc_xfer->error = XS_TIMEOUT;
    954 			wdc_atapi_reset(chp, xfer);
    955 			return 1;
    956 		}
    957 #endif
    958 		xfer->c_lenoff = len - xfer->c_bcount;
    959 		if (xfer->c_bcount < len) {
    960 			printf("wdc_atapi_intr: warning: reading only "
    961 			    "%d of %d bytes\n", xfer->c_bcount, len);
    962 			len = xfer->c_bcount;
    963 		}
    964 
    965 #if NATA_PIOBM
    966 		if (xfer->c_flags & C_PIOBM) {
    967 			/* start the busmastering PIO */
    968 			(*wdc->piobm_start)(wdc->dma_arg,
    969 			    chp->ch_channel, xfer->c_drive,
    970 			    xfer->c_skip, len, WDC_PIOBM_XFER_IRQ);
    971 			chp->ch_flags |= ATACH_DMA_WAIT | ATACH_PIOBM_WAIT;
    972 			ata_channel_unlock(chp);
    973 			return 1;
    974 		}
    975 #endif
    976 		wdc->datain_pio(chp, drvp->drive_flags,
    977 		    (char *)xfer->c_databuf + xfer->c_skip, len);
    978 
    979 #if NATA_PIOBM
    980 	end_piobm_datain:
    981 #endif
    982 		if (xfer->c_lenoff > 0)
    983 			wdcbit_bucket(chp, xfer->c_lenoff);
    984 
    985 		xfer->c_skip += len;
    986 		xfer->c_bcount -= len;
    987 		ata_channel_unlock(chp);
    988 		return 1;
    989 
    990 	case PHASE_ABORTED:
    991 	case PHASE_COMPLETED:
    992 		ATADEBUG_PRINT(("PHASE_COMPLETED\n"), DEBUG_INTR);
    993 #if NATA_DMA
    994 		if (xfer->c_flags & C_DMA) {
    995 			xfer->c_bcount -= sc_xfer->datalen;
    996 		}
    997 #endif
    998 		sc_xfer->resid = xfer->c_bcount;
    999 		/* this will unlock channel lock too */
   1000 		wdc_atapi_phase_complete(xfer);
   1001 		return(1);
   1002 
   1003 	default:
   1004 		if (++retries<500) {
   1005 			DELAY(100);
   1006 			tfd = ATACH_ERR_ST(
   1007 			    bus_space_read_1(wdr->cmd_iot,
   1008 				wdr->cmd_iohs[wd_error], 0),
   1009 			    bus_space_read_1(wdr->cmd_iot,
   1010 				wdr->cmd_iohs[wd_status], 0)
   1011 			);
   1012 			goto again;
   1013 		}
   1014 		printf("wdc_atapi_intr: unknown phase 0x%x\n", phase);
   1015 		if (ATACH_ST(tfd) & WDCS_ERR) {
   1016 			sc_xfer->error = XS_SHORTSENSE;
   1017 			sc_xfer->sense.atapi_sense = ATACH_ERR(tfd);
   1018 		} else {
   1019 			ata_channel_unlock(chp);
   1020 #if NATA_DMA
   1021 			if (xfer->c_flags & C_DMA) {
   1022 				ata_dmaerr(drvp,
   1023 				    (xfer->c_flags & C_POLL) ? AT_POLL : 0);
   1024 			}
   1025 #endif
   1026 			sc_xfer->error = XS_RESET;
   1027 			wdc_atapi_reset(chp, xfer);
   1028 			return (1);
   1029 		}
   1030 	}
   1031 	ATADEBUG_PRINT(("wdc_atapi_intr: wdc_atapi_done() (end), error 0x%x "
   1032 	    "sense 0x%x\n", sc_xfer->error, sc_xfer->sense.atapi_sense),
   1033 	    DEBUG_INTR);
   1034 	ata_channel_unlock(chp);
   1035 	wdc_atapi_done(chp, xfer);
   1036 	return (1);
   1037 }
   1038 
   1039 static void
   1040 wdc_atapi_phase_complete(struct ata_xfer *xfer)
   1041 {
   1042 	struct ata_channel *chp = xfer->c_chp;
   1043 	struct atac_softc *atac = chp->ch_atac;
   1044 #if NATA_DMA || NATA_PIOBM
   1045 	struct wdc_softc *wdc = CHAN_TO_WDC(chp);
   1046 #endif
   1047 	struct scsipi_xfer *sc_xfer = xfer->c_scsipi;
   1048 	struct ata_drive_datas *drvp = &chp->ch_drive[xfer->c_drive];
   1049 	int tfd = 0;
   1050 
   1051 	ata_channel_lock_owned(chp);
   1052 
   1053 	/* wait for DSC if needed */
   1054 	if (drvp->drive_flags & ATA_DRIVE_ATAPIDSCW) {
   1055 		ATADEBUG_PRINT(("wdc_atapi_phase_complete(%s:%d:%d) "
   1056 		    "polldsc %d\n", device_xname(atac->atac_dev),
   1057 		    chp->ch_channel,
   1058 		    xfer->c_drive, xfer->c_dscpoll), DEBUG_XFERS);
   1059 #if 1
   1060 		if (cold)
   1061 			panic("wdc_atapi_phase_complete: cold");
   1062 #endif
   1063 		if (wdcwait(chp, WDCS_DSC, WDCS_DSC, 10,
   1064 		    AT_POLL, &tfd) == WDCWAIT_TOUT) {
   1065 			/* 10ms not enough, try again in 1 tick */
   1066 			if (xfer->c_dscpoll++ >
   1067 			    mstohz(sc_xfer->timeout)) {
   1068 				printf("%s:%d:%d: wait_for_dsc "
   1069 				    "failed\n",
   1070 				    device_xname(atac->atac_dev),
   1071 				    chp->ch_channel, xfer->c_drive);
   1072 				ata_channel_unlock(chp);
   1073 				sc_xfer->error = XS_TIMEOUT;
   1074 				wdc_atapi_reset(chp, xfer);
   1075 			} else {
   1076 				callout_reset(&xfer->c_timo_callout, 1,
   1077 				    wdc_atapi_polldsc, xfer);
   1078 				ata_channel_unlock(chp);
   1079 			}
   1080 			return;
   1081 		}
   1082 	}
   1083 
   1084 
   1085 	/*
   1086 	 * Some drive occasionally set WDCS_ERR with
   1087 	 * "ATA illegal length indication" in the error
   1088 	 * register. If we read some data the sense is valid
   1089 	 * anyway, so don't report the error.
   1090 	 */
   1091 	if (ATACH_ST(tfd) & WDCS_ERR &&
   1092 	    ((sc_xfer->xs_control & XS_CTL_REQSENSE) == 0 ||
   1093 	    sc_xfer->resid == sc_xfer->datalen)) {
   1094 		/* save the short sense */
   1095 		sc_xfer->error = XS_SHORTSENSE;
   1096 		sc_xfer->sense.atapi_sense = ATACH_ERR(tfd);
   1097 		if ((sc_xfer->xs_periph->periph_quirks &
   1098 		    PQUIRK_NOSENSE) == 0) {
   1099 			/* ask scsipi to send a REQUEST_SENSE */
   1100 			sc_xfer->error = XS_BUSY;
   1101 			sc_xfer->status = SCSI_CHECK;
   1102 		}
   1103 #if NATA_DMA || NATA_PIOBM
   1104 		else if (wdc->dma_status &
   1105 		    (WDC_DMAST_NOIRQ | WDC_DMAST_ERR)) {
   1106 			ata_channel_unlock(chp);
   1107 #if NATA_DMA
   1108 			ata_dmaerr(drvp,
   1109 			    (xfer->c_flags & C_POLL) ? AT_POLL : 0);
   1110 #endif
   1111 			sc_xfer->error = XS_RESET;
   1112 			wdc_atapi_reset(chp, xfer);
   1113 			return;
   1114 		}
   1115 #endif
   1116 	}
   1117 	if (xfer->c_bcount != 0) {
   1118 		ATADEBUG_PRINT(("wdc_atapi_intr: bcount value is "
   1119 		    "%d after io\n", xfer->c_bcount), DEBUG_XFERS);
   1120 	}
   1121 #ifdef DIAGNOSTIC
   1122 	if (xfer->c_bcount < 0) {
   1123 		printf("wdc_atapi_intr warning: bcount value "
   1124 		    "is %d after io\n", xfer->c_bcount);
   1125 	}
   1126 #endif
   1127 	ATADEBUG_PRINT(("wdc_atapi_phase_complete: wdc_atapi_done(), "
   1128 	    "error 0x%x sense 0x%x\n", sc_xfer->error,
   1129 	    sc_xfer->sense.atapi_sense), DEBUG_INTR);
   1130 	ata_channel_unlock(chp);
   1131 	wdc_atapi_done(chp, xfer);
   1132 }
   1133 
   1134 static void
   1135 wdc_atapi_done(struct ata_channel *chp, struct ata_xfer *xfer)
   1136 {
   1137 	struct atac_softc *atac = chp->ch_atac;
   1138 	struct scsipi_xfer *sc_xfer = xfer->c_scsipi;
   1139 
   1140 	ATADEBUG_PRINT(("wdc_atapi_done %s:%d:%d: flags 0x%x\n",
   1141 	    device_xname(atac->atac_dev), chp->ch_channel, xfer->c_drive,
   1142 	    (u_int)xfer->c_flags), DEBUG_XFERS);
   1143 
   1144 	if (ata_waitdrain_xfer_check(chp, xfer))
   1145 		return;
   1146 
   1147 	ata_deactivate_xfer(chp, xfer);
   1148 	ata_free_xfer(chp, xfer);
   1149 
   1150 	ATADEBUG_PRINT(("wdc_atapi_done: scsipi_done\n"), DEBUG_XFERS);
   1151 	scsipi_done(sc_xfer);
   1152 	ATADEBUG_PRINT(("atastart from wdc_atapi_done, flags 0x%x\n",
   1153 	    chp->ch_flags), DEBUG_XFERS);
   1154 	atastart(chp);
   1155 }
   1156 
   1157 static void
   1158 wdc_atapi_reset(struct ata_channel *chp, struct ata_xfer *xfer)
   1159 {
   1160 	struct atac_softc *atac = chp->ch_atac;
   1161 	struct ata_drive_datas *drvp = &chp->ch_drive[xfer->c_drive];
   1162 	struct scsipi_xfer *sc_xfer = xfer->c_scsipi;
   1163 	int tfd;
   1164 
   1165 	ata_channel_lock(chp);
   1166 	wdccommandshort(chp, xfer->c_drive, ATAPI_SOFT_RESET);
   1167 	drvp->state = 0;
   1168 	if (wdc_wait_for_unbusy(chp, WDC_RESET_WAIT, AT_POLL, &tfd) != 0) {
   1169 		printf("%s:%d:%d: reset failed\n",
   1170 		    device_xname(atac->atac_dev), chp->ch_channel,
   1171 		    xfer->c_drive);
   1172 		sc_xfer->error = XS_SELTIMEOUT;
   1173 	}
   1174 	ata_channel_unlock(chp);
   1175 	wdc_atapi_done(chp, xfer);
   1176 	return;
   1177 }
   1178 
   1179 static void
   1180 wdc_atapi_polldsc(void *arg)
   1181 {
   1182 	struct ata_xfer *xfer = arg;
   1183 	struct ata_channel *chp = xfer->c_chp;
   1184 
   1185 	ata_channel_lock(chp);
   1186 
   1187 	/* this will unlock channel lock too */
   1188 	wdc_atapi_phase_complete(xfer);
   1189 }
   1190