Home | History | Annotate | Line # | Download | only in ic
ahcisata_core.c revision 1.8
      1 /*	$NetBSD: ahcisata_core.c,v 1.8 2007/11/12 21:25:14 bouyer Exp $	*/
      2 
      3 /*
      4  * Copyright (c) 2006 Manuel Bouyer.
      5  *
      6  * Redistribution and use in source and binary forms, with or without
      7  * modification, are permitted provided that the following conditions
      8  * are met:
      9  * 1. Redistributions of source code must retain the above copyright
     10  *    notice, this list of conditions and the following disclaimer.
     11  * 2. Redistributions in binary form must reproduce the above copyright
     12  *    notice, this list of conditions and the following disclaimer in the
     13  *    documentation and/or other materials provided with the distribution.
     14  * 3. All advertising materials mentioning features or use of this software
     15  *    must display the following acknowledgement:
     16  *	This product includes software developed by Manuel Bouyer.
     17  * 4. The name of the author may not be used to endorse or promote products
     18  *    derived from this software without specific prior written permission.
     19  *
     20  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
     21  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
     22  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
     23  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
     24  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
     25  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
     26  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
     27  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
     28  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
     29  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     30  *
     31  */
     32 
     33 #include <sys/cdefs.h>
     34 __KERNEL_RCSID(0, "$NetBSD: ahcisata_core.c,v 1.8 2007/11/12 21:25:14 bouyer Exp $");
     35 
     36 #include <sys/types.h>
     37 #include <sys/malloc.h>
     38 #include <sys/param.h>
     39 #include <sys/kernel.h>
     40 #include <sys/systm.h>
     41 #include <sys/disklabel.h>
     42 #include <sys/proc.h>
     43 #include <sys/buf.h>
     44 
     45 #include <uvm/uvm_extern.h>
     46 
     47 #include <dev/ic/wdcreg.h>
     48 #include <dev/ata/atareg.h>
     49 #include <dev/ata/satavar.h>
     50 #include <dev/ata/satareg.h>
     51 #include <dev/ic/ahcisatavar.h>
     52 
     53 #include "atapibus.h"
     54 
     55 #ifdef AHCI_DEBUG
     56 int ahcidebug_mask = 0x0;
     57 #endif
     58 
     59 void ahci_probe_drive(struct ata_channel *);
     60 void ahci_setup_channel(struct ata_channel *);
     61 
     62 int  ahci_ata_bio(struct ata_drive_datas *, struct ata_bio *);
     63 void ahci_reset_drive(struct ata_drive_datas *, int);
     64 void ahci_reset_channel(struct ata_channel *, int);
     65 int  ahci_exec_command(struct ata_drive_datas *, struct ata_command *);
     66 int  ahci_ata_addref(struct ata_drive_datas *);
     67 void ahci_ata_delref(struct ata_drive_datas *);
     68 void ahci_killpending(struct ata_drive_datas *);
     69 
     70 void ahci_cmd_start(struct ata_channel *, struct ata_xfer *);
     71 int  ahci_cmd_complete(struct ata_channel *, struct ata_xfer *, int);
     72 void ahci_cmd_done(struct ata_channel *, struct ata_xfer *, int);
     73 void ahci_cmd_kill_xfer(struct ata_channel *, struct ata_xfer *, int) ;
     74 void ahci_bio_start(struct ata_channel *, struct ata_xfer *);
     75 int  ahci_bio_complete(struct ata_channel *, struct ata_xfer *, int);
     76 void ahci_bio_kill_xfer(struct ata_channel *, struct ata_xfer *, int) ;
     77 void ahci_channel_stop(struct ahci_softc *, struct ata_channel *, int);
     78 void ahci_channel_start(struct ahci_softc *, struct ata_channel *);
     79 void ahci_timeout(void *);
     80 int  ahci_dma_setup(struct ata_channel *, int, void *, size_t, int);
     81 
     82 #if NATAPIBUS > 0
     83 void ahci_atapibus_attach(struct atabus_softc *);
     84 void ahci_atapi_kill_pending(struct scsipi_periph *);
     85 void ahci_atapi_minphys(struct buf *);
     86 void ahci_atapi_scsipi_request(struct scsipi_channel *,
     87     scsipi_adapter_req_t, void *);
     88 void ahci_atapi_start(struct ata_channel *, struct ata_xfer *);
     89 int  ahci_atapi_complete(struct ata_channel *, struct ata_xfer *, int);
     90 void ahci_atapi_kill_xfer(struct ata_channel *, struct ata_xfer *, int);
     91 void ahci_atapi_probe_device(struct atapibus_softc *, int);
     92 
     93 static const struct scsipi_bustype ahci_atapi_bustype = {
     94 	SCSIPI_BUSTYPE_ATAPI,
     95 	atapi_scsipi_cmd,
     96 	atapi_interpret_sense,
     97 	atapi_print_addr,
     98 	ahci_atapi_kill_pending,
     99 };
    100 #endif /* NATAPIBUS */
    101 
    102 #define ATA_DELAY 10000 /* 10s for a drive I/O */
    103 
    104 const struct ata_bustype ahci_ata_bustype = {
    105 	SCSIPI_BUSTYPE_ATA,
    106 	ahci_ata_bio,
    107 	ahci_reset_drive,
    108 	ahci_reset_channel,
    109 	ahci_exec_command,
    110 	ata_get_params,
    111 	ahci_ata_addref,
    112 	ahci_ata_delref,
    113 	ahci_killpending
    114 };
    115 
    116 void ahci_intr_port(struct ahci_softc *, struct ahci_channel *);
    117 
    118 static void ahci_setup_port(struct ahci_softc *sc, int i);
    119 
    120 int
    121 ahci_reset(struct ahci_softc *sc)
    122 {
    123 	int i;
    124 
    125 	/* reset controller */
    126 	AHCI_WRITE(sc, AHCI_GHC, AHCI_GHC_HR);
    127 	/* wait up to 1s for reset to complete */
    128 	for (i = 0; i < 1000; i++) {
    129 		delay(1000);
    130 		if ((AHCI_READ(sc, AHCI_GHC) & AHCI_GHC_HR) == 0)
    131 			break;
    132 	}
    133 	if ((AHCI_READ(sc, AHCI_GHC) & AHCI_GHC_HR)) {
    134 		aprint_error("%s: reset failed\n", AHCINAME(sc));
    135 		return -1;
    136 	}
    137 	/* enable ahci mode */
    138 	AHCI_WRITE(sc, AHCI_GHC, AHCI_GHC_AE);
    139 	return 0;
    140 }
    141 
    142 void
    143 ahci_setup_ports(struct ahci_softc *sc)
    144 {
    145 	u_int32_t ahci_ports;
    146 	int i, port;
    147 
    148 	ahci_ports = AHCI_READ(sc, AHCI_PI);
    149 	for (i = 0, port = 0; i < AHCI_MAX_PORTS; i++) {
    150 		if ((ahci_ports & (1 << i)) == 0)
    151 			continue;
    152 		if (port >= sc->sc_atac.atac_nchannels) {
    153 			aprint_error("%s: more ports than announced\n",
    154 			    AHCINAME(sc));
    155 			break;
    156 		}
    157 		ahci_setup_port(sc, i);
    158 	}
    159 }
    160 
    161 void
    162 ahci_reprobe_drives(struct ahci_softc *sc)
    163 {
    164 	u_int32_t ahci_ports;
    165 	int i, port;
    166 	struct ahci_channel *achp;
    167 	struct ata_channel *chp;
    168 
    169 	ahci_ports = AHCI_READ(sc, AHCI_PI);
    170 	for (i = 0, port = 0; i < AHCI_MAX_PORTS; i++) {
    171 		if ((ahci_ports & (1 << i)) == 0)
    172 			continue;
    173 		if (port >= sc->sc_atac.atac_nchannels) {
    174 			aprint_error("%s: more ports than announced\n",
    175 			    AHCINAME(sc));
    176 			break;
    177 		}
    178 		achp = &sc->sc_channels[i];
    179 		chp = &achp->ata_channel;
    180 
    181 		ahci_probe_drive(chp);
    182 	}
    183 }
    184 
    185 static void
    186 ahci_setup_port(struct ahci_softc *sc, int i)
    187 {
    188 	struct ahci_channel *achp;
    189 
    190 	achp = &sc->sc_channels[i];
    191 
    192 	AHCI_WRITE(sc, AHCI_P_CLB(i), achp->ahcic_bus_cmdh);
    193 	AHCI_WRITE(sc, AHCI_P_CLBU(i), 0);
    194 	AHCI_WRITE(sc, AHCI_P_FB(i), achp->ahcic_bus_rfis);
    195 	AHCI_WRITE(sc, AHCI_P_FBU(i), 0);
    196 }
    197 
    198 void
    199 ahci_enable_intrs(struct ahci_softc *sc)
    200 {
    201 
    202 	/* clear interrupts */
    203 	AHCI_WRITE(sc, AHCI_IS, AHCI_READ(sc, AHCI_IS));
    204 	/* enable interrupts */
    205 	AHCI_WRITE(sc, AHCI_GHC, AHCI_READ(sc, AHCI_GHC) | AHCI_GHC_IE);
    206 }
    207 
    208 void
    209 ahci_attach(struct ahci_softc *sc)
    210 {
    211 	u_int32_t ahci_cap, ahci_rev, ahci_ports;
    212 	int i, j, port;
    213 	struct ahci_channel *achp;
    214 	struct ata_channel *chp;
    215 	int error;
    216 	bus_dma_segment_t seg;
    217 	int rseg;
    218 	int dmasize;
    219 	void *cmdhp;
    220 	void *cmdtblp;
    221 
    222 	if (ahci_reset(sc) != 0)
    223 		return;
    224 
    225 	ahci_cap = AHCI_READ(sc, AHCI_CAP);
    226 	sc->sc_atac.atac_nchannels = (ahci_cap & AHCI_CAP_NPMASK) + 1;
    227 	sc->sc_ncmds = ((ahci_cap & AHCI_CAP_NCS) >> 8) + 1;
    228 	ahci_rev = AHCI_READ(sc, AHCI_VS);
    229 	aprint_normal("%s: AHCI revision ", AHCINAME(sc));
    230 	switch(ahci_rev) {
    231 	case AHCI_VS_10:
    232 		aprint_normal("1.0");
    233 		break;
    234 	case AHCI_VS_11:
    235 		aprint_normal("1.1");
    236 		break;
    237 	default:
    238 		aprint_normal("0x%x", ahci_rev);
    239 		break;
    240 	}
    241 
    242 	aprint_normal(", %d ports, %d command slots, features 0x%x\n",
    243 	    sc->sc_atac.atac_nchannels, sc->sc_ncmds,
    244 	    ahci_cap & ~(AHCI_CAP_NPMASK|AHCI_CAP_NCS));
    245 	sc->sc_atac.atac_cap = ATAC_CAP_DATA16 | ATAC_CAP_DMA | ATAC_CAP_UDMA;
    246 	sc->sc_atac.atac_pio_cap = 4;
    247 	sc->sc_atac.atac_dma_cap = 2;
    248 	sc->sc_atac.atac_udma_cap = 6;
    249 	sc->sc_atac.atac_channels = sc->sc_chanarray;
    250 	sc->sc_atac.atac_probe = ahci_probe_drive;
    251 	sc->sc_atac.atac_bustype_ata = &ahci_ata_bustype;
    252 	sc->sc_atac.atac_set_modes = ahci_setup_channel;
    253 #if NATAPIBUS > 0
    254 	sc->sc_atac.atac_atapibus_attach = ahci_atapibus_attach;
    255 #endif
    256 
    257 	dmasize =
    258 	    (AHCI_RFIS_SIZE + AHCI_CMDH_SIZE) * sc->sc_atac.atac_nchannels;
    259 	error = bus_dmamem_alloc(sc->sc_dmat, dmasize, PAGE_SIZE, 0,
    260 	    &seg, 1, &rseg, BUS_DMA_NOWAIT);
    261 	if (error) {
    262 		aprint_error("%s: unable to allocate command header memory"
    263 		    ", error=%d\n", AHCINAME(sc), error);
    264 		return;
    265 	}
    266 	error = bus_dmamem_map(sc->sc_dmat, &seg, rseg, dmasize,
    267 	    &cmdhp, BUS_DMA_NOWAIT|BUS_DMA_COHERENT);
    268 	if (error) {
    269 		aprint_error("%s: unable to map command header memory"
    270 		    ", error=%d\n", AHCINAME(sc), error);
    271 		return;
    272 	}
    273 	error = bus_dmamap_create(sc->sc_dmat, dmasize, 1, dmasize, 0,
    274 	    BUS_DMA_NOWAIT, &sc->sc_cmd_hdrd);
    275 	if (error) {
    276 		aprint_error("%s: unable to create command header map"
    277 		    ", error=%d\n", AHCINAME(sc), error);
    278 		return;
    279 	}
    280 	error = bus_dmamap_load(sc->sc_dmat, sc->sc_cmd_hdrd,
    281 	    cmdhp, dmasize, NULL, BUS_DMA_NOWAIT);
    282 	if (error) {
    283 		aprint_error("%s: unable to load command header map"
    284 		    ", error=%d\n", AHCINAME(sc), error);
    285 		return;
    286 	}
    287 	sc->sc_cmd_hdr = cmdhp;
    288 
    289 	ahci_enable_intrs(sc);
    290 
    291 	ahci_ports = AHCI_READ(sc, AHCI_PI);
    292 	for (i = 0, port = 0; i < AHCI_MAX_PORTS; i++) {
    293 		if ((ahci_ports & (1 << i)) == 0)
    294 			continue;
    295 		if (port >= sc->sc_atac.atac_nchannels) {
    296 			aprint_error("%s: more ports than announced\n",
    297 			    AHCINAME(sc));
    298 			break;
    299 		}
    300 		achp = &sc->sc_channels[i];
    301 		chp = (struct ata_channel *)achp;
    302 		sc->sc_chanarray[i] = chp;
    303 		chp->ch_channel = i;
    304 		chp->ch_atac = &sc->sc_atac;
    305 		chp->ch_queue = malloc(sizeof(struct ata_queue),
    306 		    M_DEVBUF, M_NOWAIT);
    307 		if (chp->ch_queue == NULL) {
    308 			aprint_error("%s port %d: can't allocate memory for "
    309 			    "command queue", AHCINAME(sc), i);
    310 			break;
    311 		}
    312 		dmasize = AHCI_CMDTBL_SIZE * sc->sc_ncmds;
    313 		error = bus_dmamem_alloc(sc->sc_dmat, dmasize, PAGE_SIZE, 0,
    314 		    &seg, 1, &rseg, BUS_DMA_NOWAIT);
    315 		if (error) {
    316 			aprint_error("%s: unable to allocate command table "
    317 			    "memory, error=%d\n", AHCINAME(sc), error);
    318 			break;
    319 		}
    320 		error = bus_dmamem_map(sc->sc_dmat, &seg, rseg, dmasize,
    321 		    &cmdtblp, BUS_DMA_NOWAIT|BUS_DMA_COHERENT);
    322 		if (error) {
    323 			aprint_error("%s: unable to map command table memory"
    324 			    ", error=%d\n", AHCINAME(sc), error);
    325 			break;
    326 		}
    327 		error = bus_dmamap_create(sc->sc_dmat, dmasize, 1, dmasize, 0,
    328 		    BUS_DMA_NOWAIT, &achp->ahcic_cmd_tbld);
    329 		if (error) {
    330 			aprint_error("%s: unable to create command table map"
    331 			    ", error=%d\n", AHCINAME(sc), error);
    332 			break;
    333 		}
    334 		error = bus_dmamap_load(sc->sc_dmat, achp->ahcic_cmd_tbld,
    335 		    cmdtblp, dmasize, NULL, BUS_DMA_NOWAIT);
    336 		if (error) {
    337 			aprint_error("%s: unable to load command table map"
    338 			    ", error=%d\n", AHCINAME(sc), error);
    339 			break;
    340 		}
    341 		achp->ahcic_cmdh  = (struct ahci_cmd_header *)
    342 		    ((char *)cmdhp + AHCI_CMDH_SIZE * port);
    343 		achp->ahcic_bus_cmdh = sc->sc_cmd_hdrd->dm_segs[0].ds_addr +
    344 		    AHCI_CMDH_SIZE * port;
    345 		achp->ahcic_rfis = (struct ahci_r_fis *)
    346 		    ((char *)cmdhp +
    347 		     AHCI_CMDH_SIZE * sc->sc_atac.atac_nchannels +
    348 		     AHCI_RFIS_SIZE * port);
    349 		achp->ahcic_bus_rfis = sc->sc_cmd_hdrd->dm_segs[0].ds_addr +
    350 		     AHCI_CMDH_SIZE * sc->sc_atac.atac_nchannels +
    351 		     AHCI_RFIS_SIZE * port;
    352 		AHCIDEBUG_PRINT(("port %d cmdh %p (0x%x) rfis %p (0x%x)\n", i,
    353 		   achp->ahcic_cmdh, (u_int)achp->ahcic_bus_cmdh,
    354 		   achp->ahcic_rfis, (u_int)achp->ahcic_bus_rfis),
    355 		   DEBUG_PROBE);
    356 
    357 		for (j = 0; j < sc->sc_ncmds; j++) {
    358 			achp->ahcic_cmd_tbl[j] = (struct ahci_cmd_tbl *)
    359 			    ((char *)cmdtblp + AHCI_CMDTBL_SIZE * j);
    360 			achp->ahcic_bus_cmd_tbl[j] =
    361 			     achp->ahcic_cmd_tbld->dm_segs[0].ds_addr +
    362 			     AHCI_CMDTBL_SIZE * j;
    363 			achp->ahcic_cmdh[j].cmdh_cmdtba =
    364 			    htole32(achp->ahcic_bus_cmd_tbl[j]);
    365 			achp->ahcic_cmdh[j].cmdh_cmdtbau = htole32(0);
    366 			AHCIDEBUG_PRINT(("port %d/%d tbl %p (0x%x)\n", i, j,
    367 			    achp->ahcic_cmd_tbl[j],
    368 			    (u_int)achp->ahcic_bus_cmd_tbl[j]), DEBUG_PROBE);
    369 			/* The xfer DMA map */
    370 			error = bus_dmamap_create(sc->sc_dmat, MAXPHYS,
    371 			    AHCI_NPRD, 0x400000 /* 4MB */, 0,
    372 			    BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW,
    373 			    &achp->ahcic_datad[j]);
    374 			if (error) {
    375 				aprint_error("%s: couldn't alloc xfer DMA map, "
    376 				    "error=%d\n", AHCINAME(sc), error);
    377 				goto end;
    378 			}
    379 		}
    380 		ahci_setup_port(sc, i);
    381 		chp->ch_ndrive = 1;
    382 		if (bus_space_subregion(sc->sc_ahcit, sc->sc_ahcih,
    383 		    AHCI_P_SSTS(i), 1,  &achp->ahcic_sstatus) != 0) {
    384 			aprint_error("%s: couldn't map channel %d "
    385 			    "sata_status regs\n", AHCINAME(sc), i);
    386 			break;
    387 		}
    388 		if (bus_space_subregion(sc->sc_ahcit, sc->sc_ahcih,
    389 		    AHCI_P_SCTL(i), 1,  &achp->ahcic_scontrol) != 0) {
    390 			aprint_error("%s: couldn't map channel %d "
    391 			    "sata_control regs\n", AHCINAME(sc), i);
    392 			break;
    393 		}
    394 		if (bus_space_subregion(sc->sc_ahcit, sc->sc_ahcih,
    395 		    AHCI_P_SERR(i), 1,  &achp->ahcic_serror) != 0) {
    396 			aprint_error("%s: couldn't map channel %d "
    397 			    "sata_error regs\n", AHCINAME(sc), i);
    398 			break;
    399 		}
    400 		ata_channel_attach(chp);
    401 		port++;
    402 end:
    403 		continue;
    404 	}
    405 }
    406 
    407 int
    408 ahci_intr(void *v)
    409 {
    410 	struct ahci_softc *sc = v;
    411 	u_int32_t is;
    412 	int i, r = 0;
    413 
    414 	while ((is = AHCI_READ(sc, AHCI_IS))) {
    415 		AHCIDEBUG_PRINT(("%s ahci_intr 0x%x\n", AHCINAME(sc), is),
    416 		    DEBUG_INTR);
    417 		r = 1;
    418 		AHCI_WRITE(sc, AHCI_IS, is);
    419 		for (i = 0; i < AHCI_MAX_PORTS; i++)
    420 			if (is & (1 << i))
    421 				ahci_intr_port(sc, &sc->sc_channels[i]);
    422 	}
    423 	return r;
    424 }
    425 
    426 void
    427 ahci_intr_port(struct ahci_softc *sc, struct ahci_channel *achp)
    428 {
    429 	u_int32_t is, tfd;
    430 	struct ata_channel *chp = &achp->ata_channel;
    431 	struct ata_xfer *xfer = chp->ch_queue->active_xfer;
    432 	int slot;
    433 
    434 	is = AHCI_READ(sc, AHCI_P_IS(chp->ch_channel));
    435 	AHCI_WRITE(sc, AHCI_P_IS(chp->ch_channel), is);
    436 	AHCIDEBUG_PRINT(("ahci_intr_port %s port %d is 0x%x CI 0x%x\n", AHCINAME(sc),
    437 	    chp->ch_channel, is, AHCI_READ(sc, AHCI_P_CI(chp->ch_channel))),
    438 	    DEBUG_INTR);
    439 
    440 	if (is & (AHCI_P_IX_TFES | AHCI_P_IX_HBFS | AHCI_P_IX_IFS |
    441 	    AHCI_P_IX_OFS | AHCI_P_IX_UFS)) {
    442 		slot = (AHCI_READ(sc, AHCI_P_CMD(chp->ch_channel))
    443 			& AHCI_P_CMD_CCS_MASK) >> AHCI_P_CMD_CCS_SHIFT;
    444 		if ((achp->ahcic_cmds_active & (1 << slot)) == 0)
    445 			return;
    446 		/* stop channel */
    447 		ahci_channel_stop(sc, chp, 0);
    448 		if (slot != 0) {
    449 			printf("ahci_intr_port: slot %d\n", slot);
    450 			panic("ahci_intr_port");
    451 		}
    452 		if (is & AHCI_P_IX_TFES) {
    453 			tfd = AHCI_READ(sc, AHCI_P_TFD(chp->ch_channel));
    454 			chp->ch_error =
    455 			    (tfd & AHCI_P_TFD_ERR_MASK) >> AHCI_P_TFD_ERR_SHIFT;
    456 			chp->ch_status = (tfd & 0xff);
    457 		} else {
    458 			/* emulate a CRC error */
    459 			chp->ch_error = WDCE_CRC;
    460 			chp->ch_status = WDCS_ERR;
    461 		}
    462 		xfer->c_intr(chp, xfer, is);
    463 		/* if channel has not been restarted, do it now */
    464 		if ((AHCI_READ(sc, AHCI_P_CMD(chp->ch_channel)) & AHCI_P_CMD_CR)
    465 		    == 0)
    466 			ahci_channel_start(sc, chp);
    467 	} else {
    468 		slot = 0; /* XXX */
    469 		is = AHCI_READ(sc, AHCI_P_IS(chp->ch_channel));
    470 		AHCIDEBUG_PRINT(("ahci_intr_port port %d is 0x%x act 0x%x CI 0x%x\n",
    471 		    chp->ch_channel, is, achp->ahcic_cmds_active,
    472 		    AHCI_READ(sc, AHCI_P_CI(chp->ch_channel))), DEBUG_INTR);
    473 		if ((achp->ahcic_cmds_active & (1 << slot)) == 0)
    474 			return;
    475 		if ((AHCI_READ(sc, AHCI_P_CI(chp->ch_channel)) & (1 << slot))
    476 		    == 0) {
    477 			xfer->c_intr(chp, xfer, 0);
    478 		}
    479 	}
    480 }
    481 
    482 void
    483 ahci_reset_drive(struct ata_drive_datas *drvp, int flags)
    484 {
    485 	struct ata_channel *chp = drvp->chnl_softc;
    486 	ata_reset_channel(chp, flags);
    487 	return;
    488 }
    489 
    490 void
    491 ahci_reset_channel(struct ata_channel *chp, int flags)
    492 {
    493 	struct ahci_softc *sc = (struct ahci_softc *)chp->ch_atac;
    494 	struct ahci_channel *achp = (struct ahci_channel *)chp;
    495 
    496 	ahci_channel_stop(sc, chp, flags);
    497 	if (sata_reset_interface(chp, sc->sc_ahcit, achp->ahcic_scontrol,
    498 	    achp->ahcic_sstatus) != SStatus_DET_DEV) {
    499 		printf("%s: port reset failed\n", AHCINAME(sc));
    500 		/* XXX and then ? */
    501 	}
    502 	AHCI_WRITE(sc, AHCI_P_SERR(chp->ch_channel),
    503 	    AHCI_READ(sc, AHCI_P_SERR(chp->ch_channel)));
    504 	if (chp->ch_queue->active_xfer) {
    505 		chp->ch_queue->active_xfer->c_kill_xfer(chp,
    506 		    chp->ch_queue->active_xfer, KILL_RESET);
    507 	}
    508 	ahci_channel_start(sc, chp);
    509 #if 0
    510 	/* Wait 15s for device to host FIS to arrive. */
    511 	for (i = 0; i <1500; i++) {
    512 		if (AHCI_READ(sc, AHCI_P_IS(chp->ch_channel)) & AHCI_P_IX_DHRS)
    513 			break;
    514 		if (flags & AT_WAIT)
    515 			tsleep(&sc, PRIBIO, "ahcid2h", mstohz(10));
    516 		else
    517 			delay (10000);
    518 	}
    519 	if (i == 1500)
    520 		aprint_error("%s port %d: D2H FIS never arrived\n", AHCINAME(sc));
    521 #endif
    522 	/* clear port interrupt register */
    523 	AHCI_WRITE(sc, AHCI_P_IS(chp->ch_channel), 0xffffffff);
    524 
    525 	return;
    526 }
    527 
    528 int
    529 ahci_ata_addref(struct ata_drive_datas *drvp)
    530 {
    531 	return 0;
    532 }
    533 
    534 void
    535 ahci_ata_delref(struct ata_drive_datas *drvp)
    536 {
    537 	return;
    538 }
    539 
    540 void
    541 ahci_killpending(struct ata_drive_datas *drvp)
    542 {
    543 	return;
    544 }
    545 
    546 void
    547 ahci_probe_drive(struct ata_channel *chp)
    548 {
    549 	struct ahci_softc *sc = (struct ahci_softc *)chp->ch_atac;
    550 	struct ahci_channel *achp = (struct ahci_channel *)chp;
    551 	int i, s;
    552 	u_int32_t sig;
    553 
    554 	/* XXX This should be done by other code. */
    555 	for (i = 0; i < chp->ch_ndrive; i++) {
    556 		chp->ch_drive[i].chnl_softc = chp;
    557 		chp->ch_drive[i].drive = i;
    558 	}
    559 
    560 	/* bring interface up, power up and spin up device */
    561 	AHCI_WRITE(sc, AHCI_P_CMD(chp->ch_channel),
    562 	    AHCI_P_CMD_ICC_AC | AHCI_P_CMD_POD | AHCI_P_CMD_SUD);
    563 	/* reset the PHY and bring online */
    564 	switch (sata_reset_interface(chp, sc->sc_ahcit, achp->ahcic_scontrol,
    565 	    achp->ahcic_sstatus)) {
    566 	case SStatus_DET_DEV:
    567 		AHCI_WRITE(sc, AHCI_P_SERR(chp->ch_channel),
    568 		    AHCI_READ(sc, AHCI_P_SERR(chp->ch_channel)));
    569 #if 0
    570 		/* wait 15s for d2h FIS */
    571 		for (i = 0; i <1500; i++) {
    572 			if (AHCI_READ(sc, AHCI_P_IS(chp->ch_channel))
    573 			    & AHCI_P_IX_DHRS)
    574 				break;
    575 			tsleep(&sc, PRIBIO, "ahcid2h", mstohz(10));
    576 		}
    577 		if (i == 1500)
    578 			aprint_error("%s: D2H FIS never arrived\n",
    579 			    AHCINAME(sc));
    580 #endif
    581 
    582 		sig = AHCI_READ(sc, AHCI_P_SIG(chp->ch_channel));
    583 		AHCIDEBUG_PRINT(("%s: port %d: sig=0x%x CMD=0x%x\n",
    584 		    AHCINAME(sc), chp->ch_channel, sig,
    585 		    AHCI_READ(sc, AHCI_P_CMD(chp->ch_channel))), DEBUG_PROBE);
    586 		/*
    587 		 * scnt and sn are supposed to be 0x1 for ATAPI, but in some
    588 		 * cases we get wrong values here, so ignore it.
    589 		 */
    590 		s = splbio();
    591 		if ((sig & 0xffff0000) == 0xeb140000) {
    592 			chp->ch_drive[0].drive_flags |= DRIVE_ATAPI;
    593 		} else
    594 			chp->ch_drive[0].drive_flags |= DRIVE_ATA;
    595 		splx(s);
    596 		/* enable interrupts */
    597 		AHCI_WRITE(sc, AHCI_P_IE(chp->ch_channel),
    598 		    AHCI_P_IX_TFES | AHCI_P_IX_HBFS | AHCI_P_IX_IFS |
    599 		    AHCI_P_IX_OFS | AHCI_P_IX_DPS | AHCI_P_IX_UFS |
    600 		    AHCI_P_IX_DHRS);
    601 		/* and start operations */
    602 		ahci_channel_start(sc, chp);
    603 		break;
    604 
    605 	default:
    606 		break;
    607 	}
    608 }
    609 
    610 void
    611 ahci_setup_channel(struct ata_channel *chp)
    612 {
    613 	return;
    614 }
    615 
    616 int
    617 ahci_exec_command(struct ata_drive_datas *drvp, struct ata_command *ata_c)
    618 {
    619 	struct ata_channel *chp = drvp->chnl_softc;
    620 	struct ata_xfer *xfer;
    621 	int ret;
    622 	int s;
    623 
    624 	struct ahci_softc *sc = (struct ahci_softc *)chp->ch_atac;
    625 	AHCIDEBUG_PRINT(("ahci_exec_command port %d CI 0x%x\n",
    626 	    chp->ch_channel, AHCI_READ(sc, AHCI_P_CI(chp->ch_channel))),
    627 	    DEBUG_XFERS);
    628 	xfer = ata_get_xfer(ata_c->flags & AT_WAIT ? ATAXF_CANSLEEP :
    629 	    ATAXF_NOSLEEP);
    630 	if (xfer == NULL) {
    631 		return ATACMD_TRY_AGAIN;
    632 	}
    633 	if (ata_c->flags & AT_POLL)
    634 		xfer->c_flags |= C_POLL;
    635 	if (ata_c->flags & AT_WAIT)
    636 		xfer->c_flags |= C_WAIT;
    637 	xfer->c_drive = drvp->drive;
    638 	xfer->c_databuf = ata_c->data;
    639 	xfer->c_bcount = ata_c->bcount;
    640 	xfer->c_cmd = ata_c;
    641 	xfer->c_start = ahci_cmd_start;
    642 	xfer->c_intr = ahci_cmd_complete;
    643 	xfer->c_kill_xfer = ahci_cmd_kill_xfer;
    644 	s = splbio();
    645 	ata_exec_xfer(chp, xfer);
    646 #ifdef DIAGNOSTIC
    647 	if ((ata_c->flags & AT_POLL) != 0 &&
    648 	    (ata_c->flags & AT_DONE) == 0)
    649 		panic("ahci_exec_command: polled command not done");
    650 #endif
    651 	if (ata_c->flags & AT_DONE) {
    652 		ret = ATACMD_COMPLETE;
    653 	} else {
    654 		if (ata_c->flags & AT_WAIT) {
    655 			while ((ata_c->flags & AT_DONE) == 0) {
    656 				tsleep(ata_c, PRIBIO, "ahcicmd", 0);
    657 			}
    658 			ret = ATACMD_COMPLETE;
    659 		} else {
    660 			ret = ATACMD_QUEUED;
    661 		}
    662 	}
    663 	splx(s);
    664 	return ret;
    665 }
    666 
    667 void
    668 ahci_cmd_start(struct ata_channel *chp, struct ata_xfer *xfer)
    669 {
    670 	struct ahci_softc *sc = (struct ahci_softc *)chp->ch_atac;
    671 	struct ahci_channel *achp = (struct ahci_channel *)chp;
    672 	struct ata_command *ata_c = xfer->c_cmd;
    673 	int slot = 0 /* XXX slot */;
    674 	struct ahci_cmd_tbl *cmd_tbl;
    675 	struct ahci_cmd_header *cmd_h;
    676 	u_int8_t *fis;
    677 	int i;
    678 	int channel = chp->ch_channel;
    679 
    680 	AHCIDEBUG_PRINT(("ahci_cmd_start CI 0x%x\n",
    681 	    AHCI_READ(sc, AHCI_P_CI(chp->ch_channel))), DEBUG_XFERS);
    682 
    683 	cmd_tbl = achp->ahcic_cmd_tbl[slot];
    684 	AHCIDEBUG_PRINT(("%s port %d tbl %p\n", AHCINAME(sc), chp->ch_channel,
    685 	      cmd_tbl), DEBUG_XFERS);
    686 	fis = cmd_tbl->cmdt_cfis;
    687 
    688 	fis[0] = 0x27;  /* host to device */
    689 	fis[1] = 0x80;  /* command FIS */
    690 	fis[2] = ata_c->r_command;
    691 	fis[3] = ata_c->r_features;
    692 	fis[4] = ata_c->r_sector;
    693 	fis[5] = ata_c->r_cyl & 0xff;
    694 	fis[6] = (ata_c->r_cyl >> 8) & 0xff;
    695 	fis[7] = ata_c->r_head & 0x0f;
    696 	fis[8] = 0;
    697 	fis[9] = 0;
    698 	fis[10] = 0;
    699 	fis[11] = 0;
    700 	fis[12] = ata_c->r_count;
    701 	fis[13] = 0;
    702 	fis[14] = 0;
    703 	fis[15] = WDCTL_4BIT;
    704 	fis[16] = 0;
    705 	fis[17] = 0;
    706 	fis[18] = 0;
    707 	fis[19] = 0;
    708 
    709 	cmd_h = &achp->ahcic_cmdh[slot];
    710 	AHCIDEBUG_PRINT(("%s port %d header %p\n", AHCINAME(sc),
    711 	    chp->ch_channel, cmd_h), DEBUG_XFERS);
    712 	if (ahci_dma_setup(chp, slot,
    713 	    (ata_c->flags & (AT_READ|AT_WRITE)) ? ata_c->data : NULL,
    714 	    ata_c->bcount,
    715 	    (ata_c->flags & AT_READ) ? BUS_DMA_READ : BUS_DMA_WRITE)) {
    716 		ata_c->flags |= AT_DF;
    717 		ahci_cmd_complete(chp, xfer, slot);
    718 		return;
    719 	}
    720 	cmd_h->cmdh_flags = htole16(
    721 	    ((ata_c->flags & AT_WRITE) ? AHCI_CMDH_F_WR : 0) |
    722 	    20 /* fis lenght */ / 4);
    723 	cmd_h->cmdh_prdbc = 0;
    724 	AHCI_CMDH_SYNC(sc, achp, slot,
    725 	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
    726 
    727 	if (ata_c->flags & AT_POLL) {
    728 		/* polled command, disable interrupts */
    729 		AHCI_WRITE(sc, AHCI_GHC,
    730 		    AHCI_READ(sc, AHCI_GHC) & ~AHCI_GHC_IE);
    731 	}
    732 	chp->ch_flags |= ATACH_IRQ_WAIT;
    733 	chp->ch_status = 0;
    734 	/* start command */
    735 	AHCI_WRITE(sc, AHCI_P_CI(chp->ch_channel), 1 << slot);
    736 	/* and says we started this command */
    737 	achp->ahcic_cmds_active |= 1 << slot;
    738 
    739 	if ((ata_c->flags & AT_POLL) == 0) {
    740 		chp->ch_flags |= ATACH_IRQ_WAIT; /* wait for interrupt */
    741 		callout_reset(&chp->ch_callout, mstohz(ata_c->timeout),
    742 		    ahci_timeout, chp);
    743 		return;
    744 	}
    745 	/*
    746 	 * Polled command.
    747 	 */
    748 	for (i = 0; i < ata_c->timeout / 10; i++) {
    749 		if (ata_c->flags & AT_DONE)
    750 			break;
    751 		ahci_intr_port(sc, achp);
    752 		if (ata_c->flags & AT_WAIT)
    753 			tsleep(&xfer, PRIBIO, "ahcipl", mstohz(10));
    754 		else
    755 			delay(10000);
    756 	}
    757 	AHCIDEBUG_PRINT(("%s port %d poll end GHC 0x%x IS 0x%x list 0x%x%x fis 0x%x%x CMD 0x%x CI 0x%x\n", AHCINAME(sc), channel,
    758 	    AHCI_READ(sc, AHCI_GHC), AHCI_READ(sc, AHCI_IS),
    759 	    AHCI_READ(sc, AHCI_P_CLBU(channel)), AHCI_READ(sc, AHCI_P_CLB(channel)),
    760 	    AHCI_READ(sc, AHCI_P_FBU(channel)), AHCI_READ(sc, AHCI_P_FB(channel)),
    761 	    AHCI_READ(sc, AHCI_P_CMD(channel)), AHCI_READ(sc, AHCI_P_CI(channel))),
    762 	    DEBUG_XFERS);
    763 	if ((ata_c->flags & AT_DONE) == 0) {
    764 		ata_c->flags |= AT_TIMEOU;
    765 		ahci_cmd_complete(chp, xfer, slot);
    766 	}
    767 	/* reenable interrupts */
    768 	AHCI_WRITE(sc, AHCI_GHC, AHCI_READ(sc, AHCI_GHC) | AHCI_GHC_IE);
    769 }
    770 
    771 void
    772 ahci_cmd_kill_xfer(struct ata_channel *chp, struct ata_xfer *xfer, int reason)
    773 {
    774 	struct ata_command *ata_c = xfer->c_cmd;
    775 	AHCIDEBUG_PRINT(("ahci_cmd_kill_xfer channel %d\n", chp->ch_channel),
    776 	    DEBUG_FUNCS);
    777 
    778 	switch (reason) {
    779 	case KILL_GONE:
    780 		ata_c->flags |= AT_GONE;
    781 		break;
    782 	case KILL_RESET:
    783 		ata_c->flags |= AT_RESET;
    784 		break;
    785 	default:
    786 		printf("ahci_cmd_kill_xfer: unknown reason %d\n", reason);
    787 		panic("ahci_cmd_kill_xfer");
    788 	}
    789 	ahci_cmd_done(chp, xfer, 0 /* XXX slot */);
    790 }
    791 
    792 int
    793 ahci_cmd_complete(struct ata_channel *chp, struct ata_xfer *xfer, int is)
    794 {
    795 	int slot = 0; /* XXX slot */
    796 	struct ata_command *ata_c = xfer->c_cmd;
    797 	struct ahci_softc *sc = (struct ahci_softc *)chp->ch_atac;
    798 
    799 	AHCIDEBUG_PRINT(("ahci_cmd_complete channel %d CMD 0x%x CI 0x%x\n",
    800 	    chp->ch_channel, AHCI_READ(sc, AHCI_P_CMD(chp->ch_channel)),
    801 	    AHCI_READ(sc, AHCI_P_CI(chp->ch_channel))),
    802 	    DEBUG_FUNCS);
    803 	chp->ch_flags &= ~ATACH_IRQ_WAIT;
    804 	if (xfer->c_flags & C_TIMEOU) {
    805 		ata_c->flags |= AT_TIMEOU;
    806 	} else
    807 		callout_stop(&chp->ch_callout);
    808 
    809 	chp->ch_queue->active_xfer = NULL;
    810 
    811 	if (chp->ch_drive[xfer->c_drive].drive_flags & DRIVE_WAITDRAIN) {
    812 		ahci_cmd_kill_xfer(chp, xfer, KILL_GONE);
    813 		chp->ch_drive[xfer->c_drive].drive_flags &= ~DRIVE_WAITDRAIN;
    814 		wakeup(&chp->ch_queue->active_xfer);
    815 		return 0;
    816 	}
    817 	if (is) {
    818 		ata_c->r_head = 0;
    819 		ata_c->r_count = 0;
    820 		ata_c->r_sector = 0;
    821 		ata_c->r_cyl = 0;
    822 		if (chp->ch_status & WDCS_BSY) {
    823 			ata_c->flags |= AT_TIMEOU;
    824 		} else if (chp->ch_status & WDCS_ERR) {
    825 			ata_c->r_error = chp->ch_error;
    826 			ata_c->flags |= AT_ERROR;
    827 		}
    828 	}
    829 	ahci_cmd_done(chp, xfer, slot);
    830 	return 0;
    831 }
    832 
    833 void
    834 ahci_cmd_done(struct ata_channel *chp, struct ata_xfer *xfer, int slot)
    835 {
    836 	struct ahci_softc *sc = (struct ahci_softc *)chp->ch_atac;
    837 	struct ahci_channel *achp = (struct ahci_channel *)chp;
    838 	struct ata_command *ata_c = xfer->c_cmd;
    839 
    840 	AHCIDEBUG_PRINT(("ahci_cmd_done channel %d\n", chp->ch_channel),
    841 	    DEBUG_FUNCS);
    842 
    843 	/* this comamnd is not active any more */
    844 	achp->ahcic_cmds_active &= ~(1 << slot);
    845 
    846 	if (ata_c->flags & (AT_READ|AT_WRITE)) {
    847 		bus_dmamap_sync(sc->sc_dmat, achp->ahcic_datad[slot], 0,
    848 		    achp->ahcic_datad[slot]->dm_mapsize,
    849 		    (ata_c->flags & AT_READ) ? BUS_DMASYNC_POSTREAD :
    850 		    BUS_DMASYNC_POSTWRITE);
    851 		bus_dmamap_unload(sc->sc_dmat, achp->ahcic_datad[slot]);
    852 	}
    853 
    854 	AHCI_CMDH_SYNC(sc, achp, slot,
    855 	    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
    856 
    857 	ata_c->flags |= AT_DONE;
    858 	if (achp->ahcic_cmdh[slot].cmdh_prdbc)
    859 		ata_c->flags |= AT_XFDONE;
    860 
    861 	ata_free_xfer(chp, xfer);
    862 	if (ata_c->flags & AT_WAIT)
    863 		wakeup(ata_c);
    864 	else if (ata_c->callback)
    865 		ata_c->callback(ata_c->callback_arg);
    866 	atastart(chp);
    867 	return;
    868 }
    869 
    870 int
    871 ahci_ata_bio(struct ata_drive_datas *drvp, struct ata_bio *ata_bio)
    872 {
    873 	struct ata_channel *chp = drvp->chnl_softc;
    874 	struct ata_xfer *xfer;
    875 
    876 	struct ahci_softc *sc = (struct ahci_softc *)chp->ch_atac;
    877 	AHCIDEBUG_PRINT(("ahci_ata_bio port %d CI 0x%x\n",
    878 	    chp->ch_channel, AHCI_READ(sc, AHCI_P_CI(chp->ch_channel))),
    879 	    DEBUG_XFERS);
    880 	xfer = ata_get_xfer(ATAXF_NOSLEEP);
    881 	if (xfer == NULL) {
    882 		return ATACMD_TRY_AGAIN;
    883 	}
    884 	if (ata_bio->flags & ATA_POLL)
    885 		xfer->c_flags |= C_POLL;
    886 	xfer->c_drive = drvp->drive;
    887 	xfer->c_cmd = ata_bio;
    888 	xfer->c_databuf = ata_bio->databuf;
    889 	xfer->c_bcount = ata_bio->bcount;
    890 	xfer->c_start = ahci_bio_start;
    891 	xfer->c_intr = ahci_bio_complete;
    892 	xfer->c_kill_xfer = ahci_bio_kill_xfer;
    893 	ata_exec_xfer(chp, xfer);
    894 	return (ata_bio->flags & ATA_ITSDONE) ? ATACMD_COMPLETE : ATACMD_QUEUED;
    895 }
    896 
    897 void
    898 ahci_bio_start(struct ata_channel *chp, struct ata_xfer *xfer)
    899 {
    900 	struct ahci_softc *sc = (struct ahci_softc *)chp->ch_atac;
    901 	struct ahci_channel *achp = (struct ahci_channel *)chp;
    902 	struct ata_bio *ata_bio = xfer->c_cmd;
    903 	int slot = 0 /* XXX slot */;
    904 	struct ahci_cmd_tbl *cmd_tbl;
    905 	struct ahci_cmd_header *cmd_h;
    906 	u_int8_t *fis;
    907 	int i, nblks;
    908 	int channel = chp->ch_channel;
    909 
    910 	AHCIDEBUG_PRINT(("ahci_bio_start CI 0x%x\n",
    911 	    AHCI_READ(sc, AHCI_P_CI(chp->ch_channel))), DEBUG_XFERS);
    912 
    913 	nblks = xfer->c_bcount / ata_bio->lp->d_secsize;
    914 
    915 	cmd_tbl = achp->ahcic_cmd_tbl[slot];
    916 	AHCIDEBUG_PRINT(("%s port %d tbl %p\n", AHCINAME(sc), chp->ch_channel,
    917 	      cmd_tbl), DEBUG_XFERS);
    918 	fis = cmd_tbl->cmdt_cfis;
    919 
    920 	fis[0] = 0x27;  /* host to device */
    921 	fis[1] = 0x80;  /* command FIS */
    922 	if (ata_bio->flags & ATA_LBA48) {
    923 		fis[2] = (ata_bio->flags & ATA_READ) ?
    924 		    WDCC_READDMA_EXT : WDCC_WRITEDMA_EXT;
    925 	} else {
    926 		fis[2] =
    927 		    (ata_bio->flags & ATA_READ) ? WDCC_READDMA : WDCC_WRITEDMA;
    928 	}
    929 	fis[3] = 0; /* features */
    930 	fis[4] = ata_bio->blkno & 0xff;
    931 	fis[5] = (ata_bio->blkno >> 8) & 0xff;
    932 	fis[6] = (ata_bio->blkno >> 16) & 0xff;
    933 	if (ata_bio->flags & ATA_LBA48) {
    934 		fis[7] = WDSD_LBA;
    935 		fis[8] = (ata_bio->blkno >> 24) & 0xff;
    936 		fis[9] = (ata_bio->blkno >> 32) & 0xff;
    937 		fis[10] = (ata_bio->blkno >> 40) & 0xff;
    938 	} else {
    939 		fis[7] = ((ata_bio->blkno >> 24) & 0x0f) | WDSD_LBA;
    940 		fis[8] = 0;
    941 		fis[9] = 0;
    942 		fis[10] = 0;
    943 	}
    944 	fis[11] = 0; /* ext features */
    945 	fis[12] = nblks & 0xff;
    946 	fis[13] = (ata_bio->flags & ATA_LBA48) ?
    947 	    ((nblks >> 8) & 0xff) : 0;
    948 	fis[14] = 0;
    949 	fis[15] = WDCTL_4BIT;
    950 	fis[16] = 0;
    951 	fis[17] = 0;
    952 	fis[18] = 0;
    953 	fis[19] = 0;
    954 
    955 	cmd_h = &achp->ahcic_cmdh[slot];
    956 	AHCIDEBUG_PRINT(("%s port %d header %p\n", AHCINAME(sc),
    957 	    chp->ch_channel, cmd_h), DEBUG_XFERS);
    958 	if (ahci_dma_setup(chp, slot, ata_bio->databuf, ata_bio->bcount,
    959 	    (ata_bio->flags & ATA_READ) ? BUS_DMA_READ : BUS_DMA_WRITE)) {
    960 		ata_bio->error = ERR_DMA;
    961 		ata_bio->r_error = 0;
    962 		ahci_bio_complete(chp, xfer, slot);
    963 		return;
    964 	}
    965 	cmd_h->cmdh_flags = htole16(
    966 	    ((ata_bio->flags & ATA_READ) ? 0 :  AHCI_CMDH_F_WR) |
    967 	    20 /* fis lenght */ / 4);
    968 	cmd_h->cmdh_prdbc = 0;
    969 	AHCI_CMDH_SYNC(sc, achp, slot,
    970 	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
    971 
    972 	if (xfer->c_flags & C_POLL) {
    973 		/* polled command, disable interrupts */
    974 		AHCI_WRITE(sc, AHCI_GHC,
    975 		    AHCI_READ(sc, AHCI_GHC) & ~AHCI_GHC_IE);
    976 	}
    977 	chp->ch_flags |= ATACH_IRQ_WAIT;
    978 	chp->ch_status = 0;
    979 	/* start command */
    980 	AHCI_WRITE(sc, AHCI_P_CI(chp->ch_channel), 1 << slot);
    981 	/* and says we started this command */
    982 	achp->ahcic_cmds_active |= 1 << slot;
    983 
    984 	if ((xfer->c_flags & C_POLL) == 0) {
    985 		chp->ch_flags |= ATACH_IRQ_WAIT; /* wait for interrupt */
    986 		callout_reset(&chp->ch_callout, mstohz(ATA_DELAY),
    987 		    ahci_timeout, chp);
    988 		return;
    989 	}
    990 	/*
    991 	 * Polled command.
    992 	 */
    993 	for (i = 0; i < ATA_DELAY / 10; i++) {
    994 		if (ata_bio->flags & ATA_ITSDONE)
    995 			break;
    996 		ahci_intr_port(sc, achp);
    997 		if (ata_bio->flags & ATA_NOSLEEP)
    998 			delay(10000);
    999 		else
   1000 			tsleep(&xfer, PRIBIO, "ahcipl", mstohz(10));
   1001 	}
   1002 	AHCIDEBUG_PRINT(("%s port %d poll end GHC 0x%x IS 0x%x list 0x%x%x fis 0x%x%x CMD 0x%x CI 0x%x\n", AHCINAME(sc), channel,
   1003 	    AHCI_READ(sc, AHCI_GHC), AHCI_READ(sc, AHCI_IS),
   1004 	    AHCI_READ(sc, AHCI_P_CLBU(channel)), AHCI_READ(sc, AHCI_P_CLB(channel)),
   1005 	    AHCI_READ(sc, AHCI_P_FBU(channel)), AHCI_READ(sc, AHCI_P_FB(channel)),
   1006 	    AHCI_READ(sc, AHCI_P_CMD(channel)), AHCI_READ(sc, AHCI_P_CI(channel))),
   1007 	    DEBUG_XFERS);
   1008 	if ((ata_bio->flags & ATA_ITSDONE) == 0) {
   1009 		ata_bio->error = TIMEOUT;
   1010 		ahci_bio_complete(chp, xfer, slot);
   1011 	}
   1012 	/* reenable interrupts */
   1013 	AHCI_WRITE(sc, AHCI_GHC, AHCI_READ(sc, AHCI_GHC) | AHCI_GHC_IE);
   1014 }
   1015 
   1016 void
   1017 ahci_bio_kill_xfer(struct ata_channel *chp, struct ata_xfer *xfer, int reason)
   1018 {
   1019 	int slot = 0;  /* XXX slot */
   1020 	int drive = xfer->c_drive;
   1021 	struct ata_bio *ata_bio = xfer->c_cmd;
   1022 	struct ahci_channel *achp = (struct ahci_channel *)chp;
   1023 	AHCIDEBUG_PRINT(("ahci_bio_kill_xfer channel %d\n", chp->ch_channel),
   1024 	    DEBUG_FUNCS);
   1025 
   1026 	achp->ahcic_cmds_active &= ~(1 << slot);
   1027 	ata_free_xfer(chp, xfer);
   1028 	ata_bio->flags |= ATA_ITSDONE;
   1029 	switch (reason) {
   1030 	case KILL_GONE:
   1031 		ata_bio->error = ERR_NODEV;
   1032 		break;
   1033 	case KILL_RESET:
   1034 		ata_bio->error = ERR_RESET;
   1035 		break;
   1036 	default:
   1037 		printf("ahci_bio_kill_xfer: unknown reason %d\n", reason);
   1038 		panic("ahci_bio_kill_xfer");
   1039 	}
   1040 	ata_bio->r_error = WDCE_ABRT;
   1041 	(*chp->ch_drive[drive].drv_done)(chp->ch_drive[drive].drv_softc);
   1042 }
   1043 
   1044 int
   1045 ahci_bio_complete(struct ata_channel *chp, struct ata_xfer *xfer, int is)
   1046 {
   1047 	int slot = 0; /* XXX slot */
   1048 	struct ata_bio *ata_bio = xfer->c_cmd;
   1049 	int drive = xfer->c_drive;
   1050 	struct ahci_channel *achp = (struct ahci_channel *)chp;
   1051 	struct ahci_softc *sc = (struct ahci_softc *)chp->ch_atac;
   1052 
   1053 	AHCIDEBUG_PRINT(("ahci_bio_complete channel %d\n", chp->ch_channel),
   1054 	    DEBUG_FUNCS);
   1055 
   1056 	achp->ahcic_cmds_active &= ~(1 << slot);
   1057 	chp->ch_flags &= ~ATACH_IRQ_WAIT;
   1058 	if (xfer->c_flags & C_TIMEOU) {
   1059 		ata_bio->error = TIMEOUT;
   1060 	} else {
   1061 		callout_stop(&chp->ch_callout);
   1062 		ata_bio->error = 0;
   1063 	}
   1064 
   1065 	chp->ch_queue->active_xfer = NULL;
   1066 	bus_dmamap_sync(sc->sc_dmat, achp->ahcic_datad[slot], 0,
   1067 	    achp->ahcic_datad[slot]->dm_mapsize,
   1068 	    (ata_bio->flags & ATA_READ) ? BUS_DMASYNC_POSTREAD :
   1069 	    BUS_DMASYNC_POSTWRITE);
   1070 	bus_dmamap_unload(sc->sc_dmat, achp->ahcic_datad[slot]);
   1071 
   1072 	if (chp->ch_drive[xfer->c_drive].drive_flags & DRIVE_WAITDRAIN) {
   1073 		ahci_bio_kill_xfer(chp, xfer, KILL_GONE);
   1074 		chp->ch_drive[xfer->c_drive].drive_flags &= ~DRIVE_WAITDRAIN;
   1075 		wakeup(&chp->ch_queue->active_xfer);
   1076 		return 0;
   1077 	}
   1078 	ata_free_xfer(chp, xfer);
   1079 	ata_bio->flags |= ATA_ITSDONE;
   1080 	if (chp->ch_status & WDCS_DWF) {
   1081 		ata_bio->error = ERR_DF;
   1082 	} else if (chp->ch_status & WDCS_ERR) {
   1083 		ata_bio->error = ERROR;
   1084 		ata_bio->r_error = chp->ch_error;
   1085 	} else if (chp->ch_status & WDCS_CORR)
   1086 		ata_bio->flags |= ATA_CORR;
   1087 
   1088 	AHCI_CMDH_SYNC(sc, achp, slot,
   1089 	    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
   1090 	AHCIDEBUG_PRINT(("ahci_bio_complete bcount %ld",
   1091 	    ata_bio->bcount), DEBUG_XFERS);
   1092 	ata_bio->bcount -= le32toh(achp->ahcic_cmdh[slot].cmdh_prdbc);
   1093 	AHCIDEBUG_PRINT((" now %ld\n", ata_bio->bcount), DEBUG_XFERS);
   1094 	(*chp->ch_drive[drive].drv_done)(chp->ch_drive[drive].drv_softc);
   1095 	atastart(chp);
   1096 	return 0;
   1097 }
   1098 
   1099 void
   1100 ahci_channel_stop(struct ahci_softc *sc, struct ata_channel *chp, int flags)
   1101 {
   1102 	int i;
   1103 	/* stop channel */
   1104 	AHCI_WRITE(sc, AHCI_P_CMD(chp->ch_channel),
   1105 	    AHCI_READ(sc, AHCI_P_CMD(chp->ch_channel)) & ~AHCI_P_CMD_ST);
   1106 	/* wait 1s for channel to stop */
   1107 	for (i = 0; i <100; i++) {
   1108 		if ((AHCI_READ(sc, AHCI_P_CMD(chp->ch_channel)) & AHCI_P_CMD_CR)
   1109 		    == 0)
   1110 			break;
   1111 		if (flags & AT_WAIT)
   1112 			tsleep(&sc, PRIBIO, "ahcirst", mstohz(10));
   1113 		else
   1114 			delay(10000);
   1115 	}
   1116 	if (AHCI_READ(sc, AHCI_P_CMD(chp->ch_channel)) & AHCI_P_CMD_CR) {
   1117 		printf("%s: channel wouldn't stop\n", AHCINAME(sc));
   1118 		/* XXX controller reset ? */
   1119 		return;
   1120 	}
   1121 }
   1122 
   1123 void
   1124 ahci_channel_start(struct ahci_softc *sc, struct ata_channel *chp)
   1125 {
   1126 	/* clear error */
   1127 	AHCI_WRITE(sc, AHCI_P_SERR(chp->ch_channel), 0);
   1128 
   1129 	/* and start controller */
   1130 	AHCI_WRITE(sc, AHCI_P_CMD(chp->ch_channel),
   1131 	    AHCI_P_CMD_ICC_AC | AHCI_P_CMD_POD | AHCI_P_CMD_SUD |
   1132 	    AHCI_P_CMD_FRE | AHCI_P_CMD_ST);
   1133 }
   1134 
   1135 void
   1136 ahci_timeout(void *v)
   1137 {
   1138 	struct ata_channel *chp = (struct ata_channel *)v;
   1139 	struct ata_xfer *xfer = chp->ch_queue->active_xfer;
   1140 	int s = splbio();
   1141 	AHCIDEBUG_PRINT(("ahci_timeout xfer %p\n", xfer), DEBUG_INTR);
   1142 	if ((chp->ch_flags & ATACH_IRQ_WAIT) != 0) {
   1143 		xfer->c_flags |= C_TIMEOU;
   1144 		xfer->c_intr(chp, xfer, 0);
   1145 	}
   1146 	splx(s);
   1147 }
   1148 
   1149 int
   1150 ahci_dma_setup(struct ata_channel *chp, int slot, void *data,
   1151     size_t count, int op)
   1152 {
   1153 	int error, seg;
   1154 	struct ahci_softc *sc = (struct ahci_softc *)chp->ch_atac;
   1155 	struct ahci_channel *achp = (struct ahci_channel *)chp;
   1156 	struct ahci_cmd_tbl *cmd_tbl;
   1157 	struct ahci_cmd_header *cmd_h;
   1158 
   1159 	cmd_h = &achp->ahcic_cmdh[slot];
   1160 	cmd_tbl = achp->ahcic_cmd_tbl[slot];
   1161 
   1162 	if (data == NULL) {
   1163 		cmd_h->cmdh_prdtl = 0;
   1164 		goto end;
   1165 	}
   1166 
   1167 	error = bus_dmamap_load(sc->sc_dmat, achp->ahcic_datad[slot],
   1168 	    data, count, NULL,
   1169 	    BUS_DMA_NOWAIT | BUS_DMA_STREAMING | op);
   1170 	if (error) {
   1171 		printf("%s port %d: failed to load xfer: %d\n",
   1172 		    AHCINAME(sc), chp->ch_channel, error);
   1173 		return error;
   1174 	}
   1175 	bus_dmamap_sync(sc->sc_dmat, achp->ahcic_datad[slot], 0,
   1176 	    achp->ahcic_datad[slot]->dm_mapsize,
   1177 	    (op == BUS_DMA_READ) ? BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE);
   1178 	for (seg = 0; seg <  achp->ahcic_datad[slot]->dm_nsegs; seg++) {
   1179 		cmd_tbl->cmdt_prd[seg].prd_dba = htole32(
   1180 		     achp->ahcic_datad[slot]->dm_segs[seg].ds_addr);
   1181 		cmd_tbl->cmdt_prd[seg].prd_dbau = 0;
   1182 		cmd_tbl->cmdt_prd[seg].prd_dbc = htole32(
   1183 		    achp->ahcic_datad[slot]->dm_segs[seg].ds_len - 1);
   1184 	}
   1185 	cmd_tbl->cmdt_prd[seg - 1].prd_dbc |= htole32(AHCI_PRD_DBC_IPC);
   1186 	cmd_h->cmdh_prdtl = htole16(achp->ahcic_datad[slot]->dm_nsegs);
   1187 end:
   1188 	AHCI_CMDTBL_SYNC(sc, achp, slot, BUS_DMASYNC_PREWRITE);
   1189 	return 0;
   1190 }
   1191 
   1192 #if NATAPIBUS > 0
   1193 void
   1194 ahci_atapibus_attach(struct atabus_softc * ata_sc)
   1195 {
   1196 	struct ata_channel *chp = ata_sc->sc_chan;
   1197 	struct atac_softc *atac = chp->ch_atac;
   1198 	struct scsipi_adapter *adapt = &atac->atac_atapi_adapter._generic;
   1199 	struct scsipi_channel *chan = &chp->ch_atapi_channel;
   1200 	/*
   1201 	 * Fill in the scsipi_adapter.
   1202 	 */
   1203 	adapt->adapt_dev = &atac->atac_dev;
   1204 	adapt->adapt_nchannels = atac->atac_nchannels;
   1205 	adapt->adapt_request = ahci_atapi_scsipi_request;
   1206 	adapt->adapt_minphys = ahci_atapi_minphys;
   1207 	atac->atac_atapi_adapter.atapi_probe_device = ahci_atapi_probe_device;
   1208 
   1209 	/*
   1210 	 * Fill in the scsipi_channel.
   1211 	 */
   1212 	memset(chan, 0, sizeof(*chan));
   1213 	chan->chan_adapter = adapt;
   1214 	chan->chan_bustype = &ahci_atapi_bustype;
   1215 	chan->chan_channel = chp->ch_channel;
   1216 	chan->chan_flags = SCSIPI_CHAN_OPENINGS;
   1217 	chan->chan_openings = 1;
   1218 	chan->chan_max_periph = 1;
   1219 	chan->chan_ntargets = 1;
   1220 	chan->chan_nluns = 1;
   1221 	chp->atapibus = config_found_ia(&ata_sc->sc_dev, "atapi", chan,
   1222 		atapiprint);
   1223 }
   1224 
   1225 void
   1226 ahci_atapi_minphys(struct buf *bp)
   1227 {
   1228 	if (bp->b_bcount > MAXPHYS)
   1229 		bp->b_bcount = MAXPHYS;
   1230 	minphys(bp);
   1231 }
   1232 
   1233 /*
   1234  * Kill off all pending xfers for a periph.
   1235  *
   1236  * Must be called at splbio().
   1237  */
   1238 void
   1239 ahci_atapi_kill_pending(struct scsipi_periph *periph)
   1240 {
   1241 	struct atac_softc *atac =
   1242 	    (void *)periph->periph_channel->chan_adapter->adapt_dev;
   1243 	struct ata_channel *chp =
   1244 	    atac->atac_channels[periph->periph_channel->chan_channel];
   1245 
   1246 	ata_kill_pending(&chp->ch_drive[periph->periph_target]);
   1247 }
   1248 
   1249 void
   1250 ahci_atapi_scsipi_request(struct scsipi_channel *chan,
   1251     scsipi_adapter_req_t req, void *arg)
   1252 {
   1253 	struct scsipi_adapter *adapt = chan->chan_adapter;
   1254 	struct scsipi_periph *periph;
   1255 	struct scsipi_xfer *sc_xfer;
   1256 	struct ahci_softc *sc = (void *)adapt->adapt_dev;
   1257 	struct atac_softc *atac = &sc->sc_atac;
   1258 	struct ata_xfer *xfer;
   1259 	int channel = chan->chan_channel;
   1260 	int drive, s;
   1261 
   1262 	switch (req) {
   1263 	case ADAPTER_REQ_RUN_XFER:
   1264 		sc_xfer = arg;
   1265 		periph = sc_xfer->xs_periph;
   1266 		drive = periph->periph_target;
   1267 		if (!device_is_active(&atac->atac_dev)) {
   1268 			sc_xfer->error = XS_DRIVER_STUFFUP;
   1269 			scsipi_done(sc_xfer);
   1270 			return;
   1271 		}
   1272 		xfer = ata_get_xfer(ATAXF_NOSLEEP);
   1273 		if (xfer == NULL) {
   1274 			sc_xfer->error = XS_RESOURCE_SHORTAGE;
   1275 			scsipi_done(sc_xfer);
   1276 			return;
   1277 		}
   1278 
   1279 		if (sc_xfer->xs_control & XS_CTL_POLL)
   1280 			xfer->c_flags |= C_POLL;
   1281 		xfer->c_drive = drive;
   1282 		xfer->c_flags |= C_ATAPI;
   1283 		xfer->c_cmd = sc_xfer;
   1284 		xfer->c_databuf = sc_xfer->data;
   1285 		xfer->c_bcount = sc_xfer->datalen;
   1286 		xfer->c_start = ahci_atapi_start;
   1287 		xfer->c_intr = ahci_atapi_complete;
   1288 		xfer->c_kill_xfer = ahci_atapi_kill_xfer;
   1289 		xfer->c_dscpoll = 0;
   1290 		s = splbio();
   1291 		ata_exec_xfer(atac->atac_channels[channel], xfer);
   1292 #ifdef DIAGNOSTIC
   1293 		if ((sc_xfer->xs_control & XS_CTL_POLL) != 0 &&
   1294 		    (sc_xfer->xs_status & XS_STS_DONE) == 0)
   1295 			panic("ahci_atapi_scsipi_request: polled command "
   1296 			    "not done");
   1297 #endif
   1298 		splx(s);
   1299 		return;
   1300 	default:
   1301 		/* Not supported, nothing to do. */
   1302 		;
   1303 	}
   1304 }
   1305 
   1306 void
   1307 ahci_atapi_start(struct ata_channel *chp, struct ata_xfer *xfer)
   1308 {
   1309 	struct ahci_softc *sc = (struct ahci_softc *)chp->ch_atac;
   1310 	struct ahci_channel *achp = (struct ahci_channel *)chp;
   1311 	struct scsipi_xfer *sc_xfer = xfer->c_cmd;
   1312 	int slot = 0 /* XXX slot */;
   1313 	struct ahci_cmd_tbl *cmd_tbl;
   1314 	struct ahci_cmd_header *cmd_h;
   1315 	u_int8_t *fis;
   1316 	int i;
   1317 	int channel = chp->ch_channel;
   1318 
   1319 	AHCIDEBUG_PRINT(("ahci_atapi_start CI 0x%x\n",
   1320 	    AHCI_READ(sc, AHCI_P_CI(chp->ch_channel))), DEBUG_XFERS);
   1321 
   1322 	cmd_tbl = achp->ahcic_cmd_tbl[slot];
   1323 	AHCIDEBUG_PRINT(("%s port %d tbl %p\n", AHCINAME(sc), chp->ch_channel,
   1324 	      cmd_tbl), DEBUG_XFERS);
   1325 	fis = cmd_tbl->cmdt_cfis;
   1326 
   1327 	fis[0] = 0x27;  /* host to device */
   1328 	fis[1] = 0x80;  /* command FIS */
   1329 	fis[2] = ATAPI_PKT_CMD;
   1330 	memset(&cmd_tbl->cmdt_acmd, 0, sizeof(cmd_tbl->cmdt_acmd));
   1331 	memcpy(cmd_tbl->cmdt_acmd, sc_xfer->cmd, sc_xfer->cmdlen);
   1332 	fis[3] = (sc_xfer->datalen ? ATAPI_PKT_CMD_FTRE_DMA : 0);
   1333 	fis[4] = 0;
   1334 	fis[5] = 0;
   1335 	fis[6] = 0;
   1336 	fis[7] = WDSD_LBA;
   1337 	fis[8] = 0;
   1338 	fis[9] = 0;
   1339 	fis[10] = 0;
   1340 	fis[11] = 0; /* ext features */
   1341 	fis[12] = 0;
   1342 	fis[13] = 0;
   1343 	fis[14] = 0;
   1344 	fis[15] = WDCTL_4BIT;
   1345 	fis[16] = 0;
   1346 	fis[17] = 0;
   1347 	fis[18] = 0;
   1348 	fis[19] = 0;
   1349 
   1350 	cmd_h = &achp->ahcic_cmdh[slot];
   1351 	AHCIDEBUG_PRINT(("%s port %d header %p\n", AHCINAME(sc),
   1352 	    chp->ch_channel, cmd_h), DEBUG_XFERS);
   1353 	if (ahci_dma_setup(chp, slot, sc_xfer->datalen ? sc_xfer->data : NULL,
   1354 	    sc_xfer->datalen,
   1355 	    (sc_xfer->xs_control & XS_CTL_DATA_IN) ?
   1356 	    BUS_DMA_READ : BUS_DMA_WRITE)) {
   1357 		sc_xfer->error = XS_DRIVER_STUFFUP;
   1358 		ahci_atapi_complete(chp, xfer, slot);
   1359 		return;
   1360 	}
   1361 	cmd_h->cmdh_flags = htole16(
   1362 	    ((sc_xfer->xs_control & XS_CTL_DATA_OUT) ? AHCI_CMDH_F_WR : 0) |
   1363 	    20 /* fis lenght */ / 4 | AHCI_CMDH_F_A);
   1364 	cmd_h->cmdh_prdbc = 0;
   1365 	AHCI_CMDH_SYNC(sc, achp, slot,
   1366 	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
   1367 
   1368 	if (xfer->c_flags & C_POLL) {
   1369 		/* polled command, disable interrupts */
   1370 		AHCI_WRITE(sc, AHCI_GHC,
   1371 		    AHCI_READ(sc, AHCI_GHC) & ~AHCI_GHC_IE);
   1372 	}
   1373 	chp->ch_flags |= ATACH_IRQ_WAIT;
   1374 	chp->ch_status = 0;
   1375 	/* start command */
   1376 	AHCI_WRITE(sc, AHCI_P_CI(chp->ch_channel), 1 << slot);
   1377 	/* and says we started this command */
   1378 	achp->ahcic_cmds_active |= 1 << slot;
   1379 
   1380 	if ((xfer->c_flags & C_POLL) == 0) {
   1381 		chp->ch_flags |= ATACH_IRQ_WAIT; /* wait for interrupt */
   1382 		callout_reset(&chp->ch_callout, mstohz(sc_xfer->timeout),
   1383 		    ahci_timeout, chp);
   1384 		return;
   1385 	}
   1386 	/*
   1387 	 * Polled command.
   1388 	 */
   1389 	for (i = 0; i < ATA_DELAY / 10; i++) {
   1390 		if (sc_xfer->xs_status & XS_STS_DONE)
   1391 			break;
   1392 		ahci_intr_port(sc, achp);
   1393 		delay(10000);
   1394 	}
   1395 	AHCIDEBUG_PRINT(("%s port %d poll end GHC 0x%x IS 0x%x list 0x%x%x fis 0x%x%x CMD 0x%x CI 0x%x\n", AHCINAME(sc), channel,
   1396 	    AHCI_READ(sc, AHCI_GHC), AHCI_READ(sc, AHCI_IS),
   1397 	    AHCI_READ(sc, AHCI_P_CLBU(channel)), AHCI_READ(sc, AHCI_P_CLB(channel)),
   1398 	    AHCI_READ(sc, AHCI_P_FBU(channel)), AHCI_READ(sc, AHCI_P_FB(channel)),
   1399 	    AHCI_READ(sc, AHCI_P_CMD(channel)), AHCI_READ(sc, AHCI_P_CI(channel))),
   1400 	    DEBUG_XFERS);
   1401 	if ((sc_xfer->xs_status & XS_STS_DONE) == 0) {
   1402 		sc_xfer->error = XS_TIMEOUT;
   1403 		ahci_atapi_complete(chp, xfer, slot);
   1404 	}
   1405 	/* reenable interrupts */
   1406 	AHCI_WRITE(sc, AHCI_GHC, AHCI_READ(sc, AHCI_GHC) | AHCI_GHC_IE);
   1407 }
   1408 
   1409 int
   1410 ahci_atapi_complete(struct ata_channel *chp, struct ata_xfer *xfer, int irq)
   1411 {
   1412 	int slot = 0; /* XXX slot */
   1413 	struct scsipi_xfer *sc_xfer = xfer->c_cmd;
   1414 	int drive = xfer->c_drive;
   1415 	struct ahci_channel *achp = (struct ahci_channel *)chp;
   1416 	struct ahci_softc *sc = (struct ahci_softc *)chp->ch_atac;
   1417 
   1418 	AHCIDEBUG_PRINT(("ahci_atapi_complete channel %d\n", chp->ch_channel),
   1419 	    DEBUG_FUNCS);
   1420 
   1421 	achp->ahcic_cmds_active &= ~(1 << slot);
   1422 	chp->ch_flags &= ~ATACH_IRQ_WAIT;
   1423 	if (xfer->c_flags & C_TIMEOU) {
   1424 		sc_xfer->error = XS_TIMEOUT;
   1425 	} else {
   1426 		callout_stop(&chp->ch_callout);
   1427 		sc_xfer->error = 0;
   1428 	}
   1429 
   1430 	chp->ch_queue->active_xfer = NULL;
   1431 	bus_dmamap_sync(sc->sc_dmat, achp->ahcic_datad[slot], 0,
   1432 	    achp->ahcic_datad[slot]->dm_mapsize,
   1433 	    (sc_xfer->xs_control & XS_CTL_DATA_IN) ? BUS_DMASYNC_POSTREAD :
   1434 	    BUS_DMASYNC_POSTWRITE);
   1435 	bus_dmamap_unload(sc->sc_dmat, achp->ahcic_datad[slot]);
   1436 
   1437 	if (chp->ch_drive[drive].drive_flags & DRIVE_WAITDRAIN) {
   1438 		ahci_atapi_kill_xfer(chp, xfer, KILL_GONE);
   1439 		chp->ch_drive[drive].drive_flags &= ~DRIVE_WAITDRAIN;
   1440 		wakeup(&chp->ch_queue->active_xfer);
   1441 		return 0;
   1442 	}
   1443 	ata_free_xfer(chp, xfer);
   1444 
   1445 	if (chp->ch_status & WDCS_ERR) {
   1446 		sc_xfer->error = XS_SHORTSENSE;
   1447 		sc_xfer->sense.atapi_sense = chp->ch_error;
   1448 	}
   1449 
   1450 	AHCI_CMDH_SYNC(sc, achp, slot,
   1451 	    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
   1452 	sc_xfer->resid = sc_xfer->datalen;
   1453 	sc_xfer->resid -= le32toh(achp->ahcic_cmdh[slot].cmdh_prdbc);
   1454 	AHCIDEBUG_PRINT(("ahci_atapi_complete datalen %d resid %d\n",
   1455 	    sc_xfer->datalen, sc_xfer->resid), DEBUG_XFERS);
   1456 	scsipi_done(sc_xfer);
   1457 	atastart(chp);
   1458 	return 0;
   1459 }
   1460 
   1461 void
   1462 ahci_atapi_kill_xfer(struct ata_channel *chp, struct ata_xfer *xfer, int reason)
   1463 {
   1464 	struct scsipi_xfer *sc_xfer = xfer->c_cmd;
   1465 	struct ahci_channel *achp = (struct ahci_channel *)chp;
   1466 	int slot = 0; /* XXX slot */
   1467 
   1468 	achp->ahcic_cmds_active &= ~(1 << slot);
   1469 
   1470 	/* remove this command from xfer queue */
   1471 	switch (reason) {
   1472 	case KILL_GONE:
   1473 		sc_xfer->error = XS_DRIVER_STUFFUP;
   1474 		break;
   1475 	case KILL_RESET:
   1476 		sc_xfer->error = XS_RESET;
   1477 		break;
   1478 	default:
   1479 		printf("ahci_ata_atapi_kill_xfer: unknown reason %d\n", reason);
   1480 		panic("ahci_ata_atapi_kill_xfer");
   1481 	}
   1482 	ata_free_xfer(chp, xfer);
   1483 	scsipi_done(sc_xfer);
   1484 }
   1485 
   1486 void
   1487 ahci_atapi_probe_device(struct atapibus_softc *sc, int target)
   1488 {
   1489 	struct scsipi_channel *chan = sc->sc_channel;
   1490 	struct scsipi_periph *periph;
   1491 	struct ataparams ids;
   1492 	struct ataparams *id = &ids;
   1493 	struct ahci_softc *ahcic = (void *)chan->chan_adapter->adapt_dev;
   1494 	struct atac_softc *atac = &ahcic->sc_atac;
   1495 	struct ata_channel *chp = atac->atac_channels[chan->chan_channel];
   1496 	struct ata_drive_datas *drvp = &chp->ch_drive[target];
   1497 	struct scsipibus_attach_args sa;
   1498 	char serial_number[21], model[41], firmware_revision[9];
   1499 	int s;
   1500 
   1501 	/* skip if already attached */
   1502 	if (scsipi_lookup_periph(chan, target, 0) != NULL)
   1503 		return;
   1504 
   1505 	/* if no ATAPI device detected at attach time, skip */
   1506 	if ((drvp->drive_flags & DRIVE_ATAPI) == 0) {
   1507 		AHCIDEBUG_PRINT(("ahci_atapi_probe_device: drive %d "
   1508 		    "not present\n", target), DEBUG_PROBE);
   1509 		return;
   1510 	}
   1511 
   1512 	/* Some ATAPI devices need a bit more time after software reset. */
   1513 	delay(5000);
   1514 	if (ata_get_params(drvp,  AT_WAIT, id) == 0) {
   1515 #ifdef ATAPI_DEBUG_PROBE
   1516 		printf("%s drive %d: cmdsz 0x%x drqtype 0x%x\n",
   1517 		    AHCINAME(sc), target,
   1518 		    id->atap_config & ATAPI_CFG_CMD_MASK,
   1519 		    id->atap_config & ATAPI_CFG_DRQ_MASK);
   1520 #endif
   1521 		periph = scsipi_alloc_periph(M_NOWAIT);
   1522 		if (periph == NULL) {
   1523 			printf("%s: unable to allocate periph for drive %d\n",
   1524 			    sc->sc_dev.dv_xname, target);
   1525 			return;
   1526 		}
   1527 		periph->periph_dev = NULL;
   1528 		periph->periph_channel = chan;
   1529 		periph->periph_switch = &atapi_probe_periphsw;
   1530 		periph->periph_target = target;
   1531 		periph->periph_lun = 0;
   1532 		periph->periph_quirks = PQUIRK_ONLYBIG;
   1533 
   1534 #ifdef SCSIPI_DEBUG
   1535 		if (SCSIPI_DEBUG_TYPE == SCSIPI_BUSTYPE_ATAPI &&
   1536 		    SCSIPI_DEBUG_TARGET == target)
   1537 			periph->periph_dbflags |= SCSIPI_DEBUG_FLAGS;
   1538 #endif
   1539 		periph->periph_type = ATAPI_CFG_TYPE(id->atap_config);
   1540 		if (id->atap_config & ATAPI_CFG_REMOV)
   1541 			periph->periph_flags |= PERIPH_REMOVABLE;
   1542 		if (periph->periph_type == T_SEQUENTIAL) {
   1543 			s = splbio();
   1544 			drvp->drive_flags |= DRIVE_ATAPIST;
   1545 			splx(s);
   1546 		}
   1547 
   1548 		sa.sa_periph = periph;
   1549 		sa.sa_inqbuf.type =  ATAPI_CFG_TYPE(id->atap_config);
   1550 		sa.sa_inqbuf.removable = id->atap_config & ATAPI_CFG_REMOV ?
   1551 		    T_REMOV : T_FIXED;
   1552 		scsipi_strvis((u_char *)model, 40, id->atap_model, 40);
   1553 		scsipi_strvis((u_char *)serial_number, 20, id->atap_serial,
   1554 		    20);
   1555 		scsipi_strvis((u_char *)firmware_revision, 8,
   1556 		    id->atap_revision, 8);
   1557 		sa.sa_inqbuf.vendor = model;
   1558 		sa.sa_inqbuf.product = serial_number;
   1559 		sa.sa_inqbuf.revision = firmware_revision;
   1560 
   1561 		/*
   1562 		 * Determine the operating mode capabilities of the device.
   1563 		 */
   1564 		if ((id->atap_config & ATAPI_CFG_CMD_MASK) == ATAPI_CFG_CMD_16)
   1565 			periph->periph_cap |= PERIPH_CAP_CMD16;
   1566 		/* XXX This is gross. */
   1567 		periph->periph_cap |= (id->atap_config & ATAPI_CFG_DRQ_MASK);
   1568 
   1569 		drvp->drv_softc = atapi_probe_device(sc, target, periph, &sa);
   1570 
   1571 		if (drvp->drv_softc)
   1572 			ata_probe_caps(drvp);
   1573 		else {
   1574 			s = splbio();
   1575 			drvp->drive_flags &= ~DRIVE_ATAPI;
   1576 			splx(s);
   1577 		}
   1578 	} else {
   1579 		AHCIDEBUG_PRINT(("ahci_atapi_get_params: ATAPI_IDENTIFY_DEVICE "
   1580 		    "failed for drive %s:%d:%d: error 0x%x\n",
   1581 		    AHCINAME(ahcic), chp->ch_channel, target,
   1582 		    chp->ch_error), DEBUG_PROBE);
   1583 		s = splbio();
   1584 		drvp->drive_flags &= ~DRIVE_ATAPI;
   1585 		splx(s);
   1586 	}
   1587 }
   1588 #endif /* NATAPIBUS */
   1589