Home | History | Annotate | Line # | Download | only in sdmmc
sdhc.c revision 1.57
      1 /*	$NetBSD: sdhc.c,v 1.57 2015/05/03 11:46:25 jmcneill Exp $	*/
      2 /*	$OpenBSD: sdhc.c,v 1.25 2009/01/13 19:44:20 grange Exp $	*/
      3 
      4 /*
      5  * Copyright (c) 2006 Uwe Stuehler <uwe (at) openbsd.org>
      6  *
      7  * Permission to use, copy, modify, and distribute this software for any
      8  * purpose with or without fee is hereby granted, provided that the above
      9  * copyright notice and this permission notice appear in all copies.
     10  *
     11  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
     12  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
     13  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
     14  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
     15  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
     16  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
     17  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
     18  */
     19 
     20 /*
     21  * SD Host Controller driver based on the SD Host Controller Standard
     22  * Simplified Specification Version 1.00 (www.sdcard.com).
     23  */
     24 
     25 #include <sys/cdefs.h>
     26 __KERNEL_RCSID(0, "$NetBSD: sdhc.c,v 1.57 2015/05/03 11:46:25 jmcneill Exp $");
     27 
     28 #ifdef _KERNEL_OPT
     29 #include "opt_sdmmc.h"
     30 #endif
     31 
     32 #include <sys/param.h>
     33 #include <sys/device.h>
     34 #include <sys/kernel.h>
     35 #include <sys/kthread.h>
     36 #include <sys/malloc.h>
     37 #include <sys/systm.h>
     38 #include <sys/mutex.h>
     39 #include <sys/condvar.h>
     40 
     41 #include <dev/sdmmc/sdhcreg.h>
     42 #include <dev/sdmmc/sdhcvar.h>
     43 #include <dev/sdmmc/sdmmcchip.h>
     44 #include <dev/sdmmc/sdmmcreg.h>
     45 #include <dev/sdmmc/sdmmcvar.h>
     46 
     47 #ifdef SDHC_DEBUG
     48 int sdhcdebug = 1;
     49 #define DPRINTF(n,s)	do { if ((n) <= sdhcdebug) printf s; } while (0)
     50 void	sdhc_dump_regs(struct sdhc_host *);
     51 #else
     52 #define DPRINTF(n,s)	do {} while (0)
     53 #endif
     54 
     55 #define SDHC_COMMAND_TIMEOUT	hz
     56 #define SDHC_BUFFER_TIMEOUT	hz
     57 #define SDHC_TRANSFER_TIMEOUT	hz
     58 #define SDHC_DMA_TIMEOUT	hz
     59 
     60 struct sdhc_host {
     61 	struct sdhc_softc *sc;		/* host controller device */
     62 
     63 	bus_space_tag_t iot;		/* host register set tag */
     64 	bus_space_handle_t ioh;		/* host register set handle */
     65 	bus_size_t ios;			/* host register space size */
     66 	bus_dma_tag_t dmat;		/* host DMA tag */
     67 
     68 	device_t sdmmc;			/* generic SD/MMC device */
     69 
     70 	struct kmutex host_mtx;
     71 
     72 	u_int clkbase;			/* base clock frequency in KHz */
     73 	int maxblklen;			/* maximum block length */
     74 	uint32_t ocr;			/* OCR value from capabilities */
     75 
     76 	uint8_t regs[14];		/* host controller state */
     77 
     78 	uint16_t intr_status;		/* soft interrupt status */
     79 	uint16_t intr_error_status;	/* soft error status */
     80 	struct kmutex intr_mtx;
     81 	struct kcondvar intr_cv;
     82 
     83 	int specver;			/* spec. version */
     84 
     85 	uint32_t flags;			/* flags for this host */
     86 #define SHF_USE_DMA		0x0001
     87 #define SHF_USE_4BIT_MODE	0x0002
     88 #define SHF_USE_8BIT_MODE	0x0004
     89 #define SHF_MODE_DMAEN		0x0008 /* needs SDHC_DMA_ENABLE in mode */
     90 };
     91 
     92 #define HDEVNAME(hp)	(device_xname((hp)->sc->sc_dev))
     93 
     94 static uint8_t
     95 hread1(struct sdhc_host *hp, bus_size_t reg)
     96 {
     97 
     98 	if (!ISSET(hp->sc->sc_flags, SDHC_FLAG_32BIT_ACCESS))
     99 		return bus_space_read_1(hp->iot, hp->ioh, reg);
    100 	return bus_space_read_4(hp->iot, hp->ioh, reg & -4) >> (8 * (reg & 3));
    101 }
    102 
    103 static uint16_t
    104 hread2(struct sdhc_host *hp, bus_size_t reg)
    105 {
    106 
    107 	if (!ISSET(hp->sc->sc_flags, SDHC_FLAG_32BIT_ACCESS))
    108 		return bus_space_read_2(hp->iot, hp->ioh, reg);
    109 	return bus_space_read_4(hp->iot, hp->ioh, reg & -4) >> (8 * (reg & 2));
    110 }
    111 
    112 #define HREAD1(hp, reg)		hread1(hp, reg)
    113 #define HREAD2(hp, reg)		hread2(hp, reg)
    114 #define HREAD4(hp, reg)		\
    115 	(bus_space_read_4((hp)->iot, (hp)->ioh, (reg)))
    116 
    117 
    118 static void
    119 hwrite1(struct sdhc_host *hp, bus_size_t o, uint8_t val)
    120 {
    121 
    122 	if (!ISSET(hp->sc->sc_flags, SDHC_FLAG_32BIT_ACCESS)) {
    123 		bus_space_write_1(hp->iot, hp->ioh, o, val);
    124 	} else {
    125 		const size_t shift = 8 * (o & 3);
    126 		o &= -4;
    127 		uint32_t tmp = bus_space_read_4(hp->iot, hp->ioh, o);
    128 		tmp = (val << shift) | (tmp & ~(0xff << shift));
    129 		bus_space_write_4(hp->iot, hp->ioh, o, tmp);
    130 	}
    131 }
    132 
    133 static void
    134 hwrite2(struct sdhc_host *hp, bus_size_t o, uint16_t val)
    135 {
    136 
    137 	if (!ISSET(hp->sc->sc_flags, SDHC_FLAG_32BIT_ACCESS)) {
    138 		bus_space_write_2(hp->iot, hp->ioh, o, val);
    139 	} else {
    140 		const size_t shift = 8 * (o & 2);
    141 		o &= -4;
    142 		uint32_t tmp = bus_space_read_4(hp->iot, hp->ioh, o);
    143 		tmp = (val << shift) | (tmp & ~(0xffff << shift));
    144 		bus_space_write_4(hp->iot, hp->ioh, o, tmp);
    145 	}
    146 }
    147 
    148 #define HWRITE1(hp, reg, val)		hwrite1(hp, reg, val)
    149 #define HWRITE2(hp, reg, val)		hwrite2(hp, reg, val)
    150 #define HWRITE4(hp, reg, val)						\
    151 	bus_space_write_4((hp)->iot, (hp)->ioh, (reg), (val))
    152 
    153 #define HCLR1(hp, reg, bits)						\
    154 	do if (bits) HWRITE1((hp), (reg), HREAD1((hp), (reg)) & ~(bits)); while (0)
    155 #define HCLR2(hp, reg, bits)						\
    156 	do if (bits) HWRITE2((hp), (reg), HREAD2((hp), (reg)) & ~(bits)); while (0)
    157 #define HCLR4(hp, reg, bits)						\
    158 	do if (bits) HWRITE4((hp), (reg), HREAD4((hp), (reg)) & ~(bits)); while (0)
    159 #define HSET1(hp, reg, bits)						\
    160 	do if (bits) HWRITE1((hp), (reg), HREAD1((hp), (reg)) | (bits)); while (0)
    161 #define HSET2(hp, reg, bits)						\
    162 	do if (bits) HWRITE2((hp), (reg), HREAD2((hp), (reg)) | (bits)); while (0)
    163 #define HSET4(hp, reg, bits)						\
    164 	do if (bits) HWRITE4((hp), (reg), HREAD4((hp), (reg)) | (bits)); while (0)
    165 
    166 static int	sdhc_host_reset(sdmmc_chipset_handle_t);
    167 static int	sdhc_host_reset1(sdmmc_chipset_handle_t);
    168 static uint32_t	sdhc_host_ocr(sdmmc_chipset_handle_t);
    169 static int	sdhc_host_maxblklen(sdmmc_chipset_handle_t);
    170 static int	sdhc_card_detect(sdmmc_chipset_handle_t);
    171 static int	sdhc_write_protect(sdmmc_chipset_handle_t);
    172 static int	sdhc_bus_power(sdmmc_chipset_handle_t, uint32_t);
    173 static int	sdhc_bus_clock(sdmmc_chipset_handle_t, int);
    174 static int	sdhc_bus_width(sdmmc_chipset_handle_t, int);
    175 static int	sdhc_bus_rod(sdmmc_chipset_handle_t, int);
    176 static void	sdhc_card_enable_intr(sdmmc_chipset_handle_t, int);
    177 static void	sdhc_card_intr_ack(sdmmc_chipset_handle_t);
    178 static void	sdhc_exec_command(sdmmc_chipset_handle_t,
    179 		    struct sdmmc_command *);
    180 static int	sdhc_start_command(struct sdhc_host *, struct sdmmc_command *);
    181 static int	sdhc_wait_state(struct sdhc_host *, uint32_t, uint32_t);
    182 static int	sdhc_soft_reset(struct sdhc_host *, int);
    183 static int	sdhc_wait_intr(struct sdhc_host *, int, int);
    184 static void	sdhc_transfer_data(struct sdhc_host *, struct sdmmc_command *);
    185 static int	sdhc_transfer_data_dma(struct sdhc_host *, struct sdmmc_command *);
    186 static int	sdhc_transfer_data_pio(struct sdhc_host *, struct sdmmc_command *);
    187 static void	sdhc_read_data_pio(struct sdhc_host *, uint8_t *, u_int);
    188 static void	sdhc_write_data_pio(struct sdhc_host *, uint8_t *, u_int);
    189 static void	esdhc_read_data_pio(struct sdhc_host *, uint8_t *, u_int);
    190 static void	esdhc_write_data_pio(struct sdhc_host *, uint8_t *, u_int);
    191 
    192 
    193 static struct sdmmc_chip_functions sdhc_functions = {
    194 	/* host controller reset */
    195 	sdhc_host_reset,
    196 
    197 	/* host controller capabilities */
    198 	sdhc_host_ocr,
    199 	sdhc_host_maxblklen,
    200 
    201 	/* card detection */
    202 	sdhc_card_detect,
    203 
    204 	/* write protect */
    205 	sdhc_write_protect,
    206 
    207 	/* bus power, clock frequency and width */
    208 	sdhc_bus_power,
    209 	sdhc_bus_clock,
    210 	sdhc_bus_width,
    211 	sdhc_bus_rod,
    212 
    213 	/* command execution */
    214 	sdhc_exec_command,
    215 
    216 	/* card interrupt */
    217 	sdhc_card_enable_intr,
    218 	sdhc_card_intr_ack
    219 };
    220 
    221 static int
    222 sdhc_cfprint(void *aux, const char *pnp)
    223 {
    224 	const struct sdmmcbus_attach_args * const saa = aux;
    225 	const struct sdhc_host * const hp = saa->saa_sch;
    226 
    227 	if (pnp) {
    228 		aprint_normal("sdmmc at %s", pnp);
    229 	}
    230 	for (size_t host = 0; host < hp->sc->sc_nhosts; host++) {
    231 		if (hp->sc->sc_host[host] == hp) {
    232 			aprint_normal(" slot %zu", host);
    233 		}
    234 	}
    235 
    236 	return UNCONF;
    237 }
    238 
    239 /*
    240  * Called by attachment driver.  For each SD card slot there is one SD
    241  * host controller standard register set. (1.3)
    242  */
    243 int
    244 sdhc_host_found(struct sdhc_softc *sc, bus_space_tag_t iot,
    245     bus_space_handle_t ioh, bus_size_t iosize)
    246 {
    247 	struct sdmmcbus_attach_args saa;
    248 	struct sdhc_host *hp;
    249 	uint32_t caps;
    250 	uint16_t sdhcver;
    251 
    252 	/* Allocate one more host structure. */
    253 	hp = malloc(sizeof(struct sdhc_host), M_DEVBUF, M_WAITOK|M_ZERO);
    254 	if (hp == NULL) {
    255 		aprint_error_dev(sc->sc_dev,
    256 		    "couldn't alloc memory (sdhc host)\n");
    257 		goto err1;
    258 	}
    259 	sc->sc_host[sc->sc_nhosts++] = hp;
    260 
    261 	/* Fill in the new host structure. */
    262 	hp->sc = sc;
    263 	hp->iot = iot;
    264 	hp->ioh = ioh;
    265 	hp->ios = iosize;
    266 	hp->dmat = sc->sc_dmat;
    267 
    268 	mutex_init(&hp->host_mtx, MUTEX_DEFAULT, IPL_SDMMC);
    269 	mutex_init(&hp->intr_mtx, MUTEX_DEFAULT, IPL_SDMMC);
    270 	cv_init(&hp->intr_cv, "sdhcintr");
    271 
    272 	if (ISSET(hp->sc->sc_flags, SDHC_FLAG_ENHANCED)) {
    273 		sdhcver = HREAD4(hp, SDHC_ESDHC_HOST_CTL_VERSION);
    274 	} else {
    275 		sdhcver = HREAD2(hp, SDHC_HOST_CTL_VERSION);
    276 	}
    277 	aprint_normal_dev(sc->sc_dev, "SD Host Specification ");
    278 	hp->specver = SDHC_SPEC_VERSION(sdhcver);
    279 	switch (SDHC_SPEC_VERSION(sdhcver)) {
    280 	case SDHC_SPEC_VERS_100:
    281 		aprint_normal("1.0");
    282 		break;
    283 
    284 	case SDHC_SPEC_VERS_200:
    285 		aprint_normal("2.0");
    286 		break;
    287 
    288 	case SDHC_SPEC_VERS_300:
    289 		aprint_normal("3.0");
    290 		break;
    291 
    292 	case SDHC_SPEC_VERS_400:
    293 		aprint_normal("4.0");
    294 		break;
    295 
    296 	default:
    297 		aprint_normal("unknown version(0x%x)",
    298 		    SDHC_SPEC_VERSION(sdhcver));
    299 		break;
    300 	}
    301 	aprint_normal(", rev.%u\n", SDHC_VENDOR_VERSION(sdhcver));
    302 
    303 	/*
    304 	 * Reset the host controller and enable interrupts.
    305 	 */
    306 	(void)sdhc_host_reset(hp);
    307 
    308 	/* Determine host capabilities. */
    309 	if (ISSET(sc->sc_flags, SDHC_FLAG_HOSTCAPS)) {
    310 		caps = sc->sc_caps;
    311 	} else {
    312 		mutex_enter(&hp->host_mtx);
    313 		caps = HREAD4(hp, SDHC_CAPABILITIES);
    314 		mutex_exit(&hp->host_mtx);
    315 	}
    316 
    317 	/*
    318 	 * Use DMA if the host system and the controller support it.
    319 	 * Suports integrated or external DMA egine, with or without
    320 	 * SDHC_DMA_ENABLE in the command.
    321 	 */
    322 	if (ISSET(sc->sc_flags, SDHC_FLAG_FORCE_DMA) ||
    323 	    (ISSET(sc->sc_flags, SDHC_FLAG_USE_DMA &&
    324 	     ISSET(caps, SDHC_DMA_SUPPORT)))) {
    325 		SET(hp->flags, SHF_USE_DMA);
    326 		if (!ISSET(sc->sc_flags, SDHC_FLAG_EXTERNAL_DMA) ||
    327 		    ISSET(sc->sc_flags, SDHC_FLAG_EXTDMA_DMAEN))
    328 			SET(hp->flags, SHF_MODE_DMAEN);
    329 
    330 		aprint_normal_dev(sc->sc_dev, "using DMA transfer\n");
    331 	}
    332 
    333 	/*
    334 	 * Determine the base clock frequency. (2.2.24)
    335 	 */
    336 	if (hp->specver >= SDHC_SPEC_VERS_300) {
    337 		hp->clkbase = SDHC_BASE_V3_FREQ_KHZ(caps);
    338 	} else {
    339 		hp->clkbase = SDHC_BASE_FREQ_KHZ(caps);
    340 	}
    341 	if (hp->clkbase == 0 ||
    342 	    ISSET(sc->sc_flags, SDHC_FLAG_NO_CLKBASE)) {
    343 		if (sc->sc_clkbase == 0) {
    344 			/* The attachment driver must tell us. */
    345 			aprint_error_dev(sc->sc_dev,
    346 			    "unknown base clock frequency\n");
    347 			goto err;
    348 		}
    349 		hp->clkbase = sc->sc_clkbase;
    350 	}
    351 	if (hp->clkbase < 10000 || hp->clkbase > 10000 * 256) {
    352 		/* SDHC 1.0 supports only 10-63 MHz. */
    353 		aprint_error_dev(sc->sc_dev,
    354 		    "base clock frequency out of range: %u MHz\n",
    355 		    hp->clkbase / 1000);
    356 		goto err;
    357 	}
    358 	DPRINTF(1,("%s: base clock frequency %u MHz\n",
    359 	    device_xname(sc->sc_dev), hp->clkbase / 1000));
    360 
    361 	/*
    362 	 * XXX Set the data timeout counter value according to
    363 	 * capabilities. (2.2.15)
    364 	 */
    365 	HWRITE1(hp, SDHC_TIMEOUT_CTL, SDHC_TIMEOUT_MAX);
    366 #if 1
    367 	if (ISSET(hp->sc->sc_flags, SDHC_FLAG_ENHANCED))
    368 		HWRITE4(hp, SDHC_NINTR_STATUS, SDHC_CMD_TIMEOUT_ERROR << 16);
    369 #endif
    370 
    371 	/*
    372 	 * Determine SD bus voltage levels supported by the controller.
    373 	 */
    374 	if (ISSET(caps, SDHC_VOLTAGE_SUPP_1_8V) &&
    375 	    (hp->specver < SDHC_SPEC_VERS_300 ||
    376 	     ISSET(caps, SDHC_EMBEDDED_SLOT))) {
    377 		SET(hp->ocr, MMC_OCR_1_7V_1_8V | MMC_OCR_1_8V_1_9V);
    378 	}
    379 	if (ISSET(caps, SDHC_VOLTAGE_SUPP_3_0V)) {
    380 		SET(hp->ocr, MMC_OCR_2_9V_3_0V | MMC_OCR_3_0V_3_1V);
    381 	}
    382 	if (ISSET(caps, SDHC_VOLTAGE_SUPP_3_3V)) {
    383 		SET(hp->ocr, MMC_OCR_3_2V_3_3V | MMC_OCR_3_3V_3_4V);
    384 	}
    385 
    386 	/*
    387 	 * Determine the maximum block length supported by the host
    388 	 * controller. (2.2.24)
    389 	 */
    390 	switch((caps >> SDHC_MAX_BLK_LEN_SHIFT) & SDHC_MAX_BLK_LEN_MASK) {
    391 	case SDHC_MAX_BLK_LEN_512:
    392 		hp->maxblklen = 512;
    393 		break;
    394 
    395 	case SDHC_MAX_BLK_LEN_1024:
    396 		hp->maxblklen = 1024;
    397 		break;
    398 
    399 	case SDHC_MAX_BLK_LEN_2048:
    400 		hp->maxblklen = 2048;
    401 		break;
    402 
    403 	case SDHC_MAX_BLK_LEN_4096:
    404 		hp->maxblklen = 4096;
    405 		break;
    406 
    407 	default:
    408 		aprint_error_dev(sc->sc_dev, "max block length unknown\n");
    409 		goto err;
    410 	}
    411 	DPRINTF(1, ("%s: max block length %u byte%s\n",
    412 	    device_xname(sc->sc_dev), hp->maxblklen,
    413 	    hp->maxblklen > 1 ? "s" : ""));
    414 
    415 	/*
    416 	 * Attach the generic SD/MMC bus driver.  (The bus driver must
    417 	 * not invoke any chipset functions before it is attached.)
    418 	 */
    419 	memset(&saa, 0, sizeof(saa));
    420 	saa.saa_busname = "sdmmc";
    421 	saa.saa_sct = &sdhc_functions;
    422 	saa.saa_sch = hp;
    423 	saa.saa_dmat = hp->dmat;
    424 	saa.saa_clkmax = hp->clkbase;
    425 	if (ISSET(sc->sc_flags, SDHC_FLAG_HAVE_CGM))
    426 		saa.saa_clkmin = hp->clkbase / 256 / 2046;
    427 	else if (ISSET(sc->sc_flags, SDHC_FLAG_HAVE_DVS))
    428 		saa.saa_clkmin = hp->clkbase / 256 / 16;
    429 	else if (hp->sc->sc_clkmsk != 0)
    430 		saa.saa_clkmin = hp->clkbase / (hp->sc->sc_clkmsk >>
    431 		    (ffs(hp->sc->sc_clkmsk) - 1));
    432 	else if (hp->specver >= SDHC_SPEC_VERS_300)
    433 		saa.saa_clkmin = hp->clkbase / 0x3ff;
    434 	else
    435 		saa.saa_clkmin = hp->clkbase / 256;
    436 	saa.saa_caps = SMC_CAPS_4BIT_MODE|SMC_CAPS_AUTO_STOP;
    437 	if (ISSET(sc->sc_flags, SDHC_FLAG_8BIT_MODE))
    438 		saa.saa_caps |= SMC_CAPS_8BIT_MODE;
    439 	if (ISSET(caps, SDHC_HIGH_SPEED_SUPP))
    440 		saa.saa_caps |= SMC_CAPS_SD_HIGHSPEED;
    441 	if (ISSET(hp->flags, SHF_USE_DMA)) {
    442 		saa.saa_caps |= SMC_CAPS_DMA;
    443 		if (!ISSET(hp->sc->sc_flags, SDHC_FLAG_ENHANCED))
    444 			saa.saa_caps |= SMC_CAPS_MULTI_SEG_DMA;
    445 	}
    446 	if (ISSET(sc->sc_flags, SDHC_FLAG_SINGLE_ONLY))
    447 		saa.saa_caps |= SMC_CAPS_SINGLE_ONLY;
    448 	hp->sdmmc = config_found(sc->sc_dev, &saa, sdhc_cfprint);
    449 
    450 	return 0;
    451 
    452 err:
    453 	cv_destroy(&hp->intr_cv);
    454 	mutex_destroy(&hp->intr_mtx);
    455 	mutex_destroy(&hp->host_mtx);
    456 	free(hp, M_DEVBUF);
    457 	sc->sc_host[--sc->sc_nhosts] = NULL;
    458 err1:
    459 	return 1;
    460 }
    461 
    462 int
    463 sdhc_detach(struct sdhc_softc *sc, int flags)
    464 {
    465 	struct sdhc_host *hp;
    466 	int rv = 0;
    467 
    468 	for (size_t n = 0; n < sc->sc_nhosts; n++) {
    469 		hp = sc->sc_host[n];
    470 		if (hp == NULL)
    471 			continue;
    472 		if (hp->sdmmc != NULL) {
    473 			rv = config_detach(hp->sdmmc, flags);
    474 			if (rv)
    475 				break;
    476 			hp->sdmmc = NULL;
    477 		}
    478 		/* disable interrupts */
    479 		if ((flags & DETACH_FORCE) == 0) {
    480 			if (ISSET(hp->sc->sc_flags, SDHC_FLAG_32BIT_ACCESS)) {
    481 				HWRITE4(hp, SDHC_NINTR_SIGNAL_EN, 0);
    482 			} else {
    483 				HWRITE2(hp, SDHC_NINTR_SIGNAL_EN, 0);
    484 			}
    485 			sdhc_soft_reset(hp, SDHC_RESET_ALL);
    486 		}
    487 		cv_destroy(&hp->intr_cv);
    488 		mutex_destroy(&hp->intr_mtx);
    489 		mutex_destroy(&hp->host_mtx);
    490 		if (hp->ios > 0) {
    491 			bus_space_unmap(hp->iot, hp->ioh, hp->ios);
    492 			hp->ios = 0;
    493 		}
    494 		free(hp, M_DEVBUF);
    495 		sc->sc_host[n] = NULL;
    496 	}
    497 
    498 	return rv;
    499 }
    500 
    501 bool
    502 sdhc_suspend(device_t dev, const pmf_qual_t *qual)
    503 {
    504 	struct sdhc_softc *sc = device_private(dev);
    505 	struct sdhc_host *hp;
    506 	size_t i;
    507 
    508 	/* XXX poll for command completion or suspend command
    509 	 * in progress */
    510 
    511 	/* Save the host controller state. */
    512 	for (size_t n = 0; n < sc->sc_nhosts; n++) {
    513 		hp = sc->sc_host[n];
    514 		if (ISSET(sc->sc_flags, SDHC_FLAG_32BIT_ACCESS)) {
    515 			for (i = 0; i < sizeof hp->regs; i += 4) {
    516 				uint32_t v = HREAD4(hp, i);
    517 				hp->regs[i + 0] = (v >> 0);
    518 				hp->regs[i + 1] = (v >> 8);
    519 				if (i + 3 < sizeof hp->regs) {
    520 					hp->regs[i + 2] = (v >> 16);
    521 					hp->regs[i + 3] = (v >> 24);
    522 				}
    523 			}
    524 		} else {
    525 			for (i = 0; i < sizeof hp->regs; i++) {
    526 				hp->regs[i] = HREAD1(hp, i);
    527 			}
    528 		}
    529 	}
    530 	return true;
    531 }
    532 
    533 bool
    534 sdhc_resume(device_t dev, const pmf_qual_t *qual)
    535 {
    536 	struct sdhc_softc *sc = device_private(dev);
    537 	struct sdhc_host *hp;
    538 	size_t i;
    539 
    540 	/* Restore the host controller state. */
    541 	for (size_t n = 0; n < sc->sc_nhosts; n++) {
    542 		hp = sc->sc_host[n];
    543 		(void)sdhc_host_reset(hp);
    544 		if (ISSET(sc->sc_flags, SDHC_FLAG_32BIT_ACCESS)) {
    545 			for (i = 0; i < sizeof hp->regs; i += 4) {
    546 				if (i + 3 < sizeof hp->regs) {
    547 					HWRITE4(hp, i,
    548 					    (hp->regs[i + 0] << 0)
    549 					    | (hp->regs[i + 1] << 8)
    550 					    | (hp->regs[i + 2] << 16)
    551 					    | (hp->regs[i + 3] << 24));
    552 				} else {
    553 					HWRITE4(hp, i,
    554 					    (hp->regs[i + 0] << 0)
    555 					    | (hp->regs[i + 1] << 8));
    556 				}
    557 			}
    558 		} else {
    559 			for (i = 0; i < sizeof hp->regs; i++) {
    560 				HWRITE1(hp, i, hp->regs[i]);
    561 			}
    562 		}
    563 	}
    564 	return true;
    565 }
    566 
    567 bool
    568 sdhc_shutdown(device_t dev, int flags)
    569 {
    570 	struct sdhc_softc *sc = device_private(dev);
    571 	struct sdhc_host *hp;
    572 
    573 	/* XXX chip locks up if we don't disable it before reboot. */
    574 	for (size_t i = 0; i < sc->sc_nhosts; i++) {
    575 		hp = sc->sc_host[i];
    576 		(void)sdhc_host_reset(hp);
    577 	}
    578 	return true;
    579 }
    580 
    581 /*
    582  * Reset the host controller.  Called during initialization, when
    583  * cards are removed, upon resume, and during error recovery.
    584  */
    585 static int
    586 sdhc_host_reset1(sdmmc_chipset_handle_t sch)
    587 {
    588 	struct sdhc_host *hp = (struct sdhc_host *)sch;
    589 	uint32_t sdhcimask;
    590 	int error;
    591 
    592 	/* Don't lock. */
    593 
    594 	/* Disable all interrupts. */
    595 	if (ISSET(hp->sc->sc_flags, SDHC_FLAG_32BIT_ACCESS)) {
    596 		HWRITE4(hp, SDHC_NINTR_SIGNAL_EN, 0);
    597 	} else {
    598 		HWRITE2(hp, SDHC_NINTR_SIGNAL_EN, 0);
    599 	}
    600 
    601 	/*
    602 	 * Reset the entire host controller and wait up to 100ms for
    603 	 * the controller to clear the reset bit.
    604 	 */
    605 	error = sdhc_soft_reset(hp, SDHC_RESET_ALL);
    606 	if (error)
    607 		goto out;
    608 
    609 	/* Set data timeout counter value to max for now. */
    610 	HWRITE1(hp, SDHC_TIMEOUT_CTL, SDHC_TIMEOUT_MAX);
    611 #if 1
    612 	if (ISSET(hp->sc->sc_flags, SDHC_FLAG_ENHANCED))
    613 		HWRITE4(hp, SDHC_NINTR_STATUS, SDHC_CMD_TIMEOUT_ERROR << 16);
    614 #endif
    615 
    616 	/* Enable interrupts. */
    617 	mutex_enter(&hp->intr_mtx);
    618 	sdhcimask = SDHC_CARD_REMOVAL | SDHC_CARD_INSERTION |
    619 	    SDHC_BUFFER_READ_READY | SDHC_BUFFER_WRITE_READY |
    620 	    SDHC_DMA_INTERRUPT | SDHC_BLOCK_GAP_EVENT |
    621 	    SDHC_TRANSFER_COMPLETE | SDHC_COMMAND_COMPLETE;
    622 	if (ISSET(hp->sc->sc_flags, SDHC_FLAG_32BIT_ACCESS)) {
    623 		sdhcimask |= SDHC_EINTR_STATUS_MASK << 16;
    624 		HWRITE4(hp, SDHC_NINTR_STATUS_EN, sdhcimask);
    625 		sdhcimask ^=
    626 		    (SDHC_EINTR_STATUS_MASK ^ SDHC_EINTR_SIGNAL_MASK) << 16;
    627 		sdhcimask ^= SDHC_BUFFER_READ_READY ^ SDHC_BUFFER_WRITE_READY;
    628 		HWRITE4(hp, SDHC_NINTR_SIGNAL_EN, sdhcimask);
    629 	} else {
    630 		HWRITE2(hp, SDHC_NINTR_STATUS_EN, sdhcimask);
    631 		HWRITE2(hp, SDHC_EINTR_STATUS_EN, SDHC_EINTR_STATUS_MASK);
    632 		sdhcimask ^= SDHC_BUFFER_READ_READY ^ SDHC_BUFFER_WRITE_READY;
    633 		HWRITE2(hp, SDHC_NINTR_SIGNAL_EN, sdhcimask);
    634 		HWRITE2(hp, SDHC_EINTR_SIGNAL_EN, SDHC_EINTR_SIGNAL_MASK);
    635 	}
    636 	mutex_exit(&hp->intr_mtx);
    637 
    638 out:
    639 	return error;
    640 }
    641 
    642 static int
    643 sdhc_host_reset(sdmmc_chipset_handle_t sch)
    644 {
    645 	struct sdhc_host *hp = (struct sdhc_host *)sch;
    646 	int error;
    647 
    648 	mutex_enter(&hp->host_mtx);
    649 	error = sdhc_host_reset1(sch);
    650 	mutex_exit(&hp->host_mtx);
    651 
    652 	return error;
    653 }
    654 
    655 static uint32_t
    656 sdhc_host_ocr(sdmmc_chipset_handle_t sch)
    657 {
    658 	struct sdhc_host *hp = (struct sdhc_host *)sch;
    659 
    660 	return hp->ocr;
    661 }
    662 
    663 static int
    664 sdhc_host_maxblklen(sdmmc_chipset_handle_t sch)
    665 {
    666 	struct sdhc_host *hp = (struct sdhc_host *)sch;
    667 
    668 	return hp->maxblklen;
    669 }
    670 
    671 /*
    672  * Return non-zero if the card is currently inserted.
    673  */
    674 static int
    675 sdhc_card_detect(sdmmc_chipset_handle_t sch)
    676 {
    677 	struct sdhc_host *hp = (struct sdhc_host *)sch;
    678 	int r;
    679 
    680 	if (hp->sc->sc_vendor_card_detect)
    681 		return (*hp->sc->sc_vendor_card_detect)(hp->sc);
    682 
    683 	mutex_enter(&hp->host_mtx);
    684 	r = ISSET(HREAD4(hp, SDHC_PRESENT_STATE), SDHC_CARD_INSERTED);
    685 	mutex_exit(&hp->host_mtx);
    686 
    687 	return r ? 1 : 0;
    688 }
    689 
    690 /*
    691  * Return non-zero if the card is currently write-protected.
    692  */
    693 static int
    694 sdhc_write_protect(sdmmc_chipset_handle_t sch)
    695 {
    696 	struct sdhc_host *hp = (struct sdhc_host *)sch;
    697 	int r;
    698 
    699 	if (hp->sc->sc_vendor_write_protect)
    700 		return (*hp->sc->sc_vendor_write_protect)(hp->sc);
    701 
    702 	mutex_enter(&hp->host_mtx);
    703 	r = ISSET(HREAD4(hp, SDHC_PRESENT_STATE), SDHC_WRITE_PROTECT_SWITCH);
    704 	mutex_exit(&hp->host_mtx);
    705 
    706 	return r ? 0 : 1;
    707 }
    708 
    709 /*
    710  * Set or change SD bus voltage and enable or disable SD bus power.
    711  * Return zero on success.
    712  */
    713 static int
    714 sdhc_bus_power(sdmmc_chipset_handle_t sch, uint32_t ocr)
    715 {
    716 	struct sdhc_host *hp = (struct sdhc_host *)sch;
    717 	uint8_t vdd;
    718 	int error = 0;
    719 	const uint32_t pcmask =
    720 	    ~(SDHC_BUS_POWER | (SDHC_VOLTAGE_MASK << SDHC_VOLTAGE_SHIFT));
    721 
    722 	mutex_enter(&hp->host_mtx);
    723 
    724 	/*
    725 	 * Disable bus power before voltage change.
    726 	 */
    727 	if (!ISSET(hp->sc->sc_flags, SDHC_FLAG_32BIT_ACCESS)
    728 	    && !ISSET(hp->sc->sc_flags, SDHC_FLAG_NO_PWR0))
    729 		HWRITE1(hp, SDHC_POWER_CTL, 0);
    730 
    731 	/* If power is disabled, reset the host and return now. */
    732 	if (ocr == 0) {
    733 		(void)sdhc_host_reset1(hp);
    734 		goto out;
    735 	}
    736 
    737 	/*
    738 	 * Select the lowest voltage according to capabilities.
    739 	 */
    740 	ocr &= hp->ocr;
    741 	if (ISSET(ocr, MMC_OCR_1_7V_1_8V|MMC_OCR_1_8V_1_9V)) {
    742 		vdd = SDHC_VOLTAGE_1_8V;
    743 	} else if (ISSET(ocr, MMC_OCR_2_9V_3_0V|MMC_OCR_3_0V_3_1V)) {
    744 		vdd = SDHC_VOLTAGE_3_0V;
    745 	} else if (ISSET(ocr, MMC_OCR_3_2V_3_3V|MMC_OCR_3_3V_3_4V)) {
    746 		vdd = SDHC_VOLTAGE_3_3V;
    747 	} else {
    748 		/* Unsupported voltage level requested. */
    749 		error = EINVAL;
    750 		goto out;
    751 	}
    752 
    753 	if (!ISSET(hp->sc->sc_flags, SDHC_FLAG_ENHANCED)) {
    754 		/*
    755 		 * Enable bus power.  Wait at least 1 ms (or 74 clocks) plus
    756 		 * voltage ramp until power rises.
    757 		 */
    758 
    759 		if (ISSET(hp->sc->sc_flags, SDHC_FLAG_SINGLE_POWER_WRITE)) {
    760 			HWRITE1(hp, SDHC_POWER_CTL,
    761 			    (vdd << SDHC_VOLTAGE_SHIFT) | SDHC_BUS_POWER);
    762 		} else {
    763 			HWRITE1(hp, SDHC_POWER_CTL,
    764 			    HREAD1(hp, SDHC_POWER_CTL) & pcmask);
    765 			sdmmc_delay(1);
    766 			HWRITE1(hp, SDHC_POWER_CTL,
    767 			    (vdd << SDHC_VOLTAGE_SHIFT));
    768 			sdmmc_delay(1);
    769 			HSET1(hp, SDHC_POWER_CTL, SDHC_BUS_POWER);
    770 			sdmmc_delay(10000);
    771 		}
    772 
    773 		/*
    774 		 * The host system may not power the bus due to battery low,
    775 		 * etc.  In that case, the host controller should clear the
    776 		 * bus power bit.
    777 		 */
    778 		if (!ISSET(HREAD1(hp, SDHC_POWER_CTL), SDHC_BUS_POWER)) {
    779 			error = ENXIO;
    780 			goto out;
    781 		}
    782 	}
    783 
    784 out:
    785 	mutex_exit(&hp->host_mtx);
    786 
    787 	return error;
    788 }
    789 
    790 /*
    791  * Return the smallest possible base clock frequency divisor value
    792  * for the CLOCK_CTL register to produce `freq' (KHz).
    793  */
    794 static bool
    795 sdhc_clock_divisor(struct sdhc_host *hp, u_int freq, u_int *divp)
    796 {
    797 	u_int div;
    798 
    799 	if (ISSET(hp->sc->sc_flags, SDHC_FLAG_HAVE_CGM)) {
    800 		for (div = hp->clkbase / freq; div <= 0x3ff; div++) {
    801 			if ((hp->clkbase / div) <= freq) {
    802 				*divp = SDHC_SDCLK_CGM
    803 				    | ((div & 0x300) << SDHC_SDCLK_XDIV_SHIFT)
    804 				    | ((div & 0x0ff) << SDHC_SDCLK_DIV_SHIFT);
    805 				//freq = hp->clkbase / div;
    806 				return true;
    807 			}
    808 		}
    809 		/* No divisor found. */
    810 		return false;
    811 	}
    812 	if (ISSET(hp->sc->sc_flags, SDHC_FLAG_HAVE_DVS)) {
    813 		u_int dvs = (hp->clkbase + freq - 1) / freq;
    814 		u_int roundup = dvs & 1;
    815 		for (dvs >>= 1, div = 1; div <= 256; div <<= 1, dvs >>= 1) {
    816 			if (dvs + roundup <= 16) {
    817 				dvs += roundup - 1;
    818 				*divp = (div << SDHC_SDCLK_DIV_SHIFT)
    819 				    |   (dvs << SDHC_SDCLK_DVS_SHIFT);
    820 				DPRINTF(2,
    821 				    ("%s: divisor for freq %u is %u * %u\n",
    822 				    HDEVNAME(hp), freq, div * 2, dvs + 1));
    823 				//freq = hp->clkbase / (div * 2) * (dvs + 1);
    824 				return true;
    825 			}
    826 			/*
    827 			 * If we drop bits, we need to round up the divisor.
    828 			 */
    829 			roundup |= dvs & 1;
    830 		}
    831 		/* No divisor found. */
    832 		return false;
    833 	}
    834 	if (hp->sc->sc_clkmsk != 0) {
    835 		div = howmany(hp->clkbase, freq);
    836 		if (div > (hp->sc->sc_clkmsk >> (ffs(hp->sc->sc_clkmsk) - 1)))
    837 			return false;
    838 		*divp = div << (ffs(hp->sc->sc_clkmsk) - 1);
    839 		//freq = hp->clkbase / div;
    840 		return true;
    841 	}
    842 	if (hp->specver >= SDHC_SPEC_VERS_300) {
    843 		div = howmany(hp->clkbase, freq);
    844 		div = div > 1 ? howmany(div, 2) : 0;
    845 		if (div > 0x3ff)
    846 			return false;
    847 		*divp = (((div >> 8) & SDHC_SDCLK_XDIV_MASK)
    848 			 << SDHC_SDCLK_XDIV_SHIFT) |
    849 			(((div >> 0) & SDHC_SDCLK_DIV_MASK)
    850 			 << SDHC_SDCLK_DIV_SHIFT);
    851 		//freq = hp->clkbase / div;
    852 		return true;
    853 	} else {
    854 		for (div = 1; div <= 256; div *= 2) {
    855 			if ((hp->clkbase / div) <= freq) {
    856 				*divp = (div / 2) << SDHC_SDCLK_DIV_SHIFT;
    857 				//freq = hp->clkbase / div;
    858 				return true;
    859 			}
    860 		}
    861 		/* No divisor found. */
    862 		return false;
    863 	}
    864 	/* No divisor found. */
    865 	return false;
    866 }
    867 
    868 /*
    869  * Set or change SDCLK frequency or disable the SD clock.
    870  * Return zero on success.
    871  */
    872 static int
    873 sdhc_bus_clock(sdmmc_chipset_handle_t sch, int freq)
    874 {
    875 	struct sdhc_host *hp = (struct sdhc_host *)sch;
    876 	u_int div;
    877 	u_int timo;
    878 	int16_t reg;
    879 	int error = 0;
    880 #ifdef DIAGNOSTIC
    881 	bool present;
    882 
    883 	mutex_enter(&hp->host_mtx);
    884 	present = ISSET(HREAD4(hp, SDHC_PRESENT_STATE), SDHC_CMD_INHIBIT_MASK);
    885 	mutex_exit(&hp->host_mtx);
    886 
    887 	/* Must not stop the clock if commands are in progress. */
    888 	if (present && sdhc_card_detect(hp)) {
    889 		aprint_normal_dev(hp->sc->sc_dev,
    890 		    "%s: command in progress\n", __func__);
    891 	}
    892 #endif
    893 
    894 	mutex_enter(&hp->host_mtx);
    895 
    896 	if (hp->sc->sc_vendor_bus_clock) {
    897 		error = (*hp->sc->sc_vendor_bus_clock)(hp->sc, freq);
    898 		if (error != 0)
    899 			goto out;
    900 	}
    901 
    902 	/*
    903 	 * Stop SD clock before changing the frequency.
    904 	 */
    905 	if (ISSET(hp->sc->sc_flags, SDHC_FLAG_ENHANCED)) {
    906 		HCLR4(hp, SDHC_CLOCK_CTL, 0xfff8);
    907 		if (freq == SDMMC_SDCLK_OFF) {
    908 			HSET4(hp, SDHC_CLOCK_CTL, 0x80f0);
    909 			goto out;
    910 		}
    911 	} else {
    912 		HCLR2(hp, SDHC_CLOCK_CTL, SDHC_SDCLK_ENABLE);
    913 		if (freq == SDMMC_SDCLK_OFF)
    914 			goto out;
    915 	}
    916 
    917 	/*
    918 	 * Set the minimum base clock frequency divisor.
    919 	 */
    920 	if (!sdhc_clock_divisor(hp, freq, &div)) {
    921 		/* Invalid base clock frequency or `freq' value. */
    922 		error = EINVAL;
    923 		goto out;
    924 	}
    925 	if (ISSET(hp->sc->sc_flags, SDHC_FLAG_ENHANCED)) {
    926 		HWRITE4(hp, SDHC_CLOCK_CTL,
    927 		    div | (SDHC_TIMEOUT_MAX << 16));
    928 	} else {
    929 		reg = HREAD2(hp, SDHC_CLOCK_CTL);
    930 		reg &= (SDHC_INTCLK_STABLE | SDHC_INTCLK_ENABLE);
    931 		HWRITE2(hp, SDHC_CLOCK_CTL, reg | div);
    932 	}
    933 
    934 	/*
    935 	 * Start internal clock.  Wait 10ms for stabilization.
    936 	 */
    937 	if (ISSET(hp->sc->sc_flags, SDHC_FLAG_ENHANCED)) {
    938 		sdmmc_delay(10000);
    939 		HSET4(hp, SDHC_CLOCK_CTL,
    940 		    8 | SDHC_INTCLK_ENABLE | SDHC_INTCLK_STABLE);
    941 	} else {
    942 		HSET2(hp, SDHC_CLOCK_CTL, SDHC_INTCLK_ENABLE);
    943 		for (timo = 1000; timo > 0; timo--) {
    944 			if (ISSET(HREAD2(hp, SDHC_CLOCK_CTL),
    945 			    SDHC_INTCLK_STABLE))
    946 				break;
    947 			sdmmc_delay(10);
    948 		}
    949 		if (timo == 0) {
    950 			error = ETIMEDOUT;
    951 			goto out;
    952 		}
    953 	}
    954 
    955 	if (ISSET(hp->sc->sc_flags, SDHC_FLAG_ENHANCED)) {
    956 		HSET1(hp, SDHC_SOFTWARE_RESET, SDHC_INIT_ACTIVE);
    957 		/*
    958 		 * Sending 80 clocks at 400kHz takes 200us.
    959 		 * So delay for that time + slop and then
    960 		 * check a few times for completion.
    961 		 */
    962 		sdmmc_delay(210);
    963 		for (timo = 10; timo > 0; timo--) {
    964 			if (!ISSET(HREAD1(hp, SDHC_SOFTWARE_RESET),
    965 			    SDHC_INIT_ACTIVE))
    966 				break;
    967 			sdmmc_delay(10);
    968 		}
    969 		DPRINTF(2,("%s: %u init spins\n", __func__, 10 - timo));
    970 
    971 		/*
    972 		 * Enable SD clock.
    973 		 */
    974 		HSET4(hp, SDHC_CLOCK_CTL, SDHC_SDCLK_ENABLE);
    975 	} else {
    976 		/*
    977 		 * Enable SD clock.
    978 		 */
    979 		HSET2(hp, SDHC_CLOCK_CTL, SDHC_SDCLK_ENABLE);
    980 
    981 		if (freq > 25000 &&
    982 		    !ISSET(hp->sc->sc_flags, SDHC_FLAG_NO_HS_BIT))
    983 			HSET1(hp, SDHC_HOST_CTL, SDHC_HIGH_SPEED);
    984 		else
    985 			HCLR1(hp, SDHC_HOST_CTL, SDHC_HIGH_SPEED);
    986 	}
    987 
    988 out:
    989 	mutex_exit(&hp->host_mtx);
    990 
    991 	return error;
    992 }
    993 
    994 static int
    995 sdhc_bus_width(sdmmc_chipset_handle_t sch, int width)
    996 {
    997 	struct sdhc_host *hp = (struct sdhc_host *)sch;
    998 	int reg;
    999 
   1000 	switch (width) {
   1001 	case 1:
   1002 	case 4:
   1003 		break;
   1004 
   1005 	case 8:
   1006 		if (ISSET(hp->sc->sc_flags, SDHC_FLAG_8BIT_MODE))
   1007 			break;
   1008 		/* FALLTHROUGH */
   1009 	default:
   1010 		DPRINTF(0,("%s: unsupported bus width (%d)\n",
   1011 		    HDEVNAME(hp), width));
   1012 		return 1;
   1013 	}
   1014 
   1015 	mutex_enter(&hp->host_mtx);
   1016 	reg = HREAD1(hp, SDHC_HOST_CTL);
   1017 	if (ISSET(hp->sc->sc_flags, SDHC_FLAG_ENHANCED)) {
   1018 		reg &= ~(SDHC_4BIT_MODE|SDHC_ESDHC_8BIT_MODE);
   1019 		if (width == 4)
   1020 			reg |= SDHC_4BIT_MODE;
   1021 		else if (width == 8)
   1022 			reg |= SDHC_ESDHC_8BIT_MODE;
   1023 	} else {
   1024 		reg &= ~SDHC_4BIT_MODE;
   1025 		if (width == 4)
   1026 			reg |= SDHC_4BIT_MODE;
   1027 	}
   1028 	HWRITE1(hp, SDHC_HOST_CTL, reg);
   1029 	mutex_exit(&hp->host_mtx);
   1030 
   1031 	return 0;
   1032 }
   1033 
   1034 static int
   1035 sdhc_bus_rod(sdmmc_chipset_handle_t sch, int on)
   1036 {
   1037 	struct sdhc_host *hp = (struct sdhc_host *)sch;
   1038 
   1039 	if (hp->sc->sc_vendor_rod)
   1040 		return (*hp->sc->sc_vendor_rod)(hp->sc, on);
   1041 
   1042 	return 0;
   1043 }
   1044 
   1045 static void
   1046 sdhc_card_enable_intr(sdmmc_chipset_handle_t sch, int enable)
   1047 {
   1048 	struct sdhc_host *hp = (struct sdhc_host *)sch;
   1049 
   1050 	if (!ISSET(hp->sc->sc_flags, SDHC_FLAG_ENHANCED)) {
   1051 		mutex_enter(&hp->intr_mtx);
   1052 		if (enable) {
   1053 			HSET2(hp, SDHC_NINTR_STATUS_EN, SDHC_CARD_INTERRUPT);
   1054 			HSET2(hp, SDHC_NINTR_SIGNAL_EN, SDHC_CARD_INTERRUPT);
   1055 		} else {
   1056 			HCLR2(hp, SDHC_NINTR_SIGNAL_EN, SDHC_CARD_INTERRUPT);
   1057 			HCLR2(hp, SDHC_NINTR_STATUS_EN, SDHC_CARD_INTERRUPT);
   1058 		}
   1059 		mutex_exit(&hp->intr_mtx);
   1060 	}
   1061 }
   1062 
   1063 static void
   1064 sdhc_card_intr_ack(sdmmc_chipset_handle_t sch)
   1065 {
   1066 	struct sdhc_host *hp = (struct sdhc_host *)sch;
   1067 
   1068 	if (!ISSET(hp->sc->sc_flags, SDHC_FLAG_ENHANCED)) {
   1069 		mutex_enter(&hp->intr_mtx);
   1070 		HSET2(hp, SDHC_NINTR_STATUS_EN, SDHC_CARD_INTERRUPT);
   1071 		mutex_exit(&hp->intr_mtx);
   1072 	}
   1073 }
   1074 
   1075 static int
   1076 sdhc_wait_state(struct sdhc_host *hp, uint32_t mask, uint32_t value)
   1077 {
   1078 	uint32_t state;
   1079 	int timeout;
   1080 
   1081 	for (timeout = 10; timeout > 0; timeout--) {
   1082 		if (((state = HREAD4(hp, SDHC_PRESENT_STATE)) & mask) == value)
   1083 			return 0;
   1084 		sdmmc_delay(10000);
   1085 	}
   1086 	DPRINTF(0,("%s: timeout waiting for %x (state=%x)\n", HDEVNAME(hp),
   1087 	    value, state));
   1088 	return ETIMEDOUT;
   1089 }
   1090 
   1091 static void
   1092 sdhc_exec_command(sdmmc_chipset_handle_t sch, struct sdmmc_command *cmd)
   1093 {
   1094 	struct sdhc_host *hp = (struct sdhc_host *)sch;
   1095 	int error;
   1096 
   1097 	if (cmd->c_data && ISSET(hp->sc->sc_flags, SDHC_FLAG_ENHANCED)) {
   1098 		const uint16_t ready = SDHC_BUFFER_READ_READY | SDHC_BUFFER_WRITE_READY;
   1099 		mutex_enter(&hp->intr_mtx);
   1100 		if (ISSET(hp->flags, SHF_USE_DMA)) {
   1101 			HCLR2(hp, SDHC_NINTR_SIGNAL_EN, ready);
   1102 			HCLR2(hp, SDHC_NINTR_STATUS_EN, ready);
   1103 		} else {
   1104 			HSET2(hp, SDHC_NINTR_SIGNAL_EN, ready);
   1105 			HSET2(hp, SDHC_NINTR_STATUS_EN, ready);
   1106 		}
   1107 		mutex_exit(&hp->intr_mtx);
   1108 	}
   1109 
   1110 	/*
   1111 	 * Start the MMC command, or mark `cmd' as failed and return.
   1112 	 */
   1113 	error = sdhc_start_command(hp, cmd);
   1114 	if (error) {
   1115 		cmd->c_error = error;
   1116 		goto out;
   1117 	}
   1118 
   1119 	/*
   1120 	 * Wait until the command phase is done, or until the command
   1121 	 * is marked done for any other reason.
   1122 	 */
   1123 	if (!sdhc_wait_intr(hp, SDHC_COMMAND_COMPLETE, SDHC_COMMAND_TIMEOUT)) {
   1124 		cmd->c_error = ETIMEDOUT;
   1125 		goto out;
   1126 	}
   1127 
   1128 	/*
   1129 	 * The host controller removes bits [0:7] from the response
   1130 	 * data (CRC) and we pass the data up unchanged to the bus
   1131 	 * driver (without padding).
   1132 	 */
   1133 	mutex_enter(&hp->host_mtx);
   1134 	if (cmd->c_error == 0 && ISSET(cmd->c_flags, SCF_RSP_PRESENT)) {
   1135 		cmd->c_resp[0] = HREAD4(hp, SDHC_RESPONSE + 0);
   1136 		if (ISSET(cmd->c_flags, SCF_RSP_136)) {
   1137 			cmd->c_resp[1] = HREAD4(hp, SDHC_RESPONSE + 4);
   1138 			cmd->c_resp[2] = HREAD4(hp, SDHC_RESPONSE + 8);
   1139 			cmd->c_resp[3] = HREAD4(hp, SDHC_RESPONSE + 12);
   1140 			if (ISSET(hp->sc->sc_flags, SDHC_FLAG_RSP136_CRC)) {
   1141 				cmd->c_resp[0] = (cmd->c_resp[0] >> 8) |
   1142 				    (cmd->c_resp[1] << 24);
   1143 				cmd->c_resp[1] = (cmd->c_resp[1] >> 8) |
   1144 				    (cmd->c_resp[2] << 24);
   1145 				cmd->c_resp[2] = (cmd->c_resp[2] >> 8) |
   1146 				    (cmd->c_resp[3] << 24);
   1147 				cmd->c_resp[3] = (cmd->c_resp[3] >> 8);
   1148 			}
   1149 		}
   1150 	}
   1151 	mutex_exit(&hp->host_mtx);
   1152 	DPRINTF(1,("%s: resp = %08x\n", HDEVNAME(hp), cmd->c_resp[0]));
   1153 
   1154 	/*
   1155 	 * If the command has data to transfer in any direction,
   1156 	 * execute the transfer now.
   1157 	 */
   1158 	if (cmd->c_error == 0 && cmd->c_data != NULL)
   1159 		sdhc_transfer_data(hp, cmd);
   1160 	else if (ISSET(cmd->c_flags, SCF_RSP_BSY)) {
   1161 		if (!sdhc_wait_intr(hp, SDHC_TRANSFER_COMPLETE, hz * 10)) {
   1162 			cmd->c_error = ETIMEDOUT;
   1163 			goto out;
   1164 		}
   1165 	}
   1166 
   1167 out:
   1168 	if (!ISSET(hp->sc->sc_flags, SDHC_FLAG_ENHANCED)
   1169 	    && !ISSET(hp->sc->sc_flags, SDHC_FLAG_NO_LED_ON)) {
   1170 		mutex_enter(&hp->host_mtx);
   1171 		/* Turn off the LED. */
   1172 		HCLR1(hp, SDHC_HOST_CTL, SDHC_LED_ON);
   1173 		mutex_exit(&hp->host_mtx);
   1174 	}
   1175 	SET(cmd->c_flags, SCF_ITSDONE);
   1176 
   1177 	DPRINTF(1,("%s: cmd %d %s (flags=%08x error=%d)\n", HDEVNAME(hp),
   1178 	    cmd->c_opcode, (cmd->c_error == 0) ? "done" : "abort",
   1179 	    cmd->c_flags, cmd->c_error));
   1180 }
   1181 
   1182 static int
   1183 sdhc_start_command(struct sdhc_host *hp, struct sdmmc_command *cmd)
   1184 {
   1185 	struct sdhc_softc * const sc = hp->sc;
   1186 	uint16_t blksize = 0;
   1187 	uint16_t blkcount = 0;
   1188 	uint16_t mode;
   1189 	uint16_t command;
   1190 	int error;
   1191 
   1192 	DPRINTF(1,("%s: start cmd %d arg=%08x data=%p dlen=%d flags=%08x, status=%#x\n",
   1193 	    HDEVNAME(hp), cmd->c_opcode, cmd->c_arg, cmd->c_data,
   1194 	    cmd->c_datalen, cmd->c_flags, HREAD4(hp, SDHC_NINTR_STATUS)));
   1195 
   1196 	/*
   1197 	 * The maximum block length for commands should be the minimum
   1198 	 * of the host buffer size and the card buffer size. (1.7.2)
   1199 	 */
   1200 
   1201 	/* Fragment the data into proper blocks. */
   1202 	if (cmd->c_datalen > 0) {
   1203 		blksize = MIN(cmd->c_datalen, cmd->c_blklen);
   1204 		blkcount = cmd->c_datalen / blksize;
   1205 		if (cmd->c_datalen % blksize > 0) {
   1206 			/* XXX: Split this command. (1.7.4) */
   1207 			aprint_error_dev(sc->sc_dev,
   1208 			    "data not a multiple of %u bytes\n", blksize);
   1209 			return EINVAL;
   1210 		}
   1211 	}
   1212 
   1213 	/* Check limit imposed by 9-bit block count. (1.7.2) */
   1214 	if (blkcount > SDHC_BLOCK_COUNT_MAX) {
   1215 		aprint_error_dev(sc->sc_dev, "too much data\n");
   1216 		return EINVAL;
   1217 	}
   1218 
   1219 	/* Prepare transfer mode register value. (2.2.5) */
   1220 	mode = SDHC_BLOCK_COUNT_ENABLE;
   1221 	if (ISSET(cmd->c_flags, SCF_CMD_READ))
   1222 		mode |= SDHC_READ_MODE;
   1223 	if (blkcount > 1) {
   1224 		mode |= SDHC_MULTI_BLOCK_MODE;
   1225 		/* XXX only for memory commands? */
   1226 		mode |= SDHC_AUTO_CMD12_ENABLE;
   1227 	}
   1228 	if (cmd->c_dmamap != NULL && cmd->c_datalen > 0 &&
   1229 	    ISSET(hp->flags,  SHF_MODE_DMAEN)) {
   1230 		mode |= SDHC_DMA_ENABLE;
   1231 	}
   1232 
   1233 	/*
   1234 	 * Prepare command register value. (2.2.6)
   1235 	 */
   1236 	command = (cmd->c_opcode & SDHC_COMMAND_INDEX_MASK) << SDHC_COMMAND_INDEX_SHIFT;
   1237 
   1238 	if (ISSET(cmd->c_flags, SCF_RSP_CRC))
   1239 		command |= SDHC_CRC_CHECK_ENABLE;
   1240 	if (ISSET(cmd->c_flags, SCF_RSP_IDX))
   1241 		command |= SDHC_INDEX_CHECK_ENABLE;
   1242 	if (cmd->c_data != NULL)
   1243 		command |= SDHC_DATA_PRESENT_SELECT;
   1244 
   1245 	if (!ISSET(cmd->c_flags, SCF_RSP_PRESENT))
   1246 		command |= SDHC_NO_RESPONSE;
   1247 	else if (ISSET(cmd->c_flags, SCF_RSP_136))
   1248 		command |= SDHC_RESP_LEN_136;
   1249 	else if (ISSET(cmd->c_flags, SCF_RSP_BSY))
   1250 		command |= SDHC_RESP_LEN_48_CHK_BUSY;
   1251 	else
   1252 		command |= SDHC_RESP_LEN_48;
   1253 
   1254 	/* Wait until command and data inhibit bits are clear. (1.5) */
   1255 	error = sdhc_wait_state(hp, SDHC_CMD_INHIBIT_MASK, 0);
   1256 	if (error)
   1257 		return error;
   1258 
   1259 	DPRINTF(1,("%s: writing cmd: blksize=%d blkcnt=%d mode=%04x cmd=%04x\n",
   1260 	    HDEVNAME(hp), blksize, blkcount, mode, command));
   1261 
   1262 	if (!ISSET(hp->sc->sc_flags, SDHC_FLAG_ENHANCED)) {
   1263 		blksize |= (MAX(0, PAGE_SHIFT - 12) & SDHC_DMA_BOUNDARY_MASK) <<
   1264 		    SDHC_DMA_BOUNDARY_SHIFT;	/* PAGE_SIZE DMA boundary */
   1265 	}
   1266 
   1267 	mutex_enter(&hp->host_mtx);
   1268 
   1269 	if (!ISSET(hp->sc->sc_flags, SDHC_FLAG_ENHANCED)) {
   1270 		/* Alert the user not to remove the card. */
   1271 		HSET1(hp, SDHC_HOST_CTL, SDHC_LED_ON);
   1272 	}
   1273 
   1274 	/* Set DMA start address. */
   1275 	if (ISSET(mode, SDHC_DMA_ENABLE) &&
   1276 	    !ISSET(sc->sc_flags, SDHC_FLAG_EXTERNAL_DMA))
   1277 		HWRITE4(hp, SDHC_DMA_ADDR, cmd->c_dmamap->dm_segs[0].ds_addr);
   1278 
   1279 	/*
   1280 	 * Start a CPU data transfer.  Writing to the high order byte
   1281 	 * of the SDHC_COMMAND register triggers the SD command. (1.5)
   1282 	 */
   1283 	if (ISSET(hp->sc->sc_flags, SDHC_FLAG_32BIT_ACCESS)) {
   1284 		HWRITE4(hp, SDHC_BLOCK_SIZE, blksize | (blkcount << 16));
   1285 		HWRITE4(hp, SDHC_ARGUMENT, cmd->c_arg);
   1286 		HWRITE4(hp, SDHC_TRANSFER_MODE, mode | (command << 16));
   1287 	} else {
   1288 		HWRITE2(hp, SDHC_BLOCK_SIZE, blksize);
   1289 		HWRITE2(hp, SDHC_BLOCK_COUNT, blkcount);
   1290 		HWRITE4(hp, SDHC_ARGUMENT, cmd->c_arg);
   1291 		HWRITE2(hp, SDHC_TRANSFER_MODE, mode);
   1292 		HWRITE2(hp, SDHC_COMMAND, command);
   1293 	}
   1294 
   1295 	mutex_exit(&hp->host_mtx);
   1296 
   1297 	return 0;
   1298 }
   1299 
   1300 static void
   1301 sdhc_transfer_data(struct sdhc_host *hp, struct sdmmc_command *cmd)
   1302 {
   1303 	struct sdhc_softc *sc = hp->sc;
   1304 	int error;
   1305 
   1306 	DPRINTF(1,("%s: data transfer: resp=%08x datalen=%u\n", HDEVNAME(hp),
   1307 	    MMC_R1(cmd->c_resp), cmd->c_datalen));
   1308 
   1309 #ifdef SDHC_DEBUG
   1310 	/* XXX I forgot why I wanted to know when this happens :-( */
   1311 	if ((cmd->c_opcode == 52 || cmd->c_opcode == 53) &&
   1312 	    ISSET(MMC_R1(cmd->c_resp), 0xcb00)) {
   1313 		aprint_error_dev(hp->sc->sc_dev,
   1314 		    "CMD52/53 error response flags %#x\n",
   1315 		    MMC_R1(cmd->c_resp) & 0xff00);
   1316 	}
   1317 #endif
   1318 
   1319 	if (cmd->c_dmamap != NULL) {
   1320 		if (hp->sc->sc_vendor_transfer_data_dma != NULL) {
   1321 			error = hp->sc->sc_vendor_transfer_data_dma(sc, cmd);
   1322 			if (error == 0 && !sdhc_wait_intr(hp,
   1323 			    SDHC_TRANSFER_COMPLETE, SDHC_TRANSFER_TIMEOUT)) {
   1324 				error = ETIMEDOUT;
   1325 			}
   1326 		} else {
   1327 			error = sdhc_transfer_data_dma(hp, cmd);
   1328 		}
   1329 	} else
   1330 		error = sdhc_transfer_data_pio(hp, cmd);
   1331 	if (error)
   1332 		cmd->c_error = error;
   1333 	SET(cmd->c_flags, SCF_ITSDONE);
   1334 
   1335 	DPRINTF(1,("%s: data transfer done (error=%d)\n",
   1336 	    HDEVNAME(hp), cmd->c_error));
   1337 }
   1338 
   1339 static int
   1340 sdhc_transfer_data_dma(struct sdhc_host *hp, struct sdmmc_command *cmd)
   1341 {
   1342 	bus_dma_segment_t *dm_segs = cmd->c_dmamap->dm_segs;
   1343 	bus_addr_t posaddr;
   1344 	bus_addr_t segaddr;
   1345 	bus_size_t seglen;
   1346 	u_int seg = 0;
   1347 	int error = 0;
   1348 	int status;
   1349 
   1350 	KASSERT(HREAD2(hp, SDHC_NINTR_STATUS_EN) & SDHC_DMA_INTERRUPT);
   1351 	KASSERT(HREAD2(hp, SDHC_NINTR_SIGNAL_EN) & SDHC_DMA_INTERRUPT);
   1352 	KASSERT(HREAD2(hp, SDHC_NINTR_STATUS_EN) & SDHC_TRANSFER_COMPLETE);
   1353 	KASSERT(HREAD2(hp, SDHC_NINTR_SIGNAL_EN) & SDHC_TRANSFER_COMPLETE);
   1354 
   1355 	for (;;) {
   1356 		status = sdhc_wait_intr(hp,
   1357 		    SDHC_DMA_INTERRUPT|SDHC_TRANSFER_COMPLETE,
   1358 		    SDHC_DMA_TIMEOUT);
   1359 
   1360 		if (status & SDHC_TRANSFER_COMPLETE) {
   1361 			break;
   1362 		}
   1363 		if (!status) {
   1364 			error = ETIMEDOUT;
   1365 			break;
   1366 		}
   1367 		if ((status & SDHC_DMA_INTERRUPT) == 0) {
   1368 			continue;
   1369 		}
   1370 
   1371 		/* DMA Interrupt (boundary crossing) */
   1372 
   1373 		segaddr = dm_segs[seg].ds_addr;
   1374 		seglen = dm_segs[seg].ds_len;
   1375 		mutex_enter(&hp->host_mtx);
   1376 		posaddr = HREAD4(hp, SDHC_DMA_ADDR);
   1377 		mutex_exit(&hp->host_mtx);
   1378 
   1379 		if ((seg == (cmd->c_dmamap->dm_nsegs-1)) && (posaddr == (segaddr + seglen))) {
   1380 			continue;
   1381 		}
   1382 		mutex_enter(&hp->host_mtx);
   1383 		if ((posaddr >= segaddr) && (posaddr < (segaddr + seglen)))
   1384 			HWRITE4(hp, SDHC_DMA_ADDR, posaddr);
   1385 		else if ((posaddr >= segaddr) && (posaddr == (segaddr + seglen)) && (seg + 1) < cmd->c_dmamap->dm_nsegs)
   1386 			HWRITE4(hp, SDHC_DMA_ADDR, dm_segs[++seg].ds_addr);
   1387 		mutex_exit(&hp->host_mtx);
   1388 		KASSERT(seg < cmd->c_dmamap->dm_nsegs);
   1389 	}
   1390 
   1391 	return error;
   1392 }
   1393 
   1394 static int
   1395 sdhc_transfer_data_pio(struct sdhc_host *hp, struct sdmmc_command *cmd)
   1396 {
   1397 	uint8_t *data = cmd->c_data;
   1398 	void (*pio_func)(struct sdhc_host *, uint8_t *, u_int);
   1399 	u_int len, datalen;
   1400 	u_int imask;
   1401 	u_int pmask;
   1402 	int error = 0;
   1403 
   1404 	if (ISSET(cmd->c_flags, SCF_CMD_READ)) {
   1405 		imask = SDHC_BUFFER_READ_READY;
   1406 		pmask = SDHC_BUFFER_READ_ENABLE;
   1407 		if (ISSET(hp->sc->sc_flags, SDHC_FLAG_ENHANCED)) {
   1408 			pio_func = esdhc_read_data_pio;
   1409 		} else {
   1410 			pio_func = sdhc_read_data_pio;
   1411 		}
   1412 	} else {
   1413 		imask = SDHC_BUFFER_WRITE_READY;
   1414 		pmask = SDHC_BUFFER_WRITE_ENABLE;
   1415 		if (ISSET(hp->sc->sc_flags, SDHC_FLAG_ENHANCED)) {
   1416 			pio_func = esdhc_write_data_pio;
   1417 		} else {
   1418 			pio_func = sdhc_write_data_pio;
   1419 		}
   1420 	}
   1421 	datalen = cmd->c_datalen;
   1422 
   1423 	KASSERT(HREAD2(hp, SDHC_NINTR_STATUS_EN) & imask);
   1424 	KASSERT(HREAD2(hp, SDHC_NINTR_STATUS_EN) & SDHC_TRANSFER_COMPLETE);
   1425 	KASSERT(HREAD2(hp, SDHC_NINTR_SIGNAL_EN) & SDHC_TRANSFER_COMPLETE);
   1426 
   1427 	while (datalen > 0) {
   1428 		if (!ISSET(HREAD4(hp, SDHC_PRESENT_STATE), imask)) {
   1429 			mutex_enter(&hp->intr_mtx);
   1430 			if (ISSET(hp->sc->sc_flags, SDHC_FLAG_32BIT_ACCESS)) {
   1431 				HSET4(hp, SDHC_NINTR_SIGNAL_EN, imask);
   1432 			} else {
   1433 				HSET2(hp, SDHC_NINTR_SIGNAL_EN, imask);
   1434 			}
   1435 			mutex_exit(&hp->intr_mtx);
   1436 			if (!sdhc_wait_intr(hp, imask, SDHC_BUFFER_TIMEOUT)) {
   1437 				error = ETIMEDOUT;
   1438 				break;
   1439 			}
   1440 
   1441 			error = sdhc_wait_state(hp, pmask, pmask);
   1442 			if (error)
   1443 				break;
   1444 		}
   1445 
   1446 		len = MIN(datalen, cmd->c_blklen);
   1447 		(*pio_func)(hp, data, len);
   1448 		DPRINTF(2,("%s: pio data transfer %u @ %p\n",
   1449 		    HDEVNAME(hp), len, data));
   1450 
   1451 		data += len;
   1452 		datalen -= len;
   1453 	}
   1454 
   1455 	if (error == 0 && !sdhc_wait_intr(hp, SDHC_TRANSFER_COMPLETE,
   1456 	    SDHC_TRANSFER_TIMEOUT))
   1457 		error = ETIMEDOUT;
   1458 
   1459 	return error;
   1460 }
   1461 
   1462 static void
   1463 sdhc_read_data_pio(struct sdhc_host *hp, uint8_t *data, u_int datalen)
   1464 {
   1465 
   1466 	if (((__uintptr_t)data & 3) == 0) {
   1467 		while (datalen > 3) {
   1468 			*(uint32_t *)data = le32toh(HREAD4(hp, SDHC_DATA));
   1469 			data += 4;
   1470 			datalen -= 4;
   1471 		}
   1472 		if (datalen > 1) {
   1473 			*(uint16_t *)data = le16toh(HREAD2(hp, SDHC_DATA));
   1474 			data += 2;
   1475 			datalen -= 2;
   1476 		}
   1477 		if (datalen > 0) {
   1478 			*data = HREAD1(hp, SDHC_DATA);
   1479 			data += 1;
   1480 			datalen -= 1;
   1481 		}
   1482 	} else if (((__uintptr_t)data & 1) == 0) {
   1483 		while (datalen > 1) {
   1484 			*(uint16_t *)data = le16toh(HREAD2(hp, SDHC_DATA));
   1485 			data += 2;
   1486 			datalen -= 2;
   1487 		}
   1488 		if (datalen > 0) {
   1489 			*data = HREAD1(hp, SDHC_DATA);
   1490 			data += 1;
   1491 			datalen -= 1;
   1492 		}
   1493 	} else {
   1494 		while (datalen > 0) {
   1495 			*data = HREAD1(hp, SDHC_DATA);
   1496 			data += 1;
   1497 			datalen -= 1;
   1498 		}
   1499 	}
   1500 }
   1501 
   1502 static void
   1503 sdhc_write_data_pio(struct sdhc_host *hp, uint8_t *data, u_int datalen)
   1504 {
   1505 
   1506 	if (((__uintptr_t)data & 3) == 0) {
   1507 		while (datalen > 3) {
   1508 			HWRITE4(hp, SDHC_DATA, htole32(*(uint32_t *)data));
   1509 			data += 4;
   1510 			datalen -= 4;
   1511 		}
   1512 		if (datalen > 1) {
   1513 			HWRITE2(hp, SDHC_DATA, htole16(*(uint16_t *)data));
   1514 			data += 2;
   1515 			datalen -= 2;
   1516 		}
   1517 		if (datalen > 0) {
   1518 			HWRITE1(hp, SDHC_DATA, *data);
   1519 			data += 1;
   1520 			datalen -= 1;
   1521 		}
   1522 	} else if (((__uintptr_t)data & 1) == 0) {
   1523 		while (datalen > 1) {
   1524 			HWRITE2(hp, SDHC_DATA, htole16(*(uint16_t *)data));
   1525 			data += 2;
   1526 			datalen -= 2;
   1527 		}
   1528 		if (datalen > 0) {
   1529 			HWRITE1(hp, SDHC_DATA, *data);
   1530 			data += 1;
   1531 			datalen -= 1;
   1532 		}
   1533 	} else {
   1534 		while (datalen > 0) {
   1535 			HWRITE1(hp, SDHC_DATA, *data);
   1536 			data += 1;
   1537 			datalen -= 1;
   1538 		}
   1539 	}
   1540 }
   1541 
   1542 static void
   1543 esdhc_read_data_pio(struct sdhc_host *hp, uint8_t *data, u_int datalen)
   1544 {
   1545 	uint16_t status = HREAD2(hp, SDHC_NINTR_STATUS);
   1546 	uint32_t v;
   1547 
   1548 	const size_t watermark = (HREAD4(hp, SDHC_WATERMARK_LEVEL) >> SDHC_WATERMARK_READ_SHIFT) & SDHC_WATERMARK_READ_MASK;
   1549 	size_t count = 0;
   1550 
   1551 	while (datalen > 3 && !ISSET(status, SDHC_TRANSFER_COMPLETE)) {
   1552 		if (count == 0) {
   1553 			/*
   1554 			 * If we've drained "watermark" words, we need to wait
   1555 			 * a little bit so the read FIFO can refill.
   1556 			 */
   1557 			sdmmc_delay(10);
   1558 			count = watermark;
   1559 		}
   1560 		v = HREAD4(hp, SDHC_DATA);
   1561 		v = le32toh(v);
   1562 		*(uint32_t *)data = v;
   1563 		data += 4;
   1564 		datalen -= 4;
   1565 		status = HREAD2(hp, SDHC_NINTR_STATUS);
   1566 		count--;
   1567 	}
   1568 	if (datalen > 0 && !ISSET(status, SDHC_TRANSFER_COMPLETE)) {
   1569 		if (count == 0) {
   1570 			sdmmc_delay(10);
   1571 		}
   1572 		v = HREAD4(hp, SDHC_DATA);
   1573 		v = le32toh(v);
   1574 		do {
   1575 			*data++ = v;
   1576 			v >>= 8;
   1577 		} while (--datalen > 0);
   1578 	}
   1579 }
   1580 
   1581 static void
   1582 esdhc_write_data_pio(struct sdhc_host *hp, uint8_t *data, u_int datalen)
   1583 {
   1584 	uint16_t status = HREAD2(hp, SDHC_NINTR_STATUS);
   1585 	uint32_t v;
   1586 
   1587 	const size_t watermark = (HREAD4(hp, SDHC_WATERMARK_LEVEL) >> SDHC_WATERMARK_WRITE_SHIFT) & SDHC_WATERMARK_WRITE_MASK;
   1588 	size_t count = watermark;
   1589 
   1590 	while (datalen > 3 && !ISSET(status, SDHC_TRANSFER_COMPLETE)) {
   1591 		if (count == 0) {
   1592 			sdmmc_delay(10);
   1593 			count = watermark;
   1594 		}
   1595 		v = *(uint32_t *)data;
   1596 		v = htole32(v);
   1597 		HWRITE4(hp, SDHC_DATA, v);
   1598 		data += 4;
   1599 		datalen -= 4;
   1600 		status = HREAD2(hp, SDHC_NINTR_STATUS);
   1601 		count--;
   1602 	}
   1603 	if (datalen > 0 && !ISSET(status, SDHC_TRANSFER_COMPLETE)) {
   1604 		if (count == 0) {
   1605 			sdmmc_delay(10);
   1606 		}
   1607 		v = *(uint32_t *)data;
   1608 		v = htole32(v);
   1609 		HWRITE4(hp, SDHC_DATA, v);
   1610 	}
   1611 }
   1612 
   1613 /* Prepare for another command. */
   1614 static int
   1615 sdhc_soft_reset(struct sdhc_host *hp, int mask)
   1616 {
   1617 	int timo;
   1618 
   1619 	DPRINTF(1,("%s: software reset reg=%08x\n", HDEVNAME(hp), mask));
   1620 
   1621 	/* Request the reset.  */
   1622 	HWRITE1(hp, SDHC_SOFTWARE_RESET, mask);
   1623 
   1624 	/*
   1625 	 * If necessary, wait for the controller to set the bits to
   1626 	 * acknowledge the reset.
   1627 	 */
   1628 	if (ISSET(hp->sc->sc_flags, SDHC_FLAG_WAIT_RESET) &&
   1629 	    ISSET(mask, (SDHC_RESET_DAT | SDHC_RESET_CMD))) {
   1630 		for (timo = 10000; timo > 0; timo--) {
   1631 			if (ISSET(HREAD1(hp, SDHC_SOFTWARE_RESET), mask))
   1632 				break;
   1633 			/* Short delay because I worry we may miss it...  */
   1634 			sdmmc_delay(1);
   1635 		}
   1636 		if (timo == 0)
   1637 			return ETIMEDOUT;
   1638 	}
   1639 
   1640 	/*
   1641 	 * Wait for the controller to clear the bits to indicate that
   1642 	 * the reset has completed.
   1643 	 */
   1644 	for (timo = 10; timo > 0; timo--) {
   1645 		if (!ISSET(HREAD1(hp, SDHC_SOFTWARE_RESET), mask))
   1646 			break;
   1647 		sdmmc_delay(10000);
   1648 	}
   1649 	if (timo == 0) {
   1650 		DPRINTF(1,("%s: timeout reg=%08x\n", HDEVNAME(hp),
   1651 		    HREAD1(hp, SDHC_SOFTWARE_RESET)));
   1652 		return ETIMEDOUT;
   1653 	}
   1654 
   1655 	if (ISSET(hp->sc->sc_flags, SDHC_FLAG_ENHANCED)) {
   1656 		HSET4(hp, SDHC_DMA_CTL, SDHC_DMA_SNOOP);
   1657 	}
   1658 
   1659 	return 0;
   1660 }
   1661 
   1662 static int
   1663 sdhc_wait_intr(struct sdhc_host *hp, int mask, int timo)
   1664 {
   1665 	int status;
   1666 
   1667 	mask |= SDHC_ERROR_INTERRUPT;
   1668 
   1669 	mutex_enter(&hp->intr_mtx);
   1670 	status = hp->intr_status & mask;
   1671 	while (status == 0) {
   1672 		if (cv_timedwait(&hp->intr_cv, &hp->intr_mtx, timo)
   1673 		    == EWOULDBLOCK) {
   1674 			status |= SDHC_ERROR_INTERRUPT;
   1675 			break;
   1676 		}
   1677 		status = hp->intr_status & mask;
   1678 	}
   1679 	hp->intr_status &= ~status;
   1680 
   1681 	DPRINTF(2,("%s: intr status %#x error %#x\n", HDEVNAME(hp), status,
   1682 	    hp->intr_error_status));
   1683 
   1684 	/* Command timeout has higher priority than command complete. */
   1685 	if (ISSET(status, SDHC_ERROR_INTERRUPT) || hp->intr_error_status) {
   1686 		hp->intr_error_status = 0;
   1687 		hp->intr_status &= ~SDHC_ERROR_INTERRUPT;
   1688 		if (!ISSET(hp->sc->sc_flags, SDHC_FLAG_ENHANCED)) {
   1689 		    (void)sdhc_soft_reset(hp, SDHC_RESET_DAT|SDHC_RESET_CMD);
   1690 		}
   1691 		status = 0;
   1692 	}
   1693 	mutex_exit(&hp->intr_mtx);
   1694 
   1695 	return status;
   1696 }
   1697 
   1698 /*
   1699  * Established by attachment driver at interrupt priority IPL_SDMMC.
   1700  */
   1701 int
   1702 sdhc_intr(void *arg)
   1703 {
   1704 	struct sdhc_softc *sc = (struct sdhc_softc *)arg;
   1705 	struct sdhc_host *hp;
   1706 	int done = 0;
   1707 	uint16_t status;
   1708 	uint16_t error;
   1709 
   1710 	/* We got an interrupt, but we don't know from which slot. */
   1711 	for (size_t host = 0; host < sc->sc_nhosts; host++) {
   1712 		hp = sc->sc_host[host];
   1713 		if (hp == NULL)
   1714 			continue;
   1715 
   1716 		if (ISSET(sc->sc_flags, SDHC_FLAG_32BIT_ACCESS)) {
   1717 			/* Find out which interrupts are pending. */
   1718 			uint32_t xstatus = HREAD4(hp, SDHC_NINTR_STATUS);
   1719 			status = xstatus;
   1720 			error = xstatus >> 16;
   1721 			if (error)
   1722 				xstatus |= SDHC_ERROR_INTERRUPT;
   1723 			else if (!ISSET(status, SDHC_NINTR_STATUS_MASK))
   1724 				continue; /* no interrupt for us */
   1725 			/* Acknowledge the interrupts we are about to handle. */
   1726 			HWRITE4(hp, SDHC_NINTR_STATUS, xstatus);
   1727 		} else {
   1728 			/* Find out which interrupts are pending. */
   1729 			error = 0;
   1730 			status = HREAD2(hp, SDHC_NINTR_STATUS);
   1731 			if (!ISSET(status, SDHC_NINTR_STATUS_MASK))
   1732 				continue; /* no interrupt for us */
   1733 			/* Acknowledge the interrupts we are about to handle. */
   1734 			HWRITE2(hp, SDHC_NINTR_STATUS, status);
   1735 			if (ISSET(status, SDHC_ERROR_INTERRUPT)) {
   1736 				/* Acknowledge error interrupts. */
   1737 				error = HREAD2(hp, SDHC_EINTR_STATUS);
   1738 				HWRITE2(hp, SDHC_EINTR_STATUS, error);
   1739 			}
   1740 		}
   1741 
   1742 		DPRINTF(2,("%s: interrupt status=%x error=%x\n", HDEVNAME(hp),
   1743 		    status, error));
   1744 
   1745 		mutex_enter(&hp->intr_mtx);
   1746 
   1747 		/* Claim this interrupt. */
   1748 		done = 1;
   1749 
   1750 		/*
   1751 		 * Service error interrupts.
   1752 		 */
   1753 		if (ISSET(error, SDHC_CMD_TIMEOUT_ERROR|
   1754 		    SDHC_DATA_TIMEOUT_ERROR)) {
   1755 			hp->intr_error_status |= error;
   1756 			hp->intr_status |= status;
   1757 			cv_broadcast(&hp->intr_cv);
   1758 		}
   1759 
   1760 		/*
   1761 		 * Wake up the sdmmc event thread to scan for cards.
   1762 		 */
   1763 		if (ISSET(status, SDHC_CARD_REMOVAL|SDHC_CARD_INSERTION)) {
   1764 			if (hp->sdmmc != NULL) {
   1765 				sdmmc_needs_discover(hp->sdmmc);
   1766 			}
   1767 			if (ISSET(sc->sc_flags, SDHC_FLAG_ENHANCED)) {
   1768 				HCLR4(hp, SDHC_NINTR_STATUS_EN,
   1769 				    status & (SDHC_CARD_REMOVAL|SDHC_CARD_INSERTION));
   1770 				HCLR4(hp, SDHC_NINTR_SIGNAL_EN,
   1771 				    status & (SDHC_CARD_REMOVAL|SDHC_CARD_INSERTION));
   1772 			}
   1773 		}
   1774 
   1775 		/*
   1776 		 * Wake up the blocking process to service command
   1777 		 * related interrupt(s).
   1778 		 */
   1779 		if (ISSET(status, SDHC_COMMAND_COMPLETE|
   1780 		    SDHC_BUFFER_READ_READY|SDHC_BUFFER_WRITE_READY|
   1781 		    SDHC_TRANSFER_COMPLETE|SDHC_DMA_INTERRUPT)) {
   1782 			hp->intr_status |= status;
   1783 			if (ISSET(sc->sc_flags, SDHC_FLAG_ENHANCED)) {
   1784 				HCLR4(hp, SDHC_NINTR_SIGNAL_EN,
   1785 				    status & (SDHC_BUFFER_READ_READY|SDHC_BUFFER_WRITE_READY));
   1786 			}
   1787 			cv_broadcast(&hp->intr_cv);
   1788 		}
   1789 
   1790 		/*
   1791 		 * Service SD card interrupts.
   1792 		 */
   1793 		if (!ISSET(sc->sc_flags, SDHC_FLAG_ENHANCED)
   1794 		    && ISSET(status, SDHC_CARD_INTERRUPT)) {
   1795 			DPRINTF(0,("%s: card interrupt\n", HDEVNAME(hp)));
   1796 			HCLR2(hp, SDHC_NINTR_STATUS_EN, SDHC_CARD_INTERRUPT);
   1797 			sdmmc_card_intr(hp->sdmmc);
   1798 		}
   1799 		mutex_exit(&hp->intr_mtx);
   1800 	}
   1801 
   1802 	return done;
   1803 }
   1804 
   1805 #ifdef SDHC_DEBUG
   1806 void
   1807 sdhc_dump_regs(struct sdhc_host *hp)
   1808 {
   1809 
   1810 	printf("0x%02x PRESENT_STATE:    %x\n", SDHC_PRESENT_STATE,
   1811 	    HREAD4(hp, SDHC_PRESENT_STATE));
   1812 	if (!ISSET(hp->sc->sc_flags, SDHC_FLAG_ENHANCED))
   1813 		printf("0x%02x POWER_CTL:        %x\n", SDHC_POWER_CTL,
   1814 		    HREAD1(hp, SDHC_POWER_CTL));
   1815 	printf("0x%02x NINTR_STATUS:     %x\n", SDHC_NINTR_STATUS,
   1816 	    HREAD2(hp, SDHC_NINTR_STATUS));
   1817 	printf("0x%02x EINTR_STATUS:     %x\n", SDHC_EINTR_STATUS,
   1818 	    HREAD2(hp, SDHC_EINTR_STATUS));
   1819 	printf("0x%02x NINTR_STATUS_EN:  %x\n", SDHC_NINTR_STATUS_EN,
   1820 	    HREAD2(hp, SDHC_NINTR_STATUS_EN));
   1821 	printf("0x%02x EINTR_STATUS_EN:  %x\n", SDHC_EINTR_STATUS_EN,
   1822 	    HREAD2(hp, SDHC_EINTR_STATUS_EN));
   1823 	printf("0x%02x NINTR_SIGNAL_EN:  %x\n", SDHC_NINTR_SIGNAL_EN,
   1824 	    HREAD2(hp, SDHC_NINTR_SIGNAL_EN));
   1825 	printf("0x%02x EINTR_SIGNAL_EN:  %x\n", SDHC_EINTR_SIGNAL_EN,
   1826 	    HREAD2(hp, SDHC_EINTR_SIGNAL_EN));
   1827 	printf("0x%02x CAPABILITIES:     %x\n", SDHC_CAPABILITIES,
   1828 	    HREAD4(hp, SDHC_CAPABILITIES));
   1829 	printf("0x%02x MAX_CAPABILITIES: %x\n", SDHC_MAX_CAPABILITIES,
   1830 	    HREAD4(hp, SDHC_MAX_CAPABILITIES));
   1831 }
   1832 #endif
   1833