Home | History | Annotate | Line # | Download | only in sdmmc
sdhc.c revision 1.82
      1 /*	$NetBSD: sdhc.c,v 1.82 2015/08/09 13:24:39 mlelstv Exp $	*/
      2 /*	$OpenBSD: sdhc.c,v 1.25 2009/01/13 19:44:20 grange Exp $	*/
      3 
      4 /*
      5  * Copyright (c) 2006 Uwe Stuehler <uwe (at) openbsd.org>
      6  *
      7  * Permission to use, copy, modify, and distribute this software for any
      8  * purpose with or without fee is hereby granted, provided that the above
      9  * copyright notice and this permission notice appear in all copies.
     10  *
     11  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
     12  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
     13  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
     14  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
     15  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
     16  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
     17  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
     18  */
     19 
     20 /*
     21  * SD Host Controller driver based on the SD Host Controller Standard
     22  * Simplified Specification Version 1.00 (www.sdcard.com).
     23  */
     24 
     25 #include <sys/cdefs.h>
     26 __KERNEL_RCSID(0, "$NetBSD: sdhc.c,v 1.82 2015/08/09 13:24:39 mlelstv Exp $");
     27 
     28 #ifdef _KERNEL_OPT
     29 #include "opt_sdmmc.h"
     30 #endif
     31 
     32 #include <sys/param.h>
     33 #include <sys/device.h>
     34 #include <sys/kernel.h>
     35 #include <sys/malloc.h>
     36 #include <sys/systm.h>
     37 #include <sys/mutex.h>
     38 #include <sys/condvar.h>
     39 #include <sys/atomic.h>
     40 
     41 #include <dev/sdmmc/sdhcreg.h>
     42 #include <dev/sdmmc/sdhcvar.h>
     43 #include <dev/sdmmc/sdmmcchip.h>
     44 #include <dev/sdmmc/sdmmcreg.h>
     45 #include <dev/sdmmc/sdmmcvar.h>
     46 
     47 #ifdef SDHC_DEBUG
     48 int sdhcdebug = 1;
     49 #define DPRINTF(n,s)	do { if ((n) <= sdhcdebug) printf s; } while (0)
     50 void	sdhc_dump_regs(struct sdhc_host *);
     51 #else
     52 #define DPRINTF(n,s)	do {} while (0)
     53 #endif
     54 
     55 #define SDHC_COMMAND_TIMEOUT	hz
     56 #define SDHC_BUFFER_TIMEOUT	hz
     57 #define SDHC_TRANSFER_TIMEOUT	hz
     58 #define SDHC_DMA_TIMEOUT	(hz*3)
     59 #define SDHC_TUNING_TIMEOUT	hz
     60 
     61 struct sdhc_host {
     62 	struct sdhc_softc *sc;		/* host controller device */
     63 
     64 	bus_space_tag_t iot;		/* host register set tag */
     65 	bus_space_handle_t ioh;		/* host register set handle */
     66 	bus_size_t ios;			/* host register space size */
     67 	bus_dma_tag_t dmat;		/* host DMA tag */
     68 
     69 	device_t sdmmc;			/* generic SD/MMC device */
     70 
     71 	u_int clkbase;			/* base clock frequency in KHz */
     72 	int maxblklen;			/* maximum block length */
     73 	uint32_t ocr;			/* OCR value from capabilities */
     74 
     75 	uint8_t regs[14];		/* host controller state */
     76 
     77 	uint16_t intr_status;		/* soft interrupt status */
     78 	uint16_t intr_error_status;	/* soft error status */
     79 	kmutex_t intr_lock;
     80 	kcondvar_t intr_cv;
     81 
     82 	callout_t tuning_timer;
     83 	int tuning_timing;
     84 	u_int tuning_timer_count;
     85 	u_int tuning_timer_pending;
     86 
     87 	int specver;			/* spec. version */
     88 
     89 	uint32_t flags;			/* flags for this host */
     90 #define SHF_USE_DMA		0x0001
     91 #define SHF_USE_4BIT_MODE	0x0002
     92 #define SHF_USE_8BIT_MODE	0x0004
     93 #define SHF_MODE_DMAEN		0x0008 /* needs SDHC_DMA_ENABLE in mode */
     94 #define SHF_USE_ADMA2_32	0x0010
     95 #define SHF_USE_ADMA2_64	0x0020
     96 #define SHF_USE_ADMA2_MASK	0x0030
     97 
     98 	bus_dmamap_t		adma_map;
     99 	bus_dma_segment_t	adma_segs[1];
    100 	void			*adma2;
    101 };
    102 
    103 #define HDEVNAME(hp)	(device_xname((hp)->sc->sc_dev))
    104 
    105 static uint8_t
    106 hread1(struct sdhc_host *hp, bus_size_t reg)
    107 {
    108 
    109 	if (!ISSET(hp->sc->sc_flags, SDHC_FLAG_32BIT_ACCESS))
    110 		return bus_space_read_1(hp->iot, hp->ioh, reg);
    111 	return bus_space_read_4(hp->iot, hp->ioh, reg & -4) >> (8 * (reg & 3));
    112 }
    113 
    114 static uint16_t
    115 hread2(struct sdhc_host *hp, bus_size_t reg)
    116 {
    117 
    118 	if (!ISSET(hp->sc->sc_flags, SDHC_FLAG_32BIT_ACCESS))
    119 		return bus_space_read_2(hp->iot, hp->ioh, reg);
    120 	return bus_space_read_4(hp->iot, hp->ioh, reg & -4) >> (8 * (reg & 2));
    121 }
    122 
    123 #define HREAD1(hp, reg)		hread1(hp, reg)
    124 #define HREAD2(hp, reg)		hread2(hp, reg)
    125 #define HREAD4(hp, reg)		\
    126 	(bus_space_read_4((hp)->iot, (hp)->ioh, (reg)))
    127 
    128 
    129 static void
    130 hwrite1(struct sdhc_host *hp, bus_size_t o, uint8_t val)
    131 {
    132 
    133 	if (!ISSET(hp->sc->sc_flags, SDHC_FLAG_32BIT_ACCESS)) {
    134 		bus_space_write_1(hp->iot, hp->ioh, o, val);
    135 	} else {
    136 		const size_t shift = 8 * (o & 3);
    137 		o &= -4;
    138 		uint32_t tmp = bus_space_read_4(hp->iot, hp->ioh, o);
    139 		tmp = (val << shift) | (tmp & ~(0xff << shift));
    140 		bus_space_write_4(hp->iot, hp->ioh, o, tmp);
    141 	}
    142 }
    143 
    144 static void
    145 hwrite2(struct sdhc_host *hp, bus_size_t o, uint16_t val)
    146 {
    147 
    148 	if (!ISSET(hp->sc->sc_flags, SDHC_FLAG_32BIT_ACCESS)) {
    149 		bus_space_write_2(hp->iot, hp->ioh, o, val);
    150 	} else {
    151 		const size_t shift = 8 * (o & 2);
    152 		o &= -4;
    153 		uint32_t tmp = bus_space_read_4(hp->iot, hp->ioh, o);
    154 		tmp = (val << shift) | (tmp & ~(0xffff << shift));
    155 		bus_space_write_4(hp->iot, hp->ioh, o, tmp);
    156 	}
    157 }
    158 
    159 #define HWRITE1(hp, reg, val)		hwrite1(hp, reg, val)
    160 #define HWRITE2(hp, reg, val)		hwrite2(hp, reg, val)
    161 #define HWRITE4(hp, reg, val)						\
    162 	bus_space_write_4((hp)->iot, (hp)->ioh, (reg), (val))
    163 
    164 #define HCLR1(hp, reg, bits)						\
    165 	do if (bits) HWRITE1((hp), (reg), HREAD1((hp), (reg)) & ~(bits)); while (0)
    166 #define HCLR2(hp, reg, bits)						\
    167 	do if (bits) HWRITE2((hp), (reg), HREAD2((hp), (reg)) & ~(bits)); while (0)
    168 #define HCLR4(hp, reg, bits)						\
    169 	do if (bits) HWRITE4((hp), (reg), HREAD4((hp), (reg)) & ~(bits)); while (0)
    170 #define HSET1(hp, reg, bits)						\
    171 	do if (bits) HWRITE1((hp), (reg), HREAD1((hp), (reg)) | (bits)); while (0)
    172 #define HSET2(hp, reg, bits)						\
    173 	do if (bits) HWRITE2((hp), (reg), HREAD2((hp), (reg)) | (bits)); while (0)
    174 #define HSET4(hp, reg, bits)						\
    175 	do if (bits) HWRITE4((hp), (reg), HREAD4((hp), (reg)) | (bits)); while (0)
    176 
    177 static int	sdhc_host_reset(sdmmc_chipset_handle_t);
    178 static int	sdhc_host_reset1(sdmmc_chipset_handle_t);
    179 static uint32_t	sdhc_host_ocr(sdmmc_chipset_handle_t);
    180 static int	sdhc_host_maxblklen(sdmmc_chipset_handle_t);
    181 static int	sdhc_card_detect(sdmmc_chipset_handle_t);
    182 static int	sdhc_write_protect(sdmmc_chipset_handle_t);
    183 static int	sdhc_bus_power(sdmmc_chipset_handle_t, uint32_t);
    184 static int	sdhc_bus_clock_ddr(sdmmc_chipset_handle_t, int, bool);
    185 static int	sdhc_bus_width(sdmmc_chipset_handle_t, int);
    186 static int	sdhc_bus_rod(sdmmc_chipset_handle_t, int);
    187 static void	sdhc_card_enable_intr(sdmmc_chipset_handle_t, int);
    188 static void	sdhc_card_intr_ack(sdmmc_chipset_handle_t);
    189 static void	sdhc_exec_command(sdmmc_chipset_handle_t,
    190 		    struct sdmmc_command *);
    191 static int	sdhc_signal_voltage(sdmmc_chipset_handle_t, int);
    192 static int	sdhc_execute_tuning(sdmmc_chipset_handle_t, int);
    193 static void	sdhc_tuning_timer(void *);
    194 static int	sdhc_start_command(struct sdhc_host *, struct sdmmc_command *);
    195 static int	sdhc_wait_state(struct sdhc_host *, uint32_t, uint32_t);
    196 static int	sdhc_soft_reset(struct sdhc_host *, int);
    197 static int	sdhc_wait_intr(struct sdhc_host *, int, int);
    198 static void	sdhc_transfer_data(struct sdhc_host *, struct sdmmc_command *);
    199 static int	sdhc_transfer_data_dma(struct sdhc_host *, struct sdmmc_command *);
    200 static int	sdhc_transfer_data_pio(struct sdhc_host *, struct sdmmc_command *);
    201 static void	sdhc_read_data_pio(struct sdhc_host *, uint8_t *, u_int);
    202 static void	sdhc_write_data_pio(struct sdhc_host *, uint8_t *, u_int);
    203 static void	esdhc_read_data_pio(struct sdhc_host *, uint8_t *, u_int);
    204 static void	esdhc_write_data_pio(struct sdhc_host *, uint8_t *, u_int);
    205 
    206 static struct sdmmc_chip_functions sdhc_functions = {
    207 	/* host controller reset */
    208 	.host_reset = sdhc_host_reset,
    209 
    210 	/* host controller capabilities */
    211 	.host_ocr = sdhc_host_ocr,
    212 	.host_maxblklen = sdhc_host_maxblklen,
    213 
    214 	/* card detection */
    215 	.card_detect = sdhc_card_detect,
    216 
    217 	/* write protect */
    218 	.write_protect = sdhc_write_protect,
    219 
    220 	/* bus power, clock frequency, width and ROD(OpenDrain/PushPull) */
    221 	.bus_power = sdhc_bus_power,
    222 	.bus_clock = NULL,	/* see sdhc_bus_clock_ddr */
    223 	.bus_width = sdhc_bus_width,
    224 	.bus_rod = sdhc_bus_rod,
    225 
    226 	/* command execution */
    227 	.exec_command = sdhc_exec_command,
    228 
    229 	/* card interrupt */
    230 	.card_enable_intr = sdhc_card_enable_intr,
    231 	.card_intr_ack = sdhc_card_intr_ack,
    232 
    233 	/* UHS functions */
    234 	.signal_voltage = sdhc_signal_voltage,
    235 	.bus_clock_ddr = sdhc_bus_clock_ddr,
    236 	.execute_tuning = sdhc_execute_tuning,
    237 };
    238 
    239 static int
    240 sdhc_cfprint(void *aux, const char *pnp)
    241 {
    242 	const struct sdmmcbus_attach_args * const saa = aux;
    243 	const struct sdhc_host * const hp = saa->saa_sch;
    244 
    245 	if (pnp) {
    246 		aprint_normal("sdmmc at %s", pnp);
    247 	}
    248 	for (size_t host = 0; host < hp->sc->sc_nhosts; host++) {
    249 		if (hp->sc->sc_host[host] == hp) {
    250 			aprint_normal(" slot %zu", host);
    251 		}
    252 	}
    253 
    254 	return UNCONF;
    255 }
    256 
    257 /*
    258  * Called by attachment driver.  For each SD card slot there is one SD
    259  * host controller standard register set. (1.3)
    260  */
    261 int
    262 sdhc_host_found(struct sdhc_softc *sc, bus_space_tag_t iot,
    263     bus_space_handle_t ioh, bus_size_t iosize)
    264 {
    265 	struct sdmmcbus_attach_args saa;
    266 	struct sdhc_host *hp;
    267 	uint32_t caps, caps2;
    268 	uint16_t sdhcver;
    269 	int error;
    270 
    271 	/* Allocate one more host structure. */
    272 	hp = malloc(sizeof(struct sdhc_host), M_DEVBUF, M_WAITOK|M_ZERO);
    273 	if (hp == NULL) {
    274 		aprint_error_dev(sc->sc_dev,
    275 		    "couldn't alloc memory (sdhc host)\n");
    276 		goto err1;
    277 	}
    278 	sc->sc_host[sc->sc_nhosts++] = hp;
    279 
    280 	/* Fill in the new host structure. */
    281 	hp->sc = sc;
    282 	hp->iot = iot;
    283 	hp->ioh = ioh;
    284 	hp->ios = iosize;
    285 	hp->dmat = sc->sc_dmat;
    286 
    287 	mutex_init(&hp->intr_lock, MUTEX_DEFAULT, IPL_SDMMC);
    288 	cv_init(&hp->intr_cv, "sdhcintr");
    289 	callout_init(&hp->tuning_timer, CALLOUT_MPSAFE);
    290 	callout_setfunc(&hp->tuning_timer, sdhc_tuning_timer, hp);
    291 
    292 	if (ISSET(hp->sc->sc_flags, SDHC_FLAG_ENHANCED)) {
    293 		sdhcver = HREAD4(hp, SDHC_ESDHC_HOST_CTL_VERSION);
    294 	} else {
    295 		sdhcver = HREAD2(hp, SDHC_HOST_CTL_VERSION);
    296 	}
    297 	aprint_normal_dev(sc->sc_dev, "SDHC ");
    298 	hp->specver = SDHC_SPEC_VERSION(sdhcver);
    299 	switch (SDHC_SPEC_VERSION(sdhcver)) {
    300 	case SDHC_SPEC_VERS_100:
    301 		aprint_normal("1.0");
    302 		break;
    303 
    304 	case SDHC_SPEC_VERS_200:
    305 		aprint_normal("2.0");
    306 		break;
    307 
    308 	case SDHC_SPEC_VERS_300:
    309 		aprint_normal("3.0");
    310 		break;
    311 
    312 	case SDHC_SPEC_VERS_400:
    313 		aprint_normal("4.0");
    314 		break;
    315 
    316 	default:
    317 		aprint_normal("unknown version(0x%x)",
    318 		    SDHC_SPEC_VERSION(sdhcver));
    319 		break;
    320 	}
    321 	aprint_normal(", rev %u", SDHC_VENDOR_VERSION(sdhcver));
    322 
    323 	/*
    324 	 * Reset the host controller and enable interrupts.
    325 	 */
    326 	(void)sdhc_host_reset(hp);
    327 
    328 	/* Determine host capabilities. */
    329 	if (ISSET(sc->sc_flags, SDHC_FLAG_HOSTCAPS)) {
    330 		caps = sc->sc_caps;
    331 		caps2 = sc->sc_caps2;
    332 	} else {
    333 		caps = sc->sc_caps = HREAD4(hp, SDHC_CAPABILITIES);
    334 		if (hp->specver >= SDHC_SPEC_VERS_300) {
    335 			caps2 = sc->sc_caps2 = HREAD4(hp, SDHC_CAPABILITIES2);
    336 		} else {
    337 			caps2 = sc->sc_caps2 = 0;
    338 		}
    339 	}
    340 
    341 	const u_int retuning_mode = (caps2 >> SDHC_RETUNING_MODES_SHIFT) &
    342 	    SDHC_RETUNING_MODES_MASK;
    343 	if (retuning_mode == SDHC_RETUNING_MODE_1) {
    344 		hp->tuning_timer_count = (caps2 >> SDHC_TIMER_COUNT_SHIFT) &
    345 		    SDHC_TIMER_COUNT_MASK;
    346 		if (hp->tuning_timer_count == 0xf)
    347 			hp->tuning_timer_count = 0;
    348 		if (hp->tuning_timer_count)
    349 			hp->tuning_timer_count =
    350 			    1 << (hp->tuning_timer_count - 1);
    351 	}
    352 
    353 	/*
    354 	 * Use DMA if the host system and the controller support it.
    355 	 * Suports integrated or external DMA egine, with or without
    356 	 * SDHC_DMA_ENABLE in the command.
    357 	 */
    358 	if (ISSET(sc->sc_flags, SDHC_FLAG_FORCE_DMA) ||
    359 	    (ISSET(sc->sc_flags, SDHC_FLAG_USE_DMA &&
    360 	     ISSET(caps, SDHC_DMA_SUPPORT)))) {
    361 		SET(hp->flags, SHF_USE_DMA);
    362 
    363 		if (ISSET(sc->sc_flags, SDHC_FLAG_USE_ADMA2) &&
    364 		    ISSET(caps, SDHC_ADMA2_SUPP)) {
    365 			SET(hp->flags, SHF_MODE_DMAEN);
    366 			/*
    367 			 * 64-bit mode was present in the 2.00 spec, removed
    368 			 * from 3.00, and re-added in 4.00 with a different
    369 			 * descriptor layout. We only support 2.00 and 3.00
    370 			 * descriptors for now.
    371 			 */
    372 			if (hp->specver == SDHC_SPEC_VERS_200 &&
    373 			    ISSET(caps, SDHC_64BIT_SYS_BUS)) {
    374 				SET(hp->flags, SHF_USE_ADMA2_64);
    375 				aprint_normal(", 64-bit ADMA2");
    376 			} else {
    377 				SET(hp->flags, SHF_USE_ADMA2_32);
    378 				aprint_normal(", 32-bit ADMA2");
    379 			}
    380 		} else {
    381 			if (!ISSET(sc->sc_flags, SDHC_FLAG_EXTERNAL_DMA) ||
    382 			    ISSET(sc->sc_flags, SDHC_FLAG_EXTDMA_DMAEN))
    383 				SET(hp->flags, SHF_MODE_DMAEN);
    384 			if (sc->sc_vendor_transfer_data_dma) {
    385 				aprint_normal(", platform DMA");
    386 			} else {
    387 				aprint_normal(", SDMA");
    388 			}
    389 		}
    390 	} else {
    391 		aprint_normal(", PIO");
    392 	}
    393 
    394 	/*
    395 	 * Determine the base clock frequency. (2.2.24)
    396 	 */
    397 	if (hp->specver >= SDHC_SPEC_VERS_300) {
    398 		hp->clkbase = SDHC_BASE_V3_FREQ_KHZ(caps);
    399 	} else {
    400 		hp->clkbase = SDHC_BASE_FREQ_KHZ(caps);
    401 	}
    402 	if (hp->clkbase == 0 ||
    403 	    ISSET(sc->sc_flags, SDHC_FLAG_NO_CLKBASE)) {
    404 		if (sc->sc_clkbase == 0) {
    405 			/* The attachment driver must tell us. */
    406 			aprint_error_dev(sc->sc_dev,
    407 			    "unknown base clock frequency\n");
    408 			goto err;
    409 		}
    410 		hp->clkbase = sc->sc_clkbase;
    411 	}
    412 	if (hp->clkbase < 10000 || hp->clkbase > 10000 * 256) {
    413 		/* SDHC 1.0 supports only 10-63 MHz. */
    414 		aprint_error_dev(sc->sc_dev,
    415 		    "base clock frequency out of range: %u MHz\n",
    416 		    hp->clkbase / 1000);
    417 		goto err;
    418 	}
    419 	aprint_normal(", %u kHz", hp->clkbase);
    420 
    421 	/*
    422 	 * XXX Set the data timeout counter value according to
    423 	 * capabilities. (2.2.15)
    424 	 */
    425 	HWRITE1(hp, SDHC_TIMEOUT_CTL, SDHC_TIMEOUT_MAX);
    426 #if 1
    427 	if (ISSET(hp->sc->sc_flags, SDHC_FLAG_ENHANCED))
    428 		HWRITE4(hp, SDHC_NINTR_STATUS, SDHC_CMD_TIMEOUT_ERROR << 16);
    429 #endif
    430 
    431 	if (ISSET(caps, SDHC_EMBEDDED_SLOT))
    432 		aprint_normal(", embedded slot");
    433 
    434 	/*
    435 	 * Determine SD bus voltage levels supported by the controller.
    436 	 */
    437 	aprint_normal(",");
    438 	if (ISSET(caps, SDHC_HIGH_SPEED_SUPP)) {
    439 		SET(hp->ocr, MMC_OCR_HCS);
    440 		aprint_normal(" HS");
    441 	}
    442 	if (ISSET(caps2, SDHC_SDR50_SUPP)) {
    443 		SET(hp->ocr, MMC_OCR_S18A);
    444 		aprint_normal(" SDR50");
    445 	}
    446 	if (ISSET(caps2, SDHC_DDR50_SUPP)) {
    447 		SET(hp->ocr, MMC_OCR_S18A);
    448 		aprint_normal(" DDR50");
    449 	}
    450 	if (ISSET(caps2, SDHC_SDR104_SUPP)) {
    451 		SET(hp->ocr, MMC_OCR_S18A);
    452 		aprint_normal(" SDR104 HS200");
    453 	}
    454 	if (ISSET(caps, SDHC_VOLTAGE_SUPP_1_8V)) {
    455 		SET(hp->ocr, MMC_OCR_1_7V_1_8V | MMC_OCR_1_8V_1_9V);
    456 		aprint_normal(" 1.8V");
    457 	}
    458 	if (ISSET(caps, SDHC_VOLTAGE_SUPP_3_0V)) {
    459 		SET(hp->ocr, MMC_OCR_2_9V_3_0V | MMC_OCR_3_0V_3_1V);
    460 		aprint_normal(" 3.0V");
    461 	}
    462 	if (ISSET(caps, SDHC_VOLTAGE_SUPP_3_3V)) {
    463 		SET(hp->ocr, MMC_OCR_3_2V_3_3V | MMC_OCR_3_3V_3_4V);
    464 		aprint_normal(" 3.3V");
    465 	}
    466 	if (hp->specver >= SDHC_SPEC_VERS_300) {
    467 		aprint_normal(", re-tuning mode %d", retuning_mode + 1);
    468 		if (hp->tuning_timer_count)
    469 			aprint_normal(" (%us timer)", hp->tuning_timer_count);
    470 	}
    471 
    472 	/*
    473 	 * Determine the maximum block length supported by the host
    474 	 * controller. (2.2.24)
    475 	 */
    476 	switch((caps >> SDHC_MAX_BLK_LEN_SHIFT) & SDHC_MAX_BLK_LEN_MASK) {
    477 	case SDHC_MAX_BLK_LEN_512:
    478 		hp->maxblklen = 512;
    479 		break;
    480 
    481 	case SDHC_MAX_BLK_LEN_1024:
    482 		hp->maxblklen = 1024;
    483 		break;
    484 
    485 	case SDHC_MAX_BLK_LEN_2048:
    486 		hp->maxblklen = 2048;
    487 		break;
    488 
    489 	case SDHC_MAX_BLK_LEN_4096:
    490 		hp->maxblklen = 4096;
    491 		break;
    492 
    493 	default:
    494 		aprint_error_dev(sc->sc_dev, "max block length unknown\n");
    495 		goto err;
    496 	}
    497 	aprint_normal(", %u byte blocks", hp->maxblklen);
    498 	aprint_normal("\n");
    499 
    500 	if (ISSET(hp->flags, SHF_USE_ADMA2_MASK)) {
    501 		int rseg;
    502 
    503 		/* Allocate ADMA2 descriptor memory */
    504 		error = bus_dmamem_alloc(sc->sc_dmat, PAGE_SIZE, PAGE_SIZE,
    505 		    PAGE_SIZE, hp->adma_segs, 1, &rseg, BUS_DMA_WAITOK);
    506 		if (error) {
    507 			aprint_error_dev(sc->sc_dev,
    508 			    "ADMA2 dmamem_alloc failed (%d)\n", error);
    509 			goto adma_done;
    510 		}
    511 		error = bus_dmamem_map(sc->sc_dmat, hp->adma_segs, rseg,
    512 		    PAGE_SIZE, (void **)&hp->adma2, BUS_DMA_WAITOK);
    513 		if (error) {
    514 			aprint_error_dev(sc->sc_dev,
    515 			    "ADMA2 dmamem_map failed (%d)\n", error);
    516 			goto adma_done;
    517 		}
    518 		error = bus_dmamap_create(sc->sc_dmat, PAGE_SIZE, 1, PAGE_SIZE,
    519 		    0, BUS_DMA_WAITOK, &hp->adma_map);
    520 		if (error) {
    521 			aprint_error_dev(sc->sc_dev,
    522 			    "ADMA2 dmamap_create failed (%d)\n", error);
    523 			goto adma_done;
    524 		}
    525 		error = bus_dmamap_load(sc->sc_dmat, hp->adma_map,
    526 		    hp->adma2, PAGE_SIZE, NULL,
    527 		    BUS_DMA_WAITOK|BUS_DMA_WRITE);
    528 		if (error) {
    529 			aprint_error_dev(sc->sc_dev,
    530 			    "ADMA2 dmamap_load failed (%d)\n", error);
    531 			goto adma_done;
    532 		}
    533 
    534 		memset(hp->adma2, 0, PAGE_SIZE);
    535 
    536 adma_done:
    537 		if (error)
    538 			CLR(hp->flags, SHF_USE_ADMA2_MASK);
    539 	}
    540 
    541 	/*
    542 	 * Attach the generic SD/MMC bus driver.  (The bus driver must
    543 	 * not invoke any chipset functions before it is attached.)
    544 	 */
    545 	memset(&saa, 0, sizeof(saa));
    546 	saa.saa_busname = "sdmmc";
    547 	saa.saa_sct = &sdhc_functions;
    548 	saa.saa_sch = hp;
    549 	saa.saa_dmat = hp->dmat;
    550 	saa.saa_clkmax = hp->clkbase;
    551 	if (ISSET(sc->sc_flags, SDHC_FLAG_HAVE_CGM))
    552 		saa.saa_clkmin = hp->clkbase / 256 / 2046;
    553 	else if (ISSET(sc->sc_flags, SDHC_FLAG_HAVE_DVS))
    554 		saa.saa_clkmin = hp->clkbase / 256 / 16;
    555 	else if (hp->sc->sc_clkmsk != 0)
    556 		saa.saa_clkmin = hp->clkbase / (hp->sc->sc_clkmsk >>
    557 		    (ffs(hp->sc->sc_clkmsk) - 1));
    558 	else if (hp->specver >= SDHC_SPEC_VERS_300)
    559 		saa.saa_clkmin = hp->clkbase / 0x3ff;
    560 	else
    561 		saa.saa_clkmin = hp->clkbase / 256;
    562 	saa.saa_caps = SMC_CAPS_4BIT_MODE|SMC_CAPS_AUTO_STOP;
    563 	if (ISSET(sc->sc_flags, SDHC_FLAG_8BIT_MODE))
    564 		saa.saa_caps |= SMC_CAPS_8BIT_MODE;
    565 	if (ISSET(caps, SDHC_HIGH_SPEED_SUPP))
    566 		saa.saa_caps |= SMC_CAPS_SD_HIGHSPEED;
    567 	if (ISSET(caps2, SDHC_SDR104_SUPP))
    568 		saa.saa_caps |= SMC_CAPS_UHS_SDR104 |
    569 				SMC_CAPS_UHS_SDR50 |
    570 				SMC_CAPS_MMC_HS200;
    571 	if (ISSET(caps2, SDHC_SDR50_SUPP))
    572 		saa.saa_caps |= SMC_CAPS_UHS_SDR50;
    573 	if (ISSET(caps2, SDHC_DDR50_SUPP))
    574 		saa.saa_caps |= SMC_CAPS_UHS_DDR50;
    575 	if (ISSET(hp->flags, SHF_USE_DMA)) {
    576 		saa.saa_caps |= SMC_CAPS_DMA;
    577 		if (!ISSET(hp->sc->sc_flags, SDHC_FLAG_ENHANCED))
    578 			saa.saa_caps |= SMC_CAPS_MULTI_SEG_DMA;
    579 	}
    580 	if (ISSET(sc->sc_flags, SDHC_FLAG_SINGLE_ONLY))
    581 		saa.saa_caps |= SMC_CAPS_SINGLE_ONLY;
    582 	if (ISSET(sc->sc_flags, SDHC_FLAG_POLL_CARD_DET))
    583 		saa.saa_caps |= SMC_CAPS_POLL_CARD_DET;
    584 	hp->sdmmc = config_found(sc->sc_dev, &saa, sdhc_cfprint);
    585 
    586 	return 0;
    587 
    588 err:
    589 	callout_destroy(&hp->tuning_timer);
    590 	cv_destroy(&hp->intr_cv);
    591 	mutex_destroy(&hp->intr_lock);
    592 	free(hp, M_DEVBUF);
    593 	sc->sc_host[--sc->sc_nhosts] = NULL;
    594 err1:
    595 	return 1;
    596 }
    597 
    598 int
    599 sdhc_detach(struct sdhc_softc *sc, int flags)
    600 {
    601 	struct sdhc_host *hp;
    602 	int rv = 0;
    603 
    604 	for (size_t n = 0; n < sc->sc_nhosts; n++) {
    605 		hp = sc->sc_host[n];
    606 		if (hp == NULL)
    607 			continue;
    608 		if (hp->sdmmc != NULL) {
    609 			rv = config_detach(hp->sdmmc, flags);
    610 			if (rv)
    611 				break;
    612 			hp->sdmmc = NULL;
    613 		}
    614 		/* disable interrupts */
    615 		if ((flags & DETACH_FORCE) == 0) {
    616 			mutex_enter(&hp->intr_lock);
    617 			if (ISSET(hp->sc->sc_flags, SDHC_FLAG_32BIT_ACCESS)) {
    618 				HWRITE4(hp, SDHC_NINTR_SIGNAL_EN, 0);
    619 			} else {
    620 				HWRITE2(hp, SDHC_NINTR_SIGNAL_EN, 0);
    621 			}
    622 			sdhc_soft_reset(hp, SDHC_RESET_ALL);
    623 			mutex_exit(&hp->intr_lock);
    624 		}
    625 		callout_halt(&hp->tuning_timer, NULL);
    626 		callout_destroy(&hp->tuning_timer);
    627 		cv_destroy(&hp->intr_cv);
    628 		mutex_destroy(&hp->intr_lock);
    629 		if (hp->ios > 0) {
    630 			bus_space_unmap(hp->iot, hp->ioh, hp->ios);
    631 			hp->ios = 0;
    632 		}
    633 		if (ISSET(hp->flags, SHF_USE_ADMA2_MASK)) {
    634 			bus_dmamap_unload(sc->sc_dmat, hp->adma_map);
    635 			bus_dmamap_destroy(sc->sc_dmat, hp->adma_map);
    636 			bus_dmamem_unmap(sc->sc_dmat, hp->adma2, PAGE_SIZE);
    637 			bus_dmamem_free(sc->sc_dmat, hp->adma_segs, 1);
    638 		}
    639 		free(hp, M_DEVBUF);
    640 		sc->sc_host[n] = NULL;
    641 	}
    642 
    643 	return rv;
    644 }
    645 
    646 bool
    647 sdhc_suspend(device_t dev, const pmf_qual_t *qual)
    648 {
    649 	struct sdhc_softc *sc = device_private(dev);
    650 	struct sdhc_host *hp;
    651 	size_t i;
    652 
    653 	/* XXX poll for command completion or suspend command
    654 	 * in progress */
    655 
    656 	/* Save the host controller state. */
    657 	for (size_t n = 0; n < sc->sc_nhosts; n++) {
    658 		hp = sc->sc_host[n];
    659 		if (ISSET(sc->sc_flags, SDHC_FLAG_32BIT_ACCESS)) {
    660 			for (i = 0; i < sizeof hp->regs; i += 4) {
    661 				uint32_t v = HREAD4(hp, i);
    662 				hp->regs[i + 0] = (v >> 0);
    663 				hp->regs[i + 1] = (v >> 8);
    664 				if (i + 3 < sizeof hp->regs) {
    665 					hp->regs[i + 2] = (v >> 16);
    666 					hp->regs[i + 3] = (v >> 24);
    667 				}
    668 			}
    669 		} else {
    670 			for (i = 0; i < sizeof hp->regs; i++) {
    671 				hp->regs[i] = HREAD1(hp, i);
    672 			}
    673 		}
    674 	}
    675 	return true;
    676 }
    677 
    678 bool
    679 sdhc_resume(device_t dev, const pmf_qual_t *qual)
    680 {
    681 	struct sdhc_softc *sc = device_private(dev);
    682 	struct sdhc_host *hp;
    683 	size_t i;
    684 
    685 	/* Restore the host controller state. */
    686 	for (size_t n = 0; n < sc->sc_nhosts; n++) {
    687 		hp = sc->sc_host[n];
    688 		(void)sdhc_host_reset(hp);
    689 		if (ISSET(sc->sc_flags, SDHC_FLAG_32BIT_ACCESS)) {
    690 			for (i = 0; i < sizeof hp->regs; i += 4) {
    691 				if (i + 3 < sizeof hp->regs) {
    692 					HWRITE4(hp, i,
    693 					    (hp->regs[i + 0] << 0)
    694 					    | (hp->regs[i + 1] << 8)
    695 					    | (hp->regs[i + 2] << 16)
    696 					    | (hp->regs[i + 3] << 24));
    697 				} else {
    698 					HWRITE4(hp, i,
    699 					    (hp->regs[i + 0] << 0)
    700 					    | (hp->regs[i + 1] << 8));
    701 				}
    702 			}
    703 		} else {
    704 			for (i = 0; i < sizeof hp->regs; i++) {
    705 				HWRITE1(hp, i, hp->regs[i]);
    706 			}
    707 		}
    708 	}
    709 	return true;
    710 }
    711 
    712 bool
    713 sdhc_shutdown(device_t dev, int flags)
    714 {
    715 	struct sdhc_softc *sc = device_private(dev);
    716 	struct sdhc_host *hp;
    717 
    718 	/* XXX chip locks up if we don't disable it before reboot. */
    719 	for (size_t i = 0; i < sc->sc_nhosts; i++) {
    720 		hp = sc->sc_host[i];
    721 		(void)sdhc_host_reset(hp);
    722 	}
    723 	return true;
    724 }
    725 
    726 /*
    727  * Reset the host controller.  Called during initialization, when
    728  * cards are removed, upon resume, and during error recovery.
    729  */
    730 static int
    731 sdhc_host_reset1(sdmmc_chipset_handle_t sch)
    732 {
    733 	struct sdhc_host *hp = (struct sdhc_host *)sch;
    734 	uint32_t sdhcimask;
    735 	int error;
    736 
    737 	KASSERT(mutex_owned(&hp->intr_lock));
    738 
    739 	/* Disable all interrupts. */
    740 	if (ISSET(hp->sc->sc_flags, SDHC_FLAG_32BIT_ACCESS)) {
    741 		HWRITE4(hp, SDHC_NINTR_SIGNAL_EN, 0);
    742 	} else {
    743 		HWRITE2(hp, SDHC_NINTR_SIGNAL_EN, 0);
    744 	}
    745 
    746 	/*
    747 	 * Reset the entire host controller and wait up to 100ms for
    748 	 * the controller to clear the reset bit.
    749 	 */
    750 	error = sdhc_soft_reset(hp, SDHC_RESET_ALL);
    751 	if (error)
    752 		goto out;
    753 
    754 	/* Set data timeout counter value to max for now. */
    755 	HWRITE1(hp, SDHC_TIMEOUT_CTL, SDHC_TIMEOUT_MAX);
    756 #if 1
    757 	if (ISSET(hp->sc->sc_flags, SDHC_FLAG_ENHANCED))
    758 		HWRITE4(hp, SDHC_NINTR_STATUS, SDHC_CMD_TIMEOUT_ERROR << 16);
    759 #endif
    760 
    761 	/* Enable interrupts. */
    762 	sdhcimask = SDHC_CARD_REMOVAL | SDHC_CARD_INSERTION |
    763 	    SDHC_BUFFER_READ_READY | SDHC_BUFFER_WRITE_READY |
    764 	    SDHC_DMA_INTERRUPT | SDHC_BLOCK_GAP_EVENT |
    765 	    SDHC_TRANSFER_COMPLETE | SDHC_COMMAND_COMPLETE;
    766 	if (ISSET(hp->sc->sc_flags, SDHC_FLAG_32BIT_ACCESS)) {
    767 		sdhcimask |= SDHC_EINTR_STATUS_MASK << 16;
    768 		HWRITE4(hp, SDHC_NINTR_STATUS_EN, sdhcimask);
    769 		sdhcimask ^=
    770 		    (SDHC_EINTR_STATUS_MASK ^ SDHC_EINTR_SIGNAL_MASK) << 16;
    771 		sdhcimask ^= SDHC_BUFFER_READ_READY ^ SDHC_BUFFER_WRITE_READY;
    772 		HWRITE4(hp, SDHC_NINTR_SIGNAL_EN, sdhcimask);
    773 	} else {
    774 		HWRITE2(hp, SDHC_NINTR_STATUS_EN, sdhcimask);
    775 		HWRITE2(hp, SDHC_EINTR_STATUS_EN, SDHC_EINTR_STATUS_MASK);
    776 		sdhcimask ^= SDHC_BUFFER_READ_READY ^ SDHC_BUFFER_WRITE_READY;
    777 		HWRITE2(hp, SDHC_NINTR_SIGNAL_EN, sdhcimask);
    778 		HWRITE2(hp, SDHC_EINTR_SIGNAL_EN, SDHC_EINTR_SIGNAL_MASK);
    779 	}
    780 
    781 out:
    782 	return error;
    783 }
    784 
    785 static int
    786 sdhc_host_reset(sdmmc_chipset_handle_t sch)
    787 {
    788 	struct sdhc_host *hp = (struct sdhc_host *)sch;
    789 	int error;
    790 
    791 	mutex_enter(&hp->intr_lock);
    792 	error = sdhc_host_reset1(sch);
    793 	mutex_exit(&hp->intr_lock);
    794 
    795 	return error;
    796 }
    797 
    798 static uint32_t
    799 sdhc_host_ocr(sdmmc_chipset_handle_t sch)
    800 {
    801 	struct sdhc_host *hp = (struct sdhc_host *)sch;
    802 
    803 	return hp->ocr;
    804 }
    805 
    806 static int
    807 sdhc_host_maxblklen(sdmmc_chipset_handle_t sch)
    808 {
    809 	struct sdhc_host *hp = (struct sdhc_host *)sch;
    810 
    811 	return hp->maxblklen;
    812 }
    813 
    814 /*
    815  * Return non-zero if the card is currently inserted.
    816  */
    817 static int
    818 sdhc_card_detect(sdmmc_chipset_handle_t sch)
    819 {
    820 	struct sdhc_host *hp = (struct sdhc_host *)sch;
    821 	int r;
    822 
    823 	if (hp->sc->sc_vendor_card_detect)
    824 		return (*hp->sc->sc_vendor_card_detect)(hp->sc);
    825 
    826 	r = ISSET(HREAD4(hp, SDHC_PRESENT_STATE), SDHC_CARD_INSERTED);
    827 
    828 	return r ? 1 : 0;
    829 }
    830 
    831 /*
    832  * Return non-zero if the card is currently write-protected.
    833  */
    834 static int
    835 sdhc_write_protect(sdmmc_chipset_handle_t sch)
    836 {
    837 	struct sdhc_host *hp = (struct sdhc_host *)sch;
    838 	int r;
    839 
    840 	if (hp->sc->sc_vendor_write_protect)
    841 		return (*hp->sc->sc_vendor_write_protect)(hp->sc);
    842 
    843 	r = ISSET(HREAD4(hp, SDHC_PRESENT_STATE), SDHC_WRITE_PROTECT_SWITCH);
    844 
    845 	return r ? 0 : 1;
    846 }
    847 
    848 /*
    849  * Set or change SD bus voltage and enable or disable SD bus power.
    850  * Return zero on success.
    851  */
    852 static int
    853 sdhc_bus_power(sdmmc_chipset_handle_t sch, uint32_t ocr)
    854 {
    855 	struct sdhc_host *hp = (struct sdhc_host *)sch;
    856 	uint8_t vdd;
    857 	int error = 0;
    858 	const uint32_t pcmask =
    859 	    ~(SDHC_BUS_POWER | (SDHC_VOLTAGE_MASK << SDHC_VOLTAGE_SHIFT));
    860 
    861 	mutex_enter(&hp->intr_lock);
    862 
    863 	/*
    864 	 * Disable bus power before voltage change.
    865 	 */
    866 	if (!ISSET(hp->sc->sc_flags, SDHC_FLAG_32BIT_ACCESS)
    867 	    && !ISSET(hp->sc->sc_flags, SDHC_FLAG_NO_PWR0))
    868 		HWRITE1(hp, SDHC_POWER_CTL, 0);
    869 
    870 	/* If power is disabled, reset the host and return now. */
    871 	if (ocr == 0) {
    872 		(void)sdhc_host_reset1(hp);
    873 		callout_halt(&hp->tuning_timer, &hp->intr_lock);
    874 		goto out;
    875 	}
    876 
    877 	/*
    878 	 * Select the lowest voltage according to capabilities.
    879 	 */
    880 	ocr &= hp->ocr;
    881 	if (ISSET(ocr, MMC_OCR_1_7V_1_8V|MMC_OCR_1_8V_1_9V)) {
    882 		vdd = SDHC_VOLTAGE_1_8V;
    883 	} else if (ISSET(ocr, MMC_OCR_2_9V_3_0V|MMC_OCR_3_0V_3_1V)) {
    884 		vdd = SDHC_VOLTAGE_3_0V;
    885 	} else if (ISSET(ocr, MMC_OCR_3_2V_3_3V|MMC_OCR_3_3V_3_4V)) {
    886 		vdd = SDHC_VOLTAGE_3_3V;
    887 	} else {
    888 		/* Unsupported voltage level requested. */
    889 		error = EINVAL;
    890 		goto out;
    891 	}
    892 
    893 	if (!ISSET(hp->sc->sc_flags, SDHC_FLAG_ENHANCED)) {
    894 		/*
    895 		 * Enable bus power.  Wait at least 1 ms (or 74 clocks) plus
    896 		 * voltage ramp until power rises.
    897 		 */
    898 
    899 		if (ISSET(hp->sc->sc_flags, SDHC_FLAG_SINGLE_POWER_WRITE)) {
    900 			HWRITE1(hp, SDHC_POWER_CTL,
    901 			    (vdd << SDHC_VOLTAGE_SHIFT) | SDHC_BUS_POWER);
    902 		} else {
    903 			HWRITE1(hp, SDHC_POWER_CTL,
    904 			    HREAD1(hp, SDHC_POWER_CTL) & pcmask);
    905 			sdmmc_delay(1);
    906 			HWRITE1(hp, SDHC_POWER_CTL,
    907 			    (vdd << SDHC_VOLTAGE_SHIFT));
    908 			sdmmc_delay(1);
    909 			HSET1(hp, SDHC_POWER_CTL, SDHC_BUS_POWER);
    910 			sdmmc_delay(10000);
    911 		}
    912 
    913 		/*
    914 		 * The host system may not power the bus due to battery low,
    915 		 * etc.  In that case, the host controller should clear the
    916 		 * bus power bit.
    917 		 */
    918 		if (!ISSET(HREAD1(hp, SDHC_POWER_CTL), SDHC_BUS_POWER)) {
    919 			error = ENXIO;
    920 			goto out;
    921 		}
    922 	}
    923 
    924 out:
    925 	mutex_exit(&hp->intr_lock);
    926 
    927 	return error;
    928 }
    929 
    930 /*
    931  * Return the smallest possible base clock frequency divisor value
    932  * for the CLOCK_CTL register to produce `freq' (KHz).
    933  */
    934 static bool
    935 sdhc_clock_divisor(struct sdhc_host *hp, u_int freq, u_int *divp)
    936 {
    937 	u_int div;
    938 
    939 	if (ISSET(hp->sc->sc_flags, SDHC_FLAG_HAVE_CGM)) {
    940 		for (div = hp->clkbase / freq; div <= 0x3ff; div++) {
    941 			if ((hp->clkbase / div) <= freq) {
    942 				*divp = SDHC_SDCLK_CGM
    943 				    | ((div & 0x300) << SDHC_SDCLK_XDIV_SHIFT)
    944 				    | ((div & 0x0ff) << SDHC_SDCLK_DIV_SHIFT);
    945 				//freq = hp->clkbase / div;
    946 				return true;
    947 			}
    948 		}
    949 		/* No divisor found. */
    950 		return false;
    951 	}
    952 	if (ISSET(hp->sc->sc_flags, SDHC_FLAG_HAVE_DVS)) {
    953 		u_int dvs = (hp->clkbase + freq - 1) / freq;
    954 		u_int roundup = dvs & 1;
    955 		for (dvs >>= 1, div = 1; div <= 256; div <<= 1, dvs >>= 1) {
    956 			if (dvs + roundup <= 16) {
    957 				dvs += roundup - 1;
    958 				*divp = (div << SDHC_SDCLK_DIV_SHIFT)
    959 				    |   (dvs << SDHC_SDCLK_DVS_SHIFT);
    960 				DPRINTF(2,
    961 				    ("%s: divisor for freq %u is %u * %u\n",
    962 				    HDEVNAME(hp), freq, div * 2, dvs + 1));
    963 				//freq = hp->clkbase / (div * 2) * (dvs + 1);
    964 				return true;
    965 			}
    966 			/*
    967 			 * If we drop bits, we need to round up the divisor.
    968 			 */
    969 			roundup |= dvs & 1;
    970 		}
    971 		/* No divisor found. */
    972 		return false;
    973 	}
    974 	if (hp->sc->sc_clkmsk != 0) {
    975 		div = howmany(hp->clkbase, freq);
    976 		if (div > (hp->sc->sc_clkmsk >> (ffs(hp->sc->sc_clkmsk) - 1)))
    977 			return false;
    978 		*divp = div << (ffs(hp->sc->sc_clkmsk) - 1);
    979 		//freq = hp->clkbase / div;
    980 		return true;
    981 	}
    982 	if (hp->specver >= SDHC_SPEC_VERS_300) {
    983 		div = howmany(hp->clkbase, freq);
    984 		div = div > 1 ? howmany(div, 2) : 0;
    985 		if (div > 0x3ff)
    986 			return false;
    987 		*divp = (((div >> 8) & SDHC_SDCLK_XDIV_MASK)
    988 			 << SDHC_SDCLK_XDIV_SHIFT) |
    989 			(((div >> 0) & SDHC_SDCLK_DIV_MASK)
    990 			 << SDHC_SDCLK_DIV_SHIFT);
    991 		//freq = hp->clkbase / (div ? div * 2 : 1);
    992 		return true;
    993 	} else {
    994 		for (div = 1; div <= 256; div *= 2) {
    995 			if ((hp->clkbase / div) <= freq) {
    996 				*divp = (div / 2) << SDHC_SDCLK_DIV_SHIFT;
    997 				//freq = hp->clkbase / div;
    998 				return true;
    999 			}
   1000 		}
   1001 		/* No divisor found. */
   1002 		return false;
   1003 	}
   1004 	/* No divisor found. */
   1005 	return false;
   1006 }
   1007 
   1008 /*
   1009  * Set or change SDCLK frequency or disable the SD clock.
   1010  * Return zero on success.
   1011  */
   1012 static int
   1013 sdhc_bus_clock_ddr(sdmmc_chipset_handle_t sch, int freq, bool ddr)
   1014 {
   1015 	struct sdhc_host *hp = (struct sdhc_host *)sch;
   1016 	u_int div;
   1017 	u_int timo;
   1018 	int16_t reg;
   1019 	int error = 0;
   1020 	bool present __diagused;
   1021 
   1022 	mutex_enter(&hp->intr_lock);
   1023 
   1024 #ifdef DIAGNOSTIC
   1025 	present = ISSET(HREAD4(hp, SDHC_PRESENT_STATE), SDHC_CMD_INHIBIT_MASK);
   1026 
   1027 	/* Must not stop the clock if commands are in progress. */
   1028 	if (present && sdhc_card_detect(hp)) {
   1029 		aprint_normal_dev(hp->sc->sc_dev,
   1030 		    "%s: command in progress\n", __func__);
   1031 	}
   1032 #endif
   1033 
   1034 	if (hp->sc->sc_vendor_bus_clock) {
   1035 		error = (*hp->sc->sc_vendor_bus_clock)(hp->sc, freq);
   1036 		if (error != 0)
   1037 			goto out;
   1038 	}
   1039 
   1040 	/*
   1041 	 * Stop SD clock before changing the frequency.
   1042 	 */
   1043 	if (ISSET(hp->sc->sc_flags, SDHC_FLAG_ENHANCED)) {
   1044 		HCLR4(hp, SDHC_CLOCK_CTL, 0xfff8);
   1045 		if (freq == SDMMC_SDCLK_OFF) {
   1046 			HSET4(hp, SDHC_CLOCK_CTL, 0x80f0);
   1047 			goto out;
   1048 		}
   1049 	} else {
   1050 		HCLR2(hp, SDHC_CLOCK_CTL, SDHC_SDCLK_ENABLE);
   1051 		if (freq == SDMMC_SDCLK_OFF)
   1052 			goto out;
   1053 	}
   1054 
   1055 	if (hp->specver >= SDHC_SPEC_VERS_300) {
   1056 		HCLR2(hp, SDHC_HOST_CTL2, SDHC_UHS_MODE_SELECT_MASK);
   1057 		if (freq > 100000) {
   1058 			HSET2(hp, SDHC_HOST_CTL2, SDHC_UHS_MODE_SELECT_SDR104);
   1059 		} else if (freq > 50000) {
   1060 			HSET2(hp, SDHC_HOST_CTL2, SDHC_UHS_MODE_SELECT_SDR50);
   1061 		} else if (freq > 25000) {
   1062 			if (ddr) {
   1063 				HSET2(hp, SDHC_HOST_CTL2,
   1064 				    SDHC_UHS_MODE_SELECT_DDR50);
   1065 			} else {
   1066 				HSET2(hp, SDHC_HOST_CTL2,
   1067 				    SDHC_UHS_MODE_SELECT_SDR25);
   1068 			}
   1069 		} else if (freq > 400) {
   1070 			HSET2(hp, SDHC_HOST_CTL2, SDHC_UHS_MODE_SELECT_SDR12);
   1071 		}
   1072 	}
   1073 
   1074 	/*
   1075 	 * Slow down Ricoh 5U823 controller that isn't reliable
   1076 	 * at 100MHz bus clock.
   1077 	 */
   1078 	if (ISSET(hp->sc->sc_flags, SDHC_FLAG_SLOW_SDR50)) {
   1079 		if (freq == 100000)
   1080 			--freq;
   1081 	}
   1082 
   1083 	/*
   1084 	 * Set the minimum base clock frequency divisor.
   1085 	 */
   1086 	if (!sdhc_clock_divisor(hp, freq, &div)) {
   1087 		/* Invalid base clock frequency or `freq' value. */
   1088 		aprint_error_dev(hp->sc->sc_dev,
   1089 			"Invalid bus clock %d kHz\n", freq);
   1090 		error = EINVAL;
   1091 		goto out;
   1092 	}
   1093 	if (ISSET(hp->sc->sc_flags, SDHC_FLAG_ENHANCED)) {
   1094 		HWRITE4(hp, SDHC_CLOCK_CTL,
   1095 		    div | (SDHC_TIMEOUT_MAX << 16));
   1096 	} else {
   1097 		reg = HREAD2(hp, SDHC_CLOCK_CTL);
   1098 		reg &= (SDHC_INTCLK_STABLE | SDHC_INTCLK_ENABLE);
   1099 		HWRITE2(hp, SDHC_CLOCK_CTL, reg | div);
   1100 	}
   1101 
   1102 	/*
   1103 	 * Start internal clock.  Wait 10ms for stabilization.
   1104 	 */
   1105 	if (ISSET(hp->sc->sc_flags, SDHC_FLAG_ENHANCED)) {
   1106 		sdmmc_delay(10000);
   1107 		HSET4(hp, SDHC_CLOCK_CTL,
   1108 		    8 | SDHC_INTCLK_ENABLE | SDHC_INTCLK_STABLE);
   1109 	} else {
   1110 		HSET2(hp, SDHC_CLOCK_CTL, SDHC_INTCLK_ENABLE);
   1111 		for (timo = 1000; timo > 0; timo--) {
   1112 			if (ISSET(HREAD2(hp, SDHC_CLOCK_CTL),
   1113 			    SDHC_INTCLK_STABLE))
   1114 				break;
   1115 			sdmmc_delay(10);
   1116 		}
   1117 		if (timo == 0) {
   1118 			error = ETIMEDOUT;
   1119 			goto out;
   1120 		}
   1121 	}
   1122 
   1123 	if (ISSET(hp->sc->sc_flags, SDHC_FLAG_ENHANCED)) {
   1124 		HSET1(hp, SDHC_SOFTWARE_RESET, SDHC_INIT_ACTIVE);
   1125 		/*
   1126 		 * Sending 80 clocks at 400kHz takes 200us.
   1127 		 * So delay for that time + slop and then
   1128 		 * check a few times for completion.
   1129 		 */
   1130 		sdmmc_delay(210);
   1131 		for (timo = 10; timo > 0; timo--) {
   1132 			if (!ISSET(HREAD1(hp, SDHC_SOFTWARE_RESET),
   1133 			    SDHC_INIT_ACTIVE))
   1134 				break;
   1135 			sdmmc_delay(10);
   1136 		}
   1137 		DPRINTF(2,("%s: %u init spins\n", __func__, 10 - timo));
   1138 
   1139 		/*
   1140 		 * Enable SD clock.
   1141 		 */
   1142 		HSET4(hp, SDHC_CLOCK_CTL, SDHC_SDCLK_ENABLE);
   1143 	} else {
   1144 		/*
   1145 		 * Enable SD clock.
   1146 		 */
   1147 		HSET2(hp, SDHC_CLOCK_CTL, SDHC_SDCLK_ENABLE);
   1148 
   1149 		if (freq > 25000 &&
   1150 		    !ISSET(hp->sc->sc_flags, SDHC_FLAG_NO_HS_BIT))
   1151 			HSET1(hp, SDHC_HOST_CTL, SDHC_HIGH_SPEED);
   1152 		else
   1153 			HCLR1(hp, SDHC_HOST_CTL, SDHC_HIGH_SPEED);
   1154 	}
   1155 
   1156 out:
   1157 	mutex_exit(&hp->intr_lock);
   1158 
   1159 	return error;
   1160 }
   1161 
   1162 static int
   1163 sdhc_bus_width(sdmmc_chipset_handle_t sch, int width)
   1164 {
   1165 	struct sdhc_host *hp = (struct sdhc_host *)sch;
   1166 	int reg;
   1167 
   1168 	switch (width) {
   1169 	case 1:
   1170 	case 4:
   1171 		break;
   1172 
   1173 	case 8:
   1174 		if (ISSET(hp->sc->sc_flags, SDHC_FLAG_8BIT_MODE))
   1175 			break;
   1176 		/* FALLTHROUGH */
   1177 	default:
   1178 		DPRINTF(0,("%s: unsupported bus width (%d)\n",
   1179 		    HDEVNAME(hp), width));
   1180 		return 1;
   1181 	}
   1182 
   1183 	mutex_enter(&hp->intr_lock);
   1184 
   1185 	reg = HREAD1(hp, SDHC_HOST_CTL);
   1186 	if (ISSET(hp->sc->sc_flags, SDHC_FLAG_ENHANCED)) {
   1187 		reg &= ~(SDHC_4BIT_MODE|SDHC_ESDHC_8BIT_MODE);
   1188 		if (width == 4)
   1189 			reg |= SDHC_4BIT_MODE;
   1190 		else if (width == 8)
   1191 			reg |= SDHC_ESDHC_8BIT_MODE;
   1192 	} else {
   1193 		reg &= ~SDHC_4BIT_MODE;
   1194 		if (hp->specver >= SDHC_SPEC_VERS_300) {
   1195 			reg &= ~SDHC_8BIT_MODE;
   1196 		}
   1197 		if (width == 4) {
   1198 			reg |= SDHC_4BIT_MODE;
   1199 		} else if (width == 8 && hp->specver >= SDHC_SPEC_VERS_300) {
   1200 			reg |= SDHC_8BIT_MODE;
   1201 		}
   1202 	}
   1203 	HWRITE1(hp, SDHC_HOST_CTL, reg);
   1204 
   1205 	mutex_exit(&hp->intr_lock);
   1206 
   1207 	return 0;
   1208 }
   1209 
   1210 static int
   1211 sdhc_bus_rod(sdmmc_chipset_handle_t sch, int on)
   1212 {
   1213 	struct sdhc_host *hp = (struct sdhc_host *)sch;
   1214 
   1215 	if (hp->sc->sc_vendor_rod)
   1216 		return (*hp->sc->sc_vendor_rod)(hp->sc, on);
   1217 
   1218 	return 0;
   1219 }
   1220 
   1221 static void
   1222 sdhc_card_enable_intr(sdmmc_chipset_handle_t sch, int enable)
   1223 {
   1224 	struct sdhc_host *hp = (struct sdhc_host *)sch;
   1225 
   1226 	if (!ISSET(hp->sc->sc_flags, SDHC_FLAG_ENHANCED)) {
   1227 		mutex_enter(&hp->intr_lock);
   1228 		if (enable) {
   1229 			HSET2(hp, SDHC_NINTR_STATUS_EN, SDHC_CARD_INTERRUPT);
   1230 			HSET2(hp, SDHC_NINTR_SIGNAL_EN, SDHC_CARD_INTERRUPT);
   1231 		} else {
   1232 			HCLR2(hp, SDHC_NINTR_SIGNAL_EN, SDHC_CARD_INTERRUPT);
   1233 			HCLR2(hp, SDHC_NINTR_STATUS_EN, SDHC_CARD_INTERRUPT);
   1234 		}
   1235 		mutex_exit(&hp->intr_lock);
   1236 	}
   1237 }
   1238 
   1239 static void
   1240 sdhc_card_intr_ack(sdmmc_chipset_handle_t sch)
   1241 {
   1242 	struct sdhc_host *hp = (struct sdhc_host *)sch;
   1243 
   1244 	if (!ISSET(hp->sc->sc_flags, SDHC_FLAG_ENHANCED)) {
   1245 		mutex_enter(&hp->intr_lock);
   1246 		HSET2(hp, SDHC_NINTR_STATUS_EN, SDHC_CARD_INTERRUPT);
   1247 		mutex_exit(&hp->intr_lock);
   1248 	}
   1249 }
   1250 
   1251 static int
   1252 sdhc_signal_voltage(sdmmc_chipset_handle_t sch, int signal_voltage)
   1253 {
   1254 	struct sdhc_host *hp = (struct sdhc_host *)sch;
   1255 
   1256 	mutex_enter(&hp->intr_lock);
   1257 	switch (signal_voltage) {
   1258 	case SDMMC_SIGNAL_VOLTAGE_180:
   1259 		HSET2(hp, SDHC_HOST_CTL2, SDHC_1_8V_SIGNAL_EN);
   1260 		break;
   1261 	case SDMMC_SIGNAL_VOLTAGE_330:
   1262 		HCLR2(hp, SDHC_HOST_CTL2, SDHC_1_8V_SIGNAL_EN);
   1263 		break;
   1264 	default:
   1265 		return EINVAL;
   1266 	}
   1267 	mutex_exit(&hp->intr_lock);
   1268 
   1269 	return 0;
   1270 }
   1271 
   1272 /*
   1273  * Sampling clock tuning procedure (UHS)
   1274  */
   1275 static int
   1276 sdhc_execute_tuning(sdmmc_chipset_handle_t sch, int timing)
   1277 {
   1278 	struct sdhc_host *hp = (struct sdhc_host *)sch;
   1279 	struct sdmmc_command cmd;
   1280 	uint8_t hostctl;
   1281 	int opcode, error, retry = 40;
   1282 
   1283 	hp->tuning_timing = timing;
   1284 
   1285 	switch (timing) {
   1286 	case SDMMC_TIMING_MMC_HS200:
   1287 		opcode = MMC_SEND_TUNING_BLOCK_HS200;
   1288 		break;
   1289 	case SDMMC_TIMING_UHS_SDR50:
   1290 		if (!ISSET(hp->sc->sc_caps2, SDHC_TUNING_SDR50))
   1291 			return 0;
   1292 		/* FALLTHROUGH */
   1293 	case SDMMC_TIMING_UHS_SDR104:
   1294 		opcode = MMC_SEND_TUNING_BLOCK;
   1295 		break;
   1296 	default:
   1297 		return EINVAL;
   1298 	}
   1299 
   1300 	hostctl = HREAD1(hp, SDHC_HOST_CTL);
   1301 
   1302 	/* enable buffer read ready interrupt */
   1303 	HSET2(hp, SDHC_NINTR_SIGNAL_EN, SDHC_BUFFER_READ_READY);
   1304 	HSET2(hp, SDHC_NINTR_STATUS_EN, SDHC_BUFFER_READ_READY);
   1305 
   1306 	/* disable DMA */
   1307 	HCLR1(hp, SDHC_HOST_CTL, SDHC_DMA_SELECT);
   1308 
   1309 	/* reset tuning circuit */
   1310 	HCLR2(hp, SDHC_HOST_CTL2, SDHC_SAMPLING_CLOCK_SEL);
   1311 
   1312 	/* start of tuning */
   1313 	HWRITE2(hp, SDHC_HOST_CTL2, SDHC_EXECUTE_TUNING);
   1314 
   1315 	mutex_enter(&hp->intr_lock);
   1316 	do {
   1317 		memset(&cmd, 0, sizeof(cmd));
   1318 		cmd.c_opcode = opcode;
   1319 		cmd.c_arg = 0;
   1320 		cmd.c_flags = SCF_CMD_ADTC | SCF_CMD_READ | SCF_RSP_R1;
   1321 		if (ISSET(hostctl, SDHC_8BIT_MODE)) {
   1322 			cmd.c_blklen = cmd.c_datalen = 128;
   1323 		} else {
   1324 			cmd.c_blklen = cmd.c_datalen = 64;
   1325 		}
   1326 
   1327 		error = sdhc_start_command(hp, &cmd);
   1328 		if (error)
   1329 			break;
   1330 
   1331 		if (!sdhc_wait_intr(hp, SDHC_BUFFER_READ_READY,
   1332 		    SDHC_TUNING_TIMEOUT)) {
   1333 			break;
   1334 		}
   1335 
   1336 		delay(1000);
   1337 	} while (HREAD2(hp, SDHC_HOST_CTL2) & SDHC_EXECUTE_TUNING && --retry);
   1338 	mutex_exit(&hp->intr_lock);
   1339 
   1340 	/* disable buffer read ready interrupt */
   1341 	HCLR2(hp, SDHC_NINTR_SIGNAL_EN, SDHC_BUFFER_READ_READY);
   1342 	HCLR2(hp, SDHC_NINTR_STATUS_EN, SDHC_BUFFER_READ_READY);
   1343 
   1344 	if (HREAD2(hp, SDHC_HOST_CTL2) & SDHC_EXECUTE_TUNING) {
   1345 		HCLR2(hp, SDHC_HOST_CTL2,
   1346 		    SDHC_SAMPLING_CLOCK_SEL|SDHC_EXECUTE_TUNING);
   1347 		sdhc_soft_reset(hp, SDHC_RESET_DAT|SDHC_RESET_CMD);
   1348 		aprint_error_dev(hp->sc->sc_dev,
   1349 		    "tuning did not complete, using fixed sampling clock\n");
   1350 		return EIO;		/* tuning did not complete */
   1351 	}
   1352 
   1353 	if ((HREAD2(hp, SDHC_HOST_CTL2) & SDHC_SAMPLING_CLOCK_SEL) == 0) {
   1354 		HCLR2(hp, SDHC_HOST_CTL2,
   1355 		    SDHC_SAMPLING_CLOCK_SEL|SDHC_EXECUTE_TUNING);
   1356 		sdhc_soft_reset(hp, SDHC_RESET_DAT|SDHC_RESET_CMD);
   1357 		aprint_error_dev(hp->sc->sc_dev,
   1358 		    "tuning failed, using fixed sampling clock\n");
   1359 		return EIO;		/* tuning failed */
   1360 	}
   1361 
   1362 	if (hp->tuning_timer_count) {
   1363 		callout_schedule(&hp->tuning_timer,
   1364 		    hz * hp->tuning_timer_count);
   1365 	}
   1366 
   1367 	return 0;		/* tuning completed */
   1368 }
   1369 
   1370 static void
   1371 sdhc_tuning_timer(void *arg)
   1372 {
   1373 	struct sdhc_host *hp = arg;
   1374 
   1375 	atomic_swap_uint(&hp->tuning_timer_pending, 1);
   1376 }
   1377 
   1378 static int
   1379 sdhc_wait_state(struct sdhc_host *hp, uint32_t mask, uint32_t value)
   1380 {
   1381 	uint32_t state;
   1382 	int timeout;
   1383 
   1384 	for (timeout = 10000; timeout > 0; timeout--) {
   1385 		if (((state = HREAD4(hp, SDHC_PRESENT_STATE)) & mask) == value)
   1386 			return 0;
   1387 		sdmmc_delay(10);
   1388 	}
   1389 	aprint_error_dev(hp->sc->sc_dev, "timeout waiting for mask %#x value %#x (state=%#x)\n",
   1390 	    mask, value, state);
   1391 	return ETIMEDOUT;
   1392 }
   1393 
   1394 static void
   1395 sdhc_exec_command(sdmmc_chipset_handle_t sch, struct sdmmc_command *cmd)
   1396 {
   1397 	struct sdhc_host *hp = (struct sdhc_host *)sch;
   1398 	int error;
   1399 
   1400 	if (atomic_cas_uint(&hp->tuning_timer_pending, 1, 0) == 1) {
   1401 		(void)sdhc_execute_tuning(hp, hp->tuning_timing);
   1402 	}
   1403 
   1404 	mutex_enter(&hp->intr_lock);
   1405 
   1406 	if (cmd->c_data && ISSET(hp->sc->sc_flags, SDHC_FLAG_ENHANCED)) {
   1407 		const uint16_t ready = SDHC_BUFFER_READ_READY | SDHC_BUFFER_WRITE_READY;
   1408 		if (ISSET(hp->flags, SHF_USE_DMA)) {
   1409 			HCLR2(hp, SDHC_NINTR_SIGNAL_EN, ready);
   1410 			HCLR2(hp, SDHC_NINTR_STATUS_EN, ready);
   1411 		} else {
   1412 			HSET2(hp, SDHC_NINTR_SIGNAL_EN, ready);
   1413 			HSET2(hp, SDHC_NINTR_STATUS_EN, ready);
   1414 		}
   1415 	}
   1416 
   1417 	if (ISSET(hp->sc->sc_flags, SDHC_FLAG_NO_TIMEOUT)) {
   1418 		const uint16_t eintr = SDHC_CMD_TIMEOUT_ERROR;
   1419 		if (cmd->c_data != NULL) {
   1420 			HCLR2(hp, SDHC_EINTR_SIGNAL_EN, eintr);
   1421 			HCLR2(hp, SDHC_EINTR_STATUS_EN, eintr);
   1422 		} else {
   1423 			HSET2(hp, SDHC_EINTR_SIGNAL_EN, eintr);
   1424 			HSET2(hp, SDHC_EINTR_STATUS_EN, eintr);
   1425 		}
   1426 	}
   1427 
   1428 	/*
   1429 	 * Start the MMC command, or mark `cmd' as failed and return.
   1430 	 */
   1431 	error = sdhc_start_command(hp, cmd);
   1432 	if (error) {
   1433 		cmd->c_error = error;
   1434 		goto out;
   1435 	}
   1436 
   1437 	/*
   1438 	 * Wait until the command phase is done, or until the command
   1439 	 * is marked done for any other reason.
   1440 	 */
   1441 	if (!sdhc_wait_intr(hp, SDHC_COMMAND_COMPLETE, SDHC_COMMAND_TIMEOUT)) {
   1442 		cmd->c_error = ETIMEDOUT;
   1443 		goto out;
   1444 	}
   1445 
   1446 	/*
   1447 	 * The host controller removes bits [0:7] from the response
   1448 	 * data (CRC) and we pass the data up unchanged to the bus
   1449 	 * driver (without padding).
   1450 	 */
   1451 	if (cmd->c_error == 0 && ISSET(cmd->c_flags, SCF_RSP_PRESENT)) {
   1452 		cmd->c_resp[0] = HREAD4(hp, SDHC_RESPONSE + 0);
   1453 		if (ISSET(cmd->c_flags, SCF_RSP_136)) {
   1454 			cmd->c_resp[1] = HREAD4(hp, SDHC_RESPONSE + 4);
   1455 			cmd->c_resp[2] = HREAD4(hp, SDHC_RESPONSE + 8);
   1456 			cmd->c_resp[3] = HREAD4(hp, SDHC_RESPONSE + 12);
   1457 			if (ISSET(hp->sc->sc_flags, SDHC_FLAG_RSP136_CRC)) {
   1458 				cmd->c_resp[0] = (cmd->c_resp[0] >> 8) |
   1459 				    (cmd->c_resp[1] << 24);
   1460 				cmd->c_resp[1] = (cmd->c_resp[1] >> 8) |
   1461 				    (cmd->c_resp[2] << 24);
   1462 				cmd->c_resp[2] = (cmd->c_resp[2] >> 8) |
   1463 				    (cmd->c_resp[3] << 24);
   1464 				cmd->c_resp[3] = (cmd->c_resp[3] >> 8);
   1465 			}
   1466 		}
   1467 	}
   1468 	DPRINTF(1,("%s: resp = %08x\n", HDEVNAME(hp), cmd->c_resp[0]));
   1469 
   1470 	/*
   1471 	 * If the command has data to transfer in any direction,
   1472 	 * execute the transfer now.
   1473 	 */
   1474 	if (cmd->c_error == 0 && cmd->c_data != NULL)
   1475 		sdhc_transfer_data(hp, cmd);
   1476 	else if (ISSET(cmd->c_flags, SCF_RSP_BSY)) {
   1477 		if (!sdhc_wait_intr(hp, SDHC_TRANSFER_COMPLETE, hz * 10)) {
   1478 			cmd->c_error = ETIMEDOUT;
   1479 			goto out;
   1480 		}
   1481 	}
   1482 
   1483 out:
   1484 	if (!ISSET(hp->sc->sc_flags, SDHC_FLAG_ENHANCED)
   1485 	    && !ISSET(hp->sc->sc_flags, SDHC_FLAG_NO_LED_ON)) {
   1486 		/* Turn off the LED. */
   1487 		HCLR1(hp, SDHC_HOST_CTL, SDHC_LED_ON);
   1488 	}
   1489 	SET(cmd->c_flags, SCF_ITSDONE);
   1490 
   1491 	mutex_exit(&hp->intr_lock);
   1492 
   1493 	DPRINTF(1,("%s: cmd %d %s (flags=%08x error=%d)\n", HDEVNAME(hp),
   1494 	    cmd->c_opcode, (cmd->c_error == 0) ? "done" : "abort",
   1495 	    cmd->c_flags, cmd->c_error));
   1496 }
   1497 
   1498 static int
   1499 sdhc_start_command(struct sdhc_host *hp, struct sdmmc_command *cmd)
   1500 {
   1501 	struct sdhc_softc * const sc = hp->sc;
   1502 	uint16_t blksize = 0;
   1503 	uint16_t blkcount = 0;
   1504 	uint16_t mode;
   1505 	uint16_t command;
   1506 	int error;
   1507 
   1508 	KASSERT(mutex_owned(&hp->intr_lock));
   1509 
   1510 	DPRINTF(1,("%s: start cmd %d arg=%08x data=%p dlen=%d flags=%08x, status=%#x\n",
   1511 	    HDEVNAME(hp), cmd->c_opcode, cmd->c_arg, cmd->c_data,
   1512 	    cmd->c_datalen, cmd->c_flags, HREAD4(hp, SDHC_NINTR_STATUS)));
   1513 
   1514 	/*
   1515 	 * The maximum block length for commands should be the minimum
   1516 	 * of the host buffer size and the card buffer size. (1.7.2)
   1517 	 */
   1518 
   1519 	/* Fragment the data into proper blocks. */
   1520 	if (cmd->c_datalen > 0) {
   1521 		blksize = MIN(cmd->c_datalen, cmd->c_blklen);
   1522 		blkcount = cmd->c_datalen / blksize;
   1523 		if (cmd->c_datalen % blksize > 0) {
   1524 			/* XXX: Split this command. (1.7.4) */
   1525 			aprint_error_dev(sc->sc_dev,
   1526 			    "data not a multiple of %u bytes\n", blksize);
   1527 			return EINVAL;
   1528 		}
   1529 	}
   1530 
   1531 	/* Check limit imposed by 9-bit block count. (1.7.2) */
   1532 	if (blkcount > SDHC_BLOCK_COUNT_MAX) {
   1533 		aprint_error_dev(sc->sc_dev, "too much data\n");
   1534 		return EINVAL;
   1535 	}
   1536 
   1537 	/* Prepare transfer mode register value. (2.2.5) */
   1538 	mode = SDHC_BLOCK_COUNT_ENABLE;
   1539 	if (ISSET(cmd->c_flags, SCF_CMD_READ))
   1540 		mode |= SDHC_READ_MODE;
   1541 	if (blkcount > 1) {
   1542 		mode |= SDHC_MULTI_BLOCK_MODE;
   1543 		/* XXX only for memory commands? */
   1544 		mode |= SDHC_AUTO_CMD12_ENABLE;
   1545 	}
   1546 	if (cmd->c_dmamap != NULL && cmd->c_datalen > 0 &&
   1547 	    ISSET(hp->flags,  SHF_MODE_DMAEN)) {
   1548 		mode |= SDHC_DMA_ENABLE;
   1549 	}
   1550 
   1551 	/*
   1552 	 * Prepare command register value. (2.2.6)
   1553 	 */
   1554 	command = (cmd->c_opcode & SDHC_COMMAND_INDEX_MASK) << SDHC_COMMAND_INDEX_SHIFT;
   1555 
   1556 	if (ISSET(cmd->c_flags, SCF_RSP_CRC))
   1557 		command |= SDHC_CRC_CHECK_ENABLE;
   1558 	if (ISSET(cmd->c_flags, SCF_RSP_IDX))
   1559 		command |= SDHC_INDEX_CHECK_ENABLE;
   1560 	if (cmd->c_datalen > 0)
   1561 		command |= SDHC_DATA_PRESENT_SELECT;
   1562 
   1563 	if (!ISSET(cmd->c_flags, SCF_RSP_PRESENT))
   1564 		command |= SDHC_NO_RESPONSE;
   1565 	else if (ISSET(cmd->c_flags, SCF_RSP_136))
   1566 		command |= SDHC_RESP_LEN_136;
   1567 	else if (ISSET(cmd->c_flags, SCF_RSP_BSY))
   1568 		command |= SDHC_RESP_LEN_48_CHK_BUSY;
   1569 	else
   1570 		command |= SDHC_RESP_LEN_48;
   1571 
   1572 	/* Wait until command and data inhibit bits are clear. (1.5) */
   1573 	error = sdhc_wait_state(hp, SDHC_CMD_INHIBIT_MASK, 0);
   1574 	if (error) {
   1575 		aprint_error_dev(sc->sc_dev, "command or data phase inhibited\n");
   1576 		return error;
   1577 	}
   1578 
   1579 	DPRINTF(1,("%s: writing cmd: blksize=%d blkcnt=%d mode=%04x cmd=%04x\n",
   1580 	    HDEVNAME(hp), blksize, blkcount, mode, command));
   1581 
   1582 	if (!ISSET(hp->sc->sc_flags, SDHC_FLAG_ENHANCED)) {
   1583 		blksize |= (MAX(0, PAGE_SHIFT - 12) & SDHC_DMA_BOUNDARY_MASK) <<
   1584 		    SDHC_DMA_BOUNDARY_SHIFT;	/* PAGE_SIZE DMA boundary */
   1585 	}
   1586 
   1587 	if (!ISSET(hp->sc->sc_flags, SDHC_FLAG_ENHANCED)) {
   1588 		/* Alert the user not to remove the card. */
   1589 		HSET1(hp, SDHC_HOST_CTL, SDHC_LED_ON);
   1590 	}
   1591 
   1592 	/* Set DMA start address. */
   1593 	if (ISSET(hp->flags, SHF_USE_ADMA2_MASK) && cmd->c_data != NULL) {
   1594 		for (int seg = 0; seg < cmd->c_dmamap->dm_nsegs; seg++) {
   1595 			bus_addr_t paddr =
   1596 			    cmd->c_dmamap->dm_segs[seg].ds_addr;
   1597 			uint16_t len =
   1598 			    cmd->c_dmamap->dm_segs[seg].ds_len == 65536 ?
   1599 			    0 : cmd->c_dmamap->dm_segs[seg].ds_len;
   1600 			uint16_t attr =
   1601 			    SDHC_ADMA2_VALID | SDHC_ADMA2_ACT_TRANS;
   1602 			if (seg == cmd->c_dmamap->dm_nsegs - 1) {
   1603 				attr |= SDHC_ADMA2_END;
   1604 			}
   1605 			if (ISSET(hp->flags, SHF_USE_ADMA2_32)) {
   1606 				struct sdhc_adma2_descriptor32 *desc =
   1607 				    hp->adma2;
   1608 				desc[seg].attribute = htole16(attr);
   1609 				desc[seg].length = htole16(len);
   1610 				desc[seg].address = htole32(paddr);
   1611 			} else {
   1612 				struct sdhc_adma2_descriptor64 *desc =
   1613 				    hp->adma2;
   1614 				desc[seg].attribute = htole16(attr);
   1615 				desc[seg].length = htole16(len);
   1616 				desc[seg].address = htole32(paddr & 0xffffffff);
   1617 				desc[seg].address_hi = htole32(
   1618 				    (uint64_t)paddr >> 32);
   1619 			}
   1620 		}
   1621 		if (ISSET(hp->flags, SHF_USE_ADMA2_32)) {
   1622 			struct sdhc_adma2_descriptor32 *desc = hp->adma2;
   1623 			desc[cmd->c_dmamap->dm_nsegs].attribute = htole16(0);
   1624 		} else {
   1625 			struct sdhc_adma2_descriptor64 *desc = hp->adma2;
   1626 			desc[cmd->c_dmamap->dm_nsegs].attribute = htole16(0);
   1627 		}
   1628 		bus_dmamap_sync(sc->sc_dmat, hp->adma_map, 0, PAGE_SIZE,
   1629 		    BUS_DMASYNC_PREWRITE);
   1630 		HCLR1(hp, SDHC_HOST_CTL, SDHC_DMA_SELECT);
   1631 		HSET1(hp, SDHC_HOST_CTL, SDHC_DMA_SELECT_ADMA2);
   1632 
   1633 		const bus_addr_t desc_addr = hp->adma_map->dm_segs[0].ds_addr;
   1634 
   1635 		HWRITE4(hp, SDHC_ADMA_SYSTEM_ADDR, desc_addr & 0xffffffff);
   1636 		if (ISSET(hp->flags, SHF_USE_ADMA2_64)) {
   1637 			HWRITE4(hp, SDHC_ADMA_SYSTEM_ADDR + 4,
   1638 			    (uint64_t)desc_addr >> 32);
   1639 		}
   1640 	} else if (ISSET(mode, SDHC_DMA_ENABLE) &&
   1641 	    !ISSET(sc->sc_flags, SDHC_FLAG_EXTERNAL_DMA)) {
   1642 		HWRITE4(hp, SDHC_DMA_ADDR, cmd->c_dmamap->dm_segs[0].ds_addr);
   1643 	}
   1644 
   1645 	/*
   1646 	 * Start a CPU data transfer.  Writing to the high order byte
   1647 	 * of the SDHC_COMMAND register triggers the SD command. (1.5)
   1648 	 */
   1649 	if (ISSET(hp->sc->sc_flags, SDHC_FLAG_32BIT_ACCESS)) {
   1650 		HWRITE4(hp, SDHC_BLOCK_SIZE, blksize | (blkcount << 16));
   1651 		HWRITE4(hp, SDHC_ARGUMENT, cmd->c_arg);
   1652 		HWRITE4(hp, SDHC_TRANSFER_MODE, mode | (command << 16));
   1653 	} else {
   1654 		HWRITE2(hp, SDHC_BLOCK_SIZE, blksize);
   1655 		HWRITE2(hp, SDHC_BLOCK_COUNT, blkcount);
   1656 		HWRITE4(hp, SDHC_ARGUMENT, cmd->c_arg);
   1657 		HWRITE2(hp, SDHC_TRANSFER_MODE, mode);
   1658 		HWRITE2(hp, SDHC_COMMAND, command);
   1659 	}
   1660 
   1661 	return 0;
   1662 }
   1663 
   1664 static void
   1665 sdhc_transfer_data(struct sdhc_host *hp, struct sdmmc_command *cmd)
   1666 {
   1667 	struct sdhc_softc *sc = hp->sc;
   1668 	int error;
   1669 
   1670 	KASSERT(mutex_owned(&hp->intr_lock));
   1671 
   1672 	DPRINTF(1,("%s: data transfer: resp=%08x datalen=%u\n", HDEVNAME(hp),
   1673 	    MMC_R1(cmd->c_resp), cmd->c_datalen));
   1674 
   1675 #ifdef SDHC_DEBUG
   1676 	/* XXX I forgot why I wanted to know when this happens :-( */
   1677 	if ((cmd->c_opcode == 52 || cmd->c_opcode == 53) &&
   1678 	    ISSET(MMC_R1(cmd->c_resp), 0xcb00)) {
   1679 		aprint_error_dev(hp->sc->sc_dev,
   1680 		    "CMD52/53 error response flags %#x\n",
   1681 		    MMC_R1(cmd->c_resp) & 0xff00);
   1682 	}
   1683 #endif
   1684 
   1685 	if (cmd->c_dmamap != NULL) {
   1686 		if (hp->sc->sc_vendor_transfer_data_dma != NULL) {
   1687 			error = hp->sc->sc_vendor_transfer_data_dma(sc, cmd);
   1688 			if (error == 0 && !sdhc_wait_intr(hp,
   1689 			    SDHC_TRANSFER_COMPLETE, SDHC_DMA_TIMEOUT)) {
   1690 				error = ETIMEDOUT;
   1691 			}
   1692 		} else {
   1693 			error = sdhc_transfer_data_dma(hp, cmd);
   1694 		}
   1695 	} else
   1696 		error = sdhc_transfer_data_pio(hp, cmd);
   1697 	if (error)
   1698 		cmd->c_error = error;
   1699 	SET(cmd->c_flags, SCF_ITSDONE);
   1700 
   1701 	DPRINTF(1,("%s: data transfer done (error=%d)\n",
   1702 	    HDEVNAME(hp), cmd->c_error));
   1703 }
   1704 
   1705 static int
   1706 sdhc_transfer_data_dma(struct sdhc_host *hp, struct sdmmc_command *cmd)
   1707 {
   1708 	bus_dma_segment_t *dm_segs = cmd->c_dmamap->dm_segs;
   1709 	bus_addr_t posaddr;
   1710 	bus_addr_t segaddr;
   1711 	bus_size_t seglen;
   1712 	u_int seg = 0;
   1713 	int error = 0;
   1714 	int status;
   1715 
   1716 	KASSERT(mutex_owned(&hp->intr_lock));
   1717 	KASSERT(HREAD2(hp, SDHC_NINTR_STATUS_EN) & SDHC_DMA_INTERRUPT);
   1718 	KASSERT(HREAD2(hp, SDHC_NINTR_SIGNAL_EN) & SDHC_DMA_INTERRUPT);
   1719 	KASSERT(HREAD2(hp, SDHC_NINTR_STATUS_EN) & SDHC_TRANSFER_COMPLETE);
   1720 	KASSERT(HREAD2(hp, SDHC_NINTR_SIGNAL_EN) & SDHC_TRANSFER_COMPLETE);
   1721 
   1722 	for (;;) {
   1723 		status = sdhc_wait_intr(hp,
   1724 		    SDHC_DMA_INTERRUPT|SDHC_TRANSFER_COMPLETE,
   1725 		    SDHC_DMA_TIMEOUT);
   1726 
   1727 		if (status & SDHC_TRANSFER_COMPLETE) {
   1728 			break;
   1729 		}
   1730 		if (!status) {
   1731 			error = ETIMEDOUT;
   1732 			break;
   1733 		}
   1734 
   1735 		if (ISSET(hp->flags, SHF_USE_ADMA2_MASK)) {
   1736 			continue;
   1737 		}
   1738 
   1739 		if ((status & SDHC_DMA_INTERRUPT) == 0) {
   1740 			continue;
   1741 		}
   1742 
   1743 		/* DMA Interrupt (boundary crossing) */
   1744 
   1745 		segaddr = dm_segs[seg].ds_addr;
   1746 		seglen = dm_segs[seg].ds_len;
   1747 		posaddr = HREAD4(hp, SDHC_DMA_ADDR);
   1748 
   1749 		if ((seg == (cmd->c_dmamap->dm_nsegs-1)) && (posaddr == (segaddr + seglen))) {
   1750 			continue;
   1751 		}
   1752 		if ((posaddr >= segaddr) && (posaddr < (segaddr + seglen)))
   1753 			HWRITE4(hp, SDHC_DMA_ADDR, posaddr);
   1754 		else if ((posaddr >= segaddr) && (posaddr == (segaddr + seglen)) && (seg + 1) < cmd->c_dmamap->dm_nsegs)
   1755 			HWRITE4(hp, SDHC_DMA_ADDR, dm_segs[++seg].ds_addr);
   1756 		KASSERT(seg < cmd->c_dmamap->dm_nsegs);
   1757 	}
   1758 
   1759 	if (ISSET(hp->flags, SHF_USE_ADMA2_MASK)) {
   1760 		bus_dmamap_sync(hp->sc->sc_dmat, hp->adma_map, 0,
   1761 		    PAGE_SIZE, BUS_DMASYNC_POSTWRITE);
   1762 	}
   1763 
   1764 	return error;
   1765 }
   1766 
   1767 static int
   1768 sdhc_transfer_data_pio(struct sdhc_host *hp, struct sdmmc_command *cmd)
   1769 {
   1770 	uint8_t *data = cmd->c_data;
   1771 	void (*pio_func)(struct sdhc_host *, uint8_t *, u_int);
   1772 	u_int len, datalen;
   1773 	u_int imask;
   1774 	u_int pmask;
   1775 	int error = 0;
   1776 
   1777 	KASSERT(mutex_owned(&hp->intr_lock));
   1778 
   1779 	if (ISSET(cmd->c_flags, SCF_CMD_READ)) {
   1780 		imask = SDHC_BUFFER_READ_READY;
   1781 		pmask = SDHC_BUFFER_READ_ENABLE;
   1782 		if (ISSET(hp->sc->sc_flags, SDHC_FLAG_ENHANCED)) {
   1783 			pio_func = esdhc_read_data_pio;
   1784 		} else {
   1785 			pio_func = sdhc_read_data_pio;
   1786 		}
   1787 	} else {
   1788 		imask = SDHC_BUFFER_WRITE_READY;
   1789 		pmask = SDHC_BUFFER_WRITE_ENABLE;
   1790 		if (ISSET(hp->sc->sc_flags, SDHC_FLAG_ENHANCED)) {
   1791 			pio_func = esdhc_write_data_pio;
   1792 		} else {
   1793 			pio_func = sdhc_write_data_pio;
   1794 		}
   1795 	}
   1796 	datalen = cmd->c_datalen;
   1797 
   1798 	KASSERT(mutex_owned(&hp->intr_lock));
   1799 	KASSERT(HREAD2(hp, SDHC_NINTR_STATUS_EN) & imask);
   1800 	KASSERT(HREAD2(hp, SDHC_NINTR_STATUS_EN) & SDHC_TRANSFER_COMPLETE);
   1801 	KASSERT(HREAD2(hp, SDHC_NINTR_SIGNAL_EN) & SDHC_TRANSFER_COMPLETE);
   1802 
   1803 	while (datalen > 0) {
   1804 		if (!ISSET(HREAD4(hp, SDHC_PRESENT_STATE), imask)) {
   1805 			if (ISSET(hp->sc->sc_flags, SDHC_FLAG_32BIT_ACCESS)) {
   1806 				HSET4(hp, SDHC_NINTR_SIGNAL_EN, imask);
   1807 			} else {
   1808 				HSET2(hp, SDHC_NINTR_SIGNAL_EN, imask);
   1809 			}
   1810 			if (!sdhc_wait_intr(hp, imask, SDHC_BUFFER_TIMEOUT)) {
   1811 				error = ETIMEDOUT;
   1812 				break;
   1813 			}
   1814 
   1815 			error = sdhc_wait_state(hp, pmask, pmask);
   1816 			if (error)
   1817 				break;
   1818 		}
   1819 
   1820 		len = MIN(datalen, cmd->c_blklen);
   1821 		(*pio_func)(hp, data, len);
   1822 		DPRINTF(2,("%s: pio data transfer %u @ %p\n",
   1823 		    HDEVNAME(hp), len, data));
   1824 
   1825 		data += len;
   1826 		datalen -= len;
   1827 	}
   1828 
   1829 	if (error == 0 && !sdhc_wait_intr(hp, SDHC_TRANSFER_COMPLETE,
   1830 	    SDHC_TRANSFER_TIMEOUT))
   1831 		error = ETIMEDOUT;
   1832 
   1833 	return error;
   1834 }
   1835 
   1836 static void
   1837 sdhc_read_data_pio(struct sdhc_host *hp, uint8_t *data, u_int datalen)
   1838 {
   1839 
   1840 	if (((__uintptr_t)data & 3) == 0) {
   1841 		while (datalen > 3) {
   1842 			*(uint32_t *)data = le32toh(HREAD4(hp, SDHC_DATA));
   1843 			data += 4;
   1844 			datalen -= 4;
   1845 		}
   1846 		if (datalen > 1) {
   1847 			*(uint16_t *)data = le16toh(HREAD2(hp, SDHC_DATA));
   1848 			data += 2;
   1849 			datalen -= 2;
   1850 		}
   1851 		if (datalen > 0) {
   1852 			*data = HREAD1(hp, SDHC_DATA);
   1853 			data += 1;
   1854 			datalen -= 1;
   1855 		}
   1856 	} else if (((__uintptr_t)data & 1) == 0) {
   1857 		while (datalen > 1) {
   1858 			*(uint16_t *)data = le16toh(HREAD2(hp, SDHC_DATA));
   1859 			data += 2;
   1860 			datalen -= 2;
   1861 		}
   1862 		if (datalen > 0) {
   1863 			*data = HREAD1(hp, SDHC_DATA);
   1864 			data += 1;
   1865 			datalen -= 1;
   1866 		}
   1867 	} else {
   1868 		while (datalen > 0) {
   1869 			*data = HREAD1(hp, SDHC_DATA);
   1870 			data += 1;
   1871 			datalen -= 1;
   1872 		}
   1873 	}
   1874 }
   1875 
   1876 static void
   1877 sdhc_write_data_pio(struct sdhc_host *hp, uint8_t *data, u_int datalen)
   1878 {
   1879 
   1880 	if (((__uintptr_t)data & 3) == 0) {
   1881 		while (datalen > 3) {
   1882 			HWRITE4(hp, SDHC_DATA, htole32(*(uint32_t *)data));
   1883 			data += 4;
   1884 			datalen -= 4;
   1885 		}
   1886 		if (datalen > 1) {
   1887 			HWRITE2(hp, SDHC_DATA, htole16(*(uint16_t *)data));
   1888 			data += 2;
   1889 			datalen -= 2;
   1890 		}
   1891 		if (datalen > 0) {
   1892 			HWRITE1(hp, SDHC_DATA, *data);
   1893 			data += 1;
   1894 			datalen -= 1;
   1895 		}
   1896 	} else if (((__uintptr_t)data & 1) == 0) {
   1897 		while (datalen > 1) {
   1898 			HWRITE2(hp, SDHC_DATA, htole16(*(uint16_t *)data));
   1899 			data += 2;
   1900 			datalen -= 2;
   1901 		}
   1902 		if (datalen > 0) {
   1903 			HWRITE1(hp, SDHC_DATA, *data);
   1904 			data += 1;
   1905 			datalen -= 1;
   1906 		}
   1907 	} else {
   1908 		while (datalen > 0) {
   1909 			HWRITE1(hp, SDHC_DATA, *data);
   1910 			data += 1;
   1911 			datalen -= 1;
   1912 		}
   1913 	}
   1914 }
   1915 
   1916 static void
   1917 esdhc_read_data_pio(struct sdhc_host *hp, uint8_t *data, u_int datalen)
   1918 {
   1919 	uint16_t status = HREAD2(hp, SDHC_NINTR_STATUS);
   1920 	uint32_t v;
   1921 
   1922 	const size_t watermark = (HREAD4(hp, SDHC_WATERMARK_LEVEL) >> SDHC_WATERMARK_READ_SHIFT) & SDHC_WATERMARK_READ_MASK;
   1923 	size_t count = 0;
   1924 
   1925 	while (datalen > 3 && !ISSET(status, SDHC_TRANSFER_COMPLETE)) {
   1926 		if (count == 0) {
   1927 			/*
   1928 			 * If we've drained "watermark" words, we need to wait
   1929 			 * a little bit so the read FIFO can refill.
   1930 			 */
   1931 			sdmmc_delay(10);
   1932 			count = watermark;
   1933 		}
   1934 		v = HREAD4(hp, SDHC_DATA);
   1935 		v = le32toh(v);
   1936 		*(uint32_t *)data = v;
   1937 		data += 4;
   1938 		datalen -= 4;
   1939 		status = HREAD2(hp, SDHC_NINTR_STATUS);
   1940 		count--;
   1941 	}
   1942 	if (datalen > 0 && !ISSET(status, SDHC_TRANSFER_COMPLETE)) {
   1943 		if (count == 0) {
   1944 			sdmmc_delay(10);
   1945 		}
   1946 		v = HREAD4(hp, SDHC_DATA);
   1947 		v = le32toh(v);
   1948 		do {
   1949 			*data++ = v;
   1950 			v >>= 8;
   1951 		} while (--datalen > 0);
   1952 	}
   1953 }
   1954 
   1955 static void
   1956 esdhc_write_data_pio(struct sdhc_host *hp, uint8_t *data, u_int datalen)
   1957 {
   1958 	uint16_t status = HREAD2(hp, SDHC_NINTR_STATUS);
   1959 	uint32_t v;
   1960 
   1961 	const size_t watermark = (HREAD4(hp, SDHC_WATERMARK_LEVEL) >> SDHC_WATERMARK_WRITE_SHIFT) & SDHC_WATERMARK_WRITE_MASK;
   1962 	size_t count = watermark;
   1963 
   1964 	while (datalen > 3 && !ISSET(status, SDHC_TRANSFER_COMPLETE)) {
   1965 		if (count == 0) {
   1966 			sdmmc_delay(10);
   1967 			count = watermark;
   1968 		}
   1969 		v = *(uint32_t *)data;
   1970 		v = htole32(v);
   1971 		HWRITE4(hp, SDHC_DATA, v);
   1972 		data += 4;
   1973 		datalen -= 4;
   1974 		status = HREAD2(hp, SDHC_NINTR_STATUS);
   1975 		count--;
   1976 	}
   1977 	if (datalen > 0 && !ISSET(status, SDHC_TRANSFER_COMPLETE)) {
   1978 		if (count == 0) {
   1979 			sdmmc_delay(10);
   1980 		}
   1981 		v = *(uint32_t *)data;
   1982 		v = htole32(v);
   1983 		HWRITE4(hp, SDHC_DATA, v);
   1984 	}
   1985 }
   1986 
   1987 /* Prepare for another command. */
   1988 static int
   1989 sdhc_soft_reset(struct sdhc_host *hp, int mask)
   1990 {
   1991 	int timo;
   1992 
   1993 	KASSERT(mutex_owned(&hp->intr_lock));
   1994 
   1995 	DPRINTF(1,("%s: software reset reg=%08x\n", HDEVNAME(hp), mask));
   1996 
   1997 	/* Request the reset.  */
   1998 	HWRITE1(hp, SDHC_SOFTWARE_RESET, mask);
   1999 
   2000 	/*
   2001 	 * If necessary, wait for the controller to set the bits to
   2002 	 * acknowledge the reset.
   2003 	 */
   2004 	if (ISSET(hp->sc->sc_flags, SDHC_FLAG_WAIT_RESET) &&
   2005 	    ISSET(mask, (SDHC_RESET_DAT | SDHC_RESET_CMD))) {
   2006 		for (timo = 10000; timo > 0; timo--) {
   2007 			if (ISSET(HREAD1(hp, SDHC_SOFTWARE_RESET), mask))
   2008 				break;
   2009 			/* Short delay because I worry we may miss it...  */
   2010 			sdmmc_delay(1);
   2011 		}
   2012 		if (timo == 0)
   2013 			return ETIMEDOUT;
   2014 	}
   2015 
   2016 	/*
   2017 	 * Wait for the controller to clear the bits to indicate that
   2018 	 * the reset has completed.
   2019 	 */
   2020 	for (timo = 10; timo > 0; timo--) {
   2021 		if (!ISSET(HREAD1(hp, SDHC_SOFTWARE_RESET), mask))
   2022 			break;
   2023 		sdmmc_delay(10000);
   2024 	}
   2025 	if (timo == 0) {
   2026 		DPRINTF(1,("%s: timeout reg=%08x\n", HDEVNAME(hp),
   2027 		    HREAD1(hp, SDHC_SOFTWARE_RESET)));
   2028 		return ETIMEDOUT;
   2029 	}
   2030 
   2031 	if (ISSET(hp->sc->sc_flags, SDHC_FLAG_ENHANCED)) {
   2032 		HSET4(hp, SDHC_DMA_CTL, SDHC_DMA_SNOOP);
   2033 	}
   2034 
   2035 	return 0;
   2036 }
   2037 
   2038 static int
   2039 sdhc_wait_intr(struct sdhc_host *hp, int mask, int timo)
   2040 {
   2041 	int status;
   2042 
   2043 	KASSERT(mutex_owned(&hp->intr_lock));
   2044 
   2045 	mask |= SDHC_ERROR_INTERRUPT;
   2046 
   2047 	status = hp->intr_status & mask;
   2048 	while (status == 0) {
   2049 		if (cv_timedwait(&hp->intr_cv, &hp->intr_lock, timo)
   2050 		    == EWOULDBLOCK) {
   2051 			status |= SDHC_ERROR_INTERRUPT;
   2052 			break;
   2053 		}
   2054 		status = hp->intr_status & mask;
   2055 	}
   2056 	hp->intr_status &= ~status;
   2057 
   2058 	DPRINTF(2,("%s: intr status %#x error %#x\n", HDEVNAME(hp), status,
   2059 	    hp->intr_error_status));
   2060 
   2061 	/* Command timeout has higher priority than command complete. */
   2062 	if (ISSET(status, SDHC_ERROR_INTERRUPT) || hp->intr_error_status) {
   2063 		hp->intr_error_status = 0;
   2064 		hp->intr_status &= ~SDHC_ERROR_INTERRUPT;
   2065 		if (!ISSET(hp->sc->sc_flags, SDHC_FLAG_ENHANCED)) {
   2066 		    (void)sdhc_soft_reset(hp, SDHC_RESET_DAT|SDHC_RESET_CMD);
   2067 		}
   2068 		status = 0;
   2069 	}
   2070 
   2071 	return status;
   2072 }
   2073 
   2074 /*
   2075  * Established by attachment driver at interrupt priority IPL_SDMMC.
   2076  */
   2077 int
   2078 sdhc_intr(void *arg)
   2079 {
   2080 	struct sdhc_softc *sc = (struct sdhc_softc *)arg;
   2081 	struct sdhc_host *hp;
   2082 	int done = 0;
   2083 	uint16_t status;
   2084 	uint16_t error;
   2085 
   2086 	/* We got an interrupt, but we don't know from which slot. */
   2087 	for (size_t host = 0; host < sc->sc_nhosts; host++) {
   2088 		hp = sc->sc_host[host];
   2089 		if (hp == NULL)
   2090 			continue;
   2091 
   2092 		mutex_enter(&hp->intr_lock);
   2093 
   2094 		if (ISSET(sc->sc_flags, SDHC_FLAG_32BIT_ACCESS)) {
   2095 			/* Find out which interrupts are pending. */
   2096 			uint32_t xstatus = HREAD4(hp, SDHC_NINTR_STATUS);
   2097 			status = xstatus;
   2098 			error = xstatus >> 16;
   2099 			if (error)
   2100 				xstatus |= SDHC_ERROR_INTERRUPT;
   2101 			else if (!ISSET(status, SDHC_NINTR_STATUS_MASK))
   2102 				goto next_port; /* no interrupt for us */
   2103 			/* Acknowledge the interrupts we are about to handle. */
   2104 			HWRITE4(hp, SDHC_NINTR_STATUS, xstatus);
   2105 		} else {
   2106 			/* Find out which interrupts are pending. */
   2107 			error = 0;
   2108 			status = HREAD2(hp, SDHC_NINTR_STATUS);
   2109 			if (!ISSET(status, SDHC_NINTR_STATUS_MASK))
   2110 				goto next_port; /* no interrupt for us */
   2111 			/* Acknowledge the interrupts we are about to handle. */
   2112 			HWRITE2(hp, SDHC_NINTR_STATUS, status);
   2113 			if (ISSET(status, SDHC_ERROR_INTERRUPT)) {
   2114 				/* Acknowledge error interrupts. */
   2115 				error = HREAD2(hp, SDHC_EINTR_STATUS);
   2116 				HWRITE2(hp, SDHC_EINTR_STATUS, error);
   2117 			}
   2118 		}
   2119 
   2120 		DPRINTF(2,("%s: interrupt status=%x error=%x\n", HDEVNAME(hp),
   2121 		    status, error));
   2122 
   2123 		/* Claim this interrupt. */
   2124 		done = 1;
   2125 
   2126 		if (ISSET(error, SDHC_ADMA_ERROR)) {
   2127 			uint8_t adma_err = HREAD1(hp, SDHC_ADMA_ERROR_STATUS);
   2128 			printf("%s: ADMA error, status %02x\n", HDEVNAME(hp),
   2129 			    adma_err);
   2130 		}
   2131 
   2132 		/*
   2133 		 * Service error interrupts.
   2134 		 */
   2135 		if (ISSET(error, SDHC_CMD_TIMEOUT_ERROR|
   2136 		    SDHC_DATA_TIMEOUT_ERROR)) {
   2137 			hp->intr_error_status |= error;
   2138 			hp->intr_status |= status;
   2139 			cv_broadcast(&hp->intr_cv);
   2140 		}
   2141 
   2142 		/*
   2143 		 * Wake up the sdmmc event thread to scan for cards.
   2144 		 */
   2145 		if (ISSET(status, SDHC_CARD_REMOVAL|SDHC_CARD_INSERTION)) {
   2146 			if (hp->sdmmc != NULL) {
   2147 				sdmmc_needs_discover(hp->sdmmc);
   2148 			}
   2149 			if (ISSET(sc->sc_flags, SDHC_FLAG_ENHANCED)) {
   2150 				HCLR4(hp, SDHC_NINTR_STATUS_EN,
   2151 				    status & (SDHC_CARD_REMOVAL|SDHC_CARD_INSERTION));
   2152 				HCLR4(hp, SDHC_NINTR_SIGNAL_EN,
   2153 				    status & (SDHC_CARD_REMOVAL|SDHC_CARD_INSERTION));
   2154 			}
   2155 		}
   2156 
   2157 		/*
   2158 		 * Schedule re-tuning process (UHS).
   2159 		 */
   2160 		if (ISSET(status, SDHC_RETUNING_EVENT)) {
   2161 			atomic_swap_uint(&hp->tuning_timer_pending, 1);
   2162 		}
   2163 
   2164 		/*
   2165 		 * Wake up the blocking process to service command
   2166 		 * related interrupt(s).
   2167 		 */
   2168 		if (ISSET(status, SDHC_COMMAND_COMPLETE|
   2169 		    SDHC_BUFFER_READ_READY|SDHC_BUFFER_WRITE_READY|
   2170 		    SDHC_TRANSFER_COMPLETE|SDHC_DMA_INTERRUPT)) {
   2171 			hp->intr_status |= status;
   2172 			if (ISSET(sc->sc_flags, SDHC_FLAG_ENHANCED)) {
   2173 				HCLR4(hp, SDHC_NINTR_SIGNAL_EN,
   2174 				    status & (SDHC_BUFFER_READ_READY|SDHC_BUFFER_WRITE_READY));
   2175 			}
   2176 			cv_broadcast(&hp->intr_cv);
   2177 		}
   2178 
   2179 		/*
   2180 		 * Service SD card interrupts.
   2181 		 */
   2182 		if (!ISSET(sc->sc_flags, SDHC_FLAG_ENHANCED)
   2183 		    && ISSET(status, SDHC_CARD_INTERRUPT)) {
   2184 			DPRINTF(0,("%s: card interrupt\n", HDEVNAME(hp)));
   2185 			HCLR2(hp, SDHC_NINTR_STATUS_EN, SDHC_CARD_INTERRUPT);
   2186 			sdmmc_card_intr(hp->sdmmc);
   2187 		}
   2188 next_port:
   2189 		mutex_exit(&hp->intr_lock);
   2190 	}
   2191 
   2192 	return done;
   2193 }
   2194 
   2195 kmutex_t *
   2196 sdhc_host_lock(struct sdhc_host *hp)
   2197 {
   2198 	return &hp->intr_lock;
   2199 }
   2200 
   2201 #ifdef SDHC_DEBUG
   2202 void
   2203 sdhc_dump_regs(struct sdhc_host *hp)
   2204 {
   2205 
   2206 	printf("0x%02x PRESENT_STATE:    %x\n", SDHC_PRESENT_STATE,
   2207 	    HREAD4(hp, SDHC_PRESENT_STATE));
   2208 	if (!ISSET(hp->sc->sc_flags, SDHC_FLAG_ENHANCED))
   2209 		printf("0x%02x POWER_CTL:        %x\n", SDHC_POWER_CTL,
   2210 		    HREAD1(hp, SDHC_POWER_CTL));
   2211 	printf("0x%02x NINTR_STATUS:     %x\n", SDHC_NINTR_STATUS,
   2212 	    HREAD2(hp, SDHC_NINTR_STATUS));
   2213 	printf("0x%02x EINTR_STATUS:     %x\n", SDHC_EINTR_STATUS,
   2214 	    HREAD2(hp, SDHC_EINTR_STATUS));
   2215 	printf("0x%02x NINTR_STATUS_EN:  %x\n", SDHC_NINTR_STATUS_EN,
   2216 	    HREAD2(hp, SDHC_NINTR_STATUS_EN));
   2217 	printf("0x%02x EINTR_STATUS_EN:  %x\n", SDHC_EINTR_STATUS_EN,
   2218 	    HREAD2(hp, SDHC_EINTR_STATUS_EN));
   2219 	printf("0x%02x NINTR_SIGNAL_EN:  %x\n", SDHC_NINTR_SIGNAL_EN,
   2220 	    HREAD2(hp, SDHC_NINTR_SIGNAL_EN));
   2221 	printf("0x%02x EINTR_SIGNAL_EN:  %x\n", SDHC_EINTR_SIGNAL_EN,
   2222 	    HREAD2(hp, SDHC_EINTR_SIGNAL_EN));
   2223 	printf("0x%02x CAPABILITIES:     %x\n", SDHC_CAPABILITIES,
   2224 	    HREAD4(hp, SDHC_CAPABILITIES));
   2225 	printf("0x%02x MAX_CAPABILITIES: %x\n", SDHC_MAX_CAPABILITIES,
   2226 	    HREAD4(hp, SDHC_MAX_CAPABILITIES));
   2227 }
   2228 #endif
   2229