Home | History | Annotate | Line # | Download | only in acpi
acpi_cpu_cstate.c revision 1.51
      1 /* $NetBSD: acpi_cpu_cstate.c,v 1.51 2011/03/17 15:32:18 jruoho Exp $ */
      2 
      3 /*-
      4  * Copyright (c) 2010, 2011 Jukka Ruohonen <jruohonen (at) iki.fi>
      5  * All rights reserved.
      6  *
      7  * Redistribution and use in source and binary forms, with or without
      8  * modification, are permitted provided that the following conditions
      9  * are met:
     10  *
     11  * 1. Redistributions of source code must retain the above copyright
     12  *    notice, this list of conditions and the following disclaimer.
     13  * 2. Redistributions in binary form must reproduce the above copyright
     14  *    notice, this list of conditions and the following disclaimer in the
     15  *    documentation and/or other materials provided with the distribution.
     16  *
     17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
     18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
     19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
     20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
     21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
     22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
     23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
     24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
     25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
     26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
     27  * SUCH DAMAGE.
     28  */
     29 #include <sys/cdefs.h>
     30 __KERNEL_RCSID(0, "$NetBSD: acpi_cpu_cstate.c,v 1.51 2011/03/17 15:32:18 jruoho Exp $");
     31 
     32 #include <sys/param.h>
     33 #include <sys/cpu.h>
     34 #include <sys/device.h>
     35 #include <sys/kernel.h>
     36 #include <sys/once.h>
     37 #include <sys/mutex.h>
     38 #include <sys/timetc.h>
     39 
     40 #include <dev/acpi/acpireg.h>
     41 #include <dev/acpi/acpivar.h>
     42 #include <dev/acpi/acpi_cpu.h>
     43 #include <dev/acpi/acpi_timer.h>
     44 
     45 #include <machine/acpi_machdep.h>
     46 
     47 #define _COMPONENT	 ACPI_BUS_COMPONENT
     48 ACPI_MODULE_NAME	 ("acpi_cpu_cstate")
     49 
     50 static ACPI_STATUS	 acpicpu_cstate_cst(struct acpicpu_softc *);
     51 static ACPI_STATUS	 acpicpu_cstate_cst_add(struct acpicpu_softc *,
     52 						ACPI_OBJECT *, int );
     53 static void		 acpicpu_cstate_cst_bios(void);
     54 static void		 acpicpu_cstate_memset(struct acpicpu_softc *);
     55 static ACPI_STATUS	 acpicpu_cstate_dep(struct acpicpu_softc *);
     56 static void		 acpicpu_cstate_fadt(struct acpicpu_softc *);
     57 static void		 acpicpu_cstate_quirks(struct acpicpu_softc *);
     58 static int		 acpicpu_cstate_latency(struct acpicpu_softc *);
     59 static bool		 acpicpu_cstate_bm_check(void);
     60 static void		 acpicpu_cstate_idle_enter(struct acpicpu_softc *,int);
     61 
     62 extern struct acpicpu_softc **acpicpu_sc;
     63 
     64 /*
     65  * XXX:	The local APIC timer (as well as TSC) is typically stopped in C3.
     66  *	For now, we cannot but disable C3. But there appears to be timer-
     67  *	related interrupt issues also in C2. The only entirely safe option
     68  *	at the moment is to use C1.
     69  */
     70 #ifdef ACPICPU_ENABLE_C3
     71 static int cs_state_max = ACPI_STATE_C3;
     72 #else
     73 static int cs_state_max = ACPI_STATE_C1;
     74 #endif
     75 
     76 void
     77 acpicpu_cstate_attach(device_t self)
     78 {
     79 	struct acpicpu_softc *sc = device_private(self);
     80 	ACPI_STATUS rv;
     81 
     82 	/*
     83 	 * Either use the preferred _CST or resort to FADT.
     84 	 */
     85 	rv = acpicpu_cstate_cst(sc);
     86 
     87 	switch (rv) {
     88 
     89 	case AE_OK:
     90 		acpicpu_cstate_cst_bios();
     91 		break;
     92 
     93 	default:
     94 		sc->sc_flags |= ACPICPU_FLAG_C_FADT;
     95 		acpicpu_cstate_fadt(sc);
     96 		break;
     97 	}
     98 
     99 	/*
    100 	 * Query the optional _CSD.
    101 	 */
    102 	rv = acpicpu_cstate_dep(sc);
    103 
    104 	if (ACPI_SUCCESS(rv))
    105 		sc->sc_flags |= ACPICPU_FLAG_C_DEP;
    106 
    107 	sc->sc_flags |= ACPICPU_FLAG_C;
    108 
    109 	acpicpu_cstate_quirks(sc);
    110 }
    111 
    112 int
    113 acpicpu_cstate_detach(device_t self)
    114 {
    115 	struct acpicpu_softc *sc = device_private(self);
    116 	static ONCE_DECL(once_detach);
    117 	int rv;
    118 
    119 	rv = RUN_ONCE(&once_detach, acpicpu_md_cstate_stop);
    120 
    121 	if (rv != 0)
    122 		return rv;
    123 
    124 	sc->sc_flags &= ~ACPICPU_FLAG_C;
    125 
    126 	return 0;
    127 }
    128 
    129 void
    130 acpicpu_cstate_start(device_t self)
    131 {
    132 	struct acpicpu_softc *sc = device_private(self);
    133 
    134 	(void)acpicpu_md_cstate_start(sc);
    135 }
    136 
    137 bool
    138 acpicpu_cstate_suspend(device_t self)
    139 {
    140 	return true;
    141 }
    142 
    143 bool
    144 acpicpu_cstate_resume(device_t self)
    145 {
    146 	static const ACPI_OSD_EXEC_CALLBACK func = acpicpu_cstate_callback;
    147 	struct acpicpu_softc *sc = device_private(self);
    148 
    149 	if ((sc->sc_flags & ACPICPU_FLAG_C_FADT) == 0)
    150 		(void)AcpiOsExecute(OSL_NOTIFY_HANDLER, func, sc->sc_dev);
    151 
    152 	return true;
    153 }
    154 
    155 void
    156 acpicpu_cstate_callback(void *aux)
    157 {
    158 	struct acpicpu_softc *sc;
    159 	device_t self = aux;
    160 
    161 	sc = device_private(self);
    162 
    163 	if ((sc->sc_flags & ACPICPU_FLAG_C_FADT) != 0)
    164 		return;
    165 
    166 	mutex_enter(&sc->sc_mtx);
    167 	(void)acpicpu_cstate_cst(sc);
    168 	mutex_exit(&sc->sc_mtx);
    169 }
    170 
    171 static ACPI_STATUS
    172 acpicpu_cstate_cst(struct acpicpu_softc *sc)
    173 {
    174 	ACPI_OBJECT *elm, *obj;
    175 	ACPI_BUFFER buf;
    176 	ACPI_STATUS rv;
    177 	uint32_t i, n;
    178 	uint8_t count;
    179 
    180 	rv = acpi_eval_struct(sc->sc_node->ad_handle, "_CST", &buf);
    181 
    182 	if (ACPI_FAILURE(rv))
    183 		return rv;
    184 
    185 	obj = buf.Pointer;
    186 
    187 	if (obj->Type != ACPI_TYPE_PACKAGE) {
    188 		rv = AE_TYPE;
    189 		goto out;
    190 	}
    191 
    192 	if (obj->Package.Count < 2) {
    193 		rv = AE_LIMIT;
    194 		goto out;
    195 	}
    196 
    197 	elm = obj->Package.Elements;
    198 
    199 	if (elm[0].Type != ACPI_TYPE_INTEGER) {
    200 		rv = AE_TYPE;
    201 		goto out;
    202 	}
    203 
    204 	n = elm[0].Integer.Value;
    205 
    206 	if (n != obj->Package.Count - 1) {
    207 		rv = AE_BAD_VALUE;
    208 		goto out;
    209 	}
    210 
    211 	if (n > ACPI_C_STATES_MAX) {
    212 		rv = AE_LIMIT;
    213 		goto out;
    214 	}
    215 
    216 	acpicpu_cstate_memset(sc);
    217 
    218 	CTASSERT(ACPI_STATE_C0 == 0 && ACPI_STATE_C1 == 1);
    219 	CTASSERT(ACPI_STATE_C2 == 2 && ACPI_STATE_C3 == 3);
    220 
    221 	for (count = 0, i = 1; i <= n; i++) {
    222 
    223 		elm = &obj->Package.Elements[i];
    224 		rv = acpicpu_cstate_cst_add(sc, elm, i);
    225 
    226 		if (ACPI_SUCCESS(rv))
    227 			count++;
    228 	}
    229 
    230 	rv = (count != 0) ? AE_OK : AE_NOT_EXIST;
    231 
    232 out:
    233 	if (buf.Pointer != NULL)
    234 		ACPI_FREE(buf.Pointer);
    235 
    236 	return rv;
    237 }
    238 
    239 static ACPI_STATUS
    240 acpicpu_cstate_cst_add(struct acpicpu_softc *sc, ACPI_OBJECT *elm, int i)
    241 {
    242 	struct acpicpu_cstate *cs = sc->sc_cstate;
    243 	struct acpicpu_cstate state;
    244 	struct acpicpu_reg *reg;
    245 	ACPI_STATUS rv = AE_OK;
    246 	ACPI_OBJECT *obj;
    247 	uint32_t type;
    248 
    249 	(void)memset(&state, 0, sizeof(*cs));
    250 
    251 	state.cs_flags = ACPICPU_FLAG_C_BM_STS;
    252 
    253 	if (elm->Type != ACPI_TYPE_PACKAGE) {
    254 		rv = AE_TYPE;
    255 		goto out;
    256 	}
    257 
    258 	if (elm->Package.Count != 4) {
    259 		rv = AE_LIMIT;
    260 		goto out;
    261 	}
    262 
    263 	/*
    264 	 * Type.
    265 	 */
    266 	obj = &elm->Package.Elements[1];
    267 
    268 	if (obj->Type != ACPI_TYPE_INTEGER) {
    269 		rv = AE_TYPE;
    270 		goto out;
    271 	}
    272 
    273 	type = obj->Integer.Value;
    274 
    275 	if (type < ACPI_STATE_C1 || type > ACPI_STATE_C3) {
    276 		rv = AE_TYPE;
    277 		goto out;
    278 	}
    279 
    280 	/*
    281 	 * Latency.
    282 	 */
    283 	obj = &elm->Package.Elements[2];
    284 
    285 	if (obj->Type != ACPI_TYPE_INTEGER) {
    286 		rv = AE_TYPE;
    287 		goto out;
    288 	}
    289 
    290 	state.cs_latency = obj->Integer.Value;
    291 
    292 	/*
    293 	 * Power.
    294 	 */
    295 	obj = &elm->Package.Elements[3];
    296 
    297 	if (obj->Type != ACPI_TYPE_INTEGER) {
    298 		rv = AE_TYPE;
    299 		goto out;
    300 	}
    301 
    302 	state.cs_power = obj->Integer.Value;
    303 
    304 	/*
    305 	 * Register.
    306 	 */
    307 	obj = &elm->Package.Elements[0];
    308 
    309 	if (obj->Type != ACPI_TYPE_BUFFER) {
    310 		rv = AE_TYPE;
    311 		goto out;
    312 	}
    313 
    314 	CTASSERT(sizeof(struct acpicpu_reg) == 15);
    315 
    316 	if (obj->Buffer.Length < sizeof(struct acpicpu_reg)) {
    317 		rv = AE_LIMIT;
    318 		goto out;
    319 	}
    320 
    321 	reg = (struct acpicpu_reg *)obj->Buffer.Pointer;
    322 
    323 	switch (reg->reg_spaceid) {
    324 
    325 	case ACPI_ADR_SPACE_SYSTEM_IO:
    326 		state.cs_method = ACPICPU_C_STATE_SYSIO;
    327 
    328 		if (reg->reg_addr == 0) {
    329 			rv = AE_AML_ILLEGAL_ADDRESS;
    330 			goto out;
    331 		}
    332 
    333 		if (reg->reg_bitwidth != 8) {
    334 			rv = AE_AML_BAD_RESOURCE_LENGTH;
    335 			goto out;
    336 		}
    337 
    338 		state.cs_addr = reg->reg_addr;
    339 		break;
    340 
    341 	case ACPI_ADR_SPACE_FIXED_HARDWARE:
    342 		state.cs_method = ACPICPU_C_STATE_FFH;
    343 
    344 		switch (type) {
    345 
    346 		case ACPI_STATE_C1:
    347 
    348 			/*
    349 			 * If ACPI wants native access (FFH), but the
    350 			 * MD code does not support MONITOR/MWAIT, use
    351 			 * HLT for C1 and error out for higher C-states.
    352 			 */
    353 			if ((sc->sc_flags & ACPICPU_FLAG_C_FFH) == 0)
    354 				state.cs_method = ACPICPU_C_STATE_HALT;
    355 
    356 			break;
    357 
    358 		default:
    359 
    360 			if ((sc->sc_flags & ACPICPU_FLAG_C_FFH) == 0) {
    361 				rv = AE_SUPPORT;
    362 				goto out;
    363 			}
    364 		}
    365 
    366 		if (sc->sc_cap != 0) {
    367 
    368 			/*
    369 			 * The _CST FFH GAS encoding may contain
    370 			 * additional hints on Intel processors.
    371 			 * Use these to determine whether we can
    372 			 * avoid the bus master activity check.
    373 			 */
    374 			if ((reg->reg_accesssize & ACPICPU_PDC_GAS_BM) == 0)
    375 				state.cs_flags &= ~ACPICPU_FLAG_C_BM_STS;
    376 		}
    377 
    378 		break;
    379 
    380 	default:
    381 		rv = AE_AML_INVALID_SPACE_ID;
    382 		goto out;
    383 	}
    384 
    385 	/*
    386 	 * As some systems define the type arbitrarily,
    387 	 * we use a sequential counter instead of the
    388 	 * BIOS data. For instance, AMD family 14h is
    389 	 * instructed to only use the value 2; see
    390 	 *
    391 	 *	Advanced Micro Devices: BIOS and Kernel
    392 	 *	Developer's Guide (BKDG) for AMD Family
    393 	 *	14h Models 00h-0Fh Processors. Revision
    394 	 *	3.00, January 4, 2011.
    395 	 */
    396 	if (i != (int)type) {
    397 
    398 		ACPI_DEBUG_PRINT((ACPI_DB_INFO,
    399 			"C%d != C%u from BIOS", i, type));
    400 	}
    401 
    402 	KASSERT(cs[i].cs_method == 0);
    403 
    404 	cs[i].cs_addr = state.cs_addr;
    405 	cs[i].cs_power = state.cs_power;
    406 	cs[i].cs_flags = state.cs_flags;
    407 	cs[i].cs_method = state.cs_method;
    408 	cs[i].cs_latency = state.cs_latency;
    409 
    410 out:
    411 	if (ACPI_FAILURE(rv))
    412 		aprint_error_dev(sc->sc_dev, "failed to add "
    413 		    "C-state: %s\n", AcpiFormatException(rv));
    414 
    415 	return rv;
    416 }
    417 
    418 static void
    419 acpicpu_cstate_cst_bios(void)
    420 {
    421 	const uint8_t val = AcpiGbl_FADT.CstControl;
    422 	const uint32_t addr = AcpiGbl_FADT.SmiCommand;
    423 
    424 	if (addr == 0 || val == 0)
    425 		return;
    426 
    427 	(void)AcpiOsWritePort(addr, val, 8);
    428 }
    429 
    430 static void
    431 acpicpu_cstate_memset(struct acpicpu_softc *sc)
    432 {
    433 	uint8_t i = 0;
    434 
    435 	while (i < __arraycount(sc->sc_cstate)) {
    436 
    437 		sc->sc_cstate[i].cs_addr = 0;
    438 		sc->sc_cstate[i].cs_power = 0;
    439 		sc->sc_cstate[i].cs_flags = 0;
    440 		sc->sc_cstate[i].cs_method = 0;
    441 		sc->sc_cstate[i].cs_latency = 0;
    442 
    443 		i++;
    444 	}
    445 }
    446 
    447 static ACPI_STATUS
    448 acpicpu_cstate_dep(struct acpicpu_softc *sc)
    449 {
    450 	ACPI_OBJECT *elm, *obj;
    451 	ACPI_BUFFER buf;
    452 	ACPI_STATUS rv;
    453 	uint32_t val;
    454 	uint8_t i, n;
    455 
    456 	rv = acpi_eval_struct(sc->sc_node->ad_handle, "_CSD", &buf);
    457 
    458 	if (ACPI_FAILURE(rv))
    459 		goto out;
    460 
    461 	obj = buf.Pointer;
    462 
    463 	if (obj->Type != ACPI_TYPE_PACKAGE) {
    464 		rv = AE_TYPE;
    465 		goto out;
    466 	}
    467 
    468 	if (obj->Package.Count != 1) {
    469 		rv = AE_LIMIT;
    470 		goto out;
    471 	}
    472 
    473 	elm = &obj->Package.Elements[0];
    474 
    475 	if (obj->Type != ACPI_TYPE_PACKAGE) {
    476 		rv = AE_TYPE;
    477 		goto out;
    478 	}
    479 
    480 	n = elm->Package.Count;
    481 
    482 	if (n != 6) {
    483 		rv = AE_LIMIT;
    484 		goto out;
    485 	}
    486 
    487 	elm = elm->Package.Elements;
    488 
    489 	for (i = 0; i < n; i++) {
    490 
    491 		if (elm[i].Type != ACPI_TYPE_INTEGER) {
    492 			rv = AE_TYPE;
    493 			goto out;
    494 		}
    495 
    496 		if (elm[i].Integer.Value > UINT32_MAX) {
    497 			rv = AE_AML_NUMERIC_OVERFLOW;
    498 			goto out;
    499 		}
    500 	}
    501 
    502 	val = elm[1].Integer.Value;
    503 
    504 	if (val != 0)
    505 		aprint_debug_dev(sc->sc_dev, "invalid revision in _CSD\n");
    506 
    507 	val = elm[3].Integer.Value;
    508 
    509 	if (val < ACPICPU_DEP_SW_ALL || val > ACPICPU_DEP_HW_ALL) {
    510 		rv = AE_AML_BAD_RESOURCE_VALUE;
    511 		goto out;
    512 	}
    513 
    514 	val = elm[4].Integer.Value;
    515 
    516 	if (val > sc->sc_ncpus) {
    517 		rv = AE_BAD_VALUE;
    518 		goto out;
    519 	}
    520 
    521 	sc->sc_cstate_dep.dep_domain = elm[2].Integer.Value;
    522 	sc->sc_cstate_dep.dep_type   = elm[3].Integer.Value;
    523 	sc->sc_cstate_dep.dep_ncpus  = elm[4].Integer.Value;
    524 	sc->sc_cstate_dep.dep_index  = elm[5].Integer.Value;
    525 
    526 out:
    527 	if (ACPI_FAILURE(rv) && rv != AE_NOT_FOUND)
    528 		aprint_debug_dev(sc->sc_dev, "failed to evaluate "
    529 		    "_CSD: %s\n", AcpiFormatException(rv));
    530 
    531 	if (buf.Pointer != NULL)
    532 		ACPI_FREE(buf.Pointer);
    533 
    534 	return rv;
    535 }
    536 
    537 static void
    538 acpicpu_cstate_fadt(struct acpicpu_softc *sc)
    539 {
    540 	struct acpicpu_cstate *cs = sc->sc_cstate;
    541 
    542 	acpicpu_cstate_memset(sc);
    543 
    544 	/*
    545 	 * All x86 processors should support C1 (a.k.a. HALT).
    546 	 */
    547 	cs[ACPI_STATE_C1].cs_method = ACPICPU_C_STATE_HALT;
    548 
    549 	if ((AcpiGbl_FADT.Flags & ACPI_FADT_C1_SUPPORTED) == 0)
    550 		aprint_debug_dev(sc->sc_dev, "HALT not supported?\n");
    551 
    552 	if (sc->sc_object.ao_pblkaddr == 0)
    553 		return;
    554 
    555 	if (sc->sc_ncpus > 1) {
    556 
    557 		if ((AcpiGbl_FADT.Flags & ACPI_FADT_C2_MP_SUPPORTED) == 0)
    558 			return;
    559 	}
    560 
    561 	cs[ACPI_STATE_C2].cs_method = ACPICPU_C_STATE_SYSIO;
    562 	cs[ACPI_STATE_C3].cs_method = ACPICPU_C_STATE_SYSIO;
    563 
    564 	cs[ACPI_STATE_C2].cs_latency = AcpiGbl_FADT.C2Latency;
    565 	cs[ACPI_STATE_C3].cs_latency = AcpiGbl_FADT.C3Latency;
    566 
    567 	cs[ACPI_STATE_C2].cs_addr = sc->sc_object.ao_pblkaddr + 4;
    568 	cs[ACPI_STATE_C3].cs_addr = sc->sc_object.ao_pblkaddr + 5;
    569 
    570 	/*
    571 	 * The P_BLK length should always be 6. If it
    572 	 * is not, reduce functionality accordingly.
    573 	 */
    574 	if (sc->sc_object.ao_pblklen < 5)
    575 		cs[ACPI_STATE_C2].cs_method = 0;
    576 
    577 	if (sc->sc_object.ao_pblklen < 6)
    578 		cs[ACPI_STATE_C3].cs_method = 0;
    579 
    580 	/*
    581 	 * Sanity check the latency levels in FADT.
    582 	 * Values above the thresholds are used to
    583 	 * inform that C-states are not supported.
    584 	 */
    585 	CTASSERT(ACPICPU_C_C2_LATENCY_MAX == 100);
    586 	CTASSERT(ACPICPU_C_C3_LATENCY_MAX == 1000);
    587 
    588 	if (AcpiGbl_FADT.C2Latency > ACPICPU_C_C2_LATENCY_MAX)
    589 		cs[ACPI_STATE_C2].cs_method = 0;
    590 
    591 	if (AcpiGbl_FADT.C3Latency > ACPICPU_C_C3_LATENCY_MAX)
    592 		cs[ACPI_STATE_C3].cs_method = 0;
    593 }
    594 
    595 static void
    596 acpicpu_cstate_quirks(struct acpicpu_softc *sc)
    597 {
    598 	const uint32_t reg = AcpiGbl_FADT.Pm2ControlBlock;
    599 	const uint32_t len = AcpiGbl_FADT.Pm2ControlLength;
    600 
    601 	/*
    602 	 * Disable C3 for PIIX4.
    603 	 */
    604 	if ((sc->sc_flags & ACPICPU_FLAG_PIIX4) != 0) {
    605 		sc->sc_cstate[ACPI_STATE_C3].cs_method = 0;
    606 		return;
    607 	}
    608 
    609 	/*
    610 	 * Check bus master arbitration. If ARB_DIS
    611 	 * is not available, processor caches must be
    612 	 * flushed before C3 (ACPI 4.0, section 8.2).
    613 	 */
    614 	if (reg != 0 && len != 0) {
    615 		sc->sc_flags |= ACPICPU_FLAG_C_ARB;
    616 		return;
    617 	}
    618 
    619 	/*
    620 	 * Disable C3 entirely if WBINVD is not present.
    621 	 */
    622 	if ((AcpiGbl_FADT.Flags & ACPI_FADT_WBINVD) == 0)
    623 		sc->sc_cstate[ACPI_STATE_C3].cs_method = 0;
    624 	else {
    625 		/*
    626 		 * If WBINVD is present and functioning properly,
    627 		 * flush all processor caches before entering C3.
    628 		 */
    629 		if ((AcpiGbl_FADT.Flags & ACPI_FADT_WBINVD_FLUSH) == 0)
    630 			sc->sc_flags &= ~ACPICPU_FLAG_C_BM;
    631 		else
    632 			sc->sc_cstate[ACPI_STATE_C3].cs_method = 0;
    633 	}
    634 }
    635 
    636 static int
    637 acpicpu_cstate_latency(struct acpicpu_softc *sc)
    638 {
    639 	static const uint32_t cs_factor = 3;
    640 	struct acpicpu_cstate *cs;
    641 	int i;
    642 
    643 	for (i = cs_state_max; i > 0; i--) {
    644 
    645 		cs = &sc->sc_cstate[i];
    646 
    647 		if (__predict_false(cs->cs_method == 0))
    648 			continue;
    649 
    650 		/*
    651 		 * Choose a state if we have previously slept
    652 		 * longer than the worst case latency of the
    653 		 * state times an arbitrary multiplier.
    654 		 */
    655 		if (sc->sc_cstate_sleep > cs->cs_latency * cs_factor)
    656 			return i;
    657 	}
    658 
    659 	return ACPI_STATE_C1;
    660 }
    661 
    662 /*
    663  * The main idle loop.
    664  */
    665 void
    666 acpicpu_cstate_idle(void)
    667 {
    668 	struct cpu_info *ci = curcpu();
    669 	struct acpicpu_softc *sc;
    670 	int state;
    671 
    672 	acpi_md_OsDisableInterrupt();
    673 
    674 	if (__predict_false(ci->ci_want_resched != 0))
    675 		goto out;
    676 
    677 	KASSERT(acpicpu_sc != NULL);
    678 	KASSERT(ci->ci_acpiid < maxcpus);
    679 
    680 	sc = acpicpu_sc[ci->ci_acpiid];
    681 
    682 	if (__predict_false(sc == NULL))
    683 		goto out;
    684 
    685 	KASSERT(ci->ci_ilevel == IPL_NONE);
    686 	KASSERT((sc->sc_flags & ACPICPU_FLAG_C) != 0);
    687 
    688 	if (__predict_false(sc->sc_cold != false))
    689 		goto out;
    690 
    691 	if (__predict_false(mutex_tryenter(&sc->sc_mtx) == 0))
    692 		goto out;
    693 
    694 	mutex_exit(&sc->sc_mtx);
    695 	state = acpicpu_cstate_latency(sc);
    696 
    697 	/*
    698 	 * Apply AMD C1E quirk.
    699 	 */
    700 	if ((sc->sc_flags & ACPICPU_FLAG_C_C1E) != 0)
    701 		acpicpu_md_quirk_c1e();
    702 
    703 	/*
    704 	 * Check for bus master activity. Note that particularly usb(4)
    705 	 * causes high activity, which may prevent the use of C3 states.
    706 	 */
    707 	if ((sc->sc_cstate[state].cs_flags & ACPICPU_FLAG_C_BM_STS) != 0) {
    708 
    709 		if (acpicpu_cstate_bm_check() != false)
    710 			state--;
    711 
    712 		if (__predict_false(sc->sc_cstate[state].cs_method == 0))
    713 			state = ACPI_STATE_C1;
    714 	}
    715 
    716 	KASSERT(state != ACPI_STATE_C0);
    717 
    718 	if (state != ACPI_STATE_C3) {
    719 		acpicpu_cstate_idle_enter(sc, state);
    720 		return;
    721 	}
    722 
    723 	/*
    724 	 * On all recent (Intel) CPUs caches are shared
    725 	 * by CPUs and bus master control is required to
    726 	 * keep these coherent while in C3. Flushing the
    727 	 * CPU caches is only the last resort.
    728 	 */
    729 	if ((sc->sc_flags & ACPICPU_FLAG_C_BM) == 0)
    730 		ACPI_FLUSH_CPU_CACHE();
    731 
    732 	/*
    733 	 * Allow the bus master to request that any given
    734 	 * CPU should return immediately to C0 from C3.
    735 	 */
    736 	if ((sc->sc_flags & ACPICPU_FLAG_C_BM) != 0)
    737 		(void)AcpiWriteBitRegister(ACPI_BITREG_BUS_MASTER_RLD, 1);
    738 
    739 	/*
    740 	 * It may be necessary to disable bus master arbitration
    741 	 * to ensure that bus master cycles do not occur while
    742 	 * sleeping in C3 (see ACPI 4.0, section 8.1.4).
    743 	 */
    744 	if ((sc->sc_flags & ACPICPU_FLAG_C_ARB) != 0)
    745 		(void)AcpiWriteBitRegister(ACPI_BITREG_ARB_DISABLE, 1);
    746 
    747 	acpicpu_cstate_idle_enter(sc, state);
    748 
    749 	/*
    750 	 * Disable bus master wake and re-enable the arbiter.
    751 	 */
    752 	if ((sc->sc_flags & ACPICPU_FLAG_C_BM) != 0)
    753 		(void)AcpiWriteBitRegister(ACPI_BITREG_BUS_MASTER_RLD, 0);
    754 
    755 	if ((sc->sc_flags & ACPICPU_FLAG_C_ARB) != 0)
    756 		(void)AcpiWriteBitRegister(ACPI_BITREG_ARB_DISABLE, 0);
    757 
    758 	return;
    759 
    760 out:
    761 	acpi_md_OsEnableInterrupt();
    762 }
    763 
    764 static void
    765 acpicpu_cstate_idle_enter(struct acpicpu_softc *sc, int state)
    766 {
    767 	struct acpicpu_cstate *cs = &sc->sc_cstate[state];
    768 	uint32_t end, start, val;
    769 
    770 	start = acpitimer_read_fast(NULL);
    771 
    772 	switch (cs->cs_method) {
    773 
    774 	case ACPICPU_C_STATE_FFH:
    775 	case ACPICPU_C_STATE_HALT:
    776 		acpicpu_md_cstate_enter(cs->cs_method, state);
    777 		break;
    778 
    779 	case ACPICPU_C_STATE_SYSIO:
    780 		(void)AcpiOsReadPort(cs->cs_addr, &val, 8);
    781 		break;
    782 	}
    783 
    784 	acpi_md_OsEnableInterrupt();
    785 
    786 	cs->cs_evcnt.ev_count++;
    787 	end = acpitimer_read_fast(NULL);
    788 	sc->sc_cstate_sleep = hztoms(acpitimer_delta(end, start)) * 1000;
    789 }
    790 
    791 static bool
    792 acpicpu_cstate_bm_check(void)
    793 {
    794 	uint32_t val = 0;
    795 	ACPI_STATUS rv;
    796 
    797 	rv = AcpiReadBitRegister(ACPI_BITREG_BUS_MASTER_STATUS, &val);
    798 
    799 	if (ACPI_FAILURE(rv) || val == 0)
    800 		return false;
    801 
    802 	(void)AcpiWriteBitRegister(ACPI_BITREG_BUS_MASTER_STATUS, 1);
    803 
    804 	return true;
    805 }
    806