Home | History | Annotate | Line # | Download | only in x86
cpu.c revision 1.136
      1 /*	$NetBSD: cpu.c,v 1.136 2017/09/28 17:48:20 maxv Exp $	*/
      2 
      3 /*
      4  * Copyright (c) 2000-2012 NetBSD Foundation, Inc.
      5  * All rights reserved.
      6  *
      7  * This code is derived from software contributed to The NetBSD Foundation
      8  * by Bill Sommerfeld of RedBack Networks Inc, and by Andrew Doran.
      9  *
     10  * Redistribution and use in source and binary forms, with or without
     11  * modification, are permitted provided that the following conditions
     12  * are met:
     13  * 1. Redistributions of source code must retain the above copyright
     14  *    notice, this list of conditions and the following disclaimer.
     15  * 2. Redistributions in binary form must reproduce the above copyright
     16  *    notice, this list of conditions and the following disclaimer in the
     17  *    documentation and/or other materials provided with the distribution.
     18  *
     19  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     20  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     21  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     22  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     23  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     24  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     25  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     26  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     27  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     28  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     29  * POSSIBILITY OF SUCH DAMAGE.
     30  */
     31 
     32 /*
     33  * Copyright (c) 1999 Stefan Grefen
     34  *
     35  * Redistribution and use in source and binary forms, with or without
     36  * modification, are permitted provided that the following conditions
     37  * are met:
     38  * 1. Redistributions of source code must retain the above copyright
     39  *    notice, this list of conditions and the following disclaimer.
     40  * 2. Redistributions in binary form must reproduce the above copyright
     41  *    notice, this list of conditions and the following disclaimer in the
     42  *    documentation and/or other materials provided with the distribution.
     43  * 3. All advertising materials mentioning features or use of this software
     44  *    must display the following acknowledgement:
     45  *      This product includes software developed by the NetBSD
     46  *      Foundation, Inc. and its contributors.
     47  * 4. Neither the name of The NetBSD Foundation nor the names of its
     48  *    contributors may be used to endorse or promote products derived
     49  *    from this software without specific prior written permission.
     50  *
     51  * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND ANY
     52  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
     53  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
     54  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR AND CONTRIBUTORS BE LIABLE
     55  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
     56  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
     57  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
     58  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
     59  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
     60  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
     61  * SUCH DAMAGE.
     62  */
     63 
     64 #include <sys/cdefs.h>
     65 __KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.136 2017/09/28 17:48:20 maxv Exp $");
     66 
     67 #include "opt_ddb.h"
     68 #include "opt_mpbios.h"		/* for MPDEBUG */
     69 #include "opt_mtrr.h"
     70 #include "opt_multiprocessor.h"
     71 
     72 #include "lapic.h"
     73 #include "ioapic.h"
     74 
     75 #include <sys/param.h>
     76 #include <sys/proc.h>
     77 #include <sys/systm.h>
     78 #include <sys/device.h>
     79 #include <sys/kmem.h>
     80 #include <sys/cpu.h>
     81 #include <sys/cpufreq.h>
     82 #include <sys/idle.h>
     83 #include <sys/atomic.h>
     84 #include <sys/reboot.h>
     85 
     86 #include <uvm/uvm.h>
     87 
     88 #include "acpica.h"		/* for NACPICA, for mp_verbose */
     89 
     90 #include <machine/cpufunc.h>
     91 #include <machine/cpuvar.h>
     92 #include <machine/pmap.h>
     93 #include <machine/vmparam.h>
     94 #if defined(MULTIPROCESSOR)
     95 #include <machine/mpbiosvar.h>
     96 #endif
     97 #include <machine/mpconfig.h>		/* for mp_verbose */
     98 #include <machine/pcb.h>
     99 #include <machine/specialreg.h>
    100 #include <machine/segments.h>
    101 #include <machine/gdt.h>
    102 #include <machine/mtrr.h>
    103 #include <machine/pio.h>
    104 #include <machine/cpu_counter.h>
    105 
    106 #include <x86/fpu.h>
    107 
    108 #if NLAPIC > 0
    109 #include <machine/apicvar.h>
    110 #include <machine/i82489reg.h>
    111 #include <machine/i82489var.h>
    112 #endif
    113 
    114 #include <dev/ic/mc146818reg.h>
    115 #include <i386/isa/nvram.h>
    116 #include <dev/isa/isareg.h>
    117 
    118 #include "tsc.h"
    119 
    120 static int	cpu_match(device_t, cfdata_t, void *);
    121 static void	cpu_attach(device_t, device_t, void *);
    122 static void	cpu_defer(device_t);
    123 static int	cpu_rescan(device_t, const char *, const int *);
    124 static void	cpu_childdetached(device_t, device_t);
    125 static bool	cpu_stop(device_t);
    126 static bool	cpu_suspend(device_t, const pmf_qual_t *);
    127 static bool	cpu_resume(device_t, const pmf_qual_t *);
    128 static bool	cpu_shutdown(device_t, int);
    129 
    130 struct cpu_softc {
    131 	device_t sc_dev;		/* device tree glue */
    132 	struct cpu_info *sc_info;	/* pointer to CPU info */
    133 	bool sc_wasonline;
    134 };
    135 
    136 #ifdef MULTIPROCESSOR
    137 int mp_cpu_start(struct cpu_info *, paddr_t);
    138 void mp_cpu_start_cleanup(struct cpu_info *);
    139 const struct cpu_functions mp_cpu_funcs = { mp_cpu_start, NULL,
    140 					    mp_cpu_start_cleanup };
    141 #endif
    142 
    143 
    144 CFATTACH_DECL2_NEW(cpu, sizeof(struct cpu_softc),
    145     cpu_match, cpu_attach, NULL, NULL, cpu_rescan, cpu_childdetached);
    146 
    147 /*
    148  * Statically-allocated CPU info for the primary CPU (or the only
    149  * CPU, on uniprocessors).  The CPU info list is initialized to
    150  * point at it.
    151  */
    152 struct cpu_info cpu_info_primary __aligned(CACHE_LINE_SIZE) = {
    153 	.ci_dev = 0,
    154 	.ci_self = &cpu_info_primary,
    155 	.ci_idepth = -1,
    156 	.ci_curlwp = &lwp0,
    157 	.ci_curldt = -1,
    158 };
    159 
    160 struct cpu_info *cpu_info_list = &cpu_info_primary;
    161 
    162 #ifdef i386
    163 void		cpu_set_tss_gates(struct cpu_info *);
    164 #endif
    165 
    166 static void	cpu_init_idle_lwp(struct cpu_info *);
    167 
    168 uint32_t cpu_feature[7] __read_mostly; /* X86 CPUID feature bits */
    169 			/* [0] basic features cpuid.1:%edx
    170 			 * [1] basic features cpuid.1:%ecx (CPUID2_xxx bits)
    171 			 * [2] extended features cpuid:80000001:%edx
    172 			 * [3] extended features cpuid:80000001:%ecx
    173 			 * [4] VIA padlock features
    174 			 * [5] structured extended features cpuid.7:%ebx
    175 			 * [6] structured extended features cpuid.7:%ecx
    176 			 */
    177 
    178 #ifdef MULTIPROCESSOR
    179 bool x86_mp_online;
    180 paddr_t mp_trampoline_paddr = MP_TRAMPOLINE;
    181 #endif
    182 #if NLAPIC > 0
    183 static vaddr_t cmos_data_mapping;
    184 #endif
    185 struct cpu_info *cpu_starting;
    186 
    187 #ifdef MULTIPROCESSOR
    188 void    	cpu_hatch(void *);
    189 static void    	cpu_boot_secondary(struct cpu_info *ci);
    190 static void    	cpu_start_secondary(struct cpu_info *ci);
    191 #endif
    192 #if NLAPIC > 0
    193 static void	cpu_copy_trampoline(paddr_t);
    194 #endif
    195 
    196 /*
    197  * Runs once per boot once multiprocessor goo has been detected and
    198  * the local APIC on the boot processor has been mapped.
    199  *
    200  * Called from lapic_boot_init() (from mpbios_scan()).
    201  */
    202 #if NLAPIC > 0
    203 void
    204 cpu_init_first(void)
    205 {
    206 
    207 	cpu_info_primary.ci_cpuid = lapic_cpu_number();
    208 
    209 	cmos_data_mapping = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, UVM_KMF_VAONLY);
    210 	if (cmos_data_mapping == 0)
    211 		panic("No KVA for page 0");
    212 	pmap_kenter_pa(cmos_data_mapping, 0, VM_PROT_READ|VM_PROT_WRITE, 0);
    213 	pmap_update(pmap_kernel());
    214 }
    215 #endif
    216 
    217 static int
    218 cpu_match(device_t parent, cfdata_t match, void *aux)
    219 {
    220 
    221 	return 1;
    222 }
    223 
    224 static void
    225 cpu_vm_init(struct cpu_info *ci)
    226 {
    227 	int ncolors = 2, i;
    228 
    229 	for (i = CAI_ICACHE; i <= CAI_L2CACHE; i++) {
    230 		struct x86_cache_info *cai;
    231 		int tcolors;
    232 
    233 		cai = &ci->ci_cinfo[i];
    234 
    235 		tcolors = atop(cai->cai_totalsize);
    236 		switch(cai->cai_associativity) {
    237 		case 0xff:
    238 			tcolors = 1; /* fully associative */
    239 			break;
    240 		case 0:
    241 		case 1:
    242 			break;
    243 		default:
    244 			tcolors /= cai->cai_associativity;
    245 		}
    246 		ncolors = max(ncolors, tcolors);
    247 		/*
    248 		 * If the desired number of colors is not a power of
    249 		 * two, it won't be good.  Find the greatest power of
    250 		 * two which is an even divisor of the number of colors,
    251 		 * to preserve even coloring of pages.
    252 		 */
    253 		if (ncolors & (ncolors - 1) ) {
    254 			int try, picked = 1;
    255 			for (try = 1; try < ncolors; try *= 2) {
    256 				if (ncolors % try == 0) picked = try;
    257 			}
    258 			if (picked == 1) {
    259 				panic("desired number of cache colors %d is "
    260 			      	" > 1, but not even!", ncolors);
    261 			}
    262 			ncolors = picked;
    263 		}
    264 	}
    265 
    266 	/*
    267 	 * Knowing the size of the largest cache on this CPU, potentially
    268 	 * re-color our pages.
    269 	 */
    270 	aprint_debug_dev(ci->ci_dev, "%d page colors\n", ncolors);
    271 	uvm_page_recolor(ncolors);
    272 
    273 	pmap_tlb_cpu_init(ci);
    274 #ifndef __HAVE_DIRECT_MAP
    275 	pmap_vpage_cpu_init(ci);
    276 #endif
    277 }
    278 
    279 static void
    280 cpu_attach(device_t parent, device_t self, void *aux)
    281 {
    282 	struct cpu_softc *sc = device_private(self);
    283 	struct cpu_attach_args *caa = aux;
    284 	struct cpu_info *ci;
    285 	uintptr_t ptr;
    286 #if NLAPIC > 0
    287 	int cpunum = caa->cpu_number;
    288 #endif
    289 	static bool again;
    290 
    291 	sc->sc_dev = self;
    292 
    293 	if (ncpu == maxcpus) {
    294 #ifndef _LP64
    295 		aprint_error(": too many CPUs, please use NetBSD/amd64\n");
    296 #else
    297 		aprint_error(": too many CPUs\n");
    298 #endif
    299 		return;
    300 	}
    301 
    302 	/*
    303 	 * If we're an Application Processor, allocate a cpu_info
    304 	 * structure, otherwise use the primary's.
    305 	 */
    306 	if (caa->cpu_role == CPU_ROLE_AP) {
    307 		if ((boothowto & RB_MD1) != 0) {
    308 			aprint_error(": multiprocessor boot disabled\n");
    309 			if (!pmf_device_register(self, NULL, NULL))
    310 				aprint_error_dev(self,
    311 				    "couldn't establish power handler\n");
    312 			return;
    313 		}
    314 		aprint_naive(": Application Processor\n");
    315 		ptr = (uintptr_t)kmem_zalloc(sizeof(*ci) + CACHE_LINE_SIZE - 1,
    316 		    KM_SLEEP);
    317 		ci = (struct cpu_info *)roundup2(ptr, CACHE_LINE_SIZE);
    318 		ci->ci_curldt = -1;
    319 	} else {
    320 		aprint_naive(": %s Processor\n",
    321 		    caa->cpu_role == CPU_ROLE_SP ? "Single" : "Boot");
    322 		ci = &cpu_info_primary;
    323 #if NLAPIC > 0
    324 		if (cpunum != lapic_cpu_number()) {
    325 			/* XXX should be done earlier. */
    326 			uint32_t reg;
    327 			aprint_verbose("\n");
    328 			aprint_verbose_dev(self, "running CPU at apic %d"
    329 			    " instead of at expected %d", lapic_cpu_number(),
    330 			    cpunum);
    331 			reg = lapic_readreg(LAPIC_ID);
    332 			lapic_writereg(LAPIC_ID, (reg & ~LAPIC_ID_MASK) |
    333 			    (cpunum << LAPIC_ID_SHIFT));
    334 		}
    335 		if (cpunum != lapic_cpu_number()) {
    336 			aprint_error_dev(self, "unable to reset apic id\n");
    337 		}
    338 #endif
    339 	}
    340 
    341 	ci->ci_self = ci;
    342 	sc->sc_info = ci;
    343 	ci->ci_dev = self;
    344 	ci->ci_acpiid = caa->cpu_id;
    345 	ci->ci_cpuid = caa->cpu_number;
    346 	ci->ci_func = caa->cpu_func;
    347 	aprint_normal("\n");
    348 
    349 	/* Must be before mi_cpu_attach(). */
    350 	cpu_vm_init(ci);
    351 
    352 	if (caa->cpu_role == CPU_ROLE_AP) {
    353 		int error;
    354 
    355 		error = mi_cpu_attach(ci);
    356 		if (error != 0) {
    357 			aprint_error_dev(self,
    358 			    "mi_cpu_attach failed with %d\n", error);
    359 			return;
    360 		}
    361 		cpu_init_tss(ci);
    362 	} else {
    363 		KASSERT(ci->ci_data.cpu_idlelwp != NULL);
    364 	}
    365 
    366 	pmap_reference(pmap_kernel());
    367 	ci->ci_pmap = pmap_kernel();
    368 	ci->ci_tlbstate = TLBSTATE_STALE;
    369 
    370 	/*
    371 	 * Boot processor may not be attached first, but the below
    372 	 * must be done to allow booting other processors.
    373 	 */
    374 	if (!again) {
    375 		atomic_or_32(&ci->ci_flags, CPUF_PRESENT | CPUF_PRIMARY);
    376 		/* Basic init. */
    377 		cpu_intr_init(ci);
    378 		cpu_get_tsc_freq(ci);
    379 		cpu_init(ci);
    380 #ifdef i386
    381 		cpu_set_tss_gates(ci);
    382 #endif
    383 		pmap_cpu_init_late(ci);
    384 #if NLAPIC > 0
    385 		if (caa->cpu_role != CPU_ROLE_SP) {
    386 			/* Enable lapic. */
    387 			lapic_enable();
    388 			lapic_set_lvt();
    389 			lapic_calibrate_timer(ci);
    390 		}
    391 #endif
    392 		/* Make sure DELAY() is initialized. */
    393 		DELAY(1);
    394 		again = true;
    395 	}
    396 
    397 	/* further PCB init done later. */
    398 
    399 	switch (caa->cpu_role) {
    400 	case CPU_ROLE_SP:
    401 		atomic_or_32(&ci->ci_flags, CPUF_SP);
    402 		cpu_identify(ci);
    403 		x86_errata();
    404 		x86_cpu_idle_init();
    405 		break;
    406 
    407 	case CPU_ROLE_BP:
    408 		atomic_or_32(&ci->ci_flags, CPUF_BSP);
    409 		cpu_identify(ci);
    410 		x86_errata();
    411 		x86_cpu_idle_init();
    412 		break;
    413 
    414 #ifdef MULTIPROCESSOR
    415 	case CPU_ROLE_AP:
    416 		/*
    417 		 * report on an AP
    418 		 */
    419 		cpu_intr_init(ci);
    420 		gdt_alloc_cpu(ci);
    421 #ifdef i386
    422 		cpu_set_tss_gates(ci);
    423 #endif
    424 		pmap_cpu_init_late(ci);
    425 		cpu_start_secondary(ci);
    426 		if (ci->ci_flags & CPUF_PRESENT) {
    427 			struct cpu_info *tmp;
    428 
    429 			cpu_identify(ci);
    430 			tmp = cpu_info_list;
    431 			while (tmp->ci_next)
    432 				tmp = tmp->ci_next;
    433 
    434 			tmp->ci_next = ci;
    435 		}
    436 		break;
    437 #endif
    438 
    439 	default:
    440 		panic("unknown processor type??\n");
    441 	}
    442 
    443 	pat_init(ci);
    444 
    445 	if (!pmf_device_register1(self, cpu_suspend, cpu_resume, cpu_shutdown))
    446 		aprint_error_dev(self, "couldn't establish power handler\n");
    447 
    448 #ifdef MULTIPROCESSOR
    449 	if (mp_verbose) {
    450 		struct lwp *l = ci->ci_data.cpu_idlelwp;
    451 		struct pcb *pcb = lwp_getpcb(l);
    452 
    453 		aprint_verbose_dev(self,
    454 		    "idle lwp at %p, idle sp at %p\n",
    455 		    l,
    456 #ifdef i386
    457 		    (void *)pcb->pcb_esp
    458 #else
    459 		    (void *)pcb->pcb_rsp
    460 #endif
    461 		);
    462 	}
    463 #endif
    464 
    465 	/*
    466 	 * Postpone the "cpufeaturebus" scan.
    467 	 * It is safe to scan the pseudo-bus
    468 	 * only after all CPUs have attached.
    469 	 */
    470 	(void)config_defer(self, cpu_defer);
    471 }
    472 
    473 static void
    474 cpu_defer(device_t self)
    475 {
    476 	cpu_rescan(self, NULL, NULL);
    477 }
    478 
    479 static int
    480 cpu_rescan(device_t self, const char *ifattr, const int *locators)
    481 {
    482 	struct cpu_softc *sc = device_private(self);
    483 	struct cpufeature_attach_args cfaa;
    484 	struct cpu_info *ci = sc->sc_info;
    485 
    486 	memset(&cfaa, 0, sizeof(cfaa));
    487 	cfaa.ci = ci;
    488 
    489 	if (ifattr_match(ifattr, "cpufeaturebus")) {
    490 		if (ci->ci_frequency == NULL) {
    491 			cfaa.name = "frequency";
    492 			ci->ci_frequency = config_found_ia(self,
    493 			    "cpufeaturebus", &cfaa, NULL);
    494 		}
    495 
    496 		if (ci->ci_padlock == NULL) {
    497 			cfaa.name = "padlock";
    498 			ci->ci_padlock = config_found_ia(self,
    499 			    "cpufeaturebus", &cfaa, NULL);
    500 		}
    501 
    502 		if (ci->ci_temperature == NULL) {
    503 			cfaa.name = "temperature";
    504 			ci->ci_temperature = config_found_ia(self,
    505 			    "cpufeaturebus", &cfaa, NULL);
    506 		}
    507 
    508 		if (ci->ci_vm == NULL) {
    509 			cfaa.name = "vm";
    510 			ci->ci_vm = config_found_ia(self,
    511 			    "cpufeaturebus", &cfaa, NULL);
    512 		}
    513 	}
    514 
    515 	return 0;
    516 }
    517 
    518 static void
    519 cpu_childdetached(device_t self, device_t child)
    520 {
    521 	struct cpu_softc *sc = device_private(self);
    522 	struct cpu_info *ci = sc->sc_info;
    523 
    524 	if (ci->ci_frequency == child)
    525 		ci->ci_frequency = NULL;
    526 
    527 	if (ci->ci_padlock == child)
    528 		ci->ci_padlock = NULL;
    529 
    530 	if (ci->ci_temperature == child)
    531 		ci->ci_temperature = NULL;
    532 
    533 	if (ci->ci_vm == child)
    534 		ci->ci_vm = NULL;
    535 }
    536 
    537 /*
    538  * Initialize the processor appropriately.
    539  */
    540 
    541 void
    542 cpu_init(struct cpu_info *ci)
    543 {
    544 	uint32_t cr4 = 0;
    545 
    546 	lcr0(rcr0() | CR0_WP);
    547 
    548 	/*
    549 	 * On a P6 or above, enable global TLB caching if the
    550 	 * hardware supports it.
    551 	 */
    552 	if (cpu_feature[0] & CPUID_PGE)
    553 		cr4 |= CR4_PGE;	/* enable global TLB caching */
    554 
    555 	/*
    556 	 * If we have FXSAVE/FXRESTOR, use them.
    557 	 */
    558 	if (cpu_feature[0] & CPUID_FXSR) {
    559 		cr4 |= CR4_OSFXSR;
    560 
    561 		/*
    562 		 * If we have SSE/SSE2, enable XMM exceptions.
    563 		 */
    564 		if (cpu_feature[0] & (CPUID_SSE|CPUID_SSE2))
    565 			cr4 |= CR4_OSXMMEXCPT;
    566 	}
    567 
    568 	/* If xsave is supported, enable it */
    569 	if (cpu_feature[1] & CPUID2_XSAVE)
    570 		cr4 |= CR4_OSXSAVE;
    571 
    572 	/* If SMEP is supported, enable it */
    573 	if (cpu_feature[5] & CPUID_SEF_SMEP)
    574 		cr4 |= CR4_SMEP;
    575 
    576 	if (cr4) {
    577 		cr4 |= rcr4();
    578 		lcr4(cr4);
    579 	}
    580 
    581 	/* If xsave is enabled, enable all fpu features */
    582 	if (cr4 & CR4_OSXSAVE)
    583 		wrxcr(0, x86_xsave_features & XCR0_FPU);
    584 
    585 #ifdef MTRR
    586 	/*
    587 	 * On a P6 or above, initialize MTRR's if the hardware supports them.
    588 	 */
    589 	if (cpu_feature[0] & CPUID_MTRR) {
    590 		if ((ci->ci_flags & CPUF_AP) == 0)
    591 			i686_mtrr_init_first();
    592 		mtrr_init_cpu(ci);
    593 	}
    594 
    595 #ifdef i386
    596 	if (strcmp((char *)(ci->ci_vendor), "AuthenticAMD") == 0) {
    597 		/*
    598 		 * Must be a K6-2 Step >= 7 or a K6-III.
    599 		 */
    600 		if (CPUID_TO_FAMILY(ci->ci_signature) == 5) {
    601 			if (CPUID_TO_MODEL(ci->ci_signature) > 8 ||
    602 			    (CPUID_TO_MODEL(ci->ci_signature) == 8 &&
    603 			     CPUID_TO_STEPPING(ci->ci_signature) >= 7)) {
    604 				mtrr_funcs = &k6_mtrr_funcs;
    605 				k6_mtrr_init_first();
    606 				mtrr_init_cpu(ci);
    607 			}
    608 		}
    609 	}
    610 #endif	/* i386 */
    611 #endif /* MTRR */
    612 
    613 	if (ci != &cpu_info_primary) {
    614 		/* Synchronize TSC again, and check for drift. */
    615 		wbinvd();
    616 		atomic_or_32(&ci->ci_flags, CPUF_RUNNING);
    617 		tsc_sync_ap(ci);
    618 	} else {
    619 		atomic_or_32(&ci->ci_flags, CPUF_RUNNING);
    620 	}
    621 }
    622 
    623 #ifdef MULTIPROCESSOR
    624 void
    625 cpu_boot_secondary_processors(void)
    626 {
    627 	struct cpu_info *ci;
    628 	kcpuset_t *cpus;
    629 	u_long i;
    630 
    631 	/* Now that we know the number of CPUs, patch the text segment. */
    632 	x86_patch(false);
    633 
    634 	kcpuset_create(&cpus, true);
    635 	kcpuset_set(cpus, cpu_index(curcpu()));
    636 	for (i = 0; i < maxcpus; i++) {
    637 		ci = cpu_lookup(i);
    638 		if (ci == NULL)
    639 			continue;
    640 		if (ci->ci_data.cpu_idlelwp == NULL)
    641 			continue;
    642 		if ((ci->ci_flags & CPUF_PRESENT) == 0)
    643 			continue;
    644 		if (ci->ci_flags & (CPUF_BSP|CPUF_SP|CPUF_PRIMARY))
    645 			continue;
    646 		cpu_boot_secondary(ci);
    647 		kcpuset_set(cpus, cpu_index(ci));
    648 	}
    649 	while (!kcpuset_match(cpus, kcpuset_running))
    650 		;
    651 	kcpuset_destroy(cpus);
    652 
    653 	x86_mp_online = true;
    654 
    655 	/* Now that we know about the TSC, attach the timecounter. */
    656 	tsc_tc_init();
    657 
    658 	/* Enable zeroing of pages in the idle loop if we have SSE2. */
    659 	vm_page_zero_enable = ((cpu_feature[0] & CPUID_SSE2) != 0);
    660 }
    661 #endif
    662 
    663 static void
    664 cpu_init_idle_lwp(struct cpu_info *ci)
    665 {
    666 	struct lwp *l = ci->ci_data.cpu_idlelwp;
    667 	struct pcb *pcb = lwp_getpcb(l);
    668 
    669 	pcb->pcb_cr0 = rcr0();
    670 }
    671 
    672 void
    673 cpu_init_idle_lwps(void)
    674 {
    675 	struct cpu_info *ci;
    676 	u_long i;
    677 
    678 	for (i = 0; i < maxcpus; i++) {
    679 		ci = cpu_lookup(i);
    680 		if (ci == NULL)
    681 			continue;
    682 		if (ci->ci_data.cpu_idlelwp == NULL)
    683 			continue;
    684 		if ((ci->ci_flags & CPUF_PRESENT) == 0)
    685 			continue;
    686 		cpu_init_idle_lwp(ci);
    687 	}
    688 }
    689 
    690 #ifdef MULTIPROCESSOR
    691 void
    692 cpu_start_secondary(struct cpu_info *ci)
    693 {
    694 	paddr_t mp_pdirpa;
    695 	u_long psl;
    696 	int i;
    697 
    698 	mp_pdirpa = pmap_init_tmp_pgtbl(mp_trampoline_paddr);
    699 	cpu_copy_trampoline(mp_pdirpa);
    700 
    701 	atomic_or_32(&ci->ci_flags, CPUF_AP);
    702 	ci->ci_curlwp = ci->ci_data.cpu_idlelwp;
    703 	if (CPU_STARTUP(ci, mp_trampoline_paddr) != 0) {
    704 		return;
    705 	}
    706 
    707 	/*
    708 	 * Wait for it to become ready.   Setting cpu_starting opens the
    709 	 * initial gate and allows the AP to start soft initialization.
    710 	 */
    711 	KASSERT(cpu_starting == NULL);
    712 	cpu_starting = ci;
    713 	for (i = 100000; (!(ci->ci_flags & CPUF_PRESENT)) && i > 0; i--) {
    714 		i8254_delay(10);
    715 	}
    716 
    717 	if ((ci->ci_flags & CPUF_PRESENT) == 0) {
    718 		aprint_error_dev(ci->ci_dev, "failed to become ready\n");
    719 #if defined(MPDEBUG) && defined(DDB)
    720 		printf("dropping into debugger; continue from here to resume boot\n");
    721 		Debugger();
    722 #endif
    723 	} else {
    724 		/*
    725 		 * Synchronize time stamp counters. Invalidate cache and do
    726 		 * twice to try and minimize possible cache effects. Disable
    727 		 * interrupts to try and rule out any external interference.
    728 		 */
    729 		psl = x86_read_psl();
    730 		x86_disable_intr();
    731 		wbinvd();
    732 		tsc_sync_bp(ci);
    733 		x86_write_psl(psl);
    734 	}
    735 
    736 	CPU_START_CLEANUP(ci);
    737 	cpu_starting = NULL;
    738 }
    739 
    740 void
    741 cpu_boot_secondary(struct cpu_info *ci)
    742 {
    743 	int64_t drift;
    744 	u_long psl;
    745 	int i;
    746 
    747 	atomic_or_32(&ci->ci_flags, CPUF_GO);
    748 	for (i = 100000; (!(ci->ci_flags & CPUF_RUNNING)) && i > 0; i--) {
    749 		i8254_delay(10);
    750 	}
    751 	if ((ci->ci_flags & CPUF_RUNNING) == 0) {
    752 		aprint_error_dev(ci->ci_dev, "failed to start\n");
    753 #if defined(MPDEBUG) && defined(DDB)
    754 		printf("dropping into debugger; continue from here to resume boot\n");
    755 		Debugger();
    756 #endif
    757 	} else {
    758 		/* Synchronize TSC again, check for drift. */
    759 		drift = ci->ci_data.cpu_cc_skew;
    760 		psl = x86_read_psl();
    761 		x86_disable_intr();
    762 		wbinvd();
    763 		tsc_sync_bp(ci);
    764 		x86_write_psl(psl);
    765 		drift -= ci->ci_data.cpu_cc_skew;
    766 		aprint_debug_dev(ci->ci_dev, "TSC skew=%lld drift=%lld\n",
    767 		    (long long)ci->ci_data.cpu_cc_skew, (long long)drift);
    768 		tsc_sync_drift(drift);
    769 	}
    770 }
    771 
    772 /*
    773  * The CPU ends up here when it's ready to run.
    774  * This is called from code in mptramp.s; at this point, we are running
    775  * in the idle pcb/idle stack of the new CPU.  When this function returns,
    776  * this processor will enter the idle loop and start looking for work.
    777  */
    778 void
    779 cpu_hatch(void *v)
    780 {
    781 	struct cpu_info *ci = (struct cpu_info *)v;
    782 	struct pcb *pcb;
    783 	int s, i;
    784 
    785 	cpu_init_msrs(ci, true);
    786 	cpu_probe(ci);
    787 
    788 	ci->ci_data.cpu_cc_freq = cpu_info_primary.ci_data.cpu_cc_freq;
    789 	/* cpu_get_tsc_freq(ci); */
    790 
    791 	KDASSERT((ci->ci_flags & CPUF_PRESENT) == 0);
    792 
    793 	/*
    794 	 * Synchronize time stamp counters.  Invalidate cache and do twice
    795 	 * to try and minimize possible cache effects.  Note that interrupts
    796 	 * are off at this point.
    797 	 */
    798 	wbinvd();
    799 	atomic_or_32(&ci->ci_flags, CPUF_PRESENT);
    800 	tsc_sync_ap(ci);
    801 
    802 	/*
    803 	 * Wait to be brought online.  Use 'monitor/mwait' if available,
    804 	 * in order to make the TSC drift as much as possible. so that
    805 	 * we can detect it later.  If not available, try 'pause'.
    806 	 * We'd like to use 'hlt', but we have interrupts off.
    807 	 */
    808 	while ((ci->ci_flags & CPUF_GO) == 0) {
    809 		if ((cpu_feature[1] & CPUID2_MONITOR) != 0) {
    810 			x86_monitor(&ci->ci_flags, 0, 0);
    811 			if ((ci->ci_flags & CPUF_GO) != 0) {
    812 				continue;
    813 			}
    814 			x86_mwait(0, 0);
    815 		} else {
    816 	/*
    817 	 * XXX The loop repetition count could be a lot higher, but
    818 	 * XXX currently qemu emulator takes a _very_long_time_ to
    819 	 * XXX execute the pause instruction.  So for now, use a low
    820 	 * XXX value to allow the cpu to hatch before timing out.
    821 	 */
    822 			for (i = 50; i != 0; i--) {
    823 				x86_pause();
    824 			}
    825 		}
    826 	}
    827 
    828 	/* Because the text may have been patched in x86_patch(). */
    829 	wbinvd();
    830 	x86_flush();
    831 	tlbflushg();
    832 
    833 	KASSERT((ci->ci_flags & CPUF_RUNNING) == 0);
    834 
    835 #ifdef PAE
    836 	pd_entry_t * l3_pd = ci->ci_pae_l3_pdir;
    837 	for (i = 0 ; i < PDP_SIZE; i++) {
    838 		l3_pd[i] = pmap_kernel()->pm_pdirpa[i] | PG_V;
    839 	}
    840 	lcr3(ci->ci_pae_l3_pdirpa);
    841 #else
    842 	lcr3(pmap_pdirpa(pmap_kernel(), 0));
    843 #endif
    844 
    845 	pcb = lwp_getpcb(curlwp);
    846 	pcb->pcb_cr3 = rcr3();
    847 	pcb = lwp_getpcb(ci->ci_data.cpu_idlelwp);
    848 	lcr0(pcb->pcb_cr0);
    849 
    850 	cpu_init_idt();
    851 	gdt_init_cpu(ci);
    852 #if NLAPIC > 0
    853 	lapic_enable();
    854 	lapic_set_lvt();
    855 	lapic_initclocks();
    856 #endif
    857 
    858 	fpuinit(ci);
    859 	lldt(GSYSSEL(GLDT_SEL, SEL_KPL));
    860 	ltr(ci->ci_tss_sel);
    861 
    862 	cpu_init(ci);
    863 	cpu_get_tsc_freq(ci);
    864 
    865 	s = splhigh();
    866 	lapic_write_tpri(0);
    867 	x86_enable_intr();
    868 	splx(s);
    869 	x86_errata();
    870 
    871 	aprint_debug_dev(ci->ci_dev, "running\n");
    872 
    873 	idle_loop(NULL);
    874 	KASSERT(false);
    875 }
    876 #endif
    877 
    878 #if defined(DDB)
    879 
    880 #include <ddb/db_output.h>
    881 #include <machine/db_machdep.h>
    882 
    883 /*
    884  * Dump CPU information from ddb.
    885  */
    886 void
    887 cpu_debug_dump(void)
    888 {
    889 	struct cpu_info *ci;
    890 	CPU_INFO_ITERATOR cii;
    891 
    892 	db_printf("addr		dev	id	flags	ipis	curlwp 		fpcurlwp\n");
    893 	for (CPU_INFO_FOREACH(cii, ci)) {
    894 		db_printf("%p	%s	%ld	%x	%x	%10p	%10p\n",
    895 		    ci,
    896 		    ci->ci_dev == NULL ? "BOOT" : device_xname(ci->ci_dev),
    897 		    (long)ci->ci_cpuid,
    898 		    ci->ci_flags, ci->ci_ipis,
    899 		    ci->ci_curlwp,
    900 		    ci->ci_fpcurlwp);
    901 	}
    902 }
    903 #endif
    904 
    905 #if NLAPIC > 0
    906 static void
    907 cpu_copy_trampoline(paddr_t pdir_pa)
    908 {
    909 	extern uint32_t nox_flag;
    910 	extern u_char cpu_spinup_trampoline[];
    911 	extern u_char cpu_spinup_trampoline_end[];
    912 	vaddr_t mp_trampoline_vaddr;
    913 	struct {
    914 		uint32_t large;
    915 		uint32_t nox;
    916 		uint32_t pdir;
    917 	} smp_data;
    918 	CTASSERT(sizeof(smp_data) == 3 * 4);
    919 
    920 	smp_data.large = (pmap_largepages != 0);
    921 	smp_data.nox = nox_flag;
    922 	smp_data.pdir = (uint32_t)(pdir_pa & 0xFFFFFFFF);
    923 
    924 	/* Enter the physical address */
    925 	mp_trampoline_vaddr = uvm_km_alloc(kernel_map, PAGE_SIZE, 0,
    926 	    UVM_KMF_VAONLY);
    927 	pmap_kenter_pa(mp_trampoline_vaddr, mp_trampoline_paddr,
    928 	    VM_PROT_READ | VM_PROT_WRITE, 0);
    929 	pmap_update(pmap_kernel());
    930 
    931 	/* Copy boot code */
    932 	memcpy((void *)mp_trampoline_vaddr,
    933 	    cpu_spinup_trampoline,
    934 	    cpu_spinup_trampoline_end - cpu_spinup_trampoline);
    935 
    936 	/* Copy smp_data at the end */
    937 	memcpy((void *)(mp_trampoline_vaddr + PAGE_SIZE - sizeof(smp_data)),
    938 	    &smp_data, sizeof(smp_data));
    939 
    940 	pmap_kremove(mp_trampoline_vaddr, PAGE_SIZE);
    941 	pmap_update(pmap_kernel());
    942 	uvm_km_free(kernel_map, mp_trampoline_vaddr, PAGE_SIZE, UVM_KMF_VAONLY);
    943 }
    944 #endif
    945 
    946 #ifdef MULTIPROCESSOR
    947 int
    948 mp_cpu_start(struct cpu_info *ci, paddr_t target)
    949 {
    950 	unsigned short dwordptr[2];
    951 	int error;
    952 
    953 	/*
    954 	 * Bootstrap code must be addressable in real mode
    955 	 * and it must be page aligned.
    956 	 */
    957 	KASSERT(target < 0x10000 && target % PAGE_SIZE == 0);
    958 
    959 	/*
    960 	 * "The BSP must initialize CMOS shutdown code to 0Ah ..."
    961 	 */
    962 
    963 	outb(IO_RTC, NVRAM_RESET);
    964 	outb(IO_RTC+1, NVRAM_RESET_JUMP);
    965 
    966 	/*
    967 	 * "and the warm reset vector (DWORD based at 40:67) to point
    968 	 * to the AP startup code ..."
    969 	 */
    970 
    971 	dwordptr[0] = 0;
    972 	dwordptr[1] = target >> 4;
    973 
    974 #if NLAPIC > 0
    975 	memcpy((uint8_t *)cmos_data_mapping + 0x467, dwordptr, 4);
    976 #endif
    977 
    978 	if ((cpu_feature[0] & CPUID_APIC) == 0) {
    979 		aprint_error("mp_cpu_start: CPU does not have APIC\n");
    980 		return ENODEV;
    981 	}
    982 
    983 	/*
    984 	 * ... prior to executing the following sequence:".  We'll also add in
    985 	 * local cache flush, in case the BIOS has left the AP with its cache
    986 	 * disabled.  It may not be able to cope with MP coherency.
    987 	 */
    988 	wbinvd();
    989 
    990 	if (ci->ci_flags & CPUF_AP) {
    991 		error = x86_ipi_init(ci->ci_cpuid);
    992 		if (error != 0) {
    993 			aprint_error_dev(ci->ci_dev, "%s: IPI not taken (1)\n",
    994 			    __func__);
    995 			return error;
    996 		}
    997 		i8254_delay(10000);
    998 
    999 		error = x86_ipi_startup(ci->ci_cpuid, target / PAGE_SIZE);
   1000 		if (error != 0) {
   1001 			aprint_error_dev(ci->ci_dev, "%s: IPI not taken (2)\n",
   1002 			    __func__);
   1003 			return error;
   1004 		}
   1005 		i8254_delay(200);
   1006 
   1007 		error = x86_ipi_startup(ci->ci_cpuid, target / PAGE_SIZE);
   1008 		if (error != 0) {
   1009 			aprint_error_dev(ci->ci_dev, "%s: IPI not taken (3)\n",
   1010 			    __func__);
   1011 			return error;
   1012 		}
   1013 		i8254_delay(200);
   1014 	}
   1015 
   1016 	return 0;
   1017 }
   1018 
   1019 void
   1020 mp_cpu_start_cleanup(struct cpu_info *ci)
   1021 {
   1022 	/*
   1023 	 * Ensure the NVRAM reset byte contains something vaguely sane.
   1024 	 */
   1025 
   1026 	outb(IO_RTC, NVRAM_RESET);
   1027 	outb(IO_RTC+1, NVRAM_RESET_RST);
   1028 }
   1029 #endif
   1030 
   1031 #ifdef __x86_64__
   1032 typedef void (vector)(void);
   1033 extern vector Xsyscall, Xsyscall32;
   1034 #endif
   1035 
   1036 void
   1037 cpu_init_msrs(struct cpu_info *ci, bool full)
   1038 {
   1039 #ifdef __x86_64__
   1040 	wrmsr(MSR_STAR,
   1041 	    ((uint64_t)GSEL(GCODE_SEL, SEL_KPL) << 32) |
   1042 	    ((uint64_t)LSEL(LSYSRETBASE_SEL, SEL_UPL) << 48));
   1043 	wrmsr(MSR_LSTAR, (uint64_t)Xsyscall);
   1044 	wrmsr(MSR_CSTAR, (uint64_t)Xsyscall32);
   1045 	wrmsr(MSR_SFMASK, PSL_NT|PSL_T|PSL_I|PSL_C);
   1046 
   1047 	if (full) {
   1048 		wrmsr(MSR_FSBASE, 0);
   1049 		wrmsr(MSR_GSBASE, (uint64_t)ci);
   1050 		wrmsr(MSR_KERNELGSBASE, 0);
   1051 	}
   1052 #endif	/* __x86_64__ */
   1053 
   1054 	if (cpu_feature[2] & CPUID_NOX)
   1055 		wrmsr(MSR_EFER, rdmsr(MSR_EFER) | EFER_NXE);
   1056 }
   1057 
   1058 void
   1059 cpu_offline_md(void)
   1060 {
   1061 	int s;
   1062 
   1063 	s = splhigh();
   1064 	fpusave_cpu(true);
   1065 	splx(s);
   1066 }
   1067 
   1068 /* XXX joerg restructure and restart CPUs individually */
   1069 static bool
   1070 cpu_stop(device_t dv)
   1071 {
   1072 	struct cpu_softc *sc = device_private(dv);
   1073 	struct cpu_info *ci = sc->sc_info;
   1074 	int err;
   1075 
   1076 	KASSERT((ci->ci_flags & CPUF_PRESENT) != 0);
   1077 
   1078 	if ((ci->ci_flags & CPUF_PRIMARY) != 0)
   1079 		return true;
   1080 
   1081 	if (ci->ci_data.cpu_idlelwp == NULL)
   1082 		return true;
   1083 
   1084 	sc->sc_wasonline = !(ci->ci_schedstate.spc_flags & SPCF_OFFLINE);
   1085 
   1086 	if (sc->sc_wasonline) {
   1087 		mutex_enter(&cpu_lock);
   1088 		err = cpu_setstate(ci, false);
   1089 		mutex_exit(&cpu_lock);
   1090 
   1091 		if (err != 0)
   1092 			return false;
   1093 	}
   1094 
   1095 	return true;
   1096 }
   1097 
   1098 static bool
   1099 cpu_suspend(device_t dv, const pmf_qual_t *qual)
   1100 {
   1101 	struct cpu_softc *sc = device_private(dv);
   1102 	struct cpu_info *ci = sc->sc_info;
   1103 
   1104 	if ((ci->ci_flags & CPUF_PRESENT) == 0)
   1105 		return true;
   1106 	else {
   1107 		cpufreq_suspend(ci);
   1108 	}
   1109 
   1110 	return cpu_stop(dv);
   1111 }
   1112 
   1113 static bool
   1114 cpu_resume(device_t dv, const pmf_qual_t *qual)
   1115 {
   1116 	struct cpu_softc *sc = device_private(dv);
   1117 	struct cpu_info *ci = sc->sc_info;
   1118 	int err = 0;
   1119 
   1120 	if ((ci->ci_flags & CPUF_PRESENT) == 0)
   1121 		return true;
   1122 
   1123 	if ((ci->ci_flags & CPUF_PRIMARY) != 0)
   1124 		goto out;
   1125 
   1126 	if (ci->ci_data.cpu_idlelwp == NULL)
   1127 		goto out;
   1128 
   1129 	if (sc->sc_wasonline) {
   1130 		mutex_enter(&cpu_lock);
   1131 		err = cpu_setstate(ci, true);
   1132 		mutex_exit(&cpu_lock);
   1133 	}
   1134 
   1135 out:
   1136 	if (err != 0)
   1137 		return false;
   1138 
   1139 	cpufreq_resume(ci);
   1140 
   1141 	return true;
   1142 }
   1143 
   1144 static bool
   1145 cpu_shutdown(device_t dv, int how)
   1146 {
   1147 	struct cpu_softc *sc = device_private(dv);
   1148 	struct cpu_info *ci = sc->sc_info;
   1149 
   1150 	if ((ci->ci_flags & CPUF_BSP) != 0)
   1151 		return false;
   1152 
   1153 	if ((ci->ci_flags & CPUF_PRESENT) == 0)
   1154 		return true;
   1155 
   1156 	return cpu_stop(dv);
   1157 }
   1158 
   1159 void
   1160 cpu_get_tsc_freq(struct cpu_info *ci)
   1161 {
   1162 	uint64_t last_tsc;
   1163 
   1164 	if (cpu_hascounter()) {
   1165 		last_tsc = cpu_counter_serializing();
   1166 		i8254_delay(100000);
   1167 		ci->ci_data.cpu_cc_freq =
   1168 		    (cpu_counter_serializing() - last_tsc) * 10;
   1169 	}
   1170 }
   1171 
   1172 void
   1173 x86_cpu_idle_mwait(void)
   1174 {
   1175 	struct cpu_info *ci = curcpu();
   1176 
   1177 	KASSERT(ci->ci_ilevel == IPL_NONE);
   1178 
   1179 	x86_monitor(&ci->ci_want_resched, 0, 0);
   1180 	if (__predict_false(ci->ci_want_resched)) {
   1181 		return;
   1182 	}
   1183 	x86_mwait(0, 0);
   1184 }
   1185 
   1186 void
   1187 x86_cpu_idle_halt(void)
   1188 {
   1189 	struct cpu_info *ci = curcpu();
   1190 
   1191 	KASSERT(ci->ci_ilevel == IPL_NONE);
   1192 
   1193 	x86_disable_intr();
   1194 	if (!__predict_false(ci->ci_want_resched)) {
   1195 		x86_stihlt();
   1196 	} else {
   1197 		x86_enable_intr();
   1198 	}
   1199 }
   1200 
   1201 /*
   1202  * Loads pmap for the current CPU.
   1203  */
   1204 void
   1205 cpu_load_pmap(struct pmap *pmap, struct pmap *oldpmap)
   1206 {
   1207 #ifdef PAE
   1208 	struct cpu_info *ci = curcpu();
   1209 	bool interrupts_enabled;
   1210 	pd_entry_t *l3_pd = ci->ci_pae_l3_pdir;
   1211 	int i;
   1212 
   1213 	/*
   1214 	 * disable interrupts to block TLB shootdowns, which can reload cr3.
   1215 	 * while this doesn't block NMIs, it's probably ok as NMIs unlikely
   1216 	 * reload cr3.
   1217 	 */
   1218 	interrupts_enabled = (x86_read_flags() & PSL_I) != 0;
   1219 	if (interrupts_enabled)
   1220 		x86_disable_intr();
   1221 
   1222 	for (i = 0 ; i < PDP_SIZE; i++) {
   1223 		l3_pd[i] = pmap->pm_pdirpa[i] | PG_V;
   1224 	}
   1225 
   1226 	if (interrupts_enabled)
   1227 		x86_enable_intr();
   1228 	tlbflush();
   1229 #else /* PAE */
   1230 	lcr3(pmap_pdirpa(pmap, 0));
   1231 #endif /* PAE */
   1232 }
   1233 
   1234 /*
   1235  * Notify all other cpus to halt.
   1236  */
   1237 
   1238 void
   1239 cpu_broadcast_halt(void)
   1240 {
   1241 	x86_broadcast_ipi(X86_IPI_HALT);
   1242 }
   1243 
   1244 /*
   1245  * Send a dummy ipi to a cpu to force it to run splraise()/spllower()
   1246  */
   1247 
   1248 void
   1249 cpu_kick(struct cpu_info *ci)
   1250 {
   1251 	x86_send_ipi(ci, 0);
   1252 }
   1253