Home | History | Annotate | Line # | Download | only in x86
cpu.c revision 1.93
      1 /*	$NetBSD: cpu.c,v 1.93 2011/09/28 15:38:21 jruoho Exp $	*/
      2 
      3 /*-
      4  * Copyright (c) 2000, 2006, 2007, 2008 The NetBSD Foundation, Inc.
      5  * All rights reserved.
      6  *
      7  * This code is derived from software contributed to The NetBSD Foundation
      8  * by Bill Sommerfeld of RedBack Networks Inc, and by Andrew Doran.
      9  *
     10  * Redistribution and use in source and binary forms, with or without
     11  * modification, are permitted provided that the following conditions
     12  * are met:
     13  * 1. Redistributions of source code must retain the above copyright
     14  *    notice, this list of conditions and the following disclaimer.
     15  * 2. Redistributions in binary form must reproduce the above copyright
     16  *    notice, this list of conditions and the following disclaimer in the
     17  *    documentation and/or other materials provided with the distribution.
     18  *
     19  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     20  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     21  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     22  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     23  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     24  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     25  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     26  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     27  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     28  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     29  * POSSIBILITY OF SUCH DAMAGE.
     30  */
     31 
     32 /*
     33  * Copyright (c) 1999 Stefan Grefen
     34  *
     35  * Redistribution and use in source and binary forms, with or without
     36  * modification, are permitted provided that the following conditions
     37  * are met:
     38  * 1. Redistributions of source code must retain the above copyright
     39  *    notice, this list of conditions and the following disclaimer.
     40  * 2. Redistributions in binary form must reproduce the above copyright
     41  *    notice, this list of conditions and the following disclaimer in the
     42  *    documentation and/or other materials provided with the distribution.
     43  * 3. All advertising materials mentioning features or use of this software
     44  *    must display the following acknowledgement:
     45  *      This product includes software developed by the NetBSD
     46  *      Foundation, Inc. and its contributors.
     47  * 4. Neither the name of The NetBSD Foundation nor the names of its
     48  *    contributors may be used to endorse or promote products derived
     49  *    from this software without specific prior written permission.
     50  *
     51  * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND ANY
     52  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
     53  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
     54  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR AND CONTRIBUTORS BE LIABLE
     55  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
     56  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
     57  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
     58  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
     59  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
     60  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
     61  * SUCH DAMAGE.
     62  */
     63 
     64 #include <sys/cdefs.h>
     65 __KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.93 2011/09/28 15:38:21 jruoho Exp $");
     66 
     67 #include "opt_ddb.h"
     68 #include "opt_mpbios.h"		/* for MPDEBUG */
     69 #include "opt_mtrr.h"
     70 
     71 #include "lapic.h"
     72 #include "ioapic.h"
     73 
     74 #ifdef i386
     75 #include "npx.h"
     76 #endif
     77 
     78 #include <sys/param.h>
     79 #include <sys/proc.h>
     80 #include <sys/systm.h>
     81 #include <sys/device.h>
     82 #include <sys/kmem.h>
     83 #include <sys/cpu.h>
     84 #include <sys/cpufreq.h>
     85 #include <sys/atomic.h>
     86 #include <sys/reboot.h>
     87 
     88 #include <uvm/uvm.h>
     89 
     90 #include <machine/cpufunc.h>
     91 #include <machine/cpuvar.h>
     92 #include <machine/pmap.h>
     93 #include <machine/vmparam.h>
     94 #include <machine/mpbiosvar.h>
     95 #include <machine/pcb.h>
     96 #include <machine/specialreg.h>
     97 #include <machine/segments.h>
     98 #include <machine/gdt.h>
     99 #include <machine/mtrr.h>
    100 #include <machine/pio.h>
    101 #include <machine/cpu_counter.h>
    102 
    103 #ifdef i386
    104 #include <machine/tlog.h>
    105 #endif
    106 
    107 #include <machine/apicvar.h>
    108 #include <machine/i82489reg.h>
    109 #include <machine/i82489var.h>
    110 
    111 #include <dev/ic/mc146818reg.h>
    112 #include <i386/isa/nvram.h>
    113 #include <dev/isa/isareg.h>
    114 
    115 #include "tsc.h"
    116 
    117 #if MAXCPUS > 32
    118 #error cpu_info contains 32bit bitmasks
    119 #endif
    120 
    121 static int	cpu_match(device_t, cfdata_t, void *);
    122 static void	cpu_attach(device_t, device_t, void *);
    123 static void	cpu_defer(device_t);
    124 static int	cpu_rescan(device_t, const char *, const int *);
    125 static void	cpu_childdetached(device_t, device_t);
    126 static bool	cpu_suspend(device_t, const pmf_qual_t *);
    127 static bool	cpu_resume(device_t, const pmf_qual_t *);
    128 static bool	cpu_shutdown(device_t, int);
    129 
    130 struct cpu_softc {
    131 	device_t sc_dev;		/* device tree glue */
    132 	struct cpu_info *sc_info;	/* pointer to CPU info */
    133 	bool sc_wasonline;
    134 };
    135 
    136 int mp_cpu_start(struct cpu_info *, paddr_t);
    137 void mp_cpu_start_cleanup(struct cpu_info *);
    138 const struct cpu_functions mp_cpu_funcs = { mp_cpu_start, NULL,
    139 					    mp_cpu_start_cleanup };
    140 
    141 
    142 CFATTACH_DECL2_NEW(cpu, sizeof(struct cpu_softc),
    143     cpu_match, cpu_attach, NULL, NULL, cpu_rescan, cpu_childdetached);
    144 
    145 /*
    146  * Statically-allocated CPU info for the primary CPU (or the only
    147  * CPU, on uniprocessors).  The CPU info list is initialized to
    148  * point at it.
    149  */
    150 #ifdef TRAPLOG
    151 struct tlog tlog_primary;
    152 #endif
    153 struct cpu_info cpu_info_primary __aligned(CACHE_LINE_SIZE) = {
    154 	.ci_dev = 0,
    155 	.ci_self = &cpu_info_primary,
    156 	.ci_idepth = -1,
    157 	.ci_curlwp = &lwp0,
    158 	.ci_curldt = -1,
    159 	.ci_cpumask = 1,
    160 #ifdef TRAPLOG
    161 	.ci_tlog_base = &tlog_primary,
    162 #endif /* !TRAPLOG */
    163 };
    164 
    165 struct cpu_info *cpu_info_list = &cpu_info_primary;
    166 
    167 static void	cpu_set_tss_gates(struct cpu_info *);
    168 
    169 #ifdef i386
    170 static void	tss_init(struct i386tss *, void *, void *);
    171 #endif
    172 
    173 static void	cpu_init_idle_lwp(struct cpu_info *);
    174 
    175 uint32_t cpus_attached = 0;
    176 uint32_t cpus_running = 1;
    177 
    178 uint32_t cpu_feature[5]; /* X86 CPUID feature bits
    179 			  *	[0] basic features %edx
    180 			  *	[1] basic features %ecx
    181 			  *	[2] extended features %edx
    182 			  *	[3] extended features %ecx
    183 			  *	[4] VIA padlock features
    184 			  */
    185 
    186 extern char x86_64_doubleflt_stack[];
    187 
    188 bool x86_mp_online;
    189 paddr_t mp_trampoline_paddr = MP_TRAMPOLINE;
    190 static vaddr_t cmos_data_mapping;
    191 struct cpu_info *cpu_starting;
    192 
    193 void    	cpu_hatch(void *);
    194 static void    	cpu_boot_secondary(struct cpu_info *ci);
    195 static void    	cpu_start_secondary(struct cpu_info *ci);
    196 static void	cpu_copy_trampoline(void);
    197 
    198 /*
    199  * Runs once per boot once multiprocessor goo has been detected and
    200  * the local APIC on the boot processor has been mapped.
    201  *
    202  * Called from lapic_boot_init() (from mpbios_scan()).
    203  */
    204 void
    205 cpu_init_first(void)
    206 {
    207 
    208 	cpu_info_primary.ci_cpuid = lapic_cpu_number();
    209 	cpu_copy_trampoline();
    210 
    211 	cmos_data_mapping = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, UVM_KMF_VAONLY);
    212 	if (cmos_data_mapping == 0)
    213 		panic("No KVA for page 0");
    214 	pmap_kenter_pa(cmos_data_mapping, 0, VM_PROT_READ|VM_PROT_WRITE, 0);
    215 	pmap_update(pmap_kernel());
    216 }
    217 
    218 static int
    219 cpu_match(device_t parent, cfdata_t match, void *aux)
    220 {
    221 
    222 	return 1;
    223 }
    224 
    225 static void
    226 cpu_vm_init(struct cpu_info *ci)
    227 {
    228 	int ncolors = 2, i;
    229 
    230 	for (i = CAI_ICACHE; i <= CAI_L2CACHE; i++) {
    231 		struct x86_cache_info *cai;
    232 		int tcolors;
    233 
    234 		cai = &ci->ci_cinfo[i];
    235 
    236 		tcolors = atop(cai->cai_totalsize);
    237 		switch(cai->cai_associativity) {
    238 		case 0xff:
    239 			tcolors = 1; /* fully associative */
    240 			break;
    241 		case 0:
    242 		case 1:
    243 			break;
    244 		default:
    245 			tcolors /= cai->cai_associativity;
    246 		}
    247 		ncolors = max(ncolors, tcolors);
    248 		/*
    249 		 * If the desired number of colors is not a power of
    250 		 * two, it won't be good.  Find the greatest power of
    251 		 * two which is an even divisor of the number of colors,
    252 		 * to preserve even coloring of pages.
    253 		 */
    254 		if (ncolors & (ncolors - 1) ) {
    255 			int try, picked = 1;
    256 			for (try = 1; try < ncolors; try *= 2) {
    257 				if (ncolors % try == 0) picked = try;
    258 			}
    259 			if (picked == 1) {
    260 				panic("desired number of cache colors %d is "
    261 			      	" > 1, but not even!", ncolors);
    262 			}
    263 			ncolors = picked;
    264 		}
    265 	}
    266 
    267 	/*
    268 	 * Knowing the size of the largest cache on this CPU, re-color
    269 	 * our pages.
    270 	 */
    271 	if (ncolors <= uvmexp.ncolors)
    272 		return;
    273 	aprint_debug_dev(ci->ci_dev, "%d page colors\n", ncolors);
    274 	uvm_page_recolor(ncolors);
    275 }
    276 
    277 
    278 static void
    279 cpu_attach(device_t parent, device_t self, void *aux)
    280 {
    281 	struct cpu_softc *sc = device_private(self);
    282 	struct cpu_attach_args *caa = aux;
    283 	struct cpu_info *ci;
    284 	uintptr_t ptr;
    285 	int cpunum = caa->cpu_number;
    286 	static bool again;
    287 
    288 	sc->sc_dev = self;
    289 
    290 	if (cpus_attached == ~0) {
    291 		aprint_error(": increase MAXCPUS\n");
    292 		return;
    293 	}
    294 
    295 	/*
    296 	 * If we're an Application Processor, allocate a cpu_info
    297 	 * structure, otherwise use the primary's.
    298 	 */
    299 	if (caa->cpu_role == CPU_ROLE_AP) {
    300 		if ((boothowto & RB_MD1) != 0) {
    301 			aprint_error(": multiprocessor boot disabled\n");
    302 			if (!pmf_device_register(self, NULL, NULL))
    303 				aprint_error_dev(self,
    304 				    "couldn't establish power handler\n");
    305 			return;
    306 		}
    307 		aprint_naive(": Application Processor\n");
    308 		ptr = (uintptr_t)kmem_zalloc(sizeof(*ci) + CACHE_LINE_SIZE - 1,
    309 		    KM_SLEEP);
    310 		ci = (struct cpu_info *)roundup2(ptr, CACHE_LINE_SIZE);
    311 		ci->ci_curldt = -1;
    312 #ifdef TRAPLOG
    313 		ci->ci_tlog_base = kmem_zalloc(sizeof(struct tlog), KM_SLEEP);
    314 #endif
    315 	} else {
    316 		aprint_naive(": %s Processor\n",
    317 		    caa->cpu_role == CPU_ROLE_SP ? "Single" : "Boot");
    318 		ci = &cpu_info_primary;
    319 		if (cpunum != lapic_cpu_number()) {
    320 			/* XXX should be done earlier. */
    321 			uint32_t reg;
    322 			aprint_verbose("\n");
    323 			aprint_verbose_dev(self, "running CPU at apic %d"
    324 			    " instead of at expected %d", lapic_cpu_number(),
    325 			    cpunum);
    326 			reg = i82489_readreg(LAPIC_ID);
    327 			i82489_writereg(LAPIC_ID, (reg & ~LAPIC_ID_MASK) |
    328 			    (cpunum << LAPIC_ID_SHIFT));
    329 		}
    330 		if (cpunum != lapic_cpu_number()) {
    331 			aprint_error_dev(self, "unable to reset apic id\n");
    332 		}
    333 	}
    334 
    335 	ci->ci_self = ci;
    336 	sc->sc_info = ci;
    337 	ci->ci_dev = self;
    338 	ci->ci_acpiid = caa->cpu_id;
    339 	ci->ci_cpuid = caa->cpu_number;
    340 	ci->ci_func = caa->cpu_func;
    341 
    342 	/* Must be before mi_cpu_attach(). */
    343 	cpu_vm_init(ci);
    344 
    345 	if (caa->cpu_role == CPU_ROLE_AP) {
    346 		int error;
    347 
    348 		error = mi_cpu_attach(ci);
    349 		if (error != 0) {
    350 			aprint_normal("\n");
    351 			aprint_error_dev(self,
    352 			    "mi_cpu_attach failed with %d\n", error);
    353 			return;
    354 		}
    355 		cpu_init_tss(ci);
    356 	} else {
    357 		KASSERT(ci->ci_data.cpu_idlelwp != NULL);
    358 	}
    359 
    360 	ci->ci_cpumask = (1 << cpu_index(ci));
    361 	pmap_reference(pmap_kernel());
    362 	ci->ci_pmap = pmap_kernel();
    363 	ci->ci_tlbstate = TLBSTATE_STALE;
    364 
    365 	/*
    366 	 * Boot processor may not be attached first, but the below
    367 	 * must be done to allow booting other processors.
    368 	 */
    369 	if (!again) {
    370 		atomic_or_32(&ci->ci_flags, CPUF_PRESENT | CPUF_PRIMARY);
    371 		/* Basic init. */
    372 		cpu_intr_init(ci);
    373 		cpu_get_tsc_freq(ci);
    374 		cpu_init(ci);
    375 		cpu_set_tss_gates(ci);
    376 		pmap_cpu_init_late(ci);
    377 		if (caa->cpu_role != CPU_ROLE_SP) {
    378 			/* Enable lapic. */
    379 			lapic_enable();
    380 			lapic_set_lvt();
    381 			lapic_calibrate_timer(ci);
    382 		}
    383 		/* Make sure DELAY() is initialized. */
    384 		DELAY(1);
    385 		again = true;
    386 	}
    387 
    388 	/* further PCB init done later. */
    389 
    390 	switch (caa->cpu_role) {
    391 	case CPU_ROLE_SP:
    392 		atomic_or_32(&ci->ci_flags, CPUF_SP);
    393 		cpu_identify(ci);
    394 		x86_errata();
    395 		x86_cpu_idle_init();
    396 		break;
    397 
    398 	case CPU_ROLE_BP:
    399 		atomic_or_32(&ci->ci_flags, CPUF_BSP);
    400 		cpu_identify(ci);
    401 		x86_errata();
    402 		x86_cpu_idle_init();
    403 		break;
    404 
    405 	case CPU_ROLE_AP:
    406 		/*
    407 		 * report on an AP
    408 		 */
    409 		cpu_intr_init(ci);
    410 		gdt_alloc_cpu(ci);
    411 		cpu_set_tss_gates(ci);
    412 		pmap_cpu_init_late(ci);
    413 		cpu_start_secondary(ci);
    414 		if (ci->ci_flags & CPUF_PRESENT) {
    415 			struct cpu_info *tmp;
    416 
    417 			cpu_identify(ci);
    418 			tmp = cpu_info_list;
    419 			while (tmp->ci_next)
    420 				tmp = tmp->ci_next;
    421 
    422 			tmp->ci_next = ci;
    423 		}
    424 		break;
    425 
    426 	default:
    427 		aprint_normal("\n");
    428 		panic("unknown processor type??\n");
    429 	}
    430 
    431 	pat_init(ci);
    432 	atomic_or_32(&cpus_attached, ci->ci_cpumask);
    433 
    434 	if (!pmf_device_register1(self, cpu_suspend, cpu_resume, cpu_shutdown))
    435 		aprint_error_dev(self, "couldn't establish power handler\n");
    436 
    437 	if (mp_verbose) {
    438 		struct lwp *l = ci->ci_data.cpu_idlelwp;
    439 		struct pcb *pcb = lwp_getpcb(l);
    440 
    441 		aprint_verbose_dev(self,
    442 		    "idle lwp at %p, idle sp at %p\n",
    443 		    l,
    444 #ifdef i386
    445 		    (void *)pcb->pcb_esp
    446 #else
    447 		    (void *)pcb->pcb_rsp
    448 #endif
    449 		);
    450 	}
    451 
    452 	/*
    453 	 * Postpone the "cpufeaturebus" scan.
    454 	 * It is safe to scan the pseudo-bus
    455 	 * only after all CPUs have attached.
    456 	 */
    457 	(void)config_defer(self, cpu_defer);
    458 }
    459 
    460 static void
    461 cpu_defer(device_t self)
    462 {
    463 	cpu_rescan(self, NULL, NULL);
    464 }
    465 
    466 static int
    467 cpu_rescan(device_t self, const char *ifattr, const int *locators)
    468 {
    469 	struct cpu_softc *sc = device_private(self);
    470 	struct cpufeature_attach_args cfaa;
    471 	struct cpu_info *ci = sc->sc_info;
    472 
    473 	memset(&cfaa, 0, sizeof(cfaa));
    474 	cfaa.ci = ci;
    475 
    476 	if (ifattr_match(ifattr, "cpufeaturebus")) {
    477 
    478 		if (ci->ci_frequency == NULL) {
    479 			cfaa.name = "frequency";
    480 			ci->ci_frequency = config_found_ia(self,
    481 			    "cpufeaturebus", &cfaa, NULL);
    482 		}
    483 
    484 		if (ci->ci_padlock == NULL) {
    485 			cfaa.name = "padlock";
    486 			ci->ci_padlock = config_found_ia(self,
    487 			    "cpufeaturebus", &cfaa, NULL);
    488 		}
    489 
    490 		if (ci->ci_temperature == NULL) {
    491 			cfaa.name = "temperature";
    492 			ci->ci_temperature = config_found_ia(self,
    493 			    "cpufeaturebus", &cfaa, NULL);
    494 		}
    495 	}
    496 
    497 	return 0;
    498 }
    499 
    500 static void
    501 cpu_childdetached(device_t self, device_t child)
    502 {
    503 	struct cpu_softc *sc = device_private(self);
    504 	struct cpu_info *ci = sc->sc_info;
    505 
    506 	if (ci->ci_frequency == child)
    507 		ci->ci_frequency = NULL;
    508 
    509 	if (ci->ci_padlock == child)
    510 		ci->ci_padlock = NULL;
    511 
    512 	if (ci->ci_temperature == child)
    513 		ci->ci_temperature = NULL;
    514 }
    515 
    516 /*
    517  * Initialize the processor appropriately.
    518  */
    519 
    520 void
    521 cpu_init(struct cpu_info *ci)
    522 {
    523 
    524 	lcr0(rcr0() | CR0_WP);
    525 
    526 	/*
    527 	 * On a P6 or above, enable global TLB caching if the
    528 	 * hardware supports it.
    529 	 */
    530 	if (cpu_feature[0] & CPUID_PGE)
    531 		lcr4(rcr4() | CR4_PGE);	/* enable global TLB caching */
    532 
    533 	/*
    534 	 * If we have FXSAVE/FXRESTOR, use them.
    535 	 */
    536 	if (cpu_feature[0] & CPUID_FXSR) {
    537 		lcr4(rcr4() | CR4_OSFXSR);
    538 
    539 		/*
    540 		 * If we have SSE/SSE2, enable XMM exceptions.
    541 		 */
    542 		if (cpu_feature[0] & (CPUID_SSE|CPUID_SSE2))
    543 			lcr4(rcr4() | CR4_OSXMMEXCPT);
    544 	}
    545 
    546 #ifdef MTRR
    547 	/*
    548 	 * On a P6 or above, initialize MTRR's if the hardware supports them.
    549 	 */
    550 	if (cpu_feature[0] & CPUID_MTRR) {
    551 		if ((ci->ci_flags & CPUF_AP) == 0)
    552 			i686_mtrr_init_first();
    553 		mtrr_init_cpu(ci);
    554 	}
    555 
    556 #ifdef i386
    557 	if (strcmp((char *)(ci->ci_vendor), "AuthenticAMD") == 0) {
    558 		/*
    559 		 * Must be a K6-2 Step >= 7 or a K6-III.
    560 		 */
    561 		if (CPUID2FAMILY(ci->ci_signature) == 5) {
    562 			if (CPUID2MODEL(ci->ci_signature) > 8 ||
    563 			    (CPUID2MODEL(ci->ci_signature) == 8 &&
    564 			     CPUID2STEPPING(ci->ci_signature) >= 7)) {
    565 				mtrr_funcs = &k6_mtrr_funcs;
    566 				k6_mtrr_init_first();
    567 				mtrr_init_cpu(ci);
    568 			}
    569 		}
    570 	}
    571 #endif	/* i386 */
    572 #endif /* MTRR */
    573 
    574 	atomic_or_32(&cpus_running, ci->ci_cpumask);
    575 
    576 	if (ci != &cpu_info_primary) {
    577 		/* Synchronize TSC again, and check for drift. */
    578 		wbinvd();
    579 		atomic_or_32(&ci->ci_flags, CPUF_RUNNING);
    580 		tsc_sync_ap(ci);
    581 	} else {
    582 		atomic_or_32(&ci->ci_flags, CPUF_RUNNING);
    583 	}
    584 }
    585 
    586 void
    587 cpu_boot_secondary_processors(void)
    588 {
    589 	struct cpu_info *ci;
    590 	u_long i;
    591 
    592 	/* Now that we know the number of CPUs, patch the text segment. */
    593 	x86_patch(false);
    594 
    595 	for (i=0; i < maxcpus; i++) {
    596 		ci = cpu_lookup(i);
    597 		if (ci == NULL)
    598 			continue;
    599 		if (ci->ci_data.cpu_idlelwp == NULL)
    600 			continue;
    601 		if ((ci->ci_flags & CPUF_PRESENT) == 0)
    602 			continue;
    603 		if (ci->ci_flags & (CPUF_BSP|CPUF_SP|CPUF_PRIMARY))
    604 			continue;
    605 		cpu_boot_secondary(ci);
    606 	}
    607 
    608 	x86_mp_online = true;
    609 
    610 	/* Now that we know about the TSC, attach the timecounter. */
    611 	tsc_tc_init();
    612 
    613 	/* Enable zeroing of pages in the idle loop if we have SSE2. */
    614 	vm_page_zero_enable = ((cpu_feature[0] & CPUID_SSE2) != 0);
    615 }
    616 
    617 static void
    618 cpu_init_idle_lwp(struct cpu_info *ci)
    619 {
    620 	struct lwp *l = ci->ci_data.cpu_idlelwp;
    621 	struct pcb *pcb = lwp_getpcb(l);
    622 
    623 	pcb->pcb_cr0 = rcr0();
    624 }
    625 
    626 void
    627 cpu_init_idle_lwps(void)
    628 {
    629 	struct cpu_info *ci;
    630 	u_long i;
    631 
    632 	for (i = 0; i < maxcpus; i++) {
    633 		ci = cpu_lookup(i);
    634 		if (ci == NULL)
    635 			continue;
    636 		if (ci->ci_data.cpu_idlelwp == NULL)
    637 			continue;
    638 		if ((ci->ci_flags & CPUF_PRESENT) == 0)
    639 			continue;
    640 		cpu_init_idle_lwp(ci);
    641 	}
    642 }
    643 
    644 void
    645 cpu_start_secondary(struct cpu_info *ci)
    646 {
    647 	extern paddr_t mp_pdirpa;
    648 	u_long psl;
    649 	int i;
    650 
    651 	mp_pdirpa = pmap_init_tmp_pgtbl(mp_trampoline_paddr);
    652 	atomic_or_32(&ci->ci_flags, CPUF_AP);
    653 	ci->ci_curlwp = ci->ci_data.cpu_idlelwp;
    654 	if (CPU_STARTUP(ci, mp_trampoline_paddr) != 0) {
    655 		return;
    656 	}
    657 
    658 	/*
    659 	 * Wait for it to become ready.   Setting cpu_starting opens the
    660 	 * initial gate and allows the AP to start soft initialization.
    661 	 */
    662 	KASSERT(cpu_starting == NULL);
    663 	cpu_starting = ci;
    664 	for (i = 100000; (!(ci->ci_flags & CPUF_PRESENT)) && i > 0; i--) {
    665 #ifdef MPDEBUG
    666 		extern int cpu_trace[3];
    667 		static int otrace[3];
    668 		if (memcmp(otrace, cpu_trace, sizeof(otrace)) != 0) {
    669 			aprint_debug_dev(ci->ci_dev, "trace %02x %02x %02x\n",
    670 			    cpu_trace[0], cpu_trace[1], cpu_trace[2]);
    671 			memcpy(otrace, cpu_trace, sizeof(otrace));
    672 		}
    673 #endif
    674 		i8254_delay(10);
    675 	}
    676 
    677 	if ((ci->ci_flags & CPUF_PRESENT) == 0) {
    678 		aprint_error_dev(ci->ci_dev, "failed to become ready\n");
    679 #if defined(MPDEBUG) && defined(DDB)
    680 		printf("dropping into debugger; continue from here to resume boot\n");
    681 		Debugger();
    682 #endif
    683 	} else {
    684 		/*
    685 		 * Synchronize time stamp counters. Invalidate cache and do
    686 		 * twice to try and minimize possible cache effects. Disable
    687 		 * interrupts to try and rule out any external interference.
    688 		 */
    689 		psl = x86_read_psl();
    690 		x86_disable_intr();
    691 		wbinvd();
    692 		tsc_sync_bp(ci);
    693 		x86_write_psl(psl);
    694 	}
    695 
    696 	CPU_START_CLEANUP(ci);
    697 	cpu_starting = NULL;
    698 }
    699 
    700 void
    701 cpu_boot_secondary(struct cpu_info *ci)
    702 {
    703 	int64_t drift;
    704 	u_long psl;
    705 	int i;
    706 
    707 	atomic_or_32(&ci->ci_flags, CPUF_GO);
    708 	for (i = 100000; (!(ci->ci_flags & CPUF_RUNNING)) && i > 0; i--) {
    709 		i8254_delay(10);
    710 	}
    711 	if ((ci->ci_flags & CPUF_RUNNING) == 0) {
    712 		aprint_error_dev(ci->ci_dev, "failed to start\n");
    713 #if defined(MPDEBUG) && defined(DDB)
    714 		printf("dropping into debugger; continue from here to resume boot\n");
    715 		Debugger();
    716 #endif
    717 	} else {
    718 		/* Synchronize TSC again, check for drift. */
    719 		drift = ci->ci_data.cpu_cc_skew;
    720 		psl = x86_read_psl();
    721 		x86_disable_intr();
    722 		wbinvd();
    723 		tsc_sync_bp(ci);
    724 		x86_write_psl(psl);
    725 		drift -= ci->ci_data.cpu_cc_skew;
    726 		aprint_debug_dev(ci->ci_dev, "TSC skew=%lld drift=%lld\n",
    727 		    (long long)ci->ci_data.cpu_cc_skew, (long long)drift);
    728 		tsc_sync_drift(drift);
    729 	}
    730 }
    731 
    732 /*
    733  * The CPU ends up here when its ready to run
    734  * This is called from code in mptramp.s; at this point, we are running
    735  * in the idle pcb/idle stack of the new CPU.  When this function returns,
    736  * this processor will enter the idle loop and start looking for work.
    737  */
    738 void
    739 cpu_hatch(void *v)
    740 {
    741 	struct cpu_info *ci = (struct cpu_info *)v;
    742 	struct pcb *pcb;
    743 	int s, i;
    744 
    745 	cpu_init_msrs(ci, true);
    746 	cpu_probe(ci);
    747 
    748 	ci->ci_data.cpu_cc_freq = cpu_info_primary.ci_data.cpu_cc_freq;
    749 	/* cpu_get_tsc_freq(ci); */
    750 
    751 	KDASSERT((ci->ci_flags & CPUF_PRESENT) == 0);
    752 
    753 	/*
    754 	 * Synchronize time stamp counters.  Invalidate cache and do twice
    755 	 * to try and minimize possible cache effects.  Note that interrupts
    756 	 * are off at this point.
    757 	 */
    758 	wbinvd();
    759 	atomic_or_32(&ci->ci_flags, CPUF_PRESENT);
    760 	tsc_sync_ap(ci);
    761 
    762 	/*
    763 	 * Wait to be brought online.  Use 'monitor/mwait' if available,
    764 	 * in order to make the TSC drift as much as possible. so that
    765 	 * we can detect it later.  If not available, try 'pause'.
    766 	 * We'd like to use 'hlt', but we have interrupts off.
    767 	 */
    768 	while ((ci->ci_flags & CPUF_GO) == 0) {
    769 		if ((cpu_feature[1] & CPUID2_MONITOR) != 0) {
    770 			x86_monitor(&ci->ci_flags, 0, 0);
    771 			if ((ci->ci_flags & CPUF_GO) != 0) {
    772 				continue;
    773 			}
    774 			x86_mwait(0, 0);
    775 		} else {
    776 			for (i = 10000; i != 0; i--) {
    777 				x86_pause();
    778 			}
    779 		}
    780 	}
    781 
    782 	/* Because the text may have been patched in x86_patch(). */
    783 	wbinvd();
    784 	x86_flush();
    785 	tlbflushg();
    786 
    787 	KASSERT((ci->ci_flags & CPUF_RUNNING) == 0);
    788 
    789 #ifdef PAE
    790 	pd_entry_t * l3_pd = ci->ci_pae_l3_pdir;
    791 	for (i = 0 ; i < PDP_SIZE; i++) {
    792 		l3_pd[i] = pmap_kernel()->pm_pdirpa[i] | PG_V;
    793 	}
    794 	lcr3(ci->ci_pae_l3_pdirpa);
    795 #else
    796 	lcr3(pmap_pdirpa(pmap_kernel(), 0));
    797 #endif
    798 
    799 	pcb = lwp_getpcb(curlwp);
    800 	pcb->pcb_cr3 = rcr3();
    801 	pcb = lwp_getpcb(ci->ci_data.cpu_idlelwp);
    802 	lcr0(pcb->pcb_cr0);
    803 
    804 	cpu_init_idt();
    805 	gdt_init_cpu(ci);
    806 	lapic_enable();
    807 	lapic_set_lvt();
    808 	lapic_initclocks();
    809 
    810 #ifdef i386
    811 #if NNPX > 0
    812 	npxinit(ci);
    813 #endif
    814 #else
    815 	fpuinit(ci);
    816 #endif
    817 	lldt(GSYSSEL(GLDT_SEL, SEL_KPL));
    818 	ltr(ci->ci_tss_sel);
    819 
    820 	cpu_init(ci);
    821 	cpu_get_tsc_freq(ci);
    822 
    823 	s = splhigh();
    824 #ifdef i386
    825 	lapic_tpr = 0;
    826 #else
    827 	lcr8(0);
    828 #endif
    829 	x86_enable_intr();
    830 	splx(s);
    831 	x86_errata();
    832 
    833 	aprint_debug_dev(ci->ci_dev, "running\n");
    834 }
    835 
    836 #if defined(DDB)
    837 
    838 #include <ddb/db_output.h>
    839 #include <machine/db_machdep.h>
    840 
    841 /*
    842  * Dump CPU information from ddb.
    843  */
    844 void
    845 cpu_debug_dump(void)
    846 {
    847 	struct cpu_info *ci;
    848 	CPU_INFO_ITERATOR cii;
    849 
    850 	db_printf("addr		dev	id	flags	ipis	curlwp 		fpcurlwp\n");
    851 	for (CPU_INFO_FOREACH(cii, ci)) {
    852 		db_printf("%p	%s	%ld	%x	%x	%10p	%10p\n",
    853 		    ci,
    854 		    ci->ci_dev == NULL ? "BOOT" : device_xname(ci->ci_dev),
    855 		    (long)ci->ci_cpuid,
    856 		    ci->ci_flags, ci->ci_ipis,
    857 		    ci->ci_curlwp,
    858 		    ci->ci_fpcurlwp);
    859 	}
    860 }
    861 #endif
    862 
    863 static void
    864 cpu_copy_trampoline(void)
    865 {
    866 	/*
    867 	 * Copy boot code.
    868 	 */
    869 	extern u_char cpu_spinup_trampoline[];
    870 	extern u_char cpu_spinup_trampoline_end[];
    871 
    872 	vaddr_t mp_trampoline_vaddr;
    873 
    874 	mp_trampoline_vaddr = uvm_km_alloc(kernel_map, PAGE_SIZE, 0,
    875 	    UVM_KMF_VAONLY);
    876 
    877 	pmap_kenter_pa(mp_trampoline_vaddr, mp_trampoline_paddr,
    878 	    VM_PROT_READ | VM_PROT_WRITE, 0);
    879 	pmap_update(pmap_kernel());
    880 	memcpy((void *)mp_trampoline_vaddr,
    881 	    cpu_spinup_trampoline,
    882 	    cpu_spinup_trampoline_end - cpu_spinup_trampoline);
    883 
    884 	pmap_kremove(mp_trampoline_vaddr, PAGE_SIZE);
    885 	pmap_update(pmap_kernel());
    886 	uvm_km_free(kernel_map, mp_trampoline_vaddr, PAGE_SIZE, UVM_KMF_VAONLY);
    887 }
    888 
    889 #ifdef i386
    890 static void
    891 tss_init(struct i386tss *tss, void *stack, void *func)
    892 {
    893 	KASSERT(curcpu()->ci_pmap == pmap_kernel());
    894 
    895 	memset(tss, 0, sizeof *tss);
    896 	tss->tss_esp0 = tss->tss_esp = (int)((char *)stack + USPACE - 16);
    897 	tss->tss_ss0 = GSEL(GDATA_SEL, SEL_KPL);
    898 	tss->__tss_cs = GSEL(GCODE_SEL, SEL_KPL);
    899 	tss->tss_fs = GSEL(GCPU_SEL, SEL_KPL);
    900 	tss->tss_gs = tss->__tss_es = tss->__tss_ds =
    901 	    tss->__tss_ss = GSEL(GDATA_SEL, SEL_KPL);
    902 	/* %cr3 contains the value associated to pmap_kernel */
    903 	tss->tss_cr3 = rcr3();
    904 	tss->tss_esp = (int)((char *)stack + USPACE - 16);
    905 	tss->tss_ldt = GSEL(GLDT_SEL, SEL_KPL);
    906 	tss->__tss_eflags = PSL_MBO | PSL_NT;	/* XXX not needed? */
    907 	tss->__tss_eip = (int)func;
    908 }
    909 
    910 /* XXX */
    911 #define IDTVEC(name)	__CONCAT(X, name)
    912 typedef void (vector)(void);
    913 extern vector IDTVEC(tss_trap08);
    914 #ifdef DDB
    915 extern vector Xintrddbipi;
    916 extern int ddb_vec;
    917 #endif
    918 
    919 static void
    920 cpu_set_tss_gates(struct cpu_info *ci)
    921 {
    922 	struct segment_descriptor sd;
    923 
    924 	ci->ci_doubleflt_stack = (char *)uvm_km_alloc(kernel_map, USPACE, 0,
    925 	    UVM_KMF_WIRED);
    926 	tss_init(&ci->ci_doubleflt_tss, ci->ci_doubleflt_stack,
    927 	    IDTVEC(tss_trap08));
    928 	setsegment(&sd, &ci->ci_doubleflt_tss, sizeof(struct i386tss) - 1,
    929 	    SDT_SYS386TSS, SEL_KPL, 0, 0);
    930 	ci->ci_gdt[GTRAPTSS_SEL].sd = sd;
    931 	setgate(&idt[8], NULL, 0, SDT_SYSTASKGT, SEL_KPL,
    932 	    GSEL(GTRAPTSS_SEL, SEL_KPL));
    933 
    934 #if defined(DDB)
    935 	/*
    936 	 * Set up separate handler for the DDB IPI, so that it doesn't
    937 	 * stomp on a possibly corrupted stack.
    938 	 *
    939 	 * XXX overwriting the gate set in db_machine_init.
    940 	 * Should rearrange the code so that it's set only once.
    941 	 */
    942 	ci->ci_ddbipi_stack = (char *)uvm_km_alloc(kernel_map, USPACE, 0,
    943 	    UVM_KMF_WIRED);
    944 	tss_init(&ci->ci_ddbipi_tss, ci->ci_ddbipi_stack, Xintrddbipi);
    945 
    946 	setsegment(&sd, &ci->ci_ddbipi_tss, sizeof(struct i386tss) - 1,
    947 	    SDT_SYS386TSS, SEL_KPL, 0, 0);
    948 	ci->ci_gdt[GIPITSS_SEL].sd = sd;
    949 
    950 	setgate(&idt[ddb_vec], NULL, 0, SDT_SYSTASKGT, SEL_KPL,
    951 	    GSEL(GIPITSS_SEL, SEL_KPL));
    952 #endif
    953 }
    954 #else
    955 static void
    956 cpu_set_tss_gates(struct cpu_info *ci)
    957 {
    958 
    959 }
    960 #endif	/* i386 */
    961 
    962 int
    963 mp_cpu_start(struct cpu_info *ci, paddr_t target)
    964 {
    965 	unsigned short dwordptr[2];
    966 	int error;
    967 
    968 	/*
    969 	 * Bootstrap code must be addressable in real mode
    970 	 * and it must be page aligned.
    971 	 */
    972 	KASSERT(target < 0x10000 && target % PAGE_SIZE == 0);
    973 
    974 	/*
    975 	 * "The BSP must initialize CMOS shutdown code to 0Ah ..."
    976 	 */
    977 
    978 	outb(IO_RTC, NVRAM_RESET);
    979 	outb(IO_RTC+1, NVRAM_RESET_JUMP);
    980 
    981 	/*
    982 	 * "and the warm reset vector (DWORD based at 40:67) to point
    983 	 * to the AP startup code ..."
    984 	 */
    985 
    986 	dwordptr[0] = 0;
    987 	dwordptr[1] = target >> 4;
    988 
    989 	memcpy((uint8_t *)cmos_data_mapping + 0x467, dwordptr, 4);
    990 
    991 	if ((cpu_feature[0] & CPUID_APIC) == 0) {
    992 		aprint_error("mp_cpu_start: CPU does not have APIC\n");
    993 		return ENODEV;
    994 	}
    995 
    996 	/*
    997 	 * ... prior to executing the following sequence:".  We'll also add in
    998 	 * local cache flush, in case the BIOS has left the AP with its cache
    999 	 * disabled.  It may not be able to cope with MP coherency.
   1000 	 */
   1001 	wbinvd();
   1002 
   1003 	if (ci->ci_flags & CPUF_AP) {
   1004 		error = x86_ipi_init(ci->ci_cpuid);
   1005 		if (error != 0) {
   1006 			aprint_error_dev(ci->ci_dev, "%s: IPI not taken (1)\n",
   1007 			    __func__);
   1008 			return error;
   1009 		}
   1010 		i8254_delay(10000);
   1011 
   1012 		error = x86_ipi_startup(ci->ci_cpuid, target / PAGE_SIZE);
   1013 		if (error != 0) {
   1014 			aprint_error_dev(ci->ci_dev, "%s: IPI not taken (2)\n",
   1015 			    __func__);
   1016 			return error;
   1017 		}
   1018 		i8254_delay(200);
   1019 
   1020 		error = x86_ipi_startup(ci->ci_cpuid, target / PAGE_SIZE);
   1021 		if (error != 0) {
   1022 			aprint_error_dev(ci->ci_dev, "%s: IPI not taken (3)\n",
   1023 			    __func__);
   1024 			return error;
   1025 		}
   1026 		i8254_delay(200);
   1027 	}
   1028 
   1029 	return 0;
   1030 }
   1031 
   1032 void
   1033 mp_cpu_start_cleanup(struct cpu_info *ci)
   1034 {
   1035 	/*
   1036 	 * Ensure the NVRAM reset byte contains something vaguely sane.
   1037 	 */
   1038 
   1039 	outb(IO_RTC, NVRAM_RESET);
   1040 	outb(IO_RTC+1, NVRAM_RESET_RST);
   1041 }
   1042 
   1043 #ifdef __x86_64__
   1044 typedef void (vector)(void);
   1045 extern vector Xsyscall, Xsyscall32;
   1046 #endif
   1047 
   1048 void
   1049 cpu_init_msrs(struct cpu_info *ci, bool full)
   1050 {
   1051 #ifdef __x86_64__
   1052 	wrmsr(MSR_STAR,
   1053 	    ((uint64_t)GSEL(GCODE_SEL, SEL_KPL) << 32) |
   1054 	    ((uint64_t)LSEL(LSYSRETBASE_SEL, SEL_UPL) << 48));
   1055 	wrmsr(MSR_LSTAR, (uint64_t)Xsyscall);
   1056 	wrmsr(MSR_CSTAR, (uint64_t)Xsyscall32);
   1057 	wrmsr(MSR_SFMASK, PSL_NT|PSL_T|PSL_I|PSL_C);
   1058 
   1059 	if (full) {
   1060 		wrmsr(MSR_FSBASE, 0);
   1061 		wrmsr(MSR_GSBASE, (uint64_t)ci);
   1062 		wrmsr(MSR_KERNELGSBASE, 0);
   1063 	}
   1064 #endif	/* __x86_64__ */
   1065 
   1066 	if (cpu_feature[2] & CPUID_NOX)
   1067 		wrmsr(MSR_EFER, rdmsr(MSR_EFER) | EFER_NXE);
   1068 }
   1069 
   1070 void
   1071 cpu_offline_md(void)
   1072 {
   1073 	int s;
   1074 
   1075 	s = splhigh();
   1076 #ifdef i386
   1077 #if NNPX > 0
   1078 	npxsave_cpu(true);
   1079 #endif
   1080 #else
   1081 	fpusave_cpu(true);
   1082 #endif
   1083 	splx(s);
   1084 }
   1085 
   1086 /* XXX joerg restructure and restart CPUs individually */
   1087 static bool
   1088 cpu_suspend(device_t dv, const pmf_qual_t *qual)
   1089 {
   1090 	struct cpu_softc *sc = device_private(dv);
   1091 	struct cpu_info *ci = sc->sc_info;
   1092 	int err;
   1093 
   1094 	if ((ci->ci_flags & CPUF_PRESENT) == 0)
   1095 		return true;
   1096 
   1097 	cpufreq_suspend(ci);
   1098 
   1099 	if ((ci->ci_flags & CPUF_PRIMARY) != 0)
   1100 		return true;
   1101 
   1102 	if (ci->ci_data.cpu_idlelwp == NULL)
   1103 		return true;
   1104 
   1105 	sc->sc_wasonline = !(ci->ci_schedstate.spc_flags & SPCF_OFFLINE);
   1106 
   1107 	if (sc->sc_wasonline) {
   1108 		mutex_enter(&cpu_lock);
   1109 		err = cpu_setstate(ci, false);
   1110 		mutex_exit(&cpu_lock);
   1111 
   1112 		if (err != 0)
   1113 			return false;
   1114 	}
   1115 
   1116 	return true;
   1117 }
   1118 
   1119 static bool
   1120 cpu_resume(device_t dv, const pmf_qual_t *qual)
   1121 {
   1122 	struct cpu_softc *sc = device_private(dv);
   1123 	struct cpu_info *ci = sc->sc_info;
   1124 	int err = 0;
   1125 
   1126 	if ((ci->ci_flags & CPUF_PRESENT) == 0)
   1127 		return true;
   1128 
   1129 	if ((ci->ci_flags & CPUF_PRIMARY) != 0)
   1130 		goto out;
   1131 
   1132 	if (ci->ci_data.cpu_idlelwp == NULL)
   1133 		goto out;
   1134 
   1135 	if (sc->sc_wasonline) {
   1136 		mutex_enter(&cpu_lock);
   1137 		err = cpu_setstate(ci, true);
   1138 		mutex_exit(&cpu_lock);
   1139 	}
   1140 
   1141 out:
   1142 	if (err != 0)
   1143 		return false;
   1144 
   1145 	cpufreq_resume(ci);
   1146 
   1147 	return true;
   1148 }
   1149 
   1150 static bool
   1151 cpu_shutdown(device_t dv, int how)
   1152 {
   1153 	struct cpu_softc *sc = device_private(dv);
   1154 	struct cpu_info *ci = sc->sc_info;
   1155 
   1156 	if (ci->ci_flags & CPUF_BSP)
   1157 		return false;
   1158 
   1159 	return cpu_suspend(dv, NULL);
   1160 }
   1161 
   1162 void
   1163 cpu_get_tsc_freq(struct cpu_info *ci)
   1164 {
   1165 	uint64_t last_tsc;
   1166 
   1167 	if (cpu_hascounter()) {
   1168 		last_tsc = cpu_counter_serializing();
   1169 		i8254_delay(100000);
   1170 		ci->ci_data.cpu_cc_freq =
   1171 		    (cpu_counter_serializing() - last_tsc) * 10;
   1172 	}
   1173 }
   1174 
   1175 void
   1176 x86_cpu_idle_mwait(void)
   1177 {
   1178 	struct cpu_info *ci = curcpu();
   1179 
   1180 	KASSERT(ci->ci_ilevel == IPL_NONE);
   1181 
   1182 	x86_monitor(&ci->ci_want_resched, 0, 0);
   1183 	if (__predict_false(ci->ci_want_resched)) {
   1184 		return;
   1185 	}
   1186 	x86_mwait(0, 0);
   1187 }
   1188 
   1189 void
   1190 x86_cpu_idle_halt(void)
   1191 {
   1192 	struct cpu_info *ci = curcpu();
   1193 
   1194 	KASSERT(ci->ci_ilevel == IPL_NONE);
   1195 
   1196 	x86_disable_intr();
   1197 	if (!__predict_false(ci->ci_want_resched)) {
   1198 		x86_stihlt();
   1199 	} else {
   1200 		x86_enable_intr();
   1201 	}
   1202 }
   1203 
   1204 /*
   1205  * Loads pmap for the current CPU.
   1206  */
   1207 void
   1208 cpu_load_pmap(struct pmap *pmap)
   1209 {
   1210 #ifdef PAE
   1211 	int i, s;
   1212 	struct cpu_info *ci;
   1213 
   1214 	s = splvm(); /* just to be safe */
   1215 	ci = curcpu();
   1216 	pd_entry_t *l3_pd = ci->ci_pae_l3_pdir;
   1217 	for (i = 0 ; i < PDP_SIZE; i++) {
   1218 		l3_pd[i] = pmap->pm_pdirpa[i] | PG_V;
   1219 	}
   1220 	splx(s);
   1221 	tlbflush();
   1222 #else /* PAE */
   1223 	lcr3(pmap_pdirpa(pmap, 0));
   1224 #endif /* PAE */
   1225 }
   1226 
   1227 /*
   1228  * Notify all other cpus to halt.
   1229  */
   1230 
   1231 void
   1232 cpu_broadcast_halt(void)
   1233 {
   1234 	x86_broadcast_ipi(X86_IPI_HALT);
   1235 }
   1236 
   1237 /*
   1238  * Send a dummy ipi to a cpu to force it to run splraise()/spllower()
   1239  */
   1240 
   1241 void
   1242 cpu_kick(struct cpu_info *ci)
   1243 {
   1244 	x86_send_ipi(ci, 0);
   1245 }
   1246