Home | History | Annotate | Line # | Download | only in x86
cpu.c revision 1.101
      1 /*	$NetBSD: cpu.c,v 1.101 2014/12/08 15:22:47 msaitoh Exp $	*/
      2 /* NetBSD: cpu.c,v 1.18 2004/02/20 17:35:01 yamt Exp  */
      3 
      4 /*-
      5  * Copyright (c) 2000 The NetBSD Foundation, Inc.
      6  * Copyright (c) 2002, 2006, 2007 YAMAMOTO Takashi,
      7  * All rights reserved.
      8  *
      9  * This code is derived from software contributed to The NetBSD Foundation
     10  * by RedBack Networks Inc.
     11  *
     12  * Author: Bill Sommerfeld
     13  *
     14  * Redistribution and use in source and binary forms, with or without
     15  * modification, are permitted provided that the following conditions
     16  * are met:
     17  * 1. Redistributions of source code must retain the above copyright
     18  *    notice, this list of conditions and the following disclaimer.
     19  * 2. Redistributions in binary form must reproduce the above copyright
     20  *    notice, this list of conditions and the following disclaimer in the
     21  *    documentation and/or other materials provided with the distribution.
     22  *
     23  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     24  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     25  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     26  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     27  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     28  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     29  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     30  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     31  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     32  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     33  * POSSIBILITY OF SUCH DAMAGE.
     34  */
     35 
     36 /*
     37  * Copyright (c) 1999 Stefan Grefen
     38  *
     39  * Redistribution and use in source and binary forms, with or without
     40  * modification, are permitted provided that the following conditions
     41  * are met:
     42  * 1. Redistributions of source code must retain the above copyright
     43  *    notice, this list of conditions and the following disclaimer.
     44  * 2. Redistributions in binary form must reproduce the above copyright
     45  *    notice, this list of conditions and the following disclaimer in the
     46  *    documentation and/or other materials provided with the distribution.
     47  * 3. All advertising materials mentioning features or use of this software
     48  *    must display the following acknowledgement:
     49  *      This product includes software developed by the NetBSD
     50  *      Foundation, Inc. and its contributors.
     51  * 4. Neither the name of The NetBSD Foundation nor the names of its
     52  *    contributors may be used to endorse or promote products derived
     53  *    from this software without specific prior written permission.
     54  *
     55  * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND ANY
     56  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
     57  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
     58  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR AND CONTRIBUTORS BE LIABLE
     59  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
     60  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
     61  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
     62  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
     63  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
     64  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
     65  * SUCH DAMAGE.
     66  */
     67 
     68 #include <sys/cdefs.h>
     69 __KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.101 2014/12/08 15:22:47 msaitoh Exp $");
     70 
     71 #include "opt_ddb.h"
     72 #include "opt_multiprocessor.h"
     73 #include "opt_mpbios.h"		/* for MPDEBUG */
     74 #include "opt_mtrr.h"
     75 #include "opt_xen.h"
     76 
     77 #include "lapic.h"
     78 #include "ioapic.h"
     79 
     80 #include <sys/param.h>
     81 #include <sys/proc.h>
     82 #include <sys/systm.h>
     83 #include <sys/device.h>
     84 #include <sys/kmem.h>
     85 #include <sys/cpu.h>
     86 #include <sys/cpufreq.h>
     87 #include <sys/atomic.h>
     88 #include <sys/reboot.h>
     89 #include <sys/idle.h>
     90 
     91 #include <uvm/uvm.h>
     92 
     93 #include <machine/cpufunc.h>
     94 #include <machine/cpuvar.h>
     95 #include <machine/pmap.h>
     96 #include <machine/vmparam.h>
     97 #include <machine/mpbiosvar.h>
     98 #include <machine/pcb.h>
     99 #include <machine/specialreg.h>
    100 #include <machine/segments.h>
    101 #include <machine/gdt.h>
    102 #include <machine/mtrr.h>
    103 #include <machine/pio.h>
    104 
    105 #include <x86/fpu.h>
    106 
    107 #include <xen/xen.h>
    108 #include <xen/xen-public/vcpu.h>
    109 #include <xen/vcpuvar.h>
    110 
    111 #if NLAPIC > 0
    112 #include <machine/apicvar.h>
    113 #include <machine/i82489reg.h>
    114 #include <machine/i82489var.h>
    115 #endif
    116 
    117 #include <dev/ic/mc146818reg.h>
    118 #include <dev/isa/isareg.h>
    119 
    120 static int	cpu_match(device_t, cfdata_t, void *);
    121 static void	cpu_attach(device_t, device_t, void *);
    122 static void	cpu_defer(device_t);
    123 static int	cpu_rescan(device_t, const char *, const int *);
    124 static void	cpu_childdetached(device_t, device_t);
    125 static int	vcpu_match(device_t, cfdata_t, void *);
    126 static void	vcpu_attach(device_t, device_t, void *);
    127 static void	cpu_attach_common(device_t, device_t, void *);
    128 void		cpu_offline_md(void);
    129 
    130 struct cpu_softc {
    131 	device_t sc_dev;		/* device tree glue */
    132 	struct cpu_info *sc_info;	/* pointer to CPU info */
    133 	bool sc_wasonline;
    134 };
    135 
    136 int mp_cpu_start(struct cpu_info *, vaddr_t);
    137 void mp_cpu_start_cleanup(struct cpu_info *);
    138 const struct cpu_functions mp_cpu_funcs = { mp_cpu_start, NULL,
    139 				      mp_cpu_start_cleanup };
    140 
    141 CFATTACH_DECL2_NEW(cpu, sizeof(struct cpu_softc),
    142     cpu_match, cpu_attach, NULL, NULL, cpu_rescan, cpu_childdetached);
    143 
    144 CFATTACH_DECL_NEW(vcpu, sizeof(struct cpu_softc),
    145     vcpu_match, vcpu_attach, NULL, NULL);
    146 
    147 /*
    148  * Statically-allocated CPU info for the primary CPU (or the only
    149  * CPU, on uniprocessors).  The CPU info list is initialized to
    150  * point at it.
    151  */
    152 #ifdef TRAPLOG
    153 #include <machine/tlog.h>
    154 struct tlog tlog_primary;
    155 #endif
    156 struct cpu_info cpu_info_primary __aligned(CACHE_LINE_SIZE) = {
    157 	.ci_dev = 0,
    158 	.ci_self = &cpu_info_primary,
    159 	.ci_idepth = -1,
    160 	.ci_curlwp = &lwp0,
    161 	.ci_curldt = -1,
    162 #ifdef TRAPLOG
    163 	.ci_tlog = &tlog_primary,
    164 #endif
    165 
    166 };
    167 struct cpu_info phycpu_info_primary __aligned(CACHE_LINE_SIZE) = {
    168 	.ci_dev = 0,
    169 	.ci_self = &phycpu_info_primary,
    170 };
    171 
    172 struct cpu_info *cpu_info_list = &cpu_info_primary;
    173 struct cpu_info *phycpu_info_list = &phycpu_info_primary;
    174 
    175 uint32_t cpu_feature[5]; /* X86 CPUID feature bits
    176 			  *	[0] basic features %edx
    177 			  *	[1] basic features %ecx
    178 			  *	[2] extended features %edx
    179 			  *	[3] extended features %ecx
    180 			  *	[4] VIA padlock features
    181 			  */
    182 
    183 bool x86_mp_online;
    184 paddr_t mp_trampoline_paddr = MP_TRAMPOLINE;
    185 
    186 #if defined(MULTIPROCESSOR)
    187 void    	cpu_hatch(void *);
    188 static void    	cpu_boot_secondary(struct cpu_info *ci);
    189 static void    	cpu_start_secondary(struct cpu_info *ci);
    190 #endif	/* MULTIPROCESSOR */
    191 
    192 static int
    193 cpu_match(device_t parent, cfdata_t match, void *aux)
    194 {
    195 
    196 	return 1;
    197 }
    198 
    199 static void
    200 cpu_attach(device_t parent, device_t self, void *aux)
    201 {
    202 	struct cpu_softc *sc = device_private(self);
    203 	struct cpu_attach_args *caa = aux;
    204 	struct cpu_info *ci;
    205 	uintptr_t ptr;
    206 	static int nphycpu = 0;
    207 
    208 	sc->sc_dev = self;
    209 
    210 	/*
    211 	 * If we're an Application Processor, allocate a cpu_info
    212 	 * If we're the first attached CPU use the primary cpu_info,
    213 	 * otherwise allocate a new one
    214 	 */
    215 	aprint_naive("\n");
    216 	aprint_normal("\n");
    217 	if (nphycpu > 0) {
    218 		struct cpu_info *tmp;
    219 		ptr = (uintptr_t)kmem_zalloc(sizeof(*ci) + CACHE_LINE_SIZE - 1,
    220 		    KM_SLEEP);
    221 		ci = (struct cpu_info *)roundup2(ptr, CACHE_LINE_SIZE);
    222 		ci->ci_curldt = -1;
    223 
    224 		tmp = phycpu_info_list;
    225 		while (tmp->ci_next)
    226 			tmp = tmp->ci_next;
    227 
    228 		tmp->ci_next = ci;
    229 	} else {
    230 		ci = &phycpu_info_primary;
    231 	}
    232 
    233 	ci->ci_self = ci;
    234 	sc->sc_info = ci;
    235 
    236 	ci->ci_dev = self;
    237 	ci->ci_acpiid = caa->cpu_id;
    238 	ci->ci_cpuid = caa->cpu_number;
    239 	ci->ci_vcpu = NULL;
    240 	ci->ci_index = nphycpu++;
    241 
    242 	if (!pmf_device_register(self, NULL, NULL))
    243 		aprint_error_dev(self, "couldn't establish power handler\n");
    244 
    245 	(void)config_defer(self, cpu_defer);
    246 }
    247 
    248 static void
    249 cpu_defer(device_t self)
    250 {
    251 	cpu_rescan(self, NULL, NULL);
    252 }
    253 
    254 static int
    255 cpu_rescan(device_t self, const char *ifattr, const int *locators)
    256 {
    257 	struct cpu_softc *sc = device_private(self);
    258 	struct cpufeature_attach_args cfaa;
    259 	struct cpu_info *ci = sc->sc_info;
    260 
    261 	memset(&cfaa, 0, sizeof(cfaa));
    262 	cfaa.ci = ci;
    263 
    264 	if (ifattr_match(ifattr, "cpufeaturebus")) {
    265 
    266 		if (ci->ci_frequency == NULL) {
    267 			cfaa.name = "frequency";
    268 			ci->ci_frequency = config_found_ia(self,
    269 			    "cpufeaturebus", &cfaa, NULL);
    270 		}
    271 	}
    272 
    273 	return 0;
    274 }
    275 
    276 static void
    277 cpu_childdetached(device_t self, device_t child)
    278 {
    279 	struct cpu_softc *sc = device_private(self);
    280 	struct cpu_info *ci = sc->sc_info;
    281 
    282 	if (ci->ci_frequency == child)
    283 		ci->ci_frequency = NULL;
    284 }
    285 
    286 static int
    287 vcpu_match(device_t parent, cfdata_t match, void *aux)
    288 {
    289 	struct vcpu_attach_args *vcaa = aux;
    290 	struct vcpu_runstate_info vcr;
    291 	int error;
    292 
    293 	if (strcmp(vcaa->vcaa_name, match->cf_name) == 0) {
    294 		error = HYPERVISOR_vcpu_op(VCPUOP_get_runstate_info,
    295 					   vcaa->vcaa_caa.cpu_number,
    296 					   &vcr);
    297 		switch (error) {
    298 		case 0:
    299 			return 1;
    300 		case -ENOENT:
    301 			return 0;
    302 		default:
    303 			panic("Unknown hypervisor error %d returned on vcpu runstate probe\n", error);
    304 		}
    305 	}
    306 
    307 	return 0;
    308 }
    309 
    310 static void
    311 vcpu_attach(device_t parent, device_t self, void *aux)
    312 {
    313 	struct vcpu_attach_args *vcaa = aux;
    314 
    315 	KASSERT(vcaa->vcaa_caa.cpu_func == NULL);
    316 	vcaa->vcaa_caa.cpu_func = &mp_cpu_funcs;
    317 	cpu_attach_common(parent, self, &vcaa->vcaa_caa);
    318 
    319 	if (!pmf_device_register(self, NULL, NULL))
    320 		aprint_error_dev(self, "couldn't establish power handler\n");
    321 }
    322 
    323 static int
    324 vcpu_is_up(struct cpu_info *ci)
    325 {
    326 	KASSERT(ci != NULL);
    327 	return HYPERVISOR_vcpu_op(VCPUOP_is_up, ci->ci_cpuid, NULL);
    328 }
    329 
    330 static void
    331 cpu_vm_init(struct cpu_info *ci)
    332 {
    333 	int ncolors = 2, i;
    334 
    335 	for (i = CAI_ICACHE; i <= CAI_L2CACHE; i++) {
    336 		struct x86_cache_info *cai;
    337 		int tcolors;
    338 
    339 		cai = &ci->ci_cinfo[i];
    340 
    341 		tcolors = atop(cai->cai_totalsize);
    342 		switch(cai->cai_associativity) {
    343 		case 0xff:
    344 			tcolors = 1; /* fully associative */
    345 			break;
    346 		case 0:
    347 		case 1:
    348 			break;
    349 		default:
    350 			tcolors /= cai->cai_associativity;
    351 		}
    352 		ncolors = max(ncolors, tcolors);
    353 	}
    354 
    355 	/*
    356 	 * Knowing the size of the largest cache on this CPU, potentially
    357 	 * re-color our pages.
    358 	 */
    359 	aprint_debug_dev(ci->ci_dev, "%d page colors\n", ncolors);
    360 	uvm_page_recolor(ncolors);
    361 	pmap_tlb_cpu_init(ci);
    362 }
    363 
    364 static void
    365 cpu_attach_common(device_t parent, device_t self, void *aux)
    366 {
    367 	struct cpu_softc *sc = device_private(self);
    368 	struct cpu_attach_args *caa = aux;
    369 	struct cpu_info *ci;
    370 	uintptr_t ptr;
    371 	int cpunum = caa->cpu_number;
    372 	static bool again = false;
    373 
    374 	sc->sc_dev = self;
    375 
    376 	/*
    377 	 * If we're an Application Processor, allocate a cpu_info
    378 	 * structure, otherwise use the primary's.
    379 	 */
    380 	if (caa->cpu_role == CPU_ROLE_AP) {
    381 		aprint_naive(": Application Processor\n");
    382 		ptr = (uintptr_t)kmem_alloc(sizeof(*ci) + CACHE_LINE_SIZE - 1,
    383 		    KM_SLEEP);
    384 		ci = (struct cpu_info *)roundup2(ptr, CACHE_LINE_SIZE);
    385 		memset(ci, 0, sizeof(*ci));
    386 #ifdef TRAPLOG
    387 		ci->ci_tlog_base = kmem_zalloc(sizeof(struct tlog), KM_SLEEP);
    388 #endif
    389 	} else {
    390 		aprint_naive(": %s Processor\n",
    391 		    caa->cpu_role == CPU_ROLE_SP ? "Single" : "Boot");
    392 		ci = &cpu_info_primary;
    393 	}
    394 
    395 	ci->ci_self = ci;
    396 	sc->sc_info = ci;
    397 	ci->ci_dev = self;
    398 	ci->ci_cpuid = cpunum;
    399 
    400 	KASSERT(HYPERVISOR_shared_info != NULL);
    401 	KASSERT(cpunum < XEN_LEGACY_MAX_VCPUS);
    402 	ci->ci_vcpu = &HYPERVISOR_shared_info->vcpu_info[cpunum];
    403 
    404 	KASSERT(ci->ci_func == 0);
    405 	ci->ci_func = caa->cpu_func;
    406 	aprint_normal("\n");
    407 
    408 	/* Must be called before mi_cpu_attach(). */
    409 	cpu_vm_init(ci);
    410 
    411 	if (caa->cpu_role == CPU_ROLE_AP) {
    412 		int error;
    413 
    414 		error = mi_cpu_attach(ci);
    415 
    416 		KASSERT(ci->ci_data.cpu_idlelwp != NULL);
    417 		if (error != 0) {
    418 			aprint_error_dev(self,
    419 			    "mi_cpu_attach failed with %d\n", error);
    420 			return;
    421 		}
    422 
    423 	} else {
    424 		KASSERT(ci->ci_data.cpu_idlelwp != NULL);
    425 	}
    426 
    427 	KASSERT(ci->ci_cpuid == ci->ci_index);
    428 #ifdef __x86_64__
    429 	/* No user PGD mapped for this CPU yet */
    430 	ci->ci_xen_current_user_pgd = 0;
    431 #endif
    432 #if defined(__x86_64__) || defined(PAE)
    433 	mutex_init(&ci->ci_kpm_mtx, MUTEX_DEFAULT, IPL_VM);
    434 #endif
    435 	pmap_reference(pmap_kernel());
    436 	ci->ci_pmap = pmap_kernel();
    437 	ci->ci_tlbstate = TLBSTATE_STALE;
    438 
    439 	/*
    440 	 * Boot processor may not be attached first, but the below
    441 	 * must be done to allow booting other processors.
    442 	 */
    443 	if (!again) {
    444 		atomic_or_32(&ci->ci_flags, CPUF_PRESENT | CPUF_PRIMARY);
    445 		/* Basic init. */
    446 		cpu_intr_init(ci);
    447 		cpu_get_tsc_freq(ci);
    448 		cpu_init(ci);
    449 		pmap_cpu_init_late(ci);
    450 
    451 		/* Every processor needs to init its own ipi h/w (similar to lapic) */
    452 		xen_ipi_init();
    453 
    454 		/* Make sure DELAY() is initialized. */
    455 		DELAY(1);
    456 		again = true;
    457 	}
    458 
    459 	/* further PCB init done later. */
    460 
    461 	switch (caa->cpu_role) {
    462 	case CPU_ROLE_SP:
    463 		atomic_or_32(&ci->ci_flags, CPUF_SP);
    464 		cpu_identify(ci);
    465 		x86_cpu_idle_init();
    466 
    467 		break;
    468 
    469 	case CPU_ROLE_BP:
    470 		atomic_or_32(&ci->ci_flags, CPUF_BSP);
    471 		cpu_identify(ci);
    472 		x86_cpu_idle_init();
    473 
    474 		break;
    475 
    476 	case CPU_ROLE_AP:
    477 		atomic_or_32(&ci->ci_flags, CPUF_AP);
    478 
    479 		/*
    480 		 * report on an AP
    481 		 */
    482 
    483 #if defined(MULTIPROCESSOR)
    484 		/* interrupt handler stack */
    485 		cpu_intr_init(ci);
    486 
    487 		/* Setup per-cpu memory for gdt */
    488 		gdt_alloc_cpu(ci);
    489 
    490 		pmap_cpu_init_late(ci);
    491 		cpu_start_secondary(ci);
    492 
    493 		if (ci->ci_flags & CPUF_PRESENT) {
    494 			struct cpu_info *tmp;
    495 
    496 			cpu_identify(ci);
    497 			tmp = cpu_info_list;
    498 			while (tmp->ci_next)
    499 				tmp = tmp->ci_next;
    500 
    501 			tmp->ci_next = ci;
    502 		}
    503 #else
    504 		aprint_error_dev(ci->ci_dev, "not started\n");
    505 #endif
    506 		break;
    507 
    508 	default:
    509 		panic("unknown processor type??\n");
    510 	}
    511 
    512 #ifdef MPVERBOSE
    513 	if (mp_verbose) {
    514 		struct lwp *l = ci->ci_data.cpu_idlelwp;
    515 		struct pcb *pcb = lwp_getpcb(l);
    516 
    517 		aprint_verbose_dev(self,
    518 		    "idle lwp at %p, idle sp at 0x%p\n",
    519 		    l,
    520 #ifdef i386
    521 		    (void *)pcb->pcb_esp
    522 #else /* i386 */
    523 		    (void *)pcb->pcb_rsp
    524 #endif /* i386 */
    525 		);
    526 
    527 	}
    528 #endif /* MPVERBOSE */
    529 }
    530 
    531 /*
    532  * Initialize the processor appropriately.
    533  */
    534 
    535 void
    536 cpu_init(struct cpu_info *ci)
    537 {
    538 
    539 	/*
    540 	 * If we have FXSAVE/FXRESTOR, use them.
    541 	 */
    542 	if (cpu_feature[0] & CPUID_FXSR) {
    543 		lcr4(rcr4() | CR4_OSFXSR);
    544 
    545 		/*
    546 		 * If we have SSE/SSE2, enable XMM exceptions.
    547 		 */
    548 		if (cpu_feature[0] & (CPUID_SSE|CPUID_SSE2))
    549 			lcr4(rcr4() | CR4_OSXMMEXCPT);
    550 	}
    551 
    552 	atomic_or_32(&ci->ci_flags, CPUF_RUNNING);
    553 }
    554 
    555 
    556 #ifdef MULTIPROCESSOR
    557 
    558 void
    559 cpu_boot_secondary_processors(void)
    560 {
    561 	struct cpu_info *ci;
    562 	u_long i;
    563 	for (i = 0; i < maxcpus; i++) {
    564 		ci = cpu_lookup(i);
    565 		if (ci == NULL)
    566 			continue;
    567 		if (ci->ci_data.cpu_idlelwp == NULL)
    568 			continue;
    569 		if ((ci->ci_flags & CPUF_PRESENT) == 0)
    570 			continue;
    571 		if (ci->ci_flags & (CPUF_BSP|CPUF_SP|CPUF_PRIMARY))
    572 			continue;
    573 		cpu_boot_secondary(ci);
    574 	}
    575 
    576 	x86_mp_online = true;
    577 }
    578 
    579 static void
    580 cpu_init_idle_lwp(struct cpu_info *ci)
    581 {
    582 	struct lwp *l = ci->ci_data.cpu_idlelwp;
    583 	struct pcb *pcb = lwp_getpcb(l);
    584 
    585 	pcb->pcb_cr0 = rcr0();
    586 }
    587 
    588 void
    589 cpu_init_idle_lwps(void)
    590 {
    591 	struct cpu_info *ci;
    592 	u_long i;
    593 
    594 	for (i = 0; i < maxcpus; i++) {
    595 		ci = cpu_lookup(i);
    596 		if (ci == NULL)
    597 			continue;
    598 		if (ci->ci_data.cpu_idlelwp == NULL)
    599 			continue;
    600 		if ((ci->ci_flags & CPUF_PRESENT) == 0)
    601 			continue;
    602 		cpu_init_idle_lwp(ci);
    603 	}
    604 }
    605 
    606 static void
    607 cpu_start_secondary(struct cpu_info *ci)
    608 {
    609 	int i;
    610 
    611 	aprint_debug_dev(ci->ci_dev, "starting\n");
    612 
    613 	ci->ci_curlwp = ci->ci_data.cpu_idlelwp;
    614 
    615 	if (CPU_STARTUP(ci, (vaddr_t) cpu_hatch) != 0) {
    616 		return;
    617 	}
    618 
    619 	/*
    620 	 * wait for it to become ready
    621 	 */
    622 	for (i = 100000; (!(ci->ci_flags & CPUF_PRESENT)) && i > 0; i--) {
    623 		delay(10);
    624 	}
    625 	if ((ci->ci_flags & CPUF_PRESENT) == 0) {
    626 		aprint_error_dev(ci->ci_dev, "failed to become ready\n");
    627 #if defined(MPDEBUG) && defined(DDB)
    628 		printf("dropping into debugger; continue from here to resume boot\n");
    629 		Debugger();
    630 #endif
    631 	}
    632 
    633 	CPU_START_CLEANUP(ci);
    634 }
    635 
    636 void
    637 cpu_boot_secondary(struct cpu_info *ci)
    638 {
    639 	int i;
    640 	atomic_or_32(&ci->ci_flags, CPUF_GO);
    641 	for (i = 100000; (!(ci->ci_flags & CPUF_RUNNING)) && i > 0; i--) {
    642 		delay(10);
    643 	}
    644 	if ((ci->ci_flags & CPUF_RUNNING) == 0) {
    645 		aprint_error_dev(ci->ci_dev, "CPU failed to start\n");
    646 #if defined(MPDEBUG) && defined(DDB)
    647 		printf("dropping into debugger; continue from here to resume boot\n");
    648 		Debugger();
    649 #endif
    650 	}
    651 }
    652 
    653 /*
    654  * APs end up here immediately after initialisation and VCPUOP_up in
    655  * mp_cpu_start().
    656  * At this point, we are running in the idle pcb/idle stack of the new
    657  * CPU.  This function jumps to the idle loop and starts looking for
    658  * work.
    659  */
    660 extern void x86_64_tls_switch(struct lwp *);
    661 void
    662 cpu_hatch(void *v)
    663 {
    664 	struct cpu_info *ci = (struct cpu_info *)v;
    665 	struct pcb *pcb;
    666 	int s, i;
    667 
    668 	/* Setup TLS and kernel GS/FS */
    669 	cpu_init_msrs(ci, true);
    670 	cpu_init_idt();
    671 	gdt_init_cpu(ci);
    672 
    673 	cpu_probe(ci);
    674 
    675 	atomic_or_32(&ci->ci_flags, CPUF_PRESENT);
    676 
    677 	while ((ci->ci_flags & CPUF_GO) == 0) {
    678 		/* Don't use delay, boot CPU may be patching the text. */
    679 		for (i = 10000; i != 0; i--)
    680 			x86_pause();
    681 	}
    682 
    683 	/* Because the text may have been patched in x86_patch(). */
    684 	x86_flush();
    685 	tlbflushg();
    686 
    687 	KASSERT((ci->ci_flags & CPUF_RUNNING) == 0);
    688 
    689 	pcb = lwp_getpcb(curlwp);
    690 	pcb->pcb_cr3 = pmap_pdirpa(pmap_kernel(), 0);
    691 	pcb = lwp_getpcb(ci->ci_data.cpu_idlelwp);
    692 
    693 	xen_ipi_init();
    694 
    695 	xen_initclocks();
    696 
    697 #ifdef __x86_64__
    698 	fpuinit(ci);
    699 #endif
    700 
    701 	lldt(GSEL(GLDT_SEL, SEL_KPL));
    702 
    703 	cpu_init(ci);
    704 	cpu_get_tsc_freq(ci);
    705 
    706 	s = splhigh();
    707 	x86_enable_intr();
    708 	splx(s);
    709 
    710 	aprint_debug_dev(ci->ci_dev, "running\n");
    711 
    712 	cpu_switchto(NULL, ci->ci_data.cpu_idlelwp, true);
    713 
    714 	idle_loop(NULL);
    715 	KASSERT(false);
    716 }
    717 
    718 #if defined(DDB)
    719 
    720 #include <ddb/db_output.h>
    721 #include <machine/db_machdep.h>
    722 
    723 /*
    724  * Dump CPU information from ddb.
    725  */
    726 void
    727 cpu_debug_dump(void)
    728 {
    729 	struct cpu_info *ci;
    730 	CPU_INFO_ITERATOR cii;
    731 
    732 	db_printf("addr		dev	id	flags	ipis	curlwp 		fpcurlwp\n");
    733 	for (CPU_INFO_FOREACH(cii, ci)) {
    734 		db_printf("%p	%s	%ld	%x	%x	%10p	%10p\n",
    735 		    ci,
    736 		    ci->ci_dev == NULL ? "BOOT" : device_xname(ci->ci_dev),
    737 		    (long)ci->ci_cpuid,
    738 		    ci->ci_flags, ci->ci_ipis,
    739 		    ci->ci_curlwp,
    740 		    ci->ci_fpcurlwp);
    741 	}
    742 }
    743 #endif /* DDB */
    744 
    745 #endif /* MULTIPROCESSOR */
    746 
    747 extern void hypervisor_callback(void);
    748 extern void failsafe_callback(void);
    749 #ifdef __x86_64__
    750 typedef void (vector)(void);
    751 extern vector Xsyscall, Xsyscall32;
    752 #endif
    753 
    754 /*
    755  * Setup the "trampoline". On Xen, we setup nearly all cpu context
    756  * outside a trampoline, so we prototype and call targetip like so:
    757  * void targetip(struct cpu_info *);
    758  */
    759 
    760 static void
    761 gdt_prepframes(paddr_t *frames, vaddr_t base, uint32_t entries)
    762 {
    763 	int i;
    764 	for (i = 0; i < roundup(entries, PAGE_SIZE) >> PAGE_SHIFT; i++) {
    765 
    766 		frames[i] = ((paddr_t) xpmap_ptetomach(
    767 				(pt_entry_t *) (base + (i << PAGE_SHIFT))))
    768 			>> PAGE_SHIFT;
    769 
    770 		/* Mark Read-only */
    771 		pmap_pte_clearbits(kvtopte(base + (i << PAGE_SHIFT)),
    772 		    PG_RW);
    773 	}
    774 }
    775 
    776 #ifdef __x86_64__
    777 extern char *ldtstore;
    778 
    779 static void
    780 xen_init_amd64_vcpuctxt(struct cpu_info *ci,
    781 			struct vcpu_guest_context *initctx,
    782 			void targetrip(struct cpu_info *))
    783 {
    784 	/* page frames to point at GDT */
    785 	extern int gdt_size;
    786 	paddr_t frames[16];
    787 	psize_t gdt_ents;
    788 
    789 	struct lwp *l;
    790 	struct pcb *pcb;
    791 
    792 	volatile struct vcpu_info *vci;
    793 
    794 	KASSERT(ci != NULL);
    795 	KASSERT(ci != &cpu_info_primary);
    796 	KASSERT(initctx != NULL);
    797 	KASSERT(targetrip != NULL);
    798 
    799 	memset(initctx, 0, sizeof *initctx);
    800 
    801 	gdt_ents = roundup(gdt_size, PAGE_SIZE) >> PAGE_SHIFT;
    802 	KASSERT(gdt_ents <= 16);
    803 
    804 	gdt_prepframes(frames, (vaddr_t) ci->ci_gdt, gdt_ents);
    805 
    806 	/* Initialise the vcpu context: We use idle_loop()'s pcb context. */
    807 
    808 	l = ci->ci_data.cpu_idlelwp;
    809 
    810 	KASSERT(l != NULL);
    811 	pcb = lwp_getpcb(l);
    812 	KASSERT(pcb != NULL);
    813 
    814 	/* resume with interrupts off */
    815 	vci = ci->ci_vcpu;
    816 	vci->evtchn_upcall_mask = 1;
    817 	xen_mb();
    818 
    819 	/* resume in kernel-mode */
    820 	initctx->flags = VGCF_in_kernel | VGCF_online;
    821 
    822 	/* Stack and entry points:
    823 	 * We arrange for the stack frame for cpu_hatch() to
    824 	 * appear as a callee frame of lwp_trampoline(). Being a
    825 	 * leaf frame prevents trampling on any of the MD stack setup
    826 	 * that x86/vm_machdep.c:cpu_lwp_fork() does for idle_loop()
    827 	 */
    828 
    829 	initctx->user_regs.rdi = (uint64_t) ci; /* targetrip(ci); */
    830 	initctx->user_regs.rip = (vaddr_t) targetrip;
    831 
    832 	initctx->user_regs.cs = GSEL(GCODE_SEL, SEL_KPL);
    833 
    834 	initctx->user_regs.rflags = pcb->pcb_flags;
    835 	initctx->user_regs.rsp = pcb->pcb_rsp;
    836 
    837 	/* Data segments */
    838 	initctx->user_regs.ss = GSEL(GDATA_SEL, SEL_KPL);
    839 	initctx->user_regs.es = GSEL(GDATA_SEL, SEL_KPL);
    840 	initctx->user_regs.ds = GSEL(GDATA_SEL, SEL_KPL);
    841 
    842 	/* GDT */
    843 	memcpy(initctx->gdt_frames, frames, sizeof frames);
    844 	initctx->gdt_ents = gdt_ents;
    845 
    846 	/* LDT */
    847 	initctx->ldt_base = (unsigned long) ldtstore;
    848 	initctx->ldt_ents = LDT_SIZE >> 3;
    849 
    850 	/* Kernel context state */
    851 	initctx->kernel_ss = GSEL(GDATA_SEL, SEL_KPL);
    852 	initctx->kernel_sp = pcb->pcb_rsp0;
    853 	initctx->ctrlreg[0] = pcb->pcb_cr0;
    854 	initctx->ctrlreg[1] = 0; /* "resuming" from kernel - no User cr3. */
    855 	initctx->ctrlreg[2] = (vaddr_t) targetrip;
    856 	/*
    857 	 * Use pmap_kernel() L4 PD directly, until we setup the
    858 	 * per-cpu L4 PD in pmap_cpu_init_late()
    859 	 */
    860 	initctx->ctrlreg[3] = xen_pfn_to_cr3(x86_btop(xpmap_ptom(ci->ci_kpm_pdirpa)));
    861 	initctx->ctrlreg[4] = CR4_PAE | CR4_OSFXSR | CR4_OSXMMEXCPT;
    862 
    863 
    864 	/* Xen callbacks */
    865 	initctx->event_callback_eip = (unsigned long) hypervisor_callback;
    866 	initctx->failsafe_callback_eip = (unsigned long) failsafe_callback;
    867 	initctx->syscall_callback_eip = (unsigned long) Xsyscall;
    868 
    869 	return;
    870 }
    871 #else /* i386 */
    872 extern union descriptor *ldt;
    873 extern void Xsyscall(void);
    874 
    875 static void
    876 xen_init_i386_vcpuctxt(struct cpu_info *ci,
    877 			struct vcpu_guest_context *initctx,
    878 			void targeteip(struct cpu_info *))
    879 {
    880 	/* page frames to point at GDT */
    881 	extern int gdt_size;
    882 	paddr_t frames[16];
    883 	psize_t gdt_ents;
    884 
    885 	struct lwp *l;
    886 	struct pcb *pcb;
    887 
    888 	volatile struct vcpu_info *vci;
    889 
    890 	KASSERT(ci != NULL);
    891 	KASSERT(ci != &cpu_info_primary);
    892 	KASSERT(initctx != NULL);
    893 	KASSERT(targeteip != NULL);
    894 
    895 	memset(initctx, 0, sizeof *initctx);
    896 
    897 	gdt_ents = roundup(gdt_size, PAGE_SIZE) >> PAGE_SHIFT;
    898 	KASSERT(gdt_ents <= 16);
    899 
    900 	gdt_prepframes(frames, (vaddr_t) ci->ci_gdt, gdt_ents);
    901 
    902 	/*
    903 	 * Initialise the vcpu context:
    904 	 * We use this cpu's idle_loop() pcb context.
    905 	 */
    906 
    907 	l = ci->ci_data.cpu_idlelwp;
    908 
    909 	KASSERT(l != NULL);
    910 	pcb = lwp_getpcb(l);
    911 	KASSERT(pcb != NULL);
    912 
    913 	/* resume with interrupts off */
    914 	vci = ci->ci_vcpu;
    915 	vci->evtchn_upcall_mask = 1;
    916 	xen_mb();
    917 
    918 	/* resume in kernel-mode */
    919 	initctx->flags = VGCF_in_kernel | VGCF_online;
    920 
    921 	/* Stack frame setup for cpu_hatch():
    922 	 * We arrange for the stack frame for cpu_hatch() to
    923 	 * appear as a callee frame of lwp_trampoline(). Being a
    924 	 * leaf frame prevents trampling on any of the MD stack setup
    925 	 * that x86/vm_machdep.c:cpu_lwp_fork() does for idle_loop()
    926 	 */
    927 
    928 	initctx->user_regs.esp = pcb->pcb_esp - 4; /* Leave word for
    929 						      arg1 */
    930 	{ /* targeteip(ci); */
    931 		uint32_t *arg = (uint32_t *) initctx->user_regs.esp;
    932 		arg[1] = (uint32_t) ci; /* arg1 */
    933 
    934 	}
    935 
    936 	initctx->user_regs.eip = (vaddr_t) targeteip;
    937 	initctx->user_regs.cs = GSEL(GCODE_SEL, SEL_KPL);
    938 	initctx->user_regs.eflags |= pcb->pcb_iopl;
    939 
    940 	/* Data segments */
    941 	initctx->user_regs.ss = GSEL(GDATA_SEL, SEL_KPL);
    942 	initctx->user_regs.es = GSEL(GDATA_SEL, SEL_KPL);
    943 	initctx->user_regs.ds = GSEL(GDATA_SEL, SEL_KPL);
    944 	initctx->user_regs.fs = GSEL(GDATA_SEL, SEL_KPL);
    945 
    946 	/* GDT */
    947 	memcpy(initctx->gdt_frames, frames, sizeof frames);
    948 	initctx->gdt_ents = gdt_ents;
    949 
    950 	/* LDT */
    951 	initctx->ldt_base = (unsigned long) ldt;
    952 	initctx->ldt_ents = NLDT;
    953 
    954 	/* Kernel context state */
    955 	initctx->kernel_ss = GSEL(GDATA_SEL, SEL_KPL);
    956 	initctx->kernel_sp = pcb->pcb_esp0;
    957 	initctx->ctrlreg[0] = pcb->pcb_cr0;
    958 	initctx->ctrlreg[1] = 0; /* "resuming" from kernel - no User cr3. */
    959 	initctx->ctrlreg[2] = (vaddr_t) targeteip;
    960 #ifdef PAE
    961 	initctx->ctrlreg[3] = xen_pfn_to_cr3(x86_btop(xpmap_ptom(ci->ci_pae_l3_pdirpa)));
    962 #else /* PAE */
    963 	initctx->ctrlreg[3] = xen_pfn_to_cr3(x86_btop(xpmap_ptom(pcb->pcb_cr3)));
    964 #endif /* PAE */
    965 	initctx->ctrlreg[4] = /* CR4_PAE |  */CR4_OSFXSR | CR4_OSXMMEXCPT;
    966 
    967 
    968 	/* Xen callbacks */
    969 	initctx->event_callback_eip = (unsigned long) hypervisor_callback;
    970 	initctx->event_callback_cs = GSEL(GCODE_SEL, SEL_KPL);
    971 	initctx->failsafe_callback_eip = (unsigned long) failsafe_callback;
    972 	initctx->failsafe_callback_cs = GSEL(GCODE_SEL, SEL_KPL);
    973 
    974 	return;
    975 }
    976 #endif /* __x86_64__ */
    977 
    978 int
    979 mp_cpu_start(struct cpu_info *ci, vaddr_t target)
    980 {
    981 
    982 	int hyperror;
    983 	struct vcpu_guest_context vcpuctx;
    984 
    985 	KASSERT(ci != NULL);
    986 	KASSERT(ci != &cpu_info_primary);
    987 	KASSERT(ci->ci_flags & CPUF_AP);
    988 
    989 #ifdef __x86_64__
    990 	xen_init_amd64_vcpuctxt(ci, &vcpuctx, (void (*)(struct cpu_info *))target);
    991 #else  /* i386 */
    992 	xen_init_i386_vcpuctxt(ci, &vcpuctx, (void (*)(struct cpu_info *))target);
    993 #endif /* __x86_64__ */
    994 
    995 	/* Initialise the given vcpu to execute cpu_hatch(ci); */
    996 	if ((hyperror = HYPERVISOR_vcpu_op(VCPUOP_initialise, ci->ci_cpuid, &vcpuctx))) {
    997 		aprint_error(": context initialisation failed. errno = %d\n", hyperror);
    998 		return hyperror;
    999 	}
   1000 
   1001 	/* Start it up */
   1002 
   1003 	/* First bring it down */
   1004 	if ((hyperror = HYPERVISOR_vcpu_op(VCPUOP_down, ci->ci_cpuid, NULL))) {
   1005 		aprint_error(": VCPUOP_down hypervisor command failed. errno = %d\n", hyperror);
   1006 		return hyperror;
   1007 	}
   1008 
   1009 	if ((hyperror = HYPERVISOR_vcpu_op(VCPUOP_up, ci->ci_cpuid, NULL))) {
   1010 		aprint_error(": VCPUOP_up hypervisor command failed. errno = %d\n", hyperror);
   1011 		return hyperror;
   1012 	}
   1013 
   1014 	if (!vcpu_is_up(ci)) {
   1015 		aprint_error(": did not come up\n");
   1016 		return -1;
   1017 	}
   1018 
   1019 	return 0;
   1020 }
   1021 
   1022 void
   1023 mp_cpu_start_cleanup(struct cpu_info *ci)
   1024 {
   1025 	if (vcpu_is_up(ci)) {
   1026 		aprint_debug_dev(ci->ci_dev, "is started.\n");
   1027 	}
   1028 	else {
   1029 		aprint_error_dev(ci->ci_dev, "did not start up.\n");
   1030 	}
   1031 
   1032 }
   1033 
   1034 void
   1035 cpu_init_msrs(struct cpu_info *ci, bool full)
   1036 {
   1037 #ifdef __x86_64__
   1038 	if (full) {
   1039 		HYPERVISOR_set_segment_base (SEGBASE_FS, 0);
   1040 		HYPERVISOR_set_segment_base (SEGBASE_GS_KERNEL, (uint64_t) ci);
   1041 		HYPERVISOR_set_segment_base (SEGBASE_GS_USER, 0);
   1042 	}
   1043 #endif	/* __x86_64__ */
   1044 
   1045 	if (cpu_feature[2] & CPUID_NOX)
   1046 		wrmsr(MSR_EFER, rdmsr(MSR_EFER) | EFER_NXE);
   1047 
   1048 }
   1049 
   1050 void
   1051 cpu_offline_md(void)
   1052 {
   1053         int s;
   1054 
   1055         s = splhigh();
   1056         fpusave_cpu(true);
   1057         splx(s);
   1058 }
   1059 
   1060 void
   1061 cpu_get_tsc_freq(struct cpu_info *ci)
   1062 {
   1063 	uint32_t vcpu_tversion;
   1064 	const volatile vcpu_time_info_t *tinfo = &ci->ci_vcpu->time;
   1065 
   1066 	vcpu_tversion = tinfo->version;
   1067 	while (tinfo->version == vcpu_tversion); /* Wait for a time update. XXX: timeout ? */
   1068 
   1069 	uint64_t freq = 1000000000ULL << 32;
   1070 	freq = freq / (uint64_t)tinfo->tsc_to_system_mul;
   1071 	if ( tinfo->tsc_shift < 0 )
   1072 		freq = freq << -tinfo->tsc_shift;
   1073 	else
   1074 		freq = freq >> tinfo->tsc_shift;
   1075 	ci->ci_data.cpu_cc_freq = freq;
   1076 }
   1077 
   1078 void
   1079 x86_cpu_idle_xen(void)
   1080 {
   1081 	struct cpu_info *ci = curcpu();
   1082 
   1083 	KASSERT(ci->ci_ilevel == IPL_NONE);
   1084 
   1085 	x86_disable_intr();
   1086 	if (!__predict_false(ci->ci_want_resched)) {
   1087 		idle_block();
   1088 	} else {
   1089 		x86_enable_intr();
   1090 	}
   1091 }
   1092 
   1093 /*
   1094  * Loads pmap for the current CPU.
   1095  */
   1096 void
   1097 cpu_load_pmap(struct pmap *pmap, struct pmap *oldpmap)
   1098 {
   1099 	KASSERT(pmap != pmap_kernel());
   1100 
   1101 #if defined(__x86_64__) || defined(PAE)
   1102 	struct cpu_info *ci = curcpu();
   1103 	cpuid_t cid = cpu_index(ci);
   1104 
   1105 	mutex_enter(&ci->ci_kpm_mtx);
   1106 	/* make new pmap visible to xen_kpm_sync() */
   1107 	kcpuset_atomic_set(pmap->pm_xen_ptp_cpus, cid);
   1108 #endif
   1109 #ifdef i386
   1110 #ifdef PAE
   1111 	{
   1112 		int i;
   1113 		paddr_t l3_pd = xpmap_ptom_masked(ci->ci_pae_l3_pdirpa);
   1114 		/* don't update the kernel L3 slot */
   1115 		for (i = 0 ; i < PDP_SIZE - 1; i++) {
   1116 			xpq_queue_pte_update(l3_pd + i * sizeof(pd_entry_t),
   1117 			    xpmap_ptom(pmap->pm_pdirpa[i]) | PG_V);
   1118 		}
   1119 		tlbflush();
   1120 	}
   1121 #else /* PAE */
   1122 	lcr3(pmap_pdirpa(pmap, 0));
   1123 #endif /* PAE */
   1124 #endif /* i386 */
   1125 
   1126 #ifdef __x86_64__
   1127 	{
   1128 		int i;
   1129 		pd_entry_t *new_pgd;
   1130 		paddr_t l4_pd_ma;
   1131 
   1132 		l4_pd_ma = xpmap_ptom_masked(ci->ci_kpm_pdirpa);
   1133 
   1134 		/*
   1135 		 * Map user space address in kernel space and load
   1136 		 * user cr3
   1137 		 */
   1138 		new_pgd = pmap->pm_pdir;
   1139 		KASSERT(pmap == ci->ci_pmap);
   1140 
   1141 		/* Copy user pmap L4 PDEs (in user addr. range) to per-cpu L4 */
   1142 		for (i = 0; i < PDIR_SLOT_PTE; i++) {
   1143 			KASSERT(pmap != pmap_kernel() || new_pgd[i] == 0);
   1144 			if (ci->ci_kpm_pdir[i] != new_pgd[i]) {
   1145 				xpq_queue_pte_update(
   1146 				   l4_pd_ma + i * sizeof(pd_entry_t),
   1147 				    new_pgd[i]);
   1148 			}
   1149 		}
   1150 
   1151 		xen_set_user_pgd(pmap_pdirpa(pmap, 0));
   1152 		ci->ci_xen_current_user_pgd = pmap_pdirpa(pmap, 0);
   1153 
   1154 		tlbflush();
   1155 	}
   1156 
   1157 #endif /* __x86_64__ */
   1158 #if defined(__x86_64__) || defined(PAE)
   1159 	/* old pmap no longer visible to xen_kpm_sync() */
   1160 	if (oldpmap != pmap_kernel()) {
   1161 		kcpuset_atomic_clear(oldpmap->pm_xen_ptp_cpus, cid);
   1162 	}
   1163 	mutex_exit(&ci->ci_kpm_mtx);
   1164 #endif
   1165 }
   1166 
   1167  /*
   1168   * pmap_cpu_init_late: perform late per-CPU initialization.
   1169   * Short note about percpu PDIR pages:
   1170   * Both the PAE and __x86_64__ architectures have per-cpu PDIR
   1171   * tables. This is to get around Xen's pagetable setup constraints for
   1172   * PAE (multiple L3[3]s cannot point to the same L2 - Xen
   1173   * will refuse to pin a table setup this way.) and for multiple cpus
   1174   * to map in different user pmaps on __x86_64__ (see: cpu_load_pmap())
   1175   *
   1176   * What this means for us is that the PDIR of the pmap_kernel() is
   1177   * considered to be a canonical "SHADOW" PDIR with the following
   1178   * properties:
   1179   * - Its recursive mapping points to itself
   1180   * - per-cpu recursive mappings point to themselves on __x86_64__
   1181   * - per-cpu L4 pages' kernel entries are expected to be in sync with
   1182   *   the shadow
   1183   */
   1184 
   1185 void
   1186 pmap_cpu_init_late(struct cpu_info *ci)
   1187 {
   1188 #if defined(PAE) || defined(__x86_64__)
   1189 	/*
   1190 	 * The BP has already its own PD page allocated during early
   1191 	 * MD startup.
   1192 	 */
   1193 
   1194 #if defined(__x86_64__)
   1195 	/* Setup per-cpu normal_pdes */
   1196 	int i;
   1197 	extern pd_entry_t * const normal_pdes[];
   1198 	for (i = 0;i < PTP_LEVELS - 1;i++) {
   1199 		ci->ci_normal_pdes[i] = normal_pdes[i];
   1200 	}
   1201 #endif /* __x86_64__ */
   1202 
   1203 	if (ci == &cpu_info_primary)
   1204 		return;
   1205 
   1206 	KASSERT(ci != NULL);
   1207 
   1208 #if defined(PAE)
   1209 	cpu_alloc_l3_page(ci);
   1210 	KASSERT(ci->ci_pae_l3_pdirpa != 0);
   1211 
   1212 	/* Initialise L2 entries 0 - 2: Point them to pmap_kernel() */
   1213 	int i;
   1214 	for (i = 0 ; i < PDP_SIZE - 1; i++) {
   1215 		ci->ci_pae_l3_pdir[i] =
   1216 		    xpmap_ptom_masked(pmap_kernel()->pm_pdirpa[i]) | PG_V;
   1217 	}
   1218 #endif /* PAE */
   1219 
   1220 	ci->ci_kpm_pdir = (pd_entry_t *)uvm_km_alloc(kernel_map, PAGE_SIZE, 0,
   1221 	    UVM_KMF_WIRED | UVM_KMF_ZERO | UVM_KMF_NOWAIT);
   1222 
   1223 	if (ci->ci_kpm_pdir == NULL) {
   1224 		panic("%s: failed to allocate L4 per-cpu PD for CPU %d\n",
   1225 		      __func__, cpu_index(ci));
   1226 	}
   1227 	ci->ci_kpm_pdirpa = vtophys((vaddr_t) ci->ci_kpm_pdir);
   1228 	KASSERT(ci->ci_kpm_pdirpa != 0);
   1229 
   1230 #if defined(__x86_64__)
   1231 	/*
   1232 	 * Copy over the pmap_kernel() shadow L4 entries
   1233 	 */
   1234 
   1235 	memcpy(ci->ci_kpm_pdir, pmap_kernel()->pm_pdir, PAGE_SIZE);
   1236 
   1237 	/* Recursive kernel mapping */
   1238 	ci->ci_kpm_pdir[PDIR_SLOT_PTE] = xpmap_ptom_masked(ci->ci_kpm_pdirpa) | PG_k | PG_V;
   1239 #elif defined(PAE)
   1240 	/* Copy over the pmap_kernel() shadow L2 entries that map the kernel */
   1241 	memcpy(ci->ci_kpm_pdir, pmap_kernel()->pm_pdir + PDIR_SLOT_KERN, nkptp[PTP_LEVELS - 1] * sizeof(pd_entry_t));
   1242 #endif /* __x86_64__ else PAE */
   1243 
   1244 	/* Xen wants R/O */
   1245 	pmap_protect(pmap_kernel(), (vaddr_t)ci->ci_kpm_pdir,
   1246 	    (vaddr_t)ci->ci_kpm_pdir + PAGE_SIZE, VM_PROT_READ);
   1247 	pmap_update(pmap_kernel());
   1248 #if defined(PAE)
   1249 	/* Initialise L3 entry 3. This mapping is shared across all
   1250 	 * pmaps and is static, ie; loading a new pmap will not update
   1251 	 * this entry.
   1252 	 */
   1253 
   1254 	ci->ci_pae_l3_pdir[3] = xpmap_ptom_masked(ci->ci_kpm_pdirpa) | PG_k | PG_V;
   1255 
   1256 	/* Mark L3 R/O (Xen wants this) */
   1257 	pmap_protect(pmap_kernel(), (vaddr_t)ci->ci_pae_l3_pdir,
   1258 	    (vaddr_t)ci->ci_pae_l3_pdir + PAGE_SIZE, VM_PROT_READ);
   1259 	pmap_update(pmap_kernel());
   1260 
   1261 	xpq_queue_pin_l3_table(xpmap_ptom_masked(ci->ci_pae_l3_pdirpa));
   1262 
   1263 #elif defined(__x86_64__)
   1264 	xpq_queue_pin_l4_table(xpmap_ptom_masked(ci->ci_kpm_pdirpa));
   1265 #endif /* PAE , __x86_64__ */
   1266 #endif /* defined(PAE) || defined(__x86_64__) */
   1267 }
   1268 
   1269 /*
   1270  * Notify all other cpus to halt.
   1271  */
   1272 
   1273 void
   1274 cpu_broadcast_halt(void)
   1275 {
   1276 	xen_broadcast_ipi(XEN_IPI_HALT);
   1277 }
   1278 
   1279 /*
   1280  * Send a dummy ipi to a cpu.
   1281  */
   1282 
   1283 void
   1284 cpu_kick(struct cpu_info *ci)
   1285 {
   1286 	(void)xen_send_ipi(ci, XEN_IPI_KICK);
   1287 }
   1288