Home | History | Annotate | Line # | Download | only in x86
cpu.c revision 1.12
      1 /*	$NetBSD: cpu.c,v 1.12 2008/04/17 12:24:44 cegger Exp $	*/
      2 /* NetBSD: cpu.c,v 1.18 2004/02/20 17:35:01 yamt Exp  */
      3 
      4 /*-
      5  * Copyright (c) 2000 The NetBSD Foundation, Inc.
      6  * All rights reserved.
      7  *
      8  * This code is derived from software contributed to The NetBSD Foundation
      9  * by RedBack Networks Inc.
     10  *
     11  * Author: Bill Sommerfeld
     12  *
     13  * Redistribution and use in source and binary forms, with or without
     14  * modification, are permitted provided that the following conditions
     15  * are met:
     16  * 1. Redistributions of source code must retain the above copyright
     17  *    notice, this list of conditions and the following disclaimer.
     18  * 2. Redistributions in binary form must reproduce the above copyright
     19  *    notice, this list of conditions and the following disclaimer in the
     20  *    documentation and/or other materials provided with the distribution.
     21  * 3. All advertising materials mentioning features or use of this software
     22  *    must display the following acknowledgement:
     23  *        This product includes software developed by the NetBSD
     24  *        Foundation, Inc. and its contributors.
     25  * 4. Neither the name of The NetBSD Foundation nor the names of its
     26  *    contributors may be used to endorse or promote products derived
     27  *    from this software without specific prior written permission.
     28  *
     29  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     30  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     31  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     32  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     33  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     34  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     35  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     36  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     37  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     38  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     39  * POSSIBILITY OF SUCH DAMAGE.
     40  */
     41 
     42 /*
     43  * Copyright (c) 1999 Stefan Grefen
     44  *
     45  * Redistribution and use in source and binary forms, with or without
     46  * modification, are permitted provided that the following conditions
     47  * are met:
     48  * 1. Redistributions of source code must retain the above copyright
     49  *    notice, this list of conditions and the following disclaimer.
     50  * 2. Redistributions in binary form must reproduce the above copyright
     51  *    notice, this list of conditions and the following disclaimer in the
     52  *    documentation and/or other materials provided with the distribution.
     53  * 3. All advertising materials mentioning features or use of this software
     54  *    must display the following acknowledgement:
     55  *      This product includes software developed by the NetBSD
     56  *      Foundation, Inc. and its contributors.
     57  * 4. Neither the name of The NetBSD Foundation nor the names of its
     58  *    contributors may be used to endorse or promote products derived
     59  *    from this software without specific prior written permission.
     60  *
     61  * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND ANY
     62  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
     63  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
     64  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR AND CONTRIBUTORS BE LIABLE
     65  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
     66  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
     67  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
     68  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
     69  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
     70  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
     71  * SUCH DAMAGE.
     72  */
     73 
     74 #include <sys/cdefs.h>
     75 __KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.12 2008/04/17 12:24:44 cegger Exp $");
     76 
     77 #include "opt_ddb.h"
     78 #include "opt_multiprocessor.h"
     79 #include "opt_mpbios.h"		/* for MPDEBUG */
     80 #include "opt_mtrr.h"
     81 #include "opt_xen.h"
     82 
     83 #include "lapic.h"
     84 #include "ioapic.h"
     85 
     86 #include <sys/param.h>
     87 #include <sys/proc.h>
     88 #include <sys/user.h>
     89 #include <sys/systm.h>
     90 #include <sys/device.h>
     91 #include <sys/malloc.h>
     92 #include <sys/cpu.h>
     93 #include <sys/atomic.h>
     94 
     95 #include <uvm/uvm_extern.h>
     96 
     97 #include <machine/cpufunc.h>
     98 #include <machine/cpuvar.h>
     99 #include <machine/pmap.h>
    100 #include <machine/vmparam.h>
    101 #include <machine/mpbiosvar.h>
    102 #include <machine/pcb.h>
    103 #include <machine/specialreg.h>
    104 #include <machine/segments.h>
    105 #include <machine/gdt.h>
    106 #include <machine/mtrr.h>
    107 #include <machine/pio.h>
    108 
    109 #ifdef XEN3
    110 #include <xen/vcpuvar.h>
    111 #endif
    112 
    113 #if NLAPIC > 0
    114 #include <machine/apicvar.h>
    115 #include <machine/i82489reg.h>
    116 #include <machine/i82489var.h>
    117 #endif
    118 
    119 #if NIOAPIC > 0
    120 #include <machine/i82093var.h>
    121 #endif
    122 
    123 #include <dev/ic/mc146818reg.h>
    124 #include <dev/isa/isareg.h>
    125 
    126 int     cpu_match(device_t, cfdata_t, void *);
    127 void    cpu_attach(device_t, device_t, void *);
    128 #ifdef XEN3
    129 int     vcpu_match(device_t, cfdata_t, void *);
    130 void    vcpu_attach(device_t, device_t, void *);
    131 #endif
    132 void    cpu_attach_common(device_t, device_t, void *);
    133 void	cpu_offline_md(void);
    134 
    135 struct cpu_softc {
    136 	device_t sc_dev;		/* device tree glue */
    137 	struct cpu_info *sc_info;	/* pointer to CPU info */
    138 };
    139 
    140 int mp_cpu_start(struct cpu_info *, paddr_t);
    141 void mp_cpu_start_cleanup(struct cpu_info *);
    142 const struct cpu_functions mp_cpu_funcs = { mp_cpu_start, NULL,
    143 				      mp_cpu_start_cleanup };
    144 
    145 CFATTACH_DECL_NEW(cpu, sizeof(struct cpu_softc),
    146     cpu_match, cpu_attach, NULL, NULL);
    147 #ifdef XEN3
    148 CFATTACH_DECL_NEW(vcpu, sizeof(struct cpu_softc),
    149     vcpu_match, vcpu_attach, NULL, NULL);
    150 #endif
    151 
    152 /*
    153  * Statically-allocated CPU info for the primary CPU (or the only
    154  * CPU, on uniprocessors).  The CPU info list is initialized to
    155  * point at it.
    156  */
    157 #ifdef TRAPLOG
    158 #include <machine/tlog.h>
    159 struct tlog tlog_primary;
    160 #endif
    161 struct cpu_info cpu_info_primary = {
    162 	.ci_dev = 0,
    163 	.ci_self = &cpu_info_primary,
    164 	.ci_idepth = -1,
    165 	.ci_curlwp = &lwp0,
    166 #ifdef TRAPLOG
    167 	.ci_tlog = &tlog_primary,
    168 #endif
    169 
    170 };
    171 struct cpu_info phycpu_info_primary = {
    172 	.ci_dev = 0,
    173 	.ci_self = &phycpu_info_primary,
    174 };
    175 
    176 struct cpu_info *cpu_info_list = &cpu_info_primary;
    177 
    178 static void	cpu_set_tss_gates(struct cpu_info *ci);
    179 
    180 uint32_t cpus_attached = 0;
    181 uint32_t cpus_running = 0;
    182 
    183 bool x86_mp_online;
    184 paddr_t mp_trampoline_paddr = MP_TRAMPOLINE;
    185 
    186 struct cpu_info *phycpu_info[X86_MAXPROCS] = { &cpu_info_primary };
    187 
    188 #ifdef MULTIPROCESSOR
    189 /*
    190  * Array of CPU info structures.  Must be statically-allocated because
    191  * curproc, etc. are used early.
    192  */
    193 struct cpu_info *cpu_info[X86_MAXPROCS] = { &cpu_info_primary };
    194 
    195 void    	cpu_hatch(void *);
    196 static void    	cpu_boot_secondary(struct cpu_info *ci);
    197 static void    	cpu_start_secondary(struct cpu_info *ci);
    198 static void	cpu_copy_trampoline(void);
    199 
    200 /*
    201  * Runs once per boot once multiprocessor goo has been detected and
    202  * the local APIC on the boot processor has been mapped.
    203  *
    204  * Called from lapic_boot_init() (from mpbios_scan()).
    205  */
    206 void
    207 cpu_init_first(void)
    208 {
    209 	int cpunum = lapic_cpu_number();
    210 
    211 	if (cpunum != 0) {
    212 		cpu_info[0] = NULL;
    213 		cpu_info[cpunum] = &cpu_info_primary;
    214 	}
    215 
    216 	cpu_copy_trampoline();
    217 }
    218 #endif
    219 
    220 int
    221 cpu_match(device_t parent, cfdata_t match, void *aux)
    222 {
    223 
    224 	return 1;
    225 }
    226 
    227 void
    228 cpu_attach(device_t parent, device_t self, void *aux)
    229 {
    230 #ifdef XEN3
    231 	struct cpu_softc *sc = device_private(self);
    232 	struct cpu_attach_args *caa = aux;
    233 	struct cpu_info *ci;
    234 	int cpunum = caa->cpu_number;
    235 
    236 	sc->sc_dev = self;
    237 
    238 	/*
    239 	 * If we're an Application Processor, allocate a cpu_info
    240 	 * structure, otherwise use the primary's.
    241 	 */
    242 	if (caa->cpu_role == CPU_ROLE_AP) {
    243 		ci = malloc(sizeof(*ci), M_DEVBUF, M_WAITOK | M_ZERO);
    244 		if (phycpu_info[cpunum] != NULL)
    245 			panic("cpu at apic id %d already attached?", cpunum);
    246 		phycpu_info[cpunum] = ci;
    247 	} else {
    248 		ci = &phycpu_info_primary;
    249 		if (cpunum != 0) {
    250 			phycpu_info[0] = NULL;
    251 			phycpu_info[cpunum] = ci;
    252 		}
    253 	}
    254 
    255 	ci->ci_self = ci;
    256 	sc->sc_info = ci;
    257 
    258 	ci->ci_dev = self;
    259 	ci->ci_apicid = caa->cpu_number;
    260 	ci->ci_cpuid = ci->ci_apicid;
    261 
    262 	printf(": ");
    263 	switch (caa->cpu_role) {
    264 	case CPU_ROLE_SP:
    265 		printf("(uniprocessor)\n");
    266 		ci->ci_flags |= CPUF_PRESENT | CPUF_SP | CPUF_PRIMARY;
    267 		break;
    268 
    269 	case CPU_ROLE_BP:
    270 		printf("(boot processor)\n");
    271 		ci->ci_flags |= CPUF_PRESENT | CPUF_BSP | CPUF_PRIMARY;
    272 #if NIOAPIC > 0
    273 		ioapic_bsp_id = caa->cpu_number;
    274 #endif
    275 		break;
    276 
    277 	case CPU_ROLE_AP:
    278 		/*
    279 		 * report on an AP
    280 		 */
    281 		printf("(application processor)\n");
    282 		break;
    283 
    284 	default:
    285 		panic("unknown processor type??\n");
    286 	}
    287 	return;
    288 #else
    289 	cpu_attach_common(parent, self, aux);
    290 #endif
    291 }
    292 
    293 #ifdef XEN3
    294 int
    295 vcpu_match(device_t parent, cfdata_t match, void *aux)
    296 {
    297 	struct vcpu_attach_args *vcaa = aux;
    298 
    299 	if (strcmp(vcaa->vcaa_name, match->cf_name) == 0)
    300 		return 1;
    301 	return 0;
    302 }
    303 
    304 void
    305 vcpu_attach(device_t parent, device_t self, void *aux)
    306 {
    307 	struct vcpu_attach_args *vcaa = aux;
    308 
    309 	cpu_attach_common(parent, self, &vcaa->vcaa_caa);
    310 }
    311 #endif
    312 
    313 static void
    314 cpu_vm_init(struct cpu_info *ci)
    315 {
    316 	int ncolors = 2, i;
    317 
    318 	for (i = CAI_ICACHE; i <= CAI_L2CACHE; i++) {
    319 		struct x86_cache_info *cai;
    320 		int tcolors;
    321 
    322 		cai = &ci->ci_cinfo[i];
    323 
    324 		tcolors = atop(cai->cai_totalsize);
    325 		switch(cai->cai_associativity) {
    326 		case 0xff:
    327 			tcolors = 1; /* fully associative */
    328 			break;
    329 		case 0:
    330 		case 1:
    331 			break;
    332 		default:
    333 			tcolors /= cai->cai_associativity;
    334 		}
    335 		ncolors = max(ncolors, tcolors);
    336 	}
    337 
    338 	/*
    339 	 * Knowing the size of the largest cache on this CPU, re-color
    340 	 * our pages.
    341 	 */
    342 	if (ncolors <= uvmexp.ncolors)
    343 		return;
    344 	printf("%s: %d page colors\n", device_xname(ci->ci_dev), ncolors);
    345 	uvm_page_recolor(ncolors);
    346 }
    347 
    348 void
    349 cpu_attach_common(device_t parent, device_t self, void *aux)
    350 {
    351 	struct cpu_softc *sc = device_private(self);
    352 	struct cpu_attach_args *caa = aux;
    353 	struct cpu_info *ci;
    354 	uintptr_t ptr;
    355 #if defined(MULTIPROCESSOR)
    356 	int cpunum = caa->cpu_number;
    357 #endif
    358 
    359 	sc->sc_dev = self;
    360 
    361 	/*
    362 	 * If we're an Application Processor, allocate a cpu_info
    363 	 * structure, otherwise use the primary's.
    364 	 */
    365 	if (caa->cpu_role == CPU_ROLE_AP) {
    366 		aprint_naive(": Application Processor\n");
    367 		ptr = (uintptr_t)malloc(sizeof(*ci) + CACHE_LINE_SIZE - 1,
    368 		    M_DEVBUF, M_WAITOK);
    369 		ci = (struct cpu_info *)((ptr + CACHE_LINE_SIZE - 1) &
    370 		    ~(CACHE_LINE_SIZE - 1));
    371 		memset(ci, 0, sizeof(*ci));
    372 #if defined(MULTIPROCESSOR)
    373 		if (cpu_info[cpunum] != NULL)
    374 			panic("cpu at apic id %d already attached?", cpunum);
    375 		cpu_info[cpunum] = ci;
    376 #endif
    377 #ifdef TRAPLOG
    378 		ci->ci_tlog_base = malloc(sizeof(struct tlog),
    379 		    M_DEVBUF, M_WAITOK);
    380 #endif
    381 	} else {
    382 		aprint_naive(": %s Processor\n",
    383 		    caa->cpu_role == CPU_ROLE_SP ? "Single" : "Boot");
    384 		ci = &cpu_info_primary;
    385 #if defined(MULTIPROCESSOR)
    386 		if (cpunum != lapic_cpu_number()) {
    387 			panic("%s: running CPU is at apic %d"
    388 			    " instead of at expected %d",
    389 			    device_xname(sc->sc_dev), lapic_cpu_number(), cpunum);
    390 		}
    391 #endif
    392 	}
    393 
    394 	ci->ci_self = ci;
    395 	sc->sc_info = ci;
    396 
    397 	ci->ci_dev = self;
    398 	ci->ci_apicid = caa->cpu_number;
    399 #ifdef MULTIPROCESSOR
    400 	ci->ci_cpuid = ci->ci_apicid;
    401 #else
    402 	ci->ci_cpuid = 0;	/* False for APs, but they're not used anyway */
    403 #endif
    404 	ci->ci_cpumask = (1 << ci->ci_cpuid);
    405 	ci->ci_func = caa->cpu_func;
    406 
    407 	if (caa->cpu_role == CPU_ROLE_AP) {
    408 #if defined(MULTIPROCESSOR)
    409 		int error;
    410 
    411 		error = mi_cpu_attach(ci);
    412 		if (error != 0) {
    413 			aprint_normal("\n");
    414 			aprint_error_dev(sc->sc_dev, "mi_cpu_attach failed with %d\n",
    415 			    error);
    416 			return;
    417 		}
    418 #endif
    419 	} else {
    420 		KASSERT(ci->ci_data.cpu_idlelwp != NULL);
    421 	}
    422 
    423 	pmap_reference(pmap_kernel());
    424 	ci->ci_pmap = pmap_kernel();
    425 	ci->ci_tlbstate = TLBSTATE_STALE;
    426 
    427 	/* further PCB init done later. */
    428 
    429 	switch (caa->cpu_role) {
    430 	case CPU_ROLE_SP:
    431 		aprint_normal(": (uniprocessor)\n");
    432 		atomic_or_32(&ci->ci_flags,
    433 		     CPUF_PRESENT | CPUF_SP | CPUF_PRIMARY);
    434 		cpu_intr_init(ci);
    435 		identifycpu(ci);
    436 		cpu_init(ci);
    437 		cpu_set_tss_gates(ci);
    438 		pmap_cpu_init_late(ci);
    439 #if 0
    440 		x86_errata();
    441 #endif
    442 		break;
    443 
    444 	case CPU_ROLE_BP:
    445 		aprint_normal("apid %d (boot processor)\n", caa->cpu_number);
    446 		atomic_or_32(&ci->ci_flags,
    447 		    CPUF_PRESENT | CPUF_BSP | CPUF_PRIMARY);
    448 		cpu_intr_init(ci);
    449 		identifycpu(ci);
    450 		cpu_init(ci);
    451 		cpu_set_tss_gates(ci);
    452 		pmap_cpu_init_late(ci);
    453 #if NLAPIC > 0
    454 		/*
    455 		 * Enable local apic
    456 		 */
    457 		lapic_enable();
    458 		lapic_set_lvt();
    459 		lapic_calibrate_timer(ci);
    460 #endif
    461 #if NIOAPIC > 0
    462 		ioapic_bsp_id = caa->cpu_number;
    463 #endif
    464 #if 0
    465 		x86_errata();
    466 #endif
    467 		break;
    468 
    469 	case CPU_ROLE_AP:
    470 		/*
    471 		 * report on an AP
    472 		 */
    473 		aprint_normal("apid %d (application processor)\n", caa->cpu_number);
    474 
    475 #if defined(MULTIPROCESSOR)
    476 		cpu_intr_init(ci);
    477 		gdt_alloc_cpu(ci);
    478 		cpu_set_tss_gates(ci);
    479 		pmap_cpu_init_early(ci);
    480 		pmap_cpu_init_late(ci);
    481 		cpu_start_secondary(ci);
    482 		if (ci->ci_flags & CPUF_PRESENT) {
    483 			identifycpu(ci);
    484 			ci->ci_next = cpu_info_list->ci_next;
    485 			cpu_info_list->ci_next = ci;
    486 		}
    487 #else
    488 		aprint_normal_dev(sc->sc_dev, "not started\n");
    489 #endif
    490 		break;
    491 
    492 	default:
    493 		aprint_normal("\n");
    494 		panic("unknown processor type??\n");
    495 	}
    496 	cpu_vm_init(ci);
    497 
    498 	cpus_attached |= (1 << ci->ci_cpuid);
    499 
    500 #if 0
    501 	if (!pmf_device_register(self, cpu_suspend, cpu_resume))
    502 		aprint_error_dev(self, "couldn't establish power handler\n");
    503 #endif
    504 
    505 #if defined(MULTIPROCESSOR)
    506 	if (mp_verbose) {
    507 		struct lwp *l = ci->ci_data.cpu_idlelwp;
    508 
    509 		aprint_verbose_dev(sc->sc_dev, "idle lwp at %p, idle sp at 0x%p\n",
    510 		    l,
    511 #ifdef i386
    512 		    (void *)l->l_addr->u_pcb.pcb_esp
    513 #else
    514 		    (void *)l->l_addr->u_pcb.pcb_rsp
    515 #endif
    516 		);
    517 
    518 	}
    519 #endif
    520 }
    521 
    522 /*
    523  * Initialize the processor appropriately.
    524  */
    525 
    526 void
    527 cpu_init(struct cpu_info *ci)
    528 {
    529 	/* configure the CPU if needed */
    530 	if (ci->cpu_setup != NULL)
    531 		(*ci->cpu_setup)(ci);
    532 
    533 	/*
    534 	 * On a P6 or above, enable global TLB caching if the
    535 	 * hardware supports it.
    536 	 */
    537 	if (cpu_feature & CPUID_PGE)
    538 		lcr4(rcr4() | CR4_PGE);	/* enable global TLB caching */
    539 
    540 #ifdef XXXMTRR
    541 	/*
    542 	 * On a P6 or above, initialize MTRR's if the hardware supports them.
    543 	 */
    544 	if (cpu_feature & CPUID_MTRR) {
    545 		if ((ci->ci_flags & CPUF_AP) == 0)
    546 			i686_mtrr_init_first();
    547 		mtrr_init_cpu(ci);
    548 	}
    549 #endif
    550 	/*
    551 	 * If we have FXSAVE/FXRESTOR, use them.
    552 	 */
    553 	if (cpu_feature & CPUID_FXSR) {
    554 		lcr4(rcr4() | CR4_OSFXSR);
    555 
    556 		/*
    557 		 * If we have SSE/SSE2, enable XMM exceptions.
    558 		 */
    559 		if (cpu_feature & (CPUID_SSE|CPUID_SSE2))
    560 			lcr4(rcr4() | CR4_OSXMMEXCPT);
    561 	}
    562 
    563 #ifdef MULTIPROCESSOR
    564 	atomic_or_32(&ci->ci_flags, CPUF_RUNNING);
    565 	atomic_or_32(&cpus_running, ci->ci_cpumask);
    566 #endif
    567 }
    568 
    569 
    570 #ifdef MULTIPROCESSOR
    571 void
    572 cpu_boot_secondary_processors(void)
    573 {
    574 	struct cpu_info *ci;
    575 	u_long i;
    576 
    577 	for (i = 0; i < X86_MAXPROCS; i++) {
    578 		ci = cpu_info[i];
    579 		if (ci == NULL)
    580 			continue;
    581 		if (ci->ci_data.cpu_idlelwp == NULL)
    582 			continue;
    583 		if ((ci->ci_flags & CPUF_PRESENT) == 0)
    584 			continue;
    585 		if (ci->ci_flags & (CPUF_BSP|CPUF_SP|CPUF_PRIMARY))
    586 			continue;
    587 		cpu_boot_secondary(ci);
    588 	}
    589 
    590 	x86_mp_online = true;
    591 }
    592 
    593 static void
    594 cpu_init_idle_lwp(struct cpu_info *ci)
    595 {
    596 	struct lwp *l = ci->ci_data.cpu_idlelwp;
    597 	struct pcb *pcb = &l->l_addr->u_pcb;
    598 
    599 	pcb->pcb_cr0 = rcr0();
    600 }
    601 
    602 void
    603 cpu_init_idle_lwps(void)
    604 {
    605 	struct cpu_info *ci;
    606 	u_long i;
    607 
    608 	for (i = 0; i < X86_MAXPROCS; i++) {
    609 		ci = cpu_info[i];
    610 		if (ci == NULL)
    611 			continue;
    612 		if (ci->ci_data.cpu_idlelwp == NULL)
    613 			continue;
    614 		if ((ci->ci_flags & CPUF_PRESENT) == 0)
    615 			continue;
    616 		cpu_init_idle_lwp(ci);
    617 	}
    618 }
    619 
    620 void
    621 cpu_start_secondary(struct cpu_info *ci)
    622 {
    623 	int i;
    624 	struct pmap *kpm = pmap_kernel();
    625 	extern uint32_t mp_pdirpa;
    626 
    627 	mp_pdirpa = kpm->pm_pdirpa; /* XXX move elsewhere, not per CPU. */
    628 
    629 	atomic_or_32(&ci->ci_flags, CPUF_AP);
    630 
    631 	aprint_debug_dev(ci->ci_dev, "starting\n");
    632 
    633 	ci->ci_curlwp = ci->ci_data.cpu_idlelwp;
    634 	if (CPU_STARTUP(ci, mp_trampoline_paddr) != 0)
    635 		return;
    636 
    637 	/*
    638 	 * wait for it to become ready
    639 	 */
    640 	for (i = 100000; (!(ci->ci_flags & CPUF_PRESENT)) && i > 0; i--) {
    641 #ifdef MPDEBUG
    642 		extern int cpu_trace[3];
    643 		static int otrace[3];
    644 		if (memcmp(otrace, cpu_trace, sizeof(otrace)) != 0) {
    645 			aprint_debug_dev(ci->ci_dev, "trace %02x %02x %02x\n",
    646 				cpu_trace[0], cpu_trace[1], cpu_trace[2]);
    647 			memcpy(otrace, cpu_trace, sizeof(otrace));
    648 		}
    649 #endif
    650 		delay(10);
    651 	}
    652 	if ((ci->ci_flags & CPUF_PRESENT) == 0) {
    653 		aprint_error_dev(ci->ci_dev, "failed to become ready\n");
    654 #if defined(MPDEBUG) && defined(DDB)
    655 		printf("dropping into debugger; continue from here to resume boot\n");
    656 		Debugger();
    657 #endif
    658 	}
    659 
    660 	CPU_START_CLEANUP(ci);
    661 }
    662 
    663 void
    664 cpu_boot_secondary(struct cpu_info *ci)
    665 {
    666 	int i;
    667 
    668 	atomic_or_32(&ci->ci_flags, CPUF_GO);
    669 	for (i = 100000; (!(ci->ci_flags & CPUF_RUNNING)) && i > 0; i--) {
    670 		delay(10);
    671 	}
    672 	if ((ci->ci_flags & CPUF_RUNNING) == 0) {
    673 		aprint_error_dev(ci->ci_dev, "CPU failed to start\n");
    674 #if defined(MPDEBUG) && defined(DDB)
    675 		printf("dropping into debugger; continue from here to resume boot\n");
    676 		Debugger();
    677 #endif
    678 	}
    679 }
    680 
    681 /*
    682  * The CPU ends up here when its ready to run
    683  * This is called from code in mptramp.s; at this point, we are running
    684  * in the idle pcb/idle stack of the new CPU.  When this function returns,
    685  * this processor will enter the idle loop and start looking for work.
    686  *
    687  * XXX should share some of this with init386 in machdep.c
    688  */
    689 void
    690 cpu_hatch(void *v)
    691 {
    692 	struct cpu_info *ci = (struct cpu_info *)v;
    693 	int s, i;
    694 	uint32_t blacklist_features;
    695 
    696 #ifdef __x86_64__
    697         cpu_init_msrs(ci, true);
    698 #endif
    699 
    700 	cpu_probe_features(ci);
    701 	cpu_feature &= ci->ci_feature_flags;
    702 	cpu_feature2 &= ci->ci_feature2_flags;
    703 
    704 	/* not on Xen... */
    705 	blacklist_features = ~(CPUID_PGE|CPUID_PSE|CPUID_MTRR|CPUID_FXSR|CPUID_NOX); /* XXX add CPUID_SVM */
    706 
    707 	cpu_feature &= blacklist_features;
    708 
    709 	KDASSERT((ci->ci_flags & CPUF_PRESENT) == 0);
    710 	atomic_or_32(&ci->ci_flags, CPUF_PRESENT);
    711 	while ((ci->ci_flags & CPUF_GO) == 0) {
    712 		/* Don't use delay, boot CPU may be patching the text. */
    713 		for (i = 10000; i != 0; i--)
    714 			x86_pause();
    715 	}
    716 
    717 	/* Because the text may have been patched in x86_patch(). */
    718 	wbinvd();
    719 	x86_flush();
    720 
    721 	KASSERT((ci->ci_flags & CPUF_RUNNING) == 0);
    722 
    723 	lcr3(pmap_kernel()->pm_pdirpa);
    724 	curlwp->l_addr->u_pcb.pcb_cr3 = pmap_kernel()->pm_pdirpa;
    725 	lcr0(ci->ci_data.cpu_idlelwp->l_addr->u_pcb.pcb_cr0);
    726 	cpu_init_idt();
    727 	gdt_init_cpu(ci);
    728 	lapic_enable();
    729 	lapic_set_lvt();
    730 	lapic_initclocks();
    731 
    732 #ifdef i386
    733 	npxinit(ci);
    734 #else
    735 	fpuinit(ci);
    736 #endif
    737 
    738 	lldt(GSEL(GLDT_SEL, SEL_KPL));
    739 	ltr(ci->ci_tss_sel);
    740 
    741 	cpu_init(ci);
    742 	cpu_get_tsc_freq(ci);
    743 
    744 	s = splhigh();
    745 #ifdef i386
    746 	lapic_tpr = 0;
    747 #else
    748 	lcr8(0);
    749 #endif
    750 	x86_enable_intr();
    751 	splx(s);
    752 #if 0
    753 	x86_errata();
    754 #endif
    755 
    756 	aprint_debug_dev(ci->ci_dev, "CPU %ld running\n",
    757 		(long)ci->ci_cpuid);
    758 }
    759 
    760 #if defined(DDB)
    761 
    762 #include <ddb/db_output.h>
    763 #include <machine/db_machdep.h>
    764 
    765 /*
    766  * Dump CPU information from ddb.
    767  */
    768 void
    769 cpu_debug_dump(void)
    770 {
    771 	struct cpu_info *ci;
    772 	CPU_INFO_ITERATOR cii;
    773 
    774 	db_printf("addr		dev	id	flags	ipis	curproc		fpcurproc\n");
    775 	for (CPU_INFO_FOREACH(cii, ci)) {
    776 		db_printf("%p	%s	%ld	%x	%x	%10p	%10p\n",
    777 		    ci,
    778 		    ci->ci_dev == NULL ? "BOOT" : device_xname(ci->ci_dev),
    779 		    (long)ci->ci_cpuid,
    780 		    ci->ci_flags, ci->ci_ipis,
    781 		    ci->ci_curlwp,
    782 		    ci->ci_fpcurlwp);
    783 	}
    784 }
    785 #endif
    786 
    787 static void
    788 cpu_copy_trampoline(void)
    789 {
    790 	/*
    791 	 * Copy boot code.
    792 	 */
    793 	extern u_char cpu_spinup_trampoline[];
    794 	extern u_char cpu_spinup_trampoline_end[];
    795 
    796 	vaddr_t mp_trampoline_vaddr;
    797 
    798 	mp_trampoline_vaddr = uvm_km_alloc(kernel_map, PAGE_SIZE, 0,
    799 		UVM_KMF_VAONLY);
    800 
    801 	pmap_kenter_pa(mp_trampoline_vaddr, mp_trampoline_paddr,
    802 		VM_PROT_READ | VM_PROT_WRITE);
    803 	pmap_update(pmap_kernel());
    804 	memcpy((void *)mp_trampoline_vaddr,
    805 		cpu_spinup_trampoline,
    806 		cpu_spinup_trampoline_end - cpu_spinup_trampoline);
    807 
    808 	pmap_kremove(mp_trampoline_vaddr, PAGE_SIZE);
    809 	pmap_update(pmap_kernel());
    810 	uvm_km_free(kernel_map, mp_trampoline_vaddr, PAGE_SIZE, UVM_KMF_VAONLY);
    811 }
    812 
    813 #endif
    814 
    815 #ifdef i386
    816 #if 0
    817 static void
    818 tss_init(struct i386tss *tss, void *stack, void *func)
    819 {
    820 	memset(tss, 0, sizeof *tss);
    821 	tss->tss_esp0 = tss->tss_esp = (int)((char *)stack + USPACE - 16);
    822 	tss->tss_ss0 = GSEL(GDATA_SEL, SEL_KPL);
    823 	tss->__tss_cs = GSEL(GCODE_SEL, SEL_KPL);
    824 	tss->tss_fs = GSEL(GCPU_SEL, SEL_KPL);
    825 	tss->tss_gs = tss->__tss_es = tss->__tss_ds =
    826 	    tss->__tss_ss = GSEL(GDATA_SEL, SEL_KPL);
    827 	tss->tss_cr3 = pmap_kernel()->pm_pdirpa;
    828 	tss->tss_esp = (int)((char *)stack + USPACE - 16);
    829 	tss->tss_ldt = GSEL(GLDT_SEL, SEL_KPL);
    830 	tss->__tss_eflags = PSL_MBO | PSL_NT;   /* XXX not needed? */
    831 	tss->__tss_eip = (int)func;
    832 }
    833 #endif
    834 
    835 /* XXX */
    836 #define IDTVEC(name)	__CONCAT(X, name)
    837 typedef void (vector)(void);
    838 extern vector IDTVEC(tss_trap08);
    839 #ifdef DDB
    840 extern vector Xintrddbipi;
    841 extern int ddb_vec;
    842 #endif
    843 
    844 static void
    845 cpu_set_tss_gates(struct cpu_info *ci)
    846 {
    847 #if 0
    848 	struct segment_descriptor sd;
    849 
    850 	ci->ci_doubleflt_stack = (char *)uvm_km_alloc(kernel_map, USPACE, 0,
    851 	    UVM_KMF_WIRED);
    852 	tss_init(&ci->ci_doubleflt_tss, ci->ci_doubleflt_stack,
    853 	    IDTVEC(tss_trap08));
    854 	setsegment(&sd, &ci->ci_doubleflt_tss, sizeof(struct i386tss) - 1,
    855 	    SDT_SYS386TSS, SEL_KPL, 0, 0);
    856 	ci->ci_gdt[GTRAPTSS_SEL].sd = sd;
    857 	setgate(&idt[8], NULL, 0, SDT_SYSTASKGT, SEL_KPL,
    858 	    GSEL(GTRAPTSS_SEL, SEL_KPL));
    859 #endif
    860 
    861 #if defined(DDB) && defined(MULTIPROCESSOR)
    862 	/*
    863 	 * Set up separate handler for the DDB IPI, so that it doesn't
    864 	 * stomp on a possibly corrupted stack.
    865 	 *
    866 	 * XXX overwriting the gate set in db_machine_init.
    867 	 * Should rearrange the code so that it's set only once.
    868 	 */
    869 	ci->ci_ddbipi_stack = (char *)uvm_km_alloc(kernel_map, USPACE, 0,
    870 	    UVM_KMF_WIRED);
    871 	tss_init(&ci->ci_ddbipi_tss, ci->ci_ddbipi_stack,
    872 	    Xintrddbipi);
    873 
    874 	setsegment(&sd, &ci->ci_ddbipi_tss, sizeof(struct i386tss) - 1,
    875 	    SDT_SYS386TSS, SEL_KPL, 0, 0);
    876 	ci->ci_gdt[GIPITSS_SEL].sd = sd;
    877 
    878 	setgate(&idt[ddb_vec], NULL, 0, SDT_SYSTASKGT, SEL_KPL,
    879 	    GSEL(GIPITSS_SEL, SEL_KPL));
    880 #endif
    881 }
    882 #else
    883 static void
    884 cpu_set_tss_gates(struct cpu_info *ci)
    885 {
    886 
    887 }
    888 #endif	/* i386 */
    889 
    890 int
    891 mp_cpu_start(struct cpu_info *ci, paddr_t target)
    892 {
    893 #if 0
    894 #if NLAPIC > 0
    895 	int error;
    896 #endif
    897 	unsigned short dwordptr[2];
    898 
    899 	/*
    900 	 * Bootstrap code must be addressable in real mode
    901 	 * and it must be page aligned.
    902 	 */
    903 	KASSERT(target < 0x10000 && target % PAGE_SIZE == 0);
    904 
    905 	/*
    906 	 * "The BSP must initialize CMOS shutdown code to 0Ah ..."
    907 	 */
    908 
    909 	outb(IO_RTC, NVRAM_RESET);
    910 	outb(IO_RTC+1, NVRAM_RESET_JUMP);
    911 
    912 	/*
    913 	 * "and the warm reset vector (DWORD based at 40:67) to point
    914 	 * to the AP startup code ..."
    915 	 */
    916 
    917 	dwordptr[0] = 0;
    918 	dwordptr[1] = target >> 4;
    919 
    920 	pmap_kenter_pa (0, 0, VM_PROT_READ|VM_PROT_WRITE);
    921 	memcpy ((uint8_t *) 0x467, dwordptr, 4);
    922 	pmap_kremove (0, PAGE_SIZE);
    923 
    924 #if NLAPIC > 0
    925 	/*
    926 	 * ... prior to executing the following sequence:"
    927 	 */
    928 
    929 	if (ci->ci_flags & CPUF_AP) {
    930 		if ((error = x86_ipi_init(ci->ci_apicid)) != 0)
    931 			return error;
    932 
    933 		delay(10000);
    934 
    935 		if (cpu_feature & CPUID_APIC) {
    936 			error = x86_ipi_init(ci->ci_apicid);
    937 			if (error != 0) {
    938 				aprint_error_dev(ci->ci_dev, "%s: IPI not taken (1)\n",
    939 						__func__);
    940 				return error;
    941 			}
    942 
    943 			delay(10000);
    944 
    945 			error = x86_ipi(target / PAGE_SIZE, ci->ci_apicid,
    946 					LAPIC_DLMODE_STARTUP);
    947 			if (error != 0) {
    948 				aprint_error_dev(ci->ci_dev, "%s: IPI not taken (2)\n",
    949 						__func__);
    950 				return error;
    951 			}
    952 			delay(200);
    953 
    954 			error = x86_ipi(target / PAGE_SIZE, ci->ci_apicid,
    955 					LAPIC_DLMODE_STARTUP);
    956 			if (error != 0) {
    957 				aprint_error_dev(ci->ci_dev, "%s: IPI not taken ((3)\n",
    958 						__func__);
    959 				return error;
    960 			}
    961 			delay(200);
    962 		}
    963 	}
    964 #endif
    965 #endif /* 0 */
    966 	return 0;
    967 }
    968 
    969 void
    970 mp_cpu_start_cleanup(struct cpu_info *ci)
    971 {
    972 #if 0
    973 	/*
    974 	 * Ensure the NVRAM reset byte contains something vaguely sane.
    975 	 */
    976 
    977 	outb(IO_RTC, NVRAM_RESET);
    978 	outb(IO_RTC+1, NVRAM_RESET_RST);
    979 #endif
    980 }
    981 
    982 #ifdef __x86_64__
    983 
    984 void
    985 cpu_init_msrs(struct cpu_info *ci, bool full)
    986 {
    987 	if (full) {
    988 		HYPERVISOR_set_segment_base (SEGBASE_FS, 0);
    989 		HYPERVISOR_set_segment_base (SEGBASE_GS_KERNEL, (uint64_t) ci);
    990 		HYPERVISOR_set_segment_base (SEGBASE_GS_USER, 0);
    991 	}
    992 }
    993 #endif	/* __x86_64__ */
    994 
    995 void
    996 cpu_offline_md(void)
    997 {
    998         int s;
    999 
   1000         s = splhigh();
   1001 #ifdef __i386__
   1002         npxsave_cpu(true);
   1003 #else
   1004         fpusave_cpu(true);
   1005 #endif
   1006         splx(s);
   1007 }
   1008 
   1009 #if 0
   1010 /* XXX joerg restructure and restart CPUs individually */
   1011 static bool
   1012 cpu_suspend(device_t dv PMF_FN_ARGS)
   1013 {
   1014 	struct cpu_softc *sc = device_private(dv);
   1015 	struct cpu_info *ci = sc->sc_info;
   1016 	int err;
   1017 
   1018 	if (ci->ci_flags & CPUF_PRIMARY)
   1019 		return true;
   1020 	if (ci->ci_data.cpu_idlelwp == NULL)
   1021 		return true;
   1022 	if ((ci->ci_flags & CPUF_PRESENT) == 0)
   1023 		return true;
   1024 
   1025 	sc->sc_wasonline = !(ci->ci_schedstate.spc_flags & SPCF_OFFLINE);
   1026 
   1027 	if (sc->sc_wasonline) {
   1028 		mutex_enter(&cpu_lock);
   1029 		err = cpu_setonline(ci, false);
   1030 		mutex_exit(&cpu_lock);
   1031 
   1032 		if (err)
   1033 			return false;
   1034 	}
   1035 
   1036 	return true;
   1037 }
   1038 
   1039 static bool
   1040 cpu_resume(device_t dv PMF_FN_ARGS)
   1041 {
   1042 	struct cpu_softc *sc = device_private(dv);
   1043 	struct cpu_info *ci = sc->sc_info;
   1044 	int err = 0;
   1045 
   1046 	if (ci->ci_flags & CPUF_PRIMARY)
   1047 		return true;
   1048 	if (ci->ci_data.cpu_idlelwp == NULL)
   1049 		return true;
   1050 	if ((ci->ci_flags & CPUF_PRESENT) == 0)
   1051 		return true;
   1052 
   1053 	if (sc->sc_wasonline) {
   1054 		mutex_enter(&cpu_lock);
   1055 		err = cpu_setonline(ci, true);
   1056 		mutex_exit(&cpu_lock);
   1057 	}
   1058 
   1059 	return err == 0;
   1060 }
   1061 #endif
   1062 
   1063 void
   1064 cpu_get_tsc_freq(struct cpu_info *ci)
   1065 {
   1066 #ifdef XEN3
   1067 	const volatile vcpu_time_info_t *tinfo =
   1068 		   &HYPERVISOR_shared_info->vcpu_info[0].time;
   1069 	delay(1000000);
   1070 	uint64_t freq = 1000000000ULL << 32;
   1071 	freq = freq / (uint64_t)tinfo->tsc_to_system_mul;
   1072 	if ( tinfo->tsc_shift < 0 )
   1073 		freq = freq << -tinfo->tsc_shift;
   1074 	else
   1075 		freq = freq >> tinfo->tsc_shift;
   1076 	ci->ci_tsc_freq = freq;
   1077 #else
   1078 	/* XXX this needs to read the shared_info of the CPU being probed.. */
   1079 	ci->ci_tsc_freq = HYPERVISOR_shared_info->cpu_freq;
   1080 #endif /* XEN3 */
   1081 }
   1082