cpu.c revision 1.22
1 /* $NetBSD: cpu.c,v 1.22 2008/02/29 06:27:15 dyoung Exp $ */ 2 3 /*- 4 * Copyright (c) 2000, 2006, 2007 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Bill Sommerfeld of RedBack Networks Inc, and by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 3. All advertising materials mentioning features or use of this software 19 * must display the following acknowledgement: 20 * This product includes software developed by the NetBSD 21 * Foundation, Inc. and its contributors. 22 * 4. Neither the name of The NetBSD Foundation nor the names of its 23 * contributors may be used to endorse or promote products derived 24 * from this software without specific prior written permission. 25 * 26 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 27 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 28 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 29 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 30 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 31 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 32 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 33 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 34 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 35 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 36 * POSSIBILITY OF SUCH DAMAGE. 37 */ 38 39 /* 40 * Copyright (c) 1999 Stefan Grefen 41 * 42 * Redistribution and use in source and binary forms, with or without 43 * modification, are permitted provided that the following conditions 44 * are met: 45 * 1. Redistributions of source code must retain the above copyright 46 * notice, this list of conditions and the following disclaimer. 47 * 2. Redistributions in binary form must reproduce the above copyright 48 * notice, this list of conditions and the following disclaimer in the 49 * documentation and/or other materials provided with the distribution. 50 * 3. All advertising materials mentioning features or use of this software 51 * must display the following acknowledgement: 52 * This product includes software developed by the NetBSD 53 * Foundation, Inc. and its contributors. 54 * 4. Neither the name of The NetBSD Foundation nor the names of its 55 * contributors may be used to endorse or promote products derived 56 * from this software without specific prior written permission. 57 * 58 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND ANY 59 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 60 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 61 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR AND CONTRIBUTORS BE LIABLE 62 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 63 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 64 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 65 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 66 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 67 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 68 * SUCH DAMAGE. 69 */ 70 71 #include <sys/cdefs.h> 72 __KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.22 2008/02/29 06:27:15 dyoung Exp $"); 73 74 #include "opt_ddb.h" 75 #include "opt_multiprocessor.h" 76 #include "opt_mpbios.h" /* for MPDEBUG */ 77 #include "opt_mtrr.h" 78 79 #include "lapic.h" 80 #include "ioapic.h" 81 82 #include <sys/param.h> 83 #include <sys/proc.h> 84 #include <sys/user.h> 85 #include <sys/systm.h> 86 #include <sys/device.h> 87 #include <sys/malloc.h> 88 #include <sys/cpu.h> 89 #include <sys/atomic.h> 90 91 #include <uvm/uvm_extern.h> 92 93 #include <machine/cpufunc.h> 94 #include <machine/cpuvar.h> 95 #include <machine/pmap.h> 96 #include <machine/vmparam.h> 97 #include <machine/mpbiosvar.h> 98 #include <machine/pcb.h> 99 #include <machine/specialreg.h> 100 #include <machine/segments.h> 101 #include <machine/gdt.h> 102 #include <machine/mtrr.h> 103 #include <machine/pio.h> 104 105 #ifdef i386 106 #include <machine/tlog.h> 107 #endif 108 109 #if NLAPIC > 0 110 #include <machine/apicvar.h> 111 #include <machine/i82489reg.h> 112 #include <machine/i82489var.h> 113 #endif 114 115 #if NIOAPIC > 0 116 #include <machine/i82093var.h> 117 #endif 118 119 #include <dev/ic/mc146818reg.h> 120 #include <i386/isa/nvram.h> 121 #include <dev/isa/isareg.h> 122 123 int cpu_match(struct device *, struct cfdata *, void *); 124 void cpu_attach(struct device *, struct device *, void *); 125 126 static bool cpu_suspend(device_t PMF_FN_PROTO); 127 static bool cpu_resume(device_t PMF_FN_PROTO); 128 129 struct cpu_softc { 130 struct device sc_dev; /* device tree glue */ 131 struct cpu_info *sc_info; /* pointer to CPU info */ 132 bool sc_wasonline; 133 }; 134 135 int mp_cpu_start(struct cpu_info *, paddr_t); 136 void mp_cpu_start_cleanup(struct cpu_info *); 137 const struct cpu_functions mp_cpu_funcs = { mp_cpu_start, NULL, 138 mp_cpu_start_cleanup }; 139 140 141 CFATTACH_DECL(cpu, sizeof(struct cpu_softc), 142 cpu_match, cpu_attach, NULL, NULL); 143 144 /* 145 * Statically-allocated CPU info for the primary CPU (or the only 146 * CPU, on uniprocessors). The CPU info list is initialized to 147 * point at it. 148 */ 149 #ifdef TRAPLOG 150 struct tlog tlog_primary; 151 #endif 152 struct cpu_info cpu_info_primary __aligned(CACHE_LINE_SIZE) = { 153 .ci_dev = 0, 154 .ci_self = &cpu_info_primary, 155 .ci_idepth = -1, 156 .ci_curlwp = &lwp0, 157 #ifdef TRAPLOG 158 .ci_tlog_base = &tlog_primary, 159 #endif /* !TRAPLOG */ 160 }; 161 162 struct cpu_info *cpu_info_list = &cpu_info_primary; 163 164 static void cpu_set_tss_gates(struct cpu_info *); 165 166 #ifdef i386 167 static void tss_init(struct i386tss *, void *, void *); 168 #endif 169 170 #ifdef MULTIPROCESSOR 171 static void cpu_init_idle_lwp(struct cpu_info *); 172 #endif 173 174 uint32_t cpus_attached = 0; 175 uint32_t cpus_running = 0; 176 177 extern char x86_64_doubleflt_stack[]; 178 179 bool x86_mp_online; 180 paddr_t mp_trampoline_paddr = MP_TRAMPOLINE; 181 182 static vaddr_t cmos_data_mapping; 183 184 #ifdef MULTIPROCESSOR 185 /* 186 * Array of CPU info structures. Must be statically-allocated because 187 * curproc, etc. are used early. 188 */ 189 struct cpu_info *cpu_info[X86_MAXPROCS] = { &cpu_info_primary, }; 190 191 void cpu_hatch(void *); 192 static void cpu_boot_secondary(struct cpu_info *ci); 193 static void cpu_start_secondary(struct cpu_info *ci); 194 static void cpu_copy_trampoline(void); 195 196 /* 197 * Runs once per boot once multiprocessor goo has been detected and 198 * the local APIC on the boot processor has been mapped. 199 * 200 * Called from lapic_boot_init() (from mpbios_scan()). 201 */ 202 void 203 cpu_init_first(void) 204 { 205 int cpunum = lapic_cpu_number(); 206 207 if (cpunum != 0) { 208 cpu_info[0] = NULL; 209 cpu_info[cpunum] = &cpu_info_primary; 210 } 211 212 cpu_info_primary.ci_cpuid = cpunum; 213 cpu_copy_trampoline(); 214 215 cmos_data_mapping = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, UVM_KMF_VAONLY); 216 if (cmos_data_mapping == 0) 217 panic("No KVA for page 0"); 218 pmap_kenter_pa(cmos_data_mapping, 0, VM_PROT_READ|VM_PROT_WRITE); 219 pmap_update(pmap_kernel()); 220 } 221 #endif 222 223 int 224 cpu_match(struct device *parent, struct cfdata *match, 225 void *aux) 226 { 227 228 return 1; 229 } 230 231 static void 232 cpu_vm_init(struct cpu_info *ci) 233 { 234 int ncolors = 2, i; 235 236 for (i = CAI_ICACHE; i <= CAI_L2CACHE; i++) { 237 struct x86_cache_info *cai; 238 int tcolors; 239 240 cai = &ci->ci_cinfo[i]; 241 242 tcolors = atop(cai->cai_totalsize); 243 switch(cai->cai_associativity) { 244 case 0xff: 245 tcolors = 1; /* fully associative */ 246 break; 247 case 0: 248 case 1: 249 break; 250 default: 251 tcolors /= cai->cai_associativity; 252 } 253 ncolors = max(ncolors, tcolors); 254 } 255 256 /* 257 * Knowing the size of the largest cache on this CPU, re-color 258 * our pages. 259 */ 260 if (ncolors <= uvmexp.ncolors) 261 return; 262 aprint_verbose("%s: %d page colors\n", ci->ci_dev->dv_xname, ncolors); 263 uvm_page_recolor(ncolors); 264 } 265 266 267 void 268 cpu_attach(struct device *parent, struct device *self, void *aux) 269 { 270 struct cpu_softc *sc = (void *) self; 271 struct cpu_attach_args *caa = aux; 272 struct cpu_info *ci; 273 uintptr_t ptr; 274 #if defined(MULTIPROCESSOR) 275 int cpunum = caa->cpu_number; 276 #endif 277 278 /* 279 * If we're an Application Processor, allocate a cpu_info 280 * structure, otherwise use the primary's. 281 */ 282 if (caa->cpu_role == CPU_ROLE_AP) { 283 aprint_naive(": Application Processor\n"); 284 ptr = (uintptr_t)malloc(sizeof(*ci) + CACHE_LINE_SIZE - 1, 285 M_DEVBUF, M_WAITOK); 286 ci = (struct cpu_info *)((ptr + CACHE_LINE_SIZE - 1) & 287 ~(CACHE_LINE_SIZE - 1)); 288 memset(ci, 0, sizeof(*ci)); 289 #if defined(MULTIPROCESSOR) 290 if (cpu_info[cpunum] != NULL) { 291 printf("\n"); 292 panic("cpu at apic id %d already attached?", cpunum); 293 } 294 cpu_info[cpunum] = ci; 295 #endif 296 #ifdef TRAPLOG 297 ci->ci_tlog_base = malloc(sizeof(struct tlog), 298 M_DEVBUF, M_WAITOK); 299 #endif 300 } else { 301 aprint_naive(": %s Processor\n", 302 caa->cpu_role == CPU_ROLE_SP ? "Single" : "Boot"); 303 ci = &cpu_info_primary; 304 #if defined(MULTIPROCESSOR) 305 if (cpunum != lapic_cpu_number()) { 306 printf("\n"); 307 panic("%s: running CPU is at apic %d" 308 " instead of at expected %d", 309 sc->sc_dev.dv_xname, lapic_cpu_number(), cpunum); 310 } 311 #endif 312 } 313 314 ci->ci_self = ci; 315 sc->sc_info = ci; 316 317 ci->ci_dev = self; 318 ci->ci_apicid = caa->cpu_number; 319 #ifdef MULTIPROCESSOR 320 ci->ci_cpuid = ci->ci_apicid; 321 #else 322 ci->ci_cpuid = 0; /* False for APs, but they're not used anyway */ 323 #endif 324 ci->ci_cpumask = (1 << ci->ci_cpuid); 325 ci->ci_func = caa->cpu_func; 326 327 if (caa->cpu_role == CPU_ROLE_AP) { 328 #ifdef MULTIPROCESSOR 329 int error; 330 331 error = mi_cpu_attach(ci); 332 if (error != 0) { 333 aprint_normal("\n"); 334 aprint_error("%s: mi_cpu_attach failed with %d\n", 335 sc->sc_dev.dv_xname, error); 336 return; 337 } 338 #endif 339 cpu_init_tss(ci); 340 } else { 341 KASSERT(ci->ci_data.cpu_idlelwp != NULL); 342 } 343 344 pmap_reference(pmap_kernel()); 345 ci->ci_pmap = pmap_kernel(); 346 ci->ci_tlbstate = TLBSTATE_STALE; 347 348 /* further PCB init done later. */ 349 350 switch (caa->cpu_role) { 351 case CPU_ROLE_SP: 352 aprint_normal(": (uniprocessor)\n"); 353 atomic_or_32(&ci->ci_flags, 354 CPUF_PRESENT | CPUF_SP | CPUF_PRIMARY); 355 cpu_intr_init(ci); 356 identifycpu(ci); 357 cpu_init(ci); 358 cpu_set_tss_gates(ci); 359 pmap_cpu_init_late(ci); 360 x86_errata(); 361 break; 362 363 case CPU_ROLE_BP: 364 aprint_normal(": (boot processor)\n"); 365 atomic_or_32(&ci->ci_flags, 366 CPUF_PRESENT | CPUF_BSP | CPUF_PRIMARY); 367 cpu_intr_init(ci); 368 identifycpu(ci); 369 cpu_init(ci); 370 cpu_set_tss_gates(ci); 371 pmap_cpu_init_late(ci); 372 #if NLAPIC > 0 373 /* 374 * Enable local apic 375 */ 376 lapic_enable(); 377 lapic_set_lvt(); 378 lapic_calibrate_timer(ci); 379 #endif 380 #if NIOAPIC > 0 381 ioapic_bsp_id = caa->cpu_number; 382 #endif 383 x86_errata(); 384 break; 385 386 case CPU_ROLE_AP: 387 /* 388 * report on an AP 389 */ 390 aprint_normal(": (application processor)\n"); 391 392 #if defined(MULTIPROCESSOR) 393 cpu_intr_init(ci); 394 gdt_alloc_cpu(ci); 395 cpu_set_tss_gates(ci); 396 pmap_cpu_init_early(ci); 397 pmap_cpu_init_late(ci); 398 cpu_start_secondary(ci); 399 if (ci->ci_flags & CPUF_PRESENT) { 400 identifycpu(ci); 401 ci->ci_next = cpu_info_list->ci_next; 402 cpu_info_list->ci_next = ci; 403 } 404 #else 405 aprint_normal("%s: not started\n", sc->sc_dev.dv_xname); 406 #endif 407 break; 408 409 default: 410 printf("\n"); 411 panic("unknown processor type??\n"); 412 } 413 cpu_vm_init(ci); 414 415 cpus_attached |= ci->ci_cpumask; 416 417 if (!pmf_device_register(self, cpu_suspend, cpu_resume)) 418 aprint_error_dev(self, "couldn't establish power handler\n"); 419 420 #if defined(MULTIPROCESSOR) 421 if (mp_verbose) { 422 struct lwp *l = ci->ci_data.cpu_idlelwp; 423 424 aprint_verbose( 425 "%s: idle lwp at %p, idle sp at %p\n", 426 sc->sc_dev.dv_xname, l, 427 #ifdef i386 428 (void *)l->l_addr->u_pcb.pcb_esp 429 #else 430 (void *)l->l_addr->u_pcb.pcb_rsp 431 #endif 432 ); 433 } 434 #endif 435 } 436 437 /* 438 * Initialize the processor appropriately. 439 */ 440 441 void 442 cpu_init(struct cpu_info *ci) 443 { 444 /* configure the CPU if needed */ 445 if (ci->cpu_setup != NULL) 446 (*ci->cpu_setup)(ci); 447 448 #ifdef i386 449 /* 450 * On a 486 or above, enable ring 0 write protection. 451 */ 452 if (ci->ci_cpu_class >= CPUCLASS_486) 453 lcr0(rcr0() | CR0_WP); 454 #else 455 lcr0(rcr0() | CR0_WP); 456 #endif 457 458 /* 459 * On a P6 or above, enable global TLB caching if the 460 * hardware supports it. 461 */ 462 if (cpu_feature & CPUID_PGE) 463 lcr4(rcr4() | CR4_PGE); /* enable global TLB caching */ 464 465 /* 466 * If we have FXSAVE/FXRESTOR, use them. 467 */ 468 if (cpu_feature & CPUID_FXSR) { 469 lcr4(rcr4() | CR4_OSFXSR); 470 471 /* 472 * If we have SSE/SSE2, enable XMM exceptions. 473 */ 474 if (cpu_feature & (CPUID_SSE|CPUID_SSE2)) 475 lcr4(rcr4() | CR4_OSXMMEXCPT); 476 } 477 478 #ifdef MTRR 479 /* 480 * On a P6 or above, initialize MTRR's if the hardware supports them. 481 */ 482 if (cpu_feature & CPUID_MTRR) { 483 if ((ci->ci_flags & CPUF_AP) == 0) 484 i686_mtrr_init_first(); 485 mtrr_init_cpu(ci); 486 } 487 488 #ifdef i386 489 if (strcmp((char *)(ci->ci_vendor), "AuthenticAMD") == 0) { 490 /* 491 * Must be a K6-2 Step >= 7 or a K6-III. 492 */ 493 if (CPUID2FAMILY(ci->ci_signature) == 5) { 494 if (CPUID2MODEL(ci->ci_signature) > 8 || 495 (CPUID2MODEL(ci->ci_signature) == 8 && 496 CPUID2STEPPING(ci->ci_signature) >= 7)) { 497 mtrr_funcs = &k6_mtrr_funcs; 498 k6_mtrr_init_first(); 499 mtrr_init_cpu(ci); 500 } 501 } 502 } 503 #endif /* i386 */ 504 #endif /* MTRR */ 505 506 atomic_or_32(&ci->ci_flags, CPUF_RUNNING); 507 atomic_or_32(&cpus_running, ci->ci_cpumask); 508 509 #ifndef MULTIPROCESSOR 510 /* XXX */ 511 x86_patch(); 512 #endif 513 } 514 515 #ifdef MULTIPROCESSOR 516 void 517 cpu_boot_secondary_processors(void) 518 { 519 struct cpu_info *ci; 520 u_long i; 521 522 /* Now that we know the number of CPUs, patch the text segment. */ 523 x86_patch(); 524 525 for (i=0; i < X86_MAXPROCS; i++) { 526 ci = cpu_info[i]; 527 if (ci == NULL) 528 continue; 529 if (ci->ci_data.cpu_idlelwp == NULL) 530 continue; 531 if ((ci->ci_flags & CPUF_PRESENT) == 0) 532 continue; 533 if (ci->ci_flags & (CPUF_BSP|CPUF_SP|CPUF_PRIMARY)) 534 continue; 535 cpu_boot_secondary(ci); 536 } 537 538 x86_mp_online = true; 539 } 540 541 static void 542 cpu_init_idle_lwp(struct cpu_info *ci) 543 { 544 struct lwp *l = ci->ci_data.cpu_idlelwp; 545 struct pcb *pcb = &l->l_addr->u_pcb; 546 547 pcb->pcb_cr0 = rcr0(); 548 } 549 550 void 551 cpu_init_idle_lwps(void) 552 { 553 struct cpu_info *ci; 554 u_long i; 555 556 for (i = 0; i < X86_MAXPROCS; i++) { 557 ci = cpu_info[i]; 558 if (ci == NULL) 559 continue; 560 if (ci->ci_data.cpu_idlelwp == NULL) 561 continue; 562 if ((ci->ci_flags & CPUF_PRESENT) == 0) 563 continue; 564 cpu_init_idle_lwp(ci); 565 } 566 } 567 568 void 569 cpu_start_secondary(struct cpu_info *ci) 570 { 571 int i; 572 extern paddr_t mp_pdirpa; 573 574 mp_pdirpa = pmap_init_tmp_pgtbl(mp_trampoline_paddr); 575 576 atomic_or_32(&ci->ci_flags, CPUF_AP); 577 578 aprint_debug("%s: starting\n", ci->ci_dev->dv_xname); 579 580 ci->ci_curlwp = ci->ci_data.cpu_idlelwp; 581 CPU_STARTUP(ci, mp_trampoline_paddr); 582 583 /* 584 * wait for it to become ready 585 */ 586 for (i = 100000; (!(ci->ci_flags & CPUF_PRESENT)) && i>0;i--) { 587 i8254_delay(10); 588 } 589 if ((ci->ci_flags & CPUF_PRESENT) == 0) { 590 aprint_error("%s: failed to become ready\n", 591 ci->ci_dev->dv_xname); 592 #if defined(MPDEBUG) && defined(DDB) 593 printf("dropping into debugger; continue from here to resume boot\n"); 594 Debugger(); 595 #endif 596 } 597 598 CPU_START_CLEANUP(ci); 599 } 600 601 void 602 cpu_boot_secondary(struct cpu_info *ci) 603 { 604 int i; 605 606 atomic_or_32(&ci->ci_flags, CPUF_GO); 607 for (i = 100000; (!(ci->ci_flags & CPUF_RUNNING)) && i>0;i--) { 608 i8254_delay(10); 609 } 610 if ((ci->ci_flags & CPUF_RUNNING) == 0) { 611 aprint_error("%s: failed to start\n", ci->ci_dev->dv_xname); 612 #if defined(MPDEBUG) && defined(DDB) 613 printf("dropping into debugger; continue from here to resume boot\n"); 614 Debugger(); 615 #endif 616 } 617 } 618 619 /* 620 * The CPU ends up here when its ready to run 621 * This is called from code in mptramp.s; at this point, we are running 622 * in the idle pcb/idle stack of the new CPU. When this function returns, 623 * this processor will enter the idle loop and start looking for work. 624 */ 625 void 626 cpu_hatch(void *v) 627 { 628 struct cpu_info *ci = (struct cpu_info *)v; 629 int s, i; 630 631 #ifdef __x86_64__ 632 cpu_init_msrs(ci, true); 633 #endif 634 cpu_probe_features(ci); 635 cpu_feature &= ci->ci_feature_flags; 636 cpu_feature2 &= ci->ci_feature2_flags; 637 638 KDASSERT((ci->ci_flags & CPUF_PRESENT) == 0); 639 atomic_or_32(&ci->ci_flags, CPUF_PRESENT); 640 while ((ci->ci_flags & CPUF_GO) == 0) { 641 /* Don't use delay, boot CPU may be patching the text. */ 642 for (i = 10000; i != 0; i--) 643 x86_pause(); 644 } 645 646 /* Beacuse the text may have been patched in x86_patch(). */ 647 wbinvd(); 648 x86_flush(); 649 650 KASSERT((ci->ci_flags & CPUF_RUNNING) == 0); 651 652 lcr3(pmap_kernel()->pm_pdirpa); 653 curlwp->l_addr->u_pcb.pcb_cr3 = pmap_kernel()->pm_pdirpa; 654 lcr0(ci->ci_data.cpu_idlelwp->l_addr->u_pcb.pcb_cr0); 655 cpu_init_idt(); 656 gdt_init_cpu(ci); 657 lapic_enable(); 658 lapic_set_lvt(); 659 lapic_initclocks(); 660 661 #ifdef i386 662 npxinit(ci); 663 #else 664 fpuinit(ci); 665 #endif 666 lldt(GSYSSEL(GLDT_SEL, SEL_KPL)); 667 ltr(ci->ci_tss_sel); 668 669 cpu_init(ci); 670 cpu_get_tsc_freq(ci); 671 672 s = splhigh(); 673 #ifdef i386 674 lapic_tpr = 0; 675 #else 676 lcr8(0); 677 #endif 678 x86_enable_intr(); 679 splx(s); 680 x86_errata(); 681 682 aprint_debug("%s: CPU %ld running\n", ci->ci_dev->dv_xname, 683 (long)ci->ci_cpuid); 684 } 685 686 #if defined(DDB) 687 688 #include <ddb/db_output.h> 689 #include <machine/db_machdep.h> 690 691 /* 692 * Dump CPU information from ddb. 693 */ 694 void 695 cpu_debug_dump(void) 696 { 697 struct cpu_info *ci; 698 CPU_INFO_ITERATOR cii; 699 700 db_printf("addr dev id flags ipis curproc fpcurproc\n"); 701 for (CPU_INFO_FOREACH(cii, ci)) { 702 db_printf("%p %s %ld %x %x %10p %10p\n", 703 ci, 704 ci->ci_dev == NULL ? "BOOT" : ci->ci_dev->dv_xname, 705 (long)ci->ci_cpuid, 706 ci->ci_flags, ci->ci_ipis, 707 ci->ci_curlwp, 708 ci->ci_fpcurlwp); 709 } 710 } 711 #endif 712 713 static void 714 cpu_copy_trampoline(void) 715 { 716 /* 717 * Copy boot code. 718 */ 719 extern u_char cpu_spinup_trampoline[]; 720 extern u_char cpu_spinup_trampoline_end[]; 721 722 vaddr_t mp_trampoline_vaddr; 723 724 mp_trampoline_vaddr = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, 725 UVM_KMF_VAONLY); 726 727 pmap_kenter_pa(mp_trampoline_vaddr, mp_trampoline_paddr, 728 VM_PROT_READ | VM_PROT_WRITE); 729 pmap_update(pmap_kernel()); 730 memcpy((void *)mp_trampoline_vaddr, 731 cpu_spinup_trampoline, 732 cpu_spinup_trampoline_end-cpu_spinup_trampoline); 733 734 pmap_kremove(mp_trampoline_vaddr, PAGE_SIZE); 735 pmap_update(pmap_kernel()); 736 uvm_km_free(kernel_map, mp_trampoline_vaddr, PAGE_SIZE, UVM_KMF_VAONLY); 737 } 738 739 #endif 740 741 #ifdef i386 742 static void 743 tss_init(struct i386tss *tss, void *stack, void *func) 744 { 745 memset(tss, 0, sizeof *tss); 746 tss->tss_esp0 = tss->tss_esp = (int)((char *)stack + USPACE - 16); 747 tss->tss_ss0 = GSEL(GDATA_SEL, SEL_KPL); 748 tss->__tss_cs = GSEL(GCODE_SEL, SEL_KPL); 749 tss->tss_fs = GSEL(GCPU_SEL, SEL_KPL); 750 tss->tss_gs = tss->__tss_es = tss->__tss_ds = 751 tss->__tss_ss = GSEL(GDATA_SEL, SEL_KPL); 752 tss->tss_cr3 = pmap_kernel()->pm_pdirpa; 753 tss->tss_esp = (int)((char *)stack + USPACE - 16); 754 tss->tss_ldt = GSEL(GLDT_SEL, SEL_KPL); 755 tss->__tss_eflags = PSL_MBO | PSL_NT; /* XXX not needed? */ 756 tss->__tss_eip = (int)func; 757 } 758 759 /* XXX */ 760 #define IDTVEC(name) __CONCAT(X, name) 761 typedef void (vector)(void); 762 extern vector IDTVEC(tss_trap08); 763 #ifdef DDB 764 extern vector Xintrddbipi; 765 extern int ddb_vec; 766 #endif 767 768 static void 769 cpu_set_tss_gates(struct cpu_info *ci) 770 { 771 struct segment_descriptor sd; 772 773 ci->ci_doubleflt_stack = (char *)uvm_km_alloc(kernel_map, USPACE, 0, 774 UVM_KMF_WIRED); 775 tss_init(&ci->ci_doubleflt_tss, ci->ci_doubleflt_stack, 776 IDTVEC(tss_trap08)); 777 setsegment(&sd, &ci->ci_doubleflt_tss, sizeof(struct i386tss) - 1, 778 SDT_SYS386TSS, SEL_KPL, 0, 0); 779 ci->ci_gdt[GTRAPTSS_SEL].sd = sd; 780 setgate(&idt[8], NULL, 0, SDT_SYSTASKGT, SEL_KPL, 781 GSEL(GTRAPTSS_SEL, SEL_KPL)); 782 783 #if defined(DDB) && defined(MULTIPROCESSOR) 784 /* 785 * Set up separate handler for the DDB IPI, so that it doesn't 786 * stomp on a possibly corrupted stack. 787 * 788 * XXX overwriting the gate set in db_machine_init. 789 * Should rearrange the code so that it's set only once. 790 */ 791 ci->ci_ddbipi_stack = (char *)uvm_km_alloc(kernel_map, USPACE, 0, 792 UVM_KMF_WIRED); 793 tss_init(&ci->ci_ddbipi_tss, ci->ci_ddbipi_stack, Xintrddbipi); 794 795 setsegment(&sd, &ci->ci_ddbipi_tss, sizeof(struct i386tss) - 1, 796 SDT_SYS386TSS, SEL_KPL, 0, 0); 797 ci->ci_gdt[GIPITSS_SEL].sd = sd; 798 799 setgate(&idt[ddb_vec], NULL, 0, SDT_SYSTASKGT, SEL_KPL, 800 GSEL(GIPITSS_SEL, SEL_KPL)); 801 #endif 802 } 803 #else 804 static void 805 cpu_set_tss_gates(struct cpu_info *ci) 806 { 807 808 } 809 #endif /* i386 */ 810 811 int 812 mp_cpu_start(struct cpu_info *ci, paddr_t target) 813 { 814 #if NLAPIC > 0 815 int error; 816 #endif 817 unsigned short dwordptr[2]; 818 819 /* 820 * Bootstrap code must be addressable in real mode 821 * and it must be page aligned. 822 */ 823 KASSERT(target < 0x10000 && target % PAGE_SIZE == 0); 824 825 /* 826 * "The BSP must initialize CMOS shutdown code to 0Ah ..." 827 */ 828 829 outb(IO_RTC, NVRAM_RESET); 830 outb(IO_RTC+1, NVRAM_RESET_JUMP); 831 832 /* 833 * "and the warm reset vector (DWORD based at 40:67) to point 834 * to the AP startup code ..." 835 */ 836 837 dwordptr[0] = 0; 838 dwordptr[1] = target >> 4; 839 840 memcpy((uint8_t *)(cmos_data_mapping + 0x467), dwordptr, 4); 841 842 #if NLAPIC > 0 843 /* 844 * ... prior to executing the following sequence:" 845 */ 846 847 if (ci->ci_flags & CPUF_AP) { 848 if ((error = x86_ipi_init(ci->ci_apicid)) != 0) 849 return error; 850 851 i8254_delay(10000); 852 853 if (cpu_feature & CPUID_APIC) { 854 855 if ((error = x86_ipi(target / PAGE_SIZE, 856 ci->ci_apicid, 857 LAPIC_DLMODE_STARTUP)) != 0) 858 return error; 859 i8254_delay(200); 860 861 if ((error = x86_ipi(target / PAGE_SIZE, 862 ci->ci_apicid, 863 LAPIC_DLMODE_STARTUP)) != 0) 864 return error; 865 i8254_delay(200); 866 } 867 } 868 #endif 869 return 0; 870 } 871 872 void 873 mp_cpu_start_cleanup(struct cpu_info *ci) 874 { 875 /* 876 * Ensure the NVRAM reset byte contains something vaguely sane. 877 */ 878 879 outb(IO_RTC, NVRAM_RESET); 880 outb(IO_RTC+1, NVRAM_RESET_RST); 881 } 882 883 #ifdef __x86_64__ 884 typedef void (vector)(void); 885 extern vector Xsyscall, Xsyscall32; 886 887 void 888 cpu_init_msrs(struct cpu_info *ci, bool full) 889 { 890 wrmsr(MSR_STAR, 891 ((uint64_t)GSEL(GCODE_SEL, SEL_KPL) << 32) | 892 ((uint64_t)LSEL(LSYSRETBASE_SEL, SEL_UPL) << 48)); 893 wrmsr(MSR_LSTAR, (uint64_t)Xsyscall); 894 wrmsr(MSR_CSTAR, (uint64_t)Xsyscall32); 895 wrmsr(MSR_SFMASK, PSL_NT|PSL_T|PSL_I|PSL_C); 896 897 if (full) { 898 wrmsr(MSR_FSBASE, 0); 899 wrmsr(MSR_GSBASE, (u_int64_t)ci); 900 wrmsr(MSR_KERNELGSBASE, 0); 901 } 902 903 if (cpu_feature & CPUID_NOX) 904 wrmsr(MSR_EFER, rdmsr(MSR_EFER) | EFER_NXE); 905 } 906 #endif /* __x86_64__ */ 907 908 void 909 cpu_offline_md(void) 910 { 911 int s; 912 913 s = splhigh(); 914 #ifdef __i386__ 915 npxsave_cpu(true); 916 #else 917 fpusave_cpu(true); 918 #endif 919 splx(s); 920 } 921 922 /* XXX joerg restructure and restart CPUs individually */ 923 static bool 924 cpu_suspend(device_t dv PMF_FN_ARGS) 925 { 926 struct cpu_softc *sc = device_private(dv); 927 struct cpu_info *ci = sc->sc_info; 928 int err; 929 930 if (ci->ci_flags & CPUF_PRIMARY) 931 return true; 932 if (ci->ci_data.cpu_idlelwp == NULL) 933 return true; 934 if ((ci->ci_flags & CPUF_PRESENT) == 0) 935 return true; 936 937 sc->sc_wasonline = !(ci->ci_schedstate.spc_flags & SPCF_OFFLINE); 938 939 if (sc->sc_wasonline) { 940 mutex_enter(&cpu_lock); 941 err = cpu_setonline(ci, false); 942 mutex_exit(&cpu_lock); 943 944 if (err) 945 return false; 946 } 947 948 return true; 949 } 950 951 static bool 952 cpu_resume(device_t dv PMF_FN_ARGS) 953 { 954 struct cpu_softc *sc = device_private(dv); 955 struct cpu_info *ci = sc->sc_info; 956 int err = 0; 957 958 if (ci->ci_flags & CPUF_PRIMARY) 959 return true; 960 if (ci->ci_data.cpu_idlelwp == NULL) 961 return true; 962 if ((ci->ci_flags & CPUF_PRESENT) == 0) 963 return true; 964 965 if (sc->sc_wasonline) { 966 mutex_enter(&cpu_lock); 967 err = cpu_setonline(ci, true); 968 mutex_exit(&cpu_lock); 969 } 970 971 return err == 0; 972 } 973 974 void 975 cpu_get_tsc_freq(struct cpu_info *ci) 976 { 977 uint64_t last_tsc; 978 u_int junk[4]; 979 980 if (ci->ci_feature_flags & CPUID_TSC) { 981 /* Serialize. */ 982 x86_cpuid(0, junk); 983 last_tsc = rdtsc(); 984 i8254_delay(100000); 985 ci->ci_tsc_freq = (rdtsc() - last_tsc) * 10; 986 } 987 } 988
Indexes created Tue Sep 23 14:10:03 GMT 2025