cpu.c revision 1.36
1 /* $NetBSD: cpu.c,v 1.36 2008/04/29 19:19:29 ad Exp $ */ 2 3 /*- 4 * Copyright (c) 2000, 2006, 2007 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Bill Sommerfeld of RedBack Networks Inc, and by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * Copyright (c) 1999 Stefan Grefen 34 * 35 * Redistribution and use in source and binary forms, with or without 36 * modification, are permitted provided that the following conditions 37 * are met: 38 * 1. Redistributions of source code must retain the above copyright 39 * notice, this list of conditions and the following disclaimer. 40 * 2. Redistributions in binary form must reproduce the above copyright 41 * notice, this list of conditions and the following disclaimer in the 42 * documentation and/or other materials provided with the distribution. 43 * 3. All advertising materials mentioning features or use of this software 44 * must display the following acknowledgement: 45 * This product includes software developed by the NetBSD 46 * Foundation, Inc. and its contributors. 47 * 4. Neither the name of The NetBSD Foundation nor the names of its 48 * contributors may be used to endorse or promote products derived 49 * from this software without specific prior written permission. 50 * 51 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND ANY 52 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 53 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 54 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR AND CONTRIBUTORS BE LIABLE 55 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 56 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 57 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 58 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 59 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 60 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 61 * SUCH DAMAGE. 62 */ 63 64 #include <sys/cdefs.h> 65 __KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.36 2008/04/29 19:19:29 ad Exp $"); 66 67 #include "opt_ddb.h" 68 #include "opt_multiprocessor.h" 69 #include "opt_mpbios.h" /* for MPDEBUG */ 70 #include "opt_mtrr.h" 71 72 #include "lapic.h" 73 #include "ioapic.h" 74 75 #include <sys/param.h> 76 #include <sys/proc.h> 77 #include <sys/user.h> 78 #include <sys/systm.h> 79 #include <sys/device.h> 80 #include <sys/malloc.h> 81 #include <sys/cpu.h> 82 #include <sys/atomic.h> 83 #include <sys/reboot.h> 84 85 #include <uvm/uvm_extern.h> 86 87 #include <machine/cpufunc.h> 88 #include <machine/cpuvar.h> 89 #include <machine/pmap.h> 90 #include <machine/vmparam.h> 91 #include <machine/mpbiosvar.h> 92 #include <machine/pcb.h> 93 #include <machine/specialreg.h> 94 #include <machine/segments.h> 95 #include <machine/gdt.h> 96 #include <machine/mtrr.h> 97 #include <machine/pio.h> 98 99 #ifdef i386 100 #include <machine/tlog.h> 101 #endif 102 103 #if NLAPIC > 0 104 #include <machine/apicvar.h> 105 #include <machine/i82489reg.h> 106 #include <machine/i82489var.h> 107 #endif 108 109 #include <dev/ic/mc146818reg.h> 110 #include <i386/isa/nvram.h> 111 #include <dev/isa/isareg.h> 112 113 int cpu_match(device_t, cfdata_t, void *); 114 void cpu_attach(device_t, device_t, void *); 115 116 static bool cpu_suspend(device_t PMF_FN_PROTO); 117 static bool cpu_resume(device_t PMF_FN_PROTO); 118 119 struct cpu_softc { 120 device_t sc_dev; /* device tree glue */ 121 struct cpu_info *sc_info; /* pointer to CPU info */ 122 bool sc_wasonline; 123 }; 124 125 int mp_cpu_start(struct cpu_info *, paddr_t); 126 void mp_cpu_start_cleanup(struct cpu_info *); 127 const struct cpu_functions mp_cpu_funcs = { mp_cpu_start, NULL, 128 mp_cpu_start_cleanup }; 129 130 131 CFATTACH_DECL_NEW(cpu, sizeof(struct cpu_softc), 132 cpu_match, cpu_attach, NULL, NULL); 133 134 /* 135 * Statically-allocated CPU info for the primary CPU (or the only 136 * CPU, on uniprocessors). The CPU info list is initialized to 137 * point at it. 138 */ 139 #ifdef TRAPLOG 140 struct tlog tlog_primary; 141 #endif 142 struct cpu_info cpu_info_primary __aligned(CACHE_LINE_SIZE) = { 143 .ci_dev = 0, 144 .ci_self = &cpu_info_primary, 145 .ci_idepth = -1, 146 .ci_curlwp = &lwp0, 147 #ifdef TRAPLOG 148 .ci_tlog_base = &tlog_primary, 149 #endif /* !TRAPLOG */ 150 }; 151 152 struct cpu_info *cpu_info_list = &cpu_info_primary; 153 154 static void cpu_set_tss_gates(struct cpu_info *); 155 156 #ifdef i386 157 static void tss_init(struct i386tss *, void *, void *); 158 #endif 159 160 #ifdef MULTIPROCESSOR 161 static void cpu_init_idle_lwp(struct cpu_info *); 162 #endif 163 164 uint32_t cpus_attached = 0; 165 uint32_t cpus_running = 0; 166 167 extern char x86_64_doubleflt_stack[]; 168 169 bool x86_mp_online; 170 paddr_t mp_trampoline_paddr = MP_TRAMPOLINE; 171 172 static vaddr_t cmos_data_mapping; 173 174 #ifdef MULTIPROCESSOR 175 /* 176 * Array of CPU info structures. Must be statically-allocated because 177 * curproc, etc. are used early. 178 */ 179 struct cpu_info *cpu_info[X86_MAXPROCS] = { &cpu_info_primary, }; 180 181 void cpu_hatch(void *); 182 static void cpu_boot_secondary(struct cpu_info *ci); 183 static void cpu_start_secondary(struct cpu_info *ci); 184 static void cpu_copy_trampoline(void); 185 186 /* 187 * Runs once per boot once multiprocessor goo has been detected and 188 * the local APIC on the boot processor has been mapped. 189 * 190 * Called from lapic_boot_init() (from mpbios_scan()). 191 */ 192 void 193 cpu_init_first(void) 194 { 195 int cpunum = lapic_cpu_number(); 196 197 if (cpunum != 0) { 198 cpu_info[0] = NULL; 199 cpu_info[cpunum] = &cpu_info_primary; 200 } 201 202 cpu_info_primary.ci_cpuid = cpunum; 203 cpu_copy_trampoline(); 204 205 cmos_data_mapping = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, UVM_KMF_VAONLY); 206 if (cmos_data_mapping == 0) 207 panic("No KVA for page 0"); 208 pmap_kenter_pa(cmos_data_mapping, 0, VM_PROT_READ|VM_PROT_WRITE); 209 pmap_update(pmap_kernel()); 210 } 211 #endif 212 213 int 214 cpu_match(device_t parent, cfdata_t match, void *aux) 215 { 216 217 return 1; 218 } 219 220 static void 221 cpu_vm_init(struct cpu_info *ci) 222 { 223 int ncolors = 2, i; 224 225 for (i = CAI_ICACHE; i <= CAI_L2CACHE; i++) { 226 struct x86_cache_info *cai; 227 int tcolors; 228 229 cai = &ci->ci_cinfo[i]; 230 231 tcolors = atop(cai->cai_totalsize); 232 switch(cai->cai_associativity) { 233 case 0xff: 234 tcolors = 1; /* fully associative */ 235 break; 236 case 0: 237 case 1: 238 break; 239 default: 240 tcolors /= cai->cai_associativity; 241 } 242 ncolors = max(ncolors, tcolors); 243 /* 244 * If the desired number of colors is not a power of 245 * two, it won't be good. Find the greatest power of 246 * two which is an even divisor of the number of colors, 247 * to preserve even coloring of pages. 248 */ 249 if (ncolors & (ncolors - 1) ) { 250 int try, picked = 1; 251 for (try = 1; try < ncolors; try *= 2) { 252 if (ncolors % try == 0) picked = try; 253 } 254 if (picked == 1) { 255 panic("desired number of cache colors %d is " 256 " > 1, but not even!", ncolors); 257 } 258 ncolors = picked; 259 } 260 } 261 262 /* 263 * Knowing the size of the largest cache on this CPU, re-color 264 * our pages. 265 */ 266 if (ncolors <= uvmexp.ncolors) 267 return; 268 aprint_verbose_dev(ci->ci_dev, "%d page colors\n", ncolors); 269 uvm_page_recolor(ncolors); 270 } 271 272 273 void 274 cpu_attach(device_t parent, device_t self, void *aux) 275 { 276 struct cpu_softc *sc = device_private(self); 277 struct cpu_attach_args *caa = aux; 278 struct cpu_info *ci; 279 uintptr_t ptr; 280 int cpunum = caa->cpu_number; 281 282 sc->sc_dev = self; 283 284 /* 285 * If we're an Application Processor, allocate a cpu_info 286 * structure, otherwise use the primary's. 287 */ 288 if (caa->cpu_role == CPU_ROLE_AP) { 289 if ((boothowto & RB_MD1) != 0) { 290 aprint_error(": multiprocessor boot disabled\n"); 291 return; 292 } 293 if (cpunum >= X86_MAXPROCS) { 294 aprint_error(": apic id %d ignored, " 295 "please increase X86_MAXPROCS\n", cpunum); 296 return; 297 } 298 aprint_naive(": Application Processor\n"); 299 ptr = (uintptr_t)malloc(sizeof(*ci) + CACHE_LINE_SIZE - 1, 300 M_DEVBUF, M_WAITOK); 301 ci = (struct cpu_info *)((ptr + CACHE_LINE_SIZE - 1) & 302 ~(CACHE_LINE_SIZE - 1)); 303 memset(ci, 0, sizeof(*ci)); 304 #if defined(MULTIPROCESSOR) 305 if (cpu_info[cpunum] != NULL) { 306 printf("\n"); 307 panic("cpu at apic id %d already attached?", cpunum); 308 } 309 cpu_info[cpunum] = ci; 310 #endif 311 #ifdef TRAPLOG 312 ci->ci_tlog_base = malloc(sizeof(struct tlog), 313 M_DEVBUF, M_WAITOK); 314 #endif 315 } else { 316 aprint_naive(": %s Processor\n", 317 caa->cpu_role == CPU_ROLE_SP ? "Single" : "Boot"); 318 ci = &cpu_info_primary; 319 #if defined(MULTIPROCESSOR) 320 if (cpunum != lapic_cpu_number()) { 321 printf("\n"); 322 panic("%s: running CPU is at apic %d" 323 " instead of at expected %d", 324 device_xname(sc->sc_dev), lapic_cpu_number(), 325 cpunum); 326 } 327 #endif 328 } 329 330 ci->ci_self = ci; 331 sc->sc_info = ci; 332 333 ci->ci_dev = self; 334 ci->ci_apicid = caa->cpu_number; 335 #ifdef MULTIPROCESSOR 336 ci->ci_cpuid = ci->ci_apicid; 337 #else 338 ci->ci_cpuid = 0; /* False for APs, but they're not used anyway */ 339 #endif 340 ci->ci_cpumask = (1 << ci->ci_cpuid); 341 ci->ci_func = caa->cpu_func; 342 343 if (caa->cpu_role == CPU_ROLE_AP) { 344 #ifdef MULTIPROCESSOR 345 int error; 346 347 error = mi_cpu_attach(ci); 348 if (error != 0) { 349 aprint_normal("\n"); 350 aprint_error_dev(sc->sc_dev, 351 "mi_cpu_attach failed with %d\n", error); 352 return; 353 } 354 #endif 355 cpu_init_tss(ci); 356 } else { 357 KASSERT(ci->ci_data.cpu_idlelwp != NULL); 358 } 359 360 pmap_reference(pmap_kernel()); 361 ci->ci_pmap = pmap_kernel(); 362 ci->ci_tlbstate = TLBSTATE_STALE; 363 364 /* further PCB init done later. */ 365 366 switch (caa->cpu_role) { 367 case CPU_ROLE_SP: 368 aprint_normal(": (uniprocessor)\n"); 369 atomic_or_32(&ci->ci_flags, 370 CPUF_PRESENT | CPUF_SP | CPUF_PRIMARY); 371 cpu_intr_init(ci); 372 identifycpu(ci); 373 cpu_init(ci); 374 cpu_set_tss_gates(ci); 375 pmap_cpu_init_late(ci); 376 x86_errata(); 377 break; 378 379 case CPU_ROLE_BP: 380 aprint_normal(": (boot processor)\n"); 381 atomic_or_32(&ci->ci_flags, 382 CPUF_PRESENT | CPUF_BSP | CPUF_PRIMARY); 383 cpu_intr_init(ci); 384 identifycpu(ci); 385 cpu_init(ci); 386 cpu_set_tss_gates(ci); 387 pmap_cpu_init_late(ci); 388 #if NLAPIC > 0 389 /* 390 * Enable local apic 391 */ 392 lapic_enable(); 393 lapic_set_lvt(); 394 lapic_calibrate_timer(ci); 395 #endif 396 x86_errata(); 397 break; 398 399 case CPU_ROLE_AP: 400 /* 401 * report on an AP 402 */ 403 aprint_normal(": (application processor)\n"); 404 405 #if defined(MULTIPROCESSOR) 406 cpu_intr_init(ci); 407 gdt_alloc_cpu(ci); 408 cpu_set_tss_gates(ci); 409 pmap_cpu_init_early(ci); 410 pmap_cpu_init_late(ci); 411 cpu_start_secondary(ci); 412 if (ci->ci_flags & CPUF_PRESENT) { 413 identifycpu(ci); 414 ci->ci_next = cpu_info_list->ci_next; 415 cpu_info_list->ci_next = ci; 416 } 417 #else 418 aprint_normal_dev(sc->sc_dev, "not started\n"); 419 #endif 420 break; 421 422 default: 423 aprint_normal("\n"); 424 panic("unknown processor type??\n"); 425 } 426 cpu_vm_init(ci); 427 428 cpus_attached |= ci->ci_cpumask; 429 430 if (!pmf_device_register(self, cpu_suspend, cpu_resume)) 431 aprint_error_dev(self, "couldn't establish power handler\n"); 432 433 #if defined(MULTIPROCESSOR) 434 if (mp_verbose) { 435 struct lwp *l = ci->ci_data.cpu_idlelwp; 436 437 aprint_verbose_dev(sc->sc_dev, 438 "idle lwp at %p, idle sp at %p\n", 439 l, 440 #ifdef i386 441 (void *)l->l_addr->u_pcb.pcb_esp 442 #else 443 (void *)l->l_addr->u_pcb.pcb_rsp 444 #endif 445 ); 446 } 447 #endif 448 } 449 450 /* 451 * Initialize the processor appropriately. 452 */ 453 454 void 455 cpu_init(struct cpu_info *ci) 456 { 457 /* configure the CPU if needed */ 458 if (ci->cpu_setup != NULL) 459 (*ci->cpu_setup)(ci); 460 461 #ifdef i386 462 /* 463 * On a 486 or above, enable ring 0 write protection. 464 */ 465 if (ci->ci_cpu_class >= CPUCLASS_486) 466 lcr0(rcr0() | CR0_WP); 467 #else 468 lcr0(rcr0() | CR0_WP); 469 #endif 470 471 /* 472 * On a P6 or above, enable global TLB caching if the 473 * hardware supports it. 474 */ 475 if (cpu_feature & CPUID_PGE) 476 lcr4(rcr4() | CR4_PGE); /* enable global TLB caching */ 477 478 /* 479 * If we have FXSAVE/FXRESTOR, use them. 480 */ 481 if (cpu_feature & CPUID_FXSR) { 482 lcr4(rcr4() | CR4_OSFXSR); 483 484 /* 485 * If we have SSE/SSE2, enable XMM exceptions. 486 */ 487 if (cpu_feature & (CPUID_SSE|CPUID_SSE2)) 488 lcr4(rcr4() | CR4_OSXMMEXCPT); 489 } 490 491 #ifdef MTRR 492 /* 493 * On a P6 or above, initialize MTRR's if the hardware supports them. 494 */ 495 if (cpu_feature & CPUID_MTRR) { 496 if ((ci->ci_flags & CPUF_AP) == 0) 497 i686_mtrr_init_first(); 498 mtrr_init_cpu(ci); 499 } 500 501 #ifdef i386 502 if (strcmp((char *)(ci->ci_vendor), "AuthenticAMD") == 0) { 503 /* 504 * Must be a K6-2 Step >= 7 or a K6-III. 505 */ 506 if (CPUID2FAMILY(ci->ci_signature) == 5) { 507 if (CPUID2MODEL(ci->ci_signature) > 8 || 508 (CPUID2MODEL(ci->ci_signature) == 8 && 509 CPUID2STEPPING(ci->ci_signature) >= 7)) { 510 mtrr_funcs = &k6_mtrr_funcs; 511 k6_mtrr_init_first(); 512 mtrr_init_cpu(ci); 513 } 514 } 515 } 516 #endif /* i386 */ 517 #endif /* MTRR */ 518 519 atomic_or_32(&ci->ci_flags, CPUF_RUNNING); 520 atomic_or_32(&cpus_running, ci->ci_cpumask); 521 522 #ifndef MULTIPROCESSOR 523 /* XXX */ 524 x86_patch(); 525 #endif 526 } 527 528 #ifdef MULTIPROCESSOR 529 void 530 cpu_boot_secondary_processors(void) 531 { 532 struct cpu_info *ci; 533 u_long i; 534 535 /* Now that we know the number of CPUs, patch the text segment. */ 536 x86_patch(); 537 538 for (i=0; i < X86_MAXPROCS; i++) { 539 ci = cpu_info[i]; 540 if (ci == NULL) 541 continue; 542 if (ci->ci_data.cpu_idlelwp == NULL) 543 continue; 544 if ((ci->ci_flags & CPUF_PRESENT) == 0) 545 continue; 546 if (ci->ci_flags & (CPUF_BSP|CPUF_SP|CPUF_PRIMARY)) 547 continue; 548 cpu_boot_secondary(ci); 549 } 550 551 x86_mp_online = true; 552 } 553 554 static void 555 cpu_init_idle_lwp(struct cpu_info *ci) 556 { 557 struct lwp *l = ci->ci_data.cpu_idlelwp; 558 struct pcb *pcb = &l->l_addr->u_pcb; 559 560 pcb->pcb_cr0 = rcr0(); 561 } 562 563 void 564 cpu_init_idle_lwps(void) 565 { 566 struct cpu_info *ci; 567 u_long i; 568 569 for (i = 0; i < X86_MAXPROCS; i++) { 570 ci = cpu_info[i]; 571 if (ci == NULL) 572 continue; 573 if (ci->ci_data.cpu_idlelwp == NULL) 574 continue; 575 if ((ci->ci_flags & CPUF_PRESENT) == 0) 576 continue; 577 cpu_init_idle_lwp(ci); 578 } 579 } 580 581 void 582 cpu_start_secondary(struct cpu_info *ci) 583 { 584 int i; 585 extern paddr_t mp_pdirpa; 586 587 mp_pdirpa = pmap_init_tmp_pgtbl(mp_trampoline_paddr); 588 589 atomic_or_32(&ci->ci_flags, CPUF_AP); 590 591 aprint_debug_dev(ci->ci_dev, "starting\n"); 592 593 ci->ci_curlwp = ci->ci_data.cpu_idlelwp; 594 if (CPU_STARTUP(ci, mp_trampoline_paddr) != 0) 595 return; 596 597 /* 598 * wait for it to become ready 599 */ 600 for (i = 100000; (!(ci->ci_flags & CPUF_PRESENT)) && i > 0; i--) { 601 #ifdef MPDEBUG 602 extern int cpu_trace[3]; 603 static int otrace[3]; 604 if (memcmp(otrace, cpu_trace, sizeof(otrace)) != 0) { 605 aprint_debug_dev(ci->ci_dev, "trace %02x %02x %02x\n", 606 cpu_trace[0], cpu_trace[1], cpu_trace[2]); 607 memcpy(otrace, cpu_trace, sizeof(otrace)); 608 } 609 #endif 610 i8254_delay(10); 611 } 612 if ((ci->ci_flags & CPUF_PRESENT) == 0) { 613 aprint_error_dev(ci->ci_dev, "failed to become ready\n"); 614 #if defined(MPDEBUG) && defined(DDB) 615 printf("dropping into debugger; continue from here to resume boot\n"); 616 Debugger(); 617 #endif 618 } 619 620 CPU_START_CLEANUP(ci); 621 } 622 623 void 624 cpu_boot_secondary(struct cpu_info *ci) 625 { 626 int i; 627 628 atomic_or_32(&ci->ci_flags, CPUF_GO); 629 for (i = 100000; (!(ci->ci_flags & CPUF_RUNNING)) && i > 0; i--) { 630 i8254_delay(10); 631 } 632 if ((ci->ci_flags & CPUF_RUNNING) == 0) { 633 aprint_error_dev(ci->ci_dev, "failed to start\n"); 634 #if defined(MPDEBUG) && defined(DDB) 635 printf("dropping into debugger; continue from here to resume boot\n"); 636 Debugger(); 637 #endif 638 } 639 } 640 641 /* 642 * The CPU ends up here when its ready to run 643 * This is called from code in mptramp.s; at this point, we are running 644 * in the idle pcb/idle stack of the new CPU. When this function returns, 645 * this processor will enter the idle loop and start looking for work. 646 */ 647 void 648 cpu_hatch(void *v) 649 { 650 struct cpu_info *ci = (struct cpu_info *)v; 651 int s, i; 652 653 #ifdef __x86_64__ 654 cpu_init_msrs(ci, true); 655 #endif 656 cpu_probe_features(ci); 657 cpu_feature &= ci->ci_feature_flags; 658 cpu_feature2 &= ci->ci_feature2_flags; 659 660 KDASSERT((ci->ci_flags & CPUF_PRESENT) == 0); 661 atomic_or_32(&ci->ci_flags, CPUF_PRESENT); 662 while ((ci->ci_flags & CPUF_GO) == 0) { 663 /* Don't use delay, boot CPU may be patching the text. */ 664 for (i = 10000; i != 0; i--) 665 x86_pause(); 666 } 667 668 /* Because the text may have been patched in x86_patch(). */ 669 wbinvd(); 670 x86_flush(); 671 672 KASSERT((ci->ci_flags & CPUF_RUNNING) == 0); 673 674 lcr3(pmap_kernel()->pm_pdirpa); 675 curlwp->l_addr->u_pcb.pcb_cr3 = pmap_kernel()->pm_pdirpa; 676 lcr0(ci->ci_data.cpu_idlelwp->l_addr->u_pcb.pcb_cr0); 677 cpu_init_idt(); 678 gdt_init_cpu(ci); 679 lapic_enable(); 680 lapic_set_lvt(); 681 lapic_initclocks(); 682 683 #ifdef i386 684 npxinit(ci); 685 #else 686 fpuinit(ci); 687 #endif 688 lldt(GSYSSEL(GLDT_SEL, SEL_KPL)); 689 ltr(ci->ci_tss_sel); 690 691 cpu_init(ci); 692 cpu_get_tsc_freq(ci); 693 694 s = splhigh(); 695 #ifdef i386 696 lapic_tpr = 0; 697 #else 698 lcr8(0); 699 #endif 700 x86_enable_intr(); 701 splx(s); 702 x86_errata(); 703 704 aprint_debug_dev(ci->ci_dev, "CPU %ld running\n", 705 (long)ci->ci_cpuid); 706 } 707 708 #if defined(DDB) 709 710 #include <ddb/db_output.h> 711 #include <machine/db_machdep.h> 712 713 /* 714 * Dump CPU information from ddb. 715 */ 716 void 717 cpu_debug_dump(void) 718 { 719 struct cpu_info *ci; 720 CPU_INFO_ITERATOR cii; 721 722 db_printf("addr dev id flags ipis curlwp fpcurlwp\n"); 723 for (CPU_INFO_FOREACH(cii, ci)) { 724 db_printf("%p %s %ld %x %x %10p %10p\n", 725 ci, 726 ci->ci_dev == NULL ? "BOOT" : device_xname(ci->ci_dev), 727 (long)ci->ci_cpuid, 728 ci->ci_flags, ci->ci_ipis, 729 ci->ci_curlwp, 730 ci->ci_fpcurlwp); 731 } 732 } 733 #endif 734 735 static void 736 cpu_copy_trampoline(void) 737 { 738 /* 739 * Copy boot code. 740 */ 741 extern u_char cpu_spinup_trampoline[]; 742 extern u_char cpu_spinup_trampoline_end[]; 743 744 vaddr_t mp_trampoline_vaddr; 745 746 mp_trampoline_vaddr = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, 747 UVM_KMF_VAONLY); 748 749 pmap_kenter_pa(mp_trampoline_vaddr, mp_trampoline_paddr, 750 VM_PROT_READ | VM_PROT_WRITE); 751 pmap_update(pmap_kernel()); 752 memcpy((void *)mp_trampoline_vaddr, 753 cpu_spinup_trampoline, 754 cpu_spinup_trampoline_end - cpu_spinup_trampoline); 755 756 pmap_kremove(mp_trampoline_vaddr, PAGE_SIZE); 757 pmap_update(pmap_kernel()); 758 uvm_km_free(kernel_map, mp_trampoline_vaddr, PAGE_SIZE, UVM_KMF_VAONLY); 759 } 760 761 #endif 762 763 #ifdef i386 764 static void 765 tss_init(struct i386tss *tss, void *stack, void *func) 766 { 767 memset(tss, 0, sizeof *tss); 768 tss->tss_esp0 = tss->tss_esp = (int)((char *)stack + USPACE - 16); 769 tss->tss_ss0 = GSEL(GDATA_SEL, SEL_KPL); 770 tss->__tss_cs = GSEL(GCODE_SEL, SEL_KPL); 771 tss->tss_fs = GSEL(GCPU_SEL, SEL_KPL); 772 tss->tss_gs = tss->__tss_es = tss->__tss_ds = 773 tss->__tss_ss = GSEL(GDATA_SEL, SEL_KPL); 774 tss->tss_cr3 = pmap_kernel()->pm_pdirpa; 775 tss->tss_esp = (int)((char *)stack + USPACE - 16); 776 tss->tss_ldt = GSEL(GLDT_SEL, SEL_KPL); 777 tss->__tss_eflags = PSL_MBO | PSL_NT; /* XXX not needed? */ 778 tss->__tss_eip = (int)func; 779 } 780 781 /* XXX */ 782 #define IDTVEC(name) __CONCAT(X, name) 783 typedef void (vector)(void); 784 extern vector IDTVEC(tss_trap08); 785 #ifdef DDB 786 extern vector Xintrddbipi; 787 extern int ddb_vec; 788 #endif 789 790 static void 791 cpu_set_tss_gates(struct cpu_info *ci) 792 { 793 struct segment_descriptor sd; 794 795 ci->ci_doubleflt_stack = (char *)uvm_km_alloc(kernel_map, USPACE, 0, 796 UVM_KMF_WIRED); 797 tss_init(&ci->ci_doubleflt_tss, ci->ci_doubleflt_stack, 798 IDTVEC(tss_trap08)); 799 setsegment(&sd, &ci->ci_doubleflt_tss, sizeof(struct i386tss) - 1, 800 SDT_SYS386TSS, SEL_KPL, 0, 0); 801 ci->ci_gdt[GTRAPTSS_SEL].sd = sd; 802 setgate(&idt[8], NULL, 0, SDT_SYSTASKGT, SEL_KPL, 803 GSEL(GTRAPTSS_SEL, SEL_KPL)); 804 805 #if defined(DDB) && defined(MULTIPROCESSOR) 806 /* 807 * Set up separate handler for the DDB IPI, so that it doesn't 808 * stomp on a possibly corrupted stack. 809 * 810 * XXX overwriting the gate set in db_machine_init. 811 * Should rearrange the code so that it's set only once. 812 */ 813 ci->ci_ddbipi_stack = (char *)uvm_km_alloc(kernel_map, USPACE, 0, 814 UVM_KMF_WIRED); 815 tss_init(&ci->ci_ddbipi_tss, ci->ci_ddbipi_stack, Xintrddbipi); 816 817 setsegment(&sd, &ci->ci_ddbipi_tss, sizeof(struct i386tss) - 1, 818 SDT_SYS386TSS, SEL_KPL, 0, 0); 819 ci->ci_gdt[GIPITSS_SEL].sd = sd; 820 821 setgate(&idt[ddb_vec], NULL, 0, SDT_SYSTASKGT, SEL_KPL, 822 GSEL(GIPITSS_SEL, SEL_KPL)); 823 #endif 824 } 825 #else 826 static void 827 cpu_set_tss_gates(struct cpu_info *ci) 828 { 829 830 } 831 #endif /* i386 */ 832 833 int 834 mp_cpu_start(struct cpu_info *ci, paddr_t target) 835 { 836 #if NLAPIC > 0 837 int error; 838 #endif 839 unsigned short dwordptr[2]; 840 841 /* 842 * Bootstrap code must be addressable in real mode 843 * and it must be page aligned. 844 */ 845 KASSERT(target < 0x10000 && target % PAGE_SIZE == 0); 846 847 /* 848 * "The BSP must initialize CMOS shutdown code to 0Ah ..." 849 */ 850 851 outb(IO_RTC, NVRAM_RESET); 852 outb(IO_RTC+1, NVRAM_RESET_JUMP); 853 854 /* 855 * "and the warm reset vector (DWORD based at 40:67) to point 856 * to the AP startup code ..." 857 */ 858 859 dwordptr[0] = 0; 860 dwordptr[1] = target >> 4; 861 862 memcpy((uint8_t *)cmos_data_mapping + 0x467, dwordptr, 4); 863 864 #if NLAPIC > 0 865 if ((cpu_feature & CPUID_APIC) == 0) { 866 aprint_error("mp_cpu_start: CPU does not have APIC\n"); 867 return ENODEV; 868 } 869 870 /* 871 * ... prior to executing the following sequence:" 872 */ 873 874 if (ci->ci_flags & CPUF_AP) { 875 error = x86_ipi_init(ci->ci_apicid); 876 if (error != 0) { 877 aprint_error_dev(ci->ci_dev, "%s: IPI not taken (1)\n", 878 __func__); 879 return error; 880 } 881 882 i8254_delay(10000); 883 884 error = x86_ipi(target / PAGE_SIZE, ci->ci_apicid, 885 LAPIC_DLMODE_STARTUP); 886 if (error != 0) { 887 aprint_error_dev(ci->ci_dev, "%s: IPI not taken (2)\n", 888 __func__); 889 return error; 890 } 891 i8254_delay(200); 892 893 error = x86_ipi(target / PAGE_SIZE, ci->ci_apicid, 894 LAPIC_DLMODE_STARTUP); 895 if (error != 0) { 896 aprint_error_dev(ci->ci_dev, "%s: IPI not taken (3)\n", 897 __func__); 898 return error; 899 } 900 i8254_delay(200); 901 } 902 #endif 903 return 0; 904 } 905 906 void 907 mp_cpu_start_cleanup(struct cpu_info *ci) 908 { 909 /* 910 * Ensure the NVRAM reset byte contains something vaguely sane. 911 */ 912 913 outb(IO_RTC, NVRAM_RESET); 914 outb(IO_RTC+1, NVRAM_RESET_RST); 915 } 916 917 #ifdef __x86_64__ 918 typedef void (vector)(void); 919 extern vector Xsyscall, Xsyscall32; 920 921 void 922 cpu_init_msrs(struct cpu_info *ci, bool full) 923 { 924 wrmsr(MSR_STAR, 925 ((uint64_t)GSEL(GCODE_SEL, SEL_KPL) << 32) | 926 ((uint64_t)LSEL(LSYSRETBASE_SEL, SEL_UPL) << 48)); 927 wrmsr(MSR_LSTAR, (uint64_t)Xsyscall); 928 wrmsr(MSR_CSTAR, (uint64_t)Xsyscall32); 929 wrmsr(MSR_SFMASK, PSL_NT|PSL_T|PSL_I|PSL_C); 930 931 if (full) { 932 wrmsr(MSR_FSBASE, 0); 933 wrmsr(MSR_GSBASE, (uint64_t)ci); 934 wrmsr(MSR_KERNELGSBASE, 0); 935 } 936 937 if (cpu_feature & CPUID_NOX) 938 wrmsr(MSR_EFER, rdmsr(MSR_EFER) | EFER_NXE); 939 } 940 #endif /* __x86_64__ */ 941 942 void 943 cpu_offline_md(void) 944 { 945 int s; 946 947 s = splhigh(); 948 #ifdef __i386__ 949 npxsave_cpu(true); 950 #else 951 fpusave_cpu(true); 952 #endif 953 splx(s); 954 } 955 956 /* XXX joerg restructure and restart CPUs individually */ 957 static bool 958 cpu_suspend(device_t dv PMF_FN_ARGS) 959 { 960 struct cpu_softc *sc = device_private(dv); 961 struct cpu_info *ci = sc->sc_info; 962 int err; 963 964 if (ci->ci_flags & CPUF_PRIMARY) 965 return true; 966 if (ci->ci_data.cpu_idlelwp == NULL) 967 return true; 968 if ((ci->ci_flags & CPUF_PRESENT) == 0) 969 return true; 970 971 sc->sc_wasonline = !(ci->ci_schedstate.spc_flags & SPCF_OFFLINE); 972 973 if (sc->sc_wasonline) { 974 mutex_enter(&cpu_lock); 975 err = cpu_setonline(ci, false); 976 mutex_exit(&cpu_lock); 977 978 if (err) 979 return false; 980 } 981 982 return true; 983 } 984 985 static bool 986 cpu_resume(device_t dv PMF_FN_ARGS) 987 { 988 struct cpu_softc *sc = device_private(dv); 989 struct cpu_info *ci = sc->sc_info; 990 int err = 0; 991 992 if (ci->ci_flags & CPUF_PRIMARY) 993 return true; 994 if (ci->ci_data.cpu_idlelwp == NULL) 995 return true; 996 if ((ci->ci_flags & CPUF_PRESENT) == 0) 997 return true; 998 999 if (sc->sc_wasonline) { 1000 mutex_enter(&cpu_lock); 1001 err = cpu_setonline(ci, true); 1002 mutex_exit(&cpu_lock); 1003 } 1004 1005 return err == 0; 1006 } 1007 1008 void 1009 cpu_get_tsc_freq(struct cpu_info *ci) 1010 { 1011 uint64_t last_tsc; 1012 u_int junk[4]; 1013 1014 if (ci->ci_feature_flags & CPUID_TSC) { 1015 /* Serialize. */ 1016 x86_cpuid(0, junk); 1017 last_tsc = rdtsc(); 1018 i8254_delay(100000); 1019 ci->ci_tsc_freq = (rdtsc() - last_tsc) * 10; 1020 } 1021 } 1022
Indexes created Tue Sep 23 21:09:50 GMT 2025