cpu.c revision 1.27
1 /* $NetBSD: cpu.c,v 1.27 2008/04/16 16:06:51 cegger Exp $ */ 2 3 /*- 4 * Copyright (c) 2000, 2006, 2007 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Bill Sommerfeld of RedBack Networks Inc, and by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 3. All advertising materials mentioning features or use of this software 19 * must display the following acknowledgement: 20 * This product includes software developed by the NetBSD 21 * Foundation, Inc. and its contributors. 22 * 4. Neither the name of The NetBSD Foundation nor the names of its 23 * contributors may be used to endorse or promote products derived 24 * from this software without specific prior written permission. 25 * 26 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 27 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 28 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 29 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 30 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 31 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 32 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 33 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 34 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 35 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 36 * POSSIBILITY OF SUCH DAMAGE. 37 */ 38 39 /* 40 * Copyright (c) 1999 Stefan Grefen 41 * 42 * Redistribution and use in source and binary forms, with or without 43 * modification, are permitted provided that the following conditions 44 * are met: 45 * 1. Redistributions of source code must retain the above copyright 46 * notice, this list of conditions and the following disclaimer. 47 * 2. Redistributions in binary form must reproduce the above copyright 48 * notice, this list of conditions and the following disclaimer in the 49 * documentation and/or other materials provided with the distribution. 50 * 3. All advertising materials mentioning features or use of this software 51 * must display the following acknowledgement: 52 * This product includes software developed by the NetBSD 53 * Foundation, Inc. and its contributors. 54 * 4. Neither the name of The NetBSD Foundation nor the names of its 55 * contributors may be used to endorse or promote products derived 56 * from this software without specific prior written permission. 57 * 58 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND ANY 59 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 60 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 61 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR AND CONTRIBUTORS BE LIABLE 62 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 63 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 64 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 65 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 66 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 67 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 68 * SUCH DAMAGE. 69 */ 70 71 #include <sys/cdefs.h> 72 __KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.27 2008/04/16 16:06:51 cegger Exp $"); 73 74 #include "opt_ddb.h" 75 #include "opt_multiprocessor.h" 76 #include "opt_mpbios.h" /* for MPDEBUG */ 77 #include "opt_mtrr.h" 78 79 #include "lapic.h" 80 #include "ioapic.h" 81 82 #include <sys/param.h> 83 #include <sys/proc.h> 84 #include <sys/user.h> 85 #include <sys/systm.h> 86 #include <sys/device.h> 87 #include <sys/malloc.h> 88 #include <sys/cpu.h> 89 #include <sys/atomic.h> 90 91 #include <uvm/uvm_extern.h> 92 93 #include <machine/cpufunc.h> 94 #include <machine/cpuvar.h> 95 #include <machine/pmap.h> 96 #include <machine/vmparam.h> 97 #include <machine/mpbiosvar.h> 98 #include <machine/pcb.h> 99 #include <machine/specialreg.h> 100 #include <machine/segments.h> 101 #include <machine/gdt.h> 102 #include <machine/mtrr.h> 103 #include <machine/pio.h> 104 105 #ifdef i386 106 #include <machine/tlog.h> 107 #endif 108 109 #if NLAPIC > 0 110 #include <machine/apicvar.h> 111 #include <machine/i82489reg.h> 112 #include <machine/i82489var.h> 113 #endif 114 115 #if NIOAPIC > 0 116 #include <machine/i82093var.h> 117 #endif 118 119 #include <dev/ic/mc146818reg.h> 120 #include <i386/isa/nvram.h> 121 #include <dev/isa/isareg.h> 122 123 int cpu_match(device_t, cfdata_t, void *); 124 void cpu_attach(device_t, device_t, void *); 125 126 static bool cpu_suspend(device_t PMF_FN_PROTO); 127 static bool cpu_resume(device_t PMF_FN_PROTO); 128 129 struct cpu_softc { 130 device_t sc_dev; /* device tree glue */ 131 struct cpu_info *sc_info; /* pointer to CPU info */ 132 bool sc_wasonline; 133 }; 134 135 int mp_cpu_start(struct cpu_info *, paddr_t); 136 void mp_cpu_start_cleanup(struct cpu_info *); 137 const struct cpu_functions mp_cpu_funcs = { mp_cpu_start, NULL, 138 mp_cpu_start_cleanup }; 139 140 141 CFATTACH_DECL_NEW(cpu, sizeof(struct cpu_softc), 142 cpu_match, cpu_attach, NULL, NULL); 143 144 /* 145 * Statically-allocated CPU info for the primary CPU (or the only 146 * CPU, on uniprocessors). The CPU info list is initialized to 147 * point at it. 148 */ 149 #ifdef TRAPLOG 150 struct tlog tlog_primary; 151 #endif 152 struct cpu_info cpu_info_primary __aligned(CACHE_LINE_SIZE) = { 153 .ci_dev = 0, 154 .ci_self = &cpu_info_primary, 155 .ci_idepth = -1, 156 .ci_curlwp = &lwp0, 157 #ifdef TRAPLOG 158 .ci_tlog_base = &tlog_primary, 159 #endif /* !TRAPLOG */ 160 }; 161 162 struct cpu_info *cpu_info_list = &cpu_info_primary; 163 164 static void cpu_set_tss_gates(struct cpu_info *); 165 166 #ifdef i386 167 static void tss_init(struct i386tss *, void *, void *); 168 #endif 169 170 #ifdef MULTIPROCESSOR 171 static void cpu_init_idle_lwp(struct cpu_info *); 172 #endif 173 174 uint32_t cpus_attached = 0; 175 uint32_t cpus_running = 0; 176 177 extern char x86_64_doubleflt_stack[]; 178 179 bool x86_mp_online; 180 paddr_t mp_trampoline_paddr = MP_TRAMPOLINE; 181 182 static vaddr_t cmos_data_mapping; 183 184 #ifdef MULTIPROCESSOR 185 /* 186 * Array of CPU info structures. Must be statically-allocated because 187 * curproc, etc. are used early. 188 */ 189 struct cpu_info *cpu_info[X86_MAXPROCS] = { &cpu_info_primary, }; 190 191 void cpu_hatch(void *); 192 static void cpu_boot_secondary(struct cpu_info *ci); 193 static void cpu_start_secondary(struct cpu_info *ci); 194 static void cpu_copy_trampoline(void); 195 196 /* 197 * Runs once per boot once multiprocessor goo has been detected and 198 * the local APIC on the boot processor has been mapped. 199 * 200 * Called from lapic_boot_init() (from mpbios_scan()). 201 */ 202 void 203 cpu_init_first(void) 204 { 205 int cpunum = lapic_cpu_number(); 206 207 if (cpunum != 0) { 208 cpu_info[0] = NULL; 209 cpu_info[cpunum] = &cpu_info_primary; 210 } 211 212 cpu_info_primary.ci_cpuid = cpunum; 213 cpu_copy_trampoline(); 214 215 cmos_data_mapping = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, UVM_KMF_VAONLY); 216 if (cmos_data_mapping == 0) 217 panic("No KVA for page 0"); 218 pmap_kenter_pa(cmos_data_mapping, 0, VM_PROT_READ|VM_PROT_WRITE); 219 pmap_update(pmap_kernel()); 220 } 221 #endif 222 223 int 224 cpu_match(device_t parent, cfdata_t match, void *aux) 225 { 226 227 return 1; 228 } 229 230 static void 231 cpu_vm_init(struct cpu_info *ci) 232 { 233 int ncolors = 2, i; 234 235 for (i = CAI_ICACHE; i <= CAI_L2CACHE; i++) { 236 struct x86_cache_info *cai; 237 int tcolors; 238 239 cai = &ci->ci_cinfo[i]; 240 241 tcolors = atop(cai->cai_totalsize); 242 switch(cai->cai_associativity) { 243 case 0xff: 244 tcolors = 1; /* fully associative */ 245 break; 246 case 0: 247 case 1: 248 break; 249 default: 250 tcolors /= cai->cai_associativity; 251 } 252 ncolors = max(ncolors, tcolors); 253 } 254 255 /* 256 * Knowing the size of the largest cache on this CPU, re-color 257 * our pages. 258 */ 259 if (ncolors <= uvmexp.ncolors) 260 return; 261 aprint_verbose_dev(ci->ci_dev, "%d page colors\n", ncolors); 262 uvm_page_recolor(ncolors); 263 } 264 265 266 void 267 cpu_attach(device_t parent, device_t self, void *aux) 268 { 269 struct cpu_softc *sc = device_private(self); 270 struct cpu_attach_args *caa = aux; 271 struct cpu_info *ci; 272 uintptr_t ptr; 273 #if defined(MULTIPROCESSOR) 274 int cpunum = caa->cpu_number; 275 #endif 276 277 sc->sc_dev = self; 278 279 /* 280 * If we're an Application Processor, allocate a cpu_info 281 * structure, otherwise use the primary's. 282 */ 283 if (caa->cpu_role == CPU_ROLE_AP) { 284 aprint_naive(": Application Processor\n"); 285 ptr = (uintptr_t)malloc(sizeof(*ci) + CACHE_LINE_SIZE - 1, 286 M_DEVBUF, M_WAITOK); 287 ci = (struct cpu_info *)((ptr + CACHE_LINE_SIZE - 1) & 288 ~(CACHE_LINE_SIZE - 1)); 289 memset(ci, 0, sizeof(*ci)); 290 #if defined(MULTIPROCESSOR) 291 if (cpu_info[cpunum] != NULL) { 292 printf("\n"); 293 panic("cpu at apic id %d already attached?", cpunum); 294 } 295 cpu_info[cpunum] = ci; 296 #endif 297 #ifdef TRAPLOG 298 ci->ci_tlog_base = malloc(sizeof(struct tlog), 299 M_DEVBUF, M_WAITOK); 300 #endif 301 } else { 302 aprint_naive(": %s Processor\n", 303 caa->cpu_role == CPU_ROLE_SP ? "Single" : "Boot"); 304 ci = &cpu_info_primary; 305 #if defined(MULTIPROCESSOR) 306 if (cpunum != lapic_cpu_number()) { 307 printf("\n"); 308 panic("%s: running CPU is at apic %d" 309 " instead of at expected %d", 310 device_xname(sc->sc_dev), lapic_cpu_number(), 311 cpunum); 312 } 313 #endif 314 } 315 316 ci->ci_self = ci; 317 sc->sc_info = ci; 318 319 ci->ci_dev = self; 320 ci->ci_apicid = caa->cpu_number; 321 #ifdef MULTIPROCESSOR 322 ci->ci_cpuid = ci->ci_apicid; 323 #else 324 ci->ci_cpuid = 0; /* False for APs, but they're not used anyway */ 325 #endif 326 ci->ci_cpumask = (1 << ci->ci_cpuid); 327 ci->ci_func = caa->cpu_func; 328 329 if (caa->cpu_role == CPU_ROLE_AP) { 330 #ifdef MULTIPROCESSOR 331 int error; 332 333 error = mi_cpu_attach(ci); 334 if (error != 0) { 335 aprint_normal("\n"); 336 aprint_error("%s: mi_cpu_attach failed with %d\n", 337 device_xname(sc->sc_dev), error); 338 return; 339 } 340 #endif 341 cpu_init_tss(ci); 342 } else { 343 KASSERT(ci->ci_data.cpu_idlelwp != NULL); 344 } 345 346 pmap_reference(pmap_kernel()); 347 ci->ci_pmap = pmap_kernel(); 348 ci->ci_tlbstate = TLBSTATE_STALE; 349 350 /* further PCB init done later. */ 351 352 switch (caa->cpu_role) { 353 case CPU_ROLE_SP: 354 aprint_normal(": (uniprocessor)\n"); 355 atomic_or_32(&ci->ci_flags, 356 CPUF_PRESENT | CPUF_SP | CPUF_PRIMARY); 357 cpu_intr_init(ci); 358 identifycpu(ci); 359 cpu_init(ci); 360 cpu_set_tss_gates(ci); 361 pmap_cpu_init_late(ci); 362 x86_errata(); 363 break; 364 365 case CPU_ROLE_BP: 366 aprint_normal(": (boot processor)\n"); 367 atomic_or_32(&ci->ci_flags, 368 CPUF_PRESENT | CPUF_BSP | CPUF_PRIMARY); 369 cpu_intr_init(ci); 370 identifycpu(ci); 371 cpu_init(ci); 372 cpu_set_tss_gates(ci); 373 pmap_cpu_init_late(ci); 374 #if NLAPIC > 0 375 /* 376 * Enable local apic 377 */ 378 lapic_enable(); 379 lapic_set_lvt(); 380 lapic_calibrate_timer(ci); 381 #endif 382 #if NIOAPIC > 0 383 ioapic_bsp_id = caa->cpu_number; 384 #endif 385 x86_errata(); 386 break; 387 388 case CPU_ROLE_AP: 389 /* 390 * report on an AP 391 */ 392 aprint_normal(": (application processor)\n"); 393 394 #if defined(MULTIPROCESSOR) 395 cpu_intr_init(ci); 396 gdt_alloc_cpu(ci); 397 cpu_set_tss_gates(ci); 398 pmap_cpu_init_early(ci); 399 pmap_cpu_init_late(ci); 400 cpu_start_secondary(ci); 401 if (ci->ci_flags & CPUF_PRESENT) { 402 identifycpu(ci); 403 ci->ci_next = cpu_info_list->ci_next; 404 cpu_info_list->ci_next = ci; 405 } 406 #else 407 aprint_normal("%s: not started\n", device_xname(sc->sc_dev)); 408 #endif 409 break; 410 411 default: 412 printf("\n"); 413 panic("unknown processor type??\n"); 414 } 415 cpu_vm_init(ci); 416 417 cpus_attached |= ci->ci_cpumask; 418 419 if (!pmf_device_register(self, cpu_suspend, cpu_resume)) 420 aprint_error_dev(self, "couldn't establish power handler\n"); 421 422 #if defined(MULTIPROCESSOR) 423 if (mp_verbose) { 424 struct lwp *l = ci->ci_data.cpu_idlelwp; 425 426 aprint_verbose( 427 "%s: idle lwp at %p, idle sp at %p\n", 428 device_xname(sc->sc_dev), l, 429 #ifdef i386 430 (void *)l->l_addr->u_pcb.pcb_esp 431 #else 432 (void *)l->l_addr->u_pcb.pcb_rsp 433 #endif 434 ); 435 } 436 #endif 437 } 438 439 /* 440 * Initialize the processor appropriately. 441 */ 442 443 void 444 cpu_init(struct cpu_info *ci) 445 { 446 /* configure the CPU if needed */ 447 if (ci->cpu_setup != NULL) 448 (*ci->cpu_setup)(ci); 449 450 #ifdef i386 451 /* 452 * On a 486 or above, enable ring 0 write protection. 453 */ 454 if (ci->ci_cpu_class >= CPUCLASS_486) 455 lcr0(rcr0() | CR0_WP); 456 #else 457 lcr0(rcr0() | CR0_WP); 458 #endif 459 460 /* 461 * On a P6 or above, enable global TLB caching if the 462 * hardware supports it. 463 */ 464 if (cpu_feature & CPUID_PGE) 465 lcr4(rcr4() | CR4_PGE); /* enable global TLB caching */ 466 467 /* 468 * If we have FXSAVE/FXRESTOR, use them. 469 */ 470 if (cpu_feature & CPUID_FXSR) { 471 lcr4(rcr4() | CR4_OSFXSR); 472 473 /* 474 * If we have SSE/SSE2, enable XMM exceptions. 475 */ 476 if (cpu_feature & (CPUID_SSE|CPUID_SSE2)) 477 lcr4(rcr4() | CR4_OSXMMEXCPT); 478 } 479 480 #ifdef MTRR 481 /* 482 * On a P6 or above, initialize MTRR's if the hardware supports them. 483 */ 484 if (cpu_feature & CPUID_MTRR) { 485 if ((ci->ci_flags & CPUF_AP) == 0) 486 i686_mtrr_init_first(); 487 mtrr_init_cpu(ci); 488 } 489 490 #ifdef i386 491 if (strcmp((char *)(ci->ci_vendor), "AuthenticAMD") == 0) { 492 /* 493 * Must be a K6-2 Step >= 7 or a K6-III. 494 */ 495 if (CPUID2FAMILY(ci->ci_signature) == 5) { 496 if (CPUID2MODEL(ci->ci_signature) > 8 || 497 (CPUID2MODEL(ci->ci_signature) == 8 && 498 CPUID2STEPPING(ci->ci_signature) >= 7)) { 499 mtrr_funcs = &k6_mtrr_funcs; 500 k6_mtrr_init_first(); 501 mtrr_init_cpu(ci); 502 } 503 } 504 } 505 #endif /* i386 */ 506 #endif /* MTRR */ 507 508 atomic_or_32(&ci->ci_flags, CPUF_RUNNING); 509 atomic_or_32(&cpus_running, ci->ci_cpumask); 510 511 #ifndef MULTIPROCESSOR 512 /* XXX */ 513 x86_patch(); 514 #endif 515 } 516 517 #ifdef MULTIPROCESSOR 518 void 519 cpu_boot_secondary_processors(void) 520 { 521 struct cpu_info *ci; 522 u_long i; 523 524 /* Now that we know the number of CPUs, patch the text segment. */ 525 x86_patch(); 526 527 for (i=0; i < X86_MAXPROCS; i++) { 528 ci = cpu_info[i]; 529 if (ci == NULL) 530 continue; 531 if (ci->ci_data.cpu_idlelwp == NULL) 532 continue; 533 if ((ci->ci_flags & CPUF_PRESENT) == 0) 534 continue; 535 if (ci->ci_flags & (CPUF_BSP|CPUF_SP|CPUF_PRIMARY)) 536 continue; 537 cpu_boot_secondary(ci); 538 } 539 540 x86_mp_online = true; 541 } 542 543 static void 544 cpu_init_idle_lwp(struct cpu_info *ci) 545 { 546 struct lwp *l = ci->ci_data.cpu_idlelwp; 547 struct pcb *pcb = &l->l_addr->u_pcb; 548 549 pcb->pcb_cr0 = rcr0(); 550 } 551 552 void 553 cpu_init_idle_lwps(void) 554 { 555 struct cpu_info *ci; 556 u_long i; 557 558 for (i = 0; i < X86_MAXPROCS; i++) { 559 ci = cpu_info[i]; 560 if (ci == NULL) 561 continue; 562 if (ci->ci_data.cpu_idlelwp == NULL) 563 continue; 564 if ((ci->ci_flags & CPUF_PRESENT) == 0) 565 continue; 566 cpu_init_idle_lwp(ci); 567 } 568 } 569 570 void 571 cpu_start_secondary(struct cpu_info *ci) 572 { 573 int i; 574 extern paddr_t mp_pdirpa; 575 576 mp_pdirpa = pmap_init_tmp_pgtbl(mp_trampoline_paddr); 577 578 atomic_or_32(&ci->ci_flags, CPUF_AP); 579 580 aprint_debug_dev(ci->ci_dev, "starting\n"); 581 582 ci->ci_curlwp = ci->ci_data.cpu_idlelwp; 583 if (CPU_STARTUP(ci, mp_trampoline_paddr) != 0) 584 return; 585 586 /* 587 * wait for it to become ready 588 */ 589 for (i = 100000; (!(ci->ci_flags & CPUF_PRESENT)) && i > 0; i--) { 590 #ifdef MPDEBUG 591 extern int cpu_trace[3]; 592 static int otrace[3]; 593 if (memcmp(otrace, cpu_trace, sizeof(otrace)) != 0) { 594 aprint_debug_dev(ci->ci_dev, "trace %02x %02x %02x\n", 595 cpu_trace[0], cpu_trace[1], cpu_trace[2]); 596 memcpy(otrace, cpu_trace, sizeof(otrace)); 597 } 598 #endif 599 i8254_delay(10); 600 } 601 if ((ci->ci_flags & CPUF_PRESENT) == 0) { 602 aprint_error_dev(ci->ci_dev, "failed to become ready\n"); 603 #if defined(MPDEBUG) && defined(DDB) 604 printf("dropping into debugger; continue from here to resume boot\n"); 605 Debugger(); 606 #endif 607 } 608 609 CPU_START_CLEANUP(ci); 610 } 611 612 void 613 cpu_boot_secondary(struct cpu_info *ci) 614 { 615 int i; 616 617 atomic_or_32(&ci->ci_flags, CPUF_GO); 618 for (i = 100000; (!(ci->ci_flags & CPUF_RUNNING)) && i > 0; i--) { 619 i8254_delay(10); 620 } 621 if ((ci->ci_flags & CPUF_RUNNING) == 0) { 622 aprint_error_dev(ci->ci_dev, "failed to start\n"); 623 #if defined(MPDEBUG) && defined(DDB) 624 printf("dropping into debugger; continue from here to resume boot\n"); 625 Debugger(); 626 #endif 627 } 628 } 629 630 /* 631 * The CPU ends up here when its ready to run 632 * This is called from code in mptramp.s; at this point, we are running 633 * in the idle pcb/idle stack of the new CPU. When this function returns, 634 * this processor will enter the idle loop and start looking for work. 635 */ 636 void 637 cpu_hatch(void *v) 638 { 639 struct cpu_info *ci = (struct cpu_info *)v; 640 int s, i; 641 642 #ifdef __x86_64__ 643 cpu_init_msrs(ci, true); 644 #endif 645 cpu_probe_features(ci); 646 cpu_feature &= ci->ci_feature_flags; 647 cpu_feature2 &= ci->ci_feature2_flags; 648 649 KDASSERT((ci->ci_flags & CPUF_PRESENT) == 0); 650 atomic_or_32(&ci->ci_flags, CPUF_PRESENT); 651 while ((ci->ci_flags & CPUF_GO) == 0) { 652 /* Don't use delay, boot CPU may be patching the text. */ 653 for (i = 10000; i != 0; i--) 654 x86_pause(); 655 } 656 657 /* Because the text may have been patched in x86_patch(). */ 658 wbinvd(); 659 x86_flush(); 660 661 KASSERT((ci->ci_flags & CPUF_RUNNING) == 0); 662 663 lcr3(pmap_kernel()->pm_pdirpa); 664 curlwp->l_addr->u_pcb.pcb_cr3 = pmap_kernel()->pm_pdirpa; 665 lcr0(ci->ci_data.cpu_idlelwp->l_addr->u_pcb.pcb_cr0); 666 cpu_init_idt(); 667 gdt_init_cpu(ci); 668 lapic_enable(); 669 lapic_set_lvt(); 670 lapic_initclocks(); 671 672 #ifdef i386 673 npxinit(ci); 674 #else 675 fpuinit(ci); 676 #endif 677 lldt(GSYSSEL(GLDT_SEL, SEL_KPL)); 678 ltr(ci->ci_tss_sel); 679 680 cpu_init(ci); 681 cpu_get_tsc_freq(ci); 682 683 s = splhigh(); 684 #ifdef i386 685 lapic_tpr = 0; 686 #else 687 lcr8(0); 688 #endif 689 x86_enable_intr(); 690 splx(s); 691 x86_errata(); 692 693 aprint_debug_dev(ci->ci_dev, "CPU %ld running\n", 694 (long)ci->ci_cpuid); 695 } 696 697 #if defined(DDB) 698 699 #include <ddb/db_output.h> 700 #include <machine/db_machdep.h> 701 702 /* 703 * Dump CPU information from ddb. 704 */ 705 void 706 cpu_debug_dump(void) 707 { 708 struct cpu_info *ci; 709 CPU_INFO_ITERATOR cii; 710 711 db_printf("addr dev id flags ipis curproc fpcurproc\n"); 712 for (CPU_INFO_FOREACH(cii, ci)) { 713 db_printf("%p %s %ld %x %x %10p %10p\n", 714 ci, 715 ci->ci_dev == NULL ? "BOOT" : device_xname(ci->ci_dev), 716 (long)ci->ci_cpuid, 717 ci->ci_flags, ci->ci_ipis, 718 ci->ci_curlwp, 719 ci->ci_fpcurlwp); 720 } 721 } 722 #endif 723 724 static void 725 cpu_copy_trampoline(void) 726 { 727 /* 728 * Copy boot code. 729 */ 730 extern u_char cpu_spinup_trampoline[]; 731 extern u_char cpu_spinup_trampoline_end[]; 732 733 vaddr_t mp_trampoline_vaddr; 734 735 mp_trampoline_vaddr = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, 736 UVM_KMF_VAONLY); 737 738 pmap_kenter_pa(mp_trampoline_vaddr, mp_trampoline_paddr, 739 VM_PROT_READ | VM_PROT_WRITE); 740 pmap_update(pmap_kernel()); 741 memcpy((void *)mp_trampoline_vaddr, 742 cpu_spinup_trampoline, 743 cpu_spinup_trampoline_end - cpu_spinup_trampoline); 744 745 pmap_kremove(mp_trampoline_vaddr, PAGE_SIZE); 746 pmap_update(pmap_kernel()); 747 uvm_km_free(kernel_map, mp_trampoline_vaddr, PAGE_SIZE, UVM_KMF_VAONLY); 748 } 749 750 #endif 751 752 #ifdef i386 753 static void 754 tss_init(struct i386tss *tss, void *stack, void *func) 755 { 756 memset(tss, 0, sizeof *tss); 757 tss->tss_esp0 = tss->tss_esp = (int)((char *)stack + USPACE - 16); 758 tss->tss_ss0 = GSEL(GDATA_SEL, SEL_KPL); 759 tss->__tss_cs = GSEL(GCODE_SEL, SEL_KPL); 760 tss->tss_fs = GSEL(GCPU_SEL, SEL_KPL); 761 tss->tss_gs = tss->__tss_es = tss->__tss_ds = 762 tss->__tss_ss = GSEL(GDATA_SEL, SEL_KPL); 763 tss->tss_cr3 = pmap_kernel()->pm_pdirpa; 764 tss->tss_esp = (int)((char *)stack + USPACE - 16); 765 tss->tss_ldt = GSEL(GLDT_SEL, SEL_KPL); 766 tss->__tss_eflags = PSL_MBO | PSL_NT; /* XXX not needed? */ 767 tss->__tss_eip = (int)func; 768 } 769 770 /* XXX */ 771 #define IDTVEC(name) __CONCAT(X, name) 772 typedef void (vector)(void); 773 extern vector IDTVEC(tss_trap08); 774 #ifdef DDB 775 extern vector Xintrddbipi; 776 extern int ddb_vec; 777 #endif 778 779 static void 780 cpu_set_tss_gates(struct cpu_info *ci) 781 { 782 struct segment_descriptor sd; 783 784 ci->ci_doubleflt_stack = (char *)uvm_km_alloc(kernel_map, USPACE, 0, 785 UVM_KMF_WIRED); 786 tss_init(&ci->ci_doubleflt_tss, ci->ci_doubleflt_stack, 787 IDTVEC(tss_trap08)); 788 setsegment(&sd, &ci->ci_doubleflt_tss, sizeof(struct i386tss) - 1, 789 SDT_SYS386TSS, SEL_KPL, 0, 0); 790 ci->ci_gdt[GTRAPTSS_SEL].sd = sd; 791 setgate(&idt[8], NULL, 0, SDT_SYSTASKGT, SEL_KPL, 792 GSEL(GTRAPTSS_SEL, SEL_KPL)); 793 794 #if defined(DDB) && defined(MULTIPROCESSOR) 795 /* 796 * Set up separate handler for the DDB IPI, so that it doesn't 797 * stomp on a possibly corrupted stack. 798 * 799 * XXX overwriting the gate set in db_machine_init. 800 * Should rearrange the code so that it's set only once. 801 */ 802 ci->ci_ddbipi_stack = (char *)uvm_km_alloc(kernel_map, USPACE, 0, 803 UVM_KMF_WIRED); 804 tss_init(&ci->ci_ddbipi_tss, ci->ci_ddbipi_stack, Xintrddbipi); 805 806 setsegment(&sd, &ci->ci_ddbipi_tss, sizeof(struct i386tss) - 1, 807 SDT_SYS386TSS, SEL_KPL, 0, 0); 808 ci->ci_gdt[GIPITSS_SEL].sd = sd; 809 810 setgate(&idt[ddb_vec], NULL, 0, SDT_SYSTASKGT, SEL_KPL, 811 GSEL(GIPITSS_SEL, SEL_KPL)); 812 #endif 813 } 814 #else 815 static void 816 cpu_set_tss_gates(struct cpu_info *ci) 817 { 818 819 } 820 #endif /* i386 */ 821 822 int 823 mp_cpu_start(struct cpu_info *ci, paddr_t target) 824 { 825 #if NLAPIC > 0 826 int error; 827 #endif 828 unsigned short dwordptr[2]; 829 830 /* 831 * Bootstrap code must be addressable in real mode 832 * and it must be page aligned. 833 */ 834 KASSERT(target < 0x10000 && target % PAGE_SIZE == 0); 835 836 /* 837 * "The BSP must initialize CMOS shutdown code to 0Ah ..." 838 */ 839 840 outb(IO_RTC, NVRAM_RESET); 841 outb(IO_RTC+1, NVRAM_RESET_JUMP); 842 843 /* 844 * "and the warm reset vector (DWORD based at 40:67) to point 845 * to the AP startup code ..." 846 */ 847 848 dwordptr[0] = 0; 849 dwordptr[1] = target >> 4; 850 851 memcpy((uint8_t *)cmos_data_mapping + 0x467, dwordptr, 4); 852 853 #if NLAPIC > 0 854 if ((cpu_feature & CPUID_APIC) == 0) { 855 aprint_error("mp_cpu_start: CPU does not have APIC\n"); 856 return ENODEV; 857 } 858 859 /* 860 * ... prior to executing the following sequence:" 861 */ 862 863 if (ci->ci_flags & CPUF_AP) { 864 error = x86_ipi_init(ci->ci_apicid); 865 if (error != 0) { 866 aprint_error_dev(ci->ci_dev, "%s: IPI not taken (1)\n", 867 __func__); 868 return error; 869 } 870 871 i8254_delay(10000); 872 873 error = x86_ipi(target / PAGE_SIZE, ci->ci_apicid, 874 LAPIC_DLMODE_STARTUP); 875 if (error != 0) { 876 aprint_error_dev(ci->ci_dev, "%s: IPI not taken (2)\n", 877 __func__); 878 return error; 879 } 880 i8254_delay(200); 881 882 error = x86_ipi(target / PAGE_SIZE, ci->ci_apicid, 883 LAPIC_DLMODE_STARTUP); 884 if (error != 0) { 885 aprint_error_dev(ci->ci_dev, "%s: IPI not taken (3)\n", 886 __func__); 887 return error; 888 } 889 i8254_delay(200); 890 } 891 #endif 892 return 0; 893 } 894 895 void 896 mp_cpu_start_cleanup(struct cpu_info *ci) 897 { 898 /* 899 * Ensure the NVRAM reset byte contains something vaguely sane. 900 */ 901 902 outb(IO_RTC, NVRAM_RESET); 903 outb(IO_RTC+1, NVRAM_RESET_RST); 904 } 905 906 #ifdef __x86_64__ 907 typedef void (vector)(void); 908 extern vector Xsyscall, Xsyscall32; 909 910 void 911 cpu_init_msrs(struct cpu_info *ci, bool full) 912 { 913 wrmsr(MSR_STAR, 914 ((uint64_t)GSEL(GCODE_SEL, SEL_KPL) << 32) | 915 ((uint64_t)LSEL(LSYSRETBASE_SEL, SEL_UPL) << 48)); 916 wrmsr(MSR_LSTAR, (uint64_t)Xsyscall); 917 wrmsr(MSR_CSTAR, (uint64_t)Xsyscall32); 918 wrmsr(MSR_SFMASK, PSL_NT|PSL_T|PSL_I|PSL_C); 919 920 if (full) { 921 wrmsr(MSR_FSBASE, 0); 922 wrmsr(MSR_GSBASE, (uint64_t)ci); 923 wrmsr(MSR_KERNELGSBASE, 0); 924 } 925 926 if (cpu_feature & CPUID_NOX) 927 wrmsr(MSR_EFER, rdmsr(MSR_EFER) | EFER_NXE); 928 } 929 #endif /* __x86_64__ */ 930 931 void 932 cpu_offline_md(void) 933 { 934 int s; 935 936 s = splhigh(); 937 #ifdef __i386__ 938 npxsave_cpu(true); 939 #else 940 fpusave_cpu(true); 941 #endif 942 splx(s); 943 } 944 945 /* XXX joerg restructure and restart CPUs individually */ 946 static bool 947 cpu_suspend(device_t dv PMF_FN_ARGS) 948 { 949 struct cpu_softc *sc = device_private(dv); 950 struct cpu_info *ci = sc->sc_info; 951 int err; 952 953 if (ci->ci_flags & CPUF_PRIMARY) 954 return true; 955 if (ci->ci_data.cpu_idlelwp == NULL) 956 return true; 957 if ((ci->ci_flags & CPUF_PRESENT) == 0) 958 return true; 959 960 sc->sc_wasonline = !(ci->ci_schedstate.spc_flags & SPCF_OFFLINE); 961 962 if (sc->sc_wasonline) { 963 mutex_enter(&cpu_lock); 964 err = cpu_setonline(ci, false); 965 mutex_exit(&cpu_lock); 966 967 if (err) 968 return false; 969 } 970 971 return true; 972 } 973 974 static bool 975 cpu_resume(device_t dv PMF_FN_ARGS) 976 { 977 struct cpu_softc *sc = device_private(dv); 978 struct cpu_info *ci = sc->sc_info; 979 int err = 0; 980 981 if (ci->ci_flags & CPUF_PRIMARY) 982 return true; 983 if (ci->ci_data.cpu_idlelwp == NULL) 984 return true; 985 if ((ci->ci_flags & CPUF_PRESENT) == 0) 986 return true; 987 988 if (sc->sc_wasonline) { 989 mutex_enter(&cpu_lock); 990 err = cpu_setonline(ci, true); 991 mutex_exit(&cpu_lock); 992 } 993 994 return err == 0; 995 } 996 997 void 998 cpu_get_tsc_freq(struct cpu_info *ci) 999 { 1000 uint64_t last_tsc; 1001 u_int junk[4]; 1002 1003 if (ci->ci_feature_flags & CPUID_TSC) { 1004 /* Serialize. */ 1005 x86_cpuid(0, junk); 1006 last_tsc = rdtsc(); 1007 i8254_delay(100000); 1008 ci->ci_tsc_freq = (rdtsc() - last_tsc) * 10; 1009 } 1010 } 1011
Indexes created Tue Sep 23 02:09:52 GMT 2025