cpu.c revision 1.33
1 /* $NetBSD: cpu.c,v 1.33 2008/04/24 15:59:57 jmcneill Exp $ */ 2 3 /*- 4 * Copyright (c) 2000, 2006, 2007 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Bill Sommerfeld of RedBack Networks Inc, and by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 3. All advertising materials mentioning features or use of this software 19 * must display the following acknowledgement: 20 * This product includes software developed by the NetBSD 21 * Foundation, Inc. and its contributors. 22 * 4. Neither the name of The NetBSD Foundation nor the names of its 23 * contributors may be used to endorse or promote products derived 24 * from this software without specific prior written permission. 25 * 26 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 27 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 28 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 29 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 30 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 31 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 32 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 33 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 34 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 35 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 36 * POSSIBILITY OF SUCH DAMAGE. 37 */ 38 39 /* 40 * Copyright (c) 1999 Stefan Grefen 41 * 42 * Redistribution and use in source and binary forms, with or without 43 * modification, are permitted provided that the following conditions 44 * are met: 45 * 1. Redistributions of source code must retain the above copyright 46 * notice, this list of conditions and the following disclaimer. 47 * 2. Redistributions in binary form must reproduce the above copyright 48 * notice, this list of conditions and the following disclaimer in the 49 * documentation and/or other materials provided with the distribution. 50 * 3. All advertising materials mentioning features or use of this software 51 * must display the following acknowledgement: 52 * This product includes software developed by the NetBSD 53 * Foundation, Inc. and its contributors. 54 * 4. Neither the name of The NetBSD Foundation nor the names of its 55 * contributors may be used to endorse or promote products derived 56 * from this software without specific prior written permission. 57 * 58 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND ANY 59 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 60 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 61 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR AND CONTRIBUTORS BE LIABLE 62 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 63 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 64 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 65 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 66 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 67 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 68 * SUCH DAMAGE. 69 */ 70 71 #include <sys/cdefs.h> 72 __KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.33 2008/04/24 15:59:57 jmcneill Exp $"); 73 74 #include "opt_ddb.h" 75 #include "opt_multiprocessor.h" 76 #include "opt_mpbios.h" /* for MPDEBUG */ 77 #include "opt_mtrr.h" 78 79 #include "lapic.h" 80 #include "ioapic.h" 81 82 #include <sys/param.h> 83 #include <sys/proc.h> 84 #include <sys/user.h> 85 #include <sys/systm.h> 86 #include <sys/device.h> 87 #include <sys/malloc.h> 88 #include <sys/cpu.h> 89 #include <sys/atomic.h> 90 91 #include <uvm/uvm_extern.h> 92 93 #include <machine/cpufunc.h> 94 #include <machine/cpuvar.h> 95 #include <machine/pmap.h> 96 #include <machine/vmparam.h> 97 #include <machine/mpbiosvar.h> 98 #include <machine/pcb.h> 99 #include <machine/specialreg.h> 100 #include <machine/segments.h> 101 #include <machine/gdt.h> 102 #include <machine/mtrr.h> 103 #include <machine/pio.h> 104 105 #ifdef i386 106 #include <machine/tlog.h> 107 #endif 108 109 #if NLAPIC > 0 110 #include <machine/apicvar.h> 111 #include <machine/i82489reg.h> 112 #include <machine/i82489var.h> 113 #endif 114 115 #include <dev/ic/mc146818reg.h> 116 #include <i386/isa/nvram.h> 117 #include <dev/isa/isareg.h> 118 119 int cpu_match(device_t, cfdata_t, void *); 120 void cpu_attach(device_t, device_t, void *); 121 122 static bool cpu_suspend(device_t PMF_FN_PROTO); 123 static bool cpu_resume(device_t PMF_FN_PROTO); 124 125 struct cpu_softc { 126 device_t sc_dev; /* device tree glue */ 127 struct cpu_info *sc_info; /* pointer to CPU info */ 128 bool sc_wasonline; 129 }; 130 131 int mp_cpu_start(struct cpu_info *, paddr_t); 132 void mp_cpu_start_cleanup(struct cpu_info *); 133 const struct cpu_functions mp_cpu_funcs = { mp_cpu_start, NULL, 134 mp_cpu_start_cleanup }; 135 136 137 CFATTACH_DECL_NEW(cpu, sizeof(struct cpu_softc), 138 cpu_match, cpu_attach, NULL, NULL); 139 140 /* 141 * Statically-allocated CPU info for the primary CPU (or the only 142 * CPU, on uniprocessors). The CPU info list is initialized to 143 * point at it. 144 */ 145 #ifdef TRAPLOG 146 struct tlog tlog_primary; 147 #endif 148 struct cpu_info cpu_info_primary __aligned(CACHE_LINE_SIZE) = { 149 .ci_dev = 0, 150 .ci_self = &cpu_info_primary, 151 .ci_idepth = -1, 152 .ci_curlwp = &lwp0, 153 #ifdef TRAPLOG 154 .ci_tlog_base = &tlog_primary, 155 #endif /* !TRAPLOG */ 156 }; 157 158 struct cpu_info *cpu_info_list = &cpu_info_primary; 159 160 static void cpu_set_tss_gates(struct cpu_info *); 161 162 #ifdef i386 163 static void tss_init(struct i386tss *, void *, void *); 164 #endif 165 166 #ifdef MULTIPROCESSOR 167 static void cpu_init_idle_lwp(struct cpu_info *); 168 #endif 169 170 uint32_t cpus_attached = 0; 171 uint32_t cpus_running = 0; 172 173 extern char x86_64_doubleflt_stack[]; 174 175 bool x86_mp_online; 176 paddr_t mp_trampoline_paddr = MP_TRAMPOLINE; 177 178 static vaddr_t cmos_data_mapping; 179 180 #ifdef MULTIPROCESSOR 181 /* 182 * Array of CPU info structures. Must be statically-allocated because 183 * curproc, etc. are used early. 184 */ 185 struct cpu_info *cpu_info[X86_MAXPROCS] = { &cpu_info_primary, }; 186 187 void cpu_hatch(void *); 188 static void cpu_boot_secondary(struct cpu_info *ci); 189 static void cpu_start_secondary(struct cpu_info *ci); 190 static void cpu_copy_trampoline(void); 191 192 /* 193 * Runs once per boot once multiprocessor goo has been detected and 194 * the local APIC on the boot processor has been mapped. 195 * 196 * Called from lapic_boot_init() (from mpbios_scan()). 197 */ 198 void 199 cpu_init_first(void) 200 { 201 int cpunum = lapic_cpu_number(); 202 203 if (cpunum != 0) { 204 cpu_info[0] = NULL; 205 cpu_info[cpunum] = &cpu_info_primary; 206 } 207 208 cpu_info_primary.ci_cpuid = cpunum; 209 cpu_copy_trampoline(); 210 211 cmos_data_mapping = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, UVM_KMF_VAONLY); 212 if (cmos_data_mapping == 0) 213 panic("No KVA for page 0"); 214 pmap_kenter_pa(cmos_data_mapping, 0, VM_PROT_READ|VM_PROT_WRITE); 215 pmap_update(pmap_kernel()); 216 } 217 #endif 218 219 int 220 cpu_match(device_t parent, cfdata_t match, void *aux) 221 { 222 223 return 1; 224 } 225 226 static void 227 cpu_vm_init(struct cpu_info *ci) 228 { 229 int ncolors = 2, i; 230 231 for (i = CAI_ICACHE; i <= CAI_L2CACHE; i++) { 232 struct x86_cache_info *cai; 233 int tcolors; 234 235 cai = &ci->ci_cinfo[i]; 236 237 tcolors = atop(cai->cai_totalsize); 238 switch(cai->cai_associativity) { 239 case 0xff: 240 tcolors = 1; /* fully associative */ 241 break; 242 case 0: 243 case 1: 244 break; 245 default: 246 tcolors /= cai->cai_associativity; 247 } 248 ncolors = max(ncolors, tcolors); 249 /* 250 * If the desired number of colors is not a power of 251 * two, it won't be good. Find the greatest power of 252 * two which is an even divisor of the number of colors, 253 * to preserve even coloring of pages. 254 */ 255 if (ncolors & (ncolors - 1) ) { 256 int try, picked = 1; 257 for (try = 1; try < ncolors; try *= 2) { 258 if (ncolors % try == 0) picked = try; 259 } 260 if (picked == 1) { 261 panic("desired number of cache colors %d is " 262 " > 1, but not even!", ncolors); 263 } 264 ncolors = picked; 265 } 266 } 267 268 /* 269 * Knowing the size of the largest cache on this CPU, re-color 270 * our pages. 271 */ 272 if (ncolors <= uvmexp.ncolors) 273 return; 274 aprint_verbose_dev(ci->ci_dev, "%d page colors\n", ncolors); 275 uvm_page_recolor(ncolors); 276 } 277 278 279 void 280 cpu_attach(device_t parent, device_t self, void *aux) 281 { 282 struct cpu_softc *sc = device_private(self); 283 struct cpu_attach_args *caa = aux; 284 struct cpu_info *ci; 285 uintptr_t ptr; 286 #if defined(MULTIPROCESSOR) 287 int cpunum = caa->cpu_number; 288 #endif 289 290 sc->sc_dev = self; 291 292 /* 293 * If we're an Application Processor, allocate a cpu_info 294 * structure, otherwise use the primary's. 295 */ 296 if (caa->cpu_role == CPU_ROLE_AP) { 297 #if defined(MULTIPROCESSOR) 298 if (cpunum >= X86_MAXPROCS) { 299 aprint_error(": apic id %d ignored, " 300 "please increase X86_MAXPROCS\n", cpunum); 301 return; 302 } 303 #endif 304 aprint_naive(": Application Processor\n"); 305 ptr = (uintptr_t)malloc(sizeof(*ci) + CACHE_LINE_SIZE - 1, 306 M_DEVBUF, M_WAITOK); 307 ci = (struct cpu_info *)((ptr + CACHE_LINE_SIZE - 1) & 308 ~(CACHE_LINE_SIZE - 1)); 309 memset(ci, 0, sizeof(*ci)); 310 #if defined(MULTIPROCESSOR) 311 if (cpu_info[cpunum] != NULL) { 312 printf("\n"); 313 panic("cpu at apic id %d already attached?", cpunum); 314 } 315 cpu_info[cpunum] = ci; 316 #endif 317 #ifdef TRAPLOG 318 ci->ci_tlog_base = malloc(sizeof(struct tlog), 319 M_DEVBUF, M_WAITOK); 320 #endif 321 } else { 322 aprint_naive(": %s Processor\n", 323 caa->cpu_role == CPU_ROLE_SP ? "Single" : "Boot"); 324 ci = &cpu_info_primary; 325 #if defined(MULTIPROCESSOR) 326 if (cpunum != lapic_cpu_number()) { 327 printf("\n"); 328 panic("%s: running CPU is at apic %d" 329 " instead of at expected %d", 330 device_xname(sc->sc_dev), lapic_cpu_number(), 331 cpunum); 332 } 333 #endif 334 } 335 336 ci->ci_self = ci; 337 sc->sc_info = ci; 338 339 ci->ci_dev = self; 340 ci->ci_apicid = caa->cpu_number; 341 #ifdef MULTIPROCESSOR 342 ci->ci_cpuid = ci->ci_apicid; 343 #else 344 ci->ci_cpuid = 0; /* False for APs, but they're not used anyway */ 345 #endif 346 ci->ci_cpumask = (1 << ci->ci_cpuid); 347 ci->ci_func = caa->cpu_func; 348 349 if (caa->cpu_role == CPU_ROLE_AP) { 350 #ifdef MULTIPROCESSOR 351 int error; 352 353 error = mi_cpu_attach(ci); 354 if (error != 0) { 355 aprint_normal("\n"); 356 aprint_error_dev(sc->sc_dev, 357 "mi_cpu_attach failed with %d\n", error); 358 return; 359 } 360 #endif 361 cpu_init_tss(ci); 362 } else { 363 KASSERT(ci->ci_data.cpu_idlelwp != NULL); 364 } 365 366 pmap_reference(pmap_kernel()); 367 ci->ci_pmap = pmap_kernel(); 368 ci->ci_tlbstate = TLBSTATE_STALE; 369 370 /* further PCB init done later. */ 371 372 switch (caa->cpu_role) { 373 case CPU_ROLE_SP: 374 aprint_normal(": (uniprocessor)\n"); 375 atomic_or_32(&ci->ci_flags, 376 CPUF_PRESENT | CPUF_SP | CPUF_PRIMARY); 377 cpu_intr_init(ci); 378 identifycpu(ci); 379 cpu_init(ci); 380 cpu_set_tss_gates(ci); 381 pmap_cpu_init_late(ci); 382 x86_errata(); 383 break; 384 385 case CPU_ROLE_BP: 386 aprint_normal(": (boot processor)\n"); 387 atomic_or_32(&ci->ci_flags, 388 CPUF_PRESENT | CPUF_BSP | CPUF_PRIMARY); 389 cpu_intr_init(ci); 390 identifycpu(ci); 391 cpu_init(ci); 392 cpu_set_tss_gates(ci); 393 pmap_cpu_init_late(ci); 394 #if NLAPIC > 0 395 /* 396 * Enable local apic 397 */ 398 lapic_enable(); 399 lapic_set_lvt(); 400 lapic_calibrate_timer(ci); 401 #endif 402 x86_errata(); 403 break; 404 405 case CPU_ROLE_AP: 406 /* 407 * report on an AP 408 */ 409 aprint_normal(": (application processor)\n"); 410 411 #if defined(MULTIPROCESSOR) 412 cpu_intr_init(ci); 413 gdt_alloc_cpu(ci); 414 cpu_set_tss_gates(ci); 415 pmap_cpu_init_early(ci); 416 pmap_cpu_init_late(ci); 417 cpu_start_secondary(ci); 418 if (ci->ci_flags & CPUF_PRESENT) { 419 identifycpu(ci); 420 ci->ci_next = cpu_info_list->ci_next; 421 cpu_info_list->ci_next = ci; 422 } 423 #else 424 aprint_normal_dev(sc->sc_dev, "not started\n"); 425 #endif 426 break; 427 428 default: 429 aprint_normal("\n"); 430 panic("unknown processor type??\n"); 431 } 432 cpu_vm_init(ci); 433 434 cpus_attached |= ci->ci_cpumask; 435 436 if (!pmf_device_register(self, cpu_suspend, cpu_resume)) 437 aprint_error_dev(self, "couldn't establish power handler\n"); 438 439 #if defined(MULTIPROCESSOR) 440 if (mp_verbose) { 441 struct lwp *l = ci->ci_data.cpu_idlelwp; 442 443 aprint_verbose_dev(sc->sc_dev, 444 "idle lwp at %p, idle sp at %p\n", 445 l, 446 #ifdef i386 447 (void *)l->l_addr->u_pcb.pcb_esp 448 #else 449 (void *)l->l_addr->u_pcb.pcb_rsp 450 #endif 451 ); 452 } 453 #endif 454 } 455 456 /* 457 * Initialize the processor appropriately. 458 */ 459 460 void 461 cpu_init(struct cpu_info *ci) 462 { 463 /* configure the CPU if needed */ 464 if (ci->cpu_setup != NULL) 465 (*ci->cpu_setup)(ci); 466 467 #ifdef i386 468 /* 469 * On a 486 or above, enable ring 0 write protection. 470 */ 471 if (ci->ci_cpu_class >= CPUCLASS_486) 472 lcr0(rcr0() | CR0_WP); 473 #else 474 lcr0(rcr0() | CR0_WP); 475 #endif 476 477 /* 478 * On a P6 or above, enable global TLB caching if the 479 * hardware supports it. 480 */ 481 if (cpu_feature & CPUID_PGE) 482 lcr4(rcr4() | CR4_PGE); /* enable global TLB caching */ 483 484 /* 485 * If we have FXSAVE/FXRESTOR, use them. 486 */ 487 if (cpu_feature & CPUID_FXSR) { 488 lcr4(rcr4() | CR4_OSFXSR); 489 490 /* 491 * If we have SSE/SSE2, enable XMM exceptions. 492 */ 493 if (cpu_feature & (CPUID_SSE|CPUID_SSE2)) 494 lcr4(rcr4() | CR4_OSXMMEXCPT); 495 } 496 497 #ifdef MTRR 498 /* 499 * On a P6 or above, initialize MTRR's if the hardware supports them. 500 */ 501 if (cpu_feature & CPUID_MTRR) { 502 if ((ci->ci_flags & CPUF_AP) == 0) 503 i686_mtrr_init_first(); 504 mtrr_init_cpu(ci); 505 } 506 507 #ifdef i386 508 if (strcmp((char *)(ci->ci_vendor), "AuthenticAMD") == 0) { 509 /* 510 * Must be a K6-2 Step >= 7 or a K6-III. 511 */ 512 if (CPUID2FAMILY(ci->ci_signature) == 5) { 513 if (CPUID2MODEL(ci->ci_signature) > 8 || 514 (CPUID2MODEL(ci->ci_signature) == 8 && 515 CPUID2STEPPING(ci->ci_signature) >= 7)) { 516 mtrr_funcs = &k6_mtrr_funcs; 517 k6_mtrr_init_first(); 518 mtrr_init_cpu(ci); 519 } 520 } 521 } 522 #endif /* i386 */ 523 #endif /* MTRR */ 524 525 atomic_or_32(&ci->ci_flags, CPUF_RUNNING); 526 atomic_or_32(&cpus_running, ci->ci_cpumask); 527 528 #ifndef MULTIPROCESSOR 529 /* XXX */ 530 x86_patch(); 531 #endif 532 } 533 534 #ifdef MULTIPROCESSOR 535 void 536 cpu_boot_secondary_processors(void) 537 { 538 struct cpu_info *ci; 539 u_long i; 540 541 /* Now that we know the number of CPUs, patch the text segment. */ 542 x86_patch(); 543 544 for (i=0; i < X86_MAXPROCS; i++) { 545 ci = cpu_info[i]; 546 if (ci == NULL) 547 continue; 548 if (ci->ci_data.cpu_idlelwp == NULL) 549 continue; 550 if ((ci->ci_flags & CPUF_PRESENT) == 0) 551 continue; 552 if (ci->ci_flags & (CPUF_BSP|CPUF_SP|CPUF_PRIMARY)) 553 continue; 554 cpu_boot_secondary(ci); 555 } 556 557 x86_mp_online = true; 558 } 559 560 static void 561 cpu_init_idle_lwp(struct cpu_info *ci) 562 { 563 struct lwp *l = ci->ci_data.cpu_idlelwp; 564 struct pcb *pcb = &l->l_addr->u_pcb; 565 566 pcb->pcb_cr0 = rcr0(); 567 } 568 569 void 570 cpu_init_idle_lwps(void) 571 { 572 struct cpu_info *ci; 573 u_long i; 574 575 for (i = 0; i < X86_MAXPROCS; i++) { 576 ci = cpu_info[i]; 577 if (ci == NULL) 578 continue; 579 if (ci->ci_data.cpu_idlelwp == NULL) 580 continue; 581 if ((ci->ci_flags & CPUF_PRESENT) == 0) 582 continue; 583 cpu_init_idle_lwp(ci); 584 } 585 } 586 587 void 588 cpu_start_secondary(struct cpu_info *ci) 589 { 590 int i; 591 extern paddr_t mp_pdirpa; 592 593 mp_pdirpa = pmap_init_tmp_pgtbl(mp_trampoline_paddr); 594 595 atomic_or_32(&ci->ci_flags, CPUF_AP); 596 597 aprint_debug_dev(ci->ci_dev, "starting\n"); 598 599 ci->ci_curlwp = ci->ci_data.cpu_idlelwp; 600 if (CPU_STARTUP(ci, mp_trampoline_paddr) != 0) 601 return; 602 603 /* 604 * wait for it to become ready 605 */ 606 for (i = 100000; (!(ci->ci_flags & CPUF_PRESENT)) && i > 0; i--) { 607 #ifdef MPDEBUG 608 extern int cpu_trace[3]; 609 static int otrace[3]; 610 if (memcmp(otrace, cpu_trace, sizeof(otrace)) != 0) { 611 aprint_debug_dev(ci->ci_dev, "trace %02x %02x %02x\n", 612 cpu_trace[0], cpu_trace[1], cpu_trace[2]); 613 memcpy(otrace, cpu_trace, sizeof(otrace)); 614 } 615 #endif 616 i8254_delay(10); 617 } 618 if ((ci->ci_flags & CPUF_PRESENT) == 0) { 619 aprint_error_dev(ci->ci_dev, "failed to become ready\n"); 620 #if defined(MPDEBUG) && defined(DDB) 621 printf("dropping into debugger; continue from here to resume boot\n"); 622 Debugger(); 623 #endif 624 } 625 626 CPU_START_CLEANUP(ci); 627 } 628 629 void 630 cpu_boot_secondary(struct cpu_info *ci) 631 { 632 int i; 633 634 atomic_or_32(&ci->ci_flags, CPUF_GO); 635 for (i = 100000; (!(ci->ci_flags & CPUF_RUNNING)) && i > 0; i--) { 636 i8254_delay(10); 637 } 638 if ((ci->ci_flags & CPUF_RUNNING) == 0) { 639 aprint_error_dev(ci->ci_dev, "failed to start\n"); 640 #if defined(MPDEBUG) && defined(DDB) 641 printf("dropping into debugger; continue from here to resume boot\n"); 642 Debugger(); 643 #endif 644 } 645 } 646 647 /* 648 * The CPU ends up here when its ready to run 649 * This is called from code in mptramp.s; at this point, we are running 650 * in the idle pcb/idle stack of the new CPU. When this function returns, 651 * this processor will enter the idle loop and start looking for work. 652 */ 653 void 654 cpu_hatch(void *v) 655 { 656 struct cpu_info *ci = (struct cpu_info *)v; 657 int s, i; 658 659 #ifdef __x86_64__ 660 cpu_init_msrs(ci, true); 661 #endif 662 cpu_probe_features(ci); 663 cpu_feature &= ci->ci_feature_flags; 664 cpu_feature2 &= ci->ci_feature2_flags; 665 666 KDASSERT((ci->ci_flags & CPUF_PRESENT) == 0); 667 atomic_or_32(&ci->ci_flags, CPUF_PRESENT); 668 while ((ci->ci_flags & CPUF_GO) == 0) { 669 /* Don't use delay, boot CPU may be patching the text. */ 670 for (i = 10000; i != 0; i--) 671 x86_pause(); 672 } 673 674 /* Because the text may have been patched in x86_patch(). */ 675 wbinvd(); 676 x86_flush(); 677 678 KASSERT((ci->ci_flags & CPUF_RUNNING) == 0); 679 680 lcr3(pmap_kernel()->pm_pdirpa); 681 curlwp->l_addr->u_pcb.pcb_cr3 = pmap_kernel()->pm_pdirpa; 682 lcr0(ci->ci_data.cpu_idlelwp->l_addr->u_pcb.pcb_cr0); 683 cpu_init_idt(); 684 gdt_init_cpu(ci); 685 lapic_enable(); 686 lapic_set_lvt(); 687 lapic_initclocks(); 688 689 #ifdef i386 690 npxinit(ci); 691 #else 692 fpuinit(ci); 693 #endif 694 lldt(GSYSSEL(GLDT_SEL, SEL_KPL)); 695 ltr(ci->ci_tss_sel); 696 697 cpu_init(ci); 698 cpu_get_tsc_freq(ci); 699 700 s = splhigh(); 701 #ifdef i386 702 lapic_tpr = 0; 703 #else 704 lcr8(0); 705 #endif 706 x86_enable_intr(); 707 splx(s); 708 x86_errata(); 709 710 aprint_debug_dev(ci->ci_dev, "CPU %ld running\n", 711 (long)ci->ci_cpuid); 712 } 713 714 #if defined(DDB) 715 716 #include <ddb/db_output.h> 717 #include <machine/db_machdep.h> 718 719 /* 720 * Dump CPU information from ddb. 721 */ 722 void 723 cpu_debug_dump(void) 724 { 725 struct cpu_info *ci; 726 CPU_INFO_ITERATOR cii; 727 728 db_printf("addr dev id flags ipis curlwp fpcurlwp\n"); 729 for (CPU_INFO_FOREACH(cii, ci)) { 730 db_printf("%p %s %ld %x %x %10p %10p\n", 731 ci, 732 ci->ci_dev == NULL ? "BOOT" : device_xname(ci->ci_dev), 733 (long)ci->ci_cpuid, 734 ci->ci_flags, ci->ci_ipis, 735 ci->ci_curlwp, 736 ci->ci_fpcurlwp); 737 } 738 } 739 #endif 740 741 static void 742 cpu_copy_trampoline(void) 743 { 744 /* 745 * Copy boot code. 746 */ 747 extern u_char cpu_spinup_trampoline[]; 748 extern u_char cpu_spinup_trampoline_end[]; 749 750 vaddr_t mp_trampoline_vaddr; 751 752 mp_trampoline_vaddr = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, 753 UVM_KMF_VAONLY); 754 755 pmap_kenter_pa(mp_trampoline_vaddr, mp_trampoline_paddr, 756 VM_PROT_READ | VM_PROT_WRITE); 757 pmap_update(pmap_kernel()); 758 memcpy((void *)mp_trampoline_vaddr, 759 cpu_spinup_trampoline, 760 cpu_spinup_trampoline_end - cpu_spinup_trampoline); 761 762 pmap_kremove(mp_trampoline_vaddr, PAGE_SIZE); 763 pmap_update(pmap_kernel()); 764 uvm_km_free(kernel_map, mp_trampoline_vaddr, PAGE_SIZE, UVM_KMF_VAONLY); 765 } 766 767 #endif 768 769 #ifdef i386 770 static void 771 tss_init(struct i386tss *tss, void *stack, void *func) 772 { 773 memset(tss, 0, sizeof *tss); 774 tss->tss_esp0 = tss->tss_esp = (int)((char *)stack + USPACE - 16); 775 tss->tss_ss0 = GSEL(GDATA_SEL, SEL_KPL); 776 tss->__tss_cs = GSEL(GCODE_SEL, SEL_KPL); 777 tss->tss_fs = GSEL(GCPU_SEL, SEL_KPL); 778 tss->tss_gs = tss->__tss_es = tss->__tss_ds = 779 tss->__tss_ss = GSEL(GDATA_SEL, SEL_KPL); 780 tss->tss_cr3 = pmap_kernel()->pm_pdirpa; 781 tss->tss_esp = (int)((char *)stack + USPACE - 16); 782 tss->tss_ldt = GSEL(GLDT_SEL, SEL_KPL); 783 tss->__tss_eflags = PSL_MBO | PSL_NT; /* XXX not needed? */ 784 tss->__tss_eip = (int)func; 785 } 786 787 /* XXX */ 788 #define IDTVEC(name) __CONCAT(X, name) 789 typedef void (vector)(void); 790 extern vector IDTVEC(tss_trap08); 791 #ifdef DDB 792 extern vector Xintrddbipi; 793 extern int ddb_vec; 794 #endif 795 796 static void 797 cpu_set_tss_gates(struct cpu_info *ci) 798 { 799 struct segment_descriptor sd; 800 801 ci->ci_doubleflt_stack = (char *)uvm_km_alloc(kernel_map, USPACE, 0, 802 UVM_KMF_WIRED); 803 tss_init(&ci->ci_doubleflt_tss, ci->ci_doubleflt_stack, 804 IDTVEC(tss_trap08)); 805 setsegment(&sd, &ci->ci_doubleflt_tss, sizeof(struct i386tss) - 1, 806 SDT_SYS386TSS, SEL_KPL, 0, 0); 807 ci->ci_gdt[GTRAPTSS_SEL].sd = sd; 808 setgate(&idt[8], NULL, 0, SDT_SYSTASKGT, SEL_KPL, 809 GSEL(GTRAPTSS_SEL, SEL_KPL)); 810 811 #if defined(DDB) && defined(MULTIPROCESSOR) 812 /* 813 * Set up separate handler for the DDB IPI, so that it doesn't 814 * stomp on a possibly corrupted stack. 815 * 816 * XXX overwriting the gate set in db_machine_init. 817 * Should rearrange the code so that it's set only once. 818 */ 819 ci->ci_ddbipi_stack = (char *)uvm_km_alloc(kernel_map, USPACE, 0, 820 UVM_KMF_WIRED); 821 tss_init(&ci->ci_ddbipi_tss, ci->ci_ddbipi_stack, Xintrddbipi); 822 823 setsegment(&sd, &ci->ci_ddbipi_tss, sizeof(struct i386tss) - 1, 824 SDT_SYS386TSS, SEL_KPL, 0, 0); 825 ci->ci_gdt[GIPITSS_SEL].sd = sd; 826 827 setgate(&idt[ddb_vec], NULL, 0, SDT_SYSTASKGT, SEL_KPL, 828 GSEL(GIPITSS_SEL, SEL_KPL)); 829 #endif 830 } 831 #else 832 static void 833 cpu_set_tss_gates(struct cpu_info *ci) 834 { 835 836 } 837 #endif /* i386 */ 838 839 int 840 mp_cpu_start(struct cpu_info *ci, paddr_t target) 841 { 842 #if NLAPIC > 0 843 int error; 844 #endif 845 unsigned short dwordptr[2]; 846 847 /* 848 * Bootstrap code must be addressable in real mode 849 * and it must be page aligned. 850 */ 851 KASSERT(target < 0x10000 && target % PAGE_SIZE == 0); 852 853 /* 854 * "The BSP must initialize CMOS shutdown code to 0Ah ..." 855 */ 856 857 outb(IO_RTC, NVRAM_RESET); 858 outb(IO_RTC+1, NVRAM_RESET_JUMP); 859 860 /* 861 * "and the warm reset vector (DWORD based at 40:67) to point 862 * to the AP startup code ..." 863 */ 864 865 dwordptr[0] = 0; 866 dwordptr[1] = target >> 4; 867 868 memcpy((uint8_t *)cmos_data_mapping + 0x467, dwordptr, 4); 869 870 #if NLAPIC > 0 871 if ((cpu_feature & CPUID_APIC) == 0) { 872 aprint_error("mp_cpu_start: CPU does not have APIC\n"); 873 return ENODEV; 874 } 875 876 /* 877 * ... prior to executing the following sequence:" 878 */ 879 880 if (ci->ci_flags & CPUF_AP) { 881 error = x86_ipi_init(ci->ci_apicid); 882 if (error != 0) { 883 aprint_error_dev(ci->ci_dev, "%s: IPI not taken (1)\n", 884 __func__); 885 return error; 886 } 887 888 i8254_delay(10000); 889 890 error = x86_ipi(target / PAGE_SIZE, ci->ci_apicid, 891 LAPIC_DLMODE_STARTUP); 892 if (error != 0) { 893 aprint_error_dev(ci->ci_dev, "%s: IPI not taken (2)\n", 894 __func__); 895 return error; 896 } 897 i8254_delay(200); 898 899 error = x86_ipi(target / PAGE_SIZE, ci->ci_apicid, 900 LAPIC_DLMODE_STARTUP); 901 if (error != 0) { 902 aprint_error_dev(ci->ci_dev, "%s: IPI not taken (3)\n", 903 __func__); 904 return error; 905 } 906 i8254_delay(200); 907 } 908 #endif 909 return 0; 910 } 911 912 void 913 mp_cpu_start_cleanup(struct cpu_info *ci) 914 { 915 /* 916 * Ensure the NVRAM reset byte contains something vaguely sane. 917 */ 918 919 outb(IO_RTC, NVRAM_RESET); 920 outb(IO_RTC+1, NVRAM_RESET_RST); 921 } 922 923 #ifdef __x86_64__ 924 typedef void (vector)(void); 925 extern vector Xsyscall, Xsyscall32; 926 927 void 928 cpu_init_msrs(struct cpu_info *ci, bool full) 929 { 930 wrmsr(MSR_STAR, 931 ((uint64_t)GSEL(GCODE_SEL, SEL_KPL) << 32) | 932 ((uint64_t)LSEL(LSYSRETBASE_SEL, SEL_UPL) << 48)); 933 wrmsr(MSR_LSTAR, (uint64_t)Xsyscall); 934 wrmsr(MSR_CSTAR, (uint64_t)Xsyscall32); 935 wrmsr(MSR_SFMASK, PSL_NT|PSL_T|PSL_I|PSL_C); 936 937 if (full) { 938 wrmsr(MSR_FSBASE, 0); 939 wrmsr(MSR_GSBASE, (uint64_t)ci); 940 wrmsr(MSR_KERNELGSBASE, 0); 941 } 942 943 if (cpu_feature & CPUID_NOX) 944 wrmsr(MSR_EFER, rdmsr(MSR_EFER) | EFER_NXE); 945 } 946 #endif /* __x86_64__ */ 947 948 void 949 cpu_offline_md(void) 950 { 951 int s; 952 953 s = splhigh(); 954 #ifdef __i386__ 955 npxsave_cpu(true); 956 #else 957 fpusave_cpu(true); 958 #endif 959 splx(s); 960 } 961 962 /* XXX joerg restructure and restart CPUs individually */ 963 static bool 964 cpu_suspend(device_t dv PMF_FN_ARGS) 965 { 966 struct cpu_softc *sc = device_private(dv); 967 struct cpu_info *ci = sc->sc_info; 968 int err; 969 970 if (ci->ci_flags & CPUF_PRIMARY) 971 return true; 972 if (ci->ci_data.cpu_idlelwp == NULL) 973 return true; 974 if ((ci->ci_flags & CPUF_PRESENT) == 0) 975 return true; 976 977 sc->sc_wasonline = !(ci->ci_schedstate.spc_flags & SPCF_OFFLINE); 978 979 if (sc->sc_wasonline) { 980 mutex_enter(&cpu_lock); 981 err = cpu_setonline(ci, false); 982 mutex_exit(&cpu_lock); 983 984 if (err) 985 return false; 986 } 987 988 return true; 989 } 990 991 static bool 992 cpu_resume(device_t dv PMF_FN_ARGS) 993 { 994 struct cpu_softc *sc = device_private(dv); 995 struct cpu_info *ci = sc->sc_info; 996 int err = 0; 997 998 if (ci->ci_flags & CPUF_PRIMARY) 999 return true; 1000 if (ci->ci_data.cpu_idlelwp == NULL) 1001 return true; 1002 if ((ci->ci_flags & CPUF_PRESENT) == 0) 1003 return true; 1004 1005 if (sc->sc_wasonline) { 1006 mutex_enter(&cpu_lock); 1007 err = cpu_setonline(ci, true); 1008 mutex_exit(&cpu_lock); 1009 } 1010 1011 return err == 0; 1012 } 1013 1014 void 1015 cpu_get_tsc_freq(struct cpu_info *ci) 1016 { 1017 uint64_t last_tsc; 1018 u_int junk[4]; 1019 1020 if (ci->ci_feature_flags & CPUID_TSC) { 1021 /* Serialize. */ 1022 x86_cpuid(0, junk); 1023 last_tsc = rdtsc(); 1024 i8254_delay(100000); 1025 ci->ci_tsc_freq = (rdtsc() - last_tsc) * 10; 1026 } 1027 } 1028
Indexes created Thu Sep 25 16:09:42 GMT 2025