cpu.c revision 1.19
1 /* $NetBSD: cpu.c,v 1.19 2008/01/23 20:02:16 joerg Exp $ */ 2 3 /*- 4 * Copyright (c) 2000, 2006, 2007 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Bill Sommerfeld of RedBack Networks Inc, and by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 3. All advertising materials mentioning features or use of this software 19 * must display the following acknowledgement: 20 * This product includes software developed by the NetBSD 21 * Foundation, Inc. and its contributors. 22 * 4. Neither the name of The NetBSD Foundation nor the names of its 23 * contributors may be used to endorse or promote products derived 24 * from this software without specific prior written permission. 25 * 26 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 27 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 28 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 29 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 30 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 31 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 32 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 33 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 34 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 35 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 36 * POSSIBILITY OF SUCH DAMAGE. 37 */ 38 39 /* 40 * Copyright (c) 1999 Stefan Grefen 41 * 42 * Redistribution and use in source and binary forms, with or without 43 * modification, are permitted provided that the following conditions 44 * are met: 45 * 1. Redistributions of source code must retain the above copyright 46 * notice, this list of conditions and the following disclaimer. 47 * 2. Redistributions in binary form must reproduce the above copyright 48 * notice, this list of conditions and the following disclaimer in the 49 * documentation and/or other materials provided with the distribution. 50 * 3. All advertising materials mentioning features or use of this software 51 * must display the following acknowledgement: 52 * This product includes software developed by the NetBSD 53 * Foundation, Inc. and its contributors. 54 * 4. Neither the name of The NetBSD Foundation nor the names of its 55 * contributors may be used to endorse or promote products derived 56 * from this software without specific prior written permission. 57 * 58 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND ANY 59 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 60 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 61 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR AND CONTRIBUTORS BE LIABLE 62 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 63 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 64 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 65 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 66 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 67 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 68 * SUCH DAMAGE. 69 */ 70 71 #include <sys/cdefs.h> 72 __KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.19 2008/01/23 20:02:16 joerg Exp $"); 73 74 #include "opt_ddb.h" 75 #include "opt_multiprocessor.h" 76 #include "opt_mpbios.h" /* for MPDEBUG */ 77 #include "opt_mtrr.h" 78 79 #include "lapic.h" 80 #include "ioapic.h" 81 82 #include <sys/param.h> 83 #include <sys/proc.h> 84 #include <sys/user.h> 85 #include <sys/systm.h> 86 #include <sys/device.h> 87 #include <sys/malloc.h> 88 #include <sys/cpu.h> 89 #include <sys/atomic.h> 90 91 #include <uvm/uvm_extern.h> 92 93 #include <machine/cpufunc.h> 94 #include <machine/cpuvar.h> 95 #include <machine/pmap.h> 96 #include <machine/vmparam.h> 97 #include <machine/mpbiosvar.h> 98 #include <machine/pcb.h> 99 #include <machine/specialreg.h> 100 #include <machine/segments.h> 101 #include <machine/gdt.h> 102 #include <machine/mtrr.h> 103 #include <machine/pio.h> 104 105 #ifdef i386 106 #include <machine/tlog.h> 107 #endif 108 109 #if NLAPIC > 0 110 #include <machine/apicvar.h> 111 #include <machine/i82489reg.h> 112 #include <machine/i82489var.h> 113 #endif 114 115 #if NIOAPIC > 0 116 #include <machine/i82093var.h> 117 #endif 118 119 #include <dev/ic/mc146818reg.h> 120 #include <i386/isa/nvram.h> 121 #include <dev/isa/isareg.h> 122 123 int cpu_match(struct device *, struct cfdata *, void *); 124 void cpu_attach(struct device *, struct device *, void *); 125 126 static bool cpu_suspend(device_t); 127 static bool cpu_resume(device_t); 128 129 struct cpu_softc { 130 struct device sc_dev; /* device tree glue */ 131 struct cpu_info *sc_info; /* pointer to CPU info */ 132 }; 133 134 int mp_cpu_start(struct cpu_info *, paddr_t); 135 void mp_cpu_start_cleanup(struct cpu_info *); 136 const struct cpu_functions mp_cpu_funcs = { mp_cpu_start, NULL, 137 mp_cpu_start_cleanup }; 138 139 140 CFATTACH_DECL(cpu, sizeof(struct cpu_softc), 141 cpu_match, cpu_attach, NULL, NULL); 142 143 /* 144 * Statically-allocated CPU info for the primary CPU (or the only 145 * CPU, on uniprocessors). The CPU info list is initialized to 146 * point at it. 147 */ 148 #ifdef TRAPLOG 149 struct tlog tlog_primary; 150 #endif 151 struct cpu_info cpu_info_primary = { 152 .ci_dev = 0, 153 .ci_self = &cpu_info_primary, 154 .ci_idepth = -1, 155 .ci_curlwp = &lwp0, 156 #ifdef TRAPLOG 157 .ci_tlog_base = &tlog_primary, 158 #endif /* !TRAPLOG */ 159 }; 160 161 struct cpu_info *cpu_info_list = &cpu_info_primary; 162 163 static void cpu_set_tss_gates(struct cpu_info *); 164 165 #ifdef i386 166 static void tss_init(struct i386tss *, void *, void *); 167 #endif 168 169 #ifdef MULTIPROCESSOR 170 static void cpu_init_idle_lwp(struct cpu_info *); 171 #endif 172 173 uint32_t cpus_attached = 0; 174 uint32_t cpus_running = 0; 175 176 extern char x86_64_doubleflt_stack[]; 177 178 bool x86_mp_online; 179 paddr_t mp_trampoline_paddr = MP_TRAMPOLINE; 180 181 static vaddr_t cmos_data_mapping; 182 183 #ifdef MULTIPROCESSOR 184 /* 185 * Array of CPU info structures. Must be statically-allocated because 186 * curproc, etc. are used early. 187 */ 188 struct cpu_info *cpu_info[X86_MAXPROCS] = { &cpu_info_primary, }; 189 190 void cpu_hatch(void *); 191 static void cpu_boot_secondary(struct cpu_info *ci); 192 static void cpu_start_secondary(struct cpu_info *ci); 193 static void cpu_copy_trampoline(void); 194 195 /* 196 * Runs once per boot once multiprocessor goo has been detected and 197 * the local APIC on the boot processor has been mapped. 198 * 199 * Called from lapic_boot_init() (from mpbios_scan()). 200 */ 201 void 202 cpu_init_first(void) 203 { 204 int cpunum = lapic_cpu_number(); 205 206 if (cpunum != 0) { 207 cpu_info[0] = NULL; 208 cpu_info[cpunum] = &cpu_info_primary; 209 } 210 211 cpu_info_primary.ci_cpuid = cpunum; 212 cpu_copy_trampoline(); 213 214 cmos_data_mapping = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, UVM_KMF_VAONLY); 215 if (cmos_data_mapping == 0) 216 panic("No KVA for page 0"); 217 pmap_kenter_pa(cmos_data_mapping, 0, VM_PROT_READ|VM_PROT_WRITE); 218 pmap_update(pmap_kernel()); 219 } 220 #endif 221 222 int 223 cpu_match(struct device *parent, struct cfdata *match, 224 void *aux) 225 { 226 227 return 1; 228 } 229 230 static void 231 cpu_vm_init(struct cpu_info *ci) 232 { 233 int ncolors = 2, i; 234 235 for (i = CAI_ICACHE; i <= CAI_L2CACHE; i++) { 236 struct x86_cache_info *cai; 237 int tcolors; 238 239 cai = &ci->ci_cinfo[i]; 240 241 tcolors = atop(cai->cai_totalsize); 242 switch(cai->cai_associativity) { 243 case 0xff: 244 tcolors = 1; /* fully associative */ 245 break; 246 case 0: 247 case 1: 248 break; 249 default: 250 tcolors /= cai->cai_associativity; 251 } 252 ncolors = max(ncolors, tcolors); 253 } 254 255 /* 256 * Knowing the size of the largest cache on this CPU, re-color 257 * our pages. 258 */ 259 if (ncolors <= uvmexp.ncolors) 260 return; 261 aprint_verbose("%s: %d page colors\n", ci->ci_dev->dv_xname, ncolors); 262 uvm_page_recolor(ncolors); 263 } 264 265 266 void 267 cpu_attach(struct device *parent, struct device *self, void *aux) 268 { 269 struct cpu_softc *sc = (void *) self; 270 struct cpu_attach_args *caa = aux; 271 struct cpu_info *ci; 272 #if defined(MULTIPROCESSOR) 273 int cpunum = caa->cpu_number; 274 #endif 275 276 /* 277 * If we're an Application Processor, allocate a cpu_info 278 * structure, otherwise use the primary's. 279 */ 280 if (caa->cpu_role == CPU_ROLE_AP) { 281 aprint_naive(": Application Processor\n"); 282 ci = malloc(sizeof(*ci), M_DEVBUF, M_WAITOK); 283 memset(ci, 0, sizeof(*ci)); 284 #if defined(MULTIPROCESSOR) 285 if (cpu_info[cpunum] != NULL) { 286 printf("\n"); 287 panic("cpu at apic id %d already attached?", cpunum); 288 } 289 cpu_info[cpunum] = ci; 290 #endif 291 #ifdef TRAPLOG 292 ci->ci_tlog_base = malloc(sizeof(struct tlog), 293 M_DEVBUF, M_WAITOK); 294 #endif 295 } else { 296 aprint_naive(": %s Processor\n", 297 caa->cpu_role == CPU_ROLE_SP ? "Single" : "Boot"); 298 ci = &cpu_info_primary; 299 #if defined(MULTIPROCESSOR) 300 if (cpunum != lapic_cpu_number()) { 301 printf("\n"); 302 panic("%s: running CPU is at apic %d" 303 " instead of at expected %d", 304 sc->sc_dev.dv_xname, lapic_cpu_number(), cpunum); 305 } 306 #endif 307 } 308 309 ci->ci_self = ci; 310 sc->sc_info = ci; 311 312 ci->ci_dev = self; 313 ci->ci_apicid = caa->cpu_number; 314 #ifdef MULTIPROCESSOR 315 ci->ci_cpuid = ci->ci_apicid; 316 #else 317 ci->ci_cpuid = 0; /* False for APs, but they're not used anyway */ 318 #endif 319 ci->ci_cpumask = (1 << ci->ci_cpuid); 320 ci->ci_func = caa->cpu_func; 321 322 if (caa->cpu_role == CPU_ROLE_AP) { 323 #ifdef MULTIPROCESSOR 324 int error; 325 326 error = mi_cpu_attach(ci); 327 if (error != 0) { 328 aprint_normal("\n"); 329 aprint_error("%s: mi_cpu_attach failed with %d\n", 330 sc->sc_dev.dv_xname, error); 331 return; 332 } 333 #endif 334 cpu_init_tss(ci); 335 } else { 336 KASSERT(ci->ci_data.cpu_idlelwp != NULL); 337 } 338 339 pmap_reference(pmap_kernel()); 340 ci->ci_pmap = pmap_kernel(); 341 ci->ci_tlbstate = TLBSTATE_STALE; 342 343 /* further PCB init done later. */ 344 345 switch (caa->cpu_role) { 346 case CPU_ROLE_SP: 347 aprint_normal(": (uniprocessor)\n"); 348 atomic_or_32(&ci->ci_flags, 349 CPUF_PRESENT | CPUF_SP | CPUF_PRIMARY); 350 cpu_intr_init(ci); 351 identifycpu(ci); 352 cpu_init(ci); 353 cpu_set_tss_gates(ci); 354 pmap_cpu_init_late(ci); 355 x86_errata(); 356 break; 357 358 case CPU_ROLE_BP: 359 aprint_normal(": (boot processor)\n"); 360 atomic_or_32(&ci->ci_flags, 361 CPUF_PRESENT | CPUF_BSP | CPUF_PRIMARY); 362 cpu_intr_init(ci); 363 identifycpu(ci); 364 cpu_init(ci); 365 cpu_set_tss_gates(ci); 366 pmap_cpu_init_late(ci); 367 #if NLAPIC > 0 368 /* 369 * Enable local apic 370 */ 371 lapic_enable(); 372 lapic_set_lvt(); 373 lapic_calibrate_timer(ci); 374 #endif 375 #if NIOAPIC > 0 376 ioapic_bsp_id = caa->cpu_number; 377 #endif 378 x86_errata(); 379 break; 380 381 case CPU_ROLE_AP: 382 /* 383 * report on an AP 384 */ 385 aprint_normal(": (application processor)\n"); 386 387 #if defined(MULTIPROCESSOR) 388 cpu_intr_init(ci); 389 gdt_alloc_cpu(ci); 390 cpu_set_tss_gates(ci); 391 pmap_cpu_init_early(ci); 392 pmap_cpu_init_late(ci); 393 cpu_start_secondary(ci); 394 if (ci->ci_flags & CPUF_PRESENT) { 395 identifycpu(ci); 396 ci->ci_next = cpu_info_list->ci_next; 397 cpu_info_list->ci_next = ci; 398 } 399 #else 400 aprint_normal("%s: not started\n", sc->sc_dev.dv_xname); 401 #endif 402 break; 403 404 default: 405 printf("\n"); 406 panic("unknown processor type??\n"); 407 } 408 cpu_vm_init(ci); 409 410 cpus_attached |= ci->ci_cpumask; 411 412 if (!pmf_device_register(self, cpu_suspend, cpu_resume)) 413 aprint_error_dev(self, "couldn't establish power handler\n"); 414 415 #if defined(MULTIPROCESSOR) 416 if (mp_verbose) { 417 struct lwp *l = ci->ci_data.cpu_idlelwp; 418 419 aprint_verbose( 420 "%s: idle lwp at %p, idle sp at %p\n", 421 sc->sc_dev.dv_xname, l, 422 #ifdef i386 423 (void *)l->l_addr->u_pcb.pcb_esp 424 #else 425 (void *)l->l_addr->u_pcb.pcb_rsp 426 #endif 427 ); 428 } 429 #endif 430 } 431 432 /* 433 * Initialize the processor appropriately. 434 */ 435 436 void 437 cpu_init(struct cpu_info *ci) 438 { 439 /* configure the CPU if needed */ 440 if (ci->cpu_setup != NULL) 441 (*ci->cpu_setup)(ci); 442 443 #ifdef i386 444 /* 445 * On a 486 or above, enable ring 0 write protection. 446 */ 447 if (ci->ci_cpu_class >= CPUCLASS_486) 448 lcr0(rcr0() | CR0_WP); 449 #else 450 lcr0(rcr0() | CR0_WP); 451 #endif 452 453 /* 454 * On a P6 or above, enable global TLB caching if the 455 * hardware supports it. 456 */ 457 if (cpu_feature & CPUID_PGE) 458 lcr4(rcr4() | CR4_PGE); /* enable global TLB caching */ 459 460 /* 461 * If we have FXSAVE/FXRESTOR, use them. 462 */ 463 if (cpu_feature & CPUID_FXSR) { 464 lcr4(rcr4() | CR4_OSFXSR); 465 466 /* 467 * If we have SSE/SSE2, enable XMM exceptions. 468 */ 469 if (cpu_feature & (CPUID_SSE|CPUID_SSE2)) 470 lcr4(rcr4() | CR4_OSXMMEXCPT); 471 } 472 473 #ifdef MTRR 474 /* 475 * On a P6 or above, initialize MTRR's if the hardware supports them. 476 */ 477 if (cpu_feature & CPUID_MTRR) { 478 if ((ci->ci_flags & CPUF_AP) == 0) 479 i686_mtrr_init_first(); 480 mtrr_init_cpu(ci); 481 } 482 483 #ifdef i386 484 if (strcmp((char *)(ci->ci_vendor), "AuthenticAMD") == 0) { 485 /* 486 * Must be a K6-2 Step >= 7 or a K6-III. 487 */ 488 if (CPUID2FAMILY(ci->ci_signature) == 5) { 489 if (CPUID2MODEL(ci->ci_signature) > 8 || 490 (CPUID2MODEL(ci->ci_signature) == 8 && 491 CPUID2STEPPING(ci->ci_signature) >= 7)) { 492 mtrr_funcs = &k6_mtrr_funcs; 493 k6_mtrr_init_first(); 494 mtrr_init_cpu(ci); 495 } 496 } 497 } 498 #endif /* i386 */ 499 #endif /* MTRR */ 500 501 atomic_or_32(&ci->ci_flags, CPUF_RUNNING); 502 atomic_or_32(&cpus_running, ci->ci_cpumask); 503 504 #ifndef MULTIPROCESSOR 505 /* XXX */ 506 x86_patch(); 507 #endif 508 } 509 510 #ifdef MULTIPROCESSOR 511 void 512 cpu_boot_secondary_processors(void) 513 { 514 struct cpu_info *ci; 515 u_long i; 516 517 /* Now that we know the number of CPUs, patch the text segment. */ 518 x86_patch(); 519 520 for (i=0; i < X86_MAXPROCS; i++) { 521 ci = cpu_info[i]; 522 if (ci == NULL) 523 continue; 524 if (ci->ci_data.cpu_idlelwp == NULL) 525 continue; 526 if ((ci->ci_flags & CPUF_PRESENT) == 0) 527 continue; 528 if (ci->ci_flags & (CPUF_BSP|CPUF_SP|CPUF_PRIMARY)) 529 continue; 530 cpu_boot_secondary(ci); 531 } 532 533 x86_mp_online = true; 534 } 535 536 static void 537 cpu_init_idle_lwp(struct cpu_info *ci) 538 { 539 struct lwp *l = ci->ci_data.cpu_idlelwp; 540 struct pcb *pcb = &l->l_addr->u_pcb; 541 542 pcb->pcb_cr0 = rcr0(); 543 } 544 545 void 546 cpu_init_idle_lwps(void) 547 { 548 struct cpu_info *ci; 549 u_long i; 550 551 for (i = 0; i < X86_MAXPROCS; i++) { 552 ci = cpu_info[i]; 553 if (ci == NULL) 554 continue; 555 if (ci->ci_data.cpu_idlelwp == NULL) 556 continue; 557 if ((ci->ci_flags & CPUF_PRESENT) == 0) 558 continue; 559 cpu_init_idle_lwp(ci); 560 } 561 } 562 563 void 564 cpu_start_secondary(struct cpu_info *ci) 565 { 566 int i; 567 extern paddr_t mp_pdirpa; 568 569 mp_pdirpa = pmap_init_tmp_pgtbl(mp_trampoline_paddr); 570 571 atomic_or_32(&ci->ci_flags, CPUF_AP); 572 573 aprint_debug("%s: starting\n", ci->ci_dev->dv_xname); 574 575 ci->ci_curlwp = ci->ci_data.cpu_idlelwp; 576 CPU_STARTUP(ci, mp_trampoline_paddr); 577 578 /* 579 * wait for it to become ready 580 */ 581 for (i = 100000; (!(ci->ci_flags & CPUF_PRESENT)) && i>0;i--) { 582 i8254_delay(10); 583 } 584 if ((ci->ci_flags & CPUF_PRESENT) == 0) { 585 aprint_error("%s: failed to become ready\n", 586 ci->ci_dev->dv_xname); 587 #if defined(MPDEBUG) && defined(DDB) 588 printf("dropping into debugger; continue from here to resume boot\n"); 589 Debugger(); 590 #endif 591 } 592 593 CPU_START_CLEANUP(ci); 594 } 595 596 void 597 cpu_boot_secondary(struct cpu_info *ci) 598 { 599 int i; 600 601 atomic_or_32(&ci->ci_flags, CPUF_GO); 602 for (i = 100000; (!(ci->ci_flags & CPUF_RUNNING)) && i>0;i--) { 603 i8254_delay(10); 604 } 605 if ((ci->ci_flags & CPUF_RUNNING) == 0) { 606 aprint_error("%s: failed to start\n", ci->ci_dev->dv_xname); 607 #if defined(MPDEBUG) && defined(DDB) 608 printf("dropping into debugger; continue from here to resume boot\n"); 609 Debugger(); 610 #endif 611 } 612 } 613 614 /* 615 * The CPU ends up here when its ready to run 616 * This is called from code in mptramp.s; at this point, we are running 617 * in the idle pcb/idle stack of the new CPU. When this function returns, 618 * this processor will enter the idle loop and start looking for work. 619 */ 620 void 621 cpu_hatch(void *v) 622 { 623 struct cpu_info *ci = (struct cpu_info *)v; 624 int s, i; 625 626 #ifdef __x86_64__ 627 cpu_init_msrs(ci, true); 628 #endif 629 cpu_probe_features(ci); 630 cpu_feature &= ci->ci_feature_flags; 631 cpu_feature2 &= ci->ci_feature2_flags; 632 633 KDASSERT((ci->ci_flags & CPUF_PRESENT) == 0); 634 atomic_or_32(&ci->ci_flags, CPUF_PRESENT); 635 while ((ci->ci_flags & CPUF_GO) == 0) { 636 /* Don't use delay, boot CPU may be patching the text. */ 637 for (i = 10000; i != 0; i--) 638 x86_pause(); 639 } 640 641 /* Beacuse the text may have been patched in x86_patch(). */ 642 wbinvd(); 643 x86_flush(); 644 645 KASSERT((ci->ci_flags & CPUF_RUNNING) == 0); 646 647 lcr3(pmap_kernel()->pm_pdirpa); 648 curlwp->l_addr->u_pcb.pcb_cr3 = pmap_kernel()->pm_pdirpa; 649 lcr0(ci->ci_data.cpu_idlelwp->l_addr->u_pcb.pcb_cr0); 650 cpu_init_idt(); 651 gdt_init_cpu(ci); 652 lapic_enable(); 653 lapic_set_lvt(); 654 lapic_initclocks(); 655 656 #ifdef i386 657 npxinit(ci); 658 #else 659 fpuinit(ci); 660 #endif 661 lldt(GSYSSEL(GLDT_SEL, SEL_KPL)); 662 ltr(ci->ci_tss_sel); 663 664 cpu_init(ci); 665 cpu_get_tsc_freq(ci); 666 667 s = splhigh(); 668 #ifdef i386 669 lapic_tpr = 0; 670 #else 671 lcr8(0); 672 #endif 673 x86_enable_intr(); 674 splx(s); 675 x86_errata(); 676 677 aprint_debug("%s: CPU %ld running\n", ci->ci_dev->dv_xname, 678 (long)ci->ci_cpuid); 679 } 680 681 #if defined(DDB) 682 683 #include <ddb/db_output.h> 684 #include <machine/db_machdep.h> 685 686 /* 687 * Dump CPU information from ddb. 688 */ 689 void 690 cpu_debug_dump(void) 691 { 692 struct cpu_info *ci; 693 CPU_INFO_ITERATOR cii; 694 695 db_printf("addr dev id flags ipis curproc fpcurproc\n"); 696 for (CPU_INFO_FOREACH(cii, ci)) { 697 db_printf("%p %s %ld %x %x %10p %10p\n", 698 ci, 699 ci->ci_dev == NULL ? "BOOT" : ci->ci_dev->dv_xname, 700 (long)ci->ci_cpuid, 701 ci->ci_flags, ci->ci_ipis, 702 ci->ci_curlwp, 703 ci->ci_fpcurlwp); 704 } 705 } 706 #endif 707 708 static void 709 cpu_copy_trampoline(void) 710 { 711 /* 712 * Copy boot code. 713 */ 714 extern u_char cpu_spinup_trampoline[]; 715 extern u_char cpu_spinup_trampoline_end[]; 716 717 vaddr_t mp_trampoline_vaddr; 718 719 mp_trampoline_vaddr = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, 720 UVM_KMF_VAONLY); 721 722 pmap_kenter_pa(mp_trampoline_vaddr, mp_trampoline_paddr, 723 VM_PROT_READ | VM_PROT_WRITE); 724 pmap_update(pmap_kernel()); 725 memcpy((void *)mp_trampoline_vaddr, 726 cpu_spinup_trampoline, 727 cpu_spinup_trampoline_end-cpu_spinup_trampoline); 728 729 pmap_kremove(mp_trampoline_vaddr, PAGE_SIZE); 730 pmap_update(pmap_kernel()); 731 uvm_km_free(kernel_map, mp_trampoline_vaddr, PAGE_SIZE, UVM_KMF_VAONLY); 732 } 733 734 #endif 735 736 #ifdef i386 737 static void 738 tss_init(struct i386tss *tss, void *stack, void *func) 739 { 740 memset(tss, 0, sizeof *tss); 741 tss->tss_esp0 = tss->tss_esp = (int)((char *)stack + USPACE - 16); 742 tss->tss_ss0 = GSEL(GDATA_SEL, SEL_KPL); 743 tss->__tss_cs = GSEL(GCODE_SEL, SEL_KPL); 744 tss->tss_fs = GSEL(GCPU_SEL, SEL_KPL); 745 tss->tss_gs = tss->__tss_es = tss->__tss_ds = 746 tss->__tss_ss = GSEL(GDATA_SEL, SEL_KPL); 747 tss->tss_cr3 = pmap_kernel()->pm_pdirpa; 748 tss->tss_esp = (int)((char *)stack + USPACE - 16); 749 tss->tss_ldt = GSEL(GLDT_SEL, SEL_KPL); 750 tss->__tss_eflags = PSL_MBO | PSL_NT; /* XXX not needed? */ 751 tss->__tss_eip = (int)func; 752 } 753 754 /* XXX */ 755 #define IDTVEC(name) __CONCAT(X, name) 756 typedef void (vector)(void); 757 extern vector IDTVEC(tss_trap08); 758 #ifdef DDB 759 extern vector Xintrddbipi; 760 extern int ddb_vec; 761 #endif 762 763 static void 764 cpu_set_tss_gates(struct cpu_info *ci) 765 { 766 struct segment_descriptor sd; 767 768 ci->ci_doubleflt_stack = (char *)uvm_km_alloc(kernel_map, USPACE, 0, 769 UVM_KMF_WIRED); 770 tss_init(&ci->ci_doubleflt_tss, ci->ci_doubleflt_stack, 771 IDTVEC(tss_trap08)); 772 setsegment(&sd, &ci->ci_doubleflt_tss, sizeof(struct i386tss) - 1, 773 SDT_SYS386TSS, SEL_KPL, 0, 0); 774 ci->ci_gdt[GTRAPTSS_SEL].sd = sd; 775 setgate(&idt[8], NULL, 0, SDT_SYSTASKGT, SEL_KPL, 776 GSEL(GTRAPTSS_SEL, SEL_KPL)); 777 778 #if defined(DDB) && defined(MULTIPROCESSOR) 779 /* 780 * Set up separate handler for the DDB IPI, so that it doesn't 781 * stomp on a possibly corrupted stack. 782 * 783 * XXX overwriting the gate set in db_machine_init. 784 * Should rearrange the code so that it's set only once. 785 */ 786 ci->ci_ddbipi_stack = (char *)uvm_km_alloc(kernel_map, USPACE, 0, 787 UVM_KMF_WIRED); 788 tss_init(&ci->ci_ddbipi_tss, ci->ci_ddbipi_stack, Xintrddbipi); 789 790 setsegment(&sd, &ci->ci_ddbipi_tss, sizeof(struct i386tss) - 1, 791 SDT_SYS386TSS, SEL_KPL, 0, 0); 792 ci->ci_gdt[GIPITSS_SEL].sd = sd; 793 794 setgate(&idt[ddb_vec], NULL, 0, SDT_SYSTASKGT, SEL_KPL, 795 GSEL(GIPITSS_SEL, SEL_KPL)); 796 #endif 797 } 798 #else 799 static void 800 cpu_set_tss_gates(struct cpu_info *ci) 801 { 802 803 } 804 #endif /* i386 */ 805 806 int 807 mp_cpu_start(struct cpu_info *ci, paddr_t target) 808 { 809 #if NLAPIC > 0 810 int error; 811 #endif 812 unsigned short dwordptr[2]; 813 814 /* 815 * Bootstrap code must be addressable in real mode 816 * and it must be page aligned. 817 */ 818 KASSERT(target < 0x10000 && target % PAGE_SIZE == 0); 819 820 /* 821 * "The BSP must initialize CMOS shutdown code to 0Ah ..." 822 */ 823 824 outb(IO_RTC, NVRAM_RESET); 825 outb(IO_RTC+1, NVRAM_RESET_JUMP); 826 827 /* 828 * "and the warm reset vector (DWORD based at 40:67) to point 829 * to the AP startup code ..." 830 */ 831 832 dwordptr[0] = 0; 833 dwordptr[1] = target >> 4; 834 835 memcpy((uint8_t *)(cmos_data_mapping + 0x467), dwordptr, 4); 836 837 #if NLAPIC > 0 838 /* 839 * ... prior to executing the following sequence:" 840 */ 841 842 if (ci->ci_flags & CPUF_AP) { 843 if ((error = x86_ipi_init(ci->ci_apicid)) != 0) 844 return error; 845 846 i8254_delay(10000); 847 848 if (cpu_feature & CPUID_APIC) { 849 850 if ((error = x86_ipi(target / PAGE_SIZE, 851 ci->ci_apicid, 852 LAPIC_DLMODE_STARTUP)) != 0) 853 return error; 854 i8254_delay(200); 855 856 if ((error = x86_ipi(target / PAGE_SIZE, 857 ci->ci_apicid, 858 LAPIC_DLMODE_STARTUP)) != 0) 859 return error; 860 i8254_delay(200); 861 } 862 } 863 #endif 864 return 0; 865 } 866 867 void 868 mp_cpu_start_cleanup(struct cpu_info *ci) 869 { 870 /* 871 * Ensure the NVRAM reset byte contains something vaguely sane. 872 */ 873 874 outb(IO_RTC, NVRAM_RESET); 875 outb(IO_RTC+1, NVRAM_RESET_RST); 876 } 877 878 #ifdef __x86_64__ 879 typedef void (vector)(void); 880 extern vector Xsyscall, Xsyscall32; 881 882 void 883 cpu_init_msrs(struct cpu_info *ci, bool full) 884 { 885 wrmsr(MSR_STAR, 886 ((uint64_t)GSEL(GCODE_SEL, SEL_KPL) << 32) | 887 ((uint64_t)LSEL(LSYSRETBASE_SEL, SEL_UPL) << 48)); 888 wrmsr(MSR_LSTAR, (uint64_t)Xsyscall); 889 wrmsr(MSR_CSTAR, (uint64_t)Xsyscall32); 890 wrmsr(MSR_SFMASK, PSL_NT|PSL_T|PSL_I|PSL_C); 891 892 if (full) { 893 wrmsr(MSR_FSBASE, 0); 894 wrmsr(MSR_GSBASE, (u_int64_t)ci); 895 wrmsr(MSR_KERNELGSBASE, 0); 896 } 897 898 if (cpu_feature & CPUID_NOX) 899 wrmsr(MSR_EFER, rdmsr(MSR_EFER) | EFER_NXE); 900 } 901 #endif /* __x86_64__ */ 902 903 void 904 cpu_offline_md(void) 905 { 906 int s; 907 908 s = splhigh(); 909 #ifdef __i386__ 910 npxsave_cpu(true); 911 #else 912 fpusave_cpu(true); 913 #endif 914 splx(s); 915 } 916 917 /* XXX joerg restructure and restart CPUs individually */ 918 static bool 919 cpu_suspend(device_t dv) 920 { 921 struct cpu_softc *sc = device_private(dv); 922 struct cpu_info *ci = sc->sc_info; 923 int err; 924 925 if (ci->ci_flags & CPUF_PRIMARY) 926 return true; 927 if (ci->ci_data.cpu_idlelwp == NULL) 928 return true; 929 if ((ci->ci_flags & CPUF_PRESENT) == 0) 930 return true; 931 932 mutex_enter(&cpu_lock); 933 err = cpu_setonline(ci, false); 934 mutex_exit(&cpu_lock); 935 936 if (err) 937 return false; 938 939 return true; 940 } 941 942 static bool 943 cpu_resume(device_t dv) 944 { 945 struct cpu_softc *sc = device_private(dv); 946 struct cpu_info *ci = sc->sc_info; 947 int err; 948 949 if (ci->ci_flags & CPUF_PRIMARY) 950 return true; 951 if (ci->ci_data.cpu_idlelwp == NULL) 952 return true; 953 if ((ci->ci_flags & CPUF_PRESENT) == 0) 954 return true; 955 956 mutex_enter(&cpu_lock); 957 err = cpu_setonline(ci, true); 958 mutex_exit(&cpu_lock); 959 960 return err == 0; 961 } 962 963 void 964 cpu_get_tsc_freq(struct cpu_info *ci) 965 { 966 uint64_t last_tsc; 967 u_int junk[4]; 968 969 if (ci->ci_feature_flags & CPUID_TSC) { 970 /* Serialize. */ 971 x86_cpuid(0, junk); 972 last_tsc = rdtsc(); 973 i8254_delay(100000); 974 ci->ci_tsc_freq = (rdtsc() - last_tsc) * 10; 975 } 976 } 977
Indexes created Wed Sep 24 05:09:52 GMT 2025