cpu.c revision 1.85
1 /* $NetBSD: cpu.c,v 1.85 2011/02/24 13:58:39 jruoho Exp $ */ 2 3 /*- 4 * Copyright (c) 2000, 2006, 2007, 2008 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Bill Sommerfeld of RedBack Networks Inc, and by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * Copyright (c) 1999 Stefan Grefen 34 * 35 * Redistribution and use in source and binary forms, with or without 36 * modification, are permitted provided that the following conditions 37 * are met: 38 * 1. Redistributions of source code must retain the above copyright 39 * notice, this list of conditions and the following disclaimer. 40 * 2. Redistributions in binary form must reproduce the above copyright 41 * notice, this list of conditions and the following disclaimer in the 42 * documentation and/or other materials provided with the distribution. 43 * 3. All advertising materials mentioning features or use of this software 44 * must display the following acknowledgement: 45 * This product includes software developed by the NetBSD 46 * Foundation, Inc. and its contributors. 47 * 4. Neither the name of The NetBSD Foundation nor the names of its 48 * contributors may be used to endorse or promote products derived 49 * from this software without specific prior written permission. 50 * 51 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND ANY 52 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 53 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 54 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR AND CONTRIBUTORS BE LIABLE 55 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 56 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 57 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 58 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 59 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 60 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 61 * SUCH DAMAGE. 62 */ 63 64 #include <sys/cdefs.h> 65 __KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.85 2011/02/24 13:58:39 jruoho Exp $"); 66 67 #include "opt_ddb.h" 68 #include "opt_mpbios.h" /* for MPDEBUG */ 69 #include "opt_mtrr.h" 70 71 #include "lapic.h" 72 #include "ioapic.h" 73 74 #ifdef i386 75 #include "npx.h" 76 #endif 77 78 #include <sys/param.h> 79 #include <sys/proc.h> 80 #include <sys/systm.h> 81 #include <sys/device.h> 82 #include <sys/kmem.h> 83 #include <sys/cpu.h> 84 #include <sys/atomic.h> 85 #include <sys/reboot.h> 86 87 #include <uvm/uvm.h> 88 89 #include <machine/cpufunc.h> 90 #include <machine/cpuvar.h> 91 #include <machine/pmap.h> 92 #include <machine/vmparam.h> 93 #include <machine/mpbiosvar.h> 94 #include <machine/pcb.h> 95 #include <machine/specialreg.h> 96 #include <machine/segments.h> 97 #include <machine/gdt.h> 98 #include <machine/mtrr.h> 99 #include <machine/pio.h> 100 #include <machine/cpu_counter.h> 101 102 #ifdef i386 103 #include <machine/tlog.h> 104 #endif 105 106 #include <machine/apicvar.h> 107 #include <machine/i82489reg.h> 108 #include <machine/i82489var.h> 109 110 #include <dev/ic/mc146818reg.h> 111 #include <i386/isa/nvram.h> 112 #include <dev/isa/isareg.h> 113 114 #include "tsc.h" 115 116 #if MAXCPUS > 32 117 #error cpu_info contains 32bit bitmasks 118 #endif 119 120 int cpu_match(device_t, cfdata_t, void *); 121 void cpu_attach(device_t, device_t, void *); 122 int cpu_rescan(device_t, const char *, const int *); 123 void cpu_childdetached(device_t, device_t); 124 125 126 static bool cpu_suspend(device_t, const pmf_qual_t *); 127 static bool cpu_resume(device_t, const pmf_qual_t *); 128 static bool cpu_shutdown(device_t, int); 129 130 struct cpu_softc { 131 device_t sc_dev; /* device tree glue */ 132 struct cpu_info *sc_info; /* pointer to CPU info */ 133 bool sc_wasonline; 134 }; 135 136 int mp_cpu_start(struct cpu_info *, paddr_t); 137 void mp_cpu_start_cleanup(struct cpu_info *); 138 const struct cpu_functions mp_cpu_funcs = { mp_cpu_start, NULL, 139 mp_cpu_start_cleanup }; 140 141 142 CFATTACH_DECL2_NEW(cpu, sizeof(struct cpu_softc), 143 cpu_match, cpu_attach, NULL, NULL, cpu_rescan, cpu_childdetached); 144 145 /* 146 * Statically-allocated CPU info for the primary CPU (or the only 147 * CPU, on uniprocessors). The CPU info list is initialized to 148 * point at it. 149 */ 150 #ifdef TRAPLOG 151 struct tlog tlog_primary; 152 #endif 153 struct cpu_info cpu_info_primary __aligned(CACHE_LINE_SIZE) = { 154 .ci_dev = 0, 155 .ci_self = &cpu_info_primary, 156 .ci_idepth = -1, 157 .ci_curlwp = &lwp0, 158 .ci_curldt = -1, 159 #ifdef TRAPLOG 160 .ci_tlog_base = &tlog_primary, 161 #endif /* !TRAPLOG */ 162 }; 163 164 struct cpu_info *cpu_info_list = &cpu_info_primary; 165 166 static void cpu_set_tss_gates(struct cpu_info *); 167 168 #ifdef i386 169 static void tss_init(struct i386tss *, void *, void *); 170 #endif 171 172 static void cpu_init_idle_lwp(struct cpu_info *); 173 174 uint32_t cpus_attached = 0; 175 uint32_t cpus_running = 0; 176 177 uint32_t cpu_feature[5]; /* X86 CPUID feature bits 178 * [0] basic features %edx 179 * [1] basic features %ecx 180 * [2] extended features %edx 181 * [3] extended features %ecx 182 * [4] VIA padlock features 183 */ 184 185 extern char x86_64_doubleflt_stack[]; 186 187 bool x86_mp_online; 188 paddr_t mp_trampoline_paddr = MP_TRAMPOLINE; 189 static vaddr_t cmos_data_mapping; 190 struct cpu_info *cpu_starting; 191 192 void cpu_hatch(void *); 193 static void cpu_boot_secondary(struct cpu_info *ci); 194 static void cpu_start_secondary(struct cpu_info *ci); 195 static void cpu_copy_trampoline(void); 196 197 /* 198 * Runs once per boot once multiprocessor goo has been detected and 199 * the local APIC on the boot processor has been mapped. 200 * 201 * Called from lapic_boot_init() (from mpbios_scan()). 202 */ 203 void 204 cpu_init_first(void) 205 { 206 207 cpu_info_primary.ci_cpuid = lapic_cpu_number(); 208 cpu_copy_trampoline(); 209 210 cmos_data_mapping = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, UVM_KMF_VAONLY); 211 if (cmos_data_mapping == 0) 212 panic("No KVA for page 0"); 213 pmap_kenter_pa(cmos_data_mapping, 0, VM_PROT_READ|VM_PROT_WRITE, 0); 214 pmap_update(pmap_kernel()); 215 } 216 217 int 218 cpu_match(device_t parent, cfdata_t match, void *aux) 219 { 220 221 return 1; 222 } 223 224 static void 225 cpu_vm_init(struct cpu_info *ci) 226 { 227 int ncolors = 2, i; 228 229 for (i = CAI_ICACHE; i <= CAI_L2CACHE; i++) { 230 struct x86_cache_info *cai; 231 int tcolors; 232 233 cai = &ci->ci_cinfo[i]; 234 235 tcolors = atop(cai->cai_totalsize); 236 switch(cai->cai_associativity) { 237 case 0xff: 238 tcolors = 1; /* fully associative */ 239 break; 240 case 0: 241 case 1: 242 break; 243 default: 244 tcolors /= cai->cai_associativity; 245 } 246 ncolors = max(ncolors, tcolors); 247 /* 248 * If the desired number of colors is not a power of 249 * two, it won't be good. Find the greatest power of 250 * two which is an even divisor of the number of colors, 251 * to preserve even coloring of pages. 252 */ 253 if (ncolors & (ncolors - 1) ) { 254 int try, picked = 1; 255 for (try = 1; try < ncolors; try *= 2) { 256 if (ncolors % try == 0) picked = try; 257 } 258 if (picked == 1) { 259 panic("desired number of cache colors %d is " 260 " > 1, but not even!", ncolors); 261 } 262 ncolors = picked; 263 } 264 } 265 266 /* 267 * Knowing the size of the largest cache on this CPU, re-color 268 * our pages. 269 */ 270 if (ncolors <= uvmexp.ncolors) 271 return; 272 aprint_debug_dev(ci->ci_dev, "%d page colors\n", ncolors); 273 uvm_page_recolor(ncolors); 274 } 275 276 277 void 278 cpu_attach(device_t parent, device_t self, void *aux) 279 { 280 struct cpu_softc *sc = device_private(self); 281 struct cpu_attach_args *caa = aux; 282 struct cpu_info *ci; 283 uintptr_t ptr; 284 int cpunum = caa->cpu_number; 285 static bool again; 286 287 sc->sc_dev = self; 288 289 if (cpus_attached == ~0) { 290 aprint_error(": increase MAXCPUS\n"); 291 return; 292 } 293 294 /* 295 * If we're an Application Processor, allocate a cpu_info 296 * structure, otherwise use the primary's. 297 */ 298 if (caa->cpu_role == CPU_ROLE_AP) { 299 if ((boothowto & RB_MD1) != 0) { 300 aprint_error(": multiprocessor boot disabled\n"); 301 if (!pmf_device_register(self, NULL, NULL)) 302 aprint_error_dev(self, 303 "couldn't establish power handler\n"); 304 return; 305 } 306 aprint_naive(": Application Processor\n"); 307 ptr = (uintptr_t)kmem_zalloc(sizeof(*ci) + CACHE_LINE_SIZE - 1, 308 KM_SLEEP); 309 ci = (struct cpu_info *)roundup2(ptr, CACHE_LINE_SIZE); 310 ci->ci_curldt = -1; 311 #ifdef TRAPLOG 312 ci->ci_tlog_base = kmem_zalloc(sizeof(struct tlog), KM_SLEEP); 313 #endif 314 } else { 315 aprint_naive(": %s Processor\n", 316 caa->cpu_role == CPU_ROLE_SP ? "Single" : "Boot"); 317 ci = &cpu_info_primary; 318 if (cpunum != lapic_cpu_number()) { 319 /* XXX should be done earlier. */ 320 uint32_t reg; 321 aprint_verbose("\n"); 322 aprint_verbose_dev(self, "running CPU at apic %d" 323 " instead of at expected %d", lapic_cpu_number(), 324 cpunum); 325 reg = i82489_readreg(LAPIC_ID); 326 i82489_writereg(LAPIC_ID, (reg & ~LAPIC_ID_MASK) | 327 (cpunum << LAPIC_ID_SHIFT)); 328 } 329 if (cpunum != lapic_cpu_number()) { 330 aprint_error_dev(self, "unable to reset apic id\n"); 331 } 332 } 333 334 ci->ci_self = ci; 335 sc->sc_info = ci; 336 ci->ci_dev = self; 337 ci->ci_acpiid = caa->cpu_id; 338 ci->ci_cpuid = caa->cpu_number; 339 ci->ci_func = caa->cpu_func; 340 341 /* Must be before mi_cpu_attach(). */ 342 cpu_vm_init(ci); 343 344 if (caa->cpu_role == CPU_ROLE_AP) { 345 int error; 346 347 error = mi_cpu_attach(ci); 348 if (error != 0) { 349 aprint_normal("\n"); 350 aprint_error_dev(self, 351 "mi_cpu_attach failed with %d\n", error); 352 return; 353 } 354 cpu_init_tss(ci); 355 } else { 356 KASSERT(ci->ci_data.cpu_idlelwp != NULL); 357 } 358 359 ci->ci_cpumask = (1 << cpu_index(ci)); 360 pmap_reference(pmap_kernel()); 361 ci->ci_pmap = pmap_kernel(); 362 ci->ci_tlbstate = TLBSTATE_STALE; 363 364 /* 365 * Boot processor may not be attached first, but the below 366 * must be done to allow booting other processors. 367 */ 368 if (!again) { 369 atomic_or_32(&ci->ci_flags, CPUF_PRESENT | CPUF_PRIMARY); 370 /* Basic init. */ 371 cpu_intr_init(ci); 372 cpu_get_tsc_freq(ci); 373 cpu_init(ci); 374 cpu_set_tss_gates(ci); 375 pmap_cpu_init_late(ci); 376 if (caa->cpu_role != CPU_ROLE_SP) { 377 /* Enable lapic. */ 378 lapic_enable(); 379 lapic_set_lvt(); 380 lapic_calibrate_timer(ci); 381 } 382 /* Make sure DELAY() is initialized. */ 383 DELAY(1); 384 again = true; 385 } 386 387 /* further PCB init done later. */ 388 389 switch (caa->cpu_role) { 390 case CPU_ROLE_SP: 391 atomic_or_32(&ci->ci_flags, CPUF_SP); 392 cpu_identify(ci); 393 x86_errata(); 394 x86_cpu_idle_init(); 395 break; 396 397 case CPU_ROLE_BP: 398 atomic_or_32(&ci->ci_flags, CPUF_BSP); 399 cpu_identify(ci); 400 x86_errata(); 401 x86_cpu_idle_init(); 402 break; 403 404 case CPU_ROLE_AP: 405 /* 406 * report on an AP 407 */ 408 cpu_intr_init(ci); 409 gdt_alloc_cpu(ci); 410 cpu_set_tss_gates(ci); 411 pmap_cpu_init_early(ci); 412 pmap_cpu_init_late(ci); 413 cpu_start_secondary(ci); 414 if (ci->ci_flags & CPUF_PRESENT) { 415 struct cpu_info *tmp; 416 417 cpu_identify(ci); 418 tmp = cpu_info_list; 419 while (tmp->ci_next) 420 tmp = tmp->ci_next; 421 422 tmp->ci_next = ci; 423 } 424 break; 425 426 default: 427 aprint_normal("\n"); 428 panic("unknown processor type??\n"); 429 } 430 431 pat_init(ci); 432 atomic_or_32(&cpus_attached, ci->ci_cpumask); 433 434 if (!pmf_device_register1(self, cpu_suspend, cpu_resume, cpu_shutdown)) 435 aprint_error_dev(self, "couldn't establish power handler\n"); 436 437 if (mp_verbose) { 438 struct lwp *l = ci->ci_data.cpu_idlelwp; 439 struct pcb *pcb = lwp_getpcb(l); 440 441 aprint_verbose_dev(self, 442 "idle lwp at %p, idle sp at %p\n", 443 l, 444 #ifdef i386 445 (void *)pcb->pcb_esp 446 #else 447 (void *)pcb->pcb_rsp 448 #endif 449 ); 450 } 451 452 cpu_rescan(self, NULL, NULL); 453 } 454 455 int 456 cpu_rescan(device_t self, const char *ifattr, const int *locators) 457 { 458 struct cpu_softc *sc = device_private(self); 459 struct cpufeature_attach_args cfaa; 460 struct cpu_info *ci = sc->sc_info; 461 462 memset(&cfaa, 0, sizeof(cfaa)); 463 cfaa.ci = ci; 464 465 if (ifattr_match(ifattr, "cpufeaturebus")) { 466 467 if (ci->ci_frequency == NULL) { 468 cfaa.name = "est"; 469 ci->ci_frequency = config_found_ia(self, 470 "cpufeaturebus", &cfaa, NULL); 471 } 472 473 if (ci->ci_frequency == NULL) { 474 cfaa.name = "powernow"; 475 ci->ci_frequency = config_found_ia(self, 476 "cpufeaturebus", &cfaa, NULL); 477 } 478 479 if (ci->ci_padlock == NULL) { 480 cfaa.name = "padlock"; 481 ci->ci_padlock = config_found_ia(self, 482 "cpufeaturebus", &cfaa, NULL); 483 } 484 485 if (ci->ci_tempsensor == NULL) { 486 cfaa.name = "coretemp"; 487 ci->ci_tempsensor = config_found_ia(self, 488 "cpufeaturebus", &cfaa, NULL); 489 } 490 491 if (ci->ci_tempsensor == NULL) { 492 cfaa.name = "viac7temp"; 493 ci->ci_tempsensor = config_found_ia(self, 494 "cpufeaturebus", &cfaa, NULL); 495 } 496 } 497 498 return 0; 499 } 500 501 void 502 cpu_childdetached(device_t self, device_t child) 503 { 504 struct cpu_softc *sc = device_private(self); 505 struct cpu_info *ci = sc->sc_info; 506 507 if (ci->ci_frequency == child) 508 ci->ci_frequency = NULL; 509 510 if (ci->ci_padlock == child) 511 ci->ci_padlock = NULL; 512 513 if (ci->ci_tempsensor == child) 514 ci->ci_tempsensor = NULL; 515 } 516 517 /* 518 * Initialize the processor appropriately. 519 */ 520 521 void 522 cpu_init(struct cpu_info *ci) 523 { 524 525 lcr0(rcr0() | CR0_WP); 526 527 /* 528 * On a P6 or above, enable global TLB caching if the 529 * hardware supports it. 530 */ 531 if (cpu_feature[0] & CPUID_PGE) 532 lcr4(rcr4() | CR4_PGE); /* enable global TLB caching */ 533 534 /* 535 * If we have FXSAVE/FXRESTOR, use them. 536 */ 537 if (cpu_feature[0] & CPUID_FXSR) { 538 lcr4(rcr4() | CR4_OSFXSR); 539 540 /* 541 * If we have SSE/SSE2, enable XMM exceptions. 542 */ 543 if (cpu_feature[0] & (CPUID_SSE|CPUID_SSE2)) 544 lcr4(rcr4() | CR4_OSXMMEXCPT); 545 } 546 547 #ifdef MTRR 548 /* 549 * On a P6 or above, initialize MTRR's if the hardware supports them. 550 */ 551 if (cpu_feature[0] & CPUID_MTRR) { 552 if ((ci->ci_flags & CPUF_AP) == 0) 553 i686_mtrr_init_first(); 554 mtrr_init_cpu(ci); 555 } 556 557 #ifdef i386 558 if (strcmp((char *)(ci->ci_vendor), "AuthenticAMD") == 0) { 559 /* 560 * Must be a K6-2 Step >= 7 or a K6-III. 561 */ 562 if (CPUID2FAMILY(ci->ci_signature) == 5) { 563 if (CPUID2MODEL(ci->ci_signature) > 8 || 564 (CPUID2MODEL(ci->ci_signature) == 8 && 565 CPUID2STEPPING(ci->ci_signature) >= 7)) { 566 mtrr_funcs = &k6_mtrr_funcs; 567 k6_mtrr_init_first(); 568 mtrr_init_cpu(ci); 569 } 570 } 571 } 572 #endif /* i386 */ 573 #endif /* MTRR */ 574 575 atomic_or_32(&cpus_running, ci->ci_cpumask); 576 577 if (ci != &cpu_info_primary) { 578 /* Synchronize TSC again, and check for drift. */ 579 wbinvd(); 580 atomic_or_32(&ci->ci_flags, CPUF_RUNNING); 581 tsc_sync_ap(ci); 582 } else { 583 atomic_or_32(&ci->ci_flags, CPUF_RUNNING); 584 } 585 } 586 587 void 588 cpu_boot_secondary_processors(void) 589 { 590 struct cpu_info *ci; 591 u_long i; 592 593 /* Now that we know the number of CPUs, patch the text segment. */ 594 x86_patch(false); 595 596 for (i=0; i < maxcpus; i++) { 597 ci = cpu_lookup(i); 598 if (ci == NULL) 599 continue; 600 if (ci->ci_data.cpu_idlelwp == NULL) 601 continue; 602 if ((ci->ci_flags & CPUF_PRESENT) == 0) 603 continue; 604 if (ci->ci_flags & (CPUF_BSP|CPUF_SP|CPUF_PRIMARY)) 605 continue; 606 cpu_boot_secondary(ci); 607 } 608 609 x86_mp_online = true; 610 611 /* Now that we know about the TSC, attach the timecounter. */ 612 tsc_tc_init(); 613 614 /* Enable zeroing of pages in the idle loop if we have SSE2. */ 615 vm_page_zero_enable = ((cpu_feature[0] & CPUID_SSE2) != 0); 616 } 617 618 static void 619 cpu_init_idle_lwp(struct cpu_info *ci) 620 { 621 struct lwp *l = ci->ci_data.cpu_idlelwp; 622 struct pcb *pcb = lwp_getpcb(l); 623 624 pcb->pcb_cr0 = rcr0(); 625 } 626 627 void 628 cpu_init_idle_lwps(void) 629 { 630 struct cpu_info *ci; 631 u_long i; 632 633 for (i = 0; i < maxcpus; i++) { 634 ci = cpu_lookup(i); 635 if (ci == NULL) 636 continue; 637 if (ci->ci_data.cpu_idlelwp == NULL) 638 continue; 639 if ((ci->ci_flags & CPUF_PRESENT) == 0) 640 continue; 641 cpu_init_idle_lwp(ci); 642 } 643 } 644 645 void 646 cpu_start_secondary(struct cpu_info *ci) 647 { 648 extern paddr_t mp_pdirpa; 649 u_long psl; 650 int i; 651 652 mp_pdirpa = pmap_init_tmp_pgtbl(mp_trampoline_paddr); 653 atomic_or_32(&ci->ci_flags, CPUF_AP); 654 ci->ci_curlwp = ci->ci_data.cpu_idlelwp; 655 if (CPU_STARTUP(ci, mp_trampoline_paddr) != 0) { 656 return; 657 } 658 659 /* 660 * Wait for it to become ready. Setting cpu_starting opens the 661 * initial gate and allows the AP to start soft initialization. 662 */ 663 KASSERT(cpu_starting == NULL); 664 cpu_starting = ci; 665 for (i = 100000; (!(ci->ci_flags & CPUF_PRESENT)) && i > 0; i--) { 666 #ifdef MPDEBUG 667 extern int cpu_trace[3]; 668 static int otrace[3]; 669 if (memcmp(otrace, cpu_trace, sizeof(otrace)) != 0) { 670 aprint_debug_dev(ci->ci_dev, "trace %02x %02x %02x\n", 671 cpu_trace[0], cpu_trace[1], cpu_trace[2]); 672 memcpy(otrace, cpu_trace, sizeof(otrace)); 673 } 674 #endif 675 i8254_delay(10); 676 } 677 678 if ((ci->ci_flags & CPUF_PRESENT) == 0) { 679 aprint_error_dev(ci->ci_dev, "failed to become ready\n"); 680 #if defined(MPDEBUG) && defined(DDB) 681 printf("dropping into debugger; continue from here to resume boot\n"); 682 Debugger(); 683 #endif 684 } else { 685 /* 686 * Synchronize time stamp counters. Invalidate cache and do 687 * twice to try and minimize possible cache effects. Disable 688 * interrupts to try and rule out any external interference. 689 */ 690 psl = x86_read_psl(); 691 x86_disable_intr(); 692 wbinvd(); 693 tsc_sync_bp(ci); 694 x86_write_psl(psl); 695 } 696 697 CPU_START_CLEANUP(ci); 698 cpu_starting = NULL; 699 } 700 701 void 702 cpu_boot_secondary(struct cpu_info *ci) 703 { 704 int64_t drift; 705 u_long psl; 706 int i; 707 708 atomic_or_32(&ci->ci_flags, CPUF_GO); 709 for (i = 100000; (!(ci->ci_flags & CPUF_RUNNING)) && i > 0; i--) { 710 i8254_delay(10); 711 } 712 if ((ci->ci_flags & CPUF_RUNNING) == 0) { 713 aprint_error_dev(ci->ci_dev, "failed to start\n"); 714 #if defined(MPDEBUG) && defined(DDB) 715 printf("dropping into debugger; continue from here to resume boot\n"); 716 Debugger(); 717 #endif 718 } else { 719 /* Synchronize TSC again, check for drift. */ 720 drift = ci->ci_data.cpu_cc_skew; 721 psl = x86_read_psl(); 722 x86_disable_intr(); 723 wbinvd(); 724 tsc_sync_bp(ci); 725 x86_write_psl(psl); 726 drift -= ci->ci_data.cpu_cc_skew; 727 aprint_debug_dev(ci->ci_dev, "TSC skew=%lld drift=%lld\n", 728 (long long)ci->ci_data.cpu_cc_skew, (long long)drift); 729 tsc_sync_drift(drift); 730 } 731 } 732 733 /* 734 * The CPU ends up here when its ready to run 735 * This is called from code in mptramp.s; at this point, we are running 736 * in the idle pcb/idle stack of the new CPU. When this function returns, 737 * this processor will enter the idle loop and start looking for work. 738 */ 739 void 740 cpu_hatch(void *v) 741 { 742 struct cpu_info *ci = (struct cpu_info *)v; 743 struct pcb *pcb; 744 int s, i; 745 746 cpu_init_msrs(ci, true); 747 cpu_probe(ci); 748 749 ci->ci_data.cpu_cc_freq = cpu_info_primary.ci_data.cpu_cc_freq; 750 /* cpu_get_tsc_freq(ci); */ 751 752 KDASSERT((ci->ci_flags & CPUF_PRESENT) == 0); 753 754 /* 755 * Synchronize time stamp counters. Invalidate cache and do twice 756 * to try and minimize possible cache effects. Note that interrupts 757 * are off at this point. 758 */ 759 wbinvd(); 760 atomic_or_32(&ci->ci_flags, CPUF_PRESENT); 761 tsc_sync_ap(ci); 762 763 /* 764 * Wait to be brought online. Use 'monitor/mwait' if available, 765 * in order to make the TSC drift as much as possible. so that 766 * we can detect it later. If not available, try 'pause'. 767 * We'd like to use 'hlt', but we have interrupts off. 768 */ 769 while ((ci->ci_flags & CPUF_GO) == 0) { 770 if ((cpu_feature[1] & CPUID2_MONITOR) != 0) { 771 x86_monitor(&ci->ci_flags, 0, 0); 772 if ((ci->ci_flags & CPUF_GO) != 0) { 773 continue; 774 } 775 x86_mwait(0, 0); 776 } else { 777 for (i = 10000; i != 0; i--) { 778 x86_pause(); 779 } 780 } 781 } 782 783 /* Because the text may have been patched in x86_patch(). */ 784 wbinvd(); 785 x86_flush(); 786 787 KASSERT((ci->ci_flags & CPUF_RUNNING) == 0); 788 789 #ifdef PAE 790 pd_entry_t * l3_pd = ci->ci_pae_l3_pdir; 791 for (i = 0 ; i < PDP_SIZE; i++) { 792 l3_pd[i] = pmap_kernel()->pm_pdirpa[i] | PG_V; 793 } 794 lcr3(ci->ci_pae_l3_pdirpa); 795 #else 796 lcr3(pmap_pdirpa(pmap_kernel(), 0)); 797 #endif 798 799 pcb = lwp_getpcb(curlwp); 800 pcb->pcb_cr3 = rcr3(); 801 pcb = lwp_getpcb(ci->ci_data.cpu_idlelwp); 802 lcr0(pcb->pcb_cr0); 803 804 cpu_init_idt(); 805 gdt_init_cpu(ci); 806 lapic_enable(); 807 lapic_set_lvt(); 808 lapic_initclocks(); 809 810 #ifdef i386 811 #if NNPX > 0 812 npxinit(ci); 813 #endif 814 #else 815 fpuinit(ci); 816 #endif 817 lldt(GSYSSEL(GLDT_SEL, SEL_KPL)); 818 ltr(ci->ci_tss_sel); 819 820 cpu_init(ci); 821 cpu_get_tsc_freq(ci); 822 823 s = splhigh(); 824 #ifdef i386 825 lapic_tpr = 0; 826 #else 827 lcr8(0); 828 #endif 829 x86_enable_intr(); 830 splx(s); 831 x86_errata(); 832 833 aprint_debug_dev(ci->ci_dev, "running\n"); 834 } 835 836 #if defined(DDB) 837 838 #include <ddb/db_output.h> 839 #include <machine/db_machdep.h> 840 841 /* 842 * Dump CPU information from ddb. 843 */ 844 void 845 cpu_debug_dump(void) 846 { 847 struct cpu_info *ci; 848 CPU_INFO_ITERATOR cii; 849 850 db_printf("addr dev id flags ipis curlwp fpcurlwp\n"); 851 for (CPU_INFO_FOREACH(cii, ci)) { 852 db_printf("%p %s %ld %x %x %10p %10p\n", 853 ci, 854 ci->ci_dev == NULL ? "BOOT" : device_xname(ci->ci_dev), 855 (long)ci->ci_cpuid, 856 ci->ci_flags, ci->ci_ipis, 857 ci->ci_curlwp, 858 ci->ci_fpcurlwp); 859 } 860 } 861 #endif 862 863 static void 864 cpu_copy_trampoline(void) 865 { 866 /* 867 * Copy boot code. 868 */ 869 extern u_char cpu_spinup_trampoline[]; 870 extern u_char cpu_spinup_trampoline_end[]; 871 872 vaddr_t mp_trampoline_vaddr; 873 874 mp_trampoline_vaddr = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, 875 UVM_KMF_VAONLY); 876 877 pmap_kenter_pa(mp_trampoline_vaddr, mp_trampoline_paddr, 878 VM_PROT_READ | VM_PROT_WRITE, 0); 879 pmap_update(pmap_kernel()); 880 memcpy((void *)mp_trampoline_vaddr, 881 cpu_spinup_trampoline, 882 cpu_spinup_trampoline_end - cpu_spinup_trampoline); 883 884 pmap_kremove(mp_trampoline_vaddr, PAGE_SIZE); 885 pmap_update(pmap_kernel()); 886 uvm_km_free(kernel_map, mp_trampoline_vaddr, PAGE_SIZE, UVM_KMF_VAONLY); 887 } 888 889 #ifdef i386 890 static void 891 tss_init(struct i386tss *tss, void *stack, void *func) 892 { 893 KASSERT(curcpu()->ci_pmap == pmap_kernel()); 894 895 memset(tss, 0, sizeof *tss); 896 tss->tss_esp0 = tss->tss_esp = (int)((char *)stack + USPACE - 16); 897 tss->tss_ss0 = GSEL(GDATA_SEL, SEL_KPL); 898 tss->__tss_cs = GSEL(GCODE_SEL, SEL_KPL); 899 tss->tss_fs = GSEL(GCPU_SEL, SEL_KPL); 900 tss->tss_gs = tss->__tss_es = tss->__tss_ds = 901 tss->__tss_ss = GSEL(GDATA_SEL, SEL_KPL); 902 /* %cr3 contains the value associated to pmap_kernel */ 903 tss->tss_cr3 = rcr3(); 904 tss->tss_esp = (int)((char *)stack + USPACE - 16); 905 tss->tss_ldt = GSEL(GLDT_SEL, SEL_KPL); 906 tss->__tss_eflags = PSL_MBO | PSL_NT; /* XXX not needed? */ 907 tss->__tss_eip = (int)func; 908 } 909 910 /* XXX */ 911 #define IDTVEC(name) __CONCAT(X, name) 912 typedef void (vector)(void); 913 extern vector IDTVEC(tss_trap08); 914 #ifdef DDB 915 extern vector Xintrddbipi; 916 extern int ddb_vec; 917 #endif 918 919 static void 920 cpu_set_tss_gates(struct cpu_info *ci) 921 { 922 struct segment_descriptor sd; 923 924 ci->ci_doubleflt_stack = (char *)uvm_km_alloc(kernel_map, USPACE, 0, 925 UVM_KMF_WIRED); 926 tss_init(&ci->ci_doubleflt_tss, ci->ci_doubleflt_stack, 927 IDTVEC(tss_trap08)); 928 setsegment(&sd, &ci->ci_doubleflt_tss, sizeof(struct i386tss) - 1, 929 SDT_SYS386TSS, SEL_KPL, 0, 0); 930 ci->ci_gdt[GTRAPTSS_SEL].sd = sd; 931 setgate(&idt[8], NULL, 0, SDT_SYSTASKGT, SEL_KPL, 932 GSEL(GTRAPTSS_SEL, SEL_KPL)); 933 934 #if defined(DDB) 935 /* 936 * Set up separate handler for the DDB IPI, so that it doesn't 937 * stomp on a possibly corrupted stack. 938 * 939 * XXX overwriting the gate set in db_machine_init. 940 * Should rearrange the code so that it's set only once. 941 */ 942 ci->ci_ddbipi_stack = (char *)uvm_km_alloc(kernel_map, USPACE, 0, 943 UVM_KMF_WIRED); 944 tss_init(&ci->ci_ddbipi_tss, ci->ci_ddbipi_stack, Xintrddbipi); 945 946 setsegment(&sd, &ci->ci_ddbipi_tss, sizeof(struct i386tss) - 1, 947 SDT_SYS386TSS, SEL_KPL, 0, 0); 948 ci->ci_gdt[GIPITSS_SEL].sd = sd; 949 950 setgate(&idt[ddb_vec], NULL, 0, SDT_SYSTASKGT, SEL_KPL, 951 GSEL(GIPITSS_SEL, SEL_KPL)); 952 #endif 953 } 954 #else 955 static void 956 cpu_set_tss_gates(struct cpu_info *ci) 957 { 958 959 } 960 #endif /* i386 */ 961 962 int 963 mp_cpu_start(struct cpu_info *ci, paddr_t target) 964 { 965 unsigned short dwordptr[2]; 966 int error; 967 968 /* 969 * Bootstrap code must be addressable in real mode 970 * and it must be page aligned. 971 */ 972 KASSERT(target < 0x10000 && target % PAGE_SIZE == 0); 973 974 /* 975 * "The BSP must initialize CMOS shutdown code to 0Ah ..." 976 */ 977 978 outb(IO_RTC, NVRAM_RESET); 979 outb(IO_RTC+1, NVRAM_RESET_JUMP); 980 981 /* 982 * "and the warm reset vector (DWORD based at 40:67) to point 983 * to the AP startup code ..." 984 */ 985 986 dwordptr[0] = 0; 987 dwordptr[1] = target >> 4; 988 989 memcpy((uint8_t *)cmos_data_mapping + 0x467, dwordptr, 4); 990 991 if ((cpu_feature[0] & CPUID_APIC) == 0) { 992 aprint_error("mp_cpu_start: CPU does not have APIC\n"); 993 return ENODEV; 994 } 995 996 /* 997 * ... prior to executing the following sequence:". We'll also add in 998 * local cache flush, in case the BIOS has left the AP with its cache 999 * disabled. It may not be able to cope with MP coherency. 1000 */ 1001 wbinvd(); 1002 1003 if (ci->ci_flags & CPUF_AP) { 1004 error = x86_ipi_init(ci->ci_cpuid); 1005 if (error != 0) { 1006 aprint_error_dev(ci->ci_dev, "%s: IPI not taken (1)\n", 1007 __func__); 1008 return error; 1009 } 1010 i8254_delay(10000); 1011 1012 error = x86_ipi_startup(ci->ci_cpuid, target / PAGE_SIZE); 1013 if (error != 0) { 1014 aprint_error_dev(ci->ci_dev, "%s: IPI not taken (2)\n", 1015 __func__); 1016 return error; 1017 } 1018 i8254_delay(200); 1019 1020 error = x86_ipi_startup(ci->ci_cpuid, target / PAGE_SIZE); 1021 if (error != 0) { 1022 aprint_error_dev(ci->ci_dev, "%s: IPI not taken (3)\n", 1023 __func__); 1024 return error; 1025 } 1026 i8254_delay(200); 1027 } 1028 1029 return 0; 1030 } 1031 1032 void 1033 mp_cpu_start_cleanup(struct cpu_info *ci) 1034 { 1035 /* 1036 * Ensure the NVRAM reset byte contains something vaguely sane. 1037 */ 1038 1039 outb(IO_RTC, NVRAM_RESET); 1040 outb(IO_RTC+1, NVRAM_RESET_RST); 1041 } 1042 1043 #ifdef __x86_64__ 1044 typedef void (vector)(void); 1045 extern vector Xsyscall, Xsyscall32; 1046 #endif 1047 1048 void 1049 cpu_init_msrs(struct cpu_info *ci, bool full) 1050 { 1051 #ifdef __x86_64__ 1052 wrmsr(MSR_STAR, 1053 ((uint64_t)GSEL(GCODE_SEL, SEL_KPL) << 32) | 1054 ((uint64_t)LSEL(LSYSRETBASE_SEL, SEL_UPL) << 48)); 1055 wrmsr(MSR_LSTAR, (uint64_t)Xsyscall); 1056 wrmsr(MSR_CSTAR, (uint64_t)Xsyscall32); 1057 wrmsr(MSR_SFMASK, PSL_NT|PSL_T|PSL_I|PSL_C); 1058 1059 if (full) { 1060 wrmsr(MSR_FSBASE, 0); 1061 wrmsr(MSR_GSBASE, (uint64_t)ci); 1062 wrmsr(MSR_KERNELGSBASE, 0); 1063 } 1064 #endif /* __x86_64__ */ 1065 1066 if (cpu_feature[2] & CPUID_NOX) 1067 wrmsr(MSR_EFER, rdmsr(MSR_EFER) | EFER_NXE); 1068 } 1069 1070 void 1071 cpu_offline_md(void) 1072 { 1073 int s; 1074 1075 s = splhigh(); 1076 #ifdef i386 1077 #if NNPX > 0 1078 npxsave_cpu(true); 1079 #endif 1080 #else 1081 fpusave_cpu(true); 1082 #endif 1083 splx(s); 1084 } 1085 1086 /* XXX joerg restructure and restart CPUs individually */ 1087 static bool 1088 cpu_suspend(device_t dv, const pmf_qual_t *qual) 1089 { 1090 struct cpu_softc *sc = device_private(dv); 1091 struct cpu_info *ci = sc->sc_info; 1092 int err; 1093 1094 if (ci->ci_flags & CPUF_PRIMARY) 1095 return true; 1096 if (ci->ci_data.cpu_idlelwp == NULL) 1097 return true; 1098 if ((ci->ci_flags & CPUF_PRESENT) == 0) 1099 return true; 1100 1101 sc->sc_wasonline = !(ci->ci_schedstate.spc_flags & SPCF_OFFLINE); 1102 1103 if (sc->sc_wasonline) { 1104 mutex_enter(&cpu_lock); 1105 err = cpu_setstate(ci, false); 1106 mutex_exit(&cpu_lock); 1107 1108 if (err) 1109 return false; 1110 } 1111 1112 return true; 1113 } 1114 1115 static bool 1116 cpu_resume(device_t dv, const pmf_qual_t *qual) 1117 { 1118 struct cpu_softc *sc = device_private(dv); 1119 struct cpu_info *ci = sc->sc_info; 1120 int err = 0; 1121 1122 if (ci->ci_flags & CPUF_PRIMARY) 1123 return true; 1124 if (ci->ci_data.cpu_idlelwp == NULL) 1125 return true; 1126 if ((ci->ci_flags & CPUF_PRESENT) == 0) 1127 return true; 1128 1129 if (sc->sc_wasonline) { 1130 mutex_enter(&cpu_lock); 1131 err = cpu_setstate(ci, true); 1132 mutex_exit(&cpu_lock); 1133 } 1134 1135 return err == 0; 1136 } 1137 1138 static bool 1139 cpu_shutdown(device_t dv, int how) 1140 { 1141 return cpu_suspend(dv, NULL); 1142 } 1143 1144 void 1145 cpu_get_tsc_freq(struct cpu_info *ci) 1146 { 1147 uint64_t last_tsc; 1148 1149 if (cpu_hascounter()) { 1150 last_tsc = cpu_counter_serializing(); 1151 i8254_delay(100000); 1152 ci->ci_data.cpu_cc_freq = 1153 (cpu_counter_serializing() - last_tsc) * 10; 1154 } 1155 } 1156 1157 void 1158 x86_cpu_idle_mwait(void) 1159 { 1160 struct cpu_info *ci = curcpu(); 1161 1162 KASSERT(ci->ci_ilevel == IPL_NONE); 1163 1164 x86_monitor(&ci->ci_want_resched, 0, 0); 1165 if (__predict_false(ci->ci_want_resched)) { 1166 return; 1167 } 1168 x86_mwait(0, 0); 1169 } 1170 1171 void 1172 x86_cpu_idle_halt(void) 1173 { 1174 struct cpu_info *ci = curcpu(); 1175 1176 KASSERT(ci->ci_ilevel == IPL_NONE); 1177 1178 x86_disable_intr(); 1179 if (!__predict_false(ci->ci_want_resched)) { 1180 x86_stihlt(); 1181 } else { 1182 x86_enable_intr(); 1183 } 1184 } 1185 1186 /* 1187 * Loads pmap for the current CPU. 1188 */ 1189 void 1190 cpu_load_pmap(struct pmap *pmap) 1191 { 1192 #ifdef PAE 1193 int i, s; 1194 struct cpu_info *ci; 1195 1196 s = splvm(); /* just to be safe */ 1197 ci = curcpu(); 1198 pd_entry_t *l3_pd = ci->ci_pae_l3_pdir; 1199 for (i = 0 ; i < PDP_SIZE; i++) { 1200 l3_pd[i] = pmap->pm_pdirpa[i] | PG_V; 1201 } 1202 splx(s); 1203 tlbflush(); 1204 #else /* PAE */ 1205 lcr3(pmap_pdirpa(pmap, 0)); 1206 #endif /* PAE */ 1207 } 1208
Indexes created Thu Sep 25 02:09:54 GMT 2025