cpu.c revision 1.70
1 /* $NetBSD: cpu.c,v 1.70 2010/04/18 23:47:51 jym Exp $ */ 2 3 /*- 4 * Copyright (c) 2000, 2006, 2007, 2008 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Bill Sommerfeld of RedBack Networks Inc, and by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * Copyright (c) 1999 Stefan Grefen 34 * 35 * Redistribution and use in source and binary forms, with or without 36 * modification, are permitted provided that the following conditions 37 * are met: 38 * 1. Redistributions of source code must retain the above copyright 39 * notice, this list of conditions and the following disclaimer. 40 * 2. Redistributions in binary form must reproduce the above copyright 41 * notice, this list of conditions and the following disclaimer in the 42 * documentation and/or other materials provided with the distribution. 43 * 3. All advertising materials mentioning features or use of this software 44 * must display the following acknowledgement: 45 * This product includes software developed by the NetBSD 46 * Foundation, Inc. and its contributors. 47 * 4. Neither the name of The NetBSD Foundation nor the names of its 48 * contributors may be used to endorse or promote products derived 49 * from this software without specific prior written permission. 50 * 51 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND ANY 52 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 53 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 54 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR AND CONTRIBUTORS BE LIABLE 55 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 56 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 57 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 58 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 59 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 60 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 61 * SUCH DAMAGE. 62 */ 63 64 #include <sys/cdefs.h> 65 __KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.70 2010/04/18 23:47:51 jym Exp $"); 66 67 #include "opt_ddb.h" 68 #include "opt_mpbios.h" /* for MPDEBUG */ 69 #include "opt_mtrr.h" 70 71 #include "lapic.h" 72 #include "ioapic.h" 73 74 #ifdef i386 75 #include "npx.h" 76 #endif 77 78 #include <sys/param.h> 79 #include <sys/proc.h> 80 #include <sys/systm.h> 81 #include <sys/device.h> 82 #include <sys/kmem.h> 83 #include <sys/cpu.h> 84 #include <sys/atomic.h> 85 #include <sys/reboot.h> 86 87 #include <uvm/uvm_extern.h> 88 89 #include <machine/cpufunc.h> 90 #include <machine/cpuvar.h> 91 #include <machine/pmap.h> 92 #include <machine/vmparam.h> 93 #include <machine/mpbiosvar.h> 94 #include <machine/pcb.h> 95 #include <machine/specialreg.h> 96 #include <machine/segments.h> 97 #include <machine/gdt.h> 98 #include <machine/mtrr.h> 99 #include <machine/pio.h> 100 #include <machine/cpu_counter.h> 101 102 #ifdef i386 103 #include <machine/tlog.h> 104 #endif 105 106 #include <machine/apicvar.h> 107 #include <machine/i82489reg.h> 108 #include <machine/i82489var.h> 109 110 #include <dev/ic/mc146818reg.h> 111 #include <i386/isa/nvram.h> 112 #include <dev/isa/isareg.h> 113 114 #include "tsc.h" 115 116 #if MAXCPUS > 32 117 #error cpu_info contains 32bit bitmasks 118 #endif 119 120 int cpu_match(device_t, cfdata_t, void *); 121 void cpu_attach(device_t, device_t, void *); 122 123 static bool cpu_suspend(device_t, const pmf_qual_t *); 124 static bool cpu_resume(device_t, const pmf_qual_t *); 125 126 struct cpu_softc { 127 device_t sc_dev; /* device tree glue */ 128 struct cpu_info *sc_info; /* pointer to CPU info */ 129 bool sc_wasonline; 130 }; 131 132 int mp_cpu_start(struct cpu_info *, paddr_t); 133 void mp_cpu_start_cleanup(struct cpu_info *); 134 const struct cpu_functions mp_cpu_funcs = { mp_cpu_start, NULL, 135 mp_cpu_start_cleanup }; 136 137 138 CFATTACH_DECL_NEW(cpu, sizeof(struct cpu_softc), 139 cpu_match, cpu_attach, NULL, NULL); 140 141 /* 142 * Statically-allocated CPU info for the primary CPU (or the only 143 * CPU, on uniprocessors). The CPU info list is initialized to 144 * point at it. 145 */ 146 #ifdef TRAPLOG 147 struct tlog tlog_primary; 148 #endif 149 struct cpu_info cpu_info_primary __aligned(CACHE_LINE_SIZE) = { 150 .ci_dev = 0, 151 .ci_self = &cpu_info_primary, 152 .ci_idepth = -1, 153 .ci_curlwp = &lwp0, 154 .ci_curldt = -1, 155 #ifdef TRAPLOG 156 .ci_tlog_base = &tlog_primary, 157 #endif /* !TRAPLOG */ 158 }; 159 160 struct cpu_info *cpu_info_list = &cpu_info_primary; 161 162 static void cpu_set_tss_gates(struct cpu_info *); 163 164 #ifdef i386 165 static void tss_init(struct i386tss *, void *, void *); 166 #endif 167 168 static void cpu_init_idle_lwp(struct cpu_info *); 169 170 uint32_t cpus_attached = 0; 171 uint32_t cpus_running = 0; 172 173 uint32_t cpu_feature[5]; /* X86 CPUID feature bits 174 * [0] basic features %edx 175 * [1] basic features %ecx 176 * [2] extended features %edx 177 * [3] extended features %ecx 178 * [4] VIA padlock features 179 */ 180 181 extern char x86_64_doubleflt_stack[]; 182 183 bool x86_mp_online; 184 paddr_t mp_trampoline_paddr = MP_TRAMPOLINE; 185 static vaddr_t cmos_data_mapping; 186 struct cpu_info *cpu_starting; 187 188 void cpu_hatch(void *); 189 static void cpu_boot_secondary(struct cpu_info *ci); 190 static void cpu_start_secondary(struct cpu_info *ci); 191 static void cpu_copy_trampoline(void); 192 193 /* 194 * Runs once per boot once multiprocessor goo has been detected and 195 * the local APIC on the boot processor has been mapped. 196 * 197 * Called from lapic_boot_init() (from mpbios_scan()). 198 */ 199 void 200 cpu_init_first(void) 201 { 202 203 cpu_info_primary.ci_cpuid = lapic_cpu_number(); 204 cpu_copy_trampoline(); 205 206 cmos_data_mapping = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, UVM_KMF_VAONLY); 207 if (cmos_data_mapping == 0) 208 panic("No KVA for page 0"); 209 pmap_kenter_pa(cmos_data_mapping, 0, VM_PROT_READ|VM_PROT_WRITE, 0); 210 pmap_update(pmap_kernel()); 211 } 212 213 int 214 cpu_match(device_t parent, cfdata_t match, void *aux) 215 { 216 217 return 1; 218 } 219 220 static void 221 cpu_vm_init(struct cpu_info *ci) 222 { 223 int ncolors = 2, i; 224 225 for (i = CAI_ICACHE; i <= CAI_L2CACHE; i++) { 226 struct x86_cache_info *cai; 227 int tcolors; 228 229 cai = &ci->ci_cinfo[i]; 230 231 tcolors = atop(cai->cai_totalsize); 232 switch(cai->cai_associativity) { 233 case 0xff: 234 tcolors = 1; /* fully associative */ 235 break; 236 case 0: 237 case 1: 238 break; 239 default: 240 tcolors /= cai->cai_associativity; 241 } 242 ncolors = max(ncolors, tcolors); 243 /* 244 * If the desired number of colors is not a power of 245 * two, it won't be good. Find the greatest power of 246 * two which is an even divisor of the number of colors, 247 * to preserve even coloring of pages. 248 */ 249 if (ncolors & (ncolors - 1) ) { 250 int try, picked = 1; 251 for (try = 1; try < ncolors; try *= 2) { 252 if (ncolors % try == 0) picked = try; 253 } 254 if (picked == 1) { 255 panic("desired number of cache colors %d is " 256 " > 1, but not even!", ncolors); 257 } 258 ncolors = picked; 259 } 260 } 261 262 /* 263 * Knowing the size of the largest cache on this CPU, re-color 264 * our pages. 265 */ 266 if (ncolors <= uvmexp.ncolors) 267 return; 268 aprint_debug_dev(ci->ci_dev, "%d page colors\n", ncolors); 269 uvm_page_recolor(ncolors); 270 } 271 272 273 void 274 cpu_attach(device_t parent, device_t self, void *aux) 275 { 276 struct cpu_softc *sc = device_private(self); 277 struct cpu_attach_args *caa = aux; 278 struct cpu_info *ci; 279 uintptr_t ptr; 280 int cpunum = caa->cpu_number; 281 static bool again; 282 283 sc->sc_dev = self; 284 285 if (cpus_attached == ~0) { 286 aprint_error(": increase MAXCPUS\n"); 287 return; 288 } 289 290 /* 291 * If we're an Application Processor, allocate a cpu_info 292 * structure, otherwise use the primary's. 293 */ 294 if (caa->cpu_role == CPU_ROLE_AP) { 295 if ((boothowto & RB_MD1) != 0) { 296 aprint_error(": multiprocessor boot disabled\n"); 297 if (!pmf_device_register(self, NULL, NULL)) 298 aprint_error_dev(self, 299 "couldn't establish power handler\n"); 300 return; 301 } 302 aprint_naive(": Application Processor\n"); 303 ptr = (uintptr_t)kmem_alloc(sizeof(*ci) + CACHE_LINE_SIZE - 1, 304 KM_SLEEP); 305 ci = (struct cpu_info *)roundup2(ptr, CACHE_LINE_SIZE); 306 memset(ci, 0, sizeof(*ci)); 307 ci->ci_curldt = -1; 308 #ifdef TRAPLOG 309 ci->ci_tlog_base = kmem_zalloc(sizeof(struct tlog), KM_SLEEP); 310 #endif 311 } else { 312 aprint_naive(": %s Processor\n", 313 caa->cpu_role == CPU_ROLE_SP ? "Single" : "Boot"); 314 ci = &cpu_info_primary; 315 if (cpunum != lapic_cpu_number()) { 316 /* XXX should be done earlier. */ 317 uint32_t reg; 318 aprint_verbose("\n"); 319 aprint_verbose_dev(self, "running CPU at apic %d" 320 " instead of at expected %d", lapic_cpu_number(), 321 cpunum); 322 reg = i82489_readreg(LAPIC_ID); 323 i82489_writereg(LAPIC_ID, (reg & ~LAPIC_ID_MASK) | 324 (cpunum << LAPIC_ID_SHIFT)); 325 } 326 if (cpunum != lapic_cpu_number()) { 327 aprint_error_dev(self, "unable to reset apic id\n"); 328 } 329 } 330 331 ci->ci_self = ci; 332 sc->sc_info = ci; 333 ci->ci_dev = self; 334 ci->ci_cpuid = caa->cpu_number; 335 ci->ci_func = caa->cpu_func; 336 337 /* Must be before mi_cpu_attach(). */ 338 cpu_vm_init(ci); 339 340 if (caa->cpu_role == CPU_ROLE_AP) { 341 int error; 342 343 error = mi_cpu_attach(ci); 344 if (error != 0) { 345 aprint_normal("\n"); 346 aprint_error_dev(self, 347 "mi_cpu_attach failed with %d\n", error); 348 return; 349 } 350 cpu_init_tss(ci); 351 } else { 352 KASSERT(ci->ci_data.cpu_idlelwp != NULL); 353 } 354 355 ci->ci_cpumask = (1 << cpu_index(ci)); 356 pmap_reference(pmap_kernel()); 357 ci->ci_pmap = pmap_kernel(); 358 ci->ci_tlbstate = TLBSTATE_STALE; 359 360 /* 361 * Boot processor may not be attached first, but the below 362 * must be done to allow booting other processors. 363 */ 364 if (!again) { 365 atomic_or_32(&ci->ci_flags, CPUF_PRESENT | CPUF_PRIMARY); 366 /* Basic init. */ 367 cpu_intr_init(ci); 368 cpu_get_tsc_freq(ci); 369 cpu_init(ci); 370 cpu_set_tss_gates(ci); 371 pmap_cpu_init_late(ci); 372 if (caa->cpu_role != CPU_ROLE_SP) { 373 /* Enable lapic. */ 374 lapic_enable(); 375 lapic_set_lvt(); 376 lapic_calibrate_timer(ci); 377 } 378 /* Make sure DELAY() is initialized. */ 379 DELAY(1); 380 again = true; 381 } 382 383 /* further PCB init done later. */ 384 385 switch (caa->cpu_role) { 386 case CPU_ROLE_SP: 387 atomic_or_32(&ci->ci_flags, CPUF_SP); 388 cpu_identify(ci); 389 x86_errata(); 390 x86_cpu_idle_init(); 391 break; 392 393 case CPU_ROLE_BP: 394 atomic_or_32(&ci->ci_flags, CPUF_BSP); 395 cpu_identify(ci); 396 x86_errata(); 397 x86_cpu_idle_init(); 398 break; 399 400 case CPU_ROLE_AP: 401 /* 402 * report on an AP 403 */ 404 cpu_intr_init(ci); 405 gdt_alloc_cpu(ci); 406 cpu_set_tss_gates(ci); 407 pmap_cpu_init_early(ci); 408 pmap_cpu_init_late(ci); 409 cpu_start_secondary(ci); 410 if (ci->ci_flags & CPUF_PRESENT) { 411 struct cpu_info *tmp; 412 413 cpu_identify(ci); 414 tmp = cpu_info_list; 415 while (tmp->ci_next) 416 tmp = tmp->ci_next; 417 418 tmp->ci_next = ci; 419 } 420 break; 421 422 default: 423 aprint_normal("\n"); 424 panic("unknown processor type??\n"); 425 } 426 427 atomic_or_32(&cpus_attached, ci->ci_cpumask); 428 429 if (!pmf_device_register(self, cpu_suspend, cpu_resume)) 430 aprint_error_dev(self, "couldn't establish power handler\n"); 431 432 if (mp_verbose) { 433 struct lwp *l = ci->ci_data.cpu_idlelwp; 434 struct pcb *pcb = lwp_getpcb(l); 435 436 aprint_verbose_dev(self, 437 "idle lwp at %p, idle sp at %p\n", 438 l, 439 #ifdef i386 440 (void *)pcb->pcb_esp 441 #else 442 (void *)pcb->pcb_rsp 443 #endif 444 ); 445 } 446 } 447 448 /* 449 * Initialize the processor appropriately. 450 */ 451 452 void 453 cpu_init(struct cpu_info *ci) 454 { 455 456 lcr0(rcr0() | CR0_WP); 457 458 /* 459 * On a P6 or above, enable global TLB caching if the 460 * hardware supports it. 461 */ 462 if (cpu_feature[0] & CPUID_PGE) 463 lcr4(rcr4() | CR4_PGE); /* enable global TLB caching */ 464 465 /* 466 * If we have FXSAVE/FXRESTOR, use them. 467 */ 468 if (cpu_feature[0] & CPUID_FXSR) { 469 lcr4(rcr4() | CR4_OSFXSR); 470 471 /* 472 * If we have SSE/SSE2, enable XMM exceptions. 473 */ 474 if (cpu_feature[0] & (CPUID_SSE|CPUID_SSE2)) 475 lcr4(rcr4() | CR4_OSXMMEXCPT); 476 } 477 478 #ifdef MTRR 479 /* 480 * On a P6 or above, initialize MTRR's if the hardware supports them. 481 */ 482 if (cpu_feature[0] & CPUID_MTRR) { 483 if ((ci->ci_flags & CPUF_AP) == 0) 484 i686_mtrr_init_first(); 485 mtrr_init_cpu(ci); 486 } 487 488 #ifdef i386 489 if (strcmp((char *)(ci->ci_vendor), "AuthenticAMD") == 0) { 490 /* 491 * Must be a K6-2 Step >= 7 or a K6-III. 492 */ 493 if (CPUID2FAMILY(ci->ci_signature) == 5) { 494 if (CPUID2MODEL(ci->ci_signature) > 8 || 495 (CPUID2MODEL(ci->ci_signature) == 8 && 496 CPUID2STEPPING(ci->ci_signature) >= 7)) { 497 mtrr_funcs = &k6_mtrr_funcs; 498 k6_mtrr_init_first(); 499 mtrr_init_cpu(ci); 500 } 501 } 502 } 503 #endif /* i386 */ 504 #endif /* MTRR */ 505 506 atomic_or_32(&cpus_running, ci->ci_cpumask); 507 508 if (ci != &cpu_info_primary) { 509 /* Synchronize TSC again, and check for drift. */ 510 wbinvd(); 511 atomic_or_32(&ci->ci_flags, CPUF_RUNNING); 512 tsc_sync_ap(ci); 513 } else { 514 atomic_or_32(&ci->ci_flags, CPUF_RUNNING); 515 } 516 } 517 518 void 519 cpu_boot_secondary_processors(void) 520 { 521 struct cpu_info *ci; 522 u_long i; 523 524 /* Now that we know the number of CPUs, patch the text segment. */ 525 x86_patch(false); 526 527 for (i=0; i < maxcpus; i++) { 528 ci = cpu_lookup(i); 529 if (ci == NULL) 530 continue; 531 if (ci->ci_data.cpu_idlelwp == NULL) 532 continue; 533 if ((ci->ci_flags & CPUF_PRESENT) == 0) 534 continue; 535 if (ci->ci_flags & (CPUF_BSP|CPUF_SP|CPUF_PRIMARY)) 536 continue; 537 cpu_boot_secondary(ci); 538 } 539 540 x86_mp_online = true; 541 542 /* Now that we know about the TSC, attach the timecounter. */ 543 tsc_tc_init(); 544 545 /* Enable zeroing of pages in the idle loop if we have SSE2. */ 546 vm_page_zero_enable = ((cpu_feature[0] & CPUID_SSE2) != 0); 547 } 548 549 static void 550 cpu_init_idle_lwp(struct cpu_info *ci) 551 { 552 struct lwp *l = ci->ci_data.cpu_idlelwp; 553 struct pcb *pcb = lwp_getpcb(l); 554 555 pcb->pcb_cr0 = rcr0(); 556 } 557 558 void 559 cpu_init_idle_lwps(void) 560 { 561 struct cpu_info *ci; 562 u_long i; 563 564 for (i = 0; i < maxcpus; i++) { 565 ci = cpu_lookup(i); 566 if (ci == NULL) 567 continue; 568 if (ci->ci_data.cpu_idlelwp == NULL) 569 continue; 570 if ((ci->ci_flags & CPUF_PRESENT) == 0) 571 continue; 572 cpu_init_idle_lwp(ci); 573 } 574 } 575 576 void 577 cpu_start_secondary(struct cpu_info *ci) 578 { 579 extern paddr_t mp_pdirpa; 580 u_long psl; 581 int i; 582 583 mp_pdirpa = pmap_init_tmp_pgtbl(mp_trampoline_paddr); 584 atomic_or_32(&ci->ci_flags, CPUF_AP); 585 ci->ci_curlwp = ci->ci_data.cpu_idlelwp; 586 if (CPU_STARTUP(ci, mp_trampoline_paddr) != 0) { 587 return; 588 } 589 590 /* 591 * Wait for it to become ready. Setting cpu_starting opens the 592 * initial gate and allows the AP to start soft initialization. 593 */ 594 KASSERT(cpu_starting == NULL); 595 cpu_starting = ci; 596 for (i = 100000; (!(ci->ci_flags & CPUF_PRESENT)) && i > 0; i--) { 597 #ifdef MPDEBUG 598 extern int cpu_trace[3]; 599 static int otrace[3]; 600 if (memcmp(otrace, cpu_trace, sizeof(otrace)) != 0) { 601 aprint_debug_dev(ci->ci_dev, "trace %02x %02x %02x\n", 602 cpu_trace[0], cpu_trace[1], cpu_trace[2]); 603 memcpy(otrace, cpu_trace, sizeof(otrace)); 604 } 605 #endif 606 i8254_delay(10); 607 } 608 609 if ((ci->ci_flags & CPUF_PRESENT) == 0) { 610 aprint_error_dev(ci->ci_dev, "failed to become ready\n"); 611 #if defined(MPDEBUG) && defined(DDB) 612 printf("dropping into debugger; continue from here to resume boot\n"); 613 Debugger(); 614 #endif 615 } else { 616 /* 617 * Synchronize time stamp counters. Invalidate cache and do 618 * twice to try and minimize possible cache effects. Disable 619 * interrupts to try and rule out any external interference. 620 */ 621 psl = x86_read_psl(); 622 x86_disable_intr(); 623 wbinvd(); 624 tsc_sync_bp(ci); 625 x86_write_psl(psl); 626 } 627 628 CPU_START_CLEANUP(ci); 629 cpu_starting = NULL; 630 } 631 632 void 633 cpu_boot_secondary(struct cpu_info *ci) 634 { 635 int64_t drift; 636 u_long psl; 637 int i; 638 639 atomic_or_32(&ci->ci_flags, CPUF_GO); 640 for (i = 100000; (!(ci->ci_flags & CPUF_RUNNING)) && i > 0; i--) { 641 i8254_delay(10); 642 } 643 if ((ci->ci_flags & CPUF_RUNNING) == 0) { 644 aprint_error_dev(ci->ci_dev, "failed to start\n"); 645 #if defined(MPDEBUG) && defined(DDB) 646 printf("dropping into debugger; continue from here to resume boot\n"); 647 Debugger(); 648 #endif 649 } else { 650 /* Synchronize TSC again, check for drift. */ 651 drift = ci->ci_data.cpu_cc_skew; 652 psl = x86_read_psl(); 653 x86_disable_intr(); 654 wbinvd(); 655 tsc_sync_bp(ci); 656 x86_write_psl(psl); 657 drift -= ci->ci_data.cpu_cc_skew; 658 aprint_debug_dev(ci->ci_dev, "TSC skew=%lld drift=%lld\n", 659 (long long)ci->ci_data.cpu_cc_skew, (long long)drift); 660 tsc_sync_drift(drift); 661 } 662 } 663 664 /* 665 * The CPU ends up here when its ready to run 666 * This is called from code in mptramp.s; at this point, we are running 667 * in the idle pcb/idle stack of the new CPU. When this function returns, 668 * this processor will enter the idle loop and start looking for work. 669 */ 670 void 671 cpu_hatch(void *v) 672 { 673 struct cpu_info *ci = (struct cpu_info *)v; 674 struct pcb *pcb; 675 int s, i; 676 677 cpu_init_msrs(ci, true); 678 cpu_probe(ci); 679 680 ci->ci_data.cpu_cc_freq = cpu_info_primary.ci_data.cpu_cc_freq; 681 /* cpu_get_tsc_freq(ci); */ 682 683 KDASSERT((ci->ci_flags & CPUF_PRESENT) == 0); 684 685 /* 686 * Synchronize time stamp counters. Invalidate cache and do twice 687 * to try and minimize possible cache effects. Note that interrupts 688 * are off at this point. 689 */ 690 wbinvd(); 691 atomic_or_32(&ci->ci_flags, CPUF_PRESENT); 692 tsc_sync_ap(ci); 693 694 /* 695 * Wait to be brought online. Use 'monitor/mwait' if available, 696 * in order to make the TSC drift as much as possible. so that 697 * we can detect it later. If not available, try 'pause'. 698 * We'd like to use 'hlt', but we have interrupts off. 699 */ 700 while ((ci->ci_flags & CPUF_GO) == 0) { 701 if ((cpu_feature[1] & CPUID2_MONITOR) != 0) { 702 x86_monitor(&ci->ci_flags, 0, 0); 703 if ((ci->ci_flags & CPUF_GO) != 0) { 704 continue; 705 } 706 x86_mwait(0, 0); 707 } else { 708 for (i = 10000; i != 0; i--) { 709 x86_pause(); 710 } 711 } 712 } 713 714 /* Because the text may have been patched in x86_patch(). */ 715 wbinvd(); 716 x86_flush(); 717 718 KASSERT((ci->ci_flags & CPUF_RUNNING) == 0); 719 720 lcr3(pmap_kernel()->pm_pdirpa); 721 pcb = lwp_getpcb(curlwp); 722 pcb->pcb_cr3 = pmap_kernel()->pm_pdirpa; 723 pcb = lwp_getpcb(ci->ci_data.cpu_idlelwp); 724 lcr0(pcb->pcb_cr0); 725 726 cpu_init_idt(); 727 gdt_init_cpu(ci); 728 lapic_enable(); 729 lapic_set_lvt(); 730 lapic_initclocks(); 731 732 #ifdef i386 733 #if NNPX > 0 734 npxinit(ci); 735 #endif 736 #else 737 fpuinit(ci); 738 #endif 739 lldt(GSYSSEL(GLDT_SEL, SEL_KPL)); 740 ltr(ci->ci_tss_sel); 741 742 cpu_init(ci); 743 cpu_get_tsc_freq(ci); 744 745 s = splhigh(); 746 #ifdef i386 747 lapic_tpr = 0; 748 #else 749 lcr8(0); 750 #endif 751 x86_enable_intr(); 752 splx(s); 753 x86_errata(); 754 755 aprint_debug_dev(ci->ci_dev, "running\n"); 756 } 757 758 #if defined(DDB) 759 760 #include <ddb/db_output.h> 761 #include <machine/db_machdep.h> 762 763 /* 764 * Dump CPU information from ddb. 765 */ 766 void 767 cpu_debug_dump(void) 768 { 769 struct cpu_info *ci; 770 CPU_INFO_ITERATOR cii; 771 772 db_printf("addr dev id flags ipis curlwp fpcurlwp\n"); 773 for (CPU_INFO_FOREACH(cii, ci)) { 774 db_printf("%p %s %ld %x %x %10p %10p\n", 775 ci, 776 ci->ci_dev == NULL ? "BOOT" : device_xname(ci->ci_dev), 777 (long)ci->ci_cpuid, 778 ci->ci_flags, ci->ci_ipis, 779 ci->ci_curlwp, 780 ci->ci_fpcurlwp); 781 } 782 } 783 #endif 784 785 static void 786 cpu_copy_trampoline(void) 787 { 788 /* 789 * Copy boot code. 790 */ 791 extern u_char cpu_spinup_trampoline[]; 792 extern u_char cpu_spinup_trampoline_end[]; 793 794 vaddr_t mp_trampoline_vaddr; 795 796 mp_trampoline_vaddr = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, 797 UVM_KMF_VAONLY); 798 799 pmap_kenter_pa(mp_trampoline_vaddr, mp_trampoline_paddr, 800 VM_PROT_READ | VM_PROT_WRITE, 0); 801 pmap_update(pmap_kernel()); 802 memcpy((void *)mp_trampoline_vaddr, 803 cpu_spinup_trampoline, 804 cpu_spinup_trampoline_end - cpu_spinup_trampoline); 805 806 pmap_kremove(mp_trampoline_vaddr, PAGE_SIZE); 807 pmap_update(pmap_kernel()); 808 uvm_km_free(kernel_map, mp_trampoline_vaddr, PAGE_SIZE, UVM_KMF_VAONLY); 809 } 810 811 #ifdef i386 812 static void 813 tss_init(struct i386tss *tss, void *stack, void *func) 814 { 815 memset(tss, 0, sizeof *tss); 816 tss->tss_esp0 = tss->tss_esp = (int)((char *)stack + USPACE - 16); 817 tss->tss_ss0 = GSEL(GDATA_SEL, SEL_KPL); 818 tss->__tss_cs = GSEL(GCODE_SEL, SEL_KPL); 819 tss->tss_fs = GSEL(GCPU_SEL, SEL_KPL); 820 tss->tss_gs = tss->__tss_es = tss->__tss_ds = 821 tss->__tss_ss = GSEL(GDATA_SEL, SEL_KPL); 822 tss->tss_cr3 = pmap_kernel()->pm_pdirpa; 823 tss->tss_esp = (int)((char *)stack + USPACE - 16); 824 tss->tss_ldt = GSEL(GLDT_SEL, SEL_KPL); 825 tss->__tss_eflags = PSL_MBO | PSL_NT; /* XXX not needed? */ 826 tss->__tss_eip = (int)func; 827 } 828 829 /* XXX */ 830 #define IDTVEC(name) __CONCAT(X, name) 831 typedef void (vector)(void); 832 extern vector IDTVEC(tss_trap08); 833 #ifdef DDB 834 extern vector Xintrddbipi; 835 extern int ddb_vec; 836 #endif 837 838 static void 839 cpu_set_tss_gates(struct cpu_info *ci) 840 { 841 struct segment_descriptor sd; 842 843 ci->ci_doubleflt_stack = (char *)uvm_km_alloc(kernel_map, USPACE, 0, 844 UVM_KMF_WIRED); 845 tss_init(&ci->ci_doubleflt_tss, ci->ci_doubleflt_stack, 846 IDTVEC(tss_trap08)); 847 setsegment(&sd, &ci->ci_doubleflt_tss, sizeof(struct i386tss) - 1, 848 SDT_SYS386TSS, SEL_KPL, 0, 0); 849 ci->ci_gdt[GTRAPTSS_SEL].sd = sd; 850 setgate(&idt[8], NULL, 0, SDT_SYSTASKGT, SEL_KPL, 851 GSEL(GTRAPTSS_SEL, SEL_KPL)); 852 853 #if defined(DDB) 854 /* 855 * Set up separate handler for the DDB IPI, so that it doesn't 856 * stomp on a possibly corrupted stack. 857 * 858 * XXX overwriting the gate set in db_machine_init. 859 * Should rearrange the code so that it's set only once. 860 */ 861 ci->ci_ddbipi_stack = (char *)uvm_km_alloc(kernel_map, USPACE, 0, 862 UVM_KMF_WIRED); 863 tss_init(&ci->ci_ddbipi_tss, ci->ci_ddbipi_stack, Xintrddbipi); 864 865 setsegment(&sd, &ci->ci_ddbipi_tss, sizeof(struct i386tss) - 1, 866 SDT_SYS386TSS, SEL_KPL, 0, 0); 867 ci->ci_gdt[GIPITSS_SEL].sd = sd; 868 869 setgate(&idt[ddb_vec], NULL, 0, SDT_SYSTASKGT, SEL_KPL, 870 GSEL(GIPITSS_SEL, SEL_KPL)); 871 #endif 872 } 873 #else 874 static void 875 cpu_set_tss_gates(struct cpu_info *ci) 876 { 877 878 } 879 #endif /* i386 */ 880 881 int 882 mp_cpu_start(struct cpu_info *ci, paddr_t target) 883 { 884 unsigned short dwordptr[2]; 885 int error; 886 887 /* 888 * Bootstrap code must be addressable in real mode 889 * and it must be page aligned. 890 */ 891 KASSERT(target < 0x10000 && target % PAGE_SIZE == 0); 892 893 /* 894 * "The BSP must initialize CMOS shutdown code to 0Ah ..." 895 */ 896 897 outb(IO_RTC, NVRAM_RESET); 898 outb(IO_RTC+1, NVRAM_RESET_JUMP); 899 900 /* 901 * "and the warm reset vector (DWORD based at 40:67) to point 902 * to the AP startup code ..." 903 */ 904 905 dwordptr[0] = 0; 906 dwordptr[1] = target >> 4; 907 908 memcpy((uint8_t *)cmos_data_mapping + 0x467, dwordptr, 4); 909 910 if ((cpu_feature[0] & CPUID_APIC) == 0) { 911 aprint_error("mp_cpu_start: CPU does not have APIC\n"); 912 return ENODEV; 913 } 914 915 /* 916 * ... prior to executing the following sequence:". We'll also add in 917 * local cache flush, in case the BIOS has left the AP with its cache 918 * disabled. It may not be able to cope with MP coherency. 919 */ 920 wbinvd(); 921 922 if (ci->ci_flags & CPUF_AP) { 923 error = x86_ipi_init(ci->ci_cpuid); 924 if (error != 0) { 925 aprint_error_dev(ci->ci_dev, "%s: IPI not taken (1)\n", 926 __func__); 927 return error; 928 } 929 i8254_delay(10000); 930 931 error = x86_ipi_startup(ci->ci_cpuid, target / PAGE_SIZE); 932 if (error != 0) { 933 aprint_error_dev(ci->ci_dev, "%s: IPI not taken (2)\n", 934 __func__); 935 return error; 936 } 937 i8254_delay(200); 938 939 error = x86_ipi_startup(ci->ci_cpuid, target / PAGE_SIZE); 940 if (error != 0) { 941 aprint_error_dev(ci->ci_dev, "%s: IPI not taken (3)\n", 942 __func__); 943 return error; 944 } 945 i8254_delay(200); 946 } 947 948 return 0; 949 } 950 951 void 952 mp_cpu_start_cleanup(struct cpu_info *ci) 953 { 954 /* 955 * Ensure the NVRAM reset byte contains something vaguely sane. 956 */ 957 958 outb(IO_RTC, NVRAM_RESET); 959 outb(IO_RTC+1, NVRAM_RESET_RST); 960 } 961 962 #ifdef __x86_64__ 963 typedef void (vector)(void); 964 extern vector Xsyscall, Xsyscall32; 965 #endif 966 967 void 968 cpu_init_msrs(struct cpu_info *ci, bool full) 969 { 970 #ifdef __x86_64__ 971 wrmsr(MSR_STAR, 972 ((uint64_t)GSEL(GCODE_SEL, SEL_KPL) << 32) | 973 ((uint64_t)LSEL(LSYSRETBASE_SEL, SEL_UPL) << 48)); 974 wrmsr(MSR_LSTAR, (uint64_t)Xsyscall); 975 wrmsr(MSR_CSTAR, (uint64_t)Xsyscall32); 976 wrmsr(MSR_SFMASK, PSL_NT|PSL_T|PSL_I|PSL_C); 977 978 if (full) { 979 wrmsr(MSR_FSBASE, 0); 980 wrmsr(MSR_GSBASE, (uint64_t)ci); 981 wrmsr(MSR_KERNELGSBASE, 0); 982 } 983 #endif /* __x86_64__ */ 984 985 if (cpu_feature[2] & CPUID_NOX) 986 wrmsr(MSR_EFER, rdmsr(MSR_EFER) | EFER_NXE); 987 } 988 989 void 990 cpu_offline_md(void) 991 { 992 int s; 993 994 s = splhigh(); 995 #ifdef i386 996 #if NNPX > 0 997 npxsave_cpu(true); 998 #endif 999 #else 1000 fpusave_cpu(true); 1001 #endif 1002 splx(s); 1003 } 1004 1005 /* XXX joerg restructure and restart CPUs individually */ 1006 static bool 1007 cpu_suspend(device_t dv, const pmf_qual_t *qual) 1008 { 1009 struct cpu_softc *sc = device_private(dv); 1010 struct cpu_info *ci = sc->sc_info; 1011 int err; 1012 1013 if (ci->ci_flags & CPUF_PRIMARY) 1014 return true; 1015 if (ci->ci_data.cpu_idlelwp == NULL) 1016 return true; 1017 if ((ci->ci_flags & CPUF_PRESENT) == 0) 1018 return true; 1019 1020 sc->sc_wasonline = !(ci->ci_schedstate.spc_flags & SPCF_OFFLINE); 1021 1022 if (sc->sc_wasonline) { 1023 mutex_enter(&cpu_lock); 1024 err = cpu_setstate(ci, false); 1025 mutex_exit(&cpu_lock); 1026 1027 if (err) 1028 return false; 1029 } 1030 1031 return true; 1032 } 1033 1034 static bool 1035 cpu_resume(device_t dv, const pmf_qual_t *qual) 1036 { 1037 struct cpu_softc *sc = device_private(dv); 1038 struct cpu_info *ci = sc->sc_info; 1039 int err = 0; 1040 1041 if (ci->ci_flags & CPUF_PRIMARY) 1042 return true; 1043 if (ci->ci_data.cpu_idlelwp == NULL) 1044 return true; 1045 if ((ci->ci_flags & CPUF_PRESENT) == 0) 1046 return true; 1047 1048 if (sc->sc_wasonline) { 1049 mutex_enter(&cpu_lock); 1050 err = cpu_setstate(ci, true); 1051 mutex_exit(&cpu_lock); 1052 } 1053 1054 return err == 0; 1055 } 1056 1057 void 1058 cpu_get_tsc_freq(struct cpu_info *ci) 1059 { 1060 uint64_t last_tsc; 1061 1062 if (cpu_hascounter()) { 1063 last_tsc = rdmsr(MSR_TSC); 1064 i8254_delay(100000); 1065 ci->ci_data.cpu_cc_freq = (rdmsr(MSR_TSC) - last_tsc) * 10; 1066 } 1067 } 1068 1069 void 1070 x86_cpu_idle_mwait(void) 1071 { 1072 struct cpu_info *ci = curcpu(); 1073 1074 KASSERT(ci->ci_ilevel == IPL_NONE); 1075 1076 x86_monitor(&ci->ci_want_resched, 0, 0); 1077 if (__predict_false(ci->ci_want_resched)) { 1078 return; 1079 } 1080 x86_mwait(0, 0); 1081 } 1082 1083 void 1084 x86_cpu_idle_halt(void) 1085 { 1086 struct cpu_info *ci = curcpu(); 1087 1088 KASSERT(ci->ci_ilevel == IPL_NONE); 1089 1090 x86_disable_intr(); 1091 if (!__predict_false(ci->ci_want_resched)) { 1092 x86_stihlt(); 1093 } else { 1094 x86_enable_intr(); 1095 } 1096 } 1097
Indexes created Thu Sep 25 16:09:42 GMT 2025