cpu.c revision 1.78
1 /* $NetBSD: cpu.c,v 1.78 2010/11/06 11:46:04 uebayasi Exp $ */ 2 3 /*- 4 * Copyright (c) 2000, 2006, 2007, 2008 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Bill Sommerfeld of RedBack Networks Inc, and by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * Copyright (c) 1999 Stefan Grefen 34 * 35 * Redistribution and use in source and binary forms, with or without 36 * modification, are permitted provided that the following conditions 37 * are met: 38 * 1. Redistributions of source code must retain the above copyright 39 * notice, this list of conditions and the following disclaimer. 40 * 2. Redistributions in binary form must reproduce the above copyright 41 * notice, this list of conditions and the following disclaimer in the 42 * documentation and/or other materials provided with the distribution. 43 * 3. All advertising materials mentioning features or use of this software 44 * must display the following acknowledgement: 45 * This product includes software developed by the NetBSD 46 * Foundation, Inc. and its contributors. 47 * 4. Neither the name of The NetBSD Foundation nor the names of its 48 * contributors may be used to endorse or promote products derived 49 * from this software without specific prior written permission. 50 * 51 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND ANY 52 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 53 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 54 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR AND CONTRIBUTORS BE LIABLE 55 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 56 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 57 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 58 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 59 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 60 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 61 * SUCH DAMAGE. 62 */ 63 64 #include <sys/cdefs.h> 65 __KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.78 2010/11/06 11:46:04 uebayasi Exp $"); 66 67 #include "opt_ddb.h" 68 #include "opt_mpbios.h" /* for MPDEBUG */ 69 #include "opt_mtrr.h" 70 71 #include "lapic.h" 72 #include "ioapic.h" 73 74 #ifdef i386 75 #include "npx.h" 76 #endif 77 78 #include <sys/param.h> 79 #include <sys/proc.h> 80 #include <sys/systm.h> 81 #include <sys/device.h> 82 #include <sys/kmem.h> 83 #include <sys/cpu.h> 84 #include <sys/atomic.h> 85 #include <sys/reboot.h> 86 87 #include <uvm/uvm.h> 88 89 #include <machine/cpufunc.h> 90 #include <machine/cpuvar.h> 91 #include <machine/pmap.h> 92 #include <machine/vmparam.h> 93 #include <machine/mpbiosvar.h> 94 #include <machine/pcb.h> 95 #include <machine/specialreg.h> 96 #include <machine/segments.h> 97 #include <machine/gdt.h> 98 #include <machine/mtrr.h> 99 #include <machine/pio.h> 100 #include <machine/cpu_counter.h> 101 102 #ifdef i386 103 #include <machine/tlog.h> 104 #endif 105 106 #include <machine/apicvar.h> 107 #include <machine/i82489reg.h> 108 #include <machine/i82489var.h> 109 110 #include <dev/ic/mc146818reg.h> 111 #include <i386/isa/nvram.h> 112 #include <dev/isa/isareg.h> 113 114 #include "tsc.h" 115 116 #if MAXCPUS > 32 117 #error cpu_info contains 32bit bitmasks 118 #endif 119 120 int cpu_match(device_t, cfdata_t, void *); 121 void cpu_attach(device_t, device_t, void *); 122 123 static bool cpu_suspend(device_t, const pmf_qual_t *); 124 static bool cpu_resume(device_t, const pmf_qual_t *); 125 126 struct cpu_softc { 127 device_t sc_dev; /* device tree glue */ 128 struct cpu_info *sc_info; /* pointer to CPU info */ 129 bool sc_wasonline; 130 }; 131 132 int mp_cpu_start(struct cpu_info *, paddr_t); 133 void mp_cpu_start_cleanup(struct cpu_info *); 134 const struct cpu_functions mp_cpu_funcs = { mp_cpu_start, NULL, 135 mp_cpu_start_cleanup }; 136 137 138 CFATTACH_DECL_NEW(cpu, sizeof(struct cpu_softc), 139 cpu_match, cpu_attach, NULL, NULL); 140 141 /* 142 * Statically-allocated CPU info for the primary CPU (or the only 143 * CPU, on uniprocessors). The CPU info list is initialized to 144 * point at it. 145 */ 146 #ifdef TRAPLOG 147 struct tlog tlog_primary; 148 #endif 149 struct cpu_info cpu_info_primary __aligned(CACHE_LINE_SIZE) = { 150 .ci_dev = 0, 151 .ci_self = &cpu_info_primary, 152 .ci_idepth = -1, 153 .ci_curlwp = &lwp0, 154 .ci_curldt = -1, 155 #ifdef TRAPLOG 156 .ci_tlog_base = &tlog_primary, 157 #endif /* !TRAPLOG */ 158 }; 159 160 struct cpu_info *cpu_info_list = &cpu_info_primary; 161 162 static void cpu_set_tss_gates(struct cpu_info *); 163 164 #ifdef i386 165 static void tss_init(struct i386tss *, void *, void *); 166 #endif 167 168 static void cpu_init_idle_lwp(struct cpu_info *); 169 170 uint32_t cpus_attached = 0; 171 uint32_t cpus_running = 0; 172 173 uint32_t cpu_feature[5]; /* X86 CPUID feature bits 174 * [0] basic features %edx 175 * [1] basic features %ecx 176 * [2] extended features %edx 177 * [3] extended features %ecx 178 * [4] VIA padlock features 179 */ 180 181 extern char x86_64_doubleflt_stack[]; 182 183 bool x86_mp_online; 184 paddr_t mp_trampoline_paddr = MP_TRAMPOLINE; 185 static vaddr_t cmos_data_mapping; 186 struct cpu_info *cpu_starting; 187 188 void cpu_hatch(void *); 189 static void cpu_boot_secondary(struct cpu_info *ci); 190 static void cpu_start_secondary(struct cpu_info *ci); 191 static void cpu_copy_trampoline(void); 192 193 /* 194 * Runs once per boot once multiprocessor goo has been detected and 195 * the local APIC on the boot processor has been mapped. 196 * 197 * Called from lapic_boot_init() (from mpbios_scan()). 198 */ 199 void 200 cpu_init_first(void) 201 { 202 203 cpu_info_primary.ci_cpuid = lapic_cpu_number(); 204 cpu_copy_trampoline(); 205 206 cmos_data_mapping = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, UVM_KMF_VAONLY); 207 if (cmos_data_mapping == 0) 208 panic("No KVA for page 0"); 209 pmap_kenter_pa(cmos_data_mapping, 0, VM_PROT_READ|VM_PROT_WRITE, 0); 210 pmap_update(pmap_kernel()); 211 } 212 213 int 214 cpu_match(device_t parent, cfdata_t match, void *aux) 215 { 216 217 return 1; 218 } 219 220 static void 221 cpu_vm_init(struct cpu_info *ci) 222 { 223 int ncolors = 2, i; 224 225 for (i = CAI_ICACHE; i <= CAI_L2CACHE; i++) { 226 struct x86_cache_info *cai; 227 int tcolors; 228 229 cai = &ci->ci_cinfo[i]; 230 231 tcolors = atop(cai->cai_totalsize); 232 switch(cai->cai_associativity) { 233 case 0xff: 234 tcolors = 1; /* fully associative */ 235 break; 236 case 0: 237 case 1: 238 break; 239 default: 240 tcolors /= cai->cai_associativity; 241 } 242 ncolors = max(ncolors, tcolors); 243 /* 244 * If the desired number of colors is not a power of 245 * two, it won't be good. Find the greatest power of 246 * two which is an even divisor of the number of colors, 247 * to preserve even coloring of pages. 248 */ 249 if (ncolors & (ncolors - 1) ) { 250 int try, picked = 1; 251 for (try = 1; try < ncolors; try *= 2) { 252 if (ncolors % try == 0) picked = try; 253 } 254 if (picked == 1) { 255 panic("desired number of cache colors %d is " 256 " > 1, but not even!", ncolors); 257 } 258 ncolors = picked; 259 } 260 } 261 262 /* 263 * Knowing the size of the largest cache on this CPU, re-color 264 * our pages. 265 */ 266 if (ncolors <= uvmexp.ncolors) 267 return; 268 aprint_debug_dev(ci->ci_dev, "%d page colors\n", ncolors); 269 uvm_page_recolor(ncolors); 270 } 271 272 273 void 274 cpu_attach(device_t parent, device_t self, void *aux) 275 { 276 struct cpu_softc *sc = device_private(self); 277 struct cpu_attach_args *caa = aux; 278 struct cpu_info *ci; 279 uintptr_t ptr; 280 int cpunum = caa->cpu_number; 281 static bool again; 282 283 sc->sc_dev = self; 284 285 if (cpus_attached == ~0) { 286 aprint_error(": increase MAXCPUS\n"); 287 return; 288 } 289 290 /* 291 * If we're an Application Processor, allocate a cpu_info 292 * structure, otherwise use the primary's. 293 */ 294 if (caa->cpu_role == CPU_ROLE_AP) { 295 if ((boothowto & RB_MD1) != 0) { 296 aprint_error(": multiprocessor boot disabled\n"); 297 if (!pmf_device_register(self, NULL, NULL)) 298 aprint_error_dev(self, 299 "couldn't establish power handler\n"); 300 return; 301 } 302 aprint_naive(": Application Processor\n"); 303 ptr = (uintptr_t)kmem_zalloc(sizeof(*ci) + CACHE_LINE_SIZE - 1, 304 KM_SLEEP); 305 ci = (struct cpu_info *)roundup2(ptr, CACHE_LINE_SIZE); 306 ci->ci_curldt = -1; 307 #ifdef TRAPLOG 308 ci->ci_tlog_base = kmem_zalloc(sizeof(struct tlog), KM_SLEEP); 309 #endif 310 } else { 311 aprint_naive(": %s Processor\n", 312 caa->cpu_role == CPU_ROLE_SP ? "Single" : "Boot"); 313 ci = &cpu_info_primary; 314 if (cpunum != lapic_cpu_number()) { 315 /* XXX should be done earlier. */ 316 uint32_t reg; 317 aprint_verbose("\n"); 318 aprint_verbose_dev(self, "running CPU at apic %d" 319 " instead of at expected %d", lapic_cpu_number(), 320 cpunum); 321 reg = i82489_readreg(LAPIC_ID); 322 i82489_writereg(LAPIC_ID, (reg & ~LAPIC_ID_MASK) | 323 (cpunum << LAPIC_ID_SHIFT)); 324 } 325 if (cpunum != lapic_cpu_number()) { 326 aprint_error_dev(self, "unable to reset apic id\n"); 327 } 328 } 329 330 ci->ci_self = ci; 331 sc->sc_info = ci; 332 ci->ci_dev = self; 333 ci->ci_acpiid = caa->cpu_id; 334 ci->ci_cpuid = caa->cpu_number; 335 ci->ci_func = caa->cpu_func; 336 337 /* Must be before mi_cpu_attach(). */ 338 cpu_vm_init(ci); 339 340 if (caa->cpu_role == CPU_ROLE_AP) { 341 int error; 342 343 error = mi_cpu_attach(ci); 344 if (error != 0) { 345 aprint_normal("\n"); 346 aprint_error_dev(self, 347 "mi_cpu_attach failed with %d\n", error); 348 return; 349 } 350 cpu_init_tss(ci); 351 } else { 352 KASSERT(ci->ci_data.cpu_idlelwp != NULL); 353 } 354 355 ci->ci_cpumask = (1 << cpu_index(ci)); 356 pmap_reference(pmap_kernel()); 357 ci->ci_pmap = pmap_kernel(); 358 ci->ci_tlbstate = TLBSTATE_STALE; 359 360 /* 361 * Boot processor may not be attached first, but the below 362 * must be done to allow booting other processors. 363 */ 364 if (!again) { 365 atomic_or_32(&ci->ci_flags, CPUF_PRESENT | CPUF_PRIMARY); 366 /* Basic init. */ 367 cpu_intr_init(ci); 368 cpu_get_tsc_freq(ci); 369 cpu_init(ci); 370 cpu_set_tss_gates(ci); 371 pmap_cpu_init_late(ci); 372 if (caa->cpu_role != CPU_ROLE_SP) { 373 /* Enable lapic. */ 374 lapic_enable(); 375 lapic_set_lvt(); 376 lapic_calibrate_timer(ci); 377 } 378 /* Make sure DELAY() is initialized. */ 379 DELAY(1); 380 again = true; 381 } 382 383 /* further PCB init done later. */ 384 385 switch (caa->cpu_role) { 386 case CPU_ROLE_SP: 387 atomic_or_32(&ci->ci_flags, CPUF_SP); 388 cpu_identify(ci); 389 x86_errata(); 390 x86_cpu_idle_init(); 391 break; 392 393 case CPU_ROLE_BP: 394 atomic_or_32(&ci->ci_flags, CPUF_BSP); 395 cpu_identify(ci); 396 x86_errata(); 397 x86_cpu_idle_init(); 398 break; 399 400 case CPU_ROLE_AP: 401 /* 402 * report on an AP 403 */ 404 cpu_intr_init(ci); 405 gdt_alloc_cpu(ci); 406 cpu_set_tss_gates(ci); 407 pmap_cpu_init_early(ci); 408 pmap_cpu_init_late(ci); 409 cpu_start_secondary(ci); 410 if (ci->ci_flags & CPUF_PRESENT) { 411 struct cpu_info *tmp; 412 413 cpu_identify(ci); 414 tmp = cpu_info_list; 415 while (tmp->ci_next) 416 tmp = tmp->ci_next; 417 418 tmp->ci_next = ci; 419 } 420 break; 421 422 default: 423 aprint_normal("\n"); 424 panic("unknown processor type??\n"); 425 } 426 427 pat_init(ci); 428 atomic_or_32(&cpus_attached, ci->ci_cpumask); 429 430 if (!pmf_device_register(self, cpu_suspend, cpu_resume)) 431 aprint_error_dev(self, "couldn't establish power handler\n"); 432 433 if (mp_verbose) { 434 struct lwp *l = ci->ci_data.cpu_idlelwp; 435 struct pcb *pcb = lwp_getpcb(l); 436 437 aprint_verbose_dev(self, 438 "idle lwp at %p, idle sp at %p\n", 439 l, 440 #ifdef i386 441 (void *)pcb->pcb_esp 442 #else 443 (void *)pcb->pcb_rsp 444 #endif 445 ); 446 } 447 } 448 449 /* 450 * Initialize the processor appropriately. 451 */ 452 453 void 454 cpu_init(struct cpu_info *ci) 455 { 456 457 lcr0(rcr0() | CR0_WP); 458 459 /* 460 * On a P6 or above, enable global TLB caching if the 461 * hardware supports it. 462 */ 463 if (cpu_feature[0] & CPUID_PGE) 464 lcr4(rcr4() | CR4_PGE); /* enable global TLB caching */ 465 466 /* 467 * If we have FXSAVE/FXRESTOR, use them. 468 */ 469 if (cpu_feature[0] & CPUID_FXSR) { 470 lcr4(rcr4() | CR4_OSFXSR); 471 472 /* 473 * If we have SSE/SSE2, enable XMM exceptions. 474 */ 475 if (cpu_feature[0] & (CPUID_SSE|CPUID_SSE2)) 476 lcr4(rcr4() | CR4_OSXMMEXCPT); 477 } 478 479 #ifdef MTRR 480 /* 481 * On a P6 or above, initialize MTRR's if the hardware supports them. 482 */ 483 if (cpu_feature[0] & CPUID_MTRR) { 484 if ((ci->ci_flags & CPUF_AP) == 0) 485 i686_mtrr_init_first(); 486 mtrr_init_cpu(ci); 487 } 488 489 #ifdef i386 490 if (strcmp((char *)(ci->ci_vendor), "AuthenticAMD") == 0) { 491 /* 492 * Must be a K6-2 Step >= 7 or a K6-III. 493 */ 494 if (CPUID2FAMILY(ci->ci_signature) == 5) { 495 if (CPUID2MODEL(ci->ci_signature) > 8 || 496 (CPUID2MODEL(ci->ci_signature) == 8 && 497 CPUID2STEPPING(ci->ci_signature) >= 7)) { 498 mtrr_funcs = &k6_mtrr_funcs; 499 k6_mtrr_init_first(); 500 mtrr_init_cpu(ci); 501 } 502 } 503 } 504 #endif /* i386 */ 505 #endif /* MTRR */ 506 507 atomic_or_32(&cpus_running, ci->ci_cpumask); 508 509 if (ci != &cpu_info_primary) { 510 /* Synchronize TSC again, and check for drift. */ 511 wbinvd(); 512 atomic_or_32(&ci->ci_flags, CPUF_RUNNING); 513 tsc_sync_ap(ci); 514 } else { 515 atomic_or_32(&ci->ci_flags, CPUF_RUNNING); 516 } 517 } 518 519 void 520 cpu_boot_secondary_processors(void) 521 { 522 struct cpu_info *ci; 523 u_long i; 524 525 /* Now that we know the number of CPUs, patch the text segment. */ 526 x86_patch(false); 527 528 for (i=0; i < maxcpus; i++) { 529 ci = cpu_lookup(i); 530 if (ci == NULL) 531 continue; 532 if (ci->ci_data.cpu_idlelwp == NULL) 533 continue; 534 if ((ci->ci_flags & CPUF_PRESENT) == 0) 535 continue; 536 if (ci->ci_flags & (CPUF_BSP|CPUF_SP|CPUF_PRIMARY)) 537 continue; 538 cpu_boot_secondary(ci); 539 } 540 541 x86_mp_online = true; 542 543 /* Now that we know about the TSC, attach the timecounter. */ 544 tsc_tc_init(); 545 546 /* Enable zeroing of pages in the idle loop if we have SSE2. */ 547 vm_page_zero_enable = ((cpu_feature[0] & CPUID_SSE2) != 0); 548 } 549 550 static void 551 cpu_init_idle_lwp(struct cpu_info *ci) 552 { 553 struct lwp *l = ci->ci_data.cpu_idlelwp; 554 struct pcb *pcb = lwp_getpcb(l); 555 556 pcb->pcb_cr0 = rcr0(); 557 } 558 559 void 560 cpu_init_idle_lwps(void) 561 { 562 struct cpu_info *ci; 563 u_long i; 564 565 for (i = 0; i < maxcpus; i++) { 566 ci = cpu_lookup(i); 567 if (ci == NULL) 568 continue; 569 if (ci->ci_data.cpu_idlelwp == NULL) 570 continue; 571 if ((ci->ci_flags & CPUF_PRESENT) == 0) 572 continue; 573 cpu_init_idle_lwp(ci); 574 } 575 } 576 577 void 578 cpu_start_secondary(struct cpu_info *ci) 579 { 580 extern paddr_t mp_pdirpa; 581 u_long psl; 582 int i; 583 584 mp_pdirpa = pmap_init_tmp_pgtbl(mp_trampoline_paddr); 585 atomic_or_32(&ci->ci_flags, CPUF_AP); 586 ci->ci_curlwp = ci->ci_data.cpu_idlelwp; 587 if (CPU_STARTUP(ci, mp_trampoline_paddr) != 0) { 588 return; 589 } 590 591 /* 592 * Wait for it to become ready. Setting cpu_starting opens the 593 * initial gate and allows the AP to start soft initialization. 594 */ 595 KASSERT(cpu_starting == NULL); 596 cpu_starting = ci; 597 for (i = 100000; (!(ci->ci_flags & CPUF_PRESENT)) && i > 0; i--) { 598 #ifdef MPDEBUG 599 extern int cpu_trace[3]; 600 static int otrace[3]; 601 if (memcmp(otrace, cpu_trace, sizeof(otrace)) != 0) { 602 aprint_debug_dev(ci->ci_dev, "trace %02x %02x %02x\n", 603 cpu_trace[0], cpu_trace[1], cpu_trace[2]); 604 memcpy(otrace, cpu_trace, sizeof(otrace)); 605 } 606 #endif 607 i8254_delay(10); 608 } 609 610 if ((ci->ci_flags & CPUF_PRESENT) == 0) { 611 aprint_error_dev(ci->ci_dev, "failed to become ready\n"); 612 #if defined(MPDEBUG) && defined(DDB) 613 printf("dropping into debugger; continue from here to resume boot\n"); 614 Debugger(); 615 #endif 616 } else { 617 /* 618 * Synchronize time stamp counters. Invalidate cache and do 619 * twice to try and minimize possible cache effects. Disable 620 * interrupts to try and rule out any external interference. 621 */ 622 psl = x86_read_psl(); 623 x86_disable_intr(); 624 wbinvd(); 625 tsc_sync_bp(ci); 626 x86_write_psl(psl); 627 } 628 629 CPU_START_CLEANUP(ci); 630 cpu_starting = NULL; 631 } 632 633 void 634 cpu_boot_secondary(struct cpu_info *ci) 635 { 636 int64_t drift; 637 u_long psl; 638 int i; 639 640 atomic_or_32(&ci->ci_flags, CPUF_GO); 641 for (i = 100000; (!(ci->ci_flags & CPUF_RUNNING)) && i > 0; i--) { 642 i8254_delay(10); 643 } 644 if ((ci->ci_flags & CPUF_RUNNING) == 0) { 645 aprint_error_dev(ci->ci_dev, "failed to start\n"); 646 #if defined(MPDEBUG) && defined(DDB) 647 printf("dropping into debugger; continue from here to resume boot\n"); 648 Debugger(); 649 #endif 650 } else { 651 /* Synchronize TSC again, check for drift. */ 652 drift = ci->ci_data.cpu_cc_skew; 653 psl = x86_read_psl(); 654 x86_disable_intr(); 655 wbinvd(); 656 tsc_sync_bp(ci); 657 x86_write_psl(psl); 658 drift -= ci->ci_data.cpu_cc_skew; 659 aprint_debug_dev(ci->ci_dev, "TSC skew=%lld drift=%lld\n", 660 (long long)ci->ci_data.cpu_cc_skew, (long long)drift); 661 tsc_sync_drift(drift); 662 } 663 } 664 665 /* 666 * The CPU ends up here when its ready to run 667 * This is called from code in mptramp.s; at this point, we are running 668 * in the idle pcb/idle stack of the new CPU. When this function returns, 669 * this processor will enter the idle loop and start looking for work. 670 */ 671 void 672 cpu_hatch(void *v) 673 { 674 struct cpu_info *ci = (struct cpu_info *)v; 675 struct pcb *pcb; 676 int s, i; 677 678 cpu_init_msrs(ci, true); 679 cpu_probe(ci); 680 681 ci->ci_data.cpu_cc_freq = cpu_info_primary.ci_data.cpu_cc_freq; 682 /* cpu_get_tsc_freq(ci); */ 683 684 KDASSERT((ci->ci_flags & CPUF_PRESENT) == 0); 685 686 /* 687 * Synchronize time stamp counters. Invalidate cache and do twice 688 * to try and minimize possible cache effects. Note that interrupts 689 * are off at this point. 690 */ 691 wbinvd(); 692 atomic_or_32(&ci->ci_flags, CPUF_PRESENT); 693 tsc_sync_ap(ci); 694 695 /* 696 * Wait to be brought online. Use 'monitor/mwait' if available, 697 * in order to make the TSC drift as much as possible. so that 698 * we can detect it later. If not available, try 'pause'. 699 * We'd like to use 'hlt', but we have interrupts off. 700 */ 701 while ((ci->ci_flags & CPUF_GO) == 0) { 702 if ((cpu_feature[1] & CPUID2_MONITOR) != 0) { 703 x86_monitor(&ci->ci_flags, 0, 0); 704 if ((ci->ci_flags & CPUF_GO) != 0) { 705 continue; 706 } 707 x86_mwait(0, 0); 708 } else { 709 for (i = 10000; i != 0; i--) { 710 x86_pause(); 711 } 712 } 713 } 714 715 /* Because the text may have been patched in x86_patch(). */ 716 wbinvd(); 717 x86_flush(); 718 719 KASSERT((ci->ci_flags & CPUF_RUNNING) == 0); 720 721 #ifdef PAE 722 pd_entry_t * l3_pd = ci->ci_pae_l3_pdir; 723 for (i = 0 ; i < PDP_SIZE; i++) { 724 l3_pd[i] = pmap_kernel()->pm_pdirpa[i] | PG_V; 725 } 726 lcr3(ci->ci_pae_l3_pdirpa); 727 #else 728 lcr3(pmap_pdirpa(pmap_kernel(), 0)); 729 #endif 730 731 pcb = lwp_getpcb(curlwp); 732 pcb->pcb_cr3 = rcr3(); 733 pcb = lwp_getpcb(ci->ci_data.cpu_idlelwp); 734 lcr0(pcb->pcb_cr0); 735 736 cpu_init_idt(); 737 gdt_init_cpu(ci); 738 lapic_enable(); 739 lapic_set_lvt(); 740 lapic_initclocks(); 741 742 #ifdef i386 743 #if NNPX > 0 744 npxinit(ci); 745 #endif 746 #else 747 fpuinit(ci); 748 #endif 749 lldt(GSYSSEL(GLDT_SEL, SEL_KPL)); 750 ltr(ci->ci_tss_sel); 751 752 cpu_init(ci); 753 cpu_get_tsc_freq(ci); 754 755 s = splhigh(); 756 #ifdef i386 757 lapic_tpr = 0; 758 #else 759 lcr8(0); 760 #endif 761 x86_enable_intr(); 762 splx(s); 763 x86_errata(); 764 765 aprint_debug_dev(ci->ci_dev, "running\n"); 766 } 767 768 #if defined(DDB) 769 770 #include <ddb/db_output.h> 771 #include <machine/db_machdep.h> 772 773 /* 774 * Dump CPU information from ddb. 775 */ 776 void 777 cpu_debug_dump(void) 778 { 779 struct cpu_info *ci; 780 CPU_INFO_ITERATOR cii; 781 782 db_printf("addr dev id flags ipis curlwp fpcurlwp\n"); 783 for (CPU_INFO_FOREACH(cii, ci)) { 784 db_printf("%p %s %ld %x %x %10p %10p\n", 785 ci, 786 ci->ci_dev == NULL ? "BOOT" : device_xname(ci->ci_dev), 787 (long)ci->ci_cpuid, 788 ci->ci_flags, ci->ci_ipis, 789 ci->ci_curlwp, 790 ci->ci_fpcurlwp); 791 } 792 } 793 #endif 794 795 static void 796 cpu_copy_trampoline(void) 797 { 798 /* 799 * Copy boot code. 800 */ 801 extern u_char cpu_spinup_trampoline[]; 802 extern u_char cpu_spinup_trampoline_end[]; 803 804 vaddr_t mp_trampoline_vaddr; 805 806 mp_trampoline_vaddr = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, 807 UVM_KMF_VAONLY); 808 809 pmap_kenter_pa(mp_trampoline_vaddr, mp_trampoline_paddr, 810 VM_PROT_READ | VM_PROT_WRITE, 0); 811 pmap_update(pmap_kernel()); 812 memcpy((void *)mp_trampoline_vaddr, 813 cpu_spinup_trampoline, 814 cpu_spinup_trampoline_end - cpu_spinup_trampoline); 815 816 pmap_kremove(mp_trampoline_vaddr, PAGE_SIZE); 817 pmap_update(pmap_kernel()); 818 uvm_km_free(kernel_map, mp_trampoline_vaddr, PAGE_SIZE, UVM_KMF_VAONLY); 819 } 820 821 #ifdef i386 822 static void 823 tss_init(struct i386tss *tss, void *stack, void *func) 824 { 825 KASSERT(curcpu()->ci_pmap == pmap_kernel()); 826 827 memset(tss, 0, sizeof *tss); 828 tss->tss_esp0 = tss->tss_esp = (int)((char *)stack + USPACE - 16); 829 tss->tss_ss0 = GSEL(GDATA_SEL, SEL_KPL); 830 tss->__tss_cs = GSEL(GCODE_SEL, SEL_KPL); 831 tss->tss_fs = GSEL(GCPU_SEL, SEL_KPL); 832 tss->tss_gs = tss->__tss_es = tss->__tss_ds = 833 tss->__tss_ss = GSEL(GDATA_SEL, SEL_KPL); 834 /* %cr3 contains the value associated to pmap_kernel */ 835 tss->tss_cr3 = rcr3(); 836 tss->tss_esp = (int)((char *)stack + USPACE - 16); 837 tss->tss_ldt = GSEL(GLDT_SEL, SEL_KPL); 838 tss->__tss_eflags = PSL_MBO | PSL_NT; /* XXX not needed? */ 839 tss->__tss_eip = (int)func; 840 } 841 842 /* XXX */ 843 #define IDTVEC(name) __CONCAT(X, name) 844 typedef void (vector)(void); 845 extern vector IDTVEC(tss_trap08); 846 #ifdef DDB 847 extern vector Xintrddbipi; 848 extern int ddb_vec; 849 #endif 850 851 static void 852 cpu_set_tss_gates(struct cpu_info *ci) 853 { 854 struct segment_descriptor sd; 855 856 ci->ci_doubleflt_stack = (char *)uvm_km_alloc(kernel_map, USPACE, 0, 857 UVM_KMF_WIRED); 858 tss_init(&ci->ci_doubleflt_tss, ci->ci_doubleflt_stack, 859 IDTVEC(tss_trap08)); 860 setsegment(&sd, &ci->ci_doubleflt_tss, sizeof(struct i386tss) - 1, 861 SDT_SYS386TSS, SEL_KPL, 0, 0); 862 ci->ci_gdt[GTRAPTSS_SEL].sd = sd; 863 setgate(&idt[8], NULL, 0, SDT_SYSTASKGT, SEL_KPL, 864 GSEL(GTRAPTSS_SEL, SEL_KPL)); 865 866 #if defined(DDB) 867 /* 868 * Set up separate handler for the DDB IPI, so that it doesn't 869 * stomp on a possibly corrupted stack. 870 * 871 * XXX overwriting the gate set in db_machine_init. 872 * Should rearrange the code so that it's set only once. 873 */ 874 ci->ci_ddbipi_stack = (char *)uvm_km_alloc(kernel_map, USPACE, 0, 875 UVM_KMF_WIRED); 876 tss_init(&ci->ci_ddbipi_tss, ci->ci_ddbipi_stack, Xintrddbipi); 877 878 setsegment(&sd, &ci->ci_ddbipi_tss, sizeof(struct i386tss) - 1, 879 SDT_SYS386TSS, SEL_KPL, 0, 0); 880 ci->ci_gdt[GIPITSS_SEL].sd = sd; 881 882 setgate(&idt[ddb_vec], NULL, 0, SDT_SYSTASKGT, SEL_KPL, 883 GSEL(GIPITSS_SEL, SEL_KPL)); 884 #endif 885 } 886 #else 887 static void 888 cpu_set_tss_gates(struct cpu_info *ci) 889 { 890 891 } 892 #endif /* i386 */ 893 894 int 895 mp_cpu_start(struct cpu_info *ci, paddr_t target) 896 { 897 unsigned short dwordptr[2]; 898 int error; 899 900 /* 901 * Bootstrap code must be addressable in real mode 902 * and it must be page aligned. 903 */ 904 KASSERT(target < 0x10000 && target % PAGE_SIZE == 0); 905 906 /* 907 * "The BSP must initialize CMOS shutdown code to 0Ah ..." 908 */ 909 910 outb(IO_RTC, NVRAM_RESET); 911 outb(IO_RTC+1, NVRAM_RESET_JUMP); 912 913 /* 914 * "and the warm reset vector (DWORD based at 40:67) to point 915 * to the AP startup code ..." 916 */ 917 918 dwordptr[0] = 0; 919 dwordptr[1] = target >> 4; 920 921 memcpy((uint8_t *)cmos_data_mapping + 0x467, dwordptr, 4); 922 923 if ((cpu_feature[0] & CPUID_APIC) == 0) { 924 aprint_error("mp_cpu_start: CPU does not have APIC\n"); 925 return ENODEV; 926 } 927 928 /* 929 * ... prior to executing the following sequence:". We'll also add in 930 * local cache flush, in case the BIOS has left the AP with its cache 931 * disabled. It may not be able to cope with MP coherency. 932 */ 933 wbinvd(); 934 935 if (ci->ci_flags & CPUF_AP) { 936 error = x86_ipi_init(ci->ci_cpuid); 937 if (error != 0) { 938 aprint_error_dev(ci->ci_dev, "%s: IPI not taken (1)\n", 939 __func__); 940 return error; 941 } 942 i8254_delay(10000); 943 944 error = x86_ipi_startup(ci->ci_cpuid, target / PAGE_SIZE); 945 if (error != 0) { 946 aprint_error_dev(ci->ci_dev, "%s: IPI not taken (2)\n", 947 __func__); 948 return error; 949 } 950 i8254_delay(200); 951 952 error = x86_ipi_startup(ci->ci_cpuid, target / PAGE_SIZE); 953 if (error != 0) { 954 aprint_error_dev(ci->ci_dev, "%s: IPI not taken (3)\n", 955 __func__); 956 return error; 957 } 958 i8254_delay(200); 959 } 960 961 return 0; 962 } 963 964 void 965 mp_cpu_start_cleanup(struct cpu_info *ci) 966 { 967 /* 968 * Ensure the NVRAM reset byte contains something vaguely sane. 969 */ 970 971 outb(IO_RTC, NVRAM_RESET); 972 outb(IO_RTC+1, NVRAM_RESET_RST); 973 } 974 975 #ifdef __x86_64__ 976 typedef void (vector)(void); 977 extern vector Xsyscall, Xsyscall32; 978 #endif 979 980 void 981 cpu_init_msrs(struct cpu_info *ci, bool full) 982 { 983 #ifdef __x86_64__ 984 wrmsr(MSR_STAR, 985 ((uint64_t)GSEL(GCODE_SEL, SEL_KPL) << 32) | 986 ((uint64_t)LSEL(LSYSRETBASE_SEL, SEL_UPL) << 48)); 987 wrmsr(MSR_LSTAR, (uint64_t)Xsyscall); 988 wrmsr(MSR_CSTAR, (uint64_t)Xsyscall32); 989 wrmsr(MSR_SFMASK, PSL_NT|PSL_T|PSL_I|PSL_C); 990 991 if (full) { 992 wrmsr(MSR_FSBASE, 0); 993 wrmsr(MSR_GSBASE, (uint64_t)ci); 994 wrmsr(MSR_KERNELGSBASE, 0); 995 } 996 #endif /* __x86_64__ */ 997 998 if (cpu_feature[2] & CPUID_NOX) 999 wrmsr(MSR_EFER, rdmsr(MSR_EFER) | EFER_NXE); 1000 } 1001 1002 void 1003 cpu_offline_md(void) 1004 { 1005 int s; 1006 1007 s = splhigh(); 1008 #ifdef i386 1009 #if NNPX > 0 1010 npxsave_cpu(true); 1011 #endif 1012 #else 1013 fpusave_cpu(true); 1014 #endif 1015 splx(s); 1016 } 1017 1018 /* XXX joerg restructure and restart CPUs individually */ 1019 static bool 1020 cpu_suspend(device_t dv, const pmf_qual_t *qual) 1021 { 1022 struct cpu_softc *sc = device_private(dv); 1023 struct cpu_info *ci = sc->sc_info; 1024 int err; 1025 1026 if (ci->ci_flags & CPUF_PRIMARY) 1027 return true; 1028 if (ci->ci_data.cpu_idlelwp == NULL) 1029 return true; 1030 if ((ci->ci_flags & CPUF_PRESENT) == 0) 1031 return true; 1032 1033 sc->sc_wasonline = !(ci->ci_schedstate.spc_flags & SPCF_OFFLINE); 1034 1035 if (sc->sc_wasonline) { 1036 mutex_enter(&cpu_lock); 1037 err = cpu_setstate(ci, false); 1038 mutex_exit(&cpu_lock); 1039 1040 if (err) 1041 return false; 1042 } 1043 1044 return true; 1045 } 1046 1047 static bool 1048 cpu_resume(device_t dv, const pmf_qual_t *qual) 1049 { 1050 struct cpu_softc *sc = device_private(dv); 1051 struct cpu_info *ci = sc->sc_info; 1052 int err = 0; 1053 1054 if (ci->ci_flags & CPUF_PRIMARY) 1055 return true; 1056 if (ci->ci_data.cpu_idlelwp == NULL) 1057 return true; 1058 if ((ci->ci_flags & CPUF_PRESENT) == 0) 1059 return true; 1060 1061 if (sc->sc_wasonline) { 1062 mutex_enter(&cpu_lock); 1063 err = cpu_setstate(ci, true); 1064 mutex_exit(&cpu_lock); 1065 } 1066 1067 return err == 0; 1068 } 1069 1070 void 1071 cpu_get_tsc_freq(struct cpu_info *ci) 1072 { 1073 uint64_t last_tsc; 1074 1075 if (cpu_hascounter()) { 1076 last_tsc = rdmsr(MSR_TSC); 1077 i8254_delay(100000); 1078 ci->ci_data.cpu_cc_freq = (rdmsr(MSR_TSC) - last_tsc) * 10; 1079 } 1080 } 1081 1082 void 1083 x86_cpu_idle_mwait(void) 1084 { 1085 struct cpu_info *ci = curcpu(); 1086 1087 KASSERT(ci->ci_ilevel == IPL_NONE); 1088 1089 x86_monitor(&ci->ci_want_resched, 0, 0); 1090 if (__predict_false(ci->ci_want_resched)) { 1091 return; 1092 } 1093 x86_mwait(0, 0); 1094 } 1095 1096 void 1097 x86_cpu_idle_halt(void) 1098 { 1099 struct cpu_info *ci = curcpu(); 1100 1101 KASSERT(ci->ci_ilevel == IPL_NONE); 1102 1103 x86_disable_intr(); 1104 if (!__predict_false(ci->ci_want_resched)) { 1105 x86_stihlt(); 1106 } else { 1107 x86_enable_intr(); 1108 } 1109 } 1110 1111 /* 1112 * Loads pmap for the current CPU. 1113 */ 1114 void 1115 cpu_load_pmap(struct pmap *pmap) 1116 { 1117 #ifdef PAE 1118 int i, s; 1119 struct cpu_info *ci; 1120 1121 s = splvm(); /* just to be safe */ 1122 ci = curcpu(); 1123 pd_entry_t *l3_pd = ci->ci_pae_l3_pdir; 1124 for (i = 0 ; i < PDP_SIZE; i++) { 1125 l3_pd[i] = pmap->pm_pdirpa[i] | PG_V; 1126 } 1127 splx(s); 1128 tlbflush(); 1129 #else /* PAE */ 1130 lcr3(pmap_pdirpa(pmap, 0)); 1131 #endif /* PAE */ 1132 } 1133
Indexes created Wed Sep 24 05:09:52 GMT 2025