cpu.c revision 1.143
1 /* $NetBSD: cpu.c,v 1.143 2018/01/07 10:16:13 maxv Exp $ */ 2 3 /* 4 * Copyright (c) 2000-2012 NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Bill Sommerfeld of RedBack Networks Inc, and by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * Copyright (c) 1999 Stefan Grefen 34 * 35 * Redistribution and use in source and binary forms, with or without 36 * modification, are permitted provided that the following conditions 37 * are met: 38 * 1. Redistributions of source code must retain the above copyright 39 * notice, this list of conditions and the following disclaimer. 40 * 2. Redistributions in binary form must reproduce the above copyright 41 * notice, this list of conditions and the following disclaimer in the 42 * documentation and/or other materials provided with the distribution. 43 * 3. All advertising materials mentioning features or use of this software 44 * must display the following acknowledgement: 45 * This product includes software developed by the NetBSD 46 * Foundation, Inc. and its contributors. 47 * 4. Neither the name of The NetBSD Foundation nor the names of its 48 * contributors may be used to endorse or promote products derived 49 * from this software without specific prior written permission. 50 * 51 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND ANY 52 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 53 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 54 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR AND CONTRIBUTORS BE LIABLE 55 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 56 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 57 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 58 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 59 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 60 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 61 * SUCH DAMAGE. 62 */ 63 64 #include <sys/cdefs.h> 65 __KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.143 2018/01/07 10:16:13 maxv Exp $"); 66 67 #include "opt_ddb.h" 68 #include "opt_mpbios.h" /* for MPDEBUG */ 69 #include "opt_mtrr.h" 70 #include "opt_multiprocessor.h" 71 72 #include "lapic.h" 73 #include "ioapic.h" 74 75 #include <sys/param.h> 76 #include <sys/proc.h> 77 #include <sys/systm.h> 78 #include <sys/device.h> 79 #include <sys/cpu.h> 80 #include <sys/cpufreq.h> 81 #include <sys/idle.h> 82 #include <sys/atomic.h> 83 #include <sys/reboot.h> 84 85 #include <uvm/uvm.h> 86 87 #include "acpica.h" /* for NACPICA, for mp_verbose */ 88 89 #include <machine/cpufunc.h> 90 #include <machine/cpuvar.h> 91 #include <machine/pmap.h> 92 #include <machine/vmparam.h> 93 #if defined(MULTIPROCESSOR) 94 #include <machine/mpbiosvar.h> 95 #endif 96 #include <machine/mpconfig.h> /* for mp_verbose */ 97 #include <machine/pcb.h> 98 #include <machine/specialreg.h> 99 #include <machine/segments.h> 100 #include <machine/gdt.h> 101 #include <machine/mtrr.h> 102 #include <machine/pio.h> 103 #include <machine/cpu_counter.h> 104 105 #include <x86/fpu.h> 106 107 #if NLAPIC > 0 108 #include <machine/apicvar.h> 109 #include <machine/i82489reg.h> 110 #include <machine/i82489var.h> 111 #endif 112 113 #include <dev/ic/mc146818reg.h> 114 #include <i386/isa/nvram.h> 115 #include <dev/isa/isareg.h> 116 117 #include "tsc.h" 118 119 static int cpu_match(device_t, cfdata_t, void *); 120 static void cpu_attach(device_t, device_t, void *); 121 static void cpu_defer(device_t); 122 static int cpu_rescan(device_t, const char *, const int *); 123 static void cpu_childdetached(device_t, device_t); 124 static bool cpu_stop(device_t); 125 static bool cpu_suspend(device_t, const pmf_qual_t *); 126 static bool cpu_resume(device_t, const pmf_qual_t *); 127 static bool cpu_shutdown(device_t, int); 128 129 struct cpu_softc { 130 device_t sc_dev; /* device tree glue */ 131 struct cpu_info *sc_info; /* pointer to CPU info */ 132 bool sc_wasonline; 133 }; 134 135 #ifdef MULTIPROCESSOR 136 int mp_cpu_start(struct cpu_info *, paddr_t); 137 void mp_cpu_start_cleanup(struct cpu_info *); 138 const struct cpu_functions mp_cpu_funcs = { mp_cpu_start, NULL, 139 mp_cpu_start_cleanup }; 140 #endif 141 142 143 CFATTACH_DECL2_NEW(cpu, sizeof(struct cpu_softc), 144 cpu_match, cpu_attach, NULL, NULL, cpu_rescan, cpu_childdetached); 145 146 /* 147 * Statically-allocated CPU info for the primary CPU (or the only 148 * CPU, on uniprocessors). The CPU info list is initialized to 149 * point at it. 150 */ 151 struct cpu_info cpu_info_primary __aligned(CACHE_LINE_SIZE) = { 152 .ci_dev = 0, 153 .ci_self = &cpu_info_primary, 154 .ci_idepth = -1, 155 .ci_curlwp = &lwp0, 156 .ci_curldt = -1, 157 }; 158 159 struct cpu_info *cpu_info_list = &cpu_info_primary; 160 161 #ifdef i386 162 void cpu_set_tss_gates(struct cpu_info *); 163 #endif 164 165 static void cpu_init_idle_lwp(struct cpu_info *); 166 167 uint32_t cpu_feature[7] __read_mostly; /* X86 CPUID feature bits */ 168 /* [0] basic features cpuid.1:%edx 169 * [1] basic features cpuid.1:%ecx (CPUID2_xxx bits) 170 * [2] extended features cpuid:80000001:%edx 171 * [3] extended features cpuid:80000001:%ecx 172 * [4] VIA padlock features 173 * [5] structured extended features cpuid.7:%ebx 174 * [6] structured extended features cpuid.7:%ecx 175 */ 176 177 #ifdef MULTIPROCESSOR 178 bool x86_mp_online; 179 paddr_t mp_trampoline_paddr = MP_TRAMPOLINE; 180 #endif 181 #if NLAPIC > 0 182 static vaddr_t cmos_data_mapping; 183 #endif 184 struct cpu_info *cpu_starting; 185 186 #ifdef MULTIPROCESSOR 187 void cpu_hatch(void *); 188 static void cpu_boot_secondary(struct cpu_info *ci); 189 static void cpu_start_secondary(struct cpu_info *ci); 190 #endif 191 #if NLAPIC > 0 192 static void cpu_copy_trampoline(paddr_t); 193 #endif 194 195 /* 196 * Runs once per boot once multiprocessor goo has been detected and 197 * the local APIC on the boot processor has been mapped. 198 * 199 * Called from lapic_boot_init() (from mpbios_scan()). 200 */ 201 #if NLAPIC > 0 202 void 203 cpu_init_first(void) 204 { 205 206 cpu_info_primary.ci_cpuid = lapic_cpu_number(); 207 208 cmos_data_mapping = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, UVM_KMF_VAONLY); 209 if (cmos_data_mapping == 0) 210 panic("No KVA for page 0"); 211 pmap_kenter_pa(cmos_data_mapping, 0, VM_PROT_READ|VM_PROT_WRITE, 0); 212 pmap_update(pmap_kernel()); 213 } 214 #endif 215 216 static int 217 cpu_match(device_t parent, cfdata_t match, void *aux) 218 { 219 220 return 1; 221 } 222 223 #ifdef __HAVE_PCPU_AREA 224 void 225 cpu_pcpuarea_init(struct cpu_info *ci) 226 { 227 struct vm_page *pg; 228 size_t i, npages; 229 vaddr_t base, va; 230 paddr_t pa; 231 232 CTASSERT(sizeof(struct pcpu_entry) % PAGE_SIZE == 0); 233 234 npages = sizeof(struct pcpu_entry) / PAGE_SIZE; 235 base = (vaddr_t)&pcpuarea->ent[cpu_index(ci)]; 236 237 for (i = 0; i < npages; i++) { 238 pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO); 239 if (pg == NULL) { 240 panic("failed to allocate pcpu PA"); 241 } 242 243 va = base + i * PAGE_SIZE; 244 pa = VM_PAGE_TO_PHYS(pg); 245 246 pmap_kenter_pa(va, pa, VM_PROT_READ|VM_PROT_WRITE, 0); 247 } 248 249 pmap_update(pmap_kernel()); 250 } 251 #endif 252 253 static void 254 cpu_vm_init(struct cpu_info *ci) 255 { 256 int ncolors = 2, i; 257 258 for (i = CAI_ICACHE; i <= CAI_L2CACHE; i++) { 259 struct x86_cache_info *cai; 260 int tcolors; 261 262 cai = &ci->ci_cinfo[i]; 263 264 tcolors = atop(cai->cai_totalsize); 265 switch(cai->cai_associativity) { 266 case 0xff: 267 tcolors = 1; /* fully associative */ 268 break; 269 case 0: 270 case 1: 271 break; 272 default: 273 tcolors /= cai->cai_associativity; 274 } 275 ncolors = max(ncolors, tcolors); 276 /* 277 * If the desired number of colors is not a power of 278 * two, it won't be good. Find the greatest power of 279 * two which is an even divisor of the number of colors, 280 * to preserve even coloring of pages. 281 */ 282 if (ncolors & (ncolors - 1) ) { 283 int try, picked = 1; 284 for (try = 1; try < ncolors; try *= 2) { 285 if (ncolors % try == 0) picked = try; 286 } 287 if (picked == 1) { 288 panic("desired number of cache colors %d is " 289 " > 1, but not even!", ncolors); 290 } 291 ncolors = picked; 292 } 293 } 294 295 /* 296 * Knowing the size of the largest cache on this CPU, potentially 297 * re-color our pages. 298 */ 299 aprint_debug_dev(ci->ci_dev, "%d page colors\n", ncolors); 300 uvm_page_recolor(ncolors); 301 302 pmap_tlb_cpu_init(ci); 303 #ifndef __HAVE_DIRECT_MAP 304 pmap_vpage_cpu_init(ci); 305 #endif 306 } 307 308 static void 309 cpu_attach(device_t parent, device_t self, void *aux) 310 { 311 struct cpu_softc *sc = device_private(self); 312 struct cpu_attach_args *caa = aux; 313 struct cpu_info *ci; 314 uintptr_t ptr; 315 #if NLAPIC > 0 316 int cpunum = caa->cpu_number; 317 #endif 318 static bool again; 319 320 sc->sc_dev = self; 321 322 if (ncpu == maxcpus) { 323 #ifndef _LP64 324 aprint_error(": too many CPUs, please use NetBSD/amd64\n"); 325 #else 326 aprint_error(": too many CPUs\n"); 327 #endif 328 return; 329 } 330 331 /* 332 * If we're an Application Processor, allocate a cpu_info 333 * structure, otherwise use the primary's. 334 */ 335 if (caa->cpu_role == CPU_ROLE_AP) { 336 if ((boothowto & RB_MD1) != 0) { 337 aprint_error(": multiprocessor boot disabled\n"); 338 if (!pmf_device_register(self, NULL, NULL)) 339 aprint_error_dev(self, 340 "couldn't establish power handler\n"); 341 return; 342 } 343 aprint_naive(": Application Processor\n"); 344 ptr = (uintptr_t)uvm_km_alloc(kernel_map, 345 sizeof(*ci) + CACHE_LINE_SIZE - 1, 0, 346 UVM_KMF_WIRED|UVM_KMF_ZERO); 347 ci = (struct cpu_info *)roundup2(ptr, CACHE_LINE_SIZE); 348 ci->ci_curldt = -1; 349 } else { 350 aprint_naive(": %s Processor\n", 351 caa->cpu_role == CPU_ROLE_SP ? "Single" : "Boot"); 352 ci = &cpu_info_primary; 353 #if NLAPIC > 0 354 if (cpunum != lapic_cpu_number()) { 355 /* XXX should be done earlier. */ 356 uint32_t reg; 357 aprint_verbose("\n"); 358 aprint_verbose_dev(self, "running CPU at apic %d" 359 " instead of at expected %d", lapic_cpu_number(), 360 cpunum); 361 reg = lapic_readreg(LAPIC_ID); 362 lapic_writereg(LAPIC_ID, (reg & ~LAPIC_ID_MASK) | 363 (cpunum << LAPIC_ID_SHIFT)); 364 } 365 if (cpunum != lapic_cpu_number()) { 366 aprint_error_dev(self, "unable to reset apic id\n"); 367 } 368 #endif 369 } 370 371 ci->ci_self = ci; 372 sc->sc_info = ci; 373 ci->ci_dev = self; 374 ci->ci_acpiid = caa->cpu_id; 375 ci->ci_cpuid = caa->cpu_number; 376 ci->ci_func = caa->cpu_func; 377 aprint_normal("\n"); 378 379 /* Must be before mi_cpu_attach(). */ 380 cpu_vm_init(ci); 381 382 if (caa->cpu_role == CPU_ROLE_AP) { 383 int error; 384 385 error = mi_cpu_attach(ci); 386 if (error != 0) { 387 aprint_error_dev(self, 388 "mi_cpu_attach failed with %d\n", error); 389 return; 390 } 391 #ifdef __HAVE_PCPU_AREA 392 cpu_pcpuarea_init(ci); 393 #endif 394 cpu_init_tss(ci); 395 } else { 396 KASSERT(ci->ci_data.cpu_idlelwp != NULL); 397 } 398 399 pmap_reference(pmap_kernel()); 400 ci->ci_pmap = pmap_kernel(); 401 ci->ci_tlbstate = TLBSTATE_STALE; 402 403 /* 404 * Boot processor may not be attached first, but the below 405 * must be done to allow booting other processors. 406 */ 407 if (!again) { 408 atomic_or_32(&ci->ci_flags, CPUF_PRESENT | CPUF_PRIMARY); 409 /* Basic init. */ 410 cpu_intr_init(ci); 411 cpu_get_tsc_freq(ci); 412 cpu_init(ci); 413 #ifdef i386 414 cpu_set_tss_gates(ci); 415 #endif 416 pmap_cpu_init_late(ci); 417 #if NLAPIC > 0 418 if (caa->cpu_role != CPU_ROLE_SP) { 419 /* Enable lapic. */ 420 lapic_enable(); 421 lapic_set_lvt(); 422 lapic_calibrate_timer(ci); 423 } 424 #endif 425 /* Make sure DELAY() is initialized. */ 426 DELAY(1); 427 again = true; 428 } 429 430 /* further PCB init done later. */ 431 432 switch (caa->cpu_role) { 433 case CPU_ROLE_SP: 434 atomic_or_32(&ci->ci_flags, CPUF_SP); 435 cpu_identify(ci); 436 x86_errata(); 437 x86_cpu_idle_init(); 438 break; 439 440 case CPU_ROLE_BP: 441 atomic_or_32(&ci->ci_flags, CPUF_BSP); 442 cpu_identify(ci); 443 x86_errata(); 444 x86_cpu_idle_init(); 445 break; 446 447 #ifdef MULTIPROCESSOR 448 case CPU_ROLE_AP: 449 /* 450 * report on an AP 451 */ 452 cpu_intr_init(ci); 453 gdt_alloc_cpu(ci); 454 #ifdef i386 455 cpu_set_tss_gates(ci); 456 #endif 457 pmap_cpu_init_late(ci); 458 cpu_start_secondary(ci); 459 if (ci->ci_flags & CPUF_PRESENT) { 460 struct cpu_info *tmp; 461 462 cpu_identify(ci); 463 tmp = cpu_info_list; 464 while (tmp->ci_next) 465 tmp = tmp->ci_next; 466 467 tmp->ci_next = ci; 468 } 469 break; 470 #endif 471 472 default: 473 panic("unknown processor type??\n"); 474 } 475 476 pat_init(ci); 477 478 if (!pmf_device_register1(self, cpu_suspend, cpu_resume, cpu_shutdown)) 479 aprint_error_dev(self, "couldn't establish power handler\n"); 480 481 #ifdef MULTIPROCESSOR 482 if (mp_verbose) { 483 struct lwp *l = ci->ci_data.cpu_idlelwp; 484 struct pcb *pcb = lwp_getpcb(l); 485 486 aprint_verbose_dev(self, 487 "idle lwp at %p, idle sp at %p\n", 488 l, 489 #ifdef i386 490 (void *)pcb->pcb_esp 491 #else 492 (void *)pcb->pcb_rsp 493 #endif 494 ); 495 } 496 #endif 497 498 /* 499 * Postpone the "cpufeaturebus" scan. 500 * It is safe to scan the pseudo-bus 501 * only after all CPUs have attached. 502 */ 503 (void)config_defer(self, cpu_defer); 504 } 505 506 static void 507 cpu_defer(device_t self) 508 { 509 cpu_rescan(self, NULL, NULL); 510 } 511 512 static int 513 cpu_rescan(device_t self, const char *ifattr, const int *locators) 514 { 515 struct cpu_softc *sc = device_private(self); 516 struct cpufeature_attach_args cfaa; 517 struct cpu_info *ci = sc->sc_info; 518 519 memset(&cfaa, 0, sizeof(cfaa)); 520 cfaa.ci = ci; 521 522 if (ifattr_match(ifattr, "cpufeaturebus")) { 523 if (ci->ci_frequency == NULL) { 524 cfaa.name = "frequency"; 525 ci->ci_frequency = config_found_ia(self, 526 "cpufeaturebus", &cfaa, NULL); 527 } 528 529 if (ci->ci_padlock == NULL) { 530 cfaa.name = "padlock"; 531 ci->ci_padlock = config_found_ia(self, 532 "cpufeaturebus", &cfaa, NULL); 533 } 534 535 if (ci->ci_temperature == NULL) { 536 cfaa.name = "temperature"; 537 ci->ci_temperature = config_found_ia(self, 538 "cpufeaturebus", &cfaa, NULL); 539 } 540 541 if (ci->ci_vm == NULL) { 542 cfaa.name = "vm"; 543 ci->ci_vm = config_found_ia(self, 544 "cpufeaturebus", &cfaa, NULL); 545 } 546 } 547 548 return 0; 549 } 550 551 static void 552 cpu_childdetached(device_t self, device_t child) 553 { 554 struct cpu_softc *sc = device_private(self); 555 struct cpu_info *ci = sc->sc_info; 556 557 if (ci->ci_frequency == child) 558 ci->ci_frequency = NULL; 559 560 if (ci->ci_padlock == child) 561 ci->ci_padlock = NULL; 562 563 if (ci->ci_temperature == child) 564 ci->ci_temperature = NULL; 565 566 if (ci->ci_vm == child) 567 ci->ci_vm = NULL; 568 } 569 570 /* 571 * Initialize the processor appropriately. 572 */ 573 574 void 575 cpu_init(struct cpu_info *ci) 576 { 577 extern int x86_fpu_save; 578 uint32_t cr4 = 0; 579 580 lcr0(rcr0() | CR0_WP); 581 582 /* 583 * On a P6 or above, enable global TLB caching if the 584 * hardware supports it. 585 */ 586 if (cpu_feature[0] & CPUID_PGE) 587 cr4 |= CR4_PGE; /* enable global TLB caching */ 588 589 /* 590 * If we have FXSAVE/FXRESTOR, use them. 591 */ 592 if (cpu_feature[0] & CPUID_FXSR) { 593 cr4 |= CR4_OSFXSR; 594 595 /* 596 * If we have SSE/SSE2, enable XMM exceptions. 597 */ 598 if (cpu_feature[0] & (CPUID_SSE|CPUID_SSE2)) 599 cr4 |= CR4_OSXMMEXCPT; 600 } 601 602 /* If xsave is supported, enable it */ 603 if (cpu_feature[1] & CPUID2_XSAVE) 604 cr4 |= CR4_OSXSAVE; 605 606 /* If SMEP is supported, enable it */ 607 if (cpu_feature[5] & CPUID_SEF_SMEP) 608 cr4 |= CR4_SMEP; 609 610 #ifdef amd64 611 /* If SMAP is supported, enable it */ 612 if (cpu_feature[5] & CPUID_SEF_SMAP) 613 cr4 |= CR4_SMAP; 614 #endif 615 616 if (cr4) { 617 cr4 |= rcr4(); 618 lcr4(cr4); 619 } 620 621 if (x86_fpu_save >= FPU_SAVE_FXSAVE) { 622 fpuinit_mxcsr_mask(); 623 } 624 625 /* If xsave is enabled, enable all fpu features */ 626 if (cr4 & CR4_OSXSAVE) 627 wrxcr(0, x86_xsave_features & XCR0_FPU); 628 629 #ifdef MTRR 630 /* 631 * On a P6 or above, initialize MTRR's if the hardware supports them. 632 */ 633 if (cpu_feature[0] & CPUID_MTRR) { 634 if ((ci->ci_flags & CPUF_AP) == 0) 635 i686_mtrr_init_first(); 636 mtrr_init_cpu(ci); 637 } 638 639 #ifdef i386 640 if (strcmp((char *)(ci->ci_vendor), "AuthenticAMD") == 0) { 641 /* 642 * Must be a K6-2 Step >= 7 or a K6-III. 643 */ 644 if (CPUID_TO_FAMILY(ci->ci_signature) == 5) { 645 if (CPUID_TO_MODEL(ci->ci_signature) > 8 || 646 (CPUID_TO_MODEL(ci->ci_signature) == 8 && 647 CPUID_TO_STEPPING(ci->ci_signature) >= 7)) { 648 mtrr_funcs = &k6_mtrr_funcs; 649 k6_mtrr_init_first(); 650 mtrr_init_cpu(ci); 651 } 652 } 653 } 654 #endif /* i386 */ 655 #endif /* MTRR */ 656 657 if (ci != &cpu_info_primary) { 658 /* Synchronize TSC again, and check for drift. */ 659 wbinvd(); 660 atomic_or_32(&ci->ci_flags, CPUF_RUNNING); 661 tsc_sync_ap(ci); 662 } else { 663 atomic_or_32(&ci->ci_flags, CPUF_RUNNING); 664 } 665 } 666 667 #ifdef MULTIPROCESSOR 668 void 669 cpu_boot_secondary_processors(void) 670 { 671 struct cpu_info *ci; 672 kcpuset_t *cpus; 673 u_long i; 674 675 /* Now that we know the number of CPUs, patch the text segment. */ 676 x86_patch(false); 677 678 kcpuset_create(&cpus, true); 679 kcpuset_set(cpus, cpu_index(curcpu())); 680 for (i = 0; i < maxcpus; i++) { 681 ci = cpu_lookup(i); 682 if (ci == NULL) 683 continue; 684 if (ci->ci_data.cpu_idlelwp == NULL) 685 continue; 686 if ((ci->ci_flags & CPUF_PRESENT) == 0) 687 continue; 688 if (ci->ci_flags & (CPUF_BSP|CPUF_SP|CPUF_PRIMARY)) 689 continue; 690 cpu_boot_secondary(ci); 691 kcpuset_set(cpus, cpu_index(ci)); 692 } 693 while (!kcpuset_match(cpus, kcpuset_running)) 694 ; 695 kcpuset_destroy(cpus); 696 697 x86_mp_online = true; 698 699 /* Now that we know about the TSC, attach the timecounter. */ 700 tsc_tc_init(); 701 702 /* Enable zeroing of pages in the idle loop if we have SSE2. */ 703 vm_page_zero_enable = ((cpu_feature[0] & CPUID_SSE2) != 0); 704 } 705 #endif 706 707 static void 708 cpu_init_idle_lwp(struct cpu_info *ci) 709 { 710 struct lwp *l = ci->ci_data.cpu_idlelwp; 711 struct pcb *pcb = lwp_getpcb(l); 712 713 pcb->pcb_cr0 = rcr0(); 714 } 715 716 void 717 cpu_init_idle_lwps(void) 718 { 719 struct cpu_info *ci; 720 u_long i; 721 722 for (i = 0; i < maxcpus; i++) { 723 ci = cpu_lookup(i); 724 if (ci == NULL) 725 continue; 726 if (ci->ci_data.cpu_idlelwp == NULL) 727 continue; 728 if ((ci->ci_flags & CPUF_PRESENT) == 0) 729 continue; 730 cpu_init_idle_lwp(ci); 731 } 732 } 733 734 #ifdef MULTIPROCESSOR 735 void 736 cpu_start_secondary(struct cpu_info *ci) 737 { 738 paddr_t mp_pdirpa; 739 u_long psl; 740 int i; 741 742 mp_pdirpa = pmap_init_tmp_pgtbl(mp_trampoline_paddr); 743 cpu_copy_trampoline(mp_pdirpa); 744 745 atomic_or_32(&ci->ci_flags, CPUF_AP); 746 ci->ci_curlwp = ci->ci_data.cpu_idlelwp; 747 if (CPU_STARTUP(ci, mp_trampoline_paddr) != 0) { 748 return; 749 } 750 751 /* 752 * Wait for it to become ready. Setting cpu_starting opens the 753 * initial gate and allows the AP to start soft initialization. 754 */ 755 KASSERT(cpu_starting == NULL); 756 cpu_starting = ci; 757 for (i = 100000; (!(ci->ci_flags & CPUF_PRESENT)) && i > 0; i--) { 758 i8254_delay(10); 759 } 760 761 if ((ci->ci_flags & CPUF_PRESENT) == 0) { 762 aprint_error_dev(ci->ci_dev, "failed to become ready\n"); 763 #if defined(MPDEBUG) && defined(DDB) 764 printf("dropping into debugger; continue from here to resume boot\n"); 765 Debugger(); 766 #endif 767 } else { 768 /* 769 * Synchronize time stamp counters. Invalidate cache and do 770 * twice to try and minimize possible cache effects. Disable 771 * interrupts to try and rule out any external interference. 772 */ 773 psl = x86_read_psl(); 774 x86_disable_intr(); 775 wbinvd(); 776 tsc_sync_bp(ci); 777 x86_write_psl(psl); 778 } 779 780 CPU_START_CLEANUP(ci); 781 cpu_starting = NULL; 782 } 783 784 void 785 cpu_boot_secondary(struct cpu_info *ci) 786 { 787 int64_t drift; 788 u_long psl; 789 int i; 790 791 atomic_or_32(&ci->ci_flags, CPUF_GO); 792 for (i = 100000; (!(ci->ci_flags & CPUF_RUNNING)) && i > 0; i--) { 793 i8254_delay(10); 794 } 795 if ((ci->ci_flags & CPUF_RUNNING) == 0) { 796 aprint_error_dev(ci->ci_dev, "failed to start\n"); 797 #if defined(MPDEBUG) && defined(DDB) 798 printf("dropping into debugger; continue from here to resume boot\n"); 799 Debugger(); 800 #endif 801 } else { 802 /* Synchronize TSC again, check for drift. */ 803 drift = ci->ci_data.cpu_cc_skew; 804 psl = x86_read_psl(); 805 x86_disable_intr(); 806 wbinvd(); 807 tsc_sync_bp(ci); 808 x86_write_psl(psl); 809 drift -= ci->ci_data.cpu_cc_skew; 810 aprint_debug_dev(ci->ci_dev, "TSC skew=%lld drift=%lld\n", 811 (long long)ci->ci_data.cpu_cc_skew, (long long)drift); 812 tsc_sync_drift(drift); 813 } 814 } 815 816 /* 817 * The CPU ends up here when it's ready to run. 818 * This is called from code in mptramp.s; at this point, we are running 819 * in the idle pcb/idle stack of the new CPU. When this function returns, 820 * this processor will enter the idle loop and start looking for work. 821 */ 822 void 823 cpu_hatch(void *v) 824 { 825 struct cpu_info *ci = (struct cpu_info *)v; 826 struct pcb *pcb; 827 int s, i; 828 829 cpu_init_msrs(ci, true); 830 cpu_probe(ci); 831 832 ci->ci_data.cpu_cc_freq = cpu_info_primary.ci_data.cpu_cc_freq; 833 /* cpu_get_tsc_freq(ci); */ 834 835 KDASSERT((ci->ci_flags & CPUF_PRESENT) == 0); 836 837 /* 838 * Synchronize time stamp counters. Invalidate cache and do twice 839 * to try and minimize possible cache effects. Note that interrupts 840 * are off at this point. 841 */ 842 wbinvd(); 843 atomic_or_32(&ci->ci_flags, CPUF_PRESENT); 844 tsc_sync_ap(ci); 845 846 /* 847 * Wait to be brought online. Use 'monitor/mwait' if available, 848 * in order to make the TSC drift as much as possible. so that 849 * we can detect it later. If not available, try 'pause'. 850 * We'd like to use 'hlt', but we have interrupts off. 851 */ 852 while ((ci->ci_flags & CPUF_GO) == 0) { 853 if ((cpu_feature[1] & CPUID2_MONITOR) != 0) { 854 x86_monitor(&ci->ci_flags, 0, 0); 855 if ((ci->ci_flags & CPUF_GO) != 0) { 856 continue; 857 } 858 x86_mwait(0, 0); 859 } else { 860 /* 861 * XXX The loop repetition count could be a lot higher, but 862 * XXX currently qemu emulator takes a _very_long_time_ to 863 * XXX execute the pause instruction. So for now, use a low 864 * XXX value to allow the cpu to hatch before timing out. 865 */ 866 for (i = 50; i != 0; i--) { 867 x86_pause(); 868 } 869 } 870 } 871 872 /* Because the text may have been patched in x86_patch(). */ 873 wbinvd(); 874 x86_flush(); 875 tlbflushg(); 876 877 KASSERT((ci->ci_flags & CPUF_RUNNING) == 0); 878 879 #ifdef PAE 880 pd_entry_t * l3_pd = ci->ci_pae_l3_pdir; 881 for (i = 0 ; i < PDP_SIZE; i++) { 882 l3_pd[i] = pmap_kernel()->pm_pdirpa[i] | PG_V; 883 } 884 lcr3(ci->ci_pae_l3_pdirpa); 885 #else 886 lcr3(pmap_pdirpa(pmap_kernel(), 0)); 887 #endif 888 889 pcb = lwp_getpcb(curlwp); 890 pcb->pcb_cr3 = rcr3(); 891 pcb = lwp_getpcb(ci->ci_data.cpu_idlelwp); 892 lcr0(pcb->pcb_cr0); 893 894 cpu_init_idt(); 895 gdt_init_cpu(ci); 896 #if NLAPIC > 0 897 lapic_enable(); 898 lapic_set_lvt(); 899 lapic_initclocks(); 900 #endif 901 902 fpuinit(ci); 903 lldt(GSYSSEL(GLDT_SEL, SEL_KPL)); 904 ltr(ci->ci_tss_sel); 905 906 cpu_init(ci); 907 cpu_get_tsc_freq(ci); 908 909 s = splhigh(); 910 lapic_write_tpri(0); 911 x86_enable_intr(); 912 splx(s); 913 x86_errata(); 914 915 aprint_debug_dev(ci->ci_dev, "running\n"); 916 917 idle_loop(NULL); 918 KASSERT(false); 919 } 920 #endif 921 922 #if defined(DDB) 923 924 #include <ddb/db_output.h> 925 #include <machine/db_machdep.h> 926 927 /* 928 * Dump CPU information from ddb. 929 */ 930 void 931 cpu_debug_dump(void) 932 { 933 struct cpu_info *ci; 934 CPU_INFO_ITERATOR cii; 935 936 db_printf("addr dev id flags ipis curlwp fpcurlwp\n"); 937 for (CPU_INFO_FOREACH(cii, ci)) { 938 db_printf("%p %s %ld %x %x %10p %10p\n", 939 ci, 940 ci->ci_dev == NULL ? "BOOT" : device_xname(ci->ci_dev), 941 (long)ci->ci_cpuid, 942 ci->ci_flags, ci->ci_ipis, 943 ci->ci_curlwp, 944 ci->ci_fpcurlwp); 945 } 946 } 947 #endif 948 949 #if NLAPIC > 0 950 static void 951 cpu_copy_trampoline(paddr_t pdir_pa) 952 { 953 extern uint32_t nox_flag; 954 extern u_char cpu_spinup_trampoline[]; 955 extern u_char cpu_spinup_trampoline_end[]; 956 vaddr_t mp_trampoline_vaddr; 957 struct { 958 uint32_t large; 959 uint32_t nox; 960 uint32_t pdir; 961 } smp_data; 962 CTASSERT(sizeof(smp_data) == 3 * 4); 963 964 smp_data.large = (pmap_largepages != 0); 965 smp_data.nox = nox_flag; 966 smp_data.pdir = (uint32_t)(pdir_pa & 0xFFFFFFFF); 967 968 /* Enter the physical address */ 969 mp_trampoline_vaddr = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, 970 UVM_KMF_VAONLY); 971 pmap_kenter_pa(mp_trampoline_vaddr, mp_trampoline_paddr, 972 VM_PROT_READ | VM_PROT_WRITE, 0); 973 pmap_update(pmap_kernel()); 974 975 /* Copy boot code */ 976 memcpy((void *)mp_trampoline_vaddr, 977 cpu_spinup_trampoline, 978 cpu_spinup_trampoline_end - cpu_spinup_trampoline); 979 980 /* Copy smp_data at the end */ 981 memcpy((void *)(mp_trampoline_vaddr + PAGE_SIZE - sizeof(smp_data)), 982 &smp_data, sizeof(smp_data)); 983 984 pmap_kremove(mp_trampoline_vaddr, PAGE_SIZE); 985 pmap_update(pmap_kernel()); 986 uvm_km_free(kernel_map, mp_trampoline_vaddr, PAGE_SIZE, UVM_KMF_VAONLY); 987 } 988 #endif 989 990 #ifdef MULTIPROCESSOR 991 int 992 mp_cpu_start(struct cpu_info *ci, paddr_t target) 993 { 994 unsigned short dwordptr[2]; 995 int error; 996 997 /* 998 * Bootstrap code must be addressable in real mode 999 * and it must be page aligned. 1000 */ 1001 KASSERT(target < 0x10000 && target % PAGE_SIZE == 0); 1002 1003 /* 1004 * "The BSP must initialize CMOS shutdown code to 0Ah ..." 1005 */ 1006 1007 outb(IO_RTC, NVRAM_RESET); 1008 outb(IO_RTC+1, NVRAM_RESET_JUMP); 1009 1010 /* 1011 * "and the warm reset vector (DWORD based at 40:67) to point 1012 * to the AP startup code ..." 1013 */ 1014 1015 dwordptr[0] = 0; 1016 dwordptr[1] = target >> 4; 1017 1018 #if NLAPIC > 0 1019 memcpy((uint8_t *)cmos_data_mapping + 0x467, dwordptr, 4); 1020 #endif 1021 1022 if ((cpu_feature[0] & CPUID_APIC) == 0) { 1023 aprint_error("mp_cpu_start: CPU does not have APIC\n"); 1024 return ENODEV; 1025 } 1026 1027 /* 1028 * ... prior to executing the following sequence:". We'll also add in 1029 * local cache flush, in case the BIOS has left the AP with its cache 1030 * disabled. It may not be able to cope with MP coherency. 1031 */ 1032 wbinvd(); 1033 1034 if (ci->ci_flags & CPUF_AP) { 1035 error = x86_ipi_init(ci->ci_cpuid); 1036 if (error != 0) { 1037 aprint_error_dev(ci->ci_dev, "%s: IPI not taken (1)\n", 1038 __func__); 1039 return error; 1040 } 1041 i8254_delay(10000); 1042 1043 error = x86_ipi_startup(ci->ci_cpuid, target / PAGE_SIZE); 1044 if (error != 0) { 1045 aprint_error_dev(ci->ci_dev, "%s: IPI not taken (2)\n", 1046 __func__); 1047 return error; 1048 } 1049 i8254_delay(200); 1050 1051 error = x86_ipi_startup(ci->ci_cpuid, target / PAGE_SIZE); 1052 if (error != 0) { 1053 aprint_error_dev(ci->ci_dev, "%s: IPI not taken (3)\n", 1054 __func__); 1055 return error; 1056 } 1057 i8254_delay(200); 1058 } 1059 1060 return 0; 1061 } 1062 1063 void 1064 mp_cpu_start_cleanup(struct cpu_info *ci) 1065 { 1066 /* 1067 * Ensure the NVRAM reset byte contains something vaguely sane. 1068 */ 1069 1070 outb(IO_RTC, NVRAM_RESET); 1071 outb(IO_RTC+1, NVRAM_RESET_RST); 1072 } 1073 #endif 1074 1075 #ifdef __x86_64__ 1076 typedef void (vector)(void); 1077 extern vector Xsyscall, Xsyscall32; 1078 #endif 1079 1080 void 1081 cpu_init_msrs(struct cpu_info *ci, bool full) 1082 { 1083 #ifdef __x86_64__ 1084 wrmsr(MSR_STAR, 1085 ((uint64_t)GSEL(GCODE_SEL, SEL_KPL) << 32) | 1086 ((uint64_t)LSEL(LSYSRETBASE_SEL, SEL_UPL) << 48)); 1087 wrmsr(MSR_LSTAR, (uint64_t)Xsyscall); 1088 wrmsr(MSR_CSTAR, (uint64_t)Xsyscall32); 1089 wrmsr(MSR_SFMASK, PSL_NT|PSL_T|PSL_I|PSL_C|PSL_D|PSL_AC); 1090 1091 if (full) { 1092 wrmsr(MSR_FSBASE, 0); 1093 wrmsr(MSR_GSBASE, (uint64_t)ci); 1094 wrmsr(MSR_KERNELGSBASE, 0); 1095 } 1096 #endif /* __x86_64__ */ 1097 1098 if (cpu_feature[2] & CPUID_NOX) 1099 wrmsr(MSR_EFER, rdmsr(MSR_EFER) | EFER_NXE); 1100 } 1101 1102 void 1103 cpu_offline_md(void) 1104 { 1105 int s; 1106 1107 s = splhigh(); 1108 fpusave_cpu(true); 1109 splx(s); 1110 } 1111 1112 /* XXX joerg restructure and restart CPUs individually */ 1113 static bool 1114 cpu_stop(device_t dv) 1115 { 1116 struct cpu_softc *sc = device_private(dv); 1117 struct cpu_info *ci = sc->sc_info; 1118 int err; 1119 1120 KASSERT((ci->ci_flags & CPUF_PRESENT) != 0); 1121 1122 if ((ci->ci_flags & CPUF_PRIMARY) != 0) 1123 return true; 1124 1125 if (ci->ci_data.cpu_idlelwp == NULL) 1126 return true; 1127 1128 sc->sc_wasonline = !(ci->ci_schedstate.spc_flags & SPCF_OFFLINE); 1129 1130 if (sc->sc_wasonline) { 1131 mutex_enter(&cpu_lock); 1132 err = cpu_setstate(ci, false); 1133 mutex_exit(&cpu_lock); 1134 1135 if (err != 0) 1136 return false; 1137 } 1138 1139 return true; 1140 } 1141 1142 static bool 1143 cpu_suspend(device_t dv, const pmf_qual_t *qual) 1144 { 1145 struct cpu_softc *sc = device_private(dv); 1146 struct cpu_info *ci = sc->sc_info; 1147 1148 if ((ci->ci_flags & CPUF_PRESENT) == 0) 1149 return true; 1150 else { 1151 cpufreq_suspend(ci); 1152 } 1153 1154 return cpu_stop(dv); 1155 } 1156 1157 static bool 1158 cpu_resume(device_t dv, const pmf_qual_t *qual) 1159 { 1160 struct cpu_softc *sc = device_private(dv); 1161 struct cpu_info *ci = sc->sc_info; 1162 int err = 0; 1163 1164 if ((ci->ci_flags & CPUF_PRESENT) == 0) 1165 return true; 1166 1167 if ((ci->ci_flags & CPUF_PRIMARY) != 0) 1168 goto out; 1169 1170 if (ci->ci_data.cpu_idlelwp == NULL) 1171 goto out; 1172 1173 if (sc->sc_wasonline) { 1174 mutex_enter(&cpu_lock); 1175 err = cpu_setstate(ci, true); 1176 mutex_exit(&cpu_lock); 1177 } 1178 1179 out: 1180 if (err != 0) 1181 return false; 1182 1183 cpufreq_resume(ci); 1184 1185 return true; 1186 } 1187 1188 static bool 1189 cpu_shutdown(device_t dv, int how) 1190 { 1191 struct cpu_softc *sc = device_private(dv); 1192 struct cpu_info *ci = sc->sc_info; 1193 1194 if ((ci->ci_flags & CPUF_BSP) != 0) 1195 return false; 1196 1197 if ((ci->ci_flags & CPUF_PRESENT) == 0) 1198 return true; 1199 1200 return cpu_stop(dv); 1201 } 1202 1203 void 1204 cpu_get_tsc_freq(struct cpu_info *ci) 1205 { 1206 uint64_t last_tsc; 1207 1208 if (cpu_hascounter()) { 1209 last_tsc = cpu_counter_serializing(); 1210 i8254_delay(100000); 1211 ci->ci_data.cpu_cc_freq = 1212 (cpu_counter_serializing() - last_tsc) * 10; 1213 } 1214 } 1215 1216 void 1217 x86_cpu_idle_mwait(void) 1218 { 1219 struct cpu_info *ci = curcpu(); 1220 1221 KASSERT(ci->ci_ilevel == IPL_NONE); 1222 1223 x86_monitor(&ci->ci_want_resched, 0, 0); 1224 if (__predict_false(ci->ci_want_resched)) { 1225 return; 1226 } 1227 x86_mwait(0, 0); 1228 } 1229 1230 void 1231 x86_cpu_idle_halt(void) 1232 { 1233 struct cpu_info *ci = curcpu(); 1234 1235 KASSERT(ci->ci_ilevel == IPL_NONE); 1236 1237 x86_disable_intr(); 1238 if (!__predict_false(ci->ci_want_resched)) { 1239 x86_stihlt(); 1240 } else { 1241 x86_enable_intr(); 1242 } 1243 } 1244 1245 /* 1246 * Loads pmap for the current CPU. 1247 */ 1248 void 1249 cpu_load_pmap(struct pmap *pmap, struct pmap *oldpmap) 1250 { 1251 #ifdef PAE 1252 struct cpu_info *ci = curcpu(); 1253 bool interrupts_enabled; 1254 pd_entry_t *l3_pd = ci->ci_pae_l3_pdir; 1255 int i; 1256 1257 /* 1258 * disable interrupts to block TLB shootdowns, which can reload cr3. 1259 * while this doesn't block NMIs, it's probably ok as NMIs unlikely 1260 * reload cr3. 1261 */ 1262 interrupts_enabled = (x86_read_flags() & PSL_I) != 0; 1263 if (interrupts_enabled) 1264 x86_disable_intr(); 1265 1266 for (i = 0 ; i < PDP_SIZE; i++) { 1267 l3_pd[i] = pmap->pm_pdirpa[i] | PG_V; 1268 } 1269 1270 if (interrupts_enabled) 1271 x86_enable_intr(); 1272 tlbflush(); 1273 #else /* PAE */ 1274 lcr3(pmap_pdirpa(pmap, 0)); 1275 #endif /* PAE */ 1276 } 1277 1278 /* 1279 * Notify all other cpus to halt. 1280 */ 1281 1282 void 1283 cpu_broadcast_halt(void) 1284 { 1285 x86_broadcast_ipi(X86_IPI_HALT); 1286 } 1287 1288 /* 1289 * Send a dummy ipi to a cpu to force it to run splraise()/spllower() 1290 */ 1291 1292 void 1293 cpu_kick(struct cpu_info *ci) 1294 { 1295 x86_send_ipi(ci, 0); 1296 } 1297
Indexes created Thu Sep 25 16:09:42 GMT 2025