cpu.c revision 1.166
1 /* $NetBSD: cpu.c,v 1.166 2019/02/14 08:18:25 cherry Exp $ */ 2 3 /* 4 * Copyright (c) 2000-2012 NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Bill Sommerfeld of RedBack Networks Inc, and by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * Copyright (c) 1999 Stefan Grefen 34 * 35 * Redistribution and use in source and binary forms, with or without 36 * modification, are permitted provided that the following conditions 37 * are met: 38 * 1. Redistributions of source code must retain the above copyright 39 * notice, this list of conditions and the following disclaimer. 40 * 2. Redistributions in binary form must reproduce the above copyright 41 * notice, this list of conditions and the following disclaimer in the 42 * documentation and/or other materials provided with the distribution. 43 * 3. All advertising materials mentioning features or use of this software 44 * must display the following acknowledgement: 45 * This product includes software developed by the NetBSD 46 * Foundation, Inc. and its contributors. 47 * 4. Neither the name of The NetBSD Foundation nor the names of its 48 * contributors may be used to endorse or promote products derived 49 * from this software without specific prior written permission. 50 * 51 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND ANY 52 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 53 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 54 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR AND CONTRIBUTORS BE LIABLE 55 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 56 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 57 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 58 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 59 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 60 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 61 * SUCH DAMAGE. 62 */ 63 64 #include <sys/cdefs.h> 65 __KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.166 2019/02/14 08:18:25 cherry Exp $"); 66 67 #include "opt_ddb.h" 68 #include "opt_mpbios.h" /* for MPDEBUG */ 69 #include "opt_mtrr.h" 70 #include "opt_multiprocessor.h" 71 #include "opt_svs.h" 72 73 #include "lapic.h" 74 #include "ioapic.h" 75 76 #include <sys/param.h> 77 #include <sys/proc.h> 78 #include <sys/systm.h> 79 #include <sys/device.h> 80 #include <sys/cpu.h> 81 #include <sys/cpufreq.h> 82 #include <sys/idle.h> 83 #include <sys/atomic.h> 84 #include <sys/reboot.h> 85 86 #include <uvm/uvm.h> 87 88 #include "acpica.h" /* for NACPICA, for mp_verbose */ 89 90 #include <machine/cpufunc.h> 91 #include <machine/cpuvar.h> 92 #include <machine/pmap.h> 93 #include <machine/vmparam.h> 94 #if defined(MULTIPROCESSOR) 95 #include <machine/mpbiosvar.h> 96 #endif 97 #include <machine/mpconfig.h> /* for mp_verbose */ 98 #include <machine/pcb.h> 99 #include <machine/specialreg.h> 100 #include <machine/segments.h> 101 #include <machine/gdt.h> 102 #include <machine/mtrr.h> 103 #include <machine/pio.h> 104 #include <machine/cpu_counter.h> 105 106 #include <x86/fpu.h> 107 108 #if NLAPIC > 0 109 #include <machine/apicvar.h> 110 #include <machine/i82489reg.h> 111 #include <machine/i82489var.h> 112 #endif 113 114 #include <dev/ic/mc146818reg.h> 115 #include <i386/isa/nvram.h> 116 #include <dev/isa/isareg.h> 117 118 #include "tsc.h" 119 120 static int cpu_match(device_t, cfdata_t, void *); 121 static void cpu_attach(device_t, device_t, void *); 122 static void cpu_defer(device_t); 123 static int cpu_rescan(device_t, const char *, const int *); 124 static void cpu_childdetached(device_t, device_t); 125 static bool cpu_stop(device_t); 126 static bool cpu_suspend(device_t, const pmf_qual_t *); 127 static bool cpu_resume(device_t, const pmf_qual_t *); 128 static bool cpu_shutdown(device_t, int); 129 130 struct cpu_softc { 131 device_t sc_dev; /* device tree glue */ 132 struct cpu_info *sc_info; /* pointer to CPU info */ 133 bool sc_wasonline; 134 }; 135 136 #ifdef MULTIPROCESSOR 137 int mp_cpu_start(struct cpu_info *, paddr_t); 138 void mp_cpu_start_cleanup(struct cpu_info *); 139 const struct cpu_functions mp_cpu_funcs = { mp_cpu_start, NULL, 140 mp_cpu_start_cleanup }; 141 #endif 142 143 144 CFATTACH_DECL2_NEW(cpu, sizeof(struct cpu_softc), 145 cpu_match, cpu_attach, NULL, NULL, cpu_rescan, cpu_childdetached); 146 147 /* 148 * Statically-allocated CPU info for the primary CPU (or the only 149 * CPU, on uniprocessors). The CPU info list is initialized to 150 * point at it. 151 */ 152 struct cpu_info cpu_info_primary __aligned(CACHE_LINE_SIZE) = { 153 .ci_dev = 0, 154 .ci_self = &cpu_info_primary, 155 .ci_idepth = -1, 156 .ci_curlwp = &lwp0, 157 .ci_curldt = -1, 158 }; 159 160 struct cpu_info *cpu_info_list = &cpu_info_primary; 161 162 #ifdef i386 163 void cpu_set_tss_gates(struct cpu_info *); 164 #endif 165 166 static void cpu_init_idle_lwp(struct cpu_info *); 167 168 uint32_t cpu_feature[7] __read_mostly; /* X86 CPUID feature bits */ 169 /* [0] basic features cpuid.1:%edx 170 * [1] basic features cpuid.1:%ecx (CPUID2_xxx bits) 171 * [2] extended features cpuid:80000001:%edx 172 * [3] extended features cpuid:80000001:%ecx 173 * [4] VIA padlock features 174 * [5] structured extended features cpuid.7:%ebx 175 * [6] structured extended features cpuid.7:%ecx 176 */ 177 178 #ifdef MULTIPROCESSOR 179 bool x86_mp_online; 180 paddr_t mp_trampoline_paddr = MP_TRAMPOLINE; 181 #endif 182 #if NLAPIC > 0 183 static vaddr_t cmos_data_mapping; 184 #endif 185 struct cpu_info *cpu_starting; 186 187 #ifdef MULTIPROCESSOR 188 void cpu_hatch(void *); 189 static void cpu_boot_secondary(struct cpu_info *ci); 190 static void cpu_start_secondary(struct cpu_info *ci); 191 #if NLAPIC > 0 192 static void cpu_copy_trampoline(paddr_t); 193 #endif 194 #endif /* MULTIPROCESSOR */ 195 196 /* 197 * Runs once per boot once multiprocessor goo has been detected and 198 * the local APIC on the boot processor has been mapped. 199 * 200 * Called from lapic_boot_init() (from mpbios_scan()). 201 */ 202 #if NLAPIC > 0 203 void 204 cpu_init_first(void) 205 { 206 207 cpu_info_primary.ci_cpuid = lapic_cpu_number(); 208 209 cmos_data_mapping = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, UVM_KMF_VAONLY); 210 if (cmos_data_mapping == 0) 211 panic("No KVA for page 0"); 212 pmap_kenter_pa(cmos_data_mapping, 0, VM_PROT_READ|VM_PROT_WRITE, 0); 213 pmap_update(pmap_kernel()); 214 } 215 #endif 216 217 static int 218 cpu_match(device_t parent, cfdata_t match, void *aux) 219 { 220 221 return 1; 222 } 223 224 #ifdef __HAVE_PCPU_AREA 225 void 226 cpu_pcpuarea_init(struct cpu_info *ci) 227 { 228 struct vm_page *pg; 229 size_t i, npages; 230 vaddr_t base, va; 231 paddr_t pa; 232 233 CTASSERT(sizeof(struct pcpu_entry) % PAGE_SIZE == 0); 234 235 npages = sizeof(struct pcpu_entry) / PAGE_SIZE; 236 base = (vaddr_t)&pcpuarea->ent[cpu_index(ci)]; 237 238 for (i = 0; i < npages; i++) { 239 pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO); 240 if (pg == NULL) { 241 panic("failed to allocate pcpu PA"); 242 } 243 244 va = base + i * PAGE_SIZE; 245 pa = VM_PAGE_TO_PHYS(pg); 246 247 pmap_kenter_pa(va, pa, VM_PROT_READ|VM_PROT_WRITE, 0); 248 } 249 250 pmap_update(pmap_kernel()); 251 } 252 #endif 253 254 static void 255 cpu_vm_init(struct cpu_info *ci) 256 { 257 int ncolors = 2, i; 258 259 for (i = CAI_ICACHE; i <= CAI_L2CACHE; i++) { 260 struct x86_cache_info *cai; 261 int tcolors; 262 263 cai = &ci->ci_cinfo[i]; 264 265 tcolors = atop(cai->cai_totalsize); 266 switch(cai->cai_associativity) { 267 case 0xff: 268 tcolors = 1; /* fully associative */ 269 break; 270 case 0: 271 case 1: 272 break; 273 default: 274 tcolors /= cai->cai_associativity; 275 } 276 ncolors = uimax(ncolors, tcolors); 277 /* 278 * If the desired number of colors is not a power of 279 * two, it won't be good. Find the greatest power of 280 * two which is an even divisor of the number of colors, 281 * to preserve even coloring of pages. 282 */ 283 if (ncolors & (ncolors - 1) ) { 284 int try, picked = 1; 285 for (try = 1; try < ncolors; try *= 2) { 286 if (ncolors % try == 0) picked = try; 287 } 288 if (picked == 1) { 289 panic("desired number of cache colors %d is " 290 " > 1, but not even!", ncolors); 291 } 292 ncolors = picked; 293 } 294 } 295 296 /* 297 * Knowing the size of the largest cache on this CPU, potentially 298 * re-color our pages. 299 */ 300 aprint_debug_dev(ci->ci_dev, "%d page colors\n", ncolors); 301 uvm_page_recolor(ncolors); 302 303 pmap_tlb_cpu_init(ci); 304 #ifndef __HAVE_DIRECT_MAP 305 pmap_vpage_cpu_init(ci); 306 #endif 307 } 308 309 static void 310 cpu_attach(device_t parent, device_t self, void *aux) 311 { 312 struct cpu_softc *sc = device_private(self); 313 struct cpu_attach_args *caa = aux; 314 struct cpu_info *ci; 315 uintptr_t ptr; 316 #if NLAPIC > 0 317 int cpunum = caa->cpu_number; 318 #endif 319 static bool again; 320 321 sc->sc_dev = self; 322 323 if (ncpu > maxcpus) { 324 #ifndef _LP64 325 aprint_error(": too many CPUs, please use NetBSD/amd64\n"); 326 #else 327 aprint_error(": too many CPUs\n"); 328 #endif 329 return; 330 } 331 332 /* 333 * If we're an Application Processor, allocate a cpu_info 334 * structure, otherwise use the primary's. 335 */ 336 if (caa->cpu_role == CPU_ROLE_AP) { 337 if ((boothowto & RB_MD1) != 0) { 338 aprint_error(": multiprocessor boot disabled\n"); 339 if (!pmf_device_register(self, NULL, NULL)) 340 aprint_error_dev(self, 341 "couldn't establish power handler\n"); 342 return; 343 } 344 aprint_naive(": Application Processor\n"); 345 ptr = (uintptr_t)uvm_km_alloc(kernel_map, 346 sizeof(*ci) + CACHE_LINE_SIZE - 1, 0, 347 UVM_KMF_WIRED|UVM_KMF_ZERO); 348 ci = (struct cpu_info *)roundup2(ptr, CACHE_LINE_SIZE); 349 ci->ci_curldt = -1; 350 } else { 351 aprint_naive(": %s Processor\n", 352 caa->cpu_role == CPU_ROLE_SP ? "Single" : "Boot"); 353 ci = &cpu_info_primary; 354 #if NLAPIC > 0 355 if (cpunum != lapic_cpu_number()) { 356 /* XXX should be done earlier. */ 357 uint32_t reg; 358 aprint_verbose("\n"); 359 aprint_verbose_dev(self, "running CPU at apic %d" 360 " instead of at expected %d", lapic_cpu_number(), 361 cpunum); 362 reg = lapic_readreg(LAPIC_ID); 363 lapic_writereg(LAPIC_ID, (reg & ~LAPIC_ID_MASK) | 364 (cpunum << LAPIC_ID_SHIFT)); 365 } 366 if (cpunum != lapic_cpu_number()) { 367 aprint_error_dev(self, "unable to reset apic id\n"); 368 } 369 #endif 370 } 371 372 ci->ci_self = ci; 373 sc->sc_info = ci; 374 ci->ci_dev = self; 375 ci->ci_acpiid = caa->cpu_id; 376 ci->ci_cpuid = caa->cpu_number; 377 ci->ci_func = caa->cpu_func; 378 aprint_normal("\n"); 379 380 /* Must be before mi_cpu_attach(). */ 381 cpu_vm_init(ci); 382 383 if (caa->cpu_role == CPU_ROLE_AP) { 384 int error; 385 386 error = mi_cpu_attach(ci); 387 if (error != 0) { 388 aprint_error_dev(self, 389 "mi_cpu_attach failed with %d\n", error); 390 return; 391 } 392 #ifdef __HAVE_PCPU_AREA 393 cpu_pcpuarea_init(ci); 394 #endif 395 cpu_init_tss(ci); 396 } else { 397 KASSERT(ci->ci_data.cpu_idlelwp != NULL); 398 } 399 400 #ifdef SVS 401 cpu_svs_init(ci); 402 #endif 403 404 pmap_reference(pmap_kernel()); 405 ci->ci_pmap = pmap_kernel(); 406 ci->ci_tlbstate = TLBSTATE_STALE; 407 408 /* 409 * Boot processor may not be attached first, but the below 410 * must be done to allow booting other processors. 411 */ 412 if (!again) { 413 atomic_or_32(&ci->ci_flags, CPUF_PRESENT | CPUF_PRIMARY); 414 /* Basic init. */ 415 cpu_intr_init(ci); 416 cpu_get_tsc_freq(ci); 417 cpu_init(ci); 418 #ifdef i386 419 cpu_set_tss_gates(ci); 420 #endif 421 pmap_cpu_init_late(ci); 422 #if NLAPIC > 0 423 if (caa->cpu_role != CPU_ROLE_SP) { 424 /* Enable lapic. */ 425 lapic_enable(); 426 lapic_set_lvt(); 427 lapic_calibrate_timer(ci); 428 } 429 #endif 430 /* Make sure DELAY() is initialized. */ 431 DELAY(1); 432 again = true; 433 } 434 435 /* further PCB init done later. */ 436 437 switch (caa->cpu_role) { 438 case CPU_ROLE_SP: 439 atomic_or_32(&ci->ci_flags, CPUF_SP); 440 cpu_identify(ci); 441 x86_errata(); 442 x86_cpu_idle_init(); 443 break; 444 445 case CPU_ROLE_BP: 446 atomic_or_32(&ci->ci_flags, CPUF_BSP); 447 cpu_identify(ci); 448 x86_errata(); 449 x86_cpu_idle_init(); 450 break; 451 452 #ifdef MULTIPROCESSOR 453 case CPU_ROLE_AP: 454 /* 455 * report on an AP 456 */ 457 cpu_intr_init(ci); 458 gdt_alloc_cpu(ci); 459 #ifdef i386 460 cpu_set_tss_gates(ci); 461 #endif 462 pmap_cpu_init_late(ci); 463 cpu_start_secondary(ci); 464 if (ci->ci_flags & CPUF_PRESENT) { 465 struct cpu_info *tmp; 466 467 cpu_identify(ci); 468 tmp = cpu_info_list; 469 while (tmp->ci_next) 470 tmp = tmp->ci_next; 471 472 tmp->ci_next = ci; 473 } 474 break; 475 #endif 476 477 default: 478 panic("unknown processor type??\n"); 479 } 480 481 pat_init(ci); 482 483 if (!pmf_device_register1(self, cpu_suspend, cpu_resume, cpu_shutdown)) 484 aprint_error_dev(self, "couldn't establish power handler\n"); 485 486 #ifdef MULTIPROCESSOR 487 if (mp_verbose) { 488 struct lwp *l = ci->ci_data.cpu_idlelwp; 489 struct pcb *pcb = lwp_getpcb(l); 490 491 aprint_verbose_dev(self, 492 "idle lwp at %p, idle sp at %p\n", 493 l, 494 #ifdef i386 495 (void *)pcb->pcb_esp 496 #else 497 (void *)pcb->pcb_rsp 498 #endif 499 ); 500 } 501 #endif 502 503 /* 504 * Postpone the "cpufeaturebus" scan. 505 * It is safe to scan the pseudo-bus 506 * only after all CPUs have attached. 507 */ 508 (void)config_defer(self, cpu_defer); 509 } 510 511 static void 512 cpu_defer(device_t self) 513 { 514 cpu_rescan(self, NULL, NULL); 515 } 516 517 static int 518 cpu_rescan(device_t self, const char *ifattr, const int *locators) 519 { 520 struct cpu_softc *sc = device_private(self); 521 struct cpufeature_attach_args cfaa; 522 struct cpu_info *ci = sc->sc_info; 523 524 memset(&cfaa, 0, sizeof(cfaa)); 525 cfaa.ci = ci; 526 527 if (ifattr_match(ifattr, "cpufeaturebus")) { 528 if (ci->ci_frequency == NULL) { 529 cfaa.name = "frequency"; 530 ci->ci_frequency = config_found_ia(self, 531 "cpufeaturebus", &cfaa, NULL); 532 } 533 534 if (ci->ci_padlock == NULL) { 535 cfaa.name = "padlock"; 536 ci->ci_padlock = config_found_ia(self, 537 "cpufeaturebus", &cfaa, NULL); 538 } 539 540 if (ci->ci_temperature == NULL) { 541 cfaa.name = "temperature"; 542 ci->ci_temperature = config_found_ia(self, 543 "cpufeaturebus", &cfaa, NULL); 544 } 545 546 if (ci->ci_vm == NULL) { 547 cfaa.name = "vm"; 548 ci->ci_vm = config_found_ia(self, 549 "cpufeaturebus", &cfaa, NULL); 550 } 551 } 552 553 return 0; 554 } 555 556 static void 557 cpu_childdetached(device_t self, device_t child) 558 { 559 struct cpu_softc *sc = device_private(self); 560 struct cpu_info *ci = sc->sc_info; 561 562 if (ci->ci_frequency == child) 563 ci->ci_frequency = NULL; 564 565 if (ci->ci_padlock == child) 566 ci->ci_padlock = NULL; 567 568 if (ci->ci_temperature == child) 569 ci->ci_temperature = NULL; 570 571 if (ci->ci_vm == child) 572 ci->ci_vm = NULL; 573 } 574 575 /* 576 * Initialize the processor appropriately. 577 */ 578 579 void 580 cpu_init(struct cpu_info *ci) 581 { 582 extern int x86_fpu_save; 583 uint32_t cr4 = 0; 584 585 lcr0(rcr0() | CR0_WP); 586 587 /* 588 * On a P6 or above, enable global TLB caching if the 589 * hardware supports it. 590 */ 591 if (cpu_feature[0] & CPUID_PGE) 592 #ifdef SVS 593 if (!svs_enabled) 594 #endif 595 cr4 |= CR4_PGE; /* enable global TLB caching */ 596 597 /* 598 * If we have FXSAVE/FXRESTOR, use them. 599 */ 600 if (cpu_feature[0] & CPUID_FXSR) { 601 cr4 |= CR4_OSFXSR; 602 603 /* 604 * If we have SSE/SSE2, enable XMM exceptions. 605 */ 606 if (cpu_feature[0] & (CPUID_SSE|CPUID_SSE2)) 607 cr4 |= CR4_OSXMMEXCPT; 608 } 609 610 /* If xsave is supported, enable it */ 611 if (cpu_feature[1] & CPUID2_XSAVE) 612 cr4 |= CR4_OSXSAVE; 613 614 /* If SMEP is supported, enable it */ 615 if (cpu_feature[5] & CPUID_SEF_SMEP) 616 cr4 |= CR4_SMEP; 617 618 /* If SMAP is supported, enable it */ 619 if (cpu_feature[5] & CPUID_SEF_SMAP) 620 cr4 |= CR4_SMAP; 621 622 if (cr4) { 623 cr4 |= rcr4(); 624 lcr4(cr4); 625 } 626 627 /* 628 * Changing CR4 register may change cpuid values. For example, setting 629 * CR4_OSXSAVE sets CPUID2_OSXSAVE. The CPUID2_OSXSAVE is in 630 * ci_feat_val[1], so update it. 631 * XXX Other than ci_feat_val[1] might be changed. 632 */ 633 if (cpuid_level >= 1) { 634 u_int descs[4]; 635 636 x86_cpuid(1, descs); 637 ci->ci_feat_val[1] = descs[2]; 638 } 639 640 if (x86_fpu_save >= FPU_SAVE_FXSAVE) { 641 fpuinit_mxcsr_mask(); 642 } 643 644 /* If xsave is enabled, enable all fpu features */ 645 if (cr4 & CR4_OSXSAVE) 646 wrxcr(0, x86_xsave_features & XCR0_FPU); 647 648 #ifdef MTRR 649 /* 650 * On a P6 or above, initialize MTRR's if the hardware supports them. 651 */ 652 if (cpu_feature[0] & CPUID_MTRR) { 653 if ((ci->ci_flags & CPUF_AP) == 0) 654 i686_mtrr_init_first(); 655 mtrr_init_cpu(ci); 656 } 657 658 #ifdef i386 659 if (strcmp((char *)(ci->ci_vendor), "AuthenticAMD") == 0) { 660 /* 661 * Must be a K6-2 Step >= 7 or a K6-III. 662 */ 663 if (CPUID_TO_FAMILY(ci->ci_signature) == 5) { 664 if (CPUID_TO_MODEL(ci->ci_signature) > 8 || 665 (CPUID_TO_MODEL(ci->ci_signature) == 8 && 666 CPUID_TO_STEPPING(ci->ci_signature) >= 7)) { 667 mtrr_funcs = &k6_mtrr_funcs; 668 k6_mtrr_init_first(); 669 mtrr_init_cpu(ci); 670 } 671 } 672 } 673 #endif /* i386 */ 674 #endif /* MTRR */ 675 676 if (ci != &cpu_info_primary) { 677 /* Synchronize TSC */ 678 wbinvd(); 679 atomic_or_32(&ci->ci_flags, CPUF_RUNNING); 680 tsc_sync_ap(ci); 681 } else { 682 atomic_or_32(&ci->ci_flags, CPUF_RUNNING); 683 } 684 } 685 686 #ifdef MULTIPROCESSOR 687 void 688 cpu_boot_secondary_processors(void) 689 { 690 struct cpu_info *ci; 691 kcpuset_t *cpus; 692 u_long i; 693 694 #ifndef XEN 695 /* Now that we know the number of CPUs, patch the text segment. */ 696 x86_patch(false); 697 #endif 698 699 kcpuset_create(&cpus, true); 700 kcpuset_set(cpus, cpu_index(curcpu())); 701 for (i = 0; i < maxcpus; i++) { 702 ci = cpu_lookup(i); 703 if (ci == NULL) 704 continue; 705 if (ci->ci_data.cpu_idlelwp == NULL) 706 continue; 707 if ((ci->ci_flags & CPUF_PRESENT) == 0) 708 continue; 709 if (ci->ci_flags & (CPUF_BSP|CPUF_SP|CPUF_PRIMARY)) 710 continue; 711 cpu_boot_secondary(ci); 712 kcpuset_set(cpus, cpu_index(ci)); 713 } 714 while (!kcpuset_match(cpus, kcpuset_running)) 715 ; 716 kcpuset_destroy(cpus); 717 718 x86_mp_online = true; 719 720 /* Now that we know about the TSC, attach the timecounter. */ 721 tsc_tc_init(); 722 723 /* Enable zeroing of pages in the idle loop if we have SSE2. */ 724 vm_page_zero_enable = ((cpu_feature[0] & CPUID_SSE2) != 0); 725 } 726 #endif 727 728 static void 729 cpu_init_idle_lwp(struct cpu_info *ci) 730 { 731 struct lwp *l = ci->ci_data.cpu_idlelwp; 732 struct pcb *pcb = lwp_getpcb(l); 733 734 pcb->pcb_cr0 = rcr0(); 735 } 736 737 void 738 cpu_init_idle_lwps(void) 739 { 740 struct cpu_info *ci; 741 u_long i; 742 743 for (i = 0; i < maxcpus; i++) { 744 ci = cpu_lookup(i); 745 if (ci == NULL) 746 continue; 747 if (ci->ci_data.cpu_idlelwp == NULL) 748 continue; 749 if ((ci->ci_flags & CPUF_PRESENT) == 0) 750 continue; 751 cpu_init_idle_lwp(ci); 752 } 753 } 754 755 #ifdef MULTIPROCESSOR 756 void 757 cpu_start_secondary(struct cpu_info *ci) 758 { 759 u_long psl; 760 int i; 761 762 #if NLAPIC > 0 763 paddr_t mp_pdirpa; 764 mp_pdirpa = pmap_init_tmp_pgtbl(mp_trampoline_paddr); 765 cpu_copy_trampoline(mp_pdirpa); 766 #endif 767 768 atomic_or_32(&ci->ci_flags, CPUF_AP); 769 ci->ci_curlwp = ci->ci_data.cpu_idlelwp; 770 if (CPU_STARTUP(ci, mp_trampoline_paddr) != 0) { 771 return; 772 } 773 774 /* 775 * Wait for it to become ready. Setting cpu_starting opens the 776 * initial gate and allows the AP to start soft initialization. 777 */ 778 KASSERT(cpu_starting == NULL); 779 cpu_starting = ci; 780 for (i = 100000; (!(ci->ci_flags & CPUF_PRESENT)) && i > 0; i--) { 781 i8254_delay(10); 782 } 783 784 if ((ci->ci_flags & CPUF_PRESENT) == 0) { 785 aprint_error_dev(ci->ci_dev, "failed to become ready\n"); 786 #if defined(MPDEBUG) && defined(DDB) 787 printf("dropping into debugger; continue from here to resume boot\n"); 788 Debugger(); 789 #endif 790 } else { 791 /* 792 * Synchronize time stamp counters. Invalidate cache and do 793 * twice (in tsc_sync_bp) to minimize possible cache effects. 794 * Disable interrupts to try and rule out any external 795 * interference. 796 */ 797 psl = x86_read_psl(); 798 x86_disable_intr(); 799 wbinvd(); 800 tsc_sync_bp(ci); 801 x86_write_psl(psl); 802 } 803 804 CPU_START_CLEANUP(ci); 805 cpu_starting = NULL; 806 } 807 808 void 809 cpu_boot_secondary(struct cpu_info *ci) 810 { 811 int64_t drift; 812 u_long psl; 813 int i; 814 815 atomic_or_32(&ci->ci_flags, CPUF_GO); 816 for (i = 100000; (!(ci->ci_flags & CPUF_RUNNING)) && i > 0; i--) { 817 i8254_delay(10); 818 } 819 if ((ci->ci_flags & CPUF_RUNNING) == 0) { 820 aprint_error_dev(ci->ci_dev, "failed to start\n"); 821 #if defined(MPDEBUG) && defined(DDB) 822 printf("dropping into debugger; continue from here to resume boot\n"); 823 Debugger(); 824 #endif 825 } else { 826 /* Synchronize TSC again, check for drift. */ 827 drift = ci->ci_data.cpu_cc_skew; 828 psl = x86_read_psl(); 829 x86_disable_intr(); 830 wbinvd(); 831 tsc_sync_bp(ci); 832 x86_write_psl(psl); 833 drift -= ci->ci_data.cpu_cc_skew; 834 aprint_debug_dev(ci->ci_dev, "TSC skew=%lld drift=%lld\n", 835 (long long)ci->ci_data.cpu_cc_skew, (long long)drift); 836 tsc_sync_drift(drift); 837 } 838 } 839 840 /* 841 * The CPU ends up here when it's ready to run. 842 * This is called from code in mptramp.s; at this point, we are running 843 * in the idle pcb/idle stack of the new CPU. When this function returns, 844 * this processor will enter the idle loop and start looking for work. 845 */ 846 void 847 cpu_hatch(void *v) 848 { 849 struct cpu_info *ci = (struct cpu_info *)v; 850 struct pcb *pcb; 851 int s, i; 852 853 /* ------------------------------------------------------------- */ 854 855 /* 856 * This section of code must be compiled with SSP disabled, to 857 * prevent a race against cpu0. See sys/conf/ssp.mk. 858 */ 859 860 cpu_init_msrs(ci, true); 861 cpu_probe(ci); 862 cpu_speculation_init(ci); 863 864 ci->ci_data.cpu_cc_freq = cpu_info_primary.ci_data.cpu_cc_freq; 865 /* cpu_get_tsc_freq(ci); */ 866 867 KDASSERT((ci->ci_flags & CPUF_PRESENT) == 0); 868 869 /* 870 * Synchronize the TSC for the first time. Note that interrupts are 871 * off at this point. 872 */ 873 wbinvd(); 874 atomic_or_32(&ci->ci_flags, CPUF_PRESENT); 875 tsc_sync_ap(ci); 876 877 /* ------------------------------------------------------------- */ 878 879 /* 880 * Wait to be brought online. 881 * 882 * Use MONITOR/MWAIT if available. These instructions put the CPU in 883 * a low consumption mode (C-state), and if the TSC is not invariant, 884 * this causes the TSC to drift. We want this to happen, so that we 885 * can later detect (in tsc_tc_init) any abnormal drift with invariant 886 * TSCs. That's just for safety; by definition such drifts should 887 * never occur with invariant TSCs. 888 * 889 * If not available, try PAUSE. We'd like to use HLT, but we have 890 * interrupts off. 891 */ 892 while ((ci->ci_flags & CPUF_GO) == 0) { 893 if ((cpu_feature[1] & CPUID2_MONITOR) != 0) { 894 x86_monitor(&ci->ci_flags, 0, 0); 895 if ((ci->ci_flags & CPUF_GO) != 0) { 896 continue; 897 } 898 x86_mwait(0, 0); 899 } else { 900 /* 901 * XXX The loop repetition count could be a lot higher, but 902 * XXX currently qemu emulator takes a _very_long_time_ to 903 * XXX execute the pause instruction. So for now, use a low 904 * XXX value to allow the cpu to hatch before timing out. 905 */ 906 for (i = 50; i != 0; i--) { 907 x86_pause(); 908 } 909 } 910 } 911 912 /* Because the text may have been patched in x86_patch(). */ 913 wbinvd(); 914 x86_flush(); 915 tlbflushg(); 916 917 KASSERT((ci->ci_flags & CPUF_RUNNING) == 0); 918 919 #ifdef PAE 920 pd_entry_t * l3_pd = ci->ci_pae_l3_pdir; 921 for (i = 0 ; i < PDP_SIZE; i++) { 922 l3_pd[i] = pmap_kernel()->pm_pdirpa[i] | PG_V; 923 } 924 lcr3(ci->ci_pae_l3_pdirpa); 925 #else 926 lcr3(pmap_pdirpa(pmap_kernel(), 0)); 927 #endif 928 929 pcb = lwp_getpcb(curlwp); 930 pcb->pcb_cr3 = rcr3(); 931 pcb = lwp_getpcb(ci->ci_data.cpu_idlelwp); 932 lcr0(pcb->pcb_cr0); 933 934 cpu_init_idt(); 935 gdt_init_cpu(ci); 936 #if NLAPIC > 0 937 lapic_enable(); 938 lapic_set_lvt(); 939 lapic_initclocks(); 940 #endif 941 942 fpuinit(ci); 943 lldt(GSYSSEL(GLDT_SEL, SEL_KPL)); 944 ltr(ci->ci_tss_sel); 945 946 /* 947 * cpu_init will re-synchronize the TSC, and will detect any abnormal 948 * drift that would have been caused by the use of MONITOR/MWAIT 949 * above. 950 */ 951 cpu_init(ci); 952 cpu_get_tsc_freq(ci); 953 954 s = splhigh(); 955 #if NLAPIC > 0 956 lapic_write_tpri(0); 957 #endif 958 x86_enable_intr(); 959 splx(s); 960 x86_errata(); 961 962 aprint_debug_dev(ci->ci_dev, "running\n"); 963 964 idle_loop(NULL); 965 KASSERT(false); 966 } 967 #endif 968 969 #if defined(DDB) 970 971 #include <ddb/db_output.h> 972 #include <machine/db_machdep.h> 973 974 /* 975 * Dump CPU information from ddb. 976 */ 977 void 978 cpu_debug_dump(void) 979 { 980 struct cpu_info *ci; 981 CPU_INFO_ITERATOR cii; 982 983 db_printf("addr dev id flags ipis curlwp fpcurlwp\n"); 984 for (CPU_INFO_FOREACH(cii, ci)) { 985 db_printf("%p %s %ld %x %x %10p %10p\n", 986 ci, 987 ci->ci_dev == NULL ? "BOOT" : device_xname(ci->ci_dev), 988 (long)ci->ci_cpuid, 989 ci->ci_flags, ci->ci_ipis, 990 ci->ci_curlwp, 991 ci->ci_fpcurlwp); 992 } 993 } 994 #endif 995 996 #ifdef MULTIPROCESSOR 997 #if NLAPIC > 0 998 static void 999 cpu_copy_trampoline(paddr_t pdir_pa) 1000 { 1001 extern uint32_t nox_flag; 1002 extern u_char cpu_spinup_trampoline[]; 1003 extern u_char cpu_spinup_trampoline_end[]; 1004 vaddr_t mp_trampoline_vaddr; 1005 struct { 1006 uint32_t large; 1007 uint32_t nox; 1008 uint32_t pdir; 1009 } smp_data; 1010 CTASSERT(sizeof(smp_data) == 3 * 4); 1011 1012 smp_data.large = (pmap_largepages != 0); 1013 smp_data.nox = nox_flag; 1014 smp_data.pdir = (uint32_t)(pdir_pa & 0xFFFFFFFF); 1015 1016 /* Enter the physical address */ 1017 mp_trampoline_vaddr = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, 1018 UVM_KMF_VAONLY); 1019 pmap_kenter_pa(mp_trampoline_vaddr, mp_trampoline_paddr, 1020 VM_PROT_READ | VM_PROT_WRITE, 0); 1021 pmap_update(pmap_kernel()); 1022 1023 /* Copy boot code */ 1024 memcpy((void *)mp_trampoline_vaddr, 1025 cpu_spinup_trampoline, 1026 cpu_spinup_trampoline_end - cpu_spinup_trampoline); 1027 1028 /* Copy smp_data at the end */ 1029 memcpy((void *)(mp_trampoline_vaddr + PAGE_SIZE - sizeof(smp_data)), 1030 &smp_data, sizeof(smp_data)); 1031 1032 pmap_kremove(mp_trampoline_vaddr, PAGE_SIZE); 1033 pmap_update(pmap_kernel()); 1034 uvm_km_free(kernel_map, mp_trampoline_vaddr, PAGE_SIZE, UVM_KMF_VAONLY); 1035 } 1036 #endif 1037 1038 int 1039 mp_cpu_start(struct cpu_info *ci, paddr_t target) 1040 { 1041 int error; 1042 1043 /* 1044 * Bootstrap code must be addressable in real mode 1045 * and it must be page aligned. 1046 */ 1047 KASSERT(target < 0x10000 && target % PAGE_SIZE == 0); 1048 1049 /* 1050 * "The BSP must initialize CMOS shutdown code to 0Ah ..." 1051 */ 1052 1053 outb(IO_RTC, NVRAM_RESET); 1054 outb(IO_RTC+1, NVRAM_RESET_JUMP); 1055 1056 #if NLAPIC > 0 1057 /* 1058 * "and the warm reset vector (DWORD based at 40:67) to point 1059 * to the AP startup code ..." 1060 */ 1061 unsigned short dwordptr[2]; 1062 dwordptr[0] = 0; 1063 dwordptr[1] = target >> 4; 1064 1065 memcpy((uint8_t *)cmos_data_mapping + 0x467, dwordptr, 4); 1066 #endif 1067 1068 if ((cpu_feature[0] & CPUID_APIC) == 0) { 1069 aprint_error("mp_cpu_start: CPU does not have APIC\n"); 1070 return ENODEV; 1071 } 1072 1073 /* 1074 * ... prior to executing the following sequence:". We'll also add in 1075 * local cache flush, in case the BIOS has left the AP with its cache 1076 * disabled. It may not be able to cope with MP coherency. 1077 */ 1078 wbinvd(); 1079 1080 if (ci->ci_flags & CPUF_AP) { 1081 error = x86_ipi_init(ci->ci_cpuid); 1082 if (error != 0) { 1083 aprint_error_dev(ci->ci_dev, "%s: IPI not taken (1)\n", 1084 __func__); 1085 return error; 1086 } 1087 i8254_delay(10000); 1088 1089 error = x86_ipi_startup(ci->ci_cpuid, target / PAGE_SIZE); 1090 if (error != 0) { 1091 aprint_error_dev(ci->ci_dev, "%s: IPI not taken (2)\n", 1092 __func__); 1093 return error; 1094 } 1095 i8254_delay(200); 1096 1097 error = x86_ipi_startup(ci->ci_cpuid, target / PAGE_SIZE); 1098 if (error != 0) { 1099 aprint_error_dev(ci->ci_dev, "%s: IPI not taken (3)\n", 1100 __func__); 1101 return error; 1102 } 1103 i8254_delay(200); 1104 } 1105 1106 return 0; 1107 } 1108 1109 void 1110 mp_cpu_start_cleanup(struct cpu_info *ci) 1111 { 1112 /* 1113 * Ensure the NVRAM reset byte contains something vaguely sane. 1114 */ 1115 1116 outb(IO_RTC, NVRAM_RESET); 1117 outb(IO_RTC+1, NVRAM_RESET_RST); 1118 } 1119 #endif 1120 1121 #ifdef __x86_64__ 1122 typedef void (vector)(void); 1123 extern vector Xsyscall, Xsyscall32, Xsyscall_svs; 1124 #endif 1125 1126 void 1127 cpu_init_msrs(struct cpu_info *ci, bool full) 1128 { 1129 #ifdef __x86_64__ 1130 wrmsr(MSR_STAR, 1131 ((uint64_t)GSEL(GCODE_SEL, SEL_KPL) << 32) | 1132 ((uint64_t)LSEL(LSYSRETBASE_SEL, SEL_UPL) << 48)); 1133 wrmsr(MSR_LSTAR, (uint64_t)Xsyscall); 1134 wrmsr(MSR_CSTAR, (uint64_t)Xsyscall32); 1135 wrmsr(MSR_SFMASK, PSL_NT|PSL_T|PSL_I|PSL_C|PSL_D|PSL_AC); 1136 1137 #ifdef SVS 1138 if (svs_enabled) 1139 wrmsr(MSR_LSTAR, (uint64_t)Xsyscall_svs); 1140 #endif 1141 1142 if (full) { 1143 wrmsr(MSR_FSBASE, 0); 1144 wrmsr(MSR_GSBASE, (uint64_t)ci); 1145 wrmsr(MSR_KERNELGSBASE, 0); 1146 } 1147 #endif /* __x86_64__ */ 1148 1149 if (cpu_feature[2] & CPUID_NOX) 1150 wrmsr(MSR_EFER, rdmsr(MSR_EFER) | EFER_NXE); 1151 } 1152 1153 void 1154 cpu_offline_md(void) 1155 { 1156 int s; 1157 1158 s = splhigh(); 1159 fpusave_cpu(true); 1160 splx(s); 1161 } 1162 1163 /* XXX joerg restructure and restart CPUs individually */ 1164 static bool 1165 cpu_stop(device_t dv) 1166 { 1167 struct cpu_softc *sc = device_private(dv); 1168 struct cpu_info *ci = sc->sc_info; 1169 int err; 1170 1171 KASSERT((ci->ci_flags & CPUF_PRESENT) != 0); 1172 1173 if ((ci->ci_flags & CPUF_PRIMARY) != 0) 1174 return true; 1175 1176 if (ci->ci_data.cpu_idlelwp == NULL) 1177 return true; 1178 1179 sc->sc_wasonline = !(ci->ci_schedstate.spc_flags & SPCF_OFFLINE); 1180 1181 if (sc->sc_wasonline) { 1182 mutex_enter(&cpu_lock); 1183 err = cpu_setstate(ci, false); 1184 mutex_exit(&cpu_lock); 1185 1186 if (err != 0) 1187 return false; 1188 } 1189 1190 return true; 1191 } 1192 1193 static bool 1194 cpu_suspend(device_t dv, const pmf_qual_t *qual) 1195 { 1196 struct cpu_softc *sc = device_private(dv); 1197 struct cpu_info *ci = sc->sc_info; 1198 1199 if ((ci->ci_flags & CPUF_PRESENT) == 0) 1200 return true; 1201 else { 1202 cpufreq_suspend(ci); 1203 } 1204 1205 return cpu_stop(dv); 1206 } 1207 1208 static bool 1209 cpu_resume(device_t dv, const pmf_qual_t *qual) 1210 { 1211 struct cpu_softc *sc = device_private(dv); 1212 struct cpu_info *ci = sc->sc_info; 1213 int err = 0; 1214 1215 if ((ci->ci_flags & CPUF_PRESENT) == 0) 1216 return true; 1217 1218 if ((ci->ci_flags & CPUF_PRIMARY) != 0) 1219 goto out; 1220 1221 if (ci->ci_data.cpu_idlelwp == NULL) 1222 goto out; 1223 1224 if (sc->sc_wasonline) { 1225 mutex_enter(&cpu_lock); 1226 err = cpu_setstate(ci, true); 1227 mutex_exit(&cpu_lock); 1228 } 1229 1230 out: 1231 if (err != 0) 1232 return false; 1233 1234 cpufreq_resume(ci); 1235 1236 return true; 1237 } 1238 1239 static bool 1240 cpu_shutdown(device_t dv, int how) 1241 { 1242 struct cpu_softc *sc = device_private(dv); 1243 struct cpu_info *ci = sc->sc_info; 1244 1245 if ((ci->ci_flags & CPUF_BSP) != 0) 1246 return false; 1247 1248 if ((ci->ci_flags & CPUF_PRESENT) == 0) 1249 return true; 1250 1251 return cpu_stop(dv); 1252 } 1253 1254 void 1255 cpu_get_tsc_freq(struct cpu_info *ci) 1256 { 1257 uint64_t last_tsc; 1258 1259 if (cpu_hascounter()) { 1260 last_tsc = cpu_counter_serializing(); 1261 i8254_delay(100000); 1262 ci->ci_data.cpu_cc_freq = 1263 (cpu_counter_serializing() - last_tsc) * 10; 1264 } 1265 } 1266 1267 void 1268 x86_cpu_idle_mwait(void) 1269 { 1270 struct cpu_info *ci = curcpu(); 1271 1272 KASSERT(ci->ci_ilevel == IPL_NONE); 1273 1274 x86_monitor(&ci->ci_want_resched, 0, 0); 1275 if (__predict_false(ci->ci_want_resched)) { 1276 return; 1277 } 1278 x86_mwait(0, 0); 1279 } 1280 1281 void 1282 x86_cpu_idle_halt(void) 1283 { 1284 struct cpu_info *ci = curcpu(); 1285 1286 KASSERT(ci->ci_ilevel == IPL_NONE); 1287 1288 x86_disable_intr(); 1289 if (!__predict_false(ci->ci_want_resched)) { 1290 x86_stihlt(); 1291 } else { 1292 x86_enable_intr(); 1293 } 1294 } 1295 1296 /* 1297 * Loads pmap for the current CPU. 1298 */ 1299 void 1300 cpu_load_pmap(struct pmap *pmap, struct pmap *oldpmap) 1301 { 1302 #ifdef SVS 1303 if (svs_enabled) { 1304 svs_pdir_switch(pmap); 1305 } 1306 #endif 1307 1308 #ifdef PAE 1309 struct cpu_info *ci = curcpu(); 1310 bool interrupts_enabled; 1311 pd_entry_t *l3_pd = ci->ci_pae_l3_pdir; 1312 int i; 1313 1314 /* 1315 * disable interrupts to block TLB shootdowns, which can reload cr3. 1316 * while this doesn't block NMIs, it's probably ok as NMIs unlikely 1317 * reload cr3. 1318 */ 1319 interrupts_enabled = (x86_read_flags() & PSL_I) != 0; 1320 if (interrupts_enabled) 1321 x86_disable_intr(); 1322 1323 for (i = 0 ; i < PDP_SIZE; i++) { 1324 l3_pd[i] = pmap->pm_pdirpa[i] | PG_V; 1325 } 1326 1327 if (interrupts_enabled) 1328 x86_enable_intr(); 1329 tlbflush(); 1330 #else 1331 lcr3(pmap_pdirpa(pmap, 0)); 1332 #endif 1333 } 1334 1335 /* 1336 * Notify all other cpus to halt. 1337 */ 1338 1339 void 1340 cpu_broadcast_halt(void) 1341 { 1342 x86_broadcast_ipi(X86_IPI_HALT); 1343 } 1344 1345 /* 1346 * Send a dummy ipi to a cpu to force it to run splraise()/spllower() 1347 */ 1348 1349 void 1350 cpu_kick(struct cpu_info *ci) 1351 { 1352 x86_send_ipi(ci, 0); 1353 } 1354
Indexes created Wed Sep 24 22:09:59 GMT 2025