cpu.c revision 1.192
1 /* $NetBSD: cpu.c,v 1.192 2020/05/21 21:12:30 ad Exp $ */ 2 3 /* 4 * Copyright (c) 2000-2020 NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Bill Sommerfeld of RedBack Networks Inc, and by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * Copyright (c) 1999 Stefan Grefen 34 * 35 * Redistribution and use in source and binary forms, with or without 36 * modification, are permitted provided that the following conditions 37 * are met: 38 * 1. Redistributions of source code must retain the above copyright 39 * notice, this list of conditions and the following disclaimer. 40 * 2. Redistributions in binary form must reproduce the above copyright 41 * notice, this list of conditions and the following disclaimer in the 42 * documentation and/or other materials provided with the distribution. 43 * 3. All advertising materials mentioning features or use of this software 44 * must display the following acknowledgement: 45 * This product includes software developed by the NetBSD 46 * Foundation, Inc. and its contributors. 47 * 4. Neither the name of The NetBSD Foundation nor the names of its 48 * contributors may be used to endorse or promote products derived 49 * from this software without specific prior written permission. 50 * 51 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND ANY 52 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 53 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 54 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR AND CONTRIBUTORS BE LIABLE 55 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 56 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 57 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 58 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 59 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 60 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 61 * SUCH DAMAGE. 62 */ 63 64 #include <sys/cdefs.h> 65 __KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.192 2020/05/21 21:12:30 ad Exp $"); 66 67 #include "opt_ddb.h" 68 #include "opt_mpbios.h" /* for MPDEBUG */ 69 #include "opt_mtrr.h" 70 #include "opt_multiprocessor.h" 71 #include "opt_svs.h" 72 73 #include "lapic.h" 74 #include "ioapic.h" 75 #include "acpica.h" 76 #include "hpet.h" 77 78 #include <sys/param.h> 79 #include <sys/proc.h> 80 #include <sys/systm.h> 81 #include <sys/device.h> 82 #include <sys/cpu.h> 83 #include <sys/cpufreq.h> 84 #include <sys/idle.h> 85 #include <sys/atomic.h> 86 #include <sys/reboot.h> 87 #include <sys/csan.h> 88 89 #include <uvm/uvm.h> 90 91 #include "acpica.h" /* for NACPICA, for mp_verbose */ 92 93 #include <x86/machdep.h> 94 #include <machine/cpufunc.h> 95 #include <machine/cpuvar.h> 96 #include <machine/pmap.h> 97 #include <machine/vmparam.h> 98 #if defined(MULTIPROCESSOR) 99 #include <machine/mpbiosvar.h> 100 #endif 101 #include <machine/mpconfig.h> /* for mp_verbose */ 102 #include <machine/pcb.h> 103 #include <machine/specialreg.h> 104 #include <machine/segments.h> 105 #include <machine/gdt.h> 106 #include <machine/mtrr.h> 107 #include <machine/pio.h> 108 #include <machine/cpu_counter.h> 109 110 #include <x86/fpu.h> 111 112 #if NACPICA > 0 113 #include <dev/acpi/acpi_srat.h> 114 #endif 115 116 #if NLAPIC > 0 117 #include <machine/apicvar.h> 118 #include <machine/i82489reg.h> 119 #include <machine/i82489var.h> 120 #endif 121 122 #include <dev/ic/mc146818reg.h> 123 #include <dev/ic/hpetvar.h> 124 #include <i386/isa/nvram.h> 125 #include <dev/isa/isareg.h> 126 127 #include "tsc.h" 128 129 #ifndef XENPV 130 #include "hyperv.h" 131 #if NHYPERV > 0 132 #include <x86/x86/hypervvar.h> 133 #endif 134 #endif 135 136 #ifdef XEN 137 #include <xen/hypervisor.h> 138 #endif 139 140 static int cpu_match(device_t, cfdata_t, void *); 141 static void cpu_attach(device_t, device_t, void *); 142 static void cpu_defer(device_t); 143 static int cpu_rescan(device_t, const char *, const int *); 144 static void cpu_childdetached(device_t, device_t); 145 static bool cpu_stop(device_t); 146 static bool cpu_suspend(device_t, const pmf_qual_t *); 147 static bool cpu_resume(device_t, const pmf_qual_t *); 148 static bool cpu_shutdown(device_t, int); 149 150 struct cpu_softc { 151 device_t sc_dev; /* device tree glue */ 152 struct cpu_info *sc_info; /* pointer to CPU info */ 153 bool sc_wasonline; 154 }; 155 156 #ifdef MULTIPROCESSOR 157 int mp_cpu_start(struct cpu_info *, paddr_t); 158 void mp_cpu_start_cleanup(struct cpu_info *); 159 const struct cpu_functions mp_cpu_funcs = { mp_cpu_start, NULL, 160 mp_cpu_start_cleanup }; 161 #endif 162 163 164 CFATTACH_DECL2_NEW(cpu, sizeof(struct cpu_softc), 165 cpu_match, cpu_attach, NULL, NULL, cpu_rescan, cpu_childdetached); 166 167 /* 168 * Statically-allocated CPU info for the primary CPU (or the only 169 * CPU, on uniprocessors). The CPU info list is initialized to 170 * point at it. 171 */ 172 struct cpu_info cpu_info_primary __aligned(CACHE_LINE_SIZE) = { 173 .ci_dev = 0, 174 .ci_self = &cpu_info_primary, 175 .ci_idepth = -1, 176 .ci_curlwp = &lwp0, 177 .ci_curldt = -1, 178 }; 179 180 struct cpu_info *cpu_info_list = &cpu_info_primary; 181 182 #ifdef i386 183 void cpu_set_tss_gates(struct cpu_info *); 184 #endif 185 186 static void cpu_init_idle_lwp(struct cpu_info *); 187 188 uint32_t cpu_feature[7] __read_mostly; /* X86 CPUID feature bits */ 189 /* [0] basic features cpuid.1:%edx 190 * [1] basic features cpuid.1:%ecx (CPUID2_xxx bits) 191 * [2] extended features cpuid:80000001:%edx 192 * [3] extended features cpuid:80000001:%ecx 193 * [4] VIA padlock features 194 * [5] structured extended features cpuid.7:%ebx 195 * [6] structured extended features cpuid.7:%ecx 196 */ 197 198 #ifdef MULTIPROCESSOR 199 bool x86_mp_online; 200 paddr_t mp_trampoline_paddr = MP_TRAMPOLINE; 201 #endif 202 #if NLAPIC > 0 203 static vaddr_t cmos_data_mapping; 204 #endif 205 struct cpu_info *cpu_starting; 206 207 #ifdef MULTIPROCESSOR 208 void cpu_hatch(void *); 209 static void cpu_boot_secondary(struct cpu_info *ci); 210 static void cpu_start_secondary(struct cpu_info *ci); 211 #if NLAPIC > 0 212 static void cpu_copy_trampoline(paddr_t); 213 #endif 214 #endif /* MULTIPROCESSOR */ 215 216 /* 217 * Runs once per boot once multiprocessor goo has been detected and 218 * the local APIC on the boot processor has been mapped. 219 * 220 * Called from lapic_boot_init() (from mpbios_scan()). 221 */ 222 #if NLAPIC > 0 223 void 224 cpu_init_first(void) 225 { 226 227 cpu_info_primary.ci_cpuid = lapic_cpu_number(); 228 229 cmos_data_mapping = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, UVM_KMF_VAONLY); 230 if (cmos_data_mapping == 0) 231 panic("No KVA for page 0"); 232 pmap_kenter_pa(cmos_data_mapping, 0, VM_PROT_READ|VM_PROT_WRITE, 0); 233 pmap_update(pmap_kernel()); 234 } 235 #endif 236 237 static int 238 cpu_match(device_t parent, cfdata_t match, void *aux) 239 { 240 241 return 1; 242 } 243 244 #ifdef __HAVE_PCPU_AREA 245 void 246 cpu_pcpuarea_init(struct cpu_info *ci) 247 { 248 struct vm_page *pg; 249 size_t i, npages; 250 vaddr_t base, va; 251 paddr_t pa; 252 253 CTASSERT(sizeof(struct pcpu_entry) % PAGE_SIZE == 0); 254 255 npages = sizeof(struct pcpu_entry) / PAGE_SIZE; 256 base = (vaddr_t)&pcpuarea->ent[cpu_index(ci)]; 257 258 for (i = 0; i < npages; i++) { 259 pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO); 260 if (pg == NULL) { 261 panic("failed to allocate pcpu PA"); 262 } 263 264 va = base + i * PAGE_SIZE; 265 pa = VM_PAGE_TO_PHYS(pg); 266 267 pmap_kenter_pa(va, pa, VM_PROT_READ|VM_PROT_WRITE, 0); 268 } 269 270 pmap_update(pmap_kernel()); 271 } 272 #endif 273 274 static void 275 cpu_vm_init(struct cpu_info *ci) 276 { 277 int ncolors = 2, i; 278 279 for (i = CAI_ICACHE; i <= CAI_L2CACHE; i++) { 280 struct x86_cache_info *cai; 281 int tcolors; 282 283 cai = &ci->ci_cinfo[i]; 284 285 tcolors = atop(cai->cai_totalsize); 286 switch (cai->cai_associativity) { 287 case 0xff: 288 tcolors = 1; /* fully associative */ 289 break; 290 case 0: 291 case 1: 292 break; 293 default: 294 tcolors /= cai->cai_associativity; 295 } 296 ncolors = uimax(ncolors, tcolors); 297 /* 298 * If the desired number of colors is not a power of 299 * two, it won't be good. Find the greatest power of 300 * two which is an even divisor of the number of colors, 301 * to preserve even coloring of pages. 302 */ 303 if (ncolors & (ncolors - 1) ) { 304 int try, picked = 1; 305 for (try = 1; try < ncolors; try *= 2) { 306 if (ncolors % try == 0) picked = try; 307 } 308 if (picked == 1) { 309 panic("desired number of cache colors %d is " 310 " > 1, but not even!", ncolors); 311 } 312 ncolors = picked; 313 } 314 } 315 316 /* 317 * Knowing the size of the largest cache on this CPU, potentially 318 * re-color our pages. 319 */ 320 aprint_debug_dev(ci->ci_dev, "%d page colors\n", ncolors); 321 uvm_page_recolor(ncolors); 322 323 pmap_tlb_cpu_init(ci); 324 #ifndef __HAVE_DIRECT_MAP 325 pmap_vpage_cpu_init(ci); 326 #endif 327 } 328 329 static void 330 cpu_attach(device_t parent, device_t self, void *aux) 331 { 332 struct cpu_softc *sc = device_private(self); 333 struct cpu_attach_args *caa = aux; 334 struct cpu_info *ci; 335 uintptr_t ptr; 336 #if NLAPIC > 0 337 int cpunum = caa->cpu_number; 338 #endif 339 static bool again; 340 341 sc->sc_dev = self; 342 343 if (ncpu > maxcpus) { 344 #ifndef _LP64 345 aprint_error(": too many CPUs, please use NetBSD/amd64\n"); 346 #else 347 aprint_error(": too many CPUs\n"); 348 #endif 349 return; 350 } 351 352 /* 353 * If we're an Application Processor, allocate a cpu_info 354 * structure, otherwise use the primary's. 355 */ 356 if (caa->cpu_role == CPU_ROLE_AP) { 357 if ((boothowto & RB_MD1) != 0) { 358 aprint_error(": multiprocessor boot disabled\n"); 359 if (!pmf_device_register(self, NULL, NULL)) 360 aprint_error_dev(self, 361 "couldn't establish power handler\n"); 362 return; 363 } 364 aprint_naive(": Application Processor\n"); 365 ptr = (uintptr_t)uvm_km_alloc(kernel_map, 366 sizeof(*ci) + CACHE_LINE_SIZE - 1, 0, 367 UVM_KMF_WIRED|UVM_KMF_ZERO); 368 ci = (struct cpu_info *)roundup2(ptr, CACHE_LINE_SIZE); 369 ci->ci_curldt = -1; 370 } else { 371 aprint_naive(": %s Processor\n", 372 caa->cpu_role == CPU_ROLE_SP ? "Single" : "Boot"); 373 ci = &cpu_info_primary; 374 #if NLAPIC > 0 375 if (cpunum != lapic_cpu_number()) { 376 /* XXX should be done earlier. */ 377 uint32_t reg; 378 aprint_verbose("\n"); 379 aprint_verbose_dev(self, "running CPU at apic %d" 380 " instead of at expected %d", lapic_cpu_number(), 381 cpunum); 382 reg = lapic_readreg(LAPIC_ID); 383 lapic_writereg(LAPIC_ID, (reg & ~LAPIC_ID_MASK) | 384 (cpunum << LAPIC_ID_SHIFT)); 385 } 386 if (cpunum != lapic_cpu_number()) { 387 aprint_error_dev(self, "unable to reset apic id\n"); 388 } 389 #endif 390 } 391 392 ci->ci_self = ci; 393 sc->sc_info = ci; 394 ci->ci_dev = self; 395 ci->ci_acpiid = caa->cpu_id; 396 ci->ci_cpuid = caa->cpu_number; 397 ci->ci_func = caa->cpu_func; 398 ci->ci_kfpu_spl = -1; 399 aprint_normal("\n"); 400 401 /* Must be before mi_cpu_attach(). */ 402 cpu_vm_init(ci); 403 404 if (caa->cpu_role == CPU_ROLE_AP) { 405 int error; 406 407 error = mi_cpu_attach(ci); 408 if (error != 0) { 409 aprint_error_dev(self, 410 "mi_cpu_attach failed with %d\n", error); 411 return; 412 } 413 #ifdef __HAVE_PCPU_AREA 414 cpu_pcpuarea_init(ci); 415 #endif 416 cpu_init_tss(ci); 417 } else { 418 KASSERT(ci->ci_data.cpu_idlelwp != NULL); 419 #if NACPICA > 0 420 /* Parse out NUMA info for cpu_identify(). */ 421 acpisrat_init(); 422 #endif 423 } 424 425 #ifdef SVS 426 cpu_svs_init(ci); 427 #endif 428 429 pmap_reference(pmap_kernel()); 430 ci->ci_pmap = pmap_kernel(); 431 ci->ci_tlbstate = TLBSTATE_STALE; 432 433 /* 434 * Boot processor may not be attached first, but the below 435 * must be done to allow booting other processors. 436 */ 437 if (!again) { 438 /* Make sure DELAY() (likely i8254_delay()) is initialized. */ 439 DELAY(1); 440 441 /* 442 * Basic init. Compute an approximate frequency for the TSC 443 * using the i8254. If there's a HPET we'll redo it later. 444 */ 445 atomic_or_32(&ci->ci_flags, CPUF_PRESENT | CPUF_PRIMARY); 446 cpu_intr_init(ci); 447 cpu_get_tsc_freq(ci); 448 cpu_init(ci); 449 #ifdef i386 450 cpu_set_tss_gates(ci); 451 #endif 452 pmap_cpu_init_late(ci); 453 #if NLAPIC > 0 454 if (caa->cpu_role != CPU_ROLE_SP) { 455 /* Enable lapic. */ 456 lapic_enable(); 457 lapic_set_lvt(); 458 if (!vm_guest_is_xenpvh_or_pvhvm()) 459 lapic_calibrate_timer(false); 460 } 461 #endif 462 kcsan_cpu_init(ci); 463 again = true; 464 } 465 466 /* further PCB init done later. */ 467 468 switch (caa->cpu_role) { 469 case CPU_ROLE_SP: 470 atomic_or_32(&ci->ci_flags, CPUF_SP); 471 cpu_identify(ci); 472 x86_errata(); 473 x86_cpu_idle_init(); 474 #ifdef XENPVHVM 475 xen_hvm_init_cpu(ci); 476 #endif 477 break; 478 479 case CPU_ROLE_BP: 480 atomic_or_32(&ci->ci_flags, CPUF_BSP); 481 cpu_identify(ci); 482 x86_errata(); 483 x86_cpu_idle_init(); 484 #ifdef XENPVHVM 485 xen_hvm_init_cpu(ci); 486 #endif 487 break; 488 489 #ifdef MULTIPROCESSOR 490 case CPU_ROLE_AP: 491 /* 492 * report on an AP 493 */ 494 cpu_intr_init(ci); 495 gdt_alloc_cpu(ci); 496 #ifdef i386 497 cpu_set_tss_gates(ci); 498 #endif 499 pmap_cpu_init_late(ci); 500 cpu_start_secondary(ci); 501 if (ci->ci_flags & CPUF_PRESENT) { 502 struct cpu_info *tmp; 503 504 cpu_identify(ci); 505 tmp = cpu_info_list; 506 while (tmp->ci_next) 507 tmp = tmp->ci_next; 508 509 tmp->ci_next = ci; 510 } 511 break; 512 #endif 513 514 default: 515 panic("unknown processor type??\n"); 516 } 517 518 pat_init(ci); 519 520 if (!pmf_device_register1(self, cpu_suspend, cpu_resume, cpu_shutdown)) 521 aprint_error_dev(self, "couldn't establish power handler\n"); 522 523 #ifdef MULTIPROCESSOR 524 if (mp_verbose) { 525 struct lwp *l = ci->ci_data.cpu_idlelwp; 526 struct pcb *pcb = lwp_getpcb(l); 527 528 aprint_verbose_dev(self, 529 "idle lwp at %p, idle sp at %p\n", 530 l, 531 #ifdef i386 532 (void *)pcb->pcb_esp 533 #else 534 (void *)pcb->pcb_rsp 535 #endif 536 ); 537 } 538 #endif 539 540 /* 541 * Postpone the "cpufeaturebus" scan. 542 * It is safe to scan the pseudo-bus 543 * only after all CPUs have attached. 544 */ 545 (void)config_defer(self, cpu_defer); 546 } 547 548 static void 549 cpu_defer(device_t self) 550 { 551 cpu_rescan(self, NULL, NULL); 552 } 553 554 static int 555 cpu_rescan(device_t self, const char *ifattr, const int *locators) 556 { 557 struct cpu_softc *sc = device_private(self); 558 struct cpufeature_attach_args cfaa; 559 struct cpu_info *ci = sc->sc_info; 560 561 /* 562 * If we booted with RB_MD1 to disable multiprocessor, the 563 * auto-configuration data still contains the additional 564 * CPUs. But their initialization was mostly bypassed 565 * during attach, so we have to make sure we don't look at 566 * their featurebus info, since it wasn't retrieved. 567 */ 568 if (ci == NULL) 569 return 0; 570 571 memset(&cfaa, 0, sizeof(cfaa)); 572 cfaa.ci = ci; 573 574 if (ifattr_match(ifattr, "cpufeaturebus")) { 575 if (ci->ci_frequency == NULL) { 576 cfaa.name = "frequency"; 577 ci->ci_frequency = config_found_ia(self, 578 "cpufeaturebus", &cfaa, NULL); 579 } 580 581 if (ci->ci_padlock == NULL) { 582 cfaa.name = "padlock"; 583 ci->ci_padlock = config_found_ia(self, 584 "cpufeaturebus", &cfaa, NULL); 585 } 586 587 if (ci->ci_temperature == NULL) { 588 cfaa.name = "temperature"; 589 ci->ci_temperature = config_found_ia(self, 590 "cpufeaturebus", &cfaa, NULL); 591 } 592 593 if (ci->ci_vm == NULL) { 594 cfaa.name = "vm"; 595 ci->ci_vm = config_found_ia(self, 596 "cpufeaturebus", &cfaa, NULL); 597 } 598 } 599 600 return 0; 601 } 602 603 static void 604 cpu_childdetached(device_t self, device_t child) 605 { 606 struct cpu_softc *sc = device_private(self); 607 struct cpu_info *ci = sc->sc_info; 608 609 if (ci->ci_frequency == child) 610 ci->ci_frequency = NULL; 611 612 if (ci->ci_padlock == child) 613 ci->ci_padlock = NULL; 614 615 if (ci->ci_temperature == child) 616 ci->ci_temperature = NULL; 617 618 if (ci->ci_vm == child) 619 ci->ci_vm = NULL; 620 } 621 622 /* 623 * Initialize the processor appropriately. 624 */ 625 626 void 627 cpu_init(struct cpu_info *ci) 628 { 629 extern int x86_fpu_save; 630 uint32_t cr4 = 0; 631 632 lcr0(rcr0() | CR0_WP); 633 634 /* If global TLB caching is supported, enable it */ 635 if (cpu_feature[0] & CPUID_PGE) 636 cr4 |= CR4_PGE; 637 638 /* 639 * If we have FXSAVE/FXRESTOR, use them. 640 */ 641 if (cpu_feature[0] & CPUID_FXSR) { 642 cr4 |= CR4_OSFXSR; 643 644 /* 645 * If we have SSE/SSE2, enable XMM exceptions. 646 */ 647 if (cpu_feature[0] & (CPUID_SSE|CPUID_SSE2)) 648 cr4 |= CR4_OSXMMEXCPT; 649 } 650 651 /* If xsave is supported, enable it */ 652 if (cpu_feature[1] & CPUID2_XSAVE) 653 cr4 |= CR4_OSXSAVE; 654 655 /* If SMEP is supported, enable it */ 656 if (cpu_feature[5] & CPUID_SEF_SMEP) 657 cr4 |= CR4_SMEP; 658 659 /* If SMAP is supported, enable it */ 660 if (cpu_feature[5] & CPUID_SEF_SMAP) 661 cr4 |= CR4_SMAP; 662 663 #ifdef SVS 664 /* If PCID is supported, enable it */ 665 if (svs_pcid) 666 cr4 |= CR4_PCIDE; 667 #endif 668 669 if (cr4) { 670 cr4 |= rcr4(); 671 lcr4(cr4); 672 } 673 674 /* 675 * Changing CR4 register may change cpuid values. For example, setting 676 * CR4_OSXSAVE sets CPUID2_OSXSAVE. The CPUID2_OSXSAVE is in 677 * ci_feat_val[1], so update it. 678 * XXX Other than ci_feat_val[1] might be changed. 679 */ 680 if (cpuid_level >= 1) { 681 u_int descs[4]; 682 683 x86_cpuid(1, descs); 684 ci->ci_feat_val[1] = descs[2]; 685 } 686 687 if (x86_fpu_save >= FPU_SAVE_FXSAVE) { 688 fpuinit_mxcsr_mask(); 689 } 690 691 /* If xsave is enabled, enable all fpu features */ 692 if (cr4 & CR4_OSXSAVE) 693 wrxcr(0, x86_xsave_features & XCR0_FPU); 694 695 #ifdef MTRR 696 /* 697 * On a P6 or above, initialize MTRR's if the hardware supports them. 698 */ 699 if (cpu_feature[0] & CPUID_MTRR) { 700 if ((ci->ci_flags & CPUF_AP) == 0) 701 i686_mtrr_init_first(); 702 mtrr_init_cpu(ci); 703 } 704 705 #ifdef i386 706 if (strcmp((char *)(ci->ci_vendor), "AuthenticAMD") == 0) { 707 /* 708 * Must be a K6-2 Step >= 7 or a K6-III. 709 */ 710 if (CPUID_TO_FAMILY(ci->ci_signature) == 5) { 711 if (CPUID_TO_MODEL(ci->ci_signature) > 8 || 712 (CPUID_TO_MODEL(ci->ci_signature) == 8 && 713 CPUID_TO_STEPPING(ci->ci_signature) >= 7)) { 714 mtrr_funcs = &k6_mtrr_funcs; 715 k6_mtrr_init_first(); 716 mtrr_init_cpu(ci); 717 } 718 } 719 } 720 #endif /* i386 */ 721 #endif /* MTRR */ 722 723 if (ci != &cpu_info_primary) { 724 /* Synchronize TSC */ 725 atomic_or_32(&ci->ci_flags, CPUF_RUNNING); 726 tsc_sync_ap(ci); 727 } else { 728 atomic_or_32(&ci->ci_flags, CPUF_RUNNING); 729 } 730 } 731 732 #ifdef MULTIPROCESSOR 733 void 734 cpu_boot_secondary_processors(void) 735 { 736 struct cpu_info *ci; 737 kcpuset_t *cpus; 738 u_long i; 739 740 /* Now that we know the number of CPUs, patch the text segment. */ 741 x86_patch(false); 742 743 #if NACPICA > 0 744 /* Finished with NUMA info for now. */ 745 acpisrat_exit(); 746 #endif 747 748 kcpuset_create(&cpus, true); 749 kcpuset_set(cpus, cpu_index(curcpu())); 750 for (i = 0; i < maxcpus; i++) { 751 ci = cpu_lookup(i); 752 if (ci == NULL) 753 continue; 754 if (ci->ci_data.cpu_idlelwp == NULL) 755 continue; 756 if ((ci->ci_flags & CPUF_PRESENT) == 0) 757 continue; 758 if (ci->ci_flags & (CPUF_BSP|CPUF_SP|CPUF_PRIMARY)) 759 continue; 760 cpu_boot_secondary(ci); 761 kcpuset_set(cpus, cpu_index(ci)); 762 } 763 while (!kcpuset_match(cpus, kcpuset_running)) 764 ; 765 kcpuset_destroy(cpus); 766 767 x86_mp_online = true; 768 769 /* Now that we know about the TSC, attach the timecounter. */ 770 tsc_tc_init(); 771 772 /* Enable zeroing of pages in the idle loop if we have SSE2. */ 773 vm_page_zero_enable = false; /* ((cpu_feature[0] & CPUID_SSE2) != 0); */ 774 } 775 #endif 776 777 static void 778 cpu_init_idle_lwp(struct cpu_info *ci) 779 { 780 struct lwp *l = ci->ci_data.cpu_idlelwp; 781 struct pcb *pcb = lwp_getpcb(l); 782 783 pcb->pcb_cr0 = rcr0(); 784 } 785 786 void 787 cpu_init_idle_lwps(void) 788 { 789 struct cpu_info *ci; 790 u_long i; 791 792 for (i = 0; i < maxcpus; i++) { 793 ci = cpu_lookup(i); 794 if (ci == NULL) 795 continue; 796 if (ci->ci_data.cpu_idlelwp == NULL) 797 continue; 798 if ((ci->ci_flags & CPUF_PRESENT) == 0) 799 continue; 800 cpu_init_idle_lwp(ci); 801 } 802 } 803 804 #ifdef MULTIPROCESSOR 805 void 806 cpu_start_secondary(struct cpu_info *ci) 807 { 808 u_long psl; 809 int i; 810 811 #if NLAPIC > 0 812 paddr_t mp_pdirpa; 813 mp_pdirpa = pmap_init_tmp_pgtbl(mp_trampoline_paddr); 814 cpu_copy_trampoline(mp_pdirpa); 815 #endif 816 817 atomic_or_32(&ci->ci_flags, CPUF_AP); 818 ci->ci_curlwp = ci->ci_data.cpu_idlelwp; 819 if (CPU_STARTUP(ci, mp_trampoline_paddr) != 0) { 820 return; 821 } 822 823 /* 824 * Wait for it to become ready. Setting cpu_starting opens the 825 * initial gate and allows the AP to start soft initialization. 826 */ 827 KASSERT(cpu_starting == NULL); 828 cpu_starting = ci; 829 for (i = 100000; (!(ci->ci_flags & CPUF_PRESENT)) && i > 0; i--) { 830 delay_func(10); 831 } 832 833 if ((ci->ci_flags & CPUF_PRESENT) == 0) { 834 aprint_error_dev(ci->ci_dev, "failed to become ready\n"); 835 #if defined(MPDEBUG) && defined(DDB) 836 printf("dropping into debugger; continue from here to resume boot\n"); 837 Debugger(); 838 #endif 839 } else { 840 /* 841 * Synchronize time stamp counters. Invalidate cache and do 842 * twice (in tsc_sync_bp) to minimize possible cache effects. 843 * Disable interrupts to try and rule out any external 844 * interference. 845 */ 846 psl = x86_read_psl(); 847 x86_disable_intr(); 848 tsc_sync_bp(ci); 849 x86_write_psl(psl); 850 } 851 852 CPU_START_CLEANUP(ci); 853 cpu_starting = NULL; 854 } 855 856 void 857 cpu_boot_secondary(struct cpu_info *ci) 858 { 859 int64_t drift; 860 u_long psl; 861 int i; 862 863 atomic_or_32(&ci->ci_flags, CPUF_GO); 864 for (i = 100000; (!(ci->ci_flags & CPUF_RUNNING)) && i > 0; i--) { 865 delay_func(10); 866 } 867 if ((ci->ci_flags & CPUF_RUNNING) == 0) { 868 aprint_error_dev(ci->ci_dev, "failed to start\n"); 869 #if defined(MPDEBUG) && defined(DDB) 870 printf("dropping into debugger; continue from here to resume boot\n"); 871 Debugger(); 872 #endif 873 } else { 874 /* Synchronize TSC again, check for drift. */ 875 drift = ci->ci_data.cpu_cc_skew; 876 psl = x86_read_psl(); 877 x86_disable_intr(); 878 tsc_sync_bp(ci); 879 x86_write_psl(psl); 880 drift -= ci->ci_data.cpu_cc_skew; 881 aprint_debug_dev(ci->ci_dev, "TSC skew=%lld drift=%lld\n", 882 (long long)ci->ci_data.cpu_cc_skew, (long long)drift); 883 tsc_sync_drift(drift); 884 } 885 } 886 887 /* 888 * The CPU ends up here when it's ready to run. 889 * This is called from code in mptramp.s; at this point, we are running 890 * in the idle pcb/idle stack of the new CPU. When this function returns, 891 * this processor will enter the idle loop and start looking for work. 892 */ 893 void 894 cpu_hatch(void *v) 895 { 896 struct cpu_info *ci = (struct cpu_info *)v; 897 struct pcb *pcb; 898 int s, i; 899 900 /* ------------------------------------------------------------- */ 901 902 /* 903 * This section of code must be compiled with SSP disabled, to 904 * prevent a race against cpu0. See sys/conf/ssp.mk. 905 */ 906 907 cpu_init_msrs(ci, true); 908 cpu_probe(ci); 909 cpu_speculation_init(ci); 910 #if NHYPERV > 0 911 hyperv_init_cpu(ci); 912 #endif 913 914 ci->ci_data.cpu_cc_freq = cpu_info_primary.ci_data.cpu_cc_freq; 915 /* cpu_get_tsc_freq(ci); */ 916 917 KDASSERT((ci->ci_flags & CPUF_PRESENT) == 0); 918 919 /* 920 * Synchronize the TSC for the first time. Note that interrupts are 921 * off at this point. 922 */ 923 atomic_or_32(&ci->ci_flags, CPUF_PRESENT); 924 tsc_sync_ap(ci); 925 926 /* ------------------------------------------------------------- */ 927 928 /* 929 * Wait to be brought online. 930 * 931 * Use MONITOR/MWAIT if available. These instructions put the CPU in 932 * a low consumption mode (C-state), and if the TSC is not invariant, 933 * this causes the TSC to drift. We want this to happen, so that we 934 * can later detect (in tsc_tc_init) any abnormal drift with invariant 935 * TSCs. That's just for safety; by definition such drifts should 936 * never occur with invariant TSCs. 937 * 938 * If not available, try PAUSE. We'd like to use HLT, but we have 939 * interrupts off. 940 */ 941 while ((ci->ci_flags & CPUF_GO) == 0) { 942 if ((cpu_feature[1] & CPUID2_MONITOR) != 0) { 943 x86_monitor(&ci->ci_flags, 0, 0); 944 if ((ci->ci_flags & CPUF_GO) != 0) { 945 continue; 946 } 947 x86_mwait(0, 0); 948 } else { 949 /* 950 * XXX The loop repetition count could be a lot higher, but 951 * XXX currently qemu emulator takes a _very_long_time_ to 952 * XXX execute the pause instruction. So for now, use a low 953 * XXX value to allow the cpu to hatch before timing out. 954 */ 955 for (i = 50; i != 0; i--) { 956 x86_pause(); 957 } 958 } 959 } 960 961 /* Because the text may have been patched in x86_patch(). */ 962 wbinvd(); 963 x86_flush(); 964 tlbflushg(); 965 966 KASSERT((ci->ci_flags & CPUF_RUNNING) == 0); 967 968 #ifdef PAE 969 pd_entry_t * l3_pd = ci->ci_pae_l3_pdir; 970 for (i = 0 ; i < PDP_SIZE; i++) { 971 l3_pd[i] = pmap_kernel()->pm_pdirpa[i] | PTE_P; 972 } 973 lcr3(ci->ci_pae_l3_pdirpa); 974 #else 975 lcr3(pmap_pdirpa(pmap_kernel(), 0)); 976 #endif 977 978 pcb = lwp_getpcb(curlwp); 979 pcb->pcb_cr3 = rcr3(); 980 pcb = lwp_getpcb(ci->ci_data.cpu_idlelwp); 981 lcr0(pcb->pcb_cr0); 982 983 cpu_init_idt(); 984 gdt_init_cpu(ci); 985 #if NLAPIC > 0 986 lapic_enable(); 987 lapic_set_lvt(); 988 #endif 989 990 fpuinit(ci); 991 lldt(GSYSSEL(GLDT_SEL, SEL_KPL)); 992 ltr(ci->ci_tss_sel); 993 994 /* 995 * cpu_init will re-synchronize the TSC, and will detect any abnormal 996 * drift that would have been caused by the use of MONITOR/MWAIT 997 * above. 998 */ 999 cpu_init(ci); 1000 #ifdef XENPVHVM 1001 xen_hvm_init_cpu(ci); 1002 #endif 1003 (*x86_initclock_func)(); 1004 cpu_get_tsc_freq(ci); 1005 1006 s = splhigh(); 1007 #if NLAPIC > 0 1008 lapic_write_tpri(0); 1009 #endif 1010 x86_enable_intr(); 1011 splx(s); 1012 x86_errata(); 1013 1014 aprint_debug_dev(ci->ci_dev, "running\n"); 1015 1016 kcsan_cpu_init(ci); 1017 1018 idle_loop(NULL); 1019 KASSERT(false); 1020 } 1021 #endif 1022 1023 #if defined(DDB) 1024 1025 #include <ddb/db_output.h> 1026 #include <machine/db_machdep.h> 1027 1028 /* 1029 * Dump CPU information from ddb. 1030 */ 1031 void 1032 cpu_debug_dump(void) 1033 { 1034 struct cpu_info *ci; 1035 CPU_INFO_ITERATOR cii; 1036 const char sixtyfour64space[] = 1037 #ifdef _LP64 1038 " " 1039 #endif 1040 ""; 1041 1042 db_printf("addr %sdev id flags ipis spl curlwp " 1043 "\n", sixtyfour64space); 1044 for (CPU_INFO_FOREACH(cii, ci)) { 1045 db_printf("%p %s %ld %x %x %d %10p\n", 1046 ci, 1047 ci->ci_dev == NULL ? "BOOT" : device_xname(ci->ci_dev), 1048 (long)ci->ci_cpuid, 1049 ci->ci_flags, ci->ci_ipis, ci->ci_ilevel, 1050 ci->ci_curlwp); 1051 } 1052 } 1053 #endif 1054 1055 #ifdef MULTIPROCESSOR 1056 #if NLAPIC > 0 1057 static void 1058 cpu_copy_trampoline(paddr_t pdir_pa) 1059 { 1060 extern uint32_t nox_flag; 1061 extern u_char cpu_spinup_trampoline[]; 1062 extern u_char cpu_spinup_trampoline_end[]; 1063 vaddr_t mp_trampoline_vaddr; 1064 struct { 1065 uint32_t large; 1066 uint32_t nox; 1067 uint32_t pdir; 1068 } smp_data; 1069 CTASSERT(sizeof(smp_data) == 3 * 4); 1070 1071 smp_data.large = (pmap_largepages != 0); 1072 smp_data.nox = nox_flag; 1073 smp_data.pdir = (uint32_t)(pdir_pa & 0xFFFFFFFF); 1074 1075 /* Enter the physical address */ 1076 mp_trampoline_vaddr = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, 1077 UVM_KMF_VAONLY); 1078 pmap_kenter_pa(mp_trampoline_vaddr, mp_trampoline_paddr, 1079 VM_PROT_READ | VM_PROT_WRITE, 0); 1080 pmap_update(pmap_kernel()); 1081 1082 /* Copy boot code */ 1083 memcpy((void *)mp_trampoline_vaddr, 1084 cpu_spinup_trampoline, 1085 cpu_spinup_trampoline_end - cpu_spinup_trampoline); 1086 1087 /* Copy smp_data at the end */ 1088 memcpy((void *)(mp_trampoline_vaddr + PAGE_SIZE - sizeof(smp_data)), 1089 &smp_data, sizeof(smp_data)); 1090 1091 pmap_kremove(mp_trampoline_vaddr, PAGE_SIZE); 1092 pmap_update(pmap_kernel()); 1093 uvm_km_free(kernel_map, mp_trampoline_vaddr, PAGE_SIZE, UVM_KMF_VAONLY); 1094 } 1095 #endif 1096 1097 int 1098 mp_cpu_start(struct cpu_info *ci, paddr_t target) 1099 { 1100 int error; 1101 1102 /* 1103 * Bootstrap code must be addressable in real mode 1104 * and it must be page aligned. 1105 */ 1106 KASSERT(target < 0x10000 && target % PAGE_SIZE == 0); 1107 1108 /* 1109 * "The BSP must initialize CMOS shutdown code to 0Ah ..." 1110 */ 1111 1112 outb(IO_RTC, NVRAM_RESET); 1113 outb(IO_RTC+1, NVRAM_RESET_JUMP); 1114 1115 #if NLAPIC > 0 1116 /* 1117 * "and the warm reset vector (DWORD based at 40:67) to point 1118 * to the AP startup code ..." 1119 */ 1120 unsigned short dwordptr[2]; 1121 dwordptr[0] = 0; 1122 dwordptr[1] = target >> 4; 1123 1124 memcpy((uint8_t *)cmos_data_mapping + 0x467, dwordptr, 4); 1125 #endif 1126 1127 if ((cpu_feature[0] & CPUID_APIC) == 0) { 1128 aprint_error("mp_cpu_start: CPU does not have APIC\n"); 1129 return ENODEV; 1130 } 1131 1132 /* 1133 * ... prior to executing the following sequence:". We'll also add in 1134 * local cache flush, in case the BIOS has left the AP with its cache 1135 * disabled. It may not be able to cope with MP coherency. 1136 */ 1137 wbinvd(); 1138 1139 if (ci->ci_flags & CPUF_AP) { 1140 error = x86_ipi_init(ci->ci_cpuid); 1141 if (error != 0) { 1142 aprint_error_dev(ci->ci_dev, "%s: IPI not taken (1)\n", 1143 __func__); 1144 return error; 1145 } 1146 delay_func(10000); 1147 1148 error = x86_ipi_startup(ci->ci_cpuid, target / PAGE_SIZE); 1149 if (error != 0) { 1150 aprint_error_dev(ci->ci_dev, "%s: IPI not taken (2)\n", 1151 __func__); 1152 return error; 1153 } 1154 delay_func(200); 1155 1156 error = x86_ipi_startup(ci->ci_cpuid, target / PAGE_SIZE); 1157 if (error != 0) { 1158 aprint_error_dev(ci->ci_dev, "%s: IPI not taken (3)\n", 1159 __func__); 1160 return error; 1161 } 1162 delay_func(200); 1163 } 1164 1165 return 0; 1166 } 1167 1168 void 1169 mp_cpu_start_cleanup(struct cpu_info *ci) 1170 { 1171 /* 1172 * Ensure the NVRAM reset byte contains something vaguely sane. 1173 */ 1174 1175 outb(IO_RTC, NVRAM_RESET); 1176 outb(IO_RTC+1, NVRAM_RESET_RST); 1177 } 1178 #endif 1179 1180 #ifdef __x86_64__ 1181 typedef void (vector)(void); 1182 extern vector Xsyscall, Xsyscall32, Xsyscall_svs; 1183 #endif 1184 1185 void 1186 cpu_init_msrs(struct cpu_info *ci, bool full) 1187 { 1188 #ifdef __x86_64__ 1189 wrmsr(MSR_STAR, 1190 ((uint64_t)GSEL(GCODE_SEL, SEL_KPL) << 32) | 1191 ((uint64_t)LSEL(LSYSRETBASE_SEL, SEL_UPL) << 48)); 1192 wrmsr(MSR_LSTAR, (uint64_t)Xsyscall); 1193 wrmsr(MSR_CSTAR, (uint64_t)Xsyscall32); 1194 wrmsr(MSR_SFMASK, PSL_NT|PSL_T|PSL_I|PSL_C|PSL_D|PSL_AC); 1195 1196 #ifdef SVS 1197 if (svs_enabled) 1198 wrmsr(MSR_LSTAR, (uint64_t)Xsyscall_svs); 1199 #endif 1200 1201 if (full) { 1202 wrmsr(MSR_FSBASE, 0); 1203 wrmsr(MSR_GSBASE, (uint64_t)ci); 1204 wrmsr(MSR_KERNELGSBASE, 0); 1205 } 1206 #endif /* __x86_64__ */ 1207 1208 if (cpu_feature[2] & CPUID_NOX) 1209 wrmsr(MSR_EFER, rdmsr(MSR_EFER) | EFER_NXE); 1210 } 1211 1212 void 1213 cpu_offline_md(void) 1214 { 1215 return; 1216 } 1217 1218 /* XXX joerg restructure and restart CPUs individually */ 1219 static bool 1220 cpu_stop(device_t dv) 1221 { 1222 struct cpu_softc *sc = device_private(dv); 1223 struct cpu_info *ci = sc->sc_info; 1224 int err; 1225 1226 KASSERT((ci->ci_flags & CPUF_PRESENT) != 0); 1227 1228 if ((ci->ci_flags & CPUF_PRIMARY) != 0) 1229 return true; 1230 1231 if (ci->ci_data.cpu_idlelwp == NULL) 1232 return true; 1233 1234 sc->sc_wasonline = !(ci->ci_schedstate.spc_flags & SPCF_OFFLINE); 1235 1236 if (sc->sc_wasonline) { 1237 mutex_enter(&cpu_lock); 1238 err = cpu_setstate(ci, false); 1239 mutex_exit(&cpu_lock); 1240 1241 if (err != 0) 1242 return false; 1243 } 1244 1245 return true; 1246 } 1247 1248 static bool 1249 cpu_suspend(device_t dv, const pmf_qual_t *qual) 1250 { 1251 struct cpu_softc *sc = device_private(dv); 1252 struct cpu_info *ci = sc->sc_info; 1253 1254 if ((ci->ci_flags & CPUF_PRESENT) == 0) 1255 return true; 1256 else { 1257 cpufreq_suspend(ci); 1258 } 1259 1260 return cpu_stop(dv); 1261 } 1262 1263 static bool 1264 cpu_resume(device_t dv, const pmf_qual_t *qual) 1265 { 1266 struct cpu_softc *sc = device_private(dv); 1267 struct cpu_info *ci = sc->sc_info; 1268 int err = 0; 1269 1270 if ((ci->ci_flags & CPUF_PRESENT) == 0) 1271 return true; 1272 1273 if ((ci->ci_flags & CPUF_PRIMARY) != 0) 1274 goto out; 1275 1276 if (ci->ci_data.cpu_idlelwp == NULL) 1277 goto out; 1278 1279 if (sc->sc_wasonline) { 1280 mutex_enter(&cpu_lock); 1281 err = cpu_setstate(ci, true); 1282 mutex_exit(&cpu_lock); 1283 } 1284 1285 out: 1286 if (err != 0) 1287 return false; 1288 1289 cpufreq_resume(ci); 1290 1291 return true; 1292 } 1293 1294 static bool 1295 cpu_shutdown(device_t dv, int how) 1296 { 1297 struct cpu_softc *sc = device_private(dv); 1298 struct cpu_info *ci = sc->sc_info; 1299 1300 if ((ci->ci_flags & CPUF_BSP) != 0) 1301 return false; 1302 1303 if ((ci->ci_flags & CPUF_PRESENT) == 0) 1304 return true; 1305 1306 return cpu_stop(dv); 1307 } 1308 1309 /* Get the TSC frequency and set it to ci->ci_data.cpu_cc_freq. */ 1310 void 1311 cpu_get_tsc_freq(struct cpu_info *ci) 1312 { 1313 uint64_t freq = 0, freq_from_cpuid, t0, t1; 1314 int64_t overhead; 1315 1316 if ((ci->ci_flags & CPUF_PRIMARY) != 0 && cpu_hascounter()) { 1317 /* 1318 * If it's the first call of this function, try to get TSC 1319 * freq from CPUID by calling cpu_tsc_freq_cpuid(). 1320 * The function also set lapic_per_second variable if it's 1321 * known. This is required for Intel's Comet Lake and newer 1322 * processors to set LAPIC timer correctly. 1323 */ 1324 if (ci->ci_data.cpu_cc_freq == 0) 1325 freq = freq_from_cpuid = cpu_tsc_freq_cpuid(ci); 1326 #if NHPET > 0 1327 if (freq == 0) 1328 freq = hpet_tsc_freq(); 1329 #endif 1330 if (freq == 0) { 1331 /* 1332 * Work out the approximate overhead involved below. 1333 * Discard the result of the first go around the 1334 * loop. 1335 */ 1336 overhead = 0; 1337 for (int i = 0; i <= 8; i++) { 1338 t0 = cpu_counter(); 1339 delay_func(0); 1340 t1 = cpu_counter(); 1341 if (i > 0) { 1342 overhead += (t1 - t0); 1343 } 1344 } 1345 overhead >>= 3; 1346 1347 /* Now do the calibration. */ 1348 t0 = cpu_counter(); 1349 delay_func(100000); 1350 t1 = cpu_counter(); 1351 freq = (t1 - t0 - overhead) * 10; 1352 } 1353 if (ci->ci_data.cpu_cc_freq != 0) { 1354 freq_from_cpuid = cpu_tsc_freq_cpuid(ci); 1355 if ((freq_from_cpuid != 0) 1356 && (freq != freq_from_cpuid)) 1357 aprint_verbose_dev(ci->ci_dev, "TSC freq " 1358 "calibrated %" PRIu64 " Hz\n", freq); 1359 } 1360 } else { 1361 freq = cpu_info_primary.ci_data.cpu_cc_freq; 1362 } 1363 1364 ci->ci_data.cpu_cc_freq = freq; 1365 } 1366 1367 void 1368 x86_cpu_idle_mwait(void) 1369 { 1370 struct cpu_info *ci = curcpu(); 1371 1372 KASSERT(ci->ci_ilevel == IPL_NONE); 1373 1374 x86_monitor(&ci->ci_want_resched, 0, 0); 1375 if (__predict_false(ci->ci_want_resched)) { 1376 return; 1377 } 1378 x86_mwait(0, 0); 1379 } 1380 1381 void 1382 x86_cpu_idle_halt(void) 1383 { 1384 struct cpu_info *ci = curcpu(); 1385 1386 KASSERT(ci->ci_ilevel == IPL_NONE); 1387 1388 x86_disable_intr(); 1389 if (!__predict_false(ci->ci_want_resched)) { 1390 x86_stihlt(); 1391 } else { 1392 x86_enable_intr(); 1393 } 1394 } 1395 1396 /* 1397 * Loads pmap for the current CPU. 1398 */ 1399 void 1400 cpu_load_pmap(struct pmap *pmap, struct pmap *oldpmap) 1401 { 1402 #ifdef SVS 1403 if (svs_enabled) { 1404 svs_pdir_switch(pmap); 1405 } 1406 #endif 1407 1408 #ifdef PAE 1409 struct cpu_info *ci = curcpu(); 1410 bool interrupts_enabled; 1411 pd_entry_t *l3_pd = ci->ci_pae_l3_pdir; 1412 int i; 1413 1414 /* 1415 * disable interrupts to block TLB shootdowns, which can reload cr3. 1416 * while this doesn't block NMIs, it's probably ok as NMIs unlikely 1417 * reload cr3. 1418 */ 1419 interrupts_enabled = (x86_read_flags() & PSL_I) != 0; 1420 if (interrupts_enabled) 1421 x86_disable_intr(); 1422 1423 for (i = 0 ; i < PDP_SIZE; i++) { 1424 l3_pd[i] = pmap->pm_pdirpa[i] | PTE_P; 1425 } 1426 1427 if (interrupts_enabled) 1428 x86_enable_intr(); 1429 tlbflush(); 1430 #else 1431 lcr3(pmap_pdirpa(pmap, 0)); 1432 #endif 1433 } 1434 1435 /* 1436 * Notify all other cpus to halt. 1437 */ 1438 1439 void 1440 cpu_broadcast_halt(void) 1441 { 1442 x86_broadcast_ipi(X86_IPI_HALT); 1443 } 1444 1445 /* 1446 * Send a dummy ipi to a cpu to force it to run splraise()/spllower(), 1447 * and trigger an AST on the running LWP. 1448 */ 1449 1450 void 1451 cpu_kick(struct cpu_info *ci) 1452 { 1453 x86_send_ipi(ci, X86_IPI_AST); 1454 } 1455
Indexes created Thu Sep 25 16:09:42 GMT 2025