cpu.c revision 1.194
1 /* $NetBSD: cpu.c,v 1.194 2020/06/15 09:09:24 msaitoh Exp $ */ 2 3 /* 4 * Copyright (c) 2000-2020 NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Bill Sommerfeld of RedBack Networks Inc, and by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * Copyright (c) 1999 Stefan Grefen 34 * 35 * Redistribution and use in source and binary forms, with or without 36 * modification, are permitted provided that the following conditions 37 * are met: 38 * 1. Redistributions of source code must retain the above copyright 39 * notice, this list of conditions and the following disclaimer. 40 * 2. Redistributions in binary form must reproduce the above copyright 41 * notice, this list of conditions and the following disclaimer in the 42 * documentation and/or other materials provided with the distribution. 43 * 3. All advertising materials mentioning features or use of this software 44 * must display the following acknowledgement: 45 * This product includes software developed by the NetBSD 46 * Foundation, Inc. and its contributors. 47 * 4. Neither the name of The NetBSD Foundation nor the names of its 48 * contributors may be used to endorse or promote products derived 49 * from this software without specific prior written permission. 50 * 51 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND ANY 52 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 53 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 54 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR AND CONTRIBUTORS BE LIABLE 55 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 56 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 57 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 58 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 59 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 60 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 61 * SUCH DAMAGE. 62 */ 63 64 #include <sys/cdefs.h> 65 __KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.194 2020/06/15 09:09:24 msaitoh Exp $"); 66 67 #include "opt_ddb.h" 68 #include "opt_mpbios.h" /* for MPDEBUG */ 69 #include "opt_mtrr.h" 70 #include "opt_multiprocessor.h" 71 #include "opt_svs.h" 72 73 #include "lapic.h" 74 #include "ioapic.h" 75 #include "acpica.h" 76 #include "hpet.h" 77 78 #include <sys/param.h> 79 #include <sys/proc.h> 80 #include <sys/systm.h> 81 #include <sys/device.h> 82 #include <sys/cpu.h> 83 #include <sys/cpufreq.h> 84 #include <sys/idle.h> 85 #include <sys/atomic.h> 86 #include <sys/reboot.h> 87 #include <sys/csan.h> 88 89 #include <uvm/uvm.h> 90 91 #include "acpica.h" /* for NACPICA, for mp_verbose */ 92 93 #include <x86/machdep.h> 94 #include <machine/cpufunc.h> 95 #include <machine/cpuvar.h> 96 #include <machine/pmap.h> 97 #include <machine/vmparam.h> 98 #if defined(MULTIPROCESSOR) 99 #include <machine/mpbiosvar.h> 100 #endif 101 #include <machine/mpconfig.h> /* for mp_verbose */ 102 #include <machine/pcb.h> 103 #include <machine/specialreg.h> 104 #include <machine/segments.h> 105 #include <machine/gdt.h> 106 #include <machine/mtrr.h> 107 #include <machine/pio.h> 108 #include <machine/cpu_counter.h> 109 110 #include <x86/fpu.h> 111 112 #if NACPICA > 0 113 #include <dev/acpi/acpi_srat.h> 114 #endif 115 116 #if NLAPIC > 0 117 #include <machine/apicvar.h> 118 #include <machine/i82489reg.h> 119 #include <machine/i82489var.h> 120 #endif 121 122 #include <dev/ic/mc146818reg.h> 123 #include <dev/ic/hpetvar.h> 124 #include <i386/isa/nvram.h> 125 #include <dev/isa/isareg.h> 126 127 #include "tsc.h" 128 129 #ifndef XENPV 130 #include "hyperv.h" 131 #if NHYPERV > 0 132 #include <x86/x86/hypervvar.h> 133 #endif 134 #endif 135 136 #ifdef XEN 137 #include <xen/hypervisor.h> 138 #endif 139 140 static int cpu_match(device_t, cfdata_t, void *); 141 static void cpu_attach(device_t, device_t, void *); 142 static void cpu_defer(device_t); 143 static int cpu_rescan(device_t, const char *, const int *); 144 static void cpu_childdetached(device_t, device_t); 145 static bool cpu_stop(device_t); 146 static bool cpu_suspend(device_t, const pmf_qual_t *); 147 static bool cpu_resume(device_t, const pmf_qual_t *); 148 static bool cpu_shutdown(device_t, int); 149 150 struct cpu_softc { 151 device_t sc_dev; /* device tree glue */ 152 struct cpu_info *sc_info; /* pointer to CPU info */ 153 bool sc_wasonline; 154 }; 155 156 #ifdef MULTIPROCESSOR 157 int mp_cpu_start(struct cpu_info *, paddr_t); 158 void mp_cpu_start_cleanup(struct cpu_info *); 159 const struct cpu_functions mp_cpu_funcs = { mp_cpu_start, NULL, 160 mp_cpu_start_cleanup }; 161 #endif 162 163 164 CFATTACH_DECL2_NEW(cpu, sizeof(struct cpu_softc), 165 cpu_match, cpu_attach, NULL, NULL, cpu_rescan, cpu_childdetached); 166 167 /* 168 * Statically-allocated CPU info for the primary CPU (or the only 169 * CPU, on uniprocessors). The CPU info list is initialized to 170 * point at it. 171 */ 172 struct cpu_info cpu_info_primary __aligned(CACHE_LINE_SIZE) = { 173 .ci_dev = 0, 174 .ci_self = &cpu_info_primary, 175 .ci_idepth = -1, 176 .ci_curlwp = &lwp0, 177 .ci_curldt = -1, 178 }; 179 180 struct cpu_info *cpu_info_list = &cpu_info_primary; 181 182 #ifdef i386 183 void cpu_set_tss_gates(struct cpu_info *); 184 #endif 185 186 static void cpu_init_idle_lwp(struct cpu_info *); 187 188 uint32_t cpu_feature[7] __read_mostly; /* X86 CPUID feature bits */ 189 /* [0] basic features cpuid.1:%edx 190 * [1] basic features cpuid.1:%ecx (CPUID2_xxx bits) 191 * [2] extended features cpuid:80000001:%edx 192 * [3] extended features cpuid:80000001:%ecx 193 * [4] VIA padlock features 194 * [5] structured extended features cpuid.7:%ebx 195 * [6] structured extended features cpuid.7:%ecx 196 */ 197 198 #ifdef MULTIPROCESSOR 199 bool x86_mp_online; 200 paddr_t mp_trampoline_paddr = MP_TRAMPOLINE; 201 #endif 202 #if NLAPIC > 0 203 static vaddr_t cmos_data_mapping; 204 #endif 205 struct cpu_info *cpu_starting; 206 207 #ifdef MULTIPROCESSOR 208 void cpu_hatch(void *); 209 static void cpu_boot_secondary(struct cpu_info *ci); 210 static void cpu_start_secondary(struct cpu_info *ci); 211 #if NLAPIC > 0 212 static void cpu_copy_trampoline(paddr_t); 213 #endif 214 #endif /* MULTIPROCESSOR */ 215 216 /* 217 * Runs once per boot once multiprocessor goo has been detected and 218 * the local APIC on the boot processor has been mapped. 219 * 220 * Called from lapic_boot_init() (from mpbios_scan()). 221 */ 222 #if NLAPIC > 0 223 void 224 cpu_init_first(void) 225 { 226 227 cpu_info_primary.ci_cpuid = lapic_cpu_number(); 228 229 cmos_data_mapping = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, UVM_KMF_VAONLY); 230 if (cmos_data_mapping == 0) 231 panic("No KVA for page 0"); 232 pmap_kenter_pa(cmos_data_mapping, 0, VM_PROT_READ|VM_PROT_WRITE, 0); 233 pmap_update(pmap_kernel()); 234 } 235 #endif 236 237 static int 238 cpu_match(device_t parent, cfdata_t match, void *aux) 239 { 240 241 return 1; 242 } 243 244 #ifdef __HAVE_PCPU_AREA 245 void 246 cpu_pcpuarea_init(struct cpu_info *ci) 247 { 248 struct vm_page *pg; 249 size_t i, npages; 250 vaddr_t base, va; 251 paddr_t pa; 252 253 CTASSERT(sizeof(struct pcpu_entry) % PAGE_SIZE == 0); 254 255 npages = sizeof(struct pcpu_entry) / PAGE_SIZE; 256 base = (vaddr_t)&pcpuarea->ent[cpu_index(ci)]; 257 258 for (i = 0; i < npages; i++) { 259 pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO); 260 if (pg == NULL) { 261 panic("failed to allocate pcpu PA"); 262 } 263 264 va = base + i * PAGE_SIZE; 265 pa = VM_PAGE_TO_PHYS(pg); 266 267 pmap_kenter_pa(va, pa, VM_PROT_READ|VM_PROT_WRITE, 0); 268 } 269 270 pmap_update(pmap_kernel()); 271 } 272 #endif 273 274 static void 275 cpu_vm_init(struct cpu_info *ci) 276 { 277 int ncolors = 2, i; 278 279 for (i = CAI_ICACHE; i <= CAI_L2CACHE; i++) { 280 struct x86_cache_info *cai; 281 int tcolors; 282 283 cai = &ci->ci_cinfo[i]; 284 285 tcolors = atop(cai->cai_totalsize); 286 switch (cai->cai_associativity) { 287 case 0xff: 288 tcolors = 1; /* fully associative */ 289 break; 290 case 0: 291 case 1: 292 break; 293 default: 294 tcolors /= cai->cai_associativity; 295 } 296 ncolors = uimax(ncolors, tcolors); 297 /* 298 * If the desired number of colors is not a power of 299 * two, it won't be good. Find the greatest power of 300 * two which is an even divisor of the number of colors, 301 * to preserve even coloring of pages. 302 */ 303 if (ncolors & (ncolors - 1) ) { 304 int try, picked = 1; 305 for (try = 1; try < ncolors; try *= 2) { 306 if (ncolors % try == 0) picked = try; 307 } 308 if (picked == 1) { 309 panic("desired number of cache colors %d is " 310 " > 1, but not even!", ncolors); 311 } 312 ncolors = picked; 313 } 314 } 315 316 /* 317 * Knowing the size of the largest cache on this CPU, potentially 318 * re-color our pages. 319 */ 320 aprint_debug_dev(ci->ci_dev, "%d page colors\n", ncolors); 321 uvm_page_recolor(ncolors); 322 323 pmap_tlb_cpu_init(ci); 324 #ifndef __HAVE_DIRECT_MAP 325 pmap_vpage_cpu_init(ci); 326 #endif 327 } 328 329 static void 330 cpu_attach(device_t parent, device_t self, void *aux) 331 { 332 struct cpu_softc *sc = device_private(self); 333 struct cpu_attach_args *caa = aux; 334 struct cpu_info *ci; 335 uintptr_t ptr; 336 #if NLAPIC > 0 337 int cpunum = caa->cpu_number; 338 #endif 339 static bool again; 340 341 sc->sc_dev = self; 342 343 if (ncpu > maxcpus) { 344 #ifndef _LP64 345 aprint_error(": too many CPUs, please use NetBSD/amd64\n"); 346 #else 347 aprint_error(": too many CPUs\n"); 348 #endif 349 return; 350 } 351 352 /* 353 * If we're an Application Processor, allocate a cpu_info 354 * structure, otherwise use the primary's. 355 */ 356 if (caa->cpu_role == CPU_ROLE_AP) { 357 if ((boothowto & RB_MD1) != 0) { 358 aprint_error(": multiprocessor boot disabled\n"); 359 if (!pmf_device_register(self, NULL, NULL)) 360 aprint_error_dev(self, 361 "couldn't establish power handler\n"); 362 return; 363 } 364 aprint_naive(": Application Processor\n"); 365 ptr = (uintptr_t)uvm_km_alloc(kernel_map, 366 sizeof(*ci) + CACHE_LINE_SIZE - 1, 0, 367 UVM_KMF_WIRED|UVM_KMF_ZERO); 368 ci = (struct cpu_info *)roundup2(ptr, CACHE_LINE_SIZE); 369 ci->ci_curldt = -1; 370 } else { 371 aprint_naive(": %s Processor\n", 372 caa->cpu_role == CPU_ROLE_SP ? "Single" : "Boot"); 373 ci = &cpu_info_primary; 374 #if NLAPIC > 0 375 if (cpunum != lapic_cpu_number()) { 376 /* XXX should be done earlier. */ 377 uint32_t reg; 378 aprint_verbose("\n"); 379 aprint_verbose_dev(self, "running CPU at apic %d" 380 " instead of at expected %d", lapic_cpu_number(), 381 cpunum); 382 reg = lapic_readreg(LAPIC_ID); 383 lapic_writereg(LAPIC_ID, (reg & ~LAPIC_ID_MASK) | 384 (cpunum << LAPIC_ID_SHIFT)); 385 } 386 if (cpunum != lapic_cpu_number()) { 387 aprint_error_dev(self, "unable to reset apic id\n"); 388 } 389 #endif 390 } 391 392 ci->ci_self = ci; 393 sc->sc_info = ci; 394 ci->ci_dev = self; 395 ci->ci_acpiid = caa->cpu_id; 396 ci->ci_cpuid = caa->cpu_number; 397 ci->ci_func = caa->cpu_func; 398 ci->ci_kfpu_spl = -1; 399 aprint_normal("\n"); 400 401 /* Must be before mi_cpu_attach(). */ 402 cpu_vm_init(ci); 403 404 if (caa->cpu_role == CPU_ROLE_AP) { 405 int error; 406 407 error = mi_cpu_attach(ci); 408 if (error != 0) { 409 aprint_error_dev(self, 410 "mi_cpu_attach failed with %d\n", error); 411 return; 412 } 413 #ifdef __HAVE_PCPU_AREA 414 cpu_pcpuarea_init(ci); 415 #endif 416 cpu_init_tss(ci); 417 } else { 418 KASSERT(ci->ci_data.cpu_idlelwp != NULL); 419 #if NACPICA > 0 420 /* Parse out NUMA info for cpu_identify(). */ 421 acpisrat_init(); 422 #endif 423 } 424 425 #ifdef SVS 426 cpu_svs_init(ci); 427 #endif 428 429 pmap_reference(pmap_kernel()); 430 ci->ci_pmap = pmap_kernel(); 431 ci->ci_tlbstate = TLBSTATE_STALE; 432 433 /* 434 * Boot processor may not be attached first, but the below 435 * must be done to allow booting other processors. 436 */ 437 if (!again) { 438 /* Make sure DELAY() (likely i8254_delay()) is initialized. */ 439 DELAY(1); 440 441 /* 442 * Basic init. Compute an approximate frequency for the TSC 443 * using the i8254. If there's a HPET we'll redo it later. 444 */ 445 atomic_or_32(&ci->ci_flags, CPUF_PRESENT | CPUF_PRIMARY); 446 cpu_intr_init(ci); 447 tsc_setfunc(ci); 448 cpu_get_tsc_freq(ci); 449 cpu_init(ci); 450 #ifdef i386 451 cpu_set_tss_gates(ci); 452 #endif 453 pmap_cpu_init_late(ci); 454 #if NLAPIC > 0 455 if (caa->cpu_role != CPU_ROLE_SP) { 456 /* Enable lapic. */ 457 lapic_enable(); 458 lapic_set_lvt(); 459 if (!vm_guest_is_xenpvh_or_pvhvm()) 460 lapic_calibrate_timer(false); 461 } 462 #endif 463 kcsan_cpu_init(ci); 464 again = true; 465 } 466 467 /* further PCB init done later. */ 468 469 switch (caa->cpu_role) { 470 case CPU_ROLE_SP: 471 atomic_or_32(&ci->ci_flags, CPUF_SP); 472 cpu_identify(ci); 473 x86_errata(); 474 x86_cpu_idle_init(); 475 #ifdef XENPVHVM 476 xen_hvm_init_cpu(ci); 477 #endif 478 break; 479 480 case CPU_ROLE_BP: 481 atomic_or_32(&ci->ci_flags, CPUF_BSP); 482 cpu_identify(ci); 483 x86_errata(); 484 x86_cpu_idle_init(); 485 #ifdef XENPVHVM 486 xen_hvm_init_cpu(ci); 487 #endif 488 break; 489 490 #ifdef MULTIPROCESSOR 491 case CPU_ROLE_AP: 492 /* 493 * report on an AP 494 */ 495 cpu_intr_init(ci); 496 gdt_alloc_cpu(ci); 497 #ifdef i386 498 cpu_set_tss_gates(ci); 499 #endif 500 pmap_cpu_init_late(ci); 501 cpu_start_secondary(ci); 502 if (ci->ci_flags & CPUF_PRESENT) { 503 struct cpu_info *tmp; 504 505 cpu_identify(ci); 506 tmp = cpu_info_list; 507 while (tmp->ci_next) 508 tmp = tmp->ci_next; 509 510 tmp->ci_next = ci; 511 } 512 break; 513 #endif 514 515 default: 516 panic("unknown processor type??\n"); 517 } 518 519 pat_init(ci); 520 521 if (!pmf_device_register1(self, cpu_suspend, cpu_resume, cpu_shutdown)) 522 aprint_error_dev(self, "couldn't establish power handler\n"); 523 524 #ifdef MULTIPROCESSOR 525 if (mp_verbose) { 526 struct lwp *l = ci->ci_data.cpu_idlelwp; 527 struct pcb *pcb = lwp_getpcb(l); 528 529 aprint_verbose_dev(self, 530 "idle lwp at %p, idle sp at %p\n", 531 l, 532 #ifdef i386 533 (void *)pcb->pcb_esp 534 #else 535 (void *)pcb->pcb_rsp 536 #endif 537 ); 538 } 539 #endif 540 541 /* 542 * Postpone the "cpufeaturebus" scan. 543 * It is safe to scan the pseudo-bus 544 * only after all CPUs have attached. 545 */ 546 (void)config_defer(self, cpu_defer); 547 } 548 549 static void 550 cpu_defer(device_t self) 551 { 552 cpu_rescan(self, NULL, NULL); 553 } 554 555 static int 556 cpu_rescan(device_t self, const char *ifattr, const int *locators) 557 { 558 struct cpu_softc *sc = device_private(self); 559 struct cpufeature_attach_args cfaa; 560 struct cpu_info *ci = sc->sc_info; 561 562 /* 563 * If we booted with RB_MD1 to disable multiprocessor, the 564 * auto-configuration data still contains the additional 565 * CPUs. But their initialization was mostly bypassed 566 * during attach, so we have to make sure we don't look at 567 * their featurebus info, since it wasn't retrieved. 568 */ 569 if (ci == NULL) 570 return 0; 571 572 memset(&cfaa, 0, sizeof(cfaa)); 573 cfaa.ci = ci; 574 575 if (ifattr_match(ifattr, "cpufeaturebus")) { 576 if (ci->ci_frequency == NULL) { 577 cfaa.name = "frequency"; 578 ci->ci_frequency = config_found_ia(self, 579 "cpufeaturebus", &cfaa, NULL); 580 } 581 582 if (ci->ci_padlock == NULL) { 583 cfaa.name = "padlock"; 584 ci->ci_padlock = config_found_ia(self, 585 "cpufeaturebus", &cfaa, NULL); 586 } 587 588 if (ci->ci_temperature == NULL) { 589 cfaa.name = "temperature"; 590 ci->ci_temperature = config_found_ia(self, 591 "cpufeaturebus", &cfaa, NULL); 592 } 593 594 if (ci->ci_vm == NULL) { 595 cfaa.name = "vm"; 596 ci->ci_vm = config_found_ia(self, 597 "cpufeaturebus", &cfaa, NULL); 598 } 599 } 600 601 return 0; 602 } 603 604 static void 605 cpu_childdetached(device_t self, device_t child) 606 { 607 struct cpu_softc *sc = device_private(self); 608 struct cpu_info *ci = sc->sc_info; 609 610 if (ci->ci_frequency == child) 611 ci->ci_frequency = NULL; 612 613 if (ci->ci_padlock == child) 614 ci->ci_padlock = NULL; 615 616 if (ci->ci_temperature == child) 617 ci->ci_temperature = NULL; 618 619 if (ci->ci_vm == child) 620 ci->ci_vm = NULL; 621 } 622 623 /* 624 * Initialize the processor appropriately. 625 */ 626 627 void 628 cpu_init(struct cpu_info *ci) 629 { 630 extern int x86_fpu_save; 631 uint32_t cr4 = 0; 632 633 lcr0(rcr0() | CR0_WP); 634 635 /* If global TLB caching is supported, enable it */ 636 if (cpu_feature[0] & CPUID_PGE) 637 cr4 |= CR4_PGE; 638 639 /* 640 * If we have FXSAVE/FXRESTOR, use them. 641 */ 642 if (cpu_feature[0] & CPUID_FXSR) { 643 cr4 |= CR4_OSFXSR; 644 645 /* 646 * If we have SSE/SSE2, enable XMM exceptions. 647 */ 648 if (cpu_feature[0] & (CPUID_SSE|CPUID_SSE2)) 649 cr4 |= CR4_OSXMMEXCPT; 650 } 651 652 /* If xsave is supported, enable it */ 653 if (cpu_feature[1] & CPUID2_XSAVE) 654 cr4 |= CR4_OSXSAVE; 655 656 /* If SMEP is supported, enable it */ 657 if (cpu_feature[5] & CPUID_SEF_SMEP) 658 cr4 |= CR4_SMEP; 659 660 /* If SMAP is supported, enable it */ 661 if (cpu_feature[5] & CPUID_SEF_SMAP) 662 cr4 |= CR4_SMAP; 663 664 #ifdef SVS 665 /* If PCID is supported, enable it */ 666 if (svs_pcid) 667 cr4 |= CR4_PCIDE; 668 #endif 669 670 if (cr4) { 671 cr4 |= rcr4(); 672 lcr4(cr4); 673 } 674 675 /* 676 * Changing CR4 register may change cpuid values. For example, setting 677 * CR4_OSXSAVE sets CPUID2_OSXSAVE. The CPUID2_OSXSAVE is in 678 * ci_feat_val[1], so update it. 679 * XXX Other than ci_feat_val[1] might be changed. 680 */ 681 if (cpuid_level >= 1) { 682 u_int descs[4]; 683 684 x86_cpuid(1, descs); 685 ci->ci_feat_val[1] = descs[2]; 686 } 687 688 if (x86_fpu_save >= FPU_SAVE_FXSAVE) { 689 fpuinit_mxcsr_mask(); 690 } 691 692 /* If xsave is enabled, enable all fpu features */ 693 if (cr4 & CR4_OSXSAVE) 694 wrxcr(0, x86_xsave_features & XCR0_FPU); 695 696 #ifdef MTRR 697 /* 698 * On a P6 or above, initialize MTRR's if the hardware supports them. 699 */ 700 if (cpu_feature[0] & CPUID_MTRR) { 701 if ((ci->ci_flags & CPUF_AP) == 0) 702 i686_mtrr_init_first(); 703 mtrr_init_cpu(ci); 704 } 705 706 #ifdef i386 707 if (strcmp((char *)(ci->ci_vendor), "AuthenticAMD") == 0) { 708 /* 709 * Must be a K6-2 Step >= 7 or a K6-III. 710 */ 711 if (CPUID_TO_FAMILY(ci->ci_signature) == 5) { 712 if (CPUID_TO_MODEL(ci->ci_signature) > 8 || 713 (CPUID_TO_MODEL(ci->ci_signature) == 8 && 714 CPUID_TO_STEPPING(ci->ci_signature) >= 7)) { 715 mtrr_funcs = &k6_mtrr_funcs; 716 k6_mtrr_init_first(); 717 mtrr_init_cpu(ci); 718 } 719 } 720 } 721 #endif /* i386 */ 722 #endif /* MTRR */ 723 724 if (ci != &cpu_info_primary) { 725 /* Synchronize TSC */ 726 atomic_or_32(&ci->ci_flags, CPUF_RUNNING); 727 tsc_sync_ap(ci); 728 } else { 729 atomic_or_32(&ci->ci_flags, CPUF_RUNNING); 730 } 731 } 732 733 #ifdef MULTIPROCESSOR 734 void 735 cpu_boot_secondary_processors(void) 736 { 737 struct cpu_info *ci; 738 kcpuset_t *cpus; 739 u_long i; 740 741 /* Now that we know the number of CPUs, patch the text segment. */ 742 x86_patch(false); 743 744 #if NACPICA > 0 745 /* Finished with NUMA info for now. */ 746 acpisrat_exit(); 747 #endif 748 749 kcpuset_create(&cpus, true); 750 kcpuset_set(cpus, cpu_index(curcpu())); 751 for (i = 0; i < maxcpus; i++) { 752 ci = cpu_lookup(i); 753 if (ci == NULL) 754 continue; 755 if (ci->ci_data.cpu_idlelwp == NULL) 756 continue; 757 if ((ci->ci_flags & CPUF_PRESENT) == 0) 758 continue; 759 if (ci->ci_flags & (CPUF_BSP|CPUF_SP|CPUF_PRIMARY)) 760 continue; 761 cpu_boot_secondary(ci); 762 kcpuset_set(cpus, cpu_index(ci)); 763 } 764 while (!kcpuset_match(cpus, kcpuset_running)) 765 ; 766 kcpuset_destroy(cpus); 767 768 x86_mp_online = true; 769 770 /* Now that we know about the TSC, attach the timecounter. */ 771 tsc_tc_init(); 772 } 773 #endif 774 775 static void 776 cpu_init_idle_lwp(struct cpu_info *ci) 777 { 778 struct lwp *l = ci->ci_data.cpu_idlelwp; 779 struct pcb *pcb = lwp_getpcb(l); 780 781 pcb->pcb_cr0 = rcr0(); 782 } 783 784 void 785 cpu_init_idle_lwps(void) 786 { 787 struct cpu_info *ci; 788 u_long i; 789 790 for (i = 0; i < maxcpus; i++) { 791 ci = cpu_lookup(i); 792 if (ci == NULL) 793 continue; 794 if (ci->ci_data.cpu_idlelwp == NULL) 795 continue; 796 if ((ci->ci_flags & CPUF_PRESENT) == 0) 797 continue; 798 cpu_init_idle_lwp(ci); 799 } 800 } 801 802 #ifdef MULTIPROCESSOR 803 void 804 cpu_start_secondary(struct cpu_info *ci) 805 { 806 u_long psl; 807 int i; 808 809 #if NLAPIC > 0 810 paddr_t mp_pdirpa; 811 mp_pdirpa = pmap_init_tmp_pgtbl(mp_trampoline_paddr); 812 cpu_copy_trampoline(mp_pdirpa); 813 #endif 814 815 atomic_or_32(&ci->ci_flags, CPUF_AP); 816 ci->ci_curlwp = ci->ci_data.cpu_idlelwp; 817 if (CPU_STARTUP(ci, mp_trampoline_paddr) != 0) { 818 return; 819 } 820 821 /* 822 * Wait for it to become ready. Setting cpu_starting opens the 823 * initial gate and allows the AP to start soft initialization. 824 */ 825 KASSERT(cpu_starting == NULL); 826 cpu_starting = ci; 827 for (i = 100000; (!(ci->ci_flags & CPUF_PRESENT)) && i > 0; i--) { 828 delay_func(10); 829 } 830 831 if ((ci->ci_flags & CPUF_PRESENT) == 0) { 832 aprint_error_dev(ci->ci_dev, "failed to become ready\n"); 833 #if defined(MPDEBUG) && defined(DDB) 834 printf("dropping into debugger; continue from here to resume boot\n"); 835 Debugger(); 836 #endif 837 } else { 838 /* 839 * Synchronize time stamp counters. Invalidate cache and do 840 * twice (in tsc_sync_bp) to minimize possible cache effects. 841 * Disable interrupts to try and rule out any external 842 * interference. 843 */ 844 psl = x86_read_psl(); 845 x86_disable_intr(); 846 tsc_sync_bp(ci); 847 x86_write_psl(psl); 848 } 849 850 CPU_START_CLEANUP(ci); 851 cpu_starting = NULL; 852 } 853 854 void 855 cpu_boot_secondary(struct cpu_info *ci) 856 { 857 int64_t drift; 858 u_long psl; 859 int i; 860 861 atomic_or_32(&ci->ci_flags, CPUF_GO); 862 for (i = 100000; (!(ci->ci_flags & CPUF_RUNNING)) && i > 0; i--) { 863 delay_func(10); 864 } 865 if ((ci->ci_flags & CPUF_RUNNING) == 0) { 866 aprint_error_dev(ci->ci_dev, "failed to start\n"); 867 #if defined(MPDEBUG) && defined(DDB) 868 printf("dropping into debugger; continue from here to resume boot\n"); 869 Debugger(); 870 #endif 871 } else { 872 /* Synchronize TSC again, check for drift. */ 873 drift = ci->ci_data.cpu_cc_skew; 874 psl = x86_read_psl(); 875 x86_disable_intr(); 876 tsc_sync_bp(ci); 877 x86_write_psl(psl); 878 drift -= ci->ci_data.cpu_cc_skew; 879 aprint_debug_dev(ci->ci_dev, "TSC skew=%lld drift=%lld\n", 880 (long long)ci->ci_data.cpu_cc_skew, (long long)drift); 881 tsc_sync_drift(drift); 882 } 883 } 884 885 /* 886 * The CPU ends up here when it's ready to run. 887 * This is called from code in mptramp.s; at this point, we are running 888 * in the idle pcb/idle stack of the new CPU. When this function returns, 889 * this processor will enter the idle loop and start looking for work. 890 */ 891 void 892 cpu_hatch(void *v) 893 { 894 struct cpu_info *ci = (struct cpu_info *)v; 895 struct pcb *pcb; 896 int s, i; 897 898 /* ------------------------------------------------------------- */ 899 900 /* 901 * This section of code must be compiled with SSP disabled, to 902 * prevent a race against cpu0. See sys/conf/ssp.mk. 903 */ 904 905 cpu_init_msrs(ci, true); 906 cpu_probe(ci); 907 cpu_speculation_init(ci); 908 #if NHYPERV > 0 909 hyperv_init_cpu(ci); 910 #endif 911 912 ci->ci_data.cpu_cc_freq = cpu_info_primary.ci_data.cpu_cc_freq; 913 /* cpu_get_tsc_freq(ci); */ 914 915 KDASSERT((ci->ci_flags & CPUF_PRESENT) == 0); 916 917 /* 918 * Synchronize the TSC for the first time. Note that interrupts are 919 * off at this point. 920 */ 921 atomic_or_32(&ci->ci_flags, CPUF_PRESENT); 922 tsc_sync_ap(ci); 923 924 /* ------------------------------------------------------------- */ 925 926 /* 927 * Wait to be brought online. 928 * 929 * Use MONITOR/MWAIT if available. These instructions put the CPU in 930 * a low consumption mode (C-state), and if the TSC is not invariant, 931 * this causes the TSC to drift. We want this to happen, so that we 932 * can later detect (in tsc_tc_init) any abnormal drift with invariant 933 * TSCs. That's just for safety; by definition such drifts should 934 * never occur with invariant TSCs. 935 * 936 * If not available, try PAUSE. We'd like to use HLT, but we have 937 * interrupts off. 938 */ 939 while ((ci->ci_flags & CPUF_GO) == 0) { 940 if ((cpu_feature[1] & CPUID2_MONITOR) != 0) { 941 x86_monitor(&ci->ci_flags, 0, 0); 942 if ((ci->ci_flags & CPUF_GO) != 0) { 943 continue; 944 } 945 x86_mwait(0, 0); 946 } else { 947 /* 948 * XXX The loop repetition count could be a lot higher, but 949 * XXX currently qemu emulator takes a _very_long_time_ to 950 * XXX execute the pause instruction. So for now, use a low 951 * XXX value to allow the cpu to hatch before timing out. 952 */ 953 for (i = 50; i != 0; i--) { 954 x86_pause(); 955 } 956 } 957 } 958 959 /* Because the text may have been patched in x86_patch(). */ 960 wbinvd(); 961 x86_flush(); 962 tlbflushg(); 963 964 KASSERT((ci->ci_flags & CPUF_RUNNING) == 0); 965 966 #ifdef PAE 967 pd_entry_t * l3_pd = ci->ci_pae_l3_pdir; 968 for (i = 0 ; i < PDP_SIZE; i++) { 969 l3_pd[i] = pmap_kernel()->pm_pdirpa[i] | PTE_P; 970 } 971 lcr3(ci->ci_pae_l3_pdirpa); 972 #else 973 lcr3(pmap_pdirpa(pmap_kernel(), 0)); 974 #endif 975 976 pcb = lwp_getpcb(curlwp); 977 pcb->pcb_cr3 = rcr3(); 978 pcb = lwp_getpcb(ci->ci_data.cpu_idlelwp); 979 lcr0(pcb->pcb_cr0); 980 981 cpu_init_idt(); 982 gdt_init_cpu(ci); 983 #if NLAPIC > 0 984 lapic_enable(); 985 lapic_set_lvt(); 986 #endif 987 988 fpuinit(ci); 989 lldt(GSYSSEL(GLDT_SEL, SEL_KPL)); 990 ltr(ci->ci_tss_sel); 991 992 /* 993 * cpu_init will re-synchronize the TSC, and will detect any abnormal 994 * drift that would have been caused by the use of MONITOR/MWAIT 995 * above. 996 */ 997 cpu_init(ci); 998 #ifdef XENPVHVM 999 xen_hvm_init_cpu(ci); 1000 #endif 1001 (*x86_initclock_func)(); 1002 cpu_get_tsc_freq(ci); 1003 1004 s = splhigh(); 1005 #if NLAPIC > 0 1006 lapic_write_tpri(0); 1007 #endif 1008 x86_enable_intr(); 1009 splx(s); 1010 x86_errata(); 1011 1012 aprint_debug_dev(ci->ci_dev, "running\n"); 1013 1014 kcsan_cpu_init(ci); 1015 1016 idle_loop(NULL); 1017 KASSERT(false); 1018 } 1019 #endif 1020 1021 #if defined(DDB) 1022 1023 #include <ddb/db_output.h> 1024 #include <machine/db_machdep.h> 1025 1026 /* 1027 * Dump CPU information from ddb. 1028 */ 1029 void 1030 cpu_debug_dump(void) 1031 { 1032 struct cpu_info *ci; 1033 CPU_INFO_ITERATOR cii; 1034 const char sixtyfour64space[] = 1035 #ifdef _LP64 1036 " " 1037 #endif 1038 ""; 1039 1040 db_printf("addr %sdev id flags ipis spl curlwp " 1041 "\n", sixtyfour64space); 1042 for (CPU_INFO_FOREACH(cii, ci)) { 1043 db_printf("%p %s %ld %x %x %d %10p\n", 1044 ci, 1045 ci->ci_dev == NULL ? "BOOT" : device_xname(ci->ci_dev), 1046 (long)ci->ci_cpuid, 1047 ci->ci_flags, ci->ci_ipis, ci->ci_ilevel, 1048 ci->ci_curlwp); 1049 } 1050 } 1051 #endif 1052 1053 #ifdef MULTIPROCESSOR 1054 #if NLAPIC > 0 1055 static void 1056 cpu_copy_trampoline(paddr_t pdir_pa) 1057 { 1058 extern uint32_t nox_flag; 1059 extern u_char cpu_spinup_trampoline[]; 1060 extern u_char cpu_spinup_trampoline_end[]; 1061 vaddr_t mp_trampoline_vaddr; 1062 struct { 1063 uint32_t large; 1064 uint32_t nox; 1065 uint32_t pdir; 1066 } smp_data; 1067 CTASSERT(sizeof(smp_data) == 3 * 4); 1068 1069 smp_data.large = (pmap_largepages != 0); 1070 smp_data.nox = nox_flag; 1071 smp_data.pdir = (uint32_t)(pdir_pa & 0xFFFFFFFF); 1072 1073 /* Enter the physical address */ 1074 mp_trampoline_vaddr = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, 1075 UVM_KMF_VAONLY); 1076 pmap_kenter_pa(mp_trampoline_vaddr, mp_trampoline_paddr, 1077 VM_PROT_READ | VM_PROT_WRITE, 0); 1078 pmap_update(pmap_kernel()); 1079 1080 /* Copy boot code */ 1081 memcpy((void *)mp_trampoline_vaddr, 1082 cpu_spinup_trampoline, 1083 cpu_spinup_trampoline_end - cpu_spinup_trampoline); 1084 1085 /* Copy smp_data at the end */ 1086 memcpy((void *)(mp_trampoline_vaddr + PAGE_SIZE - sizeof(smp_data)), 1087 &smp_data, sizeof(smp_data)); 1088 1089 pmap_kremove(mp_trampoline_vaddr, PAGE_SIZE); 1090 pmap_update(pmap_kernel()); 1091 uvm_km_free(kernel_map, mp_trampoline_vaddr, PAGE_SIZE, UVM_KMF_VAONLY); 1092 } 1093 #endif 1094 1095 int 1096 mp_cpu_start(struct cpu_info *ci, paddr_t target) 1097 { 1098 int error; 1099 1100 /* 1101 * Bootstrap code must be addressable in real mode 1102 * and it must be page aligned. 1103 */ 1104 KASSERT(target < 0x10000 && target % PAGE_SIZE == 0); 1105 1106 /* 1107 * "The BSP must initialize CMOS shutdown code to 0Ah ..." 1108 */ 1109 1110 outb(IO_RTC, NVRAM_RESET); 1111 outb(IO_RTC+1, NVRAM_RESET_JUMP); 1112 1113 #if NLAPIC > 0 1114 /* 1115 * "and the warm reset vector (DWORD based at 40:67) to point 1116 * to the AP startup code ..." 1117 */ 1118 unsigned short dwordptr[2]; 1119 dwordptr[0] = 0; 1120 dwordptr[1] = target >> 4; 1121 1122 memcpy((uint8_t *)cmos_data_mapping + 0x467, dwordptr, 4); 1123 #endif 1124 1125 if ((cpu_feature[0] & CPUID_APIC) == 0) { 1126 aprint_error("mp_cpu_start: CPU does not have APIC\n"); 1127 return ENODEV; 1128 } 1129 1130 /* 1131 * ... prior to executing the following sequence:". We'll also add in 1132 * local cache flush, in case the BIOS has left the AP with its cache 1133 * disabled. It may not be able to cope with MP coherency. 1134 */ 1135 wbinvd(); 1136 1137 if (ci->ci_flags & CPUF_AP) { 1138 error = x86_ipi_init(ci->ci_cpuid); 1139 if (error != 0) { 1140 aprint_error_dev(ci->ci_dev, "%s: IPI not taken (1)\n", 1141 __func__); 1142 return error; 1143 } 1144 delay_func(10000); 1145 1146 error = x86_ipi_startup(ci->ci_cpuid, target / PAGE_SIZE); 1147 if (error != 0) { 1148 aprint_error_dev(ci->ci_dev, "%s: IPI not taken (2)\n", 1149 __func__); 1150 return error; 1151 } 1152 delay_func(200); 1153 1154 error = x86_ipi_startup(ci->ci_cpuid, target / PAGE_SIZE); 1155 if (error != 0) { 1156 aprint_error_dev(ci->ci_dev, "%s: IPI not taken (3)\n", 1157 __func__); 1158 return error; 1159 } 1160 delay_func(200); 1161 } 1162 1163 return 0; 1164 } 1165 1166 void 1167 mp_cpu_start_cleanup(struct cpu_info *ci) 1168 { 1169 /* 1170 * Ensure the NVRAM reset byte contains something vaguely sane. 1171 */ 1172 1173 outb(IO_RTC, NVRAM_RESET); 1174 outb(IO_RTC+1, NVRAM_RESET_RST); 1175 } 1176 #endif 1177 1178 #ifdef __x86_64__ 1179 typedef void (vector)(void); 1180 extern vector Xsyscall, Xsyscall32, Xsyscall_svs; 1181 #endif 1182 1183 void 1184 cpu_init_msrs(struct cpu_info *ci, bool full) 1185 { 1186 #ifdef __x86_64__ 1187 wrmsr(MSR_STAR, 1188 ((uint64_t)GSEL(GCODE_SEL, SEL_KPL) << 32) | 1189 ((uint64_t)LSEL(LSYSRETBASE_SEL, SEL_UPL) << 48)); 1190 wrmsr(MSR_LSTAR, (uint64_t)Xsyscall); 1191 wrmsr(MSR_CSTAR, (uint64_t)Xsyscall32); 1192 wrmsr(MSR_SFMASK, PSL_NT|PSL_T|PSL_I|PSL_C|PSL_D|PSL_AC); 1193 1194 #ifdef SVS 1195 if (svs_enabled) 1196 wrmsr(MSR_LSTAR, (uint64_t)Xsyscall_svs); 1197 #endif 1198 1199 if (full) { 1200 wrmsr(MSR_FSBASE, 0); 1201 wrmsr(MSR_GSBASE, (uint64_t)ci); 1202 wrmsr(MSR_KERNELGSBASE, 0); 1203 } 1204 #endif /* __x86_64__ */ 1205 1206 if (cpu_feature[2] & CPUID_NOX) 1207 wrmsr(MSR_EFER, rdmsr(MSR_EFER) | EFER_NXE); 1208 } 1209 1210 void 1211 cpu_offline_md(void) 1212 { 1213 return; 1214 } 1215 1216 /* XXX joerg restructure and restart CPUs individually */ 1217 static bool 1218 cpu_stop(device_t dv) 1219 { 1220 struct cpu_softc *sc = device_private(dv); 1221 struct cpu_info *ci = sc->sc_info; 1222 int err; 1223 1224 KASSERT((ci->ci_flags & CPUF_PRESENT) != 0); 1225 1226 if ((ci->ci_flags & CPUF_PRIMARY) != 0) 1227 return true; 1228 1229 if (ci->ci_data.cpu_idlelwp == NULL) 1230 return true; 1231 1232 sc->sc_wasonline = !(ci->ci_schedstate.spc_flags & SPCF_OFFLINE); 1233 1234 if (sc->sc_wasonline) { 1235 mutex_enter(&cpu_lock); 1236 err = cpu_setstate(ci, false); 1237 mutex_exit(&cpu_lock); 1238 1239 if (err != 0) 1240 return false; 1241 } 1242 1243 return true; 1244 } 1245 1246 static bool 1247 cpu_suspend(device_t dv, const pmf_qual_t *qual) 1248 { 1249 struct cpu_softc *sc = device_private(dv); 1250 struct cpu_info *ci = sc->sc_info; 1251 1252 if ((ci->ci_flags & CPUF_PRESENT) == 0) 1253 return true; 1254 else { 1255 cpufreq_suspend(ci); 1256 } 1257 1258 return cpu_stop(dv); 1259 } 1260 1261 static bool 1262 cpu_resume(device_t dv, const pmf_qual_t *qual) 1263 { 1264 struct cpu_softc *sc = device_private(dv); 1265 struct cpu_info *ci = sc->sc_info; 1266 int err = 0; 1267 1268 if ((ci->ci_flags & CPUF_PRESENT) == 0) 1269 return true; 1270 1271 if ((ci->ci_flags & CPUF_PRIMARY) != 0) 1272 goto out; 1273 1274 if (ci->ci_data.cpu_idlelwp == NULL) 1275 goto out; 1276 1277 if (sc->sc_wasonline) { 1278 mutex_enter(&cpu_lock); 1279 err = cpu_setstate(ci, true); 1280 mutex_exit(&cpu_lock); 1281 } 1282 1283 out: 1284 if (err != 0) 1285 return false; 1286 1287 cpufreq_resume(ci); 1288 1289 return true; 1290 } 1291 1292 static bool 1293 cpu_shutdown(device_t dv, int how) 1294 { 1295 struct cpu_softc *sc = device_private(dv); 1296 struct cpu_info *ci = sc->sc_info; 1297 1298 if ((ci->ci_flags & CPUF_BSP) != 0) 1299 return false; 1300 1301 if ((ci->ci_flags & CPUF_PRESENT) == 0) 1302 return true; 1303 1304 return cpu_stop(dv); 1305 } 1306 1307 /* Get the TSC frequency and set it to ci->ci_data.cpu_cc_freq. */ 1308 void 1309 cpu_get_tsc_freq(struct cpu_info *ci) 1310 { 1311 uint64_t freq = 0, freq_from_cpuid, t0, t1; 1312 int64_t overhead; 1313 1314 if ((ci->ci_flags & CPUF_PRIMARY) != 0 && cpu_hascounter()) { 1315 /* 1316 * If it's the first call of this function, try to get TSC 1317 * freq from CPUID by calling cpu_tsc_freq_cpuid(). 1318 * The function also set lapic_per_second variable if it's 1319 * known. This is required for Intel's Comet Lake and newer 1320 * processors to set LAPIC timer correctly. 1321 */ 1322 if (ci->ci_data.cpu_cc_freq == 0) 1323 freq = freq_from_cpuid = cpu_tsc_freq_cpuid(ci); 1324 #if NHPET > 0 1325 if (freq == 0) 1326 freq = hpet_tsc_freq(); 1327 #endif 1328 if (freq == 0) { 1329 /* 1330 * Work out the approximate overhead involved below. 1331 * Discard the result of the first go around the 1332 * loop. 1333 */ 1334 overhead = 0; 1335 for (int i = 0; i <= 8; i++) { 1336 t0 = cpu_counter(); 1337 delay_func(0); 1338 t1 = cpu_counter(); 1339 if (i > 0) { 1340 overhead += (t1 - t0); 1341 } 1342 } 1343 overhead >>= 3; 1344 1345 /* Now do the calibration. */ 1346 t0 = cpu_counter(); 1347 delay_func(100000); 1348 t1 = cpu_counter(); 1349 freq = (t1 - t0 - overhead) * 10; 1350 } 1351 if (ci->ci_data.cpu_cc_freq != 0) { 1352 freq_from_cpuid = cpu_tsc_freq_cpuid(ci); 1353 if ((freq_from_cpuid != 0) 1354 && (freq != freq_from_cpuid)) 1355 aprint_verbose_dev(ci->ci_dev, "TSC freq " 1356 "calibrated %" PRIu64 " Hz\n", freq); 1357 } 1358 } else { 1359 freq = cpu_info_primary.ci_data.cpu_cc_freq; 1360 } 1361 1362 ci->ci_data.cpu_cc_freq = freq; 1363 } 1364 1365 void 1366 x86_cpu_idle_mwait(void) 1367 { 1368 struct cpu_info *ci = curcpu(); 1369 1370 KASSERT(ci->ci_ilevel == IPL_NONE); 1371 1372 x86_monitor(&ci->ci_want_resched, 0, 0); 1373 if (__predict_false(ci->ci_want_resched)) { 1374 return; 1375 } 1376 x86_mwait(0, 0); 1377 } 1378 1379 void 1380 x86_cpu_idle_halt(void) 1381 { 1382 struct cpu_info *ci = curcpu(); 1383 1384 KASSERT(ci->ci_ilevel == IPL_NONE); 1385 1386 x86_disable_intr(); 1387 if (!__predict_false(ci->ci_want_resched)) { 1388 x86_stihlt(); 1389 } else { 1390 x86_enable_intr(); 1391 } 1392 } 1393 1394 /* 1395 * Loads pmap for the current CPU. 1396 */ 1397 void 1398 cpu_load_pmap(struct pmap *pmap, struct pmap *oldpmap) 1399 { 1400 #ifdef SVS 1401 if (svs_enabled) { 1402 svs_pdir_switch(pmap); 1403 } 1404 #endif 1405 1406 #ifdef PAE 1407 struct cpu_info *ci = curcpu(); 1408 bool interrupts_enabled; 1409 pd_entry_t *l3_pd = ci->ci_pae_l3_pdir; 1410 int i; 1411 1412 /* 1413 * disable interrupts to block TLB shootdowns, which can reload cr3. 1414 * while this doesn't block NMIs, it's probably ok as NMIs unlikely 1415 * reload cr3. 1416 */ 1417 interrupts_enabled = (x86_read_flags() & PSL_I) != 0; 1418 if (interrupts_enabled) 1419 x86_disable_intr(); 1420 1421 for (i = 0 ; i < PDP_SIZE; i++) { 1422 l3_pd[i] = pmap->pm_pdirpa[i] | PTE_P; 1423 } 1424 1425 if (interrupts_enabled) 1426 x86_enable_intr(); 1427 tlbflush(); 1428 #else 1429 lcr3(pmap_pdirpa(pmap, 0)); 1430 #endif 1431 } 1432 1433 /* 1434 * Notify all other cpus to halt. 1435 */ 1436 1437 void 1438 cpu_broadcast_halt(void) 1439 { 1440 x86_broadcast_ipi(X86_IPI_HALT); 1441 } 1442 1443 /* 1444 * Send a dummy ipi to a cpu to force it to run splraise()/spllower(), 1445 * and trigger an AST on the running LWP. 1446 */ 1447 1448 void 1449 cpu_kick(struct cpu_info *ci) 1450 { 1451 x86_send_ipi(ci, X86_IPI_AST); 1452 } 1453
Indexes created Wed Sep 24 22:09:59 GMT 2025