cpu.c revision 1.186
1 /* $NetBSD: cpu.c,v 1.186 2020/04/23 21:35:18 ad Exp $ */ 2 3 /* 4 * Copyright (c) 2000-2020 NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Bill Sommerfeld of RedBack Networks Inc, and by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * Copyright (c) 1999 Stefan Grefen 34 * 35 * Redistribution and use in source and binary forms, with or without 36 * modification, are permitted provided that the following conditions 37 * are met: 38 * 1. Redistributions of source code must retain the above copyright 39 * notice, this list of conditions and the following disclaimer. 40 * 2. Redistributions in binary form must reproduce the above copyright 41 * notice, this list of conditions and the following disclaimer in the 42 * documentation and/or other materials provided with the distribution. 43 * 3. All advertising materials mentioning features or use of this software 44 * must display the following acknowledgement: 45 * This product includes software developed by the NetBSD 46 * Foundation, Inc. and its contributors. 47 * 4. Neither the name of The NetBSD Foundation nor the names of its 48 * contributors may be used to endorse or promote products derived 49 * from this software without specific prior written permission. 50 * 51 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND ANY 52 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 53 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 54 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR AND CONTRIBUTORS BE LIABLE 55 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 56 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 57 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 58 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 59 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 60 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 61 * SUCH DAMAGE. 62 */ 63 64 #include <sys/cdefs.h> 65 __KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.186 2020/04/23 21:35:18 ad Exp $"); 66 67 #include "opt_ddb.h" 68 #include "opt_mpbios.h" /* for MPDEBUG */ 69 #include "opt_mtrr.h" 70 #include "opt_multiprocessor.h" 71 #include "opt_svs.h" 72 73 #include "lapic.h" 74 #include "ioapic.h" 75 #include "acpica.h" 76 #include "hpet.h" 77 78 #include <sys/param.h> 79 #include <sys/proc.h> 80 #include <sys/systm.h> 81 #include <sys/device.h> 82 #include <sys/cpu.h> 83 #include <sys/cpufreq.h> 84 #include <sys/idle.h> 85 #include <sys/atomic.h> 86 #include <sys/reboot.h> 87 #include <sys/csan.h> 88 89 #include <uvm/uvm.h> 90 91 #include "acpica.h" /* for NACPICA, for mp_verbose */ 92 93 #include <machine/cpufunc.h> 94 #include <machine/cpuvar.h> 95 #include <machine/pmap.h> 96 #include <machine/vmparam.h> 97 #if defined(MULTIPROCESSOR) 98 #include <machine/mpbiosvar.h> 99 #endif 100 #include <machine/mpconfig.h> /* for mp_verbose */ 101 #include <machine/pcb.h> 102 #include <machine/specialreg.h> 103 #include <machine/segments.h> 104 #include <machine/gdt.h> 105 #include <machine/mtrr.h> 106 #include <machine/pio.h> 107 #include <machine/cpu_counter.h> 108 109 #include <x86/fpu.h> 110 111 #if NACPICA > 0 112 #include <dev/acpi/acpi_srat.h> 113 #endif 114 115 #if NLAPIC > 0 116 #include <machine/apicvar.h> 117 #include <machine/i82489reg.h> 118 #include <machine/i82489var.h> 119 #endif 120 121 #include <dev/ic/mc146818reg.h> 122 #include <dev/ic/hpetvar.h> 123 #include <i386/isa/nvram.h> 124 #include <dev/isa/isareg.h> 125 126 #include "tsc.h" 127 128 #ifndef XEN 129 #include "hyperv.h" 130 #if NHYPERV > 0 131 #include <x86/x86/hypervvar.h> 132 #endif 133 #endif 134 135 static int cpu_match(device_t, cfdata_t, void *); 136 static void cpu_attach(device_t, device_t, void *); 137 static void cpu_defer(device_t); 138 static int cpu_rescan(device_t, const char *, const int *); 139 static void cpu_childdetached(device_t, device_t); 140 static bool cpu_stop(device_t); 141 static bool cpu_suspend(device_t, const pmf_qual_t *); 142 static bool cpu_resume(device_t, const pmf_qual_t *); 143 static bool cpu_shutdown(device_t, int); 144 145 struct cpu_softc { 146 device_t sc_dev; /* device tree glue */ 147 struct cpu_info *sc_info; /* pointer to CPU info */ 148 bool sc_wasonline; 149 }; 150 151 #ifdef MULTIPROCESSOR 152 int mp_cpu_start(struct cpu_info *, paddr_t); 153 void mp_cpu_start_cleanup(struct cpu_info *); 154 const struct cpu_functions mp_cpu_funcs = { mp_cpu_start, NULL, 155 mp_cpu_start_cleanup }; 156 #endif 157 158 159 CFATTACH_DECL2_NEW(cpu, sizeof(struct cpu_softc), 160 cpu_match, cpu_attach, NULL, NULL, cpu_rescan, cpu_childdetached); 161 162 /* 163 * Statically-allocated CPU info for the primary CPU (or the only 164 * CPU, on uniprocessors). The CPU info list is initialized to 165 * point at it. 166 */ 167 struct cpu_info cpu_info_primary __aligned(CACHE_LINE_SIZE) = { 168 .ci_dev = 0, 169 .ci_self = &cpu_info_primary, 170 .ci_idepth = -1, 171 .ci_curlwp = &lwp0, 172 .ci_curldt = -1, 173 }; 174 175 struct cpu_info *cpu_info_list = &cpu_info_primary; 176 177 #ifdef i386 178 void cpu_set_tss_gates(struct cpu_info *); 179 #endif 180 181 static void cpu_init_idle_lwp(struct cpu_info *); 182 183 uint32_t cpu_feature[7] __read_mostly; /* X86 CPUID feature bits */ 184 /* [0] basic features cpuid.1:%edx 185 * [1] basic features cpuid.1:%ecx (CPUID2_xxx bits) 186 * [2] extended features cpuid:80000001:%edx 187 * [3] extended features cpuid:80000001:%ecx 188 * [4] VIA padlock features 189 * [5] structured extended features cpuid.7:%ebx 190 * [6] structured extended features cpuid.7:%ecx 191 */ 192 193 #ifdef MULTIPROCESSOR 194 bool x86_mp_online; 195 paddr_t mp_trampoline_paddr = MP_TRAMPOLINE; 196 #endif 197 #if NLAPIC > 0 198 static vaddr_t cmos_data_mapping; 199 #endif 200 struct cpu_info *cpu_starting; 201 202 int (*cpu_nullop_ptr)(void *) = nullop; 203 204 #ifdef MULTIPROCESSOR 205 void cpu_hatch(void *); 206 static void cpu_boot_secondary(struct cpu_info *ci); 207 static void cpu_start_secondary(struct cpu_info *ci); 208 #if NLAPIC > 0 209 static void cpu_copy_trampoline(paddr_t); 210 #endif 211 #endif /* MULTIPROCESSOR */ 212 213 /* 214 * Runs once per boot once multiprocessor goo has been detected and 215 * the local APIC on the boot processor has been mapped. 216 * 217 * Called from lapic_boot_init() (from mpbios_scan()). 218 */ 219 #if NLAPIC > 0 220 void 221 cpu_init_first(void) 222 { 223 224 cpu_info_primary.ci_cpuid = lapic_cpu_number(); 225 226 cmos_data_mapping = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, UVM_KMF_VAONLY); 227 if (cmos_data_mapping == 0) 228 panic("No KVA for page 0"); 229 pmap_kenter_pa(cmos_data_mapping, 0, VM_PROT_READ|VM_PROT_WRITE, 0); 230 pmap_update(pmap_kernel()); 231 } 232 #endif 233 234 static int 235 cpu_match(device_t parent, cfdata_t match, void *aux) 236 { 237 238 return 1; 239 } 240 241 #ifdef __HAVE_PCPU_AREA 242 void 243 cpu_pcpuarea_init(struct cpu_info *ci) 244 { 245 struct vm_page *pg; 246 size_t i, npages; 247 vaddr_t base, va; 248 paddr_t pa; 249 250 CTASSERT(sizeof(struct pcpu_entry) % PAGE_SIZE == 0); 251 252 npages = sizeof(struct pcpu_entry) / PAGE_SIZE; 253 base = (vaddr_t)&pcpuarea->ent[cpu_index(ci)]; 254 255 for (i = 0; i < npages; i++) { 256 pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO); 257 if (pg == NULL) { 258 panic("failed to allocate pcpu PA"); 259 } 260 261 va = base + i * PAGE_SIZE; 262 pa = VM_PAGE_TO_PHYS(pg); 263 264 pmap_kenter_pa(va, pa, VM_PROT_READ|VM_PROT_WRITE, 0); 265 } 266 267 pmap_update(pmap_kernel()); 268 } 269 #endif 270 271 static void 272 cpu_vm_init(struct cpu_info *ci) 273 { 274 int ncolors = 2, i; 275 276 for (i = CAI_ICACHE; i <= CAI_L2CACHE; i++) { 277 struct x86_cache_info *cai; 278 int tcolors; 279 280 cai = &ci->ci_cinfo[i]; 281 282 tcolors = atop(cai->cai_totalsize); 283 switch (cai->cai_associativity) { 284 case 0xff: 285 tcolors = 1; /* fully associative */ 286 break; 287 case 0: 288 case 1: 289 break; 290 default: 291 tcolors /= cai->cai_associativity; 292 } 293 ncolors = uimax(ncolors, tcolors); 294 /* 295 * If the desired number of colors is not a power of 296 * two, it won't be good. Find the greatest power of 297 * two which is an even divisor of the number of colors, 298 * to preserve even coloring of pages. 299 */ 300 if (ncolors & (ncolors - 1) ) { 301 int try, picked = 1; 302 for (try = 1; try < ncolors; try *= 2) { 303 if (ncolors % try == 0) picked = try; 304 } 305 if (picked == 1) { 306 panic("desired number of cache colors %d is " 307 " > 1, but not even!", ncolors); 308 } 309 ncolors = picked; 310 } 311 } 312 313 /* 314 * Knowing the size of the largest cache on this CPU, potentially 315 * re-color our pages. 316 */ 317 aprint_debug_dev(ci->ci_dev, "%d page colors\n", ncolors); 318 uvm_page_recolor(ncolors); 319 320 pmap_tlb_cpu_init(ci); 321 #ifndef __HAVE_DIRECT_MAP 322 pmap_vpage_cpu_init(ci); 323 #endif 324 } 325 326 static void 327 cpu_attach(device_t parent, device_t self, void *aux) 328 { 329 struct cpu_softc *sc = device_private(self); 330 struct cpu_attach_args *caa = aux; 331 struct cpu_info *ci; 332 uintptr_t ptr; 333 #if NLAPIC > 0 334 int cpunum = caa->cpu_number; 335 #endif 336 static bool again; 337 338 sc->sc_dev = self; 339 340 if (ncpu > maxcpus) { 341 #ifndef _LP64 342 aprint_error(": too many CPUs, please use NetBSD/amd64\n"); 343 #else 344 aprint_error(": too many CPUs\n"); 345 #endif 346 return; 347 } 348 349 /* 350 * If we're an Application Processor, allocate a cpu_info 351 * structure, otherwise use the primary's. 352 */ 353 if (caa->cpu_role == CPU_ROLE_AP) { 354 if ((boothowto & RB_MD1) != 0) { 355 aprint_error(": multiprocessor boot disabled\n"); 356 if (!pmf_device_register(self, NULL, NULL)) 357 aprint_error_dev(self, 358 "couldn't establish power handler\n"); 359 return; 360 } 361 aprint_naive(": Application Processor\n"); 362 ptr = (uintptr_t)uvm_km_alloc(kernel_map, 363 sizeof(*ci) + CACHE_LINE_SIZE - 1, 0, 364 UVM_KMF_WIRED|UVM_KMF_ZERO); 365 ci = (struct cpu_info *)roundup2(ptr, CACHE_LINE_SIZE); 366 ci->ci_curldt = -1; 367 } else { 368 aprint_naive(": %s Processor\n", 369 caa->cpu_role == CPU_ROLE_SP ? "Single" : "Boot"); 370 ci = &cpu_info_primary; 371 #if NLAPIC > 0 372 if (cpunum != lapic_cpu_number()) { 373 /* XXX should be done earlier. */ 374 uint32_t reg; 375 aprint_verbose("\n"); 376 aprint_verbose_dev(self, "running CPU at apic %d" 377 " instead of at expected %d", lapic_cpu_number(), 378 cpunum); 379 reg = lapic_readreg(LAPIC_ID); 380 lapic_writereg(LAPIC_ID, (reg & ~LAPIC_ID_MASK) | 381 (cpunum << LAPIC_ID_SHIFT)); 382 } 383 if (cpunum != lapic_cpu_number()) { 384 aprint_error_dev(self, "unable to reset apic id\n"); 385 } 386 #endif 387 } 388 389 ci->ci_self = ci; 390 sc->sc_info = ci; 391 ci->ci_dev = self; 392 ci->ci_acpiid = caa->cpu_id; 393 ci->ci_cpuid = caa->cpu_number; 394 ci->ci_func = caa->cpu_func; 395 ci->ci_kfpu_spl = -1; 396 aprint_normal("\n"); 397 398 /* Must be before mi_cpu_attach(). */ 399 cpu_vm_init(ci); 400 401 if (caa->cpu_role == CPU_ROLE_AP) { 402 int error; 403 404 error = mi_cpu_attach(ci); 405 if (error != 0) { 406 aprint_error_dev(self, 407 "mi_cpu_attach failed with %d\n", error); 408 return; 409 } 410 #ifdef __HAVE_PCPU_AREA 411 cpu_pcpuarea_init(ci); 412 #endif 413 cpu_init_tss(ci); 414 } else { 415 KASSERT(ci->ci_data.cpu_idlelwp != NULL); 416 #if NACPICA > 0 417 /* Parse out NUMA info for cpu_identify(). */ 418 acpisrat_init(); 419 #endif 420 } 421 422 #ifdef SVS 423 cpu_svs_init(ci); 424 #endif 425 426 pmap_reference(pmap_kernel()); 427 ci->ci_pmap = pmap_kernel(); 428 ci->ci_tlbstate = TLBSTATE_STALE; 429 430 /* 431 * Boot processor may not be attached first, but the below 432 * must be done to allow booting other processors. 433 */ 434 if (!again) { 435 /* Make sure DELAY() (likely i8254_delay()) is initialized. */ 436 DELAY(1); 437 438 /* Basic init. */ 439 atomic_or_32(&ci->ci_flags, CPUF_PRESENT | CPUF_PRIMARY); 440 cpu_intr_init(ci); 441 cpu_get_tsc_freq(ci); 442 cpu_init(ci); 443 #ifdef i386 444 cpu_set_tss_gates(ci); 445 #endif 446 pmap_cpu_init_late(ci); 447 #if NLAPIC > 0 448 if (caa->cpu_role != CPU_ROLE_SP) { 449 /* Enable lapic. */ 450 lapic_enable(); 451 lapic_set_lvt(); 452 lapic_calibrate_timer(ci); 453 } 454 #endif 455 kcsan_cpu_init(ci); 456 again = true; 457 } 458 459 /* further PCB init done later. */ 460 461 switch (caa->cpu_role) { 462 case CPU_ROLE_SP: 463 atomic_or_32(&ci->ci_flags, CPUF_SP); 464 cpu_identify(ci); 465 x86_errata(); 466 x86_cpu_idle_init(); 467 break; 468 469 case CPU_ROLE_BP: 470 atomic_or_32(&ci->ci_flags, CPUF_BSP); 471 cpu_identify(ci); 472 x86_errata(); 473 x86_cpu_idle_init(); 474 break; 475 476 #ifdef MULTIPROCESSOR 477 case CPU_ROLE_AP: 478 /* 479 * report on an AP 480 */ 481 cpu_intr_init(ci); 482 gdt_alloc_cpu(ci); 483 #ifdef i386 484 cpu_set_tss_gates(ci); 485 #endif 486 pmap_cpu_init_late(ci); 487 cpu_start_secondary(ci); 488 if (ci->ci_flags & CPUF_PRESENT) { 489 struct cpu_info *tmp; 490 491 cpu_identify(ci); 492 tmp = cpu_info_list; 493 while (tmp->ci_next) 494 tmp = tmp->ci_next; 495 496 tmp->ci_next = ci; 497 } 498 break; 499 #endif 500 501 default: 502 panic("unknown processor type??\n"); 503 } 504 505 pat_init(ci); 506 507 if (!pmf_device_register1(self, cpu_suspend, cpu_resume, cpu_shutdown)) 508 aprint_error_dev(self, "couldn't establish power handler\n"); 509 510 #ifdef MULTIPROCESSOR 511 if (mp_verbose) { 512 struct lwp *l = ci->ci_data.cpu_idlelwp; 513 struct pcb *pcb = lwp_getpcb(l); 514 515 aprint_verbose_dev(self, 516 "idle lwp at %p, idle sp at %p\n", 517 l, 518 #ifdef i386 519 (void *)pcb->pcb_esp 520 #else 521 (void *)pcb->pcb_rsp 522 #endif 523 ); 524 } 525 #endif 526 527 /* 528 * Postpone the "cpufeaturebus" scan. 529 * It is safe to scan the pseudo-bus 530 * only after all CPUs have attached. 531 */ 532 (void)config_defer(self, cpu_defer); 533 } 534 535 static void 536 cpu_defer(device_t self) 537 { 538 cpu_rescan(self, NULL, NULL); 539 } 540 541 static int 542 cpu_rescan(device_t self, const char *ifattr, const int *locators) 543 { 544 struct cpu_softc *sc = device_private(self); 545 struct cpufeature_attach_args cfaa; 546 struct cpu_info *ci = sc->sc_info; 547 548 /* 549 * If we booted with RB_MD1 to disable multiprocessor, the 550 * auto-configuration data still contains the additional 551 * CPUs. But their initialization was mostly bypassed 552 * during attach, so we have to make sure we don't look at 553 * their featurebus info, since it wasn't retrieved. 554 */ 555 if (ci == NULL) 556 return 0; 557 558 memset(&cfaa, 0, sizeof(cfaa)); 559 cfaa.ci = ci; 560 561 if (ifattr_match(ifattr, "cpufeaturebus")) { 562 if (ci->ci_frequency == NULL) { 563 cfaa.name = "frequency"; 564 ci->ci_frequency = config_found_ia(self, 565 "cpufeaturebus", &cfaa, NULL); 566 } 567 568 if (ci->ci_padlock == NULL) { 569 cfaa.name = "padlock"; 570 ci->ci_padlock = config_found_ia(self, 571 "cpufeaturebus", &cfaa, NULL); 572 } 573 574 if (ci->ci_temperature == NULL) { 575 cfaa.name = "temperature"; 576 ci->ci_temperature = config_found_ia(self, 577 "cpufeaturebus", &cfaa, NULL); 578 } 579 580 if (ci->ci_vm == NULL) { 581 cfaa.name = "vm"; 582 ci->ci_vm = config_found_ia(self, 583 "cpufeaturebus", &cfaa, NULL); 584 } 585 } 586 587 return 0; 588 } 589 590 static void 591 cpu_childdetached(device_t self, device_t child) 592 { 593 struct cpu_softc *sc = device_private(self); 594 struct cpu_info *ci = sc->sc_info; 595 596 if (ci->ci_frequency == child) 597 ci->ci_frequency = NULL; 598 599 if (ci->ci_padlock == child) 600 ci->ci_padlock = NULL; 601 602 if (ci->ci_temperature == child) 603 ci->ci_temperature = NULL; 604 605 if (ci->ci_vm == child) 606 ci->ci_vm = NULL; 607 } 608 609 /* 610 * Initialize the processor appropriately. 611 */ 612 613 void 614 cpu_init(struct cpu_info *ci) 615 { 616 extern int x86_fpu_save; 617 uint32_t cr4 = 0; 618 619 lcr0(rcr0() | CR0_WP); 620 621 /* If global TLB caching is supported, enable it */ 622 if (cpu_feature[0] & CPUID_PGE) 623 cr4 |= CR4_PGE; 624 625 /* 626 * If we have FXSAVE/FXRESTOR, use them. 627 */ 628 if (cpu_feature[0] & CPUID_FXSR) { 629 cr4 |= CR4_OSFXSR; 630 631 /* 632 * If we have SSE/SSE2, enable XMM exceptions. 633 */ 634 if (cpu_feature[0] & (CPUID_SSE|CPUID_SSE2)) 635 cr4 |= CR4_OSXMMEXCPT; 636 } 637 638 /* If xsave is supported, enable it */ 639 if (cpu_feature[1] & CPUID2_XSAVE) 640 cr4 |= CR4_OSXSAVE; 641 642 /* If SMEP is supported, enable it */ 643 if (cpu_feature[5] & CPUID_SEF_SMEP) 644 cr4 |= CR4_SMEP; 645 646 /* If SMAP is supported, enable it */ 647 if (cpu_feature[5] & CPUID_SEF_SMAP) 648 cr4 |= CR4_SMAP; 649 650 #ifdef SVS 651 /* If PCID is supported, enable it */ 652 if (svs_pcid) 653 cr4 |= CR4_PCIDE; 654 #endif 655 656 if (cr4) { 657 cr4 |= rcr4(); 658 lcr4(cr4); 659 } 660 661 /* 662 * Changing CR4 register may change cpuid values. For example, setting 663 * CR4_OSXSAVE sets CPUID2_OSXSAVE. The CPUID2_OSXSAVE is in 664 * ci_feat_val[1], so update it. 665 * XXX Other than ci_feat_val[1] might be changed. 666 */ 667 if (cpuid_level >= 1) { 668 u_int descs[4]; 669 670 x86_cpuid(1, descs); 671 ci->ci_feat_val[1] = descs[2]; 672 } 673 674 if (x86_fpu_save >= FPU_SAVE_FXSAVE) { 675 fpuinit_mxcsr_mask(); 676 } 677 678 /* If xsave is enabled, enable all fpu features */ 679 if (cr4 & CR4_OSXSAVE) 680 wrxcr(0, x86_xsave_features & XCR0_FPU); 681 682 #ifdef MTRR 683 /* 684 * On a P6 or above, initialize MTRR's if the hardware supports them. 685 */ 686 if (cpu_feature[0] & CPUID_MTRR) { 687 if ((ci->ci_flags & CPUF_AP) == 0) 688 i686_mtrr_init_first(); 689 mtrr_init_cpu(ci); 690 } 691 692 #ifdef i386 693 if (strcmp((char *)(ci->ci_vendor), "AuthenticAMD") == 0) { 694 /* 695 * Must be a K6-2 Step >= 7 or a K6-III. 696 */ 697 if (CPUID_TO_FAMILY(ci->ci_signature) == 5) { 698 if (CPUID_TO_MODEL(ci->ci_signature) > 8 || 699 (CPUID_TO_MODEL(ci->ci_signature) == 8 && 700 CPUID_TO_STEPPING(ci->ci_signature) >= 7)) { 701 mtrr_funcs = &k6_mtrr_funcs; 702 k6_mtrr_init_first(); 703 mtrr_init_cpu(ci); 704 } 705 } 706 } 707 #endif /* i386 */ 708 #endif /* MTRR */ 709 710 if (ci != &cpu_info_primary) { 711 /* Synchronize TSC */ 712 atomic_or_32(&ci->ci_flags, CPUF_RUNNING); 713 tsc_sync_ap(ci); 714 } else { 715 atomic_or_32(&ci->ci_flags, CPUF_RUNNING); 716 } 717 } 718 719 #ifdef MULTIPROCESSOR 720 void 721 cpu_boot_secondary_processors(void) 722 { 723 struct cpu_info *ci; 724 kcpuset_t *cpus; 725 u_long i; 726 727 #if NHPET > 0 728 /* Use HPET delay, and re-calibrate TSC on boot CPU using HPET. */ 729 if (hpet_delay_p() && x86_delay == i8254_delay) { 730 delay_func = x86_delay = hpet_delay; 731 cpu_get_tsc_freq(curcpu()); 732 } 733 #endif 734 735 #ifndef XEN 736 /* Now that we know the number of CPUs, patch the text segment. */ 737 x86_patch(false); 738 #endif 739 740 #if NACPICA > 0 741 /* Finished with NUMA info for now. */ 742 acpisrat_exit(); 743 #endif 744 745 kcpuset_create(&cpus, true); 746 kcpuset_set(cpus, cpu_index(curcpu())); 747 for (i = 0; i < maxcpus; i++) { 748 ci = cpu_lookup(i); 749 if (ci == NULL) 750 continue; 751 if (ci->ci_data.cpu_idlelwp == NULL) 752 continue; 753 if ((ci->ci_flags & CPUF_PRESENT) == 0) 754 continue; 755 if (ci->ci_flags & (CPUF_BSP|CPUF_SP|CPUF_PRIMARY)) 756 continue; 757 cpu_boot_secondary(ci); 758 kcpuset_set(cpus, cpu_index(ci)); 759 } 760 while (!kcpuset_match(cpus, kcpuset_running)) 761 ; 762 kcpuset_destroy(cpus); 763 764 x86_mp_online = true; 765 766 /* Now that we know about the TSC, attach the timecounter. */ 767 tsc_tc_init(); 768 769 /* Enable zeroing of pages in the idle loop if we have SSE2. */ 770 vm_page_zero_enable = false; /* ((cpu_feature[0] & CPUID_SSE2) != 0); */ 771 } 772 #endif 773 774 static void 775 cpu_init_idle_lwp(struct cpu_info *ci) 776 { 777 struct lwp *l = ci->ci_data.cpu_idlelwp; 778 struct pcb *pcb = lwp_getpcb(l); 779 780 pcb->pcb_cr0 = rcr0(); 781 } 782 783 void 784 cpu_init_idle_lwps(void) 785 { 786 struct cpu_info *ci; 787 u_long i; 788 789 for (i = 0; i < maxcpus; i++) { 790 ci = cpu_lookup(i); 791 if (ci == NULL) 792 continue; 793 if (ci->ci_data.cpu_idlelwp == NULL) 794 continue; 795 if ((ci->ci_flags & CPUF_PRESENT) == 0) 796 continue; 797 cpu_init_idle_lwp(ci); 798 } 799 } 800 801 #ifdef MULTIPROCESSOR 802 void 803 cpu_start_secondary(struct cpu_info *ci) 804 { 805 u_long psl; 806 int i; 807 808 #if NLAPIC > 0 809 paddr_t mp_pdirpa; 810 mp_pdirpa = pmap_init_tmp_pgtbl(mp_trampoline_paddr); 811 cpu_copy_trampoline(mp_pdirpa); 812 #endif 813 814 atomic_or_32(&ci->ci_flags, CPUF_AP); 815 ci->ci_curlwp = ci->ci_data.cpu_idlelwp; 816 if (CPU_STARTUP(ci, mp_trampoline_paddr) != 0) { 817 return; 818 } 819 820 /* 821 * Wait for it to become ready. Setting cpu_starting opens the 822 * initial gate and allows the AP to start soft initialization. 823 */ 824 KASSERT(cpu_starting == NULL); 825 cpu_starting = ci; 826 for (i = 100000; (!(ci->ci_flags & CPUF_PRESENT)) && i > 0; i--) { 827 x86_delay(10); 828 } 829 830 if ((ci->ci_flags & CPUF_PRESENT) == 0) { 831 aprint_error_dev(ci->ci_dev, "failed to become ready\n"); 832 #if defined(MPDEBUG) && defined(DDB) 833 printf("dropping into debugger; continue from here to resume boot\n"); 834 Debugger(); 835 #endif 836 } else { 837 /* 838 * Synchronize time stamp counters. Invalidate cache and do 839 * twice (in tsc_sync_bp) to minimize possible cache effects. 840 * Disable interrupts to try and rule out any external 841 * interference. 842 */ 843 psl = x86_read_psl(); 844 x86_disable_intr(); 845 tsc_sync_bp(ci); 846 x86_write_psl(psl); 847 } 848 849 CPU_START_CLEANUP(ci); 850 cpu_starting = NULL; 851 } 852 853 void 854 cpu_boot_secondary(struct cpu_info *ci) 855 { 856 int64_t drift; 857 u_long psl; 858 int i; 859 860 atomic_or_32(&ci->ci_flags, CPUF_GO); 861 for (i = 100000; (!(ci->ci_flags & CPUF_RUNNING)) && i > 0; i--) { 862 x86_delay(10); 863 } 864 if ((ci->ci_flags & CPUF_RUNNING) == 0) { 865 aprint_error_dev(ci->ci_dev, "failed to start\n"); 866 #if defined(MPDEBUG) && defined(DDB) 867 printf("dropping into debugger; continue from here to resume boot\n"); 868 Debugger(); 869 #endif 870 } else { 871 /* Synchronize TSC again, check for drift. */ 872 drift = ci->ci_data.cpu_cc_skew; 873 psl = x86_read_psl(); 874 x86_disable_intr(); 875 tsc_sync_bp(ci); 876 x86_write_psl(psl); 877 drift -= ci->ci_data.cpu_cc_skew; 878 aprint_debug_dev(ci->ci_dev, "TSC skew=%lld drift=%lld\n", 879 (long long)ci->ci_data.cpu_cc_skew, (long long)drift); 880 tsc_sync_drift(drift); 881 } 882 } 883 884 /* 885 * The CPU ends up here when it's ready to run. 886 * This is called from code in mptramp.s; at this point, we are running 887 * in the idle pcb/idle stack of the new CPU. When this function returns, 888 * this processor will enter the idle loop and start looking for work. 889 */ 890 void 891 cpu_hatch(void *v) 892 { 893 struct cpu_info *ci = (struct cpu_info *)v; 894 struct pcb *pcb; 895 int s, i; 896 897 /* ------------------------------------------------------------- */ 898 899 /* 900 * This section of code must be compiled with SSP disabled, to 901 * prevent a race against cpu0. See sys/conf/ssp.mk. 902 */ 903 904 cpu_init_msrs(ci, true); 905 cpu_probe(ci); 906 cpu_speculation_init(ci); 907 #if NHYPERV > 0 908 hyperv_init_cpu(ci); 909 #endif 910 911 ci->ci_data.cpu_cc_freq = cpu_info_primary.ci_data.cpu_cc_freq; 912 /* cpu_get_tsc_freq(ci); */ 913 914 KDASSERT((ci->ci_flags & CPUF_PRESENT) == 0); 915 916 /* 917 * Synchronize the TSC for the first time. Note that interrupts are 918 * off at this point. 919 */ 920 atomic_or_32(&ci->ci_flags, CPUF_PRESENT); 921 tsc_sync_ap(ci); 922 923 /* ------------------------------------------------------------- */ 924 925 /* 926 * Wait to be brought online. 927 * 928 * Use MONITOR/MWAIT if available. These instructions put the CPU in 929 * a low consumption mode (C-state), and if the TSC is not invariant, 930 * this causes the TSC to drift. We want this to happen, so that we 931 * can later detect (in tsc_tc_init) any abnormal drift with invariant 932 * TSCs. That's just for safety; by definition such drifts should 933 * never occur with invariant TSCs. 934 * 935 * If not available, try PAUSE. We'd like to use HLT, but we have 936 * interrupts off. 937 */ 938 while ((ci->ci_flags & CPUF_GO) == 0) { 939 if ((cpu_feature[1] & CPUID2_MONITOR) != 0) { 940 x86_monitor(&ci->ci_flags, 0, 0); 941 if ((ci->ci_flags & CPUF_GO) != 0) { 942 continue; 943 } 944 x86_mwait(0, 0); 945 } else { 946 /* 947 * XXX The loop repetition count could be a lot higher, but 948 * XXX currently qemu emulator takes a _very_long_time_ to 949 * XXX execute the pause instruction. So for now, use a low 950 * XXX value to allow the cpu to hatch before timing out. 951 */ 952 for (i = 50; i != 0; i--) { 953 x86_pause(); 954 } 955 } 956 } 957 958 /* Because the text may have been patched in x86_patch(). */ 959 wbinvd(); 960 x86_flush(); 961 tlbflushg(); 962 963 KASSERT((ci->ci_flags & CPUF_RUNNING) == 0); 964 965 #ifdef PAE 966 pd_entry_t * l3_pd = ci->ci_pae_l3_pdir; 967 for (i = 0 ; i < PDP_SIZE; i++) { 968 l3_pd[i] = pmap_kernel()->pm_pdirpa[i] | PTE_P; 969 } 970 lcr3(ci->ci_pae_l3_pdirpa); 971 #else 972 lcr3(pmap_pdirpa(pmap_kernel(), 0)); 973 #endif 974 975 pcb = lwp_getpcb(curlwp); 976 pcb->pcb_cr3 = rcr3(); 977 pcb = lwp_getpcb(ci->ci_data.cpu_idlelwp); 978 lcr0(pcb->pcb_cr0); 979 980 cpu_init_idt(); 981 gdt_init_cpu(ci); 982 #if NLAPIC > 0 983 lapic_enable(); 984 lapic_set_lvt(); 985 lapic_initclocks(); 986 #endif 987 988 fpuinit(ci); 989 lldt(GSYSSEL(GLDT_SEL, SEL_KPL)); 990 ltr(ci->ci_tss_sel); 991 992 /* 993 * cpu_init will re-synchronize the TSC, and will detect any abnormal 994 * drift that would have been caused by the use of MONITOR/MWAIT 995 * above. 996 */ 997 cpu_init(ci); 998 cpu_get_tsc_freq(ci); 999 1000 s = splhigh(); 1001 #if NLAPIC > 0 1002 lapic_write_tpri(0); 1003 #endif 1004 x86_enable_intr(); 1005 splx(s); 1006 x86_errata(); 1007 1008 aprint_debug_dev(ci->ci_dev, "running\n"); 1009 1010 kcsan_cpu_init(ci); 1011 1012 idle_loop(NULL); 1013 KASSERT(false); 1014 } 1015 #endif 1016 1017 #if defined(DDB) 1018 1019 #include <ddb/db_output.h> 1020 #include <machine/db_machdep.h> 1021 1022 /* 1023 * Dump CPU information from ddb. 1024 */ 1025 void 1026 cpu_debug_dump(void) 1027 { 1028 struct cpu_info *ci; 1029 CPU_INFO_ITERATOR cii; 1030 const char sixtyfour64space[] = 1031 #ifdef _LP64 1032 " " 1033 #endif 1034 ""; 1035 1036 db_printf("addr %sdev id flags ipis spl curlwp " 1037 "\n", sixtyfour64space); 1038 for (CPU_INFO_FOREACH(cii, ci)) { 1039 db_printf("%p %s %ld %x %x %d %10p\n", 1040 ci, 1041 ci->ci_dev == NULL ? "BOOT" : device_xname(ci->ci_dev), 1042 (long)ci->ci_cpuid, 1043 ci->ci_flags, ci->ci_ipis, ci->ci_ilevel, 1044 ci->ci_curlwp); 1045 } 1046 } 1047 #endif 1048 1049 #ifdef MULTIPROCESSOR 1050 #if NLAPIC > 0 1051 static void 1052 cpu_copy_trampoline(paddr_t pdir_pa) 1053 { 1054 extern uint32_t nox_flag; 1055 extern u_char cpu_spinup_trampoline[]; 1056 extern u_char cpu_spinup_trampoline_end[]; 1057 vaddr_t mp_trampoline_vaddr; 1058 struct { 1059 uint32_t large; 1060 uint32_t nox; 1061 uint32_t pdir; 1062 } smp_data; 1063 CTASSERT(sizeof(smp_data) == 3 * 4); 1064 1065 smp_data.large = (pmap_largepages != 0); 1066 smp_data.nox = nox_flag; 1067 smp_data.pdir = (uint32_t)(pdir_pa & 0xFFFFFFFF); 1068 1069 /* Enter the physical address */ 1070 mp_trampoline_vaddr = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, 1071 UVM_KMF_VAONLY); 1072 pmap_kenter_pa(mp_trampoline_vaddr, mp_trampoline_paddr, 1073 VM_PROT_READ | VM_PROT_WRITE, 0); 1074 pmap_update(pmap_kernel()); 1075 1076 /* Copy boot code */ 1077 memcpy((void *)mp_trampoline_vaddr, 1078 cpu_spinup_trampoline, 1079 cpu_spinup_trampoline_end - cpu_spinup_trampoline); 1080 1081 /* Copy smp_data at the end */ 1082 memcpy((void *)(mp_trampoline_vaddr + PAGE_SIZE - sizeof(smp_data)), 1083 &smp_data, sizeof(smp_data)); 1084 1085 pmap_kremove(mp_trampoline_vaddr, PAGE_SIZE); 1086 pmap_update(pmap_kernel()); 1087 uvm_km_free(kernel_map, mp_trampoline_vaddr, PAGE_SIZE, UVM_KMF_VAONLY); 1088 } 1089 #endif 1090 1091 int 1092 mp_cpu_start(struct cpu_info *ci, paddr_t target) 1093 { 1094 int error; 1095 1096 /* 1097 * Bootstrap code must be addressable in real mode 1098 * and it must be page aligned. 1099 */ 1100 KASSERT(target < 0x10000 && target % PAGE_SIZE == 0); 1101 1102 /* 1103 * "The BSP must initialize CMOS shutdown code to 0Ah ..." 1104 */ 1105 1106 outb(IO_RTC, NVRAM_RESET); 1107 outb(IO_RTC+1, NVRAM_RESET_JUMP); 1108 1109 #if NLAPIC > 0 1110 /* 1111 * "and the warm reset vector (DWORD based at 40:67) to point 1112 * to the AP startup code ..." 1113 */ 1114 unsigned short dwordptr[2]; 1115 dwordptr[0] = 0; 1116 dwordptr[1] = target >> 4; 1117 1118 memcpy((uint8_t *)cmos_data_mapping + 0x467, dwordptr, 4); 1119 #endif 1120 1121 if ((cpu_feature[0] & CPUID_APIC) == 0) { 1122 aprint_error("mp_cpu_start: CPU does not have APIC\n"); 1123 return ENODEV; 1124 } 1125 1126 /* 1127 * ... prior to executing the following sequence:". We'll also add in 1128 * local cache flush, in case the BIOS has left the AP with its cache 1129 * disabled. It may not be able to cope with MP coherency. 1130 */ 1131 wbinvd(); 1132 1133 if (ci->ci_flags & CPUF_AP) { 1134 error = x86_ipi_init(ci->ci_cpuid); 1135 if (error != 0) { 1136 aprint_error_dev(ci->ci_dev, "%s: IPI not taken (1)\n", 1137 __func__); 1138 return error; 1139 } 1140 x86_delay(10000); 1141 1142 error = x86_ipi_startup(ci->ci_cpuid, target / PAGE_SIZE); 1143 if (error != 0) { 1144 aprint_error_dev(ci->ci_dev, "%s: IPI not taken (2)\n", 1145 __func__); 1146 return error; 1147 } 1148 x86_delay(200); 1149 1150 error = x86_ipi_startup(ci->ci_cpuid, target / PAGE_SIZE); 1151 if (error != 0) { 1152 aprint_error_dev(ci->ci_dev, "%s: IPI not taken (3)\n", 1153 __func__); 1154 return error; 1155 } 1156 x86_delay(200); 1157 } 1158 1159 return 0; 1160 } 1161 1162 void 1163 mp_cpu_start_cleanup(struct cpu_info *ci) 1164 { 1165 /* 1166 * Ensure the NVRAM reset byte contains something vaguely sane. 1167 */ 1168 1169 outb(IO_RTC, NVRAM_RESET); 1170 outb(IO_RTC+1, NVRAM_RESET_RST); 1171 } 1172 #endif 1173 1174 #ifdef __x86_64__ 1175 typedef void (vector)(void); 1176 extern vector Xsyscall, Xsyscall32, Xsyscall_svs; 1177 #endif 1178 1179 void 1180 cpu_init_msrs(struct cpu_info *ci, bool full) 1181 { 1182 #ifdef __x86_64__ 1183 wrmsr(MSR_STAR, 1184 ((uint64_t)GSEL(GCODE_SEL, SEL_KPL) << 32) | 1185 ((uint64_t)LSEL(LSYSRETBASE_SEL, SEL_UPL) << 48)); 1186 wrmsr(MSR_LSTAR, (uint64_t)Xsyscall); 1187 wrmsr(MSR_CSTAR, (uint64_t)Xsyscall32); 1188 wrmsr(MSR_SFMASK, PSL_NT|PSL_T|PSL_I|PSL_C|PSL_D|PSL_AC); 1189 1190 #ifdef SVS 1191 if (svs_enabled) 1192 wrmsr(MSR_LSTAR, (uint64_t)Xsyscall_svs); 1193 #endif 1194 1195 if (full) { 1196 wrmsr(MSR_FSBASE, 0); 1197 wrmsr(MSR_GSBASE, (uint64_t)ci); 1198 wrmsr(MSR_KERNELGSBASE, 0); 1199 } 1200 #endif /* __x86_64__ */ 1201 1202 if (cpu_feature[2] & CPUID_NOX) 1203 wrmsr(MSR_EFER, rdmsr(MSR_EFER) | EFER_NXE); 1204 } 1205 1206 void 1207 cpu_offline_md(void) 1208 { 1209 return; 1210 } 1211 1212 /* XXX joerg restructure and restart CPUs individually */ 1213 static bool 1214 cpu_stop(device_t dv) 1215 { 1216 struct cpu_softc *sc = device_private(dv); 1217 struct cpu_info *ci = sc->sc_info; 1218 int err; 1219 1220 KASSERT((ci->ci_flags & CPUF_PRESENT) != 0); 1221 1222 if ((ci->ci_flags & CPUF_PRIMARY) != 0) 1223 return true; 1224 1225 if (ci->ci_data.cpu_idlelwp == NULL) 1226 return true; 1227 1228 sc->sc_wasonline = !(ci->ci_schedstate.spc_flags & SPCF_OFFLINE); 1229 1230 if (sc->sc_wasonline) { 1231 mutex_enter(&cpu_lock); 1232 err = cpu_setstate(ci, false); 1233 mutex_exit(&cpu_lock); 1234 1235 if (err != 0) 1236 return false; 1237 } 1238 1239 return true; 1240 } 1241 1242 static bool 1243 cpu_suspend(device_t dv, const pmf_qual_t *qual) 1244 { 1245 struct cpu_softc *sc = device_private(dv); 1246 struct cpu_info *ci = sc->sc_info; 1247 1248 if ((ci->ci_flags & CPUF_PRESENT) == 0) 1249 return true; 1250 else { 1251 cpufreq_suspend(ci); 1252 } 1253 1254 return cpu_stop(dv); 1255 } 1256 1257 static bool 1258 cpu_resume(device_t dv, const pmf_qual_t *qual) 1259 { 1260 struct cpu_softc *sc = device_private(dv); 1261 struct cpu_info *ci = sc->sc_info; 1262 int err = 0; 1263 1264 if ((ci->ci_flags & CPUF_PRESENT) == 0) 1265 return true; 1266 1267 if ((ci->ci_flags & CPUF_PRIMARY) != 0) 1268 goto out; 1269 1270 if (ci->ci_data.cpu_idlelwp == NULL) 1271 goto out; 1272 1273 if (sc->sc_wasonline) { 1274 mutex_enter(&cpu_lock); 1275 err = cpu_setstate(ci, true); 1276 mutex_exit(&cpu_lock); 1277 } 1278 1279 out: 1280 if (err != 0) 1281 return false; 1282 1283 cpufreq_resume(ci); 1284 1285 return true; 1286 } 1287 1288 static bool 1289 cpu_shutdown(device_t dv, int how) 1290 { 1291 struct cpu_softc *sc = device_private(dv); 1292 struct cpu_info *ci = sc->sc_info; 1293 1294 if ((ci->ci_flags & CPUF_BSP) != 0) 1295 return false; 1296 1297 if ((ci->ci_flags & CPUF_PRESENT) == 0) 1298 return true; 1299 1300 return cpu_stop(dv); 1301 } 1302 1303 /* Get the TSC frequency and set it to ci->ci_data.cpu_cc_freq. */ 1304 void 1305 cpu_get_tsc_freq(struct cpu_info *ci) 1306 { 1307 uint64_t freq = 0, t0, t1; 1308 int64_t overhead; 1309 1310 if (cpu_hascounter()) 1311 freq = cpu_tsc_freq_cpuid(ci); 1312 1313 if (freq != 0) { 1314 /* Use TSC frequency taken from CPUID. */ 1315 ci->ci_data.cpu_cc_freq = freq; 1316 } else { 1317 /* 1318 * Work out the approximate overhead involved below. 1319 * Discard the result of the first go around the loop. 1320 */ 1321 overhead = 0; 1322 for (int i = 0; i <= 8; i++) { 1323 __insn_barrier(); 1324 t0 = cpu_counter_serializing(); 1325 (*cpu_nullop_ptr)(NULL); 1326 t1 = cpu_counter_serializing(); 1327 __insn_barrier(); 1328 if (i > 0) { 1329 overhead += (t1 - t0); 1330 } 1331 } 1332 overhead >>= 3; 1333 1334 /* Now warm up x86_delay() and do the calibration. */ 1335 x86_delay(1); 1336 __insn_barrier(); 1337 t0 = cpu_counter_serializing(); 1338 x86_delay(100000); 1339 t1 = cpu_counter_serializing(); 1340 __insn_barrier(); 1341 ci->ci_data.cpu_cc_freq = (t1 - t0 - overhead) * 10; 1342 } 1343 } 1344 1345 void 1346 x86_cpu_idle_mwait(void) 1347 { 1348 struct cpu_info *ci = curcpu(); 1349 1350 KASSERT(ci->ci_ilevel == IPL_NONE); 1351 1352 x86_monitor(&ci->ci_want_resched, 0, 0); 1353 if (__predict_false(ci->ci_want_resched)) { 1354 return; 1355 } 1356 x86_mwait(0, 0); 1357 } 1358 1359 void 1360 x86_cpu_idle_halt(void) 1361 { 1362 struct cpu_info *ci = curcpu(); 1363 1364 KASSERT(ci->ci_ilevel == IPL_NONE); 1365 1366 x86_disable_intr(); 1367 if (!__predict_false(ci->ci_want_resched)) { 1368 x86_stihlt(); 1369 } else { 1370 x86_enable_intr(); 1371 } 1372 } 1373 1374 /* 1375 * Loads pmap for the current CPU. 1376 */ 1377 void 1378 cpu_load_pmap(struct pmap *pmap, struct pmap *oldpmap) 1379 { 1380 #ifdef SVS 1381 if (svs_enabled) { 1382 svs_pdir_switch(pmap); 1383 } 1384 #endif 1385 1386 #ifdef PAE 1387 struct cpu_info *ci = curcpu(); 1388 bool interrupts_enabled; 1389 pd_entry_t *l3_pd = ci->ci_pae_l3_pdir; 1390 int i; 1391 1392 /* 1393 * disable interrupts to block TLB shootdowns, which can reload cr3. 1394 * while this doesn't block NMIs, it's probably ok as NMIs unlikely 1395 * reload cr3. 1396 */ 1397 interrupts_enabled = (x86_read_flags() & PSL_I) != 0; 1398 if (interrupts_enabled) 1399 x86_disable_intr(); 1400 1401 for (i = 0 ; i < PDP_SIZE; i++) { 1402 l3_pd[i] = pmap->pm_pdirpa[i] | PTE_P; 1403 } 1404 1405 if (interrupts_enabled) 1406 x86_enable_intr(); 1407 tlbflush(); 1408 #else 1409 lcr3(pmap_pdirpa(pmap, 0)); 1410 #endif 1411 } 1412 1413 /* 1414 * Notify all other cpus to halt. 1415 */ 1416 1417 void 1418 cpu_broadcast_halt(void) 1419 { 1420 x86_broadcast_ipi(X86_IPI_HALT); 1421 } 1422 1423 /* 1424 * Send a dummy ipi to a cpu to force it to run splraise()/spllower(), 1425 * and trigger an AST on the running LWP. 1426 */ 1427 1428 void 1429 cpu_kick(struct cpu_info *ci) 1430 { 1431 x86_send_ipi(ci, X86_IPI_AST); 1432 } 1433
Indexes created Wed Sep 24 15:09:51 GMT 2025