cpu.c revision 1.60
1 /* $NetBSD: cpu.c,v 1.60 2011/07/16 14:46:18 rmind Exp $ */ 2 /* NetBSD: cpu.c,v 1.18 2004/02/20 17:35:01 yamt Exp */ 3 4 /*- 5 * Copyright (c) 2000 The NetBSD Foundation, Inc. 6 * Copyright (c) 2002, 2006, 2007 YAMAMOTO Takashi, 7 * All rights reserved. 8 * 9 * This code is derived from software contributed to The NetBSD Foundation 10 * by RedBack Networks Inc. 11 * 12 * Author: Bill Sommerfeld 13 * 14 * Redistribution and use in source and binary forms, with or without 15 * modification, are permitted provided that the following conditions 16 * are met: 17 * 1. Redistributions of source code must retain the above copyright 18 * notice, this list of conditions and the following disclaimer. 19 * 2. Redistributions in binary form must reproduce the above copyright 20 * notice, this list of conditions and the following disclaimer in the 21 * documentation and/or other materials provided with the distribution. 22 * 23 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 24 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 25 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 26 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 27 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 28 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 29 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 30 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 31 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 32 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 33 * POSSIBILITY OF SUCH DAMAGE. 34 */ 35 36 /* 37 * Copyright (c) 1999 Stefan Grefen 38 * 39 * Redistribution and use in source and binary forms, with or without 40 * modification, are permitted provided that the following conditions 41 * are met: 42 * 1. Redistributions of source code must retain the above copyright 43 * notice, this list of conditions and the following disclaimer. 44 * 2. Redistributions in binary form must reproduce the above copyright 45 * notice, this list of conditions and the following disclaimer in the 46 * documentation and/or other materials provided with the distribution. 47 * 3. All advertising materials mentioning features or use of this software 48 * must display the following acknowledgement: 49 * This product includes software developed by the NetBSD 50 * Foundation, Inc. and its contributors. 51 * 4. Neither the name of The NetBSD Foundation nor the names of its 52 * contributors may be used to endorse or promote products derived 53 * from this software without specific prior written permission. 54 * 55 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND ANY 56 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 57 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 58 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR AND CONTRIBUTORS BE LIABLE 59 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 60 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 61 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 62 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 63 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 64 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 65 * SUCH DAMAGE. 66 */ 67 68 #include <sys/cdefs.h> 69 __KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.60 2011/07/16 14:46:18 rmind Exp $"); 70 71 #include "opt_ddb.h" 72 #include "opt_multiprocessor.h" 73 #include "opt_mpbios.h" /* for MPDEBUG */ 74 #include "opt_mtrr.h" 75 #include "opt_xen.h" 76 77 #include "lapic.h" 78 #include "ioapic.h" 79 80 #include <sys/param.h> 81 #include <sys/proc.h> 82 #include <sys/systm.h> 83 #include <sys/device.h> 84 #include <sys/kmem.h> 85 #include <sys/cpu.h> 86 #include <sys/atomic.h> 87 #include <sys/reboot.h> 88 89 #include <uvm/uvm.h> 90 91 #include <machine/cpufunc.h> 92 #include <machine/cpuvar.h> 93 #include <machine/pmap.h> 94 #include <machine/vmparam.h> 95 #include <machine/mpbiosvar.h> 96 #include <machine/pcb.h> 97 #include <machine/specialreg.h> 98 #include <machine/segments.h> 99 #include <machine/gdt.h> 100 #include <machine/mtrr.h> 101 #include <machine/pio.h> 102 103 #include <xen/vcpuvar.h> 104 105 #if NLAPIC > 0 106 #include <machine/apicvar.h> 107 #include <machine/i82489reg.h> 108 #include <machine/i82489var.h> 109 #endif 110 111 #include <dev/ic/mc146818reg.h> 112 #include <dev/isa/isareg.h> 113 114 #if MAXCPUS > 32 115 #error cpu_info contains 32bit bitmasks 116 #endif 117 118 static int cpu_match(device_t, cfdata_t, void *); 119 static void cpu_attach(device_t, device_t, void *); 120 static void cpu_defer(device_t); 121 static int cpu_rescan(device_t, const char *, const int *); 122 static void cpu_childdetached(device_t, device_t); 123 static int vcpu_match(device_t, cfdata_t, void *); 124 static void vcpu_attach(device_t, device_t, void *); 125 static void cpu_attach_common(device_t, device_t, void *); 126 void cpu_offline_md(void); 127 128 struct cpu_softc { 129 device_t sc_dev; /* device tree glue */ 130 struct cpu_info *sc_info; /* pointer to CPU info */ 131 bool sc_wasonline; 132 }; 133 134 int mp_cpu_start(struct cpu_info *, paddr_t); 135 void mp_cpu_start_cleanup(struct cpu_info *); 136 const struct cpu_functions mp_cpu_funcs = { mp_cpu_start, NULL, 137 mp_cpu_start_cleanup }; 138 139 CFATTACH_DECL2_NEW(cpu, sizeof(struct cpu_softc), 140 cpu_match, cpu_attach, NULL, NULL, cpu_rescan, cpu_childdetached); 141 142 CFATTACH_DECL_NEW(vcpu, sizeof(struct cpu_softc), 143 vcpu_match, vcpu_attach, NULL, NULL); 144 145 /* 146 * Statically-allocated CPU info for the primary CPU (or the only 147 * CPU, on uniprocessors). The CPU info list is initialized to 148 * point at it. 149 */ 150 #ifdef TRAPLOG 151 #include <machine/tlog.h> 152 struct tlog tlog_primary; 153 #endif 154 struct cpu_info cpu_info_primary __aligned(CACHE_LINE_SIZE) = { 155 .ci_dev = 0, 156 .ci_self = &cpu_info_primary, 157 .ci_idepth = -1, 158 .ci_curlwp = &lwp0, 159 .ci_curldt = -1, 160 .ci_cpumask = 1, 161 #ifdef TRAPLOG 162 .ci_tlog = &tlog_primary, 163 #endif 164 165 }; 166 struct cpu_info phycpu_info_primary __aligned(CACHE_LINE_SIZE) = { 167 .ci_dev = 0, 168 .ci_self = &phycpu_info_primary, 169 }; 170 171 struct cpu_info *cpu_info_list = &cpu_info_primary; 172 struct cpu_info *phycpu_info_list = &phycpu_info_primary; 173 174 static void cpu_set_tss_gates(struct cpu_info *ci); 175 176 uint32_t cpus_attached = 1; 177 uint32_t cpus_running = 1; 178 179 uint32_t phycpus_attached = 0; 180 uint32_t phycpus_running = 0; 181 182 uint32_t cpu_feature[5]; /* X86 CPUID feature bits 183 * [0] basic features %edx 184 * [1] basic features %ecx 185 * [2] extended features %edx 186 * [3] extended features %ecx 187 * [4] VIA padlock features 188 */ 189 190 bool x86_mp_online; 191 paddr_t mp_trampoline_paddr = MP_TRAMPOLINE; 192 193 #if defined(MULTIPROCESSOR) 194 void cpu_hatch(void *); 195 static void cpu_boot_secondary(struct cpu_info *ci); 196 static void cpu_start_secondary(struct cpu_info *ci); 197 static void cpu_copy_trampoline(void); 198 199 /* 200 * Runs once per boot once multiprocessor goo has been detected and 201 * the local APIC on the boot processor has been mapped. 202 * 203 * Called from lapic_boot_init() (from mpbios_scan()). 204 */ 205 void 206 cpu_init_first(void) 207 { 208 209 cpu_info_primary.ci_cpuid = lapic_cpu_number(); 210 cpu_copy_trampoline(); 211 } 212 #endif /* MULTIPROCESSOR */ 213 214 static int 215 cpu_match(device_t parent, cfdata_t match, void *aux) 216 { 217 218 return 1; 219 } 220 221 static void 222 cpu_attach(device_t parent, device_t self, void *aux) 223 { 224 struct cpu_softc *sc = device_private(self); 225 struct cpu_attach_args *caa = aux; 226 struct cpu_info *ci; 227 uintptr_t ptr; 228 static int nphycpu = 0; 229 230 sc->sc_dev = self; 231 232 if (phycpus_attached == ~0) { 233 aprint_error(": increase MAXCPUS\n"); 234 return; 235 } 236 237 /* 238 * If we're an Application Processor, allocate a cpu_info 239 * If we're the first attached CPU use the primary cpu_info, 240 * otherwise allocate a new one 241 */ 242 aprint_naive("\n"); 243 aprint_normal("\n"); 244 if (nphycpu > 0) { 245 struct cpu_info *tmp; 246 ptr = (uintptr_t)kmem_zalloc(sizeof(*ci) + CACHE_LINE_SIZE - 1, 247 KM_SLEEP); 248 ci = (struct cpu_info *)roundup2(ptr, CACHE_LINE_SIZE); 249 ci->ci_curldt = -1; 250 251 tmp = phycpu_info_list; 252 while (tmp->ci_next) 253 tmp = tmp->ci_next; 254 255 tmp->ci_next = ci; 256 } else { 257 ci = &phycpu_info_primary; 258 } 259 260 ci->ci_self = ci; 261 sc->sc_info = ci; 262 263 ci->ci_dev = self; 264 ci->ci_acpiid = caa->cpu_id; 265 ci->ci_cpuid = caa->cpu_number; 266 ci->ci_vcpu = NULL; 267 ci->ci_index = nphycpu++; 268 ci->ci_cpumask = (1 << cpu_index(ci)); 269 270 atomic_or_32(&phycpus_attached, ci->ci_cpumask); 271 272 if (!pmf_device_register(self, NULL, NULL)) 273 aprint_error_dev(self, "couldn't establish power handler\n"); 274 275 (void)config_defer(self, cpu_defer); 276 } 277 278 static void 279 cpu_defer(device_t self) 280 { 281 cpu_rescan(self, NULL, NULL); 282 } 283 284 static int 285 cpu_rescan(device_t self, const char *ifattr, const int *locators) 286 { 287 struct cpu_softc *sc = device_private(self); 288 struct cpufeature_attach_args cfaa; 289 struct cpu_info *ci = sc->sc_info; 290 291 memset(&cfaa, 0, sizeof(cfaa)); 292 cfaa.ci = ci; 293 294 if (ifattr_match(ifattr, "cpufeaturebus")) { 295 296 if (ci->ci_frequency == NULL) { 297 cfaa.name = "frequency"; 298 ci->ci_frequency = config_found_ia(self, 299 "cpufeaturebus", &cfaa, NULL); 300 } 301 } 302 303 return 0; 304 } 305 306 static void 307 cpu_childdetached(device_t self, device_t child) 308 { 309 struct cpu_softc *sc = device_private(self); 310 struct cpu_info *ci = sc->sc_info; 311 312 if (ci->ci_frequency == child) 313 ci->ci_frequency = NULL; 314 } 315 316 static int 317 vcpu_match(device_t parent, cfdata_t match, void *aux) 318 { 319 struct vcpu_attach_args *vcaa = aux; 320 321 if (strcmp(vcaa->vcaa_name, match->cf_name) == 0) 322 return 1; 323 return 0; 324 } 325 326 static void 327 vcpu_attach(device_t parent, device_t self, void *aux) 328 { 329 struct vcpu_attach_args *vcaa = aux; 330 331 cpu_attach_common(parent, self, &vcaa->vcaa_caa); 332 } 333 334 static void 335 cpu_vm_init(struct cpu_info *ci) 336 { 337 int ncolors = 2, i; 338 339 for (i = CAI_ICACHE; i <= CAI_L2CACHE; i++) { 340 struct x86_cache_info *cai; 341 int tcolors; 342 343 cai = &ci->ci_cinfo[i]; 344 345 tcolors = atop(cai->cai_totalsize); 346 switch(cai->cai_associativity) { 347 case 0xff: 348 tcolors = 1; /* fully associative */ 349 break; 350 case 0: 351 case 1: 352 break; 353 default: 354 tcolors /= cai->cai_associativity; 355 } 356 ncolors = max(ncolors, tcolors); 357 } 358 359 /* 360 * Knowing the size of the largest cache on this CPU, re-color 361 * our pages. 362 */ 363 if (ncolors <= uvmexp.ncolors) 364 return; 365 aprint_debug_dev(ci->ci_dev, "%d page colors\n", ncolors); 366 uvm_page_recolor(ncolors); 367 } 368 369 static void 370 cpu_attach_common(device_t parent, device_t self, void *aux) 371 { 372 struct cpu_softc *sc = device_private(self); 373 struct cpu_attach_args *caa = aux; 374 struct cpu_info *ci; 375 uintptr_t ptr; 376 int cpunum = caa->cpu_number; 377 static bool again = false; 378 379 sc->sc_dev = self; 380 381 /* 382 * If we're an Application Processor, allocate a cpu_info 383 * structure, otherwise use the primary's. 384 */ 385 if (caa->cpu_role == CPU_ROLE_AP) { 386 aprint_naive(": Application Processor\n"); 387 ptr = (uintptr_t)kmem_alloc(sizeof(*ci) + CACHE_LINE_SIZE - 1, 388 KM_SLEEP); 389 ci = (struct cpu_info *)roundup2(ptr, CACHE_LINE_SIZE); 390 memset(ci, 0, sizeof(*ci)); 391 #ifdef TRAPLOG 392 ci->ci_tlog_base = kmem_zalloc(sizeof(struct tlog), KM_SLEEP); 393 #endif 394 } else { 395 aprint_naive(": %s Processor\n", 396 caa->cpu_role == CPU_ROLE_SP ? "Single" : "Boot"); 397 ci = &cpu_info_primary; 398 #if NLAPIC > 0 399 if (cpunum != lapic_cpu_number()) { 400 /* XXX should be done earlier */ 401 uint32_t reg; 402 aprint_verbose("\n"); 403 aprint_verbose_dev(self, "running CPU at apic %d" 404 " instead of at expected %d", lapic_cpu_number(), 405 cpunum); 406 reg = i82489_readreg(LAPIC_ID); 407 i82489_writereg(LAPIC_ID, (reg & ~LAPIC_ID_MASK) | 408 (cpunum << LAPIC_ID_SHIFT)); 409 } 410 if (cpunum != lapic_cpu_number()) { 411 aprint_error_dev(self, "unable to reset apic id\n"); 412 } 413 #endif 414 } 415 416 ci->ci_self = ci; 417 sc->sc_info = ci; 418 ci->ci_dev = self; 419 ci->ci_cpuid = cpunum; 420 421 KASSERT(HYPERVISOR_shared_info != NULL); 422 ci->ci_vcpu = &HYPERVISOR_shared_info->vcpu_info[cpunum]; 423 424 ci->ci_func = caa->cpu_func; 425 426 /* Must be called before mi_cpu_attach(). */ 427 cpu_vm_init(ci); 428 429 if (caa->cpu_role == CPU_ROLE_AP) { 430 int error; 431 432 error = mi_cpu_attach(ci); 433 if (error != 0) { 434 aprint_normal("\n"); 435 aprint_error_dev(self, 436 "mi_cpu_attach failed with %d\n", error); 437 return; 438 } 439 } else { 440 KASSERT(ci->ci_data.cpu_idlelwp != NULL); 441 } 442 443 ci->ci_cpumask = (1 << cpu_index(ci)); 444 pmap_reference(pmap_kernel()); 445 ci->ci_pmap = pmap_kernel(); 446 ci->ci_tlbstate = TLBSTATE_STALE; 447 448 /* 449 * Boot processor may not be attached first, but the below 450 * must be done to allow booting other processors. 451 */ 452 if (!again) { 453 atomic_or_32(&ci->ci_flags, CPUF_PRESENT | CPUF_PRIMARY); 454 /* Basic init. */ 455 cpu_intr_init(ci); 456 cpu_get_tsc_freq(ci); 457 cpu_init(ci); 458 cpu_set_tss_gates(ci); 459 #if NLAPIC > 0 460 if (caa->cpu_role != CPU_ROLE_SP) { 461 /* Enable lapic. */ 462 lapic_enable(); 463 lapic_set_lvt(); 464 lapic_calibrate_timer(); 465 } 466 #endif 467 /* Make sure DELAY() is initialized. */ 468 DELAY(1); 469 again = true; 470 } 471 472 /* further PCB init done later. */ 473 474 switch (caa->cpu_role) { 475 case CPU_ROLE_SP: 476 atomic_or_32(&ci->ci_flags, CPUF_SP); 477 cpu_identify(ci); 478 #if 0 479 x86_errata(); 480 #endif 481 x86_cpu_idle_init(); 482 break; 483 484 case CPU_ROLE_BP: 485 atomic_or_32(&ci->ci_flags, CPUF_BSP); 486 cpu_identify(ci); 487 cpu_init(ci); 488 #if 0 489 x86_errata(); 490 #endif 491 x86_cpu_idle_init(); 492 break; 493 494 case CPU_ROLE_AP: 495 /* 496 * report on an AP 497 */ 498 499 #if defined(MULTIPROCESSOR) 500 cpu_intr_init(ci); 501 gdt_alloc_cpu(ci); 502 cpu_set_tss_gates(ci); 503 cpu_start_secondary(ci); 504 if (ci->ci_flags & CPUF_PRESENT) { 505 struct cpu_info *tmp; 506 507 identifycpu(ci); 508 tmp = cpu_info_list; 509 while (tmp->ci_next) 510 tmp = tmp->ci_next; 511 512 tmp->ci_next = ci; 513 } 514 #else 515 aprint_error_dev(self, "not started\n"); 516 #endif 517 break; 518 519 default: 520 aprint_normal("\n"); 521 panic("unknown processor type??\n"); 522 } 523 524 pat_init(ci); 525 atomic_or_32(&cpus_attached, ci->ci_cpumask); 526 527 #if 0 528 if (!pmf_device_register(self, cpu_suspend, cpu_resume)) 529 aprint_error_dev(self, "couldn't establish power handler\n"); 530 #endif 531 532 #if defined(MULTIPROCESSOR) 533 if (mp_verbose) { 534 struct lwp *l = ci->ci_data.cpu_idlelwp; 535 struct pcb *pcb = lwp_getpcb(l); 536 537 aprint_verbose_dev(self, 538 "idle lwp at %p, idle sp at 0x%p\n", 539 l, 540 #ifdef i386 541 (void *)pcb->pcb_esp 542 #else 543 (void *)pcb->pcb_rsp 544 #endif 545 ); 546 547 } 548 #endif 549 } 550 551 /* 552 * Initialize the processor appropriately. 553 */ 554 555 void 556 cpu_init(struct cpu_info *ci) 557 { 558 559 /* 560 * On a P6 or above, enable global TLB caching if the 561 * hardware supports it. 562 */ 563 if (cpu_feature[0] & CPUID_PGE) 564 lcr4(rcr4() | CR4_PGE); /* enable global TLB caching */ 565 566 #ifdef XXXMTRR 567 /* 568 * On a P6 or above, initialize MTRR's if the hardware supports them. 569 */ 570 if (cpu_feature[0] & CPUID_MTRR) { 571 if ((ci->ci_flags & CPUF_AP) == 0) 572 i686_mtrr_init_first(); 573 mtrr_init_cpu(ci); 574 } 575 #endif 576 /* 577 * If we have FXSAVE/FXRESTOR, use them. 578 */ 579 if (cpu_feature[0] & CPUID_FXSR) { 580 lcr4(rcr4() | CR4_OSFXSR); 581 582 /* 583 * If we have SSE/SSE2, enable XMM exceptions. 584 */ 585 if (cpu_feature[0] & (CPUID_SSE|CPUID_SSE2)) 586 lcr4(rcr4() | CR4_OSXMMEXCPT); 587 } 588 589 #ifdef __x86_64__ 590 /* No user PGD mapped for this CPU yet */ 591 ci->ci_xen_current_user_pgd = 0; 592 #endif 593 594 atomic_or_32(&cpus_running, ci->ci_cpumask); 595 atomic_or_32(&ci->ci_flags, CPUF_RUNNING); 596 } 597 598 599 #ifdef MULTIPROCESSOR 600 void 601 cpu_boot_secondary_processors(void) 602 { 603 struct cpu_info *ci; 604 u_long i; 605 606 for (i = 0; i < maxcpus; i++) { 607 ci = cpu_lookup(i); 608 if (ci == NULL) 609 continue; 610 if (ci->ci_data.cpu_idlelwp == NULL) 611 continue; 612 if ((ci->ci_flags & CPUF_PRESENT) == 0) 613 continue; 614 if (ci->ci_flags & (CPUF_BSP|CPUF_SP|CPUF_PRIMARY)) 615 continue; 616 cpu_boot_secondary(ci); 617 } 618 619 x86_mp_online = true; 620 } 621 622 static void 623 cpu_init_idle_lwp(struct cpu_info *ci) 624 { 625 struct lwp *l = ci->ci_data.cpu_idlelwp; 626 struct pcb *pcb = lwp_getpcb(l); 627 628 pcb->pcb_cr0 = rcr0(); 629 } 630 631 void 632 cpu_init_idle_lwps(void) 633 { 634 struct cpu_info *ci; 635 u_long i; 636 637 for (i = 0; i < maxcpus; i++) { 638 ci = cpu_lookup(i); 639 if (ci == NULL) 640 continue; 641 if (ci->ci_data.cpu_idlelwp == NULL) 642 continue; 643 if ((ci->ci_flags & CPUF_PRESENT) == 0) 644 continue; 645 cpu_init_idle_lwp(ci); 646 } 647 } 648 649 void 650 cpu_start_secondary(struct cpu_info *ci) 651 { 652 int i; 653 struct pmap *kpm = pmap_kernel(); 654 extern uint32_t mp_pdirpa; 655 656 mp_pdirpa = kpm->pm_pdirpa; /* XXX move elsewhere, not per CPU. */ 657 658 atomic_or_32(&ci->ci_flags, CPUF_AP); 659 660 aprint_debug_dev(ci->ci_dev, "starting\n"); 661 662 ci->ci_curlwp = ci->ci_data.cpu_idlelwp; 663 if (CPU_STARTUP(ci, mp_trampoline_paddr) != 0) 664 return; 665 666 /* 667 * wait for it to become ready 668 */ 669 for (i = 100000; (!(ci->ci_flags & CPUF_PRESENT)) && i > 0; i--) { 670 #ifdef MPDEBUG 671 extern int cpu_trace[3]; 672 static int otrace[3]; 673 if (memcmp(otrace, cpu_trace, sizeof(otrace)) != 0) { 674 aprint_debug_dev(ci->ci_dev, "trace %02x %02x %02x\n", 675 cpu_trace[0], cpu_trace[1], cpu_trace[2]); 676 memcpy(otrace, cpu_trace, sizeof(otrace)); 677 } 678 #endif 679 delay(10); 680 } 681 if ((ci->ci_flags & CPUF_PRESENT) == 0) { 682 aprint_error_dev(ci->ci_dev, "failed to become ready\n"); 683 #if defined(MPDEBUG) && defined(DDB) 684 printf("dropping into debugger; continue from here to resume boot\n"); 685 Debugger(); 686 #endif 687 } 688 689 CPU_START_CLEANUP(ci); 690 } 691 692 void 693 cpu_boot_secondary(struct cpu_info *ci) 694 { 695 int i; 696 697 atomic_or_32(&ci->ci_flags, CPUF_GO); 698 for (i = 100000; (!(ci->ci_flags & CPUF_RUNNING)) && i > 0; i--) { 699 delay(10); 700 } 701 if ((ci->ci_flags & CPUF_RUNNING) == 0) { 702 aprint_error_dev(ci->ci_dev, "CPU failed to start\n"); 703 #if defined(MPDEBUG) && defined(DDB) 704 printf("dropping into debugger; continue from here to resume boot\n"); 705 Debugger(); 706 #endif 707 } 708 } 709 710 /* 711 * The CPU ends up here when its ready to run 712 * This is called from code in mptramp.s; at this point, we are running 713 * in the idle pcb/idle stack of the new CPU. When this function returns, 714 * this processor will enter the idle loop and start looking for work. 715 * 716 * XXX should share some of this with init386 in machdep.c 717 */ 718 void 719 cpu_hatch(void *v) 720 { 721 struct cpu_info *ci = (struct cpu_info *)v; 722 struct pcb *pcb; 723 int s, i; 724 725 cpu_probe(ci); 726 727 cpu_feature[0] &= ~CPUID_FEAT_BLACKLIST; 728 cpu_feature[2] &= ~CPUID_FEAT_EXT_BLACKLIST; 729 730 cpu_init_msrs(ci, true); 731 732 KDASSERT((ci->ci_flags & CPUF_PRESENT) == 0); 733 atomic_or_32(&ci->ci_flags, CPUF_PRESENT); 734 while ((ci->ci_flags & CPUF_GO) == 0) { 735 /* Don't use delay, boot CPU may be patching the text. */ 736 for (i = 10000; i != 0; i--) 737 x86_pause(); 738 } 739 740 /* Because the text may have been patched in x86_patch(). */ 741 wbinvd(); 742 x86_flush(); 743 tlbflushg(); 744 745 KASSERT((ci->ci_flags & CPUF_RUNNING) == 0); 746 747 pcb = lwp_getpcb(curlwp); 748 lcr3(pmap_kernel()->pm_pdirpa); 749 pcb->pcb_cr3 = pmap_kernel()->pm_pdirpa; 750 pcb = lwp_getpcb(ci->ci_data.cpu_idlelwp); 751 lcr0(pcb->pcb_cr0); 752 753 cpu_init_idt(); 754 gdt_init_cpu(ci); 755 lapic_enable(); 756 lapic_set_lvt(); 757 lapic_initclocks(); 758 759 #ifdef i386 760 npxinit(ci); 761 #else 762 fpuinit(ci); 763 #endif 764 765 lldt(GSEL(GLDT_SEL, SEL_KPL)); 766 ltr(ci->ci_tss_sel); 767 768 cpu_init(ci); 769 cpu_get_tsc_freq(ci); 770 771 s = splhigh(); 772 #ifdef i386 773 lapic_tpr = 0; 774 #else 775 lcr8(0); 776 #endif 777 x86_enable_intr(); 778 splx(s); 779 #if 0 780 x86_errata(); 781 #endif 782 783 aprint_debug_dev(ci->ci_dev, "CPU %ld running\n", 784 (long)ci->ci_cpuid); 785 } 786 787 #if defined(DDB) 788 789 #include <ddb/db_output.h> 790 #include <machine/db_machdep.h> 791 792 /* 793 * Dump CPU information from ddb. 794 */ 795 void 796 cpu_debug_dump(void) 797 { 798 struct cpu_info *ci; 799 CPU_INFO_ITERATOR cii; 800 801 db_printf("addr dev id flags ipis curlwp fpcurlwp\n"); 802 for (CPU_INFO_FOREACH(cii, ci)) { 803 db_printf("%p %s %ld %x %x %10p %10p\n", 804 ci, 805 ci->ci_dev == NULL ? "BOOT" : device_xname(ci->ci_dev), 806 (long)ci->ci_cpuid, 807 ci->ci_flags, ci->ci_ipis, 808 ci->ci_curlwp, 809 ci->ci_fpcurlwp); 810 } 811 } 812 #endif /* DDB */ 813 814 static void 815 cpu_copy_trampoline(void) 816 { 817 /* 818 * Copy boot code. 819 */ 820 extern u_char cpu_spinup_trampoline[]; 821 extern u_char cpu_spinup_trampoline_end[]; 822 823 vaddr_t mp_trampoline_vaddr; 824 825 mp_trampoline_vaddr = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, 826 UVM_KMF_VAONLY); 827 828 pmap_kenter_pa(mp_trampoline_vaddr, mp_trampoline_paddr, 829 VM_PROT_READ | VM_PROT_WRITE, 0); 830 pmap_update(pmap_kernel()); 831 memcpy((void *)mp_trampoline_vaddr, 832 cpu_spinup_trampoline, 833 cpu_spinup_trampoline_end - cpu_spinup_trampoline); 834 835 pmap_kremove(mp_trampoline_vaddr, PAGE_SIZE); 836 pmap_update(pmap_kernel()); 837 uvm_km_free(kernel_map, mp_trampoline_vaddr, PAGE_SIZE, UVM_KMF_VAONLY); 838 } 839 840 #endif /* MULTIPROCESSOR */ 841 842 #ifdef i386 843 #if 0 844 static void 845 tss_init(struct i386tss *tss, void *stack, void *func) 846 { 847 memset(tss, 0, sizeof *tss); 848 tss->tss_esp0 = tss->tss_esp = (int)((char *)stack + USPACE - 16); 849 tss->tss_ss0 = GSEL(GDATA_SEL, SEL_KPL); 850 tss->__tss_cs = GSEL(GCODE_SEL, SEL_KPL); 851 tss->tss_fs = GSEL(GCPU_SEL, SEL_KPL); 852 tss->tss_gs = tss->__tss_es = tss->__tss_ds = 853 tss->__tss_ss = GSEL(GDATA_SEL, SEL_KPL); 854 tss->tss_cr3 = pmap_kernel()->pm_pdirpa; 855 tss->tss_esp = (int)((char *)stack + USPACE - 16); 856 tss->tss_ldt = GSEL(GLDT_SEL, SEL_KPL); 857 tss->__tss_eflags = PSL_MBO | PSL_NT; /* XXX not needed? */ 858 tss->__tss_eip = (int)func; 859 } 860 #endif 861 862 /* XXX */ 863 #define IDTVEC(name) __CONCAT(X, name) 864 typedef void (vector)(void); 865 extern vector IDTVEC(tss_trap08); 866 #ifdef DDB 867 extern vector Xintrddbipi; 868 extern int ddb_vec; 869 #endif 870 871 static void 872 cpu_set_tss_gates(struct cpu_info *ci) 873 { 874 #if 0 875 struct segment_descriptor sd; 876 877 ci->ci_doubleflt_stack = (char *)uvm_km_alloc(kernel_map, USPACE, 0, 878 UVM_KMF_WIRED); 879 tss_init(&ci->ci_doubleflt_tss, ci->ci_doubleflt_stack, 880 IDTVEC(tss_trap08)); 881 setsegment(&sd, &ci->ci_doubleflt_tss, sizeof(struct i386tss) - 1, 882 SDT_SYS386TSS, SEL_KPL, 0, 0); 883 ci->ci_gdt[GTRAPTSS_SEL].sd = sd; 884 setgate(&idt[8], NULL, 0, SDT_SYSTASKGT, SEL_KPL, 885 GSEL(GTRAPTSS_SEL, SEL_KPL)); 886 #endif 887 888 #if defined(DDB) && defined(MULTIPROCESSOR) 889 /* 890 * Set up separate handler for the DDB IPI, so that it doesn't 891 * stomp on a possibly corrupted stack. 892 * 893 * XXX overwriting the gate set in db_machine_init. 894 * Should rearrange the code so that it's set only once. 895 */ 896 ci->ci_ddbipi_stack = (char *)uvm_km_alloc(kernel_map, USPACE, 0, 897 UVM_KMF_WIRED); 898 tss_init(&ci->ci_ddbipi_tss, ci->ci_ddbipi_stack, 899 Xintrddbipi); 900 901 setsegment(&sd, &ci->ci_ddbipi_tss, sizeof(struct i386tss) - 1, 902 SDT_SYS386TSS, SEL_KPL, 0, 0); 903 ci->ci_gdt[GIPITSS_SEL].sd = sd; 904 905 setgate(&idt[ddb_vec], NULL, 0, SDT_SYSTASKGT, SEL_KPL, 906 GSEL(GIPITSS_SEL, SEL_KPL)); 907 #endif 908 } 909 #else 910 static void 911 cpu_set_tss_gates(struct cpu_info *ci) 912 { 913 914 } 915 #endif /* i386 */ 916 917 int 918 mp_cpu_start(struct cpu_info *ci, paddr_t target) 919 { 920 #if 0 921 #if NLAPIC > 0 922 int error; 923 #endif 924 unsigned short dwordptr[2]; 925 926 /* 927 * Bootstrap code must be addressable in real mode 928 * and it must be page aligned. 929 */ 930 KASSERT(target < 0x10000 && target % PAGE_SIZE == 0); 931 932 /* 933 * "The BSP must initialize CMOS shutdown code to 0Ah ..." 934 */ 935 936 outb(IO_RTC, NVRAM_RESET); 937 outb(IO_RTC+1, NVRAM_RESET_JUMP); 938 939 /* 940 * "and the warm reset vector (DWORD based at 40:67) to point 941 * to the AP startup code ..." 942 */ 943 944 dwordptr[0] = 0; 945 dwordptr[1] = target >> 4; 946 947 pmap_kenter_pa (0, 0, VM_PROT_READ|VM_PROT_WRITE, 0); 948 pmap_update(pmap_kernel()); 949 950 memcpy ((uint8_t *) 0x467, dwordptr, 4); 951 952 pmap_kremove (0, PAGE_SIZE); 953 pmap_update(pmap_kernel()); 954 955 #if NLAPIC > 0 956 /* 957 * ... prior to executing the following sequence:" 958 */ 959 960 if (ci->ci_flags & CPUF_AP) { 961 if ((error = x86_ipi_init(ci->ci_cpuid)) != 0) 962 return error; 963 964 delay(10000); 965 966 if (cpu_feature & CPUID_APIC) { 967 error = x86_ipi_init(ci->ci_cpuid); 968 if (error != 0) { 969 aprint_error_dev(ci->ci_dev, "%s: IPI not taken (1)\n", 970 __func__); 971 return error; 972 } 973 974 delay(10000); 975 976 error = x86_ipi(target / PAGE_SIZE, ci->ci_cpuid, 977 LAPIC_DLMODE_STARTUP); 978 if (error != 0) { 979 aprint_error_dev(ci->ci_dev, "%s: IPI not taken (2)\n", 980 __func__); 981 return error; 982 } 983 delay(200); 984 985 error = x86_ipi(target / PAGE_SIZE, ci->ci_cpuid, 986 LAPIC_DLMODE_STARTUP); 987 if (error != 0) { 988 aprint_error_dev(ci->ci_dev, "%s: IPI not taken ((3)\n", 989 __func__); 990 return error; 991 } 992 delay(200); 993 } 994 } 995 #endif 996 #endif /* 0 */ 997 return 0; 998 } 999 1000 void 1001 mp_cpu_start_cleanup(struct cpu_info *ci) 1002 { 1003 #if 0 1004 /* 1005 * Ensure the NVRAM reset byte contains something vaguely sane. 1006 */ 1007 1008 outb(IO_RTC, NVRAM_RESET); 1009 outb(IO_RTC+1, NVRAM_RESET_RST); 1010 #endif 1011 } 1012 1013 void 1014 cpu_init_msrs(struct cpu_info *ci, bool full) 1015 { 1016 #ifdef __x86_64__ 1017 if (full) { 1018 HYPERVISOR_set_segment_base (SEGBASE_FS, 0); 1019 HYPERVISOR_set_segment_base (SEGBASE_GS_KERNEL, (uint64_t) ci); 1020 HYPERVISOR_set_segment_base (SEGBASE_GS_USER, 0); 1021 } 1022 #endif /* __x86_64__ */ 1023 1024 if (cpu_feature[2] & CPUID_NOX) 1025 wrmsr(MSR_EFER, rdmsr(MSR_EFER) | EFER_NXE); 1026 } 1027 1028 void 1029 cpu_offline_md(void) 1030 { 1031 int s; 1032 1033 s = splhigh(); 1034 #ifdef __i386__ 1035 npxsave_cpu(true); 1036 #else 1037 fpusave_cpu(true); 1038 #endif 1039 splx(s); 1040 } 1041 1042 #if 0 1043 /* XXX joerg restructure and restart CPUs individually */ 1044 static bool 1045 cpu_suspend(device_t dv, const pmf_qual_t *qual) 1046 { 1047 struct cpu_softc *sc = device_private(dv); 1048 struct cpu_info *ci = sc->sc_info; 1049 int err; 1050 1051 if (ci->ci_flags & CPUF_PRIMARY) 1052 return true; 1053 if (ci->ci_data.cpu_idlelwp == NULL) 1054 return true; 1055 if ((ci->ci_flags & CPUF_PRESENT) == 0) 1056 return true; 1057 1058 sc->sc_wasonline = !(ci->ci_schedstate.spc_flags & SPCF_OFFLINE); 1059 1060 if (sc->sc_wasonline) { 1061 mutex_enter(&cpu_lock); 1062 err = cpu_setstate(ci, false); 1063 mutex_exit(&cpu_lock); 1064 1065 if (err) 1066 return false; 1067 } 1068 1069 return true; 1070 } 1071 1072 static bool 1073 cpu_resume(device_t dv, const pmf_qual_t *qual) 1074 { 1075 struct cpu_softc *sc = device_private(dv); 1076 struct cpu_info *ci = sc->sc_info; 1077 int err = 0; 1078 1079 if (ci->ci_flags & CPUF_PRIMARY) 1080 return true; 1081 if (ci->ci_data.cpu_idlelwp == NULL) 1082 return true; 1083 if ((ci->ci_flags & CPUF_PRESENT) == 0) 1084 return true; 1085 1086 if (sc->sc_wasonline) { 1087 mutex_enter(&cpu_lock); 1088 err = cpu_setstate(ci, true); 1089 mutex_exit(&cpu_lock); 1090 } 1091 1092 return err == 0; 1093 } 1094 #endif 1095 1096 void 1097 cpu_get_tsc_freq(struct cpu_info *ci) 1098 { 1099 const volatile vcpu_time_info_t *tinfo = &ci->ci_vcpu->time; 1100 delay(1000000); 1101 uint64_t freq = 1000000000ULL << 32; 1102 freq = freq / (uint64_t)tinfo->tsc_to_system_mul; 1103 if ( tinfo->tsc_shift < 0 ) 1104 freq = freq << -tinfo->tsc_shift; 1105 else 1106 freq = freq >> tinfo->tsc_shift; 1107 ci->ci_data.cpu_cc_freq = freq; 1108 } 1109 1110 void 1111 x86_cpu_idle_xen(void) 1112 { 1113 struct cpu_info *ci = curcpu(); 1114 1115 KASSERT(ci->ci_ilevel == IPL_NONE); 1116 1117 x86_disable_intr(); 1118 if (!__predict_false(ci->ci_want_resched)) { 1119 idle_block(); 1120 } else { 1121 x86_enable_intr(); 1122 } 1123 } 1124 1125 /* 1126 * Loads pmap for the current CPU. 1127 */ 1128 void 1129 cpu_load_pmap(struct pmap *pmap) 1130 { 1131 #ifdef i386 1132 #ifdef PAE 1133 int i, s; 1134 struct cpu_info *ci; 1135 1136 s = splvm(); /* just to be safe */ 1137 ci = curcpu(); 1138 paddr_t l3_pd = xpmap_ptom_masked(ci->ci_pae_l3_pdirpa); 1139 /* don't update the kernel L3 slot */ 1140 for (i = 0 ; i < PDP_SIZE - 1; i++) { 1141 xpq_queue_pte_update(l3_pd + i * sizeof(pd_entry_t), 1142 xpmap_ptom(pmap->pm_pdirpa[i]) | PG_V); 1143 } 1144 splx(s); 1145 tlbflush(); 1146 #else /* PAE */ 1147 lcr3(pmap_pdirpa(pmap, 0)); 1148 #endif /* PAE */ 1149 #endif /* i386 */ 1150 1151 #ifdef __x86_64__ 1152 int i, s; 1153 pd_entry_t *old_pgd, *new_pgd; 1154 paddr_t addr; 1155 struct cpu_info *ci; 1156 1157 /* kernel pmap always in cr3 and should never go in user cr3 */ 1158 if (pmap_pdirpa(pmap, 0) != pmap_pdirpa(pmap_kernel(), 0)) { 1159 ci = curcpu(); 1160 /* 1161 * Map user space address in kernel space and load 1162 * user cr3 1163 */ 1164 s = splvm(); 1165 new_pgd = pmap->pm_pdir; 1166 old_pgd = pmap_kernel()->pm_pdir; 1167 addr = xpmap_ptom(pmap_pdirpa(pmap_kernel(), 0)); 1168 for (i = 0; i < PDIR_SLOT_PTE; 1169 i++, addr += sizeof(pd_entry_t)) { 1170 if ((new_pgd[i] & PG_V) || (old_pgd[i] & PG_V)) 1171 xpq_queue_pte_update(addr, new_pgd[i]); 1172 } 1173 xen_set_user_pgd(pmap_pdirpa(pmap, 0)); 1174 ci->ci_xen_current_user_pgd = pmap_pdirpa(pmap, 0); 1175 tlbflush(); 1176 splx(s); 1177 } 1178 #endif /* __x86_64__ */ 1179 } 1180
Indexes created Mon Sep 22 05:09:51 GMT 2025