cpu.c revision 1.55
1 /* $NetBSD: cpu.c,v 1.55 2011/02/24 19:00:58 jruoho Exp $ */ 2 /* NetBSD: cpu.c,v 1.18 2004/02/20 17:35:01 yamt Exp */ 3 4 /*- 5 * Copyright (c) 2000 The NetBSD Foundation, Inc. 6 * Copyright (c) 2002, 2006, 2007 YAMAMOTO Takashi, 7 * All rights reserved. 8 * 9 * This code is derived from software contributed to The NetBSD Foundation 10 * by RedBack Networks Inc. 11 * 12 * Author: Bill Sommerfeld 13 * 14 * Redistribution and use in source and binary forms, with or without 15 * modification, are permitted provided that the following conditions 16 * are met: 17 * 1. Redistributions of source code must retain the above copyright 18 * notice, this list of conditions and the following disclaimer. 19 * 2. Redistributions in binary form must reproduce the above copyright 20 * notice, this list of conditions and the following disclaimer in the 21 * documentation and/or other materials provided with the distribution. 22 * 23 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 24 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 25 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 26 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 27 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 28 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 29 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 30 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 31 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 32 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 33 * POSSIBILITY OF SUCH DAMAGE. 34 */ 35 36 /* 37 * Copyright (c) 1999 Stefan Grefen 38 * 39 * Redistribution and use in source and binary forms, with or without 40 * modification, are permitted provided that the following conditions 41 * are met: 42 * 1. Redistributions of source code must retain the above copyright 43 * notice, this list of conditions and the following disclaimer. 44 * 2. Redistributions in binary form must reproduce the above copyright 45 * notice, this list of conditions and the following disclaimer in the 46 * documentation and/or other materials provided with the distribution. 47 * 3. All advertising materials mentioning features or use of this software 48 * must display the following acknowledgement: 49 * This product includes software developed by the NetBSD 50 * Foundation, Inc. and its contributors. 51 * 4. Neither the name of The NetBSD Foundation nor the names of its 52 * contributors may be used to endorse or promote products derived 53 * from this software without specific prior written permission. 54 * 55 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND ANY 56 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 57 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 58 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR AND CONTRIBUTORS BE LIABLE 59 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 60 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 61 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 62 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 63 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 64 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 65 * SUCH DAMAGE. 66 */ 67 68 #include <sys/cdefs.h> 69 __KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.55 2011/02/24 19:00:58 jruoho Exp $"); 70 71 #include "opt_ddb.h" 72 #include "opt_multiprocessor.h" 73 #include "opt_mpbios.h" /* for MPDEBUG */ 74 #include "opt_mtrr.h" 75 #include "opt_xen.h" 76 77 #include "lapic.h" 78 #include "ioapic.h" 79 80 #include <sys/param.h> 81 #include <sys/proc.h> 82 #include <sys/systm.h> 83 #include <sys/device.h> 84 #include <sys/kmem.h> 85 #include <sys/cpu.h> 86 #include <sys/atomic.h> 87 #include <sys/reboot.h> 88 89 #include <uvm/uvm.h> 90 91 #include <machine/cpufunc.h> 92 #include <machine/cpuvar.h> 93 #include <machine/pmap.h> 94 #include <machine/vmparam.h> 95 #include <machine/mpbiosvar.h> 96 #include <machine/pcb.h> 97 #include <machine/specialreg.h> 98 #include <machine/segments.h> 99 #include <machine/gdt.h> 100 #include <machine/mtrr.h> 101 #include <machine/pio.h> 102 103 #include <xen/vcpuvar.h> 104 105 #if NLAPIC > 0 106 #include <machine/apicvar.h> 107 #include <machine/i82489reg.h> 108 #include <machine/i82489var.h> 109 #endif 110 111 #include <dev/ic/mc146818reg.h> 112 #include <dev/isa/isareg.h> 113 114 #if MAXCPUS > 32 115 #error cpu_info contains 32bit bitmasks 116 #endif 117 118 int cpu_match(device_t, cfdata_t, void *); 119 void cpu_attach(device_t, device_t, void *); 120 int cpu_rescan(device_t, const char *, const int *); 121 void cpu_childdetached(device_t, device_t); 122 int vcpu_match(device_t, cfdata_t, void *); 123 void vcpu_attach(device_t, device_t, void *); 124 void cpu_attach_common(device_t, device_t, void *); 125 void cpu_offline_md(void); 126 127 struct cpu_softc { 128 device_t sc_dev; /* device tree glue */ 129 struct cpu_info *sc_info; /* pointer to CPU info */ 130 bool sc_wasonline; 131 }; 132 133 int mp_cpu_start(struct cpu_info *, paddr_t); 134 void mp_cpu_start_cleanup(struct cpu_info *); 135 const struct cpu_functions mp_cpu_funcs = { mp_cpu_start, NULL, 136 mp_cpu_start_cleanup }; 137 138 CFATTACH_DECL2_NEW(cpu, sizeof(struct cpu_softc), 139 cpu_match, cpu_attach, NULL, NULL, cpu_rescan, cpu_childdetached); 140 141 CFATTACH_DECL_NEW(vcpu, sizeof(struct cpu_softc), 142 vcpu_match, vcpu_attach, NULL, NULL); 143 144 /* 145 * Statically-allocated CPU info for the primary CPU (or the only 146 * CPU, on uniprocessors). The CPU info list is initialized to 147 * point at it. 148 */ 149 #ifdef TRAPLOG 150 #include <machine/tlog.h> 151 struct tlog tlog_primary; 152 #endif 153 struct cpu_info cpu_info_primary __aligned(CACHE_LINE_SIZE) = { 154 .ci_dev = 0, 155 .ci_self = &cpu_info_primary, 156 .ci_idepth = -1, 157 .ci_curlwp = &lwp0, 158 .ci_curldt = -1, 159 #ifdef TRAPLOG 160 .ci_tlog = &tlog_primary, 161 #endif 162 163 }; 164 struct cpu_info phycpu_info_primary __aligned(CACHE_LINE_SIZE) = { 165 .ci_dev = 0, 166 .ci_self = &phycpu_info_primary, 167 }; 168 169 struct cpu_info *cpu_info_list = &cpu_info_primary; 170 struct cpu_info *phycpu_info_list = &phycpu_info_primary; 171 172 static void cpu_set_tss_gates(struct cpu_info *ci); 173 174 uint32_t cpus_attached = 0; 175 uint32_t cpus_running = 0; 176 177 uint32_t phycpus_attached = 0; 178 uint32_t phycpus_running = 0; 179 180 uint32_t cpu_feature[5]; /* X86 CPUID feature bits 181 * [0] basic features %edx 182 * [1] basic features %ecx 183 * [2] extended features %edx 184 * [3] extended features %ecx 185 * [4] VIA padlock features 186 */ 187 188 bool x86_mp_online; 189 paddr_t mp_trampoline_paddr = MP_TRAMPOLINE; 190 191 #if defined(MULTIPROCESSOR) 192 void cpu_hatch(void *); 193 static void cpu_boot_secondary(struct cpu_info *ci); 194 static void cpu_start_secondary(struct cpu_info *ci); 195 static void cpu_copy_trampoline(void); 196 197 /* 198 * Runs once per boot once multiprocessor goo has been detected and 199 * the local APIC on the boot processor has been mapped. 200 * 201 * Called from lapic_boot_init() (from mpbios_scan()). 202 */ 203 void 204 cpu_init_first(void) 205 { 206 207 cpu_info_primary.ci_cpuid = lapic_cpu_number(); 208 cpu_copy_trampoline(); 209 } 210 #endif /* MULTIPROCESSOR */ 211 212 int 213 cpu_match(device_t parent, cfdata_t match, void *aux) 214 { 215 216 return 1; 217 } 218 219 void 220 cpu_attach(device_t parent, device_t self, void *aux) 221 { 222 struct cpu_softc *sc = device_private(self); 223 struct cpu_attach_args *caa = aux; 224 struct cpu_info *ci; 225 uintptr_t ptr; 226 static int nphycpu = 0; 227 228 sc->sc_dev = self; 229 230 if (phycpus_attached == ~0) { 231 aprint_error(": increase MAXCPUS\n"); 232 return; 233 } 234 235 /* 236 * If we're an Application Processor, allocate a cpu_info 237 * If we're the first attached CPU use the primary cpu_info, 238 * otherwise allocate a new one 239 */ 240 aprint_naive("\n"); 241 aprint_normal("\n"); 242 if (nphycpu > 0) { 243 struct cpu_info *tmp; 244 ptr = (uintptr_t)kmem_zalloc(sizeof(*ci) + CACHE_LINE_SIZE - 1, 245 KM_SLEEP); 246 ci = (struct cpu_info *)roundup2(ptr, CACHE_LINE_SIZE); 247 ci->ci_curldt = -1; 248 249 tmp = phycpu_info_list; 250 while (tmp->ci_next) 251 tmp = tmp->ci_next; 252 253 tmp->ci_next = ci; 254 } else { 255 ci = &phycpu_info_primary; 256 } 257 258 ci->ci_self = ci; 259 sc->sc_info = ci; 260 261 ci->ci_dev = self; 262 ci->ci_acpiid = caa->cpu_id; 263 ci->ci_cpuid = caa->cpu_number; 264 ci->ci_vcpu = NULL; 265 ci->ci_index = nphycpu++; 266 ci->ci_cpumask = (1 << cpu_index(ci)); 267 268 atomic_or_32(&phycpus_attached, ci->ci_cpumask); 269 270 if (!pmf_device_register(self, NULL, NULL)) 271 aprint_error_dev(self, "couldn't establish power handler\n"); 272 273 return; 274 } 275 276 int 277 cpu_rescan(device_t self, const char *ifattr, const int *locators) 278 { 279 struct cpu_softc *sc = device_private(self); 280 struct cpufeature_attach_args cfaa; 281 struct cpu_info *ci = sc->sc_info; 282 283 memset(&cfaa, 0, sizeof(cfaa)); 284 cfaa.ci = ci; 285 286 if (ifattr_match(ifattr, "cpufeaturebus")) { 287 288 if (ci->ci_frequency == NULL) { 289 cfaa.name = "frequency"; 290 ci->ci_frequency = config_found_ia(self, 291 "cpufeaturebus", &cfaa, NULL); 292 } 293 } 294 295 return 0; 296 } 297 298 void 299 cpu_childdetached(device_t self, device_t child) 300 { 301 struct cpu_softc *sc = device_private(self); 302 struct cpu_info *ci = sc->sc_info; 303 304 if (ci->ci_frequency == child) 305 ci->ci_frequency = NULL; 306 } 307 308 int 309 vcpu_match(device_t parent, cfdata_t match, void *aux) 310 { 311 struct vcpu_attach_args *vcaa = aux; 312 313 if (strcmp(vcaa->vcaa_name, match->cf_name) == 0) 314 return 1; 315 return 0; 316 } 317 318 void 319 vcpu_attach(device_t parent, device_t self, void *aux) 320 { 321 struct vcpu_attach_args *vcaa = aux; 322 323 cpu_attach_common(parent, self, &vcaa->vcaa_caa); 324 } 325 326 static void 327 cpu_vm_init(struct cpu_info *ci) 328 { 329 int ncolors = 2, i; 330 331 for (i = CAI_ICACHE; i <= CAI_L2CACHE; i++) { 332 struct x86_cache_info *cai; 333 int tcolors; 334 335 cai = &ci->ci_cinfo[i]; 336 337 tcolors = atop(cai->cai_totalsize); 338 switch(cai->cai_associativity) { 339 case 0xff: 340 tcolors = 1; /* fully associative */ 341 break; 342 case 0: 343 case 1: 344 break; 345 default: 346 tcolors /= cai->cai_associativity; 347 } 348 ncolors = max(ncolors, tcolors); 349 } 350 351 /* 352 * Knowing the size of the largest cache on this CPU, re-color 353 * our pages. 354 */ 355 if (ncolors <= uvmexp.ncolors) 356 return; 357 aprint_debug_dev(ci->ci_dev, "%d page colors\n", ncolors); 358 uvm_page_recolor(ncolors); 359 } 360 361 void 362 cpu_attach_common(device_t parent, device_t self, void *aux) 363 { 364 struct cpu_softc *sc = device_private(self); 365 struct cpu_attach_args *caa = aux; 366 struct cpu_info *ci; 367 uintptr_t ptr; 368 int cpunum = caa->cpu_number; 369 static bool again = false; 370 371 sc->sc_dev = self; 372 373 /* 374 * If we're an Application Processor, allocate a cpu_info 375 * structure, otherwise use the primary's. 376 */ 377 if (caa->cpu_role == CPU_ROLE_AP) { 378 aprint_naive(": Application Processor\n"); 379 ptr = (uintptr_t)kmem_alloc(sizeof(*ci) + CACHE_LINE_SIZE - 1, 380 KM_SLEEP); 381 ci = (struct cpu_info *)roundup2(ptr, CACHE_LINE_SIZE); 382 memset(ci, 0, sizeof(*ci)); 383 #ifdef TRAPLOG 384 ci->ci_tlog_base = kmem_zalloc(sizeof(struct tlog), KM_SLEEP); 385 #endif 386 } else { 387 aprint_naive(": %s Processor\n", 388 caa->cpu_role == CPU_ROLE_SP ? "Single" : "Boot"); 389 ci = &cpu_info_primary; 390 #if NLAPIC > 0 391 if (cpunum != lapic_cpu_number()) { 392 /* XXX should be done earlier */ 393 uint32_t reg; 394 aprint_verbose("\n"); 395 aprint_verbose_dev(self, "running CPU at apic %d" 396 " instead of at expected %d", lapic_cpu_number(), 397 cpunum); 398 reg = i82489_readreg(LAPIC_ID); 399 i82489_writereg(LAPIC_ID, (reg & ~LAPIC_ID_MASK) | 400 (cpunum << LAPIC_ID_SHIFT)); 401 } 402 if (cpunum != lapic_cpu_number()) { 403 aprint_error_dev(self, "unable to reset apic id\n"); 404 } 405 #endif 406 } 407 408 ci->ci_self = ci; 409 sc->sc_info = ci; 410 ci->ci_dev = self; 411 ci->ci_cpuid = cpunum; 412 413 KASSERT(HYPERVISOR_shared_info != NULL); 414 ci->ci_vcpu = &HYPERVISOR_shared_info->vcpu_info[cpunum]; 415 416 ci->ci_func = caa->cpu_func; 417 418 /* Must be called before mi_cpu_attach(). */ 419 cpu_vm_init(ci); 420 421 if (caa->cpu_role == CPU_ROLE_AP) { 422 int error; 423 424 error = mi_cpu_attach(ci); 425 if (error != 0) { 426 aprint_normal("\n"); 427 aprint_error_dev(self, 428 "mi_cpu_attach failed with %d\n", error); 429 return; 430 } 431 } else { 432 KASSERT(ci->ci_data.cpu_idlelwp != NULL); 433 } 434 435 ci->ci_cpumask = (1 << cpu_index(ci)); 436 pmap_reference(pmap_kernel()); 437 ci->ci_pmap = pmap_kernel(); 438 ci->ci_tlbstate = TLBSTATE_STALE; 439 440 /* 441 * Boot processor may not be attached first, but the below 442 * must be done to allow booting other processors. 443 */ 444 if (!again) { 445 atomic_or_32(&ci->ci_flags, CPUF_PRESENT | CPUF_PRIMARY); 446 /* Basic init. */ 447 cpu_intr_init(ci); 448 cpu_get_tsc_freq(ci); 449 cpu_init(ci); 450 cpu_set_tss_gates(ci); 451 pmap_cpu_init_late(ci); 452 #if NLAPIC > 0 453 if (caa->cpu_role != CPU_ROLE_SP) { 454 /* Enable lapic. */ 455 lapic_enable(); 456 lapic_set_lvt(); 457 lapic_calibrate_timer(); 458 } 459 #endif 460 /* Make sure DELAY() is initialized. */ 461 DELAY(1); 462 again = true; 463 } 464 465 /* further PCB init done later. */ 466 467 switch (caa->cpu_role) { 468 case CPU_ROLE_SP: 469 atomic_or_32(&ci->ci_flags, CPUF_SP); 470 cpu_identify(ci); 471 #if 0 472 x86_errata(); 473 #endif 474 x86_cpu_idle_init(); 475 break; 476 477 case CPU_ROLE_BP: 478 atomic_or_32(&ci->ci_flags, CPUF_BSP); 479 cpu_identify(ci); 480 cpu_init(ci); 481 #if 0 482 x86_errata(); 483 #endif 484 x86_cpu_idle_init(); 485 break; 486 487 case CPU_ROLE_AP: 488 /* 489 * report on an AP 490 */ 491 492 #if defined(MULTIPROCESSOR) 493 cpu_intr_init(ci); 494 gdt_alloc_cpu(ci); 495 cpu_set_tss_gates(ci); 496 pmap_cpu_init_early(ci); 497 pmap_cpu_init_late(ci); 498 cpu_start_secondary(ci); 499 if (ci->ci_flags & CPUF_PRESENT) { 500 struct cpu_info *tmp; 501 502 identifycpu(ci); 503 tmp = cpu_info_list; 504 while (tmp->ci_next) 505 tmp = tmp->ci_next; 506 507 tmp->ci_next = ci; 508 } 509 #else 510 aprint_error_dev(self, "not started\n"); 511 #endif 512 break; 513 514 default: 515 aprint_normal("\n"); 516 panic("unknown processor type??\n"); 517 } 518 519 pat_init(ci); 520 atomic_or_32(&cpus_attached, ci->ci_cpumask); 521 522 #if 0 523 if (!pmf_device_register(self, cpu_suspend, cpu_resume)) 524 aprint_error_dev(self, "couldn't establish power handler\n"); 525 #endif 526 527 #if defined(MULTIPROCESSOR) 528 if (mp_verbose) { 529 struct lwp *l = ci->ci_data.cpu_idlelwp; 530 struct pcb *pcb = lwp_getpcb(l); 531 532 aprint_verbose_dev(self, 533 "idle lwp at %p, idle sp at 0x%p\n", 534 l, 535 #ifdef i386 536 (void *)pcb->pcb_esp 537 #else 538 (void *)pcb->pcb_rsp 539 #endif 540 ); 541 542 } 543 #endif 544 } 545 546 /* 547 * Initialize the processor appropriately. 548 */ 549 550 void 551 cpu_init(struct cpu_info *ci) 552 { 553 554 /* 555 * On a P6 or above, enable global TLB caching if the 556 * hardware supports it. 557 */ 558 if (cpu_feature[0] & CPUID_PGE) 559 lcr4(rcr4() | CR4_PGE); /* enable global TLB caching */ 560 561 #ifdef XXXMTRR 562 /* 563 * On a P6 or above, initialize MTRR's if the hardware supports them. 564 */ 565 if (cpu_feature[0] & CPUID_MTRR) { 566 if ((ci->ci_flags & CPUF_AP) == 0) 567 i686_mtrr_init_first(); 568 mtrr_init_cpu(ci); 569 } 570 #endif 571 /* 572 * If we have FXSAVE/FXRESTOR, use them. 573 */ 574 if (cpu_feature[0] & CPUID_FXSR) { 575 lcr4(rcr4() | CR4_OSFXSR); 576 577 /* 578 * If we have SSE/SSE2, enable XMM exceptions. 579 */ 580 if (cpu_feature[0] & (CPUID_SSE|CPUID_SSE2)) 581 lcr4(rcr4() | CR4_OSXMMEXCPT); 582 } 583 584 #ifdef __x86_64__ 585 /* No user PGD mapped for this CPU yet */ 586 ci->ci_xen_current_user_pgd = 0; 587 #endif 588 589 atomic_or_32(&cpus_running, ci->ci_cpumask); 590 atomic_or_32(&ci->ci_flags, CPUF_RUNNING); 591 } 592 593 594 #ifdef MULTIPROCESSOR 595 void 596 cpu_boot_secondary_processors(void) 597 { 598 struct cpu_info *ci; 599 u_long i; 600 601 for (i = 0; i < maxcpus; i++) { 602 ci = cpu_lookup(i); 603 if (ci == NULL) 604 continue; 605 if (ci->ci_data.cpu_idlelwp == NULL) 606 continue; 607 if ((ci->ci_flags & CPUF_PRESENT) == 0) 608 continue; 609 if (ci->ci_flags & (CPUF_BSP|CPUF_SP|CPUF_PRIMARY)) 610 continue; 611 cpu_boot_secondary(ci); 612 } 613 614 x86_mp_online = true; 615 } 616 617 static void 618 cpu_init_idle_lwp(struct cpu_info *ci) 619 { 620 struct lwp *l = ci->ci_data.cpu_idlelwp; 621 struct pcb *pcb = lwp_getpcb(l); 622 623 pcb->pcb_cr0 = rcr0(); 624 } 625 626 void 627 cpu_init_idle_lwps(void) 628 { 629 struct cpu_info *ci; 630 u_long i; 631 632 for (i = 0; i < maxcpus; i++) { 633 ci = cpu_lookup(i); 634 if (ci == NULL) 635 continue; 636 if (ci->ci_data.cpu_idlelwp == NULL) 637 continue; 638 if ((ci->ci_flags & CPUF_PRESENT) == 0) 639 continue; 640 cpu_init_idle_lwp(ci); 641 } 642 } 643 644 void 645 cpu_start_secondary(struct cpu_info *ci) 646 { 647 int i; 648 struct pmap *kpm = pmap_kernel(); 649 extern uint32_t mp_pdirpa; 650 651 mp_pdirpa = kpm->pm_pdirpa; /* XXX move elsewhere, not per CPU. */ 652 653 atomic_or_32(&ci->ci_flags, CPUF_AP); 654 655 aprint_debug_dev(ci->ci_dev, "starting\n"); 656 657 ci->ci_curlwp = ci->ci_data.cpu_idlelwp; 658 if (CPU_STARTUP(ci, mp_trampoline_paddr) != 0) 659 return; 660 661 /* 662 * wait for it to become ready 663 */ 664 for (i = 100000; (!(ci->ci_flags & CPUF_PRESENT)) && i > 0; i--) { 665 #ifdef MPDEBUG 666 extern int cpu_trace[3]; 667 static int otrace[3]; 668 if (memcmp(otrace, cpu_trace, sizeof(otrace)) != 0) { 669 aprint_debug_dev(ci->ci_dev, "trace %02x %02x %02x\n", 670 cpu_trace[0], cpu_trace[1], cpu_trace[2]); 671 memcpy(otrace, cpu_trace, sizeof(otrace)); 672 } 673 #endif 674 delay(10); 675 } 676 if ((ci->ci_flags & CPUF_PRESENT) == 0) { 677 aprint_error_dev(ci->ci_dev, "failed to become ready\n"); 678 #if defined(MPDEBUG) && defined(DDB) 679 printf("dropping into debugger; continue from here to resume boot\n"); 680 Debugger(); 681 #endif 682 } 683 684 CPU_START_CLEANUP(ci); 685 } 686 687 void 688 cpu_boot_secondary(struct cpu_info *ci) 689 { 690 int i; 691 692 atomic_or_32(&ci->ci_flags, CPUF_GO); 693 for (i = 100000; (!(ci->ci_flags & CPUF_RUNNING)) && i > 0; i--) { 694 delay(10); 695 } 696 if ((ci->ci_flags & CPUF_RUNNING) == 0) { 697 aprint_error_dev(ci->ci_dev, "CPU failed to start\n"); 698 #if defined(MPDEBUG) && defined(DDB) 699 printf("dropping into debugger; continue from here to resume boot\n"); 700 Debugger(); 701 #endif 702 } 703 } 704 705 /* 706 * The CPU ends up here when its ready to run 707 * This is called from code in mptramp.s; at this point, we are running 708 * in the idle pcb/idle stack of the new CPU. When this function returns, 709 * this processor will enter the idle loop and start looking for work. 710 * 711 * XXX should share some of this with init386 in machdep.c 712 */ 713 void 714 cpu_hatch(void *v) 715 { 716 struct cpu_info *ci = (struct cpu_info *)v; 717 struct pcb *pcb; 718 int s, i; 719 720 cpu_probe(ci); 721 722 cpu_feature[0] &= ~CPUID_FEAT_BLACKLIST; 723 cpu_feature[2] &= ~CPUID_FEAT_EXT_BLACKLIST; 724 725 cpu_init_msrs(ci, true); 726 727 KDASSERT((ci->ci_flags & CPUF_PRESENT) == 0); 728 atomic_or_32(&ci->ci_flags, CPUF_PRESENT); 729 while ((ci->ci_flags & CPUF_GO) == 0) { 730 /* Don't use delay, boot CPU may be patching the text. */ 731 for (i = 10000; i != 0; i--) 732 x86_pause(); 733 } 734 735 /* Because the text may have been patched in x86_patch(). */ 736 wbinvd(); 737 x86_flush(); 738 739 KASSERT((ci->ci_flags & CPUF_RUNNING) == 0); 740 741 pcb = lwp_getpcb(curlwp); 742 lcr3(pmap_kernel()->pm_pdirpa); 743 pcb->pcb_cr3 = pmap_kernel()->pm_pdirpa; 744 pcb = lwp_getpcb(ci->ci_data.cpu_idlelwp); 745 lcr0(pcb->pcb_cr0); 746 747 cpu_init_idt(); 748 gdt_init_cpu(ci); 749 lapic_enable(); 750 lapic_set_lvt(); 751 lapic_initclocks(); 752 753 #ifdef i386 754 npxinit(ci); 755 #else 756 fpuinit(ci); 757 #endif 758 759 lldt(GSEL(GLDT_SEL, SEL_KPL)); 760 ltr(ci->ci_tss_sel); 761 762 cpu_init(ci); 763 cpu_get_tsc_freq(ci); 764 765 s = splhigh(); 766 #ifdef i386 767 lapic_tpr = 0; 768 #else 769 lcr8(0); 770 #endif 771 x86_enable_intr(); 772 splx(s); 773 #if 0 774 x86_errata(); 775 #endif 776 777 aprint_debug_dev(ci->ci_dev, "CPU %ld running\n", 778 (long)ci->ci_cpuid); 779 } 780 781 #if defined(DDB) 782 783 #include <ddb/db_output.h> 784 #include <machine/db_machdep.h> 785 786 /* 787 * Dump CPU information from ddb. 788 */ 789 void 790 cpu_debug_dump(void) 791 { 792 struct cpu_info *ci; 793 CPU_INFO_ITERATOR cii; 794 795 db_printf("addr dev id flags ipis curlwp fpcurlwp\n"); 796 for (CPU_INFO_FOREACH(cii, ci)) { 797 db_printf("%p %s %ld %x %x %10p %10p\n", 798 ci, 799 ci->ci_dev == NULL ? "BOOT" : device_xname(ci->ci_dev), 800 (long)ci->ci_cpuid, 801 ci->ci_flags, ci->ci_ipis, 802 ci->ci_curlwp, 803 ci->ci_fpcurlwp); 804 } 805 } 806 #endif /* DDB */ 807 808 static void 809 cpu_copy_trampoline(void) 810 { 811 /* 812 * Copy boot code. 813 */ 814 extern u_char cpu_spinup_trampoline[]; 815 extern u_char cpu_spinup_trampoline_end[]; 816 817 vaddr_t mp_trampoline_vaddr; 818 819 mp_trampoline_vaddr = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, 820 UVM_KMF_VAONLY); 821 822 pmap_kenter_pa(mp_trampoline_vaddr, mp_trampoline_paddr, 823 VM_PROT_READ | VM_PROT_WRITE, 0); 824 pmap_update(pmap_kernel()); 825 memcpy((void *)mp_trampoline_vaddr, 826 cpu_spinup_trampoline, 827 cpu_spinup_trampoline_end - cpu_spinup_trampoline); 828 829 pmap_kremove(mp_trampoline_vaddr, PAGE_SIZE); 830 pmap_update(pmap_kernel()); 831 uvm_km_free(kernel_map, mp_trampoline_vaddr, PAGE_SIZE, UVM_KMF_VAONLY); 832 } 833 834 #endif /* MULTIPROCESSOR */ 835 836 #ifdef i386 837 #if 0 838 static void 839 tss_init(struct i386tss *tss, void *stack, void *func) 840 { 841 memset(tss, 0, sizeof *tss); 842 tss->tss_esp0 = tss->tss_esp = (int)((char *)stack + USPACE - 16); 843 tss->tss_ss0 = GSEL(GDATA_SEL, SEL_KPL); 844 tss->__tss_cs = GSEL(GCODE_SEL, SEL_KPL); 845 tss->tss_fs = GSEL(GCPU_SEL, SEL_KPL); 846 tss->tss_gs = tss->__tss_es = tss->__tss_ds = 847 tss->__tss_ss = GSEL(GDATA_SEL, SEL_KPL); 848 tss->tss_cr3 = pmap_kernel()->pm_pdirpa; 849 tss->tss_esp = (int)((char *)stack + USPACE - 16); 850 tss->tss_ldt = GSEL(GLDT_SEL, SEL_KPL); 851 tss->__tss_eflags = PSL_MBO | PSL_NT; /* XXX not needed? */ 852 tss->__tss_eip = (int)func; 853 } 854 #endif 855 856 /* XXX */ 857 #define IDTVEC(name) __CONCAT(X, name) 858 typedef void (vector)(void); 859 extern vector IDTVEC(tss_trap08); 860 #ifdef DDB 861 extern vector Xintrddbipi; 862 extern int ddb_vec; 863 #endif 864 865 static void 866 cpu_set_tss_gates(struct cpu_info *ci) 867 { 868 #if 0 869 struct segment_descriptor sd; 870 871 ci->ci_doubleflt_stack = (char *)uvm_km_alloc(kernel_map, USPACE, 0, 872 UVM_KMF_WIRED); 873 tss_init(&ci->ci_doubleflt_tss, ci->ci_doubleflt_stack, 874 IDTVEC(tss_trap08)); 875 setsegment(&sd, &ci->ci_doubleflt_tss, sizeof(struct i386tss) - 1, 876 SDT_SYS386TSS, SEL_KPL, 0, 0); 877 ci->ci_gdt[GTRAPTSS_SEL].sd = sd; 878 setgate(&idt[8], NULL, 0, SDT_SYSTASKGT, SEL_KPL, 879 GSEL(GTRAPTSS_SEL, SEL_KPL)); 880 #endif 881 882 #if defined(DDB) && defined(MULTIPROCESSOR) 883 /* 884 * Set up separate handler for the DDB IPI, so that it doesn't 885 * stomp on a possibly corrupted stack. 886 * 887 * XXX overwriting the gate set in db_machine_init. 888 * Should rearrange the code so that it's set only once. 889 */ 890 ci->ci_ddbipi_stack = (char *)uvm_km_alloc(kernel_map, USPACE, 0, 891 UVM_KMF_WIRED); 892 tss_init(&ci->ci_ddbipi_tss, ci->ci_ddbipi_stack, 893 Xintrddbipi); 894 895 setsegment(&sd, &ci->ci_ddbipi_tss, sizeof(struct i386tss) - 1, 896 SDT_SYS386TSS, SEL_KPL, 0, 0); 897 ci->ci_gdt[GIPITSS_SEL].sd = sd; 898 899 setgate(&idt[ddb_vec], NULL, 0, SDT_SYSTASKGT, SEL_KPL, 900 GSEL(GIPITSS_SEL, SEL_KPL)); 901 #endif 902 } 903 #else 904 static void 905 cpu_set_tss_gates(struct cpu_info *ci) 906 { 907 908 } 909 #endif /* i386 */ 910 911 int 912 mp_cpu_start(struct cpu_info *ci, paddr_t target) 913 { 914 #if 0 915 #if NLAPIC > 0 916 int error; 917 #endif 918 unsigned short dwordptr[2]; 919 920 /* 921 * Bootstrap code must be addressable in real mode 922 * and it must be page aligned. 923 */ 924 KASSERT(target < 0x10000 && target % PAGE_SIZE == 0); 925 926 /* 927 * "The BSP must initialize CMOS shutdown code to 0Ah ..." 928 */ 929 930 outb(IO_RTC, NVRAM_RESET); 931 outb(IO_RTC+1, NVRAM_RESET_JUMP); 932 933 /* 934 * "and the warm reset vector (DWORD based at 40:67) to point 935 * to the AP startup code ..." 936 */ 937 938 dwordptr[0] = 0; 939 dwordptr[1] = target >> 4; 940 941 pmap_kenter_pa (0, 0, VM_PROT_READ|VM_PROT_WRITE, 0); 942 pmap_update(pmap_kernel()); 943 944 memcpy ((uint8_t *) 0x467, dwordptr, 4); 945 946 pmap_kremove (0, PAGE_SIZE); 947 pmap_update(pmap_kernel()); 948 949 #if NLAPIC > 0 950 /* 951 * ... prior to executing the following sequence:" 952 */ 953 954 if (ci->ci_flags & CPUF_AP) { 955 if ((error = x86_ipi_init(ci->ci_cpuid)) != 0) 956 return error; 957 958 delay(10000); 959 960 if (cpu_feature & CPUID_APIC) { 961 error = x86_ipi_init(ci->ci_cpuid); 962 if (error != 0) { 963 aprint_error_dev(ci->ci_dev, "%s: IPI not taken (1)\n", 964 __func__); 965 return error; 966 } 967 968 delay(10000); 969 970 error = x86_ipi(target / PAGE_SIZE, ci->ci_cpuid, 971 LAPIC_DLMODE_STARTUP); 972 if (error != 0) { 973 aprint_error_dev(ci->ci_dev, "%s: IPI not taken (2)\n", 974 __func__); 975 return error; 976 } 977 delay(200); 978 979 error = x86_ipi(target / PAGE_SIZE, ci->ci_cpuid, 980 LAPIC_DLMODE_STARTUP); 981 if (error != 0) { 982 aprint_error_dev(ci->ci_dev, "%s: IPI not taken ((3)\n", 983 __func__); 984 return error; 985 } 986 delay(200); 987 } 988 } 989 #endif 990 #endif /* 0 */ 991 return 0; 992 } 993 994 void 995 mp_cpu_start_cleanup(struct cpu_info *ci) 996 { 997 #if 0 998 /* 999 * Ensure the NVRAM reset byte contains something vaguely sane. 1000 */ 1001 1002 outb(IO_RTC, NVRAM_RESET); 1003 outb(IO_RTC+1, NVRAM_RESET_RST); 1004 #endif 1005 } 1006 1007 void 1008 cpu_init_msrs(struct cpu_info *ci, bool full) 1009 { 1010 #ifdef __x86_64__ 1011 if (full) { 1012 HYPERVISOR_set_segment_base (SEGBASE_FS, 0); 1013 HYPERVISOR_set_segment_base (SEGBASE_GS_KERNEL, (uint64_t) ci); 1014 HYPERVISOR_set_segment_base (SEGBASE_GS_USER, 0); 1015 } 1016 #endif /* __x86_64__ */ 1017 1018 if (cpu_feature[2] & CPUID_NOX) 1019 wrmsr(MSR_EFER, rdmsr(MSR_EFER) | EFER_NXE); 1020 } 1021 1022 void 1023 cpu_offline_md(void) 1024 { 1025 int s; 1026 1027 s = splhigh(); 1028 #ifdef __i386__ 1029 npxsave_cpu(true); 1030 #else 1031 fpusave_cpu(true); 1032 #endif 1033 splx(s); 1034 } 1035 1036 #if 0 1037 /* XXX joerg restructure and restart CPUs individually */ 1038 static bool 1039 cpu_suspend(device_t dv, const pmf_qual_t *qual) 1040 { 1041 struct cpu_softc *sc = device_private(dv); 1042 struct cpu_info *ci = sc->sc_info; 1043 int err; 1044 1045 if (ci->ci_flags & CPUF_PRIMARY) 1046 return true; 1047 if (ci->ci_data.cpu_idlelwp == NULL) 1048 return true; 1049 if ((ci->ci_flags & CPUF_PRESENT) == 0) 1050 return true; 1051 1052 sc->sc_wasonline = !(ci->ci_schedstate.spc_flags & SPCF_OFFLINE); 1053 1054 if (sc->sc_wasonline) { 1055 mutex_enter(&cpu_lock); 1056 err = cpu_setstate(ci, false); 1057 mutex_exit(&cpu_lock); 1058 1059 if (err) 1060 return false; 1061 } 1062 1063 return true; 1064 } 1065 1066 static bool 1067 cpu_resume(device_t dv, const pmf_qual_t *qual) 1068 { 1069 struct cpu_softc *sc = device_private(dv); 1070 struct cpu_info *ci = sc->sc_info; 1071 int err = 0; 1072 1073 if (ci->ci_flags & CPUF_PRIMARY) 1074 return true; 1075 if (ci->ci_data.cpu_idlelwp == NULL) 1076 return true; 1077 if ((ci->ci_flags & CPUF_PRESENT) == 0) 1078 return true; 1079 1080 if (sc->sc_wasonline) { 1081 mutex_enter(&cpu_lock); 1082 err = cpu_setstate(ci, true); 1083 mutex_exit(&cpu_lock); 1084 } 1085 1086 return err == 0; 1087 } 1088 #endif 1089 1090 void 1091 cpu_get_tsc_freq(struct cpu_info *ci) 1092 { 1093 const volatile vcpu_time_info_t *tinfo = &ci->ci_vcpu->time; 1094 delay(1000000); 1095 uint64_t freq = 1000000000ULL << 32; 1096 freq = freq / (uint64_t)tinfo->tsc_to_system_mul; 1097 if ( tinfo->tsc_shift < 0 ) 1098 freq = freq << -tinfo->tsc_shift; 1099 else 1100 freq = freq >> tinfo->tsc_shift; 1101 ci->ci_data.cpu_cc_freq = freq; 1102 } 1103 1104 void 1105 x86_cpu_idle_xen(void) 1106 { 1107 struct cpu_info *ci = curcpu(); 1108 1109 KASSERT(ci->ci_ilevel == IPL_NONE); 1110 1111 x86_disable_intr(); 1112 if (!__predict_false(ci->ci_want_resched)) { 1113 idle_block(); 1114 } else { 1115 x86_enable_intr(); 1116 } 1117 } 1118 1119 /* 1120 * Loads pmap for the current CPU. 1121 */ 1122 void 1123 cpu_load_pmap(struct pmap *pmap) 1124 { 1125 #ifdef i386 1126 #ifdef PAE 1127 int i, s; 1128 struct cpu_info *ci; 1129 1130 s = splvm(); /* just to be safe */ 1131 ci = curcpu(); 1132 paddr_t l3_pd = xpmap_ptom_masked(ci->ci_pae_l3_pdirpa); 1133 /* don't update the kernel L3 slot */ 1134 for (i = 0 ; i < PDP_SIZE - 1; i++) { 1135 xpq_queue_pte_update(l3_pd + i * sizeof(pd_entry_t), 1136 xpmap_ptom(pmap->pm_pdirpa[i]) | PG_V); 1137 } 1138 splx(s); 1139 tlbflush(); 1140 #else /* PAE */ 1141 lcr3(pmap_pdirpa(pmap, 0)); 1142 #endif /* PAE */ 1143 #endif /* i386 */ 1144 1145 #ifdef __x86_64__ 1146 int i, s; 1147 pd_entry_t *old_pgd, *new_pgd; 1148 paddr_t addr; 1149 struct cpu_info *ci; 1150 1151 /* kernel pmap always in cr3 and should never go in user cr3 */ 1152 if (pmap_pdirpa(pmap, 0) != pmap_pdirpa(pmap_kernel(), 0)) { 1153 ci = curcpu(); 1154 /* 1155 * Map user space address in kernel space and load 1156 * user cr3 1157 */ 1158 s = splvm(); 1159 new_pgd = pmap->pm_pdir; 1160 old_pgd = pmap_kernel()->pm_pdir; 1161 addr = xpmap_ptom(pmap_pdirpa(pmap_kernel(), 0)); 1162 for (i = 0; i < PDIR_SLOT_PTE; 1163 i++, addr += sizeof(pd_entry_t)) { 1164 if ((new_pgd[i] & PG_V) || (old_pgd[i] & PG_V)) 1165 xpq_queue_pte_update(addr, new_pgd[i]); 1166 } 1167 tlbflush(); 1168 xen_set_user_pgd(pmap_pdirpa(pmap, 0)); 1169 ci->ci_xen_current_user_pgd = pmap_pdirpa(pmap, 0); 1170 splx(s); 1171 } 1172 #endif /* __x86_64__ */ 1173 } 1174
Indexes created Tue Sep 23 07:09:52 GMT 2025