cpu.c revision 1.31.2.7
1 /* $NetBSD: cpu.c,v 1.31.2.7 2011/01/10 00:37:38 jym Exp $ */ 2 /* NetBSD: cpu.c,v 1.18 2004/02/20 17:35:01 yamt Exp */ 3 4 /*- 5 * Copyright (c) 2000 The NetBSD Foundation, Inc. 6 * Copyright (c) 2002, 2006, 2007 YAMAMOTO Takashi, 7 * All rights reserved. 8 * 9 * This code is derived from software contributed to The NetBSD Foundation 10 * by RedBack Networks Inc. 11 * 12 * Author: Bill Sommerfeld 13 * 14 * Redistribution and use in source and binary forms, with or without 15 * modification, are permitted provided that the following conditions 16 * are met: 17 * 1. Redistributions of source code must retain the above copyright 18 * notice, this list of conditions and the following disclaimer. 19 * 2. Redistributions in binary form must reproduce the above copyright 20 * notice, this list of conditions and the following disclaimer in the 21 * documentation and/or other materials provided with the distribution. 22 * 23 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 24 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 25 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 26 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 27 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 28 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 29 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 30 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 31 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 32 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 33 * POSSIBILITY OF SUCH DAMAGE. 34 */ 35 36 /* 37 * Copyright (c) 1999 Stefan Grefen 38 * 39 * Redistribution and use in source and binary forms, with or without 40 * modification, are permitted provided that the following conditions 41 * are met: 42 * 1. Redistributions of source code must retain the above copyright 43 * notice, this list of conditions and the following disclaimer. 44 * 2. Redistributions in binary form must reproduce the above copyright 45 * notice, this list of conditions and the following disclaimer in the 46 * documentation and/or other materials provided with the distribution. 47 * 3. All advertising materials mentioning features or use of this software 48 * must display the following acknowledgement: 49 * This product includes software developed by the NetBSD 50 * Foundation, Inc. and its contributors. 51 * 4. Neither the name of The NetBSD Foundation nor the names of its 52 * contributors may be used to endorse or promote products derived 53 * from this software without specific prior written permission. 54 * 55 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND ANY 56 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 57 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 58 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR AND CONTRIBUTORS BE LIABLE 59 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 60 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 61 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 62 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 63 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 64 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 65 * SUCH DAMAGE. 66 */ 67 68 #include <sys/cdefs.h> 69 __KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.31.2.7 2011/01/10 00:37:38 jym Exp $"); 70 71 #include "opt_ddb.h" 72 #include "opt_multiprocessor.h" 73 #include "opt_mpbios.h" /* for MPDEBUG */ 74 #include "opt_mtrr.h" 75 #include "opt_xen.h" 76 77 #include "lapic.h" 78 #include "ioapic.h" 79 80 #include <sys/param.h> 81 #include <sys/proc.h> 82 #include <sys/systm.h> 83 #include <sys/device.h> 84 #include <sys/kmem.h> 85 #include <sys/cpu.h> 86 #include <sys/atomic.h> 87 #include <sys/reboot.h> 88 89 #include <uvm/uvm.h> 90 91 #include <machine/cpufunc.h> 92 #include <machine/cpuvar.h> 93 #include <machine/pmap.h> 94 #include <machine/vmparam.h> 95 #include <machine/mpbiosvar.h> 96 #include <machine/pcb.h> 97 #include <machine/specialreg.h> 98 #include <machine/segments.h> 99 #include <machine/gdt.h> 100 #include <machine/mtrr.h> 101 #include <machine/pio.h> 102 103 #include <xen/vcpuvar.h> 104 105 #if NLAPIC > 0 106 #include <machine/apicvar.h> 107 #include <machine/i82489reg.h> 108 #include <machine/i82489var.h> 109 #endif 110 111 #include <dev/ic/mc146818reg.h> 112 #include <dev/isa/isareg.h> 113 114 #if MAXCPUS > 32 115 #error cpu_info contains 32bit bitmasks 116 #endif 117 118 int cpu_match(device_t, cfdata_t, void *); 119 void cpu_attach(device_t, device_t, void *); 120 int vcpu_match(device_t, cfdata_t, void *); 121 void vcpu_attach(device_t, device_t, void *); 122 void cpu_attach_common(device_t, device_t, void *); 123 void cpu_offline_md(void); 124 125 struct cpu_softc { 126 device_t sc_dev; /* device tree glue */ 127 struct cpu_info *sc_info; /* pointer to CPU info */ 128 bool sc_wasonline; 129 }; 130 131 int mp_cpu_start(struct cpu_info *, paddr_t); 132 void mp_cpu_start_cleanup(struct cpu_info *); 133 const struct cpu_functions mp_cpu_funcs = { mp_cpu_start, NULL, 134 mp_cpu_start_cleanup }; 135 136 CFATTACH_DECL_NEW(cpu, sizeof(struct cpu_softc), 137 cpu_match, cpu_attach, NULL, NULL); 138 CFATTACH_DECL_NEW(vcpu, sizeof(struct cpu_softc), 139 vcpu_match, vcpu_attach, NULL, NULL); 140 141 /* 142 * Statically-allocated CPU info for the primary CPU (or the only 143 * CPU, on uniprocessors). The CPU info list is initialized to 144 * point at it. 145 */ 146 #ifdef TRAPLOG 147 #include <machine/tlog.h> 148 struct tlog tlog_primary; 149 #endif 150 struct cpu_info cpu_info_primary __aligned(CACHE_LINE_SIZE) = { 151 .ci_dev = 0, 152 .ci_self = &cpu_info_primary, 153 .ci_idepth = -1, 154 .ci_curlwp = &lwp0, 155 .ci_curldt = -1, 156 #ifdef TRAPLOG 157 .ci_tlog = &tlog_primary, 158 #endif 159 160 }; 161 struct cpu_info phycpu_info_primary __aligned(CACHE_LINE_SIZE) = { 162 .ci_dev = 0, 163 .ci_self = &phycpu_info_primary, 164 }; 165 166 struct cpu_info *cpu_info_list = &cpu_info_primary; 167 struct cpu_info *phycpu_info_list = &phycpu_info_primary; 168 169 static void cpu_set_tss_gates(struct cpu_info *ci); 170 171 uint32_t cpus_attached = 0; 172 uint32_t cpus_running = 0; 173 174 uint32_t phycpus_attached = 0; 175 uint32_t phycpus_running = 0; 176 177 uint32_t cpu_feature[5]; /* X86 CPUID feature bits 178 * [0] basic features %edx 179 * [1] basic features %ecx 180 * [2] extended features %edx 181 * [3] extended features %ecx 182 * [4] VIA padlock features 183 */ 184 185 bool x86_mp_online; 186 paddr_t mp_trampoline_paddr = MP_TRAMPOLINE; 187 188 #if defined(MULTIPROCESSOR) 189 void cpu_hatch(void *); 190 static void cpu_boot_secondary(struct cpu_info *ci); 191 static void cpu_start_secondary(struct cpu_info *ci); 192 static void cpu_copy_trampoline(void); 193 194 /* 195 * Runs once per boot once multiprocessor goo has been detected and 196 * the local APIC on the boot processor has been mapped. 197 * 198 * Called from lapic_boot_init() (from mpbios_scan()). 199 */ 200 void 201 cpu_init_first(void) 202 { 203 204 cpu_info_primary.ci_cpuid = lapic_cpu_number(); 205 cpu_copy_trampoline(); 206 } 207 #endif /* MULTIPROCESSOR */ 208 209 int 210 cpu_match(device_t parent, cfdata_t match, void *aux) 211 { 212 213 return 1; 214 } 215 216 void 217 cpu_attach(device_t parent, device_t self, void *aux) 218 { 219 struct cpu_softc *sc = device_private(self); 220 struct cpu_attach_args *caa = aux; 221 struct cpu_info *ci; 222 uintptr_t ptr; 223 static int nphycpu = 0; 224 225 sc->sc_dev = self; 226 227 if (phycpus_attached == ~0) { 228 aprint_error(": increase MAXCPUS\n"); 229 return; 230 } 231 232 /* 233 * If we're an Application Processor, allocate a cpu_info 234 * If we're the first attached CPU use the primary cpu_info, 235 * otherwise allocate a new one 236 */ 237 aprint_naive("\n"); 238 aprint_normal("\n"); 239 if (nphycpu > 0) { 240 struct cpu_info *tmp; 241 ptr = (uintptr_t)kmem_zalloc(sizeof(*ci) + CACHE_LINE_SIZE - 1, 242 KM_SLEEP); 243 ci = (struct cpu_info *)roundup2(ptr, CACHE_LINE_SIZE); 244 ci->ci_curldt = -1; 245 246 tmp = phycpu_info_list; 247 while (tmp->ci_next) 248 tmp = tmp->ci_next; 249 250 tmp->ci_next = ci; 251 } else { 252 ci = &phycpu_info_primary; 253 } 254 255 ci->ci_self = ci; 256 sc->sc_info = ci; 257 258 ci->ci_dev = self; 259 ci->ci_acpiid = caa->cpu_id; 260 ci->ci_cpuid = caa->cpu_number; 261 ci->ci_vcpu = NULL; 262 ci->ci_index = nphycpu++; 263 ci->ci_cpumask = (1 << cpu_index(ci)); 264 265 atomic_or_32(&phycpus_attached, ci->ci_cpumask); 266 267 if (!pmf_device_register(self, NULL, NULL)) 268 aprint_error_dev(self, "couldn't establish power handler\n"); 269 270 return; 271 } 272 273 int 274 vcpu_match(device_t parent, cfdata_t match, void *aux) 275 { 276 struct vcpu_attach_args *vcaa = aux; 277 278 if (strcmp(vcaa->vcaa_name, match->cf_name) == 0) 279 return 1; 280 return 0; 281 } 282 283 void 284 vcpu_attach(device_t parent, device_t self, void *aux) 285 { 286 struct vcpu_attach_args *vcaa = aux; 287 288 cpu_attach_common(parent, self, &vcaa->vcaa_caa); 289 290 if (!pmf_device_register(self, NULL, NULL)) 291 aprint_error_dev(self, "couldn't establish power handler\n"); 292 } 293 294 static void 295 cpu_vm_init(struct cpu_info *ci) 296 { 297 int ncolors = 2, i; 298 299 for (i = CAI_ICACHE; i <= CAI_L2CACHE; i++) { 300 struct x86_cache_info *cai; 301 int tcolors; 302 303 cai = &ci->ci_cinfo[i]; 304 305 tcolors = atop(cai->cai_totalsize); 306 switch(cai->cai_associativity) { 307 case 0xff: 308 tcolors = 1; /* fully associative */ 309 break; 310 case 0: 311 case 1: 312 break; 313 default: 314 tcolors /= cai->cai_associativity; 315 } 316 ncolors = max(ncolors, tcolors); 317 } 318 319 /* 320 * Knowing the size of the largest cache on this CPU, re-color 321 * our pages. 322 */ 323 if (ncolors <= uvmexp.ncolors) 324 return; 325 aprint_debug_dev(ci->ci_dev, "%d page colors\n", ncolors); 326 uvm_page_recolor(ncolors); 327 } 328 329 void 330 cpu_attach_common(device_t parent, device_t self, void *aux) 331 { 332 struct cpu_softc *sc = device_private(self); 333 struct cpu_attach_args *caa = aux; 334 struct cpu_info *ci; 335 uintptr_t ptr; 336 int cpunum = caa->cpu_number; 337 static bool again = false; 338 339 sc->sc_dev = self; 340 341 /* 342 * If we're an Application Processor, allocate a cpu_info 343 * structure, otherwise use the primary's. 344 */ 345 if (caa->cpu_role == CPU_ROLE_AP) { 346 aprint_naive(": Application Processor\n"); 347 ptr = (uintptr_t)kmem_alloc(sizeof(*ci) + CACHE_LINE_SIZE - 1, 348 KM_SLEEP); 349 ci = (struct cpu_info *)roundup2(ptr, CACHE_LINE_SIZE); 350 memset(ci, 0, sizeof(*ci)); 351 #ifdef TRAPLOG 352 ci->ci_tlog_base = kmem_zalloc(sizeof(struct tlog), KM_SLEEP); 353 #endif 354 } else { 355 aprint_naive(": %s Processor\n", 356 caa->cpu_role == CPU_ROLE_SP ? "Single" : "Boot"); 357 ci = &cpu_info_primary; 358 #if NLAPIC > 0 359 if (cpunum != lapic_cpu_number()) { 360 /* XXX should be done earlier */ 361 uint32_t reg; 362 aprint_verbose("\n"); 363 aprint_verbose_dev(self, "running CPU at apic %d" 364 " instead of at expected %d", lapic_cpu_number(), 365 cpunum); 366 reg = i82489_readreg(LAPIC_ID); 367 i82489_writereg(LAPIC_ID, (reg & ~LAPIC_ID_MASK) | 368 (cpunum << LAPIC_ID_SHIFT)); 369 } 370 if (cpunum != lapic_cpu_number()) { 371 aprint_error_dev(self, "unable to reset apic id\n"); 372 } 373 #endif 374 } 375 376 ci->ci_self = ci; 377 sc->sc_info = ci; 378 ci->ci_dev = self; 379 ci->ci_cpuid = cpunum; 380 381 KASSERT(HYPERVISOR_shared_info != NULL); 382 ci->ci_vcpu = &HYPERVISOR_shared_info->vcpu_info[cpunum]; 383 384 ci->ci_func = caa->cpu_func; 385 386 /* Must be called before mi_cpu_attach(). */ 387 cpu_vm_init(ci); 388 389 if (caa->cpu_role == CPU_ROLE_AP) { 390 int error; 391 392 error = mi_cpu_attach(ci); 393 if (error != 0) { 394 aprint_normal("\n"); 395 aprint_error_dev(self, 396 "mi_cpu_attach failed with %d\n", error); 397 return; 398 } 399 } else { 400 KASSERT(ci->ci_data.cpu_idlelwp != NULL); 401 } 402 403 ci->ci_cpumask = (1 << cpu_index(ci)); 404 pmap_reference(pmap_kernel()); 405 ci->ci_pmap = pmap_kernel(); 406 ci->ci_tlbstate = TLBSTATE_STALE; 407 408 /* 409 * Boot processor may not be attached first, but the below 410 * must be done to allow booting other processors. 411 */ 412 if (!again) { 413 atomic_or_32(&ci->ci_flags, CPUF_PRESENT | CPUF_PRIMARY); 414 /* Basic init. */ 415 cpu_intr_init(ci); 416 cpu_get_tsc_freq(ci); 417 cpu_init(ci); 418 cpu_set_tss_gates(ci); 419 pmap_cpu_init_late(ci); 420 #if NLAPIC > 0 421 if (caa->cpu_role != CPU_ROLE_SP) { 422 /* Enable lapic. */ 423 lapic_enable(); 424 lapic_set_lvt(); 425 lapic_calibrate_timer(); 426 } 427 #endif 428 /* Make sure DELAY() is initialized. */ 429 DELAY(1); 430 again = true; 431 } 432 433 /* further PCB init done later. */ 434 435 switch (caa->cpu_role) { 436 case CPU_ROLE_SP: 437 atomic_or_32(&ci->ci_flags, CPUF_SP); 438 cpu_identify(ci); 439 #if 0 440 x86_errata(); 441 #endif 442 x86_cpu_idle_init(); 443 break; 444 445 case CPU_ROLE_BP: 446 atomic_or_32(&ci->ci_flags, CPUF_BSP); 447 cpu_identify(ci); 448 cpu_init(ci); 449 #if 0 450 x86_errata(); 451 #endif 452 x86_cpu_idle_init(); 453 break; 454 455 case CPU_ROLE_AP: 456 /* 457 * report on an AP 458 */ 459 460 #if defined(MULTIPROCESSOR) 461 cpu_intr_init(ci); 462 gdt_alloc_cpu(ci); 463 cpu_set_tss_gates(ci); 464 pmap_cpu_init_early(ci); 465 pmap_cpu_init_late(ci); 466 cpu_start_secondary(ci); 467 if (ci->ci_flags & CPUF_PRESENT) { 468 struct cpu_info *tmp; 469 470 identifycpu(ci); 471 tmp = cpu_info_list; 472 while (tmp->ci_next) 473 tmp = tmp->ci_next; 474 475 tmp->ci_next = ci; 476 } 477 #else 478 aprint_error_dev(self, "not started\n"); 479 #endif 480 break; 481 482 default: 483 aprint_normal("\n"); 484 panic("unknown processor type??\n"); 485 } 486 487 pat_init(ci); 488 atomic_or_32(&cpus_attached, ci->ci_cpumask); 489 490 #if 0 491 if (!pmf_device_register(self, cpu_suspend, cpu_resume)) 492 aprint_error_dev(self, "couldn't establish power handler\n"); 493 #endif 494 495 #if defined(MULTIPROCESSOR) 496 if (mp_verbose) { 497 struct lwp *l = ci->ci_data.cpu_idlelwp; 498 struct pcb *pcb = lwp_getpcb(l); 499 500 aprint_verbose_dev(self, 501 "idle lwp at %p, idle sp at 0x%p\n", 502 l, 503 #ifdef i386 504 (void *)pcb->pcb_esp 505 #else 506 (void *)pcb->pcb_rsp 507 #endif 508 ); 509 510 } 511 #endif 512 } 513 514 /* 515 * Initialize the processor appropriately. 516 */ 517 518 void 519 cpu_init(struct cpu_info *ci) 520 { 521 522 /* 523 * On a P6 or above, enable global TLB caching if the 524 * hardware supports it. 525 */ 526 if (cpu_feature[0] & CPUID_PGE) 527 lcr4(rcr4() | CR4_PGE); /* enable global TLB caching */ 528 529 #ifdef XXXMTRR 530 /* 531 * On a P6 or above, initialize MTRR's if the hardware supports them. 532 */ 533 if (cpu_feature[0] & CPUID_MTRR) { 534 if ((ci->ci_flags & CPUF_AP) == 0) 535 i686_mtrr_init_first(); 536 mtrr_init_cpu(ci); 537 } 538 #endif 539 /* 540 * If we have FXSAVE/FXRESTOR, use them. 541 */ 542 if (cpu_feature[0] & CPUID_FXSR) { 543 lcr4(rcr4() | CR4_OSFXSR); 544 545 /* 546 * If we have SSE/SSE2, enable XMM exceptions. 547 */ 548 if (cpu_feature[0] & (CPUID_SSE|CPUID_SSE2)) 549 lcr4(rcr4() | CR4_OSXMMEXCPT); 550 } 551 552 #ifdef __x86_64__ 553 /* No user PGD mapped for this CPU yet */ 554 ci->ci_xen_current_user_pgd = 0; 555 #endif 556 557 atomic_or_32(&cpus_running, ci->ci_cpumask); 558 atomic_or_32(&ci->ci_flags, CPUF_RUNNING); 559 } 560 561 562 #ifdef MULTIPROCESSOR 563 void 564 cpu_boot_secondary_processors(void) 565 { 566 struct cpu_info *ci; 567 u_long i; 568 569 for (i = 0; i < maxcpus; i++) { 570 ci = cpu_lookup(i); 571 if (ci == NULL) 572 continue; 573 if (ci->ci_data.cpu_idlelwp == NULL) 574 continue; 575 if ((ci->ci_flags & CPUF_PRESENT) == 0) 576 continue; 577 if (ci->ci_flags & (CPUF_BSP|CPUF_SP|CPUF_PRIMARY)) 578 continue; 579 cpu_boot_secondary(ci); 580 } 581 582 x86_mp_online = true; 583 } 584 585 static void 586 cpu_init_idle_lwp(struct cpu_info *ci) 587 { 588 struct lwp *l = ci->ci_data.cpu_idlelwp; 589 struct pcb *pcb = lwp_getpcb(l); 590 591 pcb->pcb_cr0 = rcr0(); 592 } 593 594 void 595 cpu_init_idle_lwps(void) 596 { 597 struct cpu_info *ci; 598 u_long i; 599 600 for (i = 0; i < maxcpus; i++) { 601 ci = cpu_lookup(i); 602 if (ci == NULL) 603 continue; 604 if (ci->ci_data.cpu_idlelwp == NULL) 605 continue; 606 if ((ci->ci_flags & CPUF_PRESENT) == 0) 607 continue; 608 cpu_init_idle_lwp(ci); 609 } 610 } 611 612 void 613 cpu_start_secondary(struct cpu_info *ci) 614 { 615 int i; 616 struct pmap *kpm = pmap_kernel(); 617 extern uint32_t mp_pdirpa; 618 619 mp_pdirpa = kpm->pm_pdirpa; /* XXX move elsewhere, not per CPU. */ 620 621 atomic_or_32(&ci->ci_flags, CPUF_AP); 622 623 aprint_debug_dev(ci->ci_dev, "starting\n"); 624 625 ci->ci_curlwp = ci->ci_data.cpu_idlelwp; 626 if (CPU_STARTUP(ci, mp_trampoline_paddr) != 0) 627 return; 628 629 /* 630 * wait for it to become ready 631 */ 632 for (i = 100000; (!(ci->ci_flags & CPUF_PRESENT)) && i > 0; i--) { 633 #ifdef MPDEBUG 634 extern int cpu_trace[3]; 635 static int otrace[3]; 636 if (memcmp(otrace, cpu_trace, sizeof(otrace)) != 0) { 637 aprint_debug_dev(ci->ci_dev, "trace %02x %02x %02x\n", 638 cpu_trace[0], cpu_trace[1], cpu_trace[2]); 639 memcpy(otrace, cpu_trace, sizeof(otrace)); 640 } 641 #endif 642 delay(10); 643 } 644 if ((ci->ci_flags & CPUF_PRESENT) == 0) { 645 aprint_error_dev(ci->ci_dev, "failed to become ready\n"); 646 #if defined(MPDEBUG) && defined(DDB) 647 printf("dropping into debugger; continue from here to resume boot\n"); 648 Debugger(); 649 #endif 650 } 651 652 CPU_START_CLEANUP(ci); 653 } 654 655 void 656 cpu_boot_secondary(struct cpu_info *ci) 657 { 658 int i; 659 660 atomic_or_32(&ci->ci_flags, CPUF_GO); 661 for (i = 100000; (!(ci->ci_flags & CPUF_RUNNING)) && i > 0; i--) { 662 delay(10); 663 } 664 if ((ci->ci_flags & CPUF_RUNNING) == 0) { 665 aprint_error_dev(ci->ci_dev, "CPU failed to start\n"); 666 #if defined(MPDEBUG) && defined(DDB) 667 printf("dropping into debugger; continue from here to resume boot\n"); 668 Debugger(); 669 #endif 670 } 671 } 672 673 /* 674 * The CPU ends up here when its ready to run 675 * This is called from code in mptramp.s; at this point, we are running 676 * in the idle pcb/idle stack of the new CPU. When this function returns, 677 * this processor will enter the idle loop and start looking for work. 678 * 679 * XXX should share some of this with init386 in machdep.c 680 */ 681 void 682 cpu_hatch(void *v) 683 { 684 struct cpu_info *ci = (struct cpu_info *)v; 685 struct pcb *pcb; 686 int s, i; 687 688 cpu_probe(ci); 689 690 cpu_feature[0] &= ~CPUID_FEAT_BLACKLIST; 691 cpu_feature[2] &= ~CPUID_FEAT_EXT_BLACKLIST; 692 693 cpu_init_msrs(ci, true); 694 695 KDASSERT((ci->ci_flags & CPUF_PRESENT) == 0); 696 atomic_or_32(&ci->ci_flags, CPUF_PRESENT); 697 while ((ci->ci_flags & CPUF_GO) == 0) { 698 /* Don't use delay, boot CPU may be patching the text. */ 699 for (i = 10000; i != 0; i--) 700 x86_pause(); 701 } 702 703 /* Because the text may have been patched in x86_patch(). */ 704 wbinvd(); 705 x86_flush(); 706 707 KASSERT((ci->ci_flags & CPUF_RUNNING) == 0); 708 709 pcb = lwp_getpcb(curlwp); 710 lcr3(pmap_kernel()->pm_pdirpa); 711 pcb->pcb_cr3 = pmap_kernel()->pm_pdirpa; 712 pcb = lwp_getpcb(ci->ci_data.cpu_idlelwp); 713 lcr0(pcb->pcb_cr0); 714 715 cpu_init_idt(); 716 gdt_init_cpu(ci); 717 lapic_enable(); 718 lapic_set_lvt(); 719 lapic_initclocks(); 720 721 #ifdef i386 722 npxinit(ci); 723 #else 724 fpuinit(ci); 725 #endif 726 727 lldt(GSEL(GLDT_SEL, SEL_KPL)); 728 ltr(ci->ci_tss_sel); 729 730 cpu_init(ci); 731 cpu_get_tsc_freq(ci); 732 733 s = splhigh(); 734 #ifdef i386 735 lapic_tpr = 0; 736 #else 737 lcr8(0); 738 #endif 739 x86_enable_intr(); 740 splx(s); 741 #if 0 742 x86_errata(); 743 #endif 744 745 aprint_debug_dev(ci->ci_dev, "CPU %ld running\n", 746 (long)ci->ci_cpuid); 747 } 748 749 #if defined(DDB) 750 751 #include <ddb/db_output.h> 752 #include <machine/db_machdep.h> 753 754 /* 755 * Dump CPU information from ddb. 756 */ 757 void 758 cpu_debug_dump(void) 759 { 760 struct cpu_info *ci; 761 CPU_INFO_ITERATOR cii; 762 763 db_printf("addr dev id flags ipis curlwp fpcurlwp\n"); 764 for (CPU_INFO_FOREACH(cii, ci)) { 765 db_printf("%p %s %ld %x %x %10p %10p\n", 766 ci, 767 ci->ci_dev == NULL ? "BOOT" : device_xname(ci->ci_dev), 768 (long)ci->ci_cpuid, 769 ci->ci_flags, ci->ci_ipis, 770 ci->ci_curlwp, 771 ci->ci_fpcurlwp); 772 } 773 } 774 #endif /* DDB */ 775 776 static void 777 cpu_copy_trampoline(void) 778 { 779 /* 780 * Copy boot code. 781 */ 782 extern u_char cpu_spinup_trampoline[]; 783 extern u_char cpu_spinup_trampoline_end[]; 784 785 vaddr_t mp_trampoline_vaddr; 786 787 mp_trampoline_vaddr = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, 788 UVM_KMF_VAONLY); 789 790 pmap_kenter_pa(mp_trampoline_vaddr, mp_trampoline_paddr, 791 VM_PROT_READ | VM_PROT_WRITE, 0); 792 pmap_update(pmap_kernel()); 793 memcpy((void *)mp_trampoline_vaddr, 794 cpu_spinup_trampoline, 795 cpu_spinup_trampoline_end - cpu_spinup_trampoline); 796 797 pmap_kremove(mp_trampoline_vaddr, PAGE_SIZE); 798 pmap_update(pmap_kernel()); 799 uvm_km_free(kernel_map, mp_trampoline_vaddr, PAGE_SIZE, UVM_KMF_VAONLY); 800 } 801 802 #endif /* MULTIPROCESSOR */ 803 804 #ifdef i386 805 #if 0 806 static void 807 tss_init(struct i386tss *tss, void *stack, void *func) 808 { 809 memset(tss, 0, sizeof *tss); 810 tss->tss_esp0 = tss->tss_esp = (int)((char *)stack + USPACE - 16); 811 tss->tss_ss0 = GSEL(GDATA_SEL, SEL_KPL); 812 tss->__tss_cs = GSEL(GCODE_SEL, SEL_KPL); 813 tss->tss_fs = GSEL(GCPU_SEL, SEL_KPL); 814 tss->tss_gs = tss->__tss_es = tss->__tss_ds = 815 tss->__tss_ss = GSEL(GDATA_SEL, SEL_KPL); 816 tss->tss_cr3 = pmap_kernel()->pm_pdirpa; 817 tss->tss_esp = (int)((char *)stack + USPACE - 16); 818 tss->tss_ldt = GSEL(GLDT_SEL, SEL_KPL); 819 tss->__tss_eflags = PSL_MBO | PSL_NT; /* XXX not needed? */ 820 tss->__tss_eip = (int)func; 821 } 822 #endif 823 824 /* XXX */ 825 #define IDTVEC(name) __CONCAT(X, name) 826 typedef void (vector)(void); 827 extern vector IDTVEC(tss_trap08); 828 #ifdef DDB 829 extern vector Xintrddbipi; 830 extern int ddb_vec; 831 #endif 832 833 static void 834 cpu_set_tss_gates(struct cpu_info *ci) 835 { 836 #if 0 837 struct segment_descriptor sd; 838 839 ci->ci_doubleflt_stack = (char *)uvm_km_alloc(kernel_map, USPACE, 0, 840 UVM_KMF_WIRED); 841 tss_init(&ci->ci_doubleflt_tss, ci->ci_doubleflt_stack, 842 IDTVEC(tss_trap08)); 843 setsegment(&sd, &ci->ci_doubleflt_tss, sizeof(struct i386tss) - 1, 844 SDT_SYS386TSS, SEL_KPL, 0, 0); 845 ci->ci_gdt[GTRAPTSS_SEL].sd = sd; 846 setgate(&idt[8], NULL, 0, SDT_SYSTASKGT, SEL_KPL, 847 GSEL(GTRAPTSS_SEL, SEL_KPL)); 848 #endif 849 850 #if defined(DDB) && defined(MULTIPROCESSOR) 851 /* 852 * Set up separate handler for the DDB IPI, so that it doesn't 853 * stomp on a possibly corrupted stack. 854 * 855 * XXX overwriting the gate set in db_machine_init. 856 * Should rearrange the code so that it's set only once. 857 */ 858 ci->ci_ddbipi_stack = (char *)uvm_km_alloc(kernel_map, USPACE, 0, 859 UVM_KMF_WIRED); 860 tss_init(&ci->ci_ddbipi_tss, ci->ci_ddbipi_stack, 861 Xintrddbipi); 862 863 setsegment(&sd, &ci->ci_ddbipi_tss, sizeof(struct i386tss) - 1, 864 SDT_SYS386TSS, SEL_KPL, 0, 0); 865 ci->ci_gdt[GIPITSS_SEL].sd = sd; 866 867 setgate(&idt[ddb_vec], NULL, 0, SDT_SYSTASKGT, SEL_KPL, 868 GSEL(GIPITSS_SEL, SEL_KPL)); 869 #endif 870 } 871 #else 872 static void 873 cpu_set_tss_gates(struct cpu_info *ci) 874 { 875 876 } 877 #endif /* i386 */ 878 879 int 880 mp_cpu_start(struct cpu_info *ci, paddr_t target) 881 { 882 #if 0 883 #if NLAPIC > 0 884 int error; 885 #endif 886 unsigned short dwordptr[2]; 887 888 /* 889 * Bootstrap code must be addressable in real mode 890 * and it must be page aligned. 891 */ 892 KASSERT(target < 0x10000 && target % PAGE_SIZE == 0); 893 894 /* 895 * "The BSP must initialize CMOS shutdown code to 0Ah ..." 896 */ 897 898 outb(IO_RTC, NVRAM_RESET); 899 outb(IO_RTC+1, NVRAM_RESET_JUMP); 900 901 /* 902 * "and the warm reset vector (DWORD based at 40:67) to point 903 * to the AP startup code ..." 904 */ 905 906 dwordptr[0] = 0; 907 dwordptr[1] = target >> 4; 908 909 pmap_kenter_pa (0, 0, VM_PROT_READ|VM_PROT_WRITE, 0); 910 pmap_update(pmap_kernel()); 911 912 memcpy ((uint8_t *) 0x467, dwordptr, 4); 913 914 pmap_kremove (0, PAGE_SIZE); 915 pmap_update(pmap_kernel()); 916 917 #if NLAPIC > 0 918 /* 919 * ... prior to executing the following sequence:" 920 */ 921 922 if (ci->ci_flags & CPUF_AP) { 923 if ((error = x86_ipi_init(ci->ci_cpuid)) != 0) 924 return error; 925 926 delay(10000); 927 928 if (cpu_feature & CPUID_APIC) { 929 error = x86_ipi_init(ci->ci_cpuid); 930 if (error != 0) { 931 aprint_error_dev(ci->ci_dev, "%s: IPI not taken (1)\n", 932 __func__); 933 return error; 934 } 935 936 delay(10000); 937 938 error = x86_ipi(target / PAGE_SIZE, ci->ci_cpuid, 939 LAPIC_DLMODE_STARTUP); 940 if (error != 0) { 941 aprint_error_dev(ci->ci_dev, "%s: IPI not taken (2)\n", 942 __func__); 943 return error; 944 } 945 delay(200); 946 947 error = x86_ipi(target / PAGE_SIZE, ci->ci_cpuid, 948 LAPIC_DLMODE_STARTUP); 949 if (error != 0) { 950 aprint_error_dev(ci->ci_dev, "%s: IPI not taken ((3)\n", 951 __func__); 952 return error; 953 } 954 delay(200); 955 } 956 } 957 #endif 958 #endif /* 0 */ 959 return 0; 960 } 961 962 void 963 mp_cpu_start_cleanup(struct cpu_info *ci) 964 { 965 #if 0 966 /* 967 * Ensure the NVRAM reset byte contains something vaguely sane. 968 */ 969 970 outb(IO_RTC, NVRAM_RESET); 971 outb(IO_RTC+1, NVRAM_RESET_RST); 972 #endif 973 } 974 975 void 976 cpu_init_msrs(struct cpu_info *ci, bool full) 977 { 978 #ifdef __x86_64__ 979 if (full) { 980 HYPERVISOR_set_segment_base (SEGBASE_FS, 0); 981 HYPERVISOR_set_segment_base (SEGBASE_GS_KERNEL, (uint64_t) ci); 982 HYPERVISOR_set_segment_base (SEGBASE_GS_USER, 0); 983 } 984 #endif /* __x86_64__ */ 985 986 if (cpu_feature[2] & CPUID_NOX) 987 wrmsr(MSR_EFER, rdmsr(MSR_EFER) | EFER_NXE); 988 } 989 990 void 991 cpu_offline_md(void) 992 { 993 int s; 994 995 s = splhigh(); 996 #ifdef __i386__ 997 npxsave_cpu(true); 998 #else 999 fpusave_cpu(true); 1000 #endif 1001 splx(s); 1002 } 1003 1004 #if 0 1005 /* XXX joerg restructure and restart CPUs individually */ 1006 static bool 1007 cpu_suspend(device_t dv, const pmf_qual_t *qual) 1008 { 1009 struct cpu_softc *sc = device_private(dv); 1010 struct cpu_info *ci = sc->sc_info; 1011 int err; 1012 1013 if (ci->ci_flags & CPUF_PRIMARY) 1014 return true; 1015 if (ci->ci_data.cpu_idlelwp == NULL) 1016 return true; 1017 if ((ci->ci_flags & CPUF_PRESENT) == 0) 1018 return true; 1019 1020 sc->sc_wasonline = !(ci->ci_schedstate.spc_flags & SPCF_OFFLINE); 1021 1022 if (sc->sc_wasonline) { 1023 mutex_enter(&cpu_lock); 1024 err = cpu_setstate(ci, false); 1025 mutex_exit(&cpu_lock); 1026 1027 if (err) 1028 return false; 1029 } 1030 1031 return true; 1032 } 1033 1034 static bool 1035 cpu_resume(device_t dv, const pmf_qual_t *qual) 1036 { 1037 struct cpu_softc *sc = device_private(dv); 1038 struct cpu_info *ci = sc->sc_info; 1039 int err = 0; 1040 1041 if (ci->ci_flags & CPUF_PRIMARY) 1042 return true; 1043 if (ci->ci_data.cpu_idlelwp == NULL) 1044 return true; 1045 if ((ci->ci_flags & CPUF_PRESENT) == 0) 1046 return true; 1047 1048 if (sc->sc_wasonline) { 1049 mutex_enter(&cpu_lock); 1050 err = cpu_setstate(ci, true); 1051 mutex_exit(&cpu_lock); 1052 } 1053 1054 return err == 0; 1055 } 1056 #endif 1057 1058 void 1059 cpu_get_tsc_freq(struct cpu_info *ci) 1060 { 1061 const volatile vcpu_time_info_t *tinfo = &ci->ci_vcpu->time; 1062 delay(1000000); 1063 uint64_t freq = 1000000000ULL << 32; 1064 freq = freq / (uint64_t)tinfo->tsc_to_system_mul; 1065 if ( tinfo->tsc_shift < 0 ) 1066 freq = freq << -tinfo->tsc_shift; 1067 else 1068 freq = freq >> tinfo->tsc_shift; 1069 ci->ci_data.cpu_cc_freq = freq; 1070 } 1071 1072 void 1073 x86_cpu_idle_xen(void) 1074 { 1075 struct cpu_info *ci = curcpu(); 1076 1077 KASSERT(ci->ci_ilevel == IPL_NONE); 1078 1079 x86_disable_intr(); 1080 if (!__predict_false(ci->ci_want_resched)) { 1081 idle_block(); 1082 } else { 1083 x86_enable_intr(); 1084 } 1085 } 1086 1087 /* 1088 * Loads pmap for the current CPU. 1089 */ 1090 void 1091 cpu_load_pmap(struct pmap *pmap) 1092 { 1093 #ifdef i386 1094 #ifdef PAE 1095 int i, s; 1096 struct cpu_info *ci; 1097 1098 s = splvm(); /* just to be safe */ 1099 ci = curcpu(); 1100 paddr_t l3_pd = xpmap_ptom_masked(ci->ci_pae_l3_pdirpa); 1101 /* don't update the kernel L3 slot */ 1102 for (i = 0 ; i < PDP_SIZE - 1; i++) { 1103 xpq_queue_pte_update(l3_pd + i * sizeof(pd_entry_t), 1104 xpmap_ptom(pmap->pm_pdirpa[i]) | PG_V); 1105 } 1106 splx(s); 1107 tlbflush(); 1108 #else /* PAE */ 1109 lcr3(pmap_pdirpa(pmap, 0)); 1110 #endif /* PAE */ 1111 #endif /* i386 */ 1112 1113 #ifdef __x86_64__ 1114 int i, s; 1115 pd_entry_t *old_pgd, *new_pgd; 1116 paddr_t addr; 1117 struct cpu_info *ci; 1118 1119 /* kernel pmap always in cr3 and should never go in user cr3 */ 1120 if (pmap_pdirpa(pmap, 0) != pmap_pdirpa(pmap_kernel(), 0)) { 1121 ci = curcpu(); 1122 /* 1123 * Map user space address in kernel space and load 1124 * user cr3 1125 */ 1126 s = splvm(); 1127 new_pgd = pmap->pm_pdir; 1128 old_pgd = pmap_kernel()->pm_pdir; 1129 addr = xpmap_ptom(pmap_pdirpa(pmap_kernel(), 0)); 1130 for (i = 0; i < PDIR_SLOT_PTE; 1131 i++, addr += sizeof(pd_entry_t)) { 1132 if ((new_pgd[i] & PG_V) || (old_pgd[i] & PG_V)) 1133 xpq_queue_pte_update(addr, new_pgd[i]); 1134 } 1135 tlbflush(); 1136 xen_set_user_pgd(pmap_pdirpa(pmap, 0)); 1137 ci->ci_xen_current_user_pgd = pmap_pdirpa(pmap, 0); 1138 splx(s); 1139 } 1140 #endif /* __x86_64__ */ 1141 } 1142
Indexes created Fri Oct 03 02:09:57 GMT 2025