cpu.c revision 1.48
1 /* $NetBSD: cpu.c,v 1.48 2010/08/09 15:46:18 jruoho Exp $ */ 2 /* NetBSD: cpu.c,v 1.18 2004/02/20 17:35:01 yamt Exp */ 3 4 /*- 5 * Copyright (c) 2000 The NetBSD Foundation, Inc. 6 * Copyright (c) 2002, 2006, 2007 YAMAMOTO Takashi, 7 * All rights reserved. 8 * 9 * This code is derived from software contributed to The NetBSD Foundation 10 * by RedBack Networks Inc. 11 * 12 * Author: Bill Sommerfeld 13 * 14 * Redistribution and use in source and binary forms, with or without 15 * modification, are permitted provided that the following conditions 16 * are met: 17 * 1. Redistributions of source code must retain the above copyright 18 * notice, this list of conditions and the following disclaimer. 19 * 2. Redistributions in binary form must reproduce the above copyright 20 * notice, this list of conditions and the following disclaimer in the 21 * documentation and/or other materials provided with the distribution. 22 * 23 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 24 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 25 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 26 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 27 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 28 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 29 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 30 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 31 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 32 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 33 * POSSIBILITY OF SUCH DAMAGE. 34 */ 35 36 /* 37 * Copyright (c) 1999 Stefan Grefen 38 * 39 * Redistribution and use in source and binary forms, with or without 40 * modification, are permitted provided that the following conditions 41 * are met: 42 * 1. Redistributions of source code must retain the above copyright 43 * notice, this list of conditions and the following disclaimer. 44 * 2. Redistributions in binary form must reproduce the above copyright 45 * notice, this list of conditions and the following disclaimer in the 46 * documentation and/or other materials provided with the distribution. 47 * 3. All advertising materials mentioning features or use of this software 48 * must display the following acknowledgement: 49 * This product includes software developed by the NetBSD 50 * Foundation, Inc. and its contributors. 51 * 4. Neither the name of The NetBSD Foundation nor the names of its 52 * contributors may be used to endorse or promote products derived 53 * from this software without specific prior written permission. 54 * 55 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND ANY 56 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 57 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 58 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR AND CONTRIBUTORS BE LIABLE 59 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 60 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 61 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 62 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 63 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 64 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 65 * SUCH DAMAGE. 66 */ 67 68 #include <sys/cdefs.h> 69 __KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.48 2010/08/09 15:46:18 jruoho Exp $"); 70 71 #include "opt_ddb.h" 72 #include "opt_multiprocessor.h" 73 #include "opt_mpbios.h" /* for MPDEBUG */ 74 #include "opt_mtrr.h" 75 #include "opt_xen.h" 76 77 #include "lapic.h" 78 #include "ioapic.h" 79 80 #include <sys/param.h> 81 #include <sys/proc.h> 82 #include <sys/systm.h> 83 #include <sys/device.h> 84 #include <sys/kmem.h> 85 #include <sys/cpu.h> 86 #include <sys/atomic.h> 87 #include <sys/reboot.h> 88 89 #include <uvm/uvm_extern.h> 90 91 #include <machine/cpufunc.h> 92 #include <machine/cpuvar.h> 93 #include <machine/pmap.h> 94 #include <machine/vmparam.h> 95 #include <machine/mpbiosvar.h> 96 #include <machine/pcb.h> 97 #include <machine/specialreg.h> 98 #include <machine/segments.h> 99 #include <machine/gdt.h> 100 #include <machine/mtrr.h> 101 #include <machine/pio.h> 102 103 #include <xen/vcpuvar.h> 104 105 #if NLAPIC > 0 106 #include <machine/apicvar.h> 107 #include <machine/i82489reg.h> 108 #include <machine/i82489var.h> 109 #endif 110 111 #include <dev/ic/mc146818reg.h> 112 #include <dev/isa/isareg.h> 113 114 #if MAXCPUS > 32 115 #error cpu_info contains 32bit bitmasks 116 #endif 117 118 int cpu_match(device_t, cfdata_t, void *); 119 void cpu_attach(device_t, device_t, void *); 120 int vcpu_match(device_t, cfdata_t, void *); 121 void vcpu_attach(device_t, device_t, void *); 122 void cpu_attach_common(device_t, device_t, void *); 123 void cpu_offline_md(void); 124 125 struct cpu_softc { 126 device_t sc_dev; /* device tree glue */ 127 struct cpu_info *sc_info; /* pointer to CPU info */ 128 bool sc_wasonline; 129 }; 130 131 int mp_cpu_start(struct cpu_info *, paddr_t); 132 void mp_cpu_start_cleanup(struct cpu_info *); 133 const struct cpu_functions mp_cpu_funcs = { mp_cpu_start, NULL, 134 mp_cpu_start_cleanup }; 135 136 CFATTACH_DECL_NEW(cpu, sizeof(struct cpu_softc), 137 cpu_match, cpu_attach, NULL, NULL); 138 CFATTACH_DECL_NEW(vcpu, sizeof(struct cpu_softc), 139 vcpu_match, vcpu_attach, NULL, NULL); 140 141 /* 142 * Statically-allocated CPU info for the primary CPU (or the only 143 * CPU, on uniprocessors). The CPU info list is initialized to 144 * point at it. 145 */ 146 #ifdef TRAPLOG 147 #include <machine/tlog.h> 148 struct tlog tlog_primary; 149 #endif 150 struct cpu_info cpu_info_primary __aligned(CACHE_LINE_SIZE) = { 151 .ci_dev = 0, 152 .ci_self = &cpu_info_primary, 153 .ci_idepth = -1, 154 .ci_curlwp = &lwp0, 155 .ci_curldt = -1, 156 #ifdef TRAPLOG 157 .ci_tlog = &tlog_primary, 158 #endif 159 160 }; 161 struct cpu_info phycpu_info_primary __aligned(CACHE_LINE_SIZE) = { 162 .ci_dev = 0, 163 .ci_self = &phycpu_info_primary, 164 }; 165 166 struct cpu_info *cpu_info_list = &cpu_info_primary; 167 struct cpu_info *phycpu_info_list = &phycpu_info_primary; 168 169 static void cpu_set_tss_gates(struct cpu_info *ci); 170 171 uint32_t cpus_attached = 0; 172 uint32_t cpus_running = 0; 173 174 uint32_t phycpus_attached = 0; 175 uint32_t phycpus_running = 0; 176 177 uint32_t cpu_feature[5]; /* X86 CPUID feature bits 178 * [0] basic features %edx 179 * [1] basic features %ecx 180 * [2] extended features %edx 181 * [3] extended features %ecx 182 * [4] VIA padlock features 183 */ 184 185 bool x86_mp_online; 186 paddr_t mp_trampoline_paddr = MP_TRAMPOLINE; 187 188 void (*cpu_freq_init)(int) = NULL; 189 struct sysctllog *cpu_freq_sysctllog = NULL; 190 191 #if defined(MULTIPROCESSOR) 192 void cpu_hatch(void *); 193 static void cpu_boot_secondary(struct cpu_info *ci); 194 static void cpu_start_secondary(struct cpu_info *ci); 195 static void cpu_copy_trampoline(void); 196 197 /* 198 * Runs once per boot once multiprocessor goo has been detected and 199 * the local APIC on the boot processor has been mapped. 200 * 201 * Called from lapic_boot_init() (from mpbios_scan()). 202 */ 203 void 204 cpu_init_first(void) 205 { 206 207 cpu_info_primary.ci_cpuid = lapic_cpu_number(); 208 cpu_copy_trampoline(); 209 } 210 #endif /* MULTIPROCESSOR */ 211 212 int 213 cpu_match(device_t parent, cfdata_t match, void *aux) 214 { 215 216 return 1; 217 } 218 219 void 220 cpu_attach(device_t parent, device_t self, void *aux) 221 { 222 struct cpu_softc *sc = device_private(self); 223 struct cpu_attach_args *caa = aux; 224 struct cpu_info *ci; 225 uintptr_t ptr; 226 static bool again = false; 227 228 sc->sc_dev = self; 229 230 if (phycpus_attached == ~0) { 231 aprint_error(": increase MAXCPUS\n"); 232 return; 233 } 234 235 /* 236 * If we're an Application Processor, allocate a cpu_info 237 * structure, otherwise use the primary's. 238 */ 239 if (caa->cpu_role == CPU_ROLE_AP) { 240 if ((boothowto & RB_MD1) != 0) { 241 aprint_error(": multiprocessor boot disabled\n"); 242 if (!pmf_device_register(self, NULL, NULL)) 243 aprint_error_dev(self, 244 "couldn't establish power handler\n"); 245 return; 246 } 247 aprint_naive(": Application Processor\n"); 248 ptr = (uintptr_t)kmem_zalloc(sizeof(*ci) + CACHE_LINE_SIZE - 1, 249 KM_SLEEP); 250 ci = (struct cpu_info *)roundup2(ptr, CACHE_LINE_SIZE); 251 ci->ci_curldt = -1; 252 } else { 253 aprint_naive(": %s Processor\n", 254 caa->cpu_role == CPU_ROLE_SP ? "Single" : "Boot"); 255 ci = &phycpu_info_primary; 256 } 257 258 ci->ci_self = ci; 259 sc->sc_info = ci; 260 261 ci->ci_dev = self; 262 ci->ci_cpuid = caa->cpu_number; 263 ci->ci_vcpu = NULL; 264 265 /* 266 * Boot processor may not be attached first, but the below 267 * must be done to allow booting other processors. 268 */ 269 if (!again) { 270 atomic_or_32(&ci->ci_flags, CPUF_PRESENT | CPUF_PRIMARY); 271 /* Basic init */ 272 again = true; 273 } 274 275 printf(": "); 276 switch (caa->cpu_role) { 277 case CPU_ROLE_SP: 278 printf("(uniprocessor)\n"); 279 atomic_or_32(&ci->ci_flags, CPUF_SP); 280 break; 281 282 case CPU_ROLE_BP: 283 printf("(boot processor)\n"); 284 atomic_or_32(&ci->ci_flags, CPUF_BSP); 285 break; 286 287 case CPU_ROLE_AP: 288 /* 289 * report on an AP 290 */ 291 printf("(application processor)\n"); 292 if (ci->ci_flags & CPUF_PRESENT) { 293 struct cpu_info *tmp; 294 295 tmp = phycpu_info_list; 296 while (tmp->ci_next) 297 tmp = tmp->ci_next; 298 299 tmp->ci_next = ci; 300 } 301 break; 302 303 default: 304 panic("unknown processor type??\n"); 305 } 306 307 atomic_or_32(&phycpus_attached, ci->ci_cpumask); 308 309 return; 310 } 311 312 int 313 vcpu_match(device_t parent, cfdata_t match, void *aux) 314 { 315 struct vcpu_attach_args *vcaa = aux; 316 317 if (strcmp(vcaa->vcaa_name, match->cf_name) == 0) 318 return 1; 319 return 0; 320 } 321 322 void 323 vcpu_attach(device_t parent, device_t self, void *aux) 324 { 325 struct vcpu_attach_args *vcaa = aux; 326 327 cpu_attach_common(parent, self, &vcaa->vcaa_caa); 328 } 329 330 static void 331 cpu_vm_init(struct cpu_info *ci) 332 { 333 int ncolors = 2, i; 334 335 for (i = CAI_ICACHE; i <= CAI_L2CACHE; i++) { 336 struct x86_cache_info *cai; 337 int tcolors; 338 339 cai = &ci->ci_cinfo[i]; 340 341 tcolors = atop(cai->cai_totalsize); 342 switch(cai->cai_associativity) { 343 case 0xff: 344 tcolors = 1; /* fully associative */ 345 break; 346 case 0: 347 case 1: 348 break; 349 default: 350 tcolors /= cai->cai_associativity; 351 } 352 ncolors = max(ncolors, tcolors); 353 } 354 355 /* 356 * Knowing the size of the largest cache on this CPU, re-color 357 * our pages. 358 */ 359 if (ncolors <= uvmexp.ncolors) 360 return; 361 aprint_debug_dev(ci->ci_dev, "%d page colors\n", ncolors); 362 uvm_page_recolor(ncolors); 363 } 364 365 void 366 cpu_attach_common(device_t parent, device_t self, void *aux) 367 { 368 struct cpu_softc *sc = device_private(self); 369 struct cpu_attach_args *caa = aux; 370 struct cpu_info *ci; 371 uintptr_t ptr; 372 int cpunum = caa->cpu_number; 373 static bool again = false; 374 375 sc->sc_dev = self; 376 377 /* 378 * If we're an Application Processor, allocate a cpu_info 379 * structure, otherwise use the primary's. 380 */ 381 if (caa->cpu_role == CPU_ROLE_AP) { 382 aprint_naive(": Application Processor\n"); 383 ptr = (uintptr_t)kmem_alloc(sizeof(*ci) + CACHE_LINE_SIZE - 1, 384 KM_SLEEP); 385 ci = (struct cpu_info *)roundup2(ptr, CACHE_LINE_SIZE); 386 memset(ci, 0, sizeof(*ci)); 387 #ifdef TRAPLOG 388 ci->ci_tlog_base = kmem_zalloc(sizeof(struct tlog), KM_SLEEP); 389 #endif 390 } else { 391 aprint_naive(": %s Processor\n", 392 caa->cpu_role == CPU_ROLE_SP ? "Single" : "Boot"); 393 ci = &cpu_info_primary; 394 #if NLAPIC > 0 395 if (cpunum != lapic_cpu_number()) { 396 /* XXX should be done earlier */ 397 uint32_t reg; 398 aprint_verbose("\n"); 399 aprint_verbose_dev(self, "running CPU at apic %d" 400 " instead of at expected %d", lapic_cpu_number(), 401 cpunum); 402 reg = i82489_readreg(LAPIC_ID); 403 i82489_writereg(LAPIC_ID, (reg & ~LAPIC_ID_MASK) | 404 (cpunum << LAPIC_ID_SHIFT)); 405 } 406 if (cpunum != lapic_cpu_number()) { 407 aprint_error_dev(self, "unable to reset apic id\n"); 408 } 409 #endif 410 } 411 412 ci->ci_self = ci; 413 sc->sc_info = ci; 414 ci->ci_dev = self; 415 ci->ci_cpuid = cpunum; 416 417 KASSERT(HYPERVISOR_shared_info != NULL); 418 ci->ci_vcpu = &HYPERVISOR_shared_info->vcpu_info[cpunum]; 419 420 ci->ci_func = caa->cpu_func; 421 422 /* Must be called before mi_cpu_attach(). */ 423 cpu_vm_init(ci); 424 425 if (caa->cpu_role == CPU_ROLE_AP) { 426 int error; 427 428 error = mi_cpu_attach(ci); 429 if (error != 0) { 430 aprint_normal("\n"); 431 aprint_error_dev(self, 432 "mi_cpu_attach failed with %d\n", error); 433 return; 434 } 435 } else { 436 KASSERT(ci->ci_data.cpu_idlelwp != NULL); 437 } 438 439 ci->ci_cpumask = (1 << cpu_index(ci)); 440 pmap_reference(pmap_kernel()); 441 ci->ci_pmap = pmap_kernel(); 442 ci->ci_tlbstate = TLBSTATE_STALE; 443 444 /* 445 * Boot processor may not be attached first, but the below 446 * must be done to allow booting other processors. 447 */ 448 if (!again) { 449 atomic_or_32(&ci->ci_flags, CPUF_PRESENT | CPUF_PRIMARY); 450 /* Basic init. */ 451 cpu_intr_init(ci); 452 cpu_get_tsc_freq(ci); 453 cpu_init(ci); 454 cpu_set_tss_gates(ci); 455 pmap_cpu_init_late(ci); 456 #if NLAPIC > 0 457 if (caa->cpu_role != CPU_ROLE_SP) { 458 /* Enable lapic. */ 459 lapic_enable(); 460 lapic_set_lvt(); 461 lapic_calibrate_timer(); 462 } 463 #endif 464 /* Make sure DELAY() is initialized. */ 465 DELAY(1); 466 again = true; 467 } 468 469 /* further PCB init done later. */ 470 471 switch (caa->cpu_role) { 472 case CPU_ROLE_SP: 473 atomic_or_32(&ci->ci_flags, CPUF_SP); 474 cpu_identify(ci); 475 #if 0 476 x86_errata(); 477 #endif 478 x86_cpu_idle_init(); 479 break; 480 481 case CPU_ROLE_BP: 482 atomic_or_32(&ci->ci_flags, CPUF_BSP); 483 cpu_identify(ci); 484 cpu_init(ci); 485 #if 0 486 x86_errata(); 487 #endif 488 x86_cpu_idle_init(); 489 break; 490 491 case CPU_ROLE_AP: 492 /* 493 * report on an AP 494 */ 495 496 #if defined(MULTIPROCESSOR) 497 cpu_intr_init(ci); 498 gdt_alloc_cpu(ci); 499 cpu_set_tss_gates(ci); 500 pmap_cpu_init_early(ci); 501 pmap_cpu_init_late(ci); 502 cpu_start_secondary(ci); 503 if (ci->ci_flags & CPUF_PRESENT) { 504 struct cpu_info *tmp; 505 506 identifycpu(ci); 507 tmp = cpu_info_list; 508 while (tmp->ci_next) 509 tmp = tmp->ci_next; 510 511 tmp->ci_next = ci; 512 } 513 #else 514 aprint_error_dev(self, "not started\n"); 515 #endif 516 break; 517 518 default: 519 aprint_normal("\n"); 520 panic("unknown processor type??\n"); 521 } 522 523 pat_init(ci); 524 atomic_or_32(&cpus_attached, ci->ci_cpumask); 525 526 #if 0 527 if (!pmf_device_register(self, cpu_suspend, cpu_resume)) 528 aprint_error_dev(self, "couldn't establish power handler\n"); 529 #endif 530 531 #if defined(MULTIPROCESSOR) 532 if (mp_verbose) { 533 struct lwp *l = ci->ci_data.cpu_idlelwp; 534 struct pcb *pcb = lwp_getpcb(l); 535 536 aprint_verbose_dev(self, 537 "idle lwp at %p, idle sp at 0x%p\n", 538 l, 539 #ifdef i386 540 (void *)pcb->pcb_esp 541 #else 542 (void *)pcb->pcb_rsp 543 #endif 544 ); 545 546 } 547 #endif 548 } 549 550 /* 551 * Initialize the processor appropriately. 552 */ 553 554 void 555 cpu_init(struct cpu_info *ci) 556 { 557 558 /* 559 * On a P6 or above, enable global TLB caching if the 560 * hardware supports it. 561 */ 562 if (cpu_feature[0] & CPUID_PGE) 563 lcr4(rcr4() | CR4_PGE); /* enable global TLB caching */ 564 565 #ifdef XXXMTRR 566 /* 567 * On a P6 or above, initialize MTRR's if the hardware supports them. 568 */ 569 if (cpu_feature[0] & CPUID_MTRR) { 570 if ((ci->ci_flags & CPUF_AP) == 0) 571 i686_mtrr_init_first(); 572 mtrr_init_cpu(ci); 573 } 574 #endif 575 /* 576 * If we have FXSAVE/FXRESTOR, use them. 577 */ 578 if (cpu_feature[0] & CPUID_FXSR) { 579 lcr4(rcr4() | CR4_OSFXSR); 580 581 /* 582 * If we have SSE/SSE2, enable XMM exceptions. 583 */ 584 if (cpu_feature[0] & (CPUID_SSE|CPUID_SSE2)) 585 lcr4(rcr4() | CR4_OSXMMEXCPT); 586 } 587 588 #ifdef __x86_64__ 589 /* No user PGD mapped for this CPU yet */ 590 ci->ci_xen_current_user_pgd = 0; 591 #endif 592 593 atomic_or_32(&cpus_running, ci->ci_cpumask); 594 atomic_or_32(&ci->ci_flags, CPUF_RUNNING); 595 } 596 597 598 #ifdef MULTIPROCESSOR 599 void 600 cpu_boot_secondary_processors(void) 601 { 602 struct cpu_info *ci; 603 u_long i; 604 605 for (i = 0; i < maxcpus; i++) { 606 ci = cpu_lookup(i); 607 if (ci == NULL) 608 continue; 609 if (ci->ci_data.cpu_idlelwp == NULL) 610 continue; 611 if ((ci->ci_flags & CPUF_PRESENT) == 0) 612 continue; 613 if (ci->ci_flags & (CPUF_BSP|CPUF_SP|CPUF_PRIMARY)) 614 continue; 615 cpu_boot_secondary(ci); 616 } 617 618 x86_mp_online = true; 619 } 620 621 static void 622 cpu_init_idle_lwp(struct cpu_info *ci) 623 { 624 struct lwp *l = ci->ci_data.cpu_idlelwp; 625 struct pcb *pcb = lwp_getpcb(l); 626 627 pcb->pcb_cr0 = rcr0(); 628 } 629 630 void 631 cpu_init_idle_lwps(void) 632 { 633 struct cpu_info *ci; 634 u_long i; 635 636 for (i = 0; i < maxcpus; i++) { 637 ci = cpu_lookup(i); 638 if (ci == NULL) 639 continue; 640 if (ci->ci_data.cpu_idlelwp == NULL) 641 continue; 642 if ((ci->ci_flags & CPUF_PRESENT) == 0) 643 continue; 644 cpu_init_idle_lwp(ci); 645 } 646 } 647 648 void 649 cpu_start_secondary(struct cpu_info *ci) 650 { 651 int i; 652 struct pmap *kpm = pmap_kernel(); 653 extern uint32_t mp_pdirpa; 654 655 mp_pdirpa = kpm->pm_pdirpa; /* XXX move elsewhere, not per CPU. */ 656 657 atomic_or_32(&ci->ci_flags, CPUF_AP); 658 659 aprint_debug_dev(ci->ci_dev, "starting\n"); 660 661 ci->ci_curlwp = ci->ci_data.cpu_idlelwp; 662 if (CPU_STARTUP(ci, mp_trampoline_paddr) != 0) 663 return; 664 665 /* 666 * wait for it to become ready 667 */ 668 for (i = 100000; (!(ci->ci_flags & CPUF_PRESENT)) && i > 0; i--) { 669 #ifdef MPDEBUG 670 extern int cpu_trace[3]; 671 static int otrace[3]; 672 if (memcmp(otrace, cpu_trace, sizeof(otrace)) != 0) { 673 aprint_debug_dev(ci->ci_dev, "trace %02x %02x %02x\n", 674 cpu_trace[0], cpu_trace[1], cpu_trace[2]); 675 memcpy(otrace, cpu_trace, sizeof(otrace)); 676 } 677 #endif 678 delay(10); 679 } 680 if ((ci->ci_flags & CPUF_PRESENT) == 0) { 681 aprint_error_dev(ci->ci_dev, "failed to become ready\n"); 682 #if defined(MPDEBUG) && defined(DDB) 683 printf("dropping into debugger; continue from here to resume boot\n"); 684 Debugger(); 685 #endif 686 } 687 688 CPU_START_CLEANUP(ci); 689 } 690 691 void 692 cpu_boot_secondary(struct cpu_info *ci) 693 { 694 int i; 695 696 atomic_or_32(&ci->ci_flags, CPUF_GO); 697 for (i = 100000; (!(ci->ci_flags & CPUF_RUNNING)) && i > 0; i--) { 698 delay(10); 699 } 700 if ((ci->ci_flags & CPUF_RUNNING) == 0) { 701 aprint_error_dev(ci->ci_dev, "CPU failed to start\n"); 702 #if defined(MPDEBUG) && defined(DDB) 703 printf("dropping into debugger; continue from here to resume boot\n"); 704 Debugger(); 705 #endif 706 } 707 } 708 709 /* 710 * The CPU ends up here when its ready to run 711 * This is called from code in mptramp.s; at this point, we are running 712 * in the idle pcb/idle stack of the new CPU. When this function returns, 713 * this processor will enter the idle loop and start looking for work. 714 * 715 * XXX should share some of this with init386 in machdep.c 716 */ 717 void 718 cpu_hatch(void *v) 719 { 720 struct cpu_info *ci = (struct cpu_info *)v; 721 struct pcb *pcb; 722 int s, i; 723 724 cpu_probe(ci); 725 726 cpu_feature[0] &= ~CPUID_FEAT_BLACKLIST; 727 cpu_feature[2] &= ~CPUID_FEAT_EXT_BLACKLIST; 728 729 cpu_init_msrs(ci, true); 730 731 KDASSERT((ci->ci_flags & CPUF_PRESENT) == 0); 732 atomic_or_32(&ci->ci_flags, CPUF_PRESENT); 733 while ((ci->ci_flags & CPUF_GO) == 0) { 734 /* Don't use delay, boot CPU may be patching the text. */ 735 for (i = 10000; i != 0; i--) 736 x86_pause(); 737 } 738 739 /* Because the text may have been patched in x86_patch(). */ 740 wbinvd(); 741 x86_flush(); 742 743 KASSERT((ci->ci_flags & CPUF_RUNNING) == 0); 744 745 pcb = lwp_getpcb(curlwp); 746 lcr3(pmap_kernel()->pm_pdirpa); 747 pcb->pcb_cr3 = pmap_kernel()->pm_pdirpa; 748 pcb = lwp_getpcb(ci->ci_data.cpu_idlelwp); 749 lcr0(pcb->pcb_cr0); 750 751 cpu_init_idt(); 752 gdt_init_cpu(ci); 753 lapic_enable(); 754 lapic_set_lvt(); 755 lapic_initclocks(); 756 757 #ifdef i386 758 npxinit(ci); 759 #else 760 fpuinit(ci); 761 #endif 762 763 lldt(GSEL(GLDT_SEL, SEL_KPL)); 764 ltr(ci->ci_tss_sel); 765 766 cpu_init(ci); 767 cpu_get_tsc_freq(ci); 768 769 s = splhigh(); 770 #ifdef i386 771 lapic_tpr = 0; 772 #else 773 lcr8(0); 774 #endif 775 x86_enable_intr(); 776 splx(s); 777 #if 0 778 x86_errata(); 779 #endif 780 781 aprint_debug_dev(ci->ci_dev, "CPU %ld running\n", 782 (long)ci->ci_cpuid); 783 } 784 785 #if defined(DDB) 786 787 #include <ddb/db_output.h> 788 #include <machine/db_machdep.h> 789 790 /* 791 * Dump CPU information from ddb. 792 */ 793 void 794 cpu_debug_dump(void) 795 { 796 struct cpu_info *ci; 797 CPU_INFO_ITERATOR cii; 798 799 db_printf("addr dev id flags ipis curlwp fpcurlwp\n"); 800 for (CPU_INFO_FOREACH(cii, ci)) { 801 db_printf("%p %s %ld %x %x %10p %10p\n", 802 ci, 803 ci->ci_dev == NULL ? "BOOT" : device_xname(ci->ci_dev), 804 (long)ci->ci_cpuid, 805 ci->ci_flags, ci->ci_ipis, 806 ci->ci_curlwp, 807 ci->ci_fpcurlwp); 808 } 809 } 810 #endif /* DDB */ 811 812 static void 813 cpu_copy_trampoline(void) 814 { 815 /* 816 * Copy boot code. 817 */ 818 extern u_char cpu_spinup_trampoline[]; 819 extern u_char cpu_spinup_trampoline_end[]; 820 821 vaddr_t mp_trampoline_vaddr; 822 823 mp_trampoline_vaddr = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, 824 UVM_KMF_VAONLY); 825 826 pmap_kenter_pa(mp_trampoline_vaddr, mp_trampoline_paddr, 827 VM_PROT_READ | VM_PROT_WRITE, 0); 828 pmap_update(pmap_kernel()); 829 memcpy((void *)mp_trampoline_vaddr, 830 cpu_spinup_trampoline, 831 cpu_spinup_trampoline_end - cpu_spinup_trampoline); 832 833 pmap_kremove(mp_trampoline_vaddr, PAGE_SIZE); 834 pmap_update(pmap_kernel()); 835 uvm_km_free(kernel_map, mp_trampoline_vaddr, PAGE_SIZE, UVM_KMF_VAONLY); 836 } 837 838 #endif /* MULTIPROCESSOR */ 839 840 #ifdef i386 841 #if 0 842 static void 843 tss_init(struct i386tss *tss, void *stack, void *func) 844 { 845 memset(tss, 0, sizeof *tss); 846 tss->tss_esp0 = tss->tss_esp = (int)((char *)stack + USPACE - 16); 847 tss->tss_ss0 = GSEL(GDATA_SEL, SEL_KPL); 848 tss->__tss_cs = GSEL(GCODE_SEL, SEL_KPL); 849 tss->tss_fs = GSEL(GCPU_SEL, SEL_KPL); 850 tss->tss_gs = tss->__tss_es = tss->__tss_ds = 851 tss->__tss_ss = GSEL(GDATA_SEL, SEL_KPL); 852 tss->tss_cr3 = pmap_kernel()->pm_pdirpa; 853 tss->tss_esp = (int)((char *)stack + USPACE - 16); 854 tss->tss_ldt = GSEL(GLDT_SEL, SEL_KPL); 855 tss->__tss_eflags = PSL_MBO | PSL_NT; /* XXX not needed? */ 856 tss->__tss_eip = (int)func; 857 } 858 #endif 859 860 /* XXX */ 861 #define IDTVEC(name) __CONCAT(X, name) 862 typedef void (vector)(void); 863 extern vector IDTVEC(tss_trap08); 864 #ifdef DDB 865 extern vector Xintrddbipi; 866 extern int ddb_vec; 867 #endif 868 869 static void 870 cpu_set_tss_gates(struct cpu_info *ci) 871 { 872 #if 0 873 struct segment_descriptor sd; 874 875 ci->ci_doubleflt_stack = (char *)uvm_km_alloc(kernel_map, USPACE, 0, 876 UVM_KMF_WIRED); 877 tss_init(&ci->ci_doubleflt_tss, ci->ci_doubleflt_stack, 878 IDTVEC(tss_trap08)); 879 setsegment(&sd, &ci->ci_doubleflt_tss, sizeof(struct i386tss) - 1, 880 SDT_SYS386TSS, SEL_KPL, 0, 0); 881 ci->ci_gdt[GTRAPTSS_SEL].sd = sd; 882 setgate(&idt[8], NULL, 0, SDT_SYSTASKGT, SEL_KPL, 883 GSEL(GTRAPTSS_SEL, SEL_KPL)); 884 #endif 885 886 #if defined(DDB) && defined(MULTIPROCESSOR) 887 /* 888 * Set up separate handler for the DDB IPI, so that it doesn't 889 * stomp on a possibly corrupted stack. 890 * 891 * XXX overwriting the gate set in db_machine_init. 892 * Should rearrange the code so that it's set only once. 893 */ 894 ci->ci_ddbipi_stack = (char *)uvm_km_alloc(kernel_map, USPACE, 0, 895 UVM_KMF_WIRED); 896 tss_init(&ci->ci_ddbipi_tss, ci->ci_ddbipi_stack, 897 Xintrddbipi); 898 899 setsegment(&sd, &ci->ci_ddbipi_tss, sizeof(struct i386tss) - 1, 900 SDT_SYS386TSS, SEL_KPL, 0, 0); 901 ci->ci_gdt[GIPITSS_SEL].sd = sd; 902 903 setgate(&idt[ddb_vec], NULL, 0, SDT_SYSTASKGT, SEL_KPL, 904 GSEL(GIPITSS_SEL, SEL_KPL)); 905 #endif 906 } 907 #else 908 static void 909 cpu_set_tss_gates(struct cpu_info *ci) 910 { 911 912 } 913 #endif /* i386 */ 914 915 int 916 mp_cpu_start(struct cpu_info *ci, paddr_t target) 917 { 918 #if 0 919 #if NLAPIC > 0 920 int error; 921 #endif 922 unsigned short dwordptr[2]; 923 924 /* 925 * Bootstrap code must be addressable in real mode 926 * and it must be page aligned. 927 */ 928 KASSERT(target < 0x10000 && target % PAGE_SIZE == 0); 929 930 /* 931 * "The BSP must initialize CMOS shutdown code to 0Ah ..." 932 */ 933 934 outb(IO_RTC, NVRAM_RESET); 935 outb(IO_RTC+1, NVRAM_RESET_JUMP); 936 937 /* 938 * "and the warm reset vector (DWORD based at 40:67) to point 939 * to the AP startup code ..." 940 */ 941 942 dwordptr[0] = 0; 943 dwordptr[1] = target >> 4; 944 945 pmap_kenter_pa (0, 0, VM_PROT_READ|VM_PROT_WRITE, 0); 946 pmap_update(pmap_kernel()); 947 948 memcpy ((uint8_t *) 0x467, dwordptr, 4); 949 950 pmap_kremove (0, PAGE_SIZE); 951 pmap_update(pmap_kernel()); 952 953 #if NLAPIC > 0 954 /* 955 * ... prior to executing the following sequence:" 956 */ 957 958 if (ci->ci_flags & CPUF_AP) { 959 if ((error = x86_ipi_init(ci->ci_cpuid)) != 0) 960 return error; 961 962 delay(10000); 963 964 if (cpu_feature & CPUID_APIC) { 965 error = x86_ipi_init(ci->ci_cpuid); 966 if (error != 0) { 967 aprint_error_dev(ci->ci_dev, "%s: IPI not taken (1)\n", 968 __func__); 969 return error; 970 } 971 972 delay(10000); 973 974 error = x86_ipi(target / PAGE_SIZE, ci->ci_cpuid, 975 LAPIC_DLMODE_STARTUP); 976 if (error != 0) { 977 aprint_error_dev(ci->ci_dev, "%s: IPI not taken (2)\n", 978 __func__); 979 return error; 980 } 981 delay(200); 982 983 error = x86_ipi(target / PAGE_SIZE, ci->ci_cpuid, 984 LAPIC_DLMODE_STARTUP); 985 if (error != 0) { 986 aprint_error_dev(ci->ci_dev, "%s: IPI not taken ((3)\n", 987 __func__); 988 return error; 989 } 990 delay(200); 991 } 992 } 993 #endif 994 #endif /* 0 */ 995 return 0; 996 } 997 998 void 999 mp_cpu_start_cleanup(struct cpu_info *ci) 1000 { 1001 #if 0 1002 /* 1003 * Ensure the NVRAM reset byte contains something vaguely sane. 1004 */ 1005 1006 outb(IO_RTC, NVRAM_RESET); 1007 outb(IO_RTC+1, NVRAM_RESET_RST); 1008 #endif 1009 } 1010 1011 void 1012 cpu_init_msrs(struct cpu_info *ci, bool full) 1013 { 1014 #ifdef __x86_64__ 1015 if (full) { 1016 HYPERVISOR_set_segment_base (SEGBASE_FS, 0); 1017 HYPERVISOR_set_segment_base (SEGBASE_GS_KERNEL, (uint64_t) ci); 1018 HYPERVISOR_set_segment_base (SEGBASE_GS_USER, 0); 1019 } 1020 #endif /* __x86_64__ */ 1021 1022 if (cpu_feature[2] & CPUID_NOX) 1023 wrmsr(MSR_EFER, rdmsr(MSR_EFER) | EFER_NXE); 1024 } 1025 1026 void 1027 cpu_offline_md(void) 1028 { 1029 int s; 1030 1031 s = splhigh(); 1032 #ifdef __i386__ 1033 npxsave_cpu(true); 1034 #else 1035 fpusave_cpu(true); 1036 #endif 1037 splx(s); 1038 } 1039 1040 #if 0 1041 /* XXX joerg restructure and restart CPUs individually */ 1042 static bool 1043 cpu_suspend(device_t dv, const pmf_qual_t *qual) 1044 { 1045 struct cpu_softc *sc = device_private(dv); 1046 struct cpu_info *ci = sc->sc_info; 1047 int err; 1048 1049 if (ci->ci_flags & CPUF_PRIMARY) 1050 return true; 1051 if (ci->ci_data.cpu_idlelwp == NULL) 1052 return true; 1053 if ((ci->ci_flags & CPUF_PRESENT) == 0) 1054 return true; 1055 1056 sc->sc_wasonline = !(ci->ci_schedstate.spc_flags & SPCF_OFFLINE); 1057 1058 if (sc->sc_wasonline) { 1059 mutex_enter(&cpu_lock); 1060 err = cpu_setstate(ci, false); 1061 mutex_exit(&cpu_lock); 1062 1063 if (err) 1064 return false; 1065 } 1066 1067 return true; 1068 } 1069 1070 static bool 1071 cpu_resume(device_t dv, const pmf_qual_t *qual) 1072 { 1073 struct cpu_softc *sc = device_private(dv); 1074 struct cpu_info *ci = sc->sc_info; 1075 int err = 0; 1076 1077 if (ci->ci_flags & CPUF_PRIMARY) 1078 return true; 1079 if (ci->ci_data.cpu_idlelwp == NULL) 1080 return true; 1081 if ((ci->ci_flags & CPUF_PRESENT) == 0) 1082 return true; 1083 1084 if (sc->sc_wasonline) { 1085 mutex_enter(&cpu_lock); 1086 err = cpu_setstate(ci, true); 1087 mutex_exit(&cpu_lock); 1088 } 1089 1090 return err == 0; 1091 } 1092 #endif 1093 1094 void 1095 cpu_get_tsc_freq(struct cpu_info *ci) 1096 { 1097 const volatile vcpu_time_info_t *tinfo = &ci->ci_vcpu->time; 1098 delay(1000000); 1099 uint64_t freq = 1000000000ULL << 32; 1100 freq = freq / (uint64_t)tinfo->tsc_to_system_mul; 1101 if ( tinfo->tsc_shift < 0 ) 1102 freq = freq << -tinfo->tsc_shift; 1103 else 1104 freq = freq >> tinfo->tsc_shift; 1105 ci->ci_data.cpu_cc_freq = freq; 1106 } 1107 1108 void 1109 x86_cpu_idle_xen(void) 1110 { 1111 struct cpu_info *ci = curcpu(); 1112 1113 KASSERT(ci->ci_ilevel == IPL_NONE); 1114 1115 x86_disable_intr(); 1116 if (!__predict_false(ci->ci_want_resched)) { 1117 idle_block(); 1118 } else { 1119 x86_enable_intr(); 1120 } 1121 } 1122 1123 /* 1124 * Loads pmap for the current CPU. 1125 */ 1126 void 1127 cpu_load_pmap(struct pmap *pmap) 1128 { 1129 #ifdef i386 1130 #ifdef PAE 1131 int i, s; 1132 struct cpu_info *ci; 1133 1134 s = splvm(); /* just to be safe */ 1135 ci = curcpu(); 1136 paddr_t l3_pd = xpmap_ptom_masked(ci->ci_pae_l3_pdirpa); 1137 /* don't update the kernel L3 slot */ 1138 for (i = 0 ; i < PDP_SIZE - 1; i++) { 1139 xpq_queue_pte_update(l3_pd + i * sizeof(pd_entry_t), 1140 xpmap_ptom(pmap->pm_pdirpa[i]) | PG_V); 1141 } 1142 splx(s); 1143 tlbflush(); 1144 #else /* PAE */ 1145 lcr3(pmap_pdirpa(pmap, 0)); 1146 #endif /* PAE */ 1147 #endif /* i386 */ 1148 1149 #ifdef __x86_64__ 1150 int i, s; 1151 pd_entry_t *old_pgd, *new_pgd; 1152 paddr_t addr; 1153 struct cpu_info *ci; 1154 1155 /* kernel pmap always in cr3 and should never go in user cr3 */ 1156 if (pmap_pdirpa(pmap, 0) != pmap_pdirpa(pmap_kernel(), 0)) { 1157 ci = curcpu(); 1158 /* 1159 * Map user space address in kernel space and load 1160 * user cr3 1161 */ 1162 s = splvm(); 1163 new_pgd = pmap->pm_pdir; 1164 old_pgd = pmap_kernel()->pm_pdir; 1165 addr = xpmap_ptom(pmap_pdirpa(pmap_kernel(), 0)); 1166 for (i = 0; i < PDIR_SLOT_PTE; 1167 i++, addr += sizeof(pd_entry_t)) { 1168 if ((new_pgd[i] & PG_V) || (old_pgd[i] & PG_V)) 1169 xpq_queue_pte_update(addr, new_pgd[i]); 1170 } 1171 tlbflush(); 1172 xen_set_user_pgd(pmap_pdirpa(pmap, 0)); 1173 ci->ci_xen_current_user_pgd = pmap_pdirpa(pmap, 0); 1174 splx(s); 1175 } 1176 #endif /* __x86_64__ */ 1177 } 1178
Indexes created Fri Oct 03 15:10:06 GMT 2025