cpu.c revision 1.31.2.8
1 /* $NetBSD: cpu.c,v 1.31.2.8 2011/03/28 23:04:56 jym Exp $ */ 2 /* NetBSD: cpu.c,v 1.18 2004/02/20 17:35:01 yamt Exp */ 3 4 /*- 5 * Copyright (c) 2000 The NetBSD Foundation, Inc. 6 * Copyright (c) 2002, 2006, 2007 YAMAMOTO Takashi, 7 * All rights reserved. 8 * 9 * This code is derived from software contributed to The NetBSD Foundation 10 * by RedBack Networks Inc. 11 * 12 * Author: Bill Sommerfeld 13 * 14 * Redistribution and use in source and binary forms, with or without 15 * modification, are permitted provided that the following conditions 16 * are met: 17 * 1. Redistributions of source code must retain the above copyright 18 * notice, this list of conditions and the following disclaimer. 19 * 2. Redistributions in binary form must reproduce the above copyright 20 * notice, this list of conditions and the following disclaimer in the 21 * documentation and/or other materials provided with the distribution. 22 * 23 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 24 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 25 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 26 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 27 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 28 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 29 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 30 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 31 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 32 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 33 * POSSIBILITY OF SUCH DAMAGE. 34 */ 35 36 /* 37 * Copyright (c) 1999 Stefan Grefen 38 * 39 * Redistribution and use in source and binary forms, with or without 40 * modification, are permitted provided that the following conditions 41 * are met: 42 * 1. Redistributions of source code must retain the above copyright 43 * notice, this list of conditions and the following disclaimer. 44 * 2. Redistributions in binary form must reproduce the above copyright 45 * notice, this list of conditions and the following disclaimer in the 46 * documentation and/or other materials provided with the distribution. 47 * 3. All advertising materials mentioning features or use of this software 48 * must display the following acknowledgement: 49 * This product includes software developed by the NetBSD 50 * Foundation, Inc. and its contributors. 51 * 4. Neither the name of The NetBSD Foundation nor the names of its 52 * contributors may be used to endorse or promote products derived 53 * from this software without specific prior written permission. 54 * 55 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND ANY 56 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 57 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 58 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR AND CONTRIBUTORS BE LIABLE 59 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 60 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 61 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 62 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 63 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 64 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 65 * SUCH DAMAGE. 66 */ 67 68 #include <sys/cdefs.h> 69 __KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.31.2.8 2011/03/28 23:04:56 jym Exp $"); 70 71 #include "opt_ddb.h" 72 #include "opt_multiprocessor.h" 73 #include "opt_mpbios.h" /* for MPDEBUG */ 74 #include "opt_mtrr.h" 75 #include "opt_xen.h" 76 77 #include "lapic.h" 78 #include "ioapic.h" 79 80 #include <sys/param.h> 81 #include <sys/proc.h> 82 #include <sys/systm.h> 83 #include <sys/device.h> 84 #include <sys/kmem.h> 85 #include <sys/cpu.h> 86 #include <sys/atomic.h> 87 #include <sys/reboot.h> 88 89 #include <uvm/uvm.h> 90 91 #include <machine/cpufunc.h> 92 #include <machine/cpuvar.h> 93 #include <machine/pmap.h> 94 #include <machine/vmparam.h> 95 #include <machine/mpbiosvar.h> 96 #include <machine/pcb.h> 97 #include <machine/specialreg.h> 98 #include <machine/segments.h> 99 #include <machine/gdt.h> 100 #include <machine/mtrr.h> 101 #include <machine/pio.h> 102 103 #include <xen/vcpuvar.h> 104 105 #if NLAPIC > 0 106 #include <machine/apicvar.h> 107 #include <machine/i82489reg.h> 108 #include <machine/i82489var.h> 109 #endif 110 111 #include <dev/ic/mc146818reg.h> 112 #include <dev/isa/isareg.h> 113 114 #if MAXCPUS > 32 115 #error cpu_info contains 32bit bitmasks 116 #endif 117 118 static int cpu_match(device_t, cfdata_t, void *); 119 static void cpu_attach(device_t, device_t, void *); 120 static void cpu_defer(device_t); 121 static int cpu_rescan(device_t, const char *, const int *); 122 static void cpu_childdetached(device_t, device_t); 123 static int vcpu_match(device_t, cfdata_t, void *); 124 static void vcpu_attach(device_t, device_t, void *); 125 static void cpu_attach_common(device_t, device_t, void *); 126 void cpu_offline_md(void); 127 128 struct cpu_softc { 129 device_t sc_dev; /* device tree glue */ 130 struct cpu_info *sc_info; /* pointer to CPU info */ 131 bool sc_wasonline; 132 }; 133 134 int mp_cpu_start(struct cpu_info *, paddr_t); 135 void mp_cpu_start_cleanup(struct cpu_info *); 136 const struct cpu_functions mp_cpu_funcs = { mp_cpu_start, NULL, 137 mp_cpu_start_cleanup }; 138 139 CFATTACH_DECL2_NEW(cpu, sizeof(struct cpu_softc), 140 cpu_match, cpu_attach, NULL, NULL, cpu_rescan, cpu_childdetached); 141 142 CFATTACH_DECL_NEW(vcpu, sizeof(struct cpu_softc), 143 vcpu_match, vcpu_attach, NULL, NULL); 144 145 /* 146 * Statically-allocated CPU info for the primary CPU (or the only 147 * CPU, on uniprocessors). The CPU info list is initialized to 148 * point at it. 149 */ 150 #ifdef TRAPLOG 151 #include <machine/tlog.h> 152 struct tlog tlog_primary; 153 #endif 154 struct cpu_info cpu_info_primary __aligned(CACHE_LINE_SIZE) = { 155 .ci_dev = 0, 156 .ci_self = &cpu_info_primary, 157 .ci_idepth = -1, 158 .ci_curlwp = &lwp0, 159 .ci_curldt = -1, 160 #ifdef TRAPLOG 161 .ci_tlog = &tlog_primary, 162 #endif 163 164 }; 165 struct cpu_info phycpu_info_primary __aligned(CACHE_LINE_SIZE) = { 166 .ci_dev = 0, 167 .ci_self = &phycpu_info_primary, 168 }; 169 170 struct cpu_info *cpu_info_list = &cpu_info_primary; 171 struct cpu_info *phycpu_info_list = &phycpu_info_primary; 172 173 static void cpu_set_tss_gates(struct cpu_info *ci); 174 175 uint32_t cpus_attached = 0; 176 uint32_t cpus_running = 0; 177 178 uint32_t phycpus_attached = 0; 179 uint32_t phycpus_running = 0; 180 181 uint32_t cpu_feature[5]; /* X86 CPUID feature bits 182 * [0] basic features %edx 183 * [1] basic features %ecx 184 * [2] extended features %edx 185 * [3] extended features %ecx 186 * [4] VIA padlock features 187 */ 188 189 bool x86_mp_online; 190 paddr_t mp_trampoline_paddr = MP_TRAMPOLINE; 191 192 #if defined(MULTIPROCESSOR) 193 void cpu_hatch(void *); 194 static void cpu_boot_secondary(struct cpu_info *ci); 195 static void cpu_start_secondary(struct cpu_info *ci); 196 static void cpu_copy_trampoline(void); 197 198 /* 199 * Runs once per boot once multiprocessor goo has been detected and 200 * the local APIC on the boot processor has been mapped. 201 * 202 * Called from lapic_boot_init() (from mpbios_scan()). 203 */ 204 void 205 cpu_init_first(void) 206 { 207 208 cpu_info_primary.ci_cpuid = lapic_cpu_number(); 209 cpu_copy_trampoline(); 210 } 211 #endif /* MULTIPROCESSOR */ 212 213 static int 214 cpu_match(device_t parent, cfdata_t match, void *aux) 215 { 216 217 return 1; 218 } 219 220 static void 221 cpu_attach(device_t parent, device_t self, void *aux) 222 { 223 struct cpu_softc *sc = device_private(self); 224 struct cpu_attach_args *caa = aux; 225 struct cpu_info *ci; 226 uintptr_t ptr; 227 static int nphycpu = 0; 228 229 sc->sc_dev = self; 230 231 if (phycpus_attached == ~0) { 232 aprint_error(": increase MAXCPUS\n"); 233 return; 234 } 235 236 /* 237 * If we're an Application Processor, allocate a cpu_info 238 * If we're the first attached CPU use the primary cpu_info, 239 * otherwise allocate a new one 240 */ 241 aprint_naive("\n"); 242 aprint_normal("\n"); 243 if (nphycpu > 0) { 244 struct cpu_info *tmp; 245 ptr = (uintptr_t)kmem_zalloc(sizeof(*ci) + CACHE_LINE_SIZE - 1, 246 KM_SLEEP); 247 ci = (struct cpu_info *)roundup2(ptr, CACHE_LINE_SIZE); 248 ci->ci_curldt = -1; 249 250 tmp = phycpu_info_list; 251 while (tmp->ci_next) 252 tmp = tmp->ci_next; 253 254 tmp->ci_next = ci; 255 } else { 256 ci = &phycpu_info_primary; 257 } 258 259 ci->ci_self = ci; 260 sc->sc_info = ci; 261 262 ci->ci_dev = self; 263 ci->ci_acpiid = caa->cpu_id; 264 ci->ci_cpuid = caa->cpu_number; 265 ci->ci_vcpu = NULL; 266 ci->ci_index = nphycpu++; 267 ci->ci_cpumask = (1 << cpu_index(ci)); 268 269 atomic_or_32(&phycpus_attached, ci->ci_cpumask); 270 271 if (!pmf_device_register(self, NULL, NULL)) 272 aprint_error_dev(self, "couldn't establish power handler\n"); 273 274 (void)config_defer(self, cpu_defer); 275 } 276 277 static void 278 cpu_defer(device_t self) 279 { 280 cpu_rescan(self, NULL, NULL); 281 } 282 283 static int 284 cpu_rescan(device_t self, const char *ifattr, const int *locators) 285 { 286 struct cpu_softc *sc = device_private(self); 287 struct cpufeature_attach_args cfaa; 288 struct cpu_info *ci = sc->sc_info; 289 290 memset(&cfaa, 0, sizeof(cfaa)); 291 cfaa.ci = ci; 292 293 if (ifattr_match(ifattr, "cpufeaturebus")) { 294 295 if (ci->ci_frequency == NULL) { 296 cfaa.name = "frequency"; 297 ci->ci_frequency = config_found_ia(self, 298 "cpufeaturebus", &cfaa, NULL); 299 } 300 } 301 302 return 0; 303 } 304 305 static void 306 cpu_childdetached(device_t self, device_t child) 307 { 308 struct cpu_softc *sc = device_private(self); 309 struct cpu_info *ci = sc->sc_info; 310 311 if (ci->ci_frequency == child) 312 ci->ci_frequency = NULL; 313 } 314 315 static int 316 vcpu_match(device_t parent, cfdata_t match, void *aux) 317 { 318 struct vcpu_attach_args *vcaa = aux; 319 320 if (strcmp(vcaa->vcaa_name, match->cf_name) == 0) 321 return 1; 322 return 0; 323 } 324 325 static void 326 vcpu_attach(device_t parent, device_t self, void *aux) 327 { 328 struct vcpu_attach_args *vcaa = aux; 329 330 cpu_attach_common(parent, self, &vcaa->vcaa_caa); 331 332 if (!pmf_device_register(self, NULL, NULL)) 333 aprint_error_dev(self, "couldn't establish power handler\n"); 334 } 335 336 static void 337 cpu_vm_init(struct cpu_info *ci) 338 { 339 int ncolors = 2, i; 340 341 for (i = CAI_ICACHE; i <= CAI_L2CACHE; i++) { 342 struct x86_cache_info *cai; 343 int tcolors; 344 345 cai = &ci->ci_cinfo[i]; 346 347 tcolors = atop(cai->cai_totalsize); 348 switch(cai->cai_associativity) { 349 case 0xff: 350 tcolors = 1; /* fully associative */ 351 break; 352 case 0: 353 case 1: 354 break; 355 default: 356 tcolors /= cai->cai_associativity; 357 } 358 ncolors = max(ncolors, tcolors); 359 } 360 361 /* 362 * Knowing the size of the largest cache on this CPU, re-color 363 * our pages. 364 */ 365 if (ncolors <= uvmexp.ncolors) 366 return; 367 aprint_debug_dev(ci->ci_dev, "%d page colors\n", ncolors); 368 uvm_page_recolor(ncolors); 369 } 370 371 static void 372 cpu_attach_common(device_t parent, device_t self, void *aux) 373 { 374 struct cpu_softc *sc = device_private(self); 375 struct cpu_attach_args *caa = aux; 376 struct cpu_info *ci; 377 uintptr_t ptr; 378 int cpunum = caa->cpu_number; 379 static bool again = false; 380 381 sc->sc_dev = self; 382 383 /* 384 * If we're an Application Processor, allocate a cpu_info 385 * structure, otherwise use the primary's. 386 */ 387 if (caa->cpu_role == CPU_ROLE_AP) { 388 aprint_naive(": Application Processor\n"); 389 ptr = (uintptr_t)kmem_alloc(sizeof(*ci) + CACHE_LINE_SIZE - 1, 390 KM_SLEEP); 391 ci = (struct cpu_info *)roundup2(ptr, CACHE_LINE_SIZE); 392 memset(ci, 0, sizeof(*ci)); 393 #ifdef TRAPLOG 394 ci->ci_tlog_base = kmem_zalloc(sizeof(struct tlog), KM_SLEEP); 395 #endif 396 } else { 397 aprint_naive(": %s Processor\n", 398 caa->cpu_role == CPU_ROLE_SP ? "Single" : "Boot"); 399 ci = &cpu_info_primary; 400 #if NLAPIC > 0 401 if (cpunum != lapic_cpu_number()) { 402 /* XXX should be done earlier */ 403 uint32_t reg; 404 aprint_verbose("\n"); 405 aprint_verbose_dev(self, "running CPU at apic %d" 406 " instead of at expected %d", lapic_cpu_number(), 407 cpunum); 408 reg = i82489_readreg(LAPIC_ID); 409 i82489_writereg(LAPIC_ID, (reg & ~LAPIC_ID_MASK) | 410 (cpunum << LAPIC_ID_SHIFT)); 411 } 412 if (cpunum != lapic_cpu_number()) { 413 aprint_error_dev(self, "unable to reset apic id\n"); 414 } 415 #endif 416 } 417 418 ci->ci_self = ci; 419 sc->sc_info = ci; 420 ci->ci_dev = self; 421 ci->ci_cpuid = cpunum; 422 423 KASSERT(HYPERVISOR_shared_info != NULL); 424 ci->ci_vcpu = &HYPERVISOR_shared_info->vcpu_info[cpunum]; 425 426 ci->ci_func = caa->cpu_func; 427 428 /* Must be called before mi_cpu_attach(). */ 429 cpu_vm_init(ci); 430 431 if (caa->cpu_role == CPU_ROLE_AP) { 432 int error; 433 434 error = mi_cpu_attach(ci); 435 if (error != 0) { 436 aprint_normal("\n"); 437 aprint_error_dev(self, 438 "mi_cpu_attach failed with %d\n", error); 439 return; 440 } 441 } else { 442 KASSERT(ci->ci_data.cpu_idlelwp != NULL); 443 } 444 445 ci->ci_cpumask = (1 << cpu_index(ci)); 446 pmap_reference(pmap_kernel()); 447 ci->ci_pmap = pmap_kernel(); 448 ci->ci_tlbstate = TLBSTATE_STALE; 449 450 /* 451 * Boot processor may not be attached first, but the below 452 * must be done to allow booting other processors. 453 */ 454 if (!again) { 455 atomic_or_32(&ci->ci_flags, CPUF_PRESENT | CPUF_PRIMARY); 456 /* Basic init. */ 457 cpu_intr_init(ci); 458 cpu_get_tsc_freq(ci); 459 cpu_init(ci); 460 cpu_set_tss_gates(ci); 461 pmap_cpu_init_late(ci); 462 #if NLAPIC > 0 463 if (caa->cpu_role != CPU_ROLE_SP) { 464 /* Enable lapic. */ 465 lapic_enable(); 466 lapic_set_lvt(); 467 lapic_calibrate_timer(); 468 } 469 #endif 470 /* Make sure DELAY() is initialized. */ 471 DELAY(1); 472 again = true; 473 } 474 475 /* further PCB init done later. */ 476 477 switch (caa->cpu_role) { 478 case CPU_ROLE_SP: 479 atomic_or_32(&ci->ci_flags, CPUF_SP); 480 cpu_identify(ci); 481 #if 0 482 x86_errata(); 483 #endif 484 x86_cpu_idle_init(); 485 break; 486 487 case CPU_ROLE_BP: 488 atomic_or_32(&ci->ci_flags, CPUF_BSP); 489 cpu_identify(ci); 490 cpu_init(ci); 491 #if 0 492 x86_errata(); 493 #endif 494 x86_cpu_idle_init(); 495 break; 496 497 case CPU_ROLE_AP: 498 /* 499 * report on an AP 500 */ 501 502 #if defined(MULTIPROCESSOR) 503 cpu_intr_init(ci); 504 gdt_alloc_cpu(ci); 505 cpu_set_tss_gates(ci); 506 pmap_cpu_init_early(ci); 507 pmap_cpu_init_late(ci); 508 cpu_start_secondary(ci); 509 if (ci->ci_flags & CPUF_PRESENT) { 510 struct cpu_info *tmp; 511 512 identifycpu(ci); 513 tmp = cpu_info_list; 514 while (tmp->ci_next) 515 tmp = tmp->ci_next; 516 517 tmp->ci_next = ci; 518 } 519 #else 520 aprint_error_dev(self, "not started\n"); 521 #endif 522 break; 523 524 default: 525 aprint_normal("\n"); 526 panic("unknown processor type??\n"); 527 } 528 529 pat_init(ci); 530 atomic_or_32(&cpus_attached, ci->ci_cpumask); 531 532 #if 0 533 if (!pmf_device_register(self, cpu_suspend, cpu_resume)) 534 aprint_error_dev(self, "couldn't establish power handler\n"); 535 #endif 536 537 #if defined(MULTIPROCESSOR) 538 if (mp_verbose) { 539 struct lwp *l = ci->ci_data.cpu_idlelwp; 540 struct pcb *pcb = lwp_getpcb(l); 541 542 aprint_verbose_dev(self, 543 "idle lwp at %p, idle sp at 0x%p\n", 544 l, 545 #ifdef i386 546 (void *)pcb->pcb_esp 547 #else 548 (void *)pcb->pcb_rsp 549 #endif 550 ); 551 552 } 553 #endif 554 } 555 556 /* 557 * Initialize the processor appropriately. 558 */ 559 560 void 561 cpu_init(struct cpu_info *ci) 562 { 563 564 /* 565 * On a P6 or above, enable global TLB caching if the 566 * hardware supports it. 567 */ 568 if (cpu_feature[0] & CPUID_PGE) 569 lcr4(rcr4() | CR4_PGE); /* enable global TLB caching */ 570 571 #ifdef XXXMTRR 572 /* 573 * On a P6 or above, initialize MTRR's if the hardware supports them. 574 */ 575 if (cpu_feature[0] & CPUID_MTRR) { 576 if ((ci->ci_flags & CPUF_AP) == 0) 577 i686_mtrr_init_first(); 578 mtrr_init_cpu(ci); 579 } 580 #endif 581 /* 582 * If we have FXSAVE/FXRESTOR, use them. 583 */ 584 if (cpu_feature[0] & CPUID_FXSR) { 585 lcr4(rcr4() | CR4_OSFXSR); 586 587 /* 588 * If we have SSE/SSE2, enable XMM exceptions. 589 */ 590 if (cpu_feature[0] & (CPUID_SSE|CPUID_SSE2)) 591 lcr4(rcr4() | CR4_OSXMMEXCPT); 592 } 593 594 #ifdef __x86_64__ 595 /* No user PGD mapped for this CPU yet */ 596 ci->ci_xen_current_user_pgd = 0; 597 #endif 598 599 atomic_or_32(&cpus_running, ci->ci_cpumask); 600 atomic_or_32(&ci->ci_flags, CPUF_RUNNING); 601 } 602 603 604 #ifdef MULTIPROCESSOR 605 void 606 cpu_boot_secondary_processors(void) 607 { 608 struct cpu_info *ci; 609 u_long i; 610 611 for (i = 0; i < maxcpus; i++) { 612 ci = cpu_lookup(i); 613 if (ci == NULL) 614 continue; 615 if (ci->ci_data.cpu_idlelwp == NULL) 616 continue; 617 if ((ci->ci_flags & CPUF_PRESENT) == 0) 618 continue; 619 if (ci->ci_flags & (CPUF_BSP|CPUF_SP|CPUF_PRIMARY)) 620 continue; 621 cpu_boot_secondary(ci); 622 } 623 624 x86_mp_online = true; 625 } 626 627 static void 628 cpu_init_idle_lwp(struct cpu_info *ci) 629 { 630 struct lwp *l = ci->ci_data.cpu_idlelwp; 631 struct pcb *pcb = lwp_getpcb(l); 632 633 pcb->pcb_cr0 = rcr0(); 634 } 635 636 void 637 cpu_init_idle_lwps(void) 638 { 639 struct cpu_info *ci; 640 u_long i; 641 642 for (i = 0; i < maxcpus; i++) { 643 ci = cpu_lookup(i); 644 if (ci == NULL) 645 continue; 646 if (ci->ci_data.cpu_idlelwp == NULL) 647 continue; 648 if ((ci->ci_flags & CPUF_PRESENT) == 0) 649 continue; 650 cpu_init_idle_lwp(ci); 651 } 652 } 653 654 void 655 cpu_start_secondary(struct cpu_info *ci) 656 { 657 int i; 658 struct pmap *kpm = pmap_kernel(); 659 extern uint32_t mp_pdirpa; 660 661 mp_pdirpa = kpm->pm_pdirpa; /* XXX move elsewhere, not per CPU. */ 662 663 atomic_or_32(&ci->ci_flags, CPUF_AP); 664 665 aprint_debug_dev(ci->ci_dev, "starting\n"); 666 667 ci->ci_curlwp = ci->ci_data.cpu_idlelwp; 668 if (CPU_STARTUP(ci, mp_trampoline_paddr) != 0) 669 return; 670 671 /* 672 * wait for it to become ready 673 */ 674 for (i = 100000; (!(ci->ci_flags & CPUF_PRESENT)) && i > 0; i--) { 675 #ifdef MPDEBUG 676 extern int cpu_trace[3]; 677 static int otrace[3]; 678 if (memcmp(otrace, cpu_trace, sizeof(otrace)) != 0) { 679 aprint_debug_dev(ci->ci_dev, "trace %02x %02x %02x\n", 680 cpu_trace[0], cpu_trace[1], cpu_trace[2]); 681 memcpy(otrace, cpu_trace, sizeof(otrace)); 682 } 683 #endif 684 delay(10); 685 } 686 if ((ci->ci_flags & CPUF_PRESENT) == 0) { 687 aprint_error_dev(ci->ci_dev, "failed to become ready\n"); 688 #if defined(MPDEBUG) && defined(DDB) 689 printf("dropping into debugger; continue from here to resume boot\n"); 690 Debugger(); 691 #endif 692 } 693 694 CPU_START_CLEANUP(ci); 695 } 696 697 void 698 cpu_boot_secondary(struct cpu_info *ci) 699 { 700 int i; 701 702 atomic_or_32(&ci->ci_flags, CPUF_GO); 703 for (i = 100000; (!(ci->ci_flags & CPUF_RUNNING)) && i > 0; i--) { 704 delay(10); 705 } 706 if ((ci->ci_flags & CPUF_RUNNING) == 0) { 707 aprint_error_dev(ci->ci_dev, "CPU failed to start\n"); 708 #if defined(MPDEBUG) && defined(DDB) 709 printf("dropping into debugger; continue from here to resume boot\n"); 710 Debugger(); 711 #endif 712 } 713 } 714 715 /* 716 * The CPU ends up here when its ready to run 717 * This is called from code in mptramp.s; at this point, we are running 718 * in the idle pcb/idle stack of the new CPU. When this function returns, 719 * this processor will enter the idle loop and start looking for work. 720 * 721 * XXX should share some of this with init386 in machdep.c 722 */ 723 void 724 cpu_hatch(void *v) 725 { 726 struct cpu_info *ci = (struct cpu_info *)v; 727 struct pcb *pcb; 728 int s, i; 729 730 cpu_probe(ci); 731 732 cpu_feature[0] &= ~CPUID_FEAT_BLACKLIST; 733 cpu_feature[2] &= ~CPUID_FEAT_EXT_BLACKLIST; 734 735 cpu_init_msrs(ci, true); 736 737 KDASSERT((ci->ci_flags & CPUF_PRESENT) == 0); 738 atomic_or_32(&ci->ci_flags, CPUF_PRESENT); 739 while ((ci->ci_flags & CPUF_GO) == 0) { 740 /* Don't use delay, boot CPU may be patching the text. */ 741 for (i = 10000; i != 0; i--) 742 x86_pause(); 743 } 744 745 /* Because the text may have been patched in x86_patch(). */ 746 wbinvd(); 747 x86_flush(); 748 749 KASSERT((ci->ci_flags & CPUF_RUNNING) == 0); 750 751 pcb = lwp_getpcb(curlwp); 752 lcr3(pmap_kernel()->pm_pdirpa); 753 pcb->pcb_cr3 = pmap_kernel()->pm_pdirpa; 754 pcb = lwp_getpcb(ci->ci_data.cpu_idlelwp); 755 lcr0(pcb->pcb_cr0); 756 757 cpu_init_idt(); 758 gdt_init_cpu(ci); 759 lapic_enable(); 760 lapic_set_lvt(); 761 lapic_initclocks(); 762 763 #ifdef i386 764 npxinit(ci); 765 #else 766 fpuinit(ci); 767 #endif 768 769 lldt(GSEL(GLDT_SEL, SEL_KPL)); 770 ltr(ci->ci_tss_sel); 771 772 cpu_init(ci); 773 cpu_get_tsc_freq(ci); 774 775 s = splhigh(); 776 #ifdef i386 777 lapic_tpr = 0; 778 #else 779 lcr8(0); 780 #endif 781 x86_enable_intr(); 782 splx(s); 783 #if 0 784 x86_errata(); 785 #endif 786 787 aprint_debug_dev(ci->ci_dev, "CPU %ld running\n", 788 (long)ci->ci_cpuid); 789 } 790 791 #if defined(DDB) 792 793 #include <ddb/db_output.h> 794 #include <machine/db_machdep.h> 795 796 /* 797 * Dump CPU information from ddb. 798 */ 799 void 800 cpu_debug_dump(void) 801 { 802 struct cpu_info *ci; 803 CPU_INFO_ITERATOR cii; 804 805 db_printf("addr dev id flags ipis curlwp fpcurlwp\n"); 806 for (CPU_INFO_FOREACH(cii, ci)) { 807 db_printf("%p %s %ld %x %x %10p %10p\n", 808 ci, 809 ci->ci_dev == NULL ? "BOOT" : device_xname(ci->ci_dev), 810 (long)ci->ci_cpuid, 811 ci->ci_flags, ci->ci_ipis, 812 ci->ci_curlwp, 813 ci->ci_fpcurlwp); 814 } 815 } 816 #endif /* DDB */ 817 818 static void 819 cpu_copy_trampoline(void) 820 { 821 /* 822 * Copy boot code. 823 */ 824 extern u_char cpu_spinup_trampoline[]; 825 extern u_char cpu_spinup_trampoline_end[]; 826 827 vaddr_t mp_trampoline_vaddr; 828 829 mp_trampoline_vaddr = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, 830 UVM_KMF_VAONLY); 831 832 pmap_kenter_pa(mp_trampoline_vaddr, mp_trampoline_paddr, 833 VM_PROT_READ | VM_PROT_WRITE, 0); 834 pmap_update(pmap_kernel()); 835 memcpy((void *)mp_trampoline_vaddr, 836 cpu_spinup_trampoline, 837 cpu_spinup_trampoline_end - cpu_spinup_trampoline); 838 839 pmap_kremove(mp_trampoline_vaddr, PAGE_SIZE); 840 pmap_update(pmap_kernel()); 841 uvm_km_free(kernel_map, mp_trampoline_vaddr, PAGE_SIZE, UVM_KMF_VAONLY); 842 } 843 844 #endif /* MULTIPROCESSOR */ 845 846 #ifdef i386 847 #if 0 848 static void 849 tss_init(struct i386tss *tss, void *stack, void *func) 850 { 851 memset(tss, 0, sizeof *tss); 852 tss->tss_esp0 = tss->tss_esp = (int)((char *)stack + USPACE - 16); 853 tss->tss_ss0 = GSEL(GDATA_SEL, SEL_KPL); 854 tss->__tss_cs = GSEL(GCODE_SEL, SEL_KPL); 855 tss->tss_fs = GSEL(GCPU_SEL, SEL_KPL); 856 tss->tss_gs = tss->__tss_es = tss->__tss_ds = 857 tss->__tss_ss = GSEL(GDATA_SEL, SEL_KPL); 858 tss->tss_cr3 = pmap_kernel()->pm_pdirpa; 859 tss->tss_esp = (int)((char *)stack + USPACE - 16); 860 tss->tss_ldt = GSEL(GLDT_SEL, SEL_KPL); 861 tss->__tss_eflags = PSL_MBO | PSL_NT; /* XXX not needed? */ 862 tss->__tss_eip = (int)func; 863 } 864 #endif 865 866 /* XXX */ 867 #define IDTVEC(name) __CONCAT(X, name) 868 typedef void (vector)(void); 869 extern vector IDTVEC(tss_trap08); 870 #ifdef DDB 871 extern vector Xintrddbipi; 872 extern int ddb_vec; 873 #endif 874 875 static void 876 cpu_set_tss_gates(struct cpu_info *ci) 877 { 878 #if 0 879 struct segment_descriptor sd; 880 881 ci->ci_doubleflt_stack = (char *)uvm_km_alloc(kernel_map, USPACE, 0, 882 UVM_KMF_WIRED); 883 tss_init(&ci->ci_doubleflt_tss, ci->ci_doubleflt_stack, 884 IDTVEC(tss_trap08)); 885 setsegment(&sd, &ci->ci_doubleflt_tss, sizeof(struct i386tss) - 1, 886 SDT_SYS386TSS, SEL_KPL, 0, 0); 887 ci->ci_gdt[GTRAPTSS_SEL].sd = sd; 888 setgate(&idt[8], NULL, 0, SDT_SYSTASKGT, SEL_KPL, 889 GSEL(GTRAPTSS_SEL, SEL_KPL)); 890 #endif 891 892 #if defined(DDB) && defined(MULTIPROCESSOR) 893 /* 894 * Set up separate handler for the DDB IPI, so that it doesn't 895 * stomp on a possibly corrupted stack. 896 * 897 * XXX overwriting the gate set in db_machine_init. 898 * Should rearrange the code so that it's set only once. 899 */ 900 ci->ci_ddbipi_stack = (char *)uvm_km_alloc(kernel_map, USPACE, 0, 901 UVM_KMF_WIRED); 902 tss_init(&ci->ci_ddbipi_tss, ci->ci_ddbipi_stack, 903 Xintrddbipi); 904 905 setsegment(&sd, &ci->ci_ddbipi_tss, sizeof(struct i386tss) - 1, 906 SDT_SYS386TSS, SEL_KPL, 0, 0); 907 ci->ci_gdt[GIPITSS_SEL].sd = sd; 908 909 setgate(&idt[ddb_vec], NULL, 0, SDT_SYSTASKGT, SEL_KPL, 910 GSEL(GIPITSS_SEL, SEL_KPL)); 911 #endif 912 } 913 #else 914 static void 915 cpu_set_tss_gates(struct cpu_info *ci) 916 { 917 918 } 919 #endif /* i386 */ 920 921 int 922 mp_cpu_start(struct cpu_info *ci, paddr_t target) 923 { 924 #if 0 925 #if NLAPIC > 0 926 int error; 927 #endif 928 unsigned short dwordptr[2]; 929 930 /* 931 * Bootstrap code must be addressable in real mode 932 * and it must be page aligned. 933 */ 934 KASSERT(target < 0x10000 && target % PAGE_SIZE == 0); 935 936 /* 937 * "The BSP must initialize CMOS shutdown code to 0Ah ..." 938 */ 939 940 outb(IO_RTC, NVRAM_RESET); 941 outb(IO_RTC+1, NVRAM_RESET_JUMP); 942 943 /* 944 * "and the warm reset vector (DWORD based at 40:67) to point 945 * to the AP startup code ..." 946 */ 947 948 dwordptr[0] = 0; 949 dwordptr[1] = target >> 4; 950 951 pmap_kenter_pa (0, 0, VM_PROT_READ|VM_PROT_WRITE, 0); 952 pmap_update(pmap_kernel()); 953 954 memcpy ((uint8_t *) 0x467, dwordptr, 4); 955 956 pmap_kremove (0, PAGE_SIZE); 957 pmap_update(pmap_kernel()); 958 959 #if NLAPIC > 0 960 /* 961 * ... prior to executing the following sequence:" 962 */ 963 964 if (ci->ci_flags & CPUF_AP) { 965 if ((error = x86_ipi_init(ci->ci_cpuid)) != 0) 966 return error; 967 968 delay(10000); 969 970 if (cpu_feature & CPUID_APIC) { 971 error = x86_ipi_init(ci->ci_cpuid); 972 if (error != 0) { 973 aprint_error_dev(ci->ci_dev, "%s: IPI not taken (1)\n", 974 __func__); 975 return error; 976 } 977 978 delay(10000); 979 980 error = x86_ipi(target / PAGE_SIZE, ci->ci_cpuid, 981 LAPIC_DLMODE_STARTUP); 982 if (error != 0) { 983 aprint_error_dev(ci->ci_dev, "%s: IPI not taken (2)\n", 984 __func__); 985 return error; 986 } 987 delay(200); 988 989 error = x86_ipi(target / PAGE_SIZE, ci->ci_cpuid, 990 LAPIC_DLMODE_STARTUP); 991 if (error != 0) { 992 aprint_error_dev(ci->ci_dev, "%s: IPI not taken ((3)\n", 993 __func__); 994 return error; 995 } 996 delay(200); 997 } 998 } 999 #endif 1000 #endif /* 0 */ 1001 return 0; 1002 } 1003 1004 void 1005 mp_cpu_start_cleanup(struct cpu_info *ci) 1006 { 1007 #if 0 1008 /* 1009 * Ensure the NVRAM reset byte contains something vaguely sane. 1010 */ 1011 1012 outb(IO_RTC, NVRAM_RESET); 1013 outb(IO_RTC+1, NVRAM_RESET_RST); 1014 #endif 1015 } 1016 1017 void 1018 cpu_init_msrs(struct cpu_info *ci, bool full) 1019 { 1020 #ifdef __x86_64__ 1021 if (full) { 1022 HYPERVISOR_set_segment_base (SEGBASE_FS, 0); 1023 HYPERVISOR_set_segment_base (SEGBASE_GS_KERNEL, (uint64_t) ci); 1024 HYPERVISOR_set_segment_base (SEGBASE_GS_USER, 0); 1025 } 1026 #endif /* __x86_64__ */ 1027 1028 if (cpu_feature[2] & CPUID_NOX) 1029 wrmsr(MSR_EFER, rdmsr(MSR_EFER) | EFER_NXE); 1030 } 1031 1032 void 1033 cpu_offline_md(void) 1034 { 1035 int s; 1036 1037 s = splhigh(); 1038 #ifdef __i386__ 1039 npxsave_cpu(true); 1040 #else 1041 fpusave_cpu(true); 1042 #endif 1043 splx(s); 1044 } 1045 1046 #if 0 1047 /* XXX joerg restructure and restart CPUs individually */ 1048 static bool 1049 cpu_suspend(device_t dv, const pmf_qual_t *qual) 1050 { 1051 struct cpu_softc *sc = device_private(dv); 1052 struct cpu_info *ci = sc->sc_info; 1053 int err; 1054 1055 if (ci->ci_flags & CPUF_PRIMARY) 1056 return true; 1057 if (ci->ci_data.cpu_idlelwp == NULL) 1058 return true; 1059 if ((ci->ci_flags & CPUF_PRESENT) == 0) 1060 return true; 1061 1062 sc->sc_wasonline = !(ci->ci_schedstate.spc_flags & SPCF_OFFLINE); 1063 1064 if (sc->sc_wasonline) { 1065 mutex_enter(&cpu_lock); 1066 err = cpu_setstate(ci, false); 1067 mutex_exit(&cpu_lock); 1068 1069 if (err) 1070 return false; 1071 } 1072 1073 return true; 1074 } 1075 1076 static bool 1077 cpu_resume(device_t dv, const pmf_qual_t *qual) 1078 { 1079 struct cpu_softc *sc = device_private(dv); 1080 struct cpu_info *ci = sc->sc_info; 1081 int err = 0; 1082 1083 if (ci->ci_flags & CPUF_PRIMARY) 1084 return true; 1085 if (ci->ci_data.cpu_idlelwp == NULL) 1086 return true; 1087 if ((ci->ci_flags & CPUF_PRESENT) == 0) 1088 return true; 1089 1090 if (sc->sc_wasonline) { 1091 mutex_enter(&cpu_lock); 1092 err = cpu_setstate(ci, true); 1093 mutex_exit(&cpu_lock); 1094 } 1095 1096 return err == 0; 1097 } 1098 #endif 1099 1100 void 1101 cpu_get_tsc_freq(struct cpu_info *ci) 1102 { 1103 const volatile vcpu_time_info_t *tinfo = &ci->ci_vcpu->time; 1104 delay(1000000); 1105 uint64_t freq = 1000000000ULL << 32; 1106 freq = freq / (uint64_t)tinfo->tsc_to_system_mul; 1107 if ( tinfo->tsc_shift < 0 ) 1108 freq = freq << -tinfo->tsc_shift; 1109 else 1110 freq = freq >> tinfo->tsc_shift; 1111 ci->ci_data.cpu_cc_freq = freq; 1112 } 1113 1114 void 1115 x86_cpu_idle_xen(void) 1116 { 1117 struct cpu_info *ci = curcpu(); 1118 1119 KASSERT(ci->ci_ilevel == IPL_NONE); 1120 1121 x86_disable_intr(); 1122 if (!__predict_false(ci->ci_want_resched)) { 1123 idle_block(); 1124 } else { 1125 x86_enable_intr(); 1126 } 1127 } 1128 1129 /* 1130 * Loads pmap for the current CPU. 1131 */ 1132 void 1133 cpu_load_pmap(struct pmap *pmap) 1134 { 1135 #ifdef i386 1136 #ifdef PAE 1137 int i, s; 1138 struct cpu_info *ci; 1139 1140 s = splvm(); /* just to be safe */ 1141 ci = curcpu(); 1142 paddr_t l3_pd = xpmap_ptom_masked(ci->ci_pae_l3_pdirpa); 1143 /* don't update the kernel L3 slot */ 1144 for (i = 0 ; i < PDP_SIZE - 1; i++) { 1145 xpq_queue_pte_update(l3_pd + i * sizeof(pd_entry_t), 1146 xpmap_ptom(pmap->pm_pdirpa[i]) | PG_V); 1147 } 1148 splx(s); 1149 tlbflush(); 1150 #else /* PAE */ 1151 lcr3(pmap_pdirpa(pmap, 0)); 1152 #endif /* PAE */ 1153 #endif /* i386 */ 1154 1155 #ifdef __x86_64__ 1156 int i, s; 1157 pd_entry_t *old_pgd, *new_pgd; 1158 paddr_t addr; 1159 struct cpu_info *ci; 1160 1161 /* kernel pmap always in cr3 and should never go in user cr3 */ 1162 if (pmap_pdirpa(pmap, 0) != pmap_pdirpa(pmap_kernel(), 0)) { 1163 ci = curcpu(); 1164 /* 1165 * Map user space address in kernel space and load 1166 * user cr3 1167 */ 1168 s = splvm(); 1169 new_pgd = pmap->pm_pdir; 1170 old_pgd = pmap_kernel()->pm_pdir; 1171 addr = xpmap_ptom(pmap_pdirpa(pmap_kernel(), 0)); 1172 for (i = 0; i < PDIR_SLOT_PTE; 1173 i++, addr += sizeof(pd_entry_t)) { 1174 if ((new_pgd[i] & PG_V) || (old_pgd[i] & PG_V)) 1175 xpq_queue_pte_update(addr, new_pgd[i]); 1176 } 1177 tlbflush(); 1178 xen_set_user_pgd(pmap_pdirpa(pmap, 0)); 1179 ci->ci_xen_current_user_pgd = pmap_pdirpa(pmap, 0); 1180 splx(s); 1181 } 1182 #endif /* __x86_64__ */ 1183 } 1184
Indexes created Fri Oct 03 02:09:57 GMT 2025