cpu.c revision 1.56.2.3
1 /* $NetBSD: cpu.c,v 1.56.2.3 2011/07/16 10:59:46 cherry Exp $ */ 2 /* NetBSD: cpu.c,v 1.18 2004/02/20 17:35:01 yamt Exp */ 3 4 /*- 5 * Copyright (c) 2000 The NetBSD Foundation, Inc. 6 * Copyright (c) 2002, 2006, 2007 YAMAMOTO Takashi, 7 * All rights reserved. 8 * 9 * This code is derived from software contributed to The NetBSD Foundation 10 * by RedBack Networks Inc. 11 * 12 * Author: Bill Sommerfeld 13 * 14 * Redistribution and use in source and binary forms, with or without 15 * modification, are permitted provided that the following conditions 16 * are met: 17 * 1. Redistributions of source code must retain the above copyright 18 * notice, this list of conditions and the following disclaimer. 19 * 2. Redistributions in binary form must reproduce the above copyright 20 * notice, this list of conditions and the following disclaimer in the 21 * documentation and/or other materials provided with the distribution. 22 * 23 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 24 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 25 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 26 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 27 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 28 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 29 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 30 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 31 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 32 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 33 * POSSIBILITY OF SUCH DAMAGE. 34 */ 35 36 /* 37 * Copyright (c) 1999 Stefan Grefen 38 * 39 * Redistribution and use in source and binary forms, with or without 40 * modification, are permitted provided that the following conditions 41 * are met: 42 * 1. Redistributions of source code must retain the above copyright 43 * notice, this list of conditions and the following disclaimer. 44 * 2. Redistributions in binary form must reproduce the above copyright 45 * notice, this list of conditions and the following disclaimer in the 46 * documentation and/or other materials provided with the distribution. 47 * 3. All advertising materials mentioning features or use of this software 48 * must display the following acknowledgement: 49 * This product includes software developed by the NetBSD 50 * Foundation, Inc. and its contributors. 51 * 4. Neither the name of The NetBSD Foundation nor the names of its 52 * contributors may be used to endorse or promote products derived 53 * from this software without specific prior written permission. 54 * 55 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND ANY 56 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 57 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 58 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR AND CONTRIBUTORS BE LIABLE 59 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 60 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 61 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 62 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 63 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 64 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 65 * SUCH DAMAGE. 66 */ 67 68 #include <sys/cdefs.h> 69 __KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.56.2.3 2011/07/16 10:59:46 cherry Exp $"); 70 71 #include "opt_ddb.h" 72 #include "opt_multiprocessor.h" 73 #include "opt_mpbios.h" /* for MPDEBUG */ 74 #include "opt_mtrr.h" 75 #include "opt_xen.h" 76 77 #include "lapic.h" 78 #include "ioapic.h" 79 80 #include <sys/param.h> 81 #include <sys/proc.h> 82 #include <sys/systm.h> 83 #include <sys/device.h> 84 #include <sys/kmem.h> 85 #include <sys/cpu.h> 86 #include <sys/atomic.h> 87 #include <sys/reboot.h> 88 #include <sys/idle.h> 89 90 #include <uvm/uvm.h> 91 92 #include <machine/cpufunc.h> 93 #include <machine/cpuvar.h> 94 #include <machine/pmap.h> 95 #include <machine/vmparam.h> 96 #include <machine/mpbiosvar.h> 97 #include <machine/pcb.h> 98 #include <machine/specialreg.h> 99 #include <machine/segments.h> 100 #include <machine/gdt.h> 101 #include <machine/mtrr.h> 102 #include <machine/pio.h> 103 #include <machine/fpu.h> 104 105 #include <xen/xen.h> 106 #include <xen/xen3-public/vcpu.h> 107 #include <xen/vcpuvar.h> 108 109 #if NLAPIC > 0 110 #include <machine/apicvar.h> 111 #include <machine/i82489reg.h> 112 #include <machine/i82489var.h> 113 #endif 114 115 #include <dev/ic/mc146818reg.h> 116 #include <dev/isa/isareg.h> 117 118 #if MAXCPUS > 32 119 #error cpu_info contains 32bit bitmasks 120 #endif 121 122 static int cpu_match(device_t, cfdata_t, void *); 123 static void cpu_attach(device_t, device_t, void *); 124 static void cpu_defer(device_t); 125 static int cpu_rescan(device_t, const char *, const int *); 126 static void cpu_childdetached(device_t, device_t); 127 static int vcpu_match(device_t, cfdata_t, void *); 128 static void vcpu_attach(device_t, device_t, void *); 129 static void cpu_attach_common(device_t, device_t, void *); 130 void cpu_offline_md(void); 131 132 struct cpu_softc { 133 device_t sc_dev; /* device tree glue */ 134 struct cpu_info *sc_info; /* pointer to CPU info */ 135 bool sc_wasonline; 136 }; 137 138 int mp_cpu_start(struct cpu_info *, paddr_t); 139 void mp_cpu_start_cleanup(struct cpu_info *); 140 const struct cpu_functions mp_cpu_funcs = { mp_cpu_start, NULL, 141 mp_cpu_start_cleanup }; 142 143 CFATTACH_DECL2_NEW(cpu, sizeof(struct cpu_softc), 144 cpu_match, cpu_attach, NULL, NULL, cpu_rescan, cpu_childdetached); 145 146 CFATTACH_DECL_NEW(vcpu, sizeof(struct cpu_softc), 147 vcpu_match, vcpu_attach, NULL, NULL); 148 149 /* 150 * Statically-allocated CPU info for the primary CPU (or the only 151 * CPU, on uniprocessors). The CPU info list is initialized to 152 * point at it. 153 */ 154 #ifdef TRAPLOG 155 #include <machine/tlog.h> 156 struct tlog tlog_primary; 157 #endif 158 struct cpu_info cpu_info_primary __aligned(CACHE_LINE_SIZE) = { 159 .ci_dev = 0, 160 .ci_self = &cpu_info_primary, 161 .ci_idepth = -1, 162 .ci_curlwp = &lwp0, 163 .ci_curldt = -1, 164 .ci_cpumask = 1, 165 #ifdef TRAPLOG 166 .ci_tlog = &tlog_primary, 167 #endif 168 169 }; 170 struct cpu_info phycpu_info_primary __aligned(CACHE_LINE_SIZE) = { 171 .ci_dev = 0, 172 .ci_self = &phycpu_info_primary, 173 }; 174 175 struct cpu_info *cpu_info_list = &cpu_info_primary; 176 struct cpu_info *phycpu_info_list = &phycpu_info_primary; 177 178 static void cpu_set_tss_gates(struct cpu_info *ci); 179 180 uint32_t cpus_attached = 1; 181 uint32_t cpus_running = 0; 182 183 uint32_t phycpus_attached = 0; 184 uint32_t phycpus_running = 0; 185 186 uint32_t cpu_feature[5]; /* X86 CPUID feature bits 187 * [0] basic features %edx 188 * [1] basic features %ecx 189 * [2] extended features %edx 190 * [3] extended features %ecx 191 * [4] VIA padlock features 192 */ 193 194 bool x86_mp_online; 195 paddr_t mp_trampoline_paddr = MP_TRAMPOLINE; 196 197 #if defined(MULTIPROCESSOR) 198 void cpu_hatch(void *); 199 static void cpu_boot_secondary(struct cpu_info *ci); 200 static void cpu_start_secondary(struct cpu_info *ci); 201 #endif /* MULTIPROCESSOR */ 202 203 static int 204 cpu_match(device_t parent, cfdata_t match, void *aux) 205 { 206 207 return 1; 208 } 209 210 static void 211 cpu_attach(device_t parent, device_t self, void *aux) 212 { 213 struct cpu_softc *sc = device_private(self); 214 struct cpu_attach_args *caa = aux; 215 struct cpu_info *ci; 216 uintptr_t ptr; 217 static int nphycpu = 0; 218 219 sc->sc_dev = self; 220 221 if (phycpus_attached == ~0) { 222 aprint_error(": increase MAXCPUS\n"); 223 return; 224 } 225 226 /* 227 * If we're an Application Processor, allocate a cpu_info 228 * If we're the first attached CPU use the primary cpu_info, 229 * otherwise allocate a new one 230 */ 231 aprint_naive("\n"); 232 aprint_normal("\n"); 233 if (nphycpu > 0) { 234 struct cpu_info *tmp; 235 ptr = (uintptr_t)kmem_zalloc(sizeof(*ci) + CACHE_LINE_SIZE - 1, 236 KM_SLEEP); 237 ci = (struct cpu_info *)roundup2(ptr, CACHE_LINE_SIZE); 238 ci->ci_curldt = -1; 239 240 tmp = phycpu_info_list; 241 while (tmp->ci_next) 242 tmp = tmp->ci_next; 243 244 tmp->ci_next = ci; 245 } else { 246 ci = &phycpu_info_primary; 247 } 248 249 ci->ci_self = ci; 250 sc->sc_info = ci; 251 252 ci->ci_dev = self; 253 ci->ci_acpiid = caa->cpu_id; 254 ci->ci_cpuid = caa->cpu_number; 255 ci->ci_vcpu = NULL; 256 ci->ci_index = nphycpu++; 257 ci->ci_cpumask = (1 << cpu_index(ci)); 258 259 atomic_or_32(&phycpus_attached, ci->ci_cpumask); 260 261 if (!pmf_device_register(self, NULL, NULL)) 262 aprint_error_dev(self, "couldn't establish power handler\n"); 263 264 (void)config_defer(self, cpu_defer); 265 } 266 267 static void 268 cpu_defer(device_t self) 269 { 270 cpu_rescan(self, NULL, NULL); 271 } 272 273 static int 274 cpu_rescan(device_t self, const char *ifattr, const int *locators) 275 { 276 struct cpu_softc *sc = device_private(self); 277 struct cpufeature_attach_args cfaa; 278 struct cpu_info *ci = sc->sc_info; 279 280 memset(&cfaa, 0, sizeof(cfaa)); 281 cfaa.ci = ci; 282 283 if (ifattr_match(ifattr, "cpufeaturebus")) { 284 285 if (ci->ci_frequency == NULL) { 286 cfaa.name = "frequency"; 287 ci->ci_frequency = config_found_ia(self, 288 "cpufeaturebus", &cfaa, NULL); 289 } 290 } 291 292 return 0; 293 } 294 295 static void 296 cpu_childdetached(device_t self, device_t child) 297 { 298 struct cpu_softc *sc = device_private(self); 299 struct cpu_info *ci = sc->sc_info; 300 301 if (ci->ci_frequency == child) 302 ci->ci_frequency = NULL; 303 } 304 305 static int 306 vcpu_match(device_t parent, cfdata_t match, void *aux) 307 { 308 struct vcpu_attach_args *vcaa = aux; 309 struct vcpu_runstate_info vcr; 310 int error; 311 312 if (strcmp(vcaa->vcaa_name, match->cf_name) == 0) { 313 error = HYPERVISOR_vcpu_op(VCPUOP_get_runstate_info, 314 vcaa->vcaa_caa.cpu_number, 315 &vcr); 316 switch (error) { 317 case 0: 318 return 1; 319 case -ENOENT: 320 return 0; 321 default: 322 panic("Unknown hypervisor error %d returned on vcpu runstate probe\n", error); 323 } 324 } 325 326 return 0; 327 } 328 329 static void 330 vcpu_attach(device_t parent, device_t self, void *aux) 331 { 332 struct vcpu_attach_args *vcaa = aux; 333 334 KASSERT(vcaa->vcaa_caa.cpu_func == NULL); 335 vcaa->vcaa_caa.cpu_func = &mp_cpu_funcs; 336 cpu_attach_common(parent, self, &vcaa->vcaa_caa); 337 } 338 339 static int 340 vcpu_is_up(struct cpu_info *ci) 341 { 342 KASSERT(ci != NULL); 343 return HYPERVISOR_vcpu_op(VCPUOP_is_up, ci->ci_cpuid, NULL); 344 } 345 346 static void 347 cpu_vm_init(struct cpu_info *ci) 348 { 349 int ncolors = 2, i; 350 351 for (i = CAI_ICACHE; i <= CAI_L2CACHE; i++) { 352 struct x86_cache_info *cai; 353 int tcolors; 354 355 cai = &ci->ci_cinfo[i]; 356 357 tcolors = atop(cai->cai_totalsize); 358 switch(cai->cai_associativity) { 359 case 0xff: 360 tcolors = 1; /* fully associative */ 361 break; 362 case 0: 363 case 1: 364 break; 365 default: 366 tcolors /= cai->cai_associativity; 367 } 368 ncolors = max(ncolors, tcolors); 369 } 370 371 /* 372 * Knowing the size of the largest cache on this CPU, re-color 373 * our pages. 374 */ 375 if (ncolors <= uvmexp.ncolors) 376 return; 377 aprint_debug_dev(ci->ci_dev, "%d page colors\n", ncolors); 378 uvm_page_recolor(ncolors); 379 } 380 381 static void 382 cpu_attach_common(device_t parent, device_t self, void *aux) 383 { 384 struct cpu_softc *sc = device_private(self); 385 struct cpu_attach_args *caa = aux; 386 struct cpu_info *ci; 387 uintptr_t ptr; 388 int cpunum = caa->cpu_number; 389 static bool again = false; 390 391 sc->sc_dev = self; 392 393 /* 394 * If we're an Application Processor, allocate a cpu_info 395 * structure, otherwise use the primary's. 396 */ 397 if (caa->cpu_role == CPU_ROLE_AP) { 398 aprint_naive(": Application Processor\n"); 399 ptr = (uintptr_t)kmem_alloc(sizeof(*ci) + CACHE_LINE_SIZE - 1, 400 KM_SLEEP); 401 ci = (struct cpu_info *)roundup2(ptr, CACHE_LINE_SIZE); 402 memset(ci, 0, sizeof(*ci)); 403 #ifdef TRAPLOG 404 ci->ci_tlog_base = kmem_zalloc(sizeof(struct tlog), KM_SLEEP); 405 #endif 406 } else { 407 aprint_naive(": %s Processor\n", 408 caa->cpu_role == CPU_ROLE_SP ? "Single" : "Boot"); 409 ci = &cpu_info_primary; 410 } 411 412 ci->ci_self = ci; 413 sc->sc_info = ci; 414 ci->ci_dev = self; 415 ci->ci_cpuid = cpunum; 416 417 KASSERT(HYPERVISOR_shared_info != NULL); 418 ci->ci_vcpu = &HYPERVISOR_shared_info->vcpu_info[cpunum]; 419 420 KASSERT(ci->ci_func == 0); 421 ci->ci_func = caa->cpu_func; 422 423 /* Must be called before mi_cpu_attach(). */ 424 cpu_vm_init(ci); 425 426 if (caa->cpu_role == CPU_ROLE_AP) { 427 int error; 428 429 error = mi_cpu_attach(ci); 430 431 KASSERT(ci->ci_data.cpu_idlelwp != NULL); 432 if (error != 0) { 433 aprint_normal("\n"); 434 aprint_error_dev(self, 435 "mi_cpu_attach failed with %d\n", error); 436 return; 437 } 438 439 } else { 440 KASSERT(ci->ci_data.cpu_idlelwp != NULL); 441 } 442 443 ci->ci_cpumask = (1 << cpu_index(ci)); 444 pmap_reference(pmap_kernel()); 445 ci->ci_pmap = pmap_kernel(); 446 ci->ci_tlbstate = TLBSTATE_STALE; 447 448 /* 449 * Boot processor may not be attached first, but the below 450 * must be done to allow booting other processors. 451 */ 452 if (!again) { 453 atomic_or_32(&ci->ci_flags, CPUF_PRESENT | CPUF_PRIMARY); 454 /* Basic init. */ 455 cpu_intr_init(ci); 456 cpu_get_tsc_freq(ci); 457 cpu_init(ci); 458 cpu_set_tss_gates(ci); 459 pmap_cpu_init_late(ci); /* XXX: cosmetic */ 460 461 /* Every processor needs to init it's own ipi h/w (similar to lapic) */ 462 xen_ipi_init(); 463 /* XXX: clock_init() */ 464 465 /* Make sure DELAY() is initialized. */ 466 DELAY(1); 467 again = true; 468 } 469 470 /* further PCB init done later. */ 471 472 switch (caa->cpu_role) { 473 case CPU_ROLE_SP: 474 atomic_or_32(&ci->ci_flags, CPUF_SP); 475 cpu_identify(ci); 476 #if 0 477 x86_errata(); 478 #endif 479 x86_cpu_idle_init(); 480 481 break; 482 483 case CPU_ROLE_BP: 484 atomic_or_32(&ci->ci_flags, CPUF_BSP); 485 cpu_identify(ci); 486 cpu_init(ci); 487 #if 0 488 x86_errata(); 489 #endif 490 x86_cpu_idle_init(); 491 492 break; 493 494 case CPU_ROLE_AP: 495 atomic_or_32(&ci->ci_flags, CPUF_AP); 496 497 /* 498 * report on an AP 499 */ 500 501 #if defined(MULTIPROCESSOR) 502 /* interrupt handler stack */ 503 cpu_intr_init(ci); 504 505 /* Setup gdt */ 506 gdt_alloc_cpu(ci); 507 //gdt_init_cpu(ci); 508 509 cpu_set_tss_gates(ci); 510 pmap_cpu_init_late(ci); 511 cpu_start_secondary(ci); 512 513 if (ci->ci_flags & CPUF_PRESENT) { 514 struct cpu_info *tmp; 515 516 cpu_identify(ci); 517 tmp = cpu_info_list; 518 while (tmp->ci_next) 519 tmp = tmp->ci_next; 520 521 tmp->ci_next = ci; 522 } 523 #else 524 aprint_error(": not started\n"); 525 #endif 526 break; 527 528 default: 529 aprint_normal("\n"); 530 panic("unknown processor type??\n"); 531 } 532 533 pat_init(ci); 534 atomic_or_32(&cpus_attached, ci->ci_cpumask); 535 536 #if 0 537 if (!pmf_device_register(self, cpu_suspend, cpu_resume)) 538 aprint_error_dev(self, "couldn't establish power handler\n"); 539 #endif 540 541 #ifdef MPVERBOSE 542 if (mp_verbose) { 543 struct lwp *l = ci->ci_data.cpu_idlelwp; 544 struct pcb *pcb = lwp_getpcb(l); 545 546 aprint_verbose_dev(self, 547 "idle lwp at %p, idle sp at 0x%p\n", 548 l, 549 #ifdef i386 550 (void *)pcb->pcb_esp 551 #else /* i386 */ 552 (void *)pcb->pcb_rsp 553 #endif /* i386 */ 554 ); 555 556 } 557 #endif /* MPVERBOSE */ 558 } 559 560 /* 561 * Initialize the processor appropriately. 562 */ 563 564 void 565 cpu_init(struct cpu_info *ci) 566 { 567 568 /* 569 * On a P6 or above, enable global TLB caching if the 570 * hardware supports it. 571 */ 572 if (cpu_feature[0] & CPUID_PGE) 573 lcr4(rcr4() | CR4_PGE); /* enable global TLB caching */ 574 575 #ifdef XXXMTRR 576 /* 577 * On a P6 or above, initialize MTRR's if the hardware supports them. 578 */ 579 if (cpu_feature[0] & CPUID_MTRR) { 580 if ((ci->ci_flags & CPUF_AP) == 0) 581 i686_mtrr_init_first(); 582 mtrr_init_cpu(ci); 583 } 584 #endif 585 /* 586 * If we have FXSAVE/FXRESTOR, use them. 587 */ 588 if (cpu_feature[0] & CPUID_FXSR) { 589 lcr4(rcr4() | CR4_OSFXSR); 590 591 /* 592 * If we have SSE/SSE2, enable XMM exceptions. 593 */ 594 if (cpu_feature[0] & (CPUID_SSE|CPUID_SSE2)) 595 lcr4(rcr4() | CR4_OSXMMEXCPT); 596 } 597 598 #ifdef __x86_64__ 599 /* No user PGD mapped for this CPU yet */ 600 ci->ci_xen_current_user_pgd = 0; 601 #endif 602 603 atomic_or_32(&cpus_running, ci->ci_cpumask); 604 atomic_or_32(&ci->ci_flags, CPUF_RUNNING); 605 606 /* XXX: register vcpu_register_runstate_memory_area, and figure out how to make sure this VCPU is running ? */ 607 } 608 609 610 #ifdef MULTIPROCESSOR 611 612 void 613 cpu_boot_secondary_processors(void) 614 { 615 struct cpu_info *ci; 616 u_long i; 617 for (i = 0; i < maxcpus; i++) { 618 ci = cpu_lookup(i); 619 if (ci == NULL) 620 continue; 621 if (ci->ci_data.cpu_idlelwp == NULL) 622 continue; 623 if ((ci->ci_flags & CPUF_PRESENT) == 0) 624 continue; 625 if (ci->ci_flags & (CPUF_BSP|CPUF_SP|CPUF_PRIMARY)) 626 continue; 627 cpu_boot_secondary(ci); 628 } 629 630 x86_mp_online = true; 631 } 632 633 static void 634 cpu_init_idle_lwp(struct cpu_info *ci) 635 { 636 struct lwp *l = ci->ci_data.cpu_idlelwp; 637 struct pcb *pcb = lwp_getpcb(l); 638 639 pcb->pcb_cr0 = rcr0(); 640 } 641 642 void 643 cpu_init_idle_lwps(void) 644 { 645 struct cpu_info *ci; 646 u_long i; 647 648 for (i = 0; i < maxcpus; i++) { 649 ci = cpu_lookup(i); 650 if (ci == NULL) 651 continue; 652 if (ci->ci_data.cpu_idlelwp == NULL) 653 continue; 654 if ((ci->ci_flags & CPUF_PRESENT) == 0) 655 continue; 656 cpu_init_idle_lwp(ci); 657 } 658 } 659 660 static void 661 cpu_start_secondary(struct cpu_info *ci) 662 { 663 int i; 664 665 aprint_debug_dev(ci->ci_dev, "starting\n"); 666 667 ci->ci_curlwp = ci->ci_data.cpu_idlelwp; 668 669 if (CPU_STARTUP(ci, (vaddr_t) cpu_hatch) != 0) { 670 return; 671 } 672 673 /* 674 * wait for it to become ready 675 */ 676 for (i = 100000; (!(ci->ci_flags & CPUF_PRESENT)) && i > 0; i--) { 677 #ifdef MPDEBUG 678 extern int cpu_trace[3]; 679 static int otrace[3]; 680 if (memcmp(otrace, cpu_trace, sizeof(otrace)) != 0) { 681 aprint_debug_dev(ci->ci_dev, "trace %02x %02x %02x\n", 682 cpu_trace[0], cpu_trace[1], cpu_trace[2]); 683 memcpy(otrace, cpu_trace, sizeof(otrace)); 684 } 685 #endif 686 delay(10); 687 } 688 if ((ci->ci_flags & CPUF_PRESENT) == 0) { 689 aprint_error_dev(ci->ci_dev, "failed to become ready\n"); 690 #if defined(MPDEBUG) && defined(DDB) 691 printf("dropping into debugger; continue from here to resume boot\n"); 692 Debugger(); 693 #endif 694 } 695 696 CPU_START_CLEANUP(ci); 697 } 698 699 void 700 cpu_boot_secondary(struct cpu_info *ci) 701 { 702 int i; 703 atomic_or_32(&ci->ci_flags, CPUF_GO); 704 for (i = 100000; (!(ci->ci_flags & CPUF_RUNNING)) && i > 0; i--) { 705 delay(10); 706 } 707 if ((ci->ci_flags & CPUF_RUNNING) == 0) { 708 aprint_error_dev(ci->ci_dev, "CPU failed to start\n"); 709 #if defined(MPDEBUG) && defined(DDB) 710 printf("dropping into debugger; continue from here to resume boot\n"); 711 Debugger(); 712 #endif 713 } 714 } 715 716 /* 717 * APs end up here immediately after initialisation and VCPUOP_up in 718 * mp_cpu_start(). 719 * At this point, we are running in the idle pcb/idle stack of the new 720 * CPU. This function jumps to the idle loop and starts looking for 721 * work. 722 */ 723 extern void x86_64_tls_switch(struct lwp *); 724 void 725 cpu_hatch(void *v) 726 { 727 struct cpu_info *ci = (struct cpu_info *)v; 728 struct pcb *pcb; 729 int s, i; 730 731 /* Setup TLS and kernel GS */ 732 cpu_init_msrs(ci, true); 733 cpu_probe(ci); 734 735 atomic_or_32(&ci->ci_flags, CPUF_PRESENT); 736 737 while ((ci->ci_flags & CPUF_GO) == 0) { 738 /* Don't use delay, boot CPU may be patching the text. */ 739 for (i = 10000; i != 0; i--) 740 x86_pause(); 741 } 742 743 /* Because the text may have been patched in x86_patch(). */ 744 x86_flush(); 745 tlbflushg(); 746 747 KASSERT((ci->ci_flags & CPUF_RUNNING) == 0); 748 749 pcb = lwp_getpcb(curlwp); 750 pcb->pcb_cr3 = pmap_pdirpa(pmap_kernel(), 0); /* XXX: consider using pmap_load() ? */ 751 pcb = lwp_getpcb(ci->ci_data.cpu_idlelwp); 752 lcr0(pcb->pcb_cr0); 753 754 cpu_init_idt(); 755 gdt_init_cpu(ci); 756 xen_ipi_init(); 757 758 xen_initclocks(); 759 760 /* XXX: lapic_initclocks(); */ 761 762 #ifdef i386 763 #if NNPX > 0 764 npxinit(ci); 765 #endif 766 #else 767 fpuinit(ci); 768 /* XXX: fixme compile fpuinit(ci); */ 769 #endif 770 771 lldt(GSEL(GLDT_SEL, SEL_KPL)); 772 ltr(ci->ci_tss_sel); 773 774 cpu_init(ci); 775 cpu_get_tsc_freq(ci); 776 777 s = splhigh(); 778 x86_enable_intr(); 779 splx(s); 780 #if 0 781 x86_errata(); 782 #endif 783 784 aprint_debug_dev(ci->ci_dev, "running\n"); 785 786 cpu_switchto(NULL, ci->ci_data.cpu_idlelwp, true); 787 788 panic("switch to idle_loop context returned!\n"); 789 /* NOTREACHED */ 790 } 791 792 #if defined(DDB) 793 794 #include <ddb/db_output.h> 795 #include <machine/db_machdep.h> 796 797 /* 798 * Dump CPU information from ddb. 799 */ 800 void 801 cpu_debug_dump(void) 802 { 803 struct cpu_info *ci; 804 CPU_INFO_ITERATOR cii; 805 806 db_printf("addr dev id flags ipis curlwp fpcurlwp\n"); 807 for (CPU_INFO_FOREACH(cii, ci)) { 808 db_printf("%p %s %ld %x %x %10p %10p\n", 809 ci, 810 ci->ci_dev == NULL ? "BOOT" : device_xname(ci->ci_dev), 811 (long)ci->ci_cpuid, 812 ci->ci_flags, ci->ci_ipis, 813 ci->ci_curlwp, 814 ci->ci_fpcurlwp); 815 } 816 } 817 #endif /* DDB */ 818 819 #endif /* MULTIPROCESSOR */ 820 821 #ifdef i386 822 #if 0 823 static void 824 tss_init(struct i386tss *tss, void *stack, void *func) 825 { 826 memset(tss, 0, sizeof *tss); 827 tss->tss_esp0 = tss->tss_esp = (int)((char *)stack + USPACE - 16); 828 tss->tss_ss0 = GSEL(GDATA_SEL, SEL_KPL); 829 tss->__tss_cs = GSEL(GCODE_SEL, SEL_KPL); 830 tss->tss_fs = GSEL(GCPU_SEL, SEL_KPL); 831 tss->tss_gs = tss->__tss_es = tss->__tss_ds = 832 tss->__tss_ss = GSEL(GDATA_SEL, SEL_KPL); 833 tss->tss_cr3 = pmap_kernel()->pm_pdirpa; 834 tss->tss_esp = (int)((char *)stack + USPACE - 16); 835 tss->tss_ldt = GSEL(GLDT_SEL, SEL_KPL); 836 tss->__tss_eflags = PSL_MBO | PSL_NT; /* XXX not needed? */ 837 tss->__tss_eip = (int)func; 838 } 839 #endif 840 841 /* XXX */ 842 #define IDTVEC(name) __CONCAT(X, name) 843 typedef void (vector)(void); 844 extern vector IDTVEC(tss_trap08); 845 #ifdef DDB 846 extern vector Xintrddbipi; 847 extern int ddb_vec; 848 #endif 849 850 static void 851 cpu_set_tss_gates(struct cpu_info *ci) 852 { 853 #if 0 854 struct segment_descriptor sd; 855 856 ci->ci_doubleflt_stack = (char *)uvm_km_alloc(kernel_map, USPACE, 0, 857 UVM_KMF_WIRED); 858 tss_init(&ci->ci_doubleflt_tss, ci->ci_doubleflt_stack, 859 IDTVEC(tss_trap08)); 860 setsegment(&sd, &ci->ci_doubleflt_tss, sizeof(struct i386tss) - 1, 861 SDT_SYS386TSS, SEL_KPL, 0, 0); 862 ci->ci_gdt[GTRAPTSS_SEL].sd = sd; 863 setgate(&idt[8], NULL, 0, SDT_SYSTASKGT, SEL_KPL, 864 GSEL(GTRAPTSS_SEL, SEL_KPL)); 865 #endif 866 867 #if defined(DDB) && defined(MULTIPROCESSOR) 868 /* 869 * Set up separate handler for the DDB IPI, so that it doesn't 870 * stomp on a possibly corrupted stack. 871 * 872 * XXX overwriting the gate set in db_machine_init. 873 * Should rearrange the code so that it's set only once. 874 */ 875 ci->ci_ddbipi_stack = (char *)uvm_km_alloc(kernel_map, USPACE, 0, 876 UVM_KMF_WIRED); 877 tss_init(&ci->ci_ddbipi_tss, ci->ci_ddbipi_stack, 878 Xintrddbipi); 879 880 setsegment(&sd, &ci->ci_ddbipi_tss, sizeof(struct i386tss) - 1, 881 SDT_SYS386TSS, SEL_KPL, 0, 0); 882 ci->ci_gdt[GIPITSS_SEL].sd = sd; 883 884 setgate(&idt[ddb_vec], NULL, 0, SDT_SYSTASKGT, SEL_KPL, 885 GSEL(GIPITSS_SEL, SEL_KPL)); 886 #endif 887 } 888 #else 889 static void 890 cpu_set_tss_gates(struct cpu_info *ci) 891 { 892 893 } 894 #endif /* i386 */ 895 896 extern void hypervisor_callback(void); 897 extern void failsafe_callback(void); 898 #ifdef __x86_64__ 899 typedef void (vector)(void); 900 extern vector Xsyscall, Xsyscall32; 901 #endif 902 903 /* 904 * Setup the "trampoline". On Xen, we setup nearly all cpu context 905 * outside a trampoline, so we prototype and call targetrip like so: 906 * void targetrip(struct cpu_info *); 907 */ 908 909 static void 910 gdt_prepframes(paddr_t *frames, vaddr_t base, uint32_t entries) 911 { 912 int i; 913 for (i = 0; i < roundup(entries, PAGE_SIZE) >> PAGE_SHIFT; i++) { 914 915 frames[i] = ((paddr_t) xpmap_ptetomach( 916 (pt_entry_t *) (base + (i << PAGE_SHIFT)))) 917 >> PAGE_SHIFT; 918 919 /* Mark Read-only */ 920 pmap_pte_clearbits(kvtopte(base + (i << PAGE_SHIFT)), 921 PG_RW); 922 } 923 } 924 925 extern char *ldtstore; /* XXX: Xen MP todo */ 926 927 static void 928 xen_init_amd64_vcpuctxt(struct cpu_info *ci, 929 struct vcpu_guest_context *initctx, 930 void targetrip(struct cpu_info *)) 931 { 932 /* page frames to point at GDT */ 933 extern int gdt_size; 934 paddr_t frames[16]; 935 psize_t gdt_ents; 936 937 struct lwp *l; 938 struct pcb *pcb; 939 940 volatile struct vcpu_info *vci; 941 942 KASSERT(ci != NULL); 943 KASSERT(ci != &cpu_info_primary); 944 KASSERT(initctx != NULL); 945 KASSERT(targetrip != NULL); 946 947 memset(initctx, 0, sizeof *initctx); 948 949 gdt_ents = roundup(gdt_size, PAGE_SIZE) >> PAGE_SHIFT; /* XXX: re-investigate roundup(gdt_size... ) for gdt_ents. */ 950 KASSERT(gdt_ents <= 16); 951 952 gdt_prepframes(frames, (vaddr_t) ci->ci_gdt, gdt_ents); 953 954 /* XXX: The stuff in here is amd64 specific. move to mptramp.[Sc] ? */ 955 956 /* Initialise the vcpu context: We use idle_loop()'s pcb context. */ 957 958 l = ci->ci_data.cpu_idlelwp; 959 960 KASSERT(l != NULL); 961 pcb = lwp_getpcb(l); 962 KASSERT(pcb != NULL); 963 964 /* resume with interrupts off */ 965 vci = ci->ci_vcpu; 966 vci->evtchn_upcall_mask = 1; 967 xen_mb(); 968 969 /* resume in kernel-mode */ 970 initctx->flags = VGCF_in_kernel | VGCF_online; 971 972 /* Stack and entry points */ 973 initctx->user_regs.rbp = pcb->pcb_rbp; 974 initctx->user_regs.rdi = (uint64_t) ci; /* targetrip(ci); */ 975 initctx->user_regs.rip = (vaddr_t) targetrip; 976 977 initctx->user_regs.cs = GSEL(GCODE_SEL, SEL_KPL); 978 979 initctx->user_regs.rflags = pcb->pcb_flags; 980 initctx->user_regs.rsp = pcb->pcb_rsp; 981 982 /* Data segments */ 983 initctx->user_regs.ss = GSEL(GDATA_SEL, SEL_KPL); 984 initctx->user_regs.es = GSEL(GDATA_SEL, SEL_KPL); 985 initctx->user_regs.ds = GSEL(GDATA_SEL, SEL_KPL); 986 987 /* GDT */ 988 memcpy(initctx->gdt_frames, frames, sizeof frames); 989 initctx->gdt_ents = gdt_ents; 990 991 /* LDT */ 992 initctx->ldt_base = (unsigned long) ldtstore; 993 initctx->ldt_ents = LDT_SIZE >> 3; 994 995 /* Kernel context state */ 996 initctx->kernel_ss = GSEL(GDATA_SEL, SEL_KPL); 997 initctx->kernel_sp = pcb->pcb_rsp0; 998 initctx->ctrlreg[0] = pcb->pcb_cr0; 999 initctx->ctrlreg[1] = 0; /* "resuming" from kernel - no User cr3. */ 1000 initctx->ctrlreg[2] = pcb->pcb_cr2; /* XXX: */ 1001 /* 1002 * Use pmap_kernel() L4 PD directly, until we setup the 1003 * per-cpu L4 PD in pmap_cpu_init_late() 1004 */ 1005 initctx->ctrlreg[3] = xpmap_ptom(pcb->pcb_cr3); 1006 initctx->ctrlreg[4] = CR4_PAE | CR4_OSFXSR | CR4_OSXMMEXCPT; 1007 1008 1009 /* Xen callbacks */ 1010 initctx->event_callback_eip = (unsigned long) hypervisor_callback; 1011 initctx->failsafe_callback_eip = (unsigned long) failsafe_callback; 1012 initctx->syscall_callback_eip = (unsigned long) Xsyscall; 1013 1014 return; 1015 } 1016 1017 int 1018 mp_cpu_start(struct cpu_info *ci, vaddr_t target) 1019 { 1020 1021 int hyperror; 1022 struct vcpu_guest_context vcpuctx; 1023 1024 KASSERT(ci != NULL); 1025 KASSERT(ci != &cpu_info_primary); 1026 KASSERT(ci->ci_flags & CPUF_AP); 1027 1028 xen_init_amd64_vcpuctxt(ci, &vcpuctx, (void (*)(struct cpu_info *))target); 1029 1030 /* Initialise the given vcpu to execute cpu_hatch(ci); */ 1031 if ((hyperror = HYPERVISOR_vcpu_op(VCPUOP_initialise, ci->ci_cpuid, &vcpuctx))) { 1032 aprint_error(": context initialisation failed. errno = %d\n", hyperror); 1033 return hyperror; 1034 } 1035 1036 /* Start it up */ 1037 1038 /* First bring it down - yay, thanks Xen documentation for omitting this slight detail - lost only about 1 week reading through crap */ 1039 if ((hyperror = HYPERVISOR_vcpu_op(VCPUOP_down, ci->ci_cpuid, NULL))) { 1040 aprint_error(": VCPUOP_down hypervisor command failed. errno = %d\n", hyperror); 1041 return hyperror; 1042 } 1043 1044 if ((hyperror = HYPERVISOR_vcpu_op(VCPUOP_up, ci->ci_cpuid, NULL))) { 1045 aprint_error(": VCPUOP_up hypervisor command failed. errno = %d\n", hyperror); 1046 return hyperror; 1047 } 1048 1049 if (!vcpu_is_up(ci)) { 1050 aprint_error(": did not come up\n"); 1051 return -1; 1052 } 1053 1054 return 0; 1055 } 1056 1057 void 1058 mp_cpu_start_cleanup(struct cpu_info *ci) 1059 { 1060 #if 0 1061 /* 1062 * Ensure the NVRAM reset byte contains something vaguely sane. 1063 */ 1064 1065 outb(IO_RTC, NVRAM_RESET); 1066 outb(IO_RTC+1, NVRAM_RESET_RST); 1067 #endif 1068 if (vcpu_is_up(ci)) { 1069 aprint_debug_dev(ci->ci_dev, "is started.\n"); 1070 } 1071 else { 1072 aprint_error_dev(ci->ci_dev, "did not start up.\n"); 1073 } 1074 1075 } 1076 1077 void 1078 cpu_init_msrs(struct cpu_info *ci, bool full) 1079 { 1080 #ifdef __x86_64__ 1081 if (full) { 1082 HYPERVISOR_set_segment_base (SEGBASE_FS, 0); 1083 HYPERVISOR_set_segment_base (SEGBASE_GS_KERNEL, (uint64_t) ci); 1084 HYPERVISOR_set_segment_base (SEGBASE_GS_USER, 0); 1085 } 1086 #endif /* __x86_64__ */ 1087 1088 if (cpu_feature[2] & CPUID_NOX) 1089 wrmsr(MSR_EFER, rdmsr(MSR_EFER) | EFER_NXE); 1090 1091 } 1092 1093 void 1094 cpu_offline_md(void) 1095 { 1096 int s; 1097 1098 s = splhigh(); 1099 #ifdef __i386__ 1100 npxsave_cpu(true); 1101 #else 1102 fpusave_cpu(true); 1103 #endif 1104 splx(s); 1105 } 1106 1107 #if 0 1108 /* XXX joerg restructure and restart CPUs individually */ 1109 static bool 1110 cpu_suspend(device_t dv, const pmf_qual_t *qual) 1111 { 1112 struct cpu_softc *sc = device_private(dv); 1113 struct cpu_info *ci = sc->sc_info; 1114 int err; 1115 1116 if (ci->ci_flags & CPUF_PRIMARY) 1117 return true; 1118 if (ci->ci_data.cpu_idlelwp == NULL) 1119 return true; 1120 if ((ci->ci_flags & CPUF_PRESENT) == 0) 1121 return true; 1122 1123 sc->sc_wasonline = !(ci->ci_schedstate.spc_flags & SPCF_OFFLINE); 1124 1125 if (sc->sc_wasonline) { 1126 mutex_enter(&cpu_lock); 1127 err = cpu_setstate(ci, false); 1128 mutex_exit(&cpu_lock); 1129 1130 if (err) 1131 return false; 1132 } 1133 1134 return true; 1135 } 1136 1137 static bool 1138 cpu_resume(device_t dv, const pmf_qual_t *qual) 1139 { 1140 struct cpu_softc *sc = device_private(dv); 1141 struct cpu_info *ci = sc->sc_info; 1142 int err = 0; 1143 1144 if (ci->ci_flags & CPUF_PRIMARY) 1145 return true; 1146 if (ci->ci_data.cpu_idlelwp == NULL) 1147 return true; 1148 if ((ci->ci_flags & CPUF_PRESENT) == 0) 1149 return true; 1150 1151 if (sc->sc_wasonline) { 1152 mutex_enter(&cpu_lock); 1153 err = cpu_setstate(ci, true); 1154 mutex_exit(&cpu_lock); 1155 } 1156 1157 return err == 0; 1158 } 1159 #endif 1160 1161 void 1162 cpu_get_tsc_freq(struct cpu_info *ci) 1163 { 1164 uint32_t vcpu_tversion; 1165 const volatile vcpu_time_info_t *tinfo = &ci->ci_vcpu->time; 1166 1167 vcpu_tversion = tinfo->version; 1168 while (tinfo->version == vcpu_tversion); /* Wait for a time update. XXX: timeout ? */ 1169 1170 uint64_t freq = 1000000000ULL << 32; 1171 freq = freq / (uint64_t)tinfo->tsc_to_system_mul; 1172 if ( tinfo->tsc_shift < 0 ) 1173 freq = freq << -tinfo->tsc_shift; 1174 else 1175 freq = freq >> tinfo->tsc_shift; 1176 ci->ci_data.cpu_cc_freq = freq; 1177 } 1178 1179 void 1180 x86_cpu_idle_xen(void) 1181 { 1182 struct cpu_info *ci = curcpu(); 1183 1184 KASSERT(ci->ci_ilevel == IPL_NONE); 1185 1186 x86_disable_intr(); 1187 if (!__predict_false(ci->ci_want_resched)) { 1188 idle_block(); 1189 } else { 1190 x86_enable_intr(); 1191 } 1192 } 1193 1194 /* 1195 * Loads pmap for the current CPU. 1196 */ 1197 void 1198 cpu_load_pmap(struct pmap *pmap) 1199 { 1200 #ifdef i386 1201 #ifdef PAE 1202 int i, s; 1203 struct cpu_info *ci; 1204 1205 s = splvm(); /* just to be safe */ 1206 xpq_queue_lock(); 1207 ci = curcpu(); 1208 paddr_t l3_pd = xpmap_ptom_masked(ci->ci_pae_l3_pdirpa); 1209 /* don't update the kernel L3 slot */ 1210 for (i = 0 ; i < PDP_SIZE - 1; i++) { 1211 xpq_queue_pte_update(l3_pd + i * sizeof(pd_entry_t), 1212 xpmap_ptom(pmap->pm_pdirpa[i]) | PG_V); 1213 } 1214 xpq_queue_unlock(); 1215 splx(s); 1216 tlbflush(); 1217 #else /* PAE */ 1218 lcr3(pmap_pdirpa(pmap, 0)); 1219 #endif /* PAE */ 1220 #endif /* i386 */ 1221 1222 #ifdef __x86_64__ 1223 int i, s; 1224 pd_entry_t *new_pgd; 1225 struct cpu_info *ci; 1226 paddr_t l3_shadow_pa; 1227 1228 ci = curcpu(); 1229 l3_shadow_pa = xpmap_ptom_masked(ci->ci_kpm_pdirpa); 1230 1231 /* 1232 * Map user space address in kernel space and load 1233 * user cr3 1234 */ 1235 s = splvm(); 1236 new_pgd = pmap->pm_pdir; 1237 1238 xpq_queue_lock(); 1239 /* Copy source pmap L4 PDEs (in user addr. range) to shadow */ 1240 for (i = 0; i < PDIR_SLOT_PTE; i++) { 1241 xpq_queue_pte_update(l3_shadow_pa + i * sizeof(pd_entry_t), new_pgd[i]); 1242 } 1243 1244 /* Copy kernel mappings */ 1245 new_pgd = pmap_kernel()->pm_pdir; 1246 for (i = PDIR_SLOT_KERN; i < nkptp[PTP_LEVELS - 1]; i++) { 1247 xpq_queue_pte_update(l3_shadow_pa + i * sizeof(pd_entry_t), new_pgd[i]); 1248 } 1249 1250 xpq_queue_unlock(); 1251 tlbflush(); 1252 xpq_queue_lock(); 1253 if (__predict_true(pmap != pmap_kernel())) { 1254 xen_set_user_pgd(pmap_pdirpa(pmap, 0)); 1255 ci->ci_xen_current_user_pgd = pmap_pdirpa(pmap, 0); 1256 } 1257 else { 1258 xpq_queue_pt_switch(l3_shadow_pa); 1259 ci->ci_xen_current_user_pgd = 0; 1260 } 1261 xpq_queue_unlock(); 1262 splx(s); 1263 1264 #endif /* __x86_64__ */ 1265 } 1266
Indexes created Fri Oct 03 02:09:57 GMT 2025