1 /* $NetBSD: machdep.c,v 1.852 2026/07/10 15:11:25 riastradh Exp $ */ 2 3 /* 4 * Copyright (c) 1996, 1997, 1998, 2000, 2004, 2006, 2008, 2009, 2017 5 * The NetBSD Foundation, Inc. 6 * All rights reserved. 7 * 8 * This code is derived from software contributed to The NetBSD Foundation 9 * by Charles M. Hannum, by Jason R. Thorpe of the Numerical Aerospace 10 * Simulation Facility NASA Ames Research Center, by Julio M. Merino Vidal, 11 * by Andrew Doran, and by Maxime Villard. 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 1. Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 23 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 24 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 25 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 26 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 29 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 30 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 31 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 32 * POSSIBILITY OF SUCH DAMAGE. 33 */ 34 35 /* 36 * Copyright (c) 1982, 1987, 1990 The Regents of the University of California. 37 * All rights reserved. 38 * 39 * This code is derived from software contributed to Berkeley by 40 * William Jolitz. 41 * 42 * Redistribution and use in source and binary forms, with or without 43 * modification, are permitted provided that the following conditions 44 * are met: 45 * 1. Redistributions of source code must retain the above copyright 46 * notice, this list of conditions and the following disclaimer. 47 * 2. Redistributions in binary form must reproduce the above copyright 48 * notice, this list of conditions and the following disclaimer in the 49 * documentation and/or other materials provided with the distribution. 50 * 3. Neither the name of the University nor the names of its contributors 51 * may be used to endorse or promote products derived from this software 52 * without specific prior written permission. 53 * 54 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 55 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 56 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 57 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 58 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 59 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 60 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 61 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 62 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 63 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 64 * SUCH DAMAGE. 65 * 66 * @(#)machdep.c 7.4 (Berkeley) 6/3/91 67 */ 68 69 #include <sys/cdefs.h> 70 __KERNEL_RCSID(0, "$NetBSD: machdep.c,v 1.852 2026/07/10 15:11:25 riastradh Exp $"); 71 72 #include "opt_beep.h" 73 #include "opt_compat_freebsd.h" 74 #include "opt_compat_netbsd.h" 75 #include "opt_cpureset_delay.h" 76 #include "opt_ddb.h" 77 #include "opt_kgdb.h" 78 #include "opt_mtrr.h" 79 #include "opt_modular.h" 80 #include "opt_multiboot.h" 81 #include "opt_multiprocessor.h" 82 #include "opt_physmem.h" 83 #include "opt_realmem.h" 84 #include "opt_user_ldt.h" 85 #include "opt_xen.h" 86 #include "isa.h" 87 #include "pci.h" 88 89 #include <sys/param.h> 90 #include <sys/systm.h> 91 #include <sys/signal.h> 92 #include <sys/signalvar.h> 93 #include <sys/kernel.h> 94 #include <sys/cpu.h> 95 #include <sys/exec.h> 96 #include <sys/fcntl.h> 97 #include <sys/reboot.h> 98 #include <sys/conf.h> 99 #include <sys/kauth.h> 100 #include <sys/msgbuf.h> 101 #include <sys/mount.h> 102 #include <sys/syscallargs.h> 103 #include <sys/core.h> 104 #include <sys/kcore.h> 105 #include <sys/ucontext.h> 106 #include <sys/ras.h> 107 #include <sys/ksyms.h> 108 #include <sys/device.h> 109 #include <sys/timevar.h> 110 111 #ifdef KGDB 112 #include <sys/kgdb.h> 113 #endif 114 115 #include <dev/cons.h> 116 #include <dev/mm.h> 117 118 #include <uvm/uvm.h> 119 #include <uvm/uvm_page.h> 120 121 #include <sys/sysctl.h> 122 123 #include <x86/efi.h> 124 125 #include <machine/cpu.h> 126 #include <machine/cpu_rng.h> 127 #include <machine/cpufunc.h> 128 #include <machine/cpuvar.h> 129 #include <machine/gdt.h> 130 #include <machine/intr.h> 131 #include <machine/kcore.h> 132 #include <machine/pio.h> 133 #include <machine/psl.h> 134 #include <machine/reg.h> 135 #include <machine/specialreg.h> 136 #include <machine/bootinfo.h> 137 #include <machine/mtrr.h> 138 #include <machine/pmap_private.h> 139 #include <x86/x86/tsc.h> 140 141 #include <x86/bootspace.h> 142 #include <x86/fpu.h> 143 #include <x86/dbregs.h> 144 #include <x86/machdep.h> 145 146 #include <machine/multiboot.h> 147 148 #ifdef XEN 149 #include <xen/evtchn.h> 150 #include <xen/xen.h> 151 #include <xen/hypervisor.h> 152 #endif 153 154 #include <dev/isa/isareg.h> 155 #include <machine/isa_machdep.h> 156 #include <dev/ic/i8042reg.h> 157 158 #include <ddb/db_active.h> 159 160 #ifdef DDB 161 #include <machine/db_machdep.h> 162 #include <ddb/db_extern.h> 163 #endif 164 165 #include "acpica.h" 166 #include "bioscall.h" 167 168 #if NBIOSCALL > 0 169 #include <machine/bioscall.h> 170 #endif 171 172 #if NACPICA > 0 173 #include <dev/acpi/acpivar.h> 174 #define ACPI_MACHDEP_PRIVATE 175 #include <machine/acpi_machdep.h> 176 #else 177 #include <machine/i82489var.h> 178 #endif 179 180 #include "isa.h" 181 #include "isadma.h" 182 #include "ksyms.h" 183 184 #include "cardbus.h" 185 #if NCARDBUS > 0 186 /* For rbus_min_start hint. */ 187 #include <sys/bus.h> 188 #include <dev/cardbus/rbus.h> 189 #include <machine/rbus_machdep.h> 190 #endif 191 192 #include "mca.h" 193 #if NMCA > 0 194 #include <machine/mca_machdep.h> /* for mca_busprobe() */ 195 #endif 196 197 #ifdef MULTIPROCESSOR /* XXX */ 198 #include <machine/mpbiosvar.h> /* XXX */ 199 #endif /* XXX */ 200 201 /* the following is used externally (sysctl_hw) */ 202 char machine[] = "i386"; /* CPU "architecture" */ 203 char machine_arch[] = "i386"; /* machine == machine_arch */ 204 205 #ifdef CPURESET_DELAY 206 int cpureset_delay = CPURESET_DELAY; 207 #else 208 int cpureset_delay = 2000; /* default to 2s */ 209 #endif 210 211 #ifdef MTRR 212 const struct mtrr_funcs *mtrr_funcs; 213 #endif 214 215 int cpu_class; 216 int use_pae; 217 int i386_fpu_fdivbug; 218 219 int i386_use_fxsave; 220 int i386_has_sse; 221 int i386_has_sse2; 222 223 vaddr_t idt_vaddr; 224 paddr_t idt_paddr; 225 vaddr_t gdt_vaddr; 226 paddr_t gdt_paddr; 227 vaddr_t ldt_vaddr; 228 paddr_t ldt_paddr; 229 230 vaddr_t pentium_idt_vaddr; 231 232 struct vm_map *phys_map = NULL; 233 234 extern struct bootspace bootspace; 235 236 extern paddr_t lowmem_rsvd; 237 extern paddr_t avail_start, avail_end; 238 #ifdef XENPV 239 extern paddr_t pmap_pa_start, pmap_pa_end; 240 void hypervisor_callback(void); 241 void failsafe_callback(void); 242 #endif 243 244 /* 245 * Size of memory segments, before any memory is stolen. 246 */ 247 phys_ram_seg_t mem_clusters[VM_PHYSSEG_MAX]; 248 int mem_cluster_cnt = 0; 249 250 void init_bootspace(void); 251 void init386(paddr_t); 252 void initgdt(union descriptor *); 253 254 static void i386_proc0_pcb_ldt_init(void); 255 256 static int cpu_getmcontext_xsave(struct lwp *, mcontext_t *, unsigned *, 257 const struct xsave_header *, size_t, struct xsave_header *); 258 259 int *esym; 260 int *eblob; 261 extern int boothowto; 262 263 #ifndef XENPV 264 265 /* Base memory reported by BIOS. */ 266 #ifndef REALBASEMEM 267 int biosbasemem = 0; 268 #else 269 int biosbasemem = REALBASEMEM; 270 #endif 271 272 /* Extended memory reported by BIOS. */ 273 #ifndef REALEXTMEM 274 int biosextmem = 0; 275 #else 276 int biosextmem = REALEXTMEM; 277 #endif 278 279 /* Set if any boot-loader set biosbasemem/biosextmem. */ 280 int biosmem_implicit; 281 282 /* 283 * Representation of the bootinfo structure constructed by a NetBSD native 284 * boot loader. Only be used by native_loader(). 285 */ 286 struct bootinfo_source { 287 uint32_t bs_naddrs; 288 void *bs_addrs[1]; /* Actually longer. */ 289 }; 290 291 /* Only called by locore.S; no need to be in a header file. */ 292 void native_loader(int, int, struct bootinfo_source *, paddr_t, int, int); 293 294 /* 295 * Called as one of the very first things during system startup (just after 296 * the boot loader gave control to the kernel image), this routine is in 297 * charge of retrieving the parameters passed in by the boot loader and 298 * storing them in the appropriate kernel variables. 299 * 300 * WARNING: Because the kernel has not yet relocated itself to KERNBASE, 301 * special care has to be taken when accessing memory because absolute 302 * addresses (referring to kernel symbols) do not work. So: 303 * 304 * 1) Avoid jumps to absolute addresses (such as gotos and switches). 305 * 2) To access global variables use their physical address, which 306 * can be obtained using the RELOC macro. 307 */ 308 void 309 native_loader(int bl_boothowto, int bl_bootdev, 310 struct bootinfo_source *bl_bootinfo, paddr_t bl_esym, 311 int bl_biosextmem, int bl_biosbasemem) 312 { 313 #define RELOC(type, x) ((type)((vaddr_t)(x) - KERNBASE)) 314 315 *RELOC(int *, &boothowto) = bl_boothowto; 316 317 /* 318 * The boot loader provides a physical, non-relocated address 319 * for the symbols table's end. We need to convert it to a 320 * virtual address. 321 */ 322 if (bl_esym != 0) 323 *RELOC(int **, &esym) = (int *)((vaddr_t)bl_esym + KERNBASE); 324 else 325 *RELOC(int **, &esym) = 0; 326 327 /* 328 * Copy bootinfo entries (if any) from the boot loader's 329 * representation to the kernel's bootinfo space. 330 */ 331 if (bl_bootinfo != NULL) { 332 size_t i; 333 uint8_t *data; 334 struct bootinfo *bidest; 335 struct btinfo_modulelist *bi; 336 337 bidest = RELOC(struct bootinfo *, &bootinfo); 338 339 data = &bidest->bi_data[0]; 340 341 for (i = 0; i < bl_bootinfo->bs_naddrs; i++) { 342 struct btinfo_common *bc; 343 344 bc = bl_bootinfo->bs_addrs[i]; 345 346 if ((data + bc->len) > 347 (&bidest->bi_data[0] + BOOTINFO_MAXSIZE)) 348 break; 349 350 memcpy(data, bc, bc->len); 351 /* 352 * If any modules were loaded, record where they 353 * end. We'll need to skip over them. 354 */ 355 bi = (struct btinfo_modulelist *)data; 356 if (bi->common.type == BTINFO_MODULELIST) { 357 *RELOC(int **, &eblob) = 358 (int *)(bi->endpa + KERNBASE); 359 } 360 data += bc->len; 361 } 362 bidest->bi_nentries = i; 363 } 364 365 /* 366 * Configure biosbasemem and biosextmem only if they were not 367 * explicitly given during the kernel's build. 368 */ 369 if (*RELOC(int *, &biosbasemem) == 0) { 370 *RELOC(int *, &biosbasemem) = bl_biosbasemem; 371 *RELOC(int *, &biosmem_implicit) = 1; 372 } 373 if (*RELOC(int *, &biosextmem) == 0) { 374 *RELOC(int *, &biosextmem) = bl_biosextmem; 375 *RELOC(int *, &biosmem_implicit) = 1; 376 } 377 #undef RELOC 378 } 379 380 #endif /* XENPV */ 381 382 /* 383 * Machine-dependent startup code 384 */ 385 void 386 cpu_startup(void) 387 { 388 int x, y; 389 vaddr_t minaddr, maxaddr; 390 psize_t sz; 391 392 /* 393 * For console drivers that require uvm and pmap to be initialized, 394 * we'll give them one more chance here... 395 */ 396 consinit(); 397 398 /* 399 * Initialize error message buffer (et end of core). 400 */ 401 if (msgbuf_p_cnt == 0) 402 panic("msgbuf paddr map has not been set up"); 403 for (x = 0, sz = 0; x < msgbuf_p_cnt; sz += msgbuf_p_seg[x++].sz) 404 continue; 405 406 msgbuf_vaddr = uvm_km_alloc(kernel_map, sz, 0, UVM_KMF_VAONLY); 407 if (msgbuf_vaddr == 0) 408 panic("failed to valloc msgbuf_vaddr"); 409 410 for (y = 0, sz = 0; y < msgbuf_p_cnt; y++) { 411 for (x = 0; x < btoc(msgbuf_p_seg[y].sz); x++, sz += PAGE_SIZE) 412 pmap_kenter_pa((vaddr_t)msgbuf_vaddr + sz, 413 msgbuf_p_seg[y].paddr + x * PAGE_SIZE, 414 VM_PROT_READ|VM_PROT_WRITE, 0); 415 } 416 417 pmap_update(pmap_kernel()); 418 419 initmsgbuf((void *)msgbuf_vaddr, sz); 420 421 #ifdef MULTIBOOT 422 multiboot1_print_info(); 423 multiboot2_print_info(); 424 #endif 425 426 #if NCARDBUS > 0 427 /* Tell RBUS how much RAM we have, so it can use heuristics. */ 428 rbus_min_start_hint(ctob((psize_t)physmem)); 429 #endif 430 431 minaddr = 0; 432 433 /* 434 * Allocate a submap for physio 435 */ 436 phys_map = uvm_km_suballoc(kernel_map, &minaddr, &maxaddr, 437 VM_PHYS_SIZE, 0, false, NULL); 438 439 /* Say hello. */ 440 banner(); 441 442 /* Safe for i/o port / memory space allocation to use malloc now. */ 443 #if NISA > 0 || NPCI > 0 444 x86_bus_space_mallocok(); 445 #endif 446 447 gdt_init(); 448 i386_proc0_pcb_ldt_init(); 449 450 cpu_init_tss(&cpu_info_primary); 451 #ifndef XENPV 452 ltr(cpu_info_primary.ci_tss_sel); 453 #endif 454 455 x86_startup(); 456 } 457 458 /* 459 * Set up proc0's PCB and LDT. 460 */ 461 static void 462 i386_proc0_pcb_ldt_init(void) 463 { 464 struct lwp *l = &lwp0; 465 struct pcb *pcb = lwp_getpcb(l); 466 467 pcb->pcb_cr0 = rcr0() & ~CR0_TS; 468 pcb->pcb_esp0 = uvm_lwp_getuarea(l) + USPACE - 16; 469 pcb->pcb_iopl = IOPL_KPL; 470 l->l_md.md_regs = (struct trapframe *)pcb->pcb_esp0 - 1; 471 memcpy(&pcb->pcb_fsd, &gdtstore[GUDATA_SEL], sizeof(pcb->pcb_fsd)); 472 memcpy(&pcb->pcb_gsd, &gdtstore[GUDATA_SEL], sizeof(pcb->pcb_gsd)); 473 pcb->pcb_dbregs = NULL; 474 475 #ifndef XENPV 476 lldt(GSEL(GLDT_SEL, SEL_KPL)); 477 #else 478 HYPERVISOR_fpu_taskswitch(1); 479 HYPERVISOR_stack_switch(GSEL(GDATA_SEL, SEL_KPL), pcb->pcb_esp0); 480 #endif 481 } 482 483 #ifdef XENPV 484 /* used in assembly */ 485 void i386_switch_context(lwp_t *); 486 void i386_tls_switch(lwp_t *); 487 488 /* 489 * Switch context: 490 * - switch stack pointer for user->kernel transition 491 */ 492 void 493 i386_switch_context(lwp_t *l) 494 { 495 struct pcb *pcb; 496 497 pcb = lwp_getpcb(l); 498 499 HYPERVISOR_stack_switch(GSEL(GDATA_SEL, SEL_KPL), pcb->pcb_esp0); 500 501 struct physdev_set_iopl set_iopl; 502 set_iopl.iopl = pcb->pcb_iopl; 503 HYPERVISOR_physdev_op(PHYSDEVOP_set_iopl, &set_iopl); 504 } 505 506 void 507 i386_tls_switch(lwp_t *l) 508 { 509 struct cpu_info *ci = curcpu(); 510 struct pcb *pcb = lwp_getpcb(l); 511 512 /* 513 * Raise the IPL to IPL_HIGH. XXX Still needed? 514 */ 515 (void)splhigh(); 516 517 /* Update TLS segment pointers */ 518 update_descriptor(&ci->ci_gdt[GUFS_SEL], 519 (union descriptor *)&pcb->pcb_fsd); 520 update_descriptor(&ci->ci_gdt[GUGS_SEL], 521 (union descriptor *)&pcb->pcb_gsd); 522 } 523 #endif /* XENPV */ 524 525 /* XXX */ 526 #define IDTVEC(name) __CONCAT(X, name) 527 typedef void (vector)(void); 528 529 #ifndef XENPV 530 static void tss_init(struct i386tss *, void *, void *); 531 532 static void 533 tss_init(struct i386tss *tss, void *stack, void *func) 534 { 535 KASSERT(curcpu()->ci_pmap == pmap_kernel()); 536 537 memset(tss, 0, sizeof *tss); 538 tss->tss_esp0 = tss->tss_esp = (int)((char *)stack + USPACE - 16); 539 tss->tss_ss0 = GSEL(GDATA_SEL, SEL_KPL); 540 tss->__tss_cs = GSEL(GCODE_SEL, SEL_KPL); 541 tss->tss_fs = GSEL(GCPU_SEL, SEL_KPL); 542 tss->tss_gs = tss->__tss_es = tss->__tss_ds = 543 tss->__tss_ss = GSEL(GDATA_SEL, SEL_KPL); 544 /* %cr3 contains the value associated to pmap_kernel */ 545 tss->tss_cr3 = rcr3(); 546 tss->tss_esp = (int)((char *)stack + USPACE - 16); 547 tss->tss_ldt = GSEL(GLDT_SEL, SEL_KPL); 548 tss->__tss_eflags = PSL_MBO | PSL_NT; /* XXX not needed? */ 549 tss->__tss_eip = (int)func; 550 } 551 552 extern vector IDTVEC(tss_trap08); 553 #if defined(DDB) && defined(MULTIPROCESSOR) 554 extern vector Xintr_ddbipi, Xintr_x2apic_ddbipi; 555 extern int ddb_vec; 556 #endif 557 558 void 559 cpu_set_tss_gates(struct cpu_info *ci) 560 { 561 struct segment_descriptor sd; 562 void *doubleflt_stack; 563 idt_descriptor_t *idt; 564 565 doubleflt_stack = (void *)uvm_km_alloc(kernel_map, USPACE, 0, 566 UVM_KMF_WIRED); 567 tss_init(&ci->ci_tss->dblflt_tss, doubleflt_stack, IDTVEC(tss_trap08)); 568 569 setsegment(&sd, &ci->ci_tss->dblflt_tss, sizeof(struct i386tss) - 1, 570 SDT_SYS386TSS, SEL_KPL, 0, 0); 571 ci->ci_gdt[GTRAPTSS_SEL].sd = sd; 572 573 idt = cpu_info_primary.ci_idtvec.iv_idt; 574 set_idtgate(&idt[8], NULL, 0, SDT_SYSTASKGT, SEL_KPL, 575 GSEL(GTRAPTSS_SEL, SEL_KPL)); 576 577 #if defined(DDB) && defined(MULTIPROCESSOR) 578 /* 579 * Set up separate handler for the DDB IPI, so that it doesn't 580 * stomp on a possibly corrupted stack. 581 * 582 * XXX overwriting the gate set in db_machine_init. 583 * Should rearrange the code so that it's set only once. 584 */ 585 void *ddbipi_stack; 586 587 ddbipi_stack = (void *)uvm_km_alloc(kernel_map, USPACE, 0, 588 UVM_KMF_WIRED); 589 tss_init(&ci->ci_tss->ddbipi_tss, ddbipi_stack, 590 x2apic_mode ? Xintr_x2apic_ddbipi : Xintr_ddbipi); 591 592 setsegment(&sd, &ci->ci_tss->ddbipi_tss, sizeof(struct i386tss) - 1, 593 SDT_SYS386TSS, SEL_KPL, 0, 0); 594 ci->ci_gdt[GIPITSS_SEL].sd = sd; 595 596 set_idtgate(&idt[ddb_vec], NULL, 0, SDT_SYSTASKGT, SEL_KPL, 597 GSEL(GIPITSS_SEL, SEL_KPL)); 598 #endif 599 } 600 #endif /* XENPV */ 601 602 /* 603 * Set up TSS and I/O bitmap. 604 */ 605 void 606 cpu_init_tss(struct cpu_info *ci) 607 { 608 struct cpu_tss *cputss; 609 610 cputss = (struct cpu_tss *)uvm_km_alloc(kernel_map, 611 sizeof(struct cpu_tss), 0, UVM_KMF_WIRED|UVM_KMF_ZERO); 612 613 cputss->tss.tss_iobase = IOMAP_INVALOFF << 16; 614 #ifndef XENPV 615 cputss->tss.tss_ss0 = GSEL(GDATA_SEL, SEL_KPL); 616 cputss->tss.tss_ldt = GSEL(GLDT_SEL, SEL_KPL); 617 cputss->tss.tss_cr3 = rcr3(); 618 #endif 619 620 ci->ci_tss = cputss; 621 #ifndef XENPV 622 ci->ci_tss_sel = tss_alloc(&cputss->tss); 623 #endif 624 } 625 626 void * 627 getframe(struct lwp *l, int sig, int *onstack) 628 { 629 struct proc *p = l->l_proc; 630 struct trapframe *tf = l->l_md.md_regs; 631 632 /* Do we need to jump onto the signal stack? */ 633 *onstack = (l->l_sigstk.ss_flags & (SS_DISABLE | SS_ONSTACK)) == 0 634 && (SIGACTION(p, sig).sa_flags & SA_ONSTACK) != 0; 635 if (*onstack) 636 return (char *)l->l_sigstk.ss_sp + l->l_sigstk.ss_size; 637 return (void *)tf->tf_esp; 638 } 639 640 /* 641 * Build context to run handler in. We invoke the handler 642 * directly, only returning via the trampoline. Note the 643 * trampoline version numbers are coordinated with machine- 644 * dependent code in libc. 645 */ 646 void 647 buildcontext(struct lwp *l, int sel, void *catcher, void *fp) 648 { 649 struct trapframe *tf = l->l_md.md_regs; 650 651 tf->tf_gs = GSEL(GUGS_SEL, SEL_UPL); 652 tf->tf_fs = GSEL(GUFS_SEL, SEL_UPL); 653 tf->tf_es = GSEL(GUDATA_SEL, SEL_UPL); 654 tf->tf_ds = GSEL(GUDATA_SEL, SEL_UPL); 655 tf->tf_eip = (int)catcher; 656 tf->tf_cs = GSEL(sel, SEL_UPL); 657 tf->tf_eflags &= ~PSL_CLEARSIG; 658 tf->tf_esp = (int)fp; 659 tf->tf_ss = GSEL(GUDATA_SEL, SEL_UPL); 660 661 /* Ensure FP state is reset. */ 662 fpu_sigreset(l); 663 } 664 665 void 666 sendsig_siginfo(const ksiginfo_t *ksi, const sigset_t *mask) 667 { 668 struct lwp *l = curlwp; 669 struct proc *p = l->l_proc; 670 struct pmap *pmap = vm_map_pmap(&p->p_vmspace->vm_map); 671 int sel = pmap->pm_hiexec > I386_MAX_EXE_ADDR ? 672 GUCODEBIG_SEL : GUCODE_SEL; 673 struct sigacts *ps = p->p_sigacts; 674 int onstack, error; 675 int sig = ksi->ksi_signo; 676 struct sigframe_siginfo *fp, frame; 677 sig_t catcher = SIGACTION(p, sig).sa_handler; 678 struct trapframe *tf = l->l_md.md_regs; 679 const struct xsave_header *xsavebuf = NULL; 680 size_t xsavelen = 0; 681 struct xsave_header *user_xsave = NULL; 682 char *sp; 683 684 KASSERT(mutex_owned(p->p_lock)); 685 686 /* Do we need to jump onto the signal stack? */ 687 onstack = 688 (l->l_sigstk.ss_flags & (SS_DISABLE | SS_ONSTACK)) == 0 && 689 (SIGACTION(p, sig).sa_flags & SA_ONSTACK) != 0; 690 691 /* Allocate space for the signal handler context. */ 692 if (onstack) { 693 KASSERT(l->l_sigstk.ss_size >= MINSIGSTKSZ); 694 sp = ((char *)l->l_sigstk.ss_sp + l->l_sigstk.ss_size); 695 } else { 696 sp = (char *)tf->tf_esp; 697 } 698 699 /* 700 * The maximum amount of space we might use, including padding 701 * for alignment, had better fit in MINSIGSTKSZ. 702 * 703 * If this changes because you have increased XSAVE_MAX_BYTES, 704 * you need to work out the ABI change for MINSIGSTKSZ. 705 */ 706 __CTASSERT(STACK_ALIGNBYTES + sizeof(struct sigframe_siginfo) + 707 (XSAVE_ALIGN - 1) + XSAVE_MAX_BYTES <= MINSIGSTKSZ); 708 709 /* 710 * Find whether we need to allocate a separate XSAVE area, 711 * because the user program has used extended CPU state beyond 712 * the x87/SSE registers, or whether we can get by with just an 713 * FXSAVE area. 714 */ 715 if (process_xsave_needed_p(l)) { 716 process_read_xsave(l, &xsavebuf, &xsavelen); 717 KASSERT(xsavebuf != NULL); 718 KASSERT(xsavelen <= XSAVE_MAX_BYTES); 719 720 KASSERT(!onstack || sp >= (char *)l->l_sigstk.ss_sp); 721 KASSERT(!onstack || 722 sp - (char *)l->l_sigstk.ss_sp >= xsavelen); 723 sp -= xsavelen; 724 725 KASSERT(!onstack || sp >= (char *)l->l_sigstk.ss_sp); 726 KASSERT(!onstack || 727 sp - (char *)l->l_sigstk.ss_sp >= XSAVE_ALIGN - 1); 728 sp = (char *)((uintptr_t)sp & ~(XSAVE_ALIGN - 1)); 729 730 KASSERT(!onstack || sp >= (char *)l->l_sigstk.ss_sp); 731 KASSERT(((uintptr_t)sp & (XSAVE_ALIGN - 1)) == 0); 732 user_xsave = (void *)sp; 733 } 734 735 /* 736 * Reserve space for an aligned struct sigframe_siginfo. 737 */ 738 KASSERT(!onstack || sp >= (char *)l->l_sigstk.ss_sp); 739 KASSERT(!onstack || (size_t)(sp - (char *)l->l_sigstk.ss_sp) >= 740 STACK_ALIGNBYTES + sizeof(struct sigframe_siginfo)); 741 fp = (struct sigframe_siginfo *)sp; 742 fp--; 743 fp = (struct sigframe_siginfo *)((uintptr_t)fp & ~STACK_ALIGNBYTES); 744 745 memset(&frame, 0, sizeof(frame)); 746 frame.sf_ra = (int)ps->sa_sigdesc[sig].sd_tramp; 747 frame.sf_signum = sig; 748 frame.sf_sip = &fp->sf_si; 749 frame.sf_ucp = &fp->sf_uc; 750 frame.sf_si._info = ksi->ksi_info; 751 frame.sf_uc.uc_flags = _UC_SIGMASK|_UC_VM; 752 frame.sf_uc.uc_sigmask = *mask; 753 frame.sf_uc.uc_link = l->l_ctxlink; 754 frame.sf_uc.uc_flags |= (l->l_sigstk.ss_flags & SS_ONSTACK) 755 ? _UC_SETSTACK : _UC_CLRSTACK; 756 757 sendsig_reset(l, sig); 758 759 mutex_exit(p->p_lock); 760 cpu_getmcontext(l, &frame.sf_uc.uc_mcontext, &frame.sf_uc.uc_flags); 761 762 /* 763 * If we have to use XSAVE, copy out that area separately -- 764 * and be ready to bail if it failed. 765 */ 766 if (xsavebuf) { 767 error = cpu_getmcontext_xsave(l, &frame.sf_uc.uc_mcontext, 768 &frame.sf_uc.uc_flags, xsavebuf, xsavelen, user_xsave); 769 if (error != 0) 770 goto relock; 771 } 772 773 error = copyout(&frame, fp, sizeof(frame)); 774 relock: 775 mutex_enter(p->p_lock); 776 777 if (error != 0) { 778 /* 779 * Process has trashed its stack; give it an illegal 780 * instruction to halt it in its tracks. 781 */ 782 sigexit(l, SIGILL); 783 /* NOTREACHED */ 784 } 785 786 buildcontext(l, sel, catcher, fp); 787 788 /* Remember that we're now on the signal stack. */ 789 if (onstack) 790 l->l_sigstk.ss_flags |= SS_ONSTACK; 791 } 792 793 static void 794 maybe_dump(int howto) 795 { 796 int s; 797 798 /* Disable interrupts. */ 799 s = splhigh(); 800 801 /* Do a dump if requested. */ 802 if ((howto & (RB_DUMP | RB_HALT)) == RB_DUMP) 803 dumpsys(); 804 805 splx(s); 806 } 807 808 void 809 cpu_reboot(int howto, char *bootstr) 810 { 811 static bool syncdone = false; 812 int s = IPL_NONE; 813 814 if (cold) { 815 howto |= RB_HALT; 816 goto haltsys; 817 } 818 819 boothowto = howto; 820 821 /* XXX used to dump after vfs_shutdown() and before 822 * detaching devices / shutdown hooks / pmf_system_shutdown(). 823 */ 824 maybe_dump(howto); 825 826 /* 827 * If we've panic'd, don't make the situation potentially 828 * worse by syncing or unmounting the file systems. 829 */ 830 if ((howto & RB_NOSYNC) == 0 && panicstr == NULL) { 831 if (!syncdone) { 832 syncdone = true; 833 /* XXX used to force unmount as well, here */ 834 vfs_sync_all(curlwp); 835 } 836 837 while (vfs_unmountall1(curlwp, false, false) || 838 config_detach_all(boothowto) || 839 vfs_unmount_forceone(curlwp)) 840 ; /* do nothing */ 841 } else { 842 if (!db_active) 843 suspendsched(); 844 } 845 846 pmf_system_shutdown(boothowto); 847 848 s = splhigh(); 849 850 /* amd64 maybe_dump() */ 851 852 haltsys: 853 doshutdownhooks(); 854 855 if ((howto & RB_POWERDOWN) == RB_POWERDOWN) { 856 #if NACPICA > 0 857 if (s != IPL_NONE) 858 splx(s); 859 860 acpi_enter_sleep_state(ACPI_STATE_S5); 861 #else 862 __USE(s); 863 #endif 864 #ifdef XEN 865 if (vm_guest == VM_GUEST_XENPV || 866 vm_guest == VM_GUEST_XENPVH || 867 vm_guest == VM_GUEST_XENPVHVM) 868 HYPERVISOR_shutdown(); 869 #endif /* XEN */ 870 } 871 872 #ifdef MULTIPROCESSOR 873 cpu_broadcast_halt(); 874 #endif /* MULTIPROCESSOR */ 875 876 if (howto & RB_HALT) { 877 #if NACPICA > 0 878 acpi_disable(); 879 #endif 880 881 printf("\n"); 882 printf("The operating system has halted.\n"); 883 printf("Please press any key to reboot.\n\n"); 884 885 #ifdef BEEP_ONHALT 886 { 887 int c; 888 for (c = BEEP_ONHALT_COUNT; c > 0; c--) { 889 sysbeep(BEEP_ONHALT_PITCH, 890 BEEP_ONHALT_PERIOD * hz / 1000); 891 delay(BEEP_ONHALT_PERIOD * 1000); 892 sysbeep(0, BEEP_ONHALT_PERIOD * hz / 1000); 893 delay(BEEP_ONHALT_PERIOD * 1000); 894 } 895 } 896 #endif 897 898 cnpollc(true); /* for proper keyboard command handling */ 899 if (cngetc() == 0) { 900 /* no console attached, so just hlt */ 901 printf("No keyboard - cannot reboot after all.\n"); 902 for(;;) { 903 x86_hlt(); 904 } 905 } 906 cnpollc(false); 907 } 908 909 printf("rebooting...\n"); 910 if (cpureset_delay > 0) 911 delay(cpureset_delay * 1000); 912 cpu_reset(); 913 for(;;) ; 914 /*NOTREACHED*/ 915 } 916 917 /* 918 * Clear registers on exec 919 */ 920 void 921 setregs(struct lwp *l, struct exec_package *pack, vaddr_t stack) 922 { 923 struct pmap *pmap = vm_map_pmap(&l->l_proc->p_vmspace->vm_map); 924 struct pcb *pcb = lwp_getpcb(l); 925 struct trapframe *tf; 926 927 #ifdef USER_LDT 928 pmap_ldt_cleanup(l); 929 #endif 930 931 fpu_clear(l, pack->ep_osversion >= 699002600 932 ? __INITIAL_NPXCW__ : __NetBSD_COMPAT_NPXCW__); 933 934 memcpy(&pcb->pcb_fsd, &gdtstore[GUDATA_SEL], sizeof(pcb->pcb_fsd)); 935 memcpy(&pcb->pcb_gsd, &gdtstore[GUDATA_SEL], sizeof(pcb->pcb_gsd)); 936 937 x86_dbregs_clear(l); 938 939 tf = l->l_md.md_regs; 940 memset(tf, 0, sizeof(*tf)); 941 942 tf->tf_trapno = T_ASTFLT; 943 tf->tf_gs = GSEL(GUGS_SEL, SEL_UPL); 944 tf->tf_fs = GSEL(GUFS_SEL, SEL_UPL); 945 tf->tf_es = LSEL(LUDATA_SEL, SEL_UPL); 946 tf->tf_ds = LSEL(LUDATA_SEL, SEL_UPL); 947 tf->tf_edi = 0; 948 tf->tf_esi = 0; 949 tf->tf_ebp = 0; 950 tf->tf_ebx = l->l_proc->p_psstrp; 951 tf->tf_edx = 0; 952 tf->tf_ecx = 0; 953 tf->tf_eax = 0; 954 tf->tf_eip = pack->ep_entry; 955 tf->tf_cs = pmap->pm_hiexec > I386_MAX_EXE_ADDR ? 956 LSEL(LUCODEBIG_SEL, SEL_UPL) : LSEL(LUCODE_SEL, SEL_UPL); 957 tf->tf_eflags = PSL_USERSET; 958 tf->tf_esp = stack; 959 tf->tf_ss = LSEL(LUDATA_SEL, SEL_UPL); 960 } 961 962 /* 963 * Initialize segments and descriptor tables 964 */ 965 966 union descriptor *gdtstore, *ldtstore; 967 union descriptor *pentium_idt; 968 extern vaddr_t lwp0uarea; 969 970 void 971 setgate(struct gate_descriptor *gd, void *func, int args, int type, int dpl, 972 int sel) 973 { 974 975 gd->gd_looffset = (int)func; 976 gd->gd_selector = sel; 977 gd->gd_stkcpy = args; 978 gd->gd_xx = 0; 979 gd->gd_type = type; 980 gd->gd_dpl = dpl; 981 gd->gd_p = 1; 982 gd->gd_hioffset = (int)func >> 16; 983 } 984 985 void 986 unsetgate(struct gate_descriptor *gd) 987 { 988 989 gd->gd_p = 0; 990 gd->gd_hioffset = 0; 991 gd->gd_looffset = 0; 992 gd->gd_selector = 0; 993 gd->gd_xx = 0; 994 gd->gd_stkcpy = 0; 995 gd->gd_type = 0; 996 gd->gd_dpl = 0; 997 } 998 999 void 1000 setregion(struct region_descriptor *rd, void *base, size_t limit) 1001 { 1002 1003 rd->rd_limit = (int)limit; 1004 rd->rd_base = (int)base; 1005 } 1006 1007 void 1008 setsegment(struct segment_descriptor *sd, const void *base, size_t limit, 1009 int type, int dpl, int def32, int gran) 1010 { 1011 1012 sd->sd_lolimit = (int)limit; 1013 sd->sd_lobase = (int)base; 1014 sd->sd_type = type; 1015 sd->sd_dpl = dpl; 1016 sd->sd_p = 1; 1017 sd->sd_hilimit = (int)limit >> 16; 1018 sd->sd_xx = 0; 1019 sd->sd_def32 = def32; 1020 sd->sd_gran = gran; 1021 sd->sd_hibase = (int)base >> 24; 1022 } 1023 1024 /* XXX */ 1025 extern vector IDTVEC(syscall); 1026 extern vector *IDTVEC(exceptions)[]; 1027 #ifdef XENPV 1028 extern union descriptor tmpgdt[]; 1029 #endif 1030 1031 void 1032 cpu_init_idt(struct cpu_info *ci) 1033 { 1034 struct region_descriptor region; 1035 struct idt_vec *iv; 1036 idt_descriptor_t *idt; 1037 1038 iv = &ci->ci_idtvec; 1039 idt = iv->iv_idt_pentium; 1040 setregion(®ion, idt, NIDT * sizeof(idt[0]) - 1); 1041 lidt(®ion); 1042 } 1043 1044 /* 1045 * initgdt(tgdt) 1046 * 1047 * Initialize a temporary Global Descriptor Table (GDT) using 1048 * storage space at tgdt. 1049 * 1050 * 1. Set up segment descriptors for our purposes, including a 1051 * CPU-local segment descriptor pointing at &cpu_info_primary. 1052 * 1053 * 2. Load the address into the Global Descriptor Table Register. 1054 * 1055 * 3. Set up segment selectors for all the segment registers using 1056 * it so that %fs-relative addressing works for the CPU-local 1057 * data. 1058 * 1059 * After this put, CPUVAR(...), curcpu(), and curlwp will work. 1060 * 1061 * Eventually the kernel will switch to a second temporary GDT 1062 * allocated with pmap_bootstrap_valloc in pmap_bootstrap, and 1063 * then to permanent GDT allocated with uvm_km(9) in gdt_init. 1064 * But the first temporary GDT is needed now to get us going with 1065 * early access to curcpu() and curlwp before we enter kernel 1066 * main. 1067 * 1068 * XXX The purpose of each of the segment descriptors should be 1069 * written down somewhere in a single place that can be cross- 1070 * referenced. 1071 * 1072 * References: 1073 * 1074 * - Intel 64 and IA-32 Architectures Software Developer's Manual, 1075 * Volume 3: System Programming Guide, Order Number 325384, 1076 * April 2022, Sec. 3.5.1 `Segment Descriptor Tables', 1077 * pp. 3-14 through 3-16. 1078 */ 1079 void 1080 initgdt(union descriptor *tgdt) 1081 { 1082 KASSERT(tgdt != NULL); 1083 1084 gdtstore = tgdt; 1085 #ifdef XENPV 1086 u_long frames[16]; 1087 #else 1088 struct region_descriptor region; 1089 memset(gdtstore, 0, NGDT * sizeof(*gdtstore)); 1090 #endif 1091 1092 /* make gdt gates and memory segments */ 1093 setsegment(&gdtstore[GCODE_SEL].sd, 0, 0xfffff, 1094 SDT_MEMERA, SEL_KPL, 1, 1); 1095 setsegment(&gdtstore[GDATA_SEL].sd, 0, 0xfffff, 1096 SDT_MEMRWA, SEL_KPL, 1, 1); 1097 setsegment(&gdtstore[GUCODE_SEL].sd, 0, x86_btop(I386_MAX_EXE_ADDR) - 1, 1098 SDT_MEMERA, SEL_UPL, 1, 1); 1099 setsegment(&gdtstore[GUCODEBIG_SEL].sd, 0, 0xfffff, 1100 SDT_MEMERA, SEL_UPL, 1, 1); 1101 setsegment(&gdtstore[GUDATA_SEL].sd, 0, 0xfffff, 1102 SDT_MEMRWA, SEL_UPL, 1, 1); 1103 #if NBIOSCALL > 0 && !defined(XENPV) 1104 /* bios trampoline GDT entries */ 1105 setsegment(&gdtstore[GBIOSCODE_SEL].sd, 0, 0xfffff, 1106 SDT_MEMERA, SEL_KPL, 0, 0); 1107 setsegment(&gdtstore[GBIOSDATA_SEL].sd, 0, 0xfffff, 1108 SDT_MEMRWA, SEL_KPL, 0, 0); 1109 #endif 1110 setsegment(&gdtstore[GCPU_SEL].sd, &cpu_info_primary, 1111 sizeof(struct cpu_info) - 1, SDT_MEMRWA, SEL_KPL, 1, 0); 1112 1113 #ifndef XENPV 1114 setregion(®ion, gdtstore, NGDT * sizeof(gdtstore[0]) - 1); 1115 lgdt(®ion); 1116 #else /* !XENPV */ 1117 /* 1118 * We jumpstart the bootstrap process a bit so we can update 1119 * page permissions. This is done redundantly later from 1120 * x86_xpmap.c:xen_locore() - harmless. 1121 */ 1122 xpmap_phys_to_machine_mapping = 1123 (unsigned long *)xen_start_info.mfn_list; 1124 1125 frames[0] = xpmap_ptom((uint32_t)gdtstore - KERNBASE) >> PAGE_SHIFT; 1126 { /* 1127 * Enter the gdt page RO into the kernel map. We can't 1128 * use pmap_kenter_pa() here, because %fs is not 1129 * usable until the gdt is loaded, and %fs is used as 1130 * the base pointer for curcpu() and curlwp(), both of 1131 * which are in the callpath of pmap_kenter_pa(). 1132 * So we mash up our own - this is MD code anyway. 1133 */ 1134 extern pt_entry_t xpmap_pg_nx; 1135 pt_entry_t pte; 1136 1137 pte = pmap_pa2pte((vaddr_t)gdtstore - KERNBASE); 1138 pte |= xpmap_pg_nx | PTE_P; 1139 1140 if (HYPERVISOR_update_va_mapping((vaddr_t)gdtstore, pte, 1141 UVMF_INVLPG) < 0) { 1142 panic("gdt page RO update failed.\n"); 1143 } 1144 } 1145 1146 if (HYPERVISOR_set_gdt(frames, NGDT /* XXX is it right ? */)) 1147 panic("HYPERVISOR_set_gdt failed!\n"); 1148 1149 lgdt_finish(); 1150 #endif /* !XENPV */ 1151 } 1152 1153 #if !defined(XENPV) && NBIOSCALL > 0 1154 static void 1155 init386_pte0(void) 1156 { 1157 paddr_t paddr; 1158 vaddr_t vaddr; 1159 1160 paddr = 4 * PAGE_SIZE; 1161 vaddr = (vaddr_t)vtopte(0); 1162 pmap_kenter_pa(vaddr, paddr, VM_PROT_ALL, 0); 1163 pmap_update(pmap_kernel()); 1164 /* make sure it is clean before using */ 1165 memset((void *)vaddr, 0, PAGE_SIZE); 1166 } 1167 #endif /* !XENPV && NBIOSCALL > 0 */ 1168 1169 #ifndef XENPV 1170 static void 1171 init386_ksyms(void) 1172 { 1173 #if NKSYMS || defined(DDB) || defined(MODULAR) 1174 extern int end; 1175 struct btinfo_symtab *symtab; 1176 1177 #ifdef DDB 1178 db_machine_init(); 1179 #endif 1180 1181 #if defined(MULTIBOOT) 1182 if (multiboot1_ksyms_addsyms_elf()) 1183 return; 1184 1185 if (multiboot2_ksyms_addsyms_elf()) 1186 return; 1187 #endif 1188 1189 #ifdef XEN 1190 if (pvh_boot && vm_guest != VM_GUEST_XENPVH) { 1191 ksyms_addsyms_elf(0, ((int *)&end) + 1, esym); 1192 return; 1193 } 1194 #endif 1195 1196 if ((symtab = lookup_bootinfo(BTINFO_SYMTAB)) == NULL) { 1197 ksyms_addsyms_elf(*(int *)&end, ((int *)&end) + 1, esym); 1198 return; 1199 } 1200 1201 symtab->ssym += KERNBASE; 1202 symtab->esym += KERNBASE; 1203 ksyms_addsyms_elf(symtab->nsym, (int *)symtab->ssym, (int *)symtab->esym); 1204 #endif 1205 } 1206 #endif /* XENPV */ 1207 1208 void 1209 init_bootspace(void) 1210 { 1211 extern char __rodata_start; 1212 extern char __data_start; 1213 extern char __kernel_end; 1214 size_t i = 0; 1215 1216 memset(&bootspace, 0, sizeof(bootspace)); 1217 1218 bootspace.head.va = KERNTEXTOFF; 1219 bootspace.head.pa = KERNTEXTOFF - KERNBASE; 1220 bootspace.head.sz = 0; 1221 1222 bootspace.segs[i].type = BTSEG_TEXT; 1223 bootspace.segs[i].va = KERNTEXTOFF; 1224 bootspace.segs[i].pa = KERNTEXTOFF - KERNBASE; 1225 bootspace.segs[i].sz = (size_t)&__rodata_start - KERNTEXTOFF; 1226 i++; 1227 1228 bootspace.segs[i].type = BTSEG_RODATA; 1229 bootspace.segs[i].va = (vaddr_t)&__rodata_start; 1230 bootspace.segs[i].pa = (paddr_t)(vaddr_t)&__rodata_start - KERNBASE; 1231 bootspace.segs[i].sz = (size_t)&__data_start - (size_t)&__rodata_start; 1232 i++; 1233 1234 bootspace.segs[i].type = BTSEG_DATA; 1235 bootspace.segs[i].va = (vaddr_t)&__data_start; 1236 bootspace.segs[i].pa = (paddr_t)(vaddr_t)&__data_start - KERNBASE; 1237 bootspace.segs[i].sz = (size_t)&__kernel_end - (size_t)&__data_start; 1238 i++; 1239 1240 bootspace.boot.va = (vaddr_t)&__kernel_end; 1241 bootspace.boot.pa = (paddr_t)(vaddr_t)&__kernel_end - KERNBASE; 1242 bootspace.boot.sz = (size_t)(atdevbase + IOM_SIZE) - 1243 (size_t)&__kernel_end; 1244 1245 /* Virtual address of the top level page */ 1246 bootspace.pdir = (vaddr_t)(PDPpaddr + KERNBASE); 1247 } 1248 1249 void 1250 init386(paddr_t first_avail) 1251 { 1252 extern void consinit(void); 1253 int x; 1254 #ifndef XENPV 1255 extern paddr_t local_apic_pa; 1256 union descriptor *tgdt; 1257 struct region_descriptor region; 1258 #if NBIOSCALL > 0 1259 extern int biostramp_image_size; 1260 extern u_char biostramp_image[]; 1261 #endif 1262 #endif /* !XENPV */ 1263 struct pcb *pcb; 1264 struct idt_vec *iv; 1265 idt_descriptor_t *idt; 1266 1267 KASSERT(first_avail % PAGE_SIZE == 0); 1268 1269 #ifdef XENPV 1270 KASSERT(HYPERVISOR_shared_info != NULL); 1271 cpu_info_primary.ci_vcpu = &HYPERVISOR_shared_info->vcpu_info[0]; 1272 #endif 1273 1274 #ifdef XEN 1275 if (pvh_boot) 1276 xen_parse_cmdline(XEN_PARSE_BOOTFLAGS, NULL); 1277 #endif 1278 1279 uvm_lwp_setuarea(&lwp0, lwp0uarea); 1280 1281 cpu_probe(&cpu_info_primary); 1282 1283 /* 1284 * Initialize the no-execute bit on cpu0, if supported. 1285 * 1286 * Note: The call to cpu_init_msrs for secondary CPUs happens 1287 * in cpu_hatch. 1288 */ 1289 cpu_init_msrs(&cpu_info_primary, true); 1290 1291 #ifndef XENPV 1292 cpu_speculation_init(&cpu_info_primary); 1293 #endif 1294 1295 #ifdef PAE 1296 use_pae = 1; 1297 #else 1298 use_pae = 0; 1299 #endif 1300 1301 pcb = lwp_getpcb(&lwp0); 1302 #ifdef XENPV 1303 pcb->pcb_cr3 = PDPpaddr; 1304 #endif 1305 1306 #if defined(PAE) && !defined(XENPV) 1307 /* 1308 * Save VA and PA of L3 PD of boot processor (for Xen, this is done 1309 * in xen_locore()) 1310 */ 1311 cpu_info_primary.ci_pae_l3_pdirpa = rcr3(); 1312 cpu_info_primary.ci_pae_l3_pdir = (pd_entry_t *)(rcr3() + KERNBASE); 1313 #endif 1314 1315 uvm_md_init(); 1316 1317 /* 1318 * Start with 2 color bins -- this is just a guess to get us 1319 * started. We'll recolor when we determine the largest cache 1320 * sizes on the system. 1321 */ 1322 uvmexp.ncolors = 2; 1323 1324 avail_start = first_avail; 1325 1326 #ifndef XENPV 1327 /* 1328 * Low memory reservations: 1329 * Page 0: BIOS data 1330 * Page 1: BIOS callback 1331 * Page 2: MP bootstrap code (MP_TRAMPOLINE) 1332 * Page 3: ACPI wakeup code (ACPI_WAKEUP_ADDR) 1333 * Page 4: Temporary page table for 0MB-4MB 1334 * Page 5: Temporary page directory 1335 */ 1336 lowmem_rsvd = 6 * PAGE_SIZE; 1337 #else /* !XENPV */ 1338 /* Parse Xen command line (replace bootinfo) */ 1339 xen_parse_cmdline(XEN_PARSE_BOOTFLAGS, NULL); 1340 1341 /* Use the dummy page as a gdt */ 1342 extern vaddr_t xen_dummy_page; 1343 gdtstore = (void *)xen_dummy_page; 1344 1345 /* Determine physical address space */ 1346 avail_end = ctob((paddr_t)xen_start_info.nr_pages); 1347 pmap_pa_start = (KERNTEXTOFF - KERNBASE); 1348 pmap_pa_end = pmap_pa_start + ctob((paddr_t)xen_start_info.nr_pages); 1349 mem_clusters[0].start = avail_start; 1350 mem_clusters[0].size = avail_end - avail_start; 1351 mem_cluster_cnt++; 1352 physmem += xen_start_info.nr_pages; 1353 uvmexp.wired += atop(avail_start); 1354 1355 /* 1356 * initgdt() has to be done before consinit(), so that %fs is properly 1357 * initialised. initgdt() uses pmap_kenter_pa so it can't be called 1358 * before the above variables are set. 1359 */ 1360 initgdt(gdtstore); 1361 1362 mutex_init(&pte_lock, MUTEX_DEFAULT, IPL_VM); 1363 #endif /* XENPV */ 1364 1365 #if NISA > 0 || NPCI > 0 1366 x86_bus_space_init(); 1367 #endif 1368 1369 consinit(); /* XXX SHOULD NOT BE DONE HERE */ 1370 1371 #ifdef DEBUG_MEMLOAD 1372 printf("mem_cluster_count: %d\n", mem_cluster_cnt); 1373 #endif 1374 1375 /* 1376 * Call pmap initialization to make new kernel address space. 1377 * We must do this before loading pages into the VM system. 1378 */ 1379 pmap_bootstrap((vaddr_t)atdevbase + IOM_SIZE); 1380 1381 /* 1382 * Initialize RNG to get entropy ASAP either from CPU 1383 * RDRAND/RDSEED or from seed on disk. Constraints: 1384 * 1385 * - Must happen after cpu_init_msrs so that curcpu() and 1386 * curlwp work. 1387 * 1388 * - Must happen after consinit so we have the opportunity to 1389 * print useful feedback. 1390 * 1391 * - On KASLR kernels, must happen after pmap_bootstrap because 1392 * x86_rndseed requires access to the direct map. 1393 */ 1394 cpu_rng_init(); 1395 x86_rndseed(); 1396 1397 #ifndef XENPV 1398 /* Initialize the memory clusters. */ 1399 init_x86_clusters(); 1400 1401 /* Internalize the physical pages into the VM system. */ 1402 init_x86_vm(avail_start); 1403 #else /* !XENPV */ 1404 uvm_page_physload(atop(avail_start), atop(avail_end), 1405 atop(avail_start), atop(avail_end), 1406 VM_FREELIST_DEFAULT); 1407 1408 /* Reclaim the boot gdt page - see locore.s */ 1409 { 1410 extern pt_entry_t xpmap_pg_nx; 1411 pt_entry_t pte; 1412 1413 pte = pmap_pa2pte((vaddr_t)tmpgdt - KERNBASE); 1414 pte |= PTE_W | xpmap_pg_nx | PTE_P; 1415 1416 if (HYPERVISOR_update_va_mapping((vaddr_t)tmpgdt, pte, UVMF_INVLPG) < 0) { 1417 panic("tmpgdt page relaim RW update failed.\n"); 1418 } 1419 } 1420 #endif /* !XENPV */ 1421 1422 init_x86_msgbuf(); 1423 1424 #if !defined(XENPV) && NBIOSCALL > 0 1425 /* 1426 * XXX Remove this 1427 * 1428 * Setup a temporary Page Table Entry to allow identity mappings of 1429 * the real mode address. This is required by bioscall. 1430 */ 1431 init386_pte0(); 1432 1433 KASSERT(biostramp_image_size <= PAGE_SIZE); 1434 pmap_kenter_pa((vaddr_t)BIOSTRAMP_BASE, (paddr_t)BIOSTRAMP_BASE, 1435 VM_PROT_ALL, 0); 1436 pmap_update(pmap_kernel()); 1437 memcpy((void *)BIOSTRAMP_BASE, biostramp_image, biostramp_image_size); 1438 1439 /* Needed early, for bioscall() */ 1440 cpu_info_primary.ci_pmap = pmap_kernel(); 1441 #endif 1442 1443 #ifndef XENPV 1444 pmap_kenter_pa(local_apic_va, local_apic_pa, 1445 VM_PROT_READ|VM_PROT_WRITE, 0); 1446 pmap_update(pmap_kernel()); 1447 memset((void *)local_apic_va, 0, PAGE_SIZE); 1448 #endif 1449 1450 pmap_kenter_pa(idt_vaddr, idt_paddr, VM_PROT_READ|VM_PROT_WRITE, 0); 1451 pmap_kenter_pa(gdt_vaddr, gdt_paddr, VM_PROT_READ|VM_PROT_WRITE, 0); 1452 pmap_kenter_pa(ldt_vaddr, ldt_paddr, VM_PROT_READ|VM_PROT_WRITE, 0); 1453 pmap_update(pmap_kernel()); 1454 memset((void *)idt_vaddr, 0, PAGE_SIZE); 1455 memset((void *)gdt_vaddr, 0, PAGE_SIZE); 1456 memset((void *)ldt_vaddr, 0, PAGE_SIZE); 1457 1458 pmap_kenter_pa(pentium_idt_vaddr, idt_paddr, VM_PROT_READ, 0); 1459 pmap_update(pmap_kernel()); 1460 iv = &(cpu_info_primary.ci_idtvec); 1461 idt_vec_init_cpu_md(iv, cpu_index(&cpu_info_primary)); 1462 idt = (idt_descriptor_t *)iv->iv_idt; 1463 1464 #ifndef XENPV 1465 /* 1466 * Switch from the initial temporary GDT that was allocated on 1467 * the stack by our caller, start. That temporary GDT will be 1468 * popped off the stack when init386 returns before start calls 1469 * main, so we need to use a second temporary GDT allocated in 1470 * pmap_bootstrap with pmap_bootstrap_valloc/palloc to make 1471 * sure at least the CPU-local data area, used by CPUVAR(...), 1472 * curcpu(), and curlwp via %fs-relative addressing, will 1473 * continue to work. 1474 * 1475 * Later, in gdt_init via cpu_startup, we will finally allocate 1476 * a permanent GDT with uvm_km(9). 1477 * 1478 * The content of the second temporary GDT is the same as the 1479 * content of the initial GDT, initialized in initgdt, except 1480 * for the address of the LDT, which is also that we are also 1481 * switching to a new temporary LDT at a new address. 1482 */ 1483 tgdt = gdtstore; 1484 gdtstore = (union descriptor *)gdt_vaddr; 1485 ldtstore = (union descriptor *)ldt_vaddr; 1486 1487 memcpy(gdtstore, tgdt, NGDT * sizeof(*gdtstore)); 1488 1489 setsegment(&gdtstore[GLDT_SEL].sd, ldtstore, 1490 NLDT * sizeof(ldtstore[0]) - 1, SDT_SYSLDT, SEL_KPL, 0, 0); 1491 #else 1492 HYPERVISOR_set_callbacks( 1493 GSEL(GCODE_SEL, SEL_KPL), (unsigned long)hypervisor_callback, 1494 GSEL(GCODE_SEL, SEL_KPL), (unsigned long)failsafe_callback); 1495 1496 ldtstore = (union descriptor *)ldt_vaddr; 1497 #endif /* XENPV */ 1498 1499 /* make ldt gates and memory segments */ 1500 ldtstore[LUCODE_SEL] = gdtstore[GUCODE_SEL]; 1501 ldtstore[LUCODEBIG_SEL] = gdtstore[GUCODEBIG_SEL]; 1502 ldtstore[LUDATA_SEL] = gdtstore[GUDATA_SEL]; 1503 1504 /* exceptions */ 1505 for (x = 0; x < 32; x++) { 1506 /* Reset to default. Special cases below */ 1507 int sel; 1508 #ifdef XENPV 1509 sel = SEL_XEN; 1510 #else 1511 sel = SEL_KPL; 1512 #endif /* XENPV */ 1513 1514 idt_vec_reserve(iv, x); 1515 1516 switch (x) { 1517 #ifdef XENPV 1518 case 2: /* NMI */ 1519 case 18: /* MCA */ 1520 sel |= 0x4; /* Auto EOI/mask */ 1521 break; 1522 #endif /* XENPV */ 1523 case 3: 1524 case 4: 1525 sel = SEL_UPL; 1526 break; 1527 default: 1528 break; 1529 } 1530 set_idtgate(&idt[x], IDTVEC(exceptions)[x], 0, SDT_SYS386IGT, 1531 sel, GSEL(GCODE_SEL, SEL_KPL)); 1532 } 1533 1534 /* new-style interrupt gate for syscalls */ 1535 idt_vec_reserve(iv, 128); 1536 set_idtgate(&idt[128], &IDTVEC(syscall), 0, SDT_SYS386IGT, SEL_UPL, 1537 GSEL(GCODE_SEL, SEL_KPL)); 1538 1539 #ifndef XENPV 1540 /* 1541 * Activate the second temporary GDT, allocated in 1542 * pmap_bootstrap with pmap_bootstrap_valloc/palloc, and 1543 * initialized with the content of the initial temporary GDT in 1544 * initgdt, plus an updated LDT. 1545 * 1546 * This ensures the %fs-relative addressing for the CPU-local 1547 * area used by CPUVAR(...), curcpu(), and curlwp will continue 1548 * to work after init386 returns and the initial temporary GDT 1549 * is popped off, before we call main and later create a 1550 * permanent GDT in gdt_init via cpu_startup. 1551 */ 1552 setregion(®ion, gdtstore, NGDT * sizeof(gdtstore[0]) - 1); 1553 lgdt(®ion); 1554 #endif 1555 1556 lldt(GSEL(GLDT_SEL, SEL_KPL)); 1557 cpu_init_idt(&cpu_info_primary); 1558 1559 #ifdef XENPV 1560 xen_init_ksyms(); 1561 #else /* XENPV */ 1562 #ifdef XEN 1563 if (vm_guest == VM_GUEST_XENPVH) 1564 xen_init_ksyms(); 1565 else 1566 #endif /* XEN */ 1567 init386_ksyms(); 1568 #endif /* XENPV */ 1569 1570 #if NMCA > 0 1571 /* 1572 * check for MCA bus, needed to be done before ISA stuff - if 1573 * MCA is detected, ISA needs to use level triggered interrupts 1574 * by default 1575 * And we do not search for MCA using bioscall() on EFI systems 1576 * that lacks it (they lack MCA too, anyway). 1577 */ 1578 if (lookup_bootinfo(BTINFO_EFI) == NULL && vm_guest != VM_GUEST_XENPVH) 1579 mca_busprobe(); 1580 #endif 1581 1582 #ifdef XENPV 1583 extern int tmpstk; 1584 cpu_info_primary.ci_intrstack = &tmpstk; 1585 events_default_setup(); 1586 #else 1587 intr_default_setup(); 1588 #endif 1589 1590 splraise(IPL_HIGH); 1591 x86_enable_intr(); 1592 1593 #ifdef DDB 1594 if (boothowto & RB_KDB) 1595 Debugger(); 1596 #endif 1597 #ifdef KGDB 1598 kgdb_port_init(); 1599 if (boothowto & RB_KDB) { 1600 kgdb_debug_init = 1; 1601 kgdb_connect(1); 1602 } 1603 #endif 1604 1605 if (physmem < btoc(2 * 1024 * 1024)) { 1606 printf("warning: too little memory available; " 1607 "have %lu bytes, want %lu bytes\n" 1608 "running in degraded mode\n" 1609 "press a key to confirm\n\n", 1610 (unsigned long)ptoa(physmem), 2*1024*1024UL); 1611 cngetc(); 1612 } 1613 1614 pcb->pcb_dbregs = NULL; 1615 x86_dbregs_init(); 1616 } 1617 1618 #include <dev/ic/mc146818reg.h> /* for NVRAM POST */ 1619 #include <i386/isa/nvram.h> /* for NVRAM POST */ 1620 1621 void 1622 cpu_reset(void) 1623 { 1624 #ifdef XENPV 1625 HYPERVISOR_reboot(); 1626 for (;;); 1627 #else /* XENPV */ 1628 struct region_descriptor region; 1629 idt_descriptor_t *idt; 1630 1631 idt = (idt_descriptor_t *)cpu_info_primary.ci_idtvec.iv_idt; 1632 x86_disable_intr(); 1633 1634 /* 1635 * Ensure the NVRAM reset byte contains something vaguely sane. 1636 */ 1637 1638 outb(IO_RTC, NVRAM_RESET); 1639 outb(IO_RTC+1, NVRAM_RESET_RST); 1640 1641 /* 1642 * Reset AMD Geode SC1100. 1643 * 1644 * 1) Write PCI Configuration Address Register (0xcf8) to 1645 * select Function 0, Register 0x44: Bridge Configuration, 1646 * GPIO and LPC Configuration Register Space, Reset 1647 * Control Register. 1648 * 1649 * 2) Write 0xf to PCI Configuration Data Register (0xcfc) 1650 * to reset IDE controller, IDE bus, and PCI bus, and 1651 * to trigger a system-wide reset. 1652 * 1653 * See AMD Geode SC1100 Processor Data Book, Revision 2.0, 1654 * sections 6.3.1, 6.3.2, and 6.4.1. 1655 */ 1656 if (cpu_info_primary.ci_signature == 0x540) { 1657 outl(0xcf8, 0x80009044); 1658 outl(0xcfc, 0xf); 1659 } 1660 1661 x86_reset(); 1662 1663 /* 1664 * Try to cause a triple fault and watchdog reset by making the IDT 1665 * invalid and causing a fault. 1666 */ 1667 memset((void *)idt, 0, NIDT * sizeof(idt[0])); 1668 setregion(®ion, idt, NIDT * sizeof(idt[0]) - 1); 1669 lidt(®ion); 1670 breakpoint(); 1671 1672 #if 0 1673 /* 1674 * Try to cause a triple fault and watchdog reset by unmapping the 1675 * entire address space and doing a TLB flush. 1676 */ 1677 memset((void *)PTD, 0, PAGE_SIZE); 1678 tlbflush(); 1679 #endif 1680 1681 for (;;); 1682 #endif /* XENPV */ 1683 } 1684 1685 void 1686 cpu_getmcontext(struct lwp *l, mcontext_t *mcp, unsigned int *flags) 1687 { 1688 const struct trapframe *tf = l->l_md.md_regs; 1689 __greg_t *gr = mcp->__gregs; 1690 __greg_t ras_eip; 1691 1692 /* Save register context. */ 1693 gr[_REG_GS] = tf->tf_gs; 1694 gr[_REG_FS] = tf->tf_fs; 1695 gr[_REG_ES] = tf->tf_es; 1696 gr[_REG_DS] = tf->tf_ds; 1697 gr[_REG_EFL] = tf->tf_eflags; 1698 1699 gr[_REG_EDI] = tf->tf_edi; 1700 gr[_REG_ESI] = tf->tf_esi; 1701 gr[_REG_EBP] = tf->tf_ebp; 1702 gr[_REG_EBX] = tf->tf_ebx; 1703 gr[_REG_EDX] = tf->tf_edx; 1704 gr[_REG_ECX] = tf->tf_ecx; 1705 gr[_REG_EAX] = tf->tf_eax; 1706 gr[_REG_EIP] = tf->tf_eip; 1707 gr[_REG_CS] = tf->tf_cs; 1708 gr[_REG_ESP] = tf->tf_esp; 1709 gr[_REG_UESP] = tf->tf_esp; 1710 gr[_REG_SS] = tf->tf_ss; 1711 gr[_REG_TRAPNO] = tf->tf_trapno; 1712 gr[_REG_ERR] = tf->tf_err; 1713 1714 if ((ras_eip = (__greg_t)ras_lookup(l->l_proc, 1715 (void *) gr[_REG_EIP])) != -1) 1716 gr[_REG_EIP] = ras_eip; 1717 1718 *flags |= _UC_CPU; 1719 1720 mcp->_mc_tlsbase = (uintptr_t)l->l_private; 1721 *flags |= _UC_TLSBASE; 1722 1723 /* 1724 * Save floating point register context. 1725 * 1726 * If the cpu doesn't support fxsave we must still write to 1727 * the entire 512 byte area - otherwise we leak kernel memory 1728 * contents to userspace. 1729 * It wouldn't matter if we were doing the copyout here. 1730 * So we might as well convert to fxsave format. 1731 */ 1732 __CTASSERT(sizeof (struct fxsave) == 1733 sizeof mcp->__fpregs.__fp_reg_set.__fp_xmm_state); 1734 process_read_fpregs_xmm(l, (struct fxsave *) 1735 &mcp->__fpregs.__fp_reg_set.__fp_xmm_state); 1736 memset(&mcp->__fpregs.__fp_pad, 0, sizeof mcp->__fpregs.__fp_pad); 1737 *flags |= _UC_FXSAVE | _UC_FPU; 1738 } 1739 1740 /* 1741 * cpu_getmcontext_xsave(l, mcp, flags, xsavebuf, xsavelen, user_xsave) 1742 * 1743 * Copy out xsavebuf[0..xsavelen) to user_xsave, set mcp to point 1744 * there, and set _UC_XSAVE in flags. Caller must have already 1745 * used cpu_getmcontext to initialize mcp's FXSAVE area. 1746 * 1747 * May fail if the copyout fails. 1748 */ 1749 static int 1750 cpu_getmcontext_xsave(struct lwp *l, mcontext_t *mcp, unsigned int *flags, 1751 const struct xsave_header *xsavebuf, size_t xsavelen, 1752 struct xsave_header *user_xsave) 1753 { 1754 int error; 1755 1756 KASSERT(*flags & _UC_FPU); 1757 KASSERT(*flags & _UC_FXSAVE); 1758 1759 /* 1760 * Copy out the XSAVE area. 1761 */ 1762 error = copyout(xsavebuf, user_xsave, xsavelen); 1763 if (error != 0) 1764 return error; 1765 1766 /* 1767 * Record a pointer to the real XSAVE area in the 1768 * architecturally unused bits mcontext_t's FXSAVE area. 1769 */ 1770 mcp->__fpregs.__fp_reg_set.__xsave.__xsaveptr = 1771 (__greg_t)(uintptr_t)user_xsave; 1772 mcp->__fpregs.__fp_reg_set.__xsave.__xsavelen = (__greg_t)xsavelen; 1773 1774 /* 1775 * Set the _UC_XSAVE flag so cpu_setmcontext will be able to 1776 * restore the full state from the XSAVE area. 1777 */ 1778 *flags |= _UC_XSAVE; 1779 1780 /* Success! */ 1781 return 0; 1782 } 1783 1784 int 1785 cpu_mcontext_validate(struct lwp *l, const mcontext_t *mcp) 1786 { 1787 const __greg_t *gr = mcp->__gregs; 1788 struct trapframe *tf = l->l_md.md_regs; 1789 1790 /* 1791 * Check for security violations. If we're returning 1792 * to protected mode, the CPU will validate the segment 1793 * registers automatically and generate a trap on 1794 * violations. We handle the trap, rather than doing 1795 * all of the checking here. 1796 */ 1797 if (((gr[_REG_EFL] ^ tf->tf_eflags) & PSL_USERSTATIC) || 1798 !USERMODE(gr[_REG_CS])) 1799 return EINVAL; 1800 1801 return 0; 1802 } 1803 1804 int 1805 cpu_setmcontext(struct lwp *l, const mcontext_t *mcp, unsigned int flags) 1806 { 1807 struct trapframe *tf = l->l_md.md_regs; 1808 const __greg_t *gr = mcp->__gregs; 1809 struct proc *p = l->l_proc; 1810 struct xsave_header *xsavebuf = NULL; 1811 size_t xsavelen = 0; 1812 int error; 1813 1814 /* 1815 * If there's an external XSAVE area, copy it in and validate 1816 * it before we irreversibly modify the trapframe. 1817 * 1818 * We could check the length against the state components 1819 * included, but we currently don't: if it's truncated, it will 1820 * be as if the truncated part were zero-filled -- this is 1821 * implemented in process_write_xsave, called a little below. 1822 */ 1823 if ((flags & _UC_XSAVE) != 0) { 1824 const __greg_t user_xsave = 1825 mcp->__fpregs.__fp_reg_set.__xsave.__xsaveptr; 1826 1827 xsavelen = mcp->__fpregs.__fp_reg_set.__xsave.__xsavelen; 1828 error = process_verify_xsavelen(l, xsavelen); 1829 if (error != 0) 1830 goto out; 1831 xsavebuf = kmem_alloc(xsavelen, KM_SLEEP); 1832 error = copyin((const void *)user_xsave, xsavebuf, xsavelen); 1833 if (error != 0) 1834 goto out; 1835 error = process_verify_xsave(l, xsavebuf, xsavelen); 1836 if (error != 0) 1837 goto out; 1838 } 1839 1840 /* Restore register context, if any. */ 1841 if ((flags & _UC_CPU) != 0) { 1842 error = cpu_mcontext_validate(l, mcp); 1843 if (error) 1844 return error; 1845 1846 tf->tf_gs = gr[_REG_GS]; 1847 tf->tf_fs = gr[_REG_FS]; 1848 tf->tf_es = gr[_REG_ES]; 1849 tf->tf_ds = gr[_REG_DS]; 1850 /* Only change the user-alterable part of eflags */ 1851 tf->tf_eflags &= ~PSL_USER; 1852 tf->tf_eflags |= (gr[_REG_EFL] & PSL_USER); 1853 1854 tf->tf_edi = gr[_REG_EDI]; 1855 tf->tf_esi = gr[_REG_ESI]; 1856 tf->tf_ebp = gr[_REG_EBP]; 1857 tf->tf_ebx = gr[_REG_EBX]; 1858 tf->tf_edx = gr[_REG_EDX]; 1859 tf->tf_ecx = gr[_REG_ECX]; 1860 tf->tf_eax = gr[_REG_EAX]; 1861 tf->tf_eip = gr[_REG_EIP]; 1862 tf->tf_cs = gr[_REG_CS]; 1863 tf->tf_esp = gr[_REG_UESP]; 1864 tf->tf_ss = gr[_REG_SS]; 1865 } 1866 1867 if ((flags & _UC_TLSBASE) != 0) 1868 lwp_setprivate(l, (void *)(uintptr_t)mcp->_mc_tlsbase); 1869 1870 /* Restore floating point register context, if given. */ 1871 if ((flags & _UC_XSAVE) != 0) { 1872 KASSERT(xsavebuf != NULL); 1873 process_write_xsave(l, xsavebuf, xsavelen); 1874 } else if ((flags & _UC_FPU) != 0) { 1875 __CTASSERT(sizeof (struct fxsave) == 1876 sizeof mcp->__fpregs.__fp_reg_set.__fp_xmm_state); 1877 __CTASSERT(sizeof (struct save87) == 1878 sizeof mcp->__fpregs.__fp_reg_set.__fpchip_state); 1879 1880 if (flags & _UC_FXSAVE) { 1881 process_write_fpregs_xmm(l, (const struct fxsave *) 1882 &mcp->__fpregs.__fp_reg_set.__fp_xmm_state); 1883 } else { 1884 process_write_fpregs_s87(l, (const struct save87 *) 1885 &mcp->__fpregs.__fp_reg_set.__fpchip_state); 1886 } 1887 } 1888 1889 mutex_enter(p->p_lock); 1890 if (flags & _UC_SETSTACK) 1891 l->l_sigstk.ss_flags |= SS_ONSTACK; 1892 if (flags & _UC_CLRSTACK) 1893 l->l_sigstk.ss_flags &= ~SS_ONSTACK; 1894 mutex_exit(p->p_lock); 1895 1896 /* Success! */ 1897 error = 0; 1898 1899 out: if (xsavebuf) 1900 kmem_free(xsavebuf, xsavelen); 1901 return error; 1902 } 1903 1904 #define DEV_IO 14 /* iopl for compat_10 */ 1905 1906 int 1907 mm_md_open(dev_t dev, int flag, int mode, struct lwp *l) 1908 { 1909 1910 switch (minor(dev)) { 1911 case DEV_IO: 1912 /* 1913 * This is done by i386_iopl(3) now. 1914 * 1915 * #if defined(COMPAT_10) || defined(COMPAT_FREEBSD) 1916 */ 1917 if (flag & FWRITE) { 1918 struct trapframe *fp; 1919 int error; 1920 1921 error = kauth_authorize_machdep(l->l_cred, 1922 KAUTH_MACHDEP_IOPL, NULL, NULL, NULL, NULL); 1923 if (error) 1924 return (error); 1925 fp = curlwp->l_md.md_regs; 1926 fp->tf_eflags |= PSL_IOPL; 1927 } 1928 break; 1929 default: 1930 break; 1931 } 1932 return 0; 1933 } 1934 1935 #ifdef PAE 1936 void 1937 cpu_alloc_l3_page(struct cpu_info *ci) 1938 { 1939 int ret; 1940 struct pglist pg; 1941 struct vm_page *vmap; 1942 1943 KASSERT(ci != NULL); 1944 /* 1945 * Allocate a page for the per-CPU L3 PD. cr3 being 32 bits, PA musts 1946 * resides below the 4GB boundary. 1947 */ 1948 ret = uvm_pglistalloc(PAGE_SIZE, 0, 0x100000000ULL, 32, 0, &pg, 1, 0); 1949 vmap = TAILQ_FIRST(&pg); 1950 1951 if (ret != 0 || vmap == NULL) 1952 panic("%s: failed to allocate L3 pglist for CPU %d (ret %d)\n", 1953 __func__, cpu_index(ci), ret); 1954 1955 ci->ci_pae_l3_pdirpa = VM_PAGE_TO_PHYS(vmap); 1956 1957 ci->ci_pae_l3_pdir = (paddr_t *)uvm_km_alloc(kernel_map, PAGE_SIZE, 0, 1958 UVM_KMF_VAONLY | UVM_KMF_NOWAIT); 1959 if (ci->ci_pae_l3_pdir == NULL) 1960 panic("%s: failed to allocate L3 PD for CPU %d\n", 1961 __func__, cpu_index(ci)); 1962 1963 pmap_kenter_pa((vaddr_t)ci->ci_pae_l3_pdir, ci->ci_pae_l3_pdirpa, 1964 VM_PROT_READ | VM_PROT_WRITE, 0); 1965 1966 pmap_update(pmap_kernel()); 1967 } 1968 #endif /* PAE */ 1969 1970 static void 1971 idt_vec_copy(struct idt_vec *dst, struct idt_vec *src) 1972 { 1973 idt_descriptor_t *idt_dst; 1974 1975 idt_dst = dst->iv_idt; 1976 memcpy(idt_dst, src->iv_idt, PAGE_SIZE); 1977 memcpy(dst->iv_allocmap, src->iv_allocmap, sizeof(dst->iv_allocmap)); 1978 } 1979 1980 void 1981 idt_vec_init_cpu_md(struct idt_vec *iv, cpuid_t cid) 1982 { 1983 vaddr_t va_idt, va_pentium_idt; 1984 struct vm_page *pg; 1985 1986 if (idt_vec_is_pcpu() && 1987 cid != cpu_index(&cpu_info_primary)) { 1988 va_idt = uvm_km_alloc(kernel_map, PAGE_SIZE, 1989 0, UVM_KMF_VAONLY); 1990 pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO); 1991 if (pg == NULL) { 1992 panic("failed to allocate pcpu idt PA"); 1993 } 1994 pmap_kenter_pa(va_idt, VM_PAGE_TO_PHYS(pg), 1995 VM_PROT_READ|VM_PROT_WRITE, 0); 1996 pmap_update(pmap_kernel()); 1997 1998 memset((void *)va_idt, 0, PAGE_SIZE); 1999 2000 /* pentium f00f bug stuff */ 2001 va_pentium_idt = uvm_km_alloc(kernel_map, PAGE_SIZE, 2002 0, UVM_KMF_VAONLY); 2003 pmap_kenter_pa(va_pentium_idt, VM_PAGE_TO_PHYS(pg), 2004 VM_PROT_READ, 0); 2005 pmap_update(pmap_kernel()); 2006 2007 iv->iv_idt = (void *)va_idt; 2008 iv->iv_idt_pentium = (void *)va_pentium_idt; 2009 2010 idt_vec_copy(iv, &(cpu_info_primary.ci_idtvec)); 2011 } else { 2012 iv->iv_idt = (void *)idt_vaddr; 2013 iv->iv_idt_pentium = (void *)pentium_idt_vaddr; 2014 } 2015 } 2016
Indexes created Thu Aug 13 00:25:44 UTC 2026