1 /* $NetBSD: machdep.c,v 1.382 2026/11/07 22:20:10 thorpej Exp $ */ 2 3 /* 4 * Copyright (c) 1996, 1997, 1998, 2000, 2006, 2007, 2008, 2011 5 * The NetBSD Foundation, Inc. 6 * All rights reserved. 7 * 8 * This code is derived from software contributed to The NetBSD Foundation 9 * by Charles M. Hannum and by Jason R. Thorpe of the Numerical Aerospace 10 * Simulation Facility, NASA Ames Research Center. 11 * 12 * This code is derived from software contributed to The NetBSD Foundation 13 * by Coyote Point Systems, Inc. which was written under contract to Coyote 14 * Point by Jed Davis and Devon O'Dell. 15 * 16 * Redistribution and use in source and binary forms, with or without 17 * modification, are permitted provided that the following conditions 18 * are met: 19 * 1. Redistributions of source code must retain the above copyright 20 * notice, this list of conditions and the following disclaimer. 21 * 2. Redistributions in binary form must reproduce the above copyright 22 * notice, this list of conditions and the following disclaimer in the 23 * documentation and/or other materials provided with the distribution. 24 * 25 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 26 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 27 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 28 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 29 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 30 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 31 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 32 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 33 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 34 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 35 * POSSIBILITY OF SUCH DAMAGE. 36 */ 37 38 /* 39 * Copyright (c) 2006 Mathieu Ropert <mro (at) adviseo.fr> 40 * 41 * Permission to use, copy, modify, and distribute this software for any 42 * purpose with or without fee is hereby granted, provided that the above 43 * copyright notice and this permission notice appear in all copies. 44 * 45 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 46 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 47 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 48 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 49 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 50 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 51 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 52 */ 53 54 /* 55 * Copyright (c) 2007 Manuel Bouyer. 56 * 57 * Redistribution and use in source and binary forms, with or without 58 * modification, are permitted provided that the following conditions 59 * are met: 60 * 1. Redistributions of source code must retain the above copyright 61 * notice, this list of conditions and the following disclaimer. 62 * 2. Redistributions in binary form must reproduce the above copyright 63 * notice, this list of conditions and the following disclaimer in the 64 * documentation and/or other materials provided with the distribution. 65 * 66 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 67 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 68 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 69 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 70 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 71 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 72 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 73 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 74 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 75 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 76 */ 77 78 /* 79 * Copyright (c) 1982, 1987, 1990 The Regents of the University of California. 80 * All rights reserved. 81 * 82 * This code is derived from software contributed to Berkeley by 83 * William Jolitz. 84 * 85 * Redistribution and use in source and binary forms, with or without 86 * modification, are permitted provided that the following conditions 87 * are met: 88 * 1. Redistributions of source code must retain the above copyright 89 * notice, this list of conditions and the following disclaimer. 90 * 2. Redistributions in binary form must reproduce the above copyright 91 * notice, this list of conditions and the following disclaimer in the 92 * documentation and/or other materials provided with the distribution. 93 * 3. Neither the name of the University nor the names of its contributors 94 * may be used to endorse or promote products derived from this software 95 * without specific prior written permission. 96 * 97 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 98 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 99 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 100 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 101 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 102 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 103 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 104 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 105 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 106 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 107 * SUCH DAMAGE. 108 * 109 * @(#)machdep.c 7.4 (Berkeley) 6/3/91 110 */ 111 112 #include <sys/cdefs.h> 113 __KERNEL_RCSID(0, "$NetBSD: machdep.c,v 1.382 2026/11/07 22:20:10 thorpej Exp $"); 114 115 #include "opt_modular.h" 116 #include "opt_user_ldt.h" 117 #include "opt_ddb.h" 118 #include "opt_kgdb.h" 119 #include "opt_cpureset_delay.h" 120 #include "opt_mtrr.h" 121 #include "opt_realmem.h" 122 #include "opt_xen.h" 123 #include "opt_svs.h" 124 #include "opt_kaslr.h" 125 #ifndef XENPV 126 #include "opt_physmem.h" 127 #endif 128 #include "isa.h" 129 #include "pci.h" 130 131 #include <sys/param.h> 132 #include <sys/systm.h> 133 #include <sys/signal.h> 134 #include <sys/signalvar.h> 135 #include <sys/kernel.h> 136 #include <sys/cpu.h> 137 #include <sys/exec.h> 138 #include <sys/exec_aout.h> /* for MID_* */ 139 #include <sys/reboot.h> 140 #include <sys/conf.h> 141 #include <sys/msgbuf.h> 142 #include <sys/mount.h> 143 #include <sys/core.h> 144 #include <sys/kcore.h> 145 #include <sys/ucontext.h> 146 #include <machine/kcore.h> 147 #include <sys/ras.h> 148 #include <sys/syscallargs.h> 149 #include <sys/ksyms.h> 150 #include <sys/device.h> 151 #include <sys/lwp.h> 152 #include <sys/proc.h> 153 #include <sys/asan.h> 154 #include <sys/csan.h> 155 #include <sys/msan.h> 156 #include <sys/module.h> 157 #include <sys/timevar.h> 158 159 #ifdef KGDB 160 #include <sys/kgdb.h> 161 #endif 162 163 #include <lib/libkern/entpool.h> /* XXX */ 164 165 #include <dev/cons.h> 166 #include <dev/mm.h> 167 168 #include <uvm/uvm.h> 169 #include <uvm/uvm_page.h> 170 171 #include <sys/sysctl.h> 172 173 #include <machine/cpu.h> 174 #include <machine/cpu_rng.h> 175 #include <machine/cpufunc.h> 176 #include <machine/gdt.h> 177 #include <machine/intr.h> 178 #include <machine/pio.h> 179 #include <machine/psl.h> 180 #include <machine/reg.h> 181 #include <machine/specialreg.h> 182 #include <machine/bootinfo.h> 183 #include <x86/fpu.h> 184 #include <x86/dbregs.h> 185 #include <machine/mtrr.h> 186 #include <machine/mpbiosvar.h> 187 #include <machine/pmap_private.h> 188 189 #include <x86/bootspace.h> 190 #include <x86/cputypes.h> 191 #include <x86/cpuvar.h> 192 #include <x86/machdep.h> 193 #include <x86/x86/tsc.h> 194 195 #include <dev/isa/isareg.h> 196 #include <machine/isa_machdep.h> 197 #include <dev/ic/i8042reg.h> 198 199 #ifdef XEN 200 #include <xen/xen.h> 201 #include <xen/hypervisor.h> 202 #include <xen/evtchn.h> 203 #include <xen/include/public/version.h> 204 #include <xen/include/public/vcpu.h> 205 #endif /* XEN */ 206 207 #ifdef DDB 208 #include <machine/db_machdep.h> 209 #include <ddb/db_extern.h> 210 #include <ddb/db_output.h> 211 #include <ddb/db_interface.h> 212 #endif 213 214 #include "acpica.h" 215 216 #if NACPICA > 0 217 #include <dev/acpi/acpivar.h> 218 #define ACPI_MACHDEP_PRIVATE 219 #include <machine/acpi_machdep.h> 220 #else 221 #include <machine/i82489var.h> 222 #endif 223 224 #include "isa.h" 225 #include "isadma.h" 226 #include "ksyms.h" 227 228 /* the following is used externally (sysctl_hw) */ 229 char machine[] = "amd64"; /* CPU "architecture" */ 230 char machine_arch[] = "x86_64"; /* machine == machine_arch */ 231 232 #ifdef CPURESET_DELAY 233 int cpureset_delay = CPURESET_DELAY; 234 #else 235 int cpureset_delay = 2000; /* default to 2s */ 236 #endif 237 238 int cpu_class = CPUCLASS_686; 239 240 #ifdef MTRR 241 const struct mtrr_funcs *mtrr_funcs; 242 #endif 243 244 int cpu_class; 245 int use_pae; 246 247 #ifndef NO_SPARSE_DUMP 248 int sparse_dump = 1; 249 250 paddr_t max_paddr = 0; 251 unsigned char *sparse_dump_physmap; 252 #endif 253 254 char *dump_headerbuf, *dump_headerbuf_ptr; 255 #define dump_headerbuf_size PAGE_SIZE 256 #define dump_headerbuf_end (dump_headerbuf + dump_headerbuf_size) 257 #define dump_headerbuf_avail (dump_headerbuf_end - dump_headerbuf_ptr) 258 daddr_t dump_header_blkno; 259 260 size_t dump_nmemsegs; 261 size_t dump_npages; 262 size_t dump_header_size; 263 size_t dump_totalbytesleft; 264 265 vaddr_t idt_vaddr; 266 paddr_t idt_paddr; 267 vaddr_t gdt_vaddr; 268 paddr_t gdt_paddr; 269 vaddr_t ldt_vaddr; 270 paddr_t ldt_paddr; 271 272 static struct vm_map module_map_store; 273 extern struct bootspace bootspace; 274 extern struct slotspace slotspace; 275 276 vaddr_t vm_min_kernel_address __read_mostly = VM_MIN_KERNEL_ADDRESS_DEFAULT; 277 vaddr_t vm_max_kernel_address __read_mostly = VM_MAX_KERNEL_ADDRESS_DEFAULT; 278 pd_entry_t *pte_base __read_mostly; 279 280 struct vm_map *phys_map = NULL; 281 282 extern paddr_t lowmem_rsvd; 283 extern paddr_t avail_start, avail_end; 284 #ifdef XENPV 285 extern paddr_t pmap_pa_start, pmap_pa_end; 286 #endif 287 288 struct nmistore { 289 uint64_t cr3; 290 uint64_t scratch; 291 } __packed; 292 293 /* 294 * Size of memory segments, before any memory is stolen. 295 */ 296 phys_ram_seg_t mem_clusters[VM_PHYSSEG_MAX]; 297 int mem_cluster_cnt; 298 299 int cpu_dump(void); 300 int cpu_dumpsize(void); 301 u_long cpu_dump_mempagecnt(void); 302 void dodumpsys(void); 303 void dumpsys(void); 304 305 static void x86_64_proc0_pcb_ldt_init(void); 306 307 void dump_misc_init(void); 308 void dump_seg_prep(void); 309 int dump_seg_iter(int (*)(paddr_t, paddr_t)); 310 311 #ifndef NO_SPARSE_DUMP 312 void sparse_dump_reset(void); 313 void sparse_dump_mark(void); 314 void cpu_dump_prep_sparse(void); 315 #endif 316 317 void dump_header_start(void); 318 int dump_header_flush(void); 319 int dump_header_addbytes(const void*, size_t); 320 int dump_header_addseg(paddr_t, paddr_t); 321 int dump_header_finish(void); 322 323 int dump_seg_count_range(paddr_t, paddr_t); 324 int dumpsys_seg(paddr_t, paddr_t); 325 326 void init_bootspace(void); 327 void init_slotspace(void); 328 void init_x86_64(paddr_t); 329 330 static int cpu_getmcontext_xsave(struct lwp *, mcontext_t *, unsigned *, 331 const struct xsave_header *, size_t, struct xsave_header *); 332 333 /* 334 * Machine-dependent startup code 335 */ 336 void 337 cpu_startup(void) 338 { 339 int x, y; 340 vaddr_t minaddr, maxaddr; 341 psize_t sz; 342 343 /* 344 * For console drivers that require uvm and pmap to be initialized, 345 * we'll give them one more chance here... 346 */ 347 consinit(); 348 349 /* 350 * Initialize error message buffer (at end of core). 351 */ 352 if (msgbuf_p_cnt == 0) 353 panic("msgbuf paddr map has not been set up"); 354 for (x = 0, sz = 0; x < msgbuf_p_cnt; sz += msgbuf_p_seg[x++].sz) 355 continue; 356 357 msgbuf_vaddr = uvm_km_alloc(kernel_map, sz, 0, UVM_KMF_VAONLY); 358 if (msgbuf_vaddr == 0) 359 panic("failed to valloc msgbuf_vaddr"); 360 361 for (y = 0, sz = 0; y < msgbuf_p_cnt; y++) { 362 for (x = 0; x < btoc(msgbuf_p_seg[y].sz); x++, sz += PAGE_SIZE) 363 pmap_kenter_pa((vaddr_t)msgbuf_vaddr + sz, 364 msgbuf_p_seg[y].paddr + x * PAGE_SIZE, 365 VM_PROT_READ|VM_PROT_WRITE, 0); 366 } 367 368 pmap_update(pmap_kernel()); 369 370 initmsgbuf((void *)msgbuf_vaddr, round_page(sz)); 371 372 minaddr = 0; 373 374 /* 375 * Allocate a submap for physio. 376 */ 377 phys_map = uvm_km_suballoc(kernel_map, &minaddr, &maxaddr, 378 VM_PHYS_SIZE, 0, false, NULL); 379 380 /* 381 * Create the module map. 382 * 383 * The kernel uses RIP-relative addressing with a maximum offset of 384 * 2GB. Because of that, we can't put the kernel modules in kernel_map 385 * (like i386 does), since kernel_map is too far away in memory from 386 * the kernel sections. So we have to create a special module_map. 387 * 388 * The module map is taken as what is left of the bootstrap memory 389 * created in locore/prekern. 390 */ 391 uvm_map_setup(&module_map_store, bootspace.smodule, 392 bootspace.emodule, 0); 393 module_map_store.pmap = pmap_kernel(); 394 module_map = &module_map_store; 395 396 /* Say hello. */ 397 banner(); 398 399 #if NISA > 0 || NPCI > 0 400 /* Safe for i/o port / memory space allocation to use malloc now. */ 401 x86_bus_space_mallocok(); 402 #endif 403 404 #ifdef __HAVE_PCPU_AREA 405 cpu_pcpuarea_init(&cpu_info_primary); 406 #endif 407 gdt_init(); 408 x86_64_proc0_pcb_ldt_init(); 409 410 cpu_init_tss(&cpu_info_primary); 411 #if !defined(XENPV) 412 ltr(cpu_info_primary.ci_tss_sel); 413 #endif 414 415 x86_startup(); 416 } 417 418 #ifdef XENPV 419 /* used in assembly */ 420 void hypervisor_callback(void); 421 void failsafe_callback(void); 422 void x86_64_switch_context(struct pcb *); 423 void x86_64_tls_switch(struct lwp *); 424 425 void 426 x86_64_switch_context(struct pcb *new) 427 { 428 HYPERVISOR_stack_switch(GSEL(GDATA_SEL, SEL_KPL), new->pcb_rsp0); 429 struct physdev_set_iopl set_iopl; 430 set_iopl.iopl = new->pcb_iopl; 431 HYPERVISOR_physdev_op(PHYSDEVOP_set_iopl, &set_iopl); 432 } 433 434 void 435 x86_64_tls_switch(struct lwp *l) 436 { 437 struct cpu_info *ci = curcpu(); 438 struct pcb *pcb = lwp_getpcb(l); 439 struct trapframe *tf = l->l_md.md_regs; 440 uint64_t zero = 0; 441 442 /* 443 * Raise the IPL to IPL_HIGH. XXX Still needed? 444 */ 445 (void)splhigh(); 446 447 /* Update segment registers */ 448 if (pcb->pcb_flags & PCB_COMPAT32) { 449 update_descriptor(&ci->ci_gdt[GUFS_SEL], &pcb->pcb_fs); 450 update_descriptor(&ci->ci_gdt[GUGS_SEL], &pcb->pcb_gs); 451 setds(GSEL(GUDATA32_SEL, SEL_UPL)); 452 setes(GSEL(GUDATA32_SEL, SEL_UPL)); 453 setfs(GSEL(GUDATA32_SEL, SEL_UPL)); 454 HYPERVISOR_set_segment_base(SEGBASE_GS_USER_SEL, tf->tf_gs); 455 } else { 456 update_descriptor(&ci->ci_gdt[GUFS_SEL], &zero); 457 update_descriptor(&ci->ci_gdt[GUGS_SEL], &zero); 458 setds(GSEL(GUDATA_SEL, SEL_UPL)); 459 setes(GSEL(GUDATA_SEL, SEL_UPL)); 460 setfs(0); 461 HYPERVISOR_set_segment_base(SEGBASE_GS_USER_SEL, 0); 462 HYPERVISOR_set_segment_base(SEGBASE_FS, pcb->pcb_fs); 463 HYPERVISOR_set_segment_base(SEGBASE_GS_USER, pcb->pcb_gs); 464 } 465 } 466 #endif /* XENPV */ 467 468 /* 469 * Set up proc0's PCB and LDT. 470 */ 471 static void 472 x86_64_proc0_pcb_ldt_init(void) 473 { 474 struct lwp *l = &lwp0; 475 struct pcb *pcb = lwp_getpcb(l); 476 477 pcb->pcb_flags = 0; 478 pcb->pcb_fs = 0; 479 pcb->pcb_gs = 0; 480 pcb->pcb_rsp0 = (uvm_lwp_getuarea(l) + USPACE - 16) & ~0xf; 481 pcb->pcb_iopl = IOPL_KPL; 482 pcb->pcb_dbregs = NULL; 483 pcb->pcb_cr0 = rcr0() & ~CR0_TS; 484 l->l_md.md_regs = (struct trapframe *)pcb->pcb_rsp0 - 1; 485 486 #if !defined(XENPV) 487 lldt(GSYSSEL(GLDT_SEL, SEL_KPL)); 488 #else 489 xen_set_ldt((vaddr_t)ldtstore, LDT_SIZE >> 3); 490 /* Reset TS bit and set kernel stack for interrupt handlers */ 491 HYPERVISOR_fpu_taskswitch(1); 492 HYPERVISOR_stack_switch(GSEL(GDATA_SEL, SEL_KPL), pcb->pcb_rsp0); 493 struct physdev_set_iopl set_iopl; 494 set_iopl.iopl = pcb->pcb_iopl; 495 HYPERVISOR_physdev_op(PHYSDEVOP_set_iopl, &set_iopl); 496 #endif 497 } 498 499 /* 500 * Set up TSS and I/O bitmap. 501 */ 502 void 503 cpu_init_tss(struct cpu_info *ci) 504 { 505 #ifdef __HAVE_PCPU_AREA 506 const cpuid_t cid = cpu_index(ci); 507 #endif 508 struct cpu_tss *cputss; 509 struct nmistore *store; 510 uintptr_t p; 511 512 #ifdef __HAVE_PCPU_AREA 513 cputss = (struct cpu_tss *)&pcpuarea->ent[cid].tss; 514 #else 515 cputss = (struct cpu_tss *)uvm_km_alloc(kernel_map, 516 sizeof(struct cpu_tss), 0, UVM_KMF_WIRED|UVM_KMF_ZERO); 517 #endif 518 519 cputss->tss.tss_iobase = IOMAP_INVALOFF << 16; 520 521 /* DDB stack */ 522 #ifdef __HAVE_PCPU_AREA 523 p = (vaddr_t)&pcpuarea->ent[cid].ist0; 524 #else 525 p = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, UVM_KMF_WIRED|UVM_KMF_ZERO); 526 #endif 527 cputss->tss.tss_ist[0] = p + PAGE_SIZE - 16; 528 529 /* double fault */ 530 #ifdef __HAVE_PCPU_AREA 531 p = (vaddr_t)&pcpuarea->ent[cid].ist1; 532 #else 533 p = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, UVM_KMF_WIRED|UVM_KMF_ZERO); 534 #endif 535 cputss->tss.tss_ist[1] = p + PAGE_SIZE - 16; 536 537 /* NMI - store a structure at the top of the stack */ 538 #ifdef __HAVE_PCPU_AREA 539 p = (vaddr_t)&pcpuarea->ent[cid].ist2; 540 #else 541 p = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, UVM_KMF_WIRED|UVM_KMF_ZERO); 542 #endif 543 cputss->tss.tss_ist[2] = p + PAGE_SIZE - sizeof(struct nmistore); 544 store = (struct nmistore *)(p + PAGE_SIZE - sizeof(struct nmistore)); 545 store->cr3 = pmap_pdirpa(pmap_kernel(), 0); 546 547 /* DB */ 548 #ifdef __HAVE_PCPU_AREA 549 p = (vaddr_t)&pcpuarea->ent[cid].ist3; 550 #else 551 p = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, UVM_KMF_WIRED|UVM_KMF_ZERO); 552 #endif 553 cputss->tss.tss_ist[3] = p + PAGE_SIZE - 16; 554 555 ci->ci_tss = cputss; 556 ci->ci_tss_sel = tss_alloc(&cputss->tss); 557 } 558 559 void 560 buildcontext(struct lwp *l, void *catcher, void *f) 561 { 562 struct trapframe *tf = l->l_md.md_regs; 563 564 tf->tf_ds = GSEL(GUDATA_SEL, SEL_UPL); 565 tf->tf_es = GSEL(GUDATA_SEL, SEL_UPL); 566 tf->tf_fs = GSEL(GUDATA_SEL, SEL_UPL); 567 tf->tf_gs = GSEL(GUDATA_SEL, SEL_UPL); 568 569 tf->tf_rip = (uint64_t)catcher; 570 tf->tf_cs = GSEL(GUCODE_SEL, SEL_UPL); 571 tf->tf_rflags &= ~PSL_CLEARSIG; 572 tf->tf_rsp = (uint64_t)f; 573 tf->tf_ss = GSEL(GUDATA_SEL, SEL_UPL); 574 575 /* Ensure FP state is sane */ 576 fpu_sigreset(l); 577 } 578 579 void 580 sendsig_sigcontext(const ksiginfo_t *ksi, const sigset_t *mask) 581 { 582 583 printf("sendsig_sigcontext: illegal\n"); 584 sigexit(curlwp, SIGILL); 585 } 586 587 /* 588 * sendsig_siginfo(ksi, mask) 589 * 590 * Set up a stack frame and registers for signal delivery with the 591 * given siginfo and signal mask. 592 * 593 * The stack frame layout is as follows, with XSAVE parts omitted 594 * (and _UC_XSAVE clear in uc_flags) if the process doesn't have 595 * XSAVE state to restore: 596 * 597 * / +-----------------------+ 598 * | | (other XSAVE stuff) | 599 * | | --------------------- | 600 * | | Hi16_ZMM | 601 * | | --------------------- | 602 * XSAVE < | ZMM_Hi256 | 603 * | | --------------------- | 604 * | | YMM_Hi128 | 605 * | *-----------------------* \ 606 * | | FXSAVE in ucontext_t | | 607 * \ *-----------------------* > ucontext_t 608 * | rest of ucontext_t | | 609 * +-----------------------+ / 610 * | siginfo_t | 611 * +-----------------------+ =0 (mod 16) 612 * | return address | 613 * +-----------------------+ <--- rsp on signal handler entry 614 * 615 * Additionally, when XSAVE is in use, there is a pointer to the 616 * XSAVE area in some architecturally unused padding in the FXSAVE 617 * area. This is important because on i386, the XSAVE area 618 * _cannot_ overlap the ucontext_t owing to past ABI constraints, 619 * so applications running on both i386 and amd64 can use the same 620 * code to get the XSAVE pointer (and for 11.0, but not any 621 * earlier or later versions, the XSAVE pointer was also needed 622 * for signal handlers to write registers back on amd64 because 623 * the XSAVE area and the ucontext_t did not overlap). 624 * 625 * The overlap of the FXSAVE section in the XSAVE area and the 626 * FXSAVE section in ucontext_t is tricky but important: signal 627 * handlers use it to read, and potentially write back, the x87 628 * and xmm registers. Consider an application with two parts 629 * (e.g., a new library that uses an AVX512 optimization once it 630 * is available, and an old part running unmodified existing code 631 * predating NetBSD's AVX512 support): 632 * 633 * 1. Uses the high 16 zmm registers. 634 * 2. (a) Uses _only_ x87/xmm registers, and 635 * (b) has a signal handler that reads and writes them back. 636 * 637 * After part (1), the system will consider XSAVE necessary to 638 * save and restore the high 16 zmm registers, even if -- as is 639 * customary when using the ymm or low 16 zmm registers -- the 640 * application issues VZEROUPPER when part (1) is done. For this 641 * application to work without changing the signal handler in part 642 * (2)(b) to teach it about the XSAVE pointer -- i.e., for 643 * compatibility with existing code -- the XSAVE area and the 644 * ucontext_t MUST overlap. (11.0 would copy out the FXSAVE area 645 * to both locations so that unmodified signal handlers could 646 * _read_ the x87/xmm registers, but _writeback_ would be 647 * ignored.) 648 * 649 * For more background, see: 650 * 651 * PR kern/60539: XSAVE changes break ucontext userspace API 652 * https://gnats.NetBSD.org/60539 653 */ 654 void 655 sendsig_siginfo(const ksiginfo_t *ksi, const sigset_t *mask) 656 { 657 struct lwp *l = curlwp; 658 struct proc *p = l->l_proc; 659 struct sigacts *ps = p->p_sigacts; 660 int onstack, error; 661 int sig = ksi->ksi_signo; 662 struct sigframe_siginfo *fp, frame; 663 sig_t catcher = SIGACTION(p, sig).sa_handler; 664 struct trapframe *tf = l->l_md.md_regs; 665 const struct xsave_header *xsavebuf = NULL; 666 size_t xsavelen = 0; 667 struct xsave_header *user_xsave = NULL; 668 char *sp; 669 670 KASSERT(mutex_owned(p->p_lock)); 671 672 /* Do we need to jump onto the signal stack? */ 673 onstack = 674 (l->l_sigstk.ss_flags & (SS_DISABLE | SS_ONSTACK)) == 0 && 675 (SIGACTION(p, sig).sa_flags & SA_ONSTACK) != 0; 676 677 /* Allocate space for the signal handler context. */ 678 if (onstack) { 679 KASSERT(l->l_sigstk.ss_size >= MINSIGSTKSZ); 680 sp = ((char *)l->l_sigstk.ss_sp + l->l_sigstk.ss_size); 681 } else { 682 /* AMD64 ABI 128-bytes "red zone". */ 683 sp = (char *)tf->tf_rsp - 128; 684 } 685 686 /* 687 * The maximum amount of space we might use, including padding 688 * for alignment, had better fit in MINSIGSTKSZ. 689 * 690 * If this changes because you have increased XSAVE_MAX_BYTES, 691 * you need to work out the ABI change for MINSIGSTKSZ. 692 */ 693 __CTASSERT(8 + STACK_ALIGNBYTES + sizeof(struct sigframe_siginfo) + 694 (XSAVE_ALIGN - 1) + XSAVE_MAX_BYTES <= MINSIGSTKSZ); 695 696 /* 697 * Find whether we need to allocate a separate XSAVE area, 698 * because the user program has used extended CPU state beyond 699 * the x87/SSE registers, or whether we can get by with just an 700 * FXSAVE area. 701 */ 702 if (process_xsave_needed_p(l)) { 703 enum { 704 overlap = (sizeof(struct sigframe_siginfo) - 705 offsetof(struct sigframe_siginfo, 706 sf_uc.uc_mcontext.__fpregs)), 707 }; 708 709 process_read_xsave(l, &xsavebuf, &xsavelen); 710 KASSERT(xsavebuf != NULL); 711 KASSERT(xsavelen <= XSAVE_MAX_BYTES); 712 CTASSERT(overlap <= XSAVE_MAX_BYTES); 713 714 KASSERT(!onstack || sp >= (char *)l->l_sigstk.ss_sp); 715 KASSERT(!onstack || 716 sp - (char *)l->l_sigstk.ss_sp >= xsavelen); 717 KASSERT(!onstack || 718 sp - (char *)l->l_sigstk.ss_sp >= overlap); 719 sp -= MAX(xsavelen, overlap); 720 721 KASSERT(!onstack || sp >= (char *)l->l_sigstk.ss_sp); 722 KASSERT(!onstack || 723 sp - (char *)l->l_sigstk.ss_sp >= XSAVE_ALIGN - 1); 724 sp = (char *)((uintptr_t)sp & ~(XSAVE_ALIGN - 1)); 725 726 KASSERT(!onstack || sp >= (char *)l->l_sigstk.ss_sp); 727 KASSERT(((uintptr_t)sp & (XSAVE_ALIGN - 1)) == 0); 728 user_xsave = (void *)sp; 729 730 CTASSERT((overlap % XSAVE_ALIGN) == 0); 731 CTASSERT((overlap & STACK_ALIGNBYTES) == 0); 732 CTASSERT(((overlap - sizeof(struct sigframe_siginfo)) & 733 STACK_ALIGNBYTES) == (STACK_ALIGNBYTES + 1) - 8); 734 sp += overlap; 735 } 736 737 /* 738 * Reserve space for a struct sigframe_siginfo. The first 739 * member is a return address, as if we are entering a 740 * procedure with a CALL instruction with a stack frame aligned 741 * to a multiple of 16 bytes; the rest of the structure is that 742 * stack frame. So make sure the structure starts at an 743 * address congruent to 8 modulo 16. 744 */ 745 KASSERT(!onstack || sp >= (char *)l->l_sigstk.ss_sp); 746 KASSERT(!onstack || (size_t)(sp - (char *)l->l_sigstk.ss_sp) >= 747 8 + STACK_ALIGNBYTES + sizeof(struct sigframe_siginfo)); 748 sp -= sizeof(struct sigframe_siginfo); 749 /* Round down the stackpointer to a multiple of 16 for the ABI. */ 750 if (((uintptr_t)sp & STACK_ALIGNBYTES) == 751 (STACK_ALIGNBYTES + 1) - 8) { 752 fp = (struct sigframe_siginfo *)sp; 753 } else { 754 fp = (struct sigframe_siginfo *)(((uintptr_t)sp & 755 ~STACK_ALIGNBYTES) - 8); 756 } 757 KASSERT(!onstack || (char *)fp >= (char *)l->l_sigstk.ss_sp); 758 KASSERT(((uintptr_t)fp & STACK_ALIGNBYTES) == 759 (STACK_ALIGNBYTES + 1) - 8); 760 761 /* 762 * If we have to use XSAVE, the FXSAVE area of the ucontext_t 763 * on the user's stack must line up with the FXSAVE subarea of 764 * the XSAVE area on the user's stack. 765 */ 766 KASSERT(xsavebuf == 0 || 767 (((uintptr_t)&fp->sf_uc.uc_mcontext.__fpregs & (XSAVE_ALIGN - 1)) 768 == 0)); 769 KASSERT(xsavebuf == 0 || (uintptr_t)user_xsave == 770 (uintptr_t)&fp->sf_uc.uc_mcontext.__fpregs); 771 772 memset(&frame, 0, sizeof(frame)); 773 frame.sf_ra = (uint64_t)ps->sa_sigdesc[sig].sd_tramp; 774 frame.sf_si._info = ksi->ksi_info; 775 frame.sf_uc.uc_flags = _UC_SIGMASK; 776 frame.sf_uc.uc_sigmask = *mask; 777 frame.sf_uc.uc_link = l->l_ctxlink; 778 frame.sf_uc.uc_flags |= (l->l_sigstk.ss_flags & SS_ONSTACK) 779 ? _UC_SETSTACK : _UC_CLRSTACK; 780 sendsig_reset(l, sig); 781 782 mutex_exit(p->p_lock); 783 cpu_getmcontext(l, &frame.sf_uc.uc_mcontext, &frame.sf_uc.uc_flags); 784 785 /* 786 * If we have to use XSAVE, copy out the part of it past the 787 * FXSAVE area separately -- and be ready to bail if it failed. 788 */ 789 if (xsavebuf) { 790 KASSERT((void *)&fp->sf_uc.uc_mcontext.__fpregs == 791 (void *)user_xsave); 792 error = cpu_getmcontext_xsave(l, &frame.sf_uc.uc_mcontext, 793 &frame.sf_uc.uc_flags, xsavebuf, xsavelen, user_xsave); 794 if (error != 0) 795 goto relock; 796 } 797 798 /* Copyout all the fp regs, the signal handler might expect them. */ 799 error = copyout(&frame, fp, sizeof frame); 800 relock: 801 mutex_enter(p->p_lock); 802 803 if (error != 0) { 804 /* 805 * Process has trashed its stack; give it an illegal 806 * instruction to halt it in its tracks. 807 */ 808 sigexit(l, SIGILL); 809 /* NOTREACHED */ 810 } 811 812 buildcontext(l, catcher, fp); 813 814 tf->tf_rdi = sig; 815 tf->tf_rsi = (uint64_t)&fp->sf_si; 816 tf->tf_rdx = tf->tf_r15 = (uint64_t)&fp->sf_uc; 817 818 /* Remember that we're now on the signal stack. */ 819 if (onstack) 820 l->l_sigstk.ss_flags |= SS_ONSTACK; 821 822 if ((vaddr_t)catcher >= VM_MAXUSER_ADDRESS) { 823 /* 824 * process has given an invalid address for the 825 * handler. Stop it, but do not do it before so 826 * we can return the right info to userland (or in core dump) 827 */ 828 sigexit(l, SIGILL); 829 /* NOTREACHED */ 830 } 831 } 832 833 struct pcb dumppcb; 834 835 void 836 cpu_reboot(int howto, char *bootstr) 837 { 838 static bool syncdone = false; 839 int s = IPL_NONE; 840 __USE(s); /* ugly otherwise */ 841 842 if (cold) { 843 howto |= RB_HALT; 844 goto haltsys; 845 } 846 847 boothowto = howto; 848 849 /* i386 maybe_dump() */ 850 851 /* 852 * If we've panic'd, don't make the situation potentially 853 * worse by syncing or unmounting the file systems. 854 */ 855 if ((howto & RB_NOSYNC) == 0 && panicstr == NULL) { 856 if (!syncdone) { 857 syncdone = true; 858 /* XXX used to force unmount as well, here */ 859 vfs_sync_all(curlwp); 860 } 861 862 while (vfs_unmountall1(curlwp, false, false) || 863 config_detach_all(boothowto) || 864 vfs_unmount_forceone(curlwp)) 865 ; /* do nothing */ 866 } else { 867 if (!db_active) 868 suspendsched(); 869 } 870 871 pmf_system_shutdown(boothowto); 872 873 /* Disable interrupts. */ 874 s = splhigh(); 875 876 /* Do a dump if requested. */ 877 if ((howto & (RB_DUMP | RB_HALT)) == RB_DUMP) 878 dumpsys(); 879 880 haltsys: 881 doshutdownhooks(); 882 883 if ((howto & RB_POWERDOWN) == RB_POWERDOWN) { 884 #if NACPICA > 0 885 if (s != IPL_NONE) 886 splx(s); 887 888 acpi_enter_sleep_state(ACPI_STATE_S5); 889 #endif 890 #ifdef XEN 891 if (vm_guest == VM_GUEST_XENPV || 892 vm_guest == VM_GUEST_XENPVH || 893 vm_guest == VM_GUEST_XENPVHVM) 894 HYPERVISOR_shutdown(); 895 #endif /* XEN */ 896 } 897 898 cpu_broadcast_halt(); 899 900 if (howto & RB_HALT) { 901 #if NACPICA > 0 902 acpi_disable(); 903 #endif 904 905 printf("\n"); 906 printf("The operating system has halted.\n"); 907 printf("Please press any key to reboot.\n\n"); 908 cnpollc(true); /* for proper keyboard command handling */ 909 if (cngetc() == 0) { 910 /* no console attached, so just hlt */ 911 printf("No keyboard - cannot reboot after all.\n"); 912 for(;;) { 913 x86_hlt(); 914 } 915 } 916 cnpollc(false); 917 } 918 919 printf("rebooting...\n"); 920 if (cpureset_delay > 0) 921 delay(cpureset_delay * 1000); 922 cpu_reset(); 923 for(;;) ; 924 /*NOTREACHED*/ 925 } 926 927 /* 928 * XXXfvdl share dumpcode. 929 */ 930 931 /* 932 * Perform assorted dump-related initialization tasks. Assumes that 933 * the maximum physical memory address will not increase afterwards. 934 */ 935 void 936 dump_misc_init(void) 937 { 938 #ifndef NO_SPARSE_DUMP 939 int i; 940 #endif 941 942 if (dump_headerbuf != NULL) 943 return; /* already called */ 944 945 #ifndef NO_SPARSE_DUMP 946 for (i = 0; i < mem_cluster_cnt; ++i) { 947 paddr_t top = mem_clusters[i].start + mem_clusters[i].size; 948 if (max_paddr < top) 949 max_paddr = top; 950 } 951 #ifdef DEBUG 952 printf("dump_misc_init: max_paddr = 0x%lx\n", 953 (unsigned long)max_paddr); 954 #endif 955 if (max_paddr == 0) { 956 printf("Your machine does not initialize mem_clusters; " 957 "sparse_dumps disabled\n"); 958 sparse_dump = 0; 959 } else { 960 sparse_dump_physmap = (void *)uvm_km_alloc(kernel_map, 961 roundup(max_paddr / (PAGE_SIZE * NBBY), PAGE_SIZE), 962 PAGE_SIZE, UVM_KMF_WIRED|UVM_KMF_ZERO); 963 } 964 #endif 965 dump_headerbuf = (void *)uvm_km_alloc(kernel_map, 966 dump_headerbuf_size, 967 PAGE_SIZE, UVM_KMF_WIRED|UVM_KMF_ZERO); 968 /* XXXjld should check for failure here, disable dumps if so. */ 969 } 970 971 #ifndef NO_SPARSE_DUMP 972 /* 973 * Clear the set of pages to include in a sparse dump. 974 */ 975 void 976 sparse_dump_reset(void) 977 { 978 memset(sparse_dump_physmap, 0, 979 roundup(max_paddr / (PAGE_SIZE * NBBY), PAGE_SIZE)); 980 } 981 982 /* 983 * Include or exclude pages in a sparse dump. 984 */ 985 void 986 sparse_dump_mark(void) 987 { 988 paddr_t p, pstart, pend; 989 struct vm_page *pg; 990 int i; 991 uvm_physseg_t upm; 992 993 /* 994 * Mark all memory pages, then unmark pages that are uninteresting. 995 * Dereferenceing pg->uobject might crash again if another CPU 996 * frees the object out from under us, but we can't lock anything 997 * so it's a risk we have to take. 998 */ 999 1000 for (i = 0; i < mem_cluster_cnt; ++i) { 1001 pstart = mem_clusters[i].start / PAGE_SIZE; 1002 pend = pstart + mem_clusters[i].size / PAGE_SIZE; 1003 1004 for (p = pstart; p < pend; p++) { 1005 setbit(sparse_dump_physmap, p); 1006 } 1007 } 1008 for (upm = uvm_physseg_get_first(); 1009 uvm_physseg_valid_p(upm); 1010 upm = uvm_physseg_get_next(upm)) { 1011 paddr_t pfn; 1012 1013 /* 1014 * We assume that seg->start to seg->end are 1015 * uvm_page_physload()ed 1016 */ 1017 for (pfn = uvm_physseg_get_start(upm); 1018 pfn < uvm_physseg_get_end(upm); 1019 pfn++) { 1020 pg = PHYS_TO_VM_PAGE(ptoa(pfn)); 1021 1022 if (pg->uanon || (pg->flags & PG_FREE) || 1023 (pg->uobject && pg->uobject->pgops)) { 1024 p = VM_PAGE_TO_PHYS(pg) / PAGE_SIZE; 1025 clrbit(sparse_dump_physmap, p); 1026 } 1027 } 1028 } 1029 } 1030 1031 /* 1032 * Machine-dependently decides on the contents of a sparse dump, using 1033 * the above. 1034 */ 1035 void 1036 cpu_dump_prep_sparse(void) 1037 { 1038 sparse_dump_reset(); 1039 /* XXX could the alternate recursive page table be skipped? */ 1040 sparse_dump_mark(); 1041 /* Memory for I/O buffers could be unmarked here, for example. */ 1042 /* The kernel text could also be unmarked, but gdb would be upset. */ 1043 } 1044 #endif 1045 1046 /* 1047 * Abstractly iterate over the collection of memory segments to be 1048 * dumped; the callback lacks the customary environment-pointer 1049 * argument because none of the current users really need one. 1050 * 1051 * To be used only after dump_seg_prep is called to set things up. 1052 */ 1053 int 1054 dump_seg_iter(int (*callback)(paddr_t, paddr_t)) 1055 { 1056 int error, i; 1057 1058 #define CALLBACK(start,size) do { \ 1059 error = callback(start,size); \ 1060 if (error) \ 1061 return error; \ 1062 } while(0) 1063 1064 for (i = 0; i < mem_cluster_cnt; ++i) { 1065 #ifndef NO_SPARSE_DUMP 1066 /* 1067 * The bitmap is scanned within each memory segment, 1068 * rather than over its entire domain, in case any 1069 * pages outside of the memory proper have been mapped 1070 * into kva; they might be devices that wouldn't 1071 * appreciate being arbitrarily read, and including 1072 * them could also break the assumption that a sparse 1073 * dump will always be smaller than a full one. 1074 */ 1075 if (sparse_dump && sparse_dump_physmap) { 1076 paddr_t p, sp_start, sp_end; 1077 int lastset; 1078 1079 sp_start = mem_clusters[i].start; 1080 sp_end = sp_start + mem_clusters[i].size; 1081 sp_start = rounddown(sp_start, PAGE_SIZE); /* unnecessary? */ 1082 lastset = 0; 1083 for (p = sp_start; p < sp_end; p += PAGE_SIZE) { 1084 int thisset = isset(sparse_dump_physmap, 1085 p/PAGE_SIZE); 1086 1087 if (!lastset && thisset) 1088 sp_start = p; 1089 if (lastset && !thisset) 1090 CALLBACK(sp_start, p - sp_start); 1091 lastset = thisset; 1092 } 1093 if (lastset) 1094 CALLBACK(sp_start, p - sp_start); 1095 } else 1096 #endif 1097 CALLBACK(mem_clusters[i].start, mem_clusters[i].size); 1098 } 1099 return 0; 1100 #undef CALLBACK 1101 } 1102 1103 /* 1104 * Prepare for an impending core dump: decide what's being dumped and 1105 * how much space it will take up. 1106 */ 1107 void 1108 dump_seg_prep(void) 1109 { 1110 #ifndef NO_SPARSE_DUMP 1111 if (sparse_dump && sparse_dump_physmap) 1112 cpu_dump_prep_sparse(); 1113 #endif 1114 1115 dump_nmemsegs = 0; 1116 dump_npages = 0; 1117 dump_seg_iter(dump_seg_count_range); 1118 1119 dump_header_size = ALIGN(sizeof(kcore_seg_t)) + 1120 ALIGN(sizeof(cpu_kcore_hdr_t)) + 1121 ALIGN(dump_nmemsegs * sizeof(phys_ram_seg_t)); 1122 dump_header_size = roundup(dump_header_size, dbtob(1)); 1123 1124 /* 1125 * savecore(8) will read this to decide how many pages to 1126 * copy, and cpu_dumpconf has already used the pessimistic 1127 * value to set dumplo, so it's time to tell the truth. 1128 */ 1129 dumpsize = dump_npages; /* XXX could these just be one variable? */ 1130 } 1131 1132 int 1133 dump_seg_count_range(paddr_t start, paddr_t size) 1134 { 1135 ++dump_nmemsegs; 1136 dump_npages += size / PAGE_SIZE; 1137 return 0; 1138 } 1139 1140 /* 1141 * A sparse dump's header may be rather large, due to the number of 1142 * "segments" emitted. These routines manage a simple output buffer, 1143 * so that the header can be written to disk incrementally. 1144 */ 1145 void 1146 dump_header_start(void) 1147 { 1148 dump_headerbuf_ptr = dump_headerbuf; 1149 dump_header_blkno = dumplo; 1150 } 1151 1152 int 1153 dump_header_flush(void) 1154 { 1155 const struct bdevsw *bdev; 1156 size_t to_write; 1157 int error; 1158 1159 bdev = bdevsw_lookup(dumpdev); 1160 to_write = roundup(dump_headerbuf_ptr - dump_headerbuf, dbtob(1)); 1161 error = bdev->d_dump(dumpdev, dump_header_blkno, 1162 dump_headerbuf, to_write); 1163 dump_header_blkno += btodb(to_write); 1164 dump_headerbuf_ptr = dump_headerbuf; 1165 return error; 1166 } 1167 1168 int 1169 dump_header_addbytes(const void* vptr, size_t n) 1170 { 1171 const char* ptr = vptr; 1172 int error; 1173 1174 while (n > dump_headerbuf_avail) { 1175 memcpy(dump_headerbuf_ptr, ptr, dump_headerbuf_avail); 1176 ptr += dump_headerbuf_avail; 1177 n -= dump_headerbuf_avail; 1178 dump_headerbuf_ptr = dump_headerbuf_end; 1179 error = dump_header_flush(); 1180 if (error) 1181 return error; 1182 } 1183 memcpy(dump_headerbuf_ptr, ptr, n); 1184 dump_headerbuf_ptr += n; 1185 1186 return 0; 1187 } 1188 1189 int 1190 dump_header_addseg(paddr_t start, paddr_t size) 1191 { 1192 phys_ram_seg_t seg = { start, size }; 1193 int error; 1194 1195 error = dump_header_addbytes(&seg, sizeof(seg)); 1196 if (error) { 1197 printf("[seg 0x%"PRIxPADDR" bytes 0x%"PRIxPSIZE" failed," 1198 " error=%d] ", start, size, error); 1199 } 1200 return error; 1201 } 1202 1203 int 1204 dump_header_finish(void) 1205 { 1206 int error; 1207 1208 memset(dump_headerbuf_ptr, 0, dump_headerbuf_avail); 1209 error = dump_header_flush(); 1210 if (error) 1211 printf("[finish failed, error=%d] ", error); 1212 return error; 1213 } 1214 1215 1216 /* 1217 * These variables are needed by /sbin/savecore 1218 */ 1219 uint32_t dumpmag = 0x8fca0101; /* magic number */ 1220 int dumpsize = 0; /* pages */ 1221 long dumplo = 0; /* blocks */ 1222 1223 /* 1224 * cpu_dumpsize: calculate size of machine-dependent kernel core dump headers 1225 * for a full (non-sparse) dump. 1226 */ 1227 int 1228 cpu_dumpsize(void) 1229 { 1230 int size; 1231 1232 size = ALIGN(sizeof(kcore_seg_t)) + ALIGN(sizeof(cpu_kcore_hdr_t)) + 1233 ALIGN(mem_cluster_cnt * sizeof(phys_ram_seg_t)); 1234 if (roundup(size, dbtob(1)) != dbtob(1)) 1235 return (-1); 1236 1237 return (1); 1238 } 1239 1240 /* 1241 * cpu_dump_mempagecnt: calculate the size of RAM (in pages) to be dumped 1242 * for a full (non-sparse) dump. 1243 */ 1244 u_long 1245 cpu_dump_mempagecnt(void) 1246 { 1247 u_long i, n; 1248 1249 n = 0; 1250 for (i = 0; i < mem_cluster_cnt; i++) 1251 n += atop(mem_clusters[i].size); 1252 return (n); 1253 } 1254 1255 /* 1256 * cpu_dump: dump the machine-dependent kernel core dump headers. 1257 */ 1258 int 1259 cpu_dump(void) 1260 { 1261 kcore_seg_t seg; 1262 cpu_kcore_hdr_t cpuhdr; 1263 const struct bdevsw *bdev; 1264 int error; 1265 1266 bdev = bdevsw_lookup(dumpdev); 1267 if (bdev == NULL) { 1268 printf("[device 0x%llx ENXIO] ", (unsigned long long)dumpdev); 1269 return ENXIO; 1270 } 1271 1272 /* 1273 * Generate a segment header. 1274 */ 1275 CORE_SETMAGIC(seg, KCORE_MAGIC, MID_MACHINE, CORE_CPU); 1276 seg.c_size = dump_header_size - ALIGN(sizeof(seg)); 1277 error = dump_header_addbytes(&seg, ALIGN(sizeof(seg))); 1278 if (error) { 1279 printf("[segment header %zu bytes failed, error=%d] ", 1280 ALIGN(sizeof(seg)), error); 1281 /* blithely proceed (can't fail?) */ 1282 } 1283 1284 /* 1285 * Add the machine-dependent header info. 1286 */ 1287 cpuhdr.ptdpaddr = PDPpaddr; 1288 cpuhdr.nmemsegs = dump_nmemsegs; 1289 error = dump_header_addbytes(&cpuhdr, ALIGN(sizeof(cpuhdr))); 1290 if (error) { 1291 printf("[MD header %zu bytes failed, error=%d] ", 1292 ALIGN(sizeof(cpuhdr)), error); 1293 /* blithely proceed (can't fail?) */ 1294 } 1295 1296 /* 1297 * Write out the memory segment descriptors. 1298 */ 1299 return dump_seg_iter(dump_header_addseg); 1300 } 1301 1302 /* 1303 * Doadump comes here after turning off memory management and 1304 * getting on the dump stack, either when called above, or by 1305 * the auto-restart code. 1306 */ 1307 #define BYTES_PER_DUMP PAGE_SIZE /* must be a multiple of pagesize XXX small */ 1308 static vaddr_t dumpspace; 1309 1310 vaddr_t 1311 reserve_dumppages(vaddr_t p) 1312 { 1313 1314 dumpspace = p; 1315 return (p + BYTES_PER_DUMP); 1316 } 1317 1318 int 1319 dumpsys_seg(paddr_t maddr, paddr_t bytes) 1320 { 1321 u_long i, m, n; 1322 daddr_t blkno; 1323 const struct bdevsw *bdev; 1324 int (*dump)(dev_t, daddr_t, void *, size_t); 1325 int error; 1326 1327 if (dumpdev == NODEV) 1328 return ENODEV; 1329 bdev = bdevsw_lookup(dumpdev); 1330 if (bdev == NULL || bdev->d_psize == NULL) 1331 return ENODEV; 1332 1333 dump = bdev->d_dump; 1334 1335 blkno = dump_header_blkno; 1336 for (i = 0; i < bytes; i += n, dump_totalbytesleft -= n) { 1337 /* Print out how many MBs we have left to go. */ 1338 if ((dump_totalbytesleft % (1024*1024)) == 0) 1339 printf_nolog("%lu ", (unsigned long) 1340 (dump_totalbytesleft / (1024 * 1024))); 1341 1342 /* Limit size for next transfer. */ 1343 n = bytes - i; 1344 if (n > BYTES_PER_DUMP) 1345 n = BYTES_PER_DUMP; 1346 1347 for (m = 0; m < n; m += NBPG) 1348 pmap_kenter_pa(dumpspace + m, maddr + m, 1349 VM_PROT_READ, 0); 1350 pmap_update(pmap_kernel()); 1351 1352 error = (*dump)(dumpdev, blkno, (void *)dumpspace, n); 1353 pmap_kremove_local(dumpspace, n); 1354 if (error) 1355 return error; 1356 maddr += n; 1357 blkno += btodb(n); /* XXX? */ 1358 1359 #if 0 /* XXX this doesn't work. grr. */ 1360 /* operator aborting dump? */ 1361 if (sget() != NULL) 1362 return EINTR; 1363 #endif 1364 } 1365 dump_header_blkno = blkno; 1366 1367 return 0; 1368 } 1369 1370 void 1371 dodumpsys(void) 1372 { 1373 const struct bdevsw *bdev; 1374 int dumpend, psize; 1375 int error; 1376 1377 if (dumpdev == NODEV) 1378 return; 1379 1380 bdev = bdevsw_lookup(dumpdev); 1381 if (bdev == NULL || bdev->d_psize == NULL) 1382 return; 1383 /* 1384 * For dumps during autoconfiguration, 1385 * if dump device has already configured... 1386 */ 1387 if (dumpsize == 0) 1388 cpu_dumpconf(); 1389 1390 printf("\ndumping to dev %llu,%llu (offset=%ld, size=%d):", 1391 (unsigned long long)major(dumpdev), 1392 (unsigned long long)minor(dumpdev), dumplo, dumpsize); 1393 1394 if (dumplo <= 0 || dumpsize <= 0) { 1395 printf(" not possible\n"); 1396 return; 1397 } 1398 1399 psize = bdev_size(dumpdev); 1400 printf("\ndump "); 1401 if (psize == -1) { 1402 printf("area unavailable\n"); 1403 return; 1404 } 1405 1406 #if 0 /* XXX this doesn't work. grr. */ 1407 /* toss any characters present prior to dump */ 1408 while (sget() != NULL); /*syscons and pccons differ */ 1409 #endif 1410 1411 dump_seg_prep(); 1412 dumpend = dumplo + btodb(dump_header_size) + ctod(dump_npages); 1413 if (dumpend > psize) { 1414 printf("failed: insufficient space (%d < %d)\n", 1415 psize, dumpend); 1416 goto failed; 1417 } 1418 1419 dump_header_start(); 1420 if ((error = cpu_dump()) != 0) 1421 goto err; 1422 if ((error = dump_header_finish()) != 0) 1423 goto err; 1424 1425 if (dump_header_blkno != dumplo + btodb(dump_header_size)) { 1426 printf("BAD header size (%ld [written] != %ld [expected])\n", 1427 (long)(dump_header_blkno - dumplo), 1428 (long)btodb(dump_header_size)); 1429 goto failed; 1430 } 1431 1432 dump_totalbytesleft = roundup(ptoa(dump_npages), BYTES_PER_DUMP); 1433 error = dump_seg_iter(dumpsys_seg); 1434 1435 if (error == 0 && dump_header_blkno != dumpend) { 1436 printf("BAD dump size (%ld [written] != %ld [expected])\n", 1437 (long)(dumpend - dumplo), 1438 (long)(dump_header_blkno - dumplo)); 1439 goto failed; 1440 } 1441 1442 err: 1443 switch (error) { 1444 1445 case ENXIO: 1446 printf("device bad\n"); 1447 break; 1448 1449 case EFAULT: 1450 printf("device not ready\n"); 1451 break; 1452 1453 case EINVAL: 1454 printf("area improper\n"); 1455 break; 1456 1457 case EIO: 1458 printf("i/o error\n"); 1459 break; 1460 1461 case EINTR: 1462 printf("aborted from console\n"); 1463 break; 1464 1465 case 0: 1466 printf("succeeded\n"); 1467 break; 1468 1469 default: 1470 printf("error %d\n", error); 1471 break; 1472 } 1473 failed: 1474 printf("\n\n"); 1475 delay(5000000); /* 5 seconds */ 1476 } 1477 1478 /* 1479 * This is called by main to set dumplo and dumpsize. 1480 * Dumps always skip the first PAGE_SIZE of disk space 1481 * in case there might be a disk label stored there. 1482 * If there is extra space, put dump at the end to 1483 * reduce the chance that swapping trashes it. 1484 * 1485 * Sparse dumps can't placed as close to the end as possible, because 1486 * savecore(8) has to know where to start reading in the dump device 1487 * before it has access to any of the crashed system's state. 1488 * 1489 * Note also that a sparse dump will never be larger than a full one: 1490 * in order to add a phys_ram_seg_t to the header, at least one page 1491 * must be removed. 1492 */ 1493 void 1494 cpu_dumpconf(void) 1495 { 1496 int nblks, dumpblks; /* size of dump area */ 1497 1498 if (dumpdev == NODEV) 1499 goto bad; 1500 nblks = bdev_size(dumpdev); 1501 if (nblks <= ctod(1)) 1502 goto bad; 1503 1504 dumpblks = cpu_dumpsize(); 1505 if (dumpblks < 0) 1506 goto bad; 1507 1508 /* dumpsize is in page units, and doesn't include headers. */ 1509 dumpsize = cpu_dump_mempagecnt(); 1510 1511 dumpblks += ctod(dumpsize); 1512 1513 /* If dump won't fit (incl. room for possible label), punt. */ 1514 if (dumpblks > (nblks - ctod(1))) { 1515 #ifndef NO_SPARSE_DUMP 1516 /* A sparse dump might (and hopefully will) fit. */ 1517 dumplo = ctod(1); 1518 #else 1519 /* But if we're not configured for that, punt. */ 1520 goto bad; 1521 #endif 1522 } else { 1523 /* Put dump at end of partition */ 1524 dumplo = nblks - dumpblks; 1525 } 1526 1527 1528 /* Now that we've decided this will work, init ancillary stuff. */ 1529 dump_misc_init(); 1530 return; 1531 1532 bad: 1533 dumpsize = 0; 1534 } 1535 1536 /* 1537 * Clear registers on exec 1538 */ 1539 void 1540 setregs(struct lwp *l, struct exec_package *pack, vaddr_t stack) 1541 { 1542 struct pcb *pcb = lwp_getpcb(l); 1543 struct trapframe *tf; 1544 1545 #ifdef USER_LDT 1546 pmap_ldt_cleanup(l); 1547 #endif 1548 1549 fpu_clear(l, pack->ep_osversion >= 699002600 1550 ? __NetBSD_NPXCW__ : __NetBSD_COMPAT_NPXCW__); 1551 x86_dbregs_clear(l); 1552 1553 kpreempt_disable(); 1554 pcb->pcb_flags = 0; 1555 l->l_proc->p_flag &= ~PK_32; 1556 l->l_md.md_flags = MDL_IRET; 1557 cpu_segregs64_zero(l); 1558 kpreempt_enable(); 1559 1560 tf = l->l_md.md_regs; 1561 memset(tf, 0, sizeof(*tf)); 1562 1563 tf->tf_trapno = T_ASTFLT; 1564 tf->tf_ds = GSEL(GUDATA_SEL, SEL_UPL); 1565 tf->tf_es = GSEL(GUDATA_SEL, SEL_UPL); 1566 tf->tf_rdi = 0; 1567 tf->tf_rsi = 0; 1568 tf->tf_rbp = 0; 1569 tf->tf_rbx = l->l_proc->p_psstrp; 1570 tf->tf_rdx = 0; 1571 tf->tf_rcx = 0; 1572 tf->tf_rax = 0; 1573 tf->tf_rip = pack->ep_entry; 1574 tf->tf_cs = LSEL(LUCODE_SEL, SEL_UPL); 1575 tf->tf_rflags = PSL_USERSET; 1576 tf->tf_rsp = stack; 1577 tf->tf_ss = LSEL(LUDATA_SEL, SEL_UPL); 1578 } 1579 1580 /* 1581 * Initialize segments and descriptor tables 1582 */ 1583 char *ldtstore; 1584 char *gdtstore; 1585 1586 void 1587 setgate(struct gate_descriptor *gd, void *func, 1588 int ist, int type, int dpl, int sel) 1589 { 1590 vaddr_t vaddr; 1591 1592 vaddr = ((vaddr_t)gd) & ~PAGE_MASK; 1593 1594 kpreempt_disable(); 1595 pmap_changeprot_local(vaddr, VM_PROT_READ|VM_PROT_WRITE); 1596 1597 gd->gd_looffset = (uint64_t)func & 0xffff; 1598 gd->gd_selector = sel; 1599 gd->gd_ist = ist; 1600 gd->gd_type = type; 1601 gd->gd_dpl = dpl; 1602 gd->gd_p = 1; 1603 gd->gd_hioffset = (uint64_t)func >> 16; 1604 gd->gd_zero = 0; 1605 gd->gd_xx1 = 0; 1606 gd->gd_xx2 = 0; 1607 gd->gd_xx3 = 0; 1608 1609 pmap_changeprot_local(vaddr, VM_PROT_READ); 1610 kpreempt_enable(); 1611 } 1612 1613 void 1614 unsetgate(struct gate_descriptor *gd) 1615 { 1616 vaddr_t vaddr; 1617 1618 vaddr = ((vaddr_t)gd) & ~PAGE_MASK; 1619 1620 kpreempt_disable(); 1621 pmap_changeprot_local(vaddr, VM_PROT_READ|VM_PROT_WRITE); 1622 1623 memset(gd, 0, sizeof (*gd)); 1624 1625 pmap_changeprot_local(vaddr, VM_PROT_READ); 1626 kpreempt_enable(); 1627 } 1628 1629 void 1630 setregion(struct region_descriptor *rd, void *base, uint16_t limit) 1631 { 1632 rd->rd_limit = limit; 1633 rd->rd_base = (uint64_t)base; 1634 } 1635 1636 /* 1637 * Note that the base and limit fields are ignored in long mode. 1638 */ 1639 void 1640 set_mem_segment(struct mem_segment_descriptor *sd, void *base, size_t limit, 1641 int type, int dpl, int gran, int def32, int is64) 1642 { 1643 sd->sd_lolimit = (unsigned)limit; 1644 sd->sd_lobase = (unsigned long)base; 1645 sd->sd_type = type; 1646 sd->sd_dpl = dpl; 1647 sd->sd_p = 1; 1648 sd->sd_hilimit = (unsigned)limit >> 16; 1649 sd->sd_avl = 0; 1650 sd->sd_long = is64; 1651 sd->sd_def32 = def32; 1652 sd->sd_gran = gran; 1653 sd->sd_hibase = (unsigned long)base >> 24; 1654 } 1655 1656 void 1657 set_sys_segment(struct sys_segment_descriptor *sd, void *base, size_t limit, 1658 int type, int dpl, int gran) 1659 { 1660 memset(sd, 0, sizeof *sd); 1661 sd->sd_lolimit = (unsigned)limit; 1662 sd->sd_lobase = (uint64_t)base; 1663 sd->sd_type = type; 1664 sd->sd_dpl = dpl; 1665 sd->sd_p = 1; 1666 sd->sd_hilimit = (unsigned)limit >> 16; 1667 sd->sd_gran = gran; 1668 sd->sd_hibase = (uint64_t)base >> 24; 1669 } 1670 1671 void 1672 cpu_init_idt(struct cpu_info *ci) 1673 { 1674 struct region_descriptor region; 1675 idt_descriptor_t *idt; 1676 1677 idt = ci->ci_idtvec.iv_idt; 1678 setregion(®ion, idt, NIDT * sizeof(idt[0]) - 1); 1679 lidt(®ion); 1680 } 1681 1682 #define IDTVEC(name) __CONCAT(X, name) 1683 typedef void (vector)(void); 1684 extern vector IDTVEC(syscall); 1685 extern vector IDTVEC(syscall32); 1686 extern vector IDTVEC(osyscall); 1687 extern vector *x86_exceptions[]; 1688 1689 #ifndef XENPV 1690 static void 1691 init_x86_64_ksyms(void) 1692 { 1693 #if NKSYMS || defined(DDB) || defined(MODULAR) 1694 extern int end; 1695 extern int *esym; 1696 struct btinfo_symtab *symtab; 1697 vaddr_t tssym, tesym; 1698 1699 #ifdef DDB 1700 db_machine_init(); 1701 #endif 1702 1703 symtab = lookup_bootinfo(BTINFO_SYMTAB); 1704 if (symtab) { 1705 #ifdef KASLR 1706 tssym = bootspace.head.va; 1707 tesym = bootspace.head.va; /* (unused...) */ 1708 #else 1709 tssym = (vaddr_t)symtab->ssym + KERNBASE; 1710 tesym = (vaddr_t)symtab->esym + KERNBASE; 1711 #endif 1712 ksyms_addsyms_elf(symtab->nsym, (void *)tssym, (void *)tesym); 1713 } else { 1714 uintptr_t endp = (uintptr_t)(void *)&end; 1715 #ifdef XEN 1716 /* 1717 * cpu_probe() / identify_hypervisor() overrides VM_GUEST_GENPVH, 1718 * we can't rely on vm_guest == VM_GUEST_GENPVH 1719 */ 1720 if (pvh_boot && vm_guest != VM_GUEST_XENPVH) 1721 ksyms_addsyms_elf(0, ((long *)endp) + 1, esym); 1722 else 1723 #endif 1724 ksyms_addsyms_elf(*(long *)endp, ((long *)endp) + 1, esym); 1725 } 1726 #endif 1727 } 1728 #endif /* XENPV */ 1729 1730 void __noasan 1731 init_bootspace(void) 1732 { 1733 extern char __rodata_start; 1734 extern char __data_start; 1735 extern char __kernel_end; 1736 size_t i = 0; 1737 1738 memset(&bootspace, 0, sizeof(bootspace)); 1739 1740 bootspace.head.va = KERNTEXTOFF; 1741 bootspace.head.pa = KERNTEXTOFF - KERNBASE; 1742 bootspace.head.sz = 0; 1743 1744 bootspace.segs[i].type = BTSEG_TEXT; 1745 bootspace.segs[i].va = KERNTEXTOFF; 1746 bootspace.segs[i].pa = KERNTEXTOFF - KERNBASE; 1747 bootspace.segs[i].sz = (size_t)&__rodata_start - KERNTEXTOFF; 1748 i++; 1749 1750 bootspace.segs[i].type = BTSEG_RODATA; 1751 bootspace.segs[i].va = (vaddr_t)&__rodata_start; 1752 bootspace.segs[i].pa = (paddr_t)&__rodata_start - KERNBASE; 1753 bootspace.segs[i].sz = (size_t)&__data_start - (size_t)&__rodata_start; 1754 i++; 1755 1756 bootspace.segs[i].type = BTSEG_DATA; 1757 bootspace.segs[i].va = (vaddr_t)&__data_start; 1758 bootspace.segs[i].pa = (paddr_t)&__data_start - KERNBASE; 1759 bootspace.segs[i].sz = (size_t)&__kernel_end - (size_t)&__data_start; 1760 i++; 1761 1762 bootspace.boot.va = (vaddr_t)&__kernel_end; 1763 bootspace.boot.pa = (paddr_t)&__kernel_end - KERNBASE; 1764 bootspace.boot.sz = (size_t)(atdevbase + IOM_SIZE) - 1765 (size_t)&__kernel_end; 1766 1767 /* In locore.S, we allocated a tmp va. We will use it now. */ 1768 bootspace.spareva = KERNBASE + NKL2_KIMG_ENTRIES * NBPD_L2; 1769 1770 /* Virtual address of the L4 page. */ 1771 bootspace.pdir = (vaddr_t)(PDPpaddr + KERNBASE); 1772 1773 /* Kernel module map. */ 1774 bootspace.smodule = (vaddr_t)atdevbase + IOM_SIZE; 1775 bootspace.emodule = KERNBASE + NKL2_KIMG_ENTRIES * NBPD_L2; 1776 } 1777 1778 static void 1779 init_pte(void) 1780 { 1781 #ifndef XENPV 1782 extern uint32_t nox_flag; 1783 pd_entry_t *pdir = (pd_entry_t *)bootspace.pdir; 1784 pdir[L4_SLOT_PTE] = PDPpaddr | PTE_W | ((uint64_t)nox_flag << 32) | 1785 PTE_P; 1786 #endif 1787 1788 extern pd_entry_t *normal_pdes[3]; 1789 normal_pdes[0] = L2_BASE; 1790 normal_pdes[1] = L3_BASE; 1791 normal_pdes[2] = L4_BASE; 1792 } 1793 1794 void 1795 init_slotspace(void) 1796 { 1797 /* 1798 * XXX Too early to use cprng(9), or even entropy_extract. 1799 */ 1800 struct entpool pool; 1801 size_t randhole; 1802 vaddr_t randva; 1803 uint64_t sample; 1804 vaddr_t va; 1805 1806 memset(&pool, 0, sizeof pool); 1807 cpu_rng_early_sample(&sample); 1808 entpool_enter(&pool, &sample, sizeof sample); 1809 1810 memset(&slotspace, 0, sizeof(slotspace)); 1811 1812 /* User. [256, because we want to land in >= 256] */ 1813 slotspace.area[SLAREA_USER].sslot = 0; 1814 slotspace.area[SLAREA_USER].nslot = PDIR_SLOT_USERLIM+1; 1815 slotspace.area[SLAREA_USER].active = true; 1816 1817 #ifdef XENPV 1818 /* PTE. */ 1819 slotspace.area[SLAREA_PTE].sslot = PDIR_SLOT_PTE; 1820 slotspace.area[SLAREA_PTE].nslot = 1; 1821 slotspace.area[SLAREA_PTE].active = true; 1822 #endif 1823 1824 #ifdef __HAVE_PCPU_AREA 1825 /* Per-CPU. */ 1826 slotspace.area[SLAREA_PCPU].sslot = PDIR_SLOT_PCPU; 1827 slotspace.area[SLAREA_PCPU].nslot = 1; 1828 slotspace.area[SLAREA_PCPU].active = true; 1829 #endif 1830 1831 #ifdef __HAVE_DIRECT_MAP 1832 /* Direct Map. [Randomized later] */ 1833 slotspace.area[SLAREA_DMAP].active = false; 1834 #endif 1835 1836 #ifdef XENPV 1837 /* Hypervisor. */ 1838 slotspace.area[SLAREA_HYPV].sslot = 256; 1839 slotspace.area[SLAREA_HYPV].nslot = 17; 1840 slotspace.area[SLAREA_HYPV].active = true; 1841 #endif 1842 1843 #ifdef KASAN 1844 /* ASAN. */ 1845 slotspace.area[SLAREA_ASAN].sslot = L4_SLOT_KASAN; 1846 slotspace.area[SLAREA_ASAN].nslot = NL4_SLOT_KASAN; 1847 slotspace.area[SLAREA_ASAN].active = true; 1848 #endif 1849 1850 #ifdef KMSAN 1851 /* MSAN. */ 1852 slotspace.area[SLAREA_MSAN].sslot = L4_SLOT_KMSAN; 1853 slotspace.area[SLAREA_MSAN].nslot = NL4_SLOT_KMSAN; 1854 slotspace.area[SLAREA_MSAN].active = true; 1855 #endif 1856 1857 /* Kernel. */ 1858 slotspace.area[SLAREA_KERN].sslot = L4_SLOT_KERNBASE; 1859 slotspace.area[SLAREA_KERN].nslot = 1; 1860 slotspace.area[SLAREA_KERN].active = true; 1861 1862 /* Main. */ 1863 cpu_rng_early_sample(&sample); 1864 entpool_enter(&pool, &sample, sizeof sample); 1865 entpool_extract(&pool, &randhole, sizeof randhole); 1866 entpool_extract(&pool, &randva, sizeof randva); 1867 va = slotspace_rand(SLAREA_MAIN, NKL4_MAX_ENTRIES * NBPD_L4, 1868 NBPD_L4, randhole, randva); /* TODO: NBPD_L1 */ 1869 vm_min_kernel_address = va; 1870 vm_max_kernel_address = va + NKL4_MAX_ENTRIES * NBPD_L4; 1871 1872 #ifndef XENPV 1873 /* PTE. */ 1874 cpu_rng_early_sample(&sample); 1875 entpool_enter(&pool, &sample, sizeof sample); 1876 entpool_extract(&pool, &randhole, sizeof randhole); 1877 entpool_extract(&pool, &randva, sizeof randva); 1878 va = slotspace_rand(SLAREA_PTE, NBPD_L4, NBPD_L4, randhole, randva); 1879 pte_base = (pd_entry_t *)va; 1880 #endif 1881 1882 explicit_memset(&pool, 0, sizeof pool); 1883 } 1884 1885 void 1886 init_x86_64(paddr_t first_avail) 1887 { 1888 extern void consinit(void); 1889 struct region_descriptor region; 1890 struct mem_segment_descriptor *ldt_segp; 1891 struct idt_vec *iv; 1892 idt_descriptor_t *idt; 1893 int x; 1894 struct pcb *pcb; 1895 extern vaddr_t lwp0uarea; 1896 #ifndef XENPV 1897 extern paddr_t local_apic_pa; 1898 #endif 1899 1900 KASSERT(first_avail % PAGE_SIZE == 0); 1901 1902 #ifdef XENPV 1903 KASSERT(HYPERVISOR_shared_info != NULL); 1904 cpu_info_primary.ci_vcpu = &HYPERVISOR_shared_info->vcpu_info[0]; 1905 #endif 1906 1907 #ifdef XEN 1908 if (pvh_boot) 1909 xen_parse_cmdline(XEN_PARSE_BOOTFLAGS, NULL); 1910 #endif 1911 init_pte(); 1912 1913 uvm_lwp_setuarea(&lwp0, lwp0uarea); 1914 1915 cpu_probe(&cpu_info_primary); 1916 #ifdef SVS 1917 svs_init(); 1918 #endif 1919 1920 /* 1921 * Initialize MSRs on cpu0: 1922 * 1923 * - Enables SYSCALL/SYSRET. 1924 * 1925 * - Sets up %fs and %gs so that %gs points to the current 1926 * struct cpu_info as needed for CPUVAR(...), curcpu(), and 1927 * curlwp. 1928 * 1929 * - Enables the no-execute bit if supported. 1930 * 1931 * Thus, after this point, CPUVAR(...), curcpu(), and curlwp 1932 * will work on cpu0. 1933 * 1934 * Note: The call to cpu_init_msrs for secondary CPUs happens 1935 * in cpu_hatch. 1936 */ 1937 cpu_init_msrs(&cpu_info_primary, true); 1938 1939 #ifndef XENPV 1940 cpu_speculation_init(&cpu_info_primary); 1941 #endif 1942 1943 use_pae = 1; /* PAE always enabled in long mode */ 1944 1945 pcb = lwp_getpcb(&lwp0); 1946 #ifdef XENPV 1947 mutex_init(&pte_lock, MUTEX_DEFAULT, IPL_VM); 1948 pcb->pcb_cr3 = xen_start_info.pt_base - KERNBASE; 1949 #else 1950 pcb->pcb_cr3 = PDPpaddr; 1951 #endif 1952 1953 #if NISA > 0 || NPCI > 0 1954 x86_bus_space_init(); 1955 #endif 1956 1957 pat_init(&cpu_info_primary); 1958 1959 consinit(); /* XXX SHOULD NOT BE DONE HERE */ 1960 1961 /* 1962 * Initialize PAGE_SIZE-dependent variables. 1963 */ 1964 uvm_md_init(); 1965 1966 uvmexp.ncolors = 2; 1967 1968 avail_start = first_avail; 1969 1970 #ifndef XENPV 1971 /* 1972 * Low memory reservations: 1973 * Page 0: BIOS data 1974 * Page 1: BIOS callback (not used yet, for symmetry with i386) 1975 * Page 2: MP bootstrap code (MP_TRAMPOLINE) 1976 * Page 3: ACPI wakeup code (ACPI_WAKEUP_ADDR) 1977 * Page 4: Temporary page table for 0MB-4MB 1978 * Page 5: Temporary page directory 1979 * Page 6: Temporary page map level 3 1980 * Page 7: Temporary page map level 4 1981 */ 1982 lowmem_rsvd = 8 * PAGE_SIZE; 1983 1984 /* Initialize the memory clusters (needed in pmap_bootstrap). */ 1985 init_x86_clusters(); 1986 #else 1987 /* Parse Xen command line (replace bootinfo) */ 1988 xen_parse_cmdline(XEN_PARSE_BOOTFLAGS, NULL); 1989 1990 avail_end = ctob(xen_start_info.nr_pages); 1991 pmap_pa_start = (KERNTEXTOFF - KERNBASE); 1992 pmap_pa_end = avail_end; 1993 #endif 1994 1995 /* 1996 * Call pmap initialization to make new kernel address space. 1997 * We must do this before loading pages into the VM system. 1998 */ 1999 pmap_bootstrap(VM_MIN_KERNEL_ADDRESS); 2000 2001 /* 2002 * Initialize RNG to get entropy ASAP either from CPU 2003 * RDRAND/RDSEED or from seed on disk. Constraints: 2004 * 2005 * - Must happen after cpu_init_msrs so that curcpu() and 2006 * curlwp work. 2007 * 2008 * - Must happen after consinit so we have the opportunity to 2009 * print useful feedback. 2010 * 2011 * - On KASLR kernels, must happen after pmap_bootstrap because 2012 * x86_rndseed requires access to the direct map. 2013 */ 2014 cpu_rng_init(); 2015 x86_rndseed(); 2016 2017 #ifndef XENPV 2018 /* Internalize the physical pages into the VM system. */ 2019 init_x86_vm(avail_start); 2020 #else 2021 physmem = xen_start_info.nr_pages; 2022 uvm_page_physload(atop(avail_start), atop(avail_end), 2023 atop(avail_start), atop(avail_end), VM_FREELIST_DEFAULT); 2024 #endif 2025 2026 init_x86_msgbuf(); 2027 2028 kasan_init(); 2029 kcsan_init(); 2030 kmsan_init((void *)lwp0uarea); 2031 2032 pmap_growkernel(VM_MIN_KERNEL_ADDRESS + 32 * 1024 * 1024); 2033 2034 kpreempt_disable(); 2035 2036 #ifndef XENPV 2037 pmap_kenter_pa(local_apic_va, local_apic_pa, 2038 VM_PROT_READ|VM_PROT_WRITE, 0); 2039 pmap_update(pmap_kernel()); 2040 memset((void *)local_apic_va, 0, PAGE_SIZE); 2041 #endif 2042 2043 pmap_kenter_pa(idt_vaddr, idt_paddr, VM_PROT_READ|VM_PROT_WRITE, 0); 2044 pmap_kenter_pa(gdt_vaddr, gdt_paddr, VM_PROT_READ|VM_PROT_WRITE, 0); 2045 pmap_kenter_pa(ldt_vaddr, ldt_paddr, VM_PROT_READ|VM_PROT_WRITE, 0); 2046 pmap_update(pmap_kernel()); 2047 memset((void *)idt_vaddr, 0, PAGE_SIZE); 2048 memset((void *)gdt_vaddr, 0, PAGE_SIZE); 2049 memset((void *)ldt_vaddr, 0, PAGE_SIZE); 2050 2051 #ifndef XENPV 2052 pmap_changeprot_local(idt_vaddr, VM_PROT_READ); 2053 #endif 2054 2055 pmap_update(pmap_kernel()); 2056 2057 iv = &(cpu_info_primary.ci_idtvec); 2058 idt_vec_init_cpu_md(iv, cpu_index(&cpu_info_primary)); 2059 idt = iv->iv_idt; 2060 gdtstore = (char *)gdt_vaddr; 2061 ldtstore = (char *)ldt_vaddr; 2062 2063 /* 2064 * Make GDT gates and memory segments. 2065 */ 2066 set_mem_segment(GDT_ADDR_MEM(gdtstore, GCODE_SEL), 0, 2067 0xfffff, SDT_MEMERA, SEL_KPL, 1, 0, 1); 2068 2069 set_mem_segment(GDT_ADDR_MEM(gdtstore, GDATA_SEL), 0, 2070 0xfffff, SDT_MEMRWA, SEL_KPL, 1, 0, 1); 2071 2072 set_mem_segment(GDT_ADDR_MEM(gdtstore, GUCODE_SEL), 0, 2073 x86_btop(VM_MAXUSER_ADDRESS) - 1, SDT_MEMERA, SEL_UPL, 1, 0, 1); 2074 2075 set_mem_segment(GDT_ADDR_MEM(gdtstore, GUDATA_SEL), 0, 2076 x86_btop(VM_MAXUSER_ADDRESS) - 1, SDT_MEMRWA, SEL_UPL, 1, 0, 1); 2077 2078 #ifndef XENPV 2079 set_sys_segment(GDT_ADDR_SYS(gdtstore, GLDT_SEL), ldtstore, 2080 LDT_SIZE - 1, SDT_SYSLDT, SEL_KPL, 0); 2081 #endif 2082 2083 /* 2084 * Make LDT memory segments. 2085 */ 2086 *(struct mem_segment_descriptor *)(ldtstore + LUCODE_SEL) = 2087 *GDT_ADDR_MEM(gdtstore, GUCODE_SEL); 2088 *(struct mem_segment_descriptor *)(ldtstore + LUDATA_SEL) = 2089 *GDT_ADDR_MEM(gdtstore, GUDATA_SEL); 2090 2091 /* 2092 * 32 bit GDT entries. 2093 */ 2094 set_mem_segment(GDT_ADDR_MEM(gdtstore, GUCODE32_SEL), 0, 2095 x86_btop(VM_MAXUSER_ADDRESS32) - 1, SDT_MEMERA, SEL_UPL, 1, 1, 0); 2096 2097 set_mem_segment(GDT_ADDR_MEM(gdtstore, GUDATA32_SEL), 0, 2098 x86_btop(VM_MAXUSER_ADDRESS32) - 1, SDT_MEMRWA, SEL_UPL, 1, 1, 0); 2099 2100 set_mem_segment(GDT_ADDR_MEM(gdtstore, GUFS_SEL), 0, 2101 x86_btop(VM_MAXUSER_ADDRESS32) - 1, SDT_MEMRWA, SEL_UPL, 1, 1, 0); 2102 2103 set_mem_segment(GDT_ADDR_MEM(gdtstore, GUGS_SEL), 0, 2104 x86_btop(VM_MAXUSER_ADDRESS32) - 1, SDT_MEMRWA, SEL_UPL, 1, 1, 0); 2105 2106 /* 2107 * 32 bit LDT entries. 2108 */ 2109 ldt_segp = (struct mem_segment_descriptor *)(ldtstore + LUCODE32_SEL); 2110 set_mem_segment(ldt_segp, 0, x86_btop(VM_MAXUSER_ADDRESS32) - 1, 2111 SDT_MEMERA, SEL_UPL, 1, 1, 0); 2112 ldt_segp = (struct mem_segment_descriptor *)(ldtstore + LUDATA32_SEL); 2113 set_mem_segment(ldt_segp, 0, x86_btop(VM_MAXUSER_ADDRESS32) - 1, 2114 SDT_MEMRWA, SEL_UPL, 1, 1, 0); 2115 2116 /* CPU-specific IDT exceptions. */ 2117 for (x = 0; x < NCPUIDT; x++) { 2118 int sel, ist; 2119 2120 /* Reset to default. Special cases below */ 2121 sel = SEL_KPL; 2122 ist = 0; 2123 2124 idt_vec_reserve(iv, x); 2125 2126 switch (x) { 2127 case 1: /* DB */ 2128 ist = 4; 2129 break; 2130 case 2: /* NMI */ 2131 ist = 3; 2132 break; 2133 case 3: 2134 case 4: 2135 sel = SEL_UPL; 2136 break; 2137 case 8: /* double fault */ 2138 ist = 2; 2139 break; 2140 #ifdef XENPV 2141 case 18: /* MCA */ 2142 sel |= 0x4; /* Auto EOI/mask */ 2143 break; 2144 #endif /* XENPV */ 2145 default: 2146 break; 2147 } 2148 2149 set_idtgate(&idt[x], x86_exceptions[x], ist, SDT_SYS386IGT, 2150 sel, GSEL(GCODE_SEL, SEL_KPL)); 2151 } 2152 2153 /* new-style interrupt gate for syscalls */ 2154 idt_vec_reserve(iv, 128); 2155 set_idtgate(&idt[128], &IDTVEC(osyscall), 0, SDT_SYS386IGT, SEL_UPL, 2156 GSEL(GCODE_SEL, SEL_KPL)); 2157 2158 kpreempt_enable(); 2159 2160 setregion(®ion, gdtstore, DYNSEL_START - 1); 2161 lgdt(®ion); 2162 2163 #ifdef XENPV 2164 /* Init Xen callbacks and syscall handlers */ 2165 if (HYPERVISOR_set_callbacks( 2166 (unsigned long) hypervisor_callback, 2167 (unsigned long) failsafe_callback, 2168 (unsigned long) Xsyscall)) 2169 panic("HYPERVISOR_set_callbacks() failed"); 2170 #endif /* XENPV */ 2171 2172 cpu_init_idt(&cpu_info_primary); 2173 2174 #ifdef XENPV 2175 xen_init_ksyms(); 2176 #else /* XENPV */ 2177 #ifdef XEN 2178 if (vm_guest == VM_GUEST_XENPVH) 2179 xen_init_ksyms(); 2180 else 2181 #endif /* XEN */ 2182 init_x86_64_ksyms(); 2183 #endif /* XENPV */ 2184 2185 #ifndef XENPV 2186 intr_default_setup(); 2187 #else 2188 events_default_setup(); 2189 #endif 2190 2191 splraise(IPL_HIGH); 2192 x86_enable_intr(); 2193 2194 #ifdef DDB 2195 if (boothowto & RB_KDB) 2196 Debugger(); 2197 #endif 2198 #ifdef KGDB 2199 kgdb_port_init(); 2200 if (boothowto & RB_KDB) { 2201 kgdb_debug_init = 1; 2202 kgdb_connect(1); 2203 } 2204 #endif 2205 2206 pcb->pcb_dbregs = NULL; 2207 x86_dbregs_init(); 2208 } 2209 2210 void 2211 cpu_reset(void) 2212 { 2213 #ifndef XENPV 2214 idt_descriptor_t *idt; 2215 vaddr_t vaddr; 2216 2217 idt = cpu_info_primary.ci_idtvec.iv_idt; 2218 vaddr = (vaddr_t)idt; 2219 #endif 2220 2221 x86_disable_intr(); 2222 2223 #ifdef XENPV 2224 HYPERVISOR_reboot(); 2225 #else 2226 2227 x86_reset(); 2228 2229 /* 2230 * Try to cause a triple fault and watchdog reset by making the IDT 2231 * invalid and causing a fault. 2232 */ 2233 kpreempt_disable(); 2234 pmap_changeprot_local(vaddr, VM_PROT_READ|VM_PROT_WRITE); 2235 memset((void *)idt, 0, NIDT * sizeof(idt[0])); 2236 kpreempt_enable(); 2237 breakpoint(); 2238 2239 #if 0 2240 /* 2241 * Try to cause a triple fault and watchdog reset by unmapping the 2242 * entire address space and doing a TLB flush. 2243 */ 2244 memset((void *)PTD, 0, PAGE_SIZE); 2245 tlbflush(); 2246 #endif 2247 #endif /* XENPV */ 2248 2249 for (;;); 2250 } 2251 2252 void 2253 cpu_getmcontext(struct lwp *l, mcontext_t *mcp, unsigned int *flags) 2254 { 2255 const struct trapframe *tf = l->l_md.md_regs; 2256 __greg_t ras_rip; 2257 2258 mcp->__gregs[_REG_RDI] = tf->tf_rdi; 2259 mcp->__gregs[_REG_RSI] = tf->tf_rsi; 2260 mcp->__gregs[_REG_RDX] = tf->tf_rdx; 2261 mcp->__gregs[_REG_R10] = tf->tf_r10; 2262 mcp->__gregs[_REG_R8] = tf->tf_r8; 2263 mcp->__gregs[_REG_R9] = tf->tf_r9; 2264 /* argX not touched */ 2265 mcp->__gregs[_REG_RCX] = tf->tf_rcx; 2266 mcp->__gregs[_REG_R11] = tf->tf_r11; 2267 mcp->__gregs[_REG_R12] = tf->tf_r12; 2268 mcp->__gregs[_REG_R13] = tf->tf_r13; 2269 mcp->__gregs[_REG_R14] = tf->tf_r14; 2270 mcp->__gregs[_REG_R15] = tf->tf_r15; 2271 mcp->__gregs[_REG_RBP] = tf->tf_rbp; 2272 mcp->__gregs[_REG_RBX] = tf->tf_rbx; 2273 mcp->__gregs[_REG_RAX] = tf->tf_rax; 2274 mcp->__gregs[_REG_GS] = 0; 2275 mcp->__gregs[_REG_FS] = 0; 2276 mcp->__gregs[_REG_ES] = GSEL(GUDATA_SEL, SEL_UPL); 2277 mcp->__gregs[_REG_DS] = GSEL(GUDATA_SEL, SEL_UPL); 2278 mcp->__gregs[_REG_TRAPNO] = tf->tf_trapno; 2279 mcp->__gregs[_REG_ERR] = tf->tf_err; 2280 mcp->__gregs[_REG_RIP] = tf->tf_rip; 2281 mcp->__gregs[_REG_CS] = LSEL(LUCODE_SEL, SEL_UPL); 2282 mcp->__gregs[_REG_RFLAGS] = tf->tf_rflags; 2283 mcp->__gregs[_REG_RSP] = tf->tf_rsp; 2284 mcp->__gregs[_REG_SS] = LSEL(LUDATA_SEL, SEL_UPL); 2285 2286 if ((ras_rip = (__greg_t)ras_lookup(l->l_proc, 2287 (void *) mcp->__gregs[_REG_RIP])) != -1) 2288 mcp->__gregs[_REG_RIP] = ras_rip; 2289 2290 *flags |= _UC_CPU; 2291 2292 mcp->_mc_tlsbase = (uintptr_t)l->l_private; 2293 *flags |= _UC_TLSBASE; 2294 2295 process_read_fpregs_xmm(l, (struct fxsave *)&mcp->__fpregs.__fxsave); 2296 *flags |= _UC_FPU; 2297 } 2298 2299 /* 2300 * cpu_getmcontext_xsave(l, mcp, flags, xsavebuf, xsavelen, user_xsave) 2301 * 2302 * Copy out xsavebuf[512..xsavelen) to user_xsave[512..xsavelen), 2303 * set mcp to point at it, and set _UC_XSAVE in *flags. Caller: 2304 * 2305 * - must have already initialized the FXSAVE area of mcontext_t, 2306 * - must have already set _UC_FPU in *flags, 2307 * - must have arranged user_xsave[0..512) to overlap with the 2308 * FXSAVE area of mcontext_t, and 2309 * - must subsequently copy out the mcontext_t updated with a 2310 * pointer/length to the XSAVE area. 2311 * 2312 * May fail if the copyout fails. 2313 */ 2314 static int 2315 cpu_getmcontext_xsave(struct lwp *l, mcontext_t *mcp, unsigned int *flags, 2316 const struct xsave_header *xsavebuf, size_t xsavelen, 2317 struct xsave_header *user_xsave) 2318 { 2319 enum { fxsavelen = sizeof(mcp->__fpregs.__fxsave) }; 2320 int error; 2321 2322 CTASSERT(fxsavelen == 512); 2323 2324 KASSERT(*flags & _UC_FPU); 2325 KASSERT(fxsavelen <= xsavelen); 2326 KDASSERT(memcmp(&mcp->__fpregs.__fxsave, xsavebuf, fxsavelen) == 0); 2327 2328 /* 2329 * Copy out the part of the XSAVE area that doesn't overlap. 2330 */ 2331 error = copyout((const char *)xsavebuf + fxsavelen, 2332 (char *)user_xsave + fxsavelen, xsavelen - fxsavelen); 2333 if (error != 0) 2334 return error; 2335 2336 /* 2337 * Record a pointer to the real XSAVE area in the 2338 * architecturally unused bits mcontext_t's FXSAVE area. 2339 */ 2340 mcp->__fpregs.__xsave.__xsaveptr = (__greg_t)(uintptr_t)user_xsave; 2341 mcp->__fpregs.__xsave.__xsavelen = (__greg_t)xsavelen; 2342 2343 /* 2344 * Set the _UC_XSAVE flag so cpu_setmcontext will be able to 2345 * restore the full state from the XSAVE area. 2346 */ 2347 *flags |= _UC_XSAVE; 2348 2349 /* Success! */ 2350 return 0; 2351 } 2352 2353 int 2354 cpu_setmcontext(struct lwp *l, const mcontext_t *mcp, unsigned int flags) 2355 { 2356 struct trapframe *tf = l->l_md.md_regs; 2357 const __greg_t *gr = mcp->__gregs; 2358 struct proc *p = l->l_proc; 2359 struct xsave_header *xsavebuf = NULL; 2360 size_t xsavelen = 0; 2361 int error; 2362 int64_t rflags; 2363 2364 CTASSERT(sizeof (mcontext_t) == 26 * 8 + 8 + 512); 2365 2366 /* 2367 * If there's an external XSAVE area, copy it in and validate 2368 * it before we irreversibly modify the trapframe. 2369 * 2370 * We could check the length against the state components 2371 * included, but we currently don't: if it's truncated, it will 2372 * be as if the truncated part were zero-filled -- this is 2373 * implemented in process_write_xsave, called a little below. 2374 */ 2375 if ((flags & _UC_XSAVE) != 0) { 2376 const struct xsave_header *user_xsave = 2377 (void *)(uintptr_t)mcp->__fpregs.__xsave.__xsaveptr; 2378 2379 xsavelen = mcp->__fpregs.__xsave.__xsavelen; 2380 error = process_verify_xsavelen(l, xsavelen); 2381 if (error != 0) 2382 goto out; 2383 xsavebuf = kmem_alloc(xsavelen, KM_SLEEP); 2384 error = copyin(user_xsave, xsavebuf, xsavelen); 2385 if (error != 0) 2386 goto out; 2387 error = process_verify_xsave(l, xsavebuf, xsavelen); 2388 if (error != 0) 2389 goto out; 2390 } 2391 2392 if ((flags & _UC_CPU) != 0) { 2393 error = cpu_mcontext_validate(l, mcp); 2394 if (error != 0) 2395 goto out; 2396 2397 tf->tf_rdi = gr[_REG_RDI]; 2398 tf->tf_rsi = gr[_REG_RSI]; 2399 tf->tf_rdx = gr[_REG_RDX]; 2400 tf->tf_r10 = gr[_REG_R10]; 2401 tf->tf_r8 = gr[_REG_R8]; 2402 tf->tf_r9 = gr[_REG_R9]; 2403 /* argX not touched */ 2404 tf->tf_rcx = gr[_REG_RCX]; 2405 tf->tf_r11 = gr[_REG_R11]; 2406 tf->tf_r12 = gr[_REG_R12]; 2407 tf->tf_r13 = gr[_REG_R13]; 2408 tf->tf_r14 = gr[_REG_R14]; 2409 tf->tf_r15 = gr[_REG_R15]; 2410 tf->tf_rbp = gr[_REG_RBP]; 2411 tf->tf_rbx = gr[_REG_RBX]; 2412 tf->tf_rax = gr[_REG_RAX]; 2413 tf->tf_gs = 0; 2414 tf->tf_fs = 0; 2415 tf->tf_es = GSEL(GUDATA_SEL, SEL_UPL); 2416 tf->tf_ds = GSEL(GUDATA_SEL, SEL_UPL); 2417 /* trapno, err not touched */ 2418 tf->tf_rip = gr[_REG_RIP]; 2419 tf->tf_cs = LSEL(LUCODE_SEL, SEL_UPL); 2420 rflags = tf->tf_rflags; 2421 rflags &= ~PSL_USER; 2422 tf->tf_rflags = rflags | (gr[_REG_RFLAGS] & PSL_USER); 2423 tf->tf_rsp = gr[_REG_RSP]; 2424 tf->tf_ss = LSEL(LUDATA_SEL, SEL_UPL); 2425 2426 l->l_md.md_flags |= MDL_IRET; 2427 } 2428 2429 if ((flags & _UC_XSAVE) != 0) { 2430 KASSERT(xsavebuf != NULL); 2431 process_write_xsave(l, xsavebuf, xsavelen); 2432 } else if ((flags & _UC_FPU) != 0) { 2433 process_write_fpregs_xmm(l, 2434 (const struct fxsave *)&mcp->__fpregs.__fxsave); 2435 } 2436 2437 if ((flags & _UC_TLSBASE) != 0) 2438 lwp_setprivate(l, (void *)(uintptr_t)mcp->_mc_tlsbase); 2439 2440 mutex_enter(p->p_lock); 2441 if (flags & _UC_SETSTACK) 2442 l->l_sigstk.ss_flags |= SS_ONSTACK; 2443 if (flags & _UC_CLRSTACK) 2444 l->l_sigstk.ss_flags &= ~SS_ONSTACK; 2445 mutex_exit(p->p_lock); 2446 2447 /* Success! */ 2448 error = 0; 2449 2450 out: if (xsavebuf) 2451 kmem_free(xsavebuf, xsavelen); 2452 return error; 2453 } 2454 2455 int 2456 cpu_mcontext_validate(struct lwp *l, const mcontext_t *mcp) 2457 { 2458 struct proc *p __diagused = l->l_proc; 2459 struct trapframe *tf = l->l_md.md_regs; 2460 const __greg_t *gr; 2461 uint16_t sel; 2462 2463 KASSERT((p->p_flag & PK_32) == 0); 2464 gr = mcp->__gregs; 2465 2466 if (((gr[_REG_RFLAGS] ^ tf->tf_rflags) & PSL_USERSTATIC) != 0) 2467 return EINVAL; 2468 2469 sel = gr[_REG_ES] & 0xffff; 2470 if (sel != 0 && !VALID_USER_DSEL(sel)) 2471 return EINVAL; 2472 2473 sel = gr[_REG_FS] & 0xffff; 2474 if (sel != 0 && !VALID_USER_DSEL(sel)) 2475 return EINVAL; 2476 2477 sel = gr[_REG_GS] & 0xffff; 2478 if (sel != 0 && !VALID_USER_DSEL(sel)) 2479 return EINVAL; 2480 2481 sel = gr[_REG_DS] & 0xffff; 2482 if (!VALID_USER_DSEL(sel)) 2483 return EINVAL; 2484 2485 #ifndef XENPV 2486 sel = gr[_REG_SS] & 0xffff; 2487 if (!VALID_USER_DSEL(sel)) 2488 return EINVAL; 2489 2490 sel = gr[_REG_CS] & 0xffff; 2491 if (!VALID_USER_CSEL(sel)) 2492 return EINVAL; 2493 #endif 2494 2495 if (gr[_REG_RIP] >= VM_MAXUSER_ADDRESS) 2496 return EINVAL; 2497 2498 return 0; 2499 } 2500 2501 int 2502 mm_md_kernacc(void *ptr, vm_prot_t prot, bool *handled) 2503 { 2504 const vaddr_t v = (vaddr_t)ptr; 2505 vaddr_t kva, kva_end; 2506 size_t i; 2507 2508 kva = bootspace.head.va; 2509 kva_end = kva + bootspace.head.sz; 2510 if (v >= kva && v < kva_end) { 2511 *handled = true; 2512 return 0; 2513 } 2514 2515 for (i = 0; i < BTSPACE_NSEGS; i++) { 2516 kva = bootspace.segs[i].va; 2517 kva_end = kva + bootspace.segs[i].sz; 2518 if (v < kva || v >= kva_end) 2519 continue; 2520 *handled = true; 2521 if (bootspace.segs[i].type == BTSEG_TEXT || 2522 bootspace.segs[i].type == BTSEG_RODATA) { 2523 if (prot & VM_PROT_WRITE) { 2524 return EFAULT; 2525 } 2526 } 2527 return 0; 2528 } 2529 2530 kva = bootspace.boot.va; 2531 kva_end = kva + bootspace.boot.sz; 2532 if (v >= kva && v < kva_end) { 2533 *handled = true; 2534 return 0; 2535 } 2536 2537 if (v >= bootspace.smodule && v < bootspace.emodule) { 2538 *handled = true; 2539 vm_map_lock_read(module_map); 2540 const bool allowed = 2541 uvm_map_checkprot(module_map, v, v + 1, prot); 2542 vm_map_unlock_read(module_map); 2543 if (!allowed) { 2544 return EFAULT; 2545 } 2546 } else { 2547 *handled = false; 2548 } 2549 return 0; 2550 } 2551 2552 /* 2553 * Zero out a 64bit LWP's segments registers. Used when exec'ing a new 2554 * 64bit program. 2555 */ 2556 void 2557 cpu_segregs64_zero(struct lwp *l) 2558 { 2559 struct trapframe * const tf = l->l_md.md_regs; 2560 struct pcb *pcb; 2561 uint64_t zero = 0; 2562 2563 KASSERT(kpreempt_disabled()); 2564 KASSERT((l->l_proc->p_flag & PK_32) == 0); 2565 KASSERT(l == curlwp); 2566 2567 pcb = lwp_getpcb(l); 2568 2569 tf->tf_fs = 0; 2570 tf->tf_gs = 0; 2571 setds(GSEL(GUDATA_SEL, SEL_UPL)); 2572 setes(GSEL(GUDATA_SEL, SEL_UPL)); 2573 setfs(0); 2574 setusergs(0); 2575 2576 #ifndef XENPV 2577 wrmsr(MSR_FSBASE, 0); 2578 wrmsr(MSR_KERNELGSBASE, 0); 2579 #else 2580 HYPERVISOR_set_segment_base(SEGBASE_FS, 0); 2581 HYPERVISOR_set_segment_base(SEGBASE_GS_USER, 0); 2582 #endif 2583 2584 pcb->pcb_fs = 0; 2585 pcb->pcb_gs = 0; 2586 update_descriptor(&curcpu()->ci_gdt[GUFS_SEL], &zero); 2587 update_descriptor(&curcpu()->ci_gdt[GUGS_SEL], &zero); 2588 } 2589 2590 /* 2591 * Zero out a 32bit LWP's segments registers. Used when exec'ing a new 2592 * 32bit program. 2593 */ 2594 void 2595 cpu_segregs32_zero(struct lwp *l) 2596 { 2597 struct trapframe * const tf = l->l_md.md_regs; 2598 struct pcb *pcb; 2599 uint64_t zero = 0; 2600 2601 KASSERT(kpreempt_disabled()); 2602 KASSERT(l->l_proc->p_flag & PK_32); 2603 KASSERT(l == curlwp); 2604 2605 pcb = lwp_getpcb(l); 2606 2607 tf->tf_fs = 0; 2608 tf->tf_gs = 0; 2609 setds(GSEL(GUDATA32_SEL, SEL_UPL)); 2610 setes(GSEL(GUDATA32_SEL, SEL_UPL)); 2611 setfs(0); 2612 setusergs(0); 2613 pcb->pcb_fs = 0; 2614 pcb->pcb_gs = 0; 2615 update_descriptor(&curcpu()->ci_gdt[GUFS_SEL], &zero); 2616 update_descriptor(&curcpu()->ci_gdt[GUGS_SEL], &zero); 2617 } 2618 2619 /* 2620 * Load an LWP's TLS context, possibly changing the %fs and %gs selectors. 2621 * Used only for 32-bit processes. 2622 */ 2623 void 2624 cpu_fsgs_reload(struct lwp *l, int fssel, int gssel) 2625 { 2626 struct trapframe *tf; 2627 struct pcb *pcb; 2628 2629 KASSERT(l->l_proc->p_flag & PK_32); 2630 KASSERT(l == curlwp); 2631 2632 tf = l->l_md.md_regs; 2633 fssel &= 0xFFFF; 2634 gssel &= 0xFFFF; 2635 2636 pcb = lwp_getpcb(l); 2637 kpreempt_disable(); 2638 update_descriptor(&curcpu()->ci_gdt[GUFS_SEL], &pcb->pcb_fs); 2639 update_descriptor(&curcpu()->ci_gdt[GUGS_SEL], &pcb->pcb_gs); 2640 2641 #ifdef XENPV 2642 setusergs(gssel); 2643 #endif 2644 2645 tf->tf_fs = fssel; 2646 tf->tf_gs = gssel; 2647 kpreempt_enable(); 2648 } 2649 2650 bool 2651 mm_md_direct_mapped_io(void *addr, paddr_t *paddr) 2652 { 2653 vaddr_t va = (vaddr_t)addr; 2654 2655 #ifdef __HAVE_DIRECT_MAP 2656 if (va >= PMAP_DIRECT_BASE && va < PMAP_DIRECT_END) { 2657 *paddr = PMAP_DIRECT_UNMAP(va); 2658 return true; 2659 } 2660 #else 2661 __USE(va); 2662 #endif 2663 2664 return false; 2665 } 2666 2667 bool 2668 mm_md_direct_mapped_phys(paddr_t paddr, vaddr_t *vaddr) 2669 { 2670 #ifdef __HAVE_DIRECT_MAP 2671 *vaddr = PMAP_DIRECT_MAP(paddr); 2672 return true; 2673 #else 2674 return false; 2675 #endif 2676 } 2677 2678 static void 2679 idt_vec_copy(struct idt_vec *dst, struct idt_vec *src) 2680 { 2681 idt_descriptor_t *idt_dst; 2682 2683 idt_dst = dst->iv_idt; 2684 2685 kpreempt_disable(); 2686 pmap_changeprot_local((vaddr_t)idt_dst, VM_PROT_READ|VM_PROT_WRITE); 2687 2688 memcpy(idt_dst, src->iv_idt, PAGE_SIZE); 2689 memcpy(dst->iv_allocmap, src->iv_allocmap, sizeof(dst->iv_allocmap)); 2690 2691 pmap_changeprot_local((vaddr_t)idt_dst, VM_PROT_READ); 2692 kpreempt_enable(); 2693 } 2694 2695 void 2696 idt_vec_init_cpu_md(struct idt_vec *iv, cpuid_t cid) 2697 { 2698 vaddr_t va; 2699 2700 if (cid != cpu_index(&cpu_info_primary) && 2701 idt_vec_is_pcpu()) { 2702 #ifdef __HAVE_PCPU_AREA 2703 va = (vaddr_t)&pcpuarea->ent[cid].idt; 2704 #else 2705 struct vm_page *pg; 2706 2707 va = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, 2708 UVM_KMF_VAONLY); 2709 pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO); 2710 if (pg == NULL) { 2711 panic("failed to allocate a page for IDT"); 2712 } 2713 pmap_kenter_pa(va, VM_PAGE_TO_PHYS(pg), 2714 VM_PROT_READ|VM_PROT_WRITE, 0); 2715 pmap_update(pmap_kernel()); 2716 #endif 2717 2718 memset((void *)va, 0, PAGE_SIZE); 2719 #ifndef XENPV 2720 pmap_changeprot_local(va, VM_PROT_READ); 2721 #endif 2722 pmap_update(pmap_kernel()); 2723 2724 iv->iv_idt = (void *)va; 2725 idt_vec_copy(iv, &(cpu_info_primary.ci_idtvec)); 2726 } else { 2727 iv->iv_idt = (void *)idt_vaddr; 2728 } 2729 } 2730
Indexes created Sun Nov 08 00:25:27 UTC 2026