1 /* $NetBSD: machdep.c,v 1.381 2026/09/19 15:18:30 riastradh Exp $ */ 2 3 /* 4 * Copyright (c) 1996, 1997, 1998, 2000, 2006, 2007, 2008, 2011 5 * The NetBSD Foundation, Inc. 6 * All rights reserved. 7 * 8 * This code is derived from software contributed to The NetBSD Foundation 9 * by Charles M. Hannum and by Jason R. Thorpe of the Numerical Aerospace 10 * Simulation Facility, NASA Ames Research Center. 11 * 12 * This code is derived from software contributed to The NetBSD Foundation 13 * by Coyote Point Systems, Inc. which was written under contract to Coyote 14 * Point by Jed Davis and Devon O'Dell. 15 * 16 * Redistribution and use in source and binary forms, with or without 17 * modification, are permitted provided that the following conditions 18 * are met: 19 * 1. Redistributions of source code must retain the above copyright 20 * notice, this list of conditions and the following disclaimer. 21 * 2. Redistributions in binary form must reproduce the above copyright 22 * notice, this list of conditions and the following disclaimer in the 23 * documentation and/or other materials provided with the distribution. 24 * 25 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 26 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 27 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 28 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 29 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 30 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 31 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 32 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 33 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 34 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 35 * POSSIBILITY OF SUCH DAMAGE. 36 */ 37 38 /* 39 * Copyright (c) 2006 Mathieu Ropert <mro (at) adviseo.fr> 40 * 41 * Permission to use, copy, modify, and distribute this software for any 42 * purpose with or without fee is hereby granted, provided that the above 43 * copyright notice and this permission notice appear in all copies. 44 * 45 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 46 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 47 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 48 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 49 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 50 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 51 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 52 */ 53 54 /* 55 * Copyright (c) 2007 Manuel Bouyer. 56 * 57 * Redistribution and use in source and binary forms, with or without 58 * modification, are permitted provided that the following conditions 59 * are met: 60 * 1. Redistributions of source code must retain the above copyright 61 * notice, this list of conditions and the following disclaimer. 62 * 2. Redistributions in binary form must reproduce the above copyright 63 * notice, this list of conditions and the following disclaimer in the 64 * documentation and/or other materials provided with the distribution. 65 * 66 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 67 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 68 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 69 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 70 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 71 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 72 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 73 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 74 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 75 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 76 */ 77 78 /* 79 * Copyright (c) 1982, 1987, 1990 The Regents of the University of California. 80 * All rights reserved. 81 * 82 * This code is derived from software contributed to Berkeley by 83 * William Jolitz. 84 * 85 * Redistribution and use in source and binary forms, with or without 86 * modification, are permitted provided that the following conditions 87 * are met: 88 * 1. Redistributions of source code must retain the above copyright 89 * notice, this list of conditions and the following disclaimer. 90 * 2. Redistributions in binary form must reproduce the above copyright 91 * notice, this list of conditions and the following disclaimer in the 92 * documentation and/or other materials provided with the distribution. 93 * 3. Neither the name of the University nor the names of its contributors 94 * may be used to endorse or promote products derived from this software 95 * without specific prior written permission. 96 * 97 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 98 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 99 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 100 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 101 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 102 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 103 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 104 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 105 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 106 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 107 * SUCH DAMAGE. 108 * 109 * @(#)machdep.c 7.4 (Berkeley) 6/3/91 110 */ 111 112 #include <sys/cdefs.h> 113 __KERNEL_RCSID(0, "$NetBSD: machdep.c,v 1.381 2026/09/19 15:18:30 riastradh Exp $"); 114 115 #include "opt_modular.h" 116 #include "opt_user_ldt.h" 117 #include "opt_ddb.h" 118 #include "opt_kgdb.h" 119 #include "opt_cpureset_delay.h" 120 #include "opt_mtrr.h" 121 #include "opt_realmem.h" 122 #include "opt_xen.h" 123 #include "opt_svs.h" 124 #include "opt_kaslr.h" 125 #ifndef XENPV 126 #include "opt_physmem.h" 127 #endif 128 #include "isa.h" 129 #include "pci.h" 130 131 #include <sys/param.h> 132 #include <sys/systm.h> 133 #include <sys/signal.h> 134 #include <sys/signalvar.h> 135 #include <sys/kernel.h> 136 #include <sys/cpu.h> 137 #include <sys/exec.h> 138 #include <sys/exec_aout.h> /* for MID_* */ 139 #include <sys/reboot.h> 140 #include <sys/conf.h> 141 #include <sys/msgbuf.h> 142 #include <sys/mount.h> 143 #include <sys/core.h> 144 #include <sys/kcore.h> 145 #include <sys/ucontext.h> 146 #include <machine/kcore.h> 147 #include <sys/ras.h> 148 #include <sys/syscallargs.h> 149 #include <sys/ksyms.h> 150 #include <sys/device.h> 151 #include <sys/lwp.h> 152 #include <sys/proc.h> 153 #include <sys/asan.h> 154 #include <sys/csan.h> 155 #include <sys/msan.h> 156 #include <sys/module.h> 157 #include <sys/timevar.h> 158 159 #ifdef KGDB 160 #include <sys/kgdb.h> 161 #endif 162 163 #include <lib/libkern/entpool.h> /* XXX */ 164 165 #include <dev/cons.h> 166 #include <dev/mm.h> 167 168 #include <uvm/uvm.h> 169 #include <uvm/uvm_page.h> 170 171 #include <sys/sysctl.h> 172 173 #include <machine/cpu.h> 174 #include <machine/cpu_rng.h> 175 #include <machine/cpufunc.h> 176 #include <machine/gdt.h> 177 #include <machine/intr.h> 178 #include <machine/pio.h> 179 #include <machine/psl.h> 180 #include <machine/reg.h> 181 #include <machine/specialreg.h> 182 #include <machine/bootinfo.h> 183 #include <x86/fpu.h> 184 #include <x86/dbregs.h> 185 #include <machine/mtrr.h> 186 #include <machine/mpbiosvar.h> 187 #include <machine/pmap_private.h> 188 189 #include <x86/bootspace.h> 190 #include <x86/cputypes.h> 191 #include <x86/cpuvar.h> 192 #include <x86/machdep.h> 193 #include <x86/x86/tsc.h> 194 195 #include <dev/isa/isareg.h> 196 #include <machine/isa_machdep.h> 197 #include <dev/ic/i8042reg.h> 198 199 #ifdef XEN 200 #include <xen/xen.h> 201 #include <xen/hypervisor.h> 202 #include <xen/evtchn.h> 203 #include <xen/include/public/version.h> 204 #include <xen/include/public/vcpu.h> 205 #endif /* XEN */ 206 207 #include <ddb/db_active.h> 208 209 #ifdef DDB 210 #include <machine/db_machdep.h> 211 #include <ddb/db_extern.h> 212 #include <ddb/db_output.h> 213 #include <ddb/db_interface.h> 214 #endif 215 216 #include "acpica.h" 217 218 #if NACPICA > 0 219 #include <dev/acpi/acpivar.h> 220 #define ACPI_MACHDEP_PRIVATE 221 #include <machine/acpi_machdep.h> 222 #else 223 #include <machine/i82489var.h> 224 #endif 225 226 #include "isa.h" 227 #include "isadma.h" 228 #include "ksyms.h" 229 230 /* the following is used externally (sysctl_hw) */ 231 char machine[] = "amd64"; /* CPU "architecture" */ 232 char machine_arch[] = "x86_64"; /* machine == machine_arch */ 233 234 #ifdef CPURESET_DELAY 235 int cpureset_delay = CPURESET_DELAY; 236 #else 237 int cpureset_delay = 2000; /* default to 2s */ 238 #endif 239 240 int cpu_class = CPUCLASS_686; 241 242 #ifdef MTRR 243 const struct mtrr_funcs *mtrr_funcs; 244 #endif 245 246 int cpu_class; 247 int use_pae; 248 249 #ifndef NO_SPARSE_DUMP 250 int sparse_dump = 1; 251 252 paddr_t max_paddr = 0; 253 unsigned char *sparse_dump_physmap; 254 #endif 255 256 char *dump_headerbuf, *dump_headerbuf_ptr; 257 #define dump_headerbuf_size PAGE_SIZE 258 #define dump_headerbuf_end (dump_headerbuf + dump_headerbuf_size) 259 #define dump_headerbuf_avail (dump_headerbuf_end - dump_headerbuf_ptr) 260 daddr_t dump_header_blkno; 261 262 size_t dump_nmemsegs; 263 size_t dump_npages; 264 size_t dump_header_size; 265 size_t dump_totalbytesleft; 266 267 vaddr_t idt_vaddr; 268 paddr_t idt_paddr; 269 vaddr_t gdt_vaddr; 270 paddr_t gdt_paddr; 271 vaddr_t ldt_vaddr; 272 paddr_t ldt_paddr; 273 274 static struct vm_map module_map_store; 275 extern struct bootspace bootspace; 276 extern struct slotspace slotspace; 277 278 vaddr_t vm_min_kernel_address __read_mostly = VM_MIN_KERNEL_ADDRESS_DEFAULT; 279 vaddr_t vm_max_kernel_address __read_mostly = VM_MAX_KERNEL_ADDRESS_DEFAULT; 280 pd_entry_t *pte_base __read_mostly; 281 282 struct vm_map *phys_map = NULL; 283 284 extern paddr_t lowmem_rsvd; 285 extern paddr_t avail_start, avail_end; 286 #ifdef XENPV 287 extern paddr_t pmap_pa_start, pmap_pa_end; 288 #endif 289 290 struct nmistore { 291 uint64_t cr3; 292 uint64_t scratch; 293 } __packed; 294 295 /* 296 * Size of memory segments, before any memory is stolen. 297 */ 298 phys_ram_seg_t mem_clusters[VM_PHYSSEG_MAX]; 299 int mem_cluster_cnt; 300 301 int cpu_dump(void); 302 int cpu_dumpsize(void); 303 u_long cpu_dump_mempagecnt(void); 304 void dodumpsys(void); 305 void dumpsys(void); 306 307 static void x86_64_proc0_pcb_ldt_init(void); 308 309 void dump_misc_init(void); 310 void dump_seg_prep(void); 311 int dump_seg_iter(int (*)(paddr_t, paddr_t)); 312 313 #ifndef NO_SPARSE_DUMP 314 void sparse_dump_reset(void); 315 void sparse_dump_mark(void); 316 void cpu_dump_prep_sparse(void); 317 #endif 318 319 void dump_header_start(void); 320 int dump_header_flush(void); 321 int dump_header_addbytes(const void*, size_t); 322 int dump_header_addseg(paddr_t, paddr_t); 323 int dump_header_finish(void); 324 325 int dump_seg_count_range(paddr_t, paddr_t); 326 int dumpsys_seg(paddr_t, paddr_t); 327 328 void init_bootspace(void); 329 void init_slotspace(void); 330 void init_x86_64(paddr_t); 331 332 static int cpu_getmcontext_xsave(struct lwp *, mcontext_t *, unsigned *, 333 const struct xsave_header *, size_t, struct xsave_header *); 334 335 /* 336 * Machine-dependent startup code 337 */ 338 void 339 cpu_startup(void) 340 { 341 int x, y; 342 vaddr_t minaddr, maxaddr; 343 psize_t sz; 344 345 /* 346 * For console drivers that require uvm and pmap to be initialized, 347 * we'll give them one more chance here... 348 */ 349 consinit(); 350 351 /* 352 * Initialize error message buffer (at end of core). 353 */ 354 if (msgbuf_p_cnt == 0) 355 panic("msgbuf paddr map has not been set up"); 356 for (x = 0, sz = 0; x < msgbuf_p_cnt; sz += msgbuf_p_seg[x++].sz) 357 continue; 358 359 msgbuf_vaddr = uvm_km_alloc(kernel_map, sz, 0, UVM_KMF_VAONLY); 360 if (msgbuf_vaddr == 0) 361 panic("failed to valloc msgbuf_vaddr"); 362 363 for (y = 0, sz = 0; y < msgbuf_p_cnt; y++) { 364 for (x = 0; x < btoc(msgbuf_p_seg[y].sz); x++, sz += PAGE_SIZE) 365 pmap_kenter_pa((vaddr_t)msgbuf_vaddr + sz, 366 msgbuf_p_seg[y].paddr + x * PAGE_SIZE, 367 VM_PROT_READ|VM_PROT_WRITE, 0); 368 } 369 370 pmap_update(pmap_kernel()); 371 372 initmsgbuf((void *)msgbuf_vaddr, round_page(sz)); 373 374 minaddr = 0; 375 376 /* 377 * Allocate a submap for physio. 378 */ 379 phys_map = uvm_km_suballoc(kernel_map, &minaddr, &maxaddr, 380 VM_PHYS_SIZE, 0, false, NULL); 381 382 /* 383 * Create the module map. 384 * 385 * The kernel uses RIP-relative addressing with a maximum offset of 386 * 2GB. Because of that, we can't put the kernel modules in kernel_map 387 * (like i386 does), since kernel_map is too far away in memory from 388 * the kernel sections. So we have to create a special module_map. 389 * 390 * The module map is taken as what is left of the bootstrap memory 391 * created in locore/prekern. 392 */ 393 uvm_map_setup(&module_map_store, bootspace.smodule, 394 bootspace.emodule, 0); 395 module_map_store.pmap = pmap_kernel(); 396 module_map = &module_map_store; 397 398 /* Say hello. */ 399 banner(); 400 401 #if NISA > 0 || NPCI > 0 402 /* Safe for i/o port / memory space allocation to use malloc now. */ 403 x86_bus_space_mallocok(); 404 #endif 405 406 #ifdef __HAVE_PCPU_AREA 407 cpu_pcpuarea_init(&cpu_info_primary); 408 #endif 409 gdt_init(); 410 x86_64_proc0_pcb_ldt_init(); 411 412 cpu_init_tss(&cpu_info_primary); 413 #if !defined(XENPV) 414 ltr(cpu_info_primary.ci_tss_sel); 415 #endif 416 417 x86_startup(); 418 } 419 420 #ifdef XENPV 421 /* used in assembly */ 422 void hypervisor_callback(void); 423 void failsafe_callback(void); 424 void x86_64_switch_context(struct pcb *); 425 void x86_64_tls_switch(struct lwp *); 426 427 void 428 x86_64_switch_context(struct pcb *new) 429 { 430 HYPERVISOR_stack_switch(GSEL(GDATA_SEL, SEL_KPL), new->pcb_rsp0); 431 struct physdev_set_iopl set_iopl; 432 set_iopl.iopl = new->pcb_iopl; 433 HYPERVISOR_physdev_op(PHYSDEVOP_set_iopl, &set_iopl); 434 } 435 436 void 437 x86_64_tls_switch(struct lwp *l) 438 { 439 struct cpu_info *ci = curcpu(); 440 struct pcb *pcb = lwp_getpcb(l); 441 struct trapframe *tf = l->l_md.md_regs; 442 uint64_t zero = 0; 443 444 /* 445 * Raise the IPL to IPL_HIGH. XXX Still needed? 446 */ 447 (void)splhigh(); 448 449 /* Update segment registers */ 450 if (pcb->pcb_flags & PCB_COMPAT32) { 451 update_descriptor(&ci->ci_gdt[GUFS_SEL], &pcb->pcb_fs); 452 update_descriptor(&ci->ci_gdt[GUGS_SEL], &pcb->pcb_gs); 453 setds(GSEL(GUDATA32_SEL, SEL_UPL)); 454 setes(GSEL(GUDATA32_SEL, SEL_UPL)); 455 setfs(GSEL(GUDATA32_SEL, SEL_UPL)); 456 HYPERVISOR_set_segment_base(SEGBASE_GS_USER_SEL, tf->tf_gs); 457 } else { 458 update_descriptor(&ci->ci_gdt[GUFS_SEL], &zero); 459 update_descriptor(&ci->ci_gdt[GUGS_SEL], &zero); 460 setds(GSEL(GUDATA_SEL, SEL_UPL)); 461 setes(GSEL(GUDATA_SEL, SEL_UPL)); 462 setfs(0); 463 HYPERVISOR_set_segment_base(SEGBASE_GS_USER_SEL, 0); 464 HYPERVISOR_set_segment_base(SEGBASE_FS, pcb->pcb_fs); 465 HYPERVISOR_set_segment_base(SEGBASE_GS_USER, pcb->pcb_gs); 466 } 467 } 468 #endif /* XENPV */ 469 470 /* 471 * Set up proc0's PCB and LDT. 472 */ 473 static void 474 x86_64_proc0_pcb_ldt_init(void) 475 { 476 struct lwp *l = &lwp0; 477 struct pcb *pcb = lwp_getpcb(l); 478 479 pcb->pcb_flags = 0; 480 pcb->pcb_fs = 0; 481 pcb->pcb_gs = 0; 482 pcb->pcb_rsp0 = (uvm_lwp_getuarea(l) + USPACE - 16) & ~0xf; 483 pcb->pcb_iopl = IOPL_KPL; 484 pcb->pcb_dbregs = NULL; 485 pcb->pcb_cr0 = rcr0() & ~CR0_TS; 486 l->l_md.md_regs = (struct trapframe *)pcb->pcb_rsp0 - 1; 487 488 #if !defined(XENPV) 489 lldt(GSYSSEL(GLDT_SEL, SEL_KPL)); 490 #else 491 xen_set_ldt((vaddr_t)ldtstore, LDT_SIZE >> 3); 492 /* Reset TS bit and set kernel stack for interrupt handlers */ 493 HYPERVISOR_fpu_taskswitch(1); 494 HYPERVISOR_stack_switch(GSEL(GDATA_SEL, SEL_KPL), pcb->pcb_rsp0); 495 struct physdev_set_iopl set_iopl; 496 set_iopl.iopl = pcb->pcb_iopl; 497 HYPERVISOR_physdev_op(PHYSDEVOP_set_iopl, &set_iopl); 498 #endif 499 } 500 501 /* 502 * Set up TSS and I/O bitmap. 503 */ 504 void 505 cpu_init_tss(struct cpu_info *ci) 506 { 507 #ifdef __HAVE_PCPU_AREA 508 const cpuid_t cid = cpu_index(ci); 509 #endif 510 struct cpu_tss *cputss; 511 struct nmistore *store; 512 uintptr_t p; 513 514 #ifdef __HAVE_PCPU_AREA 515 cputss = (struct cpu_tss *)&pcpuarea->ent[cid].tss; 516 #else 517 cputss = (struct cpu_tss *)uvm_km_alloc(kernel_map, 518 sizeof(struct cpu_tss), 0, UVM_KMF_WIRED|UVM_KMF_ZERO); 519 #endif 520 521 cputss->tss.tss_iobase = IOMAP_INVALOFF << 16; 522 523 /* DDB stack */ 524 #ifdef __HAVE_PCPU_AREA 525 p = (vaddr_t)&pcpuarea->ent[cid].ist0; 526 #else 527 p = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, UVM_KMF_WIRED|UVM_KMF_ZERO); 528 #endif 529 cputss->tss.tss_ist[0] = p + PAGE_SIZE - 16; 530 531 /* double fault */ 532 #ifdef __HAVE_PCPU_AREA 533 p = (vaddr_t)&pcpuarea->ent[cid].ist1; 534 #else 535 p = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, UVM_KMF_WIRED|UVM_KMF_ZERO); 536 #endif 537 cputss->tss.tss_ist[1] = p + PAGE_SIZE - 16; 538 539 /* NMI - store a structure at the top of the stack */ 540 #ifdef __HAVE_PCPU_AREA 541 p = (vaddr_t)&pcpuarea->ent[cid].ist2; 542 #else 543 p = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, UVM_KMF_WIRED|UVM_KMF_ZERO); 544 #endif 545 cputss->tss.tss_ist[2] = p + PAGE_SIZE - sizeof(struct nmistore); 546 store = (struct nmistore *)(p + PAGE_SIZE - sizeof(struct nmistore)); 547 store->cr3 = pmap_pdirpa(pmap_kernel(), 0); 548 549 /* DB */ 550 #ifdef __HAVE_PCPU_AREA 551 p = (vaddr_t)&pcpuarea->ent[cid].ist3; 552 #else 553 p = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, UVM_KMF_WIRED|UVM_KMF_ZERO); 554 #endif 555 cputss->tss.tss_ist[3] = p + PAGE_SIZE - 16; 556 557 ci->ci_tss = cputss; 558 ci->ci_tss_sel = tss_alloc(&cputss->tss); 559 } 560 561 void 562 buildcontext(struct lwp *l, void *catcher, void *f) 563 { 564 struct trapframe *tf = l->l_md.md_regs; 565 566 tf->tf_ds = GSEL(GUDATA_SEL, SEL_UPL); 567 tf->tf_es = GSEL(GUDATA_SEL, SEL_UPL); 568 tf->tf_fs = GSEL(GUDATA_SEL, SEL_UPL); 569 tf->tf_gs = GSEL(GUDATA_SEL, SEL_UPL); 570 571 tf->tf_rip = (uint64_t)catcher; 572 tf->tf_cs = GSEL(GUCODE_SEL, SEL_UPL); 573 tf->tf_rflags &= ~PSL_CLEARSIG; 574 tf->tf_rsp = (uint64_t)f; 575 tf->tf_ss = GSEL(GUDATA_SEL, SEL_UPL); 576 577 /* Ensure FP state is sane */ 578 fpu_sigreset(l); 579 } 580 581 void 582 sendsig_sigcontext(const ksiginfo_t *ksi, const sigset_t *mask) 583 { 584 585 printf("sendsig_sigcontext: illegal\n"); 586 sigexit(curlwp, SIGILL); 587 } 588 589 /* 590 * sendsig_siginfo(ksi, mask) 591 * 592 * Set up a stack frame and registers for signal delivery with the 593 * given siginfo and signal mask. 594 * 595 * The stack frame layout is as follows, with XSAVE parts omitted 596 * (and _UC_XSAVE clear in uc_flags) if the process doesn't have 597 * XSAVE state to restore: 598 * 599 * / +-----------------------+ 600 * | | (other XSAVE stuff) | 601 * | | --------------------- | 602 * | | Hi16_ZMM | 603 * | | --------------------- | 604 * XSAVE < | ZMM_Hi256 | 605 * | | --------------------- | 606 * | | YMM_Hi128 | 607 * | *-----------------------* \ 608 * | | FXSAVE in ucontext_t | | 609 * \ *-----------------------* > ucontext_t 610 * | rest of ucontext_t | | 611 * +-----------------------+ / 612 * | siginfo_t | 613 * +-----------------------+ =0 (mod 16) 614 * | return address | 615 * +-----------------------+ <--- rsp on signal handler entry 616 * 617 * Additionally, when XSAVE is in use, there is a pointer to the 618 * XSAVE area in some architecturally unused padding in the FXSAVE 619 * area. This is important because on i386, the XSAVE area 620 * _cannot_ overlap the ucontext_t owing to past ABI constraints, 621 * so applications running on both i386 and amd64 can use the same 622 * code to get the XSAVE pointer (and for 11.0, but not any 623 * earlier or later versions, the XSAVE pointer was also needed 624 * for signal handlers to write registers back on amd64 because 625 * the XSAVE area and the ucontext_t did not overlap). 626 * 627 * The overlap of the FXSAVE section in the XSAVE area and the 628 * FXSAVE section in ucontext_t is tricky but important: signal 629 * handlers use it to read, and potentially write back, the x87 630 * and xmm registers. Consider an application with two parts 631 * (e.g., a new library that uses an AVX512 optimization once it 632 * is available, and an old part running unmodified existing code 633 * predating NetBSD's AVX512 support): 634 * 635 * 1. Uses the high 16 zmm registers. 636 * 2. (a) Uses _only_ x87/xmm registers, and 637 * (b) has a signal handler that reads and writes them back. 638 * 639 * After part (1), the system will consider XSAVE necessary to 640 * save and restore the high 16 zmm registers, even if -- as is 641 * customary when using the ymm or low 16 zmm registers -- the 642 * application issues VZEROUPPER when part (1) is done. For this 643 * application to work without changing the signal handler in part 644 * (2)(b) to teach it about the XSAVE pointer -- i.e., for 645 * compatibility with existing code -- the XSAVE area and the 646 * ucontext_t MUST overlap. (11.0 would copy out the FXSAVE area 647 * to both locations so that unmodified signal handlers could 648 * _read_ the x87/xmm registers, but _writeback_ would be 649 * ignored.) 650 * 651 * For more background, see: 652 * 653 * PR kern/60539: XSAVE changes break ucontext userspace API 654 * https://gnats.NetBSD.org/60539 655 */ 656 void 657 sendsig_siginfo(const ksiginfo_t *ksi, const sigset_t *mask) 658 { 659 struct lwp *l = curlwp; 660 struct proc *p = l->l_proc; 661 struct sigacts *ps = p->p_sigacts; 662 int onstack, error; 663 int sig = ksi->ksi_signo; 664 struct sigframe_siginfo *fp, frame; 665 sig_t catcher = SIGACTION(p, sig).sa_handler; 666 struct trapframe *tf = l->l_md.md_regs; 667 const struct xsave_header *xsavebuf = NULL; 668 size_t xsavelen = 0; 669 struct xsave_header *user_xsave = NULL; 670 char *sp; 671 672 KASSERT(mutex_owned(p->p_lock)); 673 674 /* Do we need to jump onto the signal stack? */ 675 onstack = 676 (l->l_sigstk.ss_flags & (SS_DISABLE | SS_ONSTACK)) == 0 && 677 (SIGACTION(p, sig).sa_flags & SA_ONSTACK) != 0; 678 679 /* Allocate space for the signal handler context. */ 680 if (onstack) { 681 KASSERT(l->l_sigstk.ss_size >= MINSIGSTKSZ); 682 sp = ((char *)l->l_sigstk.ss_sp + l->l_sigstk.ss_size); 683 } else { 684 /* AMD64 ABI 128-bytes "red zone". */ 685 sp = (char *)tf->tf_rsp - 128; 686 } 687 688 /* 689 * The maximum amount of space we might use, including padding 690 * for alignment, had better fit in MINSIGSTKSZ. 691 * 692 * If this changes because you have increased XSAVE_MAX_BYTES, 693 * you need to work out the ABI change for MINSIGSTKSZ. 694 */ 695 __CTASSERT(8 + STACK_ALIGNBYTES + sizeof(struct sigframe_siginfo) + 696 (XSAVE_ALIGN - 1) + XSAVE_MAX_BYTES <= MINSIGSTKSZ); 697 698 /* 699 * Find whether we need to allocate a separate XSAVE area, 700 * because the user program has used extended CPU state beyond 701 * the x87/SSE registers, or whether we can get by with just an 702 * FXSAVE area. 703 */ 704 if (process_xsave_needed_p(l)) { 705 enum { 706 overlap = (sizeof(struct sigframe_siginfo) - 707 offsetof(struct sigframe_siginfo, 708 sf_uc.uc_mcontext.__fpregs)), 709 }; 710 711 process_read_xsave(l, &xsavebuf, &xsavelen); 712 KASSERT(xsavebuf != NULL); 713 KASSERT(xsavelen <= XSAVE_MAX_BYTES); 714 CTASSERT(overlap <= XSAVE_MAX_BYTES); 715 716 KASSERT(!onstack || sp >= (char *)l->l_sigstk.ss_sp); 717 KASSERT(!onstack || 718 sp - (char *)l->l_sigstk.ss_sp >= xsavelen); 719 KASSERT(!onstack || 720 sp - (char *)l->l_sigstk.ss_sp >= overlap); 721 sp -= MAX(xsavelen, overlap); 722 723 KASSERT(!onstack || sp >= (char *)l->l_sigstk.ss_sp); 724 KASSERT(!onstack || 725 sp - (char *)l->l_sigstk.ss_sp >= XSAVE_ALIGN - 1); 726 sp = (char *)((uintptr_t)sp & ~(XSAVE_ALIGN - 1)); 727 728 KASSERT(!onstack || sp >= (char *)l->l_sigstk.ss_sp); 729 KASSERT(((uintptr_t)sp & (XSAVE_ALIGN - 1)) == 0); 730 user_xsave = (void *)sp; 731 732 CTASSERT((overlap % XSAVE_ALIGN) == 0); 733 CTASSERT((overlap & STACK_ALIGNBYTES) == 0); 734 CTASSERT(((overlap - sizeof(struct sigframe_siginfo)) & 735 STACK_ALIGNBYTES) == (STACK_ALIGNBYTES + 1) - 8); 736 sp += overlap; 737 } 738 739 /* 740 * Reserve space for a struct sigframe_siginfo. The first 741 * member is a return address, as if we are entering a 742 * procedure with a CALL instruction with a stack frame aligned 743 * to a multiple of 16 bytes; the rest of the structure is that 744 * stack frame. So make sure the structure starts at an 745 * address congruent to 8 modulo 16. 746 */ 747 KASSERT(!onstack || sp >= (char *)l->l_sigstk.ss_sp); 748 KASSERT(!onstack || (size_t)(sp - (char *)l->l_sigstk.ss_sp) >= 749 8 + STACK_ALIGNBYTES + sizeof(struct sigframe_siginfo)); 750 sp -= sizeof(struct sigframe_siginfo); 751 /* Round down the stackpointer to a multiple of 16 for the ABI. */ 752 if (((uintptr_t)sp & STACK_ALIGNBYTES) == 753 (STACK_ALIGNBYTES + 1) - 8) { 754 fp = (struct sigframe_siginfo *)sp; 755 } else { 756 fp = (struct sigframe_siginfo *)(((uintptr_t)sp & 757 ~STACK_ALIGNBYTES) - 8); 758 } 759 KASSERT(!onstack || (char *)fp >= (char *)l->l_sigstk.ss_sp); 760 KASSERT(((uintptr_t)fp & STACK_ALIGNBYTES) == 761 (STACK_ALIGNBYTES + 1) - 8); 762 763 /* 764 * If we have to use XSAVE, the FXSAVE area of the ucontext_t 765 * on the user's stack must line up with the FXSAVE subarea of 766 * the XSAVE area on the user's stack. 767 */ 768 KASSERT(xsavebuf == 0 || 769 (((uintptr_t)&fp->sf_uc.uc_mcontext.__fpregs & (XSAVE_ALIGN - 1)) 770 == 0)); 771 KASSERT(xsavebuf == 0 || (uintptr_t)user_xsave == 772 (uintptr_t)&fp->sf_uc.uc_mcontext.__fpregs); 773 774 memset(&frame, 0, sizeof(frame)); 775 frame.sf_ra = (uint64_t)ps->sa_sigdesc[sig].sd_tramp; 776 frame.sf_si._info = ksi->ksi_info; 777 frame.sf_uc.uc_flags = _UC_SIGMASK; 778 frame.sf_uc.uc_sigmask = *mask; 779 frame.sf_uc.uc_link = l->l_ctxlink; 780 frame.sf_uc.uc_flags |= (l->l_sigstk.ss_flags & SS_ONSTACK) 781 ? _UC_SETSTACK : _UC_CLRSTACK; 782 sendsig_reset(l, sig); 783 784 mutex_exit(p->p_lock); 785 cpu_getmcontext(l, &frame.sf_uc.uc_mcontext, &frame.sf_uc.uc_flags); 786 787 /* 788 * If we have to use XSAVE, copy out the part of it past the 789 * FXSAVE area separately -- and be ready to bail if it failed. 790 */ 791 if (xsavebuf) { 792 KASSERT((void *)&fp->sf_uc.uc_mcontext.__fpregs == 793 (void *)user_xsave); 794 error = cpu_getmcontext_xsave(l, &frame.sf_uc.uc_mcontext, 795 &frame.sf_uc.uc_flags, xsavebuf, xsavelen, user_xsave); 796 if (error != 0) 797 goto relock; 798 } 799 800 /* Copyout all the fp regs, the signal handler might expect them. */ 801 error = copyout(&frame, fp, sizeof frame); 802 relock: 803 mutex_enter(p->p_lock); 804 805 if (error != 0) { 806 /* 807 * Process has trashed its stack; give it an illegal 808 * instruction to halt it in its tracks. 809 */ 810 sigexit(l, SIGILL); 811 /* NOTREACHED */ 812 } 813 814 buildcontext(l, catcher, fp); 815 816 tf->tf_rdi = sig; 817 tf->tf_rsi = (uint64_t)&fp->sf_si; 818 tf->tf_rdx = tf->tf_r15 = (uint64_t)&fp->sf_uc; 819 820 /* Remember that we're now on the signal stack. */ 821 if (onstack) 822 l->l_sigstk.ss_flags |= SS_ONSTACK; 823 824 if ((vaddr_t)catcher >= VM_MAXUSER_ADDRESS) { 825 /* 826 * process has given an invalid address for the 827 * handler. Stop it, but do not do it before so 828 * we can return the right info to userland (or in core dump) 829 */ 830 sigexit(l, SIGILL); 831 /* NOTREACHED */ 832 } 833 } 834 835 struct pcb dumppcb; 836 837 void 838 cpu_reboot(int howto, char *bootstr) 839 { 840 static bool syncdone = false; 841 int s = IPL_NONE; 842 __USE(s); /* ugly otherwise */ 843 844 if (cold) { 845 howto |= RB_HALT; 846 goto haltsys; 847 } 848 849 boothowto = howto; 850 851 /* i386 maybe_dump() */ 852 853 /* 854 * If we've panic'd, don't make the situation potentially 855 * worse by syncing or unmounting the file systems. 856 */ 857 if ((howto & RB_NOSYNC) == 0 && panicstr == NULL) { 858 if (!syncdone) { 859 syncdone = true; 860 /* XXX used to force unmount as well, here */ 861 vfs_sync_all(curlwp); 862 } 863 864 while (vfs_unmountall1(curlwp, false, false) || 865 config_detach_all(boothowto) || 866 vfs_unmount_forceone(curlwp)) 867 ; /* do nothing */ 868 } else { 869 if (!db_active) 870 suspendsched(); 871 } 872 873 pmf_system_shutdown(boothowto); 874 875 /* Disable interrupts. */ 876 s = splhigh(); 877 878 /* Do a dump if requested. */ 879 if ((howto & (RB_DUMP | RB_HALT)) == RB_DUMP) 880 dumpsys(); 881 882 haltsys: 883 doshutdownhooks(); 884 885 if ((howto & RB_POWERDOWN) == RB_POWERDOWN) { 886 #if NACPICA > 0 887 if (s != IPL_NONE) 888 splx(s); 889 890 acpi_enter_sleep_state(ACPI_STATE_S5); 891 #endif 892 #ifdef XEN 893 if (vm_guest == VM_GUEST_XENPV || 894 vm_guest == VM_GUEST_XENPVH || 895 vm_guest == VM_GUEST_XENPVHVM) 896 HYPERVISOR_shutdown(); 897 #endif /* XEN */ 898 } 899 900 cpu_broadcast_halt(); 901 902 if (howto & RB_HALT) { 903 #if NACPICA > 0 904 acpi_disable(); 905 #endif 906 907 printf("\n"); 908 printf("The operating system has halted.\n"); 909 printf("Please press any key to reboot.\n\n"); 910 cnpollc(true); /* for proper keyboard command handling */ 911 if (cngetc() == 0) { 912 /* no console attached, so just hlt */ 913 printf("No keyboard - cannot reboot after all.\n"); 914 for(;;) { 915 x86_hlt(); 916 } 917 } 918 cnpollc(false); 919 } 920 921 printf("rebooting...\n"); 922 if (cpureset_delay > 0) 923 delay(cpureset_delay * 1000); 924 cpu_reset(); 925 for(;;) ; 926 /*NOTREACHED*/ 927 } 928 929 /* 930 * XXXfvdl share dumpcode. 931 */ 932 933 /* 934 * Perform assorted dump-related initialization tasks. Assumes that 935 * the maximum physical memory address will not increase afterwards. 936 */ 937 void 938 dump_misc_init(void) 939 { 940 #ifndef NO_SPARSE_DUMP 941 int i; 942 #endif 943 944 if (dump_headerbuf != NULL) 945 return; /* already called */ 946 947 #ifndef NO_SPARSE_DUMP 948 for (i = 0; i < mem_cluster_cnt; ++i) { 949 paddr_t top = mem_clusters[i].start + mem_clusters[i].size; 950 if (max_paddr < top) 951 max_paddr = top; 952 } 953 #ifdef DEBUG 954 printf("dump_misc_init: max_paddr = 0x%lx\n", 955 (unsigned long)max_paddr); 956 #endif 957 if (max_paddr == 0) { 958 printf("Your machine does not initialize mem_clusters; " 959 "sparse_dumps disabled\n"); 960 sparse_dump = 0; 961 } else { 962 sparse_dump_physmap = (void *)uvm_km_alloc(kernel_map, 963 roundup(max_paddr / (PAGE_SIZE * NBBY), PAGE_SIZE), 964 PAGE_SIZE, UVM_KMF_WIRED|UVM_KMF_ZERO); 965 } 966 #endif 967 dump_headerbuf = (void *)uvm_km_alloc(kernel_map, 968 dump_headerbuf_size, 969 PAGE_SIZE, UVM_KMF_WIRED|UVM_KMF_ZERO); 970 /* XXXjld should check for failure here, disable dumps if so. */ 971 } 972 973 #ifndef NO_SPARSE_DUMP 974 /* 975 * Clear the set of pages to include in a sparse dump. 976 */ 977 void 978 sparse_dump_reset(void) 979 { 980 memset(sparse_dump_physmap, 0, 981 roundup(max_paddr / (PAGE_SIZE * NBBY), PAGE_SIZE)); 982 } 983 984 /* 985 * Include or exclude pages in a sparse dump. 986 */ 987 void 988 sparse_dump_mark(void) 989 { 990 paddr_t p, pstart, pend; 991 struct vm_page *pg; 992 int i; 993 uvm_physseg_t upm; 994 995 /* 996 * Mark all memory pages, then unmark pages that are uninteresting. 997 * Dereferenceing pg->uobject might crash again if another CPU 998 * frees the object out from under us, but we can't lock anything 999 * so it's a risk we have to take. 1000 */ 1001 1002 for (i = 0; i < mem_cluster_cnt; ++i) { 1003 pstart = mem_clusters[i].start / PAGE_SIZE; 1004 pend = pstart + mem_clusters[i].size / PAGE_SIZE; 1005 1006 for (p = pstart; p < pend; p++) { 1007 setbit(sparse_dump_physmap, p); 1008 } 1009 } 1010 for (upm = uvm_physseg_get_first(); 1011 uvm_physseg_valid_p(upm); 1012 upm = uvm_physseg_get_next(upm)) { 1013 paddr_t pfn; 1014 1015 /* 1016 * We assume that seg->start to seg->end are 1017 * uvm_page_physload()ed 1018 */ 1019 for (pfn = uvm_physseg_get_start(upm); 1020 pfn < uvm_physseg_get_end(upm); 1021 pfn++) { 1022 pg = PHYS_TO_VM_PAGE(ptoa(pfn)); 1023 1024 if (pg->uanon || (pg->flags & PG_FREE) || 1025 (pg->uobject && pg->uobject->pgops)) { 1026 p = VM_PAGE_TO_PHYS(pg) / PAGE_SIZE; 1027 clrbit(sparse_dump_physmap, p); 1028 } 1029 } 1030 } 1031 } 1032 1033 /* 1034 * Machine-dependently decides on the contents of a sparse dump, using 1035 * the above. 1036 */ 1037 void 1038 cpu_dump_prep_sparse(void) 1039 { 1040 sparse_dump_reset(); 1041 /* XXX could the alternate recursive page table be skipped? */ 1042 sparse_dump_mark(); 1043 /* Memory for I/O buffers could be unmarked here, for example. */ 1044 /* The kernel text could also be unmarked, but gdb would be upset. */ 1045 } 1046 #endif 1047 1048 /* 1049 * Abstractly iterate over the collection of memory segments to be 1050 * dumped; the callback lacks the customary environment-pointer 1051 * argument because none of the current users really need one. 1052 * 1053 * To be used only after dump_seg_prep is called to set things up. 1054 */ 1055 int 1056 dump_seg_iter(int (*callback)(paddr_t, paddr_t)) 1057 { 1058 int error, i; 1059 1060 #define CALLBACK(start,size) do { \ 1061 error = callback(start,size); \ 1062 if (error) \ 1063 return error; \ 1064 } while(0) 1065 1066 for (i = 0; i < mem_cluster_cnt; ++i) { 1067 #ifndef NO_SPARSE_DUMP 1068 /* 1069 * The bitmap is scanned within each memory segment, 1070 * rather than over its entire domain, in case any 1071 * pages outside of the memory proper have been mapped 1072 * into kva; they might be devices that wouldn't 1073 * appreciate being arbitrarily read, and including 1074 * them could also break the assumption that a sparse 1075 * dump will always be smaller than a full one. 1076 */ 1077 if (sparse_dump && sparse_dump_physmap) { 1078 paddr_t p, sp_start, sp_end; 1079 int lastset; 1080 1081 sp_start = mem_clusters[i].start; 1082 sp_end = sp_start + mem_clusters[i].size; 1083 sp_start = rounddown(sp_start, PAGE_SIZE); /* unnecessary? */ 1084 lastset = 0; 1085 for (p = sp_start; p < sp_end; p += PAGE_SIZE) { 1086 int thisset = isset(sparse_dump_physmap, 1087 p/PAGE_SIZE); 1088 1089 if (!lastset && thisset) 1090 sp_start = p; 1091 if (lastset && !thisset) 1092 CALLBACK(sp_start, p - sp_start); 1093 lastset = thisset; 1094 } 1095 if (lastset) 1096 CALLBACK(sp_start, p - sp_start); 1097 } else 1098 #endif 1099 CALLBACK(mem_clusters[i].start, mem_clusters[i].size); 1100 } 1101 return 0; 1102 #undef CALLBACK 1103 } 1104 1105 /* 1106 * Prepare for an impending core dump: decide what's being dumped and 1107 * how much space it will take up. 1108 */ 1109 void 1110 dump_seg_prep(void) 1111 { 1112 #ifndef NO_SPARSE_DUMP 1113 if (sparse_dump && sparse_dump_physmap) 1114 cpu_dump_prep_sparse(); 1115 #endif 1116 1117 dump_nmemsegs = 0; 1118 dump_npages = 0; 1119 dump_seg_iter(dump_seg_count_range); 1120 1121 dump_header_size = ALIGN(sizeof(kcore_seg_t)) + 1122 ALIGN(sizeof(cpu_kcore_hdr_t)) + 1123 ALIGN(dump_nmemsegs * sizeof(phys_ram_seg_t)); 1124 dump_header_size = roundup(dump_header_size, dbtob(1)); 1125 1126 /* 1127 * savecore(8) will read this to decide how many pages to 1128 * copy, and cpu_dumpconf has already used the pessimistic 1129 * value to set dumplo, so it's time to tell the truth. 1130 */ 1131 dumpsize = dump_npages; /* XXX could these just be one variable? */ 1132 } 1133 1134 int 1135 dump_seg_count_range(paddr_t start, paddr_t size) 1136 { 1137 ++dump_nmemsegs; 1138 dump_npages += size / PAGE_SIZE; 1139 return 0; 1140 } 1141 1142 /* 1143 * A sparse dump's header may be rather large, due to the number of 1144 * "segments" emitted. These routines manage a simple output buffer, 1145 * so that the header can be written to disk incrementally. 1146 */ 1147 void 1148 dump_header_start(void) 1149 { 1150 dump_headerbuf_ptr = dump_headerbuf; 1151 dump_header_blkno = dumplo; 1152 } 1153 1154 int 1155 dump_header_flush(void) 1156 { 1157 const struct bdevsw *bdev; 1158 size_t to_write; 1159 int error; 1160 1161 bdev = bdevsw_lookup(dumpdev); 1162 to_write = roundup(dump_headerbuf_ptr - dump_headerbuf, dbtob(1)); 1163 error = bdev->d_dump(dumpdev, dump_header_blkno, 1164 dump_headerbuf, to_write); 1165 dump_header_blkno += btodb(to_write); 1166 dump_headerbuf_ptr = dump_headerbuf; 1167 return error; 1168 } 1169 1170 int 1171 dump_header_addbytes(const void* vptr, size_t n) 1172 { 1173 const char* ptr = vptr; 1174 int error; 1175 1176 while (n > dump_headerbuf_avail) { 1177 memcpy(dump_headerbuf_ptr, ptr, dump_headerbuf_avail); 1178 ptr += dump_headerbuf_avail; 1179 n -= dump_headerbuf_avail; 1180 dump_headerbuf_ptr = dump_headerbuf_end; 1181 error = dump_header_flush(); 1182 if (error) 1183 return error; 1184 } 1185 memcpy(dump_headerbuf_ptr, ptr, n); 1186 dump_headerbuf_ptr += n; 1187 1188 return 0; 1189 } 1190 1191 int 1192 dump_header_addseg(paddr_t start, paddr_t size) 1193 { 1194 phys_ram_seg_t seg = { start, size }; 1195 int error; 1196 1197 error = dump_header_addbytes(&seg, sizeof(seg)); 1198 if (error) { 1199 printf("[seg 0x%"PRIxPADDR" bytes 0x%"PRIxPSIZE" failed," 1200 " error=%d] ", start, size, error); 1201 } 1202 return error; 1203 } 1204 1205 int 1206 dump_header_finish(void) 1207 { 1208 int error; 1209 1210 memset(dump_headerbuf_ptr, 0, dump_headerbuf_avail); 1211 error = dump_header_flush(); 1212 if (error) 1213 printf("[finish failed, error=%d] ", error); 1214 return error; 1215 } 1216 1217 1218 /* 1219 * These variables are needed by /sbin/savecore 1220 */ 1221 uint32_t dumpmag = 0x8fca0101; /* magic number */ 1222 int dumpsize = 0; /* pages */ 1223 long dumplo = 0; /* blocks */ 1224 1225 /* 1226 * cpu_dumpsize: calculate size of machine-dependent kernel core dump headers 1227 * for a full (non-sparse) dump. 1228 */ 1229 int 1230 cpu_dumpsize(void) 1231 { 1232 int size; 1233 1234 size = ALIGN(sizeof(kcore_seg_t)) + ALIGN(sizeof(cpu_kcore_hdr_t)) + 1235 ALIGN(mem_cluster_cnt * sizeof(phys_ram_seg_t)); 1236 if (roundup(size, dbtob(1)) != dbtob(1)) 1237 return (-1); 1238 1239 return (1); 1240 } 1241 1242 /* 1243 * cpu_dump_mempagecnt: calculate the size of RAM (in pages) to be dumped 1244 * for a full (non-sparse) dump. 1245 */ 1246 u_long 1247 cpu_dump_mempagecnt(void) 1248 { 1249 u_long i, n; 1250 1251 n = 0; 1252 for (i = 0; i < mem_cluster_cnt; i++) 1253 n += atop(mem_clusters[i].size); 1254 return (n); 1255 } 1256 1257 /* 1258 * cpu_dump: dump the machine-dependent kernel core dump headers. 1259 */ 1260 int 1261 cpu_dump(void) 1262 { 1263 kcore_seg_t seg; 1264 cpu_kcore_hdr_t cpuhdr; 1265 const struct bdevsw *bdev; 1266 int error; 1267 1268 bdev = bdevsw_lookup(dumpdev); 1269 if (bdev == NULL) { 1270 printf("[device 0x%llx ENXIO] ", (unsigned long long)dumpdev); 1271 return ENXIO; 1272 } 1273 1274 /* 1275 * Generate a segment header. 1276 */ 1277 CORE_SETMAGIC(seg, KCORE_MAGIC, MID_MACHINE, CORE_CPU); 1278 seg.c_size = dump_header_size - ALIGN(sizeof(seg)); 1279 error = dump_header_addbytes(&seg, ALIGN(sizeof(seg))); 1280 if (error) { 1281 printf("[segment header %zu bytes failed, error=%d] ", 1282 ALIGN(sizeof(seg)), error); 1283 /* blithely proceed (can't fail?) */ 1284 } 1285 1286 /* 1287 * Add the machine-dependent header info. 1288 */ 1289 cpuhdr.ptdpaddr = PDPpaddr; 1290 cpuhdr.nmemsegs = dump_nmemsegs; 1291 error = dump_header_addbytes(&cpuhdr, ALIGN(sizeof(cpuhdr))); 1292 if (error) { 1293 printf("[MD header %zu bytes failed, error=%d] ", 1294 ALIGN(sizeof(cpuhdr)), error); 1295 /* blithely proceed (can't fail?) */ 1296 } 1297 1298 /* 1299 * Write out the memory segment descriptors. 1300 */ 1301 return dump_seg_iter(dump_header_addseg); 1302 } 1303 1304 /* 1305 * Doadump comes here after turning off memory management and 1306 * getting on the dump stack, either when called above, or by 1307 * the auto-restart code. 1308 */ 1309 #define BYTES_PER_DUMP PAGE_SIZE /* must be a multiple of pagesize XXX small */ 1310 static vaddr_t dumpspace; 1311 1312 vaddr_t 1313 reserve_dumppages(vaddr_t p) 1314 { 1315 1316 dumpspace = p; 1317 return (p + BYTES_PER_DUMP); 1318 } 1319 1320 int 1321 dumpsys_seg(paddr_t maddr, paddr_t bytes) 1322 { 1323 u_long i, m, n; 1324 daddr_t blkno; 1325 const struct bdevsw *bdev; 1326 int (*dump)(dev_t, daddr_t, void *, size_t); 1327 int error; 1328 1329 if (dumpdev == NODEV) 1330 return ENODEV; 1331 bdev = bdevsw_lookup(dumpdev); 1332 if (bdev == NULL || bdev->d_psize == NULL) 1333 return ENODEV; 1334 1335 dump = bdev->d_dump; 1336 1337 blkno = dump_header_blkno; 1338 for (i = 0; i < bytes; i += n, dump_totalbytesleft -= n) { 1339 /* Print out how many MBs we have left to go. */ 1340 if ((dump_totalbytesleft % (1024*1024)) == 0) 1341 printf_nolog("%lu ", (unsigned long) 1342 (dump_totalbytesleft / (1024 * 1024))); 1343 1344 /* Limit size for next transfer. */ 1345 n = bytes - i; 1346 if (n > BYTES_PER_DUMP) 1347 n = BYTES_PER_DUMP; 1348 1349 for (m = 0; m < n; m += NBPG) 1350 pmap_kenter_pa(dumpspace + m, maddr + m, 1351 VM_PROT_READ, 0); 1352 pmap_update(pmap_kernel()); 1353 1354 error = (*dump)(dumpdev, blkno, (void *)dumpspace, n); 1355 pmap_kremove_local(dumpspace, n); 1356 if (error) 1357 return error; 1358 maddr += n; 1359 blkno += btodb(n); /* XXX? */ 1360 1361 #if 0 /* XXX this doesn't work. grr. */ 1362 /* operator aborting dump? */ 1363 if (sget() != NULL) 1364 return EINTR; 1365 #endif 1366 } 1367 dump_header_blkno = blkno; 1368 1369 return 0; 1370 } 1371 1372 void 1373 dodumpsys(void) 1374 { 1375 const struct bdevsw *bdev; 1376 int dumpend, psize; 1377 int error; 1378 1379 if (dumpdev == NODEV) 1380 return; 1381 1382 bdev = bdevsw_lookup(dumpdev); 1383 if (bdev == NULL || bdev->d_psize == NULL) 1384 return; 1385 /* 1386 * For dumps during autoconfiguration, 1387 * if dump device has already configured... 1388 */ 1389 if (dumpsize == 0) 1390 cpu_dumpconf(); 1391 1392 printf("\ndumping to dev %llu,%llu (offset=%ld, size=%d):", 1393 (unsigned long long)major(dumpdev), 1394 (unsigned long long)minor(dumpdev), dumplo, dumpsize); 1395 1396 if (dumplo <= 0 || dumpsize <= 0) { 1397 printf(" not possible\n"); 1398 return; 1399 } 1400 1401 psize = bdev_size(dumpdev); 1402 printf("\ndump "); 1403 if (psize == -1) { 1404 printf("area unavailable\n"); 1405 return; 1406 } 1407 1408 #if 0 /* XXX this doesn't work. grr. */ 1409 /* toss any characters present prior to dump */ 1410 while (sget() != NULL); /*syscons and pccons differ */ 1411 #endif 1412 1413 dump_seg_prep(); 1414 dumpend = dumplo + btodb(dump_header_size) + ctod(dump_npages); 1415 if (dumpend > psize) { 1416 printf("failed: insufficient space (%d < %d)\n", 1417 psize, dumpend); 1418 goto failed; 1419 } 1420 1421 dump_header_start(); 1422 if ((error = cpu_dump()) != 0) 1423 goto err; 1424 if ((error = dump_header_finish()) != 0) 1425 goto err; 1426 1427 if (dump_header_blkno != dumplo + btodb(dump_header_size)) { 1428 printf("BAD header size (%ld [written] != %ld [expected])\n", 1429 (long)(dump_header_blkno - dumplo), 1430 (long)btodb(dump_header_size)); 1431 goto failed; 1432 } 1433 1434 dump_totalbytesleft = roundup(ptoa(dump_npages), BYTES_PER_DUMP); 1435 error = dump_seg_iter(dumpsys_seg); 1436 1437 if (error == 0 && dump_header_blkno != dumpend) { 1438 printf("BAD dump size (%ld [written] != %ld [expected])\n", 1439 (long)(dumpend - dumplo), 1440 (long)(dump_header_blkno - dumplo)); 1441 goto failed; 1442 } 1443 1444 err: 1445 switch (error) { 1446 1447 case ENXIO: 1448 printf("device bad\n"); 1449 break; 1450 1451 case EFAULT: 1452 printf("device not ready\n"); 1453 break; 1454 1455 case EINVAL: 1456 printf("area improper\n"); 1457 break; 1458 1459 case EIO: 1460 printf("i/o error\n"); 1461 break; 1462 1463 case EINTR: 1464 printf("aborted from console\n"); 1465 break; 1466 1467 case 0: 1468 printf("succeeded\n"); 1469 break; 1470 1471 default: 1472 printf("error %d\n", error); 1473 break; 1474 } 1475 failed: 1476 printf("\n\n"); 1477 delay(5000000); /* 5 seconds */ 1478 } 1479 1480 /* 1481 * This is called by main to set dumplo and dumpsize. 1482 * Dumps always skip the first PAGE_SIZE of disk space 1483 * in case there might be a disk label stored there. 1484 * If there is extra space, put dump at the end to 1485 * reduce the chance that swapping trashes it. 1486 * 1487 * Sparse dumps can't placed as close to the end as possible, because 1488 * savecore(8) has to know where to start reading in the dump device 1489 * before it has access to any of the crashed system's state. 1490 * 1491 * Note also that a sparse dump will never be larger than a full one: 1492 * in order to add a phys_ram_seg_t to the header, at least one page 1493 * must be removed. 1494 */ 1495 void 1496 cpu_dumpconf(void) 1497 { 1498 int nblks, dumpblks; /* size of dump area */ 1499 1500 if (dumpdev == NODEV) 1501 goto bad; 1502 nblks = bdev_size(dumpdev); 1503 if (nblks <= ctod(1)) 1504 goto bad; 1505 1506 dumpblks = cpu_dumpsize(); 1507 if (dumpblks < 0) 1508 goto bad; 1509 1510 /* dumpsize is in page units, and doesn't include headers. */ 1511 dumpsize = cpu_dump_mempagecnt(); 1512 1513 dumpblks += ctod(dumpsize); 1514 1515 /* If dump won't fit (incl. room for possible label), punt. */ 1516 if (dumpblks > (nblks - ctod(1))) { 1517 #ifndef NO_SPARSE_DUMP 1518 /* A sparse dump might (and hopefully will) fit. */ 1519 dumplo = ctod(1); 1520 #else 1521 /* But if we're not configured for that, punt. */ 1522 goto bad; 1523 #endif 1524 } else { 1525 /* Put dump at end of partition */ 1526 dumplo = nblks - dumpblks; 1527 } 1528 1529 1530 /* Now that we've decided this will work, init ancillary stuff. */ 1531 dump_misc_init(); 1532 return; 1533 1534 bad: 1535 dumpsize = 0; 1536 } 1537 1538 /* 1539 * Clear registers on exec 1540 */ 1541 void 1542 setregs(struct lwp *l, struct exec_package *pack, vaddr_t stack) 1543 { 1544 struct pcb *pcb = lwp_getpcb(l); 1545 struct trapframe *tf; 1546 1547 #ifdef USER_LDT 1548 pmap_ldt_cleanup(l); 1549 #endif 1550 1551 fpu_clear(l, pack->ep_osversion >= 699002600 1552 ? __NetBSD_NPXCW__ : __NetBSD_COMPAT_NPXCW__); 1553 x86_dbregs_clear(l); 1554 1555 kpreempt_disable(); 1556 pcb->pcb_flags = 0; 1557 l->l_proc->p_flag &= ~PK_32; 1558 l->l_md.md_flags = MDL_IRET; 1559 cpu_segregs64_zero(l); 1560 kpreempt_enable(); 1561 1562 tf = l->l_md.md_regs; 1563 memset(tf, 0, sizeof(*tf)); 1564 1565 tf->tf_trapno = T_ASTFLT; 1566 tf->tf_ds = GSEL(GUDATA_SEL, SEL_UPL); 1567 tf->tf_es = GSEL(GUDATA_SEL, SEL_UPL); 1568 tf->tf_rdi = 0; 1569 tf->tf_rsi = 0; 1570 tf->tf_rbp = 0; 1571 tf->tf_rbx = l->l_proc->p_psstrp; 1572 tf->tf_rdx = 0; 1573 tf->tf_rcx = 0; 1574 tf->tf_rax = 0; 1575 tf->tf_rip = pack->ep_entry; 1576 tf->tf_cs = LSEL(LUCODE_SEL, SEL_UPL); 1577 tf->tf_rflags = PSL_USERSET; 1578 tf->tf_rsp = stack; 1579 tf->tf_ss = LSEL(LUDATA_SEL, SEL_UPL); 1580 } 1581 1582 /* 1583 * Initialize segments and descriptor tables 1584 */ 1585 char *ldtstore; 1586 char *gdtstore; 1587 1588 void 1589 setgate(struct gate_descriptor *gd, void *func, 1590 int ist, int type, int dpl, int sel) 1591 { 1592 vaddr_t vaddr; 1593 1594 vaddr = ((vaddr_t)gd) & ~PAGE_MASK; 1595 1596 kpreempt_disable(); 1597 pmap_changeprot_local(vaddr, VM_PROT_READ|VM_PROT_WRITE); 1598 1599 gd->gd_looffset = (uint64_t)func & 0xffff; 1600 gd->gd_selector = sel; 1601 gd->gd_ist = ist; 1602 gd->gd_type = type; 1603 gd->gd_dpl = dpl; 1604 gd->gd_p = 1; 1605 gd->gd_hioffset = (uint64_t)func >> 16; 1606 gd->gd_zero = 0; 1607 gd->gd_xx1 = 0; 1608 gd->gd_xx2 = 0; 1609 gd->gd_xx3 = 0; 1610 1611 pmap_changeprot_local(vaddr, VM_PROT_READ); 1612 kpreempt_enable(); 1613 } 1614 1615 void 1616 unsetgate(struct gate_descriptor *gd) 1617 { 1618 vaddr_t vaddr; 1619 1620 vaddr = ((vaddr_t)gd) & ~PAGE_MASK; 1621 1622 kpreempt_disable(); 1623 pmap_changeprot_local(vaddr, VM_PROT_READ|VM_PROT_WRITE); 1624 1625 memset(gd, 0, sizeof (*gd)); 1626 1627 pmap_changeprot_local(vaddr, VM_PROT_READ); 1628 kpreempt_enable(); 1629 } 1630 1631 void 1632 setregion(struct region_descriptor *rd, void *base, uint16_t limit) 1633 { 1634 rd->rd_limit = limit; 1635 rd->rd_base = (uint64_t)base; 1636 } 1637 1638 /* 1639 * Note that the base and limit fields are ignored in long mode. 1640 */ 1641 void 1642 set_mem_segment(struct mem_segment_descriptor *sd, void *base, size_t limit, 1643 int type, int dpl, int gran, int def32, int is64) 1644 { 1645 sd->sd_lolimit = (unsigned)limit; 1646 sd->sd_lobase = (unsigned long)base; 1647 sd->sd_type = type; 1648 sd->sd_dpl = dpl; 1649 sd->sd_p = 1; 1650 sd->sd_hilimit = (unsigned)limit >> 16; 1651 sd->sd_avl = 0; 1652 sd->sd_long = is64; 1653 sd->sd_def32 = def32; 1654 sd->sd_gran = gran; 1655 sd->sd_hibase = (unsigned long)base >> 24; 1656 } 1657 1658 void 1659 set_sys_segment(struct sys_segment_descriptor *sd, void *base, size_t limit, 1660 int type, int dpl, int gran) 1661 { 1662 memset(sd, 0, sizeof *sd); 1663 sd->sd_lolimit = (unsigned)limit; 1664 sd->sd_lobase = (uint64_t)base; 1665 sd->sd_type = type; 1666 sd->sd_dpl = dpl; 1667 sd->sd_p = 1; 1668 sd->sd_hilimit = (unsigned)limit >> 16; 1669 sd->sd_gran = gran; 1670 sd->sd_hibase = (uint64_t)base >> 24; 1671 } 1672 1673 void 1674 cpu_init_idt(struct cpu_info *ci) 1675 { 1676 struct region_descriptor region; 1677 idt_descriptor_t *idt; 1678 1679 idt = ci->ci_idtvec.iv_idt; 1680 setregion(®ion, idt, NIDT * sizeof(idt[0]) - 1); 1681 lidt(®ion); 1682 } 1683 1684 #define IDTVEC(name) __CONCAT(X, name) 1685 typedef void (vector)(void); 1686 extern vector IDTVEC(syscall); 1687 extern vector IDTVEC(syscall32); 1688 extern vector IDTVEC(osyscall); 1689 extern vector *x86_exceptions[]; 1690 1691 #ifndef XENPV 1692 static void 1693 init_x86_64_ksyms(void) 1694 { 1695 #if NKSYMS || defined(DDB) || defined(MODULAR) 1696 extern int end; 1697 extern int *esym; 1698 struct btinfo_symtab *symtab; 1699 vaddr_t tssym, tesym; 1700 1701 #ifdef DDB 1702 db_machine_init(); 1703 #endif 1704 1705 symtab = lookup_bootinfo(BTINFO_SYMTAB); 1706 if (symtab) { 1707 #ifdef KASLR 1708 tssym = bootspace.head.va; 1709 tesym = bootspace.head.va; /* (unused...) */ 1710 #else 1711 tssym = (vaddr_t)symtab->ssym + KERNBASE; 1712 tesym = (vaddr_t)symtab->esym + KERNBASE; 1713 #endif 1714 ksyms_addsyms_elf(symtab->nsym, (void *)tssym, (void *)tesym); 1715 } else { 1716 uintptr_t endp = (uintptr_t)(void *)&end; 1717 #ifdef XEN 1718 /* 1719 * cpu_probe() / identify_hypervisor() overrides VM_GUEST_GENPVH, 1720 * we can't rely on vm_guest == VM_GUEST_GENPVH 1721 */ 1722 if (pvh_boot && vm_guest != VM_GUEST_XENPVH) 1723 ksyms_addsyms_elf(0, ((long *)endp) + 1, esym); 1724 else 1725 #endif 1726 ksyms_addsyms_elf(*(long *)endp, ((long *)endp) + 1, esym); 1727 } 1728 #endif 1729 } 1730 #endif /* XENPV */ 1731 1732 void __noasan 1733 init_bootspace(void) 1734 { 1735 extern char __rodata_start; 1736 extern char __data_start; 1737 extern char __kernel_end; 1738 size_t i = 0; 1739 1740 memset(&bootspace, 0, sizeof(bootspace)); 1741 1742 bootspace.head.va = KERNTEXTOFF; 1743 bootspace.head.pa = KERNTEXTOFF - KERNBASE; 1744 bootspace.head.sz = 0; 1745 1746 bootspace.segs[i].type = BTSEG_TEXT; 1747 bootspace.segs[i].va = KERNTEXTOFF; 1748 bootspace.segs[i].pa = KERNTEXTOFF - KERNBASE; 1749 bootspace.segs[i].sz = (size_t)&__rodata_start - KERNTEXTOFF; 1750 i++; 1751 1752 bootspace.segs[i].type = BTSEG_RODATA; 1753 bootspace.segs[i].va = (vaddr_t)&__rodata_start; 1754 bootspace.segs[i].pa = (paddr_t)&__rodata_start - KERNBASE; 1755 bootspace.segs[i].sz = (size_t)&__data_start - (size_t)&__rodata_start; 1756 i++; 1757 1758 bootspace.segs[i].type = BTSEG_DATA; 1759 bootspace.segs[i].va = (vaddr_t)&__data_start; 1760 bootspace.segs[i].pa = (paddr_t)&__data_start - KERNBASE; 1761 bootspace.segs[i].sz = (size_t)&__kernel_end - (size_t)&__data_start; 1762 i++; 1763 1764 bootspace.boot.va = (vaddr_t)&__kernel_end; 1765 bootspace.boot.pa = (paddr_t)&__kernel_end - KERNBASE; 1766 bootspace.boot.sz = (size_t)(atdevbase + IOM_SIZE) - 1767 (size_t)&__kernel_end; 1768 1769 /* In locore.S, we allocated a tmp va. We will use it now. */ 1770 bootspace.spareva = KERNBASE + NKL2_KIMG_ENTRIES * NBPD_L2; 1771 1772 /* Virtual address of the L4 page. */ 1773 bootspace.pdir = (vaddr_t)(PDPpaddr + KERNBASE); 1774 1775 /* Kernel module map. */ 1776 bootspace.smodule = (vaddr_t)atdevbase + IOM_SIZE; 1777 bootspace.emodule = KERNBASE + NKL2_KIMG_ENTRIES * NBPD_L2; 1778 } 1779 1780 static void 1781 init_pte(void) 1782 { 1783 #ifndef XENPV 1784 extern uint32_t nox_flag; 1785 pd_entry_t *pdir = (pd_entry_t *)bootspace.pdir; 1786 pdir[L4_SLOT_PTE] = PDPpaddr | PTE_W | ((uint64_t)nox_flag << 32) | 1787 PTE_P; 1788 #endif 1789 1790 extern pd_entry_t *normal_pdes[3]; 1791 normal_pdes[0] = L2_BASE; 1792 normal_pdes[1] = L3_BASE; 1793 normal_pdes[2] = L4_BASE; 1794 } 1795 1796 void 1797 init_slotspace(void) 1798 { 1799 /* 1800 * XXX Too early to use cprng(9), or even entropy_extract. 1801 */ 1802 struct entpool pool; 1803 size_t randhole; 1804 vaddr_t randva; 1805 uint64_t sample; 1806 vaddr_t va; 1807 1808 memset(&pool, 0, sizeof pool); 1809 cpu_rng_early_sample(&sample); 1810 entpool_enter(&pool, &sample, sizeof sample); 1811 1812 memset(&slotspace, 0, sizeof(slotspace)); 1813 1814 /* User. [256, because we want to land in >= 256] */ 1815 slotspace.area[SLAREA_USER].sslot = 0; 1816 slotspace.area[SLAREA_USER].nslot = PDIR_SLOT_USERLIM+1; 1817 slotspace.area[SLAREA_USER].active = true; 1818 1819 #ifdef XENPV 1820 /* PTE. */ 1821 slotspace.area[SLAREA_PTE].sslot = PDIR_SLOT_PTE; 1822 slotspace.area[SLAREA_PTE].nslot = 1; 1823 slotspace.area[SLAREA_PTE].active = true; 1824 #endif 1825 1826 #ifdef __HAVE_PCPU_AREA 1827 /* Per-CPU. */ 1828 slotspace.area[SLAREA_PCPU].sslot = PDIR_SLOT_PCPU; 1829 slotspace.area[SLAREA_PCPU].nslot = 1; 1830 slotspace.area[SLAREA_PCPU].active = true; 1831 #endif 1832 1833 #ifdef __HAVE_DIRECT_MAP 1834 /* Direct Map. [Randomized later] */ 1835 slotspace.area[SLAREA_DMAP].active = false; 1836 #endif 1837 1838 #ifdef XENPV 1839 /* Hypervisor. */ 1840 slotspace.area[SLAREA_HYPV].sslot = 256; 1841 slotspace.area[SLAREA_HYPV].nslot = 17; 1842 slotspace.area[SLAREA_HYPV].active = true; 1843 #endif 1844 1845 #ifdef KASAN 1846 /* ASAN. */ 1847 slotspace.area[SLAREA_ASAN].sslot = L4_SLOT_KASAN; 1848 slotspace.area[SLAREA_ASAN].nslot = NL4_SLOT_KASAN; 1849 slotspace.area[SLAREA_ASAN].active = true; 1850 #endif 1851 1852 #ifdef KMSAN 1853 /* MSAN. */ 1854 slotspace.area[SLAREA_MSAN].sslot = L4_SLOT_KMSAN; 1855 slotspace.area[SLAREA_MSAN].nslot = NL4_SLOT_KMSAN; 1856 slotspace.area[SLAREA_MSAN].active = true; 1857 #endif 1858 1859 /* Kernel. */ 1860 slotspace.area[SLAREA_KERN].sslot = L4_SLOT_KERNBASE; 1861 slotspace.area[SLAREA_KERN].nslot = 1; 1862 slotspace.area[SLAREA_KERN].active = true; 1863 1864 /* Main. */ 1865 cpu_rng_early_sample(&sample); 1866 entpool_enter(&pool, &sample, sizeof sample); 1867 entpool_extract(&pool, &randhole, sizeof randhole); 1868 entpool_extract(&pool, &randva, sizeof randva); 1869 va = slotspace_rand(SLAREA_MAIN, NKL4_MAX_ENTRIES * NBPD_L4, 1870 NBPD_L4, randhole, randva); /* TODO: NBPD_L1 */ 1871 vm_min_kernel_address = va; 1872 vm_max_kernel_address = va + NKL4_MAX_ENTRIES * NBPD_L4; 1873 1874 #ifndef XENPV 1875 /* PTE. */ 1876 cpu_rng_early_sample(&sample); 1877 entpool_enter(&pool, &sample, sizeof sample); 1878 entpool_extract(&pool, &randhole, sizeof randhole); 1879 entpool_extract(&pool, &randva, sizeof randva); 1880 va = slotspace_rand(SLAREA_PTE, NBPD_L4, NBPD_L4, randhole, randva); 1881 pte_base = (pd_entry_t *)va; 1882 #endif 1883 1884 explicit_memset(&pool, 0, sizeof pool); 1885 } 1886 1887 void 1888 init_x86_64(paddr_t first_avail) 1889 { 1890 extern void consinit(void); 1891 struct region_descriptor region; 1892 struct mem_segment_descriptor *ldt_segp; 1893 struct idt_vec *iv; 1894 idt_descriptor_t *idt; 1895 int x; 1896 struct pcb *pcb; 1897 extern vaddr_t lwp0uarea; 1898 #ifndef XENPV 1899 extern paddr_t local_apic_pa; 1900 #endif 1901 1902 KASSERT(first_avail % PAGE_SIZE == 0); 1903 1904 #ifdef XENPV 1905 KASSERT(HYPERVISOR_shared_info != NULL); 1906 cpu_info_primary.ci_vcpu = &HYPERVISOR_shared_info->vcpu_info[0]; 1907 #endif 1908 1909 #ifdef XEN 1910 if (pvh_boot) 1911 xen_parse_cmdline(XEN_PARSE_BOOTFLAGS, NULL); 1912 #endif 1913 init_pte(); 1914 1915 uvm_lwp_setuarea(&lwp0, lwp0uarea); 1916 1917 cpu_probe(&cpu_info_primary); 1918 #ifdef SVS 1919 svs_init(); 1920 #endif 1921 1922 /* 1923 * Initialize MSRs on cpu0: 1924 * 1925 * - Enables SYSCALL/SYSRET. 1926 * 1927 * - Sets up %fs and %gs so that %gs points to the current 1928 * struct cpu_info as needed for CPUVAR(...), curcpu(), and 1929 * curlwp. 1930 * 1931 * - Enables the no-execute bit if supported. 1932 * 1933 * Thus, after this point, CPUVAR(...), curcpu(), and curlwp 1934 * will work on cpu0. 1935 * 1936 * Note: The call to cpu_init_msrs for secondary CPUs happens 1937 * in cpu_hatch. 1938 */ 1939 cpu_init_msrs(&cpu_info_primary, true); 1940 1941 #ifndef XENPV 1942 cpu_speculation_init(&cpu_info_primary); 1943 #endif 1944 1945 use_pae = 1; /* PAE always enabled in long mode */ 1946 1947 pcb = lwp_getpcb(&lwp0); 1948 #ifdef XENPV 1949 mutex_init(&pte_lock, MUTEX_DEFAULT, IPL_VM); 1950 pcb->pcb_cr3 = xen_start_info.pt_base - KERNBASE; 1951 #else 1952 pcb->pcb_cr3 = PDPpaddr; 1953 #endif 1954 1955 #if NISA > 0 || NPCI > 0 1956 x86_bus_space_init(); 1957 #endif 1958 1959 pat_init(&cpu_info_primary); 1960 1961 consinit(); /* XXX SHOULD NOT BE DONE HERE */ 1962 1963 /* 1964 * Initialize PAGE_SIZE-dependent variables. 1965 */ 1966 uvm_md_init(); 1967 1968 uvmexp.ncolors = 2; 1969 1970 avail_start = first_avail; 1971 1972 #ifndef XENPV 1973 /* 1974 * Low memory reservations: 1975 * Page 0: BIOS data 1976 * Page 1: BIOS callback (not used yet, for symmetry with i386) 1977 * Page 2: MP bootstrap code (MP_TRAMPOLINE) 1978 * Page 3: ACPI wakeup code (ACPI_WAKEUP_ADDR) 1979 * Page 4: Temporary page table for 0MB-4MB 1980 * Page 5: Temporary page directory 1981 * Page 6: Temporary page map level 3 1982 * Page 7: Temporary page map level 4 1983 */ 1984 lowmem_rsvd = 8 * PAGE_SIZE; 1985 1986 /* Initialize the memory clusters (needed in pmap_bootstrap). */ 1987 init_x86_clusters(); 1988 #else 1989 /* Parse Xen command line (replace bootinfo) */ 1990 xen_parse_cmdline(XEN_PARSE_BOOTFLAGS, NULL); 1991 1992 avail_end = ctob(xen_start_info.nr_pages); 1993 pmap_pa_start = (KERNTEXTOFF - KERNBASE); 1994 pmap_pa_end = avail_end; 1995 #endif 1996 1997 /* 1998 * Call pmap initialization to make new kernel address space. 1999 * We must do this before loading pages into the VM system. 2000 */ 2001 pmap_bootstrap(VM_MIN_KERNEL_ADDRESS); 2002 2003 /* 2004 * Initialize RNG to get entropy ASAP either from CPU 2005 * RDRAND/RDSEED or from seed on disk. Constraints: 2006 * 2007 * - Must happen after cpu_init_msrs so that curcpu() and 2008 * curlwp work. 2009 * 2010 * - Must happen after consinit so we have the opportunity to 2011 * print useful feedback. 2012 * 2013 * - On KASLR kernels, must happen after pmap_bootstrap because 2014 * x86_rndseed requires access to the direct map. 2015 */ 2016 cpu_rng_init(); 2017 x86_rndseed(); 2018 2019 #ifndef XENPV 2020 /* Internalize the physical pages into the VM system. */ 2021 init_x86_vm(avail_start); 2022 #else 2023 physmem = xen_start_info.nr_pages; 2024 uvm_page_physload(atop(avail_start), atop(avail_end), 2025 atop(avail_start), atop(avail_end), VM_FREELIST_DEFAULT); 2026 #endif 2027 2028 init_x86_msgbuf(); 2029 2030 kasan_init(); 2031 kcsan_init(); 2032 kmsan_init((void *)lwp0uarea); 2033 2034 pmap_growkernel(VM_MIN_KERNEL_ADDRESS + 32 * 1024 * 1024); 2035 2036 kpreempt_disable(); 2037 2038 #ifndef XENPV 2039 pmap_kenter_pa(local_apic_va, local_apic_pa, 2040 VM_PROT_READ|VM_PROT_WRITE, 0); 2041 pmap_update(pmap_kernel()); 2042 memset((void *)local_apic_va, 0, PAGE_SIZE); 2043 #endif 2044 2045 pmap_kenter_pa(idt_vaddr, idt_paddr, VM_PROT_READ|VM_PROT_WRITE, 0); 2046 pmap_kenter_pa(gdt_vaddr, gdt_paddr, VM_PROT_READ|VM_PROT_WRITE, 0); 2047 pmap_kenter_pa(ldt_vaddr, ldt_paddr, VM_PROT_READ|VM_PROT_WRITE, 0); 2048 pmap_update(pmap_kernel()); 2049 memset((void *)idt_vaddr, 0, PAGE_SIZE); 2050 memset((void *)gdt_vaddr, 0, PAGE_SIZE); 2051 memset((void *)ldt_vaddr, 0, PAGE_SIZE); 2052 2053 #ifndef XENPV 2054 pmap_changeprot_local(idt_vaddr, VM_PROT_READ); 2055 #endif 2056 2057 pmap_update(pmap_kernel()); 2058 2059 iv = &(cpu_info_primary.ci_idtvec); 2060 idt_vec_init_cpu_md(iv, cpu_index(&cpu_info_primary)); 2061 idt = iv->iv_idt; 2062 gdtstore = (char *)gdt_vaddr; 2063 ldtstore = (char *)ldt_vaddr; 2064 2065 /* 2066 * Make GDT gates and memory segments. 2067 */ 2068 set_mem_segment(GDT_ADDR_MEM(gdtstore, GCODE_SEL), 0, 2069 0xfffff, SDT_MEMERA, SEL_KPL, 1, 0, 1); 2070 2071 set_mem_segment(GDT_ADDR_MEM(gdtstore, GDATA_SEL), 0, 2072 0xfffff, SDT_MEMRWA, SEL_KPL, 1, 0, 1); 2073 2074 set_mem_segment(GDT_ADDR_MEM(gdtstore, GUCODE_SEL), 0, 2075 x86_btop(VM_MAXUSER_ADDRESS) - 1, SDT_MEMERA, SEL_UPL, 1, 0, 1); 2076 2077 set_mem_segment(GDT_ADDR_MEM(gdtstore, GUDATA_SEL), 0, 2078 x86_btop(VM_MAXUSER_ADDRESS) - 1, SDT_MEMRWA, SEL_UPL, 1, 0, 1); 2079 2080 #ifndef XENPV 2081 set_sys_segment(GDT_ADDR_SYS(gdtstore, GLDT_SEL), ldtstore, 2082 LDT_SIZE - 1, SDT_SYSLDT, SEL_KPL, 0); 2083 #endif 2084 2085 /* 2086 * Make LDT memory segments. 2087 */ 2088 *(struct mem_segment_descriptor *)(ldtstore + LUCODE_SEL) = 2089 *GDT_ADDR_MEM(gdtstore, GUCODE_SEL); 2090 *(struct mem_segment_descriptor *)(ldtstore + LUDATA_SEL) = 2091 *GDT_ADDR_MEM(gdtstore, GUDATA_SEL); 2092 2093 /* 2094 * 32 bit GDT entries. 2095 */ 2096 set_mem_segment(GDT_ADDR_MEM(gdtstore, GUCODE32_SEL), 0, 2097 x86_btop(VM_MAXUSER_ADDRESS32) - 1, SDT_MEMERA, SEL_UPL, 1, 1, 0); 2098 2099 set_mem_segment(GDT_ADDR_MEM(gdtstore, GUDATA32_SEL), 0, 2100 x86_btop(VM_MAXUSER_ADDRESS32) - 1, SDT_MEMRWA, SEL_UPL, 1, 1, 0); 2101 2102 set_mem_segment(GDT_ADDR_MEM(gdtstore, GUFS_SEL), 0, 2103 x86_btop(VM_MAXUSER_ADDRESS32) - 1, SDT_MEMRWA, SEL_UPL, 1, 1, 0); 2104 2105 set_mem_segment(GDT_ADDR_MEM(gdtstore, GUGS_SEL), 0, 2106 x86_btop(VM_MAXUSER_ADDRESS32) - 1, SDT_MEMRWA, SEL_UPL, 1, 1, 0); 2107 2108 /* 2109 * 32 bit LDT entries. 2110 */ 2111 ldt_segp = (struct mem_segment_descriptor *)(ldtstore + LUCODE32_SEL); 2112 set_mem_segment(ldt_segp, 0, x86_btop(VM_MAXUSER_ADDRESS32) - 1, 2113 SDT_MEMERA, SEL_UPL, 1, 1, 0); 2114 ldt_segp = (struct mem_segment_descriptor *)(ldtstore + LUDATA32_SEL); 2115 set_mem_segment(ldt_segp, 0, x86_btop(VM_MAXUSER_ADDRESS32) - 1, 2116 SDT_MEMRWA, SEL_UPL, 1, 1, 0); 2117 2118 /* CPU-specific IDT exceptions. */ 2119 for (x = 0; x < NCPUIDT; x++) { 2120 int sel, ist; 2121 2122 /* Reset to default. Special cases below */ 2123 sel = SEL_KPL; 2124 ist = 0; 2125 2126 idt_vec_reserve(iv, x); 2127 2128 switch (x) { 2129 case 1: /* DB */ 2130 ist = 4; 2131 break; 2132 case 2: /* NMI */ 2133 ist = 3; 2134 break; 2135 case 3: 2136 case 4: 2137 sel = SEL_UPL; 2138 break; 2139 case 8: /* double fault */ 2140 ist = 2; 2141 break; 2142 #ifdef XENPV 2143 case 18: /* MCA */ 2144 sel |= 0x4; /* Auto EOI/mask */ 2145 break; 2146 #endif /* XENPV */ 2147 default: 2148 break; 2149 } 2150 2151 set_idtgate(&idt[x], x86_exceptions[x], ist, SDT_SYS386IGT, 2152 sel, GSEL(GCODE_SEL, SEL_KPL)); 2153 } 2154 2155 /* new-style interrupt gate for syscalls */ 2156 idt_vec_reserve(iv, 128); 2157 set_idtgate(&idt[128], &IDTVEC(osyscall), 0, SDT_SYS386IGT, SEL_UPL, 2158 GSEL(GCODE_SEL, SEL_KPL)); 2159 2160 kpreempt_enable(); 2161 2162 setregion(®ion, gdtstore, DYNSEL_START - 1); 2163 lgdt(®ion); 2164 2165 #ifdef XENPV 2166 /* Init Xen callbacks and syscall handlers */ 2167 if (HYPERVISOR_set_callbacks( 2168 (unsigned long) hypervisor_callback, 2169 (unsigned long) failsafe_callback, 2170 (unsigned long) Xsyscall)) 2171 panic("HYPERVISOR_set_callbacks() failed"); 2172 #endif /* XENPV */ 2173 2174 cpu_init_idt(&cpu_info_primary); 2175 2176 #ifdef XENPV 2177 xen_init_ksyms(); 2178 #else /* XENPV */ 2179 #ifdef XEN 2180 if (vm_guest == VM_GUEST_XENPVH) 2181 xen_init_ksyms(); 2182 else 2183 #endif /* XEN */ 2184 init_x86_64_ksyms(); 2185 #endif /* XENPV */ 2186 2187 #ifndef XENPV 2188 intr_default_setup(); 2189 #else 2190 events_default_setup(); 2191 #endif 2192 2193 splraise(IPL_HIGH); 2194 x86_enable_intr(); 2195 2196 #ifdef DDB 2197 if (boothowto & RB_KDB) 2198 Debugger(); 2199 #endif 2200 #ifdef KGDB 2201 kgdb_port_init(); 2202 if (boothowto & RB_KDB) { 2203 kgdb_debug_init = 1; 2204 kgdb_connect(1); 2205 } 2206 #endif 2207 2208 pcb->pcb_dbregs = NULL; 2209 x86_dbregs_init(); 2210 } 2211 2212 void 2213 cpu_reset(void) 2214 { 2215 #ifndef XENPV 2216 idt_descriptor_t *idt; 2217 vaddr_t vaddr; 2218 2219 idt = cpu_info_primary.ci_idtvec.iv_idt; 2220 vaddr = (vaddr_t)idt; 2221 #endif 2222 2223 x86_disable_intr(); 2224 2225 #ifdef XENPV 2226 HYPERVISOR_reboot(); 2227 #else 2228 2229 x86_reset(); 2230 2231 /* 2232 * Try to cause a triple fault and watchdog reset by making the IDT 2233 * invalid and causing a fault. 2234 */ 2235 kpreempt_disable(); 2236 pmap_changeprot_local(vaddr, VM_PROT_READ|VM_PROT_WRITE); 2237 memset((void *)idt, 0, NIDT * sizeof(idt[0])); 2238 kpreempt_enable(); 2239 breakpoint(); 2240 2241 #if 0 2242 /* 2243 * Try to cause a triple fault and watchdog reset by unmapping the 2244 * entire address space and doing a TLB flush. 2245 */ 2246 memset((void *)PTD, 0, PAGE_SIZE); 2247 tlbflush(); 2248 #endif 2249 #endif /* XENPV */ 2250 2251 for (;;); 2252 } 2253 2254 void 2255 cpu_getmcontext(struct lwp *l, mcontext_t *mcp, unsigned int *flags) 2256 { 2257 const struct trapframe *tf = l->l_md.md_regs; 2258 __greg_t ras_rip; 2259 2260 mcp->__gregs[_REG_RDI] = tf->tf_rdi; 2261 mcp->__gregs[_REG_RSI] = tf->tf_rsi; 2262 mcp->__gregs[_REG_RDX] = tf->tf_rdx; 2263 mcp->__gregs[_REG_R10] = tf->tf_r10; 2264 mcp->__gregs[_REG_R8] = tf->tf_r8; 2265 mcp->__gregs[_REG_R9] = tf->tf_r9; 2266 /* argX not touched */ 2267 mcp->__gregs[_REG_RCX] = tf->tf_rcx; 2268 mcp->__gregs[_REG_R11] = tf->tf_r11; 2269 mcp->__gregs[_REG_R12] = tf->tf_r12; 2270 mcp->__gregs[_REG_R13] = tf->tf_r13; 2271 mcp->__gregs[_REG_R14] = tf->tf_r14; 2272 mcp->__gregs[_REG_R15] = tf->tf_r15; 2273 mcp->__gregs[_REG_RBP] = tf->tf_rbp; 2274 mcp->__gregs[_REG_RBX] = tf->tf_rbx; 2275 mcp->__gregs[_REG_RAX] = tf->tf_rax; 2276 mcp->__gregs[_REG_GS] = 0; 2277 mcp->__gregs[_REG_FS] = 0; 2278 mcp->__gregs[_REG_ES] = GSEL(GUDATA_SEL, SEL_UPL); 2279 mcp->__gregs[_REG_DS] = GSEL(GUDATA_SEL, SEL_UPL); 2280 mcp->__gregs[_REG_TRAPNO] = tf->tf_trapno; 2281 mcp->__gregs[_REG_ERR] = tf->tf_err; 2282 mcp->__gregs[_REG_RIP] = tf->tf_rip; 2283 mcp->__gregs[_REG_CS] = LSEL(LUCODE_SEL, SEL_UPL); 2284 mcp->__gregs[_REG_RFLAGS] = tf->tf_rflags; 2285 mcp->__gregs[_REG_RSP] = tf->tf_rsp; 2286 mcp->__gregs[_REG_SS] = LSEL(LUDATA_SEL, SEL_UPL); 2287 2288 if ((ras_rip = (__greg_t)ras_lookup(l->l_proc, 2289 (void *) mcp->__gregs[_REG_RIP])) != -1) 2290 mcp->__gregs[_REG_RIP] = ras_rip; 2291 2292 *flags |= _UC_CPU; 2293 2294 mcp->_mc_tlsbase = (uintptr_t)l->l_private; 2295 *flags |= _UC_TLSBASE; 2296 2297 process_read_fpregs_xmm(l, (struct fxsave *)&mcp->__fpregs.__fxsave); 2298 *flags |= _UC_FPU; 2299 } 2300 2301 /* 2302 * cpu_getmcontext_xsave(l, mcp, flags, xsavebuf, xsavelen, user_xsave) 2303 * 2304 * Copy out xsavebuf[512..xsavelen) to user_xsave[512..xsavelen), 2305 * set mcp to point at it, and set _UC_XSAVE in *flags. Caller: 2306 * 2307 * - must have already initialized the FXSAVE area of mcontext_t, 2308 * - must have already set _UC_FPU in *flags, 2309 * - must have arranged user_xsave[0..512) to overlap with the 2310 * FXSAVE area of mcontext_t, and 2311 * - must subsequently copy out the mcontext_t updated with a 2312 * pointer/length to the XSAVE area. 2313 * 2314 * May fail if the copyout fails. 2315 */ 2316 static int 2317 cpu_getmcontext_xsave(struct lwp *l, mcontext_t *mcp, unsigned int *flags, 2318 const struct xsave_header *xsavebuf, size_t xsavelen, 2319 struct xsave_header *user_xsave) 2320 { 2321 enum { fxsavelen = sizeof(mcp->__fpregs.__fxsave) }; 2322 int error; 2323 2324 CTASSERT(fxsavelen == 512); 2325 2326 KASSERT(*flags & _UC_FPU); 2327 KASSERT(fxsavelen <= xsavelen); 2328 KDASSERT(memcmp(&mcp->__fpregs.__fxsave, xsavebuf, fxsavelen) == 0); 2329 2330 /* 2331 * Copy out the part of the XSAVE area that doesn't overlap. 2332 */ 2333 error = copyout((const char *)xsavebuf + fxsavelen, 2334 (char *)user_xsave + fxsavelen, xsavelen - fxsavelen); 2335 if (error != 0) 2336 return error; 2337 2338 /* 2339 * Record a pointer to the real XSAVE area in the 2340 * architecturally unused bits mcontext_t's FXSAVE area. 2341 */ 2342 mcp->__fpregs.__xsave.__xsaveptr = (__greg_t)(uintptr_t)user_xsave; 2343 mcp->__fpregs.__xsave.__xsavelen = (__greg_t)xsavelen; 2344 2345 /* 2346 * Set the _UC_XSAVE flag so cpu_setmcontext will be able to 2347 * restore the full state from the XSAVE area. 2348 */ 2349 *flags |= _UC_XSAVE; 2350 2351 /* Success! */ 2352 return 0; 2353 } 2354 2355 int 2356 cpu_setmcontext(struct lwp *l, const mcontext_t *mcp, unsigned int flags) 2357 { 2358 struct trapframe *tf = l->l_md.md_regs; 2359 const __greg_t *gr = mcp->__gregs; 2360 struct proc *p = l->l_proc; 2361 struct xsave_header *xsavebuf = NULL; 2362 size_t xsavelen = 0; 2363 int error; 2364 int64_t rflags; 2365 2366 CTASSERT(sizeof (mcontext_t) == 26 * 8 + 8 + 512); 2367 2368 /* 2369 * If there's an external XSAVE area, copy it in and validate 2370 * it before we irreversibly modify the trapframe. 2371 * 2372 * We could check the length against the state components 2373 * included, but we currently don't: if it's truncated, it will 2374 * be as if the truncated part were zero-filled -- this is 2375 * implemented in process_write_xsave, called a little below. 2376 */ 2377 if ((flags & _UC_XSAVE) != 0) { 2378 const struct xsave_header *user_xsave = 2379 (void *)(uintptr_t)mcp->__fpregs.__xsave.__xsaveptr; 2380 2381 xsavelen = mcp->__fpregs.__xsave.__xsavelen; 2382 error = process_verify_xsavelen(l, xsavelen); 2383 if (error != 0) 2384 goto out; 2385 xsavebuf = kmem_alloc(xsavelen, KM_SLEEP); 2386 error = copyin(user_xsave, xsavebuf, xsavelen); 2387 if (error != 0) 2388 goto out; 2389 error = process_verify_xsave(l, xsavebuf, xsavelen); 2390 if (error != 0) 2391 goto out; 2392 } 2393 2394 if ((flags & _UC_CPU) != 0) { 2395 error = cpu_mcontext_validate(l, mcp); 2396 if (error != 0) 2397 goto out; 2398 2399 tf->tf_rdi = gr[_REG_RDI]; 2400 tf->tf_rsi = gr[_REG_RSI]; 2401 tf->tf_rdx = gr[_REG_RDX]; 2402 tf->tf_r10 = gr[_REG_R10]; 2403 tf->tf_r8 = gr[_REG_R8]; 2404 tf->tf_r9 = gr[_REG_R9]; 2405 /* argX not touched */ 2406 tf->tf_rcx = gr[_REG_RCX]; 2407 tf->tf_r11 = gr[_REG_R11]; 2408 tf->tf_r12 = gr[_REG_R12]; 2409 tf->tf_r13 = gr[_REG_R13]; 2410 tf->tf_r14 = gr[_REG_R14]; 2411 tf->tf_r15 = gr[_REG_R15]; 2412 tf->tf_rbp = gr[_REG_RBP]; 2413 tf->tf_rbx = gr[_REG_RBX]; 2414 tf->tf_rax = gr[_REG_RAX]; 2415 tf->tf_gs = 0; 2416 tf->tf_fs = 0; 2417 tf->tf_es = GSEL(GUDATA_SEL, SEL_UPL); 2418 tf->tf_ds = GSEL(GUDATA_SEL, SEL_UPL); 2419 /* trapno, err not touched */ 2420 tf->tf_rip = gr[_REG_RIP]; 2421 tf->tf_cs = LSEL(LUCODE_SEL, SEL_UPL); 2422 rflags = tf->tf_rflags; 2423 rflags &= ~PSL_USER; 2424 tf->tf_rflags = rflags | (gr[_REG_RFLAGS] & PSL_USER); 2425 tf->tf_rsp = gr[_REG_RSP]; 2426 tf->tf_ss = LSEL(LUDATA_SEL, SEL_UPL); 2427 2428 l->l_md.md_flags |= MDL_IRET; 2429 } 2430 2431 if ((flags & _UC_XSAVE) != 0) { 2432 KASSERT(xsavebuf != NULL); 2433 process_write_xsave(l, xsavebuf, xsavelen); 2434 } else if ((flags & _UC_FPU) != 0) { 2435 process_write_fpregs_xmm(l, 2436 (const struct fxsave *)&mcp->__fpregs.__fxsave); 2437 } 2438 2439 if ((flags & _UC_TLSBASE) != 0) 2440 lwp_setprivate(l, (void *)(uintptr_t)mcp->_mc_tlsbase); 2441 2442 mutex_enter(p->p_lock); 2443 if (flags & _UC_SETSTACK) 2444 l->l_sigstk.ss_flags |= SS_ONSTACK; 2445 if (flags & _UC_CLRSTACK) 2446 l->l_sigstk.ss_flags &= ~SS_ONSTACK; 2447 mutex_exit(p->p_lock); 2448 2449 /* Success! */ 2450 error = 0; 2451 2452 out: if (xsavebuf) 2453 kmem_free(xsavebuf, xsavelen); 2454 return error; 2455 } 2456 2457 int 2458 cpu_mcontext_validate(struct lwp *l, const mcontext_t *mcp) 2459 { 2460 struct proc *p __diagused = l->l_proc; 2461 struct trapframe *tf = l->l_md.md_regs; 2462 const __greg_t *gr; 2463 uint16_t sel; 2464 2465 KASSERT((p->p_flag & PK_32) == 0); 2466 gr = mcp->__gregs; 2467 2468 if (((gr[_REG_RFLAGS] ^ tf->tf_rflags) & PSL_USERSTATIC) != 0) 2469 return EINVAL; 2470 2471 sel = gr[_REG_ES] & 0xffff; 2472 if (sel != 0 && !VALID_USER_DSEL(sel)) 2473 return EINVAL; 2474 2475 sel = gr[_REG_FS] & 0xffff; 2476 if (sel != 0 && !VALID_USER_DSEL(sel)) 2477 return EINVAL; 2478 2479 sel = gr[_REG_GS] & 0xffff; 2480 if (sel != 0 && !VALID_USER_DSEL(sel)) 2481 return EINVAL; 2482 2483 sel = gr[_REG_DS] & 0xffff; 2484 if (!VALID_USER_DSEL(sel)) 2485 return EINVAL; 2486 2487 #ifndef XENPV 2488 sel = gr[_REG_SS] & 0xffff; 2489 if (!VALID_USER_DSEL(sel)) 2490 return EINVAL; 2491 2492 sel = gr[_REG_CS] & 0xffff; 2493 if (!VALID_USER_CSEL(sel)) 2494 return EINVAL; 2495 #endif 2496 2497 if (gr[_REG_RIP] >= VM_MAXUSER_ADDRESS) 2498 return EINVAL; 2499 2500 return 0; 2501 } 2502 2503 int 2504 mm_md_kernacc(void *ptr, vm_prot_t prot, bool *handled) 2505 { 2506 const vaddr_t v = (vaddr_t)ptr; 2507 vaddr_t kva, kva_end; 2508 size_t i; 2509 2510 kva = bootspace.head.va; 2511 kva_end = kva + bootspace.head.sz; 2512 if (v >= kva && v < kva_end) { 2513 *handled = true; 2514 return 0; 2515 } 2516 2517 for (i = 0; i < BTSPACE_NSEGS; i++) { 2518 kva = bootspace.segs[i].va; 2519 kva_end = kva + bootspace.segs[i].sz; 2520 if (v < kva || v >= kva_end) 2521 continue; 2522 *handled = true; 2523 if (bootspace.segs[i].type == BTSEG_TEXT || 2524 bootspace.segs[i].type == BTSEG_RODATA) { 2525 if (prot & VM_PROT_WRITE) { 2526 return EFAULT; 2527 } 2528 } 2529 return 0; 2530 } 2531 2532 kva = bootspace.boot.va; 2533 kva_end = kva + bootspace.boot.sz; 2534 if (v >= kva && v < kva_end) { 2535 *handled = true; 2536 return 0; 2537 } 2538 2539 if (v >= bootspace.smodule && v < bootspace.emodule) { 2540 *handled = true; 2541 vm_map_lock_read(module_map); 2542 const bool allowed = 2543 uvm_map_checkprot(module_map, v, v + 1, prot); 2544 vm_map_unlock_read(module_map); 2545 if (!allowed) { 2546 return EFAULT; 2547 } 2548 } else { 2549 *handled = false; 2550 } 2551 return 0; 2552 } 2553 2554 /* 2555 * Zero out a 64bit LWP's segments registers. Used when exec'ing a new 2556 * 64bit program. 2557 */ 2558 void 2559 cpu_segregs64_zero(struct lwp *l) 2560 { 2561 struct trapframe * const tf = l->l_md.md_regs; 2562 struct pcb *pcb; 2563 uint64_t zero = 0; 2564 2565 KASSERT(kpreempt_disabled()); 2566 KASSERT((l->l_proc->p_flag & PK_32) == 0); 2567 KASSERT(l == curlwp); 2568 2569 pcb = lwp_getpcb(l); 2570 2571 tf->tf_fs = 0; 2572 tf->tf_gs = 0; 2573 setds(GSEL(GUDATA_SEL, SEL_UPL)); 2574 setes(GSEL(GUDATA_SEL, SEL_UPL)); 2575 setfs(0); 2576 setusergs(0); 2577 2578 #ifndef XENPV 2579 wrmsr(MSR_FSBASE, 0); 2580 wrmsr(MSR_KERNELGSBASE, 0); 2581 #else 2582 HYPERVISOR_set_segment_base(SEGBASE_FS, 0); 2583 HYPERVISOR_set_segment_base(SEGBASE_GS_USER, 0); 2584 #endif 2585 2586 pcb->pcb_fs = 0; 2587 pcb->pcb_gs = 0; 2588 update_descriptor(&curcpu()->ci_gdt[GUFS_SEL], &zero); 2589 update_descriptor(&curcpu()->ci_gdt[GUGS_SEL], &zero); 2590 } 2591 2592 /* 2593 * Zero out a 32bit LWP's segments registers. Used when exec'ing a new 2594 * 32bit program. 2595 */ 2596 void 2597 cpu_segregs32_zero(struct lwp *l) 2598 { 2599 struct trapframe * const tf = l->l_md.md_regs; 2600 struct pcb *pcb; 2601 uint64_t zero = 0; 2602 2603 KASSERT(kpreempt_disabled()); 2604 KASSERT(l->l_proc->p_flag & PK_32); 2605 KASSERT(l == curlwp); 2606 2607 pcb = lwp_getpcb(l); 2608 2609 tf->tf_fs = 0; 2610 tf->tf_gs = 0; 2611 setds(GSEL(GUDATA32_SEL, SEL_UPL)); 2612 setes(GSEL(GUDATA32_SEL, SEL_UPL)); 2613 setfs(0); 2614 setusergs(0); 2615 pcb->pcb_fs = 0; 2616 pcb->pcb_gs = 0; 2617 update_descriptor(&curcpu()->ci_gdt[GUFS_SEL], &zero); 2618 update_descriptor(&curcpu()->ci_gdt[GUGS_SEL], &zero); 2619 } 2620 2621 /* 2622 * Load an LWP's TLS context, possibly changing the %fs and %gs selectors. 2623 * Used only for 32-bit processes. 2624 */ 2625 void 2626 cpu_fsgs_reload(struct lwp *l, int fssel, int gssel) 2627 { 2628 struct trapframe *tf; 2629 struct pcb *pcb; 2630 2631 KASSERT(l->l_proc->p_flag & PK_32); 2632 KASSERT(l == curlwp); 2633 2634 tf = l->l_md.md_regs; 2635 fssel &= 0xFFFF; 2636 gssel &= 0xFFFF; 2637 2638 pcb = lwp_getpcb(l); 2639 kpreempt_disable(); 2640 update_descriptor(&curcpu()->ci_gdt[GUFS_SEL], &pcb->pcb_fs); 2641 update_descriptor(&curcpu()->ci_gdt[GUGS_SEL], &pcb->pcb_gs); 2642 2643 #ifdef XENPV 2644 setusergs(gssel); 2645 #endif 2646 2647 tf->tf_fs = fssel; 2648 tf->tf_gs = gssel; 2649 kpreempt_enable(); 2650 } 2651 2652 bool 2653 mm_md_direct_mapped_io(void *addr, paddr_t *paddr) 2654 { 2655 vaddr_t va = (vaddr_t)addr; 2656 2657 #ifdef __HAVE_DIRECT_MAP 2658 if (va >= PMAP_DIRECT_BASE && va < PMAP_DIRECT_END) { 2659 *paddr = PMAP_DIRECT_UNMAP(va); 2660 return true; 2661 } 2662 #else 2663 __USE(va); 2664 #endif 2665 2666 return false; 2667 } 2668 2669 bool 2670 mm_md_direct_mapped_phys(paddr_t paddr, vaddr_t *vaddr) 2671 { 2672 #ifdef __HAVE_DIRECT_MAP 2673 *vaddr = PMAP_DIRECT_MAP(paddr); 2674 return true; 2675 #else 2676 return false; 2677 #endif 2678 } 2679 2680 static void 2681 idt_vec_copy(struct idt_vec *dst, struct idt_vec *src) 2682 { 2683 idt_descriptor_t *idt_dst; 2684 2685 idt_dst = dst->iv_idt; 2686 2687 kpreempt_disable(); 2688 pmap_changeprot_local((vaddr_t)idt_dst, VM_PROT_READ|VM_PROT_WRITE); 2689 2690 memcpy(idt_dst, src->iv_idt, PAGE_SIZE); 2691 memcpy(dst->iv_allocmap, src->iv_allocmap, sizeof(dst->iv_allocmap)); 2692 2693 pmap_changeprot_local((vaddr_t)idt_dst, VM_PROT_READ); 2694 kpreempt_enable(); 2695 } 2696 2697 void 2698 idt_vec_init_cpu_md(struct idt_vec *iv, cpuid_t cid) 2699 { 2700 vaddr_t va; 2701 2702 if (cid != cpu_index(&cpu_info_primary) && 2703 idt_vec_is_pcpu()) { 2704 #ifdef __HAVE_PCPU_AREA 2705 va = (vaddr_t)&pcpuarea->ent[cid].idt; 2706 #else 2707 struct vm_page *pg; 2708 2709 va = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, 2710 UVM_KMF_VAONLY); 2711 pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO); 2712 if (pg == NULL) { 2713 panic("failed to allocate a page for IDT"); 2714 } 2715 pmap_kenter_pa(va, VM_PAGE_TO_PHYS(pg), 2716 VM_PROT_READ|VM_PROT_WRITE, 0); 2717 pmap_update(pmap_kernel()); 2718 #endif 2719 2720 memset((void *)va, 0, PAGE_SIZE); 2721 #ifndef XENPV 2722 pmap_changeprot_local(va, VM_PROT_READ); 2723 #endif 2724 pmap_update(pmap_kernel()); 2725 2726 iv->iv_idt = (void *)va; 2727 idt_vec_copy(iv, &(cpu_info_primary.ci_idtvec)); 2728 } else { 2729 iv->iv_idt = (void *)idt_vaddr; 2730 } 2731 } 2732
Indexes created Sun Oct 25 00:26:45 UTC 2026