1 /* $NetBSD: machdep.c,v 1.383 2026/11/10 03:43:25 thorpej Exp $ */ 2 3 /* 4 * Copyright (c) 1996, 1997, 1998, 2000, 2006, 2007, 2008, 2011 5 * The NetBSD Foundation, Inc. 6 * All rights reserved. 7 * 8 * This code is derived from software contributed to The NetBSD Foundation 9 * by Charles M. Hannum and by Jason R. Thorpe of the Numerical Aerospace 10 * Simulation Facility, NASA Ames Research Center. 11 * 12 * This code is derived from software contributed to The NetBSD Foundation 13 * by Coyote Point Systems, Inc. which was written under contract to Coyote 14 * Point by Jed Davis and Devon O'Dell. 15 * 16 * Redistribution and use in source and binary forms, with or without 17 * modification, are permitted provided that the following conditions 18 * are met: 19 * 1. Redistributions of source code must retain the above copyright 20 * notice, this list of conditions and the following disclaimer. 21 * 2. Redistributions in binary form must reproduce the above copyright 22 * notice, this list of conditions and the following disclaimer in the 23 * documentation and/or other materials provided with the distribution. 24 * 25 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 26 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 27 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 28 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 29 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 30 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 31 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 32 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 33 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 34 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 35 * POSSIBILITY OF SUCH DAMAGE. 36 */ 37 38 /* 39 * Copyright (c) 2006 Mathieu Ropert <mro (at) adviseo.fr> 40 * 41 * Permission to use, copy, modify, and distribute this software for any 42 * purpose with or without fee is hereby granted, provided that the above 43 * copyright notice and this permission notice appear in all copies. 44 * 45 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 46 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 47 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 48 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 49 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 50 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 51 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 52 */ 53 54 /* 55 * Copyright (c) 2007 Manuel Bouyer. 56 * 57 * Redistribution and use in source and binary forms, with or without 58 * modification, are permitted provided that the following conditions 59 * are met: 60 * 1. Redistributions of source code must retain the above copyright 61 * notice, this list of conditions and the following disclaimer. 62 * 2. Redistributions in binary form must reproduce the above copyright 63 * notice, this list of conditions and the following disclaimer in the 64 * documentation and/or other materials provided with the distribution. 65 * 66 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 67 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 68 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 69 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 70 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 71 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 72 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 73 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 74 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 75 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 76 */ 77 78 /* 79 * Copyright (c) 1982, 1987, 1990 The Regents of the University of California. 80 * All rights reserved. 81 * 82 * This code is derived from software contributed to Berkeley by 83 * William Jolitz. 84 * 85 * Redistribution and use in source and binary forms, with or without 86 * modification, are permitted provided that the following conditions 87 * are met: 88 * 1. Redistributions of source code must retain the above copyright 89 * notice, this list of conditions and the following disclaimer. 90 * 2. Redistributions in binary form must reproduce the above copyright 91 * notice, this list of conditions and the following disclaimer in the 92 * documentation and/or other materials provided with the distribution. 93 * 3. Neither the name of the University nor the names of its contributors 94 * may be used to endorse or promote products derived from this software 95 * without specific prior written permission. 96 * 97 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 98 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 99 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 100 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 101 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 102 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 103 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 104 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 105 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 106 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 107 * SUCH DAMAGE. 108 * 109 * @(#)machdep.c 7.4 (Berkeley) 6/3/91 110 */ 111 112 #include <sys/cdefs.h> 113 __KERNEL_RCSID(0, "$NetBSD: machdep.c,v 1.383 2026/11/10 03:43:25 thorpej Exp $"); 114 115 #include "opt_modular.h" 116 #include "opt_user_ldt.h" 117 #include "opt_ddb.h" 118 #include "opt_kgdb.h" 119 #include "opt_cpureset_delay.h" 120 #include "opt_mtrr.h" 121 #include "opt_realmem.h" 122 #include "opt_xen.h" 123 #include "opt_svs.h" 124 #include "opt_kaslr.h" 125 #ifndef XENPV 126 #include "opt_physmem.h" 127 #endif 128 #include "isa.h" 129 #include "pci.h" 130 131 #include <sys/param.h> 132 #include <sys/systm.h> 133 #include <sys/signal.h> 134 #include <sys/signalvar.h> 135 #include <sys/kernel.h> 136 #include <sys/cpu.h> 137 #include <sys/exec.h> 138 #include <sys/exec_aout.h> /* for MID_* */ 139 #include <sys/reboot.h> 140 #include <sys/conf.h> 141 #include <sys/msgbuf.h> 142 #include <sys/mount.h> 143 #include <sys/core.h> 144 #include <sys/kcore.h> 145 #include <sys/ucontext.h> 146 #include <machine/kcore.h> 147 #include <sys/ras.h> 148 #include <sys/syscallargs.h> 149 #include <sys/ksyms.h> 150 #include <sys/device.h> 151 #include <sys/lwp.h> 152 #include <sys/proc.h> 153 #include <sys/asan.h> 154 #include <sys/csan.h> 155 #include <sys/msan.h> 156 #include <sys/module.h> 157 #include <sys/timevar.h> 158 159 #ifdef KGDB 160 #include <sys/kgdb.h> 161 #endif 162 163 #include <lib/libkern/entpool.h> /* XXX */ 164 165 #include <dev/cons.h> 166 #include <dev/mm.h> 167 168 #include <uvm/uvm.h> 169 #include <uvm/uvm_page.h> 170 171 #include <sys/sysctl.h> 172 173 #include <machine/cpu.h> 174 #include <machine/cpu_rng.h> 175 #include <machine/cpufunc.h> 176 #include <machine/gdt.h> 177 #include <machine/intr.h> 178 #include <machine/pio.h> 179 #include <machine/psl.h> 180 #include <machine/reg.h> 181 #include <machine/specialreg.h> 182 #include <machine/bootinfo.h> 183 #include <x86/fpu.h> 184 #include <x86/dbregs.h> 185 #include <machine/mtrr.h> 186 #include <machine/mpbiosvar.h> 187 #include <machine/pmap_private.h> 188 189 #include <x86/bootspace.h> 190 #include <x86/cputypes.h> 191 #include <x86/cpuvar.h> 192 #include <x86/machdep.h> 193 #include <x86/x86/tsc.h> 194 195 #include <dev/isa/isareg.h> 196 #include <machine/isa_machdep.h> 197 #include <dev/ic/i8042reg.h> 198 199 #ifdef XEN 200 #include <xen/xen.h> 201 #include <xen/hypervisor.h> 202 #include <xen/evtchn.h> 203 #include <xen/include/public/version.h> 204 #include <xen/include/public/vcpu.h> 205 #endif /* XEN */ 206 207 #include <ddb/db_active.h> 208 #ifdef DDB 209 #include <machine/db_machdep.h> 210 #include <ddb/db_extern.h> 211 #include <ddb/db_output.h> 212 #include <ddb/db_interface.h> 213 #endif 214 215 #include "acpica.h" 216 217 #if NACPICA > 0 218 #include <dev/acpi/acpivar.h> 219 #define ACPI_MACHDEP_PRIVATE 220 #include <machine/acpi_machdep.h> 221 #else 222 #include <machine/i82489var.h> 223 #endif 224 225 #include "isa.h" 226 #include "isadma.h" 227 #include "ksyms.h" 228 229 /* the following is used externally (sysctl_hw) */ 230 char machine[] = "amd64"; /* CPU "architecture" */ 231 char machine_arch[] = "x86_64"; /* machine == machine_arch */ 232 233 #ifdef CPURESET_DELAY 234 int cpureset_delay = CPURESET_DELAY; 235 #else 236 int cpureset_delay = 2000; /* default to 2s */ 237 #endif 238 239 int cpu_class = CPUCLASS_686; 240 241 #ifdef MTRR 242 const struct mtrr_funcs *mtrr_funcs; 243 #endif 244 245 int cpu_class; 246 int use_pae; 247 248 #ifndef NO_SPARSE_DUMP 249 int sparse_dump = 1; 250 251 paddr_t max_paddr = 0; 252 unsigned char *sparse_dump_physmap; 253 #endif 254 255 char *dump_headerbuf, *dump_headerbuf_ptr; 256 #define dump_headerbuf_size PAGE_SIZE 257 #define dump_headerbuf_end (dump_headerbuf + dump_headerbuf_size) 258 #define dump_headerbuf_avail (dump_headerbuf_end - dump_headerbuf_ptr) 259 daddr_t dump_header_blkno; 260 261 size_t dump_nmemsegs; 262 size_t dump_npages; 263 size_t dump_header_size; 264 size_t dump_totalbytesleft; 265 266 vaddr_t idt_vaddr; 267 paddr_t idt_paddr; 268 vaddr_t gdt_vaddr; 269 paddr_t gdt_paddr; 270 vaddr_t ldt_vaddr; 271 paddr_t ldt_paddr; 272 273 static struct vm_map module_map_store; 274 extern struct bootspace bootspace; 275 extern struct slotspace slotspace; 276 277 vaddr_t vm_min_kernel_address __read_mostly = VM_MIN_KERNEL_ADDRESS_DEFAULT; 278 vaddr_t vm_max_kernel_address __read_mostly = VM_MAX_KERNEL_ADDRESS_DEFAULT; 279 pd_entry_t *pte_base __read_mostly; 280 281 struct vm_map *phys_map = NULL; 282 283 extern paddr_t lowmem_rsvd; 284 extern paddr_t avail_start, avail_end; 285 #ifdef XENPV 286 extern paddr_t pmap_pa_start, pmap_pa_end; 287 #endif 288 289 struct nmistore { 290 uint64_t cr3; 291 uint64_t scratch; 292 } __packed; 293 294 /* 295 * Size of memory segments, before any memory is stolen. 296 */ 297 phys_ram_seg_t mem_clusters[VM_PHYSSEG_MAX]; 298 int mem_cluster_cnt; 299 300 int cpu_dump(void); 301 int cpu_dumpsize(void); 302 u_long cpu_dump_mempagecnt(void); 303 void dodumpsys(void); 304 void dumpsys(void); 305 306 static void x86_64_proc0_pcb_ldt_init(void); 307 308 void dump_misc_init(void); 309 void dump_seg_prep(void); 310 int dump_seg_iter(int (*)(paddr_t, paddr_t)); 311 312 #ifndef NO_SPARSE_DUMP 313 void sparse_dump_reset(void); 314 void sparse_dump_mark(void); 315 void cpu_dump_prep_sparse(void); 316 #endif 317 318 void dump_header_start(void); 319 int dump_header_flush(void); 320 int dump_header_addbytes(const void*, size_t); 321 int dump_header_addseg(paddr_t, paddr_t); 322 int dump_header_finish(void); 323 324 int dump_seg_count_range(paddr_t, paddr_t); 325 int dumpsys_seg(paddr_t, paddr_t); 326 327 void init_bootspace(void); 328 void init_slotspace(void); 329 void init_x86_64(paddr_t); 330 331 static int cpu_getmcontext_xsave(struct lwp *, mcontext_t *, unsigned *, 332 const struct xsave_header *, size_t, struct xsave_header *); 333 334 /* 335 * Machine-dependent startup code 336 */ 337 void 338 cpu_startup(void) 339 { 340 int x, y; 341 vaddr_t minaddr, maxaddr; 342 psize_t sz; 343 344 /* 345 * For console drivers that require uvm and pmap to be initialized, 346 * we'll give them one more chance here... 347 */ 348 consinit(); 349 350 /* 351 * Initialize error message buffer (at end of core). 352 */ 353 if (msgbuf_p_cnt == 0) 354 panic("msgbuf paddr map has not been set up"); 355 for (x = 0, sz = 0; x < msgbuf_p_cnt; sz += msgbuf_p_seg[x++].sz) 356 continue; 357 358 msgbuf_vaddr = uvm_km_alloc(kernel_map, sz, 0, UVM_KMF_VAONLY); 359 if (msgbuf_vaddr == 0) 360 panic("failed to valloc msgbuf_vaddr"); 361 362 for (y = 0, sz = 0; y < msgbuf_p_cnt; y++) { 363 for (x = 0; x < btoc(msgbuf_p_seg[y].sz); x++, sz += PAGE_SIZE) 364 pmap_kenter_pa((vaddr_t)msgbuf_vaddr + sz, 365 msgbuf_p_seg[y].paddr + x * PAGE_SIZE, 366 VM_PROT_READ|VM_PROT_WRITE, 0); 367 } 368 369 pmap_update(pmap_kernel()); 370 371 initmsgbuf((void *)msgbuf_vaddr, round_page(sz)); 372 373 minaddr = 0; 374 375 /* 376 * Allocate a submap for physio. 377 */ 378 phys_map = uvm_km_suballoc(kernel_map, &minaddr, &maxaddr, 379 VM_PHYS_SIZE, 0, false, NULL); 380 381 /* 382 * Create the module map. 383 * 384 * The kernel uses RIP-relative addressing with a maximum offset of 385 * 2GB. Because of that, we can't put the kernel modules in kernel_map 386 * (like i386 does), since kernel_map is too far away in memory from 387 * the kernel sections. So we have to create a special module_map. 388 * 389 * The module map is taken as what is left of the bootstrap memory 390 * created in locore/prekern. 391 */ 392 uvm_map_setup(&module_map_store, bootspace.smodule, 393 bootspace.emodule, 0); 394 module_map_store.pmap = pmap_kernel(); 395 module_map = &module_map_store; 396 397 /* Say hello. */ 398 banner(); 399 400 #if NISA > 0 || NPCI > 0 401 /* Safe for i/o port / memory space allocation to use malloc now. */ 402 x86_bus_space_mallocok(); 403 #endif 404 405 #ifdef __HAVE_PCPU_AREA 406 cpu_pcpuarea_init(&cpu_info_primary); 407 #endif 408 gdt_init(); 409 x86_64_proc0_pcb_ldt_init(); 410 411 cpu_init_tss(&cpu_info_primary); 412 #if !defined(XENPV) 413 ltr(cpu_info_primary.ci_tss_sel); 414 #endif 415 416 x86_startup(); 417 } 418 419 #ifdef XENPV 420 /* used in assembly */ 421 void hypervisor_callback(void); 422 void failsafe_callback(void); 423 void x86_64_switch_context(struct pcb *); 424 void x86_64_tls_switch(struct lwp *); 425 426 void 427 x86_64_switch_context(struct pcb *new) 428 { 429 HYPERVISOR_stack_switch(GSEL(GDATA_SEL, SEL_KPL), new->pcb_rsp0); 430 struct physdev_set_iopl set_iopl; 431 set_iopl.iopl = new->pcb_iopl; 432 HYPERVISOR_physdev_op(PHYSDEVOP_set_iopl, &set_iopl); 433 } 434 435 void 436 x86_64_tls_switch(struct lwp *l) 437 { 438 struct cpu_info *ci = curcpu(); 439 struct pcb *pcb = lwp_getpcb(l); 440 struct trapframe *tf = l->l_md.md_regs; 441 uint64_t zero = 0; 442 443 /* 444 * Raise the IPL to IPL_HIGH. XXX Still needed? 445 */ 446 (void)splhigh(); 447 448 /* Update segment registers */ 449 if (pcb->pcb_flags & PCB_COMPAT32) { 450 update_descriptor(&ci->ci_gdt[GUFS_SEL], &pcb->pcb_fs); 451 update_descriptor(&ci->ci_gdt[GUGS_SEL], &pcb->pcb_gs); 452 setds(GSEL(GUDATA32_SEL, SEL_UPL)); 453 setes(GSEL(GUDATA32_SEL, SEL_UPL)); 454 setfs(GSEL(GUDATA32_SEL, SEL_UPL)); 455 HYPERVISOR_set_segment_base(SEGBASE_GS_USER_SEL, tf->tf_gs); 456 } else { 457 update_descriptor(&ci->ci_gdt[GUFS_SEL], &zero); 458 update_descriptor(&ci->ci_gdt[GUGS_SEL], &zero); 459 setds(GSEL(GUDATA_SEL, SEL_UPL)); 460 setes(GSEL(GUDATA_SEL, SEL_UPL)); 461 setfs(0); 462 HYPERVISOR_set_segment_base(SEGBASE_GS_USER_SEL, 0); 463 HYPERVISOR_set_segment_base(SEGBASE_FS, pcb->pcb_fs); 464 HYPERVISOR_set_segment_base(SEGBASE_GS_USER, pcb->pcb_gs); 465 } 466 } 467 #endif /* XENPV */ 468 469 /* 470 * Set up proc0's PCB and LDT. 471 */ 472 static void 473 x86_64_proc0_pcb_ldt_init(void) 474 { 475 struct lwp *l = &lwp0; 476 struct pcb *pcb = lwp_getpcb(l); 477 478 pcb->pcb_flags = 0; 479 pcb->pcb_fs = 0; 480 pcb->pcb_gs = 0; 481 pcb->pcb_rsp0 = (uvm_lwp_getuarea(l) + USPACE - 16) & ~0xf; 482 pcb->pcb_iopl = IOPL_KPL; 483 pcb->pcb_dbregs = NULL; 484 pcb->pcb_cr0 = rcr0() & ~CR0_TS; 485 l->l_md.md_regs = (struct trapframe *)pcb->pcb_rsp0 - 1; 486 487 #if !defined(XENPV) 488 lldt(GSYSSEL(GLDT_SEL, SEL_KPL)); 489 #else 490 xen_set_ldt((vaddr_t)ldtstore, LDT_SIZE >> 3); 491 /* Reset TS bit and set kernel stack for interrupt handlers */ 492 HYPERVISOR_fpu_taskswitch(1); 493 HYPERVISOR_stack_switch(GSEL(GDATA_SEL, SEL_KPL), pcb->pcb_rsp0); 494 struct physdev_set_iopl set_iopl; 495 set_iopl.iopl = pcb->pcb_iopl; 496 HYPERVISOR_physdev_op(PHYSDEVOP_set_iopl, &set_iopl); 497 #endif 498 } 499 500 /* 501 * Set up TSS and I/O bitmap. 502 */ 503 void 504 cpu_init_tss(struct cpu_info *ci) 505 { 506 #ifdef __HAVE_PCPU_AREA 507 const cpuid_t cid = cpu_index(ci); 508 #endif 509 struct cpu_tss *cputss; 510 struct nmistore *store; 511 uintptr_t p; 512 513 #ifdef __HAVE_PCPU_AREA 514 cputss = (struct cpu_tss *)&pcpuarea->ent[cid].tss; 515 #else 516 cputss = (struct cpu_tss *)uvm_km_alloc(kernel_map, 517 sizeof(struct cpu_tss), 0, UVM_KMF_WIRED|UVM_KMF_ZERO); 518 #endif 519 520 cputss->tss.tss_iobase = IOMAP_INVALOFF << 16; 521 522 /* DDB stack */ 523 #ifdef __HAVE_PCPU_AREA 524 p = (vaddr_t)&pcpuarea->ent[cid].ist0; 525 #else 526 p = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, UVM_KMF_WIRED|UVM_KMF_ZERO); 527 #endif 528 cputss->tss.tss_ist[0] = p + PAGE_SIZE - 16; 529 530 /* double fault */ 531 #ifdef __HAVE_PCPU_AREA 532 p = (vaddr_t)&pcpuarea->ent[cid].ist1; 533 #else 534 p = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, UVM_KMF_WIRED|UVM_KMF_ZERO); 535 #endif 536 cputss->tss.tss_ist[1] = p + PAGE_SIZE - 16; 537 538 /* NMI - store a structure at the top of the stack */ 539 #ifdef __HAVE_PCPU_AREA 540 p = (vaddr_t)&pcpuarea->ent[cid].ist2; 541 #else 542 p = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, UVM_KMF_WIRED|UVM_KMF_ZERO); 543 #endif 544 cputss->tss.tss_ist[2] = p + PAGE_SIZE - sizeof(struct nmistore); 545 store = (struct nmistore *)(p + PAGE_SIZE - sizeof(struct nmistore)); 546 store->cr3 = pmap_pdirpa(pmap_kernel(), 0); 547 548 /* DB */ 549 #ifdef __HAVE_PCPU_AREA 550 p = (vaddr_t)&pcpuarea->ent[cid].ist3; 551 #else 552 p = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, UVM_KMF_WIRED|UVM_KMF_ZERO); 553 #endif 554 cputss->tss.tss_ist[3] = p + PAGE_SIZE - 16; 555 556 ci->ci_tss = cputss; 557 ci->ci_tss_sel = tss_alloc(&cputss->tss); 558 } 559 560 void 561 buildcontext(struct lwp *l, void *catcher, void *f) 562 { 563 struct trapframe *tf = l->l_md.md_regs; 564 565 tf->tf_ds = GSEL(GUDATA_SEL, SEL_UPL); 566 tf->tf_es = GSEL(GUDATA_SEL, SEL_UPL); 567 tf->tf_fs = GSEL(GUDATA_SEL, SEL_UPL); 568 tf->tf_gs = GSEL(GUDATA_SEL, SEL_UPL); 569 570 tf->tf_rip = (uint64_t)catcher; 571 tf->tf_cs = GSEL(GUCODE_SEL, SEL_UPL); 572 tf->tf_rflags &= ~PSL_CLEARSIG; 573 tf->tf_rsp = (uint64_t)f; 574 tf->tf_ss = GSEL(GUDATA_SEL, SEL_UPL); 575 576 /* Ensure FP state is sane */ 577 fpu_sigreset(l); 578 } 579 580 void 581 sendsig_sigcontext(const ksiginfo_t *ksi, const sigset_t *mask) 582 { 583 584 printf("sendsig_sigcontext: illegal\n"); 585 sigexit(curlwp, SIGILL); 586 } 587 588 /* 589 * sendsig_siginfo(ksi, mask) 590 * 591 * Set up a stack frame and registers for signal delivery with the 592 * given siginfo and signal mask. 593 * 594 * The stack frame layout is as follows, with XSAVE parts omitted 595 * (and _UC_XSAVE clear in uc_flags) if the process doesn't have 596 * XSAVE state to restore: 597 * 598 * / +-----------------------+ 599 * | | (other XSAVE stuff) | 600 * | | --------------------- | 601 * | | Hi16_ZMM | 602 * | | --------------------- | 603 * XSAVE < | ZMM_Hi256 | 604 * | | --------------------- | 605 * | | YMM_Hi128 | 606 * | *-----------------------* \ 607 * | | FXSAVE in ucontext_t | | 608 * \ *-----------------------* > ucontext_t 609 * | rest of ucontext_t | | 610 * +-----------------------+ / 611 * | siginfo_t | 612 * +-----------------------+ =0 (mod 16) 613 * | return address | 614 * +-----------------------+ <--- rsp on signal handler entry 615 * 616 * Additionally, when XSAVE is in use, there is a pointer to the 617 * XSAVE area in some architecturally unused padding in the FXSAVE 618 * area. This is important because on i386, the XSAVE area 619 * _cannot_ overlap the ucontext_t owing to past ABI constraints, 620 * so applications running on both i386 and amd64 can use the same 621 * code to get the XSAVE pointer (and for 11.0, but not any 622 * earlier or later versions, the XSAVE pointer was also needed 623 * for signal handlers to write registers back on amd64 because 624 * the XSAVE area and the ucontext_t did not overlap). 625 * 626 * The overlap of the FXSAVE section in the XSAVE area and the 627 * FXSAVE section in ucontext_t is tricky but important: signal 628 * handlers use it to read, and potentially write back, the x87 629 * and xmm registers. Consider an application with two parts 630 * (e.g., a new library that uses an AVX512 optimization once it 631 * is available, and an old part running unmodified existing code 632 * predating NetBSD's AVX512 support): 633 * 634 * 1. Uses the high 16 zmm registers. 635 * 2. (a) Uses _only_ x87/xmm registers, and 636 * (b) has a signal handler that reads and writes them back. 637 * 638 * After part (1), the system will consider XSAVE necessary to 639 * save and restore the high 16 zmm registers, even if -- as is 640 * customary when using the ymm or low 16 zmm registers -- the 641 * application issues VZEROUPPER when part (1) is done. For this 642 * application to work without changing the signal handler in part 643 * (2)(b) to teach it about the XSAVE pointer -- i.e., for 644 * compatibility with existing code -- the XSAVE area and the 645 * ucontext_t MUST overlap. (11.0 would copy out the FXSAVE area 646 * to both locations so that unmodified signal handlers could 647 * _read_ the x87/xmm registers, but _writeback_ would be 648 * ignored.) 649 * 650 * For more background, see: 651 * 652 * PR kern/60539: XSAVE changes break ucontext userspace API 653 * https://gnats.NetBSD.org/60539 654 */ 655 void 656 sendsig_siginfo(const ksiginfo_t *ksi, const sigset_t *mask) 657 { 658 struct lwp *l = curlwp; 659 struct proc *p = l->l_proc; 660 struct sigacts *ps = p->p_sigacts; 661 int onstack, error; 662 int sig = ksi->ksi_signo; 663 struct sigframe_siginfo *fp, frame; 664 sig_t catcher = SIGACTION(p, sig).sa_handler; 665 struct trapframe *tf = l->l_md.md_regs; 666 const struct xsave_header *xsavebuf = NULL; 667 size_t xsavelen = 0; 668 struct xsave_header *user_xsave = NULL; 669 char *sp; 670 671 KASSERT(mutex_owned(p->p_lock)); 672 673 /* Do we need to jump onto the signal stack? */ 674 onstack = 675 (l->l_sigstk.ss_flags & (SS_DISABLE | SS_ONSTACK)) == 0 && 676 (SIGACTION(p, sig).sa_flags & SA_ONSTACK) != 0; 677 678 /* Allocate space for the signal handler context. */ 679 if (onstack) { 680 KASSERT(l->l_sigstk.ss_size >= MINSIGSTKSZ); 681 sp = ((char *)l->l_sigstk.ss_sp + l->l_sigstk.ss_size); 682 } else { 683 /* AMD64 ABI 128-bytes "red zone". */ 684 sp = (char *)tf->tf_rsp - 128; 685 } 686 687 /* 688 * The maximum amount of space we might use, including padding 689 * for alignment, had better fit in MINSIGSTKSZ. 690 * 691 * If this changes because you have increased XSAVE_MAX_BYTES, 692 * you need to work out the ABI change for MINSIGSTKSZ. 693 */ 694 __CTASSERT(8 + STACK_ALIGNBYTES + sizeof(struct sigframe_siginfo) + 695 (XSAVE_ALIGN - 1) + XSAVE_MAX_BYTES <= MINSIGSTKSZ); 696 697 /* 698 * Find whether we need to allocate a separate XSAVE area, 699 * because the user program has used extended CPU state beyond 700 * the x87/SSE registers, or whether we can get by with just an 701 * FXSAVE area. 702 */ 703 if (process_xsave_needed_p(l)) { 704 enum { 705 overlap = (sizeof(struct sigframe_siginfo) - 706 offsetof(struct sigframe_siginfo, 707 sf_uc.uc_mcontext.__fpregs)), 708 }; 709 710 process_read_xsave(l, &xsavebuf, &xsavelen); 711 KASSERT(xsavebuf != NULL); 712 KASSERT(xsavelen <= XSAVE_MAX_BYTES); 713 CTASSERT(overlap <= XSAVE_MAX_BYTES); 714 715 KASSERT(!onstack || sp >= (char *)l->l_sigstk.ss_sp); 716 KASSERT(!onstack || 717 sp - (char *)l->l_sigstk.ss_sp >= xsavelen); 718 KASSERT(!onstack || 719 sp - (char *)l->l_sigstk.ss_sp >= overlap); 720 sp -= MAX(xsavelen, overlap); 721 722 KASSERT(!onstack || sp >= (char *)l->l_sigstk.ss_sp); 723 KASSERT(!onstack || 724 sp - (char *)l->l_sigstk.ss_sp >= XSAVE_ALIGN - 1); 725 sp = (char *)((uintptr_t)sp & ~(XSAVE_ALIGN - 1)); 726 727 KASSERT(!onstack || sp >= (char *)l->l_sigstk.ss_sp); 728 KASSERT(((uintptr_t)sp & (XSAVE_ALIGN - 1)) == 0); 729 user_xsave = (void *)sp; 730 731 CTASSERT((overlap % XSAVE_ALIGN) == 0); 732 CTASSERT((overlap & STACK_ALIGNBYTES) == 0); 733 CTASSERT(((overlap - sizeof(struct sigframe_siginfo)) & 734 STACK_ALIGNBYTES) == (STACK_ALIGNBYTES + 1) - 8); 735 sp += overlap; 736 } 737 738 /* 739 * Reserve space for a struct sigframe_siginfo. The first 740 * member is a return address, as if we are entering a 741 * procedure with a CALL instruction with a stack frame aligned 742 * to a multiple of 16 bytes; the rest of the structure is that 743 * stack frame. So make sure the structure starts at an 744 * address congruent to 8 modulo 16. 745 */ 746 KASSERT(!onstack || sp >= (char *)l->l_sigstk.ss_sp); 747 KASSERT(!onstack || (size_t)(sp - (char *)l->l_sigstk.ss_sp) >= 748 8 + STACK_ALIGNBYTES + sizeof(struct sigframe_siginfo)); 749 sp -= sizeof(struct sigframe_siginfo); 750 /* Round down the stackpointer to a multiple of 16 for the ABI. */ 751 if (((uintptr_t)sp & STACK_ALIGNBYTES) == 752 (STACK_ALIGNBYTES + 1) - 8) { 753 fp = (struct sigframe_siginfo *)sp; 754 } else { 755 fp = (struct sigframe_siginfo *)(((uintptr_t)sp & 756 ~STACK_ALIGNBYTES) - 8); 757 } 758 KASSERT(!onstack || (char *)fp >= (char *)l->l_sigstk.ss_sp); 759 KASSERT(((uintptr_t)fp & STACK_ALIGNBYTES) == 760 (STACK_ALIGNBYTES + 1) - 8); 761 762 /* 763 * If we have to use XSAVE, the FXSAVE area of the ucontext_t 764 * on the user's stack must line up with the FXSAVE subarea of 765 * the XSAVE area on the user's stack. 766 */ 767 KASSERT(xsavebuf == 0 || 768 (((uintptr_t)&fp->sf_uc.uc_mcontext.__fpregs & (XSAVE_ALIGN - 1)) 769 == 0)); 770 KASSERT(xsavebuf == 0 || (uintptr_t)user_xsave == 771 (uintptr_t)&fp->sf_uc.uc_mcontext.__fpregs); 772 773 memset(&frame, 0, sizeof(frame)); 774 frame.sf_ra = (uint64_t)ps->sa_sigdesc[sig].sd_tramp; 775 frame.sf_si._info = ksi->ksi_info; 776 frame.sf_uc.uc_flags = _UC_SIGMASK; 777 frame.sf_uc.uc_sigmask = *mask; 778 frame.sf_uc.uc_link = l->l_ctxlink; 779 frame.sf_uc.uc_flags |= (l->l_sigstk.ss_flags & SS_ONSTACK) 780 ? _UC_SETSTACK : _UC_CLRSTACK; 781 sendsig_reset(l, sig); 782 783 mutex_exit(p->p_lock); 784 cpu_getmcontext(l, &frame.sf_uc.uc_mcontext, &frame.sf_uc.uc_flags); 785 786 /* 787 * If we have to use XSAVE, copy out the part of it past the 788 * FXSAVE area separately -- and be ready to bail if it failed. 789 */ 790 if (xsavebuf) { 791 KASSERT((void *)&fp->sf_uc.uc_mcontext.__fpregs == 792 (void *)user_xsave); 793 error = cpu_getmcontext_xsave(l, &frame.sf_uc.uc_mcontext, 794 &frame.sf_uc.uc_flags, xsavebuf, xsavelen, user_xsave); 795 if (error != 0) 796 goto relock; 797 } 798 799 /* Copyout all the fp regs, the signal handler might expect them. */ 800 error = copyout(&frame, fp, sizeof frame); 801 relock: 802 mutex_enter(p->p_lock); 803 804 if (error != 0) { 805 /* 806 * Process has trashed its stack; give it an illegal 807 * instruction to halt it in its tracks. 808 */ 809 sigexit(l, SIGILL); 810 /* NOTREACHED */ 811 } 812 813 buildcontext(l, catcher, fp); 814 815 tf->tf_rdi = sig; 816 tf->tf_rsi = (uint64_t)&fp->sf_si; 817 tf->tf_rdx = tf->tf_r15 = (uint64_t)&fp->sf_uc; 818 819 /* Remember that we're now on the signal stack. */ 820 if (onstack) 821 l->l_sigstk.ss_flags |= SS_ONSTACK; 822 823 if ((vaddr_t)catcher >= VM_MAXUSER_ADDRESS) { 824 /* 825 * process has given an invalid address for the 826 * handler. Stop it, but do not do it before so 827 * we can return the right info to userland (or in core dump) 828 */ 829 sigexit(l, SIGILL); 830 /* NOTREACHED */ 831 } 832 } 833 834 struct pcb dumppcb; 835 836 void 837 cpu_reboot(int howto, char *bootstr) 838 { 839 static bool syncdone = false; 840 int s = IPL_NONE; 841 __USE(s); /* ugly otherwise */ 842 843 if (cold) { 844 howto |= RB_HALT; 845 goto haltsys; 846 } 847 848 boothowto = howto; 849 850 /* i386 maybe_dump() */ 851 852 /* 853 * If we've panic'd, don't make the situation potentially 854 * worse by syncing or unmounting the file systems. 855 */ 856 if ((howto & RB_NOSYNC) == 0 && panicstr == NULL) { 857 if (!syncdone) { 858 syncdone = true; 859 /* XXX used to force unmount as well, here */ 860 vfs_sync_all(curlwp); 861 } 862 863 while (vfs_unmountall1(curlwp, false, false) || 864 config_detach_all(boothowto) || 865 vfs_unmount_forceone(curlwp)) 866 ; /* do nothing */ 867 } else { 868 if (!db_active) 869 suspendsched(); 870 } 871 872 pmf_system_shutdown(boothowto); 873 874 /* Disable interrupts. */ 875 s = splhigh(); 876 877 /* Do a dump if requested. */ 878 if ((howto & (RB_DUMP | RB_HALT)) == RB_DUMP) 879 dumpsys(); 880 881 haltsys: 882 doshutdownhooks(); 883 884 if ((howto & RB_POWERDOWN) == RB_POWERDOWN) { 885 #if NACPICA > 0 886 if (s != IPL_NONE) 887 splx(s); 888 889 acpi_enter_sleep_state(ACPI_STATE_S5); 890 #endif 891 #ifdef XEN 892 if (vm_guest == VM_GUEST_XENPV || 893 vm_guest == VM_GUEST_XENPVH || 894 vm_guest == VM_GUEST_XENPVHVM) 895 HYPERVISOR_shutdown(); 896 #endif /* XEN */ 897 } 898 899 cpu_broadcast_halt(); 900 901 if (howto & RB_HALT) { 902 #if NACPICA > 0 903 acpi_disable(); 904 #endif 905 906 printf("\n"); 907 printf("The operating system has halted.\n"); 908 printf("Please press any key to reboot.\n\n"); 909 cnpollc(true); /* for proper keyboard command handling */ 910 if (cngetc() == 0) { 911 /* no console attached, so just hlt */ 912 printf("No keyboard - cannot reboot after all.\n"); 913 for(;;) { 914 x86_hlt(); 915 } 916 } 917 cnpollc(false); 918 } 919 920 printf("rebooting...\n"); 921 if (cpureset_delay > 0) 922 delay(cpureset_delay * 1000); 923 cpu_reset(); 924 for(;;) ; 925 /*NOTREACHED*/ 926 } 927 928 /* 929 * XXXfvdl share dumpcode. 930 */ 931 932 /* 933 * Perform assorted dump-related initialization tasks. Assumes that 934 * the maximum physical memory address will not increase afterwards. 935 */ 936 void 937 dump_misc_init(void) 938 { 939 #ifndef NO_SPARSE_DUMP 940 int i; 941 #endif 942 943 if (dump_headerbuf != NULL) 944 return; /* already called */ 945 946 #ifndef NO_SPARSE_DUMP 947 for (i = 0; i < mem_cluster_cnt; ++i) { 948 paddr_t top = mem_clusters[i].start + mem_clusters[i].size; 949 if (max_paddr < top) 950 max_paddr = top; 951 } 952 #ifdef DEBUG 953 printf("dump_misc_init: max_paddr = 0x%lx\n", 954 (unsigned long)max_paddr); 955 #endif 956 if (max_paddr == 0) { 957 printf("Your machine does not initialize mem_clusters; " 958 "sparse_dumps disabled\n"); 959 sparse_dump = 0; 960 } else { 961 sparse_dump_physmap = (void *)uvm_km_alloc(kernel_map, 962 roundup(max_paddr / (PAGE_SIZE * NBBY), PAGE_SIZE), 963 PAGE_SIZE, UVM_KMF_WIRED|UVM_KMF_ZERO); 964 } 965 #endif 966 dump_headerbuf = (void *)uvm_km_alloc(kernel_map, 967 dump_headerbuf_size, 968 PAGE_SIZE, UVM_KMF_WIRED|UVM_KMF_ZERO); 969 /* XXXjld should check for failure here, disable dumps if so. */ 970 } 971 972 #ifndef NO_SPARSE_DUMP 973 /* 974 * Clear the set of pages to include in a sparse dump. 975 */ 976 void 977 sparse_dump_reset(void) 978 { 979 memset(sparse_dump_physmap, 0, 980 roundup(max_paddr / (PAGE_SIZE * NBBY), PAGE_SIZE)); 981 } 982 983 /* 984 * Include or exclude pages in a sparse dump. 985 */ 986 void 987 sparse_dump_mark(void) 988 { 989 paddr_t p, pstart, pend; 990 struct vm_page *pg; 991 int i; 992 uvm_physseg_t upm; 993 994 /* 995 * Mark all memory pages, then unmark pages that are uninteresting. 996 * Dereferenceing pg->uobject might crash again if another CPU 997 * frees the object out from under us, but we can't lock anything 998 * so it's a risk we have to take. 999 */ 1000 1001 for (i = 0; i < mem_cluster_cnt; ++i) { 1002 pstart = mem_clusters[i].start / PAGE_SIZE; 1003 pend = pstart + mem_clusters[i].size / PAGE_SIZE; 1004 1005 for (p = pstart; p < pend; p++) { 1006 setbit(sparse_dump_physmap, p); 1007 } 1008 } 1009 for (upm = uvm_physseg_get_first(); 1010 uvm_physseg_valid_p(upm); 1011 upm = uvm_physseg_get_next(upm)) { 1012 paddr_t pfn; 1013 1014 /* 1015 * We assume that seg->start to seg->end are 1016 * uvm_page_physload()ed 1017 */ 1018 for (pfn = uvm_physseg_get_start(upm); 1019 pfn < uvm_physseg_get_end(upm); 1020 pfn++) { 1021 pg = PHYS_TO_VM_PAGE(ptoa(pfn)); 1022 1023 if (pg->uanon || (pg->flags & PG_FREE) || 1024 (pg->uobject && pg->uobject->pgops)) { 1025 p = VM_PAGE_TO_PHYS(pg) / PAGE_SIZE; 1026 clrbit(sparse_dump_physmap, p); 1027 } 1028 } 1029 } 1030 } 1031 1032 /* 1033 * Machine-dependently decides on the contents of a sparse dump, using 1034 * the above. 1035 */ 1036 void 1037 cpu_dump_prep_sparse(void) 1038 { 1039 sparse_dump_reset(); 1040 /* XXX could the alternate recursive page table be skipped? */ 1041 sparse_dump_mark(); 1042 /* Memory for I/O buffers could be unmarked here, for example. */ 1043 /* The kernel text could also be unmarked, but gdb would be upset. */ 1044 } 1045 #endif 1046 1047 /* 1048 * Abstractly iterate over the collection of memory segments to be 1049 * dumped; the callback lacks the customary environment-pointer 1050 * argument because none of the current users really need one. 1051 * 1052 * To be used only after dump_seg_prep is called to set things up. 1053 */ 1054 int 1055 dump_seg_iter(int (*callback)(paddr_t, paddr_t)) 1056 { 1057 int error, i; 1058 1059 #define CALLBACK(start,size) do { \ 1060 error = callback(start,size); \ 1061 if (error) \ 1062 return error; \ 1063 } while(0) 1064 1065 for (i = 0; i < mem_cluster_cnt; ++i) { 1066 #ifndef NO_SPARSE_DUMP 1067 /* 1068 * The bitmap is scanned within each memory segment, 1069 * rather than over its entire domain, in case any 1070 * pages outside of the memory proper have been mapped 1071 * into kva; they might be devices that wouldn't 1072 * appreciate being arbitrarily read, and including 1073 * them could also break the assumption that a sparse 1074 * dump will always be smaller than a full one. 1075 */ 1076 if (sparse_dump && sparse_dump_physmap) { 1077 paddr_t p, sp_start, sp_end; 1078 int lastset; 1079 1080 sp_start = mem_clusters[i].start; 1081 sp_end = sp_start + mem_clusters[i].size; 1082 sp_start = rounddown(sp_start, PAGE_SIZE); /* unnecessary? */ 1083 lastset = 0; 1084 for (p = sp_start; p < sp_end; p += PAGE_SIZE) { 1085 int thisset = isset(sparse_dump_physmap, 1086 p/PAGE_SIZE); 1087 1088 if (!lastset && thisset) 1089 sp_start = p; 1090 if (lastset && !thisset) 1091 CALLBACK(sp_start, p - sp_start); 1092 lastset = thisset; 1093 } 1094 if (lastset) 1095 CALLBACK(sp_start, p - sp_start); 1096 } else 1097 #endif 1098 CALLBACK(mem_clusters[i].start, mem_clusters[i].size); 1099 } 1100 return 0; 1101 #undef CALLBACK 1102 } 1103 1104 /* 1105 * Prepare for an impending core dump: decide what's being dumped and 1106 * how much space it will take up. 1107 */ 1108 void 1109 dump_seg_prep(void) 1110 { 1111 #ifndef NO_SPARSE_DUMP 1112 if (sparse_dump && sparse_dump_physmap) 1113 cpu_dump_prep_sparse(); 1114 #endif 1115 1116 dump_nmemsegs = 0; 1117 dump_npages = 0; 1118 dump_seg_iter(dump_seg_count_range); 1119 1120 dump_header_size = ALIGN(sizeof(kcore_seg_t)) + 1121 ALIGN(sizeof(cpu_kcore_hdr_t)) + 1122 ALIGN(dump_nmemsegs * sizeof(phys_ram_seg_t)); 1123 dump_header_size = roundup(dump_header_size, dbtob(1)); 1124 1125 /* 1126 * savecore(8) will read this to decide how many pages to 1127 * copy, and cpu_dumpconf has already used the pessimistic 1128 * value to set dumplo, so it's time to tell the truth. 1129 */ 1130 dumpsize = dump_npages; /* XXX could these just be one variable? */ 1131 } 1132 1133 int 1134 dump_seg_count_range(paddr_t start, paddr_t size) 1135 { 1136 ++dump_nmemsegs; 1137 dump_npages += size / PAGE_SIZE; 1138 return 0; 1139 } 1140 1141 /* 1142 * A sparse dump's header may be rather large, due to the number of 1143 * "segments" emitted. These routines manage a simple output buffer, 1144 * so that the header can be written to disk incrementally. 1145 */ 1146 void 1147 dump_header_start(void) 1148 { 1149 dump_headerbuf_ptr = dump_headerbuf; 1150 dump_header_blkno = dumplo; 1151 } 1152 1153 int 1154 dump_header_flush(void) 1155 { 1156 const struct bdevsw *bdev; 1157 size_t to_write; 1158 int error; 1159 1160 bdev = bdevsw_lookup(dumpdev); 1161 to_write = roundup(dump_headerbuf_ptr - dump_headerbuf, dbtob(1)); 1162 error = bdev->d_dump(dumpdev, dump_header_blkno, 1163 dump_headerbuf, to_write); 1164 dump_header_blkno += btodb(to_write); 1165 dump_headerbuf_ptr = dump_headerbuf; 1166 return error; 1167 } 1168 1169 int 1170 dump_header_addbytes(const void* vptr, size_t n) 1171 { 1172 const char* ptr = vptr; 1173 int error; 1174 1175 while (n > dump_headerbuf_avail) { 1176 memcpy(dump_headerbuf_ptr, ptr, dump_headerbuf_avail); 1177 ptr += dump_headerbuf_avail; 1178 n -= dump_headerbuf_avail; 1179 dump_headerbuf_ptr = dump_headerbuf_end; 1180 error = dump_header_flush(); 1181 if (error) 1182 return error; 1183 } 1184 memcpy(dump_headerbuf_ptr, ptr, n); 1185 dump_headerbuf_ptr += n; 1186 1187 return 0; 1188 } 1189 1190 int 1191 dump_header_addseg(paddr_t start, paddr_t size) 1192 { 1193 phys_ram_seg_t seg = { start, size }; 1194 int error; 1195 1196 error = dump_header_addbytes(&seg, sizeof(seg)); 1197 if (error) { 1198 printf("[seg 0x%"PRIxPADDR" bytes 0x%"PRIxPSIZE" failed," 1199 " error=%d] ", start, size, error); 1200 } 1201 return error; 1202 } 1203 1204 int 1205 dump_header_finish(void) 1206 { 1207 int error; 1208 1209 memset(dump_headerbuf_ptr, 0, dump_headerbuf_avail); 1210 error = dump_header_flush(); 1211 if (error) 1212 printf("[finish failed, error=%d] ", error); 1213 return error; 1214 } 1215 1216 1217 /* 1218 * These variables are needed by /sbin/savecore 1219 */ 1220 uint32_t dumpmag = 0x8fca0101; /* magic number */ 1221 int dumpsize = 0; /* pages */ 1222 long dumplo = 0; /* blocks */ 1223 1224 /* 1225 * cpu_dumpsize: calculate size of machine-dependent kernel core dump headers 1226 * for a full (non-sparse) dump. 1227 */ 1228 int 1229 cpu_dumpsize(void) 1230 { 1231 int size; 1232 1233 size = ALIGN(sizeof(kcore_seg_t)) + ALIGN(sizeof(cpu_kcore_hdr_t)) + 1234 ALIGN(mem_cluster_cnt * sizeof(phys_ram_seg_t)); 1235 if (roundup(size, dbtob(1)) != dbtob(1)) 1236 return (-1); 1237 1238 return (1); 1239 } 1240 1241 /* 1242 * cpu_dump_mempagecnt: calculate the size of RAM (in pages) to be dumped 1243 * for a full (non-sparse) dump. 1244 */ 1245 u_long 1246 cpu_dump_mempagecnt(void) 1247 { 1248 u_long i, n; 1249 1250 n = 0; 1251 for (i = 0; i < mem_cluster_cnt; i++) 1252 n += atop(mem_clusters[i].size); 1253 return (n); 1254 } 1255 1256 /* 1257 * cpu_dump: dump the machine-dependent kernel core dump headers. 1258 */ 1259 int 1260 cpu_dump(void) 1261 { 1262 kcore_seg_t seg; 1263 cpu_kcore_hdr_t cpuhdr; 1264 const struct bdevsw *bdev; 1265 int error; 1266 1267 bdev = bdevsw_lookup(dumpdev); 1268 if (bdev == NULL) { 1269 printf("[device 0x%llx ENXIO] ", (unsigned long long)dumpdev); 1270 return ENXIO; 1271 } 1272 1273 /* 1274 * Generate a segment header. 1275 */ 1276 CORE_SETMAGIC(seg, KCORE_MAGIC, MID_MACHINE, CORE_CPU); 1277 seg.c_size = dump_header_size - ALIGN(sizeof(seg)); 1278 error = dump_header_addbytes(&seg, ALIGN(sizeof(seg))); 1279 if (error) { 1280 printf("[segment header %zu bytes failed, error=%d] ", 1281 ALIGN(sizeof(seg)), error); 1282 /* blithely proceed (can't fail?) */ 1283 } 1284 1285 /* 1286 * Add the machine-dependent header info. 1287 */ 1288 cpuhdr.ptdpaddr = PDPpaddr; 1289 cpuhdr.nmemsegs = dump_nmemsegs; 1290 error = dump_header_addbytes(&cpuhdr, ALIGN(sizeof(cpuhdr))); 1291 if (error) { 1292 printf("[MD header %zu bytes failed, error=%d] ", 1293 ALIGN(sizeof(cpuhdr)), error); 1294 /* blithely proceed (can't fail?) */ 1295 } 1296 1297 /* 1298 * Write out the memory segment descriptors. 1299 */ 1300 return dump_seg_iter(dump_header_addseg); 1301 } 1302 1303 /* 1304 * Doadump comes here after turning off memory management and 1305 * getting on the dump stack, either when called above, or by 1306 * the auto-restart code. 1307 */ 1308 #define BYTES_PER_DUMP PAGE_SIZE /* must be a multiple of pagesize XXX small */ 1309 static vaddr_t dumpspace; 1310 1311 vaddr_t 1312 reserve_dumppages(vaddr_t p) 1313 { 1314 1315 dumpspace = p; 1316 return (p + BYTES_PER_DUMP); 1317 } 1318 1319 int 1320 dumpsys_seg(paddr_t maddr, paddr_t bytes) 1321 { 1322 u_long i, m, n; 1323 daddr_t blkno; 1324 const struct bdevsw *bdev; 1325 int (*dump)(dev_t, daddr_t, void *, size_t); 1326 int error; 1327 1328 if (dumpdev == NODEV) 1329 return ENODEV; 1330 bdev = bdevsw_lookup(dumpdev); 1331 if (bdev == NULL || bdev->d_psize == NULL) 1332 return ENODEV; 1333 1334 dump = bdev->d_dump; 1335 1336 blkno = dump_header_blkno; 1337 for (i = 0; i < bytes; i += n, dump_totalbytesleft -= n) { 1338 /* Print out how many MBs we have left to go. */ 1339 if ((dump_totalbytesleft % (1024*1024)) == 0) 1340 printf_nolog("%lu ", (unsigned long) 1341 (dump_totalbytesleft / (1024 * 1024))); 1342 1343 /* Limit size for next transfer. */ 1344 n = bytes - i; 1345 if (n > BYTES_PER_DUMP) 1346 n = BYTES_PER_DUMP; 1347 1348 for (m = 0; m < n; m += NBPG) 1349 pmap_kenter_pa(dumpspace + m, maddr + m, 1350 VM_PROT_READ, 0); 1351 pmap_update(pmap_kernel()); 1352 1353 error = (*dump)(dumpdev, blkno, (void *)dumpspace, n); 1354 pmap_kremove_local(dumpspace, n); 1355 if (error) 1356 return error; 1357 maddr += n; 1358 blkno += btodb(n); /* XXX? */ 1359 1360 #if 0 /* XXX this doesn't work. grr. */ 1361 /* operator aborting dump? */ 1362 if (sget() != NULL) 1363 return EINTR; 1364 #endif 1365 } 1366 dump_header_blkno = blkno; 1367 1368 return 0; 1369 } 1370 1371 void 1372 dodumpsys(void) 1373 { 1374 const struct bdevsw *bdev; 1375 int dumpend, psize; 1376 int error; 1377 1378 if (dumpdev == NODEV) 1379 return; 1380 1381 bdev = bdevsw_lookup(dumpdev); 1382 if (bdev == NULL || bdev->d_psize == NULL) 1383 return; 1384 /* 1385 * For dumps during autoconfiguration, 1386 * if dump device has already configured... 1387 */ 1388 if (dumpsize == 0) 1389 cpu_dumpconf(); 1390 1391 printf("\ndumping to dev %llu,%llu (offset=%ld, size=%d):", 1392 (unsigned long long)major(dumpdev), 1393 (unsigned long long)minor(dumpdev), dumplo, dumpsize); 1394 1395 if (dumplo <= 0 || dumpsize <= 0) { 1396 printf(" not possible\n"); 1397 return; 1398 } 1399 1400 psize = bdev_size(dumpdev); 1401 printf("\ndump "); 1402 if (psize == -1) { 1403 printf("area unavailable\n"); 1404 return; 1405 } 1406 1407 #if 0 /* XXX this doesn't work. grr. */ 1408 /* toss any characters present prior to dump */ 1409 while (sget() != NULL); /*syscons and pccons differ */ 1410 #endif 1411 1412 dump_seg_prep(); 1413 dumpend = dumplo + btodb(dump_header_size) + ctod(dump_npages); 1414 if (dumpend > psize) { 1415 printf("failed: insufficient space (%d < %d)\n", 1416 psize, dumpend); 1417 goto failed; 1418 } 1419 1420 dump_header_start(); 1421 if ((error = cpu_dump()) != 0) 1422 goto err; 1423 if ((error = dump_header_finish()) != 0) 1424 goto err; 1425 1426 if (dump_header_blkno != dumplo + btodb(dump_header_size)) { 1427 printf("BAD header size (%ld [written] != %ld [expected])\n", 1428 (long)(dump_header_blkno - dumplo), 1429 (long)btodb(dump_header_size)); 1430 goto failed; 1431 } 1432 1433 dump_totalbytesleft = roundup(ptoa(dump_npages), BYTES_PER_DUMP); 1434 error = dump_seg_iter(dumpsys_seg); 1435 1436 if (error == 0 && dump_header_blkno != dumpend) { 1437 printf("BAD dump size (%ld [written] != %ld [expected])\n", 1438 (long)(dumpend - dumplo), 1439 (long)(dump_header_blkno - dumplo)); 1440 goto failed; 1441 } 1442 1443 err: 1444 switch (error) { 1445 1446 case ENXIO: 1447 printf("device bad\n"); 1448 break; 1449 1450 case EFAULT: 1451 printf("device not ready\n"); 1452 break; 1453 1454 case EINVAL: 1455 printf("area improper\n"); 1456 break; 1457 1458 case EIO: 1459 printf("i/o error\n"); 1460 break; 1461 1462 case EINTR: 1463 printf("aborted from console\n"); 1464 break; 1465 1466 case 0: 1467 printf("succeeded\n"); 1468 break; 1469 1470 default: 1471 printf("error %d\n", error); 1472 break; 1473 } 1474 failed: 1475 printf("\n\n"); 1476 delay(5000000); /* 5 seconds */ 1477 } 1478 1479 /* 1480 * This is called by main to set dumplo and dumpsize. 1481 * Dumps always skip the first PAGE_SIZE of disk space 1482 * in case there might be a disk label stored there. 1483 * If there is extra space, put dump at the end to 1484 * reduce the chance that swapping trashes it. 1485 * 1486 * Sparse dumps can't placed as close to the end as possible, because 1487 * savecore(8) has to know where to start reading in the dump device 1488 * before it has access to any of the crashed system's state. 1489 * 1490 * Note also that a sparse dump will never be larger than a full one: 1491 * in order to add a phys_ram_seg_t to the header, at least one page 1492 * must be removed. 1493 */ 1494 void 1495 cpu_dumpconf(void) 1496 { 1497 int nblks, dumpblks; /* size of dump area */ 1498 1499 if (dumpdev == NODEV) 1500 goto bad; 1501 nblks = bdev_size(dumpdev); 1502 if (nblks <= ctod(1)) 1503 goto bad; 1504 1505 dumpblks = cpu_dumpsize(); 1506 if (dumpblks < 0) 1507 goto bad; 1508 1509 /* dumpsize is in page units, and doesn't include headers. */ 1510 dumpsize = cpu_dump_mempagecnt(); 1511 1512 dumpblks += ctod(dumpsize); 1513 1514 /* If dump won't fit (incl. room for possible label), punt. */ 1515 if (dumpblks > (nblks - ctod(1))) { 1516 #ifndef NO_SPARSE_DUMP 1517 /* A sparse dump might (and hopefully will) fit. */ 1518 dumplo = ctod(1); 1519 #else 1520 /* But if we're not configured for that, punt. */ 1521 goto bad; 1522 #endif 1523 } else { 1524 /* Put dump at end of partition */ 1525 dumplo = nblks - dumpblks; 1526 } 1527 1528 1529 /* Now that we've decided this will work, init ancillary stuff. */ 1530 dump_misc_init(); 1531 return; 1532 1533 bad: 1534 dumpsize = 0; 1535 } 1536 1537 /* 1538 * Clear registers on exec 1539 */ 1540 void 1541 setregs(struct lwp *l, struct exec_package *pack, vaddr_t stack) 1542 { 1543 struct pcb *pcb = lwp_getpcb(l); 1544 struct trapframe *tf; 1545 1546 #ifdef USER_LDT 1547 pmap_ldt_cleanup(l); 1548 #endif 1549 1550 fpu_clear(l, pack->ep_osversion >= 699002600 1551 ? __NetBSD_NPXCW__ : __NetBSD_COMPAT_NPXCW__); 1552 x86_dbregs_clear(l); 1553 1554 kpreempt_disable(); 1555 pcb->pcb_flags = 0; 1556 l->l_proc->p_flag &= ~PK_32; 1557 l->l_md.md_flags = MDL_IRET; 1558 cpu_segregs64_zero(l); 1559 kpreempt_enable(); 1560 1561 tf = l->l_md.md_regs; 1562 memset(tf, 0, sizeof(*tf)); 1563 1564 tf->tf_trapno = T_ASTFLT; 1565 tf->tf_ds = GSEL(GUDATA_SEL, SEL_UPL); 1566 tf->tf_es = GSEL(GUDATA_SEL, SEL_UPL); 1567 tf->tf_rdi = 0; 1568 tf->tf_rsi = 0; 1569 tf->tf_rbp = 0; 1570 tf->tf_rbx = l->l_proc->p_psstrp; 1571 tf->tf_rdx = 0; 1572 tf->tf_rcx = 0; 1573 tf->tf_rax = 0; 1574 tf->tf_rip = pack->ep_entry; 1575 tf->tf_cs = LSEL(LUCODE_SEL, SEL_UPL); 1576 tf->tf_rflags = PSL_USERSET; 1577 tf->tf_rsp = stack; 1578 tf->tf_ss = LSEL(LUDATA_SEL, SEL_UPL); 1579 } 1580 1581 /* 1582 * Initialize segments and descriptor tables 1583 */ 1584 char *ldtstore; 1585 char *gdtstore; 1586 1587 void 1588 setgate(struct gate_descriptor *gd, void *func, 1589 int ist, int type, int dpl, int sel) 1590 { 1591 vaddr_t vaddr; 1592 1593 vaddr = ((vaddr_t)gd) & ~PAGE_MASK; 1594 1595 kpreempt_disable(); 1596 pmap_changeprot_local(vaddr, VM_PROT_READ|VM_PROT_WRITE); 1597 1598 gd->gd_looffset = (uint64_t)func & 0xffff; 1599 gd->gd_selector = sel; 1600 gd->gd_ist = ist; 1601 gd->gd_type = type; 1602 gd->gd_dpl = dpl; 1603 gd->gd_p = 1; 1604 gd->gd_hioffset = (uint64_t)func >> 16; 1605 gd->gd_zero = 0; 1606 gd->gd_xx1 = 0; 1607 gd->gd_xx2 = 0; 1608 gd->gd_xx3 = 0; 1609 1610 pmap_changeprot_local(vaddr, VM_PROT_READ); 1611 kpreempt_enable(); 1612 } 1613 1614 void 1615 unsetgate(struct gate_descriptor *gd) 1616 { 1617 vaddr_t vaddr; 1618 1619 vaddr = ((vaddr_t)gd) & ~PAGE_MASK; 1620 1621 kpreempt_disable(); 1622 pmap_changeprot_local(vaddr, VM_PROT_READ|VM_PROT_WRITE); 1623 1624 memset(gd, 0, sizeof (*gd)); 1625 1626 pmap_changeprot_local(vaddr, VM_PROT_READ); 1627 kpreempt_enable(); 1628 } 1629 1630 void 1631 setregion(struct region_descriptor *rd, void *base, uint16_t limit) 1632 { 1633 rd->rd_limit = limit; 1634 rd->rd_base = (uint64_t)base; 1635 } 1636 1637 /* 1638 * Note that the base and limit fields are ignored in long mode. 1639 */ 1640 void 1641 set_mem_segment(struct mem_segment_descriptor *sd, void *base, size_t limit, 1642 int type, int dpl, int gran, int def32, int is64) 1643 { 1644 sd->sd_lolimit = (unsigned)limit; 1645 sd->sd_lobase = (unsigned long)base; 1646 sd->sd_type = type; 1647 sd->sd_dpl = dpl; 1648 sd->sd_p = 1; 1649 sd->sd_hilimit = (unsigned)limit >> 16; 1650 sd->sd_avl = 0; 1651 sd->sd_long = is64; 1652 sd->sd_def32 = def32; 1653 sd->sd_gran = gran; 1654 sd->sd_hibase = (unsigned long)base >> 24; 1655 } 1656 1657 void 1658 set_sys_segment(struct sys_segment_descriptor *sd, void *base, size_t limit, 1659 int type, int dpl, int gran) 1660 { 1661 memset(sd, 0, sizeof *sd); 1662 sd->sd_lolimit = (unsigned)limit; 1663 sd->sd_lobase = (uint64_t)base; 1664 sd->sd_type = type; 1665 sd->sd_dpl = dpl; 1666 sd->sd_p = 1; 1667 sd->sd_hilimit = (unsigned)limit >> 16; 1668 sd->sd_gran = gran; 1669 sd->sd_hibase = (uint64_t)base >> 24; 1670 } 1671 1672 void 1673 cpu_init_idt(struct cpu_info *ci) 1674 { 1675 struct region_descriptor region; 1676 idt_descriptor_t *idt; 1677 1678 idt = ci->ci_idtvec.iv_idt; 1679 setregion(®ion, idt, NIDT * sizeof(idt[0]) - 1); 1680 lidt(®ion); 1681 } 1682 1683 #define IDTVEC(name) __CONCAT(X, name) 1684 typedef void (vector)(void); 1685 extern vector IDTVEC(syscall); 1686 extern vector IDTVEC(syscall32); 1687 extern vector IDTVEC(osyscall); 1688 extern vector *x86_exceptions[]; 1689 1690 #ifndef XENPV 1691 static void 1692 init_x86_64_ksyms(void) 1693 { 1694 #if NKSYMS || defined(DDB) || defined(MODULAR) 1695 extern int end; 1696 extern int *esym; 1697 struct btinfo_symtab *symtab; 1698 vaddr_t tssym, tesym; 1699 1700 #ifdef DDB 1701 db_machine_init(); 1702 #endif 1703 1704 symtab = lookup_bootinfo(BTINFO_SYMTAB); 1705 if (symtab) { 1706 #ifdef KASLR 1707 tssym = bootspace.head.va; 1708 tesym = bootspace.head.va; /* (unused...) */ 1709 #else 1710 tssym = (vaddr_t)symtab->ssym + KERNBASE; 1711 tesym = (vaddr_t)symtab->esym + KERNBASE; 1712 #endif 1713 ksyms_addsyms_elf(symtab->nsym, (void *)tssym, (void *)tesym); 1714 } else { 1715 uintptr_t endp = (uintptr_t)(void *)&end; 1716 #ifdef XEN 1717 /* 1718 * cpu_probe() / identify_hypervisor() overrides VM_GUEST_GENPVH, 1719 * we can't rely on vm_guest == VM_GUEST_GENPVH 1720 */ 1721 if (pvh_boot && vm_guest != VM_GUEST_XENPVH) 1722 ksyms_addsyms_elf(0, ((long *)endp) + 1, esym); 1723 else 1724 #endif 1725 ksyms_addsyms_elf(*(long *)endp, ((long *)endp) + 1, esym); 1726 } 1727 #endif 1728 } 1729 #endif /* XENPV */ 1730 1731 void __noasan 1732 init_bootspace(void) 1733 { 1734 extern char __rodata_start; 1735 extern char __data_start; 1736 extern char __kernel_end; 1737 size_t i = 0; 1738 1739 memset(&bootspace, 0, sizeof(bootspace)); 1740 1741 bootspace.head.va = KERNTEXTOFF; 1742 bootspace.head.pa = KERNTEXTOFF - KERNBASE; 1743 bootspace.head.sz = 0; 1744 1745 bootspace.segs[i].type = BTSEG_TEXT; 1746 bootspace.segs[i].va = KERNTEXTOFF; 1747 bootspace.segs[i].pa = KERNTEXTOFF - KERNBASE; 1748 bootspace.segs[i].sz = (size_t)&__rodata_start - KERNTEXTOFF; 1749 i++; 1750 1751 bootspace.segs[i].type = BTSEG_RODATA; 1752 bootspace.segs[i].va = (vaddr_t)&__rodata_start; 1753 bootspace.segs[i].pa = (paddr_t)&__rodata_start - KERNBASE; 1754 bootspace.segs[i].sz = (size_t)&__data_start - (size_t)&__rodata_start; 1755 i++; 1756 1757 bootspace.segs[i].type = BTSEG_DATA; 1758 bootspace.segs[i].va = (vaddr_t)&__data_start; 1759 bootspace.segs[i].pa = (paddr_t)&__data_start - KERNBASE; 1760 bootspace.segs[i].sz = (size_t)&__kernel_end - (size_t)&__data_start; 1761 i++; 1762 1763 bootspace.boot.va = (vaddr_t)&__kernel_end; 1764 bootspace.boot.pa = (paddr_t)&__kernel_end - KERNBASE; 1765 bootspace.boot.sz = (size_t)(atdevbase + IOM_SIZE) - 1766 (size_t)&__kernel_end; 1767 1768 /* In locore.S, we allocated a tmp va. We will use it now. */ 1769 bootspace.spareva = KERNBASE + NKL2_KIMG_ENTRIES * NBPD_L2; 1770 1771 /* Virtual address of the L4 page. */ 1772 bootspace.pdir = (vaddr_t)(PDPpaddr + KERNBASE); 1773 1774 /* Kernel module map. */ 1775 bootspace.smodule = (vaddr_t)atdevbase + IOM_SIZE; 1776 bootspace.emodule = KERNBASE + NKL2_KIMG_ENTRIES * NBPD_L2; 1777 } 1778 1779 static void 1780 init_pte(void) 1781 { 1782 #ifndef XENPV 1783 extern uint32_t nox_flag; 1784 pd_entry_t *pdir = (pd_entry_t *)bootspace.pdir; 1785 pdir[L4_SLOT_PTE] = PDPpaddr | PTE_W | ((uint64_t)nox_flag << 32) | 1786 PTE_P; 1787 #endif 1788 1789 extern pd_entry_t *normal_pdes[3]; 1790 normal_pdes[0] = L2_BASE; 1791 normal_pdes[1] = L3_BASE; 1792 normal_pdes[2] = L4_BASE; 1793 } 1794 1795 void 1796 init_slotspace(void) 1797 { 1798 /* 1799 * XXX Too early to use cprng(9), or even entropy_extract. 1800 */ 1801 struct entpool pool; 1802 size_t randhole; 1803 vaddr_t randva; 1804 uint64_t sample; 1805 vaddr_t va; 1806 1807 memset(&pool, 0, sizeof pool); 1808 cpu_rng_early_sample(&sample); 1809 entpool_enter(&pool, &sample, sizeof sample); 1810 1811 memset(&slotspace, 0, sizeof(slotspace)); 1812 1813 /* User. [256, because we want to land in >= 256] */ 1814 slotspace.area[SLAREA_USER].sslot = 0; 1815 slotspace.area[SLAREA_USER].nslot = PDIR_SLOT_USERLIM+1; 1816 slotspace.area[SLAREA_USER].active = true; 1817 1818 #ifdef XENPV 1819 /* PTE. */ 1820 slotspace.area[SLAREA_PTE].sslot = PDIR_SLOT_PTE; 1821 slotspace.area[SLAREA_PTE].nslot = 1; 1822 slotspace.area[SLAREA_PTE].active = true; 1823 #endif 1824 1825 #ifdef __HAVE_PCPU_AREA 1826 /* Per-CPU. */ 1827 slotspace.area[SLAREA_PCPU].sslot = PDIR_SLOT_PCPU; 1828 slotspace.area[SLAREA_PCPU].nslot = 1; 1829 slotspace.area[SLAREA_PCPU].active = true; 1830 #endif 1831 1832 #ifdef __HAVE_DIRECT_MAP 1833 /* Direct Map. [Randomized later] */ 1834 slotspace.area[SLAREA_DMAP].active = false; 1835 #endif 1836 1837 #ifdef XENPV 1838 /* Hypervisor. */ 1839 slotspace.area[SLAREA_HYPV].sslot = 256; 1840 slotspace.area[SLAREA_HYPV].nslot = 17; 1841 slotspace.area[SLAREA_HYPV].active = true; 1842 #endif 1843 1844 #ifdef KASAN 1845 /* ASAN. */ 1846 slotspace.area[SLAREA_ASAN].sslot = L4_SLOT_KASAN; 1847 slotspace.area[SLAREA_ASAN].nslot = NL4_SLOT_KASAN; 1848 slotspace.area[SLAREA_ASAN].active = true; 1849 #endif 1850 1851 #ifdef KMSAN 1852 /* MSAN. */ 1853 slotspace.area[SLAREA_MSAN].sslot = L4_SLOT_KMSAN; 1854 slotspace.area[SLAREA_MSAN].nslot = NL4_SLOT_KMSAN; 1855 slotspace.area[SLAREA_MSAN].active = true; 1856 #endif 1857 1858 /* Kernel. */ 1859 slotspace.area[SLAREA_KERN].sslot = L4_SLOT_KERNBASE; 1860 slotspace.area[SLAREA_KERN].nslot = 1; 1861 slotspace.area[SLAREA_KERN].active = true; 1862 1863 /* Main. */ 1864 cpu_rng_early_sample(&sample); 1865 entpool_enter(&pool, &sample, sizeof sample); 1866 entpool_extract(&pool, &randhole, sizeof randhole); 1867 entpool_extract(&pool, &randva, sizeof randva); 1868 va = slotspace_rand(SLAREA_MAIN, NKL4_MAX_ENTRIES * NBPD_L4, 1869 NBPD_L4, randhole, randva); /* TODO: NBPD_L1 */ 1870 vm_min_kernel_address = va; 1871 vm_max_kernel_address = va + NKL4_MAX_ENTRIES * NBPD_L4; 1872 1873 #ifndef XENPV 1874 /* PTE. */ 1875 cpu_rng_early_sample(&sample); 1876 entpool_enter(&pool, &sample, sizeof sample); 1877 entpool_extract(&pool, &randhole, sizeof randhole); 1878 entpool_extract(&pool, &randva, sizeof randva); 1879 va = slotspace_rand(SLAREA_PTE, NBPD_L4, NBPD_L4, randhole, randva); 1880 pte_base = (pd_entry_t *)va; 1881 #endif 1882 1883 explicit_memset(&pool, 0, sizeof pool); 1884 } 1885 1886 void 1887 init_x86_64(paddr_t first_avail) 1888 { 1889 extern void consinit(void); 1890 struct region_descriptor region; 1891 struct mem_segment_descriptor *ldt_segp; 1892 struct idt_vec *iv; 1893 idt_descriptor_t *idt; 1894 int x; 1895 struct pcb *pcb; 1896 extern vaddr_t lwp0uarea; 1897 #ifndef XENPV 1898 extern paddr_t local_apic_pa; 1899 #endif 1900 1901 KASSERT(first_avail % PAGE_SIZE == 0); 1902 1903 #ifdef XENPV 1904 KASSERT(HYPERVISOR_shared_info != NULL); 1905 cpu_info_primary.ci_vcpu = &HYPERVISOR_shared_info->vcpu_info[0]; 1906 #endif 1907 1908 #ifdef XEN 1909 if (pvh_boot) 1910 xen_parse_cmdline(XEN_PARSE_BOOTFLAGS, NULL); 1911 #endif 1912 init_pte(); 1913 1914 uvm_lwp_setuarea(&lwp0, lwp0uarea); 1915 1916 cpu_probe(&cpu_info_primary); 1917 #ifdef SVS 1918 svs_init(); 1919 #endif 1920 1921 /* 1922 * Initialize MSRs on cpu0: 1923 * 1924 * - Enables SYSCALL/SYSRET. 1925 * 1926 * - Sets up %fs and %gs so that %gs points to the current 1927 * struct cpu_info as needed for CPUVAR(...), curcpu(), and 1928 * curlwp. 1929 * 1930 * - Enables the no-execute bit if supported. 1931 * 1932 * Thus, after this point, CPUVAR(...), curcpu(), and curlwp 1933 * will work on cpu0. 1934 * 1935 * Note: The call to cpu_init_msrs for secondary CPUs happens 1936 * in cpu_hatch. 1937 */ 1938 cpu_init_msrs(&cpu_info_primary, true); 1939 1940 #ifndef XENPV 1941 cpu_speculation_init(&cpu_info_primary); 1942 #endif 1943 1944 use_pae = 1; /* PAE always enabled in long mode */ 1945 1946 pcb = lwp_getpcb(&lwp0); 1947 #ifdef XENPV 1948 mutex_init(&pte_lock, MUTEX_DEFAULT, IPL_VM); 1949 pcb->pcb_cr3 = xen_start_info.pt_base - KERNBASE; 1950 #else 1951 pcb->pcb_cr3 = PDPpaddr; 1952 #endif 1953 1954 #if NISA > 0 || NPCI > 0 1955 x86_bus_space_init(); 1956 #endif 1957 1958 pat_init(&cpu_info_primary); 1959 1960 consinit(); /* XXX SHOULD NOT BE DONE HERE */ 1961 1962 /* 1963 * Initialize PAGE_SIZE-dependent variables. 1964 */ 1965 uvm_md_init(); 1966 1967 uvmexp.ncolors = 2; 1968 1969 avail_start = first_avail; 1970 1971 #ifndef XENPV 1972 /* 1973 * Low memory reservations: 1974 * Page 0: BIOS data 1975 * Page 1: BIOS callback (not used yet, for symmetry with i386) 1976 * Page 2: MP bootstrap code (MP_TRAMPOLINE) 1977 * Page 3: ACPI wakeup code (ACPI_WAKEUP_ADDR) 1978 * Page 4: Temporary page table for 0MB-4MB 1979 * Page 5: Temporary page directory 1980 * Page 6: Temporary page map level 3 1981 * Page 7: Temporary page map level 4 1982 */ 1983 lowmem_rsvd = 8 * PAGE_SIZE; 1984 1985 /* Initialize the memory clusters (needed in pmap_bootstrap). */ 1986 init_x86_clusters(); 1987 #else 1988 /* Parse Xen command line (replace bootinfo) */ 1989 xen_parse_cmdline(XEN_PARSE_BOOTFLAGS, NULL); 1990 1991 avail_end = ctob(xen_start_info.nr_pages); 1992 pmap_pa_start = (KERNTEXTOFF - KERNBASE); 1993 pmap_pa_end = avail_end; 1994 #endif 1995 1996 /* 1997 * Call pmap initialization to make new kernel address space. 1998 * We must do this before loading pages into the VM system. 1999 */ 2000 pmap_bootstrap(VM_MIN_KERNEL_ADDRESS); 2001 2002 /* 2003 * Initialize RNG to get entropy ASAP either from CPU 2004 * RDRAND/RDSEED or from seed on disk. Constraints: 2005 * 2006 * - Must happen after cpu_init_msrs so that curcpu() and 2007 * curlwp work. 2008 * 2009 * - Must happen after consinit so we have the opportunity to 2010 * print useful feedback. 2011 * 2012 * - On KASLR kernels, must happen after pmap_bootstrap because 2013 * x86_rndseed requires access to the direct map. 2014 */ 2015 cpu_rng_init(); 2016 x86_rndseed(); 2017 2018 #ifndef XENPV 2019 /* Internalize the physical pages into the VM system. */ 2020 init_x86_vm(avail_start); 2021 #else 2022 physmem = xen_start_info.nr_pages; 2023 uvm_page_physload(atop(avail_start), atop(avail_end), 2024 atop(avail_start), atop(avail_end), VM_FREELIST_DEFAULT); 2025 #endif 2026 2027 init_x86_msgbuf(); 2028 2029 kasan_init(); 2030 kcsan_init(); 2031 kmsan_init((void *)lwp0uarea); 2032 2033 pmap_growkernel(VM_MIN_KERNEL_ADDRESS + 32 * 1024 * 1024); 2034 2035 kpreempt_disable(); 2036 2037 #ifndef XENPV 2038 pmap_kenter_pa(local_apic_va, local_apic_pa, 2039 VM_PROT_READ|VM_PROT_WRITE, 0); 2040 pmap_update(pmap_kernel()); 2041 memset((void *)local_apic_va, 0, PAGE_SIZE); 2042 #endif 2043 2044 pmap_kenter_pa(idt_vaddr, idt_paddr, VM_PROT_READ|VM_PROT_WRITE, 0); 2045 pmap_kenter_pa(gdt_vaddr, gdt_paddr, VM_PROT_READ|VM_PROT_WRITE, 0); 2046 pmap_kenter_pa(ldt_vaddr, ldt_paddr, VM_PROT_READ|VM_PROT_WRITE, 0); 2047 pmap_update(pmap_kernel()); 2048 memset((void *)idt_vaddr, 0, PAGE_SIZE); 2049 memset((void *)gdt_vaddr, 0, PAGE_SIZE); 2050 memset((void *)ldt_vaddr, 0, PAGE_SIZE); 2051 2052 #ifndef XENPV 2053 pmap_changeprot_local(idt_vaddr, VM_PROT_READ); 2054 #endif 2055 2056 pmap_update(pmap_kernel()); 2057 2058 iv = &(cpu_info_primary.ci_idtvec); 2059 idt_vec_init_cpu_md(iv, cpu_index(&cpu_info_primary)); 2060 idt = iv->iv_idt; 2061 gdtstore = (char *)gdt_vaddr; 2062 ldtstore = (char *)ldt_vaddr; 2063 2064 /* 2065 * Make GDT gates and memory segments. 2066 */ 2067 set_mem_segment(GDT_ADDR_MEM(gdtstore, GCODE_SEL), 0, 2068 0xfffff, SDT_MEMERA, SEL_KPL, 1, 0, 1); 2069 2070 set_mem_segment(GDT_ADDR_MEM(gdtstore, GDATA_SEL), 0, 2071 0xfffff, SDT_MEMRWA, SEL_KPL, 1, 0, 1); 2072 2073 set_mem_segment(GDT_ADDR_MEM(gdtstore, GUCODE_SEL), 0, 2074 x86_btop(VM_MAXUSER_ADDRESS) - 1, SDT_MEMERA, SEL_UPL, 1, 0, 1); 2075 2076 set_mem_segment(GDT_ADDR_MEM(gdtstore, GUDATA_SEL), 0, 2077 x86_btop(VM_MAXUSER_ADDRESS) - 1, SDT_MEMRWA, SEL_UPL, 1, 0, 1); 2078 2079 #ifndef XENPV 2080 set_sys_segment(GDT_ADDR_SYS(gdtstore, GLDT_SEL), ldtstore, 2081 LDT_SIZE - 1, SDT_SYSLDT, SEL_KPL, 0); 2082 #endif 2083 2084 /* 2085 * Make LDT memory segments. 2086 */ 2087 *(struct mem_segment_descriptor *)(ldtstore + LUCODE_SEL) = 2088 *GDT_ADDR_MEM(gdtstore, GUCODE_SEL); 2089 *(struct mem_segment_descriptor *)(ldtstore + LUDATA_SEL) = 2090 *GDT_ADDR_MEM(gdtstore, GUDATA_SEL); 2091 2092 /* 2093 * 32 bit GDT entries. 2094 */ 2095 set_mem_segment(GDT_ADDR_MEM(gdtstore, GUCODE32_SEL), 0, 2096 x86_btop(VM_MAXUSER_ADDRESS32) - 1, SDT_MEMERA, SEL_UPL, 1, 1, 0); 2097 2098 set_mem_segment(GDT_ADDR_MEM(gdtstore, GUDATA32_SEL), 0, 2099 x86_btop(VM_MAXUSER_ADDRESS32) - 1, SDT_MEMRWA, SEL_UPL, 1, 1, 0); 2100 2101 set_mem_segment(GDT_ADDR_MEM(gdtstore, GUFS_SEL), 0, 2102 x86_btop(VM_MAXUSER_ADDRESS32) - 1, SDT_MEMRWA, SEL_UPL, 1, 1, 0); 2103 2104 set_mem_segment(GDT_ADDR_MEM(gdtstore, GUGS_SEL), 0, 2105 x86_btop(VM_MAXUSER_ADDRESS32) - 1, SDT_MEMRWA, SEL_UPL, 1, 1, 0); 2106 2107 /* 2108 * 32 bit LDT entries. 2109 */ 2110 ldt_segp = (struct mem_segment_descriptor *)(ldtstore + LUCODE32_SEL); 2111 set_mem_segment(ldt_segp, 0, x86_btop(VM_MAXUSER_ADDRESS32) - 1, 2112 SDT_MEMERA, SEL_UPL, 1, 1, 0); 2113 ldt_segp = (struct mem_segment_descriptor *)(ldtstore + LUDATA32_SEL); 2114 set_mem_segment(ldt_segp, 0, x86_btop(VM_MAXUSER_ADDRESS32) - 1, 2115 SDT_MEMRWA, SEL_UPL, 1, 1, 0); 2116 2117 /* CPU-specific IDT exceptions. */ 2118 for (x = 0; x < NCPUIDT; x++) { 2119 int sel, ist; 2120 2121 /* Reset to default. Special cases below */ 2122 sel = SEL_KPL; 2123 ist = 0; 2124 2125 idt_vec_reserve(iv, x); 2126 2127 switch (x) { 2128 case 1: /* DB */ 2129 ist = 4; 2130 break; 2131 case 2: /* NMI */ 2132 ist = 3; 2133 break; 2134 case 3: 2135 case 4: 2136 sel = SEL_UPL; 2137 break; 2138 case 8: /* double fault */ 2139 ist = 2; 2140 break; 2141 #ifdef XENPV 2142 case 18: /* MCA */ 2143 sel |= 0x4; /* Auto EOI/mask */ 2144 break; 2145 #endif /* XENPV */ 2146 default: 2147 break; 2148 } 2149 2150 set_idtgate(&idt[x], x86_exceptions[x], ist, SDT_SYS386IGT, 2151 sel, GSEL(GCODE_SEL, SEL_KPL)); 2152 } 2153 2154 /* new-style interrupt gate for syscalls */ 2155 idt_vec_reserve(iv, 128); 2156 set_idtgate(&idt[128], &IDTVEC(osyscall), 0, SDT_SYS386IGT, SEL_UPL, 2157 GSEL(GCODE_SEL, SEL_KPL)); 2158 2159 kpreempt_enable(); 2160 2161 setregion(®ion, gdtstore, DYNSEL_START - 1); 2162 lgdt(®ion); 2163 2164 #ifdef XENPV 2165 /* Init Xen callbacks and syscall handlers */ 2166 if (HYPERVISOR_set_callbacks( 2167 (unsigned long) hypervisor_callback, 2168 (unsigned long) failsafe_callback, 2169 (unsigned long) Xsyscall)) 2170 panic("HYPERVISOR_set_callbacks() failed"); 2171 #endif /* XENPV */ 2172 2173 cpu_init_idt(&cpu_info_primary); 2174 2175 #ifdef XENPV 2176 xen_init_ksyms(); 2177 #else /* XENPV */ 2178 #ifdef XEN 2179 if (vm_guest == VM_GUEST_XENPVH) 2180 xen_init_ksyms(); 2181 else 2182 #endif /* XEN */ 2183 init_x86_64_ksyms(); 2184 #endif /* XENPV */ 2185 2186 #ifndef XENPV 2187 intr_default_setup(); 2188 #else 2189 events_default_setup(); 2190 #endif 2191 2192 splraise(IPL_HIGH); 2193 x86_enable_intr(); 2194 2195 #ifdef DDB 2196 if (boothowto & RB_KDB) 2197 Debugger(); 2198 #endif 2199 #ifdef KGDB 2200 kgdb_port_init(); 2201 if (boothowto & RB_KDB) { 2202 kgdb_debug_init = 1; 2203 kgdb_connect(1); 2204 } 2205 #endif 2206 2207 pcb->pcb_dbregs = NULL; 2208 x86_dbregs_init(); 2209 } 2210 2211 void 2212 cpu_reset(void) 2213 { 2214 #ifndef XENPV 2215 idt_descriptor_t *idt; 2216 vaddr_t vaddr; 2217 2218 idt = cpu_info_primary.ci_idtvec.iv_idt; 2219 vaddr = (vaddr_t)idt; 2220 #endif 2221 2222 x86_disable_intr(); 2223 2224 #ifdef XENPV 2225 HYPERVISOR_reboot(); 2226 #else 2227 2228 x86_reset(); 2229 2230 /* 2231 * Try to cause a triple fault and watchdog reset by making the IDT 2232 * invalid and causing a fault. 2233 */ 2234 kpreempt_disable(); 2235 pmap_changeprot_local(vaddr, VM_PROT_READ|VM_PROT_WRITE); 2236 memset((void *)idt, 0, NIDT * sizeof(idt[0])); 2237 kpreempt_enable(); 2238 breakpoint(); 2239 2240 #if 0 2241 /* 2242 * Try to cause a triple fault and watchdog reset by unmapping the 2243 * entire address space and doing a TLB flush. 2244 */ 2245 memset((void *)PTD, 0, PAGE_SIZE); 2246 tlbflush(); 2247 #endif 2248 #endif /* XENPV */ 2249 2250 for (;;); 2251 } 2252 2253 void 2254 cpu_getmcontext(struct lwp *l, mcontext_t *mcp, unsigned int *flags) 2255 { 2256 const struct trapframe *tf = l->l_md.md_regs; 2257 __greg_t ras_rip; 2258 2259 mcp->__gregs[_REG_RDI] = tf->tf_rdi; 2260 mcp->__gregs[_REG_RSI] = tf->tf_rsi; 2261 mcp->__gregs[_REG_RDX] = tf->tf_rdx; 2262 mcp->__gregs[_REG_R10] = tf->tf_r10; 2263 mcp->__gregs[_REG_R8] = tf->tf_r8; 2264 mcp->__gregs[_REG_R9] = tf->tf_r9; 2265 /* argX not touched */ 2266 mcp->__gregs[_REG_RCX] = tf->tf_rcx; 2267 mcp->__gregs[_REG_R11] = tf->tf_r11; 2268 mcp->__gregs[_REG_R12] = tf->tf_r12; 2269 mcp->__gregs[_REG_R13] = tf->tf_r13; 2270 mcp->__gregs[_REG_R14] = tf->tf_r14; 2271 mcp->__gregs[_REG_R15] = tf->tf_r15; 2272 mcp->__gregs[_REG_RBP] = tf->tf_rbp; 2273 mcp->__gregs[_REG_RBX] = tf->tf_rbx; 2274 mcp->__gregs[_REG_RAX] = tf->tf_rax; 2275 mcp->__gregs[_REG_GS] = 0; 2276 mcp->__gregs[_REG_FS] = 0; 2277 mcp->__gregs[_REG_ES] = GSEL(GUDATA_SEL, SEL_UPL); 2278 mcp->__gregs[_REG_DS] = GSEL(GUDATA_SEL, SEL_UPL); 2279 mcp->__gregs[_REG_TRAPNO] = tf->tf_trapno; 2280 mcp->__gregs[_REG_ERR] = tf->tf_err; 2281 mcp->__gregs[_REG_RIP] = tf->tf_rip; 2282 mcp->__gregs[_REG_CS] = LSEL(LUCODE_SEL, SEL_UPL); 2283 mcp->__gregs[_REG_RFLAGS] = tf->tf_rflags; 2284 mcp->__gregs[_REG_RSP] = tf->tf_rsp; 2285 mcp->__gregs[_REG_SS] = LSEL(LUDATA_SEL, SEL_UPL); 2286 2287 if ((ras_rip = (__greg_t)ras_lookup(l->l_proc, 2288 (void *) mcp->__gregs[_REG_RIP])) != -1) 2289 mcp->__gregs[_REG_RIP] = ras_rip; 2290 2291 *flags |= _UC_CPU; 2292 2293 mcp->_mc_tlsbase = (uintptr_t)l->l_private; 2294 *flags |= _UC_TLSBASE; 2295 2296 process_read_fpregs_xmm(l, (struct fxsave *)&mcp->__fpregs.__fxsave); 2297 *flags |= _UC_FPU; 2298 } 2299 2300 /* 2301 * cpu_getmcontext_xsave(l, mcp, flags, xsavebuf, xsavelen, user_xsave) 2302 * 2303 * Copy out xsavebuf[512..xsavelen) to user_xsave[512..xsavelen), 2304 * set mcp to point at it, and set _UC_XSAVE in *flags. Caller: 2305 * 2306 * - must have already initialized the FXSAVE area of mcontext_t, 2307 * - must have already set _UC_FPU in *flags, 2308 * - must have arranged user_xsave[0..512) to overlap with the 2309 * FXSAVE area of mcontext_t, and 2310 * - must subsequently copy out the mcontext_t updated with a 2311 * pointer/length to the XSAVE area. 2312 * 2313 * May fail if the copyout fails. 2314 */ 2315 static int 2316 cpu_getmcontext_xsave(struct lwp *l, mcontext_t *mcp, unsigned int *flags, 2317 const struct xsave_header *xsavebuf, size_t xsavelen, 2318 struct xsave_header *user_xsave) 2319 { 2320 enum { fxsavelen = sizeof(mcp->__fpregs.__fxsave) }; 2321 int error; 2322 2323 CTASSERT(fxsavelen == 512); 2324 2325 KASSERT(*flags & _UC_FPU); 2326 KASSERT(fxsavelen <= xsavelen); 2327 KDASSERT(memcmp(&mcp->__fpregs.__fxsave, xsavebuf, fxsavelen) == 0); 2328 2329 /* 2330 * Copy out the part of the XSAVE area that doesn't overlap. 2331 */ 2332 error = copyout((const char *)xsavebuf + fxsavelen, 2333 (char *)user_xsave + fxsavelen, xsavelen - fxsavelen); 2334 if (error != 0) 2335 return error; 2336 2337 /* 2338 * Record a pointer to the real XSAVE area in the 2339 * architecturally unused bits mcontext_t's FXSAVE area. 2340 */ 2341 mcp->__fpregs.__xsave.__xsaveptr = (__greg_t)(uintptr_t)user_xsave; 2342 mcp->__fpregs.__xsave.__xsavelen = (__greg_t)xsavelen; 2343 2344 /* 2345 * Set the _UC_XSAVE flag so cpu_setmcontext will be able to 2346 * restore the full state from the XSAVE area. 2347 */ 2348 *flags |= _UC_XSAVE; 2349 2350 /* Success! */ 2351 return 0; 2352 } 2353 2354 int 2355 cpu_setmcontext(struct lwp *l, const mcontext_t *mcp, unsigned int flags) 2356 { 2357 struct trapframe *tf = l->l_md.md_regs; 2358 const __greg_t *gr = mcp->__gregs; 2359 struct proc *p = l->l_proc; 2360 struct xsave_header *xsavebuf = NULL; 2361 size_t xsavelen = 0; 2362 int error; 2363 int64_t rflags; 2364 2365 CTASSERT(sizeof (mcontext_t) == 26 * 8 + 8 + 512); 2366 2367 /* 2368 * If there's an external XSAVE area, copy it in and validate 2369 * it before we irreversibly modify the trapframe. 2370 * 2371 * We could check the length against the state components 2372 * included, but we currently don't: if it's truncated, it will 2373 * be as if the truncated part were zero-filled -- this is 2374 * implemented in process_write_xsave, called a little below. 2375 */ 2376 if ((flags & _UC_XSAVE) != 0) { 2377 const struct xsave_header *user_xsave = 2378 (void *)(uintptr_t)mcp->__fpregs.__xsave.__xsaveptr; 2379 2380 xsavelen = mcp->__fpregs.__xsave.__xsavelen; 2381 error = process_verify_xsavelen(l, xsavelen); 2382 if (error != 0) 2383 goto out; 2384 xsavebuf = kmem_alloc(xsavelen, KM_SLEEP); 2385 error = copyin(user_xsave, xsavebuf, xsavelen); 2386 if (error != 0) 2387 goto out; 2388 error = process_verify_xsave(l, xsavebuf, xsavelen); 2389 if (error != 0) 2390 goto out; 2391 } 2392 2393 if ((flags & _UC_CPU) != 0) { 2394 error = cpu_mcontext_validate(l, mcp); 2395 if (error != 0) 2396 goto out; 2397 2398 tf->tf_rdi = gr[_REG_RDI]; 2399 tf->tf_rsi = gr[_REG_RSI]; 2400 tf->tf_rdx = gr[_REG_RDX]; 2401 tf->tf_r10 = gr[_REG_R10]; 2402 tf->tf_r8 = gr[_REG_R8]; 2403 tf->tf_r9 = gr[_REG_R9]; 2404 /* argX not touched */ 2405 tf->tf_rcx = gr[_REG_RCX]; 2406 tf->tf_r11 = gr[_REG_R11]; 2407 tf->tf_r12 = gr[_REG_R12]; 2408 tf->tf_r13 = gr[_REG_R13]; 2409 tf->tf_r14 = gr[_REG_R14]; 2410 tf->tf_r15 = gr[_REG_R15]; 2411 tf->tf_rbp = gr[_REG_RBP]; 2412 tf->tf_rbx = gr[_REG_RBX]; 2413 tf->tf_rax = gr[_REG_RAX]; 2414 tf->tf_gs = 0; 2415 tf->tf_fs = 0; 2416 tf->tf_es = GSEL(GUDATA_SEL, SEL_UPL); 2417 tf->tf_ds = GSEL(GUDATA_SEL, SEL_UPL); 2418 /* trapno, err not touched */ 2419 tf->tf_rip = gr[_REG_RIP]; 2420 tf->tf_cs = LSEL(LUCODE_SEL, SEL_UPL); 2421 rflags = tf->tf_rflags; 2422 rflags &= ~PSL_USER; 2423 tf->tf_rflags = rflags | (gr[_REG_RFLAGS] & PSL_USER); 2424 tf->tf_rsp = gr[_REG_RSP]; 2425 tf->tf_ss = LSEL(LUDATA_SEL, SEL_UPL); 2426 2427 l->l_md.md_flags |= MDL_IRET; 2428 } 2429 2430 if ((flags & _UC_XSAVE) != 0) { 2431 KASSERT(xsavebuf != NULL); 2432 process_write_xsave(l, xsavebuf, xsavelen); 2433 } else if ((flags & _UC_FPU) != 0) { 2434 process_write_fpregs_xmm(l, 2435 (const struct fxsave *)&mcp->__fpregs.__fxsave); 2436 } 2437 2438 if ((flags & _UC_TLSBASE) != 0) 2439 lwp_setprivate(l, (void *)(uintptr_t)mcp->_mc_tlsbase); 2440 2441 mutex_enter(p->p_lock); 2442 if (flags & _UC_SETSTACK) 2443 l->l_sigstk.ss_flags |= SS_ONSTACK; 2444 if (flags & _UC_CLRSTACK) 2445 l->l_sigstk.ss_flags &= ~SS_ONSTACK; 2446 mutex_exit(p->p_lock); 2447 2448 /* Success! */ 2449 error = 0; 2450 2451 out: if (xsavebuf) 2452 kmem_free(xsavebuf, xsavelen); 2453 return error; 2454 } 2455 2456 int 2457 cpu_mcontext_validate(struct lwp *l, const mcontext_t *mcp) 2458 { 2459 struct proc *p __diagused = l->l_proc; 2460 struct trapframe *tf = l->l_md.md_regs; 2461 const __greg_t *gr; 2462 uint16_t sel; 2463 2464 KASSERT((p->p_flag & PK_32) == 0); 2465 gr = mcp->__gregs; 2466 2467 if (((gr[_REG_RFLAGS] ^ tf->tf_rflags) & PSL_USERSTATIC) != 0) 2468 return EINVAL; 2469 2470 sel = gr[_REG_ES] & 0xffff; 2471 if (sel != 0 && !VALID_USER_DSEL(sel)) 2472 return EINVAL; 2473 2474 sel = gr[_REG_FS] & 0xffff; 2475 if (sel != 0 && !VALID_USER_DSEL(sel)) 2476 return EINVAL; 2477 2478 sel = gr[_REG_GS] & 0xffff; 2479 if (sel != 0 && !VALID_USER_DSEL(sel)) 2480 return EINVAL; 2481 2482 sel = gr[_REG_DS] & 0xffff; 2483 if (!VALID_USER_DSEL(sel)) 2484 return EINVAL; 2485 2486 #ifndef XENPV 2487 sel = gr[_REG_SS] & 0xffff; 2488 if (!VALID_USER_DSEL(sel)) 2489 return EINVAL; 2490 2491 sel = gr[_REG_CS] & 0xffff; 2492 if (!VALID_USER_CSEL(sel)) 2493 return EINVAL; 2494 #endif 2495 2496 if (gr[_REG_RIP] >= VM_MAXUSER_ADDRESS) 2497 return EINVAL; 2498 2499 return 0; 2500 } 2501 2502 int 2503 mm_md_kernacc(void *ptr, vm_prot_t prot, bool *handled) 2504 { 2505 const vaddr_t v = (vaddr_t)ptr; 2506 vaddr_t kva, kva_end; 2507 size_t i; 2508 2509 kva = bootspace.head.va; 2510 kva_end = kva + bootspace.head.sz; 2511 if (v >= kva && v < kva_end) { 2512 *handled = true; 2513 return 0; 2514 } 2515 2516 for (i = 0; i < BTSPACE_NSEGS; i++) { 2517 kva = bootspace.segs[i].va; 2518 kva_end = kva + bootspace.segs[i].sz; 2519 if (v < kva || v >= kva_end) 2520 continue; 2521 *handled = true; 2522 if (bootspace.segs[i].type == BTSEG_TEXT || 2523 bootspace.segs[i].type == BTSEG_RODATA) { 2524 if (prot & VM_PROT_WRITE) { 2525 return EFAULT; 2526 } 2527 } 2528 return 0; 2529 } 2530 2531 kva = bootspace.boot.va; 2532 kva_end = kva + bootspace.boot.sz; 2533 if (v >= kva && v < kva_end) { 2534 *handled = true; 2535 return 0; 2536 } 2537 2538 if (v >= bootspace.smodule && v < bootspace.emodule) { 2539 *handled = true; 2540 vm_map_lock_read(module_map); 2541 const bool allowed = 2542 uvm_map_checkprot(module_map, v, v + 1, prot); 2543 vm_map_unlock_read(module_map); 2544 if (!allowed) { 2545 return EFAULT; 2546 } 2547 } else { 2548 *handled = false; 2549 } 2550 return 0; 2551 } 2552 2553 /* 2554 * Zero out a 64bit LWP's segments registers. Used when exec'ing a new 2555 * 64bit program. 2556 */ 2557 void 2558 cpu_segregs64_zero(struct lwp *l) 2559 { 2560 struct trapframe * const tf = l->l_md.md_regs; 2561 struct pcb *pcb; 2562 uint64_t zero = 0; 2563 2564 KASSERT(kpreempt_disabled()); 2565 KASSERT((l->l_proc->p_flag & PK_32) == 0); 2566 KASSERT(l == curlwp); 2567 2568 pcb = lwp_getpcb(l); 2569 2570 tf->tf_fs = 0; 2571 tf->tf_gs = 0; 2572 setds(GSEL(GUDATA_SEL, SEL_UPL)); 2573 setes(GSEL(GUDATA_SEL, SEL_UPL)); 2574 setfs(0); 2575 setusergs(0); 2576 2577 #ifndef XENPV 2578 wrmsr(MSR_FSBASE, 0); 2579 wrmsr(MSR_KERNELGSBASE, 0); 2580 #else 2581 HYPERVISOR_set_segment_base(SEGBASE_FS, 0); 2582 HYPERVISOR_set_segment_base(SEGBASE_GS_USER, 0); 2583 #endif 2584 2585 pcb->pcb_fs = 0; 2586 pcb->pcb_gs = 0; 2587 update_descriptor(&curcpu()->ci_gdt[GUFS_SEL], &zero); 2588 update_descriptor(&curcpu()->ci_gdt[GUGS_SEL], &zero); 2589 } 2590 2591 /* 2592 * Zero out a 32bit LWP's segments registers. Used when exec'ing a new 2593 * 32bit program. 2594 */ 2595 void 2596 cpu_segregs32_zero(struct lwp *l) 2597 { 2598 struct trapframe * const tf = l->l_md.md_regs; 2599 struct pcb *pcb; 2600 uint64_t zero = 0; 2601 2602 KASSERT(kpreempt_disabled()); 2603 KASSERT(l->l_proc->p_flag & PK_32); 2604 KASSERT(l == curlwp); 2605 2606 pcb = lwp_getpcb(l); 2607 2608 tf->tf_fs = 0; 2609 tf->tf_gs = 0; 2610 setds(GSEL(GUDATA32_SEL, SEL_UPL)); 2611 setes(GSEL(GUDATA32_SEL, SEL_UPL)); 2612 setfs(0); 2613 setusergs(0); 2614 pcb->pcb_fs = 0; 2615 pcb->pcb_gs = 0; 2616 update_descriptor(&curcpu()->ci_gdt[GUFS_SEL], &zero); 2617 update_descriptor(&curcpu()->ci_gdt[GUGS_SEL], &zero); 2618 } 2619 2620 /* 2621 * Load an LWP's TLS context, possibly changing the %fs and %gs selectors. 2622 * Used only for 32-bit processes. 2623 */ 2624 void 2625 cpu_fsgs_reload(struct lwp *l, int fssel, int gssel) 2626 { 2627 struct trapframe *tf; 2628 struct pcb *pcb; 2629 2630 KASSERT(l->l_proc->p_flag & PK_32); 2631 KASSERT(l == curlwp); 2632 2633 tf = l->l_md.md_regs; 2634 fssel &= 0xFFFF; 2635 gssel &= 0xFFFF; 2636 2637 pcb = lwp_getpcb(l); 2638 kpreempt_disable(); 2639 update_descriptor(&curcpu()->ci_gdt[GUFS_SEL], &pcb->pcb_fs); 2640 update_descriptor(&curcpu()->ci_gdt[GUGS_SEL], &pcb->pcb_gs); 2641 2642 #ifdef XENPV 2643 setusergs(gssel); 2644 #endif 2645 2646 tf->tf_fs = fssel; 2647 tf->tf_gs = gssel; 2648 kpreempt_enable(); 2649 } 2650 2651 bool 2652 mm_md_direct_mapped_io(void *addr, paddr_t *paddr) 2653 { 2654 vaddr_t va = (vaddr_t)addr; 2655 2656 #ifdef __HAVE_DIRECT_MAP 2657 if (va >= PMAP_DIRECT_BASE && va < PMAP_DIRECT_END) { 2658 *paddr = PMAP_DIRECT_UNMAP(va); 2659 return true; 2660 } 2661 #else 2662 __USE(va); 2663 #endif 2664 2665 return false; 2666 } 2667 2668 bool 2669 mm_md_direct_mapped_phys(paddr_t paddr, vaddr_t *vaddr) 2670 { 2671 #ifdef __HAVE_DIRECT_MAP 2672 *vaddr = PMAP_DIRECT_MAP(paddr); 2673 return true; 2674 #else 2675 return false; 2676 #endif 2677 } 2678 2679 static void 2680 idt_vec_copy(struct idt_vec *dst, struct idt_vec *src) 2681 { 2682 idt_descriptor_t *idt_dst; 2683 2684 idt_dst = dst->iv_idt; 2685 2686 kpreempt_disable(); 2687 pmap_changeprot_local((vaddr_t)idt_dst, VM_PROT_READ|VM_PROT_WRITE); 2688 2689 memcpy(idt_dst, src->iv_idt, PAGE_SIZE); 2690 memcpy(dst->iv_allocmap, src->iv_allocmap, sizeof(dst->iv_allocmap)); 2691 2692 pmap_changeprot_local((vaddr_t)idt_dst, VM_PROT_READ); 2693 kpreempt_enable(); 2694 } 2695 2696 void 2697 idt_vec_init_cpu_md(struct idt_vec *iv, cpuid_t cid) 2698 { 2699 vaddr_t va; 2700 2701 if (cid != cpu_index(&cpu_info_primary) && 2702 idt_vec_is_pcpu()) { 2703 #ifdef __HAVE_PCPU_AREA 2704 va = (vaddr_t)&pcpuarea->ent[cid].idt; 2705 #else 2706 struct vm_page *pg; 2707 2708 va = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, 2709 UVM_KMF_VAONLY); 2710 pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO); 2711 if (pg == NULL) { 2712 panic("failed to allocate a page for IDT"); 2713 } 2714 pmap_kenter_pa(va, VM_PAGE_TO_PHYS(pg), 2715 VM_PROT_READ|VM_PROT_WRITE, 0); 2716 pmap_update(pmap_kernel()); 2717 #endif 2718 2719 memset((void *)va, 0, PAGE_SIZE); 2720 #ifndef XENPV 2721 pmap_changeprot_local(va, VM_PROT_READ); 2722 #endif 2723 pmap_update(pmap_kernel()); 2724 2725 iv->iv_idt = (void *)va; 2726 idt_vec_copy(iv, &(cpu_info_primary.ci_idtvec)); 2727 } else { 2728 iv->iv_idt = (void *)idt_vaddr; 2729 } 2730 } 2731