Home | History | Annotate | Line # | Download | only in amd64
      1 /*	$NetBSD: machdep.c,v 1.383 2026/11/10 03:43:25 thorpej Exp $	*/
      2 
      3 /*
      4  * Copyright (c) 1996, 1997, 1998, 2000, 2006, 2007, 2008, 2011
      5  *     The NetBSD Foundation, Inc.
      6  * All rights reserved.
      7  *
      8  * This code is derived from software contributed to The NetBSD Foundation
      9  * by Charles M. Hannum and by Jason R. Thorpe of the Numerical Aerospace
     10  * Simulation Facility, NASA Ames Research Center.
     11  *
     12  * This code is derived from software contributed to The NetBSD Foundation
     13  * by Coyote Point Systems, Inc. which was written under contract to Coyote
     14  * Point by Jed Davis and Devon O'Dell.
     15  *
     16  * Redistribution and use in source and binary forms, with or without
     17  * modification, are permitted provided that the following conditions
     18  * are met:
     19  * 1. Redistributions of source code must retain the above copyright
     20  *    notice, this list of conditions and the following disclaimer.
     21  * 2. Redistributions in binary form must reproduce the above copyright
     22  *    notice, this list of conditions and the following disclaimer in the
     23  *    documentation and/or other materials provided with the distribution.
     24  *
     25  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     26  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     27  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     28  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     29  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     30  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     31  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     32  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     33  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     34  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     35  * POSSIBILITY OF SUCH DAMAGE.
     36  */
     37 
     38 /*
     39  * Copyright (c) 2006 Mathieu Ropert <mro (at) adviseo.fr>
     40  *
     41  * Permission to use, copy, modify, and distribute this software for any
     42  * purpose with or without fee is hereby granted, provided that the above
     43  * copyright notice and this permission notice appear in all copies.
     44  *
     45  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
     46  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
     47  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
     48  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
     49  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
     50  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
     51  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
     52  */
     53 
     54 /*
     55  * Copyright (c) 2007 Manuel Bouyer.
     56  *
     57  * Redistribution and use in source and binary forms, with or without
     58  * modification, are permitted provided that the following conditions
     59  * are met:
     60  * 1. Redistributions of source code must retain the above copyright
     61  *    notice, this list of conditions and the following disclaimer.
     62  * 2. Redistributions in binary form must reproduce the above copyright
     63  *    notice, this list of conditions and the following disclaimer in the
     64  *    documentation and/or other materials provided with the distribution.
     65  *
     66  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
     67  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
     68  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
     69  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
     70  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
     71  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
     72  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
     73  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
     74  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
     75  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     76  */
     77 
     78 /*
     79  * Copyright (c) 1982, 1987, 1990 The Regents of the University of California.
     80  * All rights reserved.
     81  *
     82  * This code is derived from software contributed to Berkeley by
     83  * William Jolitz.
     84  *
     85  * Redistribution and use in source and binary forms, with or without
     86  * modification, are permitted provided that the following conditions
     87  * are met:
     88  * 1. Redistributions of source code must retain the above copyright
     89  *    notice, this list of conditions and the following disclaimer.
     90  * 2. Redistributions in binary form must reproduce the above copyright
     91  *    notice, this list of conditions and the following disclaimer in the
     92  *    documentation and/or other materials provided with the distribution.
     93  * 3. Neither the name of the University nor the names of its contributors
     94  *    may be used to endorse or promote products derived from this software
     95  *    without specific prior written permission.
     96  *
     97  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
     98  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
     99  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
    100  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
    101  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
    102  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
    103  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
    104  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
    105  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
    106  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
    107  * SUCH DAMAGE.
    108  *
    109  *	@(#)machdep.c	7.4 (Berkeley) 6/3/91
    110  */
    111 
    112 #include <sys/cdefs.h>
    113 __KERNEL_RCSID(0, "$NetBSD: machdep.c,v 1.383 2026/11/10 03:43:25 thorpej Exp $");
    114 
    115 #include "opt_modular.h"
    116 #include "opt_user_ldt.h"
    117 #include "opt_ddb.h"
    118 #include "opt_kgdb.h"
    119 #include "opt_cpureset_delay.h"
    120 #include "opt_mtrr.h"
    121 #include "opt_realmem.h"
    122 #include "opt_xen.h"
    123 #include "opt_svs.h"
    124 #include "opt_kaslr.h"
    125 #ifndef XENPV
    126 #include "opt_physmem.h"
    127 #endif
    128 #include "isa.h"
    129 #include "pci.h"
    130 
    131 #include <sys/param.h>
    132 #include <sys/systm.h>
    133 #include <sys/signal.h>
    134 #include <sys/signalvar.h>
    135 #include <sys/kernel.h>
    136 #include <sys/cpu.h>
    137 #include <sys/exec.h>
    138 #include <sys/exec_aout.h>	/* for MID_* */
    139 #include <sys/reboot.h>
    140 #include <sys/conf.h>
    141 #include <sys/msgbuf.h>
    142 #include <sys/mount.h>
    143 #include <sys/core.h>
    144 #include <sys/kcore.h>
    145 #include <sys/ucontext.h>
    146 #include <machine/kcore.h>
    147 #include <sys/ras.h>
    148 #include <sys/syscallargs.h>
    149 #include <sys/ksyms.h>
    150 #include <sys/device.h>
    151 #include <sys/lwp.h>
    152 #include <sys/proc.h>
    153 #include <sys/asan.h>
    154 #include <sys/csan.h>
    155 #include <sys/msan.h>
    156 #include <sys/module.h>
    157 #include <sys/timevar.h>
    158 
    159 #ifdef KGDB
    160 #include <sys/kgdb.h>
    161 #endif
    162 
    163 #include <lib/libkern/entpool.h> /* XXX */
    164 
    165 #include <dev/cons.h>
    166 #include <dev/mm.h>
    167 
    168 #include <uvm/uvm.h>
    169 #include <uvm/uvm_page.h>
    170 
    171 #include <sys/sysctl.h>
    172 
    173 #include <machine/cpu.h>
    174 #include <machine/cpu_rng.h>
    175 #include <machine/cpufunc.h>
    176 #include <machine/gdt.h>
    177 #include <machine/intr.h>
    178 #include <machine/pio.h>
    179 #include <machine/psl.h>
    180 #include <machine/reg.h>
    181 #include <machine/specialreg.h>
    182 #include <machine/bootinfo.h>
    183 #include <x86/fpu.h>
    184 #include <x86/dbregs.h>
    185 #include <machine/mtrr.h>
    186 #include <machine/mpbiosvar.h>
    187 #include <machine/pmap_private.h>
    188 
    189 #include <x86/bootspace.h>
    190 #include <x86/cputypes.h>
    191 #include <x86/cpuvar.h>
    192 #include <x86/machdep.h>
    193 #include <x86/x86/tsc.h>
    194 
    195 #include <dev/isa/isareg.h>
    196 #include <machine/isa_machdep.h>
    197 #include <dev/ic/i8042reg.h>
    198 
    199 #ifdef XEN
    200 #include <xen/xen.h>
    201 #include <xen/hypervisor.h>
    202 #include <xen/evtchn.h>
    203 #include <xen/include/public/version.h>
    204 #include <xen/include/public/vcpu.h>
    205 #endif /* XEN */
    206 
    207 #include <ddb/db_active.h>
    208 #ifdef DDB
    209 #include <machine/db_machdep.h>
    210 #include <ddb/db_extern.h>
    211 #include <ddb/db_output.h>
    212 #include <ddb/db_interface.h>
    213 #endif
    214 
    215 #include "acpica.h"
    216 
    217 #if NACPICA > 0
    218 #include <dev/acpi/acpivar.h>
    219 #define ACPI_MACHDEP_PRIVATE
    220 #include <machine/acpi_machdep.h>
    221 #else
    222 #include <machine/i82489var.h>
    223 #endif
    224 
    225 #include "isa.h"
    226 #include "isadma.h"
    227 #include "ksyms.h"
    228 
    229 /* the following is used externally (sysctl_hw) */
    230 char machine[] = "amd64";		/* CPU "architecture" */
    231 char machine_arch[] = "x86_64";		/* machine == machine_arch */
    232 
    233 #ifdef CPURESET_DELAY
    234 int cpureset_delay = CPURESET_DELAY;
    235 #else
    236 int cpureset_delay = 2000; /* default to 2s */
    237 #endif
    238 
    239 int cpu_class = CPUCLASS_686;
    240 
    241 #ifdef MTRR
    242 const struct mtrr_funcs *mtrr_funcs;
    243 #endif
    244 
    245 int cpu_class;
    246 int use_pae;
    247 
    248 #ifndef NO_SPARSE_DUMP
    249 int sparse_dump = 1;
    250 
    251 paddr_t max_paddr = 0;
    252 unsigned char *sparse_dump_physmap;
    253 #endif
    254 
    255 char *dump_headerbuf, *dump_headerbuf_ptr;
    256 #define dump_headerbuf_size PAGE_SIZE
    257 #define dump_headerbuf_end (dump_headerbuf + dump_headerbuf_size)
    258 #define dump_headerbuf_avail (dump_headerbuf_end - dump_headerbuf_ptr)
    259 daddr_t dump_header_blkno;
    260 
    261 size_t dump_nmemsegs;
    262 size_t dump_npages;
    263 size_t dump_header_size;
    264 size_t dump_totalbytesleft;
    265 
    266 vaddr_t idt_vaddr;
    267 paddr_t idt_paddr;
    268 vaddr_t gdt_vaddr;
    269 paddr_t gdt_paddr;
    270 vaddr_t ldt_vaddr;
    271 paddr_t ldt_paddr;
    272 
    273 static struct vm_map module_map_store;
    274 extern struct bootspace bootspace;
    275 extern struct slotspace slotspace;
    276 
    277 vaddr_t vm_min_kernel_address __read_mostly = VM_MIN_KERNEL_ADDRESS_DEFAULT;
    278 vaddr_t vm_max_kernel_address __read_mostly = VM_MAX_KERNEL_ADDRESS_DEFAULT;
    279 pd_entry_t *pte_base __read_mostly;
    280 
    281 struct vm_map *phys_map = NULL;
    282 
    283 extern paddr_t lowmem_rsvd;
    284 extern paddr_t avail_start, avail_end;
    285 #ifdef XENPV
    286 extern paddr_t pmap_pa_start, pmap_pa_end;
    287 #endif
    288 
    289 struct nmistore {
    290 	uint64_t cr3;
    291 	uint64_t scratch;
    292 } __packed;
    293 
    294 /*
    295  * Size of memory segments, before any memory is stolen.
    296  */
    297 phys_ram_seg_t mem_clusters[VM_PHYSSEG_MAX];
    298 int mem_cluster_cnt;
    299 
    300 int cpu_dump(void);
    301 int cpu_dumpsize(void);
    302 u_long cpu_dump_mempagecnt(void);
    303 void dodumpsys(void);
    304 void dumpsys(void);
    305 
    306 static void x86_64_proc0_pcb_ldt_init(void);
    307 
    308 void dump_misc_init(void);
    309 void dump_seg_prep(void);
    310 int dump_seg_iter(int (*)(paddr_t, paddr_t));
    311 
    312 #ifndef NO_SPARSE_DUMP
    313 void sparse_dump_reset(void);
    314 void sparse_dump_mark(void);
    315 void cpu_dump_prep_sparse(void);
    316 #endif
    317 
    318 void dump_header_start(void);
    319 int dump_header_flush(void);
    320 int dump_header_addbytes(const void*, size_t);
    321 int dump_header_addseg(paddr_t, paddr_t);
    322 int dump_header_finish(void);
    323 
    324 int dump_seg_count_range(paddr_t, paddr_t);
    325 int dumpsys_seg(paddr_t, paddr_t);
    326 
    327 void init_bootspace(void);
    328 void init_slotspace(void);
    329 void init_x86_64(paddr_t);
    330 
    331 static int cpu_getmcontext_xsave(struct lwp *, mcontext_t *, unsigned *,
    332     const struct xsave_header *, size_t, struct xsave_header *);
    333 
    334 /*
    335  * Machine-dependent startup code
    336  */
    337 void
    338 cpu_startup(void)
    339 {
    340 	int x, y;
    341 	vaddr_t minaddr, maxaddr;
    342 	psize_t sz;
    343 
    344 	/*
    345 	 * For console drivers that require uvm and pmap to be initialized,
    346 	 * we'll give them one more chance here...
    347 	 */
    348 	consinit();
    349 
    350 	/*
    351 	 * Initialize error message buffer (at end of core).
    352 	 */
    353 	if (msgbuf_p_cnt == 0)
    354 		panic("msgbuf paddr map has not been set up");
    355 	for (x = 0, sz = 0; x < msgbuf_p_cnt; sz += msgbuf_p_seg[x++].sz)
    356 		continue;
    357 
    358 	msgbuf_vaddr = uvm_km_alloc(kernel_map, sz, 0, UVM_KMF_VAONLY);
    359 	if (msgbuf_vaddr == 0)
    360 		panic("failed to valloc msgbuf_vaddr");
    361 
    362 	for (y = 0, sz = 0; y < msgbuf_p_cnt; y++) {
    363 		for (x = 0; x < btoc(msgbuf_p_seg[y].sz); x++, sz += PAGE_SIZE)
    364 			pmap_kenter_pa((vaddr_t)msgbuf_vaddr + sz,
    365 			    msgbuf_p_seg[y].paddr + x * PAGE_SIZE,
    366 			    VM_PROT_READ|VM_PROT_WRITE, 0);
    367 	}
    368 
    369 	pmap_update(pmap_kernel());
    370 
    371 	initmsgbuf((void *)msgbuf_vaddr, round_page(sz));
    372 
    373 	minaddr = 0;
    374 
    375 	/*
    376 	 * Allocate a submap for physio.
    377 	 */
    378 	phys_map = uvm_km_suballoc(kernel_map, &minaddr, &maxaddr,
    379 	    VM_PHYS_SIZE, 0, false, NULL);
    380 
    381 	/*
    382 	 * Create the module map.
    383 	 *
    384 	 * The kernel uses RIP-relative addressing with a maximum offset of
    385 	 * 2GB. Because of that, we can't put the kernel modules in kernel_map
    386 	 * (like i386 does), since kernel_map is too far away in memory from
    387 	 * the kernel sections. So we have to create a special module_map.
    388 	 *
    389 	 * The module map is taken as what is left of the bootstrap memory
    390 	 * created in locore/prekern.
    391 	 */
    392 	uvm_map_setup(&module_map_store, bootspace.smodule,
    393 	    bootspace.emodule, 0);
    394 	module_map_store.pmap = pmap_kernel();
    395 	module_map = &module_map_store;
    396 
    397 	/* Say hello. */
    398 	banner();
    399 
    400 #if NISA > 0 || NPCI > 0
    401 	/* Safe for i/o port / memory space allocation to use malloc now. */
    402 	x86_bus_space_mallocok();
    403 #endif
    404 
    405 #ifdef __HAVE_PCPU_AREA
    406 	cpu_pcpuarea_init(&cpu_info_primary);
    407 #endif
    408 	gdt_init();
    409 	x86_64_proc0_pcb_ldt_init();
    410 
    411 	cpu_init_tss(&cpu_info_primary);
    412 #if !defined(XENPV)
    413 	ltr(cpu_info_primary.ci_tss_sel);
    414 #endif
    415 
    416 	x86_startup();
    417 }
    418 
    419 #ifdef XENPV
    420 /* used in assembly */
    421 void hypervisor_callback(void);
    422 void failsafe_callback(void);
    423 void x86_64_switch_context(struct pcb *);
    424 void x86_64_tls_switch(struct lwp *);
    425 
    426 void
    427 x86_64_switch_context(struct pcb *new)
    428 {
    429 	HYPERVISOR_stack_switch(GSEL(GDATA_SEL, SEL_KPL), new->pcb_rsp0);
    430 	struct physdev_set_iopl set_iopl;
    431 	set_iopl.iopl = new->pcb_iopl;
    432 	HYPERVISOR_physdev_op(PHYSDEVOP_set_iopl, &set_iopl);
    433 }
    434 
    435 void
    436 x86_64_tls_switch(struct lwp *l)
    437 {
    438 	struct cpu_info *ci = curcpu();
    439 	struct pcb *pcb = lwp_getpcb(l);
    440 	struct trapframe *tf = l->l_md.md_regs;
    441 	uint64_t zero = 0;
    442 
    443 	/*
    444 	 * Raise the IPL to IPL_HIGH. XXX Still needed?
    445 	 */
    446 	(void)splhigh();
    447 
    448 	/* Update segment registers */
    449 	if (pcb->pcb_flags & PCB_COMPAT32) {
    450 		update_descriptor(&ci->ci_gdt[GUFS_SEL], &pcb->pcb_fs);
    451 		update_descriptor(&ci->ci_gdt[GUGS_SEL], &pcb->pcb_gs);
    452 		setds(GSEL(GUDATA32_SEL, SEL_UPL));
    453 		setes(GSEL(GUDATA32_SEL, SEL_UPL));
    454 		setfs(GSEL(GUDATA32_SEL, SEL_UPL));
    455 		HYPERVISOR_set_segment_base(SEGBASE_GS_USER_SEL, tf->tf_gs);
    456 	} else {
    457 		update_descriptor(&ci->ci_gdt[GUFS_SEL], &zero);
    458 		update_descriptor(&ci->ci_gdt[GUGS_SEL], &zero);
    459 		setds(GSEL(GUDATA_SEL, SEL_UPL));
    460 		setes(GSEL(GUDATA_SEL, SEL_UPL));
    461 		setfs(0);
    462 		HYPERVISOR_set_segment_base(SEGBASE_GS_USER_SEL, 0);
    463 		HYPERVISOR_set_segment_base(SEGBASE_FS, pcb->pcb_fs);
    464 		HYPERVISOR_set_segment_base(SEGBASE_GS_USER, pcb->pcb_gs);
    465 	}
    466 }
    467 #endif /* XENPV */
    468 
    469 /*
    470  * Set up proc0's PCB and LDT.
    471  */
    472 static void
    473 x86_64_proc0_pcb_ldt_init(void)
    474 {
    475 	struct lwp *l = &lwp0;
    476 	struct pcb *pcb = lwp_getpcb(l);
    477 
    478 	pcb->pcb_flags = 0;
    479 	pcb->pcb_fs = 0;
    480 	pcb->pcb_gs = 0;
    481 	pcb->pcb_rsp0 = (uvm_lwp_getuarea(l) + USPACE - 16) & ~0xf;
    482 	pcb->pcb_iopl = IOPL_KPL;
    483 	pcb->pcb_dbregs = NULL;
    484 	pcb->pcb_cr0 = rcr0() & ~CR0_TS;
    485 	l->l_md.md_regs = (struct trapframe *)pcb->pcb_rsp0 - 1;
    486 
    487 #if !defined(XENPV)
    488 	lldt(GSYSSEL(GLDT_SEL, SEL_KPL));
    489 #else
    490 	xen_set_ldt((vaddr_t)ldtstore, LDT_SIZE >> 3);
    491 	/* Reset TS bit and set kernel stack for interrupt handlers */
    492 	HYPERVISOR_fpu_taskswitch(1);
    493 	HYPERVISOR_stack_switch(GSEL(GDATA_SEL, SEL_KPL), pcb->pcb_rsp0);
    494 	struct physdev_set_iopl set_iopl;
    495 	set_iopl.iopl = pcb->pcb_iopl;
    496 	HYPERVISOR_physdev_op(PHYSDEVOP_set_iopl, &set_iopl);
    497 #endif
    498 }
    499 
    500 /*
    501  * Set up TSS and I/O bitmap.
    502  */
    503 void
    504 cpu_init_tss(struct cpu_info *ci)
    505 {
    506 #ifdef __HAVE_PCPU_AREA
    507 	const cpuid_t cid = cpu_index(ci);
    508 #endif
    509 	struct cpu_tss *cputss;
    510 	struct nmistore *store;
    511 	uintptr_t p;
    512 
    513 #ifdef __HAVE_PCPU_AREA
    514 	cputss = (struct cpu_tss *)&pcpuarea->ent[cid].tss;
    515 #else
    516 	cputss = (struct cpu_tss *)uvm_km_alloc(kernel_map,
    517 	    sizeof(struct cpu_tss), 0, UVM_KMF_WIRED|UVM_KMF_ZERO);
    518 #endif
    519 
    520 	cputss->tss.tss_iobase = IOMAP_INVALOFF << 16;
    521 
    522 	/* DDB stack */
    523 #ifdef __HAVE_PCPU_AREA
    524 	p = (vaddr_t)&pcpuarea->ent[cid].ist0;
    525 #else
    526 	p = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, UVM_KMF_WIRED|UVM_KMF_ZERO);
    527 #endif
    528 	cputss->tss.tss_ist[0] = p + PAGE_SIZE - 16;
    529 
    530 	/* double fault */
    531 #ifdef __HAVE_PCPU_AREA
    532 	p = (vaddr_t)&pcpuarea->ent[cid].ist1;
    533 #else
    534 	p = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, UVM_KMF_WIRED|UVM_KMF_ZERO);
    535 #endif
    536 	cputss->tss.tss_ist[1] = p + PAGE_SIZE - 16;
    537 
    538 	/* NMI - store a structure at the top of the stack */
    539 #ifdef __HAVE_PCPU_AREA
    540 	p = (vaddr_t)&pcpuarea->ent[cid].ist2;
    541 #else
    542 	p = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, UVM_KMF_WIRED|UVM_KMF_ZERO);
    543 #endif
    544 	cputss->tss.tss_ist[2] = p + PAGE_SIZE - sizeof(struct nmistore);
    545 	store = (struct nmistore *)(p + PAGE_SIZE - sizeof(struct nmistore));
    546 	store->cr3 = pmap_pdirpa(pmap_kernel(), 0);
    547 
    548 	/* DB */
    549 #ifdef __HAVE_PCPU_AREA
    550 	p = (vaddr_t)&pcpuarea->ent[cid].ist3;
    551 #else
    552 	p = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, UVM_KMF_WIRED|UVM_KMF_ZERO);
    553 #endif
    554 	cputss->tss.tss_ist[3] = p + PAGE_SIZE - 16;
    555 
    556 	ci->ci_tss = cputss;
    557 	ci->ci_tss_sel = tss_alloc(&cputss->tss);
    558 }
    559 
    560 void
    561 buildcontext(struct lwp *l, void *catcher, void *f)
    562 {
    563 	struct trapframe *tf = l->l_md.md_regs;
    564 
    565 	tf->tf_ds = GSEL(GUDATA_SEL, SEL_UPL);
    566 	tf->tf_es = GSEL(GUDATA_SEL, SEL_UPL);
    567 	tf->tf_fs = GSEL(GUDATA_SEL, SEL_UPL);
    568 	tf->tf_gs = GSEL(GUDATA_SEL, SEL_UPL);
    569 
    570 	tf->tf_rip = (uint64_t)catcher;
    571 	tf->tf_cs = GSEL(GUCODE_SEL, SEL_UPL);
    572 	tf->tf_rflags &= ~PSL_CLEARSIG;
    573 	tf->tf_rsp = (uint64_t)f;
    574 	tf->tf_ss = GSEL(GUDATA_SEL, SEL_UPL);
    575 
    576 	/* Ensure FP state is sane */
    577 	fpu_sigreset(l);
    578 }
    579 
    580 void
    581 sendsig_sigcontext(const ksiginfo_t *ksi, const sigset_t *mask)
    582 {
    583 
    584 	printf("sendsig_sigcontext: illegal\n");
    585 	sigexit(curlwp, SIGILL);
    586 }
    587 
    588 /*
    589  * sendsig_siginfo(ksi, mask)
    590  *
    591  *	Set up a stack frame and registers for signal delivery with the
    592  *	given siginfo and signal mask.
    593  *
    594  *	The stack frame layout is as follows, with XSAVE parts omitted
    595  *	(and _UC_XSAVE clear in uc_flags) if the process doesn't have
    596  *	XSAVE state to restore:
    597  *
    598  *	        / +-----------------------+
    599  *	       |  | (other XSAVE stuff)   |
    600  *	       |  | --------------------- |
    601  *	       |  | Hi16_ZMM              |
    602  *	       |  | --------------------- |
    603  *	 XSAVE <  | ZMM_Hi256             |
    604  *	       |  | --------------------- |
    605  *	       |  | YMM_Hi128             |
    606  *	       |  *-----------------------* \
    607  *	       |  | FXSAVE in ucontext_t  |  |
    608  *	        \ *-----------------------*  > ucontext_t
    609  *	          | rest of ucontext_t    |  |
    610  *	          +-----------------------+ /
    611  *	          | siginfo_t             |
    612  *	          +-----------------------+ =0 (mod 16)
    613  *	          | return address        |
    614  *	          +-----------------------+ <--- rsp on signal handler entry
    615  *
    616  *	Additionally, when XSAVE is in use, there is a pointer to the
    617  *	XSAVE area in some architecturally unused padding in the FXSAVE
    618  *	area.  This is important because on i386, the XSAVE area
    619  *	_cannot_ overlap the ucontext_t owing to past ABI constraints,
    620  *	so applications running on both i386 and amd64 can use the same
    621  *	code to get the XSAVE pointer (and for 11.0, but not any
    622  *	earlier or later versions, the XSAVE pointer was also needed
    623  *	for signal handlers to write registers back on amd64 because
    624  *	the XSAVE area and the ucontext_t did not overlap).
    625  *
    626  *	The overlap of the FXSAVE section in the XSAVE area and the
    627  *	FXSAVE section in ucontext_t is tricky but important: signal
    628  *	handlers use it to read, and potentially write back, the x87
    629  *	and xmm registers.  Consider an application with two parts
    630  *	(e.g., a new library that uses an AVX512 optimization once it
    631  *	is available, and an old part running unmodified existing code
    632  *	predating NetBSD's AVX512 support):
    633  *
    634  *	1. Uses the high 16 zmm registers.
    635  *	2. (a) Uses _only_ x87/xmm registers, and
    636  *	   (b) has a signal handler that reads and writes them back.
    637  *
    638  *	After part (1), the system will consider XSAVE necessary to
    639  *	save and restore the high 16 zmm registers, even if -- as is
    640  *	customary when using the ymm or low 16 zmm registers -- the
    641  *	application issues VZEROUPPER when part (1) is done.  For this
    642  *	application to work without changing the signal handler in part
    643  *	(2)(b) to teach it about the XSAVE pointer -- i.e., for
    644  *	compatibility with existing code -- the XSAVE area and the
    645  *	ucontext_t MUST overlap.  (11.0 would copy out the FXSAVE area
    646  *	to both locations so that unmodified signal handlers could
    647  *	_read_ the x87/xmm registers, but _writeback_ would be
    648  *	ignored.)
    649  *
    650  *	For more background, see:
    651  *
    652  *	PR kern/60539: XSAVE changes break ucontext userspace API
    653  *	https://gnats.NetBSD.org/60539
    654  */
    655 void
    656 sendsig_siginfo(const ksiginfo_t *ksi, const sigset_t *mask)
    657 {
    658 	struct lwp *l = curlwp;
    659 	struct proc *p = l->l_proc;
    660 	struct sigacts *ps = p->p_sigacts;
    661 	int onstack, error;
    662 	int sig = ksi->ksi_signo;
    663 	struct sigframe_siginfo *fp, frame;
    664 	sig_t catcher = SIGACTION(p, sig).sa_handler;
    665 	struct trapframe *tf = l->l_md.md_regs;
    666 	const struct xsave_header *xsavebuf = NULL;
    667 	size_t xsavelen = 0;
    668 	struct xsave_header *user_xsave = NULL;
    669 	char *sp;
    670 
    671 	KASSERT(mutex_owned(p->p_lock));
    672 
    673 	/* Do we need to jump onto the signal stack? */
    674 	onstack =
    675 	    (l->l_sigstk.ss_flags & (SS_DISABLE | SS_ONSTACK)) == 0 &&
    676 	    (SIGACTION(p, sig).sa_flags & SA_ONSTACK) != 0;
    677 
    678 	/* Allocate space for the signal handler context. */
    679 	if (onstack) {
    680 		KASSERT(l->l_sigstk.ss_size >= MINSIGSTKSZ);
    681 		sp = ((char *)l->l_sigstk.ss_sp + l->l_sigstk.ss_size);
    682 	} else {
    683 		/* AMD64 ABI 128-bytes "red zone". */
    684 		sp = (char *)tf->tf_rsp - 128;
    685 	}
    686 
    687 	/*
    688 	 * The maximum amount of space we might use, including padding
    689 	 * for alignment, had better fit in MINSIGSTKSZ.
    690 	 *
    691 	 * If this changes because you have increased XSAVE_MAX_BYTES,
    692 	 * you need to work out the ABI change for MINSIGSTKSZ.
    693 	 */
    694 	__CTASSERT(8 + STACK_ALIGNBYTES + sizeof(struct sigframe_siginfo) +
    695 	    (XSAVE_ALIGN - 1) + XSAVE_MAX_BYTES <= MINSIGSTKSZ);
    696 
    697 	/*
    698 	 * Find whether we need to allocate a separate XSAVE area,
    699 	 * because the user program has used extended CPU state beyond
    700 	 * the x87/SSE registers, or whether we can get by with just an
    701 	 * FXSAVE area.
    702 	 */
    703 	if (process_xsave_needed_p(l)) {
    704 		enum {
    705 			overlap = (sizeof(struct sigframe_siginfo) -
    706 			    offsetof(struct sigframe_siginfo,
    707 				sf_uc.uc_mcontext.__fpregs)),
    708 		};
    709 
    710 		process_read_xsave(l, &xsavebuf, &xsavelen);
    711 		KASSERT(xsavebuf != NULL);
    712 		KASSERT(xsavelen <= XSAVE_MAX_BYTES);
    713 		CTASSERT(overlap <= XSAVE_MAX_BYTES);
    714 
    715 		KASSERT(!onstack || sp >= (char *)l->l_sigstk.ss_sp);
    716 		KASSERT(!onstack ||
    717 		    sp - (char *)l->l_sigstk.ss_sp >= xsavelen);
    718 		KASSERT(!onstack ||
    719 		    sp - (char *)l->l_sigstk.ss_sp >= overlap);
    720 		sp -= MAX(xsavelen, overlap);
    721 
    722 		KASSERT(!onstack || sp >= (char *)l->l_sigstk.ss_sp);
    723 		KASSERT(!onstack ||
    724 		    sp - (char *)l->l_sigstk.ss_sp >= XSAVE_ALIGN - 1);
    725 		sp = (char *)((uintptr_t)sp & ~(XSAVE_ALIGN - 1));
    726 
    727 		KASSERT(!onstack || sp >= (char *)l->l_sigstk.ss_sp);
    728 		KASSERT(((uintptr_t)sp & (XSAVE_ALIGN - 1)) == 0);
    729 		user_xsave = (void *)sp;
    730 
    731 		CTASSERT((overlap % XSAVE_ALIGN) == 0);
    732 		CTASSERT((overlap & STACK_ALIGNBYTES) == 0);
    733 		CTASSERT(((overlap - sizeof(struct sigframe_siginfo)) &
    734 			STACK_ALIGNBYTES) == (STACK_ALIGNBYTES + 1) - 8);
    735 		sp += overlap;
    736 	}
    737 
    738 	/*
    739 	 * Reserve space for a struct sigframe_siginfo.  The first
    740 	 * member is a return address, as if we are entering a
    741 	 * procedure with a CALL instruction with a stack frame aligned
    742 	 * to a multiple of 16 bytes; the rest of the structure is that
    743 	 * stack frame.  So make sure the structure starts at an
    744 	 * address congruent to 8 modulo 16.
    745 	 */
    746 	KASSERT(!onstack || sp >= (char *)l->l_sigstk.ss_sp);
    747 	KASSERT(!onstack || (size_t)(sp - (char *)l->l_sigstk.ss_sp) >=
    748 	    8 + STACK_ALIGNBYTES + sizeof(struct sigframe_siginfo));
    749 	sp -= sizeof(struct sigframe_siginfo);
    750 	/* Round down the stackpointer to a multiple of 16 for the ABI. */
    751 	if (((uintptr_t)sp & STACK_ALIGNBYTES) ==
    752 	    (STACK_ALIGNBYTES + 1) - 8) {
    753 		fp = (struct sigframe_siginfo *)sp;
    754 	} else {
    755 		fp = (struct sigframe_siginfo *)(((uintptr_t)sp &
    756 			~STACK_ALIGNBYTES) - 8);
    757 	}
    758 	KASSERT(!onstack || (char *)fp >= (char *)l->l_sigstk.ss_sp);
    759 	KASSERT(((uintptr_t)fp & STACK_ALIGNBYTES) ==
    760 	    (STACK_ALIGNBYTES + 1) - 8);
    761 
    762 	/*
    763 	 * If we have to use XSAVE, the FXSAVE area of the ucontext_t
    764 	 * on the user's stack must line up with the FXSAVE subarea of
    765 	 * the XSAVE area on the user's stack.
    766 	 */
    767 	KASSERT(xsavebuf == 0 ||
    768 	    (((uintptr_t)&fp->sf_uc.uc_mcontext.__fpregs & (XSAVE_ALIGN - 1))
    769 		== 0));
    770 	KASSERT(xsavebuf == 0 || (uintptr_t)user_xsave ==
    771 	    (uintptr_t)&fp->sf_uc.uc_mcontext.__fpregs);
    772 
    773 	memset(&frame, 0, sizeof(frame));
    774 	frame.sf_ra = (uint64_t)ps->sa_sigdesc[sig].sd_tramp;
    775 	frame.sf_si._info = ksi->ksi_info;
    776 	frame.sf_uc.uc_flags = _UC_SIGMASK;
    777 	frame.sf_uc.uc_sigmask = *mask;
    778 	frame.sf_uc.uc_link = l->l_ctxlink;
    779 	frame.sf_uc.uc_flags |= (l->l_sigstk.ss_flags & SS_ONSTACK)
    780 	    ? _UC_SETSTACK : _UC_CLRSTACK;
    781 	sendsig_reset(l, sig);
    782 
    783 	mutex_exit(p->p_lock);
    784 	cpu_getmcontext(l, &frame.sf_uc.uc_mcontext, &frame.sf_uc.uc_flags);
    785 
    786 	/*
    787 	 * If we have to use XSAVE, copy out the part of it past the
    788 	 * FXSAVE area separately -- and be ready to bail if it failed.
    789 	 */
    790 	if (xsavebuf) {
    791 		KASSERT((void *)&fp->sf_uc.uc_mcontext.__fpregs ==
    792 		    (void *)user_xsave);
    793 		error = cpu_getmcontext_xsave(l, &frame.sf_uc.uc_mcontext,
    794 		    &frame.sf_uc.uc_flags, xsavebuf, xsavelen, user_xsave);
    795 		if (error != 0)
    796 			goto relock;
    797 	}
    798 
    799 	/* Copyout all the fp regs, the signal handler might expect them. */
    800 	error = copyout(&frame, fp, sizeof frame);
    801 relock:
    802 	mutex_enter(p->p_lock);
    803 
    804 	if (error != 0) {
    805 		/*
    806 		 * Process has trashed its stack; give it an illegal
    807 		 * instruction to halt it in its tracks.
    808 		 */
    809 		sigexit(l, SIGILL);
    810 		/* NOTREACHED */
    811 	}
    812 
    813 	buildcontext(l, catcher, fp);
    814 
    815 	tf->tf_rdi = sig;
    816 	tf->tf_rsi = (uint64_t)&fp->sf_si;
    817 	tf->tf_rdx = tf->tf_r15 = (uint64_t)&fp->sf_uc;
    818 
    819 	/* Remember that we're now on the signal stack. */
    820 	if (onstack)
    821 		l->l_sigstk.ss_flags |= SS_ONSTACK;
    822 
    823 	if ((vaddr_t)catcher >= VM_MAXUSER_ADDRESS) {
    824 		/*
    825 		 * process has given an invalid address for the
    826 		 * handler. Stop it, but do not do it before so
    827 		 * we can return the right info to userland (or in core dump)
    828 		 */
    829 		sigexit(l, SIGILL);
    830 		/* NOTREACHED */
    831 	}
    832 }
    833 
    834 struct pcb dumppcb;
    835 
    836 void
    837 cpu_reboot(int howto, char *bootstr)
    838 {
    839 	static bool syncdone = false;
    840 	int s = IPL_NONE;
    841 	__USE(s);	/* ugly otherwise */
    842 
    843 	if (cold) {
    844 		howto |= RB_HALT;
    845 		goto haltsys;
    846 	}
    847 
    848 	boothowto = howto;
    849 
    850 	/* i386 maybe_dump() */
    851 
    852 	/*
    853 	 * If we've panic'd, don't make the situation potentially
    854 	 * worse by syncing or unmounting the file systems.
    855 	 */
    856 	if ((howto & RB_NOSYNC) == 0 && panicstr == NULL) {
    857 		if (!syncdone) {
    858 			syncdone = true;
    859 			/* XXX used to force unmount as well, here */
    860 			vfs_sync_all(curlwp);
    861 		}
    862 
    863 		while (vfs_unmountall1(curlwp, false, false) ||
    864 		       config_detach_all(boothowto) ||
    865 		       vfs_unmount_forceone(curlwp))
    866 			;	/* do nothing */
    867 	} else {
    868 		if (!db_active)
    869 			suspendsched();
    870 	}
    871 
    872 	pmf_system_shutdown(boothowto);
    873 
    874 	/* Disable interrupts. */
    875 	s = splhigh();
    876 
    877 	/* Do a dump if requested. */
    878 	if ((howto & (RB_DUMP | RB_HALT)) == RB_DUMP)
    879 		dumpsys();
    880 
    881 haltsys:
    882 	doshutdownhooks();
    883 
    884         if ((howto & RB_POWERDOWN) == RB_POWERDOWN) {
    885 #if NACPICA > 0
    886 		if (s != IPL_NONE)
    887 			splx(s);
    888 
    889 		acpi_enter_sleep_state(ACPI_STATE_S5);
    890 #endif
    891 #ifdef XEN
    892 		if (vm_guest == VM_GUEST_XENPV ||
    893 		    vm_guest == VM_GUEST_XENPVH ||
    894 		    vm_guest == VM_GUEST_XENPVHVM)
    895 			HYPERVISOR_shutdown();
    896 #endif /* XEN */
    897 	}
    898 
    899 	cpu_broadcast_halt();
    900 
    901 	if (howto & RB_HALT) {
    902 #if NACPICA > 0
    903 		acpi_disable();
    904 #endif
    905 
    906 		printf("\n");
    907 		printf("The operating system has halted.\n");
    908 		printf("Please press any key to reboot.\n\n");
    909 		cnpollc(true);	/* for proper keyboard command handling */
    910 		if (cngetc() == 0) {
    911 			/* no console attached, so just hlt */
    912 			printf("No keyboard - cannot reboot after all.\n");
    913 			for(;;) {
    914 				x86_hlt();
    915 			}
    916 		}
    917 		cnpollc(false);
    918 	}
    919 
    920 	printf("rebooting...\n");
    921 	if (cpureset_delay > 0)
    922 		delay(cpureset_delay * 1000);
    923 	cpu_reset();
    924 	for(;;) ;
    925 	/*NOTREACHED*/
    926 }
    927 
    928 /*
    929  * XXXfvdl share dumpcode.
    930  */
    931 
    932 /*
    933  * Perform assorted dump-related initialization tasks.  Assumes that
    934  * the maximum physical memory address will not increase afterwards.
    935  */
    936 void
    937 dump_misc_init(void)
    938 {
    939 #ifndef NO_SPARSE_DUMP
    940 	int i;
    941 #endif
    942 
    943 	if (dump_headerbuf != NULL)
    944 		return; /* already called */
    945 
    946 #ifndef NO_SPARSE_DUMP
    947 	for (i = 0; i < mem_cluster_cnt; ++i) {
    948 		paddr_t top = mem_clusters[i].start + mem_clusters[i].size;
    949 		if (max_paddr < top)
    950 			max_paddr = top;
    951 	}
    952 #ifdef DEBUG
    953 	printf("dump_misc_init: max_paddr = 0x%lx\n",
    954 	    (unsigned long)max_paddr);
    955 #endif
    956 	if (max_paddr == 0) {
    957 		printf("Your machine does not initialize mem_clusters; "
    958 		    "sparse_dumps disabled\n");
    959 		sparse_dump = 0;
    960 	} else {
    961 		sparse_dump_physmap = (void *)uvm_km_alloc(kernel_map,
    962 		    roundup(max_paddr / (PAGE_SIZE * NBBY), PAGE_SIZE),
    963 		    PAGE_SIZE, UVM_KMF_WIRED|UVM_KMF_ZERO);
    964 	}
    965 #endif
    966 	dump_headerbuf = (void *)uvm_km_alloc(kernel_map,
    967 	    dump_headerbuf_size,
    968 	    PAGE_SIZE, UVM_KMF_WIRED|UVM_KMF_ZERO);
    969 	/* XXXjld should check for failure here, disable dumps if so. */
    970 }
    971 
    972 #ifndef NO_SPARSE_DUMP
    973 /*
    974  * Clear the set of pages to include in a sparse dump.
    975  */
    976 void
    977 sparse_dump_reset(void)
    978 {
    979 	memset(sparse_dump_physmap, 0,
    980 	    roundup(max_paddr / (PAGE_SIZE * NBBY), PAGE_SIZE));
    981 }
    982 
    983 /*
    984  * Include or exclude pages in a sparse dump.
    985  */
    986 void
    987 sparse_dump_mark(void)
    988 {
    989 	paddr_t p, pstart, pend;
    990 	struct vm_page *pg;
    991 	int i;
    992 	uvm_physseg_t upm;
    993 
    994 	/*
    995 	 * Mark all memory pages, then unmark pages that are uninteresting.
    996 	 * Dereferenceing pg->uobject might crash again if another CPU
    997 	 * frees the object out from under us, but we can't lock anything
    998 	 * so it's a risk we have to take.
    999 	 */
   1000 
   1001 	for (i = 0; i < mem_cluster_cnt; ++i) {
   1002 		pstart = mem_clusters[i].start / PAGE_SIZE;
   1003 		pend = pstart + mem_clusters[i].size / PAGE_SIZE;
   1004 
   1005 		for (p = pstart; p < pend; p++) {
   1006 			setbit(sparse_dump_physmap, p);
   1007 		}
   1008 	}
   1009         for (upm = uvm_physseg_get_first();
   1010 	     uvm_physseg_valid_p(upm);
   1011 	     upm = uvm_physseg_get_next(upm)) {
   1012 		paddr_t pfn;
   1013 
   1014 		/*
   1015 		 * We assume that seg->start to seg->end are
   1016 		 * uvm_page_physload()ed
   1017 		 */
   1018 		for (pfn = uvm_physseg_get_start(upm);
   1019 		     pfn < uvm_physseg_get_end(upm);
   1020 		     pfn++) {
   1021 			pg = PHYS_TO_VM_PAGE(ptoa(pfn));
   1022 
   1023 			if (pg->uanon || (pg->flags & PG_FREE) ||
   1024 			    (pg->uobject && pg->uobject->pgops)) {
   1025 				p = VM_PAGE_TO_PHYS(pg) / PAGE_SIZE;
   1026 				clrbit(sparse_dump_physmap, p);
   1027 			}
   1028 		}
   1029 	}
   1030 }
   1031 
   1032 /*
   1033  * Machine-dependently decides on the contents of a sparse dump, using
   1034  * the above.
   1035  */
   1036 void
   1037 cpu_dump_prep_sparse(void)
   1038 {
   1039 	sparse_dump_reset();
   1040 	/* XXX could the alternate recursive page table be skipped? */
   1041 	sparse_dump_mark();
   1042 	/* Memory for I/O buffers could be unmarked here, for example. */
   1043 	/* The kernel text could also be unmarked, but gdb would be upset. */
   1044 }
   1045 #endif
   1046 
   1047 /*
   1048  * Abstractly iterate over the collection of memory segments to be
   1049  * dumped; the callback lacks the customary environment-pointer
   1050  * argument because none of the current users really need one.
   1051  *
   1052  * To be used only after dump_seg_prep is called to set things up.
   1053  */
   1054 int
   1055 dump_seg_iter(int (*callback)(paddr_t, paddr_t))
   1056 {
   1057 	int error, i;
   1058 
   1059 #define CALLBACK(start,size) do {     \
   1060 	error = callback(start,size); \
   1061 	if (error)                    \
   1062 		return error;         \
   1063 } while(0)
   1064 
   1065 	for (i = 0; i < mem_cluster_cnt; ++i) {
   1066 #ifndef NO_SPARSE_DUMP
   1067 		/*
   1068 		 * The bitmap is scanned within each memory segment,
   1069 		 * rather than over its entire domain, in case any
   1070 		 * pages outside of the memory proper have been mapped
   1071 		 * into kva; they might be devices that wouldn't
   1072 		 * appreciate being arbitrarily read, and including
   1073 		 * them could also break the assumption that a sparse
   1074 		 * dump will always be smaller than a full one.
   1075 		 */
   1076 		if (sparse_dump && sparse_dump_physmap) {
   1077 			paddr_t p, sp_start, sp_end;
   1078 			int lastset;
   1079 
   1080 			sp_start = mem_clusters[i].start;
   1081 			sp_end = sp_start + mem_clusters[i].size;
   1082 			sp_start = rounddown(sp_start, PAGE_SIZE); /* unnecessary? */
   1083 			lastset = 0;
   1084 			for (p = sp_start; p < sp_end; p += PAGE_SIZE) {
   1085 				int thisset = isset(sparse_dump_physmap,
   1086 				    p/PAGE_SIZE);
   1087 
   1088 				if (!lastset && thisset)
   1089 					sp_start = p;
   1090 				if (lastset && !thisset)
   1091 					CALLBACK(sp_start, p - sp_start);
   1092 				lastset = thisset;
   1093 			}
   1094 			if (lastset)
   1095 				CALLBACK(sp_start, p - sp_start);
   1096 		} else
   1097 #endif
   1098 			CALLBACK(mem_clusters[i].start, mem_clusters[i].size);
   1099 	}
   1100 	return 0;
   1101 #undef CALLBACK
   1102 }
   1103 
   1104 /*
   1105  * Prepare for an impending core dump: decide what's being dumped and
   1106  * how much space it will take up.
   1107  */
   1108 void
   1109 dump_seg_prep(void)
   1110 {
   1111 #ifndef NO_SPARSE_DUMP
   1112 	if (sparse_dump && sparse_dump_physmap)
   1113 		cpu_dump_prep_sparse();
   1114 #endif
   1115 
   1116 	dump_nmemsegs = 0;
   1117 	dump_npages = 0;
   1118 	dump_seg_iter(dump_seg_count_range);
   1119 
   1120 	dump_header_size = ALIGN(sizeof(kcore_seg_t)) +
   1121 	    ALIGN(sizeof(cpu_kcore_hdr_t)) +
   1122 	    ALIGN(dump_nmemsegs * sizeof(phys_ram_seg_t));
   1123 	dump_header_size = roundup(dump_header_size, dbtob(1));
   1124 
   1125 	/*
   1126 	 * savecore(8) will read this to decide how many pages to
   1127 	 * copy, and cpu_dumpconf has already used the pessimistic
   1128 	 * value to set dumplo, so it's time to tell the truth.
   1129 	 */
   1130 	dumpsize = dump_npages; /* XXX could these just be one variable? */
   1131 }
   1132 
   1133 int
   1134 dump_seg_count_range(paddr_t start, paddr_t size)
   1135 {
   1136 	++dump_nmemsegs;
   1137 	dump_npages += size / PAGE_SIZE;
   1138 	return 0;
   1139 }
   1140 
   1141 /*
   1142  * A sparse dump's header may be rather large, due to the number of
   1143  * "segments" emitted.  These routines manage a simple output buffer,
   1144  * so that the header can be written to disk incrementally.
   1145  */
   1146 void
   1147 dump_header_start(void)
   1148 {
   1149 	dump_headerbuf_ptr = dump_headerbuf;
   1150 	dump_header_blkno = dumplo;
   1151 }
   1152 
   1153 int
   1154 dump_header_flush(void)
   1155 {
   1156 	const struct bdevsw *bdev;
   1157 	size_t to_write;
   1158 	int error;
   1159 
   1160 	bdev = bdevsw_lookup(dumpdev);
   1161 	to_write = roundup(dump_headerbuf_ptr - dump_headerbuf, dbtob(1));
   1162 	error = bdev->d_dump(dumpdev, dump_header_blkno,
   1163 	    dump_headerbuf, to_write);
   1164 	dump_header_blkno += btodb(to_write);
   1165 	dump_headerbuf_ptr = dump_headerbuf;
   1166 	return error;
   1167 }
   1168 
   1169 int
   1170 dump_header_addbytes(const void* vptr, size_t n)
   1171 {
   1172 	const char* ptr = vptr;
   1173 	int error;
   1174 
   1175 	while (n > dump_headerbuf_avail) {
   1176 		memcpy(dump_headerbuf_ptr, ptr, dump_headerbuf_avail);
   1177 		ptr += dump_headerbuf_avail;
   1178 		n -= dump_headerbuf_avail;
   1179 		dump_headerbuf_ptr = dump_headerbuf_end;
   1180 		error = dump_header_flush();
   1181 		if (error)
   1182 			return error;
   1183 	}
   1184 	memcpy(dump_headerbuf_ptr, ptr, n);
   1185 	dump_headerbuf_ptr += n;
   1186 
   1187 	return 0;
   1188 }
   1189 
   1190 int
   1191 dump_header_addseg(paddr_t start, paddr_t size)
   1192 {
   1193 	phys_ram_seg_t seg = { start, size };
   1194 	int error;
   1195 
   1196 	error = dump_header_addbytes(&seg, sizeof(seg));
   1197 	if (error) {
   1198 		printf("[seg 0x%"PRIxPADDR" bytes 0x%"PRIxPSIZE" failed,"
   1199 		    " error=%d] ", start, size, error);
   1200 	}
   1201 	return error;
   1202 }
   1203 
   1204 int
   1205 dump_header_finish(void)
   1206 {
   1207 	int error;
   1208 
   1209 	memset(dump_headerbuf_ptr, 0, dump_headerbuf_avail);
   1210 	error = dump_header_flush();
   1211 	if (error)
   1212 		printf("[finish failed, error=%d] ", error);
   1213 	return error;
   1214 }
   1215 
   1216 
   1217 /*
   1218  * These variables are needed by /sbin/savecore
   1219  */
   1220 uint32_t	dumpmag = 0x8fca0101;	/* magic number */
   1221 int 	dumpsize = 0;		/* pages */
   1222 long	dumplo = 0; 		/* blocks */
   1223 
   1224 /*
   1225  * cpu_dumpsize: calculate size of machine-dependent kernel core dump headers
   1226  * for a full (non-sparse) dump.
   1227  */
   1228 int
   1229 cpu_dumpsize(void)
   1230 {
   1231 	int size;
   1232 
   1233 	size = ALIGN(sizeof(kcore_seg_t)) + ALIGN(sizeof(cpu_kcore_hdr_t)) +
   1234 	    ALIGN(mem_cluster_cnt * sizeof(phys_ram_seg_t));
   1235 	if (roundup(size, dbtob(1)) != dbtob(1))
   1236 		return (-1);
   1237 
   1238 	return (1);
   1239 }
   1240 
   1241 /*
   1242  * cpu_dump_mempagecnt: calculate the size of RAM (in pages) to be dumped
   1243  * for a full (non-sparse) dump.
   1244  */
   1245 u_long
   1246 cpu_dump_mempagecnt(void)
   1247 {
   1248 	u_long i, n;
   1249 
   1250 	n = 0;
   1251 	for (i = 0; i < mem_cluster_cnt; i++)
   1252 		n += atop(mem_clusters[i].size);
   1253 	return (n);
   1254 }
   1255 
   1256 /*
   1257  * cpu_dump: dump the machine-dependent kernel core dump headers.
   1258  */
   1259 int
   1260 cpu_dump(void)
   1261 {
   1262 	kcore_seg_t seg;
   1263 	cpu_kcore_hdr_t cpuhdr;
   1264 	const struct bdevsw *bdev;
   1265 	int error;
   1266 
   1267 	bdev = bdevsw_lookup(dumpdev);
   1268 	if (bdev == NULL) {
   1269 		printf("[device 0x%llx ENXIO] ", (unsigned long long)dumpdev);
   1270 		return ENXIO;
   1271 	}
   1272 
   1273 	/*
   1274 	 * Generate a segment header.
   1275 	 */
   1276 	CORE_SETMAGIC(seg, KCORE_MAGIC, MID_MACHINE, CORE_CPU);
   1277 	seg.c_size = dump_header_size - ALIGN(sizeof(seg));
   1278 	error = dump_header_addbytes(&seg, ALIGN(sizeof(seg)));
   1279 	if (error) {
   1280 		printf("[segment header %zu bytes failed, error=%d] ",
   1281 		    ALIGN(sizeof(seg)), error);
   1282 		/* blithely proceed (can't fail?) */
   1283 	}
   1284 
   1285 	/*
   1286 	 * Add the machine-dependent header info.
   1287 	 */
   1288 	cpuhdr.ptdpaddr = PDPpaddr;
   1289 	cpuhdr.nmemsegs = dump_nmemsegs;
   1290 	error = dump_header_addbytes(&cpuhdr, ALIGN(sizeof(cpuhdr)));
   1291 	if (error) {
   1292 		printf("[MD header %zu bytes failed, error=%d] ",
   1293 		    ALIGN(sizeof(cpuhdr)), error);
   1294 		/* blithely proceed (can't fail?) */
   1295 	}
   1296 
   1297 	/*
   1298 	 * Write out the memory segment descriptors.
   1299 	 */
   1300 	return dump_seg_iter(dump_header_addseg);
   1301 }
   1302 
   1303 /*
   1304  * Doadump comes here after turning off memory management and
   1305  * getting on the dump stack, either when called above, or by
   1306  * the auto-restart code.
   1307  */
   1308 #define BYTES_PER_DUMP  PAGE_SIZE /* must be a multiple of pagesize XXX small */
   1309 static vaddr_t dumpspace;
   1310 
   1311 vaddr_t
   1312 reserve_dumppages(vaddr_t p)
   1313 {
   1314 
   1315 	dumpspace = p;
   1316 	return (p + BYTES_PER_DUMP);
   1317 }
   1318 
   1319 int
   1320 dumpsys_seg(paddr_t maddr, paddr_t bytes)
   1321 {
   1322 	u_long i, m, n;
   1323 	daddr_t blkno;
   1324 	const struct bdevsw *bdev;
   1325 	int (*dump)(dev_t, daddr_t, void *, size_t);
   1326 	int error;
   1327 
   1328 	if (dumpdev == NODEV)
   1329 		return ENODEV;
   1330 	bdev = bdevsw_lookup(dumpdev);
   1331 	if (bdev == NULL || bdev->d_psize == NULL)
   1332 		return ENODEV;
   1333 
   1334 	dump = bdev->d_dump;
   1335 
   1336 	blkno = dump_header_blkno;
   1337 	for (i = 0; i < bytes; i += n, dump_totalbytesleft -= n) {
   1338 		/* Print out how many MBs we have left to go. */
   1339 		if ((dump_totalbytesleft % (1024*1024)) == 0)
   1340 			printf_nolog("%lu ", (unsigned long)
   1341 			    (dump_totalbytesleft / (1024 * 1024)));
   1342 
   1343 		/* Limit size for next transfer. */
   1344 		n = bytes - i;
   1345 		if (n > BYTES_PER_DUMP)
   1346 			n = BYTES_PER_DUMP;
   1347 
   1348 		for (m = 0; m < n; m += NBPG)
   1349 			pmap_kenter_pa(dumpspace + m, maddr + m,
   1350 			    VM_PROT_READ, 0);
   1351 		pmap_update(pmap_kernel());
   1352 
   1353 		error = (*dump)(dumpdev, blkno, (void *)dumpspace, n);
   1354 		pmap_kremove_local(dumpspace, n);
   1355 		if (error)
   1356 			return error;
   1357 		maddr += n;
   1358 		blkno += btodb(n);		/* XXX? */
   1359 
   1360 #if 0	/* XXX this doesn't work.  grr. */
   1361 		/* operator aborting dump? */
   1362 		if (sget() != NULL)
   1363 			return EINTR;
   1364 #endif
   1365 	}
   1366 	dump_header_blkno = blkno;
   1367 
   1368 	return 0;
   1369 }
   1370 
   1371 void
   1372 dodumpsys(void)
   1373 {
   1374 	const struct bdevsw *bdev;
   1375 	int dumpend, psize;
   1376 	int error;
   1377 
   1378 	if (dumpdev == NODEV)
   1379 		return;
   1380 
   1381 	bdev = bdevsw_lookup(dumpdev);
   1382 	if (bdev == NULL || bdev->d_psize == NULL)
   1383 		return;
   1384 	/*
   1385 	 * For dumps during autoconfiguration,
   1386 	 * if dump device has already configured...
   1387 	 */
   1388 	if (dumpsize == 0)
   1389 		cpu_dumpconf();
   1390 
   1391 	printf("\ndumping to dev %llu,%llu (offset=%ld, size=%d):",
   1392 	    (unsigned long long)major(dumpdev),
   1393 	    (unsigned long long)minor(dumpdev), dumplo, dumpsize);
   1394 
   1395 	if (dumplo <= 0 || dumpsize <= 0) {
   1396 		printf(" not possible\n");
   1397 		return;
   1398 	}
   1399 
   1400 	psize = bdev_size(dumpdev);
   1401 	printf("\ndump ");
   1402 	if (psize == -1) {
   1403 		printf("area unavailable\n");
   1404 		return;
   1405 	}
   1406 
   1407 #if 0	/* XXX this doesn't work.  grr. */
   1408 	/* toss any characters present prior to dump */
   1409 	while (sget() != NULL); /*syscons and pccons differ */
   1410 #endif
   1411 
   1412 	dump_seg_prep();
   1413 	dumpend = dumplo + btodb(dump_header_size) + ctod(dump_npages);
   1414 	if (dumpend > psize) {
   1415 		printf("failed: insufficient space (%d < %d)\n",
   1416 		    psize, dumpend);
   1417 		goto failed;
   1418 	}
   1419 
   1420 	dump_header_start();
   1421 	if ((error = cpu_dump()) != 0)
   1422 		goto err;
   1423 	if ((error = dump_header_finish()) != 0)
   1424 		goto err;
   1425 
   1426 	if (dump_header_blkno != dumplo + btodb(dump_header_size)) {
   1427 		printf("BAD header size (%ld [written] != %ld [expected])\n",
   1428 		    (long)(dump_header_blkno - dumplo),
   1429 		    (long)btodb(dump_header_size));
   1430 		goto failed;
   1431 	}
   1432 
   1433 	dump_totalbytesleft = roundup(ptoa(dump_npages), BYTES_PER_DUMP);
   1434 	error = dump_seg_iter(dumpsys_seg);
   1435 
   1436 	if (error == 0 && dump_header_blkno != dumpend) {
   1437 		printf("BAD dump size (%ld [written] != %ld [expected])\n",
   1438 		    (long)(dumpend - dumplo),
   1439 		    (long)(dump_header_blkno - dumplo));
   1440 		goto failed;
   1441 	}
   1442 
   1443 err:
   1444 	switch (error) {
   1445 
   1446 	case ENXIO:
   1447 		printf("device bad\n");
   1448 		break;
   1449 
   1450 	case EFAULT:
   1451 		printf("device not ready\n");
   1452 		break;
   1453 
   1454 	case EINVAL:
   1455 		printf("area improper\n");
   1456 		break;
   1457 
   1458 	case EIO:
   1459 		printf("i/o error\n");
   1460 		break;
   1461 
   1462 	case EINTR:
   1463 		printf("aborted from console\n");
   1464 		break;
   1465 
   1466 	case 0:
   1467 		printf("succeeded\n");
   1468 		break;
   1469 
   1470 	default:
   1471 		printf("error %d\n", error);
   1472 		break;
   1473 	}
   1474 failed:
   1475 	printf("\n\n");
   1476 	delay(5000000);		/* 5 seconds */
   1477 }
   1478 
   1479 /*
   1480  * This is called by main to set dumplo and dumpsize.
   1481  * Dumps always skip the first PAGE_SIZE of disk space
   1482  * in case there might be a disk label stored there.
   1483  * If there is extra space, put dump at the end to
   1484  * reduce the chance that swapping trashes it.
   1485  *
   1486  * Sparse dumps can't placed as close to the end as possible, because
   1487  * savecore(8) has to know where to start reading in the dump device
   1488  * before it has access to any of the crashed system's state.
   1489  *
   1490  * Note also that a sparse dump will never be larger than a full one:
   1491  * in order to add a phys_ram_seg_t to the header, at least one page
   1492  * must be removed.
   1493  */
   1494 void
   1495 cpu_dumpconf(void)
   1496 {
   1497 	int nblks, dumpblks;	/* size of dump area */
   1498 
   1499 	if (dumpdev == NODEV)
   1500 		goto bad;
   1501 	nblks = bdev_size(dumpdev);
   1502 	if (nblks <= ctod(1))
   1503 		goto bad;
   1504 
   1505 	dumpblks = cpu_dumpsize();
   1506 	if (dumpblks < 0)
   1507 		goto bad;
   1508 
   1509 	/* dumpsize is in page units, and doesn't include headers. */
   1510 	dumpsize = cpu_dump_mempagecnt();
   1511 
   1512 	dumpblks += ctod(dumpsize);
   1513 
   1514 	/* If dump won't fit (incl. room for possible label), punt. */
   1515 	if (dumpblks > (nblks - ctod(1))) {
   1516 #ifndef NO_SPARSE_DUMP
   1517 		/* A sparse dump might (and hopefully will) fit. */
   1518 		dumplo = ctod(1);
   1519 #else
   1520 		/* But if we're not configured for that, punt. */
   1521 		goto bad;
   1522 #endif
   1523 	} else {
   1524 		/* Put dump at end of partition */
   1525 		dumplo = nblks - dumpblks;
   1526 	}
   1527 
   1528 
   1529 	/* Now that we've decided this will work, init ancillary stuff. */
   1530 	dump_misc_init();
   1531 	return;
   1532 
   1533  bad:
   1534 	dumpsize = 0;
   1535 }
   1536 
   1537 /*
   1538  * Clear registers on exec
   1539  */
   1540 void
   1541 setregs(struct lwp *l, struct exec_package *pack, vaddr_t stack)
   1542 {
   1543 	struct pcb *pcb = lwp_getpcb(l);
   1544 	struct trapframe *tf;
   1545 
   1546 #ifdef USER_LDT
   1547 	pmap_ldt_cleanup(l);
   1548 #endif
   1549 
   1550 	fpu_clear(l, pack->ep_osversion >= 699002600
   1551 	    ? __NetBSD_NPXCW__ : __NetBSD_COMPAT_NPXCW__);
   1552 	x86_dbregs_clear(l);
   1553 
   1554 	kpreempt_disable();
   1555 	pcb->pcb_flags = 0;
   1556 	l->l_proc->p_flag &= ~PK_32;
   1557 	l->l_md.md_flags = MDL_IRET;
   1558 	cpu_segregs64_zero(l);
   1559 	kpreempt_enable();
   1560 
   1561 	tf = l->l_md.md_regs;
   1562 	memset(tf, 0, sizeof(*tf));
   1563 
   1564 	tf->tf_trapno = T_ASTFLT;
   1565 	tf->tf_ds = GSEL(GUDATA_SEL, SEL_UPL);
   1566 	tf->tf_es = GSEL(GUDATA_SEL, SEL_UPL);
   1567 	tf->tf_rdi = 0;
   1568 	tf->tf_rsi = 0;
   1569 	tf->tf_rbp = 0;
   1570 	tf->tf_rbx = l->l_proc->p_psstrp;
   1571 	tf->tf_rdx = 0;
   1572 	tf->tf_rcx = 0;
   1573 	tf->tf_rax = 0;
   1574 	tf->tf_rip = pack->ep_entry;
   1575 	tf->tf_cs = LSEL(LUCODE_SEL, SEL_UPL);
   1576 	tf->tf_rflags = PSL_USERSET;
   1577 	tf->tf_rsp = stack;
   1578 	tf->tf_ss = LSEL(LUDATA_SEL, SEL_UPL);
   1579 }
   1580 
   1581 /*
   1582  * Initialize segments and descriptor tables
   1583  */
   1584 char *ldtstore;
   1585 char *gdtstore;
   1586 
   1587 void
   1588 setgate(struct gate_descriptor *gd, void *func,
   1589     int ist, int type, int dpl, int sel)
   1590 {
   1591 	vaddr_t vaddr;
   1592 
   1593 	vaddr = ((vaddr_t)gd) & ~PAGE_MASK;
   1594 
   1595 	kpreempt_disable();
   1596 	pmap_changeprot_local(vaddr, VM_PROT_READ|VM_PROT_WRITE);
   1597 
   1598 	gd->gd_looffset = (uint64_t)func & 0xffff;
   1599 	gd->gd_selector = sel;
   1600 	gd->gd_ist = ist;
   1601 	gd->gd_type = type;
   1602 	gd->gd_dpl = dpl;
   1603 	gd->gd_p = 1;
   1604 	gd->gd_hioffset = (uint64_t)func >> 16;
   1605 	gd->gd_zero = 0;
   1606 	gd->gd_xx1 = 0;
   1607 	gd->gd_xx2 = 0;
   1608 	gd->gd_xx3 = 0;
   1609 
   1610 	pmap_changeprot_local(vaddr, VM_PROT_READ);
   1611 	kpreempt_enable();
   1612 }
   1613 
   1614 void
   1615 unsetgate(struct gate_descriptor *gd)
   1616 {
   1617 	vaddr_t vaddr;
   1618 
   1619 	vaddr = ((vaddr_t)gd) & ~PAGE_MASK;
   1620 
   1621 	kpreempt_disable();
   1622 	pmap_changeprot_local(vaddr, VM_PROT_READ|VM_PROT_WRITE);
   1623 
   1624 	memset(gd, 0, sizeof (*gd));
   1625 
   1626 	pmap_changeprot_local(vaddr, VM_PROT_READ);
   1627 	kpreempt_enable();
   1628 }
   1629 
   1630 void
   1631 setregion(struct region_descriptor *rd, void *base, uint16_t limit)
   1632 {
   1633 	rd->rd_limit = limit;
   1634 	rd->rd_base = (uint64_t)base;
   1635 }
   1636 
   1637 /*
   1638  * Note that the base and limit fields are ignored in long mode.
   1639  */
   1640 void
   1641 set_mem_segment(struct mem_segment_descriptor *sd, void *base, size_t limit,
   1642 	int type, int dpl, int gran, int def32, int is64)
   1643 {
   1644 	sd->sd_lolimit = (unsigned)limit;
   1645 	sd->sd_lobase = (unsigned long)base;
   1646 	sd->sd_type = type;
   1647 	sd->sd_dpl = dpl;
   1648 	sd->sd_p = 1;
   1649 	sd->sd_hilimit = (unsigned)limit >> 16;
   1650 	sd->sd_avl = 0;
   1651 	sd->sd_long = is64;
   1652 	sd->sd_def32 = def32;
   1653 	sd->sd_gran = gran;
   1654 	sd->sd_hibase = (unsigned long)base >> 24;
   1655 }
   1656 
   1657 void
   1658 set_sys_segment(struct sys_segment_descriptor *sd, void *base, size_t limit,
   1659 	int type, int dpl, int gran)
   1660 {
   1661 	memset(sd, 0, sizeof *sd);
   1662 	sd->sd_lolimit = (unsigned)limit;
   1663 	sd->sd_lobase = (uint64_t)base;
   1664 	sd->sd_type = type;
   1665 	sd->sd_dpl = dpl;
   1666 	sd->sd_p = 1;
   1667 	sd->sd_hilimit = (unsigned)limit >> 16;
   1668 	sd->sd_gran = gran;
   1669 	sd->sd_hibase = (uint64_t)base >> 24;
   1670 }
   1671 
   1672 void
   1673 cpu_init_idt(struct cpu_info *ci)
   1674 {
   1675 	struct region_descriptor region;
   1676 	idt_descriptor_t *idt;
   1677 
   1678 	idt = ci->ci_idtvec.iv_idt;
   1679 	setregion(&region, idt, NIDT * sizeof(idt[0]) - 1);
   1680 	lidt(&region);
   1681 }
   1682 
   1683 #define	IDTVEC(name)	__CONCAT(X, name)
   1684 typedef void (vector)(void);
   1685 extern vector IDTVEC(syscall);
   1686 extern vector IDTVEC(syscall32);
   1687 extern vector IDTVEC(osyscall);
   1688 extern vector *x86_exceptions[];
   1689 
   1690 #ifndef XENPV
   1691 static void
   1692 init_x86_64_ksyms(void)
   1693 {
   1694 #if NKSYMS || defined(DDB) || defined(MODULAR)
   1695 	extern int end;
   1696 	extern int *esym;
   1697 	struct btinfo_symtab *symtab;
   1698 	vaddr_t tssym, tesym;
   1699 
   1700 #ifdef DDB
   1701 	db_machine_init();
   1702 #endif
   1703 
   1704 	symtab = lookup_bootinfo(BTINFO_SYMTAB);
   1705 	if (symtab) {
   1706 #ifdef KASLR
   1707 		tssym = bootspace.head.va;
   1708 		tesym = bootspace.head.va; /* (unused...) */
   1709 #else
   1710 		tssym = (vaddr_t)symtab->ssym + KERNBASE;
   1711 		tesym = (vaddr_t)symtab->esym + KERNBASE;
   1712 #endif
   1713 		ksyms_addsyms_elf(symtab->nsym, (void *)tssym, (void *)tesym);
   1714 	} else {
   1715 		uintptr_t endp = (uintptr_t)(void *)&end;
   1716 #ifdef XEN
   1717 		/*
   1718 		 * cpu_probe() / identify_hypervisor() overrides VM_GUEST_GENPVH,
   1719 		 * we can't rely on vm_guest == VM_GUEST_GENPVH
   1720 		 */
   1721 		if (pvh_boot && vm_guest != VM_GUEST_XENPVH)
   1722 			ksyms_addsyms_elf(0, ((long *)endp) + 1, esym);
   1723 		else
   1724 #endif
   1725 			ksyms_addsyms_elf(*(long *)endp, ((long *)endp) + 1, esym);
   1726 	}
   1727 #endif
   1728 }
   1729 #endif /* XENPV */
   1730 
   1731 void __noasan
   1732 init_bootspace(void)
   1733 {
   1734 	extern char __rodata_start;
   1735 	extern char __data_start;
   1736 	extern char __kernel_end;
   1737 	size_t i = 0;
   1738 
   1739 	memset(&bootspace, 0, sizeof(bootspace));
   1740 
   1741 	bootspace.head.va = KERNTEXTOFF;
   1742 	bootspace.head.pa = KERNTEXTOFF - KERNBASE;
   1743 	bootspace.head.sz = 0;
   1744 
   1745 	bootspace.segs[i].type = BTSEG_TEXT;
   1746 	bootspace.segs[i].va = KERNTEXTOFF;
   1747 	bootspace.segs[i].pa = KERNTEXTOFF - KERNBASE;
   1748 	bootspace.segs[i].sz = (size_t)&__rodata_start - KERNTEXTOFF;
   1749 	i++;
   1750 
   1751 	bootspace.segs[i].type = BTSEG_RODATA;
   1752 	bootspace.segs[i].va = (vaddr_t)&__rodata_start;
   1753 	bootspace.segs[i].pa = (paddr_t)&__rodata_start - KERNBASE;
   1754 	bootspace.segs[i].sz = (size_t)&__data_start - (size_t)&__rodata_start;
   1755 	i++;
   1756 
   1757 	bootspace.segs[i].type = BTSEG_DATA;
   1758 	bootspace.segs[i].va = (vaddr_t)&__data_start;
   1759 	bootspace.segs[i].pa = (paddr_t)&__data_start - KERNBASE;
   1760 	bootspace.segs[i].sz = (size_t)&__kernel_end - (size_t)&__data_start;
   1761 	i++;
   1762 
   1763 	bootspace.boot.va = (vaddr_t)&__kernel_end;
   1764 	bootspace.boot.pa = (paddr_t)&__kernel_end - KERNBASE;
   1765 	bootspace.boot.sz = (size_t)(atdevbase + IOM_SIZE) -
   1766 	    (size_t)&__kernel_end;
   1767 
   1768 	/* In locore.S, we allocated a tmp va. We will use it now. */
   1769 	bootspace.spareva = KERNBASE + NKL2_KIMG_ENTRIES * NBPD_L2;
   1770 
   1771 	/* Virtual address of the L4 page. */
   1772 	bootspace.pdir = (vaddr_t)(PDPpaddr + KERNBASE);
   1773 
   1774 	/* Kernel module map. */
   1775 	bootspace.smodule = (vaddr_t)atdevbase + IOM_SIZE;
   1776 	bootspace.emodule = KERNBASE + NKL2_KIMG_ENTRIES * NBPD_L2;
   1777 }
   1778 
   1779 static void
   1780 init_pte(void)
   1781 {
   1782 #ifndef XENPV
   1783 	extern uint32_t nox_flag;
   1784 	pd_entry_t *pdir = (pd_entry_t *)bootspace.pdir;
   1785 	pdir[L4_SLOT_PTE] = PDPpaddr | PTE_W | ((uint64_t)nox_flag << 32) |
   1786 	    PTE_P;
   1787 #endif
   1788 
   1789 	extern pd_entry_t *normal_pdes[3];
   1790 	normal_pdes[0] = L2_BASE;
   1791 	normal_pdes[1] = L3_BASE;
   1792 	normal_pdes[2] = L4_BASE;
   1793 }
   1794 
   1795 void
   1796 init_slotspace(void)
   1797 {
   1798 	/*
   1799 	 * XXX Too early to use cprng(9), or even entropy_extract.
   1800 	 */
   1801 	struct entpool pool;
   1802 	size_t randhole;
   1803 	vaddr_t randva;
   1804 	uint64_t sample;
   1805 	vaddr_t va;
   1806 
   1807 	memset(&pool, 0, sizeof pool);
   1808 	cpu_rng_early_sample(&sample);
   1809 	entpool_enter(&pool, &sample, sizeof sample);
   1810 
   1811 	memset(&slotspace, 0, sizeof(slotspace));
   1812 
   1813 	/* User. [256, because we want to land in >= 256] */
   1814 	slotspace.area[SLAREA_USER].sslot = 0;
   1815 	slotspace.area[SLAREA_USER].nslot = PDIR_SLOT_USERLIM+1;
   1816 	slotspace.area[SLAREA_USER].active = true;
   1817 
   1818 #ifdef XENPV
   1819 	/* PTE. */
   1820 	slotspace.area[SLAREA_PTE].sslot = PDIR_SLOT_PTE;
   1821 	slotspace.area[SLAREA_PTE].nslot = 1;
   1822 	slotspace.area[SLAREA_PTE].active = true;
   1823 #endif
   1824 
   1825 #ifdef __HAVE_PCPU_AREA
   1826 	/* Per-CPU. */
   1827 	slotspace.area[SLAREA_PCPU].sslot = PDIR_SLOT_PCPU;
   1828 	slotspace.area[SLAREA_PCPU].nslot = 1;
   1829 	slotspace.area[SLAREA_PCPU].active = true;
   1830 #endif
   1831 
   1832 #ifdef __HAVE_DIRECT_MAP
   1833 	/* Direct Map. [Randomized later] */
   1834 	slotspace.area[SLAREA_DMAP].active = false;
   1835 #endif
   1836 
   1837 #ifdef XENPV
   1838 	/* Hypervisor. */
   1839 	slotspace.area[SLAREA_HYPV].sslot = 256;
   1840 	slotspace.area[SLAREA_HYPV].nslot = 17;
   1841 	slotspace.area[SLAREA_HYPV].active = true;
   1842 #endif
   1843 
   1844 #ifdef KASAN
   1845 	/* ASAN. */
   1846 	slotspace.area[SLAREA_ASAN].sslot = L4_SLOT_KASAN;
   1847 	slotspace.area[SLAREA_ASAN].nslot = NL4_SLOT_KASAN;
   1848 	slotspace.area[SLAREA_ASAN].active = true;
   1849 #endif
   1850 
   1851 #ifdef KMSAN
   1852 	/* MSAN. */
   1853 	slotspace.area[SLAREA_MSAN].sslot = L4_SLOT_KMSAN;
   1854 	slotspace.area[SLAREA_MSAN].nslot = NL4_SLOT_KMSAN;
   1855 	slotspace.area[SLAREA_MSAN].active = true;
   1856 #endif
   1857 
   1858 	/* Kernel. */
   1859 	slotspace.area[SLAREA_KERN].sslot = L4_SLOT_KERNBASE;
   1860 	slotspace.area[SLAREA_KERN].nslot = 1;
   1861 	slotspace.area[SLAREA_KERN].active = true;
   1862 
   1863 	/* Main. */
   1864 	cpu_rng_early_sample(&sample);
   1865 	entpool_enter(&pool, &sample, sizeof sample);
   1866 	entpool_extract(&pool, &randhole, sizeof randhole);
   1867 	entpool_extract(&pool, &randva, sizeof randva);
   1868 	va = slotspace_rand(SLAREA_MAIN, NKL4_MAX_ENTRIES * NBPD_L4,
   1869 	    NBPD_L4, randhole, randva); /* TODO: NBPD_L1 */
   1870 	vm_min_kernel_address = va;
   1871 	vm_max_kernel_address = va + NKL4_MAX_ENTRIES * NBPD_L4;
   1872 
   1873 #ifndef XENPV
   1874 	/* PTE. */
   1875 	cpu_rng_early_sample(&sample);
   1876 	entpool_enter(&pool, &sample, sizeof sample);
   1877 	entpool_extract(&pool, &randhole, sizeof randhole);
   1878 	entpool_extract(&pool, &randva, sizeof randva);
   1879 	va = slotspace_rand(SLAREA_PTE, NBPD_L4, NBPD_L4, randhole, randva);
   1880 	pte_base = (pd_entry_t *)va;
   1881 #endif
   1882 
   1883 	explicit_memset(&pool, 0, sizeof pool);
   1884 }
   1885 
   1886 void
   1887 init_x86_64(paddr_t first_avail)
   1888 {
   1889 	extern void consinit(void);
   1890 	struct region_descriptor region;
   1891 	struct mem_segment_descriptor *ldt_segp;
   1892 	struct idt_vec *iv;
   1893 	idt_descriptor_t *idt;
   1894 	int x;
   1895 	struct pcb *pcb;
   1896 	extern vaddr_t lwp0uarea;
   1897 #ifndef XENPV
   1898 	extern paddr_t local_apic_pa;
   1899 #endif
   1900 
   1901 	KASSERT(first_avail % PAGE_SIZE == 0);
   1902 
   1903 #ifdef XENPV
   1904 	KASSERT(HYPERVISOR_shared_info != NULL);
   1905 	cpu_info_primary.ci_vcpu = &HYPERVISOR_shared_info->vcpu_info[0];
   1906 #endif
   1907 
   1908 #ifdef XEN
   1909 	if (pvh_boot)
   1910 		xen_parse_cmdline(XEN_PARSE_BOOTFLAGS, NULL);
   1911 #endif
   1912 	init_pte();
   1913 
   1914 	uvm_lwp_setuarea(&lwp0, lwp0uarea);
   1915 
   1916 	cpu_probe(&cpu_info_primary);
   1917 #ifdef SVS
   1918 	svs_init();
   1919 #endif
   1920 
   1921 	/*
   1922 	 * Initialize MSRs on cpu0:
   1923 	 *
   1924 	 * - Enables SYSCALL/SYSRET.
   1925 	 *
   1926 	 * - Sets up %fs and %gs so that %gs points to the current
   1927 	 *   struct cpu_info as needed for CPUVAR(...), curcpu(), and
   1928 	 *   curlwp.
   1929 	 *
   1930 	 * - Enables the no-execute bit if supported.
   1931 	 *
   1932 	 * Thus, after this point, CPUVAR(...), curcpu(), and curlwp
   1933 	 * will work on cpu0.
   1934 	 *
   1935 	 * Note: The call to cpu_init_msrs for secondary CPUs happens
   1936 	 * in cpu_hatch.
   1937 	 */
   1938 	cpu_init_msrs(&cpu_info_primary, true);
   1939 
   1940 #ifndef XENPV
   1941 	cpu_speculation_init(&cpu_info_primary);
   1942 #endif
   1943 
   1944 	use_pae = 1; /* PAE always enabled in long mode */
   1945 
   1946 	pcb = lwp_getpcb(&lwp0);
   1947 #ifdef XENPV
   1948 	mutex_init(&pte_lock, MUTEX_DEFAULT, IPL_VM);
   1949 	pcb->pcb_cr3 = xen_start_info.pt_base - KERNBASE;
   1950 #else
   1951 	pcb->pcb_cr3 = PDPpaddr;
   1952 #endif
   1953 
   1954 #if NISA > 0 || NPCI > 0
   1955 	x86_bus_space_init();
   1956 #endif
   1957 
   1958 	pat_init(&cpu_info_primary);
   1959 
   1960 	consinit();	/* XXX SHOULD NOT BE DONE HERE */
   1961 
   1962 	/*
   1963 	 * Initialize PAGE_SIZE-dependent variables.
   1964 	 */
   1965 	uvm_md_init();
   1966 
   1967 	uvmexp.ncolors = 2;
   1968 
   1969 	avail_start = first_avail;
   1970 
   1971 #ifndef XENPV
   1972 	/*
   1973 	 * Low memory reservations:
   1974 	 * Page 0:	BIOS data
   1975 	 * Page 1:	BIOS callback (not used yet, for symmetry with i386)
   1976 	 * Page 2:	MP bootstrap code (MP_TRAMPOLINE)
   1977 	 * Page 3:	ACPI wakeup code (ACPI_WAKEUP_ADDR)
   1978 	 * Page 4:	Temporary page table for 0MB-4MB
   1979 	 * Page 5:	Temporary page directory
   1980 	 * Page 6:	Temporary page map level 3
   1981 	 * Page 7:	Temporary page map level 4
   1982 	 */
   1983 	lowmem_rsvd = 8 * PAGE_SIZE;
   1984 
   1985 	/* Initialize the memory clusters (needed in pmap_bootstrap). */
   1986 	init_x86_clusters();
   1987 #else
   1988 	/* Parse Xen command line (replace bootinfo) */
   1989 	xen_parse_cmdline(XEN_PARSE_BOOTFLAGS, NULL);
   1990 
   1991 	avail_end = ctob(xen_start_info.nr_pages);
   1992 	pmap_pa_start = (KERNTEXTOFF - KERNBASE);
   1993 	pmap_pa_end = avail_end;
   1994 #endif
   1995 
   1996 	/*
   1997 	 * Call pmap initialization to make new kernel address space.
   1998 	 * We must do this before loading pages into the VM system.
   1999 	 */
   2000 	pmap_bootstrap(VM_MIN_KERNEL_ADDRESS);
   2001 
   2002 	/*
   2003 	 * Initialize RNG to get entropy ASAP either from CPU
   2004 	 * RDRAND/RDSEED or from seed on disk.  Constraints:
   2005 	 *
   2006 	 * - Must happen after cpu_init_msrs so that curcpu() and
   2007 	 *   curlwp work.
   2008 	 *
   2009 	 * - Must happen after consinit so we have the opportunity to
   2010 	 *   print useful feedback.
   2011 	 *
   2012 	 * - On KASLR kernels, must happen after pmap_bootstrap because
   2013 	 *   x86_rndseed requires access to the direct map.
   2014 	 */
   2015 	cpu_rng_init();
   2016 	x86_rndseed();
   2017 
   2018 #ifndef XENPV
   2019 	/* Internalize the physical pages into the VM system. */
   2020 	init_x86_vm(avail_start);
   2021 #else
   2022 	physmem = xen_start_info.nr_pages;
   2023 	uvm_page_physload(atop(avail_start), atop(avail_end),
   2024 	    atop(avail_start), atop(avail_end), VM_FREELIST_DEFAULT);
   2025 #endif
   2026 
   2027 	init_x86_msgbuf();
   2028 
   2029 	kasan_init();
   2030 	kcsan_init();
   2031 	kmsan_init((void *)lwp0uarea);
   2032 
   2033 	pmap_growkernel(VM_MIN_KERNEL_ADDRESS + 32 * 1024 * 1024);
   2034 
   2035 	kpreempt_disable();
   2036 
   2037 #ifndef XENPV
   2038 	pmap_kenter_pa(local_apic_va, local_apic_pa,
   2039 	    VM_PROT_READ|VM_PROT_WRITE, 0);
   2040 	pmap_update(pmap_kernel());
   2041 	memset((void *)local_apic_va, 0, PAGE_SIZE);
   2042 #endif
   2043 
   2044 	pmap_kenter_pa(idt_vaddr, idt_paddr, VM_PROT_READ|VM_PROT_WRITE, 0);
   2045 	pmap_kenter_pa(gdt_vaddr, gdt_paddr, VM_PROT_READ|VM_PROT_WRITE, 0);
   2046 	pmap_kenter_pa(ldt_vaddr, ldt_paddr, VM_PROT_READ|VM_PROT_WRITE, 0);
   2047 	pmap_update(pmap_kernel());
   2048 	memset((void *)idt_vaddr, 0, PAGE_SIZE);
   2049 	memset((void *)gdt_vaddr, 0, PAGE_SIZE);
   2050 	memset((void *)ldt_vaddr, 0, PAGE_SIZE);
   2051 
   2052 #ifndef XENPV
   2053 	pmap_changeprot_local(idt_vaddr, VM_PROT_READ);
   2054 #endif
   2055 
   2056 	pmap_update(pmap_kernel());
   2057 
   2058 	iv = &(cpu_info_primary.ci_idtvec);
   2059 	idt_vec_init_cpu_md(iv, cpu_index(&cpu_info_primary));
   2060 	idt = iv->iv_idt;
   2061 	gdtstore = (char *)gdt_vaddr;
   2062 	ldtstore = (char *)ldt_vaddr;
   2063 
   2064 	/*
   2065 	 * Make GDT gates and memory segments.
   2066 	 */
   2067 	set_mem_segment(GDT_ADDR_MEM(gdtstore, GCODE_SEL), 0,
   2068 	    0xfffff, SDT_MEMERA, SEL_KPL, 1, 0, 1);
   2069 
   2070 	set_mem_segment(GDT_ADDR_MEM(gdtstore, GDATA_SEL), 0,
   2071 	    0xfffff, SDT_MEMRWA, SEL_KPL, 1, 0, 1);
   2072 
   2073 	set_mem_segment(GDT_ADDR_MEM(gdtstore, GUCODE_SEL), 0,
   2074 	    x86_btop(VM_MAXUSER_ADDRESS) - 1, SDT_MEMERA, SEL_UPL, 1, 0, 1);
   2075 
   2076 	set_mem_segment(GDT_ADDR_MEM(gdtstore, GUDATA_SEL), 0,
   2077 	    x86_btop(VM_MAXUSER_ADDRESS) - 1, SDT_MEMRWA, SEL_UPL, 1, 0, 1);
   2078 
   2079 #ifndef XENPV
   2080 	set_sys_segment(GDT_ADDR_SYS(gdtstore, GLDT_SEL), ldtstore,
   2081 	    LDT_SIZE - 1, SDT_SYSLDT, SEL_KPL, 0);
   2082 #endif
   2083 
   2084 	/*
   2085 	 * Make LDT memory segments.
   2086 	 */
   2087 	*(struct mem_segment_descriptor *)(ldtstore + LUCODE_SEL) =
   2088 	    *GDT_ADDR_MEM(gdtstore, GUCODE_SEL);
   2089 	*(struct mem_segment_descriptor *)(ldtstore + LUDATA_SEL) =
   2090 	    *GDT_ADDR_MEM(gdtstore, GUDATA_SEL);
   2091 
   2092 	/*
   2093 	 * 32 bit GDT entries.
   2094 	 */
   2095 	set_mem_segment(GDT_ADDR_MEM(gdtstore, GUCODE32_SEL), 0,
   2096 	    x86_btop(VM_MAXUSER_ADDRESS32) - 1, SDT_MEMERA, SEL_UPL, 1, 1, 0);
   2097 
   2098 	set_mem_segment(GDT_ADDR_MEM(gdtstore, GUDATA32_SEL), 0,
   2099 	    x86_btop(VM_MAXUSER_ADDRESS32) - 1, SDT_MEMRWA, SEL_UPL, 1, 1, 0);
   2100 
   2101 	set_mem_segment(GDT_ADDR_MEM(gdtstore, GUFS_SEL), 0,
   2102 	    x86_btop(VM_MAXUSER_ADDRESS32) - 1, SDT_MEMRWA, SEL_UPL, 1, 1, 0);
   2103 
   2104 	set_mem_segment(GDT_ADDR_MEM(gdtstore, GUGS_SEL), 0,
   2105 	    x86_btop(VM_MAXUSER_ADDRESS32) - 1, SDT_MEMRWA, SEL_UPL, 1, 1, 0);
   2106 
   2107 	/*
   2108 	 * 32 bit LDT entries.
   2109 	 */
   2110 	ldt_segp = (struct mem_segment_descriptor *)(ldtstore + LUCODE32_SEL);
   2111 	set_mem_segment(ldt_segp, 0, x86_btop(VM_MAXUSER_ADDRESS32) - 1,
   2112 	    SDT_MEMERA, SEL_UPL, 1, 1, 0);
   2113 	ldt_segp = (struct mem_segment_descriptor *)(ldtstore + LUDATA32_SEL);
   2114 	set_mem_segment(ldt_segp, 0, x86_btop(VM_MAXUSER_ADDRESS32) - 1,
   2115 	    SDT_MEMRWA, SEL_UPL, 1, 1, 0);
   2116 
   2117 	/* CPU-specific IDT exceptions. */
   2118 	for (x = 0; x < NCPUIDT; x++) {
   2119 		int sel, ist;
   2120 
   2121 		/* Reset to default. Special cases below */
   2122 		sel = SEL_KPL;
   2123 		ist = 0;
   2124 
   2125 		idt_vec_reserve(iv, x);
   2126 
   2127 		switch (x) {
   2128 		case 1:	/* DB */
   2129 			ist = 4;
   2130 			break;
   2131 		case 2:	/* NMI */
   2132 			ist = 3;
   2133 			break;
   2134 		case 3:
   2135 		case 4:
   2136 			sel = SEL_UPL;
   2137 			break;
   2138 		case 8:	/* double fault */
   2139 			ist = 2;
   2140 			break;
   2141 #ifdef XENPV
   2142 		case 18: /* MCA */
   2143 			sel |= 0x4; /* Auto EOI/mask */
   2144 			break;
   2145 #endif /* XENPV */
   2146 		default:
   2147 			break;
   2148 		}
   2149 
   2150 		set_idtgate(&idt[x], x86_exceptions[x], ist, SDT_SYS386IGT,
   2151 		    sel, GSEL(GCODE_SEL, SEL_KPL));
   2152 	}
   2153 
   2154 	/* new-style interrupt gate for syscalls */
   2155 	idt_vec_reserve(iv, 128);
   2156 	set_idtgate(&idt[128], &IDTVEC(osyscall), 0, SDT_SYS386IGT, SEL_UPL,
   2157 	    GSEL(GCODE_SEL, SEL_KPL));
   2158 
   2159 	kpreempt_enable();
   2160 
   2161 	setregion(&region, gdtstore, DYNSEL_START - 1);
   2162 	lgdt(&region);
   2163 
   2164 #ifdef XENPV
   2165 	/* Init Xen callbacks and syscall handlers */
   2166 	if (HYPERVISOR_set_callbacks(
   2167 	    (unsigned long) hypervisor_callback,
   2168 	    (unsigned long) failsafe_callback,
   2169 	    (unsigned long) Xsyscall))
   2170 		panic("HYPERVISOR_set_callbacks() failed");
   2171 #endif /* XENPV */
   2172 
   2173 	cpu_init_idt(&cpu_info_primary);
   2174 
   2175 #ifdef XENPV
   2176 	xen_init_ksyms();
   2177 #else /* XENPV */
   2178 #ifdef XEN
   2179 	if (vm_guest == VM_GUEST_XENPVH)
   2180 		xen_init_ksyms();
   2181 	else
   2182 #endif /* XEN */
   2183 		init_x86_64_ksyms();
   2184 #endif /* XENPV */
   2185 
   2186 #ifndef XENPV
   2187 	intr_default_setup();
   2188 #else
   2189 	events_default_setup();
   2190 #endif
   2191 
   2192 	splraise(IPL_HIGH);
   2193 	x86_enable_intr();
   2194 
   2195 #ifdef DDB
   2196 	if (boothowto & RB_KDB)
   2197 		Debugger();
   2198 #endif
   2199 #ifdef KGDB
   2200 	kgdb_port_init();
   2201 	if (boothowto & RB_KDB) {
   2202 		kgdb_debug_init = 1;
   2203 		kgdb_connect(1);
   2204 	}
   2205 #endif
   2206 
   2207 	pcb->pcb_dbregs = NULL;
   2208 	x86_dbregs_init();
   2209 }
   2210 
   2211 void
   2212 cpu_reset(void)
   2213 {
   2214 #ifndef XENPV
   2215 	idt_descriptor_t *idt;
   2216 	vaddr_t vaddr;
   2217 
   2218 	idt = cpu_info_primary.ci_idtvec.iv_idt;
   2219 	vaddr = (vaddr_t)idt;
   2220 #endif
   2221 
   2222 	x86_disable_intr();
   2223 
   2224 #ifdef XENPV
   2225 	HYPERVISOR_reboot();
   2226 #else
   2227 
   2228 	x86_reset();
   2229 
   2230 	/*
   2231 	 * Try to cause a triple fault and watchdog reset by making the IDT
   2232 	 * invalid and causing a fault.
   2233 	 */
   2234 	kpreempt_disable();
   2235 	pmap_changeprot_local(vaddr, VM_PROT_READ|VM_PROT_WRITE);
   2236 	memset((void *)idt, 0, NIDT * sizeof(idt[0]));
   2237 	kpreempt_enable();
   2238 	breakpoint();
   2239 
   2240 #if 0
   2241 	/*
   2242 	 * Try to cause a triple fault and watchdog reset by unmapping the
   2243 	 * entire address space and doing a TLB flush.
   2244 	 */
   2245 	memset((void *)PTD, 0, PAGE_SIZE);
   2246 	tlbflush();
   2247 #endif
   2248 #endif	/* XENPV */
   2249 
   2250 	for (;;);
   2251 }
   2252 
   2253 void
   2254 cpu_getmcontext(struct lwp *l, mcontext_t *mcp, unsigned int *flags)
   2255 {
   2256 	const struct trapframe *tf = l->l_md.md_regs;
   2257 	__greg_t ras_rip;
   2258 
   2259 	mcp->__gregs[_REG_RDI] = tf->tf_rdi;
   2260 	mcp->__gregs[_REG_RSI] = tf->tf_rsi;
   2261 	mcp->__gregs[_REG_RDX] = tf->tf_rdx;
   2262 	mcp->__gregs[_REG_R10] = tf->tf_r10;
   2263 	mcp->__gregs[_REG_R8]  = tf->tf_r8;
   2264 	mcp->__gregs[_REG_R9]  = tf->tf_r9;
   2265 	/* argX not touched */
   2266 	mcp->__gregs[_REG_RCX] = tf->tf_rcx;
   2267 	mcp->__gregs[_REG_R11] = tf->tf_r11;
   2268 	mcp->__gregs[_REG_R12] = tf->tf_r12;
   2269 	mcp->__gregs[_REG_R13] = tf->tf_r13;
   2270 	mcp->__gregs[_REG_R14] = tf->tf_r14;
   2271 	mcp->__gregs[_REG_R15] = tf->tf_r15;
   2272 	mcp->__gregs[_REG_RBP] = tf->tf_rbp;
   2273 	mcp->__gregs[_REG_RBX] = tf->tf_rbx;
   2274 	mcp->__gregs[_REG_RAX] = tf->tf_rax;
   2275 	mcp->__gregs[_REG_GS]  = 0;
   2276 	mcp->__gregs[_REG_FS]  = 0;
   2277 	mcp->__gregs[_REG_ES]  = GSEL(GUDATA_SEL, SEL_UPL);
   2278 	mcp->__gregs[_REG_DS]  = GSEL(GUDATA_SEL, SEL_UPL);
   2279 	mcp->__gregs[_REG_TRAPNO] = tf->tf_trapno;
   2280 	mcp->__gregs[_REG_ERR] = tf->tf_err;
   2281 	mcp->__gregs[_REG_RIP] = tf->tf_rip;
   2282 	mcp->__gregs[_REG_CS]  = LSEL(LUCODE_SEL, SEL_UPL);
   2283 	mcp->__gregs[_REG_RFLAGS] = tf->tf_rflags;
   2284 	mcp->__gregs[_REG_RSP] = tf->tf_rsp;
   2285 	mcp->__gregs[_REG_SS]  = LSEL(LUDATA_SEL, SEL_UPL);
   2286 
   2287 	if ((ras_rip = (__greg_t)ras_lookup(l->l_proc,
   2288 	    (void *) mcp->__gregs[_REG_RIP])) != -1)
   2289 		mcp->__gregs[_REG_RIP] = ras_rip;
   2290 
   2291 	*flags |= _UC_CPU;
   2292 
   2293 	mcp->_mc_tlsbase = (uintptr_t)l->l_private;
   2294 	*flags |= _UC_TLSBASE;
   2295 
   2296 	process_read_fpregs_xmm(l, (struct fxsave *)&mcp->__fpregs.__fxsave);
   2297 	*flags |= _UC_FPU;
   2298 }
   2299 
   2300 /*
   2301  * cpu_getmcontext_xsave(l, mcp, flags, xsavebuf, xsavelen, user_xsave)
   2302  *
   2303  *	Copy out xsavebuf[512..xsavelen) to user_xsave[512..xsavelen),
   2304  *	set mcp to point at it, and set _UC_XSAVE in *flags.  Caller:
   2305  *
   2306  *	- must have already initialized the FXSAVE area of mcontext_t,
   2307  *	- must have already set _UC_FPU in *flags,
   2308  *	- must have arranged user_xsave[0..512) to overlap with the
   2309  *	  FXSAVE area of mcontext_t, and
   2310  *	- must subsequently copy out the mcontext_t updated with a
   2311  *	  pointer/length to the XSAVE area.
   2312  *
   2313  *	May fail if the copyout fails.
   2314  */
   2315 static int
   2316 cpu_getmcontext_xsave(struct lwp *l, mcontext_t *mcp, unsigned int *flags,
   2317     const struct xsave_header *xsavebuf, size_t xsavelen,
   2318     struct xsave_header *user_xsave)
   2319 {
   2320 	enum { fxsavelen = sizeof(mcp->__fpregs.__fxsave) };
   2321 	int error;
   2322 
   2323 	CTASSERT(fxsavelen == 512);
   2324 
   2325 	KASSERT(*flags & _UC_FPU);
   2326 	KASSERT(fxsavelen <= xsavelen);
   2327 	KDASSERT(memcmp(&mcp->__fpregs.__fxsave, xsavebuf, fxsavelen) == 0);
   2328 
   2329 	/*
   2330 	 * Copy out the part of the XSAVE area that doesn't overlap.
   2331 	 */
   2332 	error = copyout((const char *)xsavebuf + fxsavelen,
   2333 	    (char *)user_xsave + fxsavelen, xsavelen - fxsavelen);
   2334 	if (error != 0)
   2335 		return error;
   2336 
   2337 	/*
   2338 	 * Record a pointer to the real XSAVE area in the
   2339 	 * architecturally unused bits mcontext_t's FXSAVE area.
   2340 	 */
   2341 	mcp->__fpregs.__xsave.__xsaveptr = (__greg_t)(uintptr_t)user_xsave;
   2342 	mcp->__fpregs.__xsave.__xsavelen = (__greg_t)xsavelen;
   2343 
   2344 	/*
   2345 	 * Set the _UC_XSAVE flag so cpu_setmcontext will be able to
   2346 	 * restore the full state from the XSAVE area.
   2347 	 */
   2348 	*flags |= _UC_XSAVE;
   2349 
   2350 	/* Success! */
   2351 	return 0;
   2352 }
   2353 
   2354 int
   2355 cpu_setmcontext(struct lwp *l, const mcontext_t *mcp, unsigned int flags)
   2356 {
   2357 	struct trapframe *tf = l->l_md.md_regs;
   2358 	const __greg_t *gr = mcp->__gregs;
   2359 	struct proc *p = l->l_proc;
   2360 	struct xsave_header *xsavebuf = NULL;
   2361 	size_t xsavelen = 0;
   2362 	int error;
   2363 	int64_t rflags;
   2364 
   2365 	CTASSERT(sizeof (mcontext_t) == 26 * 8 + 8 + 512);
   2366 
   2367 	/*
   2368 	 * If there's an external XSAVE area, copy it in and validate
   2369 	 * it before we irreversibly modify the trapframe.
   2370 	 *
   2371 	 * We could check the length against the state components
   2372 	 * included, but we currently don't: if it's truncated, it will
   2373 	 * be as if the truncated part were zero-filled -- this is
   2374 	 * implemented in process_write_xsave, called a little below.
   2375 	 */
   2376 	if ((flags & _UC_XSAVE) != 0) {
   2377 		const struct xsave_header *user_xsave =
   2378 		    (void *)(uintptr_t)mcp->__fpregs.__xsave.__xsaveptr;
   2379 
   2380 		xsavelen = mcp->__fpregs.__xsave.__xsavelen;
   2381 		error = process_verify_xsavelen(l, xsavelen);
   2382 		if (error != 0)
   2383 			goto out;
   2384 		xsavebuf = kmem_alloc(xsavelen, KM_SLEEP);
   2385 		error = copyin(user_xsave, xsavebuf, xsavelen);
   2386 		if (error != 0)
   2387 			goto out;
   2388 		error = process_verify_xsave(l, xsavebuf, xsavelen);
   2389 		if (error != 0)
   2390 			goto out;
   2391 	}
   2392 
   2393 	if ((flags & _UC_CPU) != 0) {
   2394 		error = cpu_mcontext_validate(l, mcp);
   2395 		if (error != 0)
   2396 			goto out;
   2397 
   2398 		tf->tf_rdi  = gr[_REG_RDI];
   2399 		tf->tf_rsi  = gr[_REG_RSI];
   2400 		tf->tf_rdx  = gr[_REG_RDX];
   2401 		tf->tf_r10  = gr[_REG_R10];
   2402 		tf->tf_r8   = gr[_REG_R8];
   2403 		tf->tf_r9   = gr[_REG_R9];
   2404 		/* argX not touched */
   2405 		tf->tf_rcx  = gr[_REG_RCX];
   2406 		tf->tf_r11  = gr[_REG_R11];
   2407 		tf->tf_r12  = gr[_REG_R12];
   2408 		tf->tf_r13  = gr[_REG_R13];
   2409 		tf->tf_r14  = gr[_REG_R14];
   2410 		tf->tf_r15  = gr[_REG_R15];
   2411 		tf->tf_rbp  = gr[_REG_RBP];
   2412 		tf->tf_rbx  = gr[_REG_RBX];
   2413 		tf->tf_rax  = gr[_REG_RAX];
   2414 		tf->tf_gs   = 0;
   2415 		tf->tf_fs   = 0;
   2416 		tf->tf_es   = GSEL(GUDATA_SEL, SEL_UPL);
   2417 		tf->tf_ds   = GSEL(GUDATA_SEL, SEL_UPL);
   2418 		/* trapno, err not touched */
   2419 		tf->tf_rip  = gr[_REG_RIP];
   2420 		tf->tf_cs   = LSEL(LUCODE_SEL, SEL_UPL);
   2421 		rflags = tf->tf_rflags;
   2422 		rflags &= ~PSL_USER;
   2423 		tf->tf_rflags = rflags | (gr[_REG_RFLAGS] & PSL_USER);
   2424 		tf->tf_rsp  = gr[_REG_RSP];
   2425 		tf->tf_ss   = LSEL(LUDATA_SEL, SEL_UPL);
   2426 
   2427 		l->l_md.md_flags |= MDL_IRET;
   2428 	}
   2429 
   2430 	if ((flags & _UC_XSAVE) != 0) {
   2431 		KASSERT(xsavebuf != NULL);
   2432 		process_write_xsave(l, xsavebuf, xsavelen);
   2433 	} else if ((flags & _UC_FPU) != 0) {
   2434 		process_write_fpregs_xmm(l,
   2435 		    (const struct fxsave *)&mcp->__fpregs.__fxsave);
   2436 	}
   2437 
   2438 	if ((flags & _UC_TLSBASE) != 0)
   2439 		lwp_setprivate(l, (void *)(uintptr_t)mcp->_mc_tlsbase);
   2440 
   2441 	mutex_enter(p->p_lock);
   2442 	if (flags & _UC_SETSTACK)
   2443 		l->l_sigstk.ss_flags |= SS_ONSTACK;
   2444 	if (flags & _UC_CLRSTACK)
   2445 		l->l_sigstk.ss_flags &= ~SS_ONSTACK;
   2446 	mutex_exit(p->p_lock);
   2447 
   2448 	/* Success! */
   2449 	error = 0;
   2450 
   2451 out:	if (xsavebuf)
   2452 		kmem_free(xsavebuf, xsavelen);
   2453 	return error;
   2454 }
   2455 
   2456 int
   2457 cpu_mcontext_validate(struct lwp *l, const mcontext_t *mcp)
   2458 {
   2459 	struct proc *p __diagused = l->l_proc;
   2460 	struct trapframe *tf = l->l_md.md_regs;
   2461 	const __greg_t *gr;
   2462 	uint16_t sel;
   2463 
   2464 	KASSERT((p->p_flag & PK_32) == 0);
   2465 	gr = mcp->__gregs;
   2466 
   2467 	if (((gr[_REG_RFLAGS] ^ tf->tf_rflags) & PSL_USERSTATIC) != 0)
   2468 		return EINVAL;
   2469 
   2470 	sel = gr[_REG_ES] & 0xffff;
   2471 	if (sel != 0 && !VALID_USER_DSEL(sel))
   2472 		return EINVAL;
   2473 
   2474 	sel = gr[_REG_FS] & 0xffff;
   2475 	if (sel != 0 && !VALID_USER_DSEL(sel))
   2476 		return EINVAL;
   2477 
   2478 	sel = gr[_REG_GS] & 0xffff;
   2479 	if (sel != 0 && !VALID_USER_DSEL(sel))
   2480 		return EINVAL;
   2481 
   2482 	sel = gr[_REG_DS] & 0xffff;
   2483 	if (!VALID_USER_DSEL(sel))
   2484 		return EINVAL;
   2485 
   2486 #ifndef XENPV
   2487 	sel = gr[_REG_SS] & 0xffff;
   2488 	if (!VALID_USER_DSEL(sel))
   2489 		return EINVAL;
   2490 
   2491 	sel = gr[_REG_CS] & 0xffff;
   2492 	if (!VALID_USER_CSEL(sel))
   2493 		return EINVAL;
   2494 #endif
   2495 
   2496 	if (gr[_REG_RIP] >= VM_MAXUSER_ADDRESS)
   2497 		return EINVAL;
   2498 
   2499 	return 0;
   2500 }
   2501 
   2502 int
   2503 mm_md_kernacc(void *ptr, vm_prot_t prot, bool *handled)
   2504 {
   2505 	const vaddr_t v = (vaddr_t)ptr;
   2506 	vaddr_t kva, kva_end;
   2507 	size_t i;
   2508 
   2509 	kva = bootspace.head.va;
   2510 	kva_end = kva + bootspace.head.sz;
   2511 	if (v >= kva && v < kva_end) {
   2512 		*handled = true;
   2513 		return 0;
   2514 	}
   2515 
   2516 	for (i = 0; i < BTSPACE_NSEGS; i++) {
   2517 		kva = bootspace.segs[i].va;
   2518 		kva_end = kva + bootspace.segs[i].sz;
   2519 		if (v < kva || v >= kva_end)
   2520 			continue;
   2521 		*handled = true;
   2522 		if (bootspace.segs[i].type == BTSEG_TEXT ||
   2523 		    bootspace.segs[i].type == BTSEG_RODATA) {
   2524 			if (prot & VM_PROT_WRITE) {
   2525 				return EFAULT;
   2526 			}
   2527 		}
   2528 		return 0;
   2529 	}
   2530 
   2531 	kva = bootspace.boot.va;
   2532 	kva_end = kva + bootspace.boot.sz;
   2533 	if (v >= kva && v < kva_end) {
   2534 		*handled = true;
   2535 		return 0;
   2536 	}
   2537 
   2538 	if (v >= bootspace.smodule && v < bootspace.emodule) {
   2539 		*handled = true;
   2540 		vm_map_lock_read(module_map);
   2541 		const bool allowed =
   2542 		    uvm_map_checkprot(module_map, v, v + 1, prot);
   2543 		vm_map_unlock_read(module_map);
   2544 		if (!allowed) {
   2545 			return EFAULT;
   2546 		}
   2547 	} else {
   2548 		*handled = false;
   2549 	}
   2550 	return 0;
   2551 }
   2552 
   2553 /*
   2554  * Zero out a 64bit LWP's segments registers. Used when exec'ing a new
   2555  * 64bit program.
   2556  */
   2557 void
   2558 cpu_segregs64_zero(struct lwp *l)
   2559 {
   2560 	struct trapframe * const tf = l->l_md.md_regs;
   2561 	struct pcb *pcb;
   2562 	uint64_t zero = 0;
   2563 
   2564 	KASSERT(kpreempt_disabled());
   2565 	KASSERT((l->l_proc->p_flag & PK_32) == 0);
   2566 	KASSERT(l == curlwp);
   2567 
   2568 	pcb = lwp_getpcb(l);
   2569 
   2570 	tf->tf_fs = 0;
   2571 	tf->tf_gs = 0;
   2572 	setds(GSEL(GUDATA_SEL, SEL_UPL));
   2573 	setes(GSEL(GUDATA_SEL, SEL_UPL));
   2574 	setfs(0);
   2575 	setusergs(0);
   2576 
   2577 #ifndef XENPV
   2578 	wrmsr(MSR_FSBASE, 0);
   2579 	wrmsr(MSR_KERNELGSBASE, 0);
   2580 #else
   2581 	HYPERVISOR_set_segment_base(SEGBASE_FS, 0);
   2582 	HYPERVISOR_set_segment_base(SEGBASE_GS_USER, 0);
   2583 #endif
   2584 
   2585 	pcb->pcb_fs = 0;
   2586 	pcb->pcb_gs = 0;
   2587 	update_descriptor(&curcpu()->ci_gdt[GUFS_SEL], &zero);
   2588 	update_descriptor(&curcpu()->ci_gdt[GUGS_SEL], &zero);
   2589 }
   2590 
   2591 /*
   2592  * Zero out a 32bit LWP's segments registers. Used when exec'ing a new
   2593  * 32bit program.
   2594  */
   2595 void
   2596 cpu_segregs32_zero(struct lwp *l)
   2597 {
   2598 	struct trapframe * const tf = l->l_md.md_regs;
   2599 	struct pcb *pcb;
   2600 	uint64_t zero = 0;
   2601 
   2602 	KASSERT(kpreempt_disabled());
   2603 	KASSERT(l->l_proc->p_flag & PK_32);
   2604 	KASSERT(l == curlwp);
   2605 
   2606 	pcb = lwp_getpcb(l);
   2607 
   2608 	tf->tf_fs = 0;
   2609 	tf->tf_gs = 0;
   2610 	setds(GSEL(GUDATA32_SEL, SEL_UPL));
   2611 	setes(GSEL(GUDATA32_SEL, SEL_UPL));
   2612 	setfs(0);
   2613 	setusergs(0);
   2614 	pcb->pcb_fs = 0;
   2615 	pcb->pcb_gs = 0;
   2616 	update_descriptor(&curcpu()->ci_gdt[GUFS_SEL], &zero);
   2617 	update_descriptor(&curcpu()->ci_gdt[GUGS_SEL], &zero);
   2618 }
   2619 
   2620 /*
   2621  * Load an LWP's TLS context, possibly changing the %fs and %gs selectors.
   2622  * Used only for 32-bit processes.
   2623  */
   2624 void
   2625 cpu_fsgs_reload(struct lwp *l, int fssel, int gssel)
   2626 {
   2627 	struct trapframe *tf;
   2628 	struct pcb *pcb;
   2629 
   2630 	KASSERT(l->l_proc->p_flag & PK_32);
   2631 	KASSERT(l == curlwp);
   2632 
   2633 	tf = l->l_md.md_regs;
   2634 	fssel &= 0xFFFF;
   2635 	gssel &= 0xFFFF;
   2636 
   2637 	pcb = lwp_getpcb(l);
   2638 	kpreempt_disable();
   2639 	update_descriptor(&curcpu()->ci_gdt[GUFS_SEL], &pcb->pcb_fs);
   2640 	update_descriptor(&curcpu()->ci_gdt[GUGS_SEL], &pcb->pcb_gs);
   2641 
   2642 #ifdef XENPV
   2643 	setusergs(gssel);
   2644 #endif
   2645 
   2646 	tf->tf_fs = fssel;
   2647 	tf->tf_gs = gssel;
   2648 	kpreempt_enable();
   2649 }
   2650 
   2651 bool
   2652 mm_md_direct_mapped_io(void *addr, paddr_t *paddr)
   2653 {
   2654 	vaddr_t va = (vaddr_t)addr;
   2655 
   2656 #ifdef __HAVE_DIRECT_MAP
   2657 	if (va >= PMAP_DIRECT_BASE && va < PMAP_DIRECT_END) {
   2658 		*paddr = PMAP_DIRECT_UNMAP(va);
   2659 		return true;
   2660 	}
   2661 #else
   2662 	__USE(va);
   2663 #endif
   2664 
   2665 	return false;
   2666 }
   2667 
   2668 bool
   2669 mm_md_direct_mapped_phys(paddr_t paddr, vaddr_t *vaddr)
   2670 {
   2671 #ifdef __HAVE_DIRECT_MAP
   2672 	*vaddr = PMAP_DIRECT_MAP(paddr);
   2673 	return true;
   2674 #else
   2675 	return false;
   2676 #endif
   2677 }
   2678 
   2679 static void
   2680 idt_vec_copy(struct idt_vec *dst, struct idt_vec *src)
   2681 {
   2682 	idt_descriptor_t *idt_dst;
   2683 
   2684 	idt_dst = dst->iv_idt;
   2685 
   2686 	kpreempt_disable();
   2687 	pmap_changeprot_local((vaddr_t)idt_dst, VM_PROT_READ|VM_PROT_WRITE);
   2688 
   2689 	memcpy(idt_dst, src->iv_idt, PAGE_SIZE);
   2690 	memcpy(dst->iv_allocmap, src->iv_allocmap, sizeof(dst->iv_allocmap));
   2691 
   2692 	pmap_changeprot_local((vaddr_t)idt_dst, VM_PROT_READ);
   2693 	kpreempt_enable();
   2694 }
   2695 
   2696 void
   2697 idt_vec_init_cpu_md(struct idt_vec *iv, cpuid_t cid)
   2698 {
   2699 	vaddr_t va;
   2700 
   2701 	if (cid != cpu_index(&cpu_info_primary) &&
   2702 	    idt_vec_is_pcpu()) {
   2703 #ifdef __HAVE_PCPU_AREA
   2704 		va = (vaddr_t)&pcpuarea->ent[cid].idt;
   2705 #else
   2706 		struct vm_page *pg;
   2707 
   2708 		va = uvm_km_alloc(kernel_map, PAGE_SIZE, 0,
   2709 		    UVM_KMF_VAONLY);
   2710 		pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO);
   2711 		if (pg == NULL) {
   2712 			panic("failed to allocate a page for IDT");
   2713 		}
   2714 		pmap_kenter_pa(va, VM_PAGE_TO_PHYS(pg),
   2715 		    VM_PROT_READ|VM_PROT_WRITE, 0);
   2716 		pmap_update(pmap_kernel());
   2717 #endif
   2718 
   2719 		memset((void *)va, 0, PAGE_SIZE);
   2720 #ifndef XENPV
   2721 		pmap_changeprot_local(va, VM_PROT_READ);
   2722 #endif
   2723 		pmap_update(pmap_kernel());
   2724 
   2725 		iv->iv_idt = (void *)va;
   2726 		idt_vec_copy(iv, &(cpu_info_primary.ci_idtvec));
   2727 	} else {
   2728 		iv->iv_idt = (void *)idt_vaddr;
   2729 	}
   2730 }
   2731