1 /* $NetBSD: trap.c,v 1.30 2024/11/25 22:04:14 skrll Exp $ */ 2 3 /*- 4 * Copyright (c) 2014 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Matt Thomas of 3am Software Foundry. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 34 #define __PMAP_PRIVATE 35 #define __UFETCHSTORE_PRIVATE 36 37 __RCSID("$NetBSD: trap.c,v 1.30 2024/11/25 22:04:14 skrll Exp $"); 38 39 #include <sys/param.h> 40 41 #include <sys/atomic.h> 42 #include <sys/cpu.h> 43 #include <sys/kauth.h> 44 #include <sys/signal.h> 45 #include <sys/signalvar.h> 46 #include <sys/siginfo.h> 47 #include <sys/systm.h> 48 49 #include <uvm/uvm.h> 50 51 #include <machine/locore.h> 52 #include <machine/machdep.h> 53 #include <machine/db_machdep.h> 54 #include <machine/userret.h> 55 56 #define MACHINE_ECALL_TRAP_MASK (__BIT(CAUSE_MACHINE_ECALL)) 57 58 #define SUPERVISOR_ECALL_TRAP_MASK \ 59 (__BIT(CAUSE_SUPERVISOR_ECALL)) 60 61 #define USER_ECALL_TRAP_MASK (__BIT(CAUSE_USER_ECALL)) 62 63 #define SYSCALL_TRAP_MASK (__BIT(CAUSE_SYSCALL)) 64 65 #define BREAKPOINT_TRAP_MASK (__BIT(CAUSE_BREAKPOINT)) 66 67 #define INSTRUCTION_TRAP_MASK (__BIT(CAUSE_ILLEGAL_INSTRUCTION)) 68 69 #define FAULT_TRAP_MASK (__BIT(CAUSE_FETCH_ACCESS) \ 70 |__BIT(CAUSE_LOAD_ACCESS) \ 71 |__BIT(CAUSE_STORE_ACCESS) \ 72 |__BIT(CAUSE_FETCH_PAGE_FAULT) \ 73 |__BIT(CAUSE_LOAD_PAGE_FAULT) \ 74 |__BIT(CAUSE_STORE_PAGE_FAULT)) 75 76 #define MISALIGNED_TRAP_MASK (__BIT(CAUSE_FETCH_MISALIGNED) \ 77 |__BIT(CAUSE_LOAD_MISALIGNED) \ 78 |__BIT(CAUSE_STORE_MISALIGNED)) 79 80 static const char * const causenames[] = { 81 [CAUSE_FETCH_MISALIGNED] = "misaligned fetch", 82 [CAUSE_LOAD_MISALIGNED] = "misaligned load", 83 [CAUSE_STORE_MISALIGNED] = "misaligned store", 84 [CAUSE_FETCH_ACCESS] = "fetch", 85 [CAUSE_LOAD_ACCESS] = "load", 86 [CAUSE_STORE_ACCESS] = "store", 87 [CAUSE_ILLEGAL_INSTRUCTION] = "illegal instruction", 88 [CAUSE_BREAKPOINT] = "breakpoint", 89 [CAUSE_SYSCALL] = "syscall", 90 [CAUSE_FETCH_PAGE_FAULT] = "instruction page fault", 91 [CAUSE_LOAD_PAGE_FAULT] = "load page fault", 92 [CAUSE_STORE_PAGE_FAULT] = "store page fault", 93 }; 94 95 96 void 97 cpu_jump_onfault(struct trapframe *tf, const struct faultbuf *fb, int error) 98 { 99 tf->tf_a0 = error; 100 tf->tf_ra = fb->fb_reg[FB_RA]; 101 tf->tf_s0 = fb->fb_reg[FB_S0]; 102 tf->tf_s1 = fb->fb_reg[FB_S1]; 103 tf->tf_s2 = fb->fb_reg[FB_S2]; 104 tf->tf_s3 = fb->fb_reg[FB_S3]; 105 tf->tf_s4 = fb->fb_reg[FB_S4]; 106 tf->tf_s5 = fb->fb_reg[FB_S5]; 107 tf->tf_s6 = fb->fb_reg[FB_S6]; 108 tf->tf_s7 = fb->fb_reg[FB_S7]; 109 tf->tf_s8 = fb->fb_reg[FB_S8]; 110 tf->tf_s9 = fb->fb_reg[FB_S9]; 111 tf->tf_s10 = fb->fb_reg[FB_S10]; 112 tf->tf_s11 = fb->fb_reg[FB_S11]; 113 tf->tf_sp = fb->fb_reg[FB_SP]; 114 tf->tf_pc = fb->fb_reg[FB_RA]; 115 } 116 117 118 int 119 copyin(const void *uaddr, void *kaddr, size_t len) 120 { 121 struct faultbuf fb; 122 int error; 123 124 if (__predict_false(len == 0)) { 125 return 0; 126 } 127 128 // XXXNH cf. VM_MIN_ADDRESS and user_va0_disable 129 if (uaddr == NULL) 130 return EFAULT; 131 132 const vaddr_t uva = (vaddr_t)uaddr; 133 if (uva > VM_MAXUSER_ADDRESS - len) 134 return EFAULT; 135 136 csr_sstatus_set(SR_SUM); 137 if ((error = cpu_set_onfault(&fb)) == 0) { 138 memcpy(kaddr, uaddr, len); 139 cpu_unset_onfault(); 140 } 141 csr_sstatus_clear(SR_SUM); 142 143 return error; 144 } 145 146 int 147 copyout(const void *kaddr, void *uaddr, size_t len) 148 { 149 struct faultbuf fb; 150 int error; 151 152 if (__predict_false(len == 0)) { 153 return 0; 154 } 155 156 // XXXNH cf. VM_MIN_ADDRESS and user_va0_disable 157 if (uaddr == NULL) 158 return EFAULT; 159 160 const vaddr_t uva = (vaddr_t)uaddr; 161 if (uva > VM_MAXUSER_ADDRESS - len) 162 return EFAULT; 163 164 csr_sstatus_set(SR_SUM); 165 if ((error = cpu_set_onfault(&fb)) == 0) { 166 memcpy(uaddr, kaddr, len); 167 cpu_unset_onfault(); 168 } 169 csr_sstatus_clear(SR_SUM); 170 171 return error; 172 } 173 174 int 175 kcopy(const void *kfaddr, void *kdaddr, size_t len) 176 { 177 struct faultbuf fb; 178 int error; 179 180 if ((error = cpu_set_onfault(&fb)) == 0) { 181 memcpy(kdaddr, kfaddr, len); 182 cpu_unset_onfault(); 183 } 184 185 return error; 186 } 187 188 int 189 copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done) 190 { 191 struct faultbuf fb; 192 size_t retlen; 193 int error; 194 195 if (__predict_false(len == 0)) { 196 return 0; 197 } 198 199 if (__predict_false(uaddr == NULL)) 200 return EFAULT; 201 /* 202 * Can only check if starting user address is out of range here. 203 * The string may end before uva + len. 204 */ 205 const vaddr_t uva = (vaddr_t)uaddr; 206 if (uva > VM_MAXUSER_ADDRESS) 207 return EFAULT; 208 209 csr_sstatus_set(SR_SUM); 210 if ((error = cpu_set_onfault(&fb)) == 0) { 211 retlen = strlcpy(kaddr, uaddr, len); 212 cpu_unset_onfault(); 213 if (retlen >= len) { 214 error = ENAMETOOLONG; 215 } else if (done != NULL) { 216 *done = retlen + 1; 217 } 218 } 219 csr_sstatus_clear(SR_SUM); 220 221 return error; 222 } 223 224 int 225 copyoutstr(const void *kaddr, void *uaddr, size_t len, size_t *done) 226 { 227 struct faultbuf fb; 228 size_t retlen; 229 int error; 230 231 if (__predict_false(len == 0)) { 232 return 0; 233 } 234 235 if (__predict_false(uaddr == NULL)) 236 return EFAULT; 237 /* 238 * Can only check if starting user address is out of range here. 239 * The string may end before uva + len. 240 */ 241 const vaddr_t uva = (vaddr_t)uaddr; 242 if (uva > VM_MAXUSER_ADDRESS) 243 return EFAULT; 244 245 csr_sstatus_set(SR_SUM); 246 if ((error = cpu_set_onfault(&fb)) == 0) { 247 retlen = strlcpy(uaddr, kaddr, len); 248 cpu_unset_onfault(); 249 if (retlen >= len) { 250 error = ENAMETOOLONG; 251 } else if (done != NULL) { 252 *done = retlen + 1; 253 } 254 } 255 csr_sstatus_clear(SR_SUM); 256 257 return error; 258 } 259 260 static const char * 261 cause_name(register_t cause) 262 { 263 if (CAUSE_INTERRUPT_P(cause)) 264 return "interrupt"; 265 const char *name = "(unk)"; 266 if (cause < __arraycount(causenames) && causenames[cause] != NULL) 267 name = causenames[cause]; 268 269 return name; 270 } 271 272 void 273 dump_trapframe(const struct trapframe *tf, void (*pr)(const char *, ...)) 274 { 275 const char *name = cause_name(tf->tf_cause); 276 static const char *regname[] = { 277 "ra", "sp", "gp", // x0, x1, x2, x3, 278 "tp", "t0", "t1", "t2", // x4, x5, x6, x7, 279 "s0", "s1", "a0", "a1", // x8, x9, x10, x11, 280 "a2", "a3", "a4", "a5", // x12, x13, x14, x15, 281 "a6", "a7", "s2", "s3", // x16, x17, x18, x19, 282 "s4", "s5", "s6", "s7", // x20, x21, x22, x23, 283 "s8", "s9", "s10", "s11", // x24, x25, x26, x27, 284 "t3", "t4", "t5", "t6", // x28, x29, x30, x31, 285 }; 286 287 (*pr)("Trapframe @ %p " 288 "(cause=%d (%s), status=%#x, pc=%#18" PRIxREGISTER 289 ", va=%#" PRIxREGISTER "):\n", 290 tf, tf->tf_cause, name, tf->tf_sr, tf->tf_pc, tf->tf_tval); 291 292 (*pr)(" "); 293 for (unsigned reg = 1; reg < 32; reg++) { 294 (*pr)("%-3s=%#18" PRIxREGISTER " ", 295 regname[reg - 1], 296 tf->tf_regs.r_reg[reg - 1]); 297 if (reg % 4 == 3) 298 (*pr)("\n"); 299 } 300 } 301 302 static inline void 303 trap_ksi_init(ksiginfo_t *ksi, int signo, int code, vaddr_t addr, 304 register_t cause) 305 { 306 KSI_INIT_TRAP(ksi); 307 ksi->ksi_signo = signo; 308 ksi->ksi_code = code; 309 ksi->ksi_addr = (void *)addr; 310 ksi->ksi_trap = cause; 311 } 312 313 static void 314 cpu_trapsignal(struct trapframe *tf, ksiginfo_t *ksi) 315 { 316 if (cpu_printfataltraps) { 317 dump_trapframe(tf, printf); 318 } 319 (*curlwp->l_proc->p_emul->e_trapsignal)(curlwp, ksi); 320 } 321 322 static inline vm_prot_t 323 get_faulttype(register_t cause) 324 { 325 if (cause == CAUSE_LOAD_ACCESS || cause == CAUSE_LOAD_PAGE_FAULT) 326 return VM_PROT_READ; 327 if (cause == CAUSE_STORE_ACCESS || cause == CAUSE_STORE_PAGE_FAULT) 328 return VM_PROT_WRITE; 329 KASSERT(cause == CAUSE_FETCH_ACCESS || cause == CAUSE_FETCH_PAGE_FAULT); 330 return VM_PROT_EXECUTE; 331 } 332 333 static bool 334 trap_pagefault_fixup(struct trapframe *tf, struct pmap *pmap, register_t cause, 335 intptr_t addr) 336 { 337 pt_entry_t * const ptep = pmap_pte_lookup(pmap, addr); 338 struct vm_page *pg; 339 340 if (ptep == NULL) 341 return false; 342 343 pt_entry_t opte = *ptep; 344 if (!pte_valid_p(opte)) 345 return false; 346 347 pt_entry_t npte; 348 u_int attr; 349 do { 350 /* TODO: PTE_G is just the kernel PTE, but all pages 351 * can fault for CAUSE_LOAD_PAGE_FAULT and 352 * CAUSE_STORE_PAGE_FAULT...*/ 353 /* if ((opte & ~PTE_G) == 0) */ 354 /* return false; */ 355 356 pg = PHYS_TO_VM_PAGE(pte_to_paddr(opte)); 357 if (pg == NULL) 358 return false; 359 360 attr = 0; 361 npte = opte; 362 363 switch (cause) { 364 case CAUSE_LOAD_PAGE_FAULT: 365 if ((npte & PTE_R) == 0) { 366 npte |= PTE_A; 367 attr |= VM_PAGEMD_REFERENCED; 368 } 369 break; 370 case CAUSE_STORE_ACCESS: 371 if ((npte & PTE_W) != 0) { 372 npte |= PTE_A | PTE_D; 373 attr |= VM_PAGEMD_MODIFIED; 374 } 375 break; 376 case CAUSE_STORE_PAGE_FAULT: 377 if ((npte & PTE_D) == 0) { 378 npte |= PTE_A | PTE_D; 379 attr |= VM_PAGEMD_REFERENCED | VM_PAGEMD_MODIFIED; 380 } 381 break; 382 case CAUSE_FETCH_ACCESS: 383 case CAUSE_FETCH_PAGE_FAULT: 384 #if 0 385 if ((npte & PTE_NX) != 0) { 386 npte &= ~PTE_NX; 387 attr |= VM_PAGEMD_EXECPAGE; 388 } 389 #endif 390 break; 391 default: 392 panic("%s: Unhandled cause (%#" PRIxREGISTER 393 ") for addr %lx", __func__, cause, addr); 394 } 395 if (attr == 0) 396 return false; 397 } while (opte != atomic_cas_pte(ptep, opte, npte)); 398 399 pmap_page_set_attributes(VM_PAGE_TO_MD(pg), attr); 400 pmap_tlb_update_addr(pmap, addr, npte, 0); 401 402 if (attr & VM_PAGEMD_EXECPAGE) 403 pmap_md_page_syncicache(VM_PAGE_TO_MD(pg), 404 curcpu()->ci_kcpuset); 405 406 return true; 407 } 408 409 static bool 410 trap_pagefault(struct trapframe *tf, register_t epc, register_t status, 411 register_t cause, register_t tval, bool usertrap_p, ksiginfo_t *ksi) 412 { 413 struct proc * const p = curlwp->l_proc; 414 const intptr_t addr = trunc_page(tval); 415 416 if (__predict_false(usertrap_p 417 && (false 418 // Make this address is not trying to access kernel space. 419 || addr < 0 420 #ifdef _LP64 421 // If this is a process using a 32-bit address space, make 422 // sure the address is a signed 32-bit number. 423 || ((p->p_flag & PK_32) && (int32_t) addr != addr) 424 #endif 425 || false))) { 426 trap_ksi_init(ksi, SIGSEGV, SEGV_MAPERR, addr, cause); 427 return false; 428 } 429 430 struct vm_map * const map = (addr >= 0 ? 431 &p->p_vmspace->vm_map : kernel_map); 432 433 // See if this fault is for reference/modified/execpage tracking 434 if (trap_pagefault_fixup(tf, map->pmap, cause, addr)) 435 return true; 436 437 #ifdef PMAP_FAULTINFO 438 struct pcb * const pcb = lwp_getpcb(curlwp); 439 struct pcb_faultinfo * const pfi = &pcb->pcb_faultinfo; 440 441 if (p->p_pid == pfi->pfi_lastpid && addr == pfi->pfi_faultaddr) { 442 if (++pfi->pfi_repeats > 4) { 443 tlb_asid_t asid = tlb_get_asid(); 444 pt_entry_t *ptep = pfi->pfi_faultptep; 445 printf("%s: fault #%u (%s) for %#" PRIxVADDR 446 "(%#"PRIxVADDR") at pc %#"PRIxVADDR" curpid=%u/%u " 447 "ptep@%p=%#"PRIxPTE")\n", __func__, 448 pfi->pfi_repeats, cause_name(tf->tf_cause), 449 tval, addr, epc, map->pmap->pm_pai[0].pai_asid, 450 asid, ptep, ptep ? pte_value(*ptep) : 0); 451 if (pfi->pfi_repeats >= 4) { 452 cpu_Debugger(); 453 } else { 454 pfi->pfi_cause = cause; 455 } 456 } 457 } else { 458 pfi->pfi_lastpid = p->p_pid; 459 pfi->pfi_faultaddr = addr; 460 pfi->pfi_repeats = 0; 461 pfi->pfi_faultptep = NULL; 462 pfi->pfi_cause = cause; 463 } 464 #endif /* PMAP_FAULTINFO */ 465 466 const vm_prot_t ftype = get_faulttype(cause); 467 468 if (usertrap_p) { 469 int error = uvm_fault(&p->p_vmspace->vm_map, addr, ftype); 470 if (error) { 471 int signo = SIGSEGV; 472 int code = SEGV_MAPERR; 473 474 switch (error) { 475 case ENOMEM: { 476 struct lwp * const l = curlwp; 477 printf("UVM: pid %d (%s), uid %d killed: " 478 "out of swap\n", 479 l->l_proc->p_pid, l->l_proc->p_comm, 480 l->l_cred ? 481 kauth_cred_geteuid(l->l_cred) : -1); 482 signo = SIGKILL; 483 code = 0; 484 break; 485 } 486 case EACCES: 487 KASSERT(signo == SIGSEGV); 488 code = SEGV_ACCERR; 489 break; 490 case EINVAL: 491 signo = SIGBUS; 492 code = BUS_ADRERR; 493 break; 494 } 495 496 trap_ksi_init(ksi, signo, code, (intptr_t)tval, cause); 497 return false; 498 } 499 uvm_grow(p, addr); 500 501 return true; 502 } 503 504 // Page faults are not allowed while dealing with interrupts 505 if (cpu_intr_p()) 506 return false; 507 508 struct faultbuf * const fb = cpu_disable_onfault(); 509 int error = uvm_fault(map, addr, ftype); 510 cpu_enable_onfault(fb); 511 512 if (error == 0) { 513 if (map != kernel_map) { 514 uvm_grow(p, addr); 515 } 516 return true; 517 } 518 519 if (fb == NULL) { 520 return false; 521 } 522 523 cpu_jump_onfault(tf, fb, error); 524 return true; 525 } 526 527 static bool 528 trap_instruction(struct trapframe *tf, register_t epc, register_t status, 529 register_t cause, register_t tval, bool usertrap_p, ksiginfo_t *ksi) 530 { 531 if (usertrap_p) { 532 if (__SHIFTOUT(tf->tf_sr, SR_FS) == SR_FS_OFF) { 533 fpu_load(); 534 return true; 535 } 536 537 trap_ksi_init(ksi, SIGILL, ILL_ILLOPC, 538 (intptr_t)tval, cause); 539 } 540 return false; 541 } 542 543 static bool 544 trap_misalignment(struct trapframe *tf, register_t epc, register_t status, 545 register_t cause, register_t tval, bool usertrap_p, ksiginfo_t *ksi) 546 { 547 if (usertrap_p) { 548 trap_ksi_init(ksi, SIGBUS, BUS_ADRALN, 549 (intptr_t)tval, cause); 550 } 551 return false; 552 } 553 554 static bool 555 trap_breakpoint(struct trapframe *tf, register_t epc, register_t status, 556 register_t cause, register_t tval, bool usertrap_p, ksiginfo_t *ksi) 557 { 558 if (usertrap_p) { 559 trap_ksi_init(ksi, SIGTRAP, TRAP_BRKPT, 560 (intptr_t)tval, cause); 561 } else { 562 dump_trapframe(tf, printf); 563 #if defined(DDB) 564 kdb_trap(cause, tf); 565 PC_BREAK_ADVANCE(tf); 566 #else 567 panic("%s: unknown kernel trap", __func__); 568 #endif 569 return true; 570 } 571 return false; 572 } 573 574 void 575 cpu_trap(struct trapframe *tf, register_t epc, register_t status, 576 register_t cause, register_t tval) 577 { 578 const register_t code = CAUSE_CODE(cause); 579 const register_t fault_mask = __BIT(code); 580 const intptr_t addr = tval; 581 const bool usertrap_p = (status & SR_SPP) == 0; 582 bool ok = true; 583 ksiginfo_t ksi; 584 585 KASSERT(!CAUSE_INTERRUPT_P(cause)); 586 KASSERT(__SHIFTOUT(tf->tf_sr, SR_SIE) == 0); 587 588 /* We can allow interrupts now */ 589 csr_sstatus_set(SR_SIE); 590 591 if (__predict_true(fault_mask & FAULT_TRAP_MASK)) { 592 #ifndef _LP64 593 #if 0 594 // This fault may be cause the kernel's page table got a new 595 // page table page and this pmap's page table doesn't know 596 // about it. See 597 struct pmap * const pmap = curlwp->l_proc->p_vmspace->vm_map.pmap; 598 if ((intptr_t) addr < 0 599 && pmap != pmap_kernel() 600 && pmap_pdetab_fixup(pmap, addr)) { 601 return; 602 } 603 #endif 604 #endif 605 ok = trap_pagefault(tf, epc, status, cause, addr, 606 usertrap_p, &ksi); 607 } else if (fault_mask & INSTRUCTION_TRAP_MASK) { 608 ok = trap_instruction(tf, epc, status, cause, addr, 609 usertrap_p, &ksi); 610 } else if (fault_mask & SYSCALL_TRAP_MASK) { 611 panic("cpu_exception_handler failure"); 612 } else if (fault_mask & MISALIGNED_TRAP_MASK) { 613 ok = trap_misalignment(tf, epc, status, cause, addr, 614 usertrap_p, &ksi); 615 } else if (fault_mask & BREAKPOINT_TRAP_MASK) { 616 ok = trap_breakpoint(tf, epc, status, cause, addr, 617 usertrap_p, &ksi); 618 } 619 620 if (usertrap_p) { 621 if (!ok) 622 cpu_trapsignal(tf, &ksi); 623 624 userret(curlwp); 625 } else if (!ok) { 626 dump_trapframe(tf, printf); 627 panic("%s: fatal kernel trap", __func__); 628 } 629 /* 630 * Ensure interrupts are disabled in sstatus, and that interrupts 631 * will get enabled on 'sret' for userland. 632 */ 633 KASSERT(__SHIFTOUT(tf->tf_sr, SR_SIE) == 0); 634 KASSERT(__SHIFTOUT(tf->tf_sr, SR_SPIE) != 0 || 635 __SHIFTOUT(tf->tf_sr, SR_SPP) != 0); 636 } 637 638 void 639 cpu_ast(struct trapframe *tf) 640 { 641 struct lwp * const l = curlwp; 642 643 /* 644 * allow to have a chance of context switch just prior to user 645 * exception return. 646 */ 647 #ifdef __HAVE_PREEMPTION 648 kpreempt_disable(); 649 #endif 650 struct cpu_info * const ci = curcpu(); 651 652 ci->ci_data.cpu_ntrap++; 653 654 KDASSERT(ci->ci_cpl == IPL_NONE); 655 #ifdef __HAVE_PREEMPTION 656 kpreempt_enable(); 657 #endif 658 659 if (curlwp->l_pflag & LP_OWEUPC) { 660 curlwp->l_pflag &= ~LP_OWEUPC; 661 ADDUPROF(curlwp); 662 } 663 664 userret(l); 665 } 666 667 668 static int 669 fetch_user_data(const void *uaddr, void *valp, size_t size) 670 { 671 struct faultbuf fb; 672 int error; 673 674 const vaddr_t uva = (vaddr_t)uaddr; 675 if (__predict_false(uva > VM_MAXUSER_ADDRESS - size)) 676 return EFAULT; 677 678 if ((error = cpu_set_onfault(&fb)) != 0) 679 return error; 680 681 csr_sstatus_set(SR_SUM); 682 switch (size) { 683 case 1: 684 *(uint8_t *)valp = *(volatile const uint8_t *)uaddr; 685 break; 686 case 2: 687 *(uint16_t *)valp = *(volatile const uint16_t *)uaddr; 688 break; 689 case 4: 690 *(uint32_t *)valp = *(volatile const uint32_t *)uaddr; 691 break; 692 #ifdef _LP64 693 case 8: 694 *(uint64_t *)valp = *(volatile const uint64_t *)uaddr; 695 break; 696 #endif /* _LP64 */ 697 default: 698 error = EINVAL; 699 } 700 csr_sstatus_clear(SR_SUM); 701 702 cpu_unset_onfault(); 703 704 return error; 705 } 706 707 int 708 _ufetch_8(const uint8_t *uaddr, uint8_t *valp) 709 { 710 return fetch_user_data(uaddr, valp, sizeof(*valp)); 711 } 712 713 int 714 _ufetch_16(const uint16_t *uaddr, uint16_t *valp) 715 { 716 return fetch_user_data(uaddr, valp, sizeof(*valp)); 717 } 718 719 int 720 _ufetch_32(const uint32_t *uaddr, uint32_t *valp) 721 { 722 return fetch_user_data(uaddr, valp, sizeof(*valp)); 723 } 724 725 #ifdef _LP64 726 int 727 _ufetch_64(const uint64_t *uaddr, uint64_t *valp) 728 { 729 return fetch_user_data(uaddr, valp, sizeof(*valp)); 730 } 731 #endif /* _LP64 */ 732 733 static int 734 store_user_data(void *uaddr, const void *valp, size_t size) 735 { 736 struct faultbuf fb; 737 int error; 738 739 const vaddr_t uva = (vaddr_t)uaddr; 740 if (__predict_false(uva > VM_MAXUSER_ADDRESS - size)) 741 return EFAULT; 742 743 if ((error = cpu_set_onfault(&fb)) != 0) 744 return error; 745 746 csr_sstatus_set(SR_SUM); 747 switch (size) { 748 case 1: 749 *(volatile uint8_t *)uaddr = *(const uint8_t *)valp; 750 break; 751 case 2: 752 *(volatile uint16_t *)uaddr = *(const uint8_t *)valp; 753 break; 754 case 4: 755 *(volatile uint32_t *)uaddr = *(const uint32_t *)valp; 756 break; 757 #ifdef _LP64 758 case 8: 759 *(volatile uint64_t *)uaddr = *(const uint64_t *)valp; 760 break; 761 #endif /* _LP64 */ 762 default: 763 error = EINVAL; 764 } 765 csr_sstatus_clear(SR_SUM); 766 767 cpu_unset_onfault(); 768 769 return error; 770 } 771 772 int 773 _ustore_8(uint8_t *uaddr, uint8_t val) 774 { 775 return store_user_data(uaddr, &val, sizeof(val)); 776 } 777 778 int 779 _ustore_16(uint16_t *uaddr, uint16_t val) 780 { 781 return store_user_data(uaddr, &val, sizeof(val)); 782 } 783 784 int 785 _ustore_32(uint32_t *uaddr, uint32_t val) 786 { 787 return store_user_data(uaddr, &val, sizeof(val)); 788 } 789 790 #ifdef _LP64 791 int 792 _ustore_64(uint64_t *uaddr, uint64_t val) 793 { 794 return store_user_data(uaddr, &val, sizeof(val)); 795 } 796 #endif /* _LP64 */ 797
Indexes created Sat Sep 20 22:09:52 GMT 2025