1 /* $NetBSD: trap.c,v 1.103 2026/06/13 20:16:23 rkujawa Exp $ */ 2 3 /* 4 * Copyright 2001 Wasabi Systems, Inc. 5 * All rights reserved. 6 * 7 * Written by Eduardo Horvath and Simon Burge for Wasabi Systems, Inc. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 3. All advertising materials mentioning features or use of this software 18 * must display the following acknowledgement: 19 * This product includes software developed for the NetBSD Project by 20 * Wasabi Systems, Inc. 21 * 4. The name of Wasabi Systems, Inc. may not be used to endorse 22 * or promote products derived from this software without specific prior 23 * written permission. 24 * 25 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND 26 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 27 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 28 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC 29 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 30 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 31 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 32 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 33 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 34 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 35 * POSSIBILITY OF SUCH DAMAGE. 36 */ 37 38 /* 39 * Copyright (C) 1995, 1996 Wolfgang Solfrank. 40 * Copyright (C) 1995, 1996 TooLs GmbH. 41 * All rights reserved. 42 * 43 * Redistribution and use in source and binary forms, with or without 44 * modification, are permitted provided that the following conditions 45 * are met: 46 * 1. Redistributions of source code must retain the above copyright 47 * notice, this list of conditions and the following disclaimer. 48 * 2. Redistributions in binary form must reproduce the above copyright 49 * notice, this list of conditions and the following disclaimer in the 50 * documentation and/or other materials provided with the distribution. 51 * 3. All advertising materials mentioning features or use of this software 52 * must display the following acknowledgement: 53 * This product includes software developed by TooLs GmbH. 54 * 4. The name of TooLs GmbH may not be used to endorse or promote products 55 * derived from this software without specific prior written permission. 56 * 57 * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR 58 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 59 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 60 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 61 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 62 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 63 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 64 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 65 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 66 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 67 */ 68 69 #define __UFETCHSTORE_PRIVATE 70 71 #include <sys/cdefs.h> 72 __KERNEL_RCSID(0, "$NetBSD: trap.c,v 1.103 2026/06/13 20:16:23 rkujawa Exp $"); 73 74 #ifdef _KERNEL_OPT 75 #include "opt_ddb.h" 76 #include "opt_kgdb.h" 77 #include "opt_ppcarch.h" 78 #include "opt_ppcopts.h" 79 #endif 80 81 #include <sys/param.h> 82 #include <sys/cpu.h> 83 #include <sys/kauth.h> 84 #include <sys/kmem.h> 85 #include <sys/proc.h> 86 #include <sys/ptrace.h> 87 #include <sys/reboot.h> 88 #include <sys/syscall.h> 89 #include <sys/sysctl.h> 90 #include <sys/systm.h> 91 92 #if defined(KGDB) 93 #include <sys/kgdb.h> 94 #endif 95 96 #include <uvm/uvm_extern.h> 97 98 #include <dev/cons.h> 99 100 #include <machine/fpu.h> 101 #include <machine/frame.h> 102 #include <machine/pcb.h> 103 #include <machine/psl.h> 104 #include <machine/trap.h> 105 106 #include <powerpc/db_machdep.h> 107 #include <powerpc/spr.h> 108 #include <powerpc/userret.h> 109 110 #include <powerpc/ibm4xx/cpu.h> 111 #include <powerpc/ibm4xx/pmap.h> 112 #include <powerpc/ibm4xx/spr.h> 113 #include <powerpc/ibm4xx/tlb.h> 114 115 #include <powerpc/fpu/fpu_extern.h> 116 117 /* These definitions should probably be somewhere else XXX */ 118 #define FIRSTARG 3 /* first argument is in reg 3 */ 119 #define NARGREG 8 /* 8 args are in registers */ 120 #define MOREARGS(sp) ((void *)((int)(sp) + 8)) /* more args go here */ 121 122 void trap(struct trapframe *); /* Called from locore / trap_subr */ 123 #if 0 124 /* Not currently used nor exposed externally in any header file */ 125 int badaddr(void *, size_t); 126 int badaddr_read(void *, size_t, int *); 127 #endif 128 int ctx_setup(int, int); 129 130 #ifndef PPC_NO_UNALIGNED 131 static bool fix_unaligned(struct trapframe *, ksiginfo_t *); 132 #endif 133 134 #ifdef DEBUG 135 #define TDB_ALL 0x1 136 int trapdebug = /* TDB_ALL */ 0; 137 #define DBPRINTF(x, y) if (trapdebug & (x)) printf y 138 #else 139 #define DBPRINTF(x, y) 140 #endif 141 142 void 143 trap(struct trapframe *tf) 144 { 145 struct lwp *l = curlwp; 146 struct proc *p = l->l_proc; 147 struct pcb *pcb; 148 int type = tf->tf_exc; 149 int ftype, rv; 150 ksiginfo_t ksi; 151 152 KASSERT(l->l_stat == LSONPROC); 153 154 if (tf->tf_srr1 & PSL_PR) { 155 type |= EXC_USER; 156 } 157 158 ftype = VM_PROT_READ; 159 160 DBPRINTF(TDB_ALL, ("trap(%x) at %lx from frame %p &frame %p\n", 161 type, tf->tf_srr0, tf, &tf)); 162 163 switch (type) { 164 case EXC_DEBUG|EXC_USER: 165 /* We don't use hardware breakpoints for userland. */ 166 goto brain_damage; 167 168 case EXC_TRC|EXC_USER: 169 KSI_INIT_TRAP(&ksi); 170 ksi.ksi_signo = SIGTRAP; 171 ksi.ksi_trap = EXC_TRC; 172 ksi.ksi_addr = (void *)tf->tf_srr0; 173 trapsignal(l, &ksi); 174 break; 175 176 case EXC_DSI: 177 /* FALLTHROUGH */ 178 case EXC_DTMISS: 179 { 180 struct vm_map *map; 181 vaddr_t va; 182 struct faultbuf *fb; 183 184 pcb = lwp_getpcb(l); 185 fb = pcb->pcb_onfault; 186 187 if (curcpu()->ci_idepth >= 0) { 188 rv = EFAULT; 189 goto out; 190 } 191 192 va = tf->tf_dear; 193 if (tf->tf_pid == KERNEL_PID) { 194 map = kernel_map; 195 } else { 196 map = &p->p_vmspace->vm_map; 197 } 198 199 if (tf->tf_esr & (ESR_DST|ESR_DIZ)) 200 ftype = VM_PROT_WRITE; 201 202 DBPRINTF(TDB_ALL, 203 ("trap(EXC_DSI) at %lx %s fault on %p esr %x\n", 204 tf->tf_srr0, 205 (ftype & VM_PROT_WRITE) ? "write" : "read", 206 (void *)va, tf->tf_esr)); 207 208 pcb->pcb_onfault = NULL; 209 rv = uvm_fault(map, trunc_page(va), ftype); 210 pcb->pcb_onfault = fb; 211 if (rv == 0) 212 return; 213 out: 214 if (fb != NULL) { 215 tf->tf_pid = KERNEL_PID; 216 tf->tf_srr0 = fb->fb_pc; 217 tf->tf_srr1 |= PSL_IR; /* Re-enable IMMU */ 218 tf->tf_cr = fb->fb_cr; 219 tf->tf_fixreg[1] = fb->fb_sp; 220 tf->tf_fixreg[2] = fb->fb_r2; 221 tf->tf_fixreg[3] = rv; 222 memcpy(&tf->tf_fixreg[13], fb->fb_fixreg, 223 sizeof(fb->fb_fixreg)); 224 return; 225 } 226 } 227 goto brain_damage; 228 229 case EXC_DSI|EXC_USER: 230 /* FALLTHROUGH */ 231 case EXC_DTMISS|EXC_USER: 232 if (tf->tf_esr & (ESR_DST|ESR_DIZ)) 233 ftype = VM_PROT_WRITE; 234 235 DBPRINTF(TDB_ALL, 236 ("trap(EXC_DSI|EXC_USER) at %lx %s fault on %lx %x\n", 237 tf->tf_srr0, (ftype & VM_PROT_WRITE) ? "write" : "read", 238 tf->tf_dear, tf->tf_esr)); 239 KASSERT(l == curlwp && (l->l_stat == LSONPROC)); 240 // KASSERT(curpcb->pcb_onfault == NULL); 241 rv = uvm_fault(&p->p_vmspace->vm_map, trunc_page(tf->tf_dear), 242 ftype); 243 if (rv == 0) { 244 break; 245 } 246 KSI_INIT_TRAP(&ksi); 247 ksi.ksi_trap = EXC_DSI; 248 ksi.ksi_addr = (void *)tf->tf_dear; 249 vm_signal: 250 switch (rv) { 251 case EINVAL: 252 ksi.ksi_signo = SIGBUS; 253 ksi.ksi_code = BUS_ADRERR; 254 break; 255 case EACCES: 256 ksi.ksi_signo = SIGSEGV; 257 ksi.ksi_code = SEGV_ACCERR; 258 break; 259 case ENOMEM: 260 ksi.ksi_signo = SIGKILL; 261 printf("UVM: pid %d.%d (%s), uid %d killed: " 262 "out of swap\n", p->p_pid, l->l_lid, p->p_comm, 263 l->l_cred ? kauth_cred_geteuid(l->l_cred) : -1); 264 break; 265 default: 266 ksi.ksi_signo = SIGSEGV; 267 ksi.ksi_code = SEGV_MAPERR; 268 break; 269 } 270 trapsignal(l, &ksi); 271 break; 272 273 case EXC_ITMISS|EXC_USER: 274 case EXC_ISI|EXC_USER: 275 ftype = VM_PROT_EXECUTE; 276 DBPRINTF(TDB_ALL, 277 ("trap(EXC_ISI|EXC_USER) at %lx execute fault tf %p\n", 278 tf->tf_srr0, tf)); 279 // KASSERT(curpcb->pcb_onfault == NULL); 280 rv = uvm_fault(&p->p_vmspace->vm_map, trunc_page(tf->tf_srr0), 281 ftype); 282 if (rv == 0) { 283 break; 284 } 285 isi: 286 KSI_INIT_TRAP(&ksi); 287 ksi.ksi_trap = EXC_ISI; 288 ksi.ksi_addr = (void *)tf->tf_srr0; 289 goto vm_signal; 290 break; 291 292 case EXC_AST|EXC_USER: 293 cpu_ast(l, curcpu()); 294 break; 295 296 case EXC_ALI|EXC_USER: 297 if (fix_unaligned(tf, &ksi)) 298 trapsignal(l, &ksi); 299 break; 300 301 case EXC_PGM|EXC_USER: 302 curcpu()->ci_data.cpu_ntrap++; 303 304 KSI_INIT_TRAP(&ksi); 305 ksi.ksi_trap = EXC_PGM; 306 ksi.ksi_addr = (void *)tf->tf_srr0; 307 308 if (tf->tf_esr & ESR_PTR) { 309 vaddr_t va; 310 sigtrap: 311 va = (vaddr_t)tf->tf_srr0; 312 /* 313 * Restore original instruction and clear BP. 314 */ 315 if (p->p_md.md_ss_addr[0] == va || 316 p->p_md.md_ss_addr[1] == va) { 317 rv = ppc_sstep(l, 0); 318 if (rv != 0) 319 goto vm_signal; 320 ksi.ksi_code = TRAP_TRACE; 321 } else 322 ksi.ksi_code = TRAP_BRKPT; 323 if (p->p_raslist != NULL && 324 ras_lookup(p, (void *)va) != (void *)-1) { 325 tf->tf_srr0 += (ksi.ksi_code == TRAP_TRACE) ? 326 0 : 4; 327 break; 328 } 329 ksi.ksi_signo = SIGTRAP; 330 } else if (tf->tf_esr & ESR_PPR) { 331 uint32_t opcode; 332 333 rv = copyin((void *)tf->tf_srr0, &opcode, 334 sizeof(opcode)); 335 if (rv) 336 goto isi; 337 if (emulate_mxmsr(l, tf, opcode)) { 338 tf->tf_srr0 += 4; 339 break; 340 } 341 342 ksi.ksi_code = ILL_PRVOPC; 343 ksi.ksi_signo = SIGILL; 344 } else { 345 pcb = lwp_getpcb(l); 346 347 if (__predict_false(!fpu_used_p(l))) { 348 memset(&pcb->pcb_fpu, 0, sizeof(pcb->pcb_fpu)); 349 fpu_mark_used(l); 350 } 351 352 if (fpu_emulate(tf, &pcb->pcb_fpu, &ksi)) { 353 if (ksi.ksi_signo == 0) /* was emulated */ 354 break; 355 else if (ksi.ksi_signo == SIGTRAP) 356 goto sigtrap; /* XXX H/W bug? */ 357 } else { 358 ksi.ksi_code = ILL_ILLOPC; 359 ksi.ksi_signo = SIGILL; 360 } 361 } 362 363 trapsignal(l, &ksi); 364 break; 365 366 #ifdef PPC_IBM440 367 case EXC_FPU|EXC_USER: 368 curcpu()->ci_data.cpu_ntrap++; 369 #ifdef PPC_HAVE_FPU 370 fpu_load(); 371 #else 372 /* FPU-less 440: emulate like 40x */ 373 KSI_INIT_TRAP(&ksi); 374 ksi.ksi_trap = EXC_FPU; 375 ksi.ksi_addr = (void *)tf->tf_srr0; 376 377 pcb = lwp_getpcb(l); 378 379 if (__predict_false(!fpu_used_p(l))) { 380 memset(&pcb->pcb_fpu, 0, sizeof(pcb->pcb_fpu)); 381 fpu_mark_used(l); 382 } 383 384 if (fpu_emulate(tf, &pcb->pcb_fpu, &ksi)) { 385 if (ksi.ksi_signo == 0) /* was emulated */ 386 break; 387 else if (ksi.ksi_signo == SIGTRAP) 388 goto sigtrap; /* XXX H/W bug? */ 389 } else { 390 ksi.ksi_code = ILL_ILLOPC; 391 ksi.ksi_signo = SIGILL; 392 } 393 394 trapsignal(l, &ksi); 395 #endif /* PPC_HAVE_FPU */ 396 break; 397 #endif /* PPC_IBM440 */ 398 399 case EXC_MCHK: 400 { 401 struct faultbuf *fb; 402 403 pcb = lwp_getpcb(l); 404 if ((fb = pcb->pcb_onfault) != NULL) { 405 tf->tf_pid = KERNEL_PID; 406 tf->tf_srr0 = fb->fb_pc; 407 tf->tf_srr1 |= PSL_IR; /* Re-enable IMMU */ 408 tf->tf_fixreg[1] = fb->fb_sp; 409 tf->tf_fixreg[2] = fb->fb_r2; 410 tf->tf_fixreg[3] = 1; /* Return TRUE */ 411 tf->tf_cr = fb->fb_cr; 412 memcpy(&tf->tf_fixreg[13], fb->fb_fixreg, 413 sizeof(fb->fb_fixreg)); 414 return; 415 } 416 } 417 #ifdef PPC_IBM440 418 /* 419 * Unrecovered machine check, do what we can. 420 */ 421 { 422 const uint32_t mcsr = mfspr(SPR_MCSR); 423 424 printf("machine check: MCSR 0x%08x MCSRR0 0x%08lx " 425 "MCSRR1 0x%08lx\n", mcsr, 426 (u_long)mfspr(SPR_MCSRR0), 427 (u_long)mfspr(SPR_MCSRR1)); 428 printf("machine check cause:%s%s%s%s%s%s%s%s%s%s\n", 429 (mcsr & MCSR_MCS) ? " summary" : "", 430 (mcsr & MCSR_IB) ? " insn-PLB" : "", 431 (mcsr & MCSR_DRB) ? " data-read-PLB" : "", 432 (mcsr & MCSR_DWB) ? " data-write-PLB" : "", 433 (mcsr & MCSR_TLBP) ? " TLB-parity" : "", 434 (mcsr & MCSR_ICP) ? " Icache-parity" : "", 435 (mcsr & MCSR_DCSP) ? " Dcache-search-parity": "", 436 (mcsr & MCSR_DCFP) ? " Dcache-flush-parity" : "", 437 (mcsr & MCSR_IMPE) ? " imprecise" : "", 438 (mcsr == 0) ? " none" : ""); 439 /* MCSR is write-1-to-clear; clear what we read. */ 440 mtspr(SPR_MCSR, mcsr); 441 } 442 #endif /* PPC_IBM440 */ 443 goto brain_damage; 444 445 default: 446 brain_damage: 447 printf("trap type 0x%x at 0x%lx\n", type, tf->tf_srr0); 448 #if defined(DDB) || defined(KGDB) 449 if (kdb_trap(type, tf)) 450 return; 451 #endif 452 #ifdef TRAP_PANICWAIT 453 printf("Press a key to panic.\n"); 454 cngetc(); 455 #endif 456 panic("trap"); 457 } 458 459 /* Invoke powerpc userret code */ 460 userret(l, tf); 461 } 462 463 int 464 ctx_setup(int ctx, int srr1) 465 { 466 volatile struct pmap *pm; 467 468 /* Update PID if we're returning to user mode. */ 469 if (srr1 & PSL_PR) { 470 pm = curproc->p_vmspace->vm_map.pmap; 471 if (!pm->pm_ctx) { 472 ctx_alloc(__UNVOLATILE(pm)); 473 } 474 ctx = pm->pm_ctx; 475 } 476 else if (!ctx) { 477 ctx = KERNEL_PID; 478 } 479 return (ctx); 480 } 481 482 /* 483 * Used by copyin()/copyout() 484 */ 485 extern vaddr_t vmaprange(struct proc *, vaddr_t, vsize_t, int); 486 extern void vunmaprange(vaddr_t, vsize_t); 487 static int bigcopyin(const void *, void *, size_t ); 488 static int bigcopyout(const void *, void *, size_t ); 489 490 #ifdef PPC_IBM440 491 /* 492 * Optimize if copy is larger than vslock+vmaprange+blkcpy path 493 */ 494 int ibm4xx_copy_bigthresh = 8192; 495 #define COPY_BIGTHRESH ((size_t)ibm4xx_copy_bigthresh) 496 497 SYSCTL_SETUP(sysctl_ibm4xx_copy, "ibm4xx copy threshold") 498 { 499 sysctl_createv(clog, 0, NULL, NULL, 500 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 501 CTLTYPE_INT, "copy_bigthresh", 502 SYSCTL_DESCR("copyin/copyout size above which the " 503 "mapped bulk-copy path is used"), 504 NULL, 0, &ibm4xx_copy_bigthresh, 0, 505 CTL_MACHDEP, CTL_CREATE, CTL_EOL); 506 } 507 #else 508 #define COPY_BIGTHRESH 1024 509 #endif 510 511 #ifdef __clang__ 512 #pragma clang optimize off 513 #endif 514 int 515 copyin(const void *uaddr, void *kaddr, size_t len) 516 { 517 struct pmap *pm = curproc->p_vmspace->vm_map.pmap; 518 int rv, msr, pid, tmp, ctx; 519 struct faultbuf env; 520 521 /* For bigger buffers use the faster copy */ 522 if (len > COPY_BIGTHRESH) 523 return (bigcopyin(uaddr, kaddr, len)); 524 525 if ((rv = setfault(&env))) { 526 curpcb->pcb_onfault = NULL; 527 return rv; 528 } 529 530 if (!(ctx = pm->pm_ctx)) { 531 /* No context -- assign it one */ 532 ctx_alloc(pm); 533 ctx = pm->pm_ctx; 534 } 535 536 __asm volatile ( 537 "mfmsr %[msr];" /* Save MSR */ 538 "li %[tmp],0x20;" /* Disable IMMU */ 539 "andc %[tmp],%[msr],%[tmp];" 540 "mtmsr %[tmp];" 541 "isync;" 542 MFPID(%[pid]) /* Save old PID */ 543 544 "srwi. %[tmp],%[len],0x2;" /* How many words? */ 545 "beq- 2f;" /* No words. Go do bytes */ 546 "mtctr %[tmp];" 547 548 "1:" MTPID(%[ctx]) 549 "isync;" 550 #ifdef PPC_IBM403 551 "lswi %[tmp],%[uaddr],4;" /* Load user word */ 552 #else 553 "lwz %[tmp],0(%[uaddr]);" 554 #endif 555 "addi %[uaddr],%[uaddr],0x4;" /* next uaddr word */ 556 "sync;" 557 558 MTPID(%[pid]) 559 "isync;" 560 #ifdef PPC_IBM403 561 "stswi %[tmp],%[kaddr],4;" /* Store kernel word */ 562 #else 563 "stw %[tmp],0(%[kaddr]);" 564 #endif 565 "addi %[kaddr],%[kaddr],0x4;" /* next kaddr word */ 566 "sync;" 567 "bdnz 1b;" /* repeat */ 568 569 "2:" "andi. %[tmp],%[len],0x3;" /* How many remaining bytes? */ 570 "beq 10f;" 571 "mtxer %[tmp];" 572 573 MTPID(%[ctx]) 574 "isync;" 575 "lswx %[tmp],0,%[uaddr];" /* Load user bytes */ 576 "sync;" 577 578 MTPID(%[pid]) 579 "isync;" 580 "stswx %[tmp],0,%[kaddr];" /* Store kernel bytes */ 581 "sync;" 582 583 "10:" "mtmsr %[msr];" /* Restore MSR */ 584 "isync;" 585 586 : [msr] "=&r" (msr), [pid] "=&r" (pid), [tmp] "=&r" (tmp) 587 : [uaddr] "b" (uaddr), [kaddr] "b" (kaddr), 588 [ctx] "r" (ctx), [len] "r" (len) 589 : "cr0", "ctr", "xer"); 590 591 curpcb->pcb_onfault = NULL; 592 return 0; 593 } 594 #ifdef __clang__ 595 #pragma clang optimize on 596 #endif 597 598 static int 599 bigcopyin(const void *uaddr, void *kaddr, size_t len) 600 { 601 const char *up; 602 char *kp = kaddr; 603 struct lwp *l = curlwp; 604 struct proc *p; 605 struct faultbuf env; 606 int error; 607 608 p = l->l_proc; 609 610 /* 611 * Stolen from physio(): 612 */ 613 error = uvm_vslock(p->p_vmspace, __UNCONST(uaddr), len, VM_PROT_READ); 614 if (error) { 615 return error; 616 } 617 up = (char *)vmaprange(p, (vaddr_t)uaddr, len, VM_PROT_READ); 618 619 if ((error = setfault(&env)) == 0) { 620 #ifdef PPC_IBM440 621 ibm4xx_blkcpy(kp, up, len, true); 622 #else 623 memcpy(kp, up, len); 624 #endif 625 } 626 627 curpcb->pcb_onfault = NULL; 628 vunmaprange((vaddr_t)up, len); 629 uvm_vsunlock(p->p_vmspace, __UNCONST(uaddr), len); 630 631 return error; 632 } 633 634 #ifdef __clang__ 635 #pragma clang optimize off 636 #endif 637 int 638 copyout(const void *kaddr, void *uaddr, size_t len) 639 { 640 struct pmap *pm = curproc->p_vmspace->vm_map.pmap; 641 int rv, msr, pid, tmp, ctx; 642 struct faultbuf env; 643 644 /* For big copies use more efficient routine */ 645 if (len > COPY_BIGTHRESH) 646 return (bigcopyout(kaddr, uaddr, len)); 647 648 if ((rv = setfault(&env))) { 649 curpcb->pcb_onfault = NULL; 650 return rv; 651 } 652 653 if (!(ctx = pm->pm_ctx)) { 654 /* No context -- assign it one */ 655 ctx_alloc(pm); 656 ctx = pm->pm_ctx; 657 } 658 659 __asm volatile ( 660 "mfmsr %[msr];" /* Save MSR */ 661 "li %[tmp],0x20;" /* Disable IMMU */ 662 "andc %[tmp],%[msr],%[tmp];" 663 "mtmsr %[tmp];" 664 "isync;" 665 MFPID(%[pid]) /* Save old PID */ 666 667 "srwi. %[tmp],%[len],0x2;" /* How many words? */ 668 "beq- 2f;" /* No words. Go do bytes */ 669 "mtctr %[tmp];" 670 671 "1:" 672 #ifdef PPC_IBM403 673 "lswi %[tmp],%[kaddr],4;" /* Load kernel word */ 674 #else 675 "lwz %[tmp],0(%[kaddr]);" 676 #endif 677 "addi %[kaddr],%[kaddr],0x4;" /* next kaddr word */ 678 "sync;" 679 680 MTPID(%[ctx]) 681 "isync;" 682 #ifdef PPC_IBM403 683 "stswi %[tmp],%[uaddr],4;" /* Store user word */ 684 #else 685 "stw %[tmp],0(%[uaddr]);" 686 #endif 687 "addi %[uaddr],%[uaddr],0x4;" /* next uaddr word */ 688 "sync;" 689 690 MTPID(%[pid]) 691 "isync;" 692 "bdnz 1b;" /* repeat */ 693 694 "2:" "andi. %[tmp],%[len],0x3;" /* How many remaining bytes? */ 695 "beq 10f;" 696 "mtxer %[tmp];" 697 698 "lswx %[tmp],0,%[kaddr];" /* Load kernel bytes */ 699 "sync;" 700 701 MTPID(%[ctx]) 702 "isync;" 703 "stswx %[tmp],0,%[uaddr];" /* Store user bytes */ 704 "sync;" 705 706 MTPID(%[pid]) /* Restore PID and MSR */ 707 "10:" "mtmsr %[msr];" 708 "isync;" 709 710 : [msr] "=&r" (msr), [pid] "=&r" (pid), [tmp] "=&r" (tmp) 711 : [uaddr] "b" (uaddr), [kaddr] "b" (kaddr), 712 [ctx] "r" (ctx), [len] "r" (len) 713 : "cr0", "ctr", "xer"); 714 715 curpcb->pcb_onfault = NULL; 716 return 0; 717 } 718 #ifdef __clang__ 719 #pragma clang optimize on 720 #endif 721 722 static int 723 bigcopyout(const void *kaddr, void *uaddr, size_t len) 724 { 725 char *up; 726 const char *kp = (const char *)kaddr; 727 struct lwp *l = curlwp; 728 struct proc *p; 729 struct faultbuf env; 730 int error; 731 732 p = l->l_proc; 733 734 /* 735 * Stolen from physio(): 736 */ 737 error = uvm_vslock(p->p_vmspace, uaddr, len, VM_PROT_WRITE); 738 if (error) { 739 return error; 740 } 741 up = (char *)vmaprange(p, (vaddr_t)uaddr, len, 742 VM_PROT_READ | VM_PROT_WRITE); 743 744 if ((error = setfault(&env)) == 0) { 745 #ifdef PPC_IBM440 746 ibm4xx_blkcpy(up, kp, len, true); 747 #else 748 memcpy(up, kp, len); 749 #endif 750 } 751 752 curpcb->pcb_onfault = NULL; 753 vunmaprange((vaddr_t)up, len); 754 uvm_vsunlock(p->p_vmspace, uaddr, len); 755 756 return error; 757 } 758 759 #ifdef PPC_IBM440 760 /* 761 * Bulk copy tuned for the 440/460EX 762 */ 763 #ifndef PPC_4XX_NOCACHE 764 #define __BLKCPY_LOOP(ZERO) \ 765 __asm volatile( \ 766 "mtctr %[n];" \ 767 "1: dcbt %[s],%[pf];" \ 768 ZERO \ 769 "lwz %[t0],0(%[s]);" \ 770 "lwz %[t1],4(%[s]);" \ 771 "lwz %[t2],8(%[s]);" \ 772 "lwz %[t3],12(%[s]);" \ 773 "lwz %[t4],16(%[s]);" \ 774 "lwz %[t5],20(%[s]);" \ 775 "lwz %[t6],24(%[s]);" \ 776 "lwz %[t7],28(%[s]);" \ 777 "stw %[t0],0(%[d]);" \ 778 "stw %[t1],4(%[d]);" \ 779 "stw %[t2],8(%[d]);" \ 780 "stw %[t3],12(%[d]);" \ 781 "stw %[t4],16(%[d]);" \ 782 "stw %[t5],20(%[d]);" \ 783 "stw %[t6],24(%[d]);" \ 784 "stw %[t7],28(%[d]);" \ 785 "addi %[s],%[s],32;" \ 786 "addi %[d],%[d],32;" \ 787 "bdnz 1b;" \ 788 : [s] "+b" (s), [d] "+b" (d), \ 789 [t0] "=&r" (t0), [t1] "=&r" (t1), \ 790 [t2] "=&r" (t2), [t3] "=&r" (t3), \ 791 [t4] "=&r" (t4), [t5] "=&r" (t5), \ 792 [t6] "=&r" (t6), [t7] "=&r" (t7) \ 793 : [n] "r" (nlines), [pf] "b" (3 * CACHELINESIZE) \ 794 : "ctr", "memory") 795 #endif /* !PPC_4XX_NOCACHE */ 796 797 void 798 ibm4xx_blkcpy(void *dst, const void *src, size_t len, bool dcbz_ok) 799 { 800 #ifdef PPC_4XX_NOCACHE 801 /* All RAM is mapped caching-inhibited: dcbz would trap. */ 802 memcpy(dst, src, len); 803 #else 804 uint8_t *d = dst; 805 const uint8_t *s = src; 806 size_t head, nlines, tail; 807 808 /* 809 * below three cache lines the setup and prefetch buy nothing? 810 */ 811 if (len < 3 * CACHELINESIZE || 812 (((uintptr_t)d ^ (uintptr_t)s) & 3) != 0) { 813 memcpy(dst, src, len); 814 return; 815 } 816 817 head = (uintptr_t)(-(intptr_t)(uintptr_t)d) & (CACHELINESIZE - 1); 818 if (head != 0) { 819 memcpy(d, s, head); 820 d += head; 821 s += head; 822 len -= head; 823 } 824 nlines = len / CACHELINESIZE; 825 tail = len & (CACHELINESIZE - 1); 826 827 if (nlines != 0) { 828 uint32_t t0, t1, t2, t3, t4, t5, t6, t7; 829 830 if (dcbz_ok) 831 __BLKCPY_LOOP("dcbz 0,%[d];"); 832 else 833 __BLKCPY_LOOP(""); 834 } 835 836 if (tail != 0) 837 memcpy(d, s, tail); 838 #endif /* PPC_4XX_NOCACHE */ 839 } 840 841 #ifdef BLKCPY_SELFTEST 842 /* 843 * This is supposed to verify correctness of the above 440/460 blkcpy 844 */ 845 #define BLKCPY_TESTLEN 8192 846 #define BLKCPY_GUARD 64 847 848 void 849 ibm4xx_blkcpy_selftest(void) 850 { 851 static const size_t lens[] = { 852 0, 1, 3, 31, 32, 33, 63, 64, 65, 95, 96, 97, 255, 853 1023, 1024, 1025, 4095, 4096, 8000 854 }; 855 uint8_t *src, *dstbuf, *ref, *dst; 856 size_t i, len, off; 857 int soff, doff, z, cases = 0; 858 859 src = kmem_alloc(BLKCPY_TESTLEN + 8, KM_SLEEP); 860 ref = kmem_alloc(BLKCPY_TESTLEN + 8, KM_SLEEP); 861 dstbuf = kmem_alloc(BLKCPY_TESTLEN + 8 + 2 * BLKCPY_GUARD, 862 KM_SLEEP); 863 dst = dstbuf + BLKCPY_GUARD; 864 865 for (i = 0; i < BLKCPY_TESTLEN + 8; i++) 866 src[i] = (i * 251 + 13) & 0xff; 867 868 for (i = 0; i < __arraycount(lens); i++) { 869 len = lens[i]; 870 for (soff = 0; soff < 8; soff++) { 871 for (doff = 0; doff < 8; doff++) { 872 for (z = 0; z < 2; z++) { 873 memset(dstbuf, 0xa5, 874 BLKCPY_TESTLEN + 8 + 875 2 * BLKCPY_GUARD); 876 memset(ref, 0xa5, 877 BLKCPY_TESTLEN + 8); 878 memcpy(ref + doff, src + soff, len); 879 ibm4xx_blkcpy(dst + doff, src + soff, 880 len, z != 0); 881 if (memcmp(dst + doff, ref + doff, 882 len) != 0) 883 panic("blkcpy selftest: " 884 "payload len=%zu soff=%d " 885 "doff=%d dcbz=%d", 886 len, soff, doff, z); 887 for (off = 0; 888 off < BLKCPY_TESTLEN + 8 + 889 2 * BLKCPY_GUARD; off++) { 890 if (off >= BLKCPY_GUARD + doff 891 && off < BLKCPY_GUARD + 892 doff + len) 893 continue; 894 if (dstbuf[off] != 0xa5) 895 panic("blkcpy " 896 "selftest: guard " 897 "len=%zu soff=%d " 898 "doff=%d dcbz=%d " 899 "off=%zu", 900 len, soff, doff, 901 z, off); 902 } 903 cases++; 904 } 905 } 906 } 907 } 908 909 kmem_free(src, BLKCPY_TESTLEN + 8); 910 kmem_free(ref, BLKCPY_TESTLEN + 8); 911 kmem_free(dstbuf, BLKCPY_TESTLEN + 8 + 2 * BLKCPY_GUARD); 912 913 printf("ibm4xx_blkcpy: self-test passed (%d cases)\n", cases); 914 } 915 #endif /* BLKCPY_SELFTEST */ 916 #endif /* PPC_IBM440 */ 917 918 /* 919 * kcopy(const void *src, void *dst, size_t len); 920 * 921 * Copy len bytes from src to dst, aborting if we encounter a fatal 922 * page fault. 923 * 924 * kcopy() _must_ save and restore the old fault handler since it is 925 * called by uiomove(), which may be in the path of servicing a non-fatal 926 * page fault. 927 */ 928 int 929 kcopy(const void *src, void *dst, size_t len) 930 { 931 struct faultbuf env, *oldfault; 932 int rv; 933 934 oldfault = curpcb->pcb_onfault; 935 if ((rv = setfault(&env))) { 936 curpcb->pcb_onfault = oldfault; 937 return rv; 938 } 939 940 #ifdef PPC_IBM440 941 ibm4xx_blkcpy(dst, src, len, false); 942 #else 943 memcpy(dst, src, len); 944 #endif 945 946 curpcb->pcb_onfault = oldfault; 947 return 0; 948 } 949 950 #if 0 951 int 952 badaddr(void *addr, size_t size) 953 { 954 955 return badaddr_read(addr, size, NULL); 956 } 957 958 int 959 badaddr_read(void *addr, size_t size, int *rptr) 960 { 961 struct faultbuf env; 962 int x; 963 964 /* Get rid of any stale machine checks that have been waiting. */ 965 __asm volatile ("sync; isync"); 966 967 if (setfault(&env)) { 968 curpcb->pcb_onfault = NULL; 969 __asm volatile ("sync"); 970 return 1; 971 } 972 973 __asm volatile ("sync"); 974 975 switch (size) { 976 case 1: 977 x = *(volatile int8_t *)addr; 978 break; 979 case 2: 980 x = *(volatile int16_t *)addr; 981 break; 982 case 4: 983 x = *(volatile int32_t *)addr; 984 break; 985 default: 986 panic("badaddr: invalid size (%d)", size); 987 } 988 989 /* Make sure we took the machine check, if we caused one. */ 990 __asm volatile ("sync; isync"); 991 992 curpcb->pcb_onfault = NULL; 993 __asm volatile ("sync"); /* To be sure. */ 994 995 /* Use the value to avoid reorder. */ 996 if (rptr) 997 *rptr = x; 998 999 return 0; 1000 } 1001 #endif 1002 1003 #ifndef PPC_NO_UNALIGNED 1004 static bool 1005 fix_unaligned(struct trapframe *tf, ksiginfo_t *ksi) 1006 { 1007 1008 KSI_INIT_TRAP(ksi); 1009 ksi->ksi_signo = SIGBUS; 1010 ksi->ksi_trap = EXC_ALI; 1011 ksi->ksi_addr = (void *)tf->tf_dear; 1012 return true; 1013 } 1014 #endif 1015 1016 /* 1017 * XXX Extremely lame implementations of _ufetch_* / _ustore_*. IBM 4xx 1018 * experts should make versions that are good. 1019 */ 1020 1021 #define UFETCH(sz) \ 1022 int \ 1023 _ufetch_ ## sz(const uint ## sz ## _t *uaddr, uint ## sz ## _t *valp) \ 1024 { \ 1025 return copyin(uaddr, valp, sizeof(*valp)); \ 1026 } 1027 1028 UFETCH(8) 1029 UFETCH(16) 1030 UFETCH(32) 1031 1032 #define USTORE(sz) \ 1033 int \ 1034 _ustore_ ## sz(uint ## sz ## _t *uaddr, uint ## sz ## _t val) \ 1035 { \ 1036 return copyout(&val, uaddr, sizeof(val)); \ 1037 } 1038 1039 USTORE(8) 1040 USTORE(16) 1041 USTORE(32) 1042
Indexes created Tue Jul 07 00:25:39 UTC 2026