Home | History | Annotate | Line # | Download | only in arm32
fault.c revision 1.97
      1 /*	$NetBSD: fault.c,v 1.97 2014/03/05 02:10:39 matt Exp $	*/
      2 
      3 /*
      4  * Copyright 2003 Wasabi Systems, Inc.
      5  * All rights reserved.
      6  *
      7  * Written by Steve C. Woodford for Wasabi Systems, Inc.
      8  *
      9  * Redistribution and use in source and binary forms, with or without
     10  * modification, are permitted provided that the following conditions
     11  * are met:
     12  * 1. Redistributions of source code must retain the above copyright
     13  *    notice, this list of conditions and the following disclaimer.
     14  * 2. Redistributions in binary form must reproduce the above copyright
     15  *    notice, this list of conditions and the following disclaimer in the
     16  *    documentation and/or other materials provided with the distribution.
     17  * 3. All advertising materials mentioning features or use of this software
     18  *    must display the following acknowledgement:
     19  *      This product includes software developed for the NetBSD Project by
     20  *      Wasabi Systems, Inc.
     21  * 4. The name of Wasabi Systems, Inc. may not be used to endorse
     22  *    or promote products derived from this software without specific prior
     23  *    written permission.
     24  *
     25  * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND
     26  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     27  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     28  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL WASABI SYSTEMS, INC
     29  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     30  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     31  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     32  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     33  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     34  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     35  * POSSIBILITY OF SUCH DAMAGE.
     36  */
     37 /*
     38  * Copyright (c) 1994-1997 Mark Brinicombe.
     39  * Copyright (c) 1994 Brini.
     40  * All rights reserved.
     41  *
     42  * This code is derived from software written for Brini by Mark Brinicombe
     43  *
     44  * Redistribution and use in source and binary forms, with or without
     45  * modification, are permitted provided that the following conditions
     46  * are met:
     47  * 1. Redistributions of source code must retain the above copyright
     48  *    notice, this list of conditions and the following disclaimer.
     49  * 2. Redistributions in binary form must reproduce the above copyright
     50  *    notice, this list of conditions and the following disclaimer in the
     51  *    documentation and/or other materials provided with the distribution.
     52  * 3. All advertising materials mentioning features or use of this software
     53  *    must display the following acknowledgement:
     54  *	This product includes software developed by Brini.
     55  * 4. The name of the company nor the name of the author may be used to
     56  *    endorse or promote products derived from this software without specific
     57  *    prior written permission.
     58  *
     59  * THIS SOFTWARE IS PROVIDED BY BRINI ``AS IS'' AND ANY EXPRESS OR IMPLIED
     60  * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
     61  * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
     62  * IN NO EVENT SHALL BRINI OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
     63  * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
     64  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
     65  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
     66  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
     67  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
     68  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
     69  * SUCH DAMAGE.
     70  *
     71  * RiscBSD kernel project
     72  *
     73  * fault.c
     74  *
     75  * Fault handlers
     76  *
     77  * Created      : 28/11/94
     78  */
     79 
     80 #include "opt_ddb.h"
     81 #include "opt_kgdb.h"
     82 
     83 #include <sys/types.h>
     84 __KERNEL_RCSID(0, "$NetBSD: fault.c,v 1.97 2014/03/05 02:10:39 matt Exp $");
     85 
     86 #include <sys/param.h>
     87 #include <sys/systm.h>
     88 #include <sys/proc.h>
     89 #include <sys/kernel.h>
     90 #include <sys/kauth.h>
     91 #include <sys/cpu.h>
     92 #include <sys/intr.h>
     93 
     94 #include <uvm/uvm_extern.h>
     95 #include <uvm/uvm_stat.h>
     96 #ifdef UVMHIST
     97 #include <uvm/uvm.h>
     98 #endif
     99 
    100 #include <arm/locore.h>
    101 
    102 #include <arm/arm32/katelib.h>
    103 
    104 #include <machine/pcb.h>
    105 #if defined(DDB) || defined(KGDB)
    106 #include <machine/db_machdep.h>
    107 #ifdef KGDB
    108 #include <sys/kgdb.h>
    109 #endif
    110 #if !defined(DDB)
    111 #define kdb_trap	kgdb_trap
    112 #endif
    113 #endif
    114 
    115 #include <arch/arm/arm/disassem.h>
    116 #include <arm/arm32/machdep.h>
    117 
    118 extern char fusubailout[];
    119 
    120 #ifdef DEBUG
    121 int last_fault_code;	/* For the benefit of pmap_fault_fixup() */
    122 #endif
    123 
    124 #if defined(CPU_ARM3) || defined(CPU_ARM6) || \
    125     defined(CPU_ARM7) || defined(CPU_ARM7TDMI)
    126 /* These CPUs may need data/prefetch abort fixups */
    127 #define	CPU_ABORT_FIXUP_REQUIRED
    128 #endif
    129 
    130 struct data_abort {
    131 	int (*func)(trapframe_t *, u_int, u_int, struct lwp *, ksiginfo_t *);
    132 	const char *desc;
    133 };
    134 
    135 static int dab_fatal(trapframe_t *, u_int, u_int, struct lwp *, ksiginfo_t *);
    136 static int dab_align(trapframe_t *, u_int, u_int, struct lwp *, ksiginfo_t *);
    137 static int dab_buserr(trapframe_t *, u_int, u_int, struct lwp *, ksiginfo_t *);
    138 
    139 static const struct data_abort data_aborts[] = {
    140 	{dab_fatal,	"Vector Exception"},
    141 	{dab_align,	"Alignment Fault 1"},
    142 	{dab_fatal,	"Terminal Exception"},
    143 	{dab_align,	"Alignment Fault 3"},
    144 	{dab_buserr,	"External Linefetch Abort (S)"},
    145 	{NULL,		"Translation Fault (S)"},
    146 	{dab_buserr,	"External Linefetch Abort (P)"},
    147 	{NULL,		"Translation Fault (P)"},
    148 	{dab_buserr,	"External Non-Linefetch Abort (S)"},
    149 	{NULL,		"Domain Fault (S)"},
    150 	{dab_buserr,	"External Non-Linefetch Abort (P)"},
    151 	{NULL,		"Domain Fault (P)"},
    152 	{dab_buserr,	"External Translation Abort (L1)"},
    153 	{NULL,		"Permission Fault (S)"},
    154 	{dab_buserr,	"External Translation Abort (L2)"},
    155 	{NULL,		"Permission Fault (P)"}
    156 };
    157 
    158 /* Determine if 'x' is a permission fault */
    159 #define	IS_PERMISSION_FAULT(x)					\
    160 	(((1 << ((x) & FAULT_TYPE_MASK)) &			\
    161 	  ((1 << FAULT_PERM_P) | (1 << FAULT_PERM_S))) != 0)
    162 
    163 #if 0
    164 /* maybe one day we'll do emulations */
    165 #define	TRAPSIGNAL(l,k)	(*(l)->l_proc->p_emul->e_trapsignal)((l), (k))
    166 #else
    167 #define	TRAPSIGNAL(l,k)	trapsignal((l), (k))
    168 #endif
    169 
    170 static inline void
    171 call_trapsignal(struct lwp *l, const struct trapframe *tf, ksiginfo_t *ksi)
    172 {
    173 	if (l->l_proc->p_pid == 1 || cpu_printfataltraps) {
    174 		printf("%d.%d(%s): trap: signo=%d code=%d addr=%p trap=%#x\n",
    175 		    l->l_proc->p_pid, l->l_lid, l->l_proc->p_comm,
    176 		    ksi->ksi_signo, ksi->ksi_code, ksi->ksi_addr,
    177 		    ksi->ksi_trap);
    178 		printf("r0=%08x r1=%08x r2=%08x r3=%08x\n",
    179 		    tf->tf_r0, tf->tf_r1, tf->tf_r2, tf->tf_r3);
    180 		printf("r4=%08x r5=%08x r6=%08x r7=%08x\n",
    181 		    tf->tf_r4, tf->tf_r5, tf->tf_r6, tf->tf_r7);
    182 		printf("r8=%08x r9=%08x rA=%08x rB=%08x\n",
    183 		    tf->tf_r8, tf->tf_r9, tf->tf_r10, tf->tf_r11);
    184 		printf("ip=%08x sp=%08x lr=%08x pc=%08x spsr=%08x\n",
    185 		    tf->tf_r12, tf->tf_usr_sp, tf->tf_usr_lr, tf->tf_pc,
    186 		    tf->tf_spsr);
    187 	}
    188 
    189 	TRAPSIGNAL(l, ksi);
    190 }
    191 
    192 static inline int
    193 data_abort_fixup(trapframe_t *tf, u_int fsr, u_int far, struct lwp *l)
    194 {
    195 #ifdef CPU_ABORT_FIXUP_REQUIRED
    196 	int error;
    197 
    198 	/* Call the CPU specific data abort fixup routine */
    199 	error = cpu_dataabt_fixup(tf);
    200 	if (__predict_true(error != ABORT_FIXUP_FAILED))
    201 		return (error);
    202 
    203 	/*
    204 	 * Oops, couldn't fix up the instruction
    205 	 */
    206 	printf("%s: fixup for %s mode data abort failed.\n", __func__,
    207 	    TRAP_USERMODE(tf) ? "user" : "kernel");
    208 #ifdef THUMB_CODE
    209 	if (tf->tf_spsr & PSR_T_bit) {
    210 		printf("pc = 0x%08x, opcode 0x%04x, 0x%04x, insn = ",
    211 		    tf->tf_pc, *((uint16 *)(tf->tf_pc & ~1)),
    212 		    *((uint16 *)((tf->tf_pc + 2) & ~1)));
    213 	}
    214 	else
    215 #endif
    216 	{
    217 		printf("pc = 0x%08x, opcode 0x%08x, insn = ", tf->tf_pc,
    218 		    *((u_int *)tf->tf_pc));
    219 	}
    220 	disassemble(tf->tf_pc);
    221 
    222 	/* Die now if this happened in kernel mode */
    223 	if (!TRAP_USERMODE(tf))
    224 		dab_fatal(tf, fsr, far, l, NULL);
    225 
    226 	return (error);
    227 #else
    228 	return (ABORT_FIXUP_OK);
    229 #endif /* CPU_ABORT_FIXUP_REQUIRED */
    230 }
    231 
    232 void
    233 data_abort_handler(trapframe_t *tf)
    234 {
    235 	struct vm_map *map;
    236 	struct lwp * const l = curlwp;
    237 	struct cpu_info * const ci = curcpu();
    238 	u_int far, fsr;
    239 	vm_prot_t ftype;
    240 	void *onfault;
    241 	vaddr_t va;
    242 	int error;
    243 	ksiginfo_t ksi;
    244 
    245 	UVMHIST_FUNC(__func__);
    246 #ifdef UVMHIST
    247 	if (__predict_true(maphist.e)) {
    248 		UVMHIST_CALLED(maphist);
    249 	}
    250 #endif
    251 
    252 	/* Grab FAR/FSR before enabling interrupts */
    253 	far = cpu_faultaddress();
    254 	fsr = cpu_faultstatus();
    255 
    256 	/* Update vmmeter statistics */
    257 	ci->ci_data.cpu_ntrap++;
    258 
    259 	/* Re-enable interrupts if they were enabled previously */
    260 	KASSERT(!TRAP_USERMODE(tf) || (tf->tf_spsr & IF32_bits) == 0);
    261 	if (__predict_true((tf->tf_spsr & IF32_bits) != IF32_bits))
    262 		restore_interrupts(tf->tf_spsr & IF32_bits);
    263 
    264 	/* Get the current lwp structure */
    265 
    266 #ifdef UVMHIST
    267 	if (__predict_true(maphist.e)) {
    268 		UVMHIST_LOG(maphist, " (l=%#x, far=%#x, fsr=%#x",
    269 		    l, far, fsr, 0);
    270 		UVMHIST_LOG(maphist, "  tf=%#x, pc=%#x)",
    271 		    tf, tf->tf_pc, 0, 0);
    272 	}
    273 #endif
    274 
    275 	/* Data abort came from user mode? */
    276 	bool user = (TRAP_USERMODE(tf) != 0);
    277 	if (user)
    278 		LWP_CACHE_CREDS(l, l->l_proc);
    279 
    280 	/* Grab the current pcb */
    281 	struct pcb * const pcb = lwp_getpcb(l);
    282 
    283 	curcpu()->ci_abt_evs[fsr & FAULT_TYPE_MASK].ev_count++;
    284 
    285 	/* Invoke the appropriate handler, if necessary */
    286 	if (__predict_false(data_aborts[fsr & FAULT_TYPE_MASK].func != NULL)) {
    287 #ifdef DIAGNOSTIC
    288 		printf("%s: data_aborts fsr=0x%x far=0x%x\n",
    289 		    __func__, fsr, far);
    290 #endif
    291 		if ((data_aborts[fsr & FAULT_TYPE_MASK].func)(tf, fsr, far,
    292 		    l, &ksi))
    293 			goto do_trapsignal;
    294 		goto out;
    295 	}
    296 
    297 	/*
    298 	 * At this point, we're dealing with one of the following data aborts:
    299 	 *
    300 	 *  FAULT_TRANS_S  - Translation -- Section
    301 	 *  FAULT_TRANS_P  - Translation -- Page
    302 	 *  FAULT_DOMAIN_S - Domain -- Section
    303 	 *  FAULT_DOMAIN_P - Domain -- Page
    304 	 *  FAULT_PERM_S   - Permission -- Section
    305 	 *  FAULT_PERM_P   - Permission -- Page
    306 	 *
    307 	 * These are the main virtual memory-related faults signalled by
    308 	 * the MMU.
    309 	 */
    310 
    311 	/* fusubailout is used by [fs]uswintr to avoid page faulting */
    312 	if (__predict_false(pcb->pcb_onfault == fusubailout)) {
    313 		tf->tf_r0 = EFAULT;
    314 		tf->tf_pc = (intptr_t) pcb->pcb_onfault;
    315 		return;
    316 	}
    317 
    318 	if (user) {
    319 		lwp_settrapframe(l, tf);
    320 	}
    321 
    322 	/*
    323 	 * Make sure the Program Counter is sane. We could fall foul of
    324 	 * someone executing Thumb code, in which case the PC might not
    325 	 * be word-aligned. This would cause a kernel alignment fault
    326 	 * further down if we have to decode the current instruction.
    327 	 */
    328 #ifdef THUMB_CODE
    329 	/*
    330 	 * XXX: It would be nice to be able to support Thumb in the kernel
    331 	 * at some point.
    332 	 */
    333 	if (__predict_false(!user && (tf->tf_pc & 3) != 0)) {
    334 		printf("\n%s: Misaligned Kernel-mode Program Counter\n",
    335 		    __func__);
    336 		dab_fatal(tf, fsr, far, l, NULL);
    337 	}
    338 #else
    339 	if (__predict_false((tf->tf_pc & 3) != 0)) {
    340 		if (user) {
    341 			/*
    342 			 * Give the user an illegal instruction signal.
    343 			 */
    344 			/* Deliver a SIGILL to the process */
    345 			KSI_INIT_TRAP(&ksi);
    346 			ksi.ksi_signo = SIGILL;
    347 			ksi.ksi_code = ILL_ILLOPC;
    348 			ksi.ksi_addr = (uint32_t *)(intptr_t) far;
    349 			ksi.ksi_trap = fsr;
    350 			goto do_trapsignal;
    351 		}
    352 
    353 		/*
    354 		 * The kernel never executes Thumb code.
    355 		 */
    356 		printf("\n%s: Misaligned Kernel-mode Program Counter\n",
    357 		    __func__);
    358 		dab_fatal(tf, fsr, far, l, NULL);
    359 	}
    360 #endif
    361 
    362 	/* See if the CPU state needs to be fixed up */
    363 	switch (data_abort_fixup(tf, fsr, far, l)) {
    364 	case ABORT_FIXUP_RETURN:
    365 		return;
    366 	case ABORT_FIXUP_FAILED:
    367 		/* Deliver a SIGILL to the process */
    368 		KSI_INIT_TRAP(&ksi);
    369 		ksi.ksi_signo = SIGILL;
    370 		ksi.ksi_code = ILL_ILLOPC;
    371 		ksi.ksi_addr = (uint32_t *)(intptr_t) far;
    372 		ksi.ksi_trap = fsr;
    373 		goto do_trapsignal;
    374 	default:
    375 		break;
    376 	}
    377 
    378 	va = trunc_page((vaddr_t)far);
    379 
    380 	/*
    381 	 * It is only a kernel address space fault iff:
    382 	 *	1. user == 0  and
    383 	 *	2. pcb_onfault not set or
    384 	 *	3. pcb_onfault set and not LDRT/LDRBT/STRT/STRBT instruction.
    385 	 */
    386 	if (!user && (va >= VM_MIN_KERNEL_ADDRESS ||
    387 	    (va < VM_MIN_ADDRESS && vector_page == ARM_VECTORS_LOW)) &&
    388 	    __predict_true((pcb->pcb_onfault == NULL ||
    389 	     (read_insn(tf->tf_pc, false) & 0x05200000) != 0x04200000))) {
    390 		map = kernel_map;
    391 
    392 		/* Was the fault due to the FPE/IPKDB ? */
    393 		if (__predict_false((tf->tf_spsr & PSR_MODE)==PSR_UND32_MODE)) {
    394 			KSI_INIT_TRAP(&ksi);
    395 			ksi.ksi_signo = SIGSEGV;
    396 			ksi.ksi_code = SEGV_ACCERR;
    397 			ksi.ksi_addr = (uint32_t *)(intptr_t) far;
    398 			ksi.ksi_trap = fsr;
    399 
    400 			/*
    401 			 * Force exit via userret()
    402 			 * This is necessary as the FPE is an extension to
    403 			 * userland that actually runs in a priveledged mode
    404 			 * but uses USR mode permissions for its accesses.
    405 			 */
    406 			user = true;
    407 			goto do_trapsignal;
    408 		}
    409 	} else {
    410 		map = &l->l_proc->p_vmspace->vm_map;
    411 	}
    412 
    413 	/*
    414 	 * We need to know whether the page should be mapped as R or R/W.
    415 	 * Before ARMv6, the MMU did not give us the info as to whether the
    416 	 * fault was caused by a read or a write.
    417 	 *
    418 	 * However, we know that a permission fault can only be the result of
    419 	 * a write to a read-only location, so we can deal with those quickly.
    420 	 *
    421 	 * Otherwise we need to disassemble the instruction responsible to
    422 	 * determine if it was a write.
    423 	 */
    424 	if (CPU_IS_ARMV6_P() || CPU_IS_ARMV7_P()) {
    425 		ftype = (fsr & FAULT_WRITE) ? VM_PROT_WRITE : VM_PROT_READ;
    426 	} else if (IS_PERMISSION_FAULT(fsr)) {
    427 		ftype = VM_PROT_WRITE;
    428 	} else {
    429 #ifdef THUMB_CODE
    430 		/* Fast track the ARM case.  */
    431 		if (__predict_false(tf->tf_spsr & PSR_T_bit)) {
    432 			u_int insn = read_thumb_insn(tf->tf_pc, user);
    433 			u_int insn_f8 = insn & 0xf800;
    434 			u_int insn_fe = insn & 0xfe00;
    435 
    436 			if (insn_f8 == 0x6000 || /* STR(1) */
    437 			    insn_f8 == 0x7000 || /* STRB(1) */
    438 			    insn_f8 == 0x8000 || /* STRH(1) */
    439 			    insn_f8 == 0x9000 || /* STR(3) */
    440 			    insn_f8 == 0xc000 || /* STM */
    441 			    insn_fe == 0x5000 || /* STR(2) */
    442 			    insn_fe == 0x5200 || /* STRH(2) */
    443 			    insn_fe == 0x5400)   /* STRB(2) */
    444 				ftype = VM_PROT_WRITE;
    445 			else
    446 				ftype = VM_PROT_READ;
    447 		}
    448 		else
    449 #endif
    450 		{
    451 			u_int insn = read_insn(tf->tf_pc, user);
    452 
    453 			if (((insn & 0x0c100000) == 0x04000000) || /* STR[B] */
    454 			    ((insn & 0x0e1000b0) == 0x000000b0) || /* STR[HD]*/
    455 			    ((insn & 0x0a100000) == 0x08000000) || /* STM/CDT*/
    456 			    ((insn & 0x0f9000f0) == 0x01800090))   /* STREX[BDH] */
    457 				ftype = VM_PROT_WRITE;
    458 			else if ((insn & 0x0fb00ff0) == 0x01000090)/* SWP */
    459 				ftype = VM_PROT_READ | VM_PROT_WRITE;
    460 			else
    461 				ftype = VM_PROT_READ;
    462 		}
    463 	}
    464 
    465 	/*
    466 	 * See if the fault is as a result of ref/mod emulation,
    467 	 * or domain mismatch.
    468 	 */
    469 #ifdef DEBUG
    470 	last_fault_code = fsr;
    471 #endif
    472 	if (pmap_fault_fixup(map->pmap, va, ftype, user)) {
    473 #ifdef UVMHIST
    474 		if (__predict_true(maphist.e)) {
    475 			UVMHIST_LOG(maphist, " <- ref/mod emul", 0, 0, 0, 0);
    476 		}
    477 #endif
    478 		goto out;
    479 	}
    480 
    481 	if (__predict_false(curcpu()->ci_intr_depth > 0)) {
    482 		if (pcb->pcb_onfault) {
    483 			tf->tf_r0 = EINVAL;
    484 			tf->tf_pc = (register_t)(intptr_t) pcb->pcb_onfault;
    485 			return;
    486 		}
    487 		printf("\nNon-emulated page fault with intr_depth > 0\n");
    488 		dab_fatal(tf, fsr, far, l, NULL);
    489 	}
    490 
    491 	onfault = pcb->pcb_onfault;
    492 	pcb->pcb_onfault = NULL;
    493 	error = uvm_fault(map, va, ftype);
    494 	pcb->pcb_onfault = onfault;
    495 
    496 	if (__predict_true(error == 0)) {
    497 		if (user)
    498 			uvm_grow(l->l_proc, va); /* Record any stack growth */
    499 		else
    500 			ucas_ras_check(tf);
    501 #ifdef UVMHIST
    502 		if (__predict_true(maphist.e)) {
    503 			UVMHIST_LOG(maphist, " <- uvm", 0, 0, 0, 0);
    504 		}
    505 #endif
    506 		goto out;
    507 	}
    508 
    509 	if (user == 0) {
    510 		if (pcb->pcb_onfault) {
    511 			tf->tf_r0 = error;
    512 			tf->tf_pc = (register_t)(intptr_t) pcb->pcb_onfault;
    513 			return;
    514 		}
    515 
    516 		printf("\nuvm_fault(%p, %lx, %x) -> %x\n", map, va, ftype,
    517 		    error);
    518 		dab_fatal(tf, fsr, far, l, NULL);
    519 	}
    520 
    521 	KSI_INIT_TRAP(&ksi);
    522 
    523 	if (error == ENOMEM) {
    524 		printf("UVM: pid %d (%s), uid %d killed: "
    525 		    "out of swap\n", l->l_proc->p_pid, l->l_proc->p_comm,
    526 		    l->l_cred ? kauth_cred_geteuid(l->l_cred) : -1);
    527 		ksi.ksi_signo = SIGKILL;
    528 	} else
    529 		ksi.ksi_signo = SIGSEGV;
    530 
    531 	ksi.ksi_code = (error == EACCES) ? SEGV_ACCERR : SEGV_MAPERR;
    532 	ksi.ksi_addr = (uint32_t *)(intptr_t) far;
    533 	ksi.ksi_trap = fsr;
    534 #ifdef UVMHIST
    535 	if (__predict_true(maphist.e)) {
    536 		UVMHIST_LOG(maphist, " <- error (%d)", error, 0, 0, 0);
    537 	}
    538 #endif
    539 
    540 do_trapsignal:
    541 	call_trapsignal(l, tf, &ksi);
    542 out:
    543 	/* If returning to user mode, make sure to invoke userret() */
    544 	if (user)
    545 		userret(l);
    546 }
    547 
    548 /*
    549  * dab_fatal() handles the following data aborts:
    550  *
    551  *  FAULT_WRTBUF_0 - Vector Exception
    552  *  FAULT_WRTBUF_1 - Terminal Exception
    553  *
    554  * We should never see these on a properly functioning system.
    555  *
    556  * This function is also called by the other handlers if they
    557  * detect a fatal problem.
    558  *
    559  * Note: If 'l' is NULL, we assume we're dealing with a prefetch abort.
    560  */
    561 static int
    562 dab_fatal(trapframe_t *tf, u_int fsr, u_int far, struct lwp *l, ksiginfo_t *ksi)
    563 {
    564 	const char * const mode = TRAP_USERMODE(tf) ? "user" : "kernel";
    565 
    566 	if (l != NULL) {
    567 		printf("Fatal %s mode data abort: '%s'\n", mode,
    568 		    data_aborts[fsr & FAULT_TYPE_MASK].desc);
    569 		printf("trapframe: %p\nFSR=%08x, FAR=", tf, fsr);
    570 		if ((fsr & FAULT_IMPRECISE) == 0)
    571 			printf("%08x, ", far);
    572 		else
    573 			printf("Invalid,  ");
    574 		printf("spsr=%08x\n", tf->tf_spsr);
    575 	} else {
    576 		printf("Fatal %s mode prefetch abort at 0x%08x\n",
    577 		    mode, tf->tf_pc);
    578 		printf("trapframe: %p, spsr=%08x\n", tf, tf->tf_spsr);
    579 	}
    580 
    581 	printf("r0 =%08x, r1 =%08x, r2 =%08x, r3 =%08x\n",
    582 	    tf->tf_r0, tf->tf_r1, tf->tf_r2, tf->tf_r3);
    583 	printf("r4 =%08x, r5 =%08x, r6 =%08x, r7 =%08x\n",
    584 	    tf->tf_r4, tf->tf_r5, tf->tf_r6, tf->tf_r7);
    585 	printf("r8 =%08x, r9 =%08x, r10=%08x, r11=%08x\n",
    586 	    tf->tf_r8, tf->tf_r9, tf->tf_r10, tf->tf_r11);
    587 	printf("r12=%08x, ", tf->tf_r12);
    588 
    589 	if (TRAP_USERMODE(tf))
    590 		printf("usp=%08x, ulr=%08x",
    591 		    tf->tf_usr_sp, tf->tf_usr_lr);
    592 	else
    593 		printf("ssp=%08x, slr=%08x",
    594 		    tf->tf_svc_sp, tf->tf_svc_lr);
    595 	printf(", pc =%08x\n\n", tf->tf_pc);
    596 
    597 #if defined(DDB) || defined(KGDB)
    598 	kdb_trap(T_FAULT, tf);
    599 #endif
    600 	panic("Fatal abort");
    601 	/*NOTREACHED*/
    602 }
    603 
    604 /*
    605  * dab_align() handles the following data aborts:
    606  *
    607  *  FAULT_ALIGN_0 - Alignment fault
    608  *  FAULT_ALIGN_0 - Alignment fault
    609  *
    610  * These faults are fatal if they happen in kernel mode. Otherwise, we
    611  * deliver a bus error to the process.
    612  */
    613 static int
    614 dab_align(trapframe_t *tf, u_int fsr, u_int far, struct lwp *l, ksiginfo_t *ksi)
    615 {
    616 	/* Alignment faults are always fatal if they occur in kernel mode */
    617 	if (!TRAP_USERMODE(tf))
    618 		dab_fatal(tf, fsr, far, l, NULL);
    619 
    620 	/* pcb_onfault *must* be NULL at this point */
    621 	KDASSERT(((struct pcb *)lwp_getpcb(l))->pcb_onfault == NULL);
    622 
    623 	/* See if the CPU state needs to be fixed up */
    624 	(void) data_abort_fixup(tf, fsr, far, l);
    625 
    626 	/* Deliver a bus error signal to the process */
    627 	KSI_INIT_TRAP(ksi);
    628 	ksi->ksi_signo = SIGBUS;
    629 	ksi->ksi_code = BUS_ADRALN;
    630 	ksi->ksi_addr = (uint32_t *)(intptr_t)far;
    631 	ksi->ksi_trap = fsr;
    632 
    633 	lwp_settrapframe(l, tf);
    634 
    635 	return (1);
    636 }
    637 
    638 /*
    639  * dab_buserr() handles the following data aborts:
    640  *
    641  *  FAULT_BUSERR_0 - External Abort on Linefetch -- Section
    642  *  FAULT_BUSERR_1 - External Abort on Linefetch -- Page
    643  *  FAULT_BUSERR_2 - External Abort on Non-linefetch -- Section
    644  *  FAULT_BUSERR_3 - External Abort on Non-linefetch -- Page
    645  *  FAULT_BUSTRNL1 - External abort on Translation -- Level 1
    646  *  FAULT_BUSTRNL2 - External abort on Translation -- Level 2
    647  *
    648  * If pcb_onfault is set, flag the fault and return to the handler.
    649  * If the fault occurred in user mode, give the process a SIGBUS.
    650  *
    651  * Note: On XScale, FAULT_BUSERR_0, FAULT_BUSERR_1, and FAULT_BUSERR_2
    652  * can be flagged as imprecise in the FSR. This causes a real headache
    653  * since some of the machine state is lost. In this case, tf->tf_pc
    654  * may not actually point to the offending instruction. In fact, if
    655  * we've taken a double abort fault, it generally points somewhere near
    656  * the top of "data_abort_entry" in exception.S.
    657  *
    658  * In all other cases, these data aborts are considered fatal.
    659  */
    660 static int
    661 dab_buserr(trapframe_t *tf, u_int fsr, u_int far, struct lwp *l,
    662     ksiginfo_t *ksi)
    663 {
    664 	struct pcb *pcb = lwp_getpcb(l);
    665 
    666 #ifdef __XSCALE__
    667 	if ((fsr & FAULT_IMPRECISE) != 0 &&
    668 	    (tf->tf_spsr & PSR_MODE) == PSR_ABT32_MODE) {
    669 		/*
    670 		 * Oops, an imprecise, double abort fault. We've lost the
    671 		 * r14_abt/spsr_abt values corresponding to the original
    672 		 * abort, and the spsr saved in the trapframe indicates
    673 		 * ABT mode.
    674 		 */
    675 		tf->tf_spsr &= ~PSR_MODE;
    676 
    677 		/*
    678 		 * We use a simple heuristic to determine if the double abort
    679 		 * happened as a result of a kernel or user mode access.
    680 		 * If the current trapframe is at the top of the kernel stack,
    681 		 * the fault _must_ have come from user mode.
    682 		 */
    683 		if (tf != ((trapframe_t *)pcb->pcb_ksp) - 1) {
    684 			/*
    685 			 * Kernel mode. We're either about to die a
    686 			 * spectacular death, or pcb_onfault will come
    687 			 * to our rescue. Either way, the current value
    688 			 * of tf->tf_pc is irrelevant.
    689 			 */
    690 			tf->tf_spsr |= PSR_SVC32_MODE;
    691 			if (pcb->pcb_onfault == NULL)
    692 				printf("\nKernel mode double abort!\n");
    693 		} else {
    694 			/*
    695 			 * User mode. We've lost the program counter at the
    696 			 * time of the fault (not that it was accurate anyway;
    697 			 * it's not called an imprecise fault for nothing).
    698 			 * About all we can do is copy r14_usr to tf_pc and
    699 			 * hope for the best. The process is about to get a
    700 			 * SIGBUS, so it's probably history anyway.
    701 			 */
    702 			tf->tf_spsr |= PSR_USR32_MODE;
    703 			tf->tf_pc = tf->tf_usr_lr;
    704 #ifdef THUMB_CODE
    705 			tf->tf_spsr &= ~PSR_T_bit;
    706 			if (tf->tf_usr_lr & 1)
    707 				tf->tf_spsr |= PSR_T_bit;
    708 #endif
    709 		}
    710 	}
    711 
    712 	/* FAR is invalid for imprecise exceptions */
    713 	if ((fsr & FAULT_IMPRECISE) != 0)
    714 		far = 0;
    715 #endif /* __XSCALE__ */
    716 
    717 	if (pcb->pcb_onfault) {
    718 		KDASSERT(TRAP_USERMODE(tf) == 0);
    719 		tf->tf_r0 = EFAULT;
    720 		tf->tf_pc = (register_t)(intptr_t) pcb->pcb_onfault;
    721 		return (0);
    722 	}
    723 
    724 	/* See if the CPU state needs to be fixed up */
    725 	(void) data_abort_fixup(tf, fsr, far, l);
    726 
    727 	/*
    728 	 * At this point, if the fault happened in kernel mode, we're toast
    729 	 */
    730 	if (!TRAP_USERMODE(tf))
    731 		dab_fatal(tf, fsr, far, l, NULL);
    732 
    733 	/* Deliver a bus error signal to the process */
    734 	KSI_INIT_TRAP(ksi);
    735 	ksi->ksi_signo = SIGBUS;
    736 	ksi->ksi_code = BUS_ADRERR;
    737 	ksi->ksi_addr = (uint32_t *)(intptr_t)far;
    738 	ksi->ksi_trap = fsr;
    739 
    740 	lwp_settrapframe(l, tf);
    741 
    742 	return (1);
    743 }
    744 
    745 static inline int
    746 prefetch_abort_fixup(trapframe_t *tf)
    747 {
    748 #ifdef CPU_ABORT_FIXUP_REQUIRED
    749 	int error;
    750 
    751 	/* Call the CPU specific prefetch abort fixup routine */
    752 	error = cpu_prefetchabt_fixup(tf);
    753 	if (__predict_true(error != ABORT_FIXUP_FAILED))
    754 		return (error);
    755 
    756 	/*
    757 	 * Oops, couldn't fix up the instruction
    758 	 */
    759 	printf("%s: fixup for %s mode prefetch abort failed.\n", __func__,
    760 	    TRAP_USERMODE(tf) ? "user" : "kernel");
    761 #ifdef THUMB_CODE
    762 	if (tf->tf_spsr & PSR_T_bit) {
    763 		printf("pc = 0x%08x, opcode 0x%04x, 0x%04x, insn = ",
    764 		    tf->tf_pc, *((uint16 *)(tf->tf_pc & ~1)),
    765 		    *((uint16 *)((tf->tf_pc + 2) & ~1)));
    766 	}
    767 	else
    768 #endif
    769 	{
    770 		printf("pc = 0x%08x, opcode 0x%08x, insn = ", tf->tf_pc,
    771 		    *((u_int *)tf->tf_pc));
    772 	}
    773 	disassemble(tf->tf_pc);
    774 
    775 	/* Die now if this happened in kernel mode */
    776 	if (!TRAP_USERMODE(tf))
    777 		dab_fatal(tf, 0, tf->tf_pc, NULL, NULL);
    778 
    779 	return (error);
    780 #else
    781 	return (ABORT_FIXUP_OK);
    782 #endif /* CPU_ABORT_FIXUP_REQUIRED */
    783 }
    784 
    785 /*
    786  * void prefetch_abort_handler(trapframe_t *tf)
    787  *
    788  * Abort handler called when instruction execution occurs at
    789  * a non existent or restricted (access permissions) memory page.
    790  * If the address is invalid and we were in SVC mode then panic as
    791  * the kernel should never prefetch abort.
    792  * If the address is invalid and the page is mapped then the user process
    793  * does no have read permission so send it a signal.
    794  * Otherwise fault the page in and try again.
    795  */
    796 void
    797 prefetch_abort_handler(trapframe_t *tf)
    798 {
    799 	struct lwp *l;
    800 	struct pcb *pcb __diagused;
    801 	struct vm_map *map;
    802 	vaddr_t fault_pc, va;
    803 	ksiginfo_t ksi;
    804 	int error, user;
    805 
    806 	UVMHIST_FUNC(__func__);
    807 #ifdef UVMHIST
    808 	if (__predict_true(maphist.e)) {
    809 		UVMHIST_CALLED(maphist);
    810 	}
    811 #endif
    812 
    813 	/* Update vmmeter statistics */
    814 	curcpu()->ci_data.cpu_ntrap++;
    815 
    816 	l = curlwp;
    817 	pcb = lwp_getpcb(l);
    818 
    819 	if ((user = TRAP_USERMODE(tf)) != 0)
    820 		LWP_CACHE_CREDS(l, l->l_proc);
    821 
    822 	/*
    823 	 * Enable IRQ's (disabled by the abort) This always comes
    824 	 * from user mode so we know interrupts were not disabled.
    825 	 * But we check anyway.
    826 	 */
    827 	KASSERT(!TRAP_USERMODE(tf) || (tf->tf_spsr & IF32_bits) == 0);
    828 	if (__predict_true((tf->tf_spsr & I32_bit) != IF32_bits))
    829 		restore_interrupts(tf->tf_spsr & IF32_bits);
    830 
    831 	/* See if the CPU state needs to be fixed up */
    832 	switch (prefetch_abort_fixup(tf)) {
    833 	case ABORT_FIXUP_RETURN:
    834 		KASSERT(!TRAP_USERMODE(tf) || (tf->tf_spsr & IF32_bits) == 0);
    835 		return;
    836 	case ABORT_FIXUP_FAILED:
    837 		/* Deliver a SIGILL to the process */
    838 		KSI_INIT_TRAP(&ksi);
    839 		ksi.ksi_signo = SIGILL;
    840 		ksi.ksi_code = ILL_ILLOPC;
    841 		ksi.ksi_addr = (uint32_t *)(intptr_t) tf->tf_pc;
    842 		lwp_settrapframe(l, tf);
    843 		goto do_trapsignal;
    844 	default:
    845 		break;
    846 	}
    847 
    848 	/* Prefetch aborts cannot happen in kernel mode */
    849 	if (__predict_false(!user))
    850 		dab_fatal(tf, 0, tf->tf_pc, NULL, NULL);
    851 
    852 	/* Get fault address */
    853 	fault_pc = tf->tf_pc;
    854 	lwp_settrapframe(l, tf);
    855 #ifdef UVMHIST
    856 	if (__predict_true(maphist.e)) {
    857 		UVMHIST_LOG(maphist, " (pc=0x%x, l=0x%x, tf=0x%x)",
    858 		    fault_pc, l, tf, 0);
    859 	}
    860 #endif
    861 
    862 	/* Ok validate the address, can only execute in USER space */
    863 	if (__predict_false(fault_pc >= VM_MAXUSER_ADDRESS ||
    864 	    (fault_pc < VM_MIN_ADDRESS && vector_page == ARM_VECTORS_LOW))) {
    865 		KSI_INIT_TRAP(&ksi);
    866 		ksi.ksi_signo = SIGSEGV;
    867 		ksi.ksi_code = SEGV_ACCERR;
    868 		ksi.ksi_addr = (uint32_t *)(intptr_t) fault_pc;
    869 		ksi.ksi_trap = fault_pc;
    870 		goto do_trapsignal;
    871 	}
    872 
    873 	map = &l->l_proc->p_vmspace->vm_map;
    874 	va = trunc_page(fault_pc);
    875 
    876 	/*
    877 	 * See if the pmap can handle this fault on its own...
    878 	 */
    879 #ifdef DEBUG
    880 	last_fault_code = -1;
    881 #endif
    882 	if (pmap_fault_fixup(map->pmap, va, VM_PROT_READ|VM_PROT_EXECUTE, 1)) {
    883 #ifdef UVMHIST
    884 		if (__predict_true(maphist.e)) {
    885 			UVMHIST_LOG (maphist, " <- emulated", 0, 0, 0, 0);
    886 		}
    887 #endif
    888 		goto out;
    889 	}
    890 
    891 #ifdef DIAGNOSTIC
    892 	if (__predict_false(curcpu()->ci_intr_depth > 0)) {
    893 		printf("\nNon-emulated prefetch abort with intr_depth > 0\n");
    894 		dab_fatal(tf, 0, tf->tf_pc, NULL, NULL);
    895 	}
    896 #endif
    897 
    898 	KASSERT(pcb->pcb_onfault == NULL);
    899 	error = uvm_fault(map, va, VM_PROT_READ);
    900 
    901 	if (__predict_true(error == 0)) {
    902 #ifdef UVMHIST
    903 		if (__predict_true(maphist.e)) {
    904 			UVMHIST_LOG (maphist, " <- uvm", 0, 0, 0, 0);
    905 		}
    906 #endif
    907 		goto out;
    908 	}
    909 	KSI_INIT_TRAP(&ksi);
    910 
    911 #ifdef UVMHIST
    912 	if (__predict_true(maphist.e)) {
    913 		UVMHIST_LOG (maphist, " <- fatal (%d)", error, 0, 0, 0);
    914 	}
    915 #endif
    916 	if (error == ENOMEM) {
    917 		printf("UVM: pid %d (%s), uid %d killed: "
    918 		    "out of swap\n", l->l_proc->p_pid, l->l_proc->p_comm,
    919 		    l->l_cred ? kauth_cred_geteuid(l->l_cred) : -1);
    920 		ksi.ksi_signo = SIGKILL;
    921 	} else
    922 		ksi.ksi_signo = SIGSEGV;
    923 
    924 	ksi.ksi_code = SEGV_MAPERR;
    925 	ksi.ksi_addr = (uint32_t *)(intptr_t) fault_pc;
    926 	ksi.ksi_trap = fault_pc;
    927 
    928 do_trapsignal:
    929 	call_trapsignal(l, tf, &ksi);
    930 
    931 out:
    932 	KASSERT(!TRAP_USERMODE(tf) || (tf->tf_spsr & IF32_bits) == 0);
    933 	userret(l);
    934 }
    935 
    936 /*
    937  * Tentatively read an 8, 16, or 32-bit value from 'addr'.
    938  * If the read succeeds, the value is written to 'rptr' and zero is returned.
    939  * Else, return EFAULT.
    940  */
    941 int
    942 badaddr_read(void *addr, size_t size, void *rptr)
    943 {
    944 	extern int badaddr_read_1(const uint8_t *, uint8_t *);
    945 	extern int badaddr_read_2(const uint16_t *, uint16_t *);
    946 	extern int badaddr_read_4(const uint32_t *, uint32_t *);
    947 	union {
    948 		uint8_t v1;
    949 		uint16_t v2;
    950 		uint32_t v4;
    951 	} u;
    952 	int rv, s;
    953 
    954 	cpu_drain_writebuf();
    955 
    956 	s = splhigh();
    957 
    958 	/* Read from the test address. */
    959 	switch (size) {
    960 	case sizeof(uint8_t):
    961 		rv = badaddr_read_1(addr, &u.v1);
    962 		if (rv == 0 && rptr)
    963 			*(uint8_t *) rptr = u.v1;
    964 		break;
    965 
    966 	case sizeof(uint16_t):
    967 		rv = badaddr_read_2(addr, &u.v2);
    968 		if (rv == 0 && rptr)
    969 			*(uint16_t *) rptr = u.v2;
    970 		break;
    971 
    972 	case sizeof(uint32_t):
    973 		rv = badaddr_read_4(addr, &u.v4);
    974 		if (rv == 0 && rptr)
    975 			*(uint32_t *) rptr = u.v4;
    976 		break;
    977 
    978 	default:
    979 		panic("%s: invalid size (%zu)", __func__, size);
    980 	}
    981 
    982 	splx(s);
    983 
    984 	/* Return EFAULT if the address was invalid, else zero */
    985 	return (rv);
    986 }
    987