trap_subr.S revision 1.7
1 /* $NetBSD: trap_subr.S,v 1.7 2012/07/09 17:36:55 matt Exp $ */ 2 /*- 3 * Copyright (c) 2010, 2011 The NetBSD Foundation, Inc. 4 * All rights reserved. 5 * 6 * This code is derived from software contributed to The NetBSD Foundation 7 * by Raytheon BBN Technologies Corp and Defense Advanced Research Projects 8 * Agency and which was developed by Matt Thomas of 3am Software Foundry. 9 * 10 * This material is based upon work supported by the Defense Advanced Research 11 * Projects Agency and Space and Naval Warfare Systems Center, Pacific, under 12 * Contract No. N66001-09-C-2073. 13 * Approved for Public Release, Distribution Unlimited 14 * 15 * Redistribution and use in source and binary forms, with or without 16 * modification, are permitted provided that the following conditions 17 * are met: 18 * 1. Redistributions of source code must retain the above copyright 19 * notice, this list of conditions and the following disclaimer. 20 * 2. Redistributions in binary form must reproduce the above copyright 21 * notice, this list of conditions and the following disclaimer in the 22 * documentation and/or other materials provided with the distribution. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 25 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 26 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 27 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 28 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 29 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 30 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 31 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 32 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 33 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 34 * POSSIBILITY OF SUCH DAMAGE. 35 */ 36 37 RCSID("$NetBSD: trap_subr.S,v 1.7 2012/07/09 17:36:55 matt Exp $") 38 39 .globl _C_LABEL(sctrapexit), _C_LABEL(trapexit), _C_LABEL(intrcall) 40 41 /* 42 * We have a problem with critical (MSR[CE] or machine check (MSR[ME]) 43 * or debug (MSR[DE]) interrupts/exception in that they could happen 44 * inbewtween the mtsprg1 %r2 and mfsprg1 %r2. If that happens, %r2 45 * will be lost. Even if we moved to a different sprg, subsequent 46 * expceptions would use SPRG1 and its value would be lost. The only 47 * way to be safe for CE/ME/DE faults to save and restore SPRG1. 48 * 49 * Since CE/ME/DE faults may happen anytime, we need r1 to always 50 * contain a valid kernel stack pointer. Therefore we use r2 as 51 * our temporary register. 52 * 53 * To prevent %r2 being overwritten, each "level" (normal, critical, 54 * mchk) uses a unique sprg to save %r2 (sprg1, sprg4, sprg5). 55 * 56 * Since we can't control how many nested exceptions we might get, 57 * we don't use a dedicated save area. Instead we have a upwards 58 * growing "stack" of them; the pointer to which is kept in sprg3. 59 * 60 * To allocate from the stack, one fetches sprg3, adds the amount 61 * needed, saves sprg3, and then refers to the save using a 62 * displacement of -amount. 63 */ 64 #define FRAME_EXC_PROLOGUE(start, sprg, srr) \ 65 mt##sprg %r2; /* save r2 */ \ 66 mfsprg3 %r2; /* get save_area pointer */ \ 67 addi %r2,%r2,4*(32-start); \ 68 /* allocate save area */ \ 69 mtsprg3 %r2; /* save updated pointer */ \ 70 stmw %r##start,-4*(32-start)(%r2); \ 71 /* free r24-r31 for use */ \ 72 mf##sprg %r26; /* get saved r2 */ \ 73 mfcr %r27; /* get Condition Register */ \ 74 mfxer %r28; /* get XER */ \ 75 mfspr %r30, SPR_##srr##0; /* get SRR0 */ \ 76 mfspr %r31, SPR_##srr##1 /* get SRR1 */ 77 78 #define PROLOGUE_GET_DEAR mfspr %r24, SPR_DEAR 79 #define PROLOGUE_GET_ESR mfspr %r25, SPR_ESR 80 #define PROLOGUE_GET_SRRS mfsrr0 %r24; \ 81 mfsrr1 %r25 82 #define PROLOGUE_GET_SPRG1 mfsprg1 %r29 83 #define PROLOGUE_GET_DBSR mfspr %r25, SPR_DBSR 84 #define SAVE_ESR stw %r25, FRAME_ESR(%r1) 85 #define SAVE_DEAR stw %r24, FRAME_DEAR(%r1) 86 #define SAVE_DEAR_ESR SAVE_ESR; SAVE_DEAR 87 #define SAVE_SRRS SAVE_DEAR_ESR 88 #define SAVE_SPRG1 stw %r29, FRAME_SPRG1(%r1) 89 #define SAVE_DBSR stw %r25, FRAME_DBSR(%r1) 90 #define SAVE_NOTHING /* nothing */ 91 #define RESTORE_SPRG1(r) lwz r, FRAME_SPRG1(%r1); \ 92 mtsprg1 r 93 #define RESTORE_SRR0(r) lwz r, FRAME_DEAR(%r1); \ 94 mtsrr0 r 95 #define RESTORE_SRR1(r) lwz r, FRAME_ESR(%r1); \ 96 mtsrr1 r 97 98 #define FRAME_PROLOGUE \ 99 FRAME_EXC_PROLOGUE(26, sprg1, SRR) 100 101 #define FRAME_PROLOGUE_DEAR_ESR \ 102 FRAME_EXC_PROLOGUE(24, sprg1, SRR); \ 103 PROLOGUE_GET_ESR; \ 104 PROLOGUE_GET_DEAR 105 106 #define FRAME_PROLOGUE_ESR \ 107 FRAME_EXC_PROLOGUE(25, sprg1, SRR); \ 108 PROLOGUE_GET_ESR 109 110 #define FRAME_TLBPROLOGUE \ 111 FRAME_EXC_PROLOGUE(20, sprg1, SRR); \ 112 PROLOGUE_GET_ESR; \ 113 PROLOGUE_GET_DEAR 114 115 #define FRAME_INTR_PROLOGUE \ 116 FRAME_EXC_PROLOGUE(26, sprg1, SRR) 117 118 /* 119 * These need to save SRR0/SRR1 as well their SRR0/SRR1 in case normal 120 * exceptions happened during their execution. 121 */ 122 #define FRAME_CRIT_PROLOGUE \ 123 FRAME_EXC_PROLOGUE(24, sprg4, CSRR); \ 124 PROLOGUE_GET_SPRG1; \ 125 PROLOGUE_GET_SRRS 126 127 #define FRAME_MCHK_PROLOGUE \ 128 FRAME_EXC_PROLOGUE(24, sprg5, MCSRR); \ 129 PROLOGUE_GET_SPRG1; \ 130 PROLOGUE_GET_SRRS 131 132 #define FRAME_DEBUG_PROLOGUE \ 133 FRAME_EXC_PROLOGUE(24, sprg4, CSRR); \ 134 PROLOGUE_GET_SPRG1; \ 135 PROLOGUE_GET_SRRS 136 137 /* 138 * DDB expects to fetch the LR from the previous frame. But it also 139 * expects to be pointing at the instruction after the branch link. Since 140 * we didn't branch, we need to advance it by to fake out DDB. 141 */ 142 #ifdef DDB 143 #define FRAME_SAVE_SRR0_FOR_DDB \ 144 addi %r30, %r30, 4; /* ddb thinks its the next insn */ \ 145 stw %r30, FRAMELEN+SZREG(%r1); /* appease ddb stacktrace */ 146 #else 147 #define FRAME_SAVE_SRR0_FOR_DDB 148 #endif 149 150 #ifdef PPC_HAVE_SPE 151 #define FRAME_SAVE_SPEFSCR \ 152 mfspefscr %r0; /* get spefscr */ \ 153 stw %r0, FRAME_SPEFSCR(%r1) /* save into trapframe */ 154 #define FRAME_RESTORE_SPEFSCR \ 155 lwz %r0, FRAME_SPEFSCR(%r1); /* fetch from trapframe */ \ 156 mtspefscr %r0 /* save spefscr */ 157 #else 158 #define FRAME_SAVE_SPEFSCR 159 #define FRAME_RESTORE_SPEFSCR 160 #endif 161 /* 162 * Before the first memory refernence, we must have our state inside registers 163 * since the first memory access might cause an exception which would cause 164 * SRR0/SRR1 and DEAR/ESR to become unrecoverable. CR and XER also need to be 165 * saved early since they will modified by instrction flow. The saved stack 166 * pointer is also critical but LR and CTR can be deferred being saved until 167 * we are actually filling a trapframe. 168 */ 169 #define FRAME_EXC_ENTER(exc, tf, start, save_prologue) \ 170 mtcr %r31; /* user mode exception? */ \ 171 mr %r31, %r1; /* save SP (SRR1 is safe in CR) */ \ 172 bf MSR_PR, 1f; /* nope, sp is good */ \ 173 mfsprg2 %r2; /* get curlwp */ \ 174 lwz %r2, L_PCB(%r2); /* get uarea of curlwp */ \ 175 addi %r1, %r2, USPACE-CALLFRAMELEN; \ 176 /* start stack at top of it */ \ 177 1: \ 178 stw %r30, CFRAME_LR(%r1); /* save in previous callframe */ \ 179 stwu %r31, -FRAMELEN(%r1); /* get space for trapframe */ \ 180 stw %r0, FRAME_R0(%r1); /* save r0 */ \ 181 stw %r31, FRAME_R1(%r1); /* save (saved) r1 */ \ 182 stw %r26, FRAME_R2(%r1); /* save (saved) r2 */ \ 183 save_prologue; /* save SPRG1/ESR/DEAR */ \ 184 /* At this point, r26, r29, and r31 have been saved so we */ \ 185 /* can use them for LR, CTR, and SRR1. */ \ 186 mflr %r26; /* get Link Register */ \ 187 mfctr %r29; /* get CTR */ \ 188 mfcr %r31; /* get SRR1 */ \ 189 stmw %r26, FRAME_LR(%r1); /* save LR CR XER CTR SRR0/1 */ \ 190 FRAME_SAVE_SRR0_FOR_DDB; \ 191 mr %r0, %r31; /* save SRR1 for a bit */ \ 192 mfsprg3 %r2; /* get save_area pointer */ \ 193 addi %r2,%r2,-4*(32-start); /* find our save area */ \ 194 lmw %r##start,0(%r2); /* get start-r31 */ \ 195 mtsprg3 %r2; /* save updated pointer */ \ 196 stmw %r3, FRAME_R3(%r1); /* save r2-r31 */ \ 197 /* Now everything has been saved */ \ 198 mr %r31, %r0; /* move SRR1 back to r31 */ \ 199 mfsprg2 %r13; /* put curlwp in r13 */ \ 200 FRAME_SAVE_SPEFSCR; \ 201 li %r7, exc; /* load EXC_* */ \ 202 stw %r7, FRAME_EXC(%r1); /* save into trapframe */ \ 203 addi tf, %r1, FRAME_TF /* get address of trap frame */ 204 205 #define FRAME_EXC_EXIT(rfi, srr) \ 206 lmw %r26, FRAME_LR(%r1); /* get LR CR XER CTR SRR0/1 */ \ 207 oris %r31,%r31,PSL_CE@h; \ 208 mtspr SPR_##srr##1, %r31; /* restore SRR1 */ \ 209 mtspr SPR_##srr##0, %r30; /* restore SRR0 */ \ 210 FRAME_RESTORE_SPEFSCR; \ 211 mtctr %r29; /* restore CTR */ \ 212 mtxer %r28; /* restore XER */ \ 213 mtcr %r27; /* restore CR */ \ 214 mtlr %r26; /* restore LR */ \ 215 lmw %r2, FRAME_R2(%r1); /* restore r2-r31 */ \ 216 lwz %r0, FRAME_R0(%r1); /* restore r0 */ \ 217 lwz %r1, FRAME_R1(%r1); /* restore r1 */ \ 218 rfi /* return from interrupt */ 219 220 221 #define FRAME_ENTER(exc, tf) \ 222 FRAME_EXC_ENTER(exc, tf, 26, SAVE_NOTHING) 223 224 #define FRAME_ENTER_ESR(exc, tf) \ 225 FRAME_EXC_ENTER(exc, tf, 25, SAVE_ESR) 226 227 #define FRAME_ENTER_DEAR_ESR(exc, tf) \ 228 FRAME_EXC_ENTER(exc, tf, 24, SAVE_DEAR_ESR) 229 230 #define FRAME_EXIT FRAME_EXC_EXIT(rfi, SRR) 231 232 #define FRAME_TLBENTER(exc) \ 233 FRAME_EXC_ENTER(exc, %r4, 20, SAVE_DEAR_ESR) 234 #define FRAME_TLBEXIT FRAME_EXC_EXIT(rfi, SRR) 235 236 #define FRAME_MCHK_ENTER(exc) \ 237 FRAME_EXC_ENTER(exc, %r3, 26, SAVE_SPRG1; SAVE_SRRS) 238 #define FRAME_MCHK_EXIT \ 239 RESTORE_SRR0(%r28); \ 240 RESTORE_SRR1(%r27); \ 241 RESTORE_SPRG1(%r26); \ 242 FRAME_EXC_EXIT(rfmci, MCSRR) 243 244 #define FRAME_DEBUG_ENTER(exc) \ 245 FRAME_EXC_ENTER(exc, %r4, 26, SAVE_SPRG1; SAVE_SRRS) 246 #define FRAME_DEBUG_EXIT \ 247 RESTORE_SPRG1(%r26); FRAME_EXC_EXIT(rfci, CSRR) 248 249 #define FRAME_INTR_SP \ 250 bf MSR_PR, 1f; /* nope, sp is good */ \ 251 mfsprg2 %r2; /* get curlwp */ \ 252 lwz %r2, L_PCB(%r2); /* get uarea of curlwp */ \ 253 addi %r1, %r2, USPACE-CALLFRAMELEN; \ 254 /* start stack at top of it */ \ 255 1: 256 257 #define FRAME_INTR_SP_NEW(sym) \ 258 lis %r2,(sym)@ha; \ 259 addi %r1,%r2,(sym)@l 260 261 #define FRAME_INTR_XENTER(exc, start, get_intr_sp, save_prologue) \ 262 mtcr %r31; /* user mode exception? */ \ 263 mr %r31, %r1; /* save SP (SRR1 is safe in CR) */ \ 264 get_intr_sp; /* get kernel stack pointer */ \ 265 stw %r30, CFRAME_LR(%r1); /* save in .. */ \ 266 stwu %r31, -FRAMELEN(%r1); /* get space for trapframe */ \ 267 stw %r0, FRAME_R0(%r1); /* save r0 */ \ 268 stw %r31, FRAME_R1(%r1); /* save (saved) r1 */ \ 269 stw %r26, FRAME_R2(%r1); /* save (saved) r2 */ \ 270 save_prologue; /* save SPRG1 (maybe) */ \ 271 mflr %r26; /* get LR */ \ 272 mfctr %r29; /* get CTR */ \ 273 mfcr %r31; /* get SRR1 */ \ 274 stmw %r26, FRAME_LR(%r1); /* save LR CR XER CTR SRR0/1 */ \ 275 FRAME_SAVE_SRR0_FOR_DDB; \ 276 stw %r3, FRAME_R3(%r1); /* save r3 */ \ 277 stw %r4, FRAME_R4(%r1); /* save r4 */ \ 278 stw %r5, FRAME_R5(%r1); /* save r5 */ \ 279 stw %r6, FRAME_R6(%r1); /* save r6 */ \ 280 stw %r7, FRAME_R7(%r1); /* save r7 */ \ 281 stw %r8, FRAME_R8(%r1); /* save r8 */ \ 282 stw %r9, FRAME_R9(%r1); /* save r9 */ \ 283 stw %r10, FRAME_R10(%r1); /* save r10 */ \ 284 stw %r11, FRAME_R11(%r1); /* save r11 */ \ 285 stw %r12, FRAME_R12(%r1); /* save r12 */ \ 286 stw %r13, FRAME_R13(%r1); /* save r13 */ \ 287 mfsprg3 %r2; /* get save_area pointer */ \ 288 addi %r2,%r2,-4*(32-start); /* find our save area */ \ 289 lmw %r##start,0(%r2); /* get start-r31 */ \ 290 mtsprg3 %r2; /* save updated pointer */ \ 291 mfsprg2 %r13; /* put curlwp into r13 */ \ 292 li %r7, exc; /* load EXC_* */ \ 293 stw %r7, FRAME_EXC(%r1); /* save into trapframe */ \ 294 addi %r3, %r1, FRAME_TF /* only argument is trapframe */ 295 296 #define FRAME_INTR_XEXIT(rfi, srr) \ 297 lwz %r8, FRAME_LR(%r1); /* get LR */ \ 298 lwz %r9, FRAME_CR(%r1); /* get CR */ \ 299 lwz %r10, FRAME_XER(%r1); /* get XER */ \ 300 lwz %r11, FRAME_CTR(%r1); /* get CTR */ \ 301 lwz %r12, FRAME_SRR0(%r1); /* get SRR0 */ \ 302 lwz %r13, FRAME_SRR1(%r1); /* get SRR1 */ \ 303 mtspr SPR_##srr##1, %r13; /* restore SRR1 */ \ 304 mtspr SPR_##srr##0, %r12; /* restore SRR0 */ \ 305 mtctr %r11; /* restore CTR */ \ 306 mtxer %r10; /* restore XER */ \ 307 mtcr %r9; /* restore CR */ \ 308 mtlr %r8; /* restore LR */ \ 309 lwz %r13, FRAME_R13(%r1); /* restore r13 */ \ 310 lwz %r12, FRAME_R12(%r1); /* restore r12 */ \ 311 lwz %r11, FRAME_R11(%r1); /* restore r11 */ \ 312 lwz %r10, FRAME_R10(%r1); /* restore r10 */ \ 313 lwz %r9, FRAME_R9(%r1); /* restore r9 */ \ 314 lwz %r8, FRAME_R8(%r1); /* restore r8 */ \ 315 lwz %r7, FRAME_R7(%r1); /* restore r7 */ \ 316 lwz %r6, FRAME_R6(%r1); /* restore r6 */ \ 317 lwz %r5, FRAME_R5(%r1); /* restore r5 */ \ 318 lwz %r4, FRAME_R4(%r1); /* restore r4 */ \ 319 lwz %r3, FRAME_R3(%r1); /* restore r3 */ \ 320 lwz %r2, FRAME_R2(%r1); /* restore r2 */ \ 321 lwz %r0, FRAME_R0(%r1); /* restore r0 */ \ 322 lwz %r1, FRAME_R1(%r1); /* restore r1 */ \ 323 rfi /* return from interrupt */ 324 325 #define FRAME_INTR_ENTER(exc) \ 326 FRAME_INTR_XENTER(exc, 26, FRAME_INTR_SP, SAVE_NOTHING) 327 #define FRAME_INTR_EXIT \ 328 FRAME_INTR_XEXIT(rfi, SRR) 329 #define FRAME_CRIT_ENTER(exc) \ 330 FRAME_INTR_XENTER(exc, 24, FRAME_INTR_SP, SAVE_SPRG1) 331 #define FRAME_WDOG_ENTER(exc, sym) \ 332 FRAME_INTR_XENTER(exc, 24, FRAME_INTR_SP_NEW(sym), SAVE_SPRG1) 333 #define FRAME_CRIT_EXIT \ 334 RESTORE_SRR0(%r4); \ 335 RESTORE_SRR1(%r5); \ 336 RESTORE_SPRG1(%r6); \ 337 FRAME_INTR_XEXIT(rfci, CSRR) 338 339 .text 340 .p2align 4 341 _C_LABEL(critical_input_vector): 342 /* MSR[ME] is unchanged, all others cleared */ 343 FRAME_CRIT_PROLOGUE /* save SP r26-31 CR LR XER */ 344 FRAME_CRIT_ENTER(EXC_CII) 345 bl _C_LABEL(intr_critintr) /* critintr(tf) */ 346 FRAME_CRIT_EXIT 347 348 .p2align 4 349 _C_LABEL(machine_check_vector): 350 /* all MSR bits are cleared */ 351 FRAME_MCHK_PROLOGUE /* save SP r25-31 CR LR XER */ 352 FRAME_MCHK_ENTER(EXC_MCHK) 353 /* 354 * MCAR/MCSR don't need to be saved early since MSR[ME] is cleared 355 * on entry. 356 */ 357 mfspr %r7, SPR_MCAR 358 mfspr %r6, SPR_MCSR 359 stw %r6, FRAME_MCSR(%r1) 360 stw %r7, FRAME_MCAR(%r1) 361 li %r3, T_MACHINE_CHECK 362 bl _C_LABEL(trap) /* trap(T_MACHINE_CHECK, tf) */ 363 FRAME_MCHK_EXIT 364 365 .p2align 4 366 _C_LABEL(data_storage_vector): 367 /* MSR[CE], MSR[ME], MSR[DE] are unchanged, all others cleared */ 368 FRAME_PROLOGUE_DEAR_ESR /* save r2 DEAR ESR r24-31 CR XER SRR */ 369 FRAME_ENTER_DEAR_ESR(EXC_DSI, %r4) 370 li %r3, T_DSI 371 /* FRAME_ENTER leaves SRR1 in %r31 */ 372 trapenter: 373 trapagain: 374 wrtee %r31 /* restore MSR[EE] */ 375 376 bl _C_LABEL(trap) /* trap(trapcode, tf) */ 377 _C_LABEL(trapexit): 378 wrteei 0 /* disable interrupts */ 379 # andis. %r0, %r31, PSL_CE@h 380 # tweqi %r0, 0 381 andi. %r4, %r31, PSL_PR /* lets look at PSL_PR */ 382 beq trapdone /* if clear, skip to exit */ 383 lwz %r4, L_MD_ASTPENDING(%r13) /* get ast pending */ 384 cmplwi %r4, 0 /* is there an ast pending */ 385 beq+ trapdone /* nope, proceed to exit */ 386 li %r6, EXC_AST /* yes. */ 387 stw %r6, FRAME_EXC(%r1) /* pretend this is an AST */ 388 addi %r4, %r1, FRAME_TF /* get address of trap frame */ 389 li %r3, T_AST 390 b trapagain /* and deal with it */ 391 trapdone: 392 FRAME_EXIT 393 394 .p2align 4 395 _C_LABEL(instruction_storage_vector): 396 /* MSR[CE], MSR[ME], MSR[DE] are unchanged, all others cleared */ 397 FRAME_PROLOGUE_ESR /* save ESR r2 r25-31 CR XER SRR0/1 */ 398 FRAME_ENTER_ESR(EXC_ISI, %r4) 399 li %r3, T_ISI 400 b trapenter 401 402 .p2align 4 403 _ENTRY(external_input_vector) 404 /* MSR[CE], MSR[ME], MSR[DE] are unchanged, all others cleared */ 405 FRAME_INTR_PROLOGUE /* save SP r25-31 CR LR XER */ 406 FRAME_INTR_ENTER(EXC_EXI) 407 408 bl _C_LABEL(intr_extintr) 409 _C_LABEL(intrcall): 410 GET_CPUINFO(%r6) /* get curcpu() */ 411 lwz %r5, FRAME_SRR1(%r1) /* get saved SRR1 */ 412 # andis. %r0, %r5, PSL_CE@h 413 # tweqi %r0, 0 414 andi. %r4, %r5, PSL_PR /* lets look at PSL_PR */ 415 beq intrexit /* if clear, skip to exit */ 416 lwz %r4, L_MD_ASTPENDING(%r13) /* get ast pending */ 417 cmplwi %r4, 0 /* is there an ast pending */ 418 beq+ intrexit /* nope, proceed to exit */ 419 stmw %r14, FRAME_R14(%r1) /* save rest of registers */ 420 FRAME_SAVE_SPEFSCR 421 mr %r31, %r5 /* needed for trapagain */ 422 li %r4, EXC_AST /* */ 423 stw %r4, FRAME_EXC(%r1) /* pretend this is an AST */ 424 addi %r4, %r1, FRAME_TF /* get address of trap frame */ 425 li %r3, T_AST 426 b trapagain /* and deal with it */ 427 intrexit: 428 FRAME_INTR_EXIT 429 430 .p2align 4 431 _C_LABEL(alignment_vector): 432 /* MSR[CE], MSR[ME], MSR[DE] are unchanged, all others cleared */ 433 FRAME_PROLOGUE_DEAR_ESR /* save SP r25-31 CR LR XER */ 434 FRAME_ENTER_DEAR_ESR(EXC_ALI, %r4) 435 li %r3, T_ALIGNMENT 436 b trapenter 437 438 .p2align 4 439 _C_LABEL(program_vector): 440 /* MSR[CE], MSR[ME], MSR[DE] are unchanged, all others cleared */ 441 FRAME_PROLOGUE_ESR /* save SP r25-31 CR LR XER */ 442 FRAME_ENTER_ESR(EXC_PGM, %r4) 443 li %r3, T_PROGRAM 444 b trapenter 445 446 #ifdef SPR_IVOR7 447 .p2align 4 448 _C_LABEL(fp_unavailable_vector): 449 /* MSR[CE], MSR[ME], MSR[DE] are unchanged, all others cleared */ 450 FRAME_PROLOGUE_ESR /* save SP r25-31 CR LR XER */ 451 FRAME_ENTER_ESR(EXC_FPU, %r4) 452 li %r3, T_FP_UNAVAILABLE 453 b trapenter 454 #endif 455 456 .p2align 4 457 _C_LABEL(system_call_vector): 458 /* MSR[CE], MSR[ME], MSR[DE] are unchanged, all others cleared */ 459 FRAME_PROLOGUE /* save SP r26-31 CR LR XER */ 460 FRAME_ENTER(EXC_SC, %r3) 461 462 wrteei 1 /* enable interrupts */ 463 lwz %r7, L_PROC(%r13) /* get proc for lwp */ 464 lwz %r8, P_MD_SYSCALL(%r7) /* get syscall */ 465 mtlr %r8 /* need to call indirect */ 466 blrl /* syscall(tf) */ 467 _C_LABEL(sctrapexit): 468 wrteei 0 /* disable interrupts */ 469 lwz %r4, L_MD_ASTPENDING(%r13) /* get ast pending */ 470 cmplwi %r4, 0 /* is there an ast pending */ 471 beq+ trapdone /* nope, proceed to exit */ 472 li %r0, EXC_AST /* yes. */ 473 stw %r0, FRAME_EXC(%r1) /* pretend this is an AST */ 474 addi %r4, %r1, FRAME_TF /* get address of trap frame */ 475 li %r3, T_AST 476 b trapenter /* and deal with it */ 477 478 #ifdef SPR_IVOR9 479 .p2align 4 480 _C_LABEL(ap_unavailable_vector): 481 /* MSR[CE], MSR[ME], MSR[DE] are unchanged, all others cleared */ 482 FRAME_PROLOGUE /* save SP r25-31 CR LR XER */ 483 FRAME_ENTER(EXC_PGM, %r4) 484 li %r3, T_AP_UNAVAILABLE 485 b trapenter 486 #endif 487 488 .p2align 4 489 _C_LABEL(decrementer_vector): 490 /* MSR[CE], MSR[ME], MSR[DE] are unchanged, all others cleared */ 491 FRAME_INTR_PROLOGUE /* save SP r25-31 CR LR XER */ 492 FRAME_INTR_ENTER(EXC_DECR) 493 494 bl _C_LABEL(intr_decrintr) 495 b intrexit 496 497 .p2align 4 498 _C_LABEL(fixed_interval_timer_vector): 499 /* MSR[CE], MSR[ME], MSR[DE] are unchanged, all others cleared */ 500 FRAME_PROLOGUE /* save SP r25-31 CR LR XER */ 501 FRAME_INTR_ENTER(EXC_FIT) 502 503 bl _C_LABEL(intr_fitintr) 504 b intrexit 505 506 #ifdef E500_WDOG_STACK 507 .data 508 .lcomm wdogstk,4096 509 #endif 510 .text 511 .p2align 4 512 _C_LABEL(watchdog_timer_vector): 513 /* MSR[CE], MSR[ME], MSR[DE] are unchanged, all others cleared */ 514 FRAME_CRIT_PROLOGUE /* save SP r25-31 CR LR XER */ 515 #ifdef E500_WDOG_STACK 516 FRAME_WDOG_ENTER(EXC_WDOG, wdogstk+4096-CALLFRAMELEN) 517 #else 518 FRAME_CRIT_ENTER(EXC_WDOG); 519 #endif 520 521 bl _C_LABEL(intr_wdogintr) 522 FRAME_CRIT_EXIT 523 524 .p2align 4 525 _C_LABEL(data_tlb_error_vector): 526 /* MSR[CE], MSR[ME], MSR[DE] are unchanged, all others cleared */ 527 FRAME_TLBPROLOGUE 528 /* 529 * Registers as this point: 530 * 531 * r2 = cpu_info 532 * r20 = scratch 533 * r21 = scratch 534 * r22 = scratch 535 * r23 = scratch 536 * r24 = DEAR 537 * r25 = ESR 538 * r26 = saved r2 539 * r27 = CR 540 * r28 = XER 541 * r29 = scratch 542 * r30 = SRR0 543 * r31 = SRR1 544 * 545 * Except for r29, these values must be retained. However we must 546 * be cognizant of nesting. There are two cases here, both related. 547 * 548 * We get a critical input or machine check exception and the kernel 549 * stack doesn't have a TLB entry so we take an exception. The other 550 * nesting path is some page used by the exception handler will cause 551 * a TLB data error. 552 * 553 * The second case (more probable) is that the PTE loading will fail 554 * so we will have to do a hard trap to resolve it. But in doing so 555 * we need to save a trapframe which could result in another DTLB 556 * fault. 557 * 558 * In all cases, the save area stack shall protect us. 559 */ 560 /* 561 * Attempt to update the TLB from the page table. 562 */ 563 mflr %r29 /* save LR */ 564 mr %r23, %r24 /* address of exception */ 565 rlwinm %r22, %r31, /* index into ci_pmap_segtab */\ 566 MSR_DS+PTR_SCALESHIFT+1, \ 567 31-PTR_SCALESHIFT, \ 568 31-PTR_SCALESHIFT /* move PSL_DS[27] to bit 29 */ 569 bl pte_load 570 mtlr %r29 /* restore LR */ 571 /* 572 * If we returned, pte load failed so let trap deal with it but 573 * has kept the contents of r24-r31 (expect r29) intact. 574 */ 575 FRAME_TLBENTER(EXC_DSI) 576 li %r3, T_DATA_TLB_ERROR 577 b trapenter 578 579 .p2align 4 580 _C_LABEL(instruction_tlb_error_vector): 581 /* MSR[CE], MSR[ME], MSR[DE] are unchanged, all others cleared */ 582 FRAME_TLBPROLOGUE 583 /* 584 * Attempt to update the TLB from the page table. 585 */ 586 mflr %r29 /* save LR */ 587 mr %r23, %r30 /* PC of exception */ 588 rlwinm %r22, %r31, /* index into ci_pmap_segtab */\ 589 MSR_IS+PTR_SCALESHIFT+1, \ 590 31-PTR_SCALESHIFT, \ 591 31-PTR_SCALESHIFT /* move PSL_IS[26] to bit 29 */ 592 bl pte_load 593 mtlr %r29 /* restore LR */ 594 /* 595 * If we returned, pte load failed so let trap deal with it but 596 * has kept the contents of r24-r31 (expect r29) intact. 597 */ 598 FRAME_TLBENTER(EXC_ISI) 599 li %r3, T_INSTRUCTION_TLB_ERROR 600 b trapenter 601 602 .p2align 4 603 _C_LABEL(debug_vector): 604 FRAME_CRIT_PROLOGUE /* save SP r25-31 CR LR XER */ 605 FRAME_CRIT_ENTER(EXC_DEBUG) 606 mfspr %r6, SPR_DBSR 607 stw %r6, FRAME_ESR(%r1) 608 li %r3, T_DEBUG 609 bl _C_LABEL(trap) 610 FRAME_CRIT_EXIT 611 612 .p2align 4 613 _C_LABEL(spv_unavailable_vector): 614 FRAME_PROLOGUE_ESR /* save SP r25-31 CR LR XER */ 615 FRAME_ENTER_ESR(EXC_VEC, %r4) 616 li %r3, T_SPE_UNAVAILABLE 617 b trapenter 618 619 .p2align 4 620 _C_LABEL(fpdata_vector): 621 FRAME_PROLOGUE_ESR /* save SP r25-31 CR LR XER */ 622 FRAME_ENTER_ESR(EXC_FPA, %r4) 623 li %r3, T_EMBEDDED_FP_DATA 624 b trapenter 625 626 .p2align 4 627 _C_LABEL(fpround_vector): 628 FRAME_PROLOGUE_ESR /* save SP r25-31 CR LR XER */ 629 FRAME_ENTER_ESR(EXC_FPA, %r4) 630 li %r3, T_EMBEDDED_FP_ROUND 631 b trapenter 632 633 .p2align 4 634 _C_LABEL(perfmon_vector): 635 FRAME_PROLOGUE_ESR /* save SP r25-31 CR LR XER */ 636 FRAME_ENTER_ESR(EXC_PERF, %r4) 637 li %r3, T_EMBEDDED_PERF_MONITOR 638 b trapenter 639 640 .p2align 4 641 pte_load: 642 /* 643 * r2 = scratch 644 * r20 = scratch 645 * r21 = scratch 646 * r22 = index into ci_pmap_{kern,user}_segtab 647 * r23 = faulting address 648 * The rest are for reference and aren't modifiable. If the load 649 * fails, they will be used by FRAME_TLBENTER to create the trapframe. 650 * r24 = DEAR 651 * r25 = ESR 652 * r26 = saved r2 653 * r27 = CR 654 * r28 = XER 655 * r29 = LR 656 * r30 = SRR0 657 * r31 = SRR1 658 */ 659 cmplwi %cr2, %r22, 0 /* remember address space */ 660 GET_CPUINFO(%r2) 661 addi %r22, %r22, CI_PMAP_SEGTAB /* index into segtab(s) */ 662 lwzx %r20, %r22, %r2 /* load kern/user L1 PT addr */ 663 cmplwi %r20, 0 /* is segtab null? */ 664 beqlr %cr0 /* yes, return to fallback to trap */ 665 666 rlwinm %r22, %r23, NSEGPG_SCALESHIFT + PTR_SCALESHIFT, \ 667 31-(NSEGPG_SCALESHIFT + PTR_SCALESHIFT - 1), \ 668 31-PTR_SCALESHIFT /* extract addr bits [0:9] to [20:29] */ 669 lwzx %r20, %r22, %r20 /* load address of page table page */ 670 cmplwi %r20, 0 /* is page null? */ 671 beqlr %cr0 /* yes, return to fallback to trap */ 672 673 rlwinm %r22, %r23, \ 674 NSEGPG_SCALESHIFT + NPTEPG_SCALESHIFT + PTE_SCALESHIFT, \ 675 31-(NPTEPG_SCALESHIFT + PTE_SCALESHIFT - 1), \ 676 31-PTE_SCALESHIFT /* extract addr bits [10:19] to [20:29] */ 677 lwzx %r20, %r22, %r20 /* load PTE from page table page */ 678 cmplwi %r20, 0 /* is there a valid PTE? */ 679 beqlr %cr0 /* no, return to fallback to trap */ 680 681 #if (PTE_UNSYNCED << 1) != PTE_xX 682 #error PTE_UNSYNCED definition error 683 #endif 684 #if (PTE_UNMODIFIED << 1) != PTE_xW 685 #error PTE_UNMODIFIED definition error 686 #endif 687 andi. %r22, %r20, (PTE_UNSYNCED|PTE_UNMODIFIED) 688 /* Does the PTE need to be changed? */ 689 rotlwi %r22, %r22, 1 /* if so, clear the right PTE bits */ 690 andc %r20, %r20, %r22 /* pte &= ~((pte & (PTE_UNSYNCED|PTE_UNMODIFIED)) << 1)*/ 691 692 /* 693 * r24-r32 = (no touch) 694 * r23 = scratch (was fault addr) 695 * r22 = scratch 696 * r21 = scratch 697 * r20 = pte 698 * cr2 = AS 0=eq/!0=ne 699 */ 700 701 /* 702 * This is all E500 specific. We should have a patchable branch 703 * to support other BookE (440) implementations. 704 */ 705 e500_pte_load: 706 bne+ %cr2, 1f /* user access? MAS1 is ok. */ 707 mfspr %r22, SPR_MAS1 /* get MAS1 */ 708 lis %r21, MAS1_TID@h /* get TID mask */ 709 andc %r22, %r22, %r21 /* clear TID */ 710 mtspr SPR_MAS1, %r22 /* save MAS1 */ 711 1: 712 andi. %r21, %r20, PTE_WIMGE_MASK /* extract WIMGE from PTE */ 713 cmplwi %r21, PTE_M /* if just PTE_M is set, */ 714 beq+ %cr0, 2f /* skip munging mas2 */ 715 mfspr %r22, SPR_MAS2 /* get MAS2 (updated by error) */ 716 clrrwi %r22, %r22, PTE_RWX_SHIFT /* clear WIMGE bits */ 717 or %r22, %r22, %r21 /* combine with MAS2 contents */ 718 mtspr SPR_MAS2, %r22 /* put back into MAS2 */ 719 2: 720 /* 721 * r23 = fault addr 722 * r22 = scratch 723 * r21 = scratch 724 * r20 = pte 725 */ 726 727 /* 728 * In MAS3, the protection bits are in the low 6 bits: 729 * UX SX UW SW UR SR 730 * The User bits are 1 bit left of their Supervisor counterparts. 731 * Rotate the PTE protection bits left until they wrap around to become 732 * the least significant bits, where the Supervisor protection bits 733 * are located. Increase the rotate amount by 1 to place them where 734 * the User protection bits are located. We get that 1 by extracting 735 * the MAS1[TS] (set for User access) and moving it to bit 31 (LSB). 736 */ 737 mfspr %r21, SPR_MAS1 /* get MAS1 which has TS bit */ 738 extrwi %r21, %r21, 1, 31-MAS1_TS_SHIFT 739 /* extract MAS1_TS to LSB */ 740 clrrwi %r23, %r20, PAGE_SHIFT /* clear non-RPN bits from PTE */ 741 andi. %r20, %r20, PTE_RWX_MASK /* isolate protection bits */ 742 rotrwi %r20, %r20, PTE_RWX_SHIFT 743 andi. %r22, %r20, (MAS3_SW|MAS3_SR) /* user pages need to be R/W by kernel */ 744 rotlw %r20, %r20, %r21 /* rotate protection to correct loc */ 745 or %r20, %r20, %r22 /* combine system protection bits */ 746 or %r23, %r23, %r20 /* combine RPN and protection bits */ 747 mtspr SPR_MAS3, %r23 /* put into MAS3 */ 748 isync /* because ECORE500RM tells us too */ 749 tlbwe /* write the TLB entry */ 750 /* 751 * Increment a counter to show how many tlb misses we've handled here. 752 */ 753 lmw %r30, CI_EV_TLBMISS_SOFT(%r2) 754 addic %r31, %r31, 1 755 addze %r30, %r30 756 stmw %r30, CI_EV_TLBMISS_SOFT(%r2) 757 /* 758 * Cleanup and leave. We know any higher priority exception will 759 * save and restore SPRG1 and %r2 thereby preserving their values. 760 * 761 * r24 = DEAR (don't care) 762 * r25 = ESR (don't care) 763 * r26 = saved r2 764 * r27 = CR 765 * r28 = XER 766 * r29 = LR 767 * r30 = LSW of counter 768 * r31 = MSW of counter 769 */ 770 mtlr %r29 /* restore Link Register */ 771 mtxer %r28 /* restore XER */ 772 mtcr %r27 /* restore Condition Register */ 773 mtsprg1 %r26 /* save saved r2 across load multiple */ 774 mfsprg3 %r2 /* get end of save area */ 775 addi %r2,%r2,-4*(32-20) /* adjust save area down */ 776 lmw %r20,0(%r2) /* restore r20-r31 */ 777 mtsprg3 %r2 /* save new end of save area */ 778 mfsprg1 %r2 /* restore r2 */ 779 rfi 780 781 .p2align 4 782 .globl _C_LABEL(exception_init) 783 _C_LABEL(exception_init): 784 lis %r6,_C_LABEL(critical_input_vector)@h 785 mtspr SPR_IVPR, %r6 786 787 ori %r5,%r6,_C_LABEL(critical_input_vector)@l 788 mtspr SPR_IVOR0, %r5 789 790 ori %r5,%r6,_C_LABEL(machine_check_vector)@l 791 mtspr SPR_IVOR1, %r5 792 793 ori %r5,%r6,_C_LABEL(data_storage_vector)@l 794 mtspr SPR_IVOR2, %r5 795 796 ori %r5,%r6,_C_LABEL(instruction_storage_vector)@l 797 mtspr SPR_IVOR3, %r5 798 799 ori %r5,%r6,_C_LABEL(external_input_vector)@l 800 mtspr SPR_IVOR4, %r5 801 802 ori %r5,%r6,_C_LABEL(alignment_vector)@l 803 mtspr SPR_IVOR5, %r5 804 805 ori %r5,%r6,_C_LABEL(program_vector)@l 806 mtspr SPR_IVOR6, %r5 807 808 #ifdef SPR_IVOR7 809 ori %r5,%r6,_C_LABEL(fp_unavailable_vector)@l 810 mtspr SPR_IVOR7, %r5 811 #endif 812 813 ori %r5,%r6,_C_LABEL(system_call_vector)@l 814 mtspr SPR_IVOR8, %r5 815 816 #ifdef SPR_IVOR9 817 ori %r5,%r6,_C_LABEL(ap_unavailable_vector)@l 818 mtspr SPR_IVOR9, %r5 819 #endif 820 821 ori %r5,%r6,_C_LABEL(decrementer_vector)@l 822 mtspr SPR_IVOR10, %r5 823 824 ori %r5,%r6,_C_LABEL(fixed_interval_timer_vector)@l 825 mtspr SPR_IVOR11, %r5 826 827 ori %r5,%r6,_C_LABEL(watchdog_timer_vector)@l 828 mtspr SPR_IVOR12, %r5 829 830 ori %r5,%r6,_C_LABEL(data_tlb_error_vector)@l 831 mtspr SPR_IVOR13, %r5 832 833 ori %r5,%r6,_C_LABEL(instruction_tlb_error_vector)@l 834 mtspr SPR_IVOR14, %r5 835 836 ori %r5,%r6,_C_LABEL(debug_vector)@l 837 mtspr SPR_IVOR15, %r5 838 839 ori %r5,%r6,_C_LABEL(spv_unavailable_vector)@l 840 mtspr SPR_IVOR32, %r5 841 842 ori %r5,%r6,_C_LABEL(fpdata_vector)@l 843 mtspr SPR_IVOR33, %r5 844 845 ori %r5,%r6,_C_LABEL(fpround_vector)@l 846 mtspr SPR_IVOR34, %r5 847 848 ori %r5,%r6,_C_LABEL(perfmon_vector)@l 849 mtspr SPR_IVOR35, %r5 850 851 mfpir %r5 /* get Process ID register */ 852 cmplwi %r5,0 853 bnelr /* return if non-0 (non-primary) */ 854 855 lis %r5,_C_LABEL(powerpc_intrsw)@ha 856 stw %r3,_C_LABEL(powerpc_intrsw)@l(%r5) 857 858 blr 859 860 #ifdef notyet 861 .data 862 .lcomm ddbstk,4096 863 .text 864 865 _ENTRY(cpu_Debugger) 866 mflr %r0 867 stw %r0, CFRAME_LR(%r1) 868 869 mfmsr %r3 870 wrteei 0 871 mr %r4,%r1 872 lis %r10,ddbstk@ha 873 addi %r10,%r10,ddbstk@l 874 sub %r5,%r1,%r10 875 cmplwi %r5,4096 876 blt %cr0, 1f 877 addi %r1,%r10,4096-CALLFRAMELEN 878 1: 879 stwu %r4,-FRAMELEN(%r1) 880 stw %r4,FRAME_R1(%r1) 881 stmw %r13,FRAME_R13(%r1) 882 mr %r26,%r0 883 mfcr %r27 884 mfxer %r28 885 mfctr %r29 886 mr %r30,%r0 887 mr %r31,%r3 888 stmw %r26,FRAME_LR(%r1) 889 mr %r31,%r1 890 mr %r1,%r10 891 addi %r4,%r1,FRAME_TF 892 li %r3,EXC_PGM 893 stw %r3,FRAME_EXC(%r1) 894 li %r3,T_PROGRAM 895 bl _C_LABEL(trap) 896 lmw %r26,FRAME_LR(%r1) 897 mtlr %r26 898 mtcr %r27 899 mtxer %r28 900 mtctr %r29 901 mr %r0,%r31 902 lmw %r13,FRAME_R13(%r1) 903 lwz %r1,FRAME_R1(%r1) 904 wrtee %r0 905 blr 906 #endif /* notyet */ 907
Indexes created Tue Oct 21 19:09:51 GMT 2025