trap_subr.S revision 1.3
1 /* $NetBSD: trap_subr.S,v 1.3 2011/06/05 16:52:24 matt Exp $ */ 2 /*- 3 * Copyright (c) 2010, 2011 The NetBSD Foundation, Inc. 4 * All rights reserved. 5 * 6 * This code is derived from software contributed to The NetBSD Foundation 7 * by Raytheon BBN Technologies Corp and Defense Advanced Research Projects 8 * Agency and which was developed by Matt Thomas of 3am Software Foundry. 9 * 10 * This material is based upon work supported by the Defense Advanced Research 11 * Projects Agency and Space and Naval Warfare Systems Center, Pacific, under 12 * Contract No. N66001-09-C-2073. 13 * Approved for Public Release, Distribution Unlimited 14 * 15 * Redistribution and use in source and binary forms, with or without 16 * modification, are permitted provided that the following conditions 17 * are met: 18 * 1. Redistributions of source code must retain the above copyright 19 * notice, this list of conditions and the following disclaimer. 20 * 2. Redistributions in binary form must reproduce the above copyright 21 * notice, this list of conditions and the following disclaimer in the 22 * documentation and/or other materials provided with the distribution. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 25 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 26 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 27 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 28 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 29 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 30 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 31 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 32 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 33 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 34 * POSSIBILITY OF SUCH DAMAGE. 35 */ 36 37 RCSID("$NetBSD: trap_subr.S,v 1.3 2011/06/05 16:52:24 matt Exp $") 38 39 .globl _C_LABEL(critintr_call) 40 .globl _C_LABEL(extintr_call) 41 .globl _C_LABEL(decrintr_call) 42 .globl _C_LABEL(fitintr_call) 43 .globl _C_LABEL(wdogintr_call) 44 45 .globl _C_LABEL(sctrapexit), _C_LABEL(trapexit), _C_LABEL(intrcall) 46 47 /* 48 * We have a problem with critical (MSR[CE] or machine check (MSR[ME]) 49 * or debug (MSR[DE]) interrupts/exception in that they could happen 50 * inbewtween the mtsprg1 %r2 and mfsprg1 %r2. If that happens, %r2 51 * will be lost. Even if we moved to a different sprg, subsequent 52 * expceptions would use SPRG1 and its value would be lost. The only 53 * way to be safe for CE/ME/DE faults to save and restore SPRG1. 54 * 55 * Since CE/ME/DE faults may happen anytime, we need r1 to always 56 * contain a valid kernel stack pointer. Therefore we use r2 as 57 * our temporary register. 58 * 59 * To prevent %r2 being overwritten, each "level" (normal, critical, 60 * mchk) uses a unique sprg to save %r2 (sprg1, sprg4, sprg5). 61 * 62 * Since we can't control how many nested exceptions we might get, 63 * we don't use a dedicated save area. Instead we have a upwards 64 * growing "stack" of them; the pointer to which is kept in sprg3. 65 * 66 * To allocate from the stack, one fetches sprg3, adds the amount 67 * needed, saves sprg3, and then refers to the save using a 68 * displacement of -amount. 69 */ 70 #define FRAME_EXC_PROLOGUE(start, sprg, srr) \ 71 mt##sprg %r2; /* save r2 */ \ 72 mfsprg3 %r2; /* get save_area pointer */ \ 73 addi %r2,%r2,4*(32-start); \ 74 /* allocate save area */ \ 75 mtsprg3 %r2; /* save updated pointer */ \ 76 stmw %r##start,-4*(32-start)(%r2); \ 77 /* free r24-r31 for use */ \ 78 mf##sprg %r26; /* get saved r2 */ \ 79 mfcr %r27; /* get Condition Register */ \ 80 mfxer %r28; /* get XER */ \ 81 mfspr %r30, SPR_##srr##0; /* get SRR0 */ \ 82 mfspr %r31, SPR_##srr##1 /* get SRR1 */ 83 84 #define PROLOGUE_GET_DEAR mfspr %r24, SPR_DEAR 85 #define PROLOGUE_GET_ESR mfspr %r25, SPR_ESR 86 #define PROLOGUE_GET_SRRS mfsrr0 %r24; \ 87 mfsrr1 %r25 88 #define PROLOGUE_GET_SPRG1 mfsprg1 %r29 89 #define PROLOGUE_GET_DBSR mfspr %r25, SPR_DBSR 90 #define SAVE_ESR stw %r25, FRAME_ESR(%r1) 91 #define SAVE_DEAR stw %r24, FRAME_DEAR(%r1) 92 #define SAVE_DEAR_ESR SAVE_ESR; SAVE_DEAR 93 #define SAVE_SRRS SAVE_DEAR_ESR 94 #define SAVE_SPRG1 stw %r29, FRAME_SPRG1(%r1) 95 #define SAVE_DBSR stw %r25, FRAME_DBSR(%r1) 96 #define SAVE_NOTHING /* nothing */ 97 #define RESTORE_SPRG1(r) lwz r, FRAME_SPRG1(%r1); \ 98 mtsprg1 r 99 #define RESTORE_SRR0(r) lwz r, FRAME_DEAR(%r1); \ 100 mtsrr0 r 101 #define RESTORE_SRR1(r) lwz r, FRAME_ESR(%r1); \ 102 mtsrr1 r 103 104 #define FRAME_PROLOGUE \ 105 FRAME_EXC_PROLOGUE(26, sprg1, SRR) 106 107 #define FRAME_PROLOGUE_DEAR_ESR \ 108 FRAME_EXC_PROLOGUE(24, sprg1, SRR); \ 109 PROLOGUE_GET_ESR; \ 110 PROLOGUE_GET_DEAR 111 112 #define FRAME_PROLOGUE_ESR \ 113 FRAME_EXC_PROLOGUE(25, sprg1, SRR); \ 114 PROLOGUE_GET_ESR 115 116 #define FRAME_TLBPROLOGUE \ 117 FRAME_EXC_PROLOGUE(20, sprg1, SRR); \ 118 PROLOGUE_GET_ESR; \ 119 PROLOGUE_GET_DEAR 120 121 #define FRAME_INTR_PROLOGUE \ 122 FRAME_EXC_PROLOGUE(26, sprg1, SRR) 123 124 /* 125 * These need to save SRR0/SRR1 as well their SRR0/SRR1 in case normal 126 * exceptions happened during their execution. 127 */ 128 #define FRAME_CRIT_PROLOGUE \ 129 FRAME_EXC_PROLOGUE(24, sprg4, CSRR); \ 130 PROLOGUE_GET_SPRG1; \ 131 PROLOGUE_GET_SRRS 132 133 #define FRAME_MCHK_PROLOGUE \ 134 FRAME_EXC_PROLOGUE(24, sprg5, MCSRR); \ 135 PROLOGUE_GET_SPRG1; \ 136 PROLOGUE_GET_SRRS 137 138 #define FRAME_DEBUG_PROLOGUE \ 139 FRAME_EXC_PROLOGUE(24, sprg4, CSRR); \ 140 PROLOGUE_GET_SPRG1; \ 141 PROLOGUE_GET_SRRS 142 143 /* 144 * DDB expects to fetch the LR from the previous frame. But it also 145 * expects to be pointing at the instruction after the branch link. Since 146 * we didn't branch, we need to advance it by to fake out DDB. 147 */ 148 #ifdef DDB 149 #define FRAME_SAVE_SRR0_FOR_DDB \ 150 addi %r30, %r30, 4; /* ddb thinks its the next insn */ \ 151 stw %r30, FRAMELEN+SZREG(%r1); /* appease ddb stacktrace */ 152 #else 153 #define FRAME_SAVE_SRR0_FOR_DDB 154 #endif 155 156 #ifdef PPC_HAVE_SPE 157 #define FRAME_SAVE_SPEFSCR \ 158 mfspefscr %r0; /* get spefscr */ \ 159 stw %r0, FRAME_SPEFSCR(%r1) /* save into trapframe */ 160 #define FRAME_RESTORE_SPEFSCR \ 161 lwz %r0, FRAME_SPEFSCR(%r1); /* fetch from trapframe */ \ 162 mtspefscr %r0 /* save spefscr */ 163 #else 164 #define FRAME_SAVE_SPEFSCR 165 #define FRAME_RESTORE_SPEFSCR 166 #endif 167 /* 168 * Before the first memory refernence, we must have our state inside registers 169 * since the first memory access might cause an exception which would cause 170 * SRR0/SRR1 and DEAR/ESR to become unrecoverable. CR and XER also need to be 171 * saved early since they will modified by instrction flow. The saved stack 172 * pointer is also critical but LR and CTR can be deferred being saved until 173 * we are actually filling a trapframe. 174 */ 175 #define FRAME_EXC_ENTER(exc, tf, start, save_prologue) \ 176 mtcr %r31; /* user mode exception? */ \ 177 mr %r31, %r1; /* save SP (SRR1 is safe in CR) */ \ 178 bf MSR_PR, 1f; /* nope, sp is good */ \ 179 mfsprg2 %r2; /* get curlwp */ \ 180 lwz %r2, L_PCB(%r2); /* get uarea of curlwp */ \ 181 addi %r1, %r2, USPACE-CALLFRAMELEN; \ 182 /* start stack at top of it */ \ 183 1: \ 184 stw %r30, CFRAME_LR(%r1); /* save in previous callframe */ \ 185 stwu %r31, -FRAMELEN(%r1); /* get space for trapframe */ \ 186 stw %r0, FRAME_0(%r1); /* save r0 */ \ 187 stw %r31, FRAME_1(%r1); /* save (saved) r1 */ \ 188 stw %r26, FRAME_2(%r1); /* save (saved) r2 */ \ 189 save_prologue; /* save SPRG1/ESR/DEAR */ \ 190 /* At this point, r26, r29, and r31 have been saved so we */ \ 191 /* can use them for LR, CTR, and SRR1. */ \ 192 mflr %r26; /* get Link Register */ \ 193 mfctr %r29; /* get CTR */ \ 194 mfcr %r31; /* get SRR1 */ \ 195 stmw %r26, FRAME_LR(%r1); /* save LR CR XER CTR SRR0/1 */ \ 196 FRAME_SAVE_SRR0_FOR_DDB; \ 197 mr %r0, %r31; /* save SRR1 for a bit */ \ 198 mfsprg3 %r2; /* get save_area pointer */ \ 199 addi %r2,%r2,-4*(32-start); /* find our save area */ \ 200 lmw %r##start,0(%r2); /* get start-r31 */ \ 201 mtsprg3 %r2; /* save updated pointer */ \ 202 stmw %r3, FRAME_3(%r1); /* save r2-r31 */ \ 203 /* Now everything has been saved */ \ 204 mr %r31, %r0; /* move SRR1 back to r31 */ \ 205 mfsprg2 %r13; /* put curlwp in r13 */ \ 206 FRAME_SAVE_SPEFSCR; \ 207 li %r7, exc; /* load EXC_* */ \ 208 stw %r7, FRAME_EXC(%r1); /* save into trapframe */ \ 209 addi tf, %r1, FRAME_TF /* get address of trap frame */ 210 211 #define FRAME_EXC_EXIT(rfi, srr) \ 212 lmw %r26, FRAME_LR(%r1); /* get LR CR XER CTR SRR0/1 */ \ 213 oris %r31,%r31,PSL_CE@h; \ 214 mtspr SPR_##srr##1, %r31; /* restore SRR1 */ \ 215 mtspr SPR_##srr##0, %r30; /* restore SRR0 */ \ 216 FRAME_RESTORE_SPEFSCR; \ 217 mtctr %r29; /* restore CTR */ \ 218 mtxer %r28; /* restore XER */ \ 219 mtcr %r27; /* restore CR */ \ 220 mtlr %r26; /* restore LR */ \ 221 lmw %r2, FRAME_2(%r1); /* restore r2-r31 */ \ 222 lwz %r0, FRAME_0(%r1); /* restore r0 */ \ 223 lwz %r1, FRAME_1(%r1); /* restore r1 */ \ 224 rfi /* return from interrupt */ 225 226 227 #define FRAME_ENTER(exc, tf) \ 228 FRAME_EXC_ENTER(exc, tf, 26, SAVE_NOTHING) 229 230 #define FRAME_ENTER_ESR(exc, tf) \ 231 FRAME_EXC_ENTER(exc, tf, 25, SAVE_ESR) 232 233 #define FRAME_ENTER_DEAR_ESR(exc, tf) \ 234 FRAME_EXC_ENTER(exc, tf, 24, SAVE_DEAR_ESR) 235 236 #define FRAME_EXIT FRAME_EXC_EXIT(rfi, SRR) 237 238 #define FRAME_TLBENTER(exc) \ 239 FRAME_EXC_ENTER(exc, %r4, 20, SAVE_DEAR_ESR) 240 #define FRAME_TLBEXIT FRAME_EXC_EXIT(rfi, SRR) 241 242 #define FRAME_MCHK_ENTER(exc) \ 243 FRAME_EXC_ENTER(exc, %r3, 26, SAVE_SPRG1; SAVE_SRRS) 244 #define FRAME_MCHK_EXIT \ 245 RESTORE_SRR0(%r28); \ 246 RESTORE_SRR1(%r27); \ 247 RESTORE_SPRG1(%r26); \ 248 FRAME_EXC_EXIT(rfmci, MCSRR) 249 250 #define FRAME_DEBUG_ENTER(exc) \ 251 FRAME_EXC_ENTER(exc, %r4, 26, SAVE_SPRG1; SAVE_SRRS) 252 #define FRAME_DEBUG_EXIT \ 253 RESTORE_SPRG1(%r26); FRAME_EXC_EXIT(rfci, CSRR) 254 255 #define FRAME_INTR_SP \ 256 bf MSR_PR, 1f; /* nope, sp is good */ \ 257 mfsprg2 %r2; /* get curlwp */ \ 258 lwz %r2, L_PCB(%r2); /* get uarea of curlwp */ \ 259 addi %r1, %r2, USPACE-CALLFRAMELEN; \ 260 /* start stack at top of it */ \ 261 1: 262 263 #define FRAME_INTR_SP_NEW(sym) \ 264 lis %r2,(sym)@ha; \ 265 addi %r1,%r2,(sym)@l 266 267 #define FRAME_INTR_XENTER(exc, start, get_intr_sp, save_prologue) \ 268 mtcr %r31; /* user mode exception? */ \ 269 mr %r31, %r1; /* save SP (SRR1 is safe in CR) */ \ 270 get_intr_sp; /* get kernel stack pointer */ \ 271 stw %r30, CFRAME_LR(%r1); /* save in .. */ \ 272 stwu %r31, -FRAMELEN(%r1); /* get space for trapframe */ \ 273 stw %r0, FRAME_0(%r1); /* save r0 */ \ 274 stw %r31, FRAME_1(%r1); /* save (saved) r1 */ \ 275 stw %r26, FRAME_2(%r1); /* save (saved) r2 */ \ 276 save_prologue; /* save SPRG1 (maybe) */ \ 277 mflr %r26; /* get LR */ \ 278 mfctr %r29; /* get CTR */ \ 279 mfcr %r31; /* get SRR1 */ \ 280 stmw %r26, FRAME_LR(%r1); /* save LR CR XER CTR SRR0/1 */ \ 281 FRAME_SAVE_SRR0_FOR_DDB; \ 282 stw %r3, FRAME_3(%r1); /* save r3 */ \ 283 stw %r4, FRAME_4(%r1); /* save r4 */ \ 284 stw %r5, FRAME_5(%r1); /* save r5 */ \ 285 stw %r6, FRAME_6(%r1); /* save r6 */ \ 286 stw %r7, FRAME_7(%r1); /* save r7 */ \ 287 stw %r8, FRAME_8(%r1); /* save r8 */ \ 288 stw %r9, FRAME_9(%r1); /* save r9 */ \ 289 stw %r10, FRAME_10(%r1); /* save r10 */ \ 290 stw %r11, FRAME_11(%r1); /* save r11 */ \ 291 stw %r12, FRAME_12(%r1); /* save r12 */ \ 292 stw %r13, FRAME_13(%r1); /* save r13 */ \ 293 mfsprg3 %r2; /* get save_area pointer */ \ 294 addi %r2,%r2,-4*(32-start); /* find our save area */ \ 295 lmw %r##start,0(%r2); /* get start-r31 */ \ 296 mtsprg3 %r2; /* save updated pointer */ \ 297 mfsprg2 %r13; /* put curlwp into r13 */ \ 298 li %r7, exc; /* load EXC_* */ \ 299 stw %r7, FRAME_EXC(%r1); /* save into trapframe */ \ 300 addi %r3, %r1, FRAME_TF /* only argument is trapframe */ 301 302 #define FRAME_INTR_XEXIT(rfi, srr) \ 303 lwz %r8, FRAME_LR(%r1); /* get LR */ \ 304 lwz %r9, FRAME_CR(%r1); /* get CR */ \ 305 lwz %r10, FRAME_XER(%r1); /* get XER */ \ 306 lwz %r11, FRAME_CTR(%r1); /* get CTR */ \ 307 lwz %r12, FRAME_SRR0(%r1); /* get SRR0 */ \ 308 lwz %r13, FRAME_SRR1(%r1); /* get SRR1 */ \ 309 mtspr SPR_##srr##1, %r13; /* restore SRR1 */ \ 310 mtspr SPR_##srr##0, %r12; /* restore SRR0 */ \ 311 mtctr %r11; /* restore CTR */ \ 312 mtxer %r10; /* restore XER */ \ 313 mtcr %r9; /* restore CR */ \ 314 mtlr %r8; /* restore LR */ \ 315 lwz %r13, FRAME_13(%r1); /* restore r13 */ \ 316 lwz %r12, FRAME_12(%r1); /* restore r12 */ \ 317 lwz %r11, FRAME_11(%r1); /* restore r11 */ \ 318 lwz %r10, FRAME_10(%r1); /* restore r10 */ \ 319 lwz %r9, FRAME_9(%r1); /* restore r9 */ \ 320 lwz %r8, FRAME_8(%r1); /* restore r8 */ \ 321 lwz %r7, FRAME_7(%r1); /* restore r7 */ \ 322 lwz %r6, FRAME_6(%r1); /* restore r6 */ \ 323 lwz %r5, FRAME_5(%r1); /* restore r5 */ \ 324 lwz %r4, FRAME_4(%r1); /* restore r4 */ \ 325 lwz %r3, FRAME_3(%r1); /* restore r3 */ \ 326 lwz %r2, FRAME_2(%r1); /* restore r2 */ \ 327 lwz %r0, FRAME_0(%r1); /* restore r0 */ \ 328 lwz %r1, FRAME_1(%r1); /* restore r1 */ \ 329 rfi /* return from interrupt */ 330 331 #define FRAME_INTR_ENTER(exc) \ 332 FRAME_INTR_XENTER(exc, 26, FRAME_INTR_SP, SAVE_NOTHING) 333 #define FRAME_INTR_EXIT \ 334 FRAME_INTR_XEXIT(rfi, SRR) 335 #define FRAME_CRIT_ENTER(exc) \ 336 FRAME_INTR_XENTER(exc, 24, FRAME_INTR_SP, SAVE_SPRG1) 337 #define FRAME_WDOG_ENTER(exc, sym) \ 338 FRAME_INTR_XENTER(exc, 24, FRAME_INTR_SP_NEW(sym), SAVE_SPRG1) 339 #define FRAME_CRIT_EXIT \ 340 RESTORE_SRR0(%r4); \ 341 RESTORE_SRR1(%r5); \ 342 RESTORE_SPRG1(%r6); \ 343 FRAME_INTR_XEXIT(rfci, CSRR) 344 345 .text 346 .p2align 4 347 _C_LABEL(critical_input_vector): 348 /* MSR[ME] is unchanged, all others cleared */ 349 FRAME_CRIT_PROLOGUE /* save SP r26-31 CR LR XER */ 350 FRAME_CRIT_ENTER(EXC_CII) 351 _C_LABEL(critintr_call): 352 bl _C_LABEL(critintr_call) /* critintr(tf) */ 353 FRAME_CRIT_EXIT 354 355 .p2align 4 356 _C_LABEL(machine_check_vector): 357 /* all MSR bits are cleared */ 358 FRAME_MCHK_PROLOGUE /* save SP r25-31 CR LR XER */ 359 FRAME_MCHK_ENTER(EXC_MCHK) 360 /* 361 * MCAR/MCSR don't need to be saved early since MSR[ME] is cleared 362 * on entry. 363 */ 364 mfspr %r7, SPR_MCAR 365 mfspr %r6, SPR_MCSR 366 stw %r6, FRAME_MCSR(%r1) 367 stw %r7, FRAME_MCAR(%r1) 368 li %r3, T_MACHINE_CHECK 369 bl _C_LABEL(trap) /* trap(T_MACHINE_CHECK, tf) */ 370 FRAME_MCHK_EXIT 371 372 .p2align 4 373 _C_LABEL(data_storage_vector): 374 /* MSR[CE], MSR[ME], MSR[DE] are unchanged, all others cleared */ 375 FRAME_PROLOGUE_DEAR_ESR /* save r2 DEAR ESR r24-31 CR XER SRR */ 376 FRAME_ENTER_DEAR_ESR(EXC_DSI, %r4) 377 li %r3, T_DSI 378 /* FRAME_ENTER leaves SRR1 in %r31 */ 379 trapenter: 380 trapagain: 381 wrtee %r31 /* restore MSR[EE] */ 382 383 bl _C_LABEL(trap) /* trap(trapcode, tf) */ 384 _C_LABEL(trapexit): 385 wrteei 0 /* disable interrupts */ 386 # andis. %r0, %r31, PSL_CE@h 387 # tweqi %r0, 0 388 andi. %r4, %r31, PSL_PR /* lets look at PSL_PR */ 389 beq trapdone /* if clear, skip to exit */ 390 lwz %r4, L_MD_ASTPENDING(%r13) /* get ast pending */ 391 cmplwi %r4, 0 /* is there an ast pending */ 392 beq+ trapdone /* nope, proceed to exit */ 393 li %r6, EXC_AST /* yes. */ 394 stw %r6, FRAME_EXC(%r1) /* pretend this is an AST */ 395 addi %r4, %r1, FRAME_TF /* get address of trap frame */ 396 li %r3, T_AST 397 b trapagain /* and deal with it */ 398 trapdone: 399 FRAME_EXIT 400 401 .p2align 4 402 _C_LABEL(instruction_storage_vector): 403 /* MSR[CE], MSR[ME], MSR[DE] are unchanged, all others cleared */ 404 FRAME_PROLOGUE_ESR /* save ESR r2 r25-31 CR XER SRR0/1 */ 405 FRAME_ENTER_ESR(EXC_ISI, %r4) 406 li %r3, T_ISI 407 b trapenter 408 409 .p2align 4 410 _ENTRY(external_input_vector) 411 /* MSR[CE], MSR[ME], MSR[DE] are unchanged, all others cleared */ 412 FRAME_INTR_PROLOGUE /* save SP r25-31 CR LR XER */ 413 FRAME_INTR_ENTER(EXC_EXI) 414 415 _C_LABEL(extintr_call): 416 bl _C_LABEL(extintr_call) 417 _C_LABEL(intrcall): 418 GET_CPUINFO(%r6) /* get curcpu() */ 419 lwz %r5, FRAME_SRR1(%r1) /* get saved SRR1 */ 420 # andis. %r0, %r5, PSL_CE@h 421 # tweqi %r0, 0 422 andi. %r4, %r5, PSL_PR /* lets look at PSL_PR */ 423 beq intrexit /* if clear, skip to exit */ 424 lwz %r4, L_MD_ASTPENDING(%r13) /* get ast pending */ 425 cmplwi %r4, 0 /* is there an ast pending */ 426 beq+ intrexit /* nope, proceed to exit */ 427 stmw %r14, FRAME_14(%r1) /* save rest of registers */ 428 FRAME_SAVE_SPEFSCR 429 mr %r31, %r5 /* needed for trapagain */ 430 li %r4, EXC_AST /* */ 431 stw %r4, FRAME_EXC(%r1) /* pretend this is an AST */ 432 addi %r4, %r1, FRAME_TF /* get address of trap frame */ 433 li %r3, T_AST 434 b trapagain /* and deal with it */ 435 intrexit: 436 FRAME_INTR_EXIT 437 438 .p2align 4 439 _C_LABEL(alignment_vector): 440 /* MSR[CE], MSR[ME], MSR[DE] are unchanged, all others cleared */ 441 FRAME_PROLOGUE_DEAR_ESR /* save SP r25-31 CR LR XER */ 442 FRAME_ENTER_DEAR_ESR(EXC_ALI, %r4) 443 li %r3, T_ALIGNMENT 444 b trapenter 445 446 .p2align 4 447 _C_LABEL(program_vector): 448 /* MSR[CE], MSR[ME], MSR[DE] are unchanged, all others cleared */ 449 FRAME_PROLOGUE_ESR /* save SP r25-31 CR LR XER */ 450 FRAME_ENTER_ESR(EXC_PGM, %r4) 451 li %r3, T_PROGRAM 452 b trapenter 453 454 #ifdef SPR_IVOR7 455 .p2align 4 456 _C_LABEL(fp_unavailable_vector): 457 /* MSR[CE], MSR[ME], MSR[DE] are unchanged, all others cleared */ 458 FRAME_PROLOGUE_ESR /* save SP r25-31 CR LR XER */ 459 FRAME_ENTER_ESR(EXC_FPU, %r4) 460 li %r3, T_FP_UNAVAILABLE 461 b trapenter 462 #endif 463 464 .p2align 4 465 _C_LABEL(system_call_vector): 466 /* MSR[CE], MSR[ME], MSR[DE] are unchanged, all others cleared */ 467 FRAME_PROLOGUE /* save SP r26-31 CR LR XER */ 468 FRAME_ENTER(EXC_SC, %r3) 469 470 wrteei 1 /* enable interrupts */ 471 lwz %r7, L_PROC(%r13) /* get proc for lwp */ 472 lwz %r8, P_MD_SYSCALL(%r7) /* get syscall */ 473 mtlr %r8 /* need to call indirect */ 474 blrl /* syscall(tf) */ 475 _C_LABEL(sctrapexit): 476 wrteei 0 /* disable interrupts */ 477 mfsprg2 %r3 /* get curlwp */ 478 lwz %r4, L_MD_ASTPENDING(%r13) /* get ast pending */ 479 cmplwi %r4, 0 /* is there an ast pending */ 480 beq+ trapdone /* nope, proceed to exit */ 481 li %r0, EXC_AST /* yes. */ 482 stw %r0, FRAME_EXC(%r1) /* pretend this is an AST */ 483 addi %r4, %r1, FRAME_TF /* get address of trap frame */ 484 li %r3, T_AST 485 b trapenter /* and deal with it */ 486 487 #ifdef SPR_IVOR9 488 .p2align 4 489 _C_LABEL(ap_unavailable_vector): 490 /* MSR[CE], MSR[ME], MSR[DE] are unchanged, all others cleared */ 491 FRAME_PROLOGUE /* save SP r25-31 CR LR XER */ 492 FRAME_ENTER(EXC_PGM, %r4) 493 li %r3, T_AP_UNAVAILABLE 494 b trapenter 495 #endif 496 497 .p2align 4 498 _C_LABEL(decrementer_vector): 499 /* MSR[CE], MSR[ME], MSR[DE] are unchanged, all others cleared */ 500 FRAME_INTR_PROLOGUE /* save SP r25-31 CR LR XER */ 501 FRAME_INTR_ENTER(EXC_DECR) 502 503 _C_LABEL(decrintr_call): 504 bl _C_LABEL(decrintr_call) 505 b intrexit 506 507 .p2align 4 508 _C_LABEL(fixed_interval_timer_vector): 509 /* MSR[CE], MSR[ME], MSR[DE] are unchanged, all others cleared */ 510 FRAME_PROLOGUE /* save SP r25-31 CR LR XER */ 511 FRAME_INTR_ENTER(EXC_FIT) 512 513 _C_LABEL(fitintr_call): 514 bl _C_LABEL(fitintr_call) 515 b intrexit 516 517 .data 518 .lcomm wdogstk,4096 519 .text 520 .p2align 4 521 _C_LABEL(watchdog_timer_vector): 522 /* MSR[CE], MSR[ME], MSR[DE] are unchanged, all others cleared */ 523 FRAME_CRIT_PROLOGUE /* save SP r25-31 CR LR XER */ 524 FRAME_WDOG_ENTER(EXC_WDOG, wdogstk+4096-CALLFRAMELEN) 525 #if 0 526 lis %r11,0xe0004641@ha 527 addi %r11,%r11,0xe0004641@l 528 stb %r11,-65(%r11) 529 #endif 530 531 _C_LABEL(wdogintr_call): 532 bl _C_LABEL(wdogintr_call) 533 FRAME_CRIT_EXIT 534 535 .p2align 4 536 _C_LABEL(data_tlb_error_vector): 537 /* MSR[CE], MSR[ME], MSR[DE] are unchanged, all others cleared */ 538 FRAME_TLBPROLOGUE 539 /* 540 * Registers as this point: 541 * 542 * r2 = cpu_info 543 * r20 = scratch 544 * r21 = scratch 545 * r22 = scratch 546 * r23 = scratch 547 * r24 = DEAR 548 * r25 = ESR 549 * r26 = saved r2 550 * r27 = CR 551 * r28 = XER 552 * r29 = scratch 553 * r30 = SRR0 554 * r31 = SRR1 555 * 556 * Except for r29, these values must be retained. However we must 557 * be cognizant of nesting. There are two cases here, both related. 558 * 559 * We get a critical input or machine check exception and the kernel 560 * stack doesn't have a TLB entry so we take an exception. The other 561 * nesting path is some page used by the exception handler will cause 562 * a TLB data error. 563 * 564 * The second case (more probable) is that the PTE loading will fail 565 * so we will have to do a hard trap to resolve it. But in doing so 566 * we need to save a trapframe which could result in another DTLB 567 * fault. 568 * 569 * In all cases, the save area stack shall protect us. 570 */ 571 /* 572 * Attempt to update the TLB from the page table. 573 */ 574 mflr %r29 /* save LR */ 575 mr %r23, %r24 /* address of exception */ 576 rlwinm %r22, %r31, /* index into ci_pmap_segtab */\ 577 MSR_DS+PTR_SCALESHIFT+1, \ 578 31-PTR_SCALESHIFT, \ 579 31-PTR_SCALESHIFT /* move PSL_DS[27] to bit 29 */ 580 bl pte_load 581 mtlr %r29 /* restore LR */ 582 /* 583 * If we returned, pte load failed so let trap deal with it but 584 * has kept the contents of r24-r31 (expect r29) intact. 585 */ 586 FRAME_TLBENTER(EXC_DSI) 587 li %r3, T_DATA_TLB_ERROR 588 b trapenter 589 590 .p2align 4 591 _C_LABEL(instruction_tlb_error_vector): 592 /* MSR[CE], MSR[ME], MSR[DE] are unchanged, all others cleared */ 593 FRAME_TLBPROLOGUE 594 /* 595 * Attempt to update the TLB from the page table. 596 */ 597 mflr %r29 /* save LR */ 598 mr %r23, %r30 /* PC of exception */ 599 rlwinm %r22, %r31, /* index into ci_pmap_segtab */\ 600 MSR_IS+PTR_SCALESHIFT+1, \ 601 31-PTR_SCALESHIFT, \ 602 31-PTR_SCALESHIFT /* move PSL_IS[26] to bit 29 */ 603 bl pte_load 604 mtlr %r29 /* restore LR */ 605 /* 606 * If we returned, pte load failed so let trap deal with it but 607 * has kept the contents of r24-r31 (expect r29) intact. 608 */ 609 FRAME_TLBENTER(EXC_ISI) 610 li %r3, T_INSTRUCTION_TLB_ERROR 611 b trapenter 612 613 .p2align 4 614 _C_LABEL(debug_vector): 615 FRAME_CRIT_PROLOGUE /* save SP r25-31 CR LR XER */ 616 FRAME_CRIT_ENTER(EXC_DEBUG) 617 mfspr %r6, SPR_DBSR 618 stw %r6, FRAME_ESR(%r1) 619 li %r3, T_DEBUG 620 bl _C_LABEL(trap) 621 FRAME_CRIT_EXIT 622 623 .p2align 4 624 _C_LABEL(spv_unavailable_vector): 625 FRAME_PROLOGUE_ESR /* save SP r25-31 CR LR XER */ 626 FRAME_ENTER_ESR(EXC_VEC, %r4) 627 li %r3, T_SPE_UNAVAILABLE 628 b trapenter 629 630 .p2align 4 631 _C_LABEL(fpdata_vector): 632 FRAME_PROLOGUE_ESR /* save SP r25-31 CR LR XER */ 633 FRAME_ENTER_ESR(EXC_FPA, %r4) 634 li %r3, T_EMBEDDED_FP_DATA 635 b trapenter 636 637 .p2align 4 638 _C_LABEL(fpround_vector): 639 FRAME_PROLOGUE_ESR /* save SP r25-31 CR LR XER */ 640 FRAME_ENTER_ESR(EXC_FPA, %r4) 641 li %r3, T_EMBEDDED_FP_ROUND 642 b trapenter 643 644 .p2align 4 645 _C_LABEL(perfmon_vector): 646 FRAME_PROLOGUE_ESR /* save SP r25-31 CR LR XER */ 647 FRAME_ENTER_ESR(EXC_PERF, %r4) 648 li %r3, T_EMBEDDED_PERF_MONITOR 649 b trapenter 650 651 .p2align 4 652 pte_load: 653 /* 654 * r2 = scratch 655 * r20 = scratch 656 * r21 = scratch 657 * r22 = index into ci_pmap_{kern,user}_segtab 658 * r23 = faulting address 659 * The rest are for reference and aren't modifiable. If the load 660 * fails, they will be used by FRAME_TLBENTER to create the trapframe. 661 * r24 = DEAR 662 * r25 = ESR 663 * r26 = saved r2 664 * r27 = CR 665 * r28 = XER 666 * r29 = LR 667 * r30 = SRR0 668 * r31 = SRR1 669 */ 670 cmplwi %cr2, %r22, 0 /* remember address space */ 671 GET_CPUINFO(%r2) 672 addi %r22, %r22, CI_PMAP_SEGTAB /* index into segtab(s) */ 673 lwzx %r20, %r22, %r2 /* load kern/user L1 PT addr */ 674 cmplwi %r20, 0 /* is segtab null? */ 675 beqlr %cr0 /* yes, return to fallback to trap */ 676 677 rlwinm %r22, %r23, NSEGPG_SCALESHIFT + PTR_SCALESHIFT, \ 678 31-(NSEGPG_SCALESHIFT + PTR_SCALESHIFT - 1), \ 679 31-PTR_SCALESHIFT /* extract addr bits [0:9] to [20:29] */ 680 lwzx %r20, %r22, %r20 /* load address of page table page */ 681 cmplwi %r20, 0 /* is page null? */ 682 beqlr %cr0 /* yes, return to fallback to trap */ 683 684 rlwinm %r22, %r23, \ 685 NSEGPG_SCALESHIFT + NPTEPG_SCALESHIFT + PTE_SCALESHIFT, \ 686 31-(NPTEPG_SCALESHIFT + PTE_SCALESHIFT - 1), \ 687 31-PTE_SCALESHIFT /* extract addr bits [10:19] to [20:29] */ 688 lwzx %r20, %r22, %r20 /* load PTE from page table page */ 689 cmplwi %r20, 0 /* is there a valid PTE? */ 690 beqlr %cr0 /* no, return to fallback to trap */ 691 692 #if (PTE_UNSYNCED << 1) != PTE_xX 693 #error PTE_UNSYNCED definition error 694 #endif 695 #if (PTE_UNMODIFIED << 1) != PTE_xW 696 #error PTE_UNMODIFIED definition error 697 #endif 698 andi. %r22, %r20, (PTE_UNSYNCED|PTE_UNMODIFIED) 699 /* Does the PTE need to be changed? */ 700 rotlwi %r22, %r22, 1 /* if so, clear the right PTE bits */ 701 andc %r20, %r20, %r22 /* pte &= ~((pte & (PTE_UNSYNCED|PTE_UNMODIFIED)) << 1)*/ 702 703 /* 704 * r24-r32 = (no touch) 705 * r23 = scratch (was fault addr) 706 * r22 = scratch 707 * r21 = scratch 708 * r20 = pte 709 * cr2 = AS 0=eq/!0=ne 710 */ 711 712 /* 713 * This is all E500 specific. We should have a patchable branch 714 * to support other BookE (440) implementations. 715 */ 716 e500_pte_load: 717 bne+ %cr2, 1f /* user access? MAS1 is ok. */ 718 mfspr %r22, SPR_MAS1 /* get MAS1 */ 719 lis %r21, MAS1_TID@h /* get TID mask */ 720 andc %r22, %r22, %r21 /* clear TID */ 721 mtspr SPR_MAS1, %r22 /* save MAS1 */ 722 1: 723 andi. %r21, %r20, PTE_WIMGE_MASK /* extract WIMGE from PTE */ 724 cmplwi %r21, PTE_M /* if just PTE_M is set, */ 725 beq+ %cr0, 2f /* skip munging mas2 */ 726 mfspr %r22, SPR_MAS2 /* get MAS2 (updated by error) */ 727 clrrwi %r22, %r22, PTE_RWX_SHIFT /* clear WIMGE bits */ 728 or %r22, %r22, %r21 /* combine with MAS2 contents */ 729 mtspr SPR_MAS2, %r22 /* put back into MAS2 */ 730 2: 731 /* 732 * r23 = fault addr 733 * r22 = scratch 734 * r21 = scratch 735 * r20 = pte 736 */ 737 738 /* 739 * In MAS3, the protection bits are in the low 6 bits: 740 * UX SX UW SW UR SR 741 * The User bits are 1 bit left of their Supervisor counterparts. 742 * Rotate the PTE protection bits left until they wrap around to become 743 * the least significant bits, where the Supervisor protection bits 744 * are located. Increase the rotate amount by 1 to place them where 745 * the User protection bits are located. We get that 1 by extracting 746 * the MAS1[TS] (set for User access) and moving it to bit 31 (LSB). 747 */ 748 mfspr %r21, SPR_MAS1 /* get MAS1 which has TS bit */ 749 extrwi %r21, %r21, 1, 31-MAS1_TS_SHIFT 750 /* extract MAS1_TS to LSB */ 751 clrrwi %r23, %r20, PAGE_SHIFT /* clear non-RPN bits from PTE */ 752 andi. %r20, %r20, PTE_RWX_MASK /* isolate protection bits */ 753 rotrwi %r20, %r20, PTE_RWX_SHIFT 754 andi. %r22, %r20, (MAS3_SW|MAS3_SR) /* user pages need to be R/W by kernel */ 755 rotlw %r20, %r20, %r21 /* rotate protection to correct loc */ 756 or %r20, %r20, %r22 /* combine system protection bits */ 757 or %r23, %r23, %r20 /* combine RPN and protection bits */ 758 mtspr SPR_MAS3, %r23 /* put into MAS3 */ 759 isync /* because ECORE500RM tells us too */ 760 tlbwe /* write the TLB entry */ 761 /* 762 * Increment a counter to show how many tlb misses we've handled here. 763 */ 764 lmw %r30, CI_EV_TLBMISS_SOFT(%r2) 765 addic %r31, %r31, 1 766 addze %r30, %r30 767 stmw %r30, CI_EV_TLBMISS_SOFT(%r2) 768 /* 769 * Cleanup and leave. We know any higher priority exception will 770 * save and restore SPRG1 and %r2 thereby preserving their values. 771 * 772 * r24 = DEAR (don't care) 773 * r25 = ESR (don't care) 774 * r26 = saved r2 775 * r27 = CR 776 * r28 = XER 777 * r29 = LR 778 * r30 = LSW of counter 779 * r31 = MSW of counter 780 */ 781 mtlr %r29 /* restore Link Register */ 782 mtxer %r28 /* restore XER */ 783 mtcr %r27 /* restore Condition Register */ 784 mtsprg1 %r26 /* save saved r2 across load multiple */ 785 mfsprg3 %r2 /* get end of save area */ 786 addi %r2,%r2,-4*(32-20) /* adjust save area down */ 787 lmw %r20,0(%r2) /* restore r20-r31 */ 788 mtsprg3 %r2 /* save new end of save area */ 789 mfsprg1 %r2 /* restore r2 */ 790 rfi 791 792 .p2align 4 793 .globl _C_LABEL(exception_init) 794 _C_LABEL(exception_init): 795 lis %r6,_C_LABEL(critical_input_vector)@h 796 mtspr SPR_IVPR, %r6 797 798 ori %r5,%r6,_C_LABEL(critical_input_vector)@l 799 mtspr SPR_IVOR0, %r5 800 801 ori %r5,%r6,_C_LABEL(machine_check_vector)@l 802 mtspr SPR_IVOR1, %r5 803 804 ori %r5,%r6,_C_LABEL(data_storage_vector)@l 805 mtspr SPR_IVOR2, %r5 806 807 ori %r5,%r6,_C_LABEL(instruction_storage_vector)@l 808 mtspr SPR_IVOR3, %r5 809 810 ori %r5,%r6,_C_LABEL(external_input_vector)@l 811 mtspr SPR_IVOR4, %r5 812 813 ori %r5,%r6,_C_LABEL(alignment_vector)@l 814 mtspr SPR_IVOR5, %r5 815 816 ori %r5,%r6,_C_LABEL(program_vector)@l 817 mtspr SPR_IVOR6, %r5 818 819 #ifdef SPR_IVOR7 820 ori %r5,%r6,_C_LABEL(fp_unavailable_vector)@l 821 mtspr SPR_IVOR7, %r5 822 #endif 823 824 ori %r5,%r6,_C_LABEL(system_call_vector)@l 825 mtspr SPR_IVOR8, %r5 826 827 #ifdef SPR_IVOR9 828 ori %r5,%r6,_C_LABEL(ap_unavailable_vector)@l 829 mtspr SPR_IVOR9, %r5 830 #endif 831 832 ori %r5,%r6,_C_LABEL(decrementer_vector)@l 833 mtspr SPR_IVOR10, %r5 834 835 ori %r5,%r6,_C_LABEL(fixed_interval_timer_vector)@l 836 mtspr SPR_IVOR11, %r5 837 838 ori %r5,%r6,_C_LABEL(watchdog_timer_vector)@l 839 mtspr SPR_IVOR12, %r5 840 841 ori %r5,%r6,_C_LABEL(data_tlb_error_vector)@l 842 mtspr SPR_IVOR13, %r5 843 844 ori %r5,%r6,_C_LABEL(instruction_tlb_error_vector)@l 845 mtspr SPR_IVOR14, %r5 846 847 ori %r5,%r6,_C_LABEL(debug_vector)@l 848 mtspr SPR_IVOR15, %r5 849 850 ori %r5,%r6,_C_LABEL(spv_unavailable_vector)@l 851 mtspr SPR_IVOR32, %r5 852 853 ori %r5,%r6,_C_LABEL(fpdata_vector)@l 854 mtspr SPR_IVOR33, %r5 855 856 ori %r5,%r6,_C_LABEL(fpround_vector)@l 857 mtspr SPR_IVOR34, %r5 858 859 ori %r5,%r6,_C_LABEL(perfmon_vector)@l 860 mtspr SPR_IVOR35, %r5 861 862 mfpir %r5 /* get Process ID register */ 863 cmplwi %r5,0 864 bnelr /* return if non-0 (non-primary) */ 865 866 lis %r5,_C_LABEL(critintr_call)@ha 867 lwzu %r6,_C_LABEL(critintr_call)@l(%r5) 868 lwz %r7,INTRSW_CRITINTR(%r3) 869 sub %r8,%r7,%r5 870 add %r0,%r8,%r6 871 stw %r0,0(%r5) 872 dcbf 0,%r5 873 icbi 0,%r5 874 875 lis %r5,_C_LABEL(decrintr_call)@ha 876 lwzu %r6,_C_LABEL(decrintr_call)@l(%r5) 877 lwz %r7,INTRSW_DECRINTR(%r3) 878 sub %r8,%r7,%r5 879 add %r0,%r8,%r6 880 stw %r0,0(%r5) 881 dcbf 0,%r5 882 icbi 0,%r5 883 884 lis %r5,_C_LABEL(fitintr_call)@ha 885 lwzu %r6,_C_LABEL(fitintr_call)@l(%r5) 886 lwz %r7,INTRSW_FITINTR(%r3) 887 sub %r8,%r7,%r5 888 add %r0,%r8,%r6 889 stw %r0,0(%r5) 890 dcbf 0,%r5 891 icbi 0,%r5 892 893 lis %r5,_C_LABEL(wdogintr_call)@ha 894 lwzu %r6,_C_LABEL(wdogintr_call)@l(%r5) 895 lwz %r7,INTRSW_WDOGINTR(%r3) 896 sub %r8,%r7,%r5 897 add %r0,%r8,%r6 898 stw %r0,0(%r5) 899 dcbf 0,%r5 900 icbi 0,%r5 901 902 lis %r5,_C_LABEL(extintr_call)@ha 903 lwzu %r6,_C_LABEL(extintr_call)@l(%r5) 904 lwz %r7,INTRSW_EXTINTR(%r3) 905 sub %r8,%r7,%r5 906 add %r0,%r8,%r6 907 stw %r0,0(%r5) 908 dcbf 0,%r5 909 icbi 0,%r5 910 911 lis %r5,_C_LABEL(powerpc_intrsw)@ha 912 stw %r3,_C_LABEL(powerpc_intrsw)@l(%r5) 913 914 blr 915 916 #ifdef notyet 917 .data 918 .lcomm ddbstk,4096 919 .text 920 921 _ENTRY(cpu_Debugger) 922 mflr %r0 923 stw %r0, CFRAME_LR(%r1) 924 925 mfmsr %r3 926 wrteei 0 927 mr %r4,%r1 928 lis %r10,ddbstk@ha 929 addi %r10,%r10,ddbstk@l 930 sub %r5,%r1,%r10 931 cmplwi %r5,4096 932 blt %cr0, 1f 933 addi %r1,%r10,4096-CALLFRAMELEN 934 1: 935 stwu %r4,-FRAMELEN(%r1) 936 stw %r4,FRAME_1(%r1) 937 stmw %r13,FRAME_13(%r1) 938 mr %r26,%r0 939 mfcr %r27 940 mfxer %r28 941 mfctr %r29 942 mr %r30,%r0 943 mr %r31,%r3 944 stmw %r26,FRAME_LR(%r1) 945 mr %r31,%r1 946 mr %r1,%r10 947 addi %r4,%r1,FRAME_TF 948 li %r3,EXC_PGM 949 stw %r3,FRAME_EXC(%r1) 950 li %r3,T_PROGRAM 951 bl _C_LABEL(trap) 952 lmw %r26,FRAME_LR(%r1) 953 mtlr %r26 954 mtcr %r27 955 mtxer %r28 956 mtctr %r29 957 mr %r0,%r31 958 lmw %r13,FRAME_13(%r1) 959 lwz %r1,FRAME_1(%r1) 960 wrtee %r0 961 blr 962 #endif /* notyet */ 963
Indexes created Mon Oct 20 16:09:52 GMT 2025