1 /* $NetBSD: trap_subr.S,v 1.15 2022/07/05 20:15:40 andvar Exp $ */ 2 /*- 3 * Copyright (c) 2010, 2011 The NetBSD Foundation, Inc. 4 * All rights reserved. 5 * 6 * This code is derived from software contributed to The NetBSD Foundation 7 * by Raytheon BBN Technologies Corp and Defense Advanced Research Projects 8 * Agency and which was developed by Matt Thomas of 3am Software Foundry. 9 * 10 * This material is based upon work supported by the Defense Advanced Research 11 * Projects Agency and Space and Naval Warfare Systems Center, Pacific, under 12 * Contract No. N66001-09-C-2073. 13 * Approved for Public Release, Distribution Unlimited 14 * 15 * Redistribution and use in source and binary forms, with or without 16 * modification, are permitted provided that the following conditions 17 * are met: 18 * 1. Redistributions of source code must retain the above copyright 19 * notice, this list of conditions and the following disclaimer. 20 * 2. Redistributions in binary form must reproduce the above copyright 21 * notice, this list of conditions and the following disclaimer in the 22 * documentation and/or other materials provided with the distribution. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 25 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 26 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 27 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 28 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 29 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 30 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 31 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 32 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 33 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 34 * POSSIBILITY OF SUCH DAMAGE. 35 */ 36 37 RCSID("$NetBSD: trap_subr.S,v 1.15 2022/07/05 20:15:40 andvar Exp $") 38 39 #ifdef _KERNEL_OPT 40 #include "opt_altivec.h" 41 #include "opt_ddb.h" 42 #include "opt_mpc85xx.h" 43 #include "opt_multiprocessor.h" 44 #endif 45 46 .globl _C_LABEL(sctrapexit), _C_LABEL(trapexit), _C_LABEL(intrcall) 47 48 /* 49 * We have a problem with critical (MSR[CE] or machine check (MSR[ME]) 50 * or debug (MSR[DE]) interrupts/exception in that they could happen 51 * in-between the mtsprg1 %r2 and mfsprg1 %r2. If that happens, %r2 52 * will be lost. Even if we moved to a different sprg, subsequent 53 * exceptions would use SPRG1 and its value would be lost. The only 54 * way to be safe for CE/ME/DE faults to save and restore SPRG1. 55 * 56 * Since CE/ME/DE faults may happen anytime, we need r1 to always 57 * contain a valid kernel stack pointer. Therefore we use r2 as 58 * our temporary register. 59 * 60 * To prevent %r2 being overwritten, each "level" (normal, critical, 61 * mchk) uses a unique sprg to save %r2 (sprg1, sprg4, sprg5). 62 * 63 * Since we can't control how many nested exceptions we might get, 64 * we don't use a dedicated save area. Instead we have an upwards 65 * growing "stack" of them; the pointer to which is kept in sprg3. 66 * 67 * To allocate from the stack, one fetches sprg3, adds the amount 68 * needed, saves sprg3, and then refers to the save using a 69 * displacement of -amount. 70 */ 71 /* 72 * XXXclang 73 * At the moment, clang cannot correctly assemble m[ft]sprgN 74 * (N >= 4); use m[ft]spr for SPR_SPRGN instead. 75 */ 76 #define FRAME_EXC_PROLOGUE(start, sprg, srr) \ 77 mtspr sprg,%r2; /* save r2 */ \ 78 mfsprg3 %r2; /* get save_area pointer */ \ 79 addi %r2,%r2,4*(32-start); \ 80 /* allocate save area */ \ 81 mtsprg3 %r2; /* save updated pointer */ \ 82 stmw %r##start,-4*(32-start)(%r2); \ 83 /* free r24-r31 for use */ \ 84 mfspr %r26,sprg; /* get saved r2 */ \ 85 mfcr %r27; /* get Condition Register */ \ 86 mfxer %r28; /* get XER */ \ 87 mfspr %r30, SPR_##srr##0; /* get SRR0 */ \ 88 mfspr %r31, SPR_##srr##1 /* get SRR1 */ 89 90 #define PROLOGUE_GET_DEAR mfspr %r24, SPR_DEAR 91 #define PROLOGUE_GET_ESR mfspr %r25, SPR_ESR 92 #define PROLOGUE_GET_SRRS mfsrr0 %r24; \ 93 mfsrr1 %r25 94 #define PROLOGUE_GET_SPRG1 mfsprg1 %r29 95 #define PROLOGUE_GET_DBSR mfspr %r25, SPR_DBSR 96 #define SAVE_ESR stw %r25, FRAME_ESR(%r1) 97 #define SAVE_DEAR stw %r24, FRAME_DEAR(%r1) 98 #define SAVE_DEAR_ESR SAVE_ESR; SAVE_DEAR 99 #define SAVE_SRRS SAVE_DEAR_ESR 100 #define SAVE_SPRG1 stw %r29, FRAME_SPRG1(%r1) 101 #define SAVE_DBSR stw %r25, FRAME_DBSR(%r1) 102 #define SAVE_NOTHING /* nothing */ 103 #define RESTORE_SPRG1(r) lwz r, FRAME_SPRG1(%r1); \ 104 mtsprg1 r 105 #define RESTORE_SRR0(r) lwz r, FRAME_DEAR(%r1); \ 106 mtsrr0 r 107 #define RESTORE_SRR1(r) lwz r, FRAME_ESR(%r1); \ 108 mtsrr1 r 109 110 #define FRAME_PROLOGUE \ 111 FRAME_EXC_PROLOGUE(26, SPR_SPRG1, SRR) 112 113 #define FRAME_PROLOGUE_DEAR_ESR \ 114 FRAME_EXC_PROLOGUE(24, SPR_SPRG1, SRR); \ 115 PROLOGUE_GET_ESR; \ 116 PROLOGUE_GET_DEAR 117 118 #define FRAME_PROLOGUE_ESR \ 119 FRAME_EXC_PROLOGUE(25, SPR_SPRG1, SRR); \ 120 PROLOGUE_GET_ESR 121 122 #define FRAME_TLBPROLOGUE \ 123 FRAME_EXC_PROLOGUE(20, SPR_SPRG1, SRR); \ 124 PROLOGUE_GET_ESR; \ 125 PROLOGUE_GET_DEAR 126 127 #define FRAME_INTR_PROLOGUE \ 128 FRAME_EXC_PROLOGUE(26, SPR_SPRG1, SRR) 129 130 /* 131 * These need to save SRR0/SRR1 as well their SRR0/SRR1 in case normal 132 * exceptions happened during their execution. 133 */ 134 #define FRAME_CRIT_PROLOGUE \ 135 FRAME_EXC_PROLOGUE(24, SPR_SPRG4, CSRR); \ 136 PROLOGUE_GET_SPRG1; \ 137 PROLOGUE_GET_SRRS 138 139 #define FRAME_MCHK_PROLOGUE \ 140 FRAME_EXC_PROLOGUE(24, SPR_SPRG5, MCSRR); \ 141 PROLOGUE_GET_SPRG1; \ 142 PROLOGUE_GET_SRRS 143 144 #define FRAME_DEBUG_PROLOGUE \ 145 FRAME_EXC_PROLOGUE(24, SPR_SPRG4, CSRR); \ 146 PROLOGUE_GET_SPRG1; \ 147 PROLOGUE_GET_SRRS 148 149 /* 150 * DDB expects to fetch the LR from the previous frame. But it also 151 * expects to be pointing at the instruction after the branch link. Since 152 * we didn't branch, we need to advance it by to fake out DDB. But there's 153 * problem. If the routine is in either its first or last two instructions 154 * (before or after its adjusted its stack pointer), we could possibly 155 * overwrite stored return address. So that stored return address needs to 156 * saved and restored. 157 */ 158 #if defined(DDB) 159 #define FRAME_SAVE_SRR0_FOR_DDB \ 160 lwz %r29, FRAMELEN+CFRAME_LR(%r1); /* fetch old return address */\ 161 stw %r29, FRAME_CFRAME_LR(%r1); /* save it */ \ 162 addi %r30, %r30, 4; /* point to s the next insn */ \ 163 stw %r30, FRAMELEN+CFRAME_LR(%r1) /* appease ddb stacktrace */ 164 #define FRAME_RESTORE_RETURN_ADDRESS \ 165 lwz %r3, FRAME_CFRAME_LR(%r1); /* fetch old return address */ \ 166 stw %r3, FRAMELEN+CFRAME_LR(%r1) /* restore it */ 167 #else 168 #define FRAME_SAVE_SRR0_FOR_DDB 169 #define FRAME_RESTORE_RETURN_ADDRESS 170 #endif 171 172 #ifdef PPC_HAVE_SPE 173 #define FRAME_SAVE_SPEFSCR \ 174 mfspefscr %r0; /* get spefscr */ \ 175 stw %r0, FRAME_SPEFSCR(%r1) /* save into trapframe */ 176 #define FRAME_RESTORE_SPEFSCR \ 177 lwz %r0, FRAME_SPEFSCR(%r1); /* fetch from trapframe */ \ 178 mtspefscr %r0 /* save spefscr */ 179 #else 180 #define FRAME_SAVE_SPEFSCR 181 #define FRAME_RESTORE_SPEFSCR 182 #endif 183 /* 184 * Before the first memory reference, we must have our state inside registers 185 * since the first memory access might cause an exception which would cause 186 * SRR0/SRR1 and DEAR/ESR to become unrecoverable. CR and XER also need to be 187 * saved early since they will modified by instruction flow. The saved stack 188 * pointer is also critical but LR and CTR can be deferred being saved until 189 * we are actually filling a trapframe. 190 */ 191 #define FRAME_EXC_ENTER(exc, tf, start, save_prologue) \ 192 mtcr %r31; /* user mode exception? */ \ 193 mr %r31, %r1; /* save SP (SRR1 is safe in CR) */ \ 194 bf MSR_PR, 1f; /* nope, sp is good */ \ 195 mfsprg2 %r2; /* get curlwp */ \ 196 lwz %r2, L_PCB(%r2); /* get uarea of curlwp */ \ 197 addi %r1, %r2, USPACE-CALLFRAMELEN; \ 198 /* start stack at top of it */ \ 199 1: \ 200 stwu %r31, -FRAMELEN(%r1); /* get space for trapframe */ \ 201 stw %r0, FRAME_R0(%r1); /* save r0 */ \ 202 stw %r31, FRAME_R1(%r1); /* save (saved) r1 */ \ 203 stw %r26, FRAME_R2(%r1); /* save (saved) r2 */ \ 204 save_prologue; /* save SPRG1/ESR/DEAR */ \ 205 /* At this point, r26, r29, and r31 have been saved so we */ \ 206 /* can use them for LR, CTR, and SRR1. */ \ 207 mflr %r26; /* get Link Register */ \ 208 mfctr %r29; /* get CTR */ \ 209 mfcr %r31; /* get SRR1 */ \ 210 stmw %r26, FRAME_LR(%r1); /* save LR CR XER CTR SRR0/1 */ \ 211 FRAME_SAVE_SRR0_FOR_DDB; \ 212 mr %r0, %r31; /* save SRR1 for a bit */ \ 213 mfsprg3 %r2; /* get save_area pointer */ \ 214 addi %r2,%r2,-4*(32-start); /* find our save area */ \ 215 lmw %r##start,0(%r2); /* get start-r31 */ \ 216 mtsprg3 %r2; /* save updated pointer */ \ 217 stmw %r3, FRAME_R3(%r1); /* save r2-r31 */ \ 218 /* Now everything has been saved */ \ 219 mr %r31, %r0; /* move SRR1 back to r31 */ \ 220 mfsprg2 %r13; /* put curlwp in r13 */ \ 221 FRAME_SAVE_SPEFSCR; \ 222 li %r7, exc; /* load EXC_* */ \ 223 stw %r7, FRAME_EXC(%r1); /* save into trapframe */ \ 224 addi tf, %r1, FRAME_TF /* get address of trap frame */ 225 226 #define FRAME_EXC_EXIT(rfi, srr) \ 227 FRAME_RESTORE_RETURN_ADDRESS; /* restore return address */ \ 228 lmw %r26, FRAME_LR(%r1); /* get LR CR XER CTR SRR0/1 */ \ 229 oris %r31,%r31,PSL_CE@h; \ 230 mtspr SPR_##srr##1, %r31; /* restore SRR1 */ \ 231 mtspr SPR_##srr##0, %r30; /* restore SRR0 */ \ 232 FRAME_RESTORE_SPEFSCR; \ 233 mtctr %r29; /* restore CTR */ \ 234 mtxer %r28; /* restore XER */ \ 235 mtcr %r27; /* restore CR */ \ 236 mtlr %r26; /* restore LR */ \ 237 lmw %r2, FRAME_R2(%r1); /* restore r2-r31 */ \ 238 lwz %r0, FRAME_R0(%r1); /* restore r0 */ \ 239 lwz %r1, FRAME_R1(%r1); /* restore r1 */ \ 240 rfi /* return from interrupt */ 241 242 243 #define FRAME_ENTER(exc, tf) \ 244 FRAME_EXC_ENTER(exc, tf, 26, SAVE_NOTHING) 245 246 #define FRAME_ENTER_ESR(exc, tf) \ 247 FRAME_EXC_ENTER(exc, tf, 25, SAVE_ESR) 248 249 #define FRAME_ENTER_DEAR_ESR(exc, tf) \ 250 FRAME_EXC_ENTER(exc, tf, 24, SAVE_DEAR_ESR) 251 252 #define FRAME_EXIT FRAME_EXC_EXIT(rfi, SRR) 253 254 #define FRAME_TLBENTER(exc) \ 255 FRAME_EXC_ENTER(exc, %r4, 20, SAVE_DEAR_ESR) 256 #define FRAME_TLBEXIT FRAME_EXC_EXIT(rfi, SRR) 257 258 #define FRAME_MCHK_ENTER(exc) \ 259 FRAME_EXC_ENTER(exc, %r3, 26, SAVE_SPRG1; SAVE_SRRS) 260 #define FRAME_MCHK_EXIT \ 261 RESTORE_SRR0(%r28); \ 262 RESTORE_SRR1(%r27); \ 263 RESTORE_SPRG1(%r26); \ 264 FRAME_EXC_EXIT(rfmci, MCSRR) 265 266 #define FRAME_DEBUG_ENTER(exc) \ 267 FRAME_EXC_ENTER(exc, %r4, 26, SAVE_SPRG1; SAVE_SRRS) 268 #define FRAME_DEBUG_EXIT \ 269 RESTORE_SPRG1(%r26); FRAME_EXC_EXIT(rfci, CSRR) 270 271 #define FRAME_INTR_SP \ 272 bf MSR_PR, 1f; /* nope, sp is good */ \ 273 mfsprg2 %r2; /* get curlwp */ \ 274 lwz %r2, L_PCB(%r2); /* get uarea of curlwp */ \ 275 addi %r1, %r2, USPACE-CALLFRAMELEN; \ 276 /* start stack at top of it */ \ 277 1: 278 279 #define FRAME_INTR_SP_NEW(sym) \ 280 lis %r2,(sym)@ha; \ 281 addi %r1,%r2,(sym)@l 282 283 #define FRAME_INTR_XENTER(exc, start, get_intr_sp, save_prologue) \ 284 mtcr %r31; /* user mode exception? */ \ 285 mr %r31, %r1; /* save SP (SRR1 is safe in CR) */ \ 286 get_intr_sp; /* get kernel stack pointer */ \ 287 stwu %r31, -FRAMELEN(%r1); /* get space for trapframe */ \ 288 stw %r0, FRAME_R0(%r1); /* save r0 */ \ 289 stw %r31, FRAME_R1(%r1); /* save (saved) r1 */ \ 290 stw %r26, FRAME_R2(%r1); /* save (saved) r2 */ \ 291 save_prologue; /* save SPRG1 (maybe) */ \ 292 mflr %r26; /* get LR */ \ 293 mfctr %r29; /* get CTR */ \ 294 mfcr %r31; /* get SRR1 */ \ 295 stmw %r26, FRAME_LR(%r1); /* save LR CR XER CTR SRR0/1 */ \ 296 FRAME_SAVE_SRR0_FOR_DDB; \ 297 stw %r3, FRAME_R3(%r1); /* save r3 */ \ 298 stw %r4, FRAME_R4(%r1); /* save r4 */ \ 299 stw %r5, FRAME_R5(%r1); /* save r5 */ \ 300 stw %r6, FRAME_R6(%r1); /* save r6 */ \ 301 stw %r7, FRAME_R7(%r1); /* save r7 */ \ 302 stw %r8, FRAME_R8(%r1); /* save r8 */ \ 303 stw %r9, FRAME_R9(%r1); /* save r9 */ \ 304 stw %r10, FRAME_R10(%r1); /* save r10 */ \ 305 stw %r11, FRAME_R11(%r1); /* save r11 */ \ 306 stw %r12, FRAME_R12(%r1); /* save r12 */ \ 307 stw %r13, FRAME_R13(%r1); /* save r13 */ \ 308 mfsprg3 %r2; /* get save_area pointer */ \ 309 addi %r2,%r2,-4*(32-start); /* find our save area */ \ 310 lmw %r##start,0(%r2); /* get start-r31 */ \ 311 mtsprg3 %r2; /* save updated pointer */ \ 312 mfsprg2 %r13; /* put curlwp into r13 */ \ 313 li %r7, exc; /* load EXC_* */ \ 314 stw %r7, FRAME_EXC(%r1); /* save into trapframe */ \ 315 addi %r3, %r1, FRAME_TF /* only argument is trapframe */ 316 317 #define FRAME_INTR_XEXIT(rfi, srr) \ 318 FRAME_RESTORE_RETURN_ADDRESS; /* restore return address */ \ 319 lwz %r8, FRAME_LR(%r1); /* get LR */ \ 320 lwz %r9, FRAME_CR(%r1); /* get CR */ \ 321 lwz %r10, FRAME_XER(%r1); /* get XER */ \ 322 lwz %r11, FRAME_CTR(%r1); /* get CTR */ \ 323 lwz %r12, FRAME_SRR0(%r1); /* get SRR0 */ \ 324 lwz %r13, FRAME_SRR1(%r1); /* get SRR1 */ \ 325 mtspr SPR_##srr##1, %r13; /* restore SRR1 */ \ 326 mtspr SPR_##srr##0, %r12; /* restore SRR0 */ \ 327 mtctr %r11; /* restore CTR */ \ 328 mtxer %r10; /* restore XER */ \ 329 mtcr %r9; /* restore CR */ \ 330 mtlr %r8; /* restore LR */ \ 331 lwz %r13, FRAME_R13(%r1); /* restore r13 */ \ 332 lwz %r12, FRAME_R12(%r1); /* restore r12 */ \ 333 lwz %r11, FRAME_R11(%r1); /* restore r11 */ \ 334 lwz %r10, FRAME_R10(%r1); /* restore r10 */ \ 335 lwz %r9, FRAME_R9(%r1); /* restore r9 */ \ 336 lwz %r8, FRAME_R8(%r1); /* restore r8 */ \ 337 lwz %r7, FRAME_R7(%r1); /* restore r7 */ \ 338 lwz %r6, FRAME_R6(%r1); /* restore r6 */ \ 339 lwz %r5, FRAME_R5(%r1); /* restore r5 */ \ 340 lwz %r4, FRAME_R4(%r1); /* restore r4 */ \ 341 lwz %r3, FRAME_R3(%r1); /* restore r3 */ \ 342 lwz %r2, FRAME_R2(%r1); /* restore r2 */ \ 343 lwz %r0, FRAME_R0(%r1); /* restore r0 */ \ 344 lwz %r1, FRAME_R1(%r1); /* restore r1 */ \ 345 rfi /* return from interrupt */ 346 347 #define FRAME_INTR_ENTER(exc) \ 348 FRAME_INTR_XENTER(exc, 26, FRAME_INTR_SP, SAVE_NOTHING) 349 #define FRAME_INTR_EXIT \ 350 FRAME_INTR_XEXIT(rfi, SRR) 351 #define FRAME_CRIT_ENTER(exc) \ 352 FRAME_INTR_XENTER(exc, 24, FRAME_INTR_SP, SAVE_SPRG1) 353 #define FRAME_WDOG_ENTER(exc, sym) \ 354 FRAME_INTR_XENTER(exc, 24, FRAME_INTR_SP_NEW(sym), SAVE_SPRG1) 355 #define FRAME_CRIT_EXIT \ 356 RESTORE_SRR0(%r4); \ 357 RESTORE_SRR1(%r5); \ 358 RESTORE_SPRG1(%r6); \ 359 FRAME_INTR_XEXIT(rfci, CSRR) 360 361 #if defined(MULTIPROCESSOR) 362 #define FRAME_TLBMISSLOCK \ 363 GET_CPUINFO(%r23); \ 364 ldint %r22, CI_MTX_COUNT(%r23); \ 365 subi %r22, %r22, 1; \ 366 stint %r22, CI_MTX_COUNT(%r23); \ 367 isync; \ 368 cmpwi %r22, 0; \ 369 bne 1f; \ 370 ldint %r22, CI_CPL(%r23); \ 371 stint %r22, CI_MTX_OLDSPL(%r23); \ 372 1: lis %r23, _C_LABEL(pmap_tlb_miss_lock)@h; \ 373 ori %r23, %r23, _C_LABEL(pmap_tlb_miss_lock)@l; \ 374 li %r20, MTX_LOCK; \ 375 2: lwarx %r22, %r20, %r23; \ 376 cmpwi %r22, __SIMPLELOCK_UNLOCKED; \ 377 beq+ 4f; \ 378 3: lwzx %r22, %r20, %r23; \ 379 cmpwi %r22, __SIMPLELOCK_UNLOCKED; \ 380 beq+ 2b; \ 381 b 3b; \ 382 4: li %r21, __SIMPLELOCK_LOCKED; \ 383 stwcx. %r21, %r20, %r23; \ 384 bne- 2b; \ 385 isync; \ 386 msync; 387 #define FRAME_TLBMISSUNLOCK \ 388 sync; \ 389 lis %r23, _C_LABEL(pmap_tlb_miss_lock)@h; \ 390 ori %r23, %r23, _C_LABEL(pmap_tlb_miss_lock)@l; \ 391 li %r22, __SIMPLELOCK_UNLOCKED; \ 392 stw %r22, MTX_LOCK(%r23); \ 393 isync; \ 394 msync; \ 395 GET_CPUINFO(%r23); \ 396 ldint %r22, CI_MTX_COUNT(%r23); \ 397 addi %r22, %r22, 1; \ 398 stint %r22, CI_MTX_COUNT(%r23); \ 399 isync; 400 #else /* !MULTIPROCESSOR */ 401 #define FRAME_TLBMISSLOCK 402 #define FRAME_TLBMISSUNLOCK 403 #endif /* MULTIPROCESSOR */ 404 405 .text 406 .p2align 4 407 _C_LABEL(critical_input_vector): 408 /* MSR[ME] is unchanged, all others cleared */ 409 FRAME_CRIT_PROLOGUE /* save SP r26-31 CR LR XER */ 410 FRAME_CRIT_ENTER(EXC_CII) 411 bl _C_LABEL(intr_critintr) /* critintr(tf) */ 412 FRAME_CRIT_EXIT 413 414 .p2align 4 415 _C_LABEL(machine_check_vector): 416 /* all MSR bits are cleared */ 417 FRAME_MCHK_PROLOGUE /* save SP r25-31 CR LR XER */ 418 FRAME_MCHK_ENTER(EXC_MCHK) 419 /* 420 * MCAR/MCSR don't need to be saved early since MSR[ME] is cleared 421 * on entry. 422 */ 423 mfspr %r7, SPR_MCAR 424 mfspr %r6, SPR_MCSR 425 stw %r6, FRAME_MCSR(%r1) 426 stw %r7, FRAME_MCAR(%r1) 427 li %r3, T_MACHINE_CHECK 428 bl _C_LABEL(trap) /* trap(T_MACHINE_CHECK, tf) */ 429 FRAME_MCHK_EXIT 430 431 .p2align 4 432 _C_LABEL(data_storage_vector): 433 /* MSR[CE], MSR[ME], MSR[DE] are unchanged, all others cleared */ 434 FRAME_PROLOGUE_DEAR_ESR /* save r2 DEAR ESR r24-31 CR XER SRR */ 435 FRAME_ENTER_DEAR_ESR(EXC_DSI, %r4) 436 li %r3, T_DSI 437 /* FRAME_ENTER leaves SRR1 in %r31 */ 438 trapenter: 439 trapagain: 440 wrtee %r31 /* restore MSR[EE] */ 441 442 bl _C_LABEL(trap) /* trap(trapcode, tf) */ 443 _C_LABEL(trapexit): 444 wrteei 0 /* disable interrupts */ 445 # andis. %r0, %r31, PSL_CE@h 446 # tweqi %r0, 0 447 andi. %r4, %r31, PSL_PR /* lets look at PSL_PR */ 448 beq trapdone /* if clear, skip to exit */ 449 lwz %r4, L_MD_ASTPENDING(%r13) /* get ast pending */ 450 cmplwi %r4, 0 /* is there an ast pending */ 451 beq+ trapdone /* nope, proceed to exit */ 452 li %r6, EXC_AST /* yes. */ 453 stw %r6, FRAME_EXC(%r1) /* pretend this is an AST */ 454 addi %r4, %r1, FRAME_TF /* get address of trap frame */ 455 li %r3, T_AST 456 b trapagain /* and deal with it */ 457 trapdone: 458 FRAME_EXIT 459 460 .p2align 4 461 _C_LABEL(instruction_storage_vector): 462 /* MSR[CE], MSR[ME], MSR[DE] are unchanged, all others cleared */ 463 FRAME_PROLOGUE_ESR /* save ESR r2 r25-31 CR XER SRR0/1 */ 464 FRAME_ENTER_ESR(EXC_ISI, %r4) 465 li %r3, T_ISI 466 b trapenter 467 468 .p2align 4 469 _ENTRY(external_input_vector) 470 /* MSR[CE], MSR[ME], MSR[DE] are unchanged, all others cleared */ 471 FRAME_INTR_PROLOGUE /* save SP r25-31 CR LR XER */ 472 FRAME_INTR_ENTER(EXC_EXI) 473 474 bl _C_LABEL(intr_extintr) 475 _C_LABEL(intrcall): 476 GET_CPUINFO(%r6) /* get curcpu() */ 477 lwz %r5, FRAME_SRR1(%r1) /* get saved SRR1 */ 478 # andis. %r0, %r5, PSL_CE@h 479 # tweqi %r0, 0 480 andi. %r4, %r5, PSL_PR /* lets look at PSL_PR */ 481 beq intrexit /* if clear, skip to exit */ 482 lwz %r4, L_MD_ASTPENDING(%r13) /* get ast pending */ 483 cmplwi %r4, 0 /* is there an ast pending */ 484 beq+ intrexit /* nope, proceed to exit */ 485 stmw %r14, FRAME_R14(%r1) /* save rest of registers */ 486 FRAME_SAVE_SPEFSCR 487 mr %r31, %r5 /* needed for trapagain */ 488 li %r4, EXC_AST /* */ 489 stw %r4, FRAME_EXC(%r1) /* pretend this is an AST */ 490 addi %r4, %r1, FRAME_TF /* get address of trap frame */ 491 li %r3, T_AST 492 b trapagain /* and deal with it */ 493 intrexit: 494 FRAME_INTR_EXIT 495 496 .p2align 4 497 _C_LABEL(alignment_vector): 498 /* MSR[CE], MSR[ME], MSR[DE] are unchanged, all others cleared */ 499 FRAME_PROLOGUE_DEAR_ESR /* save SP r25-31 CR LR XER */ 500 FRAME_ENTER_DEAR_ESR(EXC_ALI, %r4) 501 li %r3, T_ALIGNMENT 502 b trapenter 503 504 .p2align 4 505 _C_LABEL(program_vector): 506 /* MSR[CE], MSR[ME], MSR[DE] are unchanged, all others cleared */ 507 FRAME_PROLOGUE_ESR /* save SP r25-31 CR LR XER */ 508 FRAME_ENTER_ESR(EXC_PGM, %r4) 509 li %r3, T_PROGRAM 510 b trapenter 511 512 #ifdef SPR_IVOR7 513 .p2align 4 514 _C_LABEL(fp_unavailable_vector): 515 /* MSR[CE], MSR[ME], MSR[DE] are unchanged, all others cleared */ 516 FRAME_PROLOGUE_ESR /* save SP r25-31 CR LR XER */ 517 FRAME_ENTER_ESR(EXC_FPU, %r4) 518 li %r3, T_FP_UNAVAILABLE 519 b trapenter 520 #endif 521 522 .p2align 4 523 _C_LABEL(system_call_vector): 524 /* MSR[CE], MSR[ME], MSR[DE] are unchanged, all others cleared */ 525 FRAME_PROLOGUE /* save SP r26-31 CR LR XER */ 526 FRAME_ENTER(EXC_SC, %r3) 527 528 wrteei 1 /* enable interrupts */ 529 lwz %r7, L_PROC(%r13) /* get proc for lwp */ 530 lwz %r8, P_MD_SYSCALL(%r7) /* get syscall */ 531 mtlr %r8 /* need to call indirect */ 532 blrl /* syscall(tf) */ 533 _C_LABEL(sctrapexit): 534 wrteei 0 /* disable interrupts */ 535 lwz %r4, L_MD_ASTPENDING(%r13) /* get ast pending */ 536 cmplwi %r4, 0 /* is there an ast pending */ 537 beq+ trapdone /* nope, proceed to exit */ 538 li %r0, EXC_AST /* yes. */ 539 stw %r0, FRAME_EXC(%r1) /* pretend this is an AST */ 540 addi %r4, %r1, FRAME_TF /* get address of trap frame */ 541 li %r3, T_AST 542 b trapenter /* and deal with it */ 543 544 #ifdef SPR_IVOR9 545 .p2align 4 546 _C_LABEL(ap_unavailable_vector): 547 /* MSR[CE], MSR[ME], MSR[DE] are unchanged, all others cleared */ 548 FRAME_PROLOGUE /* save SP r25-31 CR LR XER */ 549 FRAME_ENTER(EXC_PGM, %r4) 550 li %r3, T_AP_UNAVAILABLE 551 b trapenter 552 #endif 553 554 .p2align 4 555 _C_LABEL(decrementer_vector): 556 /* MSR[CE], MSR[ME], MSR[DE] are unchanged, all others cleared */ 557 FRAME_INTR_PROLOGUE /* save SP r25-31 CR LR XER */ 558 FRAME_INTR_ENTER(EXC_DECR) 559 560 bl _C_LABEL(intr_decrintr) 561 b intrexit 562 563 .p2align 4 564 _C_LABEL(fixed_interval_timer_vector): 565 /* MSR[CE], MSR[ME], MSR[DE] are unchanged, all others cleared */ 566 FRAME_PROLOGUE /* save SP r25-31 CR LR XER */ 567 FRAME_INTR_ENTER(EXC_FIT) 568 569 bl _C_LABEL(intr_fitintr) 570 b intrexit 571 572 #ifdef E500_WDOG_STACK 573 .data 574 .lcomm wdogstk,4096 575 #endif 576 .text 577 .p2align 4 578 _C_LABEL(watchdog_timer_vector): 579 /* MSR[CE], MSR[ME], MSR[DE] are unchanged, all others cleared */ 580 FRAME_CRIT_PROLOGUE /* save SP r25-31 CR LR XER */ 581 #ifdef E500_WDOG_STACK 582 FRAME_WDOG_ENTER(EXC_WDOG, wdogstk+4096-CALLFRAMELEN) 583 #else 584 FRAME_CRIT_ENTER(EXC_WDOG); 585 #endif 586 587 bl _C_LABEL(intr_wdogintr) 588 FRAME_CRIT_EXIT 589 590 .p2align 4 591 _C_LABEL(data_tlb_error_vector): 592 /* MSR[CE], MSR[ME], MSR[DE] are unchanged, all others cleared */ 593 FRAME_TLBPROLOGUE 594 FRAME_TLBMISSLOCK 595 /* 596 * Registers as this point: 597 * 598 * r2 = cpu_info 599 * r20 = scratch 600 * r21 = scratch 601 * r22 = scratch 602 * r23 = scratch 603 * r24 = DEAR 604 * r25 = ESR 605 * r26 = saved r2 606 * r27 = CR 607 * r28 = XER 608 * r29 = scratch 609 * r30 = SRR0 610 * r31 = SRR1 611 * 612 * Except for r29, these values must be retained. However we must 613 * be cognizant of nesting. There are two cases here, both related. 614 * 615 * We get a critical input or machine check exception and the kernel 616 * stack doesn't have a TLB entry so we take an exception. The other 617 * nesting path is some page used by the exception handler will cause 618 * a TLB data error. 619 * 620 * The second case (more probable) is that the PTE loading will fail 621 * so we will have to do a hard trap to resolve it. But in doing so 622 * we need to save a trapframe which could result in another DTLB 623 * fault. 624 * 625 * In all cases, the save area stack shall protect us. 626 */ 627 /* 628 * Attempt to update the TLB from the page table. 629 */ 630 mflr %r29 /* save LR */ 631 mr %r23, %r24 /* address of exception */ 632 rlwinm %r22, %r31, /* index into ci_pmap_segtab */\ 633 MSR_DS+PTR_SCALESHIFT+1, \ 634 31-PTR_SCALESHIFT, \ 635 31-PTR_SCALESHIFT /* move PSL_DS[27] to bit 29 */ 636 bl pte_load 637 FRAME_TLBMISSUNLOCK 638 mtlr %r29 /* restore LR */ 639 /* 640 * If we returned, pte load failed so let trap deal with it but 641 * has kept the contents of r24-r31 (expect r29) intact. 642 */ 643 FRAME_TLBENTER(EXC_DSI) 644 li %r3, T_DATA_TLB_ERROR 645 b trapenter 646 647 .p2align 4 648 _C_LABEL(instruction_tlb_error_vector): 649 /* MSR[CE], MSR[ME], MSR[DE] are unchanged, all others cleared */ 650 FRAME_TLBPROLOGUE 651 FRAME_TLBMISSLOCK 652 /* 653 * Attempt to update the TLB from the page table. 654 */ 655 mflr %r29 /* save LR */ 656 mr %r23, %r30 /* PC of exception */ 657 rlwinm %r22, %r31, /* index into ci_pmap_segtab */\ 658 MSR_IS+PTR_SCALESHIFT+1, \ 659 31-PTR_SCALESHIFT, \ 660 31-PTR_SCALESHIFT /* move PSL_IS[26] to bit 29 */ 661 bl pte_load 662 FRAME_TLBMISSUNLOCK 663 mtlr %r29 /* restore LR */ 664 /* 665 * If we returned, pte load failed so let trap deal with it but 666 * has kept the contents of r24-r31 (expect r29) intact. 667 */ 668 FRAME_TLBENTER(EXC_ISI) 669 li %r3, T_INSTRUCTION_TLB_ERROR 670 b trapenter 671 672 .p2align 4 673 _C_LABEL(debug_vector): 674 FRAME_CRIT_PROLOGUE /* save SP r25-31 CR LR XER */ 675 FRAME_CRIT_ENTER(EXC_DEBUG) 676 mfspr %r6, SPR_DBSR 677 stw %r6, FRAME_ESR(%r1) 678 li %r3, T_DEBUG 679 bl _C_LABEL(trap) 680 FRAME_CRIT_EXIT 681 682 .p2align 4 683 _C_LABEL(spv_unavailable_vector): 684 FRAME_PROLOGUE_ESR /* save SP r25-31 CR LR XER */ 685 FRAME_ENTER_ESR(EXC_VEC, %r4) 686 li %r3, T_SPE_UNAVAILABLE 687 b trapenter 688 689 .p2align 4 690 _C_LABEL(fpdata_vector): 691 FRAME_PROLOGUE_ESR /* save SP r25-31 CR LR XER */ 692 FRAME_ENTER_ESR(EXC_FPA, %r4) 693 li %r3, T_EMBEDDED_FP_DATA 694 b trapenter 695 696 .p2align 4 697 _C_LABEL(fpround_vector): 698 FRAME_PROLOGUE_ESR /* save SP r25-31 CR LR XER */ 699 FRAME_ENTER_ESR(EXC_FPA, %r4) 700 li %r3, T_EMBEDDED_FP_ROUND 701 b trapenter 702 703 .p2align 4 704 _C_LABEL(perfmon_vector): 705 FRAME_PROLOGUE_ESR /* save SP r25-31 CR LR XER */ 706 FRAME_ENTER_ESR(EXC_PERF, %r4) 707 li %r3, T_EMBEDDED_PERF_MONITOR 708 b trapenter 709 710 .p2align 4 711 pte_load: 712 /* 713 * r2 = scratch 714 * r20 = scratch 715 * r21 = scratch 716 * r22 = index into ci_pmap_{kern,user}_segtab 717 * r23 = faulting address 718 * The rest are for reference and aren't modifiable. If the load 719 * fails, they will be used by FRAME_TLBENTER to create the trapframe. 720 * r24 = DEAR 721 * r25 = ESR 722 * r26 = saved r2 723 * r27 = CR 724 * r28 = XER 725 * r29 = LR 726 * r30 = SRR0 727 * r31 = SRR1 728 */ 729 cmplwi %cr2, %r22, 0 /* remember address space */ 730 GET_CPUINFO(%r2) 731 addi %r22, %r22, CI_PMAP_SEGTAB /* index into segtab(s) */ 732 lwzx %r20, %r22, %r2 /* load kern/user L1 PT addr */ 733 cmplwi %r20, 0 /* is segtab null? */ 734 beqlr %cr0 /* yes, return to fallback to trap */ 735 736 rlwinm %r22, %r23, NSEGPG_SCALESHIFT + PTR_SCALESHIFT, \ 737 31-(NSEGPG_SCALESHIFT + PTR_SCALESHIFT - 1), \ 738 31-PTR_SCALESHIFT /* extract addr bits [0:9] to [20:29] */ 739 lwzx %r20, %r22, %r20 /* load address of page table page */ 740 cmplwi %r20, 0 /* is page null? */ 741 beqlr %cr0 /* yes, return to fallback to trap */ 742 743 rlwinm %r22, %r23, \ 744 NSEGPG_SCALESHIFT + NPTEPG_SCALESHIFT + PTE_SCALESHIFT, \ 745 31-(NPTEPG_SCALESHIFT + PTE_SCALESHIFT - 1), \ 746 31-PTE_SCALESHIFT /* extract addr bits [10:19] to [20:29] */ 747 lwzx %r20, %r22, %r20 /* load PTE from page table page */ 748 cmplwi %r20, 0 /* is there a valid PTE? */ 749 beqlr %cr0 /* no, return to fallback to trap */ 750 751 #if (PTE_UNSYNCED << 1) != PTE_xX 752 #error PTE_UNSYNCED definition error 753 #endif 754 #if (PTE_UNMODIFIED << 1) != PTE_xW 755 #error PTE_UNMODIFIED definition error 756 #endif 757 andi. %r22, %r20, (PTE_UNSYNCED|PTE_UNMODIFIED) 758 /* Does the PTE need to be changed? */ 759 rotlwi %r22, %r22, 1 /* if so, clear the right PTE bits */ 760 andc %r20, %r20, %r22 /* pte &= ~((pte & (PTE_UNSYNCED|PTE_UNMODIFIED)) << 1)*/ 761 762 /* 763 * r24-r32 = (no touch) 764 * r23 = scratch (was fault addr) 765 * r22 = scratch 766 * r21 = scratch 767 * r20 = pte 768 * cr2 = AS 0=eq/!0=ne 769 */ 770 771 /* 772 * This is all E500 specific. We should have a patchable branch 773 * to support other BookE (440) implementations. 774 */ 775 e500_pte_load: 776 bne+ %cr2, 1f /* user access? MAS1 is ok. */ 777 mfspr %r22, SPR_MAS1 /* get MAS1 */ 778 lis %r21, MAS1_TID@h /* get TID mask */ 779 andc %r22, %r22, %r21 /* clear TID */ 780 mtspr SPR_MAS1, %r22 /* save MAS1 */ 781 1: 782 andi. %r21, %r20, PTE_WIMGE_MASK /* extract WIMGE from PTE */ 783 cmplwi %r21, PTE_M /* if just PTE_M is set, */ 784 beq+ %cr0, 2f /* skip munging mas2 */ 785 mfspr %r22, SPR_MAS2 /* get MAS2 (updated by error) */ 786 clrrwi %r22, %r22, PTE_RWX_SHIFT /* clear WIMGE bits */ 787 or %r22, %r22, %r21 /* combine with MAS2 contents */ 788 mtspr SPR_MAS2, %r22 /* put back into MAS2 */ 789 2: 790 /* 791 * r23 = fault addr 792 * r22 = scratch 793 * r21 = scratch 794 * r20 = pte 795 */ 796 797 /* 798 * In MAS3, the protection bits are in the low 6 bits: 799 * UX SX UW SW UR SR 800 * The User bits are 1 bit left of their Supervisor counterparts. 801 * Rotate the PTE protection bits left until they wrap around to become 802 * the least significant bits, where the Supervisor protection bits 803 * are located. Increase the rotate amount by 1 to place them where 804 * the User protection bits are located. We get that 1 by extracting 805 * the MAS1[TS] (set for User access) and moving it to bit 31 (LSB). 806 */ 807 mfspr %r21, SPR_MAS1 /* get MAS1 which has TS bit */ 808 extrwi %r21, %r21, 1, 31-MAS1_TS_SHIFT 809 /* extract MAS1_TS to LSB */ 810 clrrwi %r23, %r20, PAGE_SHIFT /* clear non-RPN bits from PTE */ 811 andi. %r20, %r20, PTE_RWX_MASK /* isolate protection bits */ 812 rotrwi %r20, %r20, PTE_RWX_SHIFT 813 andi. %r22, %r20, (MAS3_SW|MAS3_SR) /* user pages need to be R/W by kernel */ 814 rotlw %r20, %r20, %r21 /* rotate protection to correct loc */ 815 or %r20, %r20, %r22 /* combine system protection bits */ 816 or %r23, %r23, %r20 /* combine RPN and protection bits */ 817 mtspr SPR_MAS3, %r23 /* put into MAS3 */ 818 isync /* because ECORE500RM tells us too */ 819 tlbwe /* write the TLB entry */ 820 /* 821 * Increment a counter to show how many tlb misses we've handled here. 822 */ 823 lmw %r30, CI_EV_TLBMISS_SOFT(%r2) 824 addic %r31, %r31, 1 825 addze %r30, %r30 826 stmw %r30, CI_EV_TLBMISS_SOFT(%r2) 827 828 FRAME_TLBMISSUNLOCK 829 830 /* 831 * Cleanup and leave. We know any higher priority exception will 832 * save and restore SPRG1 and %r2 thereby preserving their values. 833 * 834 * r24 = DEAR (don't care) 835 * r25 = ESR (don't care) 836 * r26 = saved r2 837 * r27 = CR 838 * r28 = XER 839 * r29 = LR 840 * r30 = LSW of counter 841 * r31 = MSW of counter 842 */ 843 mtlr %r29 /* restore Link Register */ 844 mtxer %r28 /* restore XER */ 845 mtcr %r27 /* restore Condition Register */ 846 mtsprg1 %r26 /* save saved r2 across load multiple */ 847 mfsprg3 %r2 /* get end of save area */ 848 addi %r2,%r2,-4*(32-20) /* adjust save area down */ 849 lmw %r20,0(%r2) /* restore r20-r31 */ 850 mtsprg3 %r2 /* save new end of save area */ 851 mfsprg1 %r2 /* restore r2 */ 852 rfi 853 854 .p2align 4 855 .globl _C_LABEL(exception_init) 856 _C_LABEL(exception_init): 857 lis %r6,_C_LABEL(critical_input_vector)@h 858 mtspr SPR_IVPR, %r6 859 860 ori %r5,%r6,_C_LABEL(critical_input_vector)@l 861 mtspr SPR_IVOR0, %r5 862 863 ori %r5,%r6,_C_LABEL(machine_check_vector)@l 864 mtspr SPR_IVOR1, %r5 865 866 ori %r5,%r6,_C_LABEL(data_storage_vector)@l 867 mtspr SPR_IVOR2, %r5 868 869 ori %r5,%r6,_C_LABEL(instruction_storage_vector)@l 870 mtspr SPR_IVOR3, %r5 871 872 ori %r5,%r6,_C_LABEL(external_input_vector)@l 873 mtspr SPR_IVOR4, %r5 874 875 ori %r5,%r6,_C_LABEL(alignment_vector)@l 876 mtspr SPR_IVOR5, %r5 877 878 ori %r5,%r6,_C_LABEL(program_vector)@l 879 mtspr SPR_IVOR6, %r5 880 881 #ifdef SPR_IVOR7 882 ori %r5,%r6,_C_LABEL(fp_unavailable_vector)@l 883 mtspr SPR_IVOR7, %r5 884 #endif 885 886 ori %r5,%r6,_C_LABEL(system_call_vector)@l 887 mtspr SPR_IVOR8, %r5 888 889 #ifdef SPR_IVOR9 890 ori %r5,%r6,_C_LABEL(ap_unavailable_vector)@l 891 mtspr SPR_IVOR9, %r5 892 #endif 893 894 ori %r5,%r6,_C_LABEL(decrementer_vector)@l 895 mtspr SPR_IVOR10, %r5 896 897 ori %r5,%r6,_C_LABEL(fixed_interval_timer_vector)@l 898 mtspr SPR_IVOR11, %r5 899 900 ori %r5,%r6,_C_LABEL(watchdog_timer_vector)@l 901 mtspr SPR_IVOR12, %r5 902 903 ori %r5,%r6,_C_LABEL(data_tlb_error_vector)@l 904 mtspr SPR_IVOR13, %r5 905 906 ori %r5,%r6,_C_LABEL(instruction_tlb_error_vector)@l 907 mtspr SPR_IVOR14, %r5 908 909 ori %r5,%r6,_C_LABEL(debug_vector)@l 910 mtspr SPR_IVOR15, %r5 911 912 ori %r5,%r6,_C_LABEL(spv_unavailable_vector)@l 913 mtspr SPR_IVOR32, %r5 914 915 ori %r5,%r6,_C_LABEL(fpdata_vector)@l 916 mtspr SPR_IVOR33, %r5 917 918 ori %r5,%r6,_C_LABEL(fpround_vector)@l 919 mtspr SPR_IVOR34, %r5 920 921 ori %r5,%r6,_C_LABEL(perfmon_vector)@l 922 mtspr SPR_IVOR35, %r5 923 924 mfspr %r5, SPR_PIR /* get Processor ID register */ 925 cmplwi %r5,0 926 bnelr /* return if non-0 (non-primary) */ 927 928 lis %r5,_C_LABEL(powerpc_intrsw)@ha 929 stw %r3,_C_LABEL(powerpc_intrsw)@l(%r5) 930 931 blr 932 933 #ifdef notyet 934 .data 935 .lcomm ddbstk,4096 936 .text 937 938 _ENTRY(cpu_Debugger) 939 mflr %r0 940 stw %r0, CFRAME_LR(%r1) 941 942 mfmsr %r3 943 wrteei 0 944 mr %r4,%r1 945 lis %r10,ddbstk@ha 946 addi %r10,%r10,ddbstk@l 947 sub %r5,%r1,%r10 948 cmplwi %r5,4096 949 blt %cr0, 1f 950 addi %r1,%r10,4096-CALLFRAMELEN 951 1: 952 stwu %r4,-FRAMELEN(%r1) 953 stw %r4,FRAME_R1(%r1) 954 stmw %r13,FRAME_R13(%r1) 955 mr %r26,%r0 956 mfcr %r27 957 mfxer %r28 958 mfctr %r29 959 mr %r30,%r0 960 mr %r31,%r3 961 stmw %r26,FRAME_LR(%r1) 962 mr %r31,%r1 963 mr %r1,%r10 964 addi %r4,%r1,FRAME_TF 965 li %r3,EXC_PGM 966 stw %r3,FRAME_EXC(%r1) 967 li %r3,T_PROGRAM 968 bl _C_LABEL(trap) 969 lmw %r26,FRAME_LR(%r1) 970 mtlr %r26 971 mtcr %r27 972 mtxer %r28 973 mtctr %r29 974 mr %r0,%r31 975 lmw %r13,FRAME_R13(%r1) 976 lwz %r1,FRAME_R1(%r1) 977 wrtee %r0 978 blr 979 #endif /* notyet */ 980
Indexes created Sun Oct 19 18:09:56 GMT 2025