1 /* $NetBSD: locore.s,v 1.436 2025/09/06 02:53:23 riastradh Exp $ */ 2 3 /* 4 * Copyright (c) 2006-2010 Matthew R. Green 5 * Copyright (c) 1996-2002 Eduardo Horvath 6 * Copyright (c) 1996 Paul Kranenburg 7 * Copyright (c) 1996 8 * The President and Fellows of Harvard College. 9 * All rights reserved. 10 * Copyright (c) 1992, 1993 11 * The Regents of the University of California. 12 * All rights reserved. 13 * 14 * This software was developed by the Computer Systems Engineering group 15 * at Lawrence Berkeley Laboratory under DARPA contract BG 91-66 and 16 * contributed to Berkeley. 17 * 18 * All advertising materials mentioning features or use of this software 19 * must display the following acknowledgement: 20 * This product includes software developed by the University of 21 * California, Lawrence Berkeley Laboratory. 22 * This product includes software developed by Harvard University. 23 * 24 * Redistribution and use in source and binary forms, with or without 25 * modification, are permitted provided that the following conditions 26 * are met: 27 * 1. Redistributions of source code must retain the above copyright 28 * notice, this list of conditions and the following disclaimer. 29 * 2. Redistributions in binary form must reproduce the above copyright 30 * notice, this list of conditions and the following disclaimer in the 31 * documentation and/or other materials provided with the 32 * distribution. 33 * 3. All advertising materials mentioning features or use of this 34 * software must display the following acknowledgement: 35 * This product includes software developed by the University of 36 * California, Berkeley and its contributors. 37 * This product includes software developed by Harvard University. 38 * This product includes software developed by Paul Kranenburg. 39 * 4. Neither the name of the University nor the names of its 40 * contributors may be used to endorse or promote products derived 41 * from this software without specific prior written permission. 42 * 43 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' 44 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 45 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A 46 * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR 47 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 48 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 49 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 50 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 51 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR 52 * TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF 53 * THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH 54 * DAMAGE. 55 * 56 * @(#)locore.s 8.4 (Berkeley) 12/10/93 57 */ 58 59 #undef PARANOID /* Extremely expensive consistency checks */ 60 #undef NO_VCACHE /* Map w/D$ disabled */ 61 #undef TRAPSTATS /* Count traps */ 62 #undef TRAPS_USE_IG /* Use Interrupt Globals for all traps */ 63 #define HWREF /* Track ref/mod bits in trap handlers */ 64 #undef DCACHE_BUG /* Flush D$ around ASI_PHYS accesses */ 65 #undef NO_TSB /* Don't use TSB */ 66 #define BB_ERRATA_1 /* writes to TICK_CMPR may fail */ 67 #undef TLB_FLUSH_LOWVA /* also flush 32-bit entries from the MMU */ 68 69 #include "opt_ddb.h" 70 #include "opt_kgdb.h" 71 #include "opt_multiprocessor.h" 72 #include "opt_compat_netbsd.h" 73 #include "opt_compat_netbsd32.h" 74 #include "opt_lockdebug.h" 75 76 #include "assym.h" 77 #include <machine/param.h> 78 #include <machine/types.h> 79 #include <sparc64/sparc64/intreg.h> 80 #include <sparc64/sparc64/timerreg.h> 81 #include <machine/ctlreg.h> 82 #include <machine/psl.h> 83 #include <machine/signal.h> 84 #include <machine/trap.h> 85 #include <machine/frame.h> 86 #include <machine/pmap.h> 87 #include <machine/intr.h> 88 #include <machine/asm.h> 89 #include <machine/locore.h> 90 #ifdef SUN4V 91 #include <machine/hypervisor.h> 92 #endif 93 #include <sys/syscall.h> 94 95 #define BLOCK_SIZE SPARC64_BLOCK_SIZE 96 #define BLOCK_ALIGN SPARC64_BLOCK_ALIGN 97 98 #ifdef SUN4V 99 #define SUN4V_N_REG_WINDOWS 8 /* As per UA2005 spec */ 100 #define SUN4V_NWINDOWS (SUN4V_N_REG_WINDOWS-1) /* This is an index number, so subtract one */ 101 #endif 102 103 #include "ksyms.h" 104 105 /* Misc. macros */ 106 107 .macro GET_MAXCWP reg 108 #ifdef SUN4V 109 sethi %hi(cputyp), \reg 110 ld [\reg + %lo(cputyp)], \reg 111 cmp \reg, CPU_SUN4V 112 bne,pt %icc, 2f 113 nop 114 /* sun4v */ 115 ba 3f 116 mov SUN4V_NWINDOWS, \reg 117 2: 118 #endif 119 /* sun4u */ 120 rdpr %ver, \reg 121 and \reg, CWP, \reg 122 3: 123 .endm 124 125 .macro SET_MMU_CONTEXTID_SUN4U ctxid,ctx 126 stxa \ctxid, [\ctx] ASI_DMMU; 127 .endm 128 129 #ifdef SUN4V 130 .macro SET_MMU_CONTEXTID_SUN4V ctxid,ctx 131 stxa \ctxid, [\ctx] ASI_MMU_CONTEXTID; 132 .endm 133 #endif 134 135 .macro SET_MMU_CONTEXTID ctxid,ctx,scratch 136 #ifdef SUN4V 137 sethi %hi(cputyp), \scratch 138 ld [\scratch + %lo(cputyp)], \scratch 139 cmp \scratch, CPU_SUN4V 140 bne,pt %icc, 2f 141 nop 142 /* sun4v */ 143 SET_MMU_CONTEXTID_SUN4V \ctxid,\ctx 144 ba 3f 145 nop 146 2: 147 #endif 148 /* sun4u */ 149 SET_MMU_CONTEXTID_SUN4U \ctxid,\ctx 150 3: 151 .endm 152 153 .macro GET_MMU_CONTEXTID_SUN4U ctxid,ctx 154 ldxa [\ctx] ASI_DMMU, \ctxid 155 .endm 156 157 #ifdef SUN4V 158 .macro GET_MMU_CONTEXTID_SUN4V ctxid,ctx 159 ldxa [\ctx] ASI_MMU_CONTEXTID, \ctxid 160 .endm 161 #endif 162 163 .macro GET_MMU_CONTEXTID ctxid,ctx,scratch 164 #ifdef SUN4V 165 sethi %hi(cputyp), \scratch 166 ld [\scratch + %lo(cputyp)], \scratch 167 cmp \scratch, CPU_SUN4V 168 bne,pt %icc, 2f 169 nop 170 /* sun4v */ 171 GET_MMU_CONTEXTID_SUN4V \ctxid,\ctx 172 ba 3f 173 nop 174 2: 175 #endif 176 /* sun4u */ 177 GET_MMU_CONTEXTID_SUN4U \ctxid,\ctx 178 3: 179 .endm 180 181 #ifdef SUN4V 182 .macro NORMAL_GLOBALS_SUN4V 183 wrpr %g0, 0, %gl ! Set globals to level 0 184 .endm 185 #endif 186 .macro NORMAL_GLOBALS_SUN4U 187 wrpr %g0, PSTATE_KERN, %pstate ! Alternate Globals (AG) bit set to zero 188 .endm 189 190 #ifdef SUN4V 191 .macro ALTERNATE_GLOBALS_SUN4V 192 wrpr %g0, 1, %gl ! Set globals to level 1 193 .endm 194 #endif 195 .macro ALTERNATE_GLOBALS_SUN4U 196 wrpr %g0, PSTATE_KERN|PSTATE_AG, %pstate ! Alternate Globals (AG) bit set to one 197 .endm 198 199 .macro ENABLE_INTERRUPTS scratch 200 rdpr %pstate, \scratch 201 or \scratch, PSTATE_IE, \scratch ! Interrupt Enable (IE) bit set to one 202 wrpr %g0, \scratch, %pstate 203 .endm 204 205 .macro DISABLE_INTERRUPTS scratch 206 rdpr %pstate, \scratch 207 and \scratch, ~PSTATE_IE, \scratch ! Interrupt Enable (IE) bit set to zero 208 wrpr %g0, \scratch, %pstate 209 .endm 210 211 212 #ifdef SUN4V 213 /* Misc. sun4v macros */ 214 215 .macro GET_MMFSA reg 216 sethi %hi(CPUINFO_VA + CI_MMUFSA), \reg 217 LDPTR [\reg + %lo(CPUINFO_VA + CI_MMUFSA)], \reg 218 .endm 219 220 .macro GET_CTXBUSY reg 221 sethi %hi(CPUINFO_VA + CI_CTXBUSY), \reg 222 LDPTR [\reg + %lo(CPUINFO_VA + CI_CTXBUSY)], \reg 223 .endm 224 225 .macro GET_TSB_DMMU reg 226 sethi %hi(CPUINFO_VA + CI_TSB_DMMU), \reg 227 LDPTR [\reg + %lo(CPUINFO_VA + CI_TSB_DMMU)], \reg 228 .endm 229 230 .macro sun4v_tl1_uspill_normal 231 ba,a,pt %xcc, spill_normal_to_user_stack 232 nop 233 .align 128 234 .endm 235 236 .macro sun4v_tl1_uspill_other 237 ba,a,pt %xcc, pcbspill_other 238 nop 239 .align 128 240 .endm 241 242 #endif 243 244 #if 1 245 /* 246 * Try to issue an elf note to ask the Solaris 247 * bootloader to align the kernel properly. 248 */ 249 .section .note 250 .word 0x0d 251 .word 4 ! Dunno why 252 .word 1 253 0: .asciz "SUNW Solaris" 254 1: 255 .align 4 256 .word 0x0400000 257 #endif 258 259 .register %g2,#scratch 260 .register %g3,#scratch 261 262 263 .data 264 .globl _C_LABEL(data_start) 265 _C_LABEL(data_start): ! Start of data segment 266 267 #ifdef KGDB 268 /* 269 * Another item that must be aligned, easiest to put it here. 270 */ 271 KGDB_STACK_SIZE = 2048 272 .globl _C_LABEL(kgdb_stack) 273 _C_LABEL(kgdb_stack): 274 .space KGDB_STACK_SIZE ! hope this is enough 275 #endif 276 277 #ifdef NOTDEF_DEBUG 278 /* 279 * This stack is used when we detect kernel stack corruption. 280 */ 281 .space USPACE 282 .align 16 283 panicstack: 284 #endif 285 286 /* 287 * romp is the prom entry pointer 288 * romtba is the prom trap table base address 289 */ 290 .globl romp 291 romp: POINTER 0 292 .globl romtba 293 romtba: POINTER 0 294 295 .globl cputyp 296 cputyp: .word CPU_SUN4U ! Default to sun4u 297 298 _ALIGN 299 .text 300 301 /* 302 * The v9 trap frame is stored in the special trap registers. The 303 * register window is only modified on window overflow, underflow, 304 * and clean window traps, where it points to the register window 305 * needing service. Traps have space for 8 instructions, except for 306 * the window overflow, underflow, and clean window traps which are 307 * 32 instructions long, large enough to in-line. 308 * 309 * The spitfire CPU (Ultra I) has 4 different sets of global registers. 310 * (blah blah...) 311 * 312 * I used to generate these numbers by address arithmetic, but gas's 313 * expression evaluator has about as much sense as your average slug 314 * (oddly enough, the code looks about as slimy too). Thus, all the 315 * trap numbers are given as arguments to the trap macros. This means 316 * there is one line per trap. Sigh. 317 * 318 * Hardware interrupt vectors can be `linked'---the linkage is to regular 319 * C code---or rewired to fast in-window handlers. The latter are good 320 * for unbuffered hardware like the Zilog serial chip and the AMD audio 321 * chip, where many interrupts can be handled trivially with pseudo-DMA 322 * or similar. Only one `fast' interrupt can be used per level, however, 323 * and direct and `fast' interrupts are incompatible. Routines in intr.c 324 * handle setting these, with optional paranoia. 325 */ 326 327 /* 328 * TA8 -- trap align for 8 instruction traps 329 * TA32 -- trap align for 32 instruction traps 330 */ 331 #define TA8 .align 32 332 #define TA32 .align 128 333 334 /* 335 * v9 trap macros: 336 * 337 * We have a problem with v9 traps; we have no registers to put the 338 * trap type into. But we do have a %tt register which already has 339 * that information. Trap types in these macros are all dummys. 340 */ 341 /* regular vectored traps */ 342 343 #define VTRAP(type, label) \ 344 ba,a,pt %icc,label; nop; NOTREACHED; TA8 345 346 /* hardware interrupts (can be linked or made `fast') */ 347 #define HARDINT4U(lev) \ 348 VTRAP(lev, _C_LABEL(sparc_interrupt)) 349 #ifdef SUN4V 350 #define HARDINT4V(lev) HARDINT4U(lev) 351 #endif 352 353 /* software interrupts (may not be made direct, sorry---but you 354 should not be using them trivially anyway) */ 355 #define SOFTINT4U(lev, bit) \ 356 HARDINT4U(lev) 357 358 /* traps that just call trap() */ 359 #define TRAP(type) VTRAP(type, slowtrap) 360 361 /* architecturally undefined traps (cause panic) */ 362 #ifndef DEBUG 363 #define UTRAP(type) sir; VTRAP(type, slowtrap) 364 #else 365 #define UTRAP(type) VTRAP(type, slowtrap) 366 #endif 367 368 /* software undefined traps (may be replaced) */ 369 #define STRAP(type) VTRAP(type, slowtrap) 370 371 /* breakpoint acts differently under kgdb */ 372 #ifdef KGDB 373 #define BPT VTRAP(T_BREAKPOINT, bpt) 374 #define BPT_KGDB_EXEC VTRAP(T_KGDB_EXEC, bpt) 375 #else 376 #define BPT TRAP(T_BREAKPOINT) 377 #define BPT_KGDB_EXEC TRAP(T_KGDB_EXEC) 378 #endif 379 380 #define SYSCALL VTRAP(0x100, syscall_setup) 381 #ifdef notyet 382 #define ZS_INTERRUPT ba,a,pt %icc, zshard; nop; TA8 383 #else 384 #define ZS_INTERRUPT4U HARDINT4U(12) 385 #endif 386 387 388 /* 389 * Macro to clear %tt so we don't get confused with old traps. 390 */ 391 #ifdef DEBUG 392 #define CLRTT wrpr %g0,0x1ff,%tt 393 #else 394 #define CLRTT 395 #endif 396 397 398 /* 399 * Some macros to load and store a register window 400 */ 401 402 .macro SPILL storer,base,size,asi 403 404 .irpc n,01234567 405 \storer %l\n, [\base + (\n * \size)] \asi 406 .endr 407 .irpc n,01234567 408 \storer %i\n, [\base + ((8+\n) * \size)] \asi 409 .endr 410 411 .endm 412 413 414 .macro FILL loader, base, size, asi 415 416 .irpc n,01234567 417 \loader [\base + (\n * \size)] \asi, %l\n 418 .endr 419 420 .irpc n,01234567 421 \loader [\base + ((8+\n) * \size)] \asi, %i\n 422 .endr 423 424 .endm 425 426 /* 427 * Here are some oft repeated traps as macros. 428 */ 429 430 /* spill a 64-bit register window */ 431 #define SPILL64(label,as) \ 432 label: \ 433 wr %g0, as, %asi; \ 434 stxa %l0, [%sp+BIAS+0x00]%asi; \ 435 stxa %l1, [%sp+BIAS+0x08]%asi; \ 436 stxa %l2, [%sp+BIAS+0x10]%asi; \ 437 stxa %l3, [%sp+BIAS+0x18]%asi; \ 438 stxa %l4, [%sp+BIAS+0x20]%asi; \ 439 stxa %l5, [%sp+BIAS+0x28]%asi; \ 440 stxa %l6, [%sp+BIAS+0x30]%asi; \ 441 \ 442 stxa %l7, [%sp+BIAS+0x38]%asi; \ 443 stxa %i0, [%sp+BIAS+0x40]%asi; \ 444 stxa %i1, [%sp+BIAS+0x48]%asi; \ 445 stxa %i2, [%sp+BIAS+0x50]%asi; \ 446 stxa %i3, [%sp+BIAS+0x58]%asi; \ 447 stxa %i4, [%sp+BIAS+0x60]%asi; \ 448 stxa %i5, [%sp+BIAS+0x68]%asi; \ 449 stxa %i6, [%sp+BIAS+0x70]%asi; \ 450 \ 451 stxa %i7, [%sp+BIAS+0x78]%asi; \ 452 saved; \ 453 CLRTT; \ 454 retry; \ 455 NOTREACHED; \ 456 TA32 457 458 /* spill a 32-bit register window */ 459 #define SPILL32(label,as) \ 460 label: \ 461 wr %g0, as, %asi; \ 462 srl %sp, 0, %sp; /* fixup 32-bit pointers */ \ 463 stwa %l0, [%sp+0x00]%asi; \ 464 stwa %l1, [%sp+0x04]%asi; \ 465 stwa %l2, [%sp+0x08]%asi; \ 466 stwa %l3, [%sp+0x0c]%asi; \ 467 stwa %l4, [%sp+0x10]%asi; \ 468 stwa %l5, [%sp+0x14]%asi; \ 469 \ 470 stwa %l6, [%sp+0x18]%asi; \ 471 stwa %l7, [%sp+0x1c]%asi; \ 472 stwa %i0, [%sp+0x20]%asi; \ 473 stwa %i1, [%sp+0x24]%asi; \ 474 stwa %i2, [%sp+0x28]%asi; \ 475 stwa %i3, [%sp+0x2c]%asi; \ 476 stwa %i4, [%sp+0x30]%asi; \ 477 stwa %i5, [%sp+0x34]%asi; \ 478 \ 479 stwa %i6, [%sp+0x38]%asi; \ 480 stwa %i7, [%sp+0x3c]%asi; \ 481 saved; \ 482 CLRTT; \ 483 retry; \ 484 NOTREACHED; \ 485 TA32 486 487 /* Spill either 32-bit or 64-bit register window. */ 488 #define SPILLBOTH(label64,label32,as) \ 489 andcc %sp, 1, %g0; \ 490 bnz,pt %xcc, label64+4; /* Is it a v9 or v8 stack? */ \ 491 wr %g0, as, %asi; \ 492 ba,pt %xcc, label32+8; \ 493 srl %sp, 0, %sp; /* fixup 32-bit pointers */ \ 494 NOTREACHED; \ 495 TA32 496 497 /* fill a 64-bit register window */ 498 #define FILL64(label,as) \ 499 label: \ 500 wr %g0, as, %asi; \ 501 ldxa [%sp+BIAS+0x00]%asi, %l0; \ 502 ldxa [%sp+BIAS+0x08]%asi, %l1; \ 503 ldxa [%sp+BIAS+0x10]%asi, %l2; \ 504 ldxa [%sp+BIAS+0x18]%asi, %l3; \ 505 ldxa [%sp+BIAS+0x20]%asi, %l4; \ 506 ldxa [%sp+BIAS+0x28]%asi, %l5; \ 507 ldxa [%sp+BIAS+0x30]%asi, %l6; \ 508 \ 509 ldxa [%sp+BIAS+0x38]%asi, %l7; \ 510 ldxa [%sp+BIAS+0x40]%asi, %i0; \ 511 ldxa [%sp+BIAS+0x48]%asi, %i1; \ 512 ldxa [%sp+BIAS+0x50]%asi, %i2; \ 513 ldxa [%sp+BIAS+0x58]%asi, %i3; \ 514 ldxa [%sp+BIAS+0x60]%asi, %i4; \ 515 ldxa [%sp+BIAS+0x68]%asi, %i5; \ 516 ldxa [%sp+BIAS+0x70]%asi, %i6; \ 517 \ 518 ldxa [%sp+BIAS+0x78]%asi, %i7; \ 519 restored; \ 520 CLRTT; \ 521 retry; \ 522 NOTREACHED; \ 523 TA32 524 525 /* fill a 32-bit register window */ 526 #define FILL32(label,as) \ 527 label: \ 528 wr %g0, as, %asi; \ 529 srl %sp, 0, %sp; /* fixup 32-bit pointers */ \ 530 lda [%sp+0x00]%asi, %l0; \ 531 lda [%sp+0x04]%asi, %l1; \ 532 lda [%sp+0x08]%asi, %l2; \ 533 lda [%sp+0x0c]%asi, %l3; \ 534 lda [%sp+0x10]%asi, %l4; \ 535 lda [%sp+0x14]%asi, %l5; \ 536 \ 537 lda [%sp+0x18]%asi, %l6; \ 538 lda [%sp+0x1c]%asi, %l7; \ 539 lda [%sp+0x20]%asi, %i0; \ 540 lda [%sp+0x24]%asi, %i1; \ 541 lda [%sp+0x28]%asi, %i2; \ 542 lda [%sp+0x2c]%asi, %i3; \ 543 lda [%sp+0x30]%asi, %i4; \ 544 lda [%sp+0x34]%asi, %i5; \ 545 \ 546 lda [%sp+0x38]%asi, %i6; \ 547 lda [%sp+0x3c]%asi, %i7; \ 548 restored; \ 549 CLRTT; \ 550 retry; \ 551 NOTREACHED; \ 552 TA32 553 554 /* fill either 32-bit or 64-bit register window. */ 555 #define FILLBOTH(label64,label32,as) \ 556 andcc %sp, 1, %i0; \ 557 bnz (label64)+4; /* See if it's a v9 stack or v8 */ \ 558 wr %g0, as, %asi; \ 559 ba (label32)+8; \ 560 srl %sp, 0, %sp; /* fixup 32-bit pointers */ \ 561 NOTREACHED; \ 562 TA32 563 564 /* handle clean window trap when trap level = 0 */ 565 .macro CLEANWIN0 566 rdpr %cleanwin, %o7 567 inc %o7 ! This handler is in-lined and cannot fault 568 #ifdef DEBUG 569 set 0xbadcafe, %l0 ! DEBUG -- compiler should not rely on zero-ed registers. 570 #else 571 clr %l0 572 #endif 573 wrpr %g0, %o7, %cleanwin ! Nucleus (trap&IRQ) code does not need clean windows 574 575 mov %l0,%l1; mov %l0,%l2 ! Clear out %l0-%l8 and %o0-%o8 and inc %cleanwin and done 576 mov %l0,%l3; mov %l0,%l4 577 #if 0 578 #ifdef DIAGNOSTIC 579 !! 580 !! Check the sp redzone 581 !! 582 !! Since we can't spill the current window, we'll just keep 583 !! track of the frame pointer. Problems occur when the routine 584 !! allocates and uses stack storage. 585 !! 586 ! rdpr %wstate, %l5 ! User stack? 587 ! cmp %l5, WSTATE_KERN 588 ! bne,pt %icc, 7f 589 sethi %hi(CPCB), %l5 590 LDPTR [%l5 + %lo(CPCB)], %l5 ! If pcb < fp < pcb+sizeof(pcb) 591 inc PCB_SIZE, %l5 ! then we have a stack overflow 592 btst %fp, 1 ! 64-bit stack? 593 sub %fp, %l5, %l7 594 bnz,a,pt %icc, 1f 595 inc BIAS, %l7 ! Remove BIAS 596 1: 597 cmp %l7, PCB_SIZE 598 blu %xcc, cleanwin_overflow 599 #endif 600 #endif 601 mov %l0, %l5 602 mov %l0, %l6; mov %l0, %l7; mov %l0, %o0; mov %l0, %o1 603 604 mov %l0, %o2; mov %l0, %o3; mov %l0, %o4; mov %l0, %o5; 605 mov %l0, %o6; mov %l0, %o7 606 CLRTT 607 retry; nop; NOTREACHED; TA32 608 .endm 609 610 /* handle clean window trap when trap level = 1 */ 611 .macro CLEANWIN1 612 clr %l0 613 #ifdef DEBUG 614 set 0xbadbeef, %l0 ! DEBUG 615 #endif 616 mov %l0, %l1; mov %l0, %l2 617 rdpr %cleanwin, %o7 ! This handler is in-lined and cannot fault 618 inc %o7; mov %l0, %l3 ! Nucleus (trap&IRQ) code does not need clean windows 619 wrpr %g0, %o7, %cleanwin ! Clear out %l0-%l8 and %o0-%o8 and inc %cleanwin and done 620 #ifdef NOT_DEBUG 621 !! 622 !! Check the sp redzone 623 !! 624 rdpr %wstate, t1 625 cmp t1, WSTATE_KERN 626 bne,pt icc, 7f 627 sethi %hi(_C_LABEL(redzone)), t1 628 ldx [t1 + %lo(_C_LABEL(redzone))], t2 629 cmp %sp, t2 ! if sp >= t2, not in red zone 630 blu panic_red ! and can continue normally 631 7: 632 #endif 633 mov %l0, %l4; mov %l0, %l5; mov %l0, %l6; mov %l0, %l7 634 mov %l0, %o0; mov %l0, %o1; mov %l0, %o2; mov %l0, %o3 635 636 mov %l0, %o4; mov %l0, %o5; mov %l0, %o6; mov %l0, %o7 637 CLRTT 638 retry; nop; TA32 639 .endm 640 641 .globl start, _C_LABEL(kernel_text) 642 _C_LABEL(kernel_text) = kernel_start ! for kvm_mkdb(8) 643 kernel_start: 644 /* Traps from TL=0 -- traps from user mode */ 645 #ifdef __STDC__ 646 #define TABLE(name) user_ ## name 647 #else 648 #define TABLE(name) user_/**/name 649 #endif 650 .globl _C_LABEL(trapbase) 651 _C_LABEL(trapbase): 652 b dostart; nop; TA8 ! 000 = reserved -- Use it to boot 653 /* We should not get the next 5 traps */ 654 UTRAP(0x001) ! 001 = POR Reset -- ROM should get this 655 UTRAP(0x002) ! 002 = WDR -- ROM should get this 656 UTRAP(0x003) ! 003 = XIR -- ROM should get this 657 UTRAP(0x004) ! 004 = SIR -- ROM should get this 658 UTRAP(0x005) ! 005 = RED state exception 659 UTRAP(0x006); UTRAP(0x007) 660 VTRAP(T_INST_EXCEPT, textfault) ! 008 = instr. access except 661 VTRAP(T_TEXTFAULT, textfault) ! 009 = instr access MMU miss 662 VTRAP(T_INST_ERROR, textfault) ! 00a = instr. access err 663 UTRAP(0x00b); UTRAP(0x00c); UTRAP(0x00d); UTRAP(0x00e); UTRAP(0x00f) 664 TRAP(T_ILLINST) ! 010 = illegal instruction 665 TRAP(T_PRIVINST) ! 011 = privileged instruction 666 UTRAP(0x012) ! 012 = unimplemented LDD 667 UTRAP(0x013) ! 013 = unimplemented STD 668 UTRAP(0x014); UTRAP(0x015); UTRAP(0x016); UTRAP(0x017); UTRAP(0x018) 669 UTRAP(0x019); UTRAP(0x01a); UTRAP(0x01b); UTRAP(0x01c); UTRAP(0x01d) 670 UTRAP(0x01e); UTRAP(0x01f) 671 TRAP(T_FPDISABLED) ! 020 = fp instr, but EF bit off in psr 672 TRAP(T_FP_IEEE_754) ! 021 = ieee 754 exception 673 TRAP(T_FP_OTHER) ! 022 = other fp exception 674 TRAP(T_TAGOF) ! 023 = tag overflow 675 CLEANWIN0 ! 024-027 = clean window trap 676 TRAP(T_DIV0) ! 028 = divide by zero 677 UTRAP(0x029) ! 029 = internal processor error 678 UTRAP(0x02a); UTRAP(0x02b); UTRAP(0x02c); UTRAP(0x02d); UTRAP(0x02e); UTRAP(0x02f) 679 VTRAP(T_DATAFAULT, winfault) ! 030 = data fetch fault 680 UTRAP(0x031) ! 031 = data MMU miss -- no MMU 681 VTRAP(T_DATA_ERROR, winfault) ! 032 = data access error 682 VTRAP(T_DATA_PROT, winfault) ! 033 = data protection fault 683 TRAP(T_ALIGN) ! 034 = address alignment error -- we could fix it inline... 684 TRAP(T_LDDF_ALIGN) ! 035 = LDDF address alignment error -- we could fix it inline... 685 TRAP(T_STDF_ALIGN) ! 036 = STDF address alignment error -- we could fix it inline... 686 TRAP(T_PRIVACT) ! 037 = privileged action 687 UTRAP(0x038); UTRAP(0x039); UTRAP(0x03a); UTRAP(0x03b); UTRAP(0x03c); 688 UTRAP(0x03d); UTRAP(0x03e); UTRAP(0x03f); 689 VTRAP(T_ASYNC_ERROR, winfault) ! 040 = data fetch fault 690 SOFTINT4U(1, IE_L1) ! 041 = level 1 interrupt 691 HARDINT4U(2) ! 042 = level 2 interrupt 692 HARDINT4U(3) ! 043 = level 3 interrupt 693 SOFTINT4U(4, IE_L4) ! 044 = level 4 interrupt 694 HARDINT4U(5) ! 045 = level 5 interrupt 695 SOFTINT4U(6, IE_L6) ! 046 = level 6 interrupt 696 HARDINT4U(7) ! 047 = level 7 interrupt 697 HARDINT4U(8) ! 048 = level 8 interrupt 698 HARDINT4U(9) ! 049 = level 9 interrupt 699 HARDINT4U(10) ! 04a = level 10 interrupt 700 HARDINT4U(11) ! 04b = level 11 interrupt 701 ZS_INTERRUPT4U ! 04c = level 12 (zs) interrupt 702 HARDINT4U(13) ! 04d = level 13 interrupt 703 HARDINT4U(14) ! 04e = level 14 interrupt 704 HARDINT4U(15) ! 04f = nonmaskable interrupt 705 UTRAP(0x050); UTRAP(0x051); UTRAP(0x052); UTRAP(0x053); UTRAP(0x054); UTRAP(0x055) 706 UTRAP(0x056); UTRAP(0x057); UTRAP(0x058); UTRAP(0x059); UTRAP(0x05a); UTRAP(0x05b) 707 UTRAP(0x05c); UTRAP(0x05d); UTRAP(0x05e); UTRAP(0x05f) 708 VTRAP(0x060, interrupt_vector); ! 060 = interrupt vector 709 TRAP(T_PA_WATCHPT) ! 061 = physical address data watchpoint 710 TRAP(T_VA_WATCHPT) ! 062 = virtual address data watchpoint 711 TRAP(T_ECCERR) ! 063 = corrected ECC error 712 ufast_IMMU_miss: ! 064 = fast instr access MMU miss 713 ldxa [%g0] ASI_IMMU_8KPTR, %g2 ! Load IMMU 8K TSB pointer 714 #ifdef NO_TSB 715 ba,a %icc, instr_miss 716 #endif 717 ldxa [%g0] ASI_IMMU, %g1 ! Load IMMU tag target register 718 ldda [%g2] ASI_NUCLEUS_QUAD_LDD, %g4 ! Load TSB tag:data into %g4:%g5 719 brgez,pn %g5, instr_miss ! Entry invalid? Punt 720 cmp %g1, %g4 ! Compare TLB tags 721 bne,pn %xcc, instr_miss ! Got right tag? 722 nop 723 CLRTT 724 stxa %g5, [%g0] ASI_IMMU_DATA_IN ! Enter new mapping 725 retry ! Try new mapping 726 1: 727 sir 728 TA32 729 ufast_DMMU_miss: ! 068 = fast data access MMU miss 730 ldxa [%g0] ASI_DMMU_8KPTR, %g2! Load DMMU 8K TSB pointer 731 #ifdef NO_TSB 732 ba,a %icc, data_miss 733 #endif 734 ldxa [%g0] ASI_DMMU, %g1 ! Load DMMU tag target register 735 ldda [%g2] ASI_NUCLEUS_QUAD_LDD, %g4 ! Load TSB tag and data into %g4 and %g5 736 brgez,pn %g5, data_miss ! Entry invalid? Punt 737 cmp %g1, %g4 ! Compare TLB tags 738 bnz,pn %xcc, data_miss ! Got right tag? 739 nop 740 CLRTT 741 #ifdef TRAPSTATS 742 sethi %hi(_C_LABEL(udhit)), %g1 743 lduw [%g1+%lo(_C_LABEL(udhit))], %g2 744 inc %g2 745 stw %g2, [%g1+%lo(_C_LABEL(udhit))] 746 #endif 747 stxa %g5, [%g0] ASI_DMMU_DATA_IN ! Enter new mapping 748 retry ! Try new mapping 749 1: 750 sir 751 TA32 752 ufast_DMMU_protection: ! 06c = fast data access MMU protection 753 #ifdef TRAPSTATS 754 sethi %hi(_C_LABEL(udprot)), %g1 755 lduw [%g1+%lo(_C_LABEL(udprot))], %g2 756 inc %g2 757 stw %g2, [%g1+%lo(_C_LABEL(udprot))] 758 #endif 759 #ifdef HWREF 760 ba,a,pt %xcc, dmmu_write_fault 761 #else 762 ba,a,pt %xcc, winfault 763 #endif 764 nop 765 TA32 766 TRAP(0x070) ! 0x070 fast_ECC_error 767 ! Implementation dependent traps 768 UTRAP(0x071); UTRAP(0x072); UTRAP(0x073); UTRAP(0x074); UTRAP(0x075); UTRAP(0x076) 769 UTRAP(0x077); UTRAP(0x078); UTRAP(0x079); UTRAP(0x07a); UTRAP(0x07b); UTRAP(0x07c) 770 UTRAP(0x07d); UTRAP(0x07e); UTRAP(0x07f) 771 TABLE(uspill): 772 SPILL64(uspill8,ASI_AIUS) ! 0x080 spill_0_normal -- used to save user windows in user mode 773 SPILL32(uspill4,ASI_AIUS) ! 0x084 spill_1_normal 774 SPILLBOTH(uspill8,uspill4,ASI_AIUS) ! 0x088 spill_2_normal 775 UTRAP(0x08c); TA32 ! 0x08c spill_3_normal 776 TABLE(kspill): 777 SPILL64(kspill8,ASI_N) ! 0x090 spill_4_normal -- used to save supervisor windows 778 SPILL32(kspill4,ASI_N) ! 0x094 spill_5_normal 779 SPILLBOTH(kspill8,kspill4,ASI_N) ! 0x098 spill_6_normal 780 UTRAP(0x09c); TA32 ! 0x09c spill_7_normal 781 TABLE(uspillk): 782 SPILL64(uspillk8,ASI_AIUS) ! 0x0a0 spill_0_other -- used to save user windows in supervisor mode 783 SPILL32(uspillk4,ASI_AIUS) ! 0x0a4 spill_1_other 784 SPILLBOTH(uspillk8,uspillk4,ASI_AIUS) ! 0x0a8 spill_2_other 785 UTRAP(0x0ac); TA32 ! 0x0ac spill_3_other 786 UTRAP(0x0b0); TA32 ! 0x0b0 spill_4_other 787 UTRAP(0x0b4); TA32 ! 0x0b4 spill_5_other 788 UTRAP(0x0b8); TA32 ! 0x0b8 spill_6_other 789 UTRAP(0x0bc); TA32 ! 0x0bc spill_7_other 790 TABLE(ufill): 791 FILL64(ufill8,ASI_AIUS) ! 0x0c0 fill_0_normal -- used to fill windows when running user mode 792 FILL32(ufill4,ASI_AIUS) ! 0x0c4 fill_1_normal 793 FILLBOTH(ufill8,ufill4,ASI_AIUS) ! 0x0c8 fill_2_normal 794 UTRAP(0x0cc); TA32 ! 0x0cc fill_3_normal 795 TABLE(kfill): 796 FILL64(kfill8,ASI_N) ! 0x0d0 fill_4_normal -- used to fill windows when running supervisor mode 797 FILL32(kfill4,ASI_N) ! 0x0d4 fill_5_normal 798 FILLBOTH(kfill8,kfill4,ASI_N) ! 0x0d8 fill_6_normal 799 UTRAP(0x0dc); TA32 ! 0x0dc fill_7_normal 800 TABLE(ufillk): 801 FILL64(ufillk8,ASI_AIUS) ! 0x0e0 fill_0_other 802 FILL32(ufillk4,ASI_AIUS) ! 0x0e4 fill_1_other 803 FILLBOTH(ufillk8,ufillk4,ASI_AIUS) ! 0x0e8 fill_2_other 804 UTRAP(0x0ec); TA32 ! 0x0ec fill_3_other 805 UTRAP(0x0f0); TA32 ! 0x0f0 fill_4_other 806 UTRAP(0x0f4); TA32 ! 0x0f4 fill_5_other 807 UTRAP(0x0f8); TA32 ! 0x0f8 fill_6_other 808 UTRAP(0x0fc); TA32 ! 0x0fc fill_7_other 809 TABLE(syscall): 810 SYSCALL ! 0x100 = sun syscall 811 BPT ! 0x101 = pseudo breakpoint instruction 812 STRAP(0x102); STRAP(0x103); STRAP(0x104); STRAP(0x105); STRAP(0x106); STRAP(0x107) 813 SYSCALL ! 0x108 = svr4 syscall 814 SYSCALL ! 0x109 = bsd syscall 815 BPT_KGDB_EXEC ! 0x10a = enter kernel gdb on kernel startup 816 STRAP(0x10b); STRAP(0x10c); STRAP(0x10d); STRAP(0x10e); STRAP(0x10f); 817 STRAP(0x110); STRAP(0x111); STRAP(0x112); STRAP(0x113); STRAP(0x114); STRAP(0x115); STRAP(0x116); STRAP(0x117) 818 STRAP(0x118); STRAP(0x119); STRAP(0x11a); STRAP(0x11b); STRAP(0x11c); STRAP(0x11d); STRAP(0x11e); STRAP(0x11f) 819 STRAP(0x120); STRAP(0x121); STRAP(0x122); STRAP(0x123); STRAP(0x124); STRAP(0x125); STRAP(0x126); STRAP(0x127) 820 STRAP(0x128); STRAP(0x129); STRAP(0x12a); STRAP(0x12b); STRAP(0x12c); STRAP(0x12d); STRAP(0x12e); STRAP(0x12f) 821 STRAP(0x130); STRAP(0x131); STRAP(0x132); STRAP(0x133); STRAP(0x134); STRAP(0x135); STRAP(0x136); STRAP(0x137) 822 STRAP(0x138); STRAP(0x139); STRAP(0x13a); STRAP(0x13b); STRAP(0x13c); STRAP(0x13d); STRAP(0x13e); STRAP(0x13f) 823 SYSCALL ! 0x140 SVID syscall (Solaris 2.7) 824 SYSCALL ! 0x141 SPARC International syscall 825 SYSCALL ! 0x142 OS Vendor syscall 826 SYSCALL ! 0x143 HW OEM syscall 827 STRAP(0x144); STRAP(0x145); STRAP(0x146); STRAP(0x147) 828 STRAP(0x148); STRAP(0x149); STRAP(0x14a); STRAP(0x14b); STRAP(0x14c); STRAP(0x14d); STRAP(0x14e); STRAP(0x14f) 829 STRAP(0x150); STRAP(0x151); STRAP(0x152); STRAP(0x153); STRAP(0x154); STRAP(0x155); STRAP(0x156); STRAP(0x157) 830 STRAP(0x158); STRAP(0x159); STRAP(0x15a); STRAP(0x15b); STRAP(0x15c); STRAP(0x15d); STRAP(0x15e); STRAP(0x15f) 831 STRAP(0x160); STRAP(0x161); STRAP(0x162); STRAP(0x163); STRAP(0x164); STRAP(0x165); STRAP(0x166); STRAP(0x167) 832 STRAP(0x168); STRAP(0x169); STRAP(0x16a); STRAP(0x16b); STRAP(0x16c); STRAP(0x16d); STRAP(0x16e); STRAP(0x16f) 833 STRAP(0x170); STRAP(0x171); STRAP(0x172); STRAP(0x173); STRAP(0x174); STRAP(0x175); STRAP(0x176); STRAP(0x177) 834 STRAP(0x178); STRAP(0x179); STRAP(0x17a); STRAP(0x17b); STRAP(0x17c); STRAP(0x17d); STRAP(0x17e); STRAP(0x17f) 835 ! Traps beyond 0x17f are reserved 836 UTRAP(0x180); UTRAP(0x181); UTRAP(0x182); UTRAP(0x183); UTRAP(0x184); UTRAP(0x185); UTRAP(0x186); UTRAP(0x187) 837 UTRAP(0x188); UTRAP(0x189); UTRAP(0x18a); UTRAP(0x18b); UTRAP(0x18c); UTRAP(0x18d); UTRAP(0x18e); UTRAP(0x18f) 838 UTRAP(0x190); UTRAP(0x191); UTRAP(0x192); UTRAP(0x193); UTRAP(0x194); UTRAP(0x195); UTRAP(0x196); UTRAP(0x197) 839 UTRAP(0x198); UTRAP(0x199); UTRAP(0x19a); UTRAP(0x19b); UTRAP(0x19c); UTRAP(0x19d); UTRAP(0x19e); UTRAP(0x19f) 840 UTRAP(0x1a0); UTRAP(0x1a1); UTRAP(0x1a2); UTRAP(0x1a3); UTRAP(0x1a4); UTRAP(0x1a5); UTRAP(0x1a6); UTRAP(0x1a7) 841 UTRAP(0x1a8); UTRAP(0x1a9); UTRAP(0x1aa); UTRAP(0x1ab); UTRAP(0x1ac); UTRAP(0x1ad); UTRAP(0x1ae); UTRAP(0x1af) 842 UTRAP(0x1b0); UTRAP(0x1b1); UTRAP(0x1b2); UTRAP(0x1b3); UTRAP(0x1b4); UTRAP(0x1b5); UTRAP(0x1b6); UTRAP(0x1b7) 843 UTRAP(0x1b8); UTRAP(0x1b9); UTRAP(0x1ba); UTRAP(0x1bb); UTRAP(0x1bc); UTRAP(0x1bd); UTRAP(0x1be); UTRAP(0x1bf) 844 UTRAP(0x1c0); UTRAP(0x1c1); UTRAP(0x1c2); UTRAP(0x1c3); UTRAP(0x1c4); UTRAP(0x1c5); UTRAP(0x1c6); UTRAP(0x1c7) 845 UTRAP(0x1c8); UTRAP(0x1c9); UTRAP(0x1ca); UTRAP(0x1cb); UTRAP(0x1cc); UTRAP(0x1cd); UTRAP(0x1ce); UTRAP(0x1cf) 846 UTRAP(0x1d0); UTRAP(0x1d1); UTRAP(0x1d2); UTRAP(0x1d3); UTRAP(0x1d4); UTRAP(0x1d5); UTRAP(0x1d6); UTRAP(0x1d7) 847 UTRAP(0x1d8); UTRAP(0x1d9); UTRAP(0x1da); UTRAP(0x1db); UTRAP(0x1dc); UTRAP(0x1dd); UTRAP(0x1de); UTRAP(0x1df) 848 UTRAP(0x1e0); UTRAP(0x1e1); UTRAP(0x1e2); UTRAP(0x1e3); UTRAP(0x1e4); UTRAP(0x1e5); UTRAP(0x1e6); UTRAP(0x1e7) 849 UTRAP(0x1e8); UTRAP(0x1e9); UTRAP(0x1ea); UTRAP(0x1eb); UTRAP(0x1ec); UTRAP(0x1ed); UTRAP(0x1ee); UTRAP(0x1ef) 850 UTRAP(0x1f0); UTRAP(0x1f1); UTRAP(0x1f2); UTRAP(0x1f3); UTRAP(0x1f4); UTRAP(0x1f5); UTRAP(0x1f6); UTRAP(0x1f7) 851 UTRAP(0x1f8); UTRAP(0x1f9); UTRAP(0x1fa); UTRAP(0x1fb); UTRAP(0x1fc); UTRAP(0x1fd); UTRAP(0x1fe); UTRAP(0x1ff) 852 853 /* Traps from TL>0 -- traps from supervisor mode */ 854 #undef TABLE 855 #ifdef __STDC__ 856 #define TABLE(name) nucleus_ ## name 857 #else 858 #define TABLE(name) nucleus_/**/name 859 #endif 860 trapbase_priv: 861 UTRAP(0x000) ! 000 = reserved -- Use it to boot 862 /* We should not get the next 5 traps */ 863 UTRAP(0x001) ! 001 = POR Reset -- ROM should get this 864 UTRAP(0x002) ! 002 = WDR Watchdog -- ROM should get this 865 UTRAP(0x003) ! 003 = XIR -- ROM should get this 866 UTRAP(0x004) ! 004 = SIR -- ROM should get this 867 UTRAP(0x005) ! 005 = RED state exception 868 UTRAP(0x006); UTRAP(0x007) 869 ktextfault: 870 VTRAP(T_INST_EXCEPT, textfault) ! 008 = instr. access except 871 VTRAP(T_TEXTFAULT, textfault) ! 009 = instr access MMU miss -- no MMU 872 VTRAP(T_INST_ERROR, textfault) ! 00a = instr. access err 873 UTRAP(0x00b); UTRAP(0x00c); UTRAP(0x00d); UTRAP(0x00e); UTRAP(0x00f) 874 TRAP(T_ILLINST) ! 010 = illegal instruction 875 TRAP(T_PRIVINST) ! 011 = privileged instruction 876 UTRAP(0x012) ! 012 = unimplemented LDD 877 UTRAP(0x013) ! 013 = unimplemented STD 878 UTRAP(0x014); UTRAP(0x015); UTRAP(0x016); UTRAP(0x017); UTRAP(0x018) 879 UTRAP(0x019); UTRAP(0x01a); UTRAP(0x01b); UTRAP(0x01c); UTRAP(0x01d) 880 UTRAP(0x01e); UTRAP(0x01f) 881 TRAP(T_FPDISABLED) ! 020 = fp instr, but EF bit off in psr 882 TRAP(T_FP_IEEE_754) ! 021 = ieee 754 exception 883 TRAP(T_FP_OTHER) ! 022 = other fp exception 884 TRAP(T_TAGOF) ! 023 = tag overflow 885 CLEANWIN1 ! 024-027 = clean window trap 886 TRAP(T_DIV0) ! 028 = divide by zero 887 UTRAP(0x029) ! 029 = internal processor error 888 UTRAP(0x02a); UTRAP(0x02b); UTRAP(0x02c); UTRAP(0x02d); UTRAP(0x02e); UTRAP(0x02f) 889 kdatafault: 890 VTRAP(T_DATAFAULT, winfault) ! 030 = data fetch fault 891 UTRAP(0x031) ! 031 = data MMU miss -- no MMU 892 VTRAP(T_DATA_ERROR, winfault) ! 032 = data fetch fault 893 VTRAP(T_DATA_PROT, winfault) ! 033 = data fetch fault 894 VTRAP(T_ALIGN, checkalign) ! 034 = address alignment error -- we could fix it inline... 895 TRAP(T_LDDF_ALIGN) ! 035 = LDDF address alignment error -- we could fix it inline... 896 TRAP(T_STDF_ALIGN) ! 036 = STDF address alignment error -- we could fix it inline... 897 TRAP(T_PRIVACT) ! 037 = privileged action 898 UTRAP(0x038); UTRAP(0x039); UTRAP(0x03a); UTRAP(0x03b); UTRAP(0x03c); 899 UTRAP(0x03d); UTRAP(0x03e); UTRAP(0x03f); 900 VTRAP(T_ASYNC_ERROR, winfault) ! 040 = data fetch fault 901 SOFTINT4U(1, IE_L1) ! 041 = level 1 interrupt 902 HARDINT4U(2) ! 042 = level 2 interrupt 903 HARDINT4U(3) ! 043 = level 3 interrupt 904 SOFTINT4U(4, IE_L4) ! 044 = level 4 interrupt 905 HARDINT4U(5) ! 045 = level 5 interrupt 906 SOFTINT4U(6, IE_L6) ! 046 = level 6 interrupt 907 HARDINT4U(7) ! 047 = level 7 interrupt 908 HARDINT4U(8) ! 048 = level 8 interrupt 909 HARDINT4U(9) ! 049 = level 9 interrupt 910 HARDINT4U(10) ! 04a = level 10 interrupt 911 HARDINT4U(11) ! 04b = level 11 interrupt 912 ZS_INTERRUPT4U ! 04c = level 12 (zs) interrupt 913 HARDINT4U(13) ! 04d = level 13 interrupt 914 HARDINT4U(14) ! 04e = level 14 interrupt 915 HARDINT4U(15) ! 04f = nonmaskable interrupt 916 UTRAP(0x050); UTRAP(0x051); UTRAP(0x052); UTRAP(0x053); UTRAP(0x054); UTRAP(0x055) 917 UTRAP(0x056); UTRAP(0x057); UTRAP(0x058); UTRAP(0x059); UTRAP(0x05a); UTRAP(0x05b) 918 UTRAP(0x05c); UTRAP(0x05d); UTRAP(0x05e); UTRAP(0x05f) 919 VTRAP(0x060, interrupt_vector); ! 060 = interrupt vector 920 TRAP(T_PA_WATCHPT) ! 061 = physical address data watchpoint 921 TRAP(T_VA_WATCHPT) ! 062 = virtual address data watchpoint 922 TRAP(T_ECCERR) ! 063 = corrected ECC error 923 kfast_IMMU_miss: ! 064 = fast instr access MMU miss 924 ldxa [%g0] ASI_IMMU_8KPTR, %g2 ! Load IMMU 8K TSB pointer 925 #ifdef NO_TSB 926 ba,a %icc, instr_miss 927 #endif 928 ldxa [%g0] ASI_IMMU, %g1 ! Load IMMU tag target register 929 ldda [%g2] ASI_NUCLEUS_QUAD_LDD, %g4 ! Load TSB tag:data into %g4:%g5 930 brgez,pn %g5, instr_miss ! Entry invalid? Punt 931 cmp %g1, %g4 ! Compare TLB tags 932 bne,pn %xcc, instr_miss ! Got right tag? 933 nop 934 CLRTT 935 stxa %g5, [%g0] ASI_IMMU_DATA_IN ! Enter new mapping 936 retry ! Try new mapping 937 1: 938 sir 939 TA32 940 kfast_DMMU_miss: ! 068 = fast data access MMU miss 941 ldxa [%g0] ASI_DMMU_8KPTR, %g2! Load DMMU 8K TSB pointer 942 #ifdef NO_TSB 943 ba,a %icc, data_miss 944 #endif 945 ldxa [%g0] ASI_DMMU, %g1 ! Load DMMU tag target register 946 ldda [%g2] ASI_NUCLEUS_QUAD_LDD, %g4 ! Load TSB tag and data into %g4 and %g5 947 brgez,pn %g5, data_miss ! Entry invalid? Punt 948 cmp %g1, %g4 ! Compare TLB tags 949 bnz,pn %xcc, data_miss ! Got right tag? 950 nop 951 CLRTT 952 #ifdef TRAPSTATS 953 sethi %hi(_C_LABEL(kdhit)), %g1 954 lduw [%g1+%lo(_C_LABEL(kdhit))], %g2 955 inc %g2 956 stw %g2, [%g1+%lo(_C_LABEL(kdhit))] 957 #endif 958 stxa %g5, [%g0] ASI_DMMU_DATA_IN ! Enter new mapping 959 retry ! Try new mapping 960 1: 961 sir 962 TA32 963 kfast_DMMU_protection: ! 06c = fast data access MMU protection 964 #ifdef TRAPSTATS 965 sethi %hi(_C_LABEL(kdprot)), %g1 966 lduw [%g1+%lo(_C_LABEL(kdprot))], %g2 967 inc %g2 968 stw %g2, [%g1+%lo(_C_LABEL(kdprot))] 969 #endif 970 #ifdef HWREF 971 ba,a,pt %xcc, dmmu_write_fault 972 #else 973 ba,a,pt %xcc, winfault 974 #endif 975 nop 976 TA32 977 TRAP(0x070) ! 0x070 fast_ECC_error 978 ! Implementation dependent traps 979 UTRAP(0x071); UTRAP(0x072); UTRAP(0x073); UTRAP(0x074); UTRAP(0x075); UTRAP(0x076) 980 UTRAP(0x077); UTRAP(0x078); UTRAP(0x079); UTRAP(0x07a); UTRAP(0x07b); UTRAP(0x07c) 981 UTRAP(0x07d); UTRAP(0x07e); UTRAP(0x07f) 982 TABLE(uspill): 983 SPILL64(1,ASI_AIUS) ! 0x080 spill_0_normal -- used to save user windows 984 SPILL32(2,ASI_AIUS) ! 0x084 spill_1_normal 985 SPILLBOTH(1b,2b,ASI_AIUS) ! 0x088 spill_2_normal 986 UTRAP(0x08c); TA32 ! 0x08c spill_3_normal 987 TABLE(kspill): 988 SPILL64(1,ASI_N) ! 0x090 spill_4_normal -- used to save supervisor windows 989 SPILL32(2,ASI_N) ! 0x094 spill_5_normal 990 SPILLBOTH(1b,2b,ASI_N) ! 0x098 spill_6_normal 991 UTRAP(0x09c); TA32 ! 0x09c spill_7_normal 992 TABLE(uspillk): 993 SPILL64(1,ASI_AIUS) ! 0x0a0 spill_0_other -- used to save user windows in nucleus mode 994 SPILL32(2,ASI_AIUS) ! 0x0a4 spill_1_other 995 SPILLBOTH(1b,2b,ASI_AIUS) ! 0x0a8 spill_2_other 996 UTRAP(0x0ac); TA32 ! 0x0ac spill_3_other 997 UTRAP(0x0b0); TA32 ! 0x0b0 spill_4_other 998 UTRAP(0x0b4); TA32 ! 0x0b4 spill_5_other 999 UTRAP(0x0b8); TA32 ! 0x0b8 spill_6_other 1000 UTRAP(0x0bc); TA32 ! 0x0bc spill_7_other 1001 TABLE(ufill): 1002 FILL64(nufill8,ASI_AIUS) ! 0x0c0 fill_0_normal -- used to fill windows when running nucleus mode from user 1003 FILL32(nufill4,ASI_AIUS) ! 0x0c4 fill_1_normal 1004 FILLBOTH(nufill8,nufill4,ASI_AIUS) ! 0x0c8 fill_2_normal 1005 UTRAP(0x0cc); TA32 ! 0x0cc fill_3_normal 1006 TABLE(sfill): 1007 FILL64(sfill8,ASI_N) ! 0x0d0 fill_4_normal -- used to fill windows when running nucleus mode from supervisor 1008 FILL32(sfill4,ASI_N) ! 0x0d4 fill_5_normal 1009 FILLBOTH(sfill8,sfill4,ASI_N) ! 0x0d8 fill_6_normal 1010 UTRAP(0x0dc); TA32 ! 0x0dc fill_7_normal 1011 TABLE(kfill): 1012 FILL64(nkfill8,ASI_AIUS) ! 0x0e0 fill_0_other -- used to fill user windows when running nucleus mode -- will we ever use this? 1013 FILL32(nkfill4,ASI_AIUS) ! 0x0e4 fill_1_other 1014 FILLBOTH(nkfill8,nkfill4,ASI_AIUS)! 0x0e8 fill_2_other 1015 UTRAP(0x0ec); TA32 ! 0x0ec fill_3_other 1016 UTRAP(0x0f0); TA32 ! 0x0f0 fill_4_other 1017 UTRAP(0x0f4); TA32 ! 0x0f4 fill_5_other 1018 UTRAP(0x0f8); TA32 ! 0x0f8 fill_6_other 1019 UTRAP(0x0fc); TA32 ! 0x0fc fill_7_other 1020 TABLE(syscall): 1021 SYSCALL ! 0x100 = sun syscall 1022 BPT ! 0x101 = pseudo breakpoint instruction 1023 STRAP(0x102); STRAP(0x103); STRAP(0x104); STRAP(0x105); STRAP(0x106); STRAP(0x107) 1024 SYSCALL ! 0x108 = svr4 syscall 1025 SYSCALL ! 0x109 = bsd syscall 1026 BPT_KGDB_EXEC ! 0x10a = enter kernel gdb on kernel startup 1027 STRAP(0x10b); STRAP(0x10c); STRAP(0x10d); STRAP(0x10e); STRAP(0x10f); 1028 STRAP(0x110); STRAP(0x111); STRAP(0x112); STRAP(0x113); STRAP(0x114); STRAP(0x115); STRAP(0x116); STRAP(0x117) 1029 STRAP(0x118); STRAP(0x119); STRAP(0x11a); STRAP(0x11b); STRAP(0x11c); STRAP(0x11d); STRAP(0x11e); STRAP(0x11f) 1030 STRAP(0x120); STRAP(0x121); STRAP(0x122); STRAP(0x123); STRAP(0x124); STRAP(0x125); STRAP(0x126); STRAP(0x127) 1031 STRAP(0x128); STRAP(0x129); STRAP(0x12a); STRAP(0x12b); STRAP(0x12c); STRAP(0x12d); STRAP(0x12e); STRAP(0x12f) 1032 STRAP(0x130); STRAP(0x131); STRAP(0x132); STRAP(0x133); STRAP(0x134); STRAP(0x135); STRAP(0x136); STRAP(0x137) 1033 STRAP(0x138); STRAP(0x139); STRAP(0x13a); STRAP(0x13b); STRAP(0x13c); STRAP(0x13d); STRAP(0x13e); STRAP(0x13f) 1034 STRAP(0x140); STRAP(0x141); STRAP(0x142); STRAP(0x143); STRAP(0x144); STRAP(0x145); STRAP(0x146); STRAP(0x147) 1035 STRAP(0x148); STRAP(0x149); STRAP(0x14a); STRAP(0x14b); STRAP(0x14c); STRAP(0x14d); STRAP(0x14e); STRAP(0x14f) 1036 STRAP(0x150); STRAP(0x151); STRAP(0x152); STRAP(0x153); STRAP(0x154); STRAP(0x155); STRAP(0x156); STRAP(0x157) 1037 STRAP(0x158); STRAP(0x159); STRAP(0x15a); STRAP(0x15b); STRAP(0x15c); STRAP(0x15d); STRAP(0x15e); STRAP(0x15f) 1038 STRAP(0x160); STRAP(0x161); STRAP(0x162); STRAP(0x163); STRAP(0x164); STRAP(0x165); STRAP(0x166); STRAP(0x167) 1039 STRAP(0x168); STRAP(0x169); STRAP(0x16a); STRAP(0x16b); STRAP(0x16c); STRAP(0x16d); STRAP(0x16e); STRAP(0x16f) 1040 STRAP(0x170); STRAP(0x171); STRAP(0x172); STRAP(0x173); STRAP(0x174); STRAP(0x175); STRAP(0x176); STRAP(0x177) 1041 STRAP(0x178); STRAP(0x179); STRAP(0x17a); STRAP(0x17b); STRAP(0x17c); STRAP(0x17d); STRAP(0x17e); STRAP(0x17f) 1042 ! Traps beyond 0x17f are reserved 1043 UTRAP(0x180); UTRAP(0x181); UTRAP(0x182); UTRAP(0x183); UTRAP(0x184); UTRAP(0x185); UTRAP(0x186); UTRAP(0x187) 1044 UTRAP(0x188); UTRAP(0x189); UTRAP(0x18a); UTRAP(0x18b); UTRAP(0x18c); UTRAP(0x18d); UTRAP(0x18e); UTRAP(0x18f) 1045 UTRAP(0x190); UTRAP(0x191); UTRAP(0x192); UTRAP(0x193); UTRAP(0x194); UTRAP(0x195); UTRAP(0x196); UTRAP(0x197) 1046 UTRAP(0x198); UTRAP(0x199); UTRAP(0x19a); UTRAP(0x19b); UTRAP(0x19c); UTRAP(0x19d); UTRAP(0x19e); UTRAP(0x19f) 1047 UTRAP(0x1a0); UTRAP(0x1a1); UTRAP(0x1a2); UTRAP(0x1a3); UTRAP(0x1a4); UTRAP(0x1a5); UTRAP(0x1a6); UTRAP(0x1a7) 1048 UTRAP(0x1a8); UTRAP(0x1a9); UTRAP(0x1aa); UTRAP(0x1ab); UTRAP(0x1ac); UTRAP(0x1ad); UTRAP(0x1ae); UTRAP(0x1af) 1049 UTRAP(0x1b0); UTRAP(0x1b1); UTRAP(0x1b2); UTRAP(0x1b3); UTRAP(0x1b4); UTRAP(0x1b5); UTRAP(0x1b6); UTRAP(0x1b7) 1050 UTRAP(0x1b8); UTRAP(0x1b9); UTRAP(0x1ba); UTRAP(0x1bb); UTRAP(0x1bc); UTRAP(0x1bd); UTRAP(0x1be); UTRAP(0x1bf) 1051 UTRAP(0x1c0); UTRAP(0x1c1); UTRAP(0x1c2); UTRAP(0x1c3); UTRAP(0x1c4); UTRAP(0x1c5); UTRAP(0x1c6); UTRAP(0x1c7) 1052 UTRAP(0x1c8); UTRAP(0x1c9); UTRAP(0x1ca); UTRAP(0x1cb); UTRAP(0x1cc); UTRAP(0x1cd); UTRAP(0x1ce); UTRAP(0x1cf) 1053 UTRAP(0x1d0); UTRAP(0x1d1); UTRAP(0x1d2); UTRAP(0x1d3); UTRAP(0x1d4); UTRAP(0x1d5); UTRAP(0x1d6); UTRAP(0x1d7) 1054 UTRAP(0x1d8); UTRAP(0x1d9); UTRAP(0x1da); UTRAP(0x1db); UTRAP(0x1dc); UTRAP(0x1dd); UTRAP(0x1de); UTRAP(0x1df) 1055 UTRAP(0x1e0); UTRAP(0x1e1); UTRAP(0x1e2); UTRAP(0x1e3); UTRAP(0x1e4); UTRAP(0x1e5); UTRAP(0x1e6); UTRAP(0x1e7) 1056 UTRAP(0x1e8); UTRAP(0x1e9); UTRAP(0x1ea); UTRAP(0x1eb); UTRAP(0x1ec); UTRAP(0x1ed); UTRAP(0x1ee); UTRAP(0x1ef) 1057 UTRAP(0x1f0); UTRAP(0x1f1); UTRAP(0x1f2); UTRAP(0x1f3); UTRAP(0x1f4); UTRAP(0x1f5); UTRAP(0x1f6); UTRAP(0x1f7) 1058 UTRAP(0x1f8); UTRAP(0x1f9); UTRAP(0x1fa); UTRAP(0x1fb); UTRAP(0x1fc); UTRAP(0x1fd); UTRAP(0x1fe); UTRAP(0x1ff) 1059 1060 #ifdef SUN4V 1061 1062 /* Macros for sun4v traps */ 1063 1064 .macro sun4v_trap_entry count 1065 .rept \count 1066 ba slowtrap 1067 nop 1068 .align 32 1069 .endr 1070 .endm 1071 1072 .macro sun4v_trap_entry_fail count 1073 .rept \count 1074 sir 1075 .align 32 1076 .endr 1077 .endm 1078 1079 .macro sun4v_trap_entry_spill_fill_fail count 1080 .rept \count 1081 sir 1082 .align 128 1083 .endr 1084 .endm 1085 1086 /* The actual trap base for sun4v */ 1087 .align 0x8000 1088 .globl _C_LABEL(trapbase_sun4v) 1089 _C_LABEL(trapbase_sun4v): 1090 ! 1091 ! trap level 0 1092 ! 1093 sun4v_trap_entry 8 ! 0x000-0x007 1094 VTRAP(T_INST_EXCEPT, sun4v_tl0_itsb_miss) ! 0x008 - inst except 1095 VTRAP(T_TEXTFAULT, sun4v_tl0_itsb_miss) ! 0x009 - inst MMU miss 1096 sun4v_trap_entry 26 ! 0x00a-0x023 1097 CLEANWIN0 ! 0x24-0x27 = clean window 1098 sun4v_trap_entry 9 ! 0x028-0x030 1099 VTRAP(T_DATA_MMU_MISS, sun4v_dtsb_miss) ! 0x031 = data MMU miss 1100 sun4v_trap_entry 2 ! 0x032-0x033 1101 TRAP(T_ALIGN) ! 0x034 = address alignment error 1102 sun4v_trap_entry 12 ! 0x035-0x040 1103 HARDINT4V(1) ! 0x041 = level 1 interrupt 1104 HARDINT4V(2) ! 0x042 = level 2 interrupt 1105 HARDINT4V(3) ! 0x043 = level 3 interrupt 1106 HARDINT4V(4) ! 0x044 = level 4 interrupt 1107 HARDINT4V(5) ! 0x045 = level 5 interrupt 1108 HARDINT4V(6) ! 0x046 = level 6 interrupt 1109 HARDINT4V(7) ! 0x047 = level 7 interrupt 1110 HARDINT4V(8) ! 0x048 = level 8 interrupt 1111 HARDINT4V(9) ! 0x049 = level 9 interrupt 1112 HARDINT4V(10) ! 0x04a = level 10 interrupt 1113 HARDINT4V(11) ! 0x04b = level 11 interrupt 1114 HARDINT4V(12) ! 0x04c = level 12 interrupt 1115 HARDINT4V(13) ! 0x04d = level 13 interrupt 1116 HARDINT4V(14) ! 0x04e = level 14 interrupt 1117 HARDINT4V(15) ! 0x04f = level 15 interrupt 1118 sun4v_trap_entry 28 ! 0x050-0x06b 1119 VTRAP(T_FDMMU_PROT, sun4v_tl0_dtsb_prot) ! 0x06c 1120 sun4v_trap_entry 15 ! 0x06d-0x07b 1121 VTRAP(T_CPU_MONDO, sun4v_cpu_mondo) ! 0x07c = cpu mondo 1122 VTRAP(T_DEV_MONDO, sun4v_dev_mondo) ! 0x07d = dev mondo 1123 sun4v_trap_entry 2 ! 0x07e-0x07f 1124 SPILL64(uspill8_sun4vt0,ASI_AIUS) ! 0x080 spill_0_normal -- used to save user windows in user mode 1125 SPILL32(uspill4_sun4vt0,ASI_AIUS) ! 0x084 spill_1_normal 1126 SPILLBOTH(uspill8_sun4vt0,uspill4_sun4vt0,ASI_AIUS) ! 0x088 spill_2_normal 1127 sun4v_trap_entry_spill_fill_fail 1 ! 0x08c spill_3_normal 1128 SPILL64(kspill8_sun4vt0,ASI_N) ! 0x090 spill_4_normal -- used to save supervisor windows 1129 SPILL32(kspill4_sun4vt0,ASI_N) ! 0x094 spill_5_normal 1130 SPILLBOTH(kspill8_sun4vt0,kspill4_sun4vt0,ASI_N) ! 0x098 spill_6_normal 1131 sun4v_trap_entry_spill_fill_fail 1 ! 0x09c spill_7_normal 1132 SPILL64(uspillk8_sun4vt0,ASI_AIUS) ! 0x0a0 spill_0_other -- used to save user windows in supervisor mode 1133 SPILL32(uspillk4_sun4vt0,ASI_AIUS) ! 0x0a4 spill_1_other 1134 SPILLBOTH(uspillk8_sun4vt0,uspillk4_sun4vt0,ASI_AIUS) ! 0x0a8 spill_2_other 1135 sun4v_trap_entry_spill_fill_fail 1 ! 0x0ac spill_3_other 1136 sun4v_trap_entry_spill_fill_fail 1 ! 0x0b0 spill_4_other 1137 sun4v_trap_entry_spill_fill_fail 1 ! 0x0b4 spill_5_other 1138 sun4v_trap_entry_spill_fill_fail 1 ! 0x0b8 spill_6_other 1139 sun4v_trap_entry_spill_fill_fail 1 ! 0x0bc spill_7_other 1140 FILL64(ufill8_sun4vt0,ASI_AIUS) ! 0x0c0 fill_0_normal -- used to fill windows when running user mode 1141 FILL32(ufill4_sun4vt0,ASI_AIUS) ! 0x0c4 fill_1_normal 1142 FILLBOTH(ufill8_sun4vt0,ufill4_sun4vt0,ASI_AIUS) ! 0x0c8 fill_2_normal 1143 sun4v_trap_entry_spill_fill_fail 1 ! 0x0cc fill_3_normal 1144 FILL64(kfill8_sun4vt0,ASI_N) ! 0x0d0 fill_4_normal -- used to fill windows when running supervisor mode 1145 FILL32(kfill4_sun4vt0,ASI_N) ! 0x0d4 fill_5_normal 1146 FILLBOTH(kfill8_sun4vt0,kfill4_sun4vt0,ASI_N) ! 0x0d8 fill_6_normal 1147 sun4v_trap_entry_spill_fill_fail 1 ! 0x0dc fill_7_normal 1148 FILL64(ufillk8_sun4vt0,ASI_AIUS) ! 0x0e0 fill_0_other 1149 FILL32(ufillk4_sun4vt0,ASI_AIUS) ! 0x0e4 fill_1_other 1150 FILLBOTH(ufillk8_sun4vt0,ufillk4_sun4vt0,ASI_AIUS) ! 0x0e8 fill_2_other 1151 sun4v_trap_entry_spill_fill_fail 1 ! 0x0ec fill_3_other 1152 sun4v_trap_entry_spill_fill_fail 1 ! 0x0f0 fill_4_other 1153 sun4v_trap_entry_spill_fill_fail 1 ! 0x0f4 fill_5_other 1154 sun4v_trap_entry_spill_fill_fail 1 ! 0x0f8 fill_6_other 1155 sun4v_trap_entry_spill_fill_fail 1 ! 0x0fc fill_7_other 1156 SYSCALL ! 0x100 = syscall 1157 BPT ! 0x101 = pseudo breakpoint instruction 1158 sun4v_trap_entry 254 ! 0x102-0x1ff 1159 ! 1160 ! trap level 1 1161 ! 1162 sun4v_trap_entry 36 ! 0x000-0x023 1163 CLEANWIN1 ! 0x24-0x27 = clean window 1164 sun4v_trap_entry 8 ! 0x028-0x02F 1165 VTRAP(T_DATAFAULT, sun4v_tl1_ptbl_miss) ! 0x030 = ??? 1166 VTRAP(T_DATA_MMU_MISS, sun4v_tl1_dtsb_miss) ! 0x031 = data MMU miss 1167 VTRAP(T_DATA_ERROR, sun4v_tl1_ptbl_miss) ! 0x032 = ??? 1168 VTRAP(T_DATA_PROT, sun4v_tl1_ptbl_miss) ! 0x033 = ??? 1169 sun4v_trap_entry 56 ! 0x034-0x06b 1170 VTRAP(T_FDMMU_PROT, sun4v_tl1_dtsb_prot) ! 0x06c 1171 sun4v_trap_entry 19 ! 0x06d-0x07f 1172 sun4v_tl1_uspill_normal ! 0x080 spill_0_normal -- save user windows 1173 sun4v_tl1_uspill_normal ! 0x084 spill_1_normal 1174 sun4v_tl1_uspill_normal ! 0x088 spill_2_normal 1175 sun4v_trap_entry_spill_fill_fail 1 ! 0x08c spill_3_normal 1176 SPILL64(kspill8_sun4vt1,ASI_N) ! 0x090 spill_4_normal -- save supervisor windows 1177 SPILL32(kspill4_sun4vt1,ASI_N) ! 0x094 spill_5_normal 1178 SPILLBOTH(kspill8_sun4vt1,kspill4_sun4vt1,ASI_N) ! 0x098 spill_6_normal 1179 sun4v_trap_entry_spill_fill_fail 1 ! 0x09c spill_7_normal 1180 sun4v_tl1_uspill_other ! 0x0a0 spill_0_other -- save user windows in nucleus mode 1181 sun4v_tl1_uspill_other ! 0x0a4 spill_1_other 1182 sun4v_tl1_uspill_other ! 0x0a8 spill_2_other 1183 sun4v_trap_entry_spill_fill_fail 1 ! 0x0ac spill_3_other 1184 sun4v_trap_entry_spill_fill_fail 1 ! 0x0b0 spill_4_other 1185 sun4v_trap_entry_spill_fill_fail 1 ! 0x0b4 spill_5_other 1186 sun4v_trap_entry_spill_fill_fail 1 ! 0x0b8 spill_6_other 1187 sun4v_trap_entry_spill_fill_fail 1 ! 0x0bc spill_7_other 1188 FILL64(ufill8_sun4vt1,ASI_AIUS) ! 0x0c0 fill_0_normal -- fill windows when running nucleus mode from user 1189 FILL32(ufill4_sun4vt1,ASI_AIUS) ! 0x0c4 fill_1_normal 1190 FILLBOTH(ufill8_sun4vt1,ufill4_sun4vt1,ASI_AIUS) ! 0x0c8 fill_2_normal 1191 sun4v_trap_entry_spill_fill_fail 1 ! 0x0cc fill_3_normal 1192 FILL64(kfill8_sun4vt1,ASI_N) ! 0x0d0 fill_4_normal -- fill windows when running nucleus mode from supervisor 1193 FILL32(kfill4_sun4vt1,ASI_N) ! 0x0d4 fill_5_normal 1194 FILLBOTH(kfill8_sun4vt1,kfill4_sun4vt1,ASI_N) ! 0x0d8 fill_6_normal 1195 sun4v_trap_entry_spill_fill_fail 1 ! 0x0dc fill_7_normal 1196 FILL64(ufillk8_sun4vt1,ASI_AIUS) ! 0x0e0 fill_0_other -- fill user windows when running nucleus mode -- will we ever use this? 1197 FILL32(ufillk4_sun4vt1,ASI_AIUS) ! 0x0e4 fill_1_other 1198 FILLBOTH(ufillk8_sun4vt1,ufillk4_sun4vt1,ASI_AIUS) ! 0x0e8 fill_2_other 1199 sun4v_trap_entry_spill_fill_fail 1 ! 0x0ec fill_3_other 1200 sun4v_trap_entry_spill_fill_fail 1 ! 0x0f0 fill_4_other 1201 sun4v_trap_entry_spill_fill_fail 1 ! 0x0f4 fill_5_other 1202 sun4v_trap_entry_spill_fill_fail 1 ! 0x0f8 fill_6_other 1203 sun4v_trap_entry_spill_fill_fail 1 ! 0x0fc fill_7_other 1204 sun4v_trap_entry_fail 256 ! 0x100-0x1ff 1205 1206 #endif 1207 1208 #if 0 1209 /* 1210 * If the cleanwin trap handler detects an overflow we come here. 1211 * We need to fix up the window registers, switch to the interrupt 1212 * stack, and then trap to the debugger. 1213 */ 1214 cleanwin_overflow: 1215 !! We've already incremented %cleanwin 1216 !! So restore %cwp 1217 rdpr %cwp, %l0 1218 dec %l0 1219 wrpr %l0, %g0, %cwp 1220 set EINTSTACK-STKB-CC64FSZ, %l0 1221 save %l0, 0, %sp 1222 1223 ta 1 ! Enter debugger 1224 sethi %hi(1f), %o0 1225 call _C_LABEL(panic) 1226 or %o0, %lo(1f), %o0 1227 restore 1228 retry 1229 .data 1230 1: 1231 .asciz "Kernel stack overflow!" 1232 _ALIGN 1233 .text 1234 #endif 1235 1236 #ifdef NOTDEF_DEBUG 1237 /* 1238 * A hardware red zone is impossible. We simulate one in software by 1239 * keeping a `red zone' pointer; if %sp becomes less than this, we panic. 1240 * This is expensive and is only enabled when debugging. 1241 */ 1242 #define REDSIZE (PCB_SIZE) /* Mark used portion of pcb structure out of bounds */ 1243 #define REDSTACK 2048 /* size of `panic: stack overflow' region */ 1244 .data 1245 _ALIGN 1246 redzone: 1247 .xword _C_LABEL(XXX) + REDSIZE 1248 redstack: 1249 .space REDSTACK 1250 eredstack: 1251 Lpanic_red: 1252 .asciz "kernel stack overflow" 1253 _ALIGN 1254 .text 1255 1256 /* set stack pointer redzone to base+minstack; alters base */ 1257 #define SET_SP_REDZONE(base, tmp) \ 1258 add base, REDSIZE, base; \ 1259 sethi %hi(_C_LABEL(redzone)), tmp; \ 1260 stx base, [tmp + %lo(_C_LABEL(redzone))] 1261 1262 /* variant with a constant */ 1263 #define SET_SP_REDZONE_CONST(const, tmp1, tmp2) \ 1264 set (const) + REDSIZE, tmp1; \ 1265 sethi %hi(_C_LABEL(redzone)), tmp2; \ 1266 stx tmp1, [tmp2 + %lo(_C_LABEL(redzone))] 1267 1268 /* check stack pointer against redzone (uses two temps) */ 1269 #define CHECK_SP_REDZONE(t1, t2) \ 1270 sethi KERNBASE, t1; \ 1271 cmp %sp, t1; \ 1272 blu,pt %xcc, 7f; \ 1273 sethi %hi(_C_LABEL(redzone)), t1; \ 1274 ldx [t1 + %lo(_C_LABEL(redzone))], t2; \ 1275 cmp %sp, t2; /* if sp >= t2, not in red zone */ \ 1276 blu panic_red; nop; /* and can continue normally */ \ 1277 7: 1278 1279 panic_red: 1280 /* move to panic stack */ 1281 stx %g0, [t1 + %lo(_C_LABEL(redzone))]; 1282 set eredstack - BIAS, %sp; 1283 /* prevent panic() from lowering ipl */ 1284 sethi %hi(_C_LABEL(panicstr)), t2; 1285 set Lpanic_red, t2; 1286 st t2, [t1 + %lo(_C_LABEL(panicstr))]; 1287 wrpr g0, 15, %pil /* t1 = splhigh() */ 1288 save %sp, -CCF64SZ, %sp; /* preserve current window */ 1289 sethi %hi(Lpanic_red), %o0; 1290 call _C_LABEL(panic); 1291 or %o0, %lo(Lpanic_red), %o0; 1292 1293 1294 #else 1295 1296 #define SET_SP_REDZONE(base, tmp) 1297 #define SET_SP_REDZONE_CONST(const, t1, t2) 1298 #define CHECK_SP_REDZONE(t1, t2) 1299 #endif 1300 1301 #define TRACESIZ 0x01000 1302 .globl _C_LABEL(trap_trace) 1303 .globl _C_LABEL(trap_trace_ptr) 1304 .globl _C_LABEL(trap_trace_end) 1305 .globl _C_LABEL(trap_trace_dis) 1306 .data 1307 _C_LABEL(trap_trace_dis): 1308 .word 1, 1 ! Starts disabled. DDB turns it on. 1309 _C_LABEL(trap_trace_ptr): 1310 .word 0, 0, 0, 0 1311 _C_LABEL(trap_trace): 1312 .space TRACESIZ 1313 _C_LABEL(trap_trace_end): 1314 .space 0x20 ! safety margin 1315 1316 1317 /* 1318 * v9 machines do not have a trap window. 1319 * 1320 * When we take a trap the trap state is pushed on to the stack of trap 1321 * registers, interrupts are disabled, then we switch to an alternate set 1322 * of global registers. 1323 * 1324 * The trap handling code needs to allocate a trap frame on the kernel, or 1325 * for interrupts, the interrupt stack, save the out registers to the trap 1326 * frame, then switch to the normal globals and save them to the trap frame 1327 * too. 1328 * 1329 * XXX it would be good to save the interrupt stack frame to the kernel 1330 * stack so we wouldn't have to copy it later if we needed to handle a AST. 1331 * 1332 * Since kernel stacks are all on one page and the interrupt stack is entirely 1333 * within the locked TLB, we can use physical addressing to save out our 1334 * trap frame so we don't trap during the TRAP_SETUP() operation. There 1335 * is unfortunately no supportable method for issuing a non-trapping save. 1336 * 1337 * However, if we use physical addresses to save our trapframe, we will need 1338 * to clear out the data cache before continuing much further. 1339 * 1340 * In short, what we need to do is: 1341 * 1342 * all preliminary processing is done using the alternate globals 1343 * 1344 * When we allocate our trap windows we must give up our globals because 1345 * their state may have changed during the save operation 1346 * 1347 * we need to save our normal globals as soon as we have a stack 1348 * 1349 * Finally, we may now call C code. 1350 * 1351 * This macro will destroy %g5-%g7. %g0-%g4 remain unchanged. 1352 * 1353 * In order to properly handle nested traps without lossage, alternate 1354 * global %g6 is used as a kernel stack pointer. It is set to the last 1355 * allocated stack pointer (trapframe) and the old value is stored in 1356 * tf_kstack. It is restored when returning from a trap. It is cleared 1357 * on entering user mode. 1358 */ 1359 1360 /* 1361 * Other misc. design criteria: 1362 * 1363 * When taking an address fault, fault info is in the sfsr, sfar, 1364 * TLB_TAG_ACCESS registers. If we take another address fault 1365 * while trying to handle the first fault then that information, 1366 * the only information that tells us what address we trapped on, 1367 * can potentially be lost. This trap can be caused when allocating 1368 * a register window with which to handle the trap because the save 1369 * may try to store or restore a register window that corresponds 1370 * to part of the stack that is not mapped. Preventing this trap, 1371 * while possible, is much too complicated to do in a trap handler, 1372 * and then we will need to do just as much work to restore the processor 1373 * window state. 1374 * 1375 * Possible solutions to the problem: 1376 * 1377 * Since we have separate AG, MG, and IG, we could have all traps 1378 * above level-1 preserve AG and use other registers. This causes 1379 * a problem for the return from trap code which is coded to use 1380 * alternate globals only. 1381 * 1382 * We could store the trapframe and trap address info to the stack 1383 * using physical addresses. Then we need to read it back using 1384 * physical addressing, or flush the D$. 1385 * 1386 * We could identify certain registers to hold address fault info. 1387 * this means that these registers need to be preserved across all 1388 * fault handling. But since we only have 7 useable globals, that 1389 * really puts a cramp in our style. 1390 * 1391 * Finally, there is the issue of returning from kernel mode to user 1392 * mode. If we need to issue a restore of a user window in kernel 1393 * mode, we need the window control registers in a user mode setup. 1394 * If the trap handlers notice the register windows are in user mode, 1395 * they will allocate a trapframe at the bottom of the kernel stack, 1396 * overwriting the frame we were trying to return to. This means that 1397 * we must complete the restoration of all registers *before* switching 1398 * to a user-mode window configuration. 1399 * 1400 * Essentially we need to be able to write re-entrant code w/no stack. 1401 */ 1402 .data 1403 trap_setup_msg: 1404 .asciz "TRAP_SETUP: tt=%x osp=%x nsp=%x tl=%x tpc=%x\n" 1405 _ALIGN 1406 intr_setup_msg: 1407 .asciz "INTR_SETUP: tt=%x osp=%x nsp=%x tl=%x tpc=%x\n" 1408 _ALIGN 1409 .text 1410 1411 #ifdef DEBUG 1412 /* Only save a snapshot of locals and ins in DEBUG kernels */ 1413 #define SAVE_LOCALS_INS \ 1414 /* Save local registers to trap frame */ \ 1415 stx %l0, [%g6 + CC64FSZ + STKB + TF_L + (0*8)]; \ 1416 stx %l1, [%g6 + CC64FSZ + STKB + TF_L + (1*8)]; \ 1417 stx %l2, [%g6 + CC64FSZ + STKB + TF_L + (2*8)]; \ 1418 stx %l3, [%g6 + CC64FSZ + STKB + TF_L + (3*8)]; \ 1419 stx %l4, [%g6 + CC64FSZ + STKB + TF_L + (4*8)]; \ 1420 stx %l5, [%g6 + CC64FSZ + STKB + TF_L + (5*8)]; \ 1421 stx %l6, [%g6 + CC64FSZ + STKB + TF_L + (6*8)]; \ 1422 stx %l7, [%g6 + CC64FSZ + STKB + TF_L + (7*8)]; \ 1423 \ 1424 /* Save in registers to trap frame */ \ 1425 stx %i0, [%g6 + CC64FSZ + STKB + TF_I + (0*8)]; \ 1426 stx %i1, [%g6 + CC64FSZ + STKB + TF_I + (1*8)]; \ 1427 stx %i2, [%g6 + CC64FSZ + STKB + TF_I + (2*8)]; \ 1428 stx %i3, [%g6 + CC64FSZ + STKB + TF_I + (3*8)]; \ 1429 stx %i4, [%g6 + CC64FSZ + STKB + TF_I + (4*8)]; \ 1430 stx %i5, [%g6 + CC64FSZ + STKB + TF_I + (5*8)]; \ 1431 stx %i6, [%g6 + CC64FSZ + STKB + TF_I + (6*8)]; \ 1432 stx %i7, [%g6 + CC64FSZ + STKB + TF_I + (7*8)]; \ 1433 \ 1434 stx %g1, [%g6 + CC64FSZ + STKB + TF_FAULT]; 1435 #else 1436 #define SAVE_LOCALS_INS 1437 #endif 1438 1439 #ifdef _LP64 1440 #define FIXUP_TRAP_STACK \ 1441 btst 1, %g6; /* Fixup 64-bit stack if necessary */ \ 1442 bnz,pt %icc, 1f; \ 1443 add %g6, %g5, %g6; /* Allocate a stack frame */ \ 1444 inc -BIAS, %g6; \ 1445 1: 1446 #else 1447 #define FIXUP_TRAP_STACK \ 1448 srl %g6, 0, %g6; /* truncate at 32-bits */ \ 1449 btst 1, %g6; /* Fixup 64-bit stack if necessary */ \ 1450 add %g6, %g5, %g6; /* Allocate a stack frame */ \ 1451 add %g6, BIAS, %g5; \ 1452 movne %icc, %g5, %g6; 1453 #endif 1454 1455 #ifdef _LP64 1456 #define TRAP_SETUP(stackspace) \ 1457 sethi %hi(CPCB), %g6; \ 1458 sethi %hi((stackspace)), %g5; \ 1459 LDPTR [%g6 + %lo(CPCB)], %g6; \ 1460 sethi %hi(USPACE), %g7; /* Always multiple of page size */ \ 1461 or %g5, %lo((stackspace)), %g5; \ 1462 add %g6, %g7, %g6; \ 1463 rdpr %wstate, %g7; /* Find if we're from user mode */ \ 1464 sra %g5, 0, %g5; /* Sign extend the damn thing */ \ 1465 \ 1466 sub %g7, WSTATE_KERN, %g7; /* Compare & leave in register */ \ 1467 movrz %g7, %sp, %g6; /* Select old (kernel) stack or base of kernel stack */ \ 1468 FIXUP_TRAP_STACK \ 1469 SAVE_LOCALS_INS \ 1470 save %g6, 0, %sp; /* If we fault we should come right back here */ \ 1471 stx %i0, [%sp + CC64FSZ + BIAS + TF_O + (0*8)]; /* Save out registers to trap frame */ \ 1472 stx %i1, [%sp + CC64FSZ + BIAS + TF_O + (1*8)]; \ 1473 stx %i2, [%sp + CC64FSZ + BIAS + TF_O + (2*8)]; \ 1474 stx %i3, [%sp + CC64FSZ + BIAS + TF_O + (3*8)]; \ 1475 stx %i4, [%sp + CC64FSZ + BIAS + TF_O + (4*8)]; \ 1476 stx %i5, [%sp + CC64FSZ + BIAS + TF_O + (5*8)]; \ 1477 \ 1478 stx %i6, [%sp + CC64FSZ + BIAS + TF_O + (6*8)]; \ 1479 brz,pt %g7, 1f; /* If we were in kernel mode start saving globals */ \ 1480 stx %i7, [%sp + CC64FSZ + BIAS + TF_O + (7*8)]; \ 1481 mov CTX_PRIMARY, %g7; \ 1482 /* came from user mode -- switch to kernel mode stack */ \ 1483 rdpr %canrestore, %g5; /* Fixup register window state registers */ \ 1484 wrpr %g0, 0, %canrestore; \ 1485 wrpr %g0, %g5, %otherwin; \ 1486 wrpr %g0, WSTATE_KERN, %wstate; /* Enable kernel mode window traps -- now we can trap again */ \ 1487 \ 1488 SET_MMU_CONTEXTID %g0, %g7,%g5; /* Switch MMU to kernel primary context */ \ 1489 sethi %hi(KERNBASE), %g5; \ 1490 flush %g5; /* Some convenient address that won't trap */ \ 1491 1: 1492 1493 /* 1494 * Interrupt setup is almost exactly like trap setup, but we need to 1495 * go to the interrupt stack if (a) we came from user mode or (b) we 1496 * came from kernel mode on the kernel stack. 1497 * 1498 * We don't guarantee any registers are preserved during this operation. 1499 * So we can be more efficient. 1500 */ 1501 #define INTR_SETUP(stackspace) \ 1502 rdpr %wstate, %g7; /* Find if we're from user mode */ \ 1503 \ 1504 sethi %hi(EINTSTACK-BIAS), %g6; \ 1505 sethi %hi(EINTSTACK-INTSTACK), %g4; \ 1506 \ 1507 or %g6, %lo(EINTSTACK-BIAS), %g6; /* Base of interrupt stack */ \ 1508 dec %g4; /* Make it into a mask */ \ 1509 \ 1510 sub %g6, %sp, %g1; /* Offset from interrupt stack */ \ 1511 sethi %hi((stackspace)), %g5; \ 1512 \ 1513 or %g5, %lo((stackspace)), %g5; \ 1514 \ 1515 andn %g1, %g4, %g4; /* Are we out of the interrupt stack range? */ \ 1516 xor %g7, WSTATE_KERN, %g3; /* Are we on the user stack ? */ \ 1517 \ 1518 sra %g5, 0, %g5; /* Sign extend the damn thing */ \ 1519 orcc %g3, %g4, %g0; /* Definitely not off the interrupt stack */ \ 1520 \ 1521 sethi %hi(CPUINFO_VA + CI_EINTSTACK), %g4; \ 1522 bz,a,pt %xcc, 1f; \ 1523 mov %sp, %g6; \ 1524 \ 1525 ldx [%g4 + %lo(CPUINFO_VA + CI_EINTSTACK)], %g4; \ 1526 movrnz %g4, %g4, %g6; /* Use saved intr stack if exists */ \ 1527 \ 1528 1: add %g6, %g5, %g5; /* Allocate a stack frame */ \ 1529 btst 1, %g6; \ 1530 bnz,pt %icc, 1f; \ 1531 \ 1532 mov %g5, %g6; \ 1533 \ 1534 add %g5, -BIAS, %g6; \ 1535 \ 1536 1: SAVE_LOCALS_INS \ 1537 save %g6, 0, %sp; /* If we fault we should come right back here */ \ 1538 stx %i0, [%sp + CC64FSZ + BIAS + TF_O + (0*8)]; /* Save out registers to trap frame */ \ 1539 stx %i1, [%sp + CC64FSZ + BIAS + TF_O + (1*8)]; \ 1540 stx %i2, [%sp + CC64FSZ + BIAS + TF_O + (2*8)]; \ 1541 stx %i3, [%sp + CC64FSZ + BIAS + TF_O + (3*8)]; \ 1542 stx %i4, [%sp + CC64FSZ + BIAS + TF_O + (4*8)]; \ 1543 \ 1544 stx %i5, [%sp + CC64FSZ + BIAS + TF_O + (5*8)]; \ 1545 stx %i6, [%sp + CC64FSZ + BIAS + TF_O + (6*8)]; \ 1546 stx %i6, [%sp + CC64FSZ + BIAS + TF_G + (0*8)]; /* Save fp in clockframe->cf_fp */ \ 1547 brz,pt %g3, 1f; /* If we were in kernel mode start saving globals */ \ 1548 stx %i7, [%sp + CC64FSZ + BIAS + TF_O + (7*8)]; \ 1549 /* came from user mode -- switch to kernel mode stack */ \ 1550 rdpr %otherwin, %g5; /* Has this already been done? */ \ 1551 \ 1552 brnz,pn %g5, 1f; /* Don't set this twice */ \ 1553 \ 1554 rdpr %canrestore, %g5; /* Fixup register window state registers */ \ 1555 \ 1556 wrpr %g0, 0, %canrestore; \ 1557 \ 1558 wrpr %g0, %g5, %otherwin; \ 1559 \ 1560 mov CTX_PRIMARY, %g7; \ 1561 \ 1562 wrpr %g0, WSTATE_KERN, %wstate; /* Enable kernel mode window traps -- now we can trap again */ \ 1563 \ 1564 SET_MMU_CONTEXTID %g0, %g7, %g5; /* Switch MMU to kernel primary context */ \ 1565 \ 1566 sethi %hi(KERNBASE), %g5; \ 1567 flush %g5; /* Some convenient address that won't trap */ \ 1568 1: 1569 1570 #else /* _LP64 */ 1571 1572 #define TRAP_SETUP(stackspace) \ 1573 sethi %hi(CPCB), %g6; \ 1574 sethi %hi((stackspace)), %g5; \ 1575 LDPTR [%g6 + %lo(CPCB)], %g6; \ 1576 sethi %hi(USPACE), %g7; \ 1577 or %g5, %lo((stackspace)), %g5; \ 1578 add %g6, %g7, %g6; \ 1579 rdpr %wstate, %g7; /* Find if we're from user mode */ \ 1580 \ 1581 sra %g5, 0, %g5; /* Sign extend the damn thing */ \ 1582 subcc %g7, WSTATE_KERN, %g7; /* Compare & leave in register */ \ 1583 movz %icc, %sp, %g6; /* Select old (kernel) stack or base of kernel stack */ \ 1584 FIXUP_TRAP_STACK \ 1585 SAVE_LOCALS_INS \ 1586 save %g6, 0, %sp; /* If we fault we should come right back here */ \ 1587 stx %i0, [%sp + CC64FSZ + STKB + TF_O + (0*8)]; /* Save out registers to trap frame */ \ 1588 stx %i1, [%sp + CC64FSZ + STKB + TF_O + (1*8)]; \ 1589 stx %i2, [%sp + CC64FSZ + STKB + TF_O + (2*8)]; \ 1590 stx %i3, [%sp + CC64FSZ + STKB + TF_O + (3*8)]; \ 1591 stx %i4, [%sp + CC64FSZ + STKB + TF_O + (4*8)]; \ 1592 stx %i5, [%sp + CC64FSZ + STKB + TF_O + (5*8)]; \ 1593 \ 1594 stx %i6, [%sp + CC64FSZ + STKB + TF_O + (6*8)]; \ 1595 brz,pn %g7, 1f; /* If we were in kernel mode start saving globals */ \ 1596 stx %i7, [%sp + CC64FSZ + STKB + TF_O + (7*8)]; \ 1597 mov CTX_PRIMARY, %g7; \ 1598 /* came from user mode -- switch to kernel mode stack */ \ 1599 rdpr %canrestore, %g5; /* Fixup register window state registers */ \ 1600 wrpr %g0, 0, %canrestore; \ 1601 wrpr %g0, %g5, %otherwin; \ 1602 wrpr %g0, WSTATE_KERN, %wstate; /* Enable kernel mode window traps -- now we can trap again */ \ 1603 \ 1604 SET_MMU_CONTEXTID %g0, %g7, %g5; /* Switch MMU to kernel primary context */ \ 1605 sethi %hi(KERNBASE), %g5; \ 1606 flush %g5; /* Some convenient address that won't trap */ \ 1607 1: 1608 1609 /* 1610 * Interrupt setup is almost exactly like trap setup, but we need to 1611 * go to the interrupt stack if (a) we came from user mode or (b) we 1612 * came from kernel mode on the kernel stack. 1613 * 1614 * We don't guarantee any registers are preserved during this operation. 1615 */ 1616 #define INTR_SETUP(stackspace) \ 1617 sethi %hi(EINTSTACK), %g1; \ 1618 sethi %hi((stackspace)), %g5; \ 1619 btst 1, %sp; \ 1620 add %sp, BIAS, %g6; \ 1621 movz %icc, %sp, %g6; \ 1622 or %g1, %lo(EINTSTACK), %g1; \ 1623 srl %g6, 0, %g6; /* truncate at 32-bits */ \ 1624 set (EINTSTACK-INTSTACK), %g7; \ 1625 or %g5, %lo((stackspace)), %g5; \ 1626 sub %g1, %g6, %g2; /* Determine if we need to switch to intr stack or not */ \ 1627 dec %g7; /* Make it into a mask */ \ 1628 sethi %hi(CPUINFO_VA + CI_EINTSTACK), %g3; \ 1629 andncc %g2, %g7, %g0; /* XXXXXXXXXX This assumes kernel addresses are unique from user addresses */ \ 1630 LDPTR [%g3 + %lo(CPUINFO_VA + CI_EINTSTACK)], %g3; \ 1631 rdpr %wstate, %g7; /* Find if we're from user mode */ \ 1632 movrnz %g3, %g3, %g1; /* Use saved intr stack if exists */ \ 1633 sra %g5, 0, %g5; /* Sign extend the damn thing */ \ 1634 movnz %xcc, %g1, %g6; /* Stay on interrupt stack? */ \ 1635 cmp %g7, WSTATE_KERN; /* User or kernel sp? */ \ 1636 movnz %icc, %g1, %g6; /* Stay on interrupt stack? */ \ 1637 add %g6, %g5, %g6; /* Allocate a stack frame */ \ 1638 \ 1639 SAVE_LOCALS_INS \ 1640 save %g6, 0, %sp; /* If we fault we should come right back here */ \ 1641 stx %i0, [%sp + CC64FSZ + STKB + TF_O + (0*8)]; /* Save out registers to trap frame */ \ 1642 stx %i1, [%sp + CC64FSZ + STKB + TF_O + (1*8)]; \ 1643 stx %i2, [%sp + CC64FSZ + STKB + TF_O + (2*8)]; \ 1644 stx %i3, [%sp + CC64FSZ + STKB + TF_O + (3*8)]; \ 1645 stx %i4, [%sp + CC64FSZ + STKB + TF_O + (4*8)]; \ 1646 stx %i5, [%sp + CC64FSZ + STKB + TF_O + (5*8)]; \ 1647 stx %i6, [%sp + CC64FSZ + STKB + TF_O + (6*8)]; \ 1648 stx %i6, [%sp + CC64FSZ + STKB + TF_G + (0*8)]; /* Save fp in clockframe->cf_fp */ \ 1649 rdpr %wstate, %g7; /* Find if we're from user mode */ \ 1650 stx %i7, [%sp + CC64FSZ + STKB + TF_O + (7*8)]; \ 1651 cmp %g7, WSTATE_KERN; /* Compare & leave in register */ \ 1652 be,pn %icc, 1f; /* If we were in kernel mode start saving globals */ \ 1653 /* came from user mode -- switch to kernel mode stack */ \ 1654 rdpr %otherwin, %g5; /* Has this already been done? */ \ 1655 tst %g5; tnz %xcc, 1; nop; /* DEBUG -- this should _NEVER_ happen */ \ 1656 brnz,pn %g5, 1f; /* Don't set this twice */ \ 1657 rdpr %canrestore, %g5; /* Fixup register window state registers */ \ 1658 wrpr %g0, 0, %canrestore; \ 1659 mov CTX_PRIMARY, %g7; \ 1660 wrpr %g0, %g5, %otherwin; \ 1661 wrpr %g0, WSTATE_KERN, %wstate; /* Enable kernel mode window traps -- now we can trap again */ \ 1662 SET_MMU_CONTEXTID %g0, %g7, %g5; /* Switch MMU to kernel primary context */ \ 1663 sethi %hi(KERNBASE), %g5; \ 1664 flush %g5; /* Some convenient address that won't trap */ \ 1665 1: 1666 #endif /* _LP64 */ 1667 1668 #ifdef DEBUG 1669 1670 /* Look up kpte to test algorithm */ 1671 .globl asmptechk 1672 asmptechk: 1673 mov %o0, %g4 ! pmap->pm_segs 1674 mov %o1, %g3 ! Addr to lookup -- mind the context 1675 1676 srax %g3, HOLESHIFT, %g5 ! Check for valid address 1677 brz,pt %g5, 0f ! Should be zero or -1 1678 inc %g5 ! Make -1 -> 0 1679 brnz,pn %g5, 1f ! Error! 1680 0: 1681 srlx %g3, STSHIFT, %g5 1682 and %g5, STMASK, %g5 1683 sll %g5, 3, %g5 1684 add %g4, %g5, %g4 1685 DLFLUSH(%g4,%g5) 1686 ldxa [%g4] ASI_PHYS_CACHED, %g4 ! Remember -- UNSIGNED 1687 DLFLUSH2(%g5) 1688 brz,pn %g4, 1f ! NULL entry? check somewhere else 1689 1690 srlx %g3, PDSHIFT, %g5 1691 and %g5, PDMASK, %g5 1692 sll %g5, 3, %g5 1693 add %g4, %g5, %g4 1694 DLFLUSH(%g4,%g5) 1695 ldxa [%g4] ASI_PHYS_CACHED, %g4 ! Remember -- UNSIGNED 1696 DLFLUSH2(%g5) 1697 brz,pn %g4, 1f ! NULL entry? check somewhere else 1698 1699 srlx %g3, PTSHIFT, %g5 ! Convert to ptab offset 1700 and %g5, PTMASK, %g5 1701 sll %g5, 3, %g5 1702 add %g4, %g5, %g4 1703 DLFLUSH(%g4,%g5) 1704 ldxa [%g4] ASI_PHYS_CACHED, %g6 1705 DLFLUSH2(%g5) 1706 brgez,pn %g6, 1f ! Entry invalid? Punt 1707 srlx %g6, 32, %o0 1708 retl 1709 srl %g6, 0, %o1 1710 1: 1711 mov %g0, %o1 1712 retl 1713 mov %g0, %o0 1714 1715 .data 1716 2: 1717 .asciz "asmptechk: %x %x %x %x:%x\n" 1718 _ALIGN 1719 .text 1720 #endif 1721 1722 /* 1723 * This is the MMU protection handler. It's too big to fit 1724 * in the trap table so I moved it here. It's relatively simple. 1725 * It looks up the page mapping in the page table associated with 1726 * the trapping context. It checks to see if the S/W writable bit 1727 * is set. If so, it sets the H/W write bit, marks the tte modified, 1728 * and enters the mapping into the MMU. Otherwise it does a regular 1729 * data fault. 1730 */ 1731 ICACHE_ALIGN 1732 dmmu_write_fault: 1733 mov TLB_TAG_ACCESS, %g3 1734 sethi %hi(0x1fff), %g6 ! 8K context mask 1735 ldxa [%g3] ASI_DMMU, %g3 ! Get fault addr from Tag Target 1736 sethi %hi(CPUINFO_VA+CI_CTXBUSY), %g4 1737 or %g6, %lo(0x1fff), %g6 1738 LDPTR [%g4 + %lo(CPUINFO_VA+CI_CTXBUSY)], %g4 1739 srax %g3, HOLESHIFT, %g5 ! Check for valid address 1740 and %g3, %g6, %g6 ! Isolate context 1741 1742 inc %g5 ! (0 or -1) -> (1 or 0) 1743 sllx %g6, 3, %g6 ! Make it into an offset into ctxbusy 1744 ldx [%g4+%g6], %g4 ! Load up our page table. 1745 srlx %g3, STSHIFT, %g6 1746 cmp %g5, 1 1747 bgu,pn %xcc, winfix ! Error! 1748 srlx %g3, PDSHIFT, %g5 1749 and %g6, STMASK, %g6 1750 sll %g6, 3, %g6 1751 1752 and %g5, PDMASK, %g5 1753 sll %g5, 3, %g5 1754 add %g6, %g4, %g4 1755 DLFLUSH(%g4,%g6) 1756 ldxa [%g4] ASI_PHYS_CACHED, %g4 1757 DLFLUSH2(%g6) 1758 srlx %g3, PTSHIFT, %g6 ! Convert to ptab offset 1759 and %g6, PTMASK, %g6 1760 add %g5, %g4, %g5 1761 brz,pn %g4, winfix ! NULL entry? check somewhere else 1762 nop 1763 1764 ldxa [%g5] ASI_PHYS_CACHED, %g4 1765 sll %g6, 3, %g6 1766 brz,pn %g4, winfix ! NULL entry? check somewhere else 1767 add %g6, %g4, %g6 1768 1: 1769 ldxa [%g6] ASI_PHYS_CACHED, %g4 1770 brgez,pn %g4, winfix ! Entry invalid? Punt 1771 or %g4, SUN4U_TTE_MODIFY|SUN4U_TTE_ACCESS|SUN4U_TTE_W, %g7 ! Update the modified bit 1772 1773 btst SUN4U_TTE_REAL_W|SUN4U_TTE_W, %g4 ! Is it a ref fault? 1774 bz,pn %xcc, winfix ! No -- really fault 1775 #ifdef DEBUG 1776 /* Make sure we don't try to replace a kernel translation */ 1777 /* This should not be necessary */ 1778 sllx %g3, 64-13, %g2 ! Isolate context bits 1779 sethi %hi(KERNBASE), %g5 ! Don't need %lo 1780 brnz,pt %g2, 0f ! Ignore context != 0 1781 set 0x0800000, %g2 ! 8MB 1782 sub %g3, %g5, %g5 1783 cmp %g5, %g2 1784 tlu %xcc, 1; nop 1785 blu,pn %xcc, winfix ! Next insn in delay slot is unimportant 1786 0: 1787 #endif 1788 /* Need to check for and handle large pages. */ 1789 srlx %g4, 61, %g5 ! Isolate the size bits 1790 ldxa [%g0] ASI_DMMU_8KPTR, %g2 ! Load DMMU 8K TSB pointer 1791 andcc %g5, 0x3, %g5 ! 8K? 1792 bnz,pn %icc, winfix ! We punt to the pmap code since we can't handle policy 1793 ldxa [%g0] ASI_DMMU, %g1 ! Load DMMU tag target register 1794 casxa [%g6] ASI_PHYS_CACHED, %g4, %g7 ! and write it out 1795 membar #StoreLoad 1796 cmp %g4, %g7 1797 bne,pn %xcc, 1b 1798 or %g4, SUN4U_TTE_MODIFY|SUN4U_TTE_ACCESS|SUN4U_TTE_W, %g4 ! Update the modified bit 1799 stx %g1, [%g2] ! Update TSB entry tag 1800 mov SFSR, %g7 1801 stx %g4, [%g2+8] ! Update TSB entry data 1802 nop 1803 1804 #ifdef TRAPSTATS 1805 sethi %hi(_C_LABEL(protfix)), %g1 1806 lduw [%g1+%lo(_C_LABEL(protfix))], %g2 1807 inc %g2 1808 stw %g2, [%g1+%lo(_C_LABEL(protfix))] 1809 #endif 1810 mov DEMAP_PAGE_SECONDARY, %g1 ! Secondary flush 1811 mov DEMAP_PAGE_NUCLEUS, %g5 ! Nucleus flush 1812 stxa %g0, [%g7] ASI_DMMU ! clear out the fault 1813 sllx %g3, (64-13), %g7 ! Need to demap old entry first 1814 andn %g3, 0xfff, %g6 1815 movrz %g7, %g5, %g1 ! Pick one 1816 or %g6, %g1, %g6 1817 membar #Sync 1818 stxa %g6, [%g6] ASI_DMMU_DEMAP ! Do the demap 1819 membar #Sync 1820 1821 stxa %g4, [%g0] ASI_DMMU_DATA_IN ! Enter new mapping 1822 membar #Sync 1823 retry 1824 1825 /* 1826 * Each memory data access fault from a fast access miss handler comes here. 1827 * We will quickly check if this is an original prom mapping before going 1828 * to the generic fault handler 1829 * 1830 * We will assume that %pil is not lost so we won't bother to save it 1831 * unless we're in an interrupt handler. 1832 * 1833 * On entry: 1834 * We are on one of the alternate set of globals 1835 * %g1 = MMU tag target 1836 * %g2 = 8Kptr 1837 * %g3 = TLB TAG ACCESS 1838 * 1839 * On return: 1840 * 1841 */ 1842 ICACHE_ALIGN 1843 data_miss: 1844 #ifdef TRAPSTATS 1845 set _C_LABEL(kdmiss), %g3 1846 set _C_LABEL(udmiss), %g4 1847 rdpr %tl, %g6 1848 dec %g6 1849 movrz %g6, %g4, %g3 1850 lduw [%g3], %g4 1851 inc %g4 1852 stw %g4, [%g3] 1853 #endif 1854 mov TLB_TAG_ACCESS, %g3 ! Get real fault page 1855 sethi %hi(0x1fff), %g6 ! 8K context mask 1856 ldxa [%g3] ASI_DMMU, %g3 ! from tag access register 1857 sethi %hi(CPUINFO_VA+CI_CTXBUSY), %g4 1858 or %g6, %lo(0x1fff), %g6 1859 LDPTR [%g4 + %lo(CPUINFO_VA+CI_CTXBUSY)], %g4 1860 srax %g3, HOLESHIFT, %g5 ! Check for valid address 1861 and %g3, %g6, %g6 ! Isolate context 1862 1863 inc %g5 ! (0 or -1) -> (1 or 0) 1864 sllx %g6, 3, %g6 ! Make it into an offset into ctxbusy 1865 ldx [%g4+%g6], %g4 ! Load up our page table. 1866 #ifdef DEBUG 1867 /* Make sure we don't try to replace a kernel translation */ 1868 /* This should not be necessary */ 1869 brnz,pt %g6, 1f ! If user context continue miss 1870 sethi %hi(KERNBASE), %g7 ! Don't need %lo 1871 set 0x0800000, %g6 ! 8MB 1872 sub %g3, %g7, %g7 1873 cmp %g7, %g6 1874 tlu %xcc, 1; nop 1875 1: 1876 #endif 1877 srlx %g3, STSHIFT, %g6 1878 cmp %g5, 1 1879 bgu,pn %xcc, winfix ! Error! 1880 srlx %g3, PDSHIFT, %g5 1881 and %g6, STMASK, %g6 1882 1883 sll %g6, 3, %g6 1884 and %g5, PDMASK, %g5 1885 sll %g5, 3, %g5 1886 add %g6, %g4, %g4 1887 ldxa [%g4] ASI_PHYS_CACHED, %g4 1888 srlx %g3, PTSHIFT, %g6 ! Convert to ptab offset 1889 and %g6, PTMASK, %g6 1890 add %g5, %g4, %g5 1891 brz,pn %g4, data_nfo ! NULL entry? check somewhere else 1892 1893 nop 1894 ldxa [%g5] ASI_PHYS_CACHED, %g4 1895 sll %g6, 3, %g6 1896 brz,pn %g4, data_nfo ! NULL entry? check somewhere else 1897 add %g6, %g4, %g6 1898 1899 1: 1900 ldxa [%g6] ASI_PHYS_CACHED, %g4 1901 brgez,pn %g4, data_nfo ! Entry invalid? Punt 1902 or %g4, SUN4U_TTE_ACCESS, %g7 ! Update the access bit 1903 1904 btst SUN4U_TTE_ACCESS, %g4 ! Need to update access git? 1905 bne,pt %xcc, 1f 1906 nop 1907 casxa [%g6] ASI_PHYS_CACHED, %g4, %g7 ! and write it out 1908 cmp %g4, %g7 1909 bne,pn %xcc, 1b 1910 or %g4, SUN4U_TTE_ACCESS, %g4 ! Update the access bit 1911 1912 1: 1913 stx %g1, [%g2] ! Update TSB entry tag 1914 stx %g4, [%g2+8] ! Update TSB entry data 1915 stxa %g4, [%g0] ASI_DMMU_DATA_IN ! Enter new mapping 1916 membar #Sync 1917 CLRTT 1918 retry 1919 NOTREACHED 1920 /* 1921 * We had a data miss but did not find a mapping. Insert 1922 * a NFO mapping to satisfy speculative loads and return. 1923 * If this had been a real load, it will re-execute and 1924 * result in a data fault or protection fault rather than 1925 * a TLB miss. We insert an 8K TTE with the valid and NFO 1926 * bits set. All others should zero. The TTE looks like this: 1927 * 1928 * 0x9000000000000000 1929 * 1930 */ 1931 data_nfo: 1932 sethi %hi(0x90000000), %g4 ! V(0x8)|NFO(0x1) 1933 sllx %g4, 32, %g4 1934 stxa %g4, [%g0] ASI_DMMU_DATA_IN ! Enter new mapping 1935 membar #Sync 1936 CLRTT 1937 retry 1938 1939 /* 1940 * Handler for making the trap window shiny clean. 1941 * 1942 * If the store that trapped was to a kernel address, panic. 1943 * 1944 * If the store that trapped was to a user address, stick it in the PCB. 1945 * Since we don't want to force user code to use the standard register 1946 * convention if we don't have to, we will not assume that %fp points to 1947 * anything valid. 1948 * 1949 * On entry: 1950 * We are on one of the alternate set of globals 1951 * %g1 = %tl - 1, tstate[tl-1], scratch - local 1952 * %g2 = %tl - local 1953 * %g3 = MMU tag access - in 1954 * %g4 = %cwp - local 1955 * %g5 = scratch - local 1956 * %g6 = cpcb - local 1957 * %g7 = scratch - local 1958 * 1959 * On return: 1960 * 1961 * NB: remove most of this from main codepath & cleanup I$ 1962 */ 1963 winfault: 1964 mov TLB_TAG_ACCESS, %g3 ! Get real fault page from tag access register 1965 ldxa [%g3] ASI_DMMU, %g3 ! And put it into the non-MMU alternate regs 1966 winfix: 1967 rdpr %tl, %g2 1968 subcc %g2, 1, %g1 1969 ble,pt %icc, datafault ! Don't go below trap level 1 1970 sethi %hi(CPCB), %g6 ! get current pcb 1971 1972 1973 wrpr %g1, 0, %tl ! Pop a trap level 1974 rdpr %tt, %g7 ! Read type of prev. trap 1975 rdpr %tstate, %g4 ! Try to restore prev %cwp if we were executing a restore 1976 andn %g7, 0x3f, %g5 ! window fill traps are all 0b 0000 11xx xxxx 1977 1978 #if 1 1979 cmp %g7, 0x30 ! If we took a datafault just before this trap 1980 bne,pt %icc, winfixfill ! our stack's probably bad so we need to switch somewhere else 1981 nop 1982 1983 !! 1984 !! Double data fault -- bad stack? 1985 !! 1986 wrpr %g2, %tl ! Restore trap level. 1987 sir ! Just issue a reset and don't try to recover. 1988 mov %fp, %l6 ! Save the frame pointer 1989 set EINTSTACK+USPACE+CC64FSZ-STKB, %fp ! Set the frame pointer to the middle of the idle stack 1990 add %fp, -CC64FSZ, %sp ! Create a stackframe 1991 wrpr %g0, 15, %pil ! Disable interrupts, too 1992 wrpr %g0, %g0, %canrestore ! Our stack is hozed and our PCB 1993 wrpr %g0, 7, %cansave ! probably is too, so blow away 1994 ba slowtrap ! all our register windows. 1995 wrpr %g0, 0x101, %tt 1996 #endif 1997 1998 winfixfill: 1999 cmp %g5, 0x0c0 ! so we mask lower bits & compare to 0b 0000 1100 0000 2000 bne,pt %icc, winfixspill ! Dump our trap frame -- we will retry the fill when the page is loaded 2001 cmp %g5, 0x080 ! window spill traps are all 0b 0000 10xx xxxx 2002 2003 !! 2004 !! This was a fill 2005 !! 2006 #ifdef TRAPSTATS 2007 set _C_LABEL(wfill), %g1 2008 lduw [%g1], %g5 2009 inc %g5 2010 stw %g5, [%g1] 2011 #endif 2012 btst TSTATE_PRIV, %g4 ! User mode? 2013 and %g4, CWP, %g5 ! %g4 = %cwp of trap 2014 wrpr %g7, 0, %tt 2015 bz,a,pt %icc, datafault ! We were in user mode -- normal fault 2016 wrpr %g5, %cwp ! Restore cwp from before fill trap -- regs should now be consistent 2017 2018 /* 2019 * We're in a pickle here. We were trying to return to user mode 2020 * and the restore of the user window failed, so now we have one valid 2021 * kernel window and a user window state. If we do a TRAP_SETUP() now, 2022 * our kernel window will be considered a user window and cause a 2023 * fault when we try to save it later due to an invalid user address. 2024 * If we return to where we faulted, our window state will not be valid 2025 * and we will fault trying to enter user with our primary context of zero. 2026 * 2027 * What we'll do is arrange to have us return to return_from_trap so we will 2028 * start the whole business over again. But first, switch to a kernel window 2029 * setup. Let's see, canrestore and otherwin are zero. Set WSTATE_KERN and 2030 * make sure we're in kernel context and we're done. 2031 */ 2032 2033 #ifdef TRAPSTATS 2034 set _C_LABEL(kwfill), %g4 2035 lduw [%g4], %g7 2036 inc %g7 2037 stw %g7, [%g4] 2038 #endif 2039 #if 0 /* Need to switch over to new stuff to fix WDR bug */ 2040 wrpr %g5, %cwp ! Restore cwp from before fill trap -- regs should now be consistent 2041 wrpr %g2, %g0, %tl ! Restore trap level -- we need to reuse it 2042 set return_from_trap, %g4 ! XXX - need to set %g1 to tstate 2043 set CTX_PRIMARY, %g7 2044 wrpr %g4, 0, %tpc 2045 stxa %g0, [%g7] ASI_DMMU 2046 inc 4, %g4 2047 membar #Sync 2048 flush %g4 ! Isn't this convenient? 2049 wrpr %g0, WSTATE_KERN, %wstate 2050 wrpr %g0, 0, %canrestore ! These should be zero but 2051 wrpr %g0, 0, %otherwin ! clear them just in case 2052 rdpr %ver, %g5 2053 and %g5, CWP, %g5 2054 wrpr %g0, 0, %cleanwin 2055 dec 1, %g5 ! NWINDOWS-1-1 2056 wrpr %g5, 0, %cansave ! Invalidate all windows 2057 ! flushw ! DEBUG 2058 ba,pt %icc, datafault 2059 wrpr %g4, 0, %tnpc 2060 #else 2061 wrpr %g2, %g0, %tl ! Restore trap level 2062 cmp %g2, 3 2063 tne %icc, 1 2064 rdpr %tt, %g5 2065 wrpr %g0, 1, %tl ! Revert to TL==1 XXX what if this wasn't in rft_user? Oh well. 2066 wrpr %g5, %g0, %tt ! Set trap type correctly 2067 /* 2068 * Here we need to implement the beginning of datafault. 2069 * TRAP_SETUP expects to come from either kernel mode or 2070 * user mode with at least one valid register window. It 2071 * will allocate a trap frame, save the out registers, and 2072 * fix the window registers to think we have one user 2073 * register window. 2074 * 2075 * However, under these circumstances we don't have any 2076 * valid register windows, so we need to clean up the window 2077 * registers to prevent garbage from being saved to either 2078 * the user stack or the PCB before calling the datafault 2079 * handler. 2080 * 2081 * We could simply jump to datafault if we could somehow 2082 * make the handler issue a `saved' instruction immediately 2083 * after creating the trapframe. 2084 * 2085 * The following is duplicated from datafault: 2086 */ 2087 #ifdef TRAPS_USE_IG 2088 wrpr %g0, PSTATE_KERN|PSTATE_IG, %pstate ! We need to save volatile stuff to interrupt globals 2089 #else 2090 wrpr %g0, PSTATE_KERN|PSTATE_AG, %pstate ! We need to save volatile stuff to alternate globals 2091 #endif 2092 wr %g0, ASI_DMMU, %asi ! We need to re-load trap info 2093 ldxa [%g0 + TLB_TAG_ACCESS] %asi, %g1 ! Get fault address from tag access register 2094 ldxa [SFAR] %asi, %g2 ! sync virt addr; must be read first 2095 ldxa [SFSR] %asi, %g3 ! get sync fault status register 2096 stxa %g0, [SFSR] %asi ! Clear out fault now 2097 2098 TRAP_SETUP(-CC64FSZ-TF_SIZE) 2099 saved ! Blow away that one register window we didn't ever use. 2100 ba,a,pt %icc, Ldatafault_internal ! Now we should return directly to user mode 2101 nop 2102 #endif 2103 winfixspill: 2104 bne,a,pt %xcc, datafault ! Was not a spill -- handle it normally 2105 wrpr %g2, 0, %tl ! Restore trap level for now XXXX 2106 2107 !! 2108 !! This was a spill 2109 !! 2110 #if 1 2111 btst TSTATE_PRIV, %g4 ! From user mode? 2112 wrpr %g2, 0, %tl ! We need to load the fault type so we can 2113 rdpr %tt, %g5 ! overwrite the lower trap and get it to the fault handler 2114 wrpr %g1, 0, %tl 2115 wrpr %g5, 0, %tt ! Copy over trap type for the fault handler 2116 and %g4, CWP, %g5 ! find %cwp from trap 2117 be,a,pt %xcc, datafault ! Let's do a regular datafault. When we try a save in datafault we'll 2118 wrpr %g5, 0, %cwp ! return here and write out all dirty windows. 2119 #endif 2120 wrpr %g2, 0, %tl ! Restore trap level for now XXXX 2121 LDPTR [%g6 + %lo(CPCB)], %g6 ! This is in the locked TLB and should not fault 2122 #ifdef TRAPSTATS 2123 set _C_LABEL(wspill), %g7 2124 lduw [%g7], %g5 2125 inc %g5 2126 stw %g5, [%g7] 2127 #endif 2128 2129 /* 2130 * Traverse kernel map to find paddr of cpcb and only us ASI_PHYS_CACHED to 2131 * prevent any faults while saving the windows. BTW if it isn't mapped, we 2132 * will trap and hopefully panic. 2133 */ 2134 2135 ! ba 0f ! DEBUG -- don't use phys addresses 2136 wr %g0, ASI_NUCLEUS, %asi ! In case of problems finding PA 2137 sethi %hi(CPUINFO_VA+CI_CTXBUSY), %g1 2138 LDPTR [%g1 + %lo(CPUINFO_VA+CI_CTXBUSY)], %g1 ! Load start of ctxbusy 2139 #ifdef DEBUG 2140 srax %g6, HOLESHIFT, %g7 ! Check for valid address 2141 brz,pt %g7, 1f ! Should be zero or -1 2142 addcc %g7, 1, %g7 ! Make -1 -> 0 2143 tnz %xcc, 1 ! Invalid address??? How did this happen? 2144 1: 2145 #endif 2146 srlx %g6, STSHIFT, %g7 2147 ldx [%g1], %g1 ! Load pointer to kernel_pmap 2148 and %g7, STMASK, %g7 2149 sll %g7, 3, %g7 2150 add %g7, %g1, %g1 2151 DLFLUSH(%g1,%g7) 2152 ldxa [%g1] ASI_PHYS_CACHED, %g1 ! Load pointer to directory 2153 DLFLUSH2(%g7) 2154 2155 srlx %g6, PDSHIFT, %g7 ! Do page directory 2156 and %g7, PDMASK, %g7 2157 sll %g7, 3, %g7 2158 brz,pn %g1, 0f 2159 add %g7, %g1, %g1 2160 DLFLUSH(%g1,%g7) 2161 ldxa [%g1] ASI_PHYS_CACHED, %g1 2162 DLFLUSH2(%g7) 2163 2164 srlx %g6, PTSHIFT, %g7 ! Convert to ptab offset 2165 and %g7, PTMASK, %g7 2166 brz %g1, 0f 2167 sll %g7, 3, %g7 2168 add %g1, %g7, %g7 2169 DLFLUSH(%g7,%g1) 2170 ldxa [%g7] ASI_PHYS_CACHED, %g7 ! This one is not 2171 DLFLUSH2(%g1) 2172 brgez %g7, 0f 2173 srlx %g7, PGSHIFT, %g7 ! Isolate PA part 2174 sll %g6, 32-PGSHIFT, %g6 ! And offset 2175 sllx %g7, PGSHIFT+23, %g7 ! There are 23 bits to the left of the PA in the TTE 2176 srl %g6, 32-PGSHIFT, %g6 2177 srax %g7, 23, %g7 2178 or %g7, %g6, %g6 ! Then combine them to form PA 2179 2180 wr %g0, ASI_PHYS_CACHED, %asi ! Use ASI_PHYS_CACHED to prevent possible page faults 2181 0: 2182 /* 2183 * Now save all user windows to cpcb. 2184 */ 2185 #ifdef NOTDEF_DEBUG 2186 add %g6, PCB_NSAVED, %g7 2187 DLFLUSH(%g7,%g5) 2188 lduba [%g6 + PCB_NSAVED] %asi, %g7 ! make sure that pcb_nsaved 2189 DLFLUSH2(%g5) 2190 brz,pt %g7, 1f ! is zero, else 2191 nop 2192 wrpr %g0, 4, %tl 2193 sir ! Force a watchdog 2194 1: 2195 #endif 2196 rdpr %otherwin, %g7 2197 brnz,pt %g7, 1f 2198 rdpr %canrestore, %g5 2199 rdpr %cansave, %g1 2200 add %g5, 1, %g7 ! add the %cwp window to the list to save 2201 ! movrnz %g1, %g5, %g7 ! If we're issuing a save 2202 ! mov %g5, %g7 ! DEBUG 2203 wrpr %g0, 0, %canrestore 2204 wrpr %g7, 0, %otherwin ! Still in user mode -- need to switch to kernel mode 2205 1: 2206 mov %g7, %g1 2207 add %g6, PCB_NSAVED, %g7 2208 DLFLUSH(%g7,%g5) 2209 lduba [%g6 + PCB_NSAVED] %asi, %g7 ! Start incrementing pcb_nsaved 2210 DLFLUSH2(%g5) 2211 2212 #ifdef DEBUG 2213 wrpr %g0, 5, %tl 2214 #endif 2215 mov %g6, %g5 2216 brz,pt %g7, winfixsave ! If it's in use, panic 2217 saved ! frob window registers 2218 2219 /* PANIC */ 2220 ! sir ! Force a watchdog 2221 #ifdef DEBUG 2222 wrpr %g2, 0, %tl 2223 #endif 2224 mov %g7, %o2 2225 rdpr %ver, %o1 2226 sethi %hi(2f), %o0 2227 and %o1, CWP, %o1 2228 wrpr %g0, %o1, %cleanwin 2229 dec 1, %o1 2230 wrpr %g0, %o1, %cansave ! kludge away any more window problems 2231 wrpr %g0, 0, %canrestore 2232 wrpr %g0, 0, %otherwin 2233 or %lo(2f), %o0, %o0 2234 wrpr %g0, WSTATE_KERN, %wstate 2235 set PANICSTACK-CC64FSZ-STKB, %sp 2236 ta 1; nop ! This helps out traptrace. 2237 call _C_LABEL(panic) ! This needs to be fixed properly but we should panic here 2238 mov %g1, %o1 2239 NOTREACHED 2240 .data 2241 2: 2242 .asciz "winfault: double invalid window at %p, nsaved=%d" 2243 _ALIGN 2244 .text 2245 3: 2246 saved 2247 save 2248 winfixsave: 2249 stxa %l0, [%g5 + PCB_RW + ( 0*8)] %asi ! Save the window in the pcb, we can schedule other stuff in here 2250 stxa %l1, [%g5 + PCB_RW + ( 1*8)] %asi 2251 stxa %l2, [%g5 + PCB_RW + ( 2*8)] %asi 2252 stxa %l3, [%g5 + PCB_RW + ( 3*8)] %asi 2253 stxa %l4, [%g5 + PCB_RW + ( 4*8)] %asi 2254 stxa %l5, [%g5 + PCB_RW + ( 5*8)] %asi 2255 stxa %l6, [%g5 + PCB_RW + ( 6*8)] %asi 2256 stxa %l7, [%g5 + PCB_RW + ( 7*8)] %asi 2257 2258 stxa %i0, [%g5 + PCB_RW + ( 8*8)] %asi 2259 stxa %i1, [%g5 + PCB_RW + ( 9*8)] %asi 2260 stxa %i2, [%g5 + PCB_RW + (10*8)] %asi 2261 stxa %i3, [%g5 + PCB_RW + (11*8)] %asi 2262 stxa %i4, [%g5 + PCB_RW + (12*8)] %asi 2263 stxa %i5, [%g5 + PCB_RW + (13*8)] %asi 2264 stxa %i6, [%g5 + PCB_RW + (14*8)] %asi 2265 stxa %i7, [%g5 + PCB_RW + (15*8)] %asi 2266 2267 ! rdpr %otherwin, %g1 ! Check to see if we's done 2268 dec %g1 2269 wrpr %g0, 7, %cleanwin ! BUGBUG -- we should not hardcode this, but I have no spare globals 2270 inc 16*8, %g5 ! Move to next window 2271 inc %g7 ! inc pcb_nsaved 2272 brnz,pt %g1, 3b 2273 stxa %o6, [%g5 + PCB_RW + (14*8)] %asi ! Save %sp so we can write these all out 2274 2275 /* fix up pcb fields */ 2276 stba %g7, [%g6 + PCB_NSAVED] %asi ! cpcb->pcb_nsaved = n 2277 #if 0 2278 mov %g7, %g5 ! fixup window registers 2279 5: 2280 dec %g5 2281 brgz,a,pt %g5, 5b 2282 restore 2283 #ifdef NOT_DEBUG 2284 rdpr %wstate, %g5 ! DEBUG 2285 wrpr %g0, WSTATE_KERN, %wstate ! DEBUG 2286 wrpr %g0, 4, %tl 2287 rdpr %cansave, %g7 2288 rdpr %canrestore, %g6 2289 flushw ! DEBUG 2290 wrpr %g2, 0, %tl 2291 wrpr %g5, 0, %wstate ! DEBUG 2292 #endif 2293 #else 2294 /* 2295 * We just issued a bunch of saves, so %cansave is now 0, 2296 * probably (if we were doing a flushw then we may have 2297 * come in with only partially full register windows and 2298 * it may not be 0). 2299 * 2300 * %g7 contains the count of the windows we just finished 2301 * saving. 2302 * 2303 * What we need to do now is move some of the windows from 2304 * %canrestore to %cansave. What we should do is take 2305 * min(%canrestore, %g7) and move that over to %cansave. 2306 * 2307 * %g7 is the number of windows we flushed, so we should 2308 * use that as a base. Clear out %otherwin, set %cansave 2309 * to min(%g7, NWINDOWS - 2), set %cleanwin to %canrestore 2310 * + %cansave and the rest follows: 2311 * 2312 * %otherwin = 0 2313 * %cansave = NWINDOWS - 2 - %canrestore 2314 */ 2315 wrpr %g0, 0, %otherwin 2316 rdpr %canrestore, %g1 2317 sub %g1, %g7, %g1 ! Calculate %canrestore - %g7 2318 movrlz %g1, %g0, %g1 ! Clamp at zero 2319 wrpr %g1, 0, %canrestore ! This is the new canrestore 2320 rdpr %ver, %g5 2321 and %g5, CWP, %g5 ! NWINDOWS-1 2322 dec %g5 ! NWINDOWS-2 2323 wrpr %g5, 0, %cleanwin ! Set cleanwin to max, since we're in-kernel 2324 sub %g5, %g1, %g5 ! NWINDOWS-2-%canrestore 2325 wrpr %g5, 0, %cansave 2326 #ifdef NOT_DEBUG 2327 rdpr %wstate, %g5 ! DEBUG 2328 wrpr %g0, WSTATE_KERN, %wstate ! DEBUG 2329 wrpr %g0, 4, %tl 2330 flushw ! DEBUG 2331 wrpr %g2, 0, %tl 2332 wrpr %g5, 0, %wstate ! DEBUG 2333 #endif 2334 #endif 2335 2336 #ifdef NOTDEF_DEBUG 2337 set panicstack-CC64FSZ, %g1 2338 save %g1, 0, %sp 2339 GLOBTOLOC 2340 rdpr %wstate, %l0 2341 wrpr %g0, WSTATE_KERN, %wstate 2342 set 8f, %o0 2343 mov %g7, %o1 2344 call printf 2345 mov %g5, %o2 2346 wrpr %l0, 0, %wstate 2347 LOCTOGLOB 2348 restore 2349 .data 2350 8: 2351 .asciz "winfix: spill fixup\n" 2352 _ALIGN 2353 .text 2354 #endif 2355 ! rdpr %tl, %g2 ! DEBUG DEBUG -- did we trap somewhere? 2356 sub %g2, 1, %g1 2357 rdpr %tt, %g2 2358 wrpr %g1, 0, %tl ! We will not attempt to re-execute the spill, so dump our trap frame permanently 2359 wrpr %g2, 0, %tt ! Move trap type from fault frame here, overwriting spill 2360 2361 /* Did we save a user or kernel window ? */ 2362 ! srax %g3, 48, %g5 ! User or kernel store? (TAG TARGET) 2363 sllx %g3, (64-13), %g5 ! User or kernel store? (TAG ACCESS) 2364 sethi %hi(dcache_size), %g7 2365 ld [%g7 + %lo(dcache_size)], %g7 2366 sethi %hi(dcache_line_size), %g6 2367 ld [%g6 + %lo(dcache_line_size)], %g6 2368 brnz,pt %g5, 1f ! User fault -- save windows to pcb 2369 sub %g7, %g6, %g7 2370 2371 and %g4, CWP, %g4 ! %g4 = %cwp of trap 2372 wrpr %g4, 0, %cwp ! Kernel fault -- restore %cwp and force and trap to debugger 2373 !! 2374 !! Here we managed to fault trying to access a kernel window 2375 !! This is a bug. Switch to the interrupt stack if we aren't 2376 !! there already and then trap into the debugger or panic. 2377 !! 2378 sethi %hi(EINTSTACK-BIAS), %g6 2379 btst 1, %sp 2380 bnz,pt %icc, 0f 2381 mov %sp, %g1 2382 add %sp, -BIAS, %g1 2383 0: 2384 or %g6, %lo(EINTSTACK-BIAS), %g6 2385 set (EINTSTACK-INTSTACK), %g7 ! XXXXXXXXXX This assumes kernel addresses are unique from user addresses 2386 sub %g6, %g1, %g2 ! Determine if we need to switch to intr stack or not 2387 dec %g7 ! Make it into a mask 2388 andncc %g2, %g7, %g0 ! XXXXXXXXXX This assumes kernel addresses are unique from user addresses */ \ 2389 movz %xcc, %g1, %g6 ! Stay on interrupt stack? 2390 add %g6, -CCFSZ, %g6 ! Allocate a stack frame 2391 mov %sp, %l6 ! XXXXX Save old stack pointer 2392 mov %g6, %sp 2393 ta 1; nop ! Enter debugger 2394 NOTREACHED 2395 1: 2396 #if 1 2397 /* Now we need to blast away the D$ to make sure we're in sync */ 2398 stxa %g0, [%g7] ASI_DCACHE_TAG 2399 brnz,pt %g7, 1b 2400 sub %g7, %g6, %g7 2401 #endif 2402 2403 #ifdef NOTDEF_DEBUG 2404 set panicstack-CC64FSZ, %g5 2405 save %g5, 0, %sp 2406 GLOBTOLOC 2407 rdpr %wstate, %l0 2408 wrpr %g0, WSTATE_KERN, %wstate 2409 set 8f, %o0 2410 call printf 2411 mov %fp, %o1 2412 wrpr %l0, 0, %wstate 2413 LOCTOGLOB 2414 restore 2415 .data 2416 8: 2417 .asciz "winfix: kernel spill retry\n" 2418 _ALIGN 2419 .text 2420 #endif 2421 #ifdef TRAPSTATS 2422 set _C_LABEL(wspillskip), %g4 2423 lduw [%g4], %g5 2424 inc %g5 2425 stw %g5, [%g4] 2426 #endif 2427 /* 2428 * If we had WSTATE_KERN then we had at least one valid kernel window. 2429 * We should re-execute the trapping save. 2430 */ 2431 rdpr %wstate, %g3 2432 mov %g3, %g3 2433 cmp %g3, WSTATE_KERN 2434 bne,pt %icc, 1f 2435 nop 2436 retry ! Now we can complete the save 2437 1: 2438 /* 2439 * Since we had a WSTATE_USER, we had no valid kernel windows. This should 2440 * only happen inside TRAP_SETUP or INTR_SETUP. Emulate 2441 * the instruction, clean up the register windows, then done. 2442 */ 2443 rdpr %cwp, %g1 2444 inc %g1 2445 rdpr %tstate, %g2 2446 wrpr %g1, %cwp 2447 andn %g2, CWP, %g2 2448 wrpr %g1, %g2, %tstate 2449 #ifdef TRAPS_USE_IG 2450 wrpr %g0, PSTATE_KERN|PSTATE_IG, %pstate ! DEBUG 2451 #else 2452 wrpr %g0, PSTATE_KERN|PSTATE_AG, %pstate 2453 #endif 2454 mov %g6, %sp 2455 done 2456 2457 /* 2458 * Each memory data access fault, from user or kernel mode, 2459 * comes here. 2460 * 2461 * We will assume that %pil is not lost so we won't bother to save it 2462 * unless we're in an interrupt handler. 2463 * 2464 * On entry: 2465 * We are on one of the alternate set of globals 2466 * %g1 = MMU tag target 2467 * %g2 = %tl 2468 * 2469 * On return: 2470 * 2471 */ 2472 datafault: 2473 #ifdef TRAPS_USE_IG 2474 wrpr %g0, PSTATE_KERN|PSTATE_IG, %pstate ! We need to save volatile stuff to interrupt globals 2475 #else 2476 wrpr %g0, PSTATE_KERN|PSTATE_AG, %pstate ! We need to save volatile stuff to alternate globals 2477 #endif 2478 wr %g0, ASI_DMMU, %asi ! We need to re-load trap info 2479 ldxa [%g0 + TLB_TAG_ACCESS] %asi, %g1 ! Get fault address from tag access register 2480 ldxa [SFAR] %asi, %g2 ! sync virt addr; must be read first 2481 ldxa [SFSR] %asi, %g3 ! get sync fault status register 2482 stxa %g0, [SFSR] %asi ! Clear out fault now 2483 2484 TRAP_SETUP(-CC64FSZ-TF_SIZE) 2485 Ldatafault_internal: 2486 INCR64(CPUINFO_VA+CI_NFAULT) ! cnt.v_faults++ (clobbers %o0,%o1) 2487 ! ldx [%sp + CC64FSZ + STKB + TF_FAULT], %g1 ! DEBUG make sure this has not changed 2488 mov %g1, %o0 ! Move these to the out regs so we can save the globals 2489 mov %g2, %o4 2490 mov %g3, %o5 2491 2492 ldxa [%g0] ASI_AFAR, %o2 ! get async fault address 2493 ldxa [%g0] ASI_AFSR, %o3 ! get async fault status 2494 mov -1, %g7 2495 stxa %g7, [%g0] ASI_AFSR ! And clear this out, too 2496 2497 wrpr %g0, PSTATE_KERN, %pstate ! Get back to normal globals 2498 2499 stx %g1, [%sp + CC64FSZ + STKB + TF_G + (1*8)] ! save g1 2500 rdpr %tt, %o1 ! find out what trap brought us here 2501 stx %g2, [%sp + CC64FSZ + STKB + TF_G + (2*8)] ! save g2 2502 rdpr %tstate, %g1 2503 stx %g3, [%sp + CC64FSZ + STKB + TF_G + (3*8)] ! (sneak g3 in here) 2504 rdpr %tpc, %g2 2505 stx %g4, [%sp + CC64FSZ + STKB + TF_G + (4*8)] ! sneak in g4 2506 rdpr %tnpc, %g3 2507 stx %g5, [%sp + CC64FSZ + STKB + TF_G + (5*8)] ! sneak in g5 2508 mov %g2, %o7 ! Make the fault address look like the return address 2509 stx %g6, [%sp + CC64FSZ + STKB + TF_G + (6*8)] ! sneak in g6 2510 rd %y, %g5 ! save y 2511 stx %g7, [%sp + CC64FSZ + STKB + TF_G + (7*8)] ! sneak in g7 2512 2513 sth %o1, [%sp + CC64FSZ + STKB + TF_TT] 2514 stx %g1, [%sp + CC64FSZ + STKB + TF_TSTATE] ! set tf.tf_psr, tf.tf_pc 2515 stx %g2, [%sp + CC64FSZ + STKB + TF_PC] ! set tf.tf_npc 2516 stx %g3, [%sp + CC64FSZ + STKB + TF_NPC] 2517 2518 rdpr %pil, %g4 2519 stb %g4, [%sp + CC64FSZ + STKB + TF_PIL] 2520 stb %g4, [%sp + CC64FSZ + STKB + TF_OLDPIL] 2521 2522 #if 1 2523 rdpr %tl, %g7 2524 dec %g7 2525 movrlz %g7, %g0, %g7 2526 wrpr %g0, %g7, %tl ! Revert to kernel mode 2527 #else 2528 wrpr %g0, 0, %tl ! Revert to kernel mode 2529 #endif 2530 /* Finish stackframe, call C trap handler */ 2531 flushw ! Get this clean so we won't take any more user faults 2532 #ifdef NOTDEF_DEBUG 2533 set CPCB, %o7 2534 LDPTR [%o7], %o7 2535 ldub [%o7 + PCB_NSAVED], %o7 2536 brz,pt %o7, 2f 2537 nop 2538 save %sp, -CC64FSZ, %sp 2539 set 1f, %o0 2540 call printf 2541 mov %i7, %o1 2542 ta 1; nop 2543 restore 2544 .data 2545 1: .asciz "datafault: nsaved = %d\n" 2546 _ALIGN 2547 .text 2548 2: 2549 #endif 2550 !! In the EMBEDANY memory model %g4 points to the start of the data segment. 2551 !! In our case we need to clear it before calling any C-code 2552 clr %g4 2553 2554 /* 2555 * Right now the registers have the following values: 2556 * 2557 * %o0 -- MMU_TAG_ACCESS 2558 * %o1 -- TT 2559 * %o2 -- afar 2560 * %o3 -- afsr 2561 * %o4 -- sfar 2562 * %o5 -- sfsr 2563 */ 2564 2565 cmp %o1, T_DATA_ERROR 2566 st %g5, [%sp + CC64FSZ + STKB + TF_Y] 2567 wr %g0, ASI_PRIMARY_NOFAULT, %asi ! Restore default ASI 2568 be,pn %icc, data_error 2569 wrpr %g0, PSTATE_INTR, %pstate ! reenable interrupts 2570 2571 mov %o0, %o3 ! (argument: trap address) 2572 mov %g2, %o2 ! (argument: trap pc) 2573 call _C_LABEL(data_access_fault) ! data_access_fault(&tf, type, 2574 ! pc, addr, sfva, sfsr) 2575 add %sp, CC64FSZ + STKB, %o0 ! (argument: &tf) 2576 2577 data_recover: 2578 #ifdef TRAPSTATS 2579 set _C_LABEL(uintrcnt), %g1 2580 stw %g0, [%g1] 2581 set _C_LABEL(iveccnt), %g1 2582 stw %g0, [%g1] 2583 #endif 2584 wrpr %g0, PSTATE_KERN, %pstate ! disable interrupts 2585 b return_from_trap ! go return 2586 ldx [%sp + CC64FSZ + STKB + TF_TSTATE], %g1 ! Load this for return_from_trap 2587 NOTREACHED 2588 2589 data_error: 2590 call _C_LABEL(data_access_error) ! data_access_error(&tf, type, 2591 ! afva, afsr, sfva, sfsr) 2592 add %sp, CC64FSZ + STKB, %o0 ! (argument: &tf) 2593 ba data_recover 2594 nop 2595 NOTREACHED 2596 2597 /* 2598 * Each memory instruction access fault from a fast access handler comes here. 2599 * We will quickly check if this is an original prom mapping before going 2600 * to the generic fault handler 2601 * 2602 * We will assume that %pil is not lost so we won't bother to save it 2603 * unless we're in an interrupt handler. 2604 * 2605 * On entry: 2606 * We are on one of the alternate set of globals 2607 * %g1 = MMU tag target 2608 * %g2 = TSB entry ptr 2609 * %g3 = TLB Tag Access 2610 * 2611 * On return: 2612 * 2613 */ 2614 2615 ICACHE_ALIGN 2616 instr_miss: 2617 #ifdef TRAPSTATS 2618 set _C_LABEL(ktmiss), %g3 2619 set _C_LABEL(utmiss), %g4 2620 rdpr %tl, %g6 2621 dec %g6 2622 movrz %g6, %g4, %g3 2623 lduw [%g3], %g4 2624 inc %g4 2625 stw %g4, [%g3] 2626 #endif 2627 mov TLB_TAG_ACCESS, %g3 ! Get real fault page 2628 sethi %hi(0x1fff), %g7 ! 8K context mask 2629 ldxa [%g3] ASI_IMMU, %g3 ! from tag access register 2630 sethi %hi(CPUINFO_VA+CI_CTXBUSY), %g4 2631 or %g7, %lo(0x1fff), %g7 2632 LDPTR [%g4 + %lo(CPUINFO_VA+CI_CTXBUSY)], %g4 2633 srax %g3, HOLESHIFT, %g5 ! Check for valid address 2634 and %g3, %g7, %g6 ! Isolate context 2635 sllx %g6, 3, %g6 ! Make it into an offset into ctxbusy 2636 inc %g5 ! (0 or -1) -> (1 or 0) 2637 ldx [%g4+%g6], %g4 ! Load up our page table. 2638 #ifdef DEBUG 2639 /* Make sure we don't try to replace a kernel translation */ 2640 /* This should not be necessary */ 2641 brnz,pt %g6, 1f ! If user context continue miss 2642 sethi %hi(KERNBASE), %g7 ! Don't need %lo 2643 set 0x0800000, %g6 ! 8MB 2644 sub %g3, %g7, %g7 2645 cmp %g7, %g6 2646 tlu %xcc, 1; nop 2647 1: 2648 #endif 2649 srlx %g3, STSHIFT, %g6 2650 cmp %g5, 1 2651 bgu,pn %xcc, textfault ! Error! 2652 srlx %g3, PDSHIFT, %g5 2653 and %g6, STMASK, %g6 2654 sll %g6, 3, %g6 2655 and %g5, PDMASK, %g5 2656 nop 2657 2658 sll %g5, 3, %g5 2659 add %g6, %g4, %g4 2660 ldxa [%g4] ASI_PHYS_CACHED, %g4 2661 srlx %g3, PTSHIFT, %g6 ! Convert to ptab offset 2662 and %g6, PTMASK, %g6 2663 add %g5, %g4, %g5 2664 brz,pn %g4, textfault ! NULL entry? check somewhere else 2665 nop 2666 2667 ldxa [%g5] ASI_PHYS_CACHED, %g4 2668 sll %g6, 3, %g6 2669 brz,pn %g4, textfault ! NULL entry? check somewhere else 2670 add %g6, %g4, %g6 2671 1: 2672 ldxa [%g6] ASI_PHYS_CACHED, %g4 2673 brgez,pn %g4, textfault 2674 nop 2675 2676 /* Check if it's an executable mapping. */ 2677 andcc %g4, SUN4U_TTE_EXEC, %g0 2678 bz,pn %xcc, textfault 2679 nop 2680 2681 or %g4, SUN4U_TTE_ACCESS, %g7 ! Update accessed bit 2682 btst SUN4U_TTE_ACCESS, %g4 ! Need to update access git? 2683 bne,pt %xcc, 1f 2684 nop 2685 casxa [%g6] ASI_PHYS_CACHED, %g4, %g7 ! and store it 2686 cmp %g4, %g7 2687 bne,pn %xcc, 1b 2688 or %g4, SUN4U_TTE_ACCESS, %g4 ! Update accessed bit 2689 1: 2690 stx %g1, [%g2] ! Update TSB entry tag 2691 stx %g4, [%g2+8] ! Update TSB entry data 2692 stxa %g4, [%g0] ASI_IMMU_DATA_IN ! Enter new mapping 2693 membar #Sync 2694 CLRTT 2695 retry 2696 NOTREACHED 2697 !! 2698 !! Check our prom mappings -- temporary 2699 !! 2700 2701 /* 2702 * Each memory text access fault, from user or kernel mode, 2703 * comes here. 2704 * 2705 * We will assume that %pil is not lost so we won't bother to save it 2706 * unless we're in an interrupt handler. 2707 * 2708 * On entry: 2709 * We are on one of the alternate set of globals 2710 * %g1 = MMU tag target 2711 * %g2 = %tl 2712 * %g3 = %tl - 1 2713 * 2714 * On return: 2715 * 2716 */ 2717 2718 textfault: 2719 #ifdef TRAPS_USE_IG 2720 wrpr %g0, PSTATE_KERN|PSTATE_IG, %pstate ! We need to save volatile stuff to interrupt globals 2721 #else 2722 wrpr %g0, PSTATE_KERN|PSTATE_AG, %pstate ! We need to save volatile stuff to alternate globals 2723 #endif 2724 wr %g0, ASI_IMMU, %asi 2725 ldxa [%g0 + TLB_TAG_ACCESS] %asi, %g1 ! Get fault address from tag access register 2726 ldxa [SFSR] %asi, %g3 ! get sync fault status register 2727 membar #LoadStore 2728 stxa %g0, [SFSR] %asi ! Clear out old info 2729 2730 TRAP_SETUP(-CC64FSZ-TF_SIZE) 2731 INCR64(CPUINFO_VA+CI_NFAULT) ! cnt.v_faults++ (clobbers %o0,%o1) 2732 2733 mov %g3, %o3 2734 2735 wrpr %g0, PSTATE_KERN, %pstate ! Switch to normal globals 2736 ldxa [%g0] ASI_AFSR, %o4 ! get async fault status 2737 ldxa [%g0] ASI_AFAR, %o5 ! get async fault address 2738 mov -1, %o0 2739 stxa %o0, [%g0] ASI_AFSR ! Clear this out 2740 stx %g1, [%sp + CC64FSZ + STKB + TF_G + (1*8)] ! save g1 2741 stx %g2, [%sp + CC64FSZ + STKB + TF_G + (2*8)] ! save g2 2742 stx %g3, [%sp + CC64FSZ + STKB + TF_G + (3*8)] ! (sneak g3 in here) 2743 rdpr %tt, %o1 ! Find out what caused this trap 2744 stx %g4, [%sp + CC64FSZ + STKB + TF_G + (4*8)] ! sneak in g4 2745 rdpr %tstate, %g1 2746 stx %g5, [%sp + CC64FSZ + STKB + TF_G + (5*8)] ! sneak in g5 2747 rdpr %tpc, %o2 ! sync virt addr; must be read first 2748 stx %g6, [%sp + CC64FSZ + STKB + TF_G + (6*8)] ! sneak in g6 2749 rdpr %tnpc, %g3 2750 stx %g7, [%sp + CC64FSZ + STKB + TF_G + (7*8)] ! sneak in g7 2751 rd %y, %g5 ! save y 2752 2753 /* Finish stackframe, call C trap handler */ 2754 stx %g1, [%sp + CC64FSZ + STKB + TF_TSTATE] ! set tf.tf_psr, tf.tf_pc 2755 sth %o1, [%sp + CC64FSZ + STKB + TF_TT] ! debug 2756 2757 stx %o2, [%sp + CC64FSZ + STKB + TF_PC] 2758 stx %g3, [%sp + CC64FSZ + STKB + TF_NPC] ! set tf.tf_npc 2759 2760 rdpr %pil, %g4 2761 stb %g4, [%sp + CC64FSZ + STKB + TF_PIL] 2762 stb %g4, [%sp + CC64FSZ + STKB + TF_OLDPIL] 2763 2764 rdpr %tl, %g7 2765 dec %g7 2766 movrlz %g7, %g0, %g7 2767 wrpr %g0, %g7, %tl ! Revert to kernel mode 2768 2769 wr %g0, ASI_PRIMARY_NOFAULT, %asi ! Restore default ASI 2770 flushw ! Get rid of any user windows so we don't deadlock 2771 2772 !! In the EMBEDANY memory model %g4 points to the start of the data segment. 2773 !! In our case we need to clear it before calling any C-code 2774 clr %g4 2775 2776 /* Use trap type to see what handler to call */ 2777 cmp %o1, T_INST_ERROR 2778 be,pn %xcc, text_error 2779 st %g5, [%sp + CC64FSZ + STKB + TF_Y] ! set tf.tf_y 2780 2781 wrpr %g0, PSTATE_INTR, %pstate ! reenable interrupts 2782 call _C_LABEL(text_access_fault) ! mem_access_fault(&tf, type, pc, sfsr) 2783 add %sp, CC64FSZ + STKB, %o0 ! (argument: &tf) 2784 text_recover: 2785 wrpr %g0, PSTATE_KERN, %pstate ! disable interrupts 2786 b return_from_trap ! go return 2787 ldx [%sp + CC64FSZ + STKB + TF_TSTATE], %g1 ! Load this for return_from_trap 2788 NOTREACHED 2789 2790 text_error: 2791 wrpr %g0, PSTATE_INTR, %pstate ! reenable interrupts 2792 call _C_LABEL(text_access_error) ! mem_access_fault(&tfm type, sfva [pc], sfsr, 2793 ! afva, afsr); 2794 add %sp, CC64FSZ + STKB, %o0 ! (argument: &tf) 2795 ba text_recover 2796 nop 2797 NOTREACHED 2798 2799 #ifdef SUN4V 2800 2801 /* 2802 * Traps for sun4v. 2803 */ 2804 2805 sun4v_dtsb_miss: 2806 GET_MMFSA %g1 ! MMU Fault status area 2807 add %g1, 0x48, %g3 2808 LDPTRA [%g3] ASI_PHYS_CACHED, %g3 ! Data fault address 2809 add %g1, 0x50, %g6 2810 LDPTRA [%g6] ASI_PHYS_CACHED, %g6 ! Data fault context 2811 2812 GET_CTXBUSY %g4 2813 sllx %g6, 3, %g6 ! Make it into an offset into ctxbusy 2814 LDPTR [%g4 + %g6], %g4 ! Load up our page table. 2815 2816 srax %g3, HOLESHIFT, %g5 ! Check for valid address 2817 brz,pt %g5, 0f ! Should be zero or -1 2818 inc %g5 ! Make -1 -> 0 2819 brnz,pn %g5, sun4v_datatrap ! Error! In hole! 2820 0: 2821 srlx %g3, STSHIFT, %g6 2822 and %g6, STMASK, %g6 ! Index into pm_segs 2823 sll %g6, 3, %g6 2824 add %g4, %g6, %g4 2825 LDPTRA [%g4] ASI_PHYS_CACHED, %g4 ! Load page directory pointer 2826 srlx %g3, PDSHIFT, %g6 2827 and %g6, PDMASK, %g6 2828 sll %g6, 3, %g6 2829 brz,pn %g4, sun4v_datatrap ! NULL entry? check somewhere else 2830 add %g4, %g6, %g4 2831 LDPTRA [%g4] ASI_PHYS_CACHED, %g4 ! Load page table pointer 2832 2833 srlx %g3, PTSHIFT, %g6 ! Convert to ptab offset 2834 and %g6, PTMASK, %g6 2835 sll %g6, 3, %g6 2836 brz,pn %g4, sun4v_datatrap ! NULL entry? check somewhere else 2837 add %g4, %g6, %g6 2838 1: 2839 LDPTRA [%g6] ASI_PHYS_CACHED, %g4 ! Fetch TTE 2840 brgez,pn %g4, sun4v_datatrap ! Entry invalid? Punt 2841 or %g4, SUN4V_TLB_ACCESS, %g7 ! Update the access bit 2842 2843 btst SUN4V_TLB_ACCESS, %g4 ! Need to update access bit? 2844 bne,pt %xcc, 2f 2845 nop 2846 casxa [%g6] ASI_PHYS_CACHED, %g4, %g7 ! and write it out 2847 cmp %g4, %g7 2848 bne,pn %xcc, 1b 2849 or %g4, SUN4V_TLB_ACCESS, %g4 ! Update the access bit 2850 2: 2851 GET_TSB_DMMU %g2 2852 2853 /* Construct TSB tag word. */ 2854 add %g1, 0x50, %g6 2855 LDPTRA [%g6] ASI_PHYS_CACHED, %g6 ! Data fault context 2856 mov %g3, %g1 ! Data fault address 2857 srlx %g1, 22, %g1 ! 63..22 of virt addr 2858 sllx %g6, 48, %g6 ! context_id in 63..48 2859 or %g1, %g6, %g1 ! construct TTE tag 2860 srlx %g3, PTSHIFT, %g3 2861 sethi %hi(_C_LABEL(tsbsize)), %g5 2862 mov 512, %g6 2863 ld [%g5 + %lo(_C_LABEL(tsbsize))], %g5 2864 sllx %g6, %g5, %g5 ! %g5 = 512 << tsbsize = TSBENTS 2865 sub %g5, 1, %g5 ! TSBENTS -> offset 2866 and %g3, %g5, %g3 ! mask out TTE index 2867 sllx %g3, 4, %g3 ! TTE size is 16 bytes 2868 add %g2, %g3, %g2 ! location of TTE in ci_tsb_dmmu 2869 2870 membar #StoreStore 2871 2872 STPTR %g4, [%g2 + 8] ! store TTE data 2873 STPTR %g1, [%g2] ! store TTE tag 2874 2875 retry 2876 NOTREACHED 2877 2878 sun4v_tl1_dtsb_miss: 2879 GET_MMFSA %g1 ! MMU Fault status area 2880 add %g1, 0x48, %g3 2881 LDPTRA [%g3] ASI_PHYS_CACHED, %g3 ! Data fault address 2882 add %g1, 0x50, %g6 2883 LDPTRA [%g6] ASI_PHYS_CACHED, %g6 ! Data fault context 2884 2885 GET_CTXBUSY %g4 2886 sllx %g6, 3, %g6 ! Make it into an offset into ctxbusy 2887 LDPTR [%g4 + %g6], %g4 ! Load up our page table. 2888 2889 srax %g3, HOLESHIFT, %g5 ! Check for valid address 2890 brz,pt %g5, 0f ! Should be zero or -1 2891 inc %g5 ! Make -1 -> 0 2892 brnz,pn %g5, sun4v_tl1_ptbl_miss ! Error! In hole! 2893 0: 2894 srlx %g3, STSHIFT, %g6 2895 and %g6, STMASK, %g6 ! Index into pm_segs 2896 sll %g6, 3, %g6 2897 add %g4, %g6, %g4 2898 LDPTRA [%g4] ASI_PHYS_CACHED, %g4 ! Load page directory pointer 2899 srlx %g3, PDSHIFT, %g6 2900 and %g6, PDMASK, %g6 2901 sll %g6, 3, %g6 2902 brz,pn %g4, sun4v_tl1_ptbl_miss ! NULL entry? check somewhere else 2903 add %g4, %g6, %g4 2904 LDPTRA [%g4] ASI_PHYS_CACHED, %g4 ! Load page table pointer 2905 2906 srlx %g3, PTSHIFT, %g6 ! Convert to ptab offset 2907 and %g6, PTMASK, %g6 2908 sll %g6, 3, %g6 2909 brz,pn %g4, sun4v_tl1_ptbl_miss ! NULL entry? check somewhere else 2910 add %g4, %g6, %g6 2911 1: 2912 LDPTRA [%g6] ASI_PHYS_CACHED, %g4 ! Fetch TTE 2913 brgez,pn %g4, sun4v_tl1_ptbl_miss ! Entry invalid? Punt 2914 or %g4, SUN4V_TLB_ACCESS, %g7 ! Update the access bit 2915 2916 btst SUN4V_TLB_ACCESS, %g4 ! Need to update access bit? 2917 bne,pt %xcc, 2f 2918 nop 2919 casxa [%g6] ASI_PHYS_CACHED, %g4, %g7 ! and write it out 2920 cmp %g4, %g7 2921 bne,pn %xcc, 1b 2922 or %g4, SUN4V_TLB_ACCESS, %g4 ! Update the access bit 2923 2: 2924 GET_TSB_DMMU %g2 2925 2926 /* Construct TSB tag word. */ 2927 add %g1, 0x50, %g6 2928 LDPTRA [%g6] ASI_PHYS_CACHED, %g6 ! Data fault context 2929 mov %g3, %g1 ! Data fault address 2930 srlx %g1, 22, %g1 ! 63..22 of virt addr 2931 sllx %g6, 48, %g6 ! context_id in 63..48 2932 or %g1, %g6, %g1 ! construct TTE tag 2933 srlx %g3, PTSHIFT, %g3 2934 sethi %hi(_C_LABEL(tsbsize)), %g5 2935 mov 512, %g6 2936 ld [%g5 + %lo(_C_LABEL(tsbsize))], %g5 2937 sllx %g6, %g5, %g5 ! %g5 = 512 << tsbsize = TSBENTS 2938 sub %g5, 1, %g5 ! TSBENTS -> offset 2939 and %g3, %g5, %g3 ! mask out TTE index 2940 sllx %g3, 4, %g3 ! TTE size is 16 bytes 2941 add %g2, %g3, %g2 ! location of TTE in ci_tsb_dmmu 2942 2943 membar #StoreStore 2944 2945 STPTR %g4, [%g2 + 8] ! store TTE data 2946 STPTR %g1, [%g2] ! store TTE tag 2947 2948 retry 2949 NOTREACHED 2950 2951 sun4v_datatrap: 2952 GET_MMFSA %g3 ! MMU Fault status area 2953 add %g3, 0x48, %g1 2954 LDPTRA [%g1] ASI_PHYS_CACHED, %g1 ! Data fault address 2955 add %g3, 0x50, %g2 2956 LDPTRA [%g2] ASI_PHYS_CACHED, %g2 ! Data fault context 2957 2958 TRAP_SETUP(-CC64FSZ-TF_SIZE) 2959 or %g1, %g2, %o3 2960 mov %g1, %o4 2961 2962 rdpr %tt, %g4 2963 rdpr %tstate, %g1 2964 rdpr %tpc, %g2 2965 rdpr %tnpc, %g3 2966 2967 stx %g1, [%sp + CC64FSZ + BIAS + TF_TSTATE] 2968 mov %g4, %o1 ! (type) 2969 stx %g2, [%sp + CC64FSZ + BIAS + TF_PC] 2970 rd %y, %g5 2971 stx %g3, [%sp + CC64FSZ + BIAS + TF_NPC] 2972 st %g5, [%sp + CC64FSZ + BIAS + TF_Y] 2973 mov %g2, %o2 ! (pc) 2974 sth %o1, [%sp + CC64FSZ + BIAS + TF_TT]! debug 2975 2976 cmp %o1, T_FDMMU_PROT 2977 bne,pn %icc, 1f 2978 mov SFSR_FV, %o5 2979 or %o5, SFSR_W, %o5 2980 2981 1: 2982 NORMAL_GLOBALS_SUN4V 2983 2984 stx %g1, [%sp + CC64FSZ + BIAS + TF_G + (1*8)] 2985 stx %g2, [%sp + CC64FSZ + BIAS + TF_G + (2*8)] 2986 add %sp, CC64FSZ + BIAS, %o0 ! (&tf) 2987 stx %g3, [%sp + CC64FSZ + BIAS + TF_G + (3*8)] 2988 stx %g4, [%sp + CC64FSZ + BIAS + TF_G + (4*8)] 2989 stx %g5, [%sp + CC64FSZ + BIAS + TF_G + (5*8)] 2990 rdpr %pil, %g5 2991 stx %g6, [%sp + CC64FSZ + BIAS + TF_G + (6*8)] 2992 stx %g7, [%sp + CC64FSZ + BIAS + TF_G + (7*8)] 2993 stb %g5, [%sp + CC64FSZ + BIAS + TF_PIL] 2994 stb %g5, [%sp + CC64FSZ + BIAS + TF_OLDPIL] 2995 2996 /* 2997 * Phew, ready to enable traps and call C code. 2998 */ 2999 wrpr %g0, 0, %tl 3000 3001 wr %g0, ASI_PRIMARY_NOFAULT, %asi ! Restore default ASI 3002 wrpr %g0, PSTATE_INTR, %pstate ! traps on again 3003 call _C_LABEL(data_access_fault) ! data_acces_fault(tf, type, ...) 3004 nop 3005 3006 ba,a,pt %icc, return_from_trap 3007 nop 3008 NOTREACHED 3009 3010 sun4v_tl0_dtsb_prot: 3011 GET_MMFSA %g1 ! MMU Fault status area 3012 add %g1, 0x48, %g3 3013 LDPTRA [%g3] ASI_PHYS_CACHED, %g3 ! Data fault address 3014 add %g1, 0x50, %g6 3015 LDPTRA [%g6] ASI_PHYS_CACHED, %g6 ! Data fault context 3016 3017 GET_CTXBUSY %g4 3018 sllx %g6, 3, %g6 ! Make it into an offset into ctxbusy 3019 LDPTR [%g4 + %g6], %g4 ! Load up our page table. 3020 3021 srax %g3, HOLESHIFT, %g5 ! Check for valid address 3022 brz,pt %g5, 0f ! Should be zero or -1 3023 inc %g5 ! Make -1 -> 0 3024 brnz,pn %g5, sun4v_datatrap ! Error! In hole! 3025 0: 3026 srlx %g3, STSHIFT, %g6 3027 and %g6, STMASK, %g6 ! Index into pm_segs 3028 sll %g6, 3, %g6 3029 add %g4, %g6, %g4 3030 LDPTRA [%g4] ASI_PHYS_CACHED, %g4 ! Load page directory pointer 3031 3032 srlx %g3, PDSHIFT, %g6 3033 and %g6, PDMASK, %g6 3034 sll %g6, 3, %g6 3035 brz,pn %g4, sun4v_datatrap ! NULL entry? check somewhere else 3036 add %g4, %g6, %g4 3037 LDPTRA [%g4] ASI_PHYS_CACHED, %g4 ! Load page table pointer 3038 3039 srlx %g3, PTSHIFT, %g6 ! Convert to ptab offset 3040 and %g6, PTMASK, %g6 3041 sll %g6, 3, %g6 3042 brz,pn %g4, sun4v_datatrap ! NULL entry? check somewhere else 3043 add %g4, %g6, %g6 3044 1: 3045 LDPTRA [%g6] ASI_PHYS_CACHED, %g4 ! Fetch TTE 3046 brgez,pn %g4, sun4v_datatrap ! Entry invalid? Punt 3047 or %g4, SUN4V_TLB_MODIFY|SUN4V_TLB_ACCESS|SUN4V_TLB_W, %g7 ! Update the modified bit 3048 3049 # btst SUN4V_TLB_REAL_W|SUN4V_TLB_W, %g4 ! Is it a ref fault? 3050 mov 1, %g2 3051 sllx %g2, 61, %g2 ! %g2 is now SUN4V_TLB_REAL_W 3052 or %g2, SUN4V_TLB_W, %g2 3053 btst %g2, %g4 3054 bz,pn %xcc, sun4v_datatrap ! No -- really fault 3055 nop 3056 casxa [%g6] ASI_PHYS_CACHED, %g4, %g7 ! and write it out 3057 cmp %g4, %g7 3058 bne,pn %xcc, 1b 3059 or %g4, SUN4V_TLB_MODIFY|SUN4V_TLB_ACCESS|SUN4V_TLB_W, %g4 ! Update the modified bit 3060 2: 3061 GET_TSB_DMMU %g2 3062 3063 mov %g1, %g7 ! save MMFSA 3064 3065 /* Construct TSB tag word. */ 3066 add %g1, 0x50, %g6 3067 LDPTRA [%g6] ASI_PHYS_CACHED, %g6 ! Data fault context 3068 mov %g3, %g1 ! Data fault address 3069 srlx %g1, 22, %g1 ! 63..22 of virt addr 3070 sllx %g6, 48, %g6 ! context_id in 63..48 3071 or %g1, %g6, %g1 ! construct TTE tag 3072 3073 srlx %g3, PTSHIFT, %g3 3074 sethi %hi(_C_LABEL(tsbsize)), %g5 3075 mov 512, %g6 3076 ld [%g5 + %lo(_C_LABEL(tsbsize))], %g5 3077 sllx %g6, %g5, %g5 ! %g5 = 512 << tsbsize = TSBENTS 3078 sub %g5, 1, %g5 ! TSBENTS -> offset 3079 and %g3, %g5, %g3 ! mask out TTE index 3080 sllx %g3, 4, %g3 ! TTE size is 16 bytes 3081 add %g2, %g3, %g2 ! location of TTE in ci_tsb_dmmu 3082 3083 membar #StoreStore 3084 3085 STPTR %g4, [%g2 + 8] ! store TTE data 3086 STPTR %g1, [%g2] ! store TTE tag 3087 3088 mov %o0, %g1 3089 mov %o1, %g2 3090 mov %o2, %g3 3091 3092 add %g7, 0x48, %o0 3093 ldxa [%o0] ASI_PHYS_CACHED, %o0 ! Data fault address 3094 add %g7, 0x50, %o1 3095 ldxa [%o1] ASI_PHYS_CACHED, %o1 ! Data fault context 3096 mov MAP_DTLB, %o2 3097 ta ST_MMU_UNMAP_ADDR 3098 3099 mov %g1, %o0 3100 mov %g2, %o1 3101 mov %g3, %o2 3102 3103 retry 3104 NOTREACHED 3105 3106 sun4v_tl0_itsb_miss: 3107 GET_MMFSA %g1 ! MMU Fault status area 3108 add %g1, 0x8, %g3 3109 LDPTRA [%g3] ASI_PHYS_CACHED, %g3 ! Instruction fault address 3110 add %g1, 0x10, %g6 3111 LDPTRA [%g6] ASI_PHYS_CACHED, %g6 ! Data fault context 3112 3113 GET_CTXBUSY %g4 3114 sllx %g6, 3, %g6 ! Make it into an offset into ctxbusy 3115 LDPTR [%g4 + %g6], %g4 ! Load up our page table. 3116 3117 srax %g3, HOLESHIFT, %g5 ! Check for valid address 3118 brz,pt %g5, 0f ! Should be zero or -1 3119 inc %g5 ! Make -1 -> 0 3120 brnz,pn %g5, sun4v_texttrap ! Error! In hole! 3121 0: 3122 srlx %g3, STSHIFT, %g6 3123 and %g6, STMASK, %g6 ! Index into pm_segs 3124 sll %g6, 3, %g6 3125 add %g4, %g6, %g4 3126 LDPTRA [%g4] ASI_PHYS_CACHED, %g4 ! Load page directory pointer 3127 3128 srlx %g3, PDSHIFT, %g6 3129 and %g6, PDMASK, %g6 3130 sll %g6, 3, %g6 3131 brz,pn %g4, sun4v_texttrap ! NULL entry? check somewhere else 3132 add %g4, %g6, %g4 3133 LDPTRA [%g4] ASI_PHYS_CACHED, %g4 ! Load page table pointer 3134 3135 srlx %g3, PTSHIFT, %g6 ! Convert to ptab offset 3136 and %g6, PTMASK, %g6 3137 sll %g6, 3, %g6 3138 brz,pn %g4, sun4v_texttrap ! NULL entry? check somewhere else 3139 add %g4, %g6, %g6 3140 1: 3141 LDPTRA [%g6] ASI_PHYS_CACHED, %g4 ! Fetch TTE 3142 brgez,pn %g4, sun4v_texttrap ! Entry invalid? Punt 3143 or %g4, SUN4V_TLB_ACCESS, %g7 ! Update the access bit 3144 3145 btst SUN4V_TLB_EXEC, %g4 ! Need to update exec bit? 3146 bz,pn %xcc, sun4v_texttrap 3147 nop 3148 btst SUN4V_TLB_ACCESS, %g4 ! Need to update access bit? 3149 bne,pt %xcc, 2f 3150 nop 3151 casxa [%g6] ASI_PHYS_CACHED, %g4, %g7 ! and write it out 3152 cmp %g4, %g7 3153 bne,pn %xcc, 1b 3154 or %g4, SUN4V_TLB_ACCESS, %g4 ! Update the modified bit 3155 2: 3156 GET_TSB_DMMU %g2 3157 3158 mov %g1, %g7 3159 /* Construct TSB tag word. */ 3160 add %g1, 0x10, %g6 3161 LDPTRA [%g6] ASI_PHYS_CACHED, %g6 ! Instruction fault context 3162 mov %g3, %g1 ! Instruction fault address 3163 srlx %g1, 22, %g1 ! 63..22 of virt addr 3164 sllx %g6, 48, %g6 ! context_id in 63..48 3165 or %g1, %g6, %g1 ! construct TTE tag 3166 3167 srlx %g3, PTSHIFT, %g3 3168 sethi %hi(_C_LABEL(tsbsize)), %g5 3169 mov 512, %g6 3170 ld [%g5 + %lo(_C_LABEL(tsbsize))], %g5 3171 sllx %g6, %g5, %g5 ! %g5 = 512 << tsbsize = TSBENTS 3172 sub %g5, 1, %g5 ! TSBENTS -> offset 3173 and %g3, %g5, %g3 ! mask out TTE index 3174 sllx %g3, 4, %g3 ! TTE size is 16 bytes 3175 add %g2, %g3, %g2 ! location of TTE in ci_tsb_dmmu (FIXME ci_tsb_immu?) 3176 3177 membar #StoreStore 3178 STPTR %g4, [%g2 + 8] ! store TTE data 3179 stx %g1, [%g2] ! store TTE tag 3180 3181 retry 3182 NOTREACHED 3183 3184 sun4v_texttrap: 3185 GET_MMFSA %g3 ! MMU Fault status area 3186 add %g3, 0x08, %g1 3187 LDPTRA [%g1] ASI_PHYS_CACHED, %g1 ! Instruction fault address 3188 add %g3, 0x10, %g2 3189 LDPTRA [%g2] ASI_PHYS_CACHED, %g2 ! Instruction fault context 3190 3191 TRAP_SETUP(-CC64FSZ-TF_SIZE) 3192 3193 or %g1, %g2, %o2 3194 clr %o3 3195 3196 rdpr %tt, %g4 3197 rdpr %tstate, %g1 3198 rdpr %tpc, %g2 3199 rdpr %tnpc, %g3 3200 3201 stx %g1, [%sp + CC64FSZ + BIAS + TF_TSTATE] 3202 mov %g4, %o1 ! (type) 3203 stx %g2, [%sp + CC64FSZ + BIAS + TF_PC] 3204 rd %y, %g5 3205 stx %g3, [%sp + CC64FSZ + BIAS + TF_NPC] 3206 st %g5, [%sp + CC64FSZ + BIAS + TF_Y] 3207 sth %o1, [%sp + CC64FSZ + BIAS + TF_TT]! debug 3208 3209 ! Get back to normal globals 3210 wrpr %g0, PSTATE_KERN, %pstate 3211 NORMAL_GLOBALS_SUN4V 3212 3213 stx %g1, [%sp + CC64FSZ + BIAS + TF_G + (1*8)] 3214 stx %g2, [%sp + CC64FSZ + BIAS + TF_G + (2*8)] 3215 add %sp, CC64FSZ + BIAS, %o0 ! (&tf) 3216 stx %g3, [%sp + CC64FSZ + BIAS + TF_G + (3*8)] 3217 stx %g4, [%sp + CC64FSZ + BIAS + TF_G + (4*8)] 3218 stx %g5, [%sp + CC64FSZ + BIAS + TF_G + (5*8)] 3219 rdpr %pil, %g5 3220 stx %g6, [%sp + CC64FSZ + BIAS + TF_G + (6*8)] 3221 stx %g7, [%sp + CC64FSZ + BIAS + TF_G + (7*8)] 3222 stb %g5, [%sp + CC64FSZ + BIAS + TF_PIL] 3223 stb %g5, [%sp + CC64FSZ + BIAS + TF_OLDPIL] 3224 3225 /* 3226 * Phew, ready to enable traps and call C code. 3227 */ 3228 wrpr %g0, 0, %tl 3229 3230 wr %g0, ASI_PRIMARY_NOFAULT, %asi ! Restore default ASI 3231 wrpr %g0, PSTATE_INTR, %pstate ! traps on again 3232 call _C_LABEL(text_access_fault) ! text_access_fault(tf, type, ...) 3233 nop 3234 3235 ba,a,pt %icc, return_from_trap 3236 nop 3237 NOTREACHED 3238 3239 sun4v_tl1_dtsb_prot: 3240 GET_MMFSA %g1 ! MMU Fault status area 3241 add %g1, 0x48, %g3 3242 LDPTRA [%g3] ASI_PHYS_CACHED, %g3 ! Data fault address 3243 add %g1, 0x50, %g6 3244 LDPTRA [%g6] ASI_PHYS_CACHED, %g6 ! Data fault context 3245 3246 GET_CTXBUSY %g4 3247 sllx %g6, 3, %g6 ! Make it into an offset into ctxbusy 3248 LDPTR [%g4 + %g6], %g4 ! Load up our page table. 3249 3250 srax %g3, HOLESHIFT, %g5 ! Check for valid address 3251 brz,pt %g5, 0f ! Should be zero or -1 3252 inc %g5 ! Make -1 -> 0 3253 brnz,pn %g5, sun4v_tl1_ptbl_miss ! Error! In hole! 3254 0: 3255 srlx %g3, STSHIFT, %g6 3256 and %g6, STMASK, %g6 ! Index into pm_segs 3257 sll %g6, 3, %g6 3258 add %g4, %g6, %g4 3259 LDPTRA [%g4] ASI_PHYS_CACHED, %g4 ! Load page directory pointer 3260 3261 srlx %g3, PDSHIFT, %g6 3262 and %g6, PDMASK, %g6 3263 sll %g6, 3, %g6 3264 brz,pn %g4, sun4v_tl1_ptbl_miss ! NULL entry? check somewhere else 3265 add %g4, %g6, %g4 3266 LDPTRA [%g4] ASI_PHYS_CACHED, %g4 ! Load page table pointer 3267 3268 srlx %g3, PTSHIFT, %g6 ! Convert to ptab offset 3269 and %g6, PTMASK, %g6 3270 sll %g6, 3, %g6 3271 brz,pn %g4, sun4v_tl1_ptbl_miss ! NULL entry? check somewhere else 3272 add %g4, %g6, %g6 3273 1: 3274 LDPTRA [%g6] ASI_PHYS_CACHED, %g4 ! Fetch TTE 3275 brgez,pn %g4, sun4v_tl1_ptbl_miss ! Entry invalid? Punt 3276 or %g4, SUN4V_TLB_MODIFY|SUN4V_TLB_ACCESS|SUN4V_TLB_W, %g7 ! Update the modified bit 3277 3278 # btst SUN4V_TLB_REAL_W|SUN4V_TLB_W, %g4 ! Is it a ref fault? 3279 mov 1, %g2 3280 sllx %g2, 61, %g2 ! %g2 is now SUN4V_TLB_REAL_W 3281 or %g2, SUN4V_TLB_W, %g2 3282 btst %g2, %g4 3283 bz,pn %xcc, sun4v_tl1_ptbl_miss ! No -- really fault 3284 nop 3285 casxa [%g6] ASI_PHYS_CACHED, %g4, %g7 ! and write it out 3286 cmp %g4, %g7 3287 bne,pn %xcc, 1b 3288 or %g4, SUN4V_TLB_MODIFY|SUN4V_TLB_ACCESS|SUN4V_TLB_W, %g4 ! Update the modified bit 3289 2: 3290 GET_TSB_DMMU %g2 3291 3292 mov %g1, %g7 ! save MMFSA 3293 3294 /* Construct TSB tag word. */ 3295 add %g1, 0x50, %g6 3296 LDPTRA [%g6] ASI_PHYS_CACHED, %g6 ! Data fault context 3297 mov %g3, %g1 ! Data fault address 3298 srlx %g1, 22, %g1 ! 63..22 of virt addr 3299 sllx %g6, 48, %g6 ! context_id in 63..48 3300 or %g1, %g6, %g1 ! construct TTE tag 3301 3302 srlx %g3, PTSHIFT, %g3 3303 sethi %hi(_C_LABEL(tsbsize)), %g5 3304 mov 512, %g6 3305 ld [%g5 + %lo(_C_LABEL(tsbsize))], %g5 3306 sllx %g6, %g5, %g5 ! %g5 = 512 << tsbsize = TSBENTS 3307 sub %g5, 1, %g5 ! TSBENTS -> offset 3308 and %g3, %g5, %g3 ! mask out TTE index 3309 sllx %g3, 4, %g3 ! TTE size is 16 bytes 3310 add %g2, %g3, %g2 ! location of TTE in ci_tsb_dmmu 3311 3312 membar #StoreStore 3313 3314 STPTR %g4, [%g2 + 8] ! store TTE data 3315 STPTR %g1, [%g2] ! store TTE tag 3316 3317 mov %o0, %g1 3318 mov %o1, %g2 3319 mov %o2, %g3 3320 3321 add %g7, 0x48, %o0 3322 ldxa [%o0] ASI_PHYS_CACHED, %o0 ! Data fault address 3323 add %g7, 0x50, %o1 3324 ldxa [%o1] ASI_PHYS_CACHED, %o1 ! Data fault context 3325 mov MAP_DTLB, %o2 3326 ta ST_MMU_UNMAP_ADDR 3327 3328 mov %g1, %o0 3329 mov %g2, %o1 3330 mov %g3, %o2 3331 3332 retry 3333 NOTREACHED 3334 3335 sun4v_tl1_ptbl_miss: 3336 rdpr %tpc, %g1 3337 3338 set rft_user_fault_start, %g2 3339 cmp %g1, %g2 3340 blu,pt %xcc, 1f 3341 set rft_user_fault_end, %g2 3342 cmp %g1, %g2 3343 bgeu,pt %xcc, 1f 3344 nop 3345 3346 /* We had a miss inside rtf_user_fault_start/rtf_user_fault_end block (FILL) 3347 3348 /* Fixup %cwp. */ 3349 rdpr %cwp, %g1 3350 inc %g1 3351 wrpr %g1, %cwp 3352 3353 rdpr %tt, %g1 3354 wrpr 1, %tl 3355 wrpr %g1, %tt 3356 rdpr %cwp, %g1 3357 set TSTATE_KERN, %g2 3358 wrpr %g1, %g2, %tstate 3359 set return_from_trap, %g1 3360 wrpr %g1, %tpc 3361 add %g1, 4, %g1 3362 wrpr %g1, %tnpc 3363 wrpr %g0, 1, %gl 3364 3365 ba,pt %xcc, sun4v_datatrap 3366 wrpr WSTATE_KERN, %wstate 3367 3368 1: 3369 rdpr %tstate, %g3 3370 rdpr %tt, %g4 3371 3372 rdpr %tl, %g1 3373 dec %g1 3374 wrpr %g1, %tl 3375 rdpr %tt, %g2 3376 inc %g1 3377 wrpr %g1, %tl 3378 3379 wrpr %g0, %g3, %tstate 3380 wrpr %g0, %g4, %tt 3381 3382 andn %g2, 0x00f, %g3 3383 cmp %g3, 0x080 3384 be,pn %icc, flush_normals 3385 nop 3386 cmp %g3, 0x0a0 3387 be,pn %icc, flush_others 3388 nop 3389 cmp %g3, 0x0c0 3390 be,pn %icc, ufill_trap 3391 nop 3392 3393 Debugger() 3394 NOTREACHED 3395 3396 flush_others: 3397 set pcbspill_others, %g1 3398 wrpr %g1, %tnpc 3399 done 3400 NOTREACHED 3401 3402 flush_normals: 3403 ufill_trap: 3404 3405 /* 3406 * Rearrange our trap state such that it appears as if we got 3407 * this trap directly from user mode. Then process it at TL = 1. 3408 * We'll take the spill/fill trap again once we return to user mode. 3409 */ 3410 rdpr %tt, %g1 3411 rdpr %tstate, %g3 3412 wrpr %g0, 1, %tl 3413 wrpr %g0, %g1, %tt 3414 rdpr %tstate, %g2 3415 wrpr %g0, 2, %tl 3416 and %g2, TSTATE_CWP, %g2 3417 andn %g3, TSTATE_CWP, %g3 3418 wrpr %g2, %g3, %tstate 3419 set sun4v_datatrap, %g4 3420 wrpr %g0, %g4, %tnpc 3421 done 3422 3423 /* 3424 * Spill user windows into the PCB. 3425 */ 3426 pcbspill_normals: 3427 ba,pt %xcc, pcbspill 3428 wrpr 0x80, %tt 3429 3430 pcbspill_others: 3431 wrpr 0xa0, %tt 3432 3433 pcbspill: 3434 set CPUINFO_VA, %g6 3435 ldx [%g6 + CI_CPCB], %g6 3436 3437 GET_CTXBUSY %g1 3438 3439 ldx [%g1], %g1 ! kernel pmap is ctx 0 3440 3441 srlx %g6, STSHIFT, %g7 3442 and %g7, STMASK, %g7 3443 sll %g7, 3, %g7 ! byte offset into ctxbusy 3444 add %g7, %g1, %g1 3445 ldxa [%g1] ASI_PHYS_CACHED, %g1 ! Load pointer to directory 3446 3447 srlx %g6, PDSHIFT, %g7 ! Do page directory 3448 and %g7, PDMASK, %g7 3449 sll %g7, 3, %g7 3450 brz,pn %g1, pcbspill_fail 3451 add %g7, %g1, %g1 3452 ldxa [%g1] ASI_PHYS_CACHED, %g1 3453 srlx %g6, PTSHIFT, %g7 ! Convert to ptab offset 3454 and %g7, PTMASK, %g7 3455 brz %g1, pcbspill_fail 3456 sll %g7, 3, %g7 3457 add %g1, %g7, %g7 3458 ldxa [%g7] ASI_PHYS_CACHED, %g7 ! This one is not 3459 brgez %g7, pcbspill_fail 3460 srlx %g7, PGSHIFT, %g7 ! Isolate PA part 3461 sll %g6, 32-PGSHIFT, %g6 ! And offset 3462 sllx %g7, PGSHIFT+8, %g7 ! There are 8 bits to the left of the PA in the TTE 3463 srl %g6, 32-PGSHIFT, %g6 3464 srax %g7, 8, %g7 3465 or %g7, %g6, %g6 ! Then combine them to form PA 3466 3467 wr %g0, ASI_PHYS_CACHED, %asi ! Use ASI_PHYS_CACHED to prevent possible page faults 3468 3469 lduba [%g6 + PCB_NSAVED] %asi, %g7 ! Fetch current nsaved from the pcb 3470 sllx %g7, 7, %g5 ! 8+8 registers each 8 bytes = 128 bytes (2^7) 3471 add %g6, %g5, %g5 ! Offset into pcb_rw 3472 SPILL stxa, %g5 + PCB_RW, 8, %asi ! Store the locals and ins 3473 saved 3474 3475 sllx %g7, 3, %g5 3476 add %g6, %g5, %g5 3477 3478 inc %g7 3479 stba %g7, [%g6 + PCB_NSAVED] %asi 3480 3481 retry 3482 NOTREACHED 3483 3484 pcbspill_fail: 3485 Debugger() 3486 NOTREACHED 3487 3488 3489 pcbspill_other: 3490 3491 set CPUINFO_VA, %g6 3492 ldx [%g6 + CI_CPCB], %g6 3493 3494 GET_CTXBUSY %g1 3495 3496 ldx [%g1], %g1 ! kernel pmap is ctx 0 3497 3498 srlx %g6, STSHIFT, %g7 3499 and %g7, STMASK, %g7 3500 sll %g7, 3, %g7 ! byte offset into ctxbusy 3501 add %g7, %g1, %g1 3502 ldxa [%g1] ASI_PHYS_CACHED, %g1 ! Load pointer to directory 3503 3504 srlx %g6, PDSHIFT, %g7 ! Do page directory 3505 and %g7, PDMASK, %g7 3506 sll %g7, 3, %g7 3507 brz,pn %g1, pcbspill_other_fail 3508 add %g7, %g1, %g1 3509 ldxa [%g1] ASI_PHYS_CACHED, %g1 3510 srlx %g6, PTSHIFT, %g7 ! Convert to ptab offset 3511 and %g7, PTMASK, %g7 3512 brz %g1, pcbspill_other_fail 3513 sll %g7, 3, %g7 3514 add %g1, %g7, %g7 3515 ldxa [%g7] ASI_PHYS_CACHED, %g7 ! This one is not 3516 brgez %g7, pcbspill_other_fail 3517 srlx %g7, PGSHIFT, %g7 ! Isolate PA part 3518 sll %g6, 32-PGSHIFT, %g6 ! And offset 3519 sllx %g7, PGSHIFT+8, %g7 ! There are 8 bits to the left of the PA in the TTE 3520 srl %g6, 32-PGSHIFT, %g6 3521 srax %g7, 8, %g7 3522 or %g7, %g6, %g6 ! Then combine them to form PA 3523 3524 wr %g0, ASI_PHYS_CACHED, %asi ! Use ASI_PHYS_CACHED to prevent possible page faults 3525 3526 lduba [%g6 + PCB_NSAVED] %asi, %g7 ! Fetch current nsaved from the pcb 3527 sllx %g7, 7, %g5 ! 8+8 registers each 8 bytes = 128 bytes (2^7) 3528 add %g6, %g5, %g5 ! Offset into pcb_rw 3529 1: 3530 SPILL stxa, %g5 + PCB_RW, 8, %asi ! Store the locals and ins 3531 3532 add %g5, 16*8, %g5 ! Next location for saved register windows 3533 3534 stxa %o6, [%g5 + PCB_RW + (14*8)] %asi ! Save %sp so we can write these all out 3535 3536 saved ! Increments %cansave and decrements %otherwin 3537 3538 rdpr %cwp, %g1 ! shift register window forward 3539 inc %g1 3540 wrpr %g1, %cwp 3541 3542 3543 inc %g7 ! increment number of saved register windows 3544 3545 rdpr %otherwin, %g1 ! Check to see if done spill'ing otherwin 3546 brnz,pt %g1, 1b 3547 nop 3548 3549 stba %g7, [%g6 + PCB_NSAVED] %asi 3550 3551 retry 3552 NOTREACHED 3553 3554 pcbspill_other_fail: 3555 Debugger() 3556 NOTREACHED 3557 3558 3559 spill_normal_to_user_stack: 3560 mov %sp, %g6 ! calculate virtual address of destination stack 3561 add %g6, BIAS, %g6 3562 3563 mov CTX_SECONDARY, %g2 ! Is this context ok or should it be CTX_PRIMARY? XXX 3564 GET_MMU_CONTEXTID %g3, %g2, %g1 3565 sllx %g3, 3, %g3 ! Make it into an offset into ctxbusy (see below) 3566 3567 GET_CTXBUSY %g1 3568 ldx [%g1 + %g3], %g1 ! Fetch pmap for current context id 3569 3570 ! Start of code to extract PA 3571 srlx %g6, STSHIFT, %g7 3572 and %g7, STMASK, %g7 3573 sll %g7, 3, %g7 ! byte offset into ctxbusy 3574 add %g7, %g1, %g1 3575 ldxa [%g1] ASI_PHYS_CACHED, %g1 ! Load pointer to directory 3576 srlx %g6, PDSHIFT, %g7 ! Do page directory 3577 and %g7, PDMASK, %g7 3578 sll %g7, 3, %g7 3579 brz,pn %g1, spill_normal_to_user_stack_fail 3580 add %g7, %g1, %g1 3581 3582 ldxa [%g1] ASI_PHYS_CACHED, %g1 3583 srlx %g6, PTSHIFT, %g7 ! Convert to ptab offset 3584 and %g7, PTMASK, %g7 3585 brz %g1, spill_normal_to_user_stack_fail 3586 sll %g7, 3, %g7 3587 3588 add %g1, %g7, %g7 3589 ldxa [%g7] ASI_PHYS_CACHED, %g7 ! This one is not 3590 brgez %g7, spill_normal_to_user_stack_fail 3591 srlx %g7, PGSHIFT, %g7 ! Isolate PA part 3592 3593 sll %g6, 32-PGSHIFT, %g6 ! And offset 3594 sllx %g7, PGSHIFT+8, %g7 ! There are 8 bits to the left of the PA in the TTE 3595 srl %g6, 32-PGSHIFT, %g6 3596 srax %g7, 8, %g7 3597 or %g7, %g6, %g6 ! Then combine them to form PA 3598 ! End of code to extract PA 3599 3600 wr %g0, ASI_PHYS_CACHED, %asi ! Use ASI_PHYS_CACHED to prevent possible page faults 3601 SPILL stxa, %g6, 8, %asi ! Store the locals and ins 3602 saved 3603 3604 retry 3605 NOTREACHED 3606 3607 spill_normal_to_user_stack_fail: 3608 sir 3609 nop 3610 3611 /* 3612 * End of traps for sun4v. 3613 */ 3614 3615 #endif 3616 3617 /* 3618 * We're here because we took an alignment fault in NUCLEUS context. 3619 * This could be a kernel bug or it could be due to saving a user 3620 * window to an invalid stack pointer. 3621 * 3622 * If the latter is the case, we could try to emulate unaligned accesses, 3623 * but we really don't know where to store the registers since we can't 3624 * determine if there's a stack bias. Or we could store all the regs 3625 * into the PCB and punt, until the user program uses up all the CPU's 3626 * register windows and we run out of places to store them. So for 3627 * simplicity we'll just blow them away and enter the trap code which 3628 * will generate a bus error. Debugging the problem will be a bit 3629 * complicated since lots of register windows will be lost, but what 3630 * can we do? 3631 */ 3632 checkalign: 3633 rdpr %tl, %g2 3634 subcc %g2, 1, %g1 3635 bneg,pn %icc, slowtrap ! Huh? 3636 sethi %hi(CPCB), %g6 ! get current pcb 3637 3638 wrpr %g1, 0, %tl 3639 rdpr %tt, %g7 3640 rdpr %tstate, %g4 3641 andn %g7, 0x3f, %g5 3642 cmp %g5, 0x080 ! window spill traps are all 0b 0000 10xx xxxx 3643 bne,a,pn %icc, slowtrap 3644 wrpr %g1, 0, %tl ! Revert TL XXX wrpr in a delay slot... 3645 3646 #ifdef DEBUG 3647 cmp %g7, 0x34 ! If we took a datafault just before this trap 3648 bne,pt %icc, checkalignspill ! our stack's probably bad so we need to switch somewhere else 3649 nop 3650 3651 !! 3652 !! Double data fault -- bad stack? 3653 !! 3654 wrpr %g2, %tl ! Restore trap level. 3655 sir ! Just issue a reset and don't try to recover. 3656 mov %fp, %l6 ! Save the frame pointer 3657 set EINTSTACK+USPACE+CC64FSZ-STKB, %fp ! Set the frame pointer to the middle of the idle stack 3658 add %fp, -CC64FSZ, %sp ! Create a stackframe 3659 wrpr %g0, 15, %pil ! Disable interrupts, too 3660 wrpr %g0, %g0, %canrestore ! Our stack is hozed and our PCB 3661 wrpr %g0, 7, %cansave ! probably is too, so blow away 3662 ba slowtrap ! all our register windows. 3663 wrpr %g0, 0x101, %tt 3664 #endif 3665 checkalignspill: 3666 /* 3667 * %g1 -- current tl 3668 * %g2 -- original tl 3669 * %g4 -- tstate 3670 * %g7 -- tt 3671 */ 3672 3673 and %g4, CWP, %g5 3674 wrpr %g5, %cwp ! Go back to the original register win 3675 3676 /* 3677 * Remember: 3678 * 3679 * %otherwin = 0 3680 * %cansave = NWINDOWS - 2 - %canrestore 3681 */ 3682 3683 rdpr %otherwin, %g6 3684 rdpr %canrestore, %g3 3685 rdpr %ver, %g5 3686 sub %g3, %g6, %g3 ! Calculate %canrestore - %g7 3687 and %g5, CWP, %g5 ! NWINDOWS-1 3688 movrlz %g3, %g0, %g3 ! Clamp at zero 3689 wrpr %g0, 0, %otherwin 3690 wrpr %g3, 0, %canrestore ! This is the new canrestore 3691 dec %g5 ! NWINDOWS-2 3692 wrpr %g5, 0, %cleanwin ! Set cleanwin to max, since we're in-kernel 3693 sub %g5, %g3, %g5 ! NWINDOWS-2-%canrestore 3694 wrpr %g5, 0, %cansave 3695 3696 wrpr %g0, T_ALIGN, %tt ! This was an alignment fault 3697 /* 3698 * Now we need to determine if this was a userland store or not. 3699 * Userland stores occur in anything other than the kernel spill 3700 * handlers (trap type 09x). 3701 */ 3702 and %g7, 0xff0, %g5 3703 cmp %g5, 0x90 3704 bz,pn %icc, slowtrap 3705 nop 3706 bclr TSTATE_PRIV, %g4 3707 wrpr %g4, 0, %tstate 3708 ba,a,pt %icc, slowtrap 3709 nop 3710 3711 /* 3712 * slowtrap() builds a trap frame and calls trap(). 3713 * This is called `slowtrap' because it *is*.... 3714 * We have to build a full frame for ptrace(), for instance. 3715 * 3716 * Registers: 3717 * 3718 */ 3719 slowtrap: 3720 #ifdef TRAPS_USE_IG 3721 wrpr %g0, PSTATE_KERN|PSTATE_IG, %pstate ! DEBUG 3722 #endif 3723 #ifdef DIAGNOSTIC 3724 /* Make sure kernel stack is aligned */ 3725 btst 0x03, %sp ! 32-bit stack OK? 3726 and %sp, 0x07, %g4 ! 64-bit stack OK? 3727 bz,pt %icc, 1f 3728 cmp %g4, 0x1 ! Must end in 0b001 3729 be,pt %icc, 1f 3730 rdpr %wstate, %g7 3731 cmp %g7, WSTATE_KERN 3732 bnz,pt %icc, 1f ! User stack -- we'll blow it away 3733 nop 3734 set PANICSTACK-CC64FSZ-STKB, %sp 3735 1: 3736 #endif 3737 rdpr %tt, %g4 3738 rdpr %tstate, %g1 3739 rdpr %tpc, %g2 3740 rdpr %tnpc, %g3 3741 3742 TRAP_SETUP(-CC64FSZ-TF_SIZE) 3743 Lslowtrap_reenter: 3744 stx %g1, [%sp + CC64FSZ + STKB + TF_TSTATE] 3745 mov %g4, %o1 ! (type) 3746 stx %g2, [%sp + CC64FSZ + STKB + TF_PC] 3747 rd %y, %g5 3748 stx %g3, [%sp + CC64FSZ + STKB + TF_NPC] 3749 mov %g1, %o3 ! (pstate) 3750 st %g5, [%sp + CC64FSZ + STKB + TF_Y] 3751 mov %g2, %o2 ! (pc) 3752 sth %o1, [%sp + CC64FSZ + STKB + TF_TT]! debug 3753 3754 ! Get back to normal globals 3755 #ifdef SUN4V 3756 sethi %hi(cputyp), %g5 3757 ld [%g5 + %lo(cputyp)], %g5 3758 cmp %g5, CPU_SUN4V 3759 bne,pt %icc, 1f 3760 nop 3761 NORMAL_GLOBALS_SUN4V 3762 ba 2f 3763 nop 3764 1: 3765 #endif 3766 NORMAL_GLOBALS_SUN4U 3767 2: 3768 stx %g1, [%sp + CC64FSZ + STKB + TF_G + (1*8)] 3769 stx %g2, [%sp + CC64FSZ + STKB + TF_G + (2*8)] 3770 add %sp, CC64FSZ + STKB, %o0 ! (&tf) 3771 stx %g3, [%sp + CC64FSZ + STKB + TF_G + (3*8)] 3772 stx %g4, [%sp + CC64FSZ + STKB + TF_G + (4*8)] 3773 stx %g5, [%sp + CC64FSZ + STKB + TF_G + (5*8)] 3774 rdpr %pil, %g5 3775 stx %g6, [%sp + CC64FSZ + STKB + TF_G + (6*8)] 3776 stx %g7, [%sp + CC64FSZ + STKB + TF_G + (7*8)] 3777 stb %g5, [%sp + CC64FSZ + STKB + TF_PIL] 3778 stb %g5, [%sp + CC64FSZ + STKB + TF_OLDPIL] 3779 /* 3780 * Phew, ready to enable traps and call C code. 3781 */ 3782 rdpr %tl, %g1 3783 dec %g1 3784 movrlz %g1, %g0, %g1 3785 wrpr %g0, %g1, %tl ! Revert to kernel mode 3786 !! In the EMBEDANY memory model %g4 points to the start of the data segment. 3787 !! In our case we need to clear it before calling any C-code 3788 clr %g4 3789 3790 wr %g0, ASI_PRIMARY_NOFAULT, %asi ! Restore default ASI 3791 wrpr %g0, PSTATE_INTR, %pstate ! traps on again 3792 call _C_LABEL(trap) ! trap(tf, type, pc, pstate) 3793 nop 3794 3795 b return_from_trap 3796 ldx [%sp + CC64FSZ + STKB + TF_TSTATE], %g1 ! Load this for return_from_trap 3797 NOTREACHED 3798 #if 1 3799 /* 3800 * This code is no longer needed. 3801 */ 3802 /* 3803 * Do a `software' trap by re-entering the trap code, possibly first 3804 * switching from interrupt stack to kernel stack. This is used for 3805 * scheduling and signal ASTs (which generally occur from softclock or 3806 * tty or net interrupts). 3807 * 3808 * We enter with the trap type in %g1. All we have to do is jump to 3809 * Lslowtrap_reenter above, but maybe after switching stacks.... 3810 * 3811 * We should be running alternate globals. The normal globals and 3812 * out registers were just loaded from the old trap frame. 3813 * 3814 * Input Params: 3815 * %g1 = tstate 3816 * %g2 = tpc 3817 * %g3 = tnpc 3818 * %g4 = tt == T_AST 3819 */ 3820 softtrap: 3821 sethi %hi(EINTSTACK-STKB), %g5 3822 sethi %hi(EINTSTACK-INTSTACK), %g7 3823 or %g5, %lo(EINTSTACK-STKB), %g5 3824 dec %g7 3825 sub %g5, %sp, %g5 3826 sethi %hi(CPCB), %g6 3827 andncc %g5, %g7, %g0 3828 bnz,pt %xcc, Lslowtrap_reenter 3829 LDPTR [%g6 + %lo(CPCB)], %g7 3830 set USPACE-CC64FSZ-TF_SIZE-STKB, %g5 3831 add %g7, %g5, %g6 3832 SET_SP_REDZONE(%g7, %g5) 3833 #ifdef DEBUG 3834 stx %g1, [%g6 + CC64FSZ + STKB + TF_FAULT] ! Generate a new trapframe 3835 #endif 3836 stx %i0, [%g6 + CC64FSZ + STKB + TF_O + (0*8)] ! but don't bother with 3837 stx %i1, [%g6 + CC64FSZ + STKB + TF_O + (1*8)] ! locals and ins 3838 stx %i2, [%g6 + CC64FSZ + STKB + TF_O + (2*8)] 3839 stx %i3, [%g6 + CC64FSZ + STKB + TF_O + (3*8)] 3840 stx %i4, [%g6 + CC64FSZ + STKB + TF_O + (4*8)] 3841 stx %i5, [%g6 + CC64FSZ + STKB + TF_O + (5*8)] 3842 stx %i6, [%g6 + CC64FSZ + STKB + TF_O + (6*8)] 3843 stx %i7, [%g6 + CC64FSZ + STKB + TF_O + (7*8)] 3844 #ifdef DEBUG 3845 ldx [%sp + CC64FSZ + STKB + TF_I + (0*8)], %l0 ! Copy over the rest of the regs 3846 ldx [%sp + CC64FSZ + STKB + TF_I + (1*8)], %l1 ! But just dirty the locals 3847 ldx [%sp + CC64FSZ + STKB + TF_I + (2*8)], %l2 3848 ldx [%sp + CC64FSZ + STKB + TF_I + (3*8)], %l3 3849 ldx [%sp + CC64FSZ + STKB + TF_I + (4*8)], %l4 3850 ldx [%sp + CC64FSZ + STKB + TF_I + (5*8)], %l5 3851 ldx [%sp + CC64FSZ + STKB + TF_I + (6*8)], %l6 3852 ldx [%sp + CC64FSZ + STKB + TF_I + (7*8)], %l7 3853 stx %l0, [%g6 + CC64FSZ + STKB + TF_I + (0*8)] 3854 stx %l1, [%g6 + CC64FSZ + STKB + TF_I + (1*8)] 3855 stx %l2, [%g6 + CC64FSZ + STKB + TF_I + (2*8)] 3856 stx %l3, [%g6 + CC64FSZ + STKB + TF_I + (3*8)] 3857 stx %l4, [%g6 + CC64FSZ + STKB + TF_I + (4*8)] 3858 stx %l5, [%g6 + CC64FSZ + STKB + TF_I + (5*8)] 3859 stx %l6, [%g6 + CC64FSZ + STKB + TF_I + (6*8)] 3860 stx %l7, [%g6 + CC64FSZ + STKB + TF_I + (7*8)] 3861 ldx [%sp + CC64FSZ + STKB + TF_L + (0*8)], %l0 3862 ldx [%sp + CC64FSZ + STKB + TF_L + (1*8)], %l1 3863 ldx [%sp + CC64FSZ + STKB + TF_L + (2*8)], %l2 3864 ldx [%sp + CC64FSZ + STKB + TF_L + (3*8)], %l3 3865 ldx [%sp + CC64FSZ + STKB + TF_L + (4*8)], %l4 3866 ldx [%sp + CC64FSZ + STKB + TF_L + (5*8)], %l5 3867 ldx [%sp + CC64FSZ + STKB + TF_L + (6*8)], %l6 3868 ldx [%sp + CC64FSZ + STKB + TF_L + (7*8)], %l7 3869 stx %l0, [%g6 + CC64FSZ + STKB + TF_L + (0*8)] 3870 stx %l1, [%g6 + CC64FSZ + STKB + TF_L + (1*8)] 3871 stx %l2, [%g6 + CC64FSZ + STKB + TF_L + (2*8)] 3872 stx %l3, [%g6 + CC64FSZ + STKB + TF_L + (3*8)] 3873 stx %l4, [%g6 + CC64FSZ + STKB + TF_L + (4*8)] 3874 stx %l5, [%g6 + CC64FSZ + STKB + TF_L + (5*8)] 3875 stx %l6, [%g6 + CC64FSZ + STKB + TF_L + (6*8)] 3876 stx %l7, [%g6 + CC64FSZ + STKB + TF_L + (7*8)] 3877 #endif 3878 ba,pt %xcc, Lslowtrap_reenter 3879 mov %g6, %sp 3880 #endif 3881 3882 #if 0 3883 /* 3884 * breakpoint: capture as much info as possible and then call DDB 3885 * or trap, as the case may be. 3886 * 3887 * First, we switch to interrupt globals, and blow away %g7. Then 3888 * switch down one stackframe -- just fiddle w/cwp, don't save or 3889 * we'll trap. Then slowly save all the globals into our static 3890 * register buffer. etc. etc. 3891 */ 3892 3893 breakpoint: 3894 wrpr %g0, PSTATE_KERN|PSTATE_IG, %pstate ! Get IG to use 3895 rdpr %cwp, %g7 3896 inc 1, %g7 ! Equivalent of save 3897 wrpr %g7, 0, %cwp ! Now we have some unused locals to fiddle with 3898 XXX ddb_regs is now ddb-regp and is a pointer not a symbol. 3899 set _C_LABEL(ddb_regs), %l0 3900 stx %g1, [%l0+DBR_IG+(1*8)] ! Save IGs 3901 stx %g2, [%l0+DBR_IG+(2*8)] 3902 stx %g3, [%l0+DBR_IG+(3*8)] 3903 stx %g4, [%l0+DBR_IG+(4*8)] 3904 stx %g5, [%l0+DBR_IG+(5*8)] 3905 stx %g6, [%l0+DBR_IG+(6*8)] 3906 stx %g7, [%l0+DBR_IG+(7*8)] 3907 wrpr %g0, PSTATE_KERN|PSTATE_MG, %pstate ! Get MG to use 3908 stx %g1, [%l0+DBR_MG+(1*8)] ! Save MGs 3909 stx %g2, [%l0+DBR_MG+(2*8)] 3910 stx %g3, [%l0+DBR_MG+(3*8)] 3911 stx %g4, [%l0+DBR_MG+(4*8)] 3912 stx %g5, [%l0+DBR_MG+(5*8)] 3913 stx %g6, [%l0+DBR_MG+(6*8)] 3914 stx %g7, [%l0+DBR_MG+(7*8)] 3915 wrpr %g0, PSTATE_KERN|PSTATE_AG, %pstate ! Get AG to use 3916 stx %g1, [%l0+DBR_AG+(1*8)] ! Save AGs 3917 stx %g2, [%l0+DBR_AG+(2*8)] 3918 stx %g3, [%l0+DBR_AG+(3*8)] 3919 stx %g4, [%l0+DBR_AG+(4*8)] 3920 stx %g5, [%l0+DBR_AG+(5*8)] 3921 stx %g6, [%l0+DBR_AG+(6*8)] 3922 stx %g7, [%l0+DBR_AG+(7*8)] 3923 wrpr %g0, PSTATE_KERN, %pstate ! Get G to use 3924 stx %g1, [%l0+DBR_G+(1*8)] ! Save Gs 3925 stx %g2, [%l0+DBR_G+(2*8)] 3926 stx %g3, [%l0+DBR_G+(3*8)] 3927 stx %g4, [%l0+DBR_G+(4*8)] 3928 stx %g5, [%l0+DBR_G+(5*8)] 3929 stx %g6, [%l0+DBR_G+(6*8)] 3930 stx %g7, [%l0+DBR_G+(7*8)] 3931 rdpr %canrestore, %l1 3932 stb %l1, [%l0+DBR_CANRESTORE] 3933 rdpr %cansave, %l2 3934 stb %l2, [%l0+DBR_CANSAVE] 3935 rdpr %cleanwin, %l3 3936 stb %l3, [%l0+DBR_CLEANWIN] 3937 rdpr %wstate, %l4 3938 stb %l4, [%l0+DBR_WSTATE] 3939 rd %y, %l5 3940 stw %l5, [%l0+DBR_Y] 3941 rdpr %tl, %l6 3942 stb %l6, [%l0+DBR_TL] 3943 dec 1, %g7 3944 #endif 3945 3946 /* 3947 * I will not touch any of the DDB or KGDB stuff until I know what's going 3948 * on with the symbol table. This is all still v7/v8 code and needs to be fixed. 3949 */ 3950 #ifdef KGDB 3951 /* 3952 * bpt is entered on all breakpoint traps. 3953 * If this is a kernel breakpoint, we do not want to call trap(). 3954 * Among other reasons, this way we can set breakpoints in trap(). 3955 */ 3956 bpt: 3957 set TSTATE_PRIV, %l4 3958 andcc %l4, %l0, %g0 ! breakpoint from kernel? 3959 bz slowtrap ! no, go do regular trap 3960 nop 3961 3962 /* 3963 * Build a trap frame for kgdb_trap_glue to copy. 3964 * Enable traps but set ipl high so that we will not 3965 * see interrupts from within breakpoints. 3966 */ 3967 save %sp, -CCFSZ-TF_SIZE, %sp ! allocate a trap frame 3968 TRAP_SETUP(-CCFSZ-TF_SIZE) 3969 or %l0, PSR_PIL, %l4 ! splhigh() 3970 wr %l4, 0, %psr ! the manual claims that this 3971 wr %l4, PSR_ET, %psr ! song and dance is necessary 3972 std %l0, [%sp + CCFSZ + 0] ! tf.tf_psr, tf.tf_pc 3973 mov %l3, %o0 ! trap type arg for kgdb_trap_glue 3974 rd %y, %l3 3975 std %l2, [%sp + CCFSZ + 8] ! tf.tf_npc, tf.tf_y 3976 rd %wim, %l3 3977 st %l3, [%sp + CCFSZ + 16] ! tf.tf_wim (a kgdb-only r/o field) 3978 st %g1, [%sp + CCFSZ + 20] ! tf.tf_global[1] 3979 std %g2, [%sp + CCFSZ + 24] ! etc 3980 std %g4, [%sp + CCFSZ + 32] 3981 std %g6, [%sp + CCFSZ + 40] 3982 std %i0, [%sp + CCFSZ + 48] ! tf.tf_in[0..1] 3983 std %i2, [%sp + CCFSZ + 56] ! etc 3984 std %i4, [%sp + CCFSZ + 64] 3985 std %i6, [%sp + CCFSZ + 72] 3986 3987 /* 3988 * Now call kgdb_trap_glue(); if it returns, call trap(). 3989 */ 3990 mov %o0, %l3 ! gotta save trap type 3991 call _C_LABEL(kgdb_trap_glue) ! kgdb_trap_glue(type, &trapframe) 3992 add %sp, CCFSZ, %o1 ! (&trapframe) 3993 3994 /* 3995 * Use slowtrap to call trap---but first erase our tracks 3996 * (put the registers back the way they were). 3997 */ 3998 mov %l3, %o0 ! slowtrap will need trap type 3999 ld [%sp + CCFSZ + 12], %l3 4000 wr %l3, 0, %y 4001 ld [%sp + CCFSZ + 20], %g1 4002 ldd [%sp + CCFSZ + 24], %g2 4003 ldd [%sp + CCFSZ + 32], %g4 4004 b Lslowtrap_reenter 4005 ldd [%sp + CCFSZ + 40], %g6 4006 4007 /* 4008 * Enter kernel breakpoint. Write all the windows (not including the 4009 * current window) into the stack, so that backtrace works. Copy the 4010 * supplied trap frame to the kgdb stack and switch stacks. 4011 * 4012 * kgdb_trap_glue(type, tf0) 4013 * int type; 4014 * struct trapframe *tf0; 4015 */ 4016 ENTRY_NOPROFILE(kgdb_trap_glue) 4017 save %sp, -CCFSZ, %sp 4018 4019 flushw ! flush all windows 4020 mov %sp, %l4 ! %l4 = current %sp 4021 4022 /* copy trapframe to top of kgdb stack */ 4023 set _C_LABEL(kgdb_stack) + KGDB_STACK_SIZE - 80, %l0 4024 ! %l0 = tfcopy -> end_of_kgdb_stack 4025 mov 80, %l1 4026 1: ldd [%i1], %l2 4027 inc 8, %i1 4028 deccc 8, %l1 4029 std %l2, [%l0] 4030 bg 1b 4031 inc 8, %l0 4032 4033 #ifdef NOTDEF_DEBUG 4034 /* save old red zone and then turn it off */ 4035 sethi %hi(_C_LABEL(redzone)), %l7 4036 ld [%l7 + %lo(_C_LABEL(redzone))], %l6 4037 st %g0, [%l7 + %lo(_C_LABEL(redzone))] 4038 #endif 4039 /* switch to kgdb stack */ 4040 add %l0, -CCFSZ-TF_SIZE, %sp 4041 4042 /* if (kgdb_trap(type, tfcopy)) kgdb_rett(tfcopy); */ 4043 mov %i0, %o0 4044 call _C_LABEL(kgdb_trap) 4045 add %l0, -80, %o1 4046 tst %o0 4047 bnz,a kgdb_rett 4048 add %l0, -80, %g1 4049 4050 /* 4051 * kgdb_trap() did not handle the trap at all so the stack is 4052 * still intact. A simple `restore' will put everything back, 4053 * after we reset the stack pointer. 4054 */ 4055 mov %l4, %sp 4056 #ifdef NOTDEF_DEBUG 4057 st %l6, [%l7 + %lo(_C_LABEL(redzone))] ! restore red zone 4058 #endif 4059 ret 4060 restore 4061 4062 /* 4063 * Return from kgdb trap. This is sort of special. 4064 * 4065 * We know that kgdb_trap_glue wrote the window above it, so that we will 4066 * be able to (and are sure to have to) load it up. We also know that we 4067 * came from kernel land and can assume that the %fp (%i6) we load here 4068 * is proper. We must also be sure not to lower ipl (it is at splhigh()) 4069 * until we have traps disabled, due to the SPARC taking traps at the 4070 * new ipl before noticing that PSR_ET has been turned off. We are on 4071 * the kgdb stack, so this could be disastrous. 4072 * 4073 * Note that the trapframe argument in %g1 points into the current stack 4074 * frame (current window). We abandon this window when we move %g1->tf_psr 4075 * into %psr, but we will not have loaded the new %sp yet, so again traps 4076 * must be disabled. 4077 */ 4078 kgdb_rett: 4079 rd %psr, %g4 ! turn off traps 4080 wr %g4, PSR_ET, %psr 4081 /* use the three-instruction delay to do something useful */ 4082 ld [%g1], %g2 ! pick up new %psr 4083 ld [%g1 + 12], %g3 ! set %y 4084 wr %g3, 0, %y 4085 #ifdef NOTDEF_DEBUG 4086 st %l6, [%l7 + %lo(_C_LABEL(redzone))] ! and restore red zone 4087 #endif 4088 wr %g0, 0, %wim ! enable window changes 4089 nop; nop; nop 4090 /* now safe to set the new psr (changes CWP, leaves traps disabled) */ 4091 wr %g2, 0, %psr ! set rett psr (including cond codes) 4092 /* 3 instruction delay before we can use the new window */ 4093 /*1*/ ldd [%g1 + 24], %g2 ! set new %g2, %g3 4094 /*2*/ ldd [%g1 + 32], %g4 ! set new %g4, %g5 4095 /*3*/ ldd [%g1 + 40], %g6 ! set new %g6, %g7 4096 4097 /* now we can use the new window */ 4098 mov %g1, %l4 4099 ld [%l4 + 4], %l1 ! get new pc 4100 ld [%l4 + 8], %l2 ! get new npc 4101 ld [%l4 + 20], %g1 ! set new %g1 4102 4103 /* set up returnee's out registers, including its %sp */ 4104 ldd [%l4 + 48], %i0 4105 ldd [%l4 + 56], %i2 4106 ldd [%l4 + 64], %i4 4107 ldd [%l4 + 72], %i6 4108 4109 /* load returnee's window, making the window above it be invalid */ 4110 restore 4111 restore %g0, 1, %l1 ! move to inval window and set %l1 = 1 4112 rd %psr, %l0 4113 srl %l1, %l0, %l1 4114 wr %l1, 0, %wim ! %wim = 1 << (%psr & 31) 4115 sethi %hi(CPCB), %l1 4116 LDPTR [%l1 + %lo(CPCB)], %l1 4117 and %l0, 31, %l0 ! CWP = %psr & 31; 4118 ! st %l0, [%l1 + PCB_WIM] ! cpcb->pcb_wim = CWP; 4119 save %g0, %g0, %g0 ! back to window to reload 4120 ! LOADWIN(%sp) 4121 save %g0, %g0, %g0 ! back to trap window 4122 /* note, we have not altered condition codes; safe to just rett */ 4123 RETT 4124 #endif 4125 4126 /* 4127 * syscall_setup() builds a trap frame and calls syscall(). 4128 * sun_syscall is same but delivers sun system call number 4129 * XXX should not have to save&reload ALL the registers just for 4130 * ptrace... 4131 */ 4132 syscall_setup: 4133 #ifdef TRAPS_USE_IG 4134 wrpr %g0, PSTATE_KERN|PSTATE_IG, %pstate ! DEBUG 4135 #endif 4136 TRAP_SETUP(-CC64FSZ-TF_SIZE) 4137 4138 #ifdef DEBUG 4139 rdpr %tt, %o1 ! debug 4140 sth %o1, [%sp + CC64FSZ + STKB + TF_TT]! debug 4141 #endif 4142 4143 ! Get back to normal globals 4144 #ifdef SUN4V 4145 sethi %hi(cputyp), %g5 4146 ld [%g5 + %lo(cputyp)], %g5 4147 cmp %g5, CPU_SUN4V 4148 bne,pt %icc, 1f 4149 nop 4150 NORMAL_GLOBALS_SUN4V 4151 ba 2f 4152 nop 4153 1: 4154 #endif 4155 NORMAL_GLOBALS_SUN4U 4156 2: 4157 4158 stx %g1, [%sp + CC64FSZ + STKB + TF_G + ( 1*8)] 4159 mov %g1, %o1 ! code 4160 rdpr %tpc, %o2 ! (pc) 4161 stx %g2, [%sp + CC64FSZ + STKB + TF_G + ( 2*8)] 4162 rdpr %tstate, %g1 4163 stx %g3, [%sp + CC64FSZ + STKB + TF_G + ( 3*8)] 4164 rdpr %tnpc, %o3 4165 stx %g4, [%sp + CC64FSZ + STKB + TF_G + ( 4*8)] 4166 rd %y, %o4 4167 stx %g5, [%sp + CC64FSZ + STKB + TF_G + ( 5*8)] 4168 stx %g6, [%sp + CC64FSZ + STKB + TF_G + ( 6*8)] 4169 wrpr %g0, 0, %tl ! return to tl=0 4170 stx %g7, [%sp + CC64FSZ + STKB + TF_G + ( 7*8)] 4171 add %sp, CC64FSZ + STKB, %o0 ! (&tf) 4172 4173 stx %g1, [%sp + CC64FSZ + STKB + TF_TSTATE] 4174 stx %o2, [%sp + CC64FSZ + STKB + TF_PC] 4175 stx %o3, [%sp + CC64FSZ + STKB + TF_NPC] 4176 st %o4, [%sp + CC64FSZ + STKB + TF_Y] 4177 4178 rdpr %pil, %g5 4179 stb %g5, [%sp + CC64FSZ + STKB + TF_PIL] 4180 stb %g5, [%sp + CC64FSZ + STKB + TF_OLDPIL] 4181 4182 !! In the EMBEDANY memory model %g4 points to the start of the data segment. 4183 !! In our case we need to clear it before calling any C-code 4184 clr %g4 4185 wr %g0, ASI_PRIMARY_NOFAULT, %asi ! Restore default ASI 4186 4187 sethi %hi(CURLWP), %l1 4188 LDPTR [%l1 + %lo(CURLWP)], %l1 4189 LDPTR [%l1 + L_PROC], %l1 ! now %l1 points to p 4190 LDPTR [%l1 + P_MD_SYSCALL], %l1 4191 call %l1 4192 wrpr %g0, PSTATE_INTR, %pstate ! turn on interrupts 4193 4194 /* see `lwp_trampoline' for the reason for this label */ 4195 return_from_syscall: 4196 wrpr %g0, PSTATE_KERN, %pstate ! Disable interrupts 4197 wrpr %g0, 0, %tl ! Return to tl==0 4198 b return_from_trap 4199 nop 4200 NOTREACHED 4201 4202 /* 4203 * interrupt_vector: 4204 * 4205 * Spitfire chips never get level interrupts directly from H/W. 4206 * Instead, all interrupts come in as interrupt_vector traps. 4207 * The interrupt number or handler address is an 11 bit number 4208 * encoded in the first interrupt data word. Additional words 4209 * are application specific and used primarily for cross-calls. 4210 * 4211 * The interrupt vector handler then needs to identify the 4212 * interrupt source from the interrupt number and arrange to 4213 * invoke the interrupt handler. This can either be done directly 4214 * from here, or a softint at a particular level can be issued. 4215 * 4216 * To call an interrupt directly and not overflow the trap stack, 4217 * the trap registers should be saved on the stack, registers 4218 * cleaned, trap-level decremented, the handler called, and then 4219 * the process must be reversed. 4220 * 4221 * To simplify life all we do here is issue an appropriate softint. 4222 * 4223 * Note: It is impossible to identify or change a device's 4224 * interrupt number until it is probed. That's the 4225 * purpose for all the funny interrupt acknowledge 4226 * code. 4227 * 4228 */ 4229 4230 /* 4231 * Vectored interrupts: 4232 * 4233 * When an interrupt comes in, interrupt_vector uses the interrupt 4234 * vector number to lookup the appropriate intrhand from the intrlev 4235 * array. It then looks up the interrupt level from the intrhand 4236 * structure. It uses the level to index the intrpending array, 4237 * which is 8 slots for each possible interrupt level (so we can 4238 * shift instead of multiply for address calculation). It hunts for 4239 * any available slot at that level. Available slots are NULL. 4240 * 4241 * Then interrupt_vector uses the interrupt level in the intrhand 4242 * to issue a softint of the appropriate level. The softint handler 4243 * figures out what level interrupt it's handling and pulls the first 4244 * intrhand pointer out of the intrpending array for that interrupt 4245 * level, puts a NULL in its place, clears the interrupt generator, 4246 * and invokes the interrupt handler. 4247 */ 4248 4249 /* intrpending array is now in per-CPU structure. */ 4250 4251 #ifdef DEBUG 4252 #define INTRDEBUG_VECTOR 0x1 4253 #define INTRDEBUG_LEVEL 0x2 4254 #define INTRDEBUG_FUNC 0x4 4255 #define INTRDEBUG_SPUR 0x8 4256 .data 4257 .globl _C_LABEL(intrdebug) 4258 _C_LABEL(intrdebug): .word 0x0 4259 /* 4260 * Note: we use the local label `97' to branch forward to, to skip 4261 * actual debugging code following a `intrdebug' bit test. 4262 */ 4263 #endif 4264 .text 4265 interrupt_vector: 4266 #ifdef TRAPSTATS 4267 set _C_LABEL(kiveccnt), %g1 4268 set _C_LABEL(iveccnt), %g2 4269 rdpr %tl, %g3 4270 dec %g3 4271 movrz %g3, %g2, %g1 4272 lduw [%g1], %g2 4273 inc %g2 4274 stw %g2, [%g1] 4275 #endif 4276 ldxa [%g0] ASI_IRSR, %g1 4277 mov IRDR_0H, %g7 4278 ldxa [%g7] ASI_IRDR, %g7 ! Get interrupt number 4279 membar #Sync 4280 4281 btst IRSR_BUSY, %g1 4282 bz,pn %icc, 3f ! spurious interrupt 4283 #ifdef MULTIPROCESSOR 4284 sethi %hi(KERNBASE), %g1 4285 4286 cmp %g7, %g1 4287 bl,a,pt %xcc, Lsoftint_regular ! >= KERNBASE is a fast cross-call 4288 and %g7, (MAXINTNUM-1), %g7 ! XXX make sun4us work 4289 4290 mov IRDR_1H, %g2 4291 ldxa [%g2] ASI_IRDR, %g2 ! Get IPI handler argument 1 4292 mov IRDR_2H, %g3 4293 ldxa [%g3] ASI_IRDR, %g3 ! Get IPI handler argument 2 4294 4295 stxa %g0, [%g0] ASI_IRSR ! Ack IRQ 4296 membar #Sync ! Should not be needed due to retry 4297 4298 jmpl %g7, %g0 4299 nop 4300 #else 4301 and %g7, (MAXINTNUM-1), %g7 ! XXX make sun4us work 4302 #endif 4303 4304 Lsoftint_regular: 4305 stxa %g0, [%g0] ASI_IRSR ! Ack IRQ 4306 membar #Sync ! Should not be needed due to retry 4307 sethi %hi(_C_LABEL(intrlev)), %g3 4308 sllx %g7, PTRSHFT, %g5 ! Calculate entry number 4309 or %g3, %lo(_C_LABEL(intrlev)), %g3 4310 LDPTR [%g3 + %g5], %g5 ! We have a pointer to the handler 4311 brz,pn %g5, 3f ! NULL means it isn't registered yet. Skip it. 4312 nop 4313 4314 ! increment per-ivec counter 4315 ldx [%g5 + IH_CNT], %g1 4316 inc %g1 4317 stx %g1, [%g5 + IH_CNT] 4318 4319 setup_sparcintr: 4320 LDPTR [%g5+IH_PEND], %g6 ! Read pending flag 4321 brnz,pn %g6, ret_from_intr_vector ! Skip it if it's running 4322 ldub [%g5+IH_PIL], %g6 ! Read interrupt level 4323 sethi %hi(CPUINFO_VA+CI_INTRPENDING), %g1 4324 sll %g6, PTRSHFT, %g3 ! Find start of table for this IPL 4325 or %g1, %lo(CPUINFO_VA+CI_INTRPENDING), %g1 4326 add %g1, %g3, %g1 4327 1: 4328 LDPTR [%g1], %g3 ! Load list head 4329 STPTR %g3, [%g5+IH_PEND] ! Link our intrhand node in 4330 mov %g5, %g7 4331 CASPTRA [%g1] ASI_N, %g3, %g7 4332 cmp %g7, %g3 ! Did it work? 4333 bne,pn CCCR, 1b ! No, try again 4334 .empty 4335 2: 4336 #ifdef NOT_DEBUG 4337 set _C_LABEL(intrdebug), %g7 4338 ld [%g7], %g7 4339 btst INTRDEBUG_VECTOR, %g7 4340 bz,pt %icc, 97f 4341 nop 4342 4343 cmp %g6, 0xa ! ignore clock interrupts? 4344 bz,pt %icc, 97f 4345 nop 4346 4347 STACKFRAME(-CC64FSZ) ! Get a clean register window 4348 LOAD_ASCIZ(%o0,\ 4349 "interrupt_vector: number %lx softint mask %lx pil %lu slot %p\n") 4350 mov %g2, %o1 4351 rdpr %pil, %o3 4352 mov %g1, %o4 4353 GLOBTOLOC 4354 clr %g4 4355 call prom_printf 4356 mov %g6, %o2 4357 LOCTOGLOB 4358 restore 4359 97: 4360 #endif 4361 mov 1, %g7 4362 sll %g7, %g6, %g6 4363 wr %g6, 0, SET_SOFTINT ! Invoke a softint 4364 4365 .global ret_from_intr_vector 4366 ret_from_intr_vector: 4367 retry 4368 NOTREACHED 4369 4370 3: 4371 #ifdef NOT_DEBUG /* always do this */ 4372 set _C_LABEL(intrdebug), %g6 4373 ld [%g6], %g6 4374 btst INTRDEBUG_SPUR, %g6 4375 bz,pt %icc, 97f 4376 nop 4377 #endif 4378 #if 1 4379 set PANICSTACK-STKB, %g1 ! Use panic stack temporarily 4380 save %g1, -CC64FSZ, %sp ! Get a clean register window 4381 LOAD_ASCIZ(%o0, "interrupt_vector: spurious vector %lx at pil %d\n") 4382 mov %g7, %o1 4383 GLOBTOLOC 4384 clr %g4 4385 call prom_printf 4386 rdpr %pil, %o2 4387 LOCTOGLOB 4388 restore 4389 97: 4390 #endif 4391 ba,a ret_from_intr_vector 4392 nop ! XXX spitfire bug? 4393 4394 sun4v_cpu_mondo: 4395 ! XXX Rework this when a UP kernel works - crash for now 4396 sir 4397 mov 0x3c0, %g1 ! CPU Mondo Queue Head 4398 ldxa [%g1] ASI_QUEUE, %g2 ! fetch index value for head 4399 set CPUINFO_VA, %g3 4400 ldx [%g3 + CI_PADDR], %g3 4401 add %g3, CI_CPUMQ, %g3 4402 ldxa [%g3] ASI_PHYS_CACHED, %g3 ! fetch head element 4403 ldxa [%g3 + %g2] ASI_PHYS_CACHED, %g4 ! fetch func 4404 add %g2, 8, %g5 4405 ldxa [%g3 + %g5] ASI_PHYS_CACHED, %g5 ! fetch arg1 4406 add %g2, 16, %g6 4407 ldxa [%g3 + %g6] ASI_PHYS_CACHED, %g6 ! fetch arg2 4408 add %g2, 64, %g2 ! point to next element in queue 4409 and %g2, 0x7ff, %g2 ! modulo queue size 2048 (32*64) 4410 stxa %g2, [%g1] ASI_QUEUE ! update head index 4411 membar #Sync 4412 4413 mov %g4, %g2 4414 mov %g5, %g3 4415 mov %g6, %g5 4416 jmpl %g2, %g0 4417 nop ! No store here! 4418 retry 4419 NOTREACHED 4420 4421 sun4v_dev_mondo: 4422 mov 0x3d0, %g1 ! Dev Mondo Queue Head 4423 ldxa [%g1] ASI_QUEUE, %g2 ! fetch index value 4424 mov 0x3d8, %g1 ! Dev Mondo Queue Tail 4425 ldxa [%g1] ASI_QUEUE, %g4 ! fetch index value 4426 cmp %g2, %g4 ! head = queue? 4427 bne,pt %xcc, 2f ! unusually not the case 4428 nop 4429 retry ! unlikely, ignore interrupt 4430 2: 4431 set CPUINFO_VA, %g3 ! fetch cpuinfo pa 4432 ldx [%g3 + CI_PADDR], %g3 ! fetch intstack pa 4433 set CPUINFO_VA-INTSTACK, %g4 ! offset to cpuinfo 4434 add %g4, %g3, %g3 ! %g3 is now cpuifo 4435 add %g3, CI_DEVMQ, %g3 ! calc offset to devmq 4436 ldxa [%g3] ASI_PHYS_CACHED, %g3 ! fetch address of devmq 4437 ldxa [%g3 + %g2] ASI_PHYS_CACHED, %g5 ! 4438 add %g2, 64, %g2 ! each element is 64 bytes 4439 and %g2, 0x7ff, %g2 ! assume 32 elements 4440 mov 0x3d0, %g1 ! Dev Mondo Queue Head 4441 stxa %g2, [%g1] ASI_QUEUE ! adjust head index value 4442 membar #Sync 4443 4444 cmp %g5, MAXINTNUM ! Handle both sun4v legacy (sysino) and cookies. 4445 bgeu,pn %xcc, 1f ! See UltraSPARC Virtual Machine Specification 4446 nop ! version 3 chapter 6 (Interrupt model) 4447 4448 sethi %hi(_C_LABEL(intrlev)), %g3 4449 sllx %g5, PTRSHFT, %g5 ! Calculate entry number 4450 or %g3, %lo(_C_LABEL(intrlev)), %g3 4451 LDPTR [%g3 + %g5], %g5 ! We have a pointer to the handler 4452 1: 4453 brnz,pt %g5, setup_sparcintr ! branch if valid handle 4454 nop 4455 4456 ba,a 3b ! log if invalid handle 4457 nop 4458 4459 /* 4460 * Ultra1 and Ultra2 CPUs use soft interrupts for everything. What we do 4461 * on a soft interrupt, is we should check which bits in SOFTINT(%asr22) 4462 * are set, handle those interrupts, then clear them by setting the 4463 * appropriate bits in CLEAR_SOFTINT(%asr21). 4464 * 4465 * We have an array of 8 interrupt vector slots for each of 15 interrupt 4466 * levels. If a vectored interrupt can be dispatched, the dispatch 4467 * routine will place a pointer to an intrhand structure in one of 4468 * the slots. The interrupt handler will go through the list to look 4469 * for an interrupt to dispatch. If it finds one it will pull it off 4470 * the list, free the entry, and call the handler. The code is like 4471 * this: 4472 * 4473 * for (i=0; i<8; i++) 4474 * if (ih = intrpending[intlev][i]) { 4475 * intrpending[intlev][i] = NULL; 4476 * if ((*ih->ih_fun)(ih->ih_arg ? ih->ih_arg : &frame)) 4477 * return; 4478 * strayintr(&frame); 4479 * return; 4480 * } 4481 * 4482 * Otherwise we go back to the old style of polled interrupts. 4483 * 4484 * After preliminary setup work, the interrupt is passed to each 4485 * registered handler in turn. These are expected to return nonzero if 4486 * they took care of the interrupt. If a handler claims the interrupt, 4487 * we exit (hardware interrupts are latched in the requestor so we'll 4488 * just take another interrupt in the unlikely event of simultaneous 4489 * interrupts from two different devices at the same level). If we go 4490 * through all the registered handlers and no one claims it, we report a 4491 * stray interrupt. This is more or less done as: 4492 * 4493 * for (ih = intrhand[intlev]; ih; ih = ih->ih_next) 4494 * if ((*ih->ih_fun)(ih->ih_arg ? ih->ih_arg : &frame)) 4495 * return; 4496 * strayintr(&frame); 4497 * 4498 * Inputs: 4499 * %l0 = %tstate 4500 * %l1 = return pc 4501 * %l2 = return npc 4502 * %l3 = interrupt level 4503 * (software interrupt only) %l4 = bits to clear in interrupt register 4504 * 4505 * Internal: 4506 * %l4, %l5: local variables 4507 * %l6 = %y 4508 * %l7 = %g1 4509 * %g2..%g7 go to stack 4510 * 4511 * An interrupt frame is built in the space for a full trapframe; 4512 * this contains the psr, pc, npc, and interrupt level. 4513 * 4514 * The level of this interrupt is determined by: 4515 * 4516 * IRQ# = %tt - 0x40 4517 */ 4518 4519 ENTRY_NOPROFILE(sparc_interrupt) 4520 #ifdef TRAPS_USE_IG 4521 ! This is for interrupt debugging 4522 wrpr %g0, PSTATE_KERN|PSTATE_IG, %pstate ! DEBUG 4523 #endif 4524 /* 4525 * If this is a %tick or %stick softint, clear it then call 4526 * interrupt_vector. Only one of them should be enabled at any given 4527 * time. 4528 */ 4529 rd SOFTINT, %g1 4530 set TICK_INT|STICK_INT, %g5 4531 andcc %g5, %g1, %g5 4532 bz,pt %icc, 0f 4533 sethi %hi(CPUINFO_VA+CI_TICK_IH), %g3 4534 wr %g0, %g5, CLEAR_SOFTINT 4535 ba,pt %icc, setup_sparcintr 4536 LDPTR [%g3 + %lo(CPUINFO_VA+CI_TICK_IH)], %g5 4537 0: 4538 4539 #ifdef TRAPSTATS 4540 sethi %hi(_C_LABEL(kintrcnt)), %g1 4541 sethi %hi(_C_LABEL(uintrcnt)), %g2 4542 or %g1, %lo(_C_LABEL(kintrcnt)), %g1 4543 or %g1, %lo(_C_LABEL(uintrcnt)), %g2 4544 rdpr %tl, %g3 4545 dec %g3 4546 movrz %g3, %g2, %g1 4547 lduw [%g1], %g2 4548 inc %g2 4549 stw %g2, [%g1] 4550 /* See if we're on the interrupt stack already. */ 4551 set EINTSTACK, %g2 4552 set (EINTSTACK-INTSTACK), %g1 4553 btst 1, %sp 4554 add %sp, BIAS, %g3 4555 movz %icc, %sp, %g3 4556 srl %g3, 0, %g3 4557 sub %g2, %g3, %g3 4558 cmp %g3, %g1 4559 bgu 1f 4560 set _C_LABEL(intristk), %g1 4561 lduw [%g1], %g2 4562 inc %g2 4563 stw %g2, [%g1] 4564 1: 4565 #endif 4566 INTR_SETUP(-CC64FSZ-TF_SIZE) 4567 4568 ! Switch to normal globals so we can save them 4569 #ifdef SUN4V 4570 sethi %hi(cputyp), %g5 4571 ld [%g5 + %lo(cputyp)], %g5 4572 cmp %g5, CPU_SUN4V 4573 bne,pt %icc, 1f 4574 nop 4575 NORMAL_GLOBALS_SUN4V 4576 ! Save the normal globals 4577 stx %g1, [%sp + CC64FSZ + STKB + TF_G + ( 1*8)] 4578 stx %g2, [%sp + CC64FSZ + STKB + TF_G + ( 2*8)] 4579 stx %g3, [%sp + CC64FSZ + STKB + TF_G + ( 3*8)] 4580 stx %g4, [%sp + CC64FSZ + STKB + TF_G + ( 4*8)] 4581 stx %g5, [%sp + CC64FSZ + STKB + TF_G + ( 5*8)] 4582 stx %g6, [%sp + CC64FSZ + STKB + TF_G + ( 6*8)] 4583 stx %g7, [%sp + CC64FSZ + STKB + TF_G + ( 7*8)] 4584 4585 /* 4586 * In the EMBEDANY memory model %g4 points to the start of the 4587 * data segment. In our case we need to clear it before calling 4588 * any C-code. 4589 */ 4590 clr %g4 4591 4592 ba 2f 4593 nop 4594 1: 4595 #endif 4596 NORMAL_GLOBALS_SUN4U 4597 ! Save the normal globals 4598 stx %g1, [%sp + CC64FSZ + STKB + TF_G + ( 1*8)] 4599 stx %g2, [%sp + CC64FSZ + STKB + TF_G + ( 2*8)] 4600 stx %g3, [%sp + CC64FSZ + STKB + TF_G + ( 3*8)] 4601 stx %g4, [%sp + CC64FSZ + STKB + TF_G + ( 4*8)] 4602 stx %g5, [%sp + CC64FSZ + STKB + TF_G + ( 5*8)] 4603 stx %g6, [%sp + CC64FSZ + STKB + TF_G + ( 6*8)] 4604 stx %g7, [%sp + CC64FSZ + STKB + TF_G + ( 7*8)] 4605 4606 /* 4607 * In the EMBEDANY memory model %g4 points to the start of the 4608 * data segment. In our case we need to clear it before calling 4609 * any C-code. 4610 */ 4611 clr %g4 4612 4613 flushw ! Do not remove this insn -- causes interrupt loss 4614 4615 2: 4616 rd %y, %l6 4617 INCR64(CPUINFO_VA+CI_NINTR) ! cnt.v_ints++ (clobbers %o0,%o1) 4618 rdpr %tt, %l5 ! Find out our current IPL 4619 rdpr %tstate, %l0 4620 rdpr %tpc, %l1 4621 rdpr %tnpc, %l2 4622 rdpr %tl, %l3 ! Dump our trap frame now we have taken the IRQ 4623 stw %l6, [%sp + CC64FSZ + STKB + TF_Y] ! Silly, but we need to save this for rft 4624 dec %l3 4625 wrpr %g0, %l3, %tl 4626 sth %l5, [%sp + CC64FSZ + STKB + TF_TT]! debug 4627 stx %l0, [%sp + CC64FSZ + STKB + TF_TSTATE] ! set up intrframe/clockframe 4628 stx %l1, [%sp + CC64FSZ + STKB + TF_PC] 4629 btst TSTATE_PRIV, %l0 ! User mode? 4630 stx %l2, [%sp + CC64FSZ + STKB + TF_NPC] 4631 4632 sub %l5, 0x40, %l6 ! Convert to interrupt level 4633 sethi %hi(_C_LABEL(intr_evcnts)), %l4 4634 stb %l6, [%sp + CC64FSZ + STKB + TF_PIL] ! set up intrframe/clockframe 4635 rdpr %pil, %o1 4636 mulx %l6, EVC_SIZE, %l3 4637 or %l4, %lo(_C_LABEL(intr_evcnts)), %l4 ! intrcnt[intlev]++; 4638 stb %o1, [%sp + CC64FSZ + STKB + TF_OLDPIL] ! old %pil 4639 ldx [%l4 + %l3], %o0 4640 add %l4, %l3, %l4 4641 clr %l5 ! Zero handled count 4642 #ifdef MULTIPROCESSOR 4643 mov 1, %l3 ! Ack softint 4644 1: add %o0, 1, %l7 4645 casxa [%l4] ASI_N, %o0, %l7 4646 cmp %o0, %l7 4647 bne,a,pn %xcc, 1b ! retry if changed 4648 mov %l7, %o0 4649 #else 4650 inc %o0 4651 mov 1, %l3 ! Ack softint 4652 stx %o0, [%l4] 4653 #endif 4654 sll %l3, %l6, %l3 ! Generate IRQ mask 4655 4656 wrpr %l6, %pil 4657 4658 #define SOFTINT_INT \ 4659 (1<<IPL_SOFTCLOCK|1<<IPL_SOFTBIO|1<<IPL_SOFTNET|1<<IPL_SOFTSERIAL) 4660 4661 ! Increment the per-cpu interrupt depth in case of hardintrs 4662 btst SOFTINT_INT, %l3 4663 bnz,pn %icc, sparc_intr_retry 4664 sethi %hi(CPUINFO_VA+CI_IDEPTH), %l1 4665 ld [%l1 + %lo(CPUINFO_VA+CI_IDEPTH)], %l2 4666 inc %l2 4667 st %l2, [%l1 + %lo(CPUINFO_VA+CI_IDEPTH)] 4668 4669 sparc_intr_retry: 4670 wr %l3, 0, CLEAR_SOFTINT ! (don't clear possible %tick IRQ) 4671 sethi %hi(CPUINFO_VA+CI_INTRPENDING), %l4 4672 sll %l6, PTRSHFT, %l2 4673 or %l4, %lo(CPUINFO_VA+CI_INTRPENDING), %l4 4674 add %l2, %l4, %l4 4675 4676 1: 4677 membar #StoreLoad ! Make sure any failed casxa insns complete 4678 LDPTR [%l4], %l2 ! Check a slot 4679 cmp %l2, -1 4680 beq,pn CCCR, intrcmplt ! Empty list? 4681 mov -1, %l7 4682 membar #LoadStore 4683 CASPTRA [%l4] ASI_N, %l2, %l7 ! Grab the entire list 4684 cmp %l7, %l2 4685 bne,pn CCCR, 1b 4686 add %sp, CC64FSZ+STKB, %o2 ! tf = %sp + CC64FSZ + STKB 4687 LDPTR [%l2 + IH_PEND], %l7 4688 cmp %l7, -1 ! Last slot? 4689 be,pt CCCR, 3f 4690 membar #LoadStore 4691 4692 /* 4693 * Reverse a pending list since setup_sparcintr/send_softint 4694 * makes it in a LIFO order. 4695 */ 4696 mov -1, %o0 ! prev = -1 4697 1: STPTR %o0, [%l2 + IH_PEND] ! ih->ih_pending = prev 4698 mov %l2, %o0 ! prev = ih 4699 mov %l7, %l2 ! ih = ih->ih_pending 4700 LDPTR [%l2 + IH_PEND], %l7 4701 cmp %l7, -1 ! Last slot? 4702 bne,pn CCCR, 1b 4703 membar #LoadStore 4704 ba,pt CCCR, 3f 4705 mov %o0, %l7 ! save ih->ih_pending 4706 4707 2: 4708 add %sp, CC64FSZ+STKB, %o2 ! tf = %sp + CC64FSZ + STKB 4709 LDPTR [%l2 + IH_PEND], %l7 ! save ih->ih_pending 4710 membar #LoadStore 4711 3: 4712 STPTR %g0, [%l2 + IH_PEND] ! Clear pending flag 4713 membar #Sync 4714 LDPTR [%l2 + IH_FUN], %o4 ! ih->ih_fun 4715 LDPTR [%l2 + IH_ARG], %o0 ! ih->ih_arg 4716 4717 #ifdef NOT_DEBUG 4718 set _C_LABEL(intrdebug), %o3 4719 ld [%o2], %o3 4720 btst INTRDEBUG_FUNC, %o3 4721 bz,a,pt %icc, 97f 4722 nop 4723 4724 cmp %l6, 0xa ! ignore clock interrupts? 4725 bz,pt %icc, 97f 4726 nop 4727 4728 STACKFRAME(-CC64FSZ) ! Get a clean register window 4729 LOAD_ASCIZ(%o0, "sparc_interrupt: func %p arg %p\n") 4730 mov %i0, %o2 ! arg 4731 GLOBTOLOC 4732 call prom_printf 4733 mov %i4, %o1 ! func 4734 LOCTOGLOB 4735 restore 4736 97: 4737 mov %l4, %o1 4738 #endif 4739 4740 wrpr %g0, PSTATE_INTR, %pstate ! Reenable interrupts 4741 jmpl %o4, %o7 ! handled = (*ih->ih_fun)(...) 4742 movrz %o0, %o2, %o0 ! arg = (arg == 0) ? arg : tf 4743 wrpr %g0, PSTATE_KERN, %pstate ! Disable interrupts 4744 LDPTR [%l2 + IH_CLR], %l1 4745 membar #Sync 4746 4747 brz,pn %l1, 0f 4748 add %l5, %o0, %l5 4749 stx %g0, [%l1] ! Clear intr source 4750 membar #Sync ! Should not be needed 4751 0: 4752 LDPTR [%l2 + IH_ACK], %l1 ! ih->ih_ack 4753 brz,pn %l1, 1f 4754 nop 4755 jmpl %l1, %o7 ! (*ih->ih_ack)(ih) 4756 mov %l2, %o0 4757 1: 4758 cmp %l7, -1 4759 bne,pn CCCR, 2b ! 'Nother? 4760 mov %l7, %l2 4761 4762 intrcmplt: 4763 /* 4764 * Re-read SOFTINT to see if any new pending interrupts 4765 * at this level. 4766 */ 4767 mov 1, %l3 ! Ack softint 4768 rd SOFTINT, %l7 ! %l5 contains #intr handled. 4769 sll %l3, %l6, %l3 ! Generate IRQ mask 4770 btst %l3, %l7 ! leave mask in %l3 for retry code 4771 bnz,pn %icc, sparc_intr_retry 4772 mov 1, %l5 ! initialize intr count for next run 4773 4774 ! Decrement this cpu's interrupt depth in case of hardintrs 4775 btst SOFTINT_INT, %l3 4776 bnz,pn %icc, 1f 4777 sethi %hi(CPUINFO_VA+CI_IDEPTH), %l4 4778 ld [%l4 + %lo(CPUINFO_VA+CI_IDEPTH)], %l5 4779 dec %l5 4780 st %l5, [%l4 + %lo(CPUINFO_VA+CI_IDEPTH)] 4781 1: 4782 4783 #ifdef NOT_DEBUG 4784 set _C_LABEL(intrdebug), %o2 4785 ld [%o2], %o2 4786 btst INTRDEBUG_FUNC, %o2 4787 bz,a,pt %icc, 97f 4788 nop 4789 4790 cmp %l6, 0xa ! ignore clock interrupts? 4791 bz,pt %icc, 97f 4792 nop 4793 4794 STACKFRAME(-CC64FSZ) ! Get a clean register window 4795 LOAD_ASCIZ(%o0, "sparc_interrupt: done\n") 4796 GLOBTOLOC 4797 call prom_printf 4798 nop 4799 LOCTOGLOB 4800 restore 4801 97: 4802 #endif 4803 4804 ldub [%sp + CC64FSZ + STKB + TF_OLDPIL], %l3 ! restore old %pil 4805 wrpr %l3, 0, %pil 4806 4807 b return_from_trap 4808 ldx [%sp + CC64FSZ + STKB + TF_TSTATE], %g1 ! Load this for return_from_trap 4809 4810 #ifdef notyet 4811 /* 4812 * Level 12 (ZS serial) interrupt. Handle it quickly, schedule a 4813 * software interrupt, and get out. Do the software interrupt directly 4814 * if we would just take it on the way out. 4815 * 4816 * Input: 4817 * %l0 = %psr 4818 * %l1 = return pc 4819 * %l2 = return npc 4820 * Internal: 4821 * %l3 = zs device 4822 * %l4, %l5 = temporary 4823 * %l6 = rr3 (or temporary data) + 0x100 => need soft int 4824 * %l7 = zs soft status 4825 */ 4826 zshard: 4827 #endif /* notyet */ 4828 4829 .globl return_from_trap, rft_kernel, rft_user 4830 .globl softtrap, slowtrap 4831 4832 /* 4833 * Various return-from-trap routines (see return_from_trap). 4834 */ 4835 4836 /* 4837 * Return from trap. 4838 * registers are: 4839 * 4840 * [%sp + CC64FSZ + STKB] => trap frame 4841 * %g1 => tstate from trap frame 4842 * 4843 * We must load all global, out, and trap registers from the trap frame. 4844 * 4845 * If returning to kernel, we should be at the proper trap level because 4846 * we don't touch %tl. 4847 * 4848 * When returning to user mode, the trap level does not matter, as it 4849 * will be set explicitly. 4850 * 4851 * If we are returning to user code, we must: 4852 * 1. Check for register windows in the pcb that belong on the stack. 4853 * If there are any, reload them 4854 */ 4855 return_from_trap: 4856 #ifdef DEBUG 4857 !! Make sure we don't have pc == npc == 0 or we suck. 4858 ldx [%sp + CC64FSZ + STKB + TF_PC], %g2 4859 ldx [%sp + CC64FSZ + STKB + TF_NPC], %g3 4860 orcc %g2, %g3, %g0 4861 tz %icc, 1 4862 #endif 4863 4864 !! 4865 !! We'll make sure we flush our pcb here, rather than later. 4866 !! 4867 ! ldx [%sp + CC64FSZ + STKB + TF_TSTATE], %g1 ! already passed in, no need to reload 4868 btst TSTATE_PRIV, %g1 ! returning to userland? 4869 4870 !! 4871 !! Let all pending interrupts drain before returning to userland 4872 !! 4873 bnz,pn %icc, 1f ! Returning to userland? 4874 nop 4875 ENABLE_INTERRUPTS %g5 4876 wrpr %g0, %g0, %pil ! Lower IPL 4877 1: 4878 !! Make sure we have no IRQs 4879 DISABLE_INTERRUPTS %g5 4880 4881 #ifdef SUN4V 4882 sethi %hi(cputyp), %g5 4883 ld [%g5 + %lo(cputyp)], %g5 4884 cmp %g5, CPU_SUN4V 4885 bne,pt %icc, 1f 4886 nop 4887 !! Make sure we have normal globals 4888 NORMAL_GLOBALS_SUN4V 4889 /* Restore normal globals */ 4890 ldx [%sp + CC64FSZ + STKB + TF_G + (1*8)], %g1 4891 ldx [%sp + CC64FSZ + STKB + TF_G + (2*8)], %g2 4892 ldx [%sp + CC64FSZ + STKB + TF_G + (3*8)], %g3 4893 ldx [%sp + CC64FSZ + STKB + TF_G + (4*8)], %g4 4894 ldx [%sp + CC64FSZ + STKB + TF_G + (5*8)], %g5 4895 ldx [%sp + CC64FSZ + STKB + TF_G + (6*8)], %g6 4896 ldx [%sp + CC64FSZ + STKB + TF_G + (7*8)], %g7 4897 /* Switch to alternate globals */ 4898 ALTERNATE_GLOBALS_SUN4V 4899 ba 2f 4900 nop 4901 1: 4902 #endif 4903 !! Make sure we have normal globals 4904 NORMAL_GLOBALS_SUN4U 4905 /* Restore normal globals */ 4906 ldx [%sp + CC64FSZ + STKB + TF_G + (1*8)], %g1 4907 ldx [%sp + CC64FSZ + STKB + TF_G + (2*8)], %g2 4908 ldx [%sp + CC64FSZ + STKB + TF_G + (3*8)], %g3 4909 ldx [%sp + CC64FSZ + STKB + TF_G + (4*8)], %g4 4910 ldx [%sp + CC64FSZ + STKB + TF_G + (5*8)], %g5 4911 ldx [%sp + CC64FSZ + STKB + TF_G + (6*8)], %g6 4912 ldx [%sp + CC64FSZ + STKB + TF_G + (7*8)], %g7 4913 /* Switch to alternate globals */ 4914 #ifdef TRAPS_USE_IG 4915 wrpr %g0, PSTATE_KERN|PSTATE_IG, %pstate ! DEBUG 4916 #else 4917 ALTERNATE_GLOBALS_SUN4U 4918 #endif 4919 2: 4920 4921 /* Load outs */ 4922 ldx [%sp + CC64FSZ + STKB + TF_O + (0*8)], %i0 4923 ldx [%sp + CC64FSZ + STKB + TF_O + (1*8)], %i1 4924 ldx [%sp + CC64FSZ + STKB + TF_O + (2*8)], %i2 4925 ldx [%sp + CC64FSZ + STKB + TF_O + (3*8)], %i3 4926 ldx [%sp + CC64FSZ + STKB + TF_O + (4*8)], %i4 4927 ldx [%sp + CC64FSZ + STKB + TF_O + (5*8)], %i5 4928 ldx [%sp + CC64FSZ + STKB + TF_O + (6*8)], %i6 4929 ldx [%sp + CC64FSZ + STKB + TF_O + (7*8)], %i7 4930 /* Now load trap registers into alternate globals */ 4931 ld [%sp + CC64FSZ + STKB + TF_Y], %g4 4932 ldx [%sp + CC64FSZ + STKB + TF_TSTATE], %g1 ! load new values 4933 wr %g4, 0, %y 4934 ldx [%sp + CC64FSZ + STKB + TF_PC], %g2 4935 ldx [%sp + CC64FSZ + STKB + TF_NPC], %g3 4936 4937 #ifdef NOTDEF_DEBUG 4938 ldub [%sp + CC64FSZ + STKB + TF_PIL], %g5 ! restore %pil 4939 wrpr %g5, %pil ! DEBUG 4940 #endif 4941 4942 /* Returning to user mode or kernel mode? */ 4943 btst TSTATE_PRIV, %g1 ! returning to userland? 4944 bz,pt %icc, rft_user 4945 sethi %hi(CPUINFO_VA+CI_WANT_AST), %g7 ! first instr of rft_user 4946 4947 /* 4948 * Return from trap, to kernel. 4949 * 4950 * We will assume, for the moment, that all kernel traps are properly stacked 4951 * in the trap registers, so all we have to do is insert the (possibly modified) 4952 * register values into the trap registers then do a retry. 4953 * 4954 */ 4955 rft_kernel: 4956 rdpr %tl, %g4 ! Grab a set of trap registers 4957 inc %g4 4958 wrpr %g4, %g0, %tl 4959 wrpr %g3, 0, %tnpc 4960 wrpr %g2, 0, %tpc 4961 wrpr %g1, 0, %tstate 4962 4963 rdpr %canrestore, %g2 4964 brnz %g2, 1f 4965 nop 4966 4967 wr %g0, ASI_NUCLEUS, %asi 4968 rdpr %cwp, %g1 4969 dec %g1 4970 wrpr %g1, %cwp 4971 #ifdef _LP64 4972 FILL ldxa, %sp+BIAS, 8, %asi 4973 #else 4974 FILL lda, %sp, 4, %asi 4975 #endif 4976 restored 4977 inc %g1 4978 wrpr %g1, %cwp 4979 1: 4980 restore 4981 rdpr %tstate, %g1 ! Since we may have trapped our regs may be toast 4982 rdpr %cwp, %g2 4983 andn %g1, CWP, %g1 4984 wrpr %g1, %g2, %tstate ! Put %cwp in %tstate 4985 CLRTT 4986 #ifdef TRAPSTATS 4987 rdpr %tl, %g2 4988 set _C_LABEL(rftkcnt), %g1 4989 sllx %g2, 2, %g2 4990 add %g1, %g2, %g1 4991 lduw [%g1], %g2 4992 inc %g2 4993 stw %g2, [%g1] 4994 #endif 4995 #if 0 4996 wrpr %g0, 0, %cleanwin ! DEBUG 4997 #endif 4998 #if defined(DDB) && defined(MULTIPROCESSOR) 4999 set sparc64_ipi_pause_trap_point, %g1 5000 rdpr %tpc, %g2 5001 cmp %g1, %g2 5002 bne,pt %icc, 0f 5003 nop 5004 done 5005 0: 5006 #endif 5007 retry 5008 NOTREACHED 5009 /* 5010 * Return from trap, to user. Checks for scheduling trap (`ast') first; 5011 * will re-enter trap() if set. Note that we may have to switch from 5012 * the interrupt stack to the kernel stack in this case. 5013 * %g1 = %tstate 5014 * %g2 = return %pc 5015 * %g3 = return %npc 5016 * If returning to a valid window, just set psr and return. 5017 */ 5018 .data 5019 rft_wcnt: .word 0 5020 .text 5021 5022 rft_user: 5023 ! sethi %hi(CPUINFO_VA+CI_WANT_AST), %g7 ! (done above) 5024 lduw [%g7 + %lo(CPUINFO_VA+CI_WANT_AST)], %g7! want AST trap? 5025 brnz,pn %g7, softtrap ! yes, re-enter trap with type T_AST 5026 mov T_AST, %g4 5027 5028 #ifdef NOTDEF_DEBUG 5029 sethi %hi(CPCB), %g4 5030 LDPTR [%g4 + %lo(CPCB)], %g4 5031 ldub [%g4 + PCB_NSAVED], %g4 ! nsaved 5032 brz,pt %g4, 2f ! Only print if nsaved <> 0 5033 nop 5034 5035 set 1f, %o0 5036 mov %g4, %o1 5037 mov %g2, %o2 ! pc 5038 wr %g0, ASI_DMMU, %asi ! restore the user context 5039 ldxa [CTX_SECONDARY] %asi, %o3 ! ctx 5040 GLOBTOLOC 5041 mov %g3, %o5 5042 call printf 5043 mov %i6, %o4 ! sp 5044 ! wrpr %g0, PSTATE_INTR, %pstate ! Allow IRQ service 5045 ! wrpr %g0, PSTATE_KERN, %pstate ! DenyIRQ service 5046 LOCTOGLOB 5047 1: 5048 .data 5049 .asciz "rft_user: nsaved=%x pc=%d ctx=%x sp=%x npc=%p\n" 5050 _ALIGN 5051 .text 5052 #endif 5053 5054 /* 5055 * NB: only need to do this after a cache miss 5056 */ 5057 #ifdef TRAPSTATS 5058 set _C_LABEL(rftucnt), %g6 5059 lduw [%g6], %g7 5060 inc %g7 5061 stw %g7, [%g6] 5062 #endif 5063 /* 5064 * Now check to see if any regs are saved in the pcb and restore them. 5065 * 5066 * Here we need to undo the damage caused by switching to a kernel 5067 * stack. 5068 * 5069 * We will use alternate globals %g4..%g7 because %g1..%g3 are used 5070 * by the data fault trap handlers and we don't want possible conflict. 5071 */ 5072 5073 sethi %hi(CPCB), %g6 5074 rdpr %otherwin, %g7 ! restore register window controls 5075 #ifdef DEBUG 5076 rdpr %canrestore, %g5 ! DEBUG 5077 tst %g5 ! DEBUG 5078 tnz %icc, 1; nop ! DEBUG 5079 ! mov %g0, %g5 ! There should be *NO* %canrestore 5080 add %g7, %g5, %g7 ! DEBUG 5081 #endif 5082 wrpr %g0, %g7, %canrestore 5083 LDPTR [%g6 + %lo(CPCB)], %g6 5084 wrpr %g0, 0, %otherwin 5085 5086 ldub [%g6 + PCB_NSAVED], %g7 ! Any saved reg windows? 5087 wrpr %g0, WSTATE_USER, %wstate ! Need to know where our sp points 5088 5089 #ifdef DEBUG 5090 set rft_wcnt, %g4 ! Keep track of all the windows we restored 5091 stw %g7, [%g4] 5092 #endif 5093 5094 brz,pt %g7, 5f ! No saved reg wins 5095 nop 5096 dec %g7 ! We can do this now or later. Move to last entry 5097 5098 #ifdef DEBUG 5099 rdpr %canrestore, %g4 ! DEBUG Make sure we've restored everything 5100 brnz,a,pn %g4, 0f ! DEBUG 5101 sir ! DEBUG we should NOT have any usable windows here 5102 0: ! DEBUG 5103 wrpr %g0, 5, %tl 5104 #endif 5105 rdpr %otherwin, %g4 5106 sll %g7, 7, %g5 ! calculate ptr into rw64 array 8*16 == 128 or 7 bits 5107 brz,pt %g4, 6f ! We should not have any user windows left 5108 add %g5, %g6, %g5 5109 5110 set 1f, %o0 5111 mov %g7, %o1 5112 mov %g4, %o2 5113 call printf 5114 wrpr %g0, PSTATE_KERN, %pstate 5115 set 2f, %o0 5116 call panic 5117 nop 5118 NOTREACHED 5119 .data 5120 1: .asciz "pcb_nsaved=%x and otherwin=%x\n" 5121 2: .asciz "rft_user\n" 5122 _ALIGN 5123 .text 5124 6: 5125 3: 5126 restored ! Load in the window 5127 restore ! This should not trap! 5128 ldx [%g5 + PCB_RW + ( 0*8)], %l0 ! Load the window from the pcb 5129 ldx [%g5 + PCB_RW + ( 1*8)], %l1 5130 ldx [%g5 + PCB_RW + ( 2*8)], %l2 5131 ldx [%g5 + PCB_RW + ( 3*8)], %l3 5132 ldx [%g5 + PCB_RW + ( 4*8)], %l4 5133 ldx [%g5 + PCB_RW + ( 5*8)], %l5 5134 ldx [%g5 + PCB_RW + ( 6*8)], %l6 5135 ldx [%g5 + PCB_RW + ( 7*8)], %l7 5136 5137 ldx [%g5 + PCB_RW + ( 8*8)], %i0 5138 ldx [%g5 + PCB_RW + ( 9*8)], %i1 5139 ldx [%g5 + PCB_RW + (10*8)], %i2 5140 ldx [%g5 + PCB_RW + (11*8)], %i3 5141 ldx [%g5 + PCB_RW + (12*8)], %i4 5142 ldx [%g5 + PCB_RW + (13*8)], %i5 5143 ldx [%g5 + PCB_RW + (14*8)], %i6 5144 ldx [%g5 + PCB_RW + (15*8)], %i7 5145 5146 #ifdef DEBUG 5147 stx %g0, [%g5 + PCB_RW + (14*8)] ! DEBUG mark that we've saved this one 5148 #endif 5149 5150 cmp %g5, %g6 5151 bgu,pt %xcc, 3b ! Next one? 5152 dec 8*16, %g5 5153 5154 stb %g0, [%g6 + PCB_NSAVED] ! Clear them out so we won't do this again 5155 GET_MAXCWP %g5 5156 add %g5, %g7, %g4 5157 dec 1, %g5 ! NWINDOWS-1-1 5158 wrpr %g5, 0, %cansave 5159 wrpr %g0, 0, %canrestore ! Make sure we have no freeloaders XXX 5160 wrpr %g0, WSTATE_USER, %wstate ! Save things to user space 5161 mov %g7, %g5 ! We already did one restore 5162 4: 5163 rdpr %canrestore, %g4 5164 inc %g4 5165 deccc %g5 5166 wrpr %g4, 0, %cleanwin ! Make *sure* we don't trap to cleanwin 5167 bge,a,pt %xcc, 4b ! return to starting regwin 5168 save %g0, %g0, %g0 ! This may force a datafault 5169 5170 #ifdef DEBUG 5171 wrpr %g0, 0, %tl 5172 #endif 5173 #ifdef TRAPSTATS 5174 set _C_LABEL(rftuld), %g5 5175 lduw [%g5], %g4 5176 inc %g4 5177 stw %g4, [%g5] 5178 #endif 5179 !! 5180 !! We can't take any save faults in here 'cause they will never be serviced 5181 !! 5182 5183 #ifdef DEBUG 5184 sethi %hi(CPCB), %g5 5185 LDPTR [%g5 + %lo(CPCB)], %g5 5186 ldub [%g5 + PCB_NSAVED], %g5 ! Any saved reg windows? 5187 tst %g5 5188 tnz %icc, 1; nop ! Debugger if we still have saved windows 5189 bne,a rft_user ! Try starting over again 5190 sethi %hi(CPUINFO_VA+CI_WANT_AST), %g7 5191 #endif 5192 /* 5193 * Set up our return trapframe so we can recover if we trap from here 5194 * on in. 5195 */ 5196 wrpr %g0, 1, %tl ! Set up the trap state 5197 wrpr %g2, 0, %tpc 5198 wrpr %g3, 0, %tnpc 5199 ba,pt %icc, 6f 5200 wrpr %g1, %g0, %tstate 5201 5202 5: 5203 /* 5204 * Set up our return trapframe so we can recover if we trap from here 5205 * on in. 5206 */ 5207 wrpr %g0, 1, %tl ! Set up the trap state 5208 wrpr %g2, 0, %tpc 5209 wrpr %g3, 0, %tnpc 5210 wrpr %g1, %g0, %tstate 5211 5212 /* 5213 * The restore instruction further down may cause the trap level 5214 * to exceed the maximum trap level on sun4v, so a manual fill 5215 * may be necessary. 5216 */ 5217 5218 #ifdef SUN4V 5219 sethi %hi(cputyp), %g5 5220 ld [%g5 + %lo(cputyp)], %g5 5221 cmp %g5, CPU_SUN4V 5222 bne,pt %icc, 1f 5223 nop 5224 5225 ! Only manual fill if the restore instruction will cause a fill trap 5226 rdpr %canrestore, %g5 5227 brnz %g5, 1f 5228 nop 5229 5230 ! Do a manual fill 5231 wr %g0, ASI_AIUS, %asi 5232 rdpr %cwp, %g4 5233 dec %g4 5234 wrpr %g4, 0, %cwp 5235 rft_user_fault_start: 5236 FILL ldxa, %sp+BIAS, 8, %asi 5237 rft_user_fault_end: 5238 restored 5239 inc %g4 5240 wrpr %g4, 0, %cwp 5241 1: 5242 #endif 5243 restore 5244 6: 5245 rdpr %canrestore, %g5 5246 wrpr %g5, 0, %cleanwin ! Force cleanup of kernel windows 5247 5248 #ifdef NOTDEF_DEBUG 5249 ldx [%g6 + CC64FSZ + STKB + TF_L + (0*8)], %g5! DEBUG -- get proper value for %l0 5250 cmp %l0, %g5 5251 be,a,pt %icc, 1f 5252 nop 5253 ! sir ! WATCHDOG 5254 set badregs, %g1 ! Save the suspect regs 5255 stw %l0, [%g1+(4*0)] 5256 stw %l1, [%g1+(4*1)] 5257 stw %l2, [%g1+(4*2)] 5258 stw %l3, [%g1+(4*3)] 5259 stw %l4, [%g1+(4*4)] 5260 stw %l5, [%g1+(4*5)] 5261 stw %l6, [%g1+(4*6)] 5262 stw %l7, [%g1+(4*7)] 5263 stw %i0, [%g1+(4*8)+(4*0)] 5264 stw %i1, [%g1+(4*8)+(4*1)] 5265 stw %i2, [%g1+(4*8)+(4*2)] 5266 stw %i3, [%g1+(4*8)+(4*3)] 5267 stw %i4, [%g1+(4*8)+(4*4)] 5268 stw %i5, [%g1+(4*8)+(4*5)] 5269 stw %i6, [%g1+(4*8)+(4*6)] 5270 stw %i7, [%g1+(4*8)+(4*7)] 5271 save 5272 inc %g7 5273 wrpr %g7, 0, %otherwin 5274 wrpr %g0, 0, %canrestore 5275 wrpr %g0, WSTATE_KERN, %wstate ! Need to know where our sp points 5276 set rft_wcnt, %g4 ! Restore nsaved before trapping 5277 sethi %hi(CPCB), %g6 5278 LDPTR [%g6 + %lo(CPCB)], %g6 5279 lduw [%g4], %g4 5280 stb %g4, [%g6 + PCB_NSAVED] 5281 ta 1 5282 sir 5283 .data 5284 badregs: 5285 .space 16*4 5286 .text 5287 1: 5288 #endif 5289 5290 rdpr %tstate, %g1 5291 rdpr %cwp, %g7 ! Find our cur window 5292 andn %g1, CWP, %g1 ! Clear it from %tstate 5293 wrpr %g1, %g7, %tstate ! Set %tstate with %cwp 5294 mov CTX_SECONDARY, %g1 ! Restore the user context 5295 GET_MMU_CONTEXTID %g4, %g1, %g3 5296 mov CTX_PRIMARY, %g2 5297 SET_MMU_CONTEXTID %g4, %g2, %g3 5298 sethi %hi(KERNBASE), %g7 ! Should not be needed due to retry 5299 membar #Sync ! Should not be needed due to retry 5300 flush %g7 ! Should not be needed due to retry 5301 5302 CLRTT 5303 #ifdef TRAPSTATS 5304 set _C_LABEL(rftudone), %g1 5305 lduw [%g1], %g2 5306 inc %g2 5307 stw %g2, [%g1] 5308 #endif 5309 #ifdef DEBUG 5310 sethi %hi(CPCB), %g5 5311 LDPTR [%g5 + %lo(CPCB)], %g5 5312 ldub [%g5 + PCB_NSAVED], %g5 ! Any saved reg windows? 5313 tst %g5 5314 tnz %icc, 1; nop ! Debugger if we still have saved windows! 5315 #endif 5316 wrpr %g0, 0, %pil ! Enable all interrupts 5317 retry 5318 5319 ! exported end marker for kernel gdb 5320 .globl _C_LABEL(endtrapcode) 5321 _C_LABEL(endtrapcode): 5322 5323 /* 5324 * Kernel entry point. 5325 * 5326 * The contract between bootloader and kernel is: 5327 * 5328 * %o0 OpenFirmware entry point, to keep Sun's updaters happy 5329 * %o1 Address of boot information vector (see bootinfo.h) 5330 * %o2 Length of the vector, in bytes 5331 * %o3 OpenFirmware entry point, to mimic Sun bootloader behavior 5332 * %o4 OpenFirmware, to meet earlier NetBSD kernels expectations 5333 */ 5334 .align 8 5335 start: 5336 dostart: 5337 /* 5338 * Startup. 5339 * 5340 * The Sun FCODE bootloader is nice and loads us where we want 5341 * to be. We have a full set of mappings already set up for us. 5342 * 5343 * I think we end up having an entire 16M allocated to us. 5344 * 5345 * We enter with the prom entry vector in %o0, dvec in %o1, 5346 * and the bootops vector in %o2. 5347 * 5348 * All we need to do is: 5349 * 5350 * 1: Save the prom vector 5351 * 5352 * 2: Create a decent stack for ourselves 5353 * 5354 * 3: Install the permanent 4MB kernel mapping 5355 * 5356 * 4: Call the C language initialization code 5357 * 5358 */ 5359 5360 /* 5361 * Set the psr into a known state: 5362 * Set supervisor mode, interrupt level >= 13, traps enabled 5363 */ 5364 wrpr %g0, 13, %pil 5365 wrpr %g0, PSTATE_INTR|PSTATE_PEF, %pstate 5366 wr %g0, FPRS_FEF, %fprs ! Turn on FPU 5367 5368 /* 5369 * Step 2: Set up a v8-like stack if we need to 5370 */ 5371 5372 #ifdef _LP64 5373 btst 1, %sp 5374 bnz,pt %icc, 0f 5375 nop 5376 add %sp, -BIAS, %sp 5377 #else 5378 btst 1, %sp 5379 bz,pt %icc, 0f 5380 nop 5381 add %sp, BIAS, %sp 5382 #endif 5383 0: 5384 5385 call _C_LABEL(bootstrap) 5386 clr %g4 ! Clear data segment pointer 5387 5388 /* 5389 * Initialize the boot CPU. Basically: 5390 * 5391 * Locate the cpu_info structure for this CPU. 5392 * Establish a locked mapping for interrupt stack. 5393 * Switch to the initial stack. 5394 * Call the routine passed in in cpu_info->ci_spinup 5395 */ 5396 5397 #ifdef NO_VCACHE 5398 #define SUN4U_TTE_DATABITS SUN4U_TTE_L|SUN4U_TTE_CP|SUN4U_TTE_P|SUN4U_TTE_W 5399 #else 5400 #define SUN4U_TTE_DATABITS SUN4U_TTE_L|SUN4U_TTE_CP|SUN4U_TTE_CV|SUN4U_TTE_P|SUN4U_TTE_W 5401 #endif 5402 5403 5404 ENTRY_NOPROFILE(cpu_initialize) /* for cosmetic reasons - nicer backtrace */ 5405 5406 /* Cache the cputyp in %l6 for later use below */ 5407 sethi %hi(cputyp), %l6 5408 ld [%l6 + %lo(cputyp)], %l6 5409 5410 /* 5411 * Step 5: is no more. 5412 */ 5413 5414 /* 5415 * Step 6: hunt through cpus list and find the one that matches our cpuid 5416 */ 5417 5418 call _C_LABEL(cpu_myid) ! Retrieve cpuid in %o0 5419 mov %g0, %o0 5420 5421 sethi %hi(_C_LABEL(cpus)), %l1 5422 LDPTR [%l1 + %lo(_C_LABEL(cpus))], %l1 5423 0: 5424 ld [%l1 + CI_CPUID], %l3 ! Load CPUID 5425 cmp %l3, %o0 ! Does it match? 5426 bne,a,pt %icc, 0b ! no 5427 LDPTR [%l1 + CI_NEXT], %l1 ! Load next cpu_info pointer 5428 5429 /* 5430 * Get pointer to our cpu_info struct 5431 */ 5432 mov %l1, %l7 ! save cpu_info pointer 5433 ldx [%l1 + CI_PADDR], %l1 ! Load the interrupt stack's PA 5434 #ifdef SUN4V 5435 cmp %l6, CPU_SUN4V 5436 bne,pt %icc, 3f 5437 nop 5438 5439 /* sun4v */ 5440 call _C_LABEL(pmap_setup_intstack_sun4v) ! Call nice C function for mapping INTSTACK 5441 mov %l1, %o0 5442 ba 4f 5443 nop 5444 3: 5445 #endif 5446 /* sun4u */ 5447 sethi %hi(0xa0000000), %l2 ! V=1|SZ=01|NFO=0|IE=0 5448 sllx %l2, 32, %l2 ! Shift it into place 5449 5450 mov -1, %l3 ! Create a nice mask 5451 sllx %l3, 43, %l4 ! Mask off high bits 5452 or %l4, 0xfff, %l4 ! We can just load this in 12 (of 13) bits 5453 5454 andn %l1, %l4, %l1 ! Mask the phys page number 5455 5456 or %l2, %l1, %l1 ! Now take care of the high bits 5457 or %l1, SUN4U_TTE_DATABITS, %l2 ! And low bits: L=1|CP=1|CV=?|E=0|P=1|W=1|G=0 5458 5459 !! 5460 !! Now, map in the interrupt stack as context==0 5461 !! 5462 set TLB_TAG_ACCESS, %l5 5463 set INTSTACK, %l0 5464 stxa %l0, [%l5] ASI_DMMU ! Make DMMU point to it 5465 stxa %l2, [%g0] ASI_DMMU_DATA_IN ! Store it 5466 membar #Sync 5467 4: 5468 5469 !! Setup kernel stack (we rely on curlwp on this cpu 5470 !! being lwp0 here and its uarea is mapped special 5471 !! and already accessible here) 5472 flushw 5473 LDPTR [%l7 + CI_CPCB], %l0 ! load PCB/uarea pointer 5474 set 2*USPACE - TF_SIZE - CC64FSZ, %l1 5475 add %l1, %l0, %l0 5476 #ifdef _LP64 5477 andn %l0, 0x0f, %l0 ! Needs to be 16-byte aligned 5478 sub %l0, BIAS, %l0 ! and biased 5479 #endif 5480 mov %l0, %sp 5481 flushw 5482 5483 #ifdef DEBUG 5484 set _C_LABEL(pmapdebug), %o1 5485 ld [%o1], %o1 5486 sethi %hi(0x40000), %o2 5487 btst %o2, %o1 5488 bz 0f 5489 5490 set 1f, %o0 ! Debug printf 5491 call _C_LABEL(prom_printf) 5492 nop 5493 .data 5494 1: 5495 .asciz "Setting trap base...\n" 5496 _ALIGN 5497 .text 5498 0: 5499 #endif 5500 /* 5501 * Step 7: change the trap base register, and install our TSB pointers 5502 */ 5503 5504 /* 5505 * install our TSB pointers 5506 */ 5507 5508 #ifdef SUN4V 5509 cmp %l6, CPU_SUN4V 5510 bne,pt %icc, 5f 5511 nop 5512 5513 /* sun4v */ 5514 LDPTR [%l7 + CI_TSB_DESC], %o0 5515 call _C_LABEL(pmap_setup_tsb_sun4v) 5516 nop 5517 ba 1f 5518 nop 5519 5: 5520 #endif 5521 /* sun4u */ 5522 sethi %hi(_C_LABEL(tsbsize)), %l2 5523 sethi %hi(0x1fff), %l3 5524 sethi %hi(TSB), %l4 5525 LDPTR [%l7 + CI_TSB_DMMU], %l0 5526 LDPTR [%l7 + CI_TSB_IMMU], %l1 5527 ld [%l2 + %lo(_C_LABEL(tsbsize))], %l2 5528 or %l3, %lo(0x1fff), %l3 5529 or %l4, %lo(TSB), %l4 5530 5531 andn %l0, %l3, %l0 ! Mask off size and split bits 5532 or %l0, %l2, %l0 ! Make a TSB pointer 5533 stxa %l0, [%l4] ASI_DMMU ! Install data TSB pointer 5534 5535 andn %l1, %l3, %l1 ! Mask off size and split bits 5536 or %l1, %l2, %l1 ! Make a TSB pointer 5537 stxa %l1, [%l4] ASI_IMMU ! Install instruction TSB pointer 5538 membar #Sync 5539 set 1f, %l1 5540 flush %l1 5541 1: 5542 5543 /* set trap table */ 5544 #ifdef SUN4V 5545 cmp %l6, CPU_SUN4V 5546 bne,pt %icc, 6f 5547 nop 5548 /* sun4v */ 5549 set _C_LABEL(trapbase_sun4v), %l1 5550 GET_MMFSA %o1 5551 call _C_LABEL(prom_set_trap_table_sun4v) ! Now we should be running 100% from our handlers 5552 mov %l1, %o0 5553 5554 ba 7f 5555 nop 5556 6: 5557 #endif 5558 /* sun4u */ 5559 set _C_LABEL(trapbase), %l1 5560 call _C_LABEL(prom_set_trap_table_sun4u) ! Now we should be running 100% from our handlers 5561 mov %l1, %o0 5562 7: 5563 wrpr %l1, 0, %tba ! Make sure the PROM didn't foul up. 5564 5565 /* 5566 * Switch to the kernel mode and run away. 5567 */ 5568 wrpr %g0, WSTATE_KERN, %wstate 5569 5570 #ifdef DEBUG 5571 wrpr %g0, 1, %tl ! Debug -- start at tl==3 so we'll watchdog 5572 wrpr %g0, 0x1ff, %tt ! Debug -- clear out unused trap regs 5573 wrpr %g0, 0, %tpc 5574 wrpr %g0, 0, %tnpc 5575 wrpr %g0, 0, %tstate 5576 wrpr %g0, 0, %tl 5577 #endif 5578 5579 #ifdef DEBUG 5580 set _C_LABEL(pmapdebug), %o1 5581 ld [%o1], %o1 5582 sethi %hi(0x40000), %o2 5583 btst %o2, %o1 5584 bz 0f 5585 5586 LDPTR [%l7 + CI_SPINUP], %o1 5587 set 1f, %o0 ! Debug printf 5588 call _C_LABEL(prom_printf) 5589 mov %sp, %o2 5590 5591 .data 5592 1: 5593 .asciz "Calling startup routine %p with stack at %p...\n" 5594 _ALIGN 5595 .text 5596 0: 5597 #endif 5598 /* 5599 * Call our startup routine. 5600 */ 5601 5602 LDPTR [%l7 + CI_SPINUP], %o1 5603 5604 call %o1 ! Call routine 5605 clr %o0 ! our frame arg is ignored 5606 5607 set 1f, %o0 ! Main should never come back here 5608 call _C_LABEL(panic) 5609 nop 5610 .data 5611 1: 5612 .asciz "main() returned\n" 5613 _ALIGN 5614 .text 5615 5616 .align 8 5617 ENTRY(get_romtba) 5618 retl 5619 rdpr %tba, %o0 5620 5621 ENTRY(setcputyp) 5622 sethi %hi(cputyp), %o1 ! Trash %o1 assuming this is ok 5623 st %o0, [%o1 + %lo(cputyp)] 5624 retl 5625 nop 5626 5627 #ifdef MULTIPROCESSOR 5628 /* 5629 * cpu_mp_startup is called with: 5630 * 5631 * %g2 = cpu_args 5632 */ 5633 ENTRY(cpu_mp_startup) 5634 mov 1, %o0 5635 sllx %o0, 63, %o0 5636 wr %o0, TICK_CMPR ! XXXXXXX clear and disable %tick_cmpr for now 5637 wrpr %g0, 0, %cleanwin 5638 wrpr %g0, 0, %tl ! Make sure we're not in NUCLEUS mode 5639 wrpr %g0, WSTATE_KERN, %wstate 5640 wrpr %g0, PSTATE_KERN, %pstate 5641 flushw 5642 5643 /* Cache the cputyp in %l6 for later use below */ 5644 sethi %hi(cputyp), %l6 5645 ld [%l6 + %lo(cputyp)], %l6 5646 5647 /* 5648 * Get pointer to our cpu_info struct 5649 */ 5650 ldx [%g2 + CBA_CPUINFO], %l1 ! Load the interrupt stack's PA 5651 5652 #ifdef SUN4V 5653 cmp %l6, CPU_SUN4V 5654 bne,pt %icc, 3f 5655 nop 5656 5657 /* sun4v */ 5658 5659 sethi %hi(0x80000000), %l2 ! V=1|NFO=0|SW=0 5660 sllx %l2, 32, %l2 ! Shift it into place 5661 mov -1, %l3 ! Create a nice mask 5662 sllx %l3, 56, %l4 ! Mask off high 8 bits 5663 or %l4, 0xfff, %l4 ! We can just load this in 12 (of 13) bits 5664 andn %l1, %l4, %l1 ! Mask the phys page number into RA 5665 or %l2, %l1, %l1 ! Now take care of the 8 high bits V|NFO|SW 5666 or %l1, 0x0741, %l2 ! And low 13 bits IE=0|E=0|CP=1|CV=1|P=1| 5667 ! X=0|W=1|SW=00|SZ=0001 5668 5669 /* 5670 * Now, map in the interrupt stack & cpu_info as context==0 5671 */ 5672 5673 set INTSTACK, %o0 ! vaddr 5674 clr %o1 ! reserved 5675 mov %l2, %o2 ! tte 5676 mov MAP_DTLB, %o3 ! flags 5677 mov FT_MMU_MAP_PERM_ADDR, %o5 ! hv fast trap function 5678 ta ST_FAST_TRAP 5679 cmp %o0, 0 5680 be,pt %icc, 5f 5681 nop 5682 sir ! crash if mapping fails 5683 5: 5684 5685 /* 5686 * Set 0 as primary context XXX 5687 */ 5688 5689 mov CTX_PRIMARY, %o0 5690 SET_MMU_CONTEXTID_SUN4V %g0, %o0 5691 5692 ba 4f 5693 nop 5694 3: 5695 #endif 5696 5697 /* sun4u */ 5698 5699 sethi %hi(0xa0000000), %l2 ! V=1|SZ=01|NFO=0|IE=0 5700 sllx %l2, 32, %l2 ! Shift it into place 5701 mov -1, %l3 ! Create a nice mask 5702 sllx %l3, 43, %l4 ! Mask off high bits 5703 or %l4, 0xfff, %l4 ! We can just load this in 12 (of 13) bits 5704 andn %l1, %l4, %l1 ! Mask the phys page number 5705 or %l2, %l1, %l1 ! Now take care of the high bits 5706 or %l1, SUN4U_TTE_DATABITS, %l2 ! And low bits: L=1|CP=1|CV=?|E=0|P=1|W=1|G=0 5707 5708 /* 5709 * Now, map in the interrupt stack & cpu_info as context==0 5710 */ 5711 5712 set TLB_TAG_ACCESS, %l5 5713 set INTSTACK, %l0 5714 stxa %l0, [%l5] ASI_DMMU ! Make DMMU point to it 5715 stxa %l2, [%g0] ASI_DMMU_DATA_IN ! Store it 5716 5717 /* 5718 * Set 0 as primary context XXX 5719 */ 5720 5721 mov CTX_PRIMARY, %o0 5722 SET_MMU_CONTEXTID_SUN4U %g0, %o0 5723 5724 4: 5725 membar #Sync 5726 5727 /* 5728 * Temporarily use the interrupt stack 5729 */ 5730 #ifdef _LP64 5731 set ((EINTSTACK - CC64FSZ - TF_SIZE)) & ~0x0f - BIAS, %sp 5732 #else 5733 set EINTSTACK - CC64FSZ - TF_SIZE, %sp 5734 #endif 5735 set 1, %fp 5736 clr %i7 5737 5738 #ifdef SUN4V 5739 cmp %l6, CPU_SUN4V 5740 bne,pt %icc, 2f 5741 nop 5742 5743 /* sun4v */ 5744 5745 /* 5746 * install our TSB pointers 5747 */ 5748 5749 set CPUINFO_VA, %o0 5750 LDPTR [%o0 + CI_TSB_DESC], %o0 5751 call _C_LABEL(pmap_setup_tsb_sun4v) 5752 nop 5753 5754 /* set trap table */ 5755 5756 set _C_LABEL(trapbase_sun4v), %l1 5757 GET_MMFSA %o1 5758 call _C_LABEL(prom_set_trap_table_sun4v) 5759 mov %l1, %o0 5760 5761 ! Now we should be running 100% from our handlers 5762 ba 3f 5763 nop 5764 2: 5765 #endif 5766 /* sun4u */ 5767 5768 /* 5769 * install our TSB pointers 5770 */ 5771 5772 sethi %hi(CPUINFO_VA+CI_TSB_DMMU), %l0 5773 sethi %hi(CPUINFO_VA+CI_TSB_IMMU), %l1 5774 sethi %hi(_C_LABEL(tsbsize)), %l2 5775 sethi %hi(0x1fff), %l3 5776 sethi %hi(TSB), %l4 5777 LDPTR [%l0 + %lo(CPUINFO_VA+CI_TSB_DMMU)], %l0 5778 LDPTR [%l1 + %lo(CPUINFO_VA+CI_TSB_IMMU)], %l1 5779 ld [%l2 + %lo(_C_LABEL(tsbsize))], %l2 5780 or %l3, %lo(0x1fff), %l3 5781 or %l4, %lo(TSB), %l4 5782 5783 andn %l0, %l3, %l0 ! Mask off size and split bits 5784 or %l0, %l2, %l0 ! Make a TSB pointer 5785 stxa %l0, [%l4] ASI_DMMU ! Install data TSB pointer 5786 membar #Sync 5787 5788 andn %l1, %l3, %l1 ! Mask off size and split bits 5789 or %l1, %l2, %l1 ! Make a TSB pointer 5790 stxa %l1, [%l4] ASI_IMMU ! Install instruction TSB pointer 5791 membar #Sync 5792 set 1f, %o0 5793 flush %o0 5794 1: 5795 5796 /* set trap table */ 5797 5798 set _C_LABEL(trapbase), %l1 5799 call _C_LABEL(prom_set_trap_table_sun4u) 5800 mov %l1, %o0 5801 3: 5802 wrpr %l1, 0, %tba ! Make sure the PROM didn't 5803 ! foul up. 5804 /* 5805 * Use this CPUs idlelewp's uarea stack 5806 */ 5807 sethi %hi(CPUINFO_VA+CI_IDLELWP), %l0 5808 LDPTR [%l0 + %lo(CPUINFO_VA+CI_IDLELWP)], %l0 5809 set USPACE - TF_SIZE - CC64FSZ, %l1 5810 LDPTR [%l0 + L_PCB], %l0 5811 add %l0, %l1, %l0 5812 #ifdef _LP64 5813 andn %l0, 0x0f, %l0 ! Needs to be 16-byte aligned 5814 sub %l0, BIAS, %l0 ! and biased 5815 #endif 5816 mov %l0, %sp 5817 flushw 5818 5819 /* 5820 * Switch to the kernel mode and run away. 5821 */ 5822 wrpr %g0, 13, %pil 5823 wrpr %g0, PSTATE_INTR|PSTATE_PEF, %pstate 5824 wr %g0, FPRS_FEF, %fprs ! Turn on FPU 5825 5826 call _C_LABEL(cpu_hatch) 5827 clr %g4 5828 5829 b _C_LABEL(idle_loop) 5830 clr %o0 5831 5832 NOTREACHED 5833 5834 .globl cpu_mp_startup_end 5835 cpu_mp_startup_end: 5836 #endif 5837 5838 /* 5839 * openfirmware(cell* param); 5840 * 5841 * OpenFirmware entry point 5842 * 5843 * If we're running in 32-bit mode we need to convert to a 64-bit stack 5844 * and 64-bit cells. The cells we'll allocate off the stack for simplicity. 5845 */ 5846 .align 8 5847 ENTRY(openfirmware) 5848 sethi %hi(romp), %o4 5849 andcc %sp, 1, %g0 5850 bz,pt %icc, 1f 5851 LDPTR [%o4+%lo(romp)], %o4 ! v9 stack, just load the addr and callit 5852 save %sp, -CC64FSZ, %sp 5853 rdpr %pil, %i2 5854 mov PIL_HIGH, %i3 5855 cmp %i3, %i2 5856 movle %icc, %i2, %i3 5857 wrpr %g0, %i3, %pil 5858 mov %i0, %o0 5859 mov %g1, %l1 5860 mov %g2, %l2 5861 mov %g3, %l3 5862 mov %g4, %l4 5863 mov %g5, %l5 5864 mov %g6, %l6 5865 mov %g7, %l7 5866 rdpr %pstate, %l0 5867 jmpl %i4, %o7 5868 #if !defined(_LP64) 5869 wrpr %g0, PSTATE_PROM, %pstate 5870 #else 5871 wrpr %g0, PSTATE_PROM|PSTATE_IE, %pstate 5872 #endif 5873 wrpr %l0, %g0, %pstate 5874 mov %l1, %g1 5875 mov %l2, %g2 5876 mov %l3, %g3 5877 mov %l4, %g4 5878 mov %l5, %g5 5879 mov %l6, %g6 5880 mov %l7, %g7 5881 wrpr %i2, 0, %pil 5882 ret 5883 restore %o0, %g0, %o0 5884 5885 1: ! v8 -- need to screw with stack & params 5886 #ifdef NOTDEF_DEBUG 5887 mov %o7, %o5 5888 call globreg_check 5889 nop 5890 mov %o5, %o7 5891 #endif 5892 save %sp, -CC64FSZ, %sp ! Get a new 64-bit stack frame 5893 add %sp, -BIAS, %sp 5894 rdpr %pstate, %l0 5895 srl %sp, 0, %sp 5896 rdpr %pil, %i2 ! s = splx(level) 5897 mov %i0, %o0 5898 mov PIL_HIGH, %i3 5899 mov %g1, %l1 5900 mov %g2, %l2 5901 cmp %i3, %i2 5902 mov %g3, %l3 5903 mov %g4, %l4 5904 mov %g5, %l5 5905 movle %icc, %i2, %i3 5906 mov %g6, %l6 5907 mov %g7, %l7 5908 wrpr %i3, %g0, %pil 5909 jmpl %i4, %o7 5910 ! Enable 64-bit addresses for the prom 5911 #if defined(_LP64) 5912 wrpr %g0, PSTATE_PROM, %pstate 5913 #else 5914 wrpr %g0, PSTATE_PROM|PSTATE_IE, %pstate 5915 #endif 5916 wrpr %l0, 0, %pstate 5917 wrpr %i2, 0, %pil 5918 mov %l1, %g1 5919 mov %l2, %g2 5920 mov %l3, %g3 5921 mov %l4, %g4 5922 mov %l5, %g5 5923 mov %l6, %g6 5924 mov %l7, %g7 5925 ret 5926 restore %o0, %g0, %o0 5927 5928 /* 5929 * void ofw_exit(cell_t args[]) 5930 */ 5931 ENTRY(openfirmware_exit) 5932 STACKFRAME(-CC64FSZ) 5933 flushw ! Flush register windows 5934 5935 wrpr %g0, PIL_HIGH, %pil ! Disable interrupts 5936 sethi %hi(romtba), %l5 5937 LDPTR [%l5 + %lo(romtba)], %l5 5938 wrpr %l5, 0, %tba ! restore the ofw trap table 5939 5940 /* Arrange locked kernel stack as PROM stack */ 5941 set EINTSTACK - CC64FSZ, %l5 5942 5943 andn %l5, 0x0f, %l5 ! Needs to be 16-byte aligned 5944 sub %l5, BIAS, %l5 ! and biased 5945 mov %l5, %sp 5946 flushw 5947 5948 sethi %hi(romp), %l6 5949 LDPTR [%l6 + %lo(romp)], %l6 5950 5951 mov CTX_PRIMARY, %l3 ! set context 0 5952 stxa %g0, [%l3] ASI_DMMU 5953 membar #Sync 5954 5955 wrpr %g0, PSTATE_PROM, %pstate ! Disable interrupts 5956 ! and enable 64-bit addresses 5957 wrpr %g0, 0, %tl ! force trap level 0 5958 call %l6 5959 mov %i0, %o0 5960 NOTREACHED 5961 5962 /* 5963 * sp_tlb_flush_pte_us(vaddr_t va, int ctx) 5964 * sp_tlb_flush_pte_usiii(vaddr_t va, int ctx) 5965 * 5966 * Flush tte from both IMMU and DMMU. 5967 * 5968 * This uses %o0-%o5 5969 */ 5970 .align 8 5971 ENTRY(sp_tlb_flush_pte_us) 5972 #ifdef DEBUG 5973 set pmapdebug, %o3 5974 lduw [%o3], %o3 5975 ! movrz %o1, -1, %o3 ! Print on either pmapdebug & PDB_DEMAP or ctx == 0 5976 btst 0x0020, %o3 5977 bz,pt %icc, 2f 5978 nop 5979 save %sp, -CC64FSZ, %sp 5980 set 1f, %o0 5981 mov %i1, %o1 5982 andn %i0, 0xfff, %o3 5983 or %o3, 0x010, %o3 5984 call _C_LABEL(printf) 5985 mov %i0, %o2 5986 restore 5987 .data 5988 1: 5989 .asciz "sp_tlb_flush_pte_us: demap ctx=%x va=%08x res=%x\n" 5990 _ALIGN 5991 .text 5992 2: 5993 #endif 5994 #ifdef MULTIPROCESSOR 5995 rdpr %pstate, %o3 5996 andn %o3, PSTATE_IE, %o4 ! disable interrupts 5997 wrpr %o4, 0, %pstate 5998 #endif 5999 srlx %o0, PG_SHIFT4U, %o0 ! drop unused va bits 6000 mov CTX_SECONDARY, %o2 6001 sllx %o0, PG_SHIFT4U, %o0 6002 ldxa [%o2] ASI_DMMU, %o5 ! Save secondary context 6003 sethi %hi(KERNBASE), %o4 6004 membar #LoadStore 6005 stxa %o1, [%o2] ASI_DMMU ! Insert context to demap 6006 membar #Sync 6007 or %o0, DEMAP_PAGE_SECONDARY, %o0 ! Demap page from secondary context only 6008 stxa %o0, [%o0] ASI_DMMU_DEMAP ! Do the demap 6009 stxa %o0, [%o0] ASI_IMMU_DEMAP ! to both TLBs 6010 #ifdef TLB_FLUSH_LOWVA 6011 srl %o0, 0, %o0 ! and make sure it's both 32- and 64-bit entries 6012 stxa %o0, [%o0] ASI_DMMU_DEMAP ! Do the demap 6013 stxa %o0, [%o0] ASI_IMMU_DEMAP ! Do the demap 6014 #endif 6015 flush %o4 6016 stxa %o5, [%o2] ASI_DMMU ! Restore secondary context 6017 membar #Sync 6018 retl 6019 #ifdef MULTIPROCESSOR 6020 wrpr %o3, %pstate ! restore interrupts 6021 #else 6022 nop 6023 #endif 6024 6025 ENTRY(sp_tlb_flush_pte_usiii) 6026 #ifdef DEBUG 6027 set pmapdebug, %o3 6028 lduw [%o3], %o3 6029 ! movrz %o1, -1, %o3 ! Print on either pmapdebug & PDB_DEMAP or ctx == 0 6030 btst 0x0020, %o3 6031 bz,pt %icc, 2f 6032 nop 6033 save %sp, -CC64FSZ, %sp 6034 set 1f, %o0 6035 mov %i1, %o1 6036 andn %i0, 0xfff, %o3 6037 or %o3, 0x010, %o3 6038 call _C_LABEL(printf) 6039 mov %i0, %o2 6040 restore 6041 .data 6042 1: 6043 .asciz "sp_tlb_flush_pte_usiii: demap ctx=%x va=%08x res=%x\n" 6044 _ALIGN 6045 .text 6046 2: 6047 #endif 6048 ! %o0 = VA [in] 6049 ! %o1 = ctx value [in] / KERNBASE 6050 ! %o2 = CTX_PRIMARY 6051 ! %o3 = saved %tl 6052 ! %o4 = saved %pstate 6053 ! %o5 = saved primary ctx 6054 6055 ! Need this for UP as well 6056 rdpr %pstate, %o4 6057 andn %o4, PSTATE_IE, %o3 ! disable interrupts 6058 wrpr %o3, 0, %pstate 6059 6060 !! 6061 !! Cheetahs do not support flushing the IMMU from secondary context 6062 !! 6063 rdpr %tl, %o3 6064 mov CTX_PRIMARY, %o2 6065 brnz,pt %o3, 1f 6066 andn %o0, 0xfff, %o0 ! drop unused va bits 6067 wrpr %g0, 1, %tl ! Make sure we're NUCLEUS 6068 1: 6069 ldxa [%o2] ASI_DMMU, %o5 ! Save primary context 6070 membar #LoadStore 6071 stxa %o1, [%o2] ASI_DMMU ! Insert context to demap 6072 sethi %hi(KERNBASE), %o1 6073 membar #Sync 6074 or %o0, DEMAP_PAGE_PRIMARY, %o0 6075 stxa %o0, [%o0] ASI_DMMU_DEMAP ! Do the demap 6076 membar #Sync 6077 stxa %o0, [%o0] ASI_IMMU_DEMAP ! to both TLBs 6078 membar #Sync 6079 #ifdef TLB_FLUSH_LOWVA 6080 srl %o0, 0, %o0 ! and make sure it's both 32- and 64-bit entries 6081 stxa %o0, [%o0] ASI_DMMU_DEMAP ! Do the demap 6082 membar #Sync 6083 stxa %o0, [%o0] ASI_IMMU_DEMAP ! Do the demap 6084 membar #Sync 6085 #endif 6086 flush %o1 6087 stxa %o5, [%o2] ASI_DMMU ! Restore primary context 6088 membar #Sync 6089 brnz,pt %o3, 1f 6090 flush %o1 6091 wrpr %g0, %o3, %tl ! Return to kernel mode. 6092 1: 6093 retl 6094 wrpr %o4, %pstate ! restore interrupts 6095 6096 6097 /* 6098 * sp_tlb_flush_all_us(void) 6099 * sp_tlb_flush_all_usiii(void) 6100 * 6101 * Flush all user TLB entries from both IMMU and DMMU. 6102 * We have both UltraSPARC I+II, and UltraSPARC >=III versions. 6103 */ 6104 .align 8 6105 ENTRY(sp_tlb_flush_all_us) 6106 rdpr %pstate, %o3 6107 andn %o3, PSTATE_IE, %o4 ! disable interrupts 6108 wrpr %o4, 0, %pstate 6109 set ((TLB_SIZE_SPITFIRE-1) * 8), %o0 6110 set CTX_SECONDARY, %o4 6111 ldxa [%o4] ASI_DMMU, %o4 ! save secondary context 6112 set CTX_MASK, %o5 6113 membar #Sync 6114 6115 ! %o0 = loop counter 6116 ! %o1 = ctx value 6117 ! %o2 = TLB tag value 6118 ! %o3 = saved %pstate 6119 ! %o4 = saved primary ctx 6120 ! %o5 = CTX_MASK 6121 ! %xx = saved %tl 6122 6123 0: 6124 ldxa [%o0] ASI_DMMU_TLB_TAG, %o2 ! fetch the TLB tag 6125 andcc %o2, %o5, %o1 ! context 0? 6126 bz,pt %xcc, 1f ! if so, skip 6127 mov CTX_SECONDARY, %o2 6128 6129 stxa %o1, [%o2] ASI_DMMU ! set the context 6130 set DEMAP_CTX_SECONDARY, %o2 6131 membar #Sync 6132 stxa %o2, [%o2] ASI_DMMU_DEMAP ! do the demap 6133 membar #Sync 6134 6135 1: 6136 dec 8, %o0 6137 brgz,pt %o0, 0b ! loop over all entries 6138 nop 6139 6140 /* 6141 * now do the IMMU 6142 */ 6143 6144 set ((TLB_SIZE_SPITFIRE-1) * 8), %o0 6145 6146 0: 6147 ldxa [%o0] ASI_IMMU_TLB_TAG, %o2 ! fetch the TLB tag 6148 andcc %o2, %o5, %o1 ! context 0? 6149 bz,pt %xcc, 1f ! if so, skip 6150 mov CTX_SECONDARY, %o2 6151 6152 stxa %o1, [%o2] ASI_DMMU ! set the context 6153 set DEMAP_CTX_SECONDARY, %o2 6154 membar #Sync 6155 stxa %o2, [%o2] ASI_IMMU_DEMAP ! do the demap 6156 membar #Sync 6157 6158 1: 6159 dec 8, %o0 6160 brgz,pt %o0, 0b ! loop over all entries 6161 nop 6162 6163 set CTX_SECONDARY, %o2 6164 stxa %o4, [%o2] ASI_DMMU ! restore secondary ctx 6165 sethi %hi(KERNBASE), %o4 6166 membar #Sync 6167 flush %o4 6168 retl 6169 wrpr %o3, %pstate 6170 6171 .align 8 6172 ENTRY(sp_tlb_flush_all_usiii) 6173 rdpr %tl, %o5 6174 brnz,pt %o5, 1f 6175 set DEMAP_ALL, %o2 6176 wrpr 1, %tl 6177 1: 6178 rdpr %pstate, %o3 6179 andn %o3, PSTATE_IE, %o4 ! disable interrupts 6180 wrpr %o4, 0, %pstate 6181 6182 stxa %o2, [%o2] ASI_IMMU_DEMAP 6183 membar #Sync 6184 stxa %o2, [%o2] ASI_DMMU_DEMAP 6185 6186 sethi %hi(KERNBASE), %o4 6187 membar #Sync 6188 flush %o4 6189 6190 wrpr %o5, %tl 6191 retl 6192 wrpr %o3, %pstate 6193 6194 /* 6195 * sp_blast_dcache(int dcache_size, int dcache_line_size) 6196 * sp_blast_dcache_disabled(int dcache_size, int dcache_line_size) 6197 * 6198 * Clear out all of D$ regardless of contents. The latter one also 6199 * disables the D$ while doing so. 6200 */ 6201 .align 8 6202 ENTRY(sp_blast_dcache) 6203 /* 6204 * We turn off interrupts for the duration to prevent RED exceptions. 6205 */ 6206 #ifdef PROF 6207 save %sp, -CC64FSZ, %sp 6208 #endif 6209 6210 rdpr %pstate, %o3 6211 sub %o0, %o1, %o0 6212 andn %o3, PSTATE_IE, %o4 ! Turn off PSTATE_IE bit 6213 wrpr %o4, 0, %pstate 6214 1: 6215 stxa %g0, [%o0] ASI_DCACHE_TAG 6216 membar #Sync 6217 brnz,pt %o0, 1b 6218 sub %o0, %o1, %o0 6219 6220 sethi %hi(KERNBASE), %o2 6221 flush %o2 6222 membar #Sync 6223 #ifdef PROF 6224 wrpr %o3, %pstate 6225 ret 6226 restore 6227 #else 6228 retl 6229 wrpr %o3, %pstate 6230 #endif 6231 6232 .align 8 6233 ENTRY(sp_blast_dcache_disabled) 6234 /* 6235 * We turn off interrupts for the duration to prevent RED exceptions. 6236 */ 6237 #ifdef PROF 6238 save %sp, -CC64FSZ, %sp 6239 #endif 6240 6241 rdpr %pstate, %o3 6242 sub %o0, %o1, %o0 6243 andn %o3, PSTATE_IE, %o4 ! Turn off PSTATE_IE bit 6244 wrpr %o4, 0, %pstate 6245 6246 ldxa [%g0] ASI_MCCR, %o5 6247 andn %o5, MCCR_DCACHE_EN, %o4 ! Turn off the D$ 6248 stxa %o4, [%g0] ASI_MCCR 6249 flush %g0 6250 6251 1: 6252 stxa %g0, [%o0] ASI_DCACHE_TAG 6253 membar #Sync 6254 brnz,pt %o0, 1b 6255 sub %o0, %o1, %o0 6256 6257 sethi %hi(KERNBASE), %o2 6258 flush %o2 6259 membar #Sync 6260 6261 stxa %o5, [%g0] ASI_MCCR ! Restore the D$ 6262 flush %g0 6263 #ifdef PROF 6264 wrpr %o3, %pstate 6265 ret 6266 restore 6267 #else 6268 retl 6269 wrpr %o3, %pstate 6270 #endif 6271 6272 #ifdef MULTIPROCESSOR 6273 /* 6274 * void sparc64_ipi_blast_dcache(int dcache_size, int dcache_line_size) 6275 * 6276 * Clear out all of D$ regardless of contents 6277 * 6278 * On entry: 6279 * %g2 = dcache_size 6280 * %g3 = dcache_line_size 6281 */ 6282 .align 8 6283 ENTRY(sparc64_ipi_blast_dcache) 6284 sub %g2, %g3, %g2 6285 1: 6286 stxa %g0, [%g2] ASI_DCACHE_TAG 6287 membar #Sync 6288 brnz,pt %g2, 1b 6289 sub %g2, %g3, %g2 6290 6291 sethi %hi(KERNBASE), %g5 6292 flush %g5 6293 membar #Sync 6294 6295 ba,a ret_from_intr_vector 6296 nop 6297 #endif /* MULTIPROCESSOR */ 6298 6299 /* 6300 * blast_icache_us() 6301 * blast_icache_usiii() 6302 * 6303 * Clear out all of I$ regardless of contents 6304 * Does not modify %o0 6305 * 6306 * We turn off interrupts for the duration to prevent RED exceptions. 6307 * For the Cheetah version, we also have to to turn off the I$ during this as 6308 * ASI_ICACHE_TAG accesses interfere with coherency. 6309 */ 6310 .align 8 6311 ENTRY(blast_icache_us) 6312 rdpr %pstate, %o3 6313 sethi %hi(icache_size), %o1 6314 ld [%o1 + %lo(icache_size)], %o1 6315 sethi %hi(icache_line_size), %o2 6316 ld [%o2 + %lo(icache_line_size)], %o2 6317 sub %o1, %o2, %o1 6318 andn %o3, PSTATE_IE, %o4 ! Turn off PSTATE_IE bit 6319 wrpr %o4, 0, %pstate 6320 1: 6321 stxa %g0, [%o1] ASI_ICACHE_TAG 6322 brnz,pt %o1, 1b 6323 sub %o1, %o2, %o1 6324 sethi %hi(KERNBASE), %o5 6325 flush %o5 6326 membar #Sync 6327 retl 6328 wrpr %o3, %pstate 6329 6330 .align 8 6331 ENTRY(blast_icache_usiii) 6332 rdpr %pstate, %o3 6333 sethi %hi(icache_size), %o1 6334 ld [%o1 + %lo(icache_size)], %o1 6335 sethi %hi(icache_line_size), %o2 6336 ld [%o2 + %lo(icache_line_size)], %o2 6337 sub %o1, %o2, %o1 6338 andn %o3, PSTATE_IE, %o4 ! Turn off PSTATE_IE bit 6339 wrpr %o4, 0, %pstate 6340 ldxa [%g0] ASI_MCCR, %o5 6341 andn %o5, MCCR_ICACHE_EN, %o4 ! Turn off the I$ 6342 stxa %o4, [%g0] ASI_MCCR 6343 flush %g0 6344 1: 6345 stxa %g0, [%o1] ASI_ICACHE_TAG 6346 membar #Sync 6347 brnz,pt %o1, 1b 6348 sub %o1, %o2, %o1 6349 stxa %o5, [%g0] ASI_MCCR ! Restore the I$ 6350 flush %g0 6351 retl 6352 wrpr %o3, %pstate 6353 6354 /* 6355 * dcache_flush_page_us(paddr_t pa) 6356 * dcache_flush_page_usiii(paddr_t pa) 6357 * 6358 * Clear one page from D$. 6359 * 6360 */ 6361 .align 8 6362 ENTRY(dcache_flush_page_us) 6363 #ifndef _LP64 6364 COMBINE(%o0, %o1, %o0) 6365 #endif 6366 mov -1, %o1 ! Generate mask for tag: bits [29..2] 6367 srlx %o0, 13-2, %o2 ! Tag is PA bits <40:13> in bits <29:2> 6368 clr %o4 6369 srl %o1, 2, %o1 ! Now we have bits <29:0> set 6370 set (2*NBPG), %o5 6371 ba,pt %icc, 1f 6372 andn %o1, 3, %o1 ! Now we have bits <29:2> set 6373 6374 .align 8 6375 1: 6376 ldxa [%o4] ASI_DCACHE_TAG, %o3 6377 mov %o4, %o0 6378 deccc 32, %o5 6379 bl,pn %icc, 2f 6380 inc 32, %o4 6381 6382 xor %o3, %o2, %o3 6383 andcc %o3, %o1, %g0 6384 bne,pt %xcc, 1b 6385 membar #LoadStore 6386 6387 stxa %g0, [%o0] ASI_DCACHE_TAG 6388 ba,pt %icc, 1b 6389 membar #StoreLoad 6390 2: 6391 6392 sethi %hi(KERNBASE), %o5 6393 flush %o5 6394 retl 6395 membar #Sync 6396 6397 .align 8 6398 ENTRY(dcache_flush_page_usiii) 6399 #ifndef _LP64 6400 COMBINE(%o0, %o1, %o0) 6401 #endif 6402 set NBPG, %o1 6403 sethi %hi(dcache_line_size), %o2 6404 add %o0, %o1, %o1 ! end address 6405 ld [%o2 + %lo(dcache_line_size)], %o2 6406 6407 1: 6408 stxa %g0, [%o0] ASI_DCACHE_INVALIDATE 6409 add %o0, %o2, %o0 6410 cmp %o0, %o1 6411 bl,pt %xcc, 1b 6412 nop 6413 6414 sethi %hi(KERNBASE), %o5 6415 flush %o5 6416 retl 6417 membar #Sync 6418 6419 /* 6420 * cache_flush_phys_us(paddr_t, psize_t, int); 6421 * cache_flush_phys_usiii(paddr_t, psize_t, int); 6422 * 6423 * Clear a set of paddrs from the D$, I$ and if param3 is 6424 * non-zero, E$. (E$ is not supported yet). 6425 */ 6426 6427 .align 8 6428 ENTRY(cache_flush_phys_us) 6429 #ifndef _LP64 6430 COMBINE(%o0, %o1, %o0) 6431 COMBINE(%o2, %o3, %o1) 6432 mov %o4, %o2 6433 #endif 6434 #ifdef DEBUG 6435 tst %o2 ! Want to clear E$? 6436 tnz 1 ! Error! 6437 #endif 6438 add %o0, %o1, %o1 ! End PA 6439 dec %o1 6440 6441 !! 6442 !! Both D$ and I$ tags match pa bits 42-13, but 6443 !! they are shifted different amounts. So we'll 6444 !! generate a mask for bits 40-13. 6445 !! 6446 6447 mov -1, %o2 ! Generate mask for tag: bits [40..13] 6448 srl %o2, 5, %o2 ! 32-5 = [27..0] 6449 sllx %o2, 13, %o2 ! 27+13 = [40..13] 6450 6451 and %o2, %o0, %o0 ! Mask away uninteresting bits 6452 and %o2, %o1, %o1 ! (probably not necessary) 6453 6454 set (2*NBPG), %o5 6455 clr %o4 6456 1: 6457 ldxa [%o4] ASI_DCACHE_TAG, %o3 6458 sllx %o3, 40-29, %o3 ! Shift D$ tag into place 6459 and %o3, %o2, %o3 ! Mask out trash 6460 6461 cmp %o0, %o3 6462 blt,pt %xcc, 2f ! Too low 6463 cmp %o1, %o3 6464 bgt,pt %xcc, 2f ! Too high 6465 nop 6466 6467 membar #LoadStore 6468 stxa %g0, [%o4] ASI_DCACHE_TAG ! Just right 6469 membar #Sync 6470 2: 6471 ldda [%o4] ASI_ICACHE_TAG, %g0 ! Tag goes in %g1 6472 sllx %g1, 40-35, %g1 ! Shift I$ tag into place 6473 and %g1, %o2, %g1 ! Mask out trash 6474 cmp %o0, %g1 6475 blt,pt %xcc, 3f 6476 cmp %o1, %g1 6477 bgt,pt %xcc, 3f 6478 nop 6479 stxa %g0, [%o4] ASI_ICACHE_TAG 6480 3: 6481 membar #StoreLoad 6482 dec 32, %o5 6483 brgz,pt %o5, 1b 6484 inc 32, %o4 6485 6486 sethi %hi(KERNBASE), %o5 6487 flush %o5 6488 retl 6489 membar #Sync 6490 6491 .align 8 6492 ENTRY(cache_flush_phys_usiii) 6493 #ifndef _LP64 6494 COMBINE(%o0, %o1, %o0) 6495 COMBINE(%o2, %o3, %o1) 6496 mov %o4, %o2 6497 #endif 6498 #ifdef DEBUG 6499 tst %o2 ! Want to clear E$? 6500 tnz 1 ! Error! 6501 #endif 6502 add %o0, %o1, %o1 ! End PA 6503 sethi %hi(dcache_line_size), %o3 6504 ld [%o3 + %lo(dcache_line_size)], %o3 6505 sethi %hi(KERNBASE), %o5 6506 1: 6507 stxa %g0, [%o0] ASI_DCACHE_INVALIDATE 6508 add %o0, %o3, %o0 6509 cmp %o0, %o1 6510 bl,pt %xcc, 1b 6511 nop 6512 6513 /* don't need to flush the I$ on cheetah */ 6514 6515 flush %o5 6516 retl 6517 membar #Sync 6518 6519 #ifdef COMPAT_16 6520 #ifdef _LP64 6521 /* 6522 * XXXXX Still needs lotsa cleanup after sendsig is complete and offsets are known 6523 * 6524 * The following code is copied to the top of the user stack when each 6525 * process is exec'ed, and signals are `trampolined' off it. 6526 * 6527 * When this code is run, the stack looks like: 6528 * [%sp] 128 bytes to which registers can be dumped 6529 * [%sp + 128] signal number (goes in %o0) 6530 * [%sp + 128 + 4] signal code (goes in %o1) 6531 * [%sp + 128 + 8] first word of saved state (sigcontext) 6532 * . 6533 * . 6534 * . 6535 * [%sp + NNN] last word of saved state 6536 * (followed by previous stack contents or top of signal stack). 6537 * The address of the function to call is in %g1; the old %g1 and %o0 6538 * have already been saved in the sigcontext. We are running in a clean 6539 * window, all previous windows now being saved to the stack. 6540 * 6541 * Note that [%sp + 128 + 8] == %sp + 128 + 16. The copy at %sp+128+8 6542 * will eventually be removed, with a hole left in its place, if things 6543 * work out. 6544 */ 6545 ENTRY_NOPROFILE(sigcode) 6546 /* 6547 * XXX the `save' and `restore' below are unnecessary: should 6548 * replace with simple arithmetic on %sp 6549 * 6550 * Make room on the stack for 64 %f registers + %fsr. This comes 6551 * out to 64*4+8 or 264 bytes, but this must be aligned to a multiple 6552 * of 64, or 320 bytes. 6553 */ 6554 save %sp, -CC64FSZ - 320, %sp 6555 mov %g2, %l2 ! save globals in %l registers 6556 mov %g3, %l3 6557 mov %g4, %l4 6558 mov %g5, %l5 6559 mov %g6, %l6 6560 mov %g7, %l7 6561 /* 6562 * Saving the fpu registers is expensive, so do it iff it is 6563 * enabled and dirty. 6564 */ 6565 rd %fprs, %l0 6566 btst FPRS_DL|FPRS_DU, %l0 ! All clean? 6567 bz,pt %icc, 2f 6568 btst FPRS_DL, %l0 ! test dl 6569 bz,pt %icc, 1f 6570 btst FPRS_DU, %l0 ! test du 6571 6572 ! fpu is enabled, oh well 6573 stx %fsr, [%sp + CC64FSZ + BIAS + 0] 6574 add %sp, BIAS+CC64FSZ+BLOCK_SIZE, %l0 ! Generate a pointer so we can 6575 andn %l0, BLOCK_ALIGN, %l0 ! do a block store 6576 stda %f0, [%l0] ASI_BLK_P 6577 inc BLOCK_SIZE, %l0 6578 stda %f16, [%l0] ASI_BLK_P 6579 1: 6580 bz,pt %icc, 2f 6581 add %sp, BIAS+CC64FSZ+BLOCK_SIZE, %l0 ! Generate a pointer so we can 6582 andn %l0, BLOCK_ALIGN, %l0 ! do a block store 6583 add %l0, 2*BLOCK_SIZE, %l0 ! and skip what we already stored 6584 stda %f32, [%l0] ASI_BLK_P 6585 inc BLOCK_SIZE, %l0 6586 stda %f48, [%l0] ASI_BLK_P 6587 2: 6588 membar #Sync 6589 rd %fprs, %l0 ! reload fprs copy, for checking after 6590 rd %y, %l1 ! in any case, save %y 6591 lduw [%fp + BIAS + 128], %o0 ! sig 6592 lduw [%fp + BIAS + 128 + 4], %o1 ! code 6593 call %g1 ! (*sa->sa_handler)(sig,code,scp) 6594 add %fp, BIAS + 128 + 8, %o2 ! scp 6595 wr %l1, %g0, %y ! in any case, restore %y 6596 6597 /* 6598 * Now that the handler has returned, re-establish all the state 6599 * we just saved above, then do a sigreturn. 6600 */ 6601 btst FPRS_DL|FPRS_DU, %l0 ! All clean? 6602 bz,pt %icc, 2f 6603 btst FPRS_DL, %l0 ! test dl 6604 bz,pt %icc, 1f 6605 btst FPRS_DU, %l0 ! test du 6606 6607 ldx [%sp + CC64FSZ + BIAS + 0], %fsr 6608 add %sp, BIAS+CC64FSZ+BLOCK_SIZE, %l0 ! Generate a pointer so we can 6609 andn %l0, BLOCK_ALIGN, %l0 ! do a block load 6610 ldda [%l0] ASI_BLK_P, %f0 6611 inc BLOCK_SIZE, %l0 6612 ldda [%l0] ASI_BLK_P, %f16 6613 1: 6614 bz,pt %icc, 2f 6615 nop 6616 add %sp, BIAS+CC64FSZ+BLOCK_SIZE, %l0 ! Generate a pointer so we can 6617 andn %l0, BLOCK_ALIGN, %l0 ! do a block load 6618 inc 2*BLOCK_SIZE, %l0 ! and skip what we already loaded 6619 ldda [%l0] ASI_BLK_P, %f32 6620 inc BLOCK_SIZE, %l0 6621 ldda [%l0] ASI_BLK_P, %f48 6622 2: 6623 mov %l2, %g2 6624 mov %l3, %g3 6625 mov %l4, %g4 6626 mov %l5, %g5 6627 mov %l6, %g6 6628 mov %l7, %g7 6629 membar #Sync 6630 6631 restore %g0, SYS_compat_16___sigreturn14, %g1 ! get registers back & set syscall # 6632 add %sp, BIAS + 128 + 8, %o0! compute scp 6633 ! andn %o0, 0x0f, %o0 6634 t ST_SYSCALL ! sigreturn(scp) 6635 ! sigreturn does not return unless it fails 6636 mov SYS_exit, %g1 ! exit(errno) 6637 t ST_SYSCALL 6638 /* NOTREACHED */ 6639 6640 .globl _C_LABEL(esigcode) 6641 _C_LABEL(esigcode): 6642 #endif 6643 6644 #if !defined(_LP64) 6645 6646 #define SIGCODE_NAME sigcode 6647 #define ESIGCODE_NAME esigcode 6648 #define SIGRETURN_NAME SYS_compat_16___sigreturn14 6649 #define EXIT_NAME SYS_exit 6650 6651 #include "sigcode32.s" 6652 6653 #endif 6654 #endif 6655 6656 /* 6657 * getfp() - get stack frame pointer 6658 */ 6659 ENTRY(getfp) 6660 retl 6661 mov %fp, %o0 6662 6663 /* 6664 * Call optional cpu_idle handler if provided 6665 */ 6666 ENTRY(cpu_idle) 6667 set CPUINFO_VA, %o0 6668 LDPTR [%o0 + CI_IDLESPIN], %o1 6669 tst %o1 6670 bz 1f 6671 nop 6672 jmp %o1 6673 nop 6674 1: 6675 retl 6676 nop 6677 6678 /* 6679 * cpu_switchto() switches to an lwp to run and runs it, saving the 6680 * current one away. 6681 * 6682 * struct lwp * cpu_switchto(struct lwp *current, struct lwp *next) 6683 * Switch to the specified next LWP 6684 * Arguments: 6685 * i0 'struct lwp *' of the current LWP 6686 * i1 'struct lwp *' of the LWP to switch to 6687 * i2 'bool' of the flag returning to a softint LWP or not 6688 * Returns: 6689 * the old lwp switched away from 6690 */ 6691 ENTRY(cpu_switchto) 6692 save %sp, -CC64FSZ, %sp 6693 /* 6694 * REGISTER USAGE AT THIS POINT: 6695 * %l1 = newpcb 6696 * %l3 = new trapframe 6697 * %l4 = new l->l_proc 6698 * %l5 = pcb of oldlwp 6699 * %l6 = %hi(CPCB) 6700 * %l7 = %hi(CURLWP) 6701 * %i0 = oldlwp 6702 * %i1 = lwp 6703 * %i2 = returning 6704 * %o0 = tmp 1 6705 * %o1 = tmp 2 6706 * %o2 = tmp 3 6707 * %o3 = tmp 4 6708 */ 6709 6710 flushw ! save all register windows except this one 6711 wrpr %g0, PSTATE_KERN, %pstate ! make sure we're on normal globals 6712 ! with traps turned off 6713 6714 sethi %hi(CPCB), %l6 6715 6716 rdpr %pstate, %o1 ! oldpstate = %pstate; 6717 LDPTR [%i0 + L_PCB], %l5 6718 6719 stx %i7, [%l5 + PCB_PC] 6720 stx %i6, [%l5 + PCB_SP] 6721 sth %o1, [%l5 + PCB_PSTATE] 6722 6723 rdpr %cwp, %o2 ! Useless 6724 stb %o2, [%l5 + PCB_CWP] 6725 6726 sethi %hi(CURLWP), %l7 6727 6728 LDPTR [%i1 + L_PCB], %l1 ! newpcb = l->l_pcb; 6729 6730 /* 6731 * Load the new lwp. To load, we must change stacks and 6732 * alter cpcb and the window control registers, hence we must 6733 * keep interrupts disabled. 6734 * 6735 * Issue barriers to coordinate mutex_exit on this CPU with 6736 * mutex_vector_enter on another CPU. 6737 * 6738 * 1. Any prior mutex_exit by oldlwp must be visible to other 6739 * CPUs before we set ci_curlwp := newlwp on this one, 6740 * requiring a store-before-store barrier. 6741 * 6742 * 2. ci_curlwp := newlwp must be visible on all other CPUs 6743 * before any subsequent mutex_exit by newlwp can even test 6744 * whether there might be waiters, requiring a 6745 * store-before-load barrier. 6746 * 6747 * See kern_mutex.c for details -- this is necessary for 6748 * adaptive mutexes to detect whether the lwp is on the CPU in 6749 * order to safely block without requiring atomic r/m/w in 6750 * mutex_exit. 6751 */ 6752 6753 membar #StoreStore 6754 STPTR %i1, [%l7 + %lo(CURLWP)] ! curlwp = l; 6755 membar #StoreLoad 6756 STPTR %l1, [%l6 + %lo(CPCB)] ! cpcb = newpcb; 6757 6758 ldx [%l1 + PCB_SP], %i6 6759 ldx [%l1 + PCB_PC], %i7 6760 6761 wrpr %g0, 0, %otherwin ! These two insns should be redundant 6762 wrpr %g0, 0, %canrestore 6763 GET_MAXCWP %o3 6764 wrpr %g0, %o3, %cleanwin 6765 dec 1, %o3 ! CANSAVE + CANRESTORE + OTHERWIN = MAXCWP - 1 6766 /* Skip the rest if returning to a interrupted LWP. */ 6767 brnz,pn %i2, Lsw_noras 6768 wrpr %o3, %cansave 6769 6770 /* finally, enable traps */ 6771 wrpr %g0, PSTATE_INTR, %pstate 6772 6773 !flushw 6774 !membar #Sync 6775 6776 /* 6777 * Check for restartable atomic sequences (RAS) 6778 */ 6779 LDPTR [%i1 + L_PROC], %l4 ! now %l4 points to p 6780 mov %l4, %o0 ! p is first arg to ras_lookup 6781 LDPTR [%o0 + P_RASLIST], %o1 ! any RAS in p? 6782 brz,pt %o1, Lsw_noras ! no, skip RAS check 6783 LDPTR [%i1 + L_TF], %l3 ! pointer to trap frame 6784 call _C_LABEL(ras_lookup) 6785 ldx [%l3 + TF_PC], %o1 6786 cmp %o0, -1 6787 be,pt CCCR, Lsw_noras 6788 add %o0, 4, %o1 6789 stx %o0, [%l3 + TF_PC] ! store rewound %pc 6790 stx %o1, [%l3 + TF_NPC] ! and %npc 6791 6792 Lsw_noras: 6793 6794 /* 6795 * We are resuming the process that was running at the 6796 * call to switch(). Just set psr ipl and return. 6797 */ 6798 ! wrpr %g0, 0, %cleanwin ! DEBUG 6799 clr %g4 ! This needs to point to the base of the data segment 6800 wr %g0, ASI_PRIMARY_NOFAULT, %asi ! Restore default ASI 6801 !wrpr %g0, PSTATE_INTR, %pstate 6802 ret 6803 restore %i0, %g0, %o0 ! return old curlwp 6804 6805 #ifdef __HAVE_FAST_SOFTINTS 6806 /* 6807 * Switch to the LWP assigned to handle interrupts from the given 6808 * source. We borrow the VM context from the interrupted LWP. 6809 * 6810 * int softint_fastintr(void *l) 6811 * 6812 * Arguments: 6813 * i0 softint lwp 6814 */ 6815 ENTRY(softint_fastintr) 6816 save %sp, -CC64FSZ, %sp 6817 set CPUINFO_VA, %l0 ! l0 = curcpu() 6818 rdpr %pil, %l7 ! l7 = splhigh() 6819 wrpr %g0, PIL_HIGH, %pil 6820 LDPTR [%l0 + CI_EINTSTACK], %l6 ! l6 = ci_eintstack 6821 add %sp, -CC64FSZ, %l2 ! ci_eintstack = sp - CC64FSZ 6822 STPTR %l2, [%l0 + CI_EINTSTACK] ! save intstack for nested intr 6823 6824 mov %i0, %o0 ! o0/i0 = softint lwp 6825 mov %l7, %o1 ! o1/i1 = ipl 6826 save %sp, -CC64FSZ, %sp ! make one more register window 6827 flushw ! and save all 6828 6829 sethi %hi(CURLWP), %l7 6830 sethi %hi(CPCB), %l6 6831 LDPTR [%l7 + %lo(CURLWP)], %l0 ! l0 = interrupted lwp (curlwp) 6832 6833 /* save interrupted lwp/pcb info */ 6834 sethi %hi(softint_fastintr_ret - 8), %o0 ! trampoline function 6835 LDPTR [%l0 + L_PCB], %l5 ! l5 = interrupted pcb 6836 or %o0, %lo(softint_fastintr_ret - 8), %o0 6837 stx %i6, [%l5 + PCB_SP] 6838 stx %o0, [%l5 + PCB_PC] 6839 rdpr %pstate, %o1 6840 rdpr %cwp, %o2 6841 sth %o1, [%l5 + PCB_PSTATE] 6842 stb %o2, [%l5 + PCB_CWP] 6843 6844 /* switch to softint lwp */ 6845 sethi %hi(USPACE - TF_SIZE - CC64FSZ - STKB), %o3 6846 LDPTR [%i0 + L_PCB], %l1 ! l1 = softint pcb 6847 or %o3, %lo(USPACE - TF_SIZE - CC64FSZ - STKB), %o3 6848 membar #StoreStore /* for mutex_enter; see cpu_switchto */ 6849 STPTR %i0, [%l7 + %lo(CURLWP)] 6850 /* 6851 * No need for barrier after ci->ci_curlwp = softlwp -- when we 6852 * enter a softint lwp, it can't be holding any mutexes, so it 6853 * can't release any until after it has acquired them, so we 6854 * need not participate in the protocol with mutex_vector_enter 6855 * barriers here. 6856 */ 6857 add %l1, %o3, %i6 6858 STPTR %l1, [%l6 + %lo(CPCB)] 6859 stx %i6, [%l1 + PCB_SP] 6860 add %i6, -CC64FSZ, %sp ! new stack 6861 6862 /* now switched, then invoke MI dispatcher */ 6863 mov %i1, %o1 6864 call _C_LABEL(softint_dispatch) 6865 mov %l0, %o0 6866 6867 /* switch back to interrupted lwp */ 6868 ldx [%l5 + PCB_SP], %i6 6869 membar #StoreStore /* for mutex_enter; see cpu_switchto */ 6870 STPTR %l0, [%l7 + %lo(CURLWP)] 6871 membar #StoreLoad /* for mutex_enter; see cpu_switchto */ 6872 STPTR %l5, [%l6 + %lo(CPCB)] 6873 6874 restore ! rewind register window 6875 6876 STPTR %l6, [%l0 + CI_EINTSTACK] ! restore ci_eintstack 6877 wrpr %g0, %l7, %pil ! restore ipl 6878 ret 6879 restore %g0, 1, %o0 6880 6881 /* 6882 * Trampoline function that gets returned to by cpu_switchto() when 6883 * an interrupt handler blocks. 6884 * 6885 * Arguments: 6886 * o0 old lwp from cpu_switchto() 6887 * 6888 * from softint_fastintr(): 6889 * l0 CPUINFO_VA 6890 * l6 saved ci_eintstack 6891 * l7 saved ipl 6892 */ 6893 softint_fastintr_ret: 6894 /* re-adjust after mi_switch() */ 6895 ld [%l0 + CI_MTX_COUNT], %o1 6896 inc %o1 ! ci_mtx_count++ 6897 st %o1, [%l0 + CI_MTX_COUNT] 6898 6899 STPTR %l6, [%l0 + CI_EINTSTACK] ! restore ci_eintstack 6900 wrpr %g0, %l7, %pil ! restore ipl 6901 ret 6902 restore %g0, 1, %o0 6903 6904 #endif /* __HAVE_FAST_SOFTINTS */ 6905 6906 /* 6907 * Snapshot the current process so that stack frames are up to date. 6908 * Only used just before a crash dump. 6909 */ 6910 ENTRY(snapshot) 6911 rdpr %pstate, %o1 ! save psr 6912 stx %o7, [%o0 + PCB_PC] ! save pc 6913 stx %o6, [%o0 + PCB_SP] ! save sp 6914 rdpr %pil, %o2 6915 sth %o1, [%o0 + PCB_PSTATE] 6916 rdpr %cwp, %o3 6917 stb %o2, [%o0 + PCB_PIL] 6918 stb %o3, [%o0 + PCB_CWP] 6919 6920 flushw 6921 save %sp, -CC64FSZ, %sp 6922 flushw 6923 ret 6924 restore 6925 6926 /* 6927 * cpu_lwp_fork() arranges for lwp_trampoline() to run when the 6928 * nascent lwp is selected by switch(). 6929 * 6930 * The switch frame will contain pointer to struct lwp of this lwp in 6931 * %l2, a pointer to the function to call in %l0, and an argument to 6932 * pass to it in %l1 (we abuse the callee-saved registers). 6933 * 6934 * We enter lwp_trampoline as if we are "returning" from 6935 * cpu_switchto(), so %o0 contains previous lwp (the one we are 6936 * switching from) that we pass to lwp_startup(). 6937 * 6938 * If the function *(%l0) returns, we arrange for an immediate return 6939 * to user mode. This happens in two known cases: after execve(2) of 6940 * init, and when returning a child to user mode after a fork(2). 6941 * 6942 * If were setting up a kernel thread, the function *(%l0) will not 6943 * return. 6944 */ 6945 ENTRY(lwp_trampoline) 6946 /* 6947 * Note: cpu_lwp_fork() has set up a stack frame for us to run 6948 * in, so we can call other functions from here without using 6949 * `save ... restore'. 6950 */ 6951 6952 ! newlwp in %l2, oldlwp in %o0 6953 call lwp_startup 6954 mov %l2, %o1 6955 6956 call %l0 ! re-use current frame 6957 mov %l1, %o0 6958 6959 /* 6960 * Here we finish up as in syscall, but simplified. 6961 */ 6962 b return_from_trap 6963 nop 6964 6965 /* 6966 * pmap_zero_page_phys(pa) 6967 * 6968 * Zero one page physically addressed 6969 * 6970 * Block load/store ASIs do not exist for physical addresses, 6971 * so we won't use them. 6972 * 6973 * We will execute a flush at the end to sync the I$. 6974 * 6975 * This version expects to have the dcache_flush_page_all(pa) 6976 * to have been called before calling into here. 6977 */ 6978 ENTRY(pmap_zero_page_phys) 6979 #ifndef _LP64 6980 COMBINE(%o0, %o1, %o0) 6981 #endif 6982 #ifdef DEBUG 6983 set pmapdebug, %o4 6984 ld [%o4], %o4 6985 btst 0x80, %o4 ! PDB_COPY 6986 bz,pt %icc, 3f 6987 nop 6988 save %sp, -CC64FSZ, %sp 6989 set 2f, %o0 6990 call printf 6991 mov %i0, %o1 6992 ! ta 1; nop 6993 restore 6994 .data 6995 2: .asciz "pmap_zero_page(%p)\n" 6996 _ALIGN 6997 .text 6998 3: 6999 #endif 7000 set NBPG, %o2 ! Loop count 7001 wr %g0, ASI_PHYS_CACHED, %asi 7002 1: 7003 /* Unroll the loop 8 times */ 7004 stxa %g0, [%o0 + 0x00] %asi 7005 deccc 0x40, %o2 7006 stxa %g0, [%o0 + 0x08] %asi 7007 stxa %g0, [%o0 + 0x10] %asi 7008 stxa %g0, [%o0 + 0x18] %asi 7009 stxa %g0, [%o0 + 0x20] %asi 7010 stxa %g0, [%o0 + 0x28] %asi 7011 stxa %g0, [%o0 + 0x30] %asi 7012 stxa %g0, [%o0 + 0x38] %asi 7013 bg,pt %icc, 1b 7014 inc 0x40, %o0 7015 7016 sethi %hi(KERNBASE), %o3 7017 flush %o3 7018 retl 7019 wr %g0, ASI_PRIMARY_NOFAULT, %asi ! Make C code happy 7020 7021 /* 7022 * pmap_copy_page_phys(paddr_t src, paddr_t dst) 7023 * 7024 * Copy one page physically addressed 7025 * We need to use a global reg for ldxa/stxa 7026 * so the top 32-bits cannot be lost if we take 7027 * a trap and need to save our stack frame to a 7028 * 32-bit stack. We will unroll the loop by 4 to 7029 * improve performance. 7030 * 7031 * This version expects to have the dcache_flush_page_all(pa) 7032 * to have been called before calling into here. 7033 * 7034 */ 7035 ENTRY(pmap_copy_page_phys) 7036 #ifndef _LP64 7037 COMBINE(%o0, %o1, %o0) 7038 COMBINE(%o2, %o3, %o1) 7039 #endif 7040 #ifdef DEBUG 7041 set pmapdebug, %o4 7042 ld [%o4], %o4 7043 btst 0x80, %o4 ! PDB_COPY 7044 bz,pt %icc, 3f 7045 nop 7046 save %sp, -CC64FSZ, %sp 7047 mov %i0, %o1 7048 set 2f, %o0 7049 call printf 7050 mov %i1, %o2 7051 ! ta 1; nop 7052 restore 7053 .data 7054 2: .asciz "pmap_copy_page(%p,%p)\n" 7055 _ALIGN 7056 .text 7057 3: 7058 #endif 7059 #if 1 7060 set NBPG, %o2 7061 wr %g0, ASI_PHYS_CACHED, %asi 7062 1: 7063 ldxa [%o0 + 0x00] %asi, %g1 7064 ldxa [%o0 + 0x08] %asi, %o3 7065 ldxa [%o0 + 0x10] %asi, %o4 7066 ldxa [%o0 + 0x18] %asi, %o5 7067 inc 0x20, %o0 7068 deccc 0x20, %o2 7069 stxa %g1, [%o1 + 0x00] %asi 7070 stxa %o3, [%o1 + 0x08] %asi 7071 stxa %o4, [%o1 + 0x10] %asi 7072 stxa %o5, [%o1 + 0x18] %asi 7073 bg,pt %icc, 1b ! We don't care about pages >4GB 7074 inc 0x20, %o1 7075 retl 7076 wr %g0, ASI_PRIMARY_NOFAULT, %asi 7077 #else 7078 set NBPG, %o3 7079 add %o3, %o0, %o3 7080 mov %g1, %o4 ! Save g1 7081 1: 7082 ldxa [%o0] ASI_PHYS_CACHED, %g1 7083 inc 8, %o0 7084 cmp %o0, %o3 7085 stxa %g1, [%o1] ASI_PHYS_CACHED 7086 bl,pt %icc, 1b ! We don't care about pages >4GB 7087 inc 8, %o1 7088 retl 7089 mov %o4, %g1 ! Restore g1 7090 #endif 7091 7092 /* 7093 * extern int64_t pseg_get_real(struct pmap *pm, vaddr_t addr); 7094 * 7095 * Return TTE at addr in pmap. Uses physical addressing only. 7096 * pmap->pm_physaddr must by the physical address of pm_segs 7097 * 7098 */ 7099 ENTRY(pseg_get_real) 7100 ! flushw ! Make sure we don't have stack probs & lose hibits of %o 7101 #ifndef _LP64 7102 clruw %o1 ! Zero extend 7103 #endif 7104 ldx [%o0 + PM_PHYS], %o2 ! pmap->pm_segs 7105 7106 srax %o1, HOLESHIFT, %o3 ! Check for valid address 7107 brz,pt %o3, 0f ! Should be zero or -1 7108 inc %o3 ! Make -1 -> 0 7109 brnz,pn %o3, 1f ! Error! In hole! 7110 0: 7111 srlx %o1, STSHIFT, %o3 7112 and %o3, STMASK, %o3 ! Index into pm_segs 7113 sll %o3, 3, %o3 7114 add %o2, %o3, %o2 7115 DLFLUSH(%o2,%o3) 7116 ldxa [%o2] ASI_PHYS_CACHED, %o2 ! Load page directory pointer 7117 DLFLUSH2(%o3) 7118 7119 srlx %o1, PDSHIFT, %o3 7120 and %o3, PDMASK, %o3 7121 sll %o3, 3, %o3 7122 brz,pn %o2, 1f ! NULL entry? check somewhere else 7123 add %o2, %o3, %o2 7124 DLFLUSH(%o2,%o3) 7125 ldxa [%o2] ASI_PHYS_CACHED, %o2 ! Load page table pointer 7126 DLFLUSH2(%o3) 7127 7128 srlx %o1, PTSHIFT, %o3 ! Convert to ptab offset 7129 and %o3, PTMASK, %o3 7130 sll %o3, 3, %o3 7131 brz,pn %o2, 1f ! NULL entry? check somewhere else 7132 add %o2, %o3, %o2 7133 DLFLUSH(%o2,%o3) 7134 ldxa [%o2] ASI_PHYS_CACHED, %o0 7135 DLFLUSH2(%o3) 7136 brgez,pn %o0, 1f ! Entry invalid? Punt 7137 btst 1, %sp 7138 bz,pn %icc, 0f ! 64-bit mode? 7139 nop 7140 retl ! Yes, return full value 7141 nop 7142 0: 7143 #if 1 7144 srl %o0, 0, %o1 7145 retl ! No, generate a %o0:%o1 double 7146 srlx %o0, 32, %o0 7147 #else 7148 DLFLUSH(%o2,%o3) 7149 ldda [%o2] ASI_PHYS_CACHED, %o0 7150 DLFLUSH2(%o3) 7151 retl ! No, generate a %o0:%o1 double 7152 nop 7153 #endif 7154 1: 7155 #ifndef _LP64 7156 clr %o1 7157 #endif 7158 retl 7159 clr %o0 7160 7161 /* 7162 * In 32-bit mode: 7163 * 7164 * extern int pseg_set_real(struct pmap* %o0, vaddr_t addr %o1, 7165 * int64_t tte %o2:%o3, paddr_t spare %o4:%o5); 7166 * 7167 * In 64-bit mode: 7168 * 7169 * extern int pseg_set_real(struct pmap* %o0, vaddr_t addr %o1, 7170 * int64_t tte %o2, paddr_t spare %o3); 7171 * 7172 * Set a pseg entry to a particular TTE value. Return values are: 7173 * 7174 * -2 addr in hole 7175 * 0 success (spare was not used if given) 7176 * 1 failure (spare was not given, but one is needed) 7177 * 2 success (spare was given, used for L2) 7178 * 3 failure (spare was given, used for L2, another is needed for L3) 7179 * 4 success (spare was given, used for L3) 7180 * 7181 * rv == 0 success, spare not used if one was given 7182 * rv & 4 spare was used for L3 7183 * rv & 2 spare was used for L2 7184 * rv & 1 failure, spare is needed 7185 * 7186 * (NB: nobody in pmap checks for the virtual hole, so the system will hang.) 7187 * The way to call this is: first just call it without a spare page. 7188 * If that fails, allocate a page and try again, passing the paddr of the 7189 * new page as the spare. 7190 * If spare is non-zero it is assumed to be the address of a zeroed physical 7191 * page that can be used to generate a directory table or page table if needed. 7192 * 7193 * We keep track of valid (A_TLB_V bit set) and wired (A_TLB_TSB_LOCK bit set) 7194 * mappings that are set here. We check both bits on the new data entered 7195 * and increment counts, as well as decrementing counts if the bits are set 7196 * in the value replaced by this call. 7197 * The counters are 32 bit or 64 bit wide, depending on the kernel type we are 7198 * running! 7199 */ 7200 ENTRY(pseg_set_real) 7201 #ifndef _LP64 7202 clruw %o1 ! Zero extend 7203 COMBINE(%o2, %o3, %o2) 7204 COMBINE(%o4, %o5, %o3) 7205 #endif 7206 !! 7207 !! However we managed to get here we now have: 7208 !! 7209 !! %o0 = *pmap 7210 !! %o1 = addr 7211 !! %o2 = tte 7212 !! %o3 = paddr of spare page 7213 !! 7214 srax %o1, HOLESHIFT, %o4 ! Check for valid address 7215 brz,pt %o4, 0f ! Should be zero or -1 7216 inc %o4 ! Make -1 -> 0 7217 brz,pt %o4, 0f 7218 nop 7219 #ifdef DEBUG 7220 ta 1 ! Break into debugger 7221 #endif 7222 retl 7223 mov -2, %o0 ! Error -- in hole! 7224 7225 0: 7226 ldx [%o0 + PM_PHYS], %o4 ! pmap->pm_segs 7227 clr %g1 7228 srlx %o1, STSHIFT, %o5 7229 and %o5, STMASK, %o5 7230 sll %o5, 3, %o5 7231 add %o4, %o5, %o4 7232 0: 7233 DLFLUSH(%o4,%g5) 7234 ldxa [%o4] ASI_PHYS_CACHED, %o5 ! Load page directory pointer 7235 DLFLUSH2(%g5) 7236 7237 brnz,a,pt %o5, 0f ! Null pointer? 7238 mov %o5, %o4 7239 brz,pn %o3, 9f ! Have a spare? 7240 mov %o3, %o5 7241 casxa [%o4] ASI_PHYS_CACHED, %g0, %o5 7242 brnz,pn %o5, 0b ! Something changed? 7243 DLFLUSH(%o4, %o5) 7244 mov %o3, %o4 7245 mov 2, %g1 ! record spare used for L2 7246 clr %o3 ! and not available for L3 7247 0: 7248 srlx %o1, PDSHIFT, %o5 7249 and %o5, PDMASK, %o5 7250 sll %o5, 3, %o5 7251 add %o4, %o5, %o4 7252 0: 7253 DLFLUSH(%o4,%g5) 7254 ldxa [%o4] ASI_PHYS_CACHED, %o5 ! Load table directory pointer 7255 DLFLUSH2(%g5) 7256 7257 brnz,a,pt %o5, 0f ! Null pointer? 7258 mov %o5, %o4 7259 brz,pn %o3, 9f ! Have a spare? 7260 mov %o3, %o5 7261 casxa [%o4] ASI_PHYS_CACHED, %g0, %o5 7262 brnz,pn %o5, 0b ! Something changed? 7263 DLFLUSH(%o4, %o4) 7264 mov %o3, %o4 7265 mov 4, %g1 ! record spare used for L3 7266 0: 7267 srlx %o1, PTSHIFT, %o5 ! Convert to ptab offset 7268 and %o5, PTMASK, %o5 7269 sll %o5, 3, %o5 7270 add %o5, %o4, %o4 7271 7272 DLFLUSH(%o4,%g5) 7273 ldxa [%o4] ASI_PHYS_CACHED, %o5 ! save old value in %o5 7274 stxa %o2, [%o4] ASI_PHYS_CACHED ! Easier than shift+or 7275 DLFLUSH2(%g5) 7276 7277 !! at this point we have: 7278 !! %g1 = return value 7279 !! %o0 = struct pmap * (where the counts are) 7280 !! %o2 = new TTE 7281 !! %o5 = old TTE 7282 7283 !! see if stats needs an update 7284 #ifdef SUN4V 7285 sethi %hi(cputyp), %g5 7286 ld [%g5 + %lo(cputyp)], %g5 7287 cmp %g5, CPU_SUN4V 7288 bne,pt %icc, 0f 7289 nop 7290 sethi %hh(SUN4V_TLB_TSB_LOCK), %g5 7291 sllx %g5, 32, %g5 7292 ba 1f 7293 nop 7294 0: 7295 #endif 7296 set SUN4U_TLB_TSB_LOCK, %g5 7297 1: 7298 xor %o2, %o5, %o3 ! %o3 - what changed 7299 7300 brgez,pn %o3, 5f ! has resident changed? (we predict it has) 7301 btst %g5, %o3 ! has wired changed? 7302 7303 LDPTR [%o0 + PM_RESIDENT], %o1 ! gonna update resident count 7304 brlz %o2, 0f 7305 mov 1, %o4 7306 neg %o4 ! new is not resident -> decrement 7307 0: add %o1, %o4, %o1 7308 STPTR %o1, [%o0 + PM_RESIDENT] 7309 btst %g5, %o3 ! has wired changed? 7310 5: bz,pt %xcc, 8f ! we predict it's not 7311 btst %g5, %o2 ! don't waste delay slot, check if new one is wired 7312 LDPTR [%o0 + PM_WIRED], %o1 ! gonna update wired count 7313 bnz,pt %xcc, 0f ! if wired changes, we predict it increments 7314 mov 1, %o4 7315 neg %o4 ! new is not wired -> decrement 7316 0: add %o1, %o4, %o1 7317 STPTR %o1, [%o0 + PM_WIRED] 7318 8: retl 7319 mov %g1, %o0 ! return %g1 7320 7321 9: retl 7322 or %g1, 1, %o0 ! spare needed, return flags + 1 7323 7324 7325 /* 7326 * clearfpstate() 7327 * 7328 * Drops the current fpu state, without saving it. 7329 */ 7330 ENTRY(clearfpstate) 7331 rdpr %pstate, %o1 ! enable FPU 7332 wr %g0, FPRS_FEF, %fprs 7333 or %o1, PSTATE_PEF, %o1 7334 retl 7335 wrpr %o1, 0, %pstate 7336 7337 /* 7338 * savefpstate(f) struct fpstate *f; 7339 * 7340 * Store the current FPU state. 7341 * 7342 * Since the kernel may need to use the FPU and we have problems atomically 7343 * testing and enabling the FPU, we leave here with the FPRS_FEF bit set. 7344 * Normally this should be turned on in loadfpstate(). 7345 */ 7346 /* XXXXXXXXXX Assume caller created a proper stack frame */ 7347 ENTRY(savefpstate) 7348 ! flushw ! Make sure we don't have stack probs & lose hibits of %o 7349 rdpr %pstate, %o1 ! enable FP before we begin 7350 rd %fprs, %o5 7351 wr %g0, FPRS_FEF, %fprs 7352 or %o1, PSTATE_PEF, %o1 7353 wrpr %o1, 0, %pstate 7354 7355 stx %fsr, [%o0 + FS_FSR] ! f->fs_fsr = getfsr(); 7356 rd %gsr, %o4 ! Save %gsr 7357 st %o4, [%o0 + FS_GSR] 7358 7359 add %o0, FS_REGS, %o2 7360 #ifdef DIAGNOSTIC 7361 btst BLOCK_ALIGN, %o2 ! Needs to be re-executed 7362 bnz,pn %icc, 6f ! Check alignment 7363 #endif 7364 st %g0, [%o0 + FS_QSIZE] ! f->fs_qsize = 0; 7365 btst FPRS_DL|FPRS_DU, %o5 ! Both FPU halves clean? 7366 bz,pt %icc, 5f ! Then skip it 7367 7368 btst FPRS_DL, %o5 ! Lower FPU clean? 7369 membar #Sync 7370 bz,a,pt %icc, 1f ! Then skip it, but upper FPU not clean 7371 add %o2, 2*BLOCK_SIZE, %o2 ! Skip a block 7372 7373 stda %f0, [%o2] ASI_BLK_P ! f->fs_f0 = etc; 7374 inc BLOCK_SIZE, %o2 7375 stda %f16, [%o2] ASI_BLK_P 7376 7377 btst FPRS_DU, %o5 ! Upper FPU clean? 7378 bz,pt %icc, 2f ! Then skip it 7379 inc BLOCK_SIZE, %o2 7380 1: 7381 stda %f32, [%o2] ASI_BLK_P 7382 inc BLOCK_SIZE, %o2 7383 stda %f48, [%o2] ASI_BLK_P 7384 2: 7385 membar #Sync ! Finish operation so we can 7386 5: 7387 retl 7388 wr %g0, FPRS_FEF, %fprs ! Mark FPU clean 7389 7390 #ifdef DIAGNOSTIC 7391 !! 7392 !! Damn thing is *NOT* aligned on a 64-byte boundary 7393 !! 7394 6: 7395 wr %g0, FPRS_FEF, %fprs 7396 ! XXX -- we should panic instead of silently entering debugger 7397 ta 1 7398 retl 7399 nop 7400 #endif 7401 7402 /* 7403 * Load FPU state. 7404 */ 7405 /* XXXXXXXXXX Should test to see if we only need to do a partial restore */ 7406 ENTRY(loadfpstate) 7407 flushw ! Make sure we don't have stack probs & lose hibits of %o 7408 rdpr %pstate, %o1 ! enable FP before we begin 7409 ld [%o0 + FS_GSR], %o4 ! Restore %gsr 7410 set PSTATE_PEF, %o2 7411 wr %g0, FPRS_FEF, %fprs 7412 or %o1, %o2, %o1 7413 wrpr %o1, 0, %pstate 7414 ldx [%o0 + FS_FSR], %fsr ! setfsr(f->fs_fsr); 7415 add %o0, FS_REGS, %o3 ! This is zero... 7416 #ifdef DIAGNOSTIC 7417 btst BLOCK_ALIGN, %o3 7418 bne,pn %icc, 1f ! Only use block loads on aligned blocks 7419 #endif 7420 wr %o4, %g0, %gsr 7421 membar #Sync 7422 ldda [%o3] ASI_BLK_P, %f0 7423 inc BLOCK_SIZE, %o3 7424 ldda [%o3] ASI_BLK_P, %f16 7425 inc BLOCK_SIZE, %o3 7426 ldda [%o3] ASI_BLK_P, %f32 7427 inc BLOCK_SIZE, %o3 7428 ldda [%o3] ASI_BLK_P, %f48 7429 membar #Sync ! Make sure loads are complete 7430 retl 7431 wr %g0, FPRS_FEF, %fprs ! Clear dirty bits 7432 7433 #ifdef DIAGNOSTIC 7434 !! 7435 !! Damn thing is *NOT* aligned on a 64-byte boundary 7436 !! 7437 1: 7438 wr %g0, FPRS_FEF, %fprs ! Clear dirty bits 7439 ! XXX -- we should panic instead of silently entering debugger 7440 ta 1 7441 retl 7442 nop 7443 #endif 7444 7445 /* 7446 * ienab_bis(bis) int bis; 7447 * ienab_bic(bic) int bic; 7448 * 7449 * Set and clear bits in the interrupt register. 7450 */ 7451 7452 /* 7453 * sun4u has separate asr's for clearing/setting the interrupt mask. 7454 */ 7455 ENTRY(ienab_bis) 7456 retl 7457 wr %o0, 0, SET_SOFTINT ! SET_SOFTINT 7458 7459 ENTRY(ienab_bic) 7460 retl 7461 wr %o0, 0, CLEAR_SOFTINT ! CLEAR_SOFTINT 7462 7463 /* 7464 * send_softint(cpu, level, intrhand) 7465 * 7466 * Send a softint with an intrhand pointer so we can cause a vectored 7467 * interrupt instead of a polled interrupt. This does pretty much the same 7468 * as interrupt_vector. If cpu is -1 then send it to this CPU, if it's -2 7469 * send it to any CPU, otherwise send it to a particular CPU. 7470 * 7471 * XXXX Dispatching to different CPUs is not implemented yet. 7472 */ 7473 ENTRY(send_softint) 7474 rdpr %pstate, %g1 7475 andn %g1, PSTATE_IE, %g2 ! clear PSTATE.IE 7476 wrpr %g2, 0, %pstate 7477 7478 sethi %hi(CPUINFO_VA+CI_INTRPENDING), %o3 7479 LDPTR [%o2 + IH_PEND], %o5 7480 or %o3, %lo(CPUINFO_VA+CI_INTRPENDING), %o3 7481 brnz %o5, 1f 7482 sll %o1, PTRSHFT, %o5 ! Find start of table for this IPL 7483 add %o3, %o5, %o3 7484 2: 7485 LDPTR [%o3], %o5 ! Load list head 7486 STPTR %o5, [%o2+IH_PEND] ! Link our intrhand node in 7487 mov %o2, %o4 7488 CASPTRA [%o3] ASI_N, %o5, %o4 7489 cmp %o4, %o5 ! Did it work? 7490 bne,pn CCCR, 2b ! No, try again 7491 .empty 7492 7493 mov 1, %o4 ! Change from level to bitmask 7494 sllx %o4, %o1, %o4 7495 wr %o4, 0, SET_SOFTINT ! SET_SOFTINT 7496 1: 7497 retl 7498 wrpr %g1, 0, %pstate ! restore PSTATE.IE 7499 7500 7501 #define MICROPERSEC (1000000) 7502 7503 /* 7504 * delay function 7505 * 7506 * void delay(N) -- delay N microseconds 7507 * 7508 * Register usage: %o0 = "N" number of usecs to go (counts down to zero) 7509 * %o1 = "timerblurb" (stays constant) 7510 * %o2 = counter for 1 usec (counts down from %o1 to zero) 7511 * 7512 * 7513 * ci_cpu_clockrate should be tuned during CPU probe to the CPU 7514 * clockrate in Hz 7515 * 7516 */ 7517 ENTRY(delay) ! %o0 = n 7518 #if 1 7519 rdpr %tick, %o1 ! Take timer snapshot 7520 sethi %hi(CPUINFO_VA + CI_CLOCKRATE), %o2 7521 sethi %hi(MICROPERSEC), %o3 7522 ldx [%o2 + %lo(CPUINFO_VA + CI_CLOCKRATE + 8)], %o4 ! Get scale factor 7523 brnz,pt %o4, 0f 7524 or %o3, %lo(MICROPERSEC), %o3 7525 7526 !! Calculate ticks/usec 7527 ldx [%o2 + %lo(CPUINFO_VA + CI_CLOCKRATE)], %o4 ! No, we need to calculate it 7528 udivx %o4, %o3, %o4 7529 stx %o4, [%o2 + %lo(CPUINFO_VA + CI_CLOCKRATE + 8)] ! Save it so we don't need to divide again 7530 0: 7531 7532 mulx %o0, %o4, %o0 ! Convert usec -> ticks 7533 rdpr %tick, %o2 ! Top of next itr 7534 1: 7535 sub %o2, %o1, %o3 ! How many ticks have gone by? 7536 sub %o0, %o3, %o4 ! Decrement count by that much 7537 movrgz %o3, %o4, %o0 ! But only if we're decrementing 7538 mov %o2, %o1 ! Remember last tick 7539 brgz,pt %o0, 1b ! Done? 7540 rdpr %tick, %o2 ! Get new tick 7541 7542 retl 7543 nop 7544 #else 7545 /* This code only works if %tick does not wrap */ 7546 rdpr %tick, %g1 ! Take timer snapshot 7547 sethi %hi(CPUINFO_VA + CI_CLOCKRATE), %g2 7548 sethi %hi(MICROPERSEC), %o2 7549 ldx [%g2 + %lo(CPUINFO_VA + CI_CLOCKRATE)], %g2 ! Get scale factor 7550 or %o2, %lo(MICROPERSEC), %o2 7551 ! sethi %hi(_C_LABEL(timerblurb), %o5 ! This is if we plan to tune the clock 7552 ! ld [%o5 + %lo(_C_LABEL(timerblurb))], %o5 ! with respect to the counter/timer 7553 mulx %o0, %g2, %g2 ! Scale it: (usec * Hz) / 1 x 10^6 = ticks 7554 udivx %g2, %o2, %g2 7555 add %g1, %g2, %g2 7556 ! add %o5, %g2, %g2 5, %g2, %g2 ! But this gets complicated 7557 rdpr %tick, %g1 ! Top of next itr 7558 mov %g1, %g1 ! Erratum 50 7559 1: 7560 cmp %g1, %g2 7561 bl,a,pn %xcc, 1b ! Done? 7562 rdpr %tick, %g1 7563 7564 retl 7565 nop 7566 #endif 7567 /* 7568 * If something's wrong with the standard setup do this stupid loop 7569 * calibrated for a 143MHz processor. 7570 */ 7571 Lstupid_delay: 7572 set 142857143/MICROPERSEC, %o1 7573 Lstupid_loop: 7574 brnz,pt %o1, Lstupid_loop 7575 dec %o1 7576 brnz,pt %o0, Lstupid_delay 7577 dec %o0 7578 retl 7579 nop 7580 7581 /* 7582 * next_tick(long increment) 7583 * 7584 * Sets the %tick_cmpr register to fire off in `increment' machine 7585 * cycles in the future. Also handles %tick wraparound. In 32-bit 7586 * mode we're limited to a 32-bit increment. 7587 */ 7588 ENTRY(next_tick) 7589 rd TICK_CMPR, %o2 7590 rdpr %tick, %o1 7591 7592 mov 1, %o3 ! Mask off high bits of these registers 7593 sllx %o3, 63, %o3 7594 andn %o1, %o3, %o1 7595 andn %o2, %o3, %o2 7596 cmp %o1, %o2 ! Did we wrap? (tick < tick_cmpr) 7597 bgt,pt %icc, 1f 7598 add %o1, 1000, %o1 ! Need some slack so we don't lose intrs. 7599 7600 /* 7601 * Handle the unlikely case of %tick wrapping. 7602 * 7603 * This should only happen every 10 years or more. 7604 * 7605 * We need to increment the time base by the size of %tick in 7606 * microseconds. This will require some divides and multiplies 7607 * which can take time. So we re-read %tick. 7608 * 7609 */ 7610 7611 /* XXXXX NOT IMPLEMENTED */ 7612 7613 7614 7615 1: 7616 add %o2, %o0, %o2 7617 andn %o2, %o3, %o4 7618 brlz,pn %o4, Ltick_ovflw 7619 cmp %o2, %o1 ! Has this tick passed? 7620 blt,pn %xcc, 1b ! Yes 7621 nop 7622 7623 #ifdef BB_ERRATA_1 7624 ba,a 2f 7625 nop 7626 #else 7627 retl 7628 wr %o2, TICK_CMPR 7629 #endif 7630 7631 Ltick_ovflw: 7632 /* 7633 * When we get here tick_cmpr has wrapped, but we don't know if %tick 7634 * has wrapped. If bit 62 is set then we have not wrapped and we can 7635 * use the current value of %o4 as %tick. Otherwise we need to return 7636 * to our loop with %o4 as %tick_cmpr (%o2). 7637 */ 7638 srlx %o3, 1, %o5 7639 btst %o5, %o1 7640 bz,pn %xcc, 1b 7641 mov %o4, %o2 7642 #ifdef BB_ERRATA_1 7643 ba,a 2f 7644 nop 7645 .align 64 7646 2: wr %o2, TICK_CMPR 7647 rd TICK_CMPR, %g0 7648 retl 7649 nop 7650 #else 7651 retl 7652 wr %o2, TICK_CMPR 7653 #endif 7654 7655 /* 7656 * next_stick(long increment) 7657 * 7658 * Sets the %stick_cmpr register to fire off in `increment' machine 7659 * cycles in the future. Also handles %stick wraparound. In 32-bit 7660 * mode we're limited to a 32-bit increment. 7661 */ 7662 ENTRY(next_stick) 7663 rd STICK_CMPR, %o2 7664 rd STICK, %o1 7665 7666 mov 1, %o3 ! Mask off high bits of these registers 7667 sllx %o3, 63, %o3 7668 andn %o1, %o3, %o1 7669 andn %o2, %o3, %o2 7670 cmp %o1, %o2 ! Did we wrap? (stick < stick_cmpr) 7671 bgt,pt %xcc, 1f 7672 add %o1, 1000, %o1 ! Need some slack so we don't lose intrs. 7673 7674 /* 7675 * Handle the unlikely case of %stick wrapping. 7676 * 7677 * This should only happen every 10 years or more. 7678 * 7679 * We need to increment the time base by the size of %stick in 7680 * microseconds. This will require some divides and multiplies 7681 * which can take time. So we re-read %stick. 7682 * 7683 */ 7684 7685 /* XXXXX NOT IMPLEMENTED */ 7686 7687 7688 7689 1: 7690 add %o2, %o0, %o2 7691 andn %o2, %o3, %o4 7692 brlz,pn %o4, Lstick_ovflw 7693 cmp %o2, %o1 ! Has this stick passed? 7694 blt,pn %xcc, 1b ! Yes 7695 nop 7696 retl 7697 wr %o2, STICK_CMPR 7698 7699 Lstick_ovflw: 7700 /* 7701 * When we get here tick_cmpr has wrapped, but we don't know if %stick 7702 * has wrapped. If bit 62 is set then we have not wrapped and we can 7703 * use the current value of %o4 as %stick. Otherwise we need to return 7704 * to our loop with %o4 as %stick_cmpr (%o2). 7705 */ 7706 srlx %o3, 1, %o5 7707 btst %o5, %o1 7708 bz,pn %xcc, 1b 7709 mov %o4, %o2 7710 retl 7711 wr %o2, STICK_CMPR 7712 7713 /* 7714 * next_stick_init() 7715 * 7716 * Sets the %stick_cmpr register to the value retrieved from %stick so 7717 * next_stick() does not spend too much time in the function when called 7718 * for the first time. 7719 * This has been observed on (at least) a SPARC-T5 (sun4v) system where 7720 * the %stick_cmpr ends up being less than the %stick value and then 7721 * the stickitr() interrupt is never triggered. 7722 */ 7723 ENTRY(next_stick_init) 7724 rd STICK, %o0 7725 mov 1, %o1 ! Mask off high bits of the register 7726 sllx %o1, 63, %o1 7727 andn %o0, %o1, %o0 7728 retl 7729 wr %o0, STICK_CMPR 7730 7731 ENTRY(setjmp) 7732 save %sp, -CC64FSZ, %sp ! Need a frame to return to. 7733 flushw 7734 stx %fp, [%i0+0] ! 64-bit stack pointer 7735 stx %i7, [%i0+8] ! 64-bit return pc 7736 ret 7737 restore %g0, 0, %o0 7738 7739 .data 7740 Lpanic_ljmp: 7741 .asciz "longjmp botch" 7742 _ALIGN 7743 .text 7744 7745 ENTRY(longjmp) 7746 save %sp, -CC64FSZ, %sp ! prepare to restore to (old) frame 7747 flushw 7748 mov 1, %i2 7749 ldx [%i0+0], %fp ! get return stack 7750 ldx [%i0+8], %i7 ! get rpc 7751 ret 7752 restore %i2, 0, %o0 7753 7754 #if defined(DDB) || defined(KGDB) 7755 /* 7756 * Debug stuff. Dump the trap registers into buffer & set tl=0. 7757 * 7758 * %o0 = *ts 7759 */ 7760 ENTRY(savetstate) 7761 mov %o0, %o1 7762 rdpr %tl, %o0 7763 brz %o0, 2f 7764 mov %o0, %o2 7765 1: 7766 rdpr %tstate, %o3 7767 stx %o3, [%o1] 7768 deccc %o2 7769 inc 8, %o1 7770 rdpr %tpc, %o4 7771 stx %o4, [%o1] 7772 inc 8, %o1 7773 rdpr %tnpc, %o5 7774 stx %o5, [%o1] 7775 inc 8, %o1 7776 rdpr %tt, %o4 7777 stx %o4, [%o1] 7778 inc 8, %o1 7779 bnz 1b 7780 wrpr %o2, 0, %tl 7781 2: 7782 retl 7783 nop 7784 7785 /* 7786 * Debug stuff. Restore trap registers from buffer. 7787 * 7788 * %o0 = %tl 7789 * %o1 = *ts 7790 * 7791 * Maybe this should be re-written to increment tl instead of decrementing. 7792 */ 7793 ENTRY(restoretstate) 7794 flushw ! Make sure we don't have stack probs & lose hibits of %o 7795 brz,pn %o0, 2f 7796 mov %o0, %o2 7797 wrpr %o0, 0, %tl 7798 1: 7799 ldx [%o1], %o3 7800 deccc %o2 7801 inc 8, %o1 7802 wrpr %o3, 0, %tstate 7803 ldx [%o1], %o4 7804 inc 8, %o1 7805 wrpr %o4, 0, %tpc 7806 ldx [%o1], %o5 7807 inc 8, %o1 7808 wrpr %o5, 0, %tnpc 7809 ldx [%o1], %o4 7810 inc 8, %o1 7811 wrpr %o4, 0, %tt 7812 bnz 1b 7813 wrpr %o2, 0, %tl 7814 2: 7815 retl 7816 wrpr %o0, 0, %tl 7817 7818 /* 7819 * Switch to context in abs(%o0) 7820 */ 7821 ENTRY(switchtoctx_us) 7822 set DEMAP_CTX_SECONDARY, %o3 7823 stxa %o3, [%o3] ASI_DMMU_DEMAP 7824 mov CTX_SECONDARY, %o4 7825 stxa %o3, [%o3] ASI_IMMU_DEMAP 7826 membar #Sync 7827 stxa %o0, [%o4] ASI_DMMU ! Maybe we should invalid 7828 sethi %hi(KERNBASE), %o2 7829 membar #Sync 7830 flush %o2 7831 retl 7832 nop 7833 7834 ENTRY(switchtoctx_usiii) 7835 mov CTX_SECONDARY, %o4 7836 ldxa [%o4] ASI_DMMU, %o2 ! Load secondary context 7837 mov CTX_PRIMARY, %o5 7838 ldxa [%o5] ASI_DMMU, %o1 ! Save primary context 7839 membar #LoadStore 7840 stxa %o2, [%o5] ASI_DMMU ! Insert secondary for demap 7841 membar #Sync 7842 set DEMAP_CTX_PRIMARY, %o3 7843 stxa %o3, [%o3] ASI_DMMU_DEMAP 7844 membar #Sync 7845 stxa %o0, [%o4] ASI_DMMU ! Maybe we should invalid 7846 membar #Sync 7847 stxa %o1, [%o5] ASI_DMMU ! Restore primary context 7848 sethi %hi(KERNBASE), %o2 7849 membar #Sync 7850 flush %o2 7851 retl 7852 nop 7853 7854 #ifndef _LP64 7855 /* 7856 * Convert to 32-bit stack then call OF_sym2val() 7857 */ 7858 ENTRY(OF_sym2val32) 7859 save %sp, -CC64FSZ, %sp 7860 btst 7, %i0 7861 bnz,pn %icc, 1f 7862 add %sp, BIAS, %o1 7863 btst 1, %sp 7864 movnz %icc, %o1, %sp 7865 call _C_LABEL(OF_sym2val) 7866 mov %i0, %o0 7867 1: 7868 ret 7869 restore %o0, 0, %o0 7870 7871 /* 7872 * Convert to 32-bit stack then call OF_val2sym() 7873 */ 7874 ENTRY(OF_val2sym32) 7875 save %sp, -CC64FSZ, %sp 7876 btst 7, %i0 7877 bnz,pn %icc, 1f 7878 add %sp, BIAS, %o1 7879 btst 1, %sp 7880 movnz %icc, %o1, %sp 7881 call _C_LABEL(OF_val2sym) 7882 mov %i0, %o0 7883 1: 7884 ret 7885 restore %o0, 0, %o0 7886 #endif /* _LP64 */ 7887 #endif /* DDB */ 7888 7889 7890 #if defined(MULTIPROCESSOR) 7891 /* 7892 * IPI target function to setup a C compatible environment and call a MI function. 7893 * 7894 * On entry: 7895 * We are on one of the alternate set of globals 7896 * %g2 = function to call 7897 * %g3 = single argument to called function 7898 */ 7899 ENTRY(sparc64_ipi_ccall) 7900 #ifdef TRAPS_USE_IG 7901 wrpr %g0, PSTATE_KERN|PSTATE_IG, %pstate ! DEBUG 7902 #endif 7903 TRAP_SETUP(-CC64FSZ-TF_SIZE) 7904 7905 #ifdef DEBUG 7906 rdpr %tt, %o1 ! debug 7907 sth %o1, [%sp + CC64FSZ + STKB + TF_TT]! debug 7908 #endif 7909 mov %g3, %o0 ! save argument of function to call 7910 mov %g2, %o5 ! save function pointer 7911 7912 wrpr %g0, PSTATE_KERN, %pstate ! Get back to normal globals 7913 stx %g1, [%sp + CC64FSZ + STKB + TF_G + ( 1*8)] 7914 rdpr %tpc, %o2 ! (pc) 7915 stx %g2, [%sp + CC64FSZ + STKB + TF_G + ( 2*8)] 7916 rdpr %tstate, %g1 7917 stx %g3, [%sp + CC64FSZ + STKB + TF_G + ( 3*8)] 7918 rdpr %tnpc, %o3 7919 stx %g4, [%sp + CC64FSZ + STKB + TF_G + ( 4*8)] 7920 rd %y, %o4 7921 stx %g5, [%sp + CC64FSZ + STKB + TF_G + ( 5*8)] 7922 stx %g6, [%sp + CC64FSZ + STKB + TF_G + ( 6*8)] 7923 stx %g7, [%sp + CC64FSZ + STKB + TF_G + ( 7*8)] 7924 7925 stx %g1, [%sp + CC64FSZ + STKB + TF_TSTATE] 7926 stx %o2, [%sp + CC64FSZ + STKB + TF_PC] 7927 stx %o3, [%sp + CC64FSZ + STKB + TF_NPC] 7928 st %o4, [%sp + CC64FSZ + STKB + TF_Y] 7929 7930 rdpr %pil, %g5 7931 stb %g5, [%sp + CC64FSZ + STKB + TF_PIL] 7932 stb %g5, [%sp + CC64FSZ + STKB + TF_OLDPIL] 7933 7934 rdpr %tl, %g7 7935 dec %g7 7936 movrlz %g7, %g0, %g7 7937 wrpr %g0, %g7, %tl 7938 !! In the EMBEDANY memory model %g4 points to the start of the data segment. 7939 !! In our case we need to clear it before calling any C-code 7940 clr %g4 7941 wr %g0, ASI_NUCLEUS, %asi ! default kernel ASI 7942 7943 call %o5 ! call function 7944 nop 7945 7946 b return_from_trap ! and return from IPI 7947 ldx [%sp + CC64FSZ + STKB + TF_TSTATE], %g1 ! Load this for return_from_trap 7948 7949 #endif 7950 7951 ENTRY(paravirt_membar_sync) 7952 /* 7953 * Store-before-load ordering with respect to matching logic 7954 * on the hypervisor side. 7955 * 7956 * This is the same as membar_sync, but without patching or 7957 * conditionalizing away the MEMBAR instruction on uniprocessor 7958 * builds or boots -- because under virtualization, we still 7959 * have to coordinate with a `device' backed by a hypervisor 7960 * that is potentially on another physical CPU even if we 7961 * observe only one virtual CPU as the guest. 7962 * 7963 * See common/lib/libc/arch/sparc64/atomic/membar_ops.S for why 7964 * we avoid using the delay slot and keep this in sync with the 7965 * implementation of membar_sync there. 7966 */ 7967 membar #StoreLoad 7968 retl 7969 nop 7970 END(paravirt_membar_sync) 7971 7972 .data 7973 _ALIGN 7974 #if NKSYMS || defined(DDB) || defined(MODULAR) 7975 .globl _C_LABEL(esym) 7976 _C_LABEL(esym): 7977 POINTER 0 7978 .globl _C_LABEL(ssym) 7979 _C_LABEL(ssym): 7980 POINTER 0 7981 #endif 7982 .comm _C_LABEL(promvec), PTRSZ 7983 7984 #ifdef DEBUG 7985 .comm _C_LABEL(trapdebug), 4 7986 .comm _C_LABEL(pmapdebug), 4 7987 #endif 7988
Indexes created Sat Sep 20 22:09:52 GMT 2025